From 55a8e062cfca8f715ec1675cc6cab512dfd125bf Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Mon, 26 Oct 2026 17:31:25 +0800 Subject: [PATCH 01/74] fix(tls): preserve transport and completion ownership Checkpoint the reviewed TLS ownership repair before integrating current main. Native transport takeover, completion custody, shutdown joins, and regression coverage are retained together. The generic TLS retention case still needs qualification; no landing claim is made. --- CHANGELOG.md | 2 + src/js/node/net.ts | 258 ++-- src/runtime/api/bun/h2_frame_parser.rs | 13 +- src/runtime/node/node_net_binding.rs | 5 +- src/runtime/socket/Listener.rs | 15 +- src/runtime/socket/UpgradedDuplex.rs | 349 +++++- src/runtime/socket/socket_body.bind.ts | 15 + src/runtime/socket/socket_body.rs | 1082 +++++++++++++++-- src/runtime/socket/sockets.classes.ts | 19 +- test/js/bun/net/socket-retention.test.ts | 126 ++ .../http2/node-http2-syscall-fault.test.ts | 40 +- .../node-http2-writable-destroy-fixture.ts | 46 +- .../node-tls-duplex-close-throw-uaf.test.ts | 59 +- .../tls/node-tls-duplex-end-verify.test.ts | 362 +++++- 14 files changed, 2120 insertions(+), 271 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 275506e67da4..7fe8fafc5b7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Preserve native socket ownership when nesting TLS, including pending writes, read backpressure, shutdown completion and buried HTTP/2 listeners. Keep generic Duplex write completions and queued TLS callbacks alive through reentrant teardown and garbage collection. + - Sync oven-sh/bun through `ad4089bf49d9f4c3ab7c8d95a58f3e0f0f186054`, including lint, format, binding, WebSocket and SQL updates, while retaining the fork runtime and CI contracts. - Share identical `node:vm` script and compiled-function source buffers and source-map URI strings through a bounded per-VM weak index, including after compilation-cache eviction. diff --git a/src/js/node/net.ts b/src/js/node/net.ts index c9db4d282185..2130bc5bf1d1 100644 --- a/src/js/node/net.ts +++ b/src/js/node/net.ts @@ -282,7 +282,11 @@ const checkServerIdentityInHandshake = $bindgen("socket_body.bind.ts", "checkSer // destroy() in the handshake callback: drops the handshake flight that the native layer holds. const releaseHeldFlight = $bindgen("socket_body.bind.ts", "releaseHeldFlight"); const isNamedPipeSocket = $bindgen("socket_body.bind.ts", "isNamedPipeSocket"); +const hasNativeSocketListeners = $bindgen("socket_body.bind.ts", "hasNativeSocketListeners"); const getBufferedAmount = $bindgen("socket_body.bind.ts", "getBufferedAmount"); +const attachNativeTLSTransport = $bindgen("socket_body.bind.ts", "attachNativeTLSTransport"); +const nativeTLSPreviousWritesDone = $bindgen("socket_body.bind.ts", "nativeTLSPreviousWritesDone"); +const shutdownNativeTLSTransport = $bindgen("socket_body.bind.ts", "shutdownNativeTLSTransport"); const bunTlsSymbol = Symbol.for("::buntls::"); const bunSocketServerOptions = Symbol.for("::bunnetserveroptions::"); @@ -314,6 +318,7 @@ const kupgraded = Symbol("kupgraded"); // On the raw handle of an adopted fd: the TLS socket that adopted it. const kAdoptedTLSRaw = Symbol("kAdoptedTLSRaw"); const kAdoptedTLSTransport = Symbol("kAdoptedTLSTransport"); +const kNativeTLSTransport = Symbol("kNativeTLSTransport"); // On that TLS socket: the fd closed, and the socket it wraps waits to be closed with it. const kOwesRawClose = Symbol("kOwesRawClose"); // Emitted on a socket under the stream-level TLS engine in place of reporting a read error itself. @@ -439,6 +444,7 @@ function writeErrnoException(negErrno) { function endNT(self, socket, callback) { const current = self._handle; const handle = !socket || current?.[kAdoptedTLSRaw] ? current : socket; + if (shutdownNativeTLSTransport(handle, callback)) return; const transport = unadoptedTransport(self); if (transport) { const eos = require("internal/streams/end-of-stream"); @@ -462,6 +468,14 @@ function takeInFlightWrite(self) { self[kwriteCallback] = null; return callback; } +// A transport error event may precede its deferred write completion. Settle once even after _hadError. +function settlePendingWrite(self, error) { + const callback = self[kwriteCallback]; + if (!callback) return false; + self[kwriteCallback] = self._pendingData = null; + callback(error); + return true; +} // uv_close() fails a queued write with UV_ECANCELED: https://github.com/nodejs/node/blob/v26.3.0/deps/uv/src/unix/stream.c#L464 function cancelWriteNT(callback) { callback(new ErrnoException(uv().UV_ECANCELED, "write")); @@ -477,8 +491,8 @@ function finishDestroy(callback, err, canceledWrite) { // Shared-fd TLS pair teardown: mirrors node's close ordering, where the // close-callbacks phase runs after the check phase (lib/net.js close path in // node v26.3.0), so destroy()-time setImmediates still see the pair alive. -function closeAdoptedTLSRawNT(self, handle, isException) { - setImmediate(closeSocketHandle, self, handle, isException); +function closeAdoptedTLSRawNT(self, handle, isException, token) { + setImmediate(closeSocketHandle, self, handle, isException, token); } function detachSocket(self) { if (!self) self = this; @@ -488,21 +502,34 @@ function destroyNT(self, err) { self.destroy(err); } // Node's wrap 'close' -> destroy(): https://github.com/nodejs/node/blob/v26.3.0/lib/internal/tls/wrap.js#L739-L741 -function onUpgradedClose(self, connection) { - if (self[kupgraded] !== connection || self.destroyed) return; +function onUpgradedClose(self, connection, token) { + if (self[kNativeTLSTransport] !== token || self[kupgraded] !== connection || self.destroyed) return; // The stream-level engine reads its transport with no backpressure, so the // transport can close after the peer's EOF with plaintext still unread. - if ((self[kended] || self[kOnreadPendingEnd]) && !self.readableEnded) self.once("end", self[kOnUpgradedClose]); - else self.destroy(); + if ((self[kended] || self[kOnreadPendingEnd]) && !self.readableEnded) { + self.once("end", scheduleUpgradedClose.bind(null, self, connection, token)); + } else self.destroy(); } function closeDuplexTLSOwner(self) { const connection = self[kupgraded]; - if (connection && !self[kAdoptedTLSTransport]) onUpgradedClose(self, connection); + if (connection && !self[kAdoptedTLSTransport]) onUpgradedClose(self, connection, self[kNativeTLSTransport]); +} +function scheduleUpgradedClose(self, connection, token) { + // JSStreamSocket closes in the check phase, after earlier write completions. + if (self[kAdoptedTLSTransport]) onUpgradedClose(self, connection, token); + else setImmediate(onUpgradedClose, self, connection, token); +} +function onUpgradedTransportClose(self, connection, token) { + token.closeEmitted = true; + const complete = token.closeComplete; + token.closeComplete = undefined; + scheduleUpgradedClose(self, connection, token); + if (self[kNativeTLSTransport] === token) complete?.(); } // Armed ahead of the stream-level engine's own 'close' thunk: that thunk aborts // a pending handshake, which a socket destroyed first does not report. -function destroyWhenUpgradedCloses(self, connection) { - connection.once("close", (self[kOnUpgradedClose] = onUpgradedClose.bind(null, self, connection))); +function destroyWhenUpgradedCloses(self, connection, token) { + connection.once("close", (self[kOnUpgradedClose] = onUpgradedTransportClose.bind(null, self, connection, token))); } // Node's wrap 'error' -> _emitTLSError. // https://github.com/nodejs/node/blob/v26.3.0/lib/internal/js_stream_socket.js#L65 @@ -524,7 +551,18 @@ function linkUpgraded(self, connection: SocketInstance) { const isSocket = connection instanceof Socket; self[kupgraded] = connection; self._parent = isSocket ? connection : null; - destroyWhenUpgradedCloses(self, connection); + // Retain attachment facts after native close has detached the upper handle. + const token = (self[kNativeTLSTransport] = { + connection, + upperHandle: undefined, + lowerHandle: undefined, + native: false, + nativeReadEnded: false, + nativeShutdownCompleted: false, + closeEmitted: false, + closeComplete: undefined, + }); + destroyWhenUpgradedCloses(self, connection, token); connection.on("error", onUpgradedError.bind(null, self)); self.connecting = isSocket && (connection.connecting || !connection._handle); } @@ -539,16 +577,55 @@ function onUpgradedEnd(connection, eof) { if (!connection.destroyed) eof(); } function attachTLSEngine(self, connection, options) { + const token = self[kNativeTLSTransport]; + if (token?.connection !== connection || self[kupgraded] !== connection) return; self[kAdoptedTLSTransport] = false; const [handle, events] = upgradeDuplexToTLS(connection, { ...options, isNodeSocket: connection instanceof Socket }); - connection.on("data", events[0]); - connection.on("end", onUpgradedEnd.bind(null, connection, events[1])); - connection.on("drain", events[2]); - connection.on("close", events[3]); - connection.once(kReadError, onUpgradedReadError.bind(null, self)); - // https://github.com/nodejs/node/blob/v26.3.0/lib/internal/js_stream_socket.js#L117-L120 - connection.resume(); + // Registration and resume can re-enter or throw; destroy must already own this handle. self._handle = handle; + token.upperHandle = handle; + try { + // Native attachment also needs the JS owner's original transport read error. + connection.once(kReadError, onUpgradedReadError.bind(null, self)); + let initialData; + if (connection instanceof Socket) { + if (connection.readableLength) initialData = connection.read(connection.readableLength) || undefined; + if ( + self.destroyed || + self[kNativeTLSTransport] !== token || + self[kupgraded] !== connection || + self._handle !== handle + ) { + return; + } + // A buffered read can re-enter and replace the handle or queue another write. + const pending = hasUnflushedWrites(connection); + token.lowerHandle = connection._handle; + if (attachNativeTLSTransport(handle, token.lowerHandle, pending, initialData, token)) { + // Only already-issued JS writes use the old Writable queue. New TLS records own the handle. + if (pending) { + connection.write("", error => { + if (error) self.destroy(error); + else if (self._handle === handle) nativeTLSPreviousWritesDone(handle); + }); + } + return; + } + } + connection.on("data", events[0]); + connection.on("end", onUpgradedEnd.bind(null, connection, events[1])); + connection.on("drain", events[2]); + connection.on("close", events[3]); + if (initialData) events[0](initialData); + // https://github.com/nodejs/node/blob/v26.3.0/lib/internal/js_stream_socket.js#L117-L120 + if (!self.destroyed) connection.resume(); + } catch (error) { + try { + self.destroy(error); + } finally { + throw error; + } + } } function adoptTLSPair(self, connection, pair) { self[kAdoptedTLSTransport] = true; @@ -563,7 +640,12 @@ function attachClientTLS(self, connection, tls, upgradeDuplex) { if (self.destroyed || (!upgradeDuplex && !handle)) return; const options = { data: { self, req: {} }, tls, socket: self[khandlers], isServer: false }; try { - if (upgradeDuplex || isNamedPipeSocket(handle) || hasUnflushedWrites(connection)) { + if ( + upgradeDuplex || + isNamedPipeSocket(handle) || + hasNativeSocketListeners(handle) || + hasUnflushedWrites(connection) + ) { attachTLSEngine(self, connection, options); } else { adoptTLSPair(self, connection, upgradeTLSDeferred(handle, options)); @@ -581,7 +663,7 @@ function adoptServerTLS(self, connection, options) { } try { // A user 'connection' listener that ran after the wrap may have queued plain writes, which must flush first. - if (hasUnflushedWrites(connection)) { + if (hasNativeSocketListeners(handle) || hasUnflushedWrites(connection)) { attachTLSEngine(self, connection, options); } else { // What was read off the fd before the wrap sits in the connection's readable buffer. @@ -914,15 +996,10 @@ const SocketHandlers = { error(socket, error) { const self = socket.data; if (!self) return; + settlePendingWrite(self, error); if (self._hadError) return; self._hadError = true; - const callback = self[kwriteCallback]; - if (callback) { - self[kwriteCallback] = null; - callback(error); - } - self.emit("error", error); }, open(socket) { @@ -1452,18 +1529,14 @@ const ServerHandlers = { const data = this.data; if (!data) return; + const bunTLS = data[bunTlsSymbol]; + const tlsError = typeof bunTLS === "function" ? establishedTLSError(data, error) : undefined; + if (tlsError) error = tlsError; + const callback = settlePendingWrite(data, error); if (data._hadError) return; data._hadError = true; - const bunTLS = data[bunTlsSymbol]; if (typeof bunTLS === "function") { - const tlsError = establishedTLSError(data, error); - if (tlsError) error = tlsError; - const callback = data[kwriteCallback]; - if (callback) { - data[kwriteCallback] = null; - callback(error); - } // Mirrors node's onerror: // https://github.com/nodejs/node/blob/v26.3.0/lib/internal/tls/wrap.js#L467-L498 if (!data._secureEstablished) { @@ -1499,11 +1572,6 @@ const ServerHandlers = { // callback, then destroy with the error. destroy() owns the single // 'error' emission via the stream's errorEmitted guard; callback(error) // may have already destroyed, in which case this is a no-op. - const callback = data[kwriteCallback]; - if (callback) { - data[kwriteCallback] = null; - callback(error); - } if (!data.destroyed) { data.destroy(error); } @@ -1825,16 +1893,13 @@ const SocketHandlers2 = { $debug("Bun.Socket error"); if (socket.data === undefined) return; const { self } = socket.data; - if (self._hadError) return; - self._hadError = true; const tlsError = establishedTLSError(self, error); if (tlsError) error = tlsError; + const callback = settlePendingWrite(self, error); + if (self._hadError) return; + self._hadError = true; - const callback = self[kwriteCallback]; - if (callback) { - self[kwriteCallback] = null; - callback(error); - } else if (tlsError) { + if (!callback && tlsError) { // https://github.com/nodejs/node/blob/v26.3.0/lib/internal/tls/wrap.js#L467-L498 (self as TLSSocketInstance)._emitTLSError(error); return; @@ -2025,6 +2090,7 @@ function Socket(options?): void { this._parentWrap = null; this[kupgraded] = null; this[kAdoptedTLSTransport] = false; + this[kNativeTLSTransport] = undefined; this[kStandaloneWrap] = false; this[kOnUpgradedClose] = undefined; this[kOwesRawClose] = false; @@ -2517,6 +2583,7 @@ Socket.prototype._destroy = function _destroy(err, callback) { if (canceledWrite && !err) process.nextTick(cancelWriteNT, canceledWrite); const upgraded = this[kupgraded]; + const token = this[kNativeTLSTransport]; // Close an fd adopted for synchronous writes (node closes the wrapping // libuv handle here). Leave stdio fds 0-2 open: process.stdout/stderr and @@ -2581,14 +2648,14 @@ Socket.prototype._destroy = function _destroy(err, callback) { // Enqueue closing the socket as a microtask, so that the socket can be // accessible when an `error` event is handled in the `next tick queue`. // The native close that follows a failure reported under a JS call detaches _handle before this runs. - queueMicrotask(() => closeSocketHandle(this, currentHandle, isException, true)); + queueMicrotask(() => closeSocketHandle(this, currentHandle, isException, token, true)); } else if (currentHandle[kAdoptedTLSRaw]) { // Shared-fd TLS pair: defer the close two check-phase turns // (test-tls-socket-close); see closeAdoptedTLSRawNT. currentHandle.pause?.(); - setImmediate(closeAdoptedTLSRawNT, this, currentHandle, isException); + setImmediate(closeAdoptedTLSRawNT, this, currentHandle, isException, token); } else { - closeSocketHandle(this, currentHandle, isException); + closeSocketHandle(this, currentHandle, isException, token); } if (!this._closeAfterHandlingError) { @@ -2602,10 +2669,11 @@ Socket.prototype._destroy = function _destroy(err, callback) { } else { finishDestroy(callback, err, canceledWrite); closeOwedRaw(this, upgraded); - const transport = transportToDestroy(this); - if (transport !== undefined) abortTransport(transport); - if (transport instanceof Socket) transport.once("close", emitCloseNT.bind(null, this, err ? true : false)); - else process.nextTick(emitCloseNT, this, err ? true : false); + const isException = err ? true : false; + const onClose = () => { + if (this[kNativeTLSTransport] === token) emitCloseNT(this, isException); + }; + if (!closeTransportAndJoin(this, null, isException, token, onClose)) process.nextTick(onClose); } const server = this._server; @@ -2733,7 +2801,13 @@ Socket.prototype[Symbol.for("::bunUpgradeServerTLS::")] = function (connection, linkUpgraded(this, connection); const options = { data: this, tls, socket: serverHandlersFor(this), isServer: true }; const socket = connection._handle; - if (!socket || connection.encrypted || isNamedPipeSocket(socket) || hasUnflushedWrites(connection)) { + if ( + !socket || + connection.encrypted || + isNamedPipeSocket(socket) || + hasNativeSocketListeners(socket) || + hasUnflushedWrites(connection) + ) { // No adoptable fd (generic Duplex, named pipe, never dialed), TLS over TLS (the fd belongs to the // outer SSL layer), or pending plain writes that must flush first. attachTLSEngine(this, connection, options); @@ -3129,8 +3203,16 @@ Socket.prototype._write = function _write(chunk, encoding, callback) { callback(er); return false; } + // A JS transport can destroy the TLS socket from inside write(). Own its + // callback before entering native code so teardown can cancel it once. + const duplexWrite = this[kupgraded] && !this[kAdoptedTLSTransport]; + if (duplexWrite) this[kwriteCallback] = callback; const res = socket.$write(chunk, encoding); this[kBytesWritten] = socket.bytesWritten; + if (duplexWrite) { + if (this[kwriteCallback] !== callback) return false; + if (res) this[kwriteCallback] = null; + } if (res < 0) { // The kernel rejected the send outright (peer reset): $write returned the // negative errno; deliver it like the EBADF/EPIPE branch above. @@ -3153,7 +3235,7 @@ Socket.prototype._write = function _write(chunk, encoding, callback) { // https://github.com/nodejs/node/blob/843dc5f0d5ad/lib/net.js#L1036 callback(); } - } else if (this[kwriteCallback]) { + } else if (!duplexWrite && this[kwriteCallback]) { callback(new Error("overlapping _write()")); } else { this[kwriteCallback] = callback; @@ -4589,26 +4671,62 @@ function initSocketHandle(self) { // intercepts close on `socket._handle` and invokes it, so always pass one. function onSocketHandleClosed() {} -function closeSocketHandle(self, handle, isException, isCleanupPending = false) { +// Native shutdown bypasses the lower JS Writable: only its recorded EOF and +// completed drain let a finished upper close that exact transport and join its close. +function closeTransportAndJoin(self, handle, isException, token, onClose) { + if (self[kNativeTLSTransport] !== token) return false; + const finishedNative = + token?.native && + token.nativeReadEnded && + token.nativeShutdownCompleted && + self[kupgraded] === token.connection && + (!self._handle || self._handle === token.upperHandle) && + (!handle || handle === token.upperHandle) && + (token.connection._handle === token.lowerHandle || (token.connection.destroyed && !token.connection._handle)) && + self[kSessionAccepted] === true && + self.readableEnded && + self.writableFinished && + !isException && + !self._hadError && + !self._closeAfterHandlingError && + !self.resetAndClosing; + const transport = finishedNative ? token.connection : transportToDestroy(self); + const join = transport instanceof Socket && (finishedNative ? !token.closeEmitted : !handle); + // The existing observer also completes a join requested by an earlier close listener. + // A new once('close') here would miss that in-progress emission. + if (join) { + if (finishedNative) { + if (token.closeComplete === undefined) token.closeComplete = onClose; + } else transport.once("close", onClose); + } + if (transport !== undefined && !transport.destroyed) { + if (finishedNative) transport.destroy(); + else abortTransport(transport); + } + return join; +} + +function closeSocketHandle(self, handle, isException, token, isCleanupPending = false) { $debug("closeSocketHandle", isException, isCleanupPending); - // Ahead of the setImmediate below: the transport's 'close' precedes this socket's. - const transport = transportToDestroy(self); - if (transport !== undefined) abortTransport(transport); + const onClose = () => + setImmediate(() => { + if (self[kNativeTLSTransport] !== token) return; + $debug("emit close", isCleanupPending); + self.emit("close", isException); + if (isCleanupPending) { + // self._handle may have been detached or replaced since. + handle.onread = noop; + if (self._handle === handle) { + self._handle = null; + self._sockname = null; + } + } + }); + const joined = closeTransportAndJoin(self, handle, isException, token, onClose); // The owner of a wrapped socket destroyed it: the TLS socket goes first, so the close of their fd is nothing to report. handle[kAdoptedTLSRaw]?.destroy(); handle.close(onSocketHandleClosed); - setImmediate(() => { - $debug("emit close", isCleanupPending); - self.emit("close", isException); - if (isCleanupPending) { - // self._handle may have been detached or replaced since. - handle.onread = noop; - if (self._handle === handle) { - self._handle = null; - self._sockname = null; - } - } - }); + if (!joined) onClose(); } // Reformat a native listen error to Node's "listen : " diff --git a/src/runtime/api/bun/h2_frame_parser.rs b/src/runtime/api/bun/h2_frame_parser.rs index 6fdc51e902ef..5c326eb8c8db 100644 --- a/src/runtime/api/bun/h2_frame_parser.rs +++ b/src/runtime/api/bun/h2_frame_parser.rs @@ -110,7 +110,7 @@ enum BunSocket { /// that slot is taken, takes a ref on the socket (`*Writeonly`); `detach` / /// `Drop` undo whichever one it was. Readers take a [`BunSocket`] snapshot. #[derive(Default)] -struct NativeSocket(Cell); +struct NativeSocket(Cell, Cell<*mut H2FrameParser>); impl NativeSocket { #[inline] @@ -132,6 +132,7 @@ impl NativeSocket { // rooted by the caller's `socket_js`. let socket_nn = NonNull::new(socket).expect("NewSocket m_ctx"); let socket = bun_ptr::BackRef::from(socket_nn); + self.1.set(h2.as_ptr()); self.0 .set(if socket.attach_native_callback(NativeCallbacks::H2(h2)) { if SSL { @@ -151,9 +152,11 @@ impl NativeSocket { } fn detach(&self) { + let owner = + crate::socket::socket_body::NativeCallbackId::H2(self.1.replace(core::ptr::null_mut())); match self.0.replace(BunSocket::None) { - BunSocket::Tcp(socket) => socket.detach_native_callback(), - BunSocket::Tls(socket) => socket.detach_native_callback(), + BunSocket::Tcp(socket) => socket.remove_native_callback(owner), + BunSocket::Tls(socket) => socket.remove_native_callback(owner), // The ref `attach` took. BunSocket::TcpWriteonly(socket) => TCPSocket::deref(socket.get()), BunSocket::TlsWriteonly(socket) => TLSSocket::deref(socket.get()), @@ -2875,6 +2878,10 @@ impl H2FrameParser { } } + pub(crate) fn native_write_pending(&self) -> bool { + self.has_backpressure() + } + fn has_backpressure(&self) -> bool { self.write_buffer.get().len_u32() > 0 || self.has_nonnative_backpressure.get() } diff --git a/src/runtime/node/node_net_binding.rs b/src/runtime/node/node_net_binding.rs index c5ec94848eea..ae0f536c9cb3 100644 --- a/src/runtime/node/node_net_binding.rs +++ b/src/runtime/node/node_net_binding.rs @@ -9,7 +9,7 @@ use bun_jsc::bindgen::JsScope; use bun_jsc::{self as jsc, CallFrame, JSFunction, JSGlobalObject, JSValue, JsCell, JsResult}; use bun_uws as uws; -use crate::socket::{Listener, NativeCallbacks, NewSocket, SocketFlags, TCPSocket, TLSSocket}; +use crate::socket::{Listener, NewSocket, SocketFlags, TCPSocket, TLSSocket}; static AUTO_SELECT_FAMILY_DEFAULT: AtomicBool = AtomicBool::new(true); @@ -119,7 +119,8 @@ pub(crate) fn new_detached_socket(cx: &mut JsScope<'_>, is_ssl: bool) -> JSValue server_name: JsCell::new(None), buffered_data_for_node_net: Default::default(), bytes_written: Cell::new(0), - native_callback: JsCell::new(NativeCallbacks::None), + native_callbacks: JsCell::new(Vec::new()), + native_write_owner: Cell::new(None), twin: JsCell::new(None), verify_error: JsCell::new(None), latest_session: core::cell::Cell::new(None), diff --git a/src/runtime/socket/Listener.rs b/src/runtime/socket/Listener.rs index 82aae1ae5551..5a61e9a765af 100644 --- a/src/runtime/socket/Listener.rs +++ b/src/runtime/socket/Listener.rs @@ -703,7 +703,8 @@ impl Listener { server_name: JsCell::new(None), buffered_data_for_node_net: Default::default(), bytes_written: Cell::new(0), - native_callback: JsCell::new(crate::socket::NativeCallbacks::None), + native_callbacks: JsCell::new(Vec::new()), + native_write_owner: Cell::new(None), twin: JsCell::new(None), verify_error: JsCell::new(None), latest_session: Cell::new(None), @@ -750,7 +751,8 @@ impl Listener { server_name: JsCell::new(None), buffered_data_for_node_net: Default::default(), bytes_written: Cell::new(0), - native_callback: JsCell::new(crate::socket::NativeCallbacks::None), + native_callbacks: JsCell::new(Vec::new()), + native_write_owner: Cell::new(None), twin: JsCell::new(None), verify_error: JsCell::new(None), latest_session: Cell::new(None), @@ -1280,7 +1282,8 @@ impl Listener { ref_pollref_on_connect: Cell::new(true), buffered_data_for_node_net: Default::default(), bytes_written: Cell::new(0), - native_callback: JsCell::new(crate::socket::NativeCallbacks::None), + native_callbacks: JsCell::new(Vec::new()), + native_write_owner: Cell::new(None), twin: JsCell::new(None), verify_error: JsCell::new(None), latest_session: Cell::new(None), @@ -1374,7 +1377,8 @@ impl Listener { ref_pollref_on_connect: Cell::new(true), buffered_data_for_node_net: Default::default(), bytes_written: Cell::new(0), - native_callback: JsCell::new(crate::socket::NativeCallbacks::None), + native_callbacks: JsCell::new(Vec::new()), + native_write_owner: Cell::new(None), twin: JsCell::new(None), verify_error: JsCell::new(None), latest_session: Cell::new(None), @@ -1619,7 +1623,8 @@ fn connect_finish( ref_pollref_on_connect: Cell::new(true), buffered_data_for_node_net: Default::default(), bytes_written: Cell::new(0), - native_callback: JsCell::new(crate::socket::NativeCallbacks::None), + native_callbacks: JsCell::new(Vec::new()), + native_write_owner: Cell::new(None), twin: JsCell::new(None), verify_error: JsCell::new(None), latest_session: Cell::new(None), diff --git a/src/runtime/socket/UpgradedDuplex.rs b/src/runtime/socket/UpgradedDuplex.rs index cc5a76efd4fb..b549698fce24 100644 --- a/src/runtime/socket/UpgradedDuplex.rs +++ b/src/runtime/socket/UpgradedDuplex.rs @@ -28,7 +28,7 @@ bun_output::declare_scope!(UpgradedDuplex, visible); pub(crate) struct UpgradedDuplex { pub wrapper: JsCell>, /// The owning `JSTLSSocket` wrapper. Its `values:` slots root `origin` and - /// the four listener thunks; the `JSValue` fields below are read-side + /// the listener and completion thunks; the `JSValue` fields below are read-side /// shadows of those slots. pub js_wrapper: JSValue, pub origin: Cell, // any duplex @@ -44,6 +44,14 @@ pub(crate) struct UpgradedDuplex { pub on_end_callback: Cell, pub on_writable_callback: Cell, pub on_close_callback: Cell, + on_write_callback: Cell, + on_write_dispatch: Cell, + on_close_dispatch: Cell, + pending_writes: Cell, + shutdown_queued: Cell, + write_failed: Cell, + transport_call_depth: Cell, + context: bun_jsc::ContextId, pub event_loop_timer: JsCell, pub current_timeout: Cell, /// Transport bytes that arrived before the TLS engine existed. @@ -116,30 +124,35 @@ use crate::jsc_hooks::timer_all_mut as timer_all; /// Lazily create-and-cache a JS host-function callback in `shadow`, mirrored /// into the owning `JSTLSSocket` wrapper's visited `values:` slot (the GC -/// root). All four `get_js_handlers` slots follow the identical -/// `NewFunctionWithData(global, null, 0, fn, self)` → `ensureStillAlive` → -/// `setFunctionData(self)` → store pattern. +/// root). A bound callback traces the owner; the unbound target remains in +/// the shadow so live teardown can neuter it before freeing native storage. #[inline] fn lazy_js_handler( shadow: &Cell, js_wrapper: JSValue, + get_slot: fn(JSValue) -> Option, set_slot: fn(JSValue, &JSGlobalObject, JSValue), global: &JSGlobalObject, func: host_fn::JsHostFn, this_ptr: *mut c_void, -) -> JSValue { +) -> JsResult { if !shadow.get().is_empty() { - return shadow.get(); + return Ok(get_slot(js_wrapper).expect("cached duplex callback")); } let callback = host_fn::new_function_with_data(global, None, 0, func, this_ptr); callback.ensure_still_alive(); - host_fn::set_function_data(callback, Some(this_ptr)); - set_slot(js_wrapper, global, callback); + let bound = callback.bind(global, js_wrapper, &bun_core::String::EMPTY, 0.0, &[])?; + set_slot(js_wrapper, global, bound); shadow.set(callback); - callback + Ok(bound) } impl UpgradedDuplex { + pub(super) fn context_pin(&self) -> bun_ptr::RefPtr { + // The allocation outlives callbacks even if context stop releases its attachment. + unsafe { &*self.handlers.ctx.cast::() }.ref_guard() + } + // SAFETY (all handlers): the SSLWrapper handlers ctx is `self as *mut // Self`, live for the wrapper's lifetime. @@ -220,6 +233,7 @@ impl UpgradedDuplex { } pub(super) fn finish_close(&self) { + let _context = self.context_pin(); // Keep the wrapper (and so its visited `duplex*` slots) reachable // across `handlers.on_close`, which downgrades the socket's own strong // self-reference and re-enters JS. @@ -240,20 +254,43 @@ impl UpgradedDuplex { /// https://github.com/nodejs/node/blob/v26.3.0/lib/internal/js_stream_socket.js#L117-L125 #[uws_callback(export = "UpgradedDuplex__pause_stream")] pub(crate) fn pause_stream(&self) -> bool { - // Like a connecting fd: `on_open` forgets the owner's pause, so nothing would resume this one. - let paused = self.wrapper_ref().is_some() && self.call_origin("pause"); + let context = self.context_pin(); + // Like a connecting fd: on_open forgets the owner's pause, so nothing would resume this one. + if self.wrapper_ref().is_none() { + return false; + } + if context.uses_native_transport() { + // Intent survives displacement; publish it before the lower stream can re-enter. + self.paused_origin.set(true); + return context.reconcile_native_reads(); + } + let paused = self.call_origin("pause"); self.paused_origin.set(self.paused_origin.get() || paused); paused } #[uws_callback(export = "UpgradedDuplex__resume_stream")] pub(crate) fn resume_stream(&self) -> bool { + let context = self.context_pin(); self.paused_origin.set(false); - self.call_origin("resume") + if context.uses_native_transport() { + context.reconcile_native_reads() + } else { + self.call_origin("resume") + } + } + + pub(super) fn native_read_paused(&self) -> bool { + self.paused_origin.get() + } + + pub(super) fn take_native_pause(&self) -> bool { + self.paused_origin.replace(false) } /// `origin[name]()`. A throw goes to `on_error`. False when the call did not complete. fn call_origin(&self, name: &str) -> bool { + let _context = self.context_pin(); let duplex = self.origin.get(); if duplex.is_empty() { return false; @@ -276,6 +313,7 @@ impl UpgradedDuplex { } fn call_write_or_end(&self, data: Option<&[u8]>, msg_more: bool) { + let _context = self.context_pin(); // No JS duplex to talk to: the zeroed placeholder, or the owning // socket's finalizer abandoned it (`abandon_js_side`). let duplex = self.origin.get(); @@ -285,12 +323,9 @@ impl UpgradedDuplex { // global is set in `from()` whenever origin is set. let Some(global) = self.global else { return }; - // Teardown-phase bytes (close_notify / the trailing end()) aimed at a - // duplex whose write side already ended (TLS-inception teardown) only - // surface a spurious EPIPE - drop them. Ordinary data writes skip the - // probe so write-after-end still errors like node. - // A peer's close_notify still leaves our writable half open. - let teardown = data.is_none() || self.is_shutdown(); + // Drop close_notify aimed at an already-ended transport, but still register end's + // completion callback: native owners join that callback before reporting shutdown. + let teardown = data.is_some() && self.is_shutdown(); if teardown { // A readable EOF still permits close_notify while the transport is writable. // Node ends no destroyed stream. @@ -321,26 +356,94 @@ impl UpgradedDuplex { } }; - if let Some(data) = data { + let callback = match lazy_js_handler( + &self.on_write_callback, + self.js_wrapper, + js_TLSSocket::duplex_on_write_get_cached, + js_TLSSocket::duplex_on_write_set_cached, + &global, + __jsc_host_on_write_complete, + std::ptr::from_ref(self).cast_mut().cast(), + ) { + Ok(callback) => callback, + Err(err) => { + self.write_failed.set(true); + (self.handlers.on_error)(self.handlers.ctx, global.take_error(err)); + return; + } + }; + let chunk = if let Some(data) = data { let buffer = match bun_jsc::array_buffer::BinaryType::Buffer.to_js(data, &global) { Ok(b) => b, Err(err) => { + self.write_failed.set(true); (self.handlers.on_error)(self.handlers.ctx, global.take_error(err)); return; } }; - buffer.ensure_still_alive(); + self.pending_writes.set(self.pending_writes.get() + 1); + buffer + } else { + // shutdown() counted this end before queueing it, so a join cannot observe a gap. + JSValue::NULL + }; + self.transport_call_depth + .set(self.transport_call_depth.get() + 1); + let result = write_or_end.call(&global, duplex, &[chunk, callback]); + self.transport_call_depth + .set(self.transport_call_depth.get() - 1); + chunk.ensure_still_alive(); + if let Err(err) = result { + // A synchronous callback may already be queued. Its deferred dispatch owns the count. + self.write_failed.set(true); + (self.handlers.on_error)(self.handlers.ctx, global.take_error(err)); + } + } - if let Err(err) = write_or_end.call(&global, duplex, &[buffer]) { - (self.handlers.on_error)(self.handlers.ctx, global.take_error(err)); - } + pub(super) fn transport_idle(&self) -> bool { + let context = self.context_pin(); + if context.uses_native_transport() { + context.native_idle() } else { - if let Err(err) = write_or_end.call(&global, duplex, &[JSValue::NULL]) { - (self.handlers.on_error)(self.handlers.ctx, global.take_error(err)); - } + self.pending_writes.get() == 0 && !self.write_failed.get() } } + fn queue_write_completion(&self, global: &JSGlobalObject, status: i32) -> JsResult<()> { + let callback = lazy_js_handler( + &self.on_write_dispatch, + self.js_wrapper, + js_TLSSocket::duplex_write_dispatch_get_cached, + js_TLSSocket::duplex_write_dispatch_set_cached, + global, + __jsc_host_dispatch_write_complete, + std::ptr::from_ref(self).cast_mut().cast(), + )?; + // Keep the ciphertext pending through the check phase, even for a synchronous origin callback. + let context = global.bun_vm().context_of(self.context); + crate::timer::All::set_immediate( + &global.js_thread(context), + callback, + JSValue::from(status), + )?; + Ok(()) + } + + fn queue_close(&self, global: &JSGlobalObject) -> JsResult<()> { + let callback = lazy_js_handler( + &self.on_close_dispatch, + self.js_wrapper, + js_TLSSocket::duplex_close_dispatch_get_cached, + js_TLSSocket::duplex_close_dispatch_set_cached, + global, + __jsc_host_dispatch_close, + std::ptr::from_ref(self).cast_mut().cast(), + )?; + let context = global.bun_vm().context_of(self.context); + crate::timer::All::set_immediate(&global.js_thread(context), callback, JSValue::UNDEFINED)?; + Ok(()) + } + fn internal_write(this: *mut Self, encoded_data: &[u8]) { // SAFETY: see handler note above. unsafe { &*this }.write_encrypted(encoded_data); @@ -348,6 +451,11 @@ impl UpgradedDuplex { fn write_encrypted(&self, encoded_data: &[u8]) { self.reset_timeout(); + let context = self.context_pin(); + if context.uses_native_transport() { + context.write_native(encoded_data); + return; + } // Possible scenarios: // Scenario 1: will not write if vm is shutting down (we cannot do anything about it) @@ -359,12 +467,23 @@ impl UpgradedDuplex { #[uws_callback(export = "UpgradedDuplex__flush")] pub(crate) fn flush(&self) { + let _context = self.context_pin(); if let Some(w) = self.wrapper_ref() { let _ = w.flush(); } } + pub(super) fn receive_native_data(&self, data: &[u8]) { + self.on_internal_receive_data(data); + } + + pub(super) fn receive_native_end(&self) { + let _context = self.context_pin(); + (self.handlers.on_end)(self.handlers.ctx); + } + fn on_internal_receive_data(&self, data: &[u8]) { + let _context = self.context_pin(); if let Some(w) = self.wrapper_ref() { self.reset_timeout(); w.receive_data(data); @@ -380,6 +499,7 @@ impl UpgradedDuplex { /// finished its bookkeeping, so the replay is indistinguishable from an /// ordinary post-start delivery. pub(super) fn drain_pending(&self) { + let _context = self.context_pin(); // Nothing to replay, or the engine never came up (the socket died // before `StartTLS`). Bail before taking so the bytes are not // destroyed by a drain that could not deliver them. @@ -407,6 +527,7 @@ impl UpgradedDuplex { } pub(crate) fn on_timeout(&self) { + let _context = self.context_pin(); bun_output::scoped_log!(UpgradedDuplex, "onTimeout"); let has_been_cleared = self.event_loop_timer.get().state == EventLoopTimerState::CANCELLED @@ -431,6 +552,7 @@ impl UpgradedDuplex { js_wrapper: JSValue, origin: JSValue, is_node_socket: bool, + context: bun_jsc::ContextId, handlers: Handlers, ) -> UpgradedDuplex { js_TLSSocket::duplex_origin_set_cached(js_wrapper, global, origin); @@ -446,6 +568,14 @@ impl UpgradedDuplex { on_end_callback: Cell::new(JSValue::ZERO), on_writable_callback: Cell::new(JSValue::ZERO), on_close_callback: Cell::new(JSValue::ZERO), + on_write_callback: Cell::new(JSValue::ZERO), + on_write_dispatch: Cell::new(JSValue::ZERO), + on_close_dispatch: Cell::new(JSValue::ZERO), + pending_writes: Cell::new(0), + shutdown_queued: Cell::new(false), + write_failed: Cell::new(false), + transport_call_depth: Cell::new(0), + context, event_loop_timer: JsCell::new(EventLoopTimer::init_paused( EventLoopTimerTag::UpgradedDuplex, )), @@ -470,11 +600,12 @@ impl UpgradedDuplex { lazy_js_handler( &self.on_data_callback, js_wrapper, + js_TLSSocket::duplex_on_data_get_cached, js_TLSSocket::duplex_on_data_set_cached, global, __jsc_host_on_received_data, this_ptr, - ), + )?, )?; array.put_index( global, @@ -482,11 +613,12 @@ impl UpgradedDuplex { lazy_js_handler( &self.on_end_callback, js_wrapper, + js_TLSSocket::duplex_on_end_get_cached, js_TLSSocket::duplex_on_end_set_cached, global, __jsc_host_on_end, this_ptr, - ), + )?, )?; array.put_index( global, @@ -494,11 +626,12 @@ impl UpgradedDuplex { lazy_js_handler( &self.on_writable_callback, js_wrapper, + js_TLSSocket::duplex_on_writable_get_cached, js_TLSSocket::duplex_on_writable_set_cached, global, __jsc_host_on_writable, this_ptr, - ), + )?, )?; array.put_index( global, @@ -506,11 +639,12 @@ impl UpgradedDuplex { lazy_js_handler( &self.on_close_callback, js_wrapper, + js_TLSSocket::duplex_on_close_get_cached, js_TLSSocket::duplex_on_close_set_cached, global, __jsc_host_on_close_js, this_ptr, - ), + )?, )?; Ok(array) @@ -532,6 +666,7 @@ impl UpgradedDuplex { } fn install_and_start(&self, wrapper: WrapperType, verify: ServerVerify) { + let _context = self.context_pin(); self.wrapper.set(Some(wrapper)); let w = self.wrapper_ref().unwrap(); w.set_server_verify(verify.request_cert, verify.reject_unauthorized); @@ -565,6 +700,7 @@ impl UpgradedDuplex { #[uws_callback(export = "UpgradedDuplex__encode_and_write")] pub(crate) fn encode_and_write(&self, data: &[u8]) -> i32 { + let _context = self.context_pin(); bun_output::scoped_log!(UpgradedDuplex, "encodeAndWrite (len: {})", data.len()); if let Some(w) = self.wrapper_ref() { let written = w.write_data(data).unwrap_or(0); @@ -575,6 +711,7 @@ impl UpgradedDuplex { #[uws_callback(export = "UpgradedDuplex__raw_write")] pub(crate) fn raw_write(&self, encoded_data: &[u8]) -> i32 { + let _context = self.context_pin(); self.write_encrypted(encoded_data); i32::try_from(encoded_data.len()).expect("int cast") } @@ -585,14 +722,33 @@ impl UpgradedDuplex { /// native side. #[uws_callback(export = "UpgradedDuplex__abandon_js_side", no_catch)] pub(crate) fn abandon_js_side(&self) { + let context = self.context_pin(); + context.unlink_native(true); self.origin.set(JSValue::ZERO); + // Finalization may follow collection of the targets. Do not inspect their JS cells. + for callback in [ + &self.on_data_callback, + &self.on_end_callback, + &self.on_writable_callback, + &self.on_close_callback, + &self.on_write_callback, + &self.on_write_dispatch, + &self.on_close_dispatch, + ] { + callback.set(JSValue::ZERO); + } } #[uws_callback(export = "UpgradedDuplex__close")] pub(crate) fn close(&self) { + let _context = self.context_pin(); let Some(w) = self.wrapper_ref() else { - // `start_tls` is still queued: no SSL, and no close callback yet. - self.pending_close.set(true); + if _context.uses_native_transport() { + // Native termination can suppress StartTLS; explicit upper close still owns teardown. + self.finish_close(); + } else { + self.pending_close.set(true); + } return; }; let _ = w.shutdown(true); @@ -601,15 +757,21 @@ impl UpgradedDuplex { /// Half-close like `us_internal_ssl_shutdown`: close_notify (none mid-handshake), then end(). #[uws_callback(export = "UpgradedDuplex__shutdown")] pub(crate) fn shutdown(&self) { + let _context = self.context_pin(); let Some(w) = self.wrapper_ref() else { self.pending_shutdown.set(true); return; }; let _ = w.shutdown(false); - if self.origin.get().is_empty() { + if _context.uses_native_transport() { + _context.shutdown_native(); + return; + } + if self.origin.get().is_empty() || self.shutdown_queued.replace(true) { return; } let Some(global) = self.global else { return }; + self.pending_writes.set(self.pending_writes.get() + 1); let callback = bun_jsc::JSFunction::create( &global, "", @@ -618,12 +780,14 @@ impl UpgradedDuplex { Default::default(), ); if let Err(err) = JSValue::call_next_tick_1(callback, &global, self.js_wrapper) { + self.write_failed.set(true); (self.handlers.on_error)(self.handlers.ctx, global.take_error(err)); } } #[uws_callback(export = "UpgradedDuplex__shutdown_read")] pub(crate) fn shutdown_read(&self) { + let _context = self.context_pin(); if let Some(w) = self.wrapper_ref() { w.shutdown_read(); } @@ -709,6 +873,10 @@ impl UpgradedDuplex { /// are neutered while the wrapper is still strongly reachable. pub(super) fn teardown(&self) { bun_output::scoped_log!(UpgradedDuplex, "deinit"); + if !self.handlers.ctx.is_null() { + let context = unsafe { &*self.handlers.ctx.cast::() }; + context.teardown_native(self.origin.get().is_empty()); + } // clear the timer self.set_timeout(0); @@ -737,6 +905,9 @@ impl UpgradedDuplex { &self.on_end_callback, &self.on_writable_callback, &self.on_close_callback, + &self.on_write_callback, + &self.on_write_dispatch, + &self.on_close_dispatch, ] { let value = cb.get(); if !value.is_empty() { @@ -748,6 +919,9 @@ impl UpgradedDuplex { self.pending_data.set(Vec::new()); self.pending_close.set(false); self.pending_shutdown.set(false); + self.pending_writes.set(0); + self.shutdown_queued.set(false); + self.write_failed.set(false); } } @@ -787,6 +961,7 @@ fn on_received_data(global: &JSGlobalObject, frame: &CallFrame) -> JsResult() }; + let _context = this.context_pin(); if frame.arguments_count() >= 1 { if !this.origin.get().is_empty() { if data_arg.is_empty_or_undefined_or_null() { @@ -821,6 +996,7 @@ fn on_end(_global: &JSGlobalObject, frame: &CallFrame) -> JsResult { if let Some(self_ptr) = host_fn::get_function_data(function) { // SAFETY: see host-fn note above. let this = unsafe { &*self_ptr.cast::() }; + let _context = this.context_pin(); // Like node's JSStreamSocket. Ahead of staged bytes too: no handshake can complete after it. (this.handlers.on_end)(this.handlers.ctx); @@ -837,17 +1013,92 @@ fn on_writable(_global: &JSGlobalObject, frame: &CallFrame) -> JsResult if let Some(self_ptr) = host_fn::get_function_data(function) { // SAFETY: see host-fn note above. let this = unsafe { &*self_ptr.cast::() }; - // flush pending data - this.flush(); - // call onWritable (will flush on demand) - (this.handlers.on_writable)(this.handlers.ctx); + let _context = this.context_pin(); + // An origin may emit drain inside write/end; the current TLS flush still owns its input. + if this.transport_call_depth.get() == 0 && !this.write_failed.get() { + this.flush(); + (this.handlers.on_writable)(this.handlers.ctx); + } } Ok(JSValue::UNDEFINED) } #[bun_jsc::host_fn] -fn on_close_js(_global: &JSGlobalObject, frame: &CallFrame) -> JsResult { +fn on_write_complete(global: &JSGlobalObject, frame: &CallFrame) -> JsResult { + if let Some(pointer) = host_fn::get_function_data(frame.callee()) { + // The bound callback roots its owner; live teardown neuters this target. + let this = unsafe { &*pointer.cast::() }; + let _context = this.context_pin(); + let [error] = frame.arguments_as_array::<1>(); + let queued = (|| -> JsResult<()> { + // Snapshot before yielding: an origin error listener can mutate err.code. + let mut status = 0; + if !error.is_empty() && error.to_boolean() { + status = -bun_errno::uv_e::PIPE; + if let Some(code) = error.get(global, "code")? { + let code = code.to_bun_string(global)?; + if let Some((value, _)) = bun_errno::uv_e::ENTRIES + .iter() + .find(|(_, name)| code.eq_ascii(name.as_bytes())) + { + status = *value; + } + } + } + // Reading err.code can re-enter destroy and neuter every target. + if this.origin.get().is_empty() { + return Ok(()); + } + this.queue_write_completion(global, status) + })(); + if let Err(err) = queued { + // No dispatcher owns this completion. Leave it failed until teardown, never idle. + this.write_failed.set(true); + if matches!(err, bun_jsc::JsError::Terminated) { + return Err(err); + } + (this.handlers.on_error)(this.handlers.ctx, global.take_error(err)); + } + } + Ok(JSValue::UNDEFINED) +} + +#[bun_jsc::host_fn] +fn dispatch_write_complete(global: &JSGlobalObject, frame: &CallFrame) -> JsResult { + let Some(pointer) = host_fn::get_function_data(frame.callee()) else { + return Ok(JSValue::UNDEFINED); + }; + // The queued bound dispatcher roots the owner until the numeric completion is delivered. + let this = unsafe { &*pointer.cast::() }; + let _context = this.context_pin(); + if this.pending_writes.get() == 0 { + return Ok(JSValue::UNDEFINED); + } + this.pending_writes.set(this.pending_writes.get() - 1); + let [status] = frame.arguments_as_array::<1>(); + let status = status.to_int32(); + if status != 0 { + if !this.write_failed.replace(true) { + let name = bun_errno::uv_e::name(status).expect("mapped write errno"); + let error = bun_jsc::SystemError { + errno: status, + code: bun_core::String::static_(name), + message: bun_core::String::clone_utf8(format!("write {name}").as_bytes()), + syscall: bun_core::String::static_("write"), + ..Default::default() + } + .to_error_instance(global); + (this.handlers.on_error)(this.handlers.ctx, error); + } + } else if this.transport_idle() && !this.origin.get().is_empty() { + (this.handlers.on_writable)(this.handlers.ctx); + } + Ok(JSValue::UNDEFINED) +} + +#[bun_jsc::host_fn] +fn on_close_js(global: &JSGlobalObject, frame: &CallFrame) -> JsResult { bun_output::scoped_log!(UpgradedDuplex, "onCloseJS"); let function = frame.callee(); @@ -855,13 +1106,31 @@ fn on_close_js(_global: &JSGlobalObject, frame: &CallFrame) -> JsResult if let Some(self_ptr) = host_fn::get_function_data(function) { // SAFETY: see host-fn note above. let this = unsafe { &*self_ptr.cast::() }; - // flush pending data - this.close(); + let _context = this.context_pin(); + // Preserve earlier queued write completions before closing this JS-driven transport. + if let Err(err) = this.queue_close(global) { + this.write_failed.set(true); + if matches!(err, bun_jsc::JsError::Terminated) { + return Err(err); + } + (this.handlers.on_error)(this.handlers.ctx, global.take_error(err)); + } } Ok(JSValue::UNDEFINED) } +#[bun_jsc::host_fn] +fn dispatch_close(_global: &JSGlobalObject, frame: &CallFrame) -> JsResult { + if let Some(pointer) = host_fn::get_function_data(frame.callee()) { + // Live teardown neuters the target; the bound dispatcher keeps its owner traced. + let this = unsafe { &*pointer.cast::() }; + let _context = this.context_pin(); + this.close(); + } + Ok(JSValue::UNDEFINED) +} + // ────────────────────────────────────────────────────────────────────────── // `bun_uws::UpgradedDuplex` link-time-dispatch shims (cycle break). // diff --git a/src/runtime/socket/socket_body.bind.ts b/src/runtime/socket/socket_body.bind.ts index 0a4c855be8e7..6c7f8040b898 100644 --- a/src/runtime/socket/socket_body.bind.ts +++ b/src/runtime/socket/socket_body.bind.ts @@ -19,4 +19,19 @@ export const releaseHeldFlight = b.fn({ args: { socket: SocketHandle } }); export const checkServerIdentityInHandshake = b.fn({ args: { socket: SocketHandle, inHandshake: b.bool } }); export const firstFlightBeforeFin = b.fn({ args: { socket: SocketHandle } }); export const isNamedPipeSocket = b.fn({ args: { socket: SocketHandle }, returns: b.bool }); +export const hasNativeSocketListeners = b.fn({ args: { socket: SocketHandle }, returns: b.bool }); export const getBufferedAmount = b.fn({ args: { socket: SocketHandle }, returns: b.usize }); + +// Internal StreamBase ownership for node:tls; generic Duplexes retain JSStreamSocket. +export const attachNativeTLSTransport = b.fn({ + args: { socket: TLSSocket, transport: SocketHandle, previousWrites: b.bool, initialData: b.any, token: b.any }, + returns: b.bool, + capability: "js", + throws: true, +}); +export const nativeTLSPreviousWritesDone = b.fn({ args: { socket: TLSSocket } }); +export const shutdownNativeTLSTransport = b.fn({ + args: { socket: SocketHandle, callback: b.any }, + returns: b.bool, + capability: "js", +}); diff --git a/src/runtime/socket/socket_body.rs b/src/runtime/socket/socket_body.rs index 16a903dc802b..a57d3a421b6f 100644 --- a/src/runtime/socket/socket_body.rs +++ b/src/runtime/socket/socket_body.rs @@ -349,7 +349,9 @@ pub(crate) struct NewSocket { pub(crate) buffered_data_for_node_net: JsCell>, pub(crate) bytes_written: Cell, - pub(crate) native_callback: JsCell, + // Ordered StreamListener ownership: a displaced H2/TLS listener remains removable. + pub(crate) native_callbacks: JsCell>>, + pub(crate) native_write_owner: Cell>, /// `upgradeTLS` produces two `TLSSocket` wrappers over one /// `us_socket_t` (the encrypted view + the raw-bytes view node:net /// expects at index 0). The encrypted half holds a ref on the raw half @@ -595,19 +597,254 @@ impl NewSocket { /// On `false` the rejected `callback` (and the ref it holds) is dropped. pub(crate) fn attach_native_callback(&self, callback: NativeCallbacks) -> bool { - if !matches!(self.native_callback.get(), NativeCallbacks::None) { + if self.socket.get().is_detached() { return false; } - self.native_callback.set(callback); + self.native_callbacks + .with_mut(|entries| entries.push(Some(callback))); true } - pub(crate) fn detach_native_callback(&self) { - if let NativeCallbacks::H2(h2) = self.native_callback.replace(NativeCallbacks::None) { - h2.on_native_close(); + fn attach_native_duplex( + &self, + global: &JSGlobalObject, + context: &DuplexUpgradeContext, + ) -> JsResult<()> { + let wrapper = self.get_this_value(global); + let roots = match self.native_attachment_roots() { + Some(roots) => roots, + None => JSValue::create_empty_array(global, 0)?, + }; + roots.ensure_still_alive(); + let index = self.native_callbacks.get().len(); + // Allocate/root before publishing the BackRef. Removal leaves a hole, never shifts identities. + self.native_callbacks.with_mut(|entries| entries.reserve(1)); + roots.put_index(global, index as u32, context.upgrade.js_wrapper)?; + if SSL { + js_TLSSocket::native_attachments_set_cached(wrapper, global, roots); + } else { + js_TCPSocket::native_attachments_set_cached(wrapper, global, roots); + } + if self.native_write_owner.get().is_none() { + if let Some(previous) = self.native_callback() { + if previous.write_pending() { + self.native_write_owner.set(Some(previous.id())); + } + } + } + self.native_callbacks.with_mut(|entries| { + entries.push(Some(NativeCallbacks::Duplex(bun_ptr::BackRef::new( + context, + )))) + }); + Ok(()) + } + + fn native_transport_error(&self) -> i32 { + let socket = self.socket.get(); + if socket.is_detached() || socket.is_closed() { + return sys::SystemErrno::ECANCELED as i32; + } + if SSL { + if let Some(raw) = socket.socket.get() { + // Same fatal-error producer as write_check_error; these are read-only observations. + let fatal = unsafe { us_internal_ssl_fatal_write_error(raw) }; + if fatal != 0 { + return native_write_errno(fatal); + } + if unsafe { us_socket_ssl_spill_dropped(raw) } != 0 { + return sys::SystemErrno::EPIPE as i32; + } + } + } + 0 + } + + fn native_transport_drained(&self) -> bool { + if self.native_transport_error() != 0 { + return false; + } + let spill = if SSL { + self.socket.get().socket.get().map_or(0, |socket| { + // The live socket is pinned by NativeTransport. This observes, never changes, fd policy. + unsafe { us_socket_ssl_spill_pending(socket) } + }) + } else { + 0 + }; + spill == 0 && self.buffered_data_for_node_net.get().is_empty() && self.transport_idle() + } + + fn native_transport_write(&self, owner: NativeCallbackId, data: &[u8]) -> i32 { + self.native_write_owner.set(Some(owner)); + self.write_maybe_corked(data) + } + + fn native_callback(&self) -> Option { + self.native_callbacks + .get() + .iter() + .rev() + .flatten() + .next() + .map(NativeCallbacks::pin) + } + + fn native_callback_by_id(&self, id: NativeCallbackId) -> Option { + self.native_callbacks + .get() + .iter() + .flatten() + .find(|entry| entry.id() == id) + .map(NativeCallbacks::pin) + } + + fn native_attachment_roots(&self) -> Option { + let wrapper = self.this_value.get().try_get()?; + if SSL { + js_TLSSocket::native_attachments_get_cached(wrapper) + } else { + js_TCPSocket::native_attachments_get_cached(wrapper) + } + } + + fn clear_native_attachment_root(&self, index: usize) { + // Finalizers must not inspect an array whose cells may already have been swept. + if self.flags.get().contains(Flags::FINALIZING) || !self.has_handlers() { + return; + } + if let Some(roots) = self.native_attachment_roots() { + let global = self.get_handlers().global_object; + crate::dispatch::fold(roots.put_index(&global, index as u32, JSValue::UNDEFINED)); + } + } + + fn unlink_native_duplex(&self, id: NativeCallbackId, finalizing: bool) { + if finalizing { + // No JS cell access from a collected upper wrapper's finalizer. + self.native_callbacks.with_mut(|entries| { + if let Some(entry) = entries + .iter_mut() + .find(|entry| entry.as_ref().is_some_and(|entry| entry.id() == id)) + { + *entry = None; + } + while entries.last().is_some_and(Option::is_none) { + entries.pop(); + } + }); + if self.native_write_owner.get() == Some(id) { + self.native_write_owner.set(None); + } + } else { + self.remove_native_callback(id); + } + } + + pub(crate) fn remove_native_callback(&self, id: NativeCallbackId) { + let active = self.native_callback().filter(|owner| owner.id() == id); + let wrapper = active.as_ref().and_then(NativeDispatch::js_owner); + let resume_if_empty = active.as_ref().is_some_and(NativeDispatch::read_paused); + let removed = self.native_callbacks.with_mut(|entries| { + let index = entries + .iter() + .position(|entry| entry.as_ref().is_some_and(|entry| entry.id() == id))?; + let entry = entries[index].take(); + while entries.last().is_some_and(Option::is_none) { + entries.pop(); + } + Some((index, entry)) + }); + if let Some((index, entry)) = removed { + if self.native_write_owner.get() == Some(id) { + self.native_write_owner.set(None); + } + self.clear_native_attachment_root(index); + // Drop the native owner only after unlinking; it can re-enter this list. + drop(entry); + if active.is_some() { + self.reconcile_native_reads(resume_if_empty); + } + } + if let Some(wrapper) = wrapper { + wrapper.ensure_still_alive(); + } + } + + fn reconcile_native_reads(&self, mut resume_if_empty: bool) -> bool { + let _pin = self.ref_guard(); + loop { + if self.socket.get().is_detached() || self.flags.get().contains(Flags::FINALIZING) { + return false; + } + let owner = self.native_callback(); + let target = owner + .as_ref() + .map(|owner| (owner.id(), owner.read_paused())); + let wrapper = owner.as_ref().and_then(NativeDispatch::js_owner); + let applied = match target { + Some((_, true)) => { + resume_if_empty = true; + self.pause_reads() + } + Some((_, false)) => self.resume_reads(), + None if resume_if_empty => self.resume_reads(), + None => return false, + }; + if let Some(wrapper) = wrapper { + wrapper.ensure_still_alive(); + } + // A generic lower stream can change the listener or its intent inside pause/resume. + // Correct the outer IS_PAUSED write only when that authoritative target changed. + let current = self + .native_callback() + .map(|owner| (owner.id(), owner.read_paused())); + if current == target { + return applied; + } + } + } + + fn reconcile_native_reader(&self, id: NativeCallbackId) -> bool { + if self.native_callback().is_some_and(|owner| owner.id() == id) { + self.reconcile_native_reads(false) + } else { + // Buried owners retain their intent without changing the active reader. + self.native_callback_by_id(id).is_some() } } + fn close_native_callbacks(&self) { + // Pop before entering the owner. Reentrant close may remove a buried entry. + while let Some((index, callback)) = self.native_callbacks.with_mut(|entries| { + let index = entries.iter().rposition(Option::is_some)?; + let entry = entries[index].take().unwrap(); + while entries.last().is_some_and(Option::is_none) { + entries.pop(); + } + Some((index, entry)) + }) { + let dispatch = callback.pin(); + let finalizing = self.flags.get().contains(Flags::FINALIZING); + let wrapper = if finalizing { + None + } else { + dispatch.js_owner() + }; + self.clear_native_attachment_root(index); + drop(callback); + dispatch.on_close(finalizing); + if let Some(wrapper) = wrapper { + wrapper.ensure_still_alive(); + } + } + self.native_write_owner.set(None); + } + + pub(crate) fn detach_native_callback(&self) { + self.close_native_callbacks(); + } + /// Connect to `self.connection` (must be `Some`). Reads the field directly /// rather than taking it by-ref so the single caller in `connect_finish` /// doesn't need a disjoint borrow. @@ -739,21 +976,26 @@ impl NewSocket { _frame: &CallFrame, ) -> JsResult { jsc::mark_binding!(); - if this.socket.get().is_detached() { - return Ok(JSValue::UNDEFINED); - } log!("resume"); + this.resume_reads(); + Ok(JSValue::UNDEFINED) + } + + fn resume_reads(&self) -> bool { + if self.socket.get().is_detached() { + return false; + } // The raw half of an upgradeTLS pair is an observation tap; flow // control belongs to the TLS half. Pausing the shared fd here would // wedge the TLS read path (#15438). - if this.flags.get().contains(Flags::BYPASS_TLS) { - return Ok(JSValue::UNDEFINED); + if self.flags.get().contains(Flags::BYPASS_TLS) { + return false; } - if this.flags.get().contains(Flags::IS_PAUSED) { - let resumed = this.socket.get().resume_stream(); - this.update_flags(|f| f.set(Flags::IS_PAUSED, !resumed)); + if self.flags.get().contains(Flags::IS_PAUSED) { + let resumed = self.socket.get().resume_stream(); + self.update_flags(|f| f.set(Flags::IS_PAUSED, !resumed)); } - Ok(JSValue::UNDEFINED) + !self.flags.get().contains(Flags::IS_PAUSED) } #[bun_jsc::host_fn(method)] @@ -774,11 +1016,15 @@ impl NewSocket { Ok(JSValue::UNDEFINED) } - fn pause_reads(&self) { + fn pause_reads(&self) -> bool { + if self.socket.get().is_detached() || self.flags.get().contains(Flags::BYPASS_TLS) { + return false; + } if !self.flags.get().contains(Flags::IS_PAUSED) { let paused = self.socket.get().pause_stream(); self.update_flags(|f| f.set(Flags::IS_PAUSED, paused)); } + self.flags.get().contains(Flags::IS_PAUSED) } /// Consumes `PAUSE_ON_CONNECT`: a TLS client dispatches another handshake after a renegotiation. @@ -941,6 +1187,41 @@ impl NewSocket { called } + fn dispatch_native_writable(&self) -> bool { + let mut delivered = None; + if let Some(id) = self.native_write_owner.get() { + if let Some(previous) = self.native_callback_by_id(id) { + previous.on_writable(); + delivered = Some(id); + if self.native_write_owner.get() == Some(id) { + if previous.write_pending() { + return true; + } + self.native_write_owner.set(None); + } + if self.socket.get().is_detached() { + return true; + } + } else { + self.native_write_owner.set(None); + } + } + let Some(owner) = self.native_callback() else { + return delivered.is_some(); + }; + if delivered == Some(owner.id()) { + return true; + } + if let NativeDispatch::Duplex(context) = &owner { + // The old net.Socket write must still reach its own JS drain and empty-write barrier. + if context.native_previous_js_write.get() { + return false; + } + } + owner.on_writable(); + true + } + /// Takes `ThisPtr`, not `&mut self`: `callback.call(...)` re-enters /// JS which can call `socket.write()`/`end()`/`reload()` on this same /// wrapper via the JS object's `m_ptr`, re-deriving a borrow and mutating @@ -963,18 +1244,24 @@ impl NewSocket { if this.socket.get().is_detached() { return Ok(()); } - if this.native_callback.get().on_writable() { + // Native HTTP2 and transport flush callbacks can re-enter and detach the socket. + let _guard = RefPtr::from_this(this); + let native_handled = this.dispatch_native_writable(); + if !this.has_handlers() || this.socket.get().is_detached() { return Ok(()); } - let handlers = this.get_handlers(); - let callback = handlers.on_writable(); - if callback.is_empty() { + // H2 flushes its own queue, but a JS transport can still own net's write callback. + if native_handled + && (matches!(this.native_callback(), Some(NativeDispatch::Duplex(_))) + || !matches!( + this.socket.get().socket, + uws::InternalSocket::UpgradedDuplex(_) + )) + { return Ok(()); } - - // Hold the socket alive for the rest of the dispatch: `internal_flush` - // and the drain callback can both re-enter JS and close it. - let _guard = RefPtr::from_this(this); + let handlers = this.get_handlers(); + let callback = handlers.on_writable(); // NOTE (Windows): the drain dispatch deliberately does not depend on // whether the flush hit a fatal send error. Skipping it on fatal // (tried in f0325bddf2) made Windows servers reset FIN-terminated @@ -1019,7 +1306,11 @@ impl NewSocket { this.buffered_data_for_node_net.get().len() ); // is not writable if we have buffered data or if we are already detached - if this.buffered_data_for_node_net.get().len() > 0 || this.socket.get().is_detached() { + if this.buffered_data_for_node_net.get().len() > 0 + || this.socket.get().is_detached() + || !this.transport_idle() + || callback.is_empty() + { return Ok(()); } @@ -1757,6 +2048,9 @@ impl NewSocket { if this.socket.get().is_detached() { return Ok(()); } + if this.native_callback().is_some_and(|owner| owner.on_end()) { + return Ok(()); + } let handlers = this.get_handlers(); log!( "onEnd {}", @@ -2215,8 +2509,8 @@ impl NewSocket { // dispatches either. mark_inactive is not needed: handlers being // null means the previous teardown already ran it. if !this.has_handlers() { - this.detach_native_callback(); this.socket.set(SocketHandler::::DETACHED); + this.detach_native_callback(); this.get().deref(); return Ok(()); } @@ -2229,8 +2523,8 @@ impl NewSocket { "C" } ); - this.detach_native_callback(); this.socket.set(SocketHandler::::DETACHED); + this.close_native_callbacks(); // Declared first so it drops last: ticks drain ahead of the teardown. let _cleanup = CloseTeardown { socket: this, @@ -2322,7 +2616,8 @@ impl NewSocket { if this.socket.get().is_detached() { return Ok(()); } - if this.native_callback.get().on_data(data)? { + if let Some(owner) = this.native_callback() { + owner.on_data(data)?; return Ok(()); } let handlers = this.get_handlers(); @@ -2730,7 +3025,9 @@ impl NewSocket { // node:net fails the write like Node's onWriteComplete; // -1 stays the legacy closed/shutdown sentinel. JSValue::js_number(f64::from(wrote)) - } else if usize::try_from(wrote.max(0)).expect("int cast") == total { + } else if usize::try_from(wrote.max(0)).expect("int cast") == total + && this.transport_idle() + { JSValue::TRUE } else { JSValue::FALSE @@ -2762,7 +3059,10 @@ impl NewSocket { let _ = this.internal_flush(); } - JSValue::from(usize::try_from(wrote.max(0)).expect("int cast") == total) + JSValue::from( + usize::try_from(wrote.max(0)).expect("int cast") == total + && this.transport_idle(), + ) } }; Ok(result) @@ -3147,12 +3447,23 @@ impl NewSocket { false } + fn transport_idle(&self) -> bool { + match self.socket.get().socket { + uws::InternalSocket::UpgradedDuplex(pointer) => { + // Installed by DuplexUpgradeContext::duplex_socket and live until detach. + unsafe { &*pointer.cast::() }.transport_idle() + } + _ => true, + } + } + fn can_end_after_flush(&self) -> bool { let flags = self.flags.get(); flags.contains(Flags::IS_ACTIVE) && flags.contains(Flags::END_AFTER_FLUSH) && !flags.contains(Flags::EMPTY_PACKET_PENDING) && self.buffered_data_for_node_net.get().len() == 0 + && self.transport_idle() } /// Flushes the node:net buffered tail. Returns 0, or the positive errno of @@ -3641,7 +3952,8 @@ impl NewSocket { ref_pollref_on_connect: Cell::new(true), buffered_data_for_node_net: JsCell::new(Vec::new()), bytes_written: Cell::new(0), - native_callback: JsCell::new(NativeCallbacks::None), + native_callbacks: JsCell::new(Vec::new()), + native_write_owner: Cell::new(None), twin: JsCell::new(None), verify_error: JsCell::new(None), latest_session: Cell::new(None), @@ -3766,7 +4078,8 @@ impl NewSocket { ref_pollref_on_connect: Cell::new(true), buffered_data_for_node_net: JsCell::new(Vec::new()), bytes_written: Cell::new(0), - native_callback: JsCell::new(NativeCallbacks::None), + native_callbacks: JsCell::new(Vec::new()), + native_write_owner: Cell::new(None), twin: JsCell::new(None), verify_error: JsCell::new(None), latest_session: Cell::new(None), @@ -4138,39 +4451,234 @@ impl_socket_js_class!(TCPSocket, js_TCPSocket); impl_socket_js_class!(TLSSocket, js_TLSSocket); // ────────────────────────────────────────────────────────────────────────── -// NativeCallbacks — direct callbacks on HTTP2 when available +// NativeCallbacks — ordered TLS and HTTP/2 stream listeners // ────────────────────────────────────────────────────────────────────────── +unsafe extern "C" { + fn us_socket_ssl_spill_pending(socket: *mut uws::us_socket_t) -> c_uint; + fn us_socket_ssl_spill_dropped(socket: *mut uws::us_socket_t) -> c_int; + fn us_internal_ssl_fatal_write_error(socket: *mut uws::us_socket_t) -> c_int; +} + +fn native_write_errno(code: c_int) -> i32 { + if code == -1 { + return sys::SystemErrno::EPIPE as i32; + } + #[cfg(windows)] + { + // C writes report WSA or UV codes; Error stores SystemErrno discriminants. + sys::SystemErrno::init(code).unwrap_or(sys::SystemErrno::EIO) as i32 + } + #[cfg(not(windows))] + { + code.abs() + } +} + +#[derive(Clone)] +enum NativeTransport { + Tcp(RefPtr), + Tls(RefPtr), +} + +impl NativeTransport { + fn attach(&self, global: &JSGlobalObject, context: &DuplexUpgradeContext) -> JsResult<()> { + match self { + Self::Tcp(socket) => socket.attach_native_duplex(global, context), + Self::Tls(socket) => socket.attach_native_duplex(global, context), + } + } + fn remove(&self, owner: NativeCallbackId, finalizing: bool) { + match self { + Self::Tcp(socket) => socket.unlink_native_duplex(owner, finalizing), + Self::Tls(socket) => socket.unlink_native_duplex(owner, finalizing), + } + } + fn previous_write_pending(&self, owner: NativeCallbackId) -> bool { + let pending = match self { + Self::Tcp(socket) => socket.native_write_owner.get(), + Self::Tls(socket) => socket.native_write_owner.get(), + }; + pending.is_some_and(|pending| pending != owner) + } + fn write(&self, owner: NativeCallbackId, data: &[u8]) -> i32 { + match self { + Self::Tcp(socket) => socket.native_transport_write(owner, data), + Self::Tls(socket) => socket.native_transport_write(owner, data), + } + } + fn error(&self) -> i32 { + match self { + Self::Tcp(socket) => socket.native_transport_error(), + Self::Tls(socket) => socket.native_transport_error(), + } + } + fn drained(&self) -> bool { + match self { + Self::Tcp(socket) => socket.native_transport_drained(), + Self::Tls(socket) => socket.native_transport_drained(), + } + } + fn clear_writer(&self, owner: NativeCallbackId) { + match self { + Self::Tcp(socket) => { + if socket.native_write_owner.get() == Some(owner) { + socket.native_write_owner.set(None); + } + } + Self::Tls(socket) => { + if socket.native_write_owner.get() == Some(owner) { + socket.native_write_owner.set(None); + } + } + } + } + fn flush(&self) { + match self { + Self::Tcp(socket) => socket.socket.get().flush(), + Self::Tls(socket) => socket.socket.get().flush(), + } + } + fn shutdown(&self) { + match self { + Self::Tcp(socket) => socket.socket.get().shutdown(), + Self::Tls(socket) => socket.socket.get().shutdown(), + } + } + fn reconcile_reads(&self, owner: NativeCallbackId) -> bool { + match self { + Self::Tcp(socket) => socket.reconcile_native_reader(owner), + Self::Tls(socket) => socket.reconcile_native_reader(owner), + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum NativeCallbackId { + H2(*mut H2FrameParser), + Duplex(*mut DuplexUpgradeContext), +} + pub(crate) enum NativeCallbacks { H2(RefPtr), - None, + // The context unlinks before releasing its attachment reference. Never a native cycle. + Duplex(bun_ptr::BackRef), } impl NativeCallbacks { - /// `&self` borrows the socket's `JsCell`; the dispatch - /// re-enters JS, which can `detach_native_callback` and overwrite that cell. - /// Copy the parser pointer out first; the callee's `keepalive()` holds it. - /// `Ok(false)`: nothing attached, dispatch to the JS handlers. - pub(crate) fn on_data(&self, data: &[u8]) -> JsResult { - let h2 = match self { - NativeCallbacks::H2(h2) => h2.as_ptr(), - NativeCallbacks::None => return Ok(false), + fn id(&self) -> NativeCallbackId { + match self { + Self::H2(h2) => NativeCallbackId::H2(h2.as_ptr()), + Self::Duplex(context) => { + NativeCallbackId::Duplex(ptr::from_ref(context.get()).cast_mut()) + } + } + } + + fn pin(&self) -> NativeDispatch { + match self { + Self::H2(h2) => NativeDispatch::H2(h2.clone()), + Self::Duplex(context) => NativeDispatch::Duplex(context.ref_guard()), + } + } +} + +// A dispatch snapshot owns its allocation after the list borrow ends, including JS reentry. +enum NativeDispatch { + H2(RefPtr), + Duplex(RefPtr), +} + +impl NativeDispatch { + fn id(&self) -> NativeCallbackId { + match self { + Self::H2(h2) => NativeCallbackId::H2(h2.as_ptr()), + Self::Duplex(context) => NativeCallbackId::Duplex(context.as_ptr()), + } + } + + fn js_owner(&self) -> Option { + match self { + // H2's RefPtr already owns strong_this. + Self::H2(_) => None, + Self::Duplex(context) => context + .tls + .get() + .as_ref() + .and_then(|tls| tls.this_value.get().try_get()), + } + } + + fn read_paused(&self) -> bool { + match self { + // H2 receive-window flow control does not pause its native parser's transport. + Self::H2(_) => false, + Self::Duplex(context) => context.upgrade.native_read_paused(), + } + } + + fn write_pending(&self) -> bool { + match self { + Self::H2(h2) => h2.native_write_pending(), + Self::Duplex(context) => !context.native_idle(), + } + } + + fn on_data(&self, data: &[u8]) -> JsResult<()> { + let wrapper = self.js_owner(); + let result = match self { + Self::H2(h2) => h2.on_native_read(data), + Self::Duplex(context) => { + context.upgrade.receive_native_data(data); + Ok(()) + } }; - // SAFETY: `on_native_read` takes a keepalive; `h2` stays live across re-entry. - unsafe { (*h2).on_native_read(data) }?; - Ok(true) + if let Some(wrapper) = wrapper { + wrapper.ensure_still_alive(); + } + result + } + + fn on_writable(&self) { + let wrapper = self.js_owner(); + match self { + Self::H2(h2) => h2.on_native_writable(), + Self::Duplex(context) => context.on_native_writable(), + } + if let Some(wrapper) = wrapper { + wrapper.ensure_still_alive(); + } } - pub(crate) fn on_writable(&self) -> bool { - let h2 = match self { - NativeCallbacks::H2(h2) => h2.as_ptr(), - NativeCallbacks::None => return false, + + fn on_end(&self) -> bool { + let wrapper = self.js_owner(); + let result = match self { + // Preserve H2's existing JS EOF owner. + Self::H2(_) => false, + Self::Duplex(context) => { + let token = context.publish_native_transport_fact("nativeReadEnded"); + context.upgrade.receive_native_end(); + if let Some(token) = token { + token.ensure_still_alive(); + } + true + } }; - // SAFETY: `on_native_writable` takes a keepalive; `h2` stays live across re-entry. - unsafe { (*h2).on_native_writable() }; - true + if let Some(wrapper) = wrapper { + wrapper.ensure_still_alive(); + } + result + } + + fn on_close(&self, finalizing: bool) { + match self { + Self::H2(h2) => h2.on_native_close(), + Self::Duplex(context) => context.on_native_close(finalizing), + } } } +// ────────────────────────────────────────────────────────────────────────── // ────────────────────────────────────────────────────────────────────────── // WriteResult // ────────────────────────────────────────────────────────────────────────── @@ -4308,6 +4816,8 @@ impl bun_event_loop::Taskable for DuplexUpgradeContext { unsafe fn release_unrun(this: *mut Self) { // SAFETY: fn contract; the single queue entry for `this`. let ctx = unsafe { bun_ptr::ThisPtr::new(this) }; + // Each queued task owns a reference independently of the attachment. + let _task = unsafe { RefPtr::from_raw(this) }; ctx.queued.set(false); if let Some(tls) = ctx.tls.replace(None) { let socket = ctx.duplex_socket(); @@ -4325,8 +4835,19 @@ impl bun_event_loop::Taskable for DuplexUpgradeContext { } } +#[derive(bun_ptr::RefCounted)] pub(crate) struct DuplexUpgradeContext { + ref_count: bun_ptr::RefCount, + base_released: Cell, pub upgrade: UpgradedDuplex, + native_transport: JsCell>, + native_output: JsCell>, + native_flushing: Cell, + native_previous_js_write: Cell, + native_shutdown_requested: Cell, + native_shutdown_started: Cell, + native_shutdown_callback: Cell, + native_failed: Cell, // We only us a tls and not a raw socket when upgrading a Duplex, Duplex dont support socketpairs pub tls: JsCell>>, // task used to deinit the context in the next tick, vm is used to enqueue the task @@ -4373,6 +4894,274 @@ pub(super) enum EventState { } impl DuplexUpgradeContext { + fn native_id(&self) -> NativeCallbackId { + NativeCallbackId::Duplex(ptr::from_ref(self).cast_mut()) + } + + pub(super) fn uses_native_transport(&self) -> bool { + // Detachment must not reroute a terminal native stream through origin.write/end. + self.native_transport.get().is_some() || self.native_failed.get() + } + + pub(super) fn reconcile_native_reads(&self) -> bool { + if self.native_failed.get() { + return false; + } + self.native_transport + .get() + .clone() + .is_some_and(|transport| { + transport.reconcile_reads(self.native_id()) + && !self.native_failed.get() + && self.native_transport.get().is_some() + }) + } + + pub(super) fn native_idle(&self) -> bool { + if self.native_failed.get() + || self.native_flushing.get() + || self.native_previous_js_write.get() + || !self.native_output.get().is_empty() + { + return false; + } + self.native_transport + .get() + .clone() + .is_some_and(|transport| { + !transport.previous_write_pending(self.native_id()) && transport.drained() + }) + } + + pub(super) fn write_native(&self, data: &[u8]) { + let _pin = self.ref_guard(); + if self.native_failed.get() { + return; + } + self.native_output + .with_mut(|pending| pending.extend_from_slice(data)); + self.flush_native(); + } + + fn fail_native_write(&self, errno: i32) { + self.native_failed.set(true); + let error = + sys::Error::from_code_int(errno, sys::Tag::write).to_js(&self.upgrade.global.unwrap()); + Self::on_error( + unsafe { bun_ptr::ThisPtr::new(ptr::from_ref(self).cast_mut()) }, + error, + ); + self.finish_native_shutdown(error, false); + } + + fn flush_native(&self) { + let _pin = self.ref_guard(); + if self.native_flushing.replace(true) { + return; + } + (|| { + let Some(transport) = self.native_transport.get().clone() else { + return; + }; + loop { + if self.native_failed.get() + || self.native_previous_js_write.get() + || transport.previous_write_pending(self.native_id()) + { + return; + } + transport.flush(); + if self.native_transport.get().is_none() || self.native_failed.get() { + return; + } + let error = transport.error(); + if error != 0 { + self.fail_native_write(error); + return; + } + let output = self.native_output.replace(Vec::new()); + let mut blocked = false; + if !output.is_empty() { + let written = transport.write(self.native_id(), &output); + if self.native_transport.get().is_none() || self.native_failed.get() { + return; + } + if written < 0 { + self.fail_native_write(native_write_errno(written)); + return; + } + let consumed = (written as usize).min(output.len()); + blocked = consumed < output.len(); + if blocked { + self.native_output.with_mut(|pending| { + let newer = core::mem::take(pending); + pending.extend_from_slice(&output[consumed..]); + pending.extend_from_slice(&newer); + }); + } + } + // Reentrant writes append behind this batch. This flusher owns them until backpressure. + transport.flush(); + if self.native_transport.get().is_none() || self.native_failed.get() { + return; + } + let error = transport.error(); + if error != 0 { + self.fail_native_write(error); + return; + } + if blocked || !transport.drained() { + return; + } + if !self.native_output.get().is_empty() { + continue; + } + if self.native_shutdown_requested.get() + && !self.native_shutdown_started.replace(true) + { + // Flush close_notify first; the lower shutdown may enqueue its own ciphertext. + transport.shutdown(); + if self.native_transport.get().is_none() || self.native_failed.get() { + return; + } + let error = transport.error(); + if error != 0 { + self.fail_native_write(error); + return; + } + if !transport.drained() { + return; + } + if !self.native_output.get().is_empty() { + continue; + } + } + transport.clear_writer(self.native_id()); + if self.native_shutdown_started.get() { + let owner = self.upgrade.js_wrapper; + let token = self.publish_native_transport_fact("nativeShutdownCompleted"); + self.finish_native_shutdown(JSValue::UNDEFINED, false); + if let Some(token) = token { + token.ensure_still_alive(); + } + owner.ensure_still_alive(); + } + return; + } + })(); + self.native_flushing.set(false); + } + + fn on_native_writable(&self) { + let _pin = self.ref_guard(); + if self.native_failed.get() || self.native_flushing.get() { + return; + } + if self.upgrade.wrapper.get().is_none() && !self.native_previous_js_write.get() { + if self + .native_transport + .get() + .clone() + .is_some_and(|transport| !transport.previous_write_pending(self.native_id())) + { + Self::start_tls(unsafe { bun_ptr::ThisPtr::new(ptr::from_ref(self).cast_mut()) }); + } + return; + } + self.flush_native(); + if self.native_idle() { + Self::on_writable(unsafe { bun_ptr::ThisPtr::new(ptr::from_ref(self).cast_mut()) }); + } + } + + pub(super) fn shutdown_native(&self) { + let _pin = self.ref_guard(); + self.native_shutdown_requested.set(true); + self.flush_native(); + } + + fn canceled_native_shutdown(&self) -> JSValue { + let mut error: SystemError = sys::Error::new(sys::SystemErrno::ECANCELED, sys::Tag::write) + .to_system_error() + .into(); + error.syscall = BunString::static_("shutdown"); + error.to_error_instance(&self.upgrade.global.unwrap()) + } + + // Only live producer paths call this; finalizers never inspect the cached JS token. + fn publish_native_transport_fact(&self, name: &str) -> Option { + let owner = self.upgrade.js_wrapper; + let token = js_TLSSocket::native_transport_token_get_cached(owner)?; + token.put(&self.upgrade.global.unwrap(), name, JSValue::from(true)); + owner.ensure_still_alive(); + Some(token) + } + + fn finish_native_shutdown(&self, error: JSValue, finalizing: bool) { + if !self.native_shutdown_callback.replace(false) || finalizing { + return; + } + let owner = self.upgrade.js_wrapper; + let global = self.upgrade.global.unwrap(); + if let Some(callback) = js_TLSSocket::native_shutdown_get_cached(owner) { + // Release the visited callback before entering arbitrary JS. + js_TLSSocket::native_shutdown_set_cached(owner, &global, JSValue::UNDEFINED); + crate::dispatch::fold( + callback + .call(&global, JSValue::UNDEFINED, &[error]) + .map(|_| ()), + ); + } + } + + fn on_native_close(&self, finalizing: bool) { + let _pin = self.ref_guard(); + self.native_failed.set(true); + // The lower owner already unlinked this entry. Its JS error/close path owns upper teardown. + self.upgrade.take_native_pause(); + self.native_transport.set(None); + self.native_output.set(Vec::new()); + self.native_previous_js_write.set(false); + if finalizing { + if let Some(tls) = self.tls_this_ptr() { + tls.update_flags(|flags| flags.insert(Flags::FINALIZING)); + } + self.upgrade.abandon_js_side(); + self.upgrade.finish_close(); + } + } + + pub(super) fn teardown_native(&self, finalizing: bool) { + // Drop runs after deinit consumed these obligations; never resurrect its zero refcount. + if self.native_transport.get().is_some() || self.native_shutdown_callback.get() { + let _pin = self.ref_guard(); + self.unlink_native(finalizing); + } + } + + pub(super) fn unlink_native(&self, finalizing: bool) { + if self.native_transport.get().is_some() { + // Removal and shutdown cancellation can re-enter; terminal routing must already be sticky. + self.native_failed.set(true); + } + if let Some(transport) = self.native_transport.replace(None) { + transport.remove(self.native_id(), finalizing); + self.upgrade.take_native_pause(); + } + self.native_output.set(Vec::new()); + if finalizing { + self.finish_native_shutdown(JSValue::UNDEFINED, true); + } else if self.native_shutdown_callback.get() { + let error = self.canceled_native_shutdown(); + self.finish_native_shutdown(error, false); + } + } + + pub(super) fn ref_guard(&self) -> RefPtr { + // The attachment, queued task, or caller's dispatch pin owns this context. + unsafe { RefPtr::init_ref(ptr::from_ref(self).cast_mut()) } + } + #[inline(always)] // The `on_*` callbacks take `ThisPtr`, not `&mut self`: they are // dispatched from `UpgradedDuplex`'s engine (`start_tls` fires `on_open` @@ -4449,6 +5238,9 @@ impl DuplexUpgradeContext { } fn on_writable(this: bun_ptr::ThisPtr) { + if this.native_failed.get() { + return; + } let socket = this.duplex_socket(); if let Some(tls) = this.tls_this_ptr() { crate::dispatch::fold(TLSSocket::on_writable(tls, socket)); @@ -4523,6 +5315,8 @@ impl DuplexUpgradeContext { /// Takes `ThisPtr` because it may free `this` (via /// [`Self::deinit`]); nothing touches `this` after that call. pub(crate) fn run_event(this: bun_ptr::ThisPtr) { + // Transfer the queued task's reference to this frame before entering JS. + let _task = unsafe { RefPtr::from_raw(this.as_ptr()) }; this.queued.set(false); match this.task_event.get() { EventState::StartTLS => { @@ -4546,6 +5340,21 @@ impl DuplexUpgradeContext { this.upgrade.finish_close(); return; } + if this.native_failed.get() { + // The lower JS close path will close the upper; never reopen its native transport. + return; + } + if this.native_previous_js_write.get() + || this + .native_transport + .get() + .clone() + .is_some_and(|transport| transport.previous_write_pending(this.native_id())) + { + // The previous listener owns its completion; its drain requeues this start. + return; + } + log!( "DuplexUpgradeContext.startTLS mode={}", <&'static str>::from(this.mode) @@ -4600,6 +5409,13 @@ impl DuplexUpgradeContext { // e.g. both ends of a `duplexPair()` wrapped in-process, where // the peer's ClientHello lands before this side starts. this.upgrade.drain_pending(); + // JS origin listeners and queued completions now trace the wrapper. The startup + // pin must not keep an otherwise unreachable Duplex cycle alive forever. + if let Some(tls) = this.tls_this_ptr() { + if !matches!(tls.this_value.get(), JsRef::Finalized) { + tls.this_value.with_mut(|value| value.downgrade()); + } + } } // Previously this only called `upgrade.close()` and never `deinit`, // leaking the SSLWrapper, the strong refs, and this struct itself @@ -4619,6 +5435,8 @@ impl DuplexUpgradeContext { // Already in the queue: that entry runs the updated `task_event`. return; } + // Unlinking the attachment must not invalidate an already queued pointer. + unsafe { bun_ptr::RefCount::::ref_(this.as_ptr()) }; this.vm .event_loop_mut() .enqueue_task(jsc::Task::init(this.as_ptr())); @@ -4630,32 +5448,33 @@ impl DuplexUpgradeContext { } fn start_tls(this: bun_ptr::ThisPtr) { + if this.native_failed.get() { + return; + } this.task_event.set(EventState::StartTLS); Self::enqueue_self_task(this); } /// # Safety - /// `this` must be the unique live pointer to the heap allocation produced - /// in `js_upgrade_duplex_to_tls`. Frees the allocation; callers must not - /// hold a `&`/`&mut Self` across this call (taking `&mut self` here would - /// be a Stacked Borrows protector violation when the backing `Box` is - /// reclaimed below). + /// `this` must be a live context. Releases its attachment reference once; + /// queued tasks and reentrant dispatch frames keep the allocation alive. unsafe fn deinit(this: *mut Self) { { - // SAFETY: `this` is live; this borrow ends with the block, before the `heap::take` free below. + // SAFETY: the caller owns a live context reference. let ctx = unsafe { &*this }; + if ctx.base_released.replace(true) { + return; + } ctx.abort_handle.leave(); + ctx.unlink_native(ctx.upgrade.origin.get().is_empty()); + ctx.upgrade.teardown(); ctx.tls.set(None); // Close raced ahead of StartTLS — drop the unconsumed config / ctx. ctx.ssl_config.set(None); ctx.owned_ctx.set(None); } - // `UpgradedDuplex` cleanup - // runs via `Drop` when `heap::take(this)` frees the containing - // struct below; an explicit call here would double-free. - // SAFETY: heap-allocated in `js_upgrade_duplex_to_tls`; this is the - // matching free. No `&`/`&mut Self` survives past this point. - drop(unsafe { bun_core::heap::take(this) }); + // SAFETY: consumes only the base reference, never a task or dispatch pin. + unsafe { bun_ptr::RefCount::::deref(this) }; } } @@ -4680,6 +5499,76 @@ pub(crate) fn upgrade_tls_deferred<'a>( } } +pub(crate) fn attach_native_tls_transport( + cx: &mut JsScope<'_>, + socket: &TLSSocket, + transport: SocketHandle<'_>, + previous_writes: bool, + initial_data: JsLocal<'_>, + token: JsLocal<'_>, +) -> JsResult { + let uws::InternalSocket::UpgradedDuplex(pointer) = socket.socket.get().socket else { + return Ok(false); + }; + let upgrade = unsafe { &*pointer.cast::() }; + let context = upgrade.context_pin(); + let transport = match transport { + SocketHandle::Tcp(lower) if !lower.socket.get().is_detached() => { + NativeTransport::Tcp(lower.ref_guard()) + } + SocketHandle::Tls(lower) if !lower.socket.get().is_detached() => { + NativeTransport::Tls(lower.ref_guard()) + } + _ => return Ok(false), + }; + transport.attach(cx.global(), &context)?; + context.native_previous_js_write.set(previous_writes); + context.native_transport.set(Some(transport.clone())); + let token = token.to_js(); + js_TLSSocket::native_transport_token_set_cached(upgrade.js_wrapper, cx.global(), token); + token.put(cx.global(), "native", JSValue::from(true)); + let initial = initial_data.to_js(); + if let Some(buffer) = initial.as_array_buffer(cx.global()) { + upgrade.receive_native_data(buffer.slice()); + } + transport.reconcile_reads(context.native_id()); + token.ensure_still_alive(); + upgrade.js_wrapper.ensure_still_alive(); + Ok(true) +} + +pub(crate) fn native_tls_previous_writes_done(socket: &TLSSocket) { + if let uws::InternalSocket::UpgradedDuplex(pointer) = socket.socket.get().socket { + let upgrade = unsafe { &*pointer.cast::() }; + let context = upgrade.context_pin(); + if context.native_previous_js_write.replace(false) { + context.on_native_writable(); + } + } +} + +pub(crate) fn shutdown_native_tls_transport( + cx: &mut JsScope<'_>, + socket: SocketHandle<'_>, + callback: JsLocal<'_>, +) -> bool { + let SocketHandle::Tls(socket) = socket else { + return false; + }; + let uws::InternalSocket::UpgradedDuplex(pointer) = socket.socket.get().socket else { + return false; + }; + let upgrade = unsafe { &*pointer.cast::() }; + let context = upgrade.context_pin(); + if !context.uses_native_transport() { + return false; + } + js_TLSSocket::native_shutdown_set_cached(upgrade.js_wrapper, cx.global(), callback.to_js()); + context.native_shutdown_callback.set(true); + upgrade.shutdown(); + true +} + /// node:net's `destroy()`: drops the handshake flight held for the handshake callback. pub(crate) fn release_held_flight(socket: SocketHandle<'_>) { match socket { @@ -4851,7 +5740,8 @@ pub(crate) fn js_upgrade_duplex_to_tls( ref_pollref_on_connect: Cell::new(true), buffered_data_for_node_net: JsCell::new(Vec::new()), bytes_written: Cell::new(0), - native_callback: JsCell::new(NativeCallbacks::None), + native_callbacks: JsCell::new(Vec::new()), + native_write_owner: Cell::new(None), twin: JsCell::new(None), verify_error: JsCell::new(None), latest_session: Cell::new(None), @@ -4877,6 +5767,17 @@ pub(crate) fn js_upgrade_duplex_to_tls( // SAFETY: fresh heap allocation; every field is `ptr::write`-initialized // below before any read or `&mut DuplexUpgradeContext` is formed. unsafe { + ptr::addr_of_mut!((*duplex_context).ref_count).write(bun_ptr::RefCount::init()); + ptr::addr_of_mut!((*duplex_context).native_transport).write(JsCell::new(None)); + ptr::addr_of_mut!((*duplex_context).native_output).write(JsCell::new(Vec::new())); + ptr::addr_of_mut!((*duplex_context).native_flushing).write(Cell::new(false)); + ptr::addr_of_mut!((*duplex_context).native_previous_js_write).write(Cell::new(false)); + ptr::addr_of_mut!((*duplex_context).native_shutdown_requested).write(Cell::new(false)); + ptr::addr_of_mut!((*duplex_context).native_shutdown_started).write(Cell::new(false)); + ptr::addr_of_mut!((*duplex_context).native_shutdown_callback).write(Cell::new(false)); + ptr::addr_of_mut!((*duplex_context).native_failed).write(Cell::new(false)); + + ptr::addr_of_mut!((*duplex_context).base_released).write(Cell::new(false)); ptr::addr_of_mut!((*duplex_context).tls) .write(JsCell::new(Some(RefPtr::from_raw(tls.as_ptr())))); ptr::addr_of_mut!((*duplex_context).vm).write(VirtualMachine::get()); @@ -4907,6 +5808,7 @@ pub(crate) fn js_upgrade_duplex_to_tls( tls_js_value, duplex, is_node_socket, + context.id(), UpgradedDuplexHandlers { // SAFETY: `c` is `ctx` below — the live `DuplexUpgradeContext` heap allocation. on_open: |c: *mut ()| { @@ -4972,19 +5874,39 @@ pub(crate) fn js_upgrade_duplex_to_tls( // dangling still exits. If the underlying stream is a real socket, that // socket's own handle keeps the loop alive. - // SAFETY: non-null, fully initialised, heap-pinned; leaves its context in `deinit`. + // Prepare every fallible JS value before publishing a queued StartTLS owner. + let result = (|| { + let array = JSValue::create_empty_array(global, 2)?; + array.ensure_still_alive(); + array.put_index(global, 0, tls_js_value)?; + array.put_index( + global, + 1, + duplex_context_ref.upgrade.get_js_handlers(global)?, + )?; + Ok(array) + })(); + let array = match result { + Ok(array) => array, + Err(error) => { + duplex_context_ref.upgrade.teardown(); + if let Some(tls) = duplex_context_ref.tls.replace(None) { + // The pending exception suppresses JS close dispatch; on_close balances the native pin. + let _ = TLSSocket::on_close( + tls.into_this_ptr(), + duplex_context_ref.duplex_socket(), + 0, + None, + ); + } + // No task has been queued and the thunks are inert. + unsafe { DuplexUpgradeContext::deinit(duplex_context) }; + return Err(error); + } + }; + // SAFETY: fully initialised and heap-pinned; leaves its context in `deinit`. unsafe { bun_jsc::AbortHandle::arm_owner(duplex_context, context) }; DuplexUpgradeContext::start_tls(duplex_context_ref); - - let array = JSValue::create_empty_array(global, 2)?; - array.put_index(global, 0, tls_js_value)?; - // data, end, drain and close events must be reported - array.put_index( - global, - 1, - duplex_context_ref.upgrade.get_js_handlers(global)?, - )?; - Ok(array) } @@ -4996,6 +5918,14 @@ pub(crate) fn is_named_pipe_socket(socket: SocketHandle<'_>) -> bool { } } +pub(crate) fn has_native_socket_listeners(socket: SocketHandle<'_>) -> bool { + match socket { + SocketHandle::Tcp(this) => this.native_callbacks.get().iter().any(Option::is_some), + SocketHandle::Tls(this) => this.native_callbacks.get().iter().any(Option::is_some), + SocketHandle::Ignored(_) => false, + } +} + pub(crate) fn get_buffered_amount(socket: SocketHandle<'_>) -> usize { match socket { SocketHandle::Tcp(this) => this.buffered_data_for_node_net.get().len(), diff --git a/src/runtime/socket/sockets.classes.ts b/src/runtime/socket/sockets.classes.ts index 854e00ffebed..9cb3851e3ba2 100644 --- a/src/runtime/socket/sockets.classes.ts +++ b/src/runtime/socket/sockets.classes.ts @@ -9,12 +9,25 @@ function generate(ssl) { memoryCost: true, // Visited slot holding the shared JSSocketHandlers cell, so the callbacks // stay alive as long as any socket that can still fire them. The duplex* - // slots carry the origin stream and the four native listener thunks for a + // slots carry the origin stream and native listener/completion thunks for a // TLSSocket driven by an upgraded Duplex (UpgradedDuplex); plain TCP // sockets never populate them. values: ssl - ? ["handlers", "duplexOrigin", "duplexOnData", "duplexOnEnd", "duplexOnWritable", "duplexOnClose"] - : ["handlers"], + ? [ + "handlers", + "nativeAttachments", + "nativeShutdown", + "nativeTransportToken", + "duplexOrigin", + "duplexOnData", + "duplexOnEnd", + "duplexOnWritable", + "duplexOnClose", + "duplexOnWrite", + "duplexWriteDispatch", + "duplexCloseDispatch", + ] + : ["handlers", "nativeAttachments"], proto: { getAuthorizationError: { fn: "getAuthorizationError", diff --git a/test/js/bun/net/socket-retention.test.ts b/test/js/bun/net/socket-retention.test.ts index 29f93efb273d..930e10adc343 100644 --- a/test/js/bun/net/socket-retention.test.ts +++ b/test/js/bun/net/socket-retention.test.ts @@ -495,3 +495,129 @@ test.skipIf(!isWindows)( }, 60_000, ); + +test("TLS Duplex cycles collect after startup while a retained origin keeps its wrapper alive", async () => { + await using proc = Bun.spawn({ + cmd: [ + bunExe(), + "-e", + ` + const tls = require("node:tls"); + const { Duplex } = require("node:stream"); + const { once } = require("node:events"); + const certs = ${JSON.stringify(tlsCert)}; + const turn = () => new Promise(resolve => setImmediate(resolve)); + async function makePair(retain) { + let clientSide, serverSide; + const side = peer => new Duplex({ + allowHalfOpen: true, + read() {}, + write(chunk, encoding, callback) { peer().push(chunk); callback(); }, + }); + clientSide = side(() => serverSide); + serverSide = side(() => clientSide); + const server = new tls.TLSSocket(serverSide, { isServer: true, ...certs, allowHalfOpen: true }); + const client = tls.connect({ socket: clientSide, rejectUnauthorized: false, allowHalfOpen: true }); + server.on("error", () => {}); + client.on("error", () => {}); + // Exercise collection during startup; these locals still retain both sockets. + Bun.gc(true); + await once(client, "secureConnect"); + await turn(); + return { weak: new WeakRef(client._handle), origin: retain ? clientSide : null }; + } + let retained = await makePair(true); + const dropped = await makePair(false); + for (let i = 0; i < 30; i++) { await turn(); Bun.gc(true); } + const retainedAlive = !!retained.weak.deref(); + const droppedCollected = !dropped.weak.deref(); + retained.origin = null; + await turn(); + for (let i = 0; i < 30; i++) { await turn(); Bun.gc(true); } + console.log(JSON.stringify({ retainedAlive, droppedCollected, releasedCollected: !retained.weak.deref() })); + `, + ], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stdout.trim()).toBe('{"retainedAlive":true,"droppedCollected":true,"releasedCollected":true}'); + expect(exitCode, stderr).toBe(0); +}); + +for (const destroyBeforeCompletion of [false, true]) { + test(`a TLS Duplex completion retains its owner and is ${destroyBeforeCompletion ? "inert after destroy" : "delivered once"}`, async () => { + await using proc = Bun.spawn({ + cmd: [ + bunExe(), + "-e", + ` + const tls = require("node:tls"); + const { Duplex } = require("node:stream"); + const { once } = require("node:events"); + const certs = ${JSON.stringify(tlsCert)}; + const turn = () => new Promise(resolve => setImmediate(resolve)); + async function pending() { + let clientSide, serverSide; + const side = peer => new Duplex({ + allowHalfOpen: true, + read() {}, + write(chunk, encoding, callback) { peer().push(chunk); callback(); }, + }); + clientSide = side(() => serverSide); + serverSide = side(() => clientSide); + const server = new tls.TLSSocket(serverSide, { isServer: true, ...certs, allowHalfOpen: true }); + const client = tls.connect({ socket: clientSide, rejectUnauthorized: false, allowHalfOpen: true }); + server.on("error", () => {}); + client.on("error", () => {}); + await once(client, "secureConnect"); + await turn(); + let completion; + serverSide.write = (chunk, encoding, callback) => { + completion = typeof encoding === "function" ? encoding : callback; + return true; + }; + const settled = []; + server.write("x", error => settled.push(error?.code ?? null)); + if (typeof completion !== "function") throw new Error("ciphertext callback was not captured"); + const weak = new WeakRef(server._handle); + if (${destroyBeforeCompletion}) { + const closed = once(server, "close"); + server.destroy(); + await closed; + } + return { weak, completion, settled }; + } + const state = await pending(); + await turn(); + Bun.gc(true); + const callbackRoot = !!state.weak.deref(); + await turn(); // End the WeakRef keep-alive job before testing the queued owner. + state.completion(); + state.completion = null; + // Leave the callback invocation before collecting, without running the queued Immediate. + await Promise.resolve(); + if (!${destroyBeforeCompletion} && state.settled.length !== 0) throw new Error("completion ran before its Immediate"); + Bun.gc(true); + const queuedRoot = !!state.weak.deref(); + await turn(); + for (let i = 0; i < 30; i++) { await turn(); Bun.gc(true); } + console.log(JSON.stringify({ callbackRoot, queuedRoot, collected: !state.weak.deref(), settled: state.settled })); + `, + ], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + const { queuedRoot, ...result } = JSON.parse(stdout.trim()); + expect(result).toEqual({ + callbackRoot: true, + collected: true, + settled: [destroyBeforeCompletion ? "ECANCELED" : null], + }); + if (!destroyBeforeCompletion) expect(queuedRoot).toBe(true); + expect(exitCode, stderr).toBe(0); + }); +} diff --git a/test/js/node/http2/node-http2-syscall-fault.test.ts b/test/js/node/http2/node-http2-syscall-fault.test.ts index ccf83bfee03a..275f89bce6bf 100644 --- a/test/js/node/http2/node-http2-syscall-fault.test.ts +++ b/test/js/node/http2/node-http2-syscall-fault.test.ts @@ -217,23 +217,29 @@ describe.skipIf(skip)("node:http2 under injected syscall faults", () => { }); // Hive-pool user-poison, so only ASAN observes the freed-slot read. - test.skipIf(!isASAN)( - "send → backpressure then session.destroy() inside the drained write callback does not UAF", - async () => { - // Runs in a subprocess: the failure mode is an ASAN abort inside - // on_native_writable, not an exception the test runner can catch. - await using proc = Bun.spawn({ - cmd: [bunExe(), path.join(import.meta.dir, "node-http2-writable-destroy-fixture.ts")], - env: { ...bunEnv, ASAN_OPTIONS: [bunEnv.ASAN_OPTIONS, "symbolize=0"].filter(Boolean).join(":") }, - stdout: "pipe", - stderr: "pipe", - }); - const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); - expect(stderr).not.toContain("AddressSanitizer"); - expect(stdout.trim()).toBe("ok"); - expect(exitCode).toBe(0); - }, - ); + for (const transport of ["TCP", "TLS"]) { + test.skipIf(!isASAN)( + `send → backpressure then session.destroy() inside the drained ${transport} write callback does not UAF`, + async () => { + // Runs in a subprocess: the failure mode is an ASAN abort inside + // on_native_writable, not an exception the test runner can catch. + await using proc = Bun.spawn({ + cmd: [ + bunExe(), + path.join(import.meta.dir, "node-http2-writable-destroy-fixture.ts"), + ...(transport === "TLS" ? ["--tls"] : []), + ], + env: { ...bunEnv, ASAN_OPTIONS: [bunEnv.ASAN_OPTIONS, "symbolize=0"].filter(Boolean).join(":") }, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).not.toContain("AddressSanitizer"); + expect(stdout.trim()).toBe("ok"); + expect(exitCode).toBe(0); + }, + ); + } }); describe.skipIf(skip)("node:http2 seeded short-I/O fuzz", () => { diff --git a/test/js/node/http2/node-http2-writable-destroy-fixture.ts b/test/js/node/http2/node-http2-writable-destroy-fixture.ts index 15525727f711..c0896e730d1c 100644 --- a/test/js/node/http2/node-http2-writable-destroy-fixture.ts +++ b/test/js/node/http2/node-http2-writable-destroy-fixture.ts @@ -19,6 +19,8 @@ // the stack; when flush() returns, that keepalive is the last ref. import { socketFaultInjection as fault } from "bun:internal-for-testing"; +import { tls as certs } from "harness"; +import assert from "node:assert/strict"; import { once } from "node:events"; import http2 from "node:http2"; import type { AddressInfo } from "node:net"; @@ -28,7 +30,11 @@ if (!fault.available()) { process.exit(0); } -const server = http2.createServer(); +const secure = process.argv.includes("--tls"); +const server = secure + ? http2.createSecureServer({ ...certs, settings: { initialWindowSize: 1024 * 1024 } }) + : http2.createServer(); +if (secure) server.on("session", session => session.setLocalWindowSize(1024 * 1024)); server.on("stream", (stream: http2.ServerHttp2Stream) => { stream.respond({ ":status": 200 }); stream.resume(); @@ -40,9 +46,14 @@ server.listen(0, "127.0.0.1"); await once(server, "listening"); const { port } = server.address() as AddressInfo; -const client = http2.connect(`http://127.0.0.1:${port}`); +const client = http2.connect(`${secure ? "https" : "http"}://127.0.0.1:${port}`, { rejectUnauthorized: false }); client.on("error", () => {}); await once(client, "connect"); +if (secure) { + // The ping ACK follows the server's SETTINGS/WINDOW_UPDATE on the same stream. + await new Promise((resolve, reject) => client.ping(error => (error ? reject(error) : resolve()))); + assert(client.state.remoteWindowSize! >= 1024 * 1024); +} const req = client.request({ ":path": "/", ":method": "POST" }); req.on("error", () => {}); @@ -59,11 +70,28 @@ const { promise: destroyed, resolve } = Promise.withResolvers(); // (1) Large payload: multi-frame batch path → flush_batch_vectored → writev // returns 0, unwritten tail lands in write_buffer. Its Writable callback is // deferred via nextTick (direct path). -req.write(Buffer.alloc(40000, "a")); +let firstCallback = false; +let secondCallback = false; +let faultCleared = false; +// TLS flushes ciphertext batches at 128 KiB. With larger admitted H2 windows, +// this write spills before consuming all plaintext and leaves H2 backpressure. +req.write( + Buffer.alloc(secure ? 400 * 1024 : 40000, "a"), + secure + ? () => { + firstCallback = true; + } + : undefined, +); // (2) Buffered in the Writable until (1)'s callback runs; when _write runs // has_backpressure() is true so send_data queues this frame with destroyCb. req.write(Buffer.alloc(64, "b"), function destroyCb() { + if (secure) { + assert(faultCleared); + assert(!secondCallback); + secondCallback = true; + } try { client.destroy(); } catch {} @@ -73,7 +101,17 @@ req.write(Buffer.alloc(64, "b"), function destroyCb() { // (3) Queued after (1)'s deferred callback: by the time this runs the second // chunk is in data_frame_queue; clear the fault so the writable event drains. -process.nextTick(() => process.nextTick(() => fault.clear())); +process.nextTick(() => + process.nextTick(() => { + if (secure) { + assert(firstCallback); + assert(!secondCallback); + assert.equal(req.writableLength, 64); + } + faultCleared = true; + fault.clear(); + }), +); await destroyed; await new Promise(r => setImmediate(r)); diff --git a/test/js/node/tls/node-tls-duplex-close-throw-uaf.test.ts b/test/js/node/tls/node-tls-duplex-close-throw-uaf.test.ts index d909c945bd11..1872f37d08e2 100644 --- a/test/js/node/tls/node-tls-duplex-close-throw-uaf.test.ts +++ b/test/js/node/tls/node-tls-duplex-close-throw-uaf.test.ts @@ -7,7 +7,7 @@ // on the existing null-check. import { describe, expect, test } from "bun:test"; -import { bunEnv, bunExe, isASAN, isDebug } from "harness"; +import { bunEnv, bunExe, isASAN, isDebug, tls as certs } from "harness"; async function run(script: string, expected = "ok") { // Spawn a subprocess so an ASAN use-after-poison report shows up as a @@ -34,6 +34,63 @@ async function run(script: string, expected = "ok") { // build enables); release builds may read garbage without trapping. Each // test spawns an independent subprocess so they can run concurrently. describe.concurrent.skipIf(!isASAN && !isDebug)("tls.connect({socket: Duplex}) does not read freed Handlers", () => { + for (const phase of ["startup", "completion"]) { + test.serial(`terminating a worker releases queued TLS ${phase}`, async () => { + const workerSource = ` + const tls = require("node:tls"); + const { Duplex } = require("node:stream"); + const { once } = require("node:events"); + const { parentPort, workerData } = require("node:worker_threads"); + (async () => { + let a, b; + const side = peer => new Duplex({ + allowHalfOpen: true, + read() {}, + write(chunk, encoding, callback) { peer().push(chunk); callback(); }, + }); + a = side(() => b); + b = side(() => a); + const server = new tls.TLSSocket(b, { isServer: true, ...workerData.certs }); + const client = tls.connect({ socket: a, rejectUnauthorized: false }); + server.on("error", error => { throw error; }); + client.on("error", error => { throw error; }); + if (workerData.phase === "completion") { + await once(client, "secureConnect"); + await new Promise(resolve => setImmediate(resolve)); + let completion; + b.write = (chunk, encoding, callback) => { + completion = typeof encoding === "function" ? encoding : callback; + return true; + }; + server.write("x", () => {}); + if (typeof completion !== "function") throw new Error("missing ciphertext completion"); + completion(); + } + // Stop this invocation before StartTLS/the completion Immediate can run. + parentPort.postMessage(workerData.phase); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0); + })().catch(error => { throw error; }); + `; + await run(` + const { Worker } = require("node:worker_threads"); + const { once } = require("node:events"); + const assert = require("node:assert/strict"); + const worker = new Worker(${JSON.stringify(workerSource)}, { + eval: true, + workerData: { phase: ${JSON.stringify(phase)}, certs: ${JSON.stringify(certs)} }, + }); + try { + const [ready] = await once(worker, "message"); + assert.equal(ready, ${JSON.stringify(phase)}); + assert.equal(await worker.terminate(), 1); + console.log("ok"); + } finally { + await worker.terminate(); + } + `); + }); + } + test("closing a duplex does not call its throwing end method", async () => { // Closing once called end() after freeing the Handlers. Like Node, destruction // now avoids end() entirely, including when the transport emits close. diff --git a/test/js/node/tls/node-tls-duplex-end-verify.test.ts b/test/js/node/tls/node-tls-duplex-end-verify.test.ts index 96bf3db3c1ca..17e175102702 100644 --- a/test/js/node/tls/node-tls-duplex-end-verify.test.ts +++ b/test/js/node/tls/node-tls-duplex-end-verify.test.ts @@ -2,6 +2,7 @@ // Runs under node:test, so the same file runs on node (`node --test`) and on bun (`bun test`). import assert from "node:assert"; import fs from "node:fs"; +import { once } from "node:events"; import net from "node:net"; import { Duplex, duplexPair } from "node:stream"; import { test } from "node:test"; @@ -29,8 +30,6 @@ const clientCA = clientCert; // Both runtimes refuse an untrusted client. Bun reports the certificate check to 'tlsClientError', Node reports how // the connection ended. const isBun = process.versions.bun !== undefined; -// Inherited nested-reset notification gap; see the separate follow-up. -const inheritedResetTest = isBun ? (await import("bun:test")).test.failing : test; test("over a Duplex: responds after peer TLS shutdown and transport FIN", async () => { const [left, right] = duplexPair({ allowHalfOpen: true }); @@ -1663,9 +1662,15 @@ for (const side of ["client", "server"]) { const first = Buffer.alloc(8 * 1024 * 1024, "a"); writer.write(first, (err?: NodeJS.ErrnoException | null) => log.push(`write callback: ${err?.code}`)); writer.write("tail", (err?: NodeJS.ErrnoException | null) => log.push(`queued write callback: ${err?.code}`)); - // A turn later only what the kernel did not take is left. Node writes to the handle of the session below, past its queue. + // Output must still be queued somewhere in the writing TLS chain. await turn(); - if (isBun) assert.notStrictEqual((side === "client" ? clients : servers)[0].writableLength, 0); + if (isBun) { + const lengths = (side === "client" ? clients : servers).map(socket => socket.writableLength); + assert.ok( + lengths.some(n => n > 0), + JSON.stringify({ side, depth, lengths, received, log }), + ); + } reader.end(); await readerEnded; assert.deepStrictEqual( @@ -1696,7 +1701,13 @@ for (const [how, leave] of [ servers[1].resume(); clients[1].write(Buffer.alloc(8 * 1024 * 1024, "a")); await turn(); - if (isBun) assert.notStrictEqual(clients[0].writableLength, 0); + if (isBun) { + const lengths = clients.map(socket => socket.writableLength); + assert.ok( + lengths.some(n => n > 0), + JSON.stringify({ lengths }), + ); + } leave(servers[1], servers[0]); await Promise.all(closed); } finally { @@ -1792,51 +1803,48 @@ test("2 TLS sessions deep: the timeout of the session below still fires after th }); for (const side of ["client", "server"]) { - inheritedResetTest( - `2 TLS sessions deep: a ${side} hears of a reset that cuts what it wrote, after its TLS socket closed too`, - async () => { - const accepted = Promise.withResolvers(); - const listener = net.createServer(accepted.resolve) as Listening; - await new Promise(listening => listener.listen(0, "127.0.0.1", listening)); - const tcpClient = net.connect(listener.address().port, "127.0.0.1"); - const tcpServer = await accepted.promise; - const outerServer = new tls.TLSSocket(tcpServer, { isServer: true, key, cert }); - const outerClient = tls.connect({ socket: tcpClient, rejectUnauthorized: false }); - const innerServer = new tls.TLSSocket(outerServer, { isServer: true, key, cert }); - const innerClient = tls.connect({ socket: outerClient, rejectUnauthorized: false }); - const all = [tcpClient, tcpServer, outerClient, outerServer, innerClient, innerServer]; - const [inner, outer, peerTcp, peers] = - side === "client" - ? [innerClient, outerClient, tcpServer, [innerServer, outerServer, tcpServer]] - : [innerServer, outerServer, tcpClient, [innerClient, outerClient, tcpClient]]; - try { - const told: string[] = []; - for (const socket of all) socket.on("error", () => {}); - await Promise.all([ - new Promise(secured => innerClient.once("secureConnect", secured)), - new Promise(secured => innerServer.once("secure", secured)), - ]); - // The peer reads nothing, so most of this never leaves. - for (const socket of peers) socket.pause(); - inner.on("error", (err: NodeJS.ErrnoException) => told.push(`TLS socket 'error': ${err.code}`)); - outer.on("error", (err: NodeJS.ErrnoException) => told.push(`session below 'error': ${err.code}`)); - inner.write( - Buffer.alloc(8 * 1024 * 1024, "a"), - (err?: NodeJS.ErrnoException | null) => err && told.push(`write callback: ${err.code}`), - ); - const innerClosed = new Promise(resolve => inner.once("close", resolve)); - const outerClosed = new Promise(resolve => outer.once("close", resolve)); - inner.destroy(); - await innerClosed; - peerTcp.resetAndDestroy(); - await outerClosed; - assert.notDeepStrictEqual(told, []); - } finally { - for (const socket of all) socket.destroy(); - listener.close(); - } - }, - ); + test(`2 TLS sessions deep: a ${side} hears of a reset that cuts what it wrote, after its TLS socket closed too`, async () => { + const accepted = Promise.withResolvers(); + const listener = net.createServer(accepted.resolve) as Listening; + await new Promise(listening => listener.listen(0, "127.0.0.1", listening)); + const tcpClient = net.connect(listener.address().port, "127.0.0.1"); + const tcpServer = await accepted.promise; + const outerServer = new tls.TLSSocket(tcpServer, { isServer: true, key, cert }); + const outerClient = tls.connect({ socket: tcpClient, rejectUnauthorized: false }); + const innerServer = new tls.TLSSocket(outerServer, { isServer: true, key, cert }); + const innerClient = tls.connect({ socket: outerClient, rejectUnauthorized: false }); + const all = [tcpClient, tcpServer, outerClient, outerServer, innerClient, innerServer]; + const [inner, outer, peerTcp, peers] = + side === "client" + ? [innerClient, outerClient, tcpServer, [innerServer, outerServer, tcpServer]] + : [innerServer, outerServer, tcpClient, [innerClient, outerClient, tcpClient]]; + try { + const told: string[] = []; + for (const socket of all) socket.on("error", () => {}); + await Promise.all([ + new Promise(secured => innerClient.once("secureConnect", secured)), + new Promise(secured => innerServer.once("secure", secured)), + ]); + // The peer reads nothing, so most of this never leaves. + for (const socket of peers) socket.pause(); + inner.on("error", (err: NodeJS.ErrnoException) => told.push(`TLS socket 'error': ${err.code}`)); + outer.on("error", (err: NodeJS.ErrnoException) => told.push(`session below 'error': ${err.code}`)); + inner.write( + Buffer.alloc(8 * 1024 * 1024, "a"), + (err?: NodeJS.ErrnoException | null) => err && told.push(`write callback: ${err.code}`), + ); + const innerClosed = new Promise(resolve => inner.once("close", resolve)); + const outerClosed = new Promise(resolve => outer.once("close", resolve)); + inner.destroy(); + await innerClosed; + peerTcp.resetAndDestroy(); + await outerClosed; + assert.notDeepStrictEqual(told, []); + } finally { + for (const socket of all) socket.destroy(); + listener.close(); + } + }); } // No session the application could have written over: the connection goes at once, whatever its peer does. @@ -1964,3 +1972,257 @@ test("3 TLS sessions deep: the timeout of the session over TCP still fires after close(); } }); + +// The pre-upgrade plain TCP queue is the native pending-write observation used by node-net.test.ts. +// Windows can complete the OS write before an immediate takeover, so it cannot pin this boundary. +for (const side of ["client", "server"] as const) { + test( + `native takeover: a ${side} keeps its unfinished plain write ahead of TLS`, + { skip: process.platform === "win32" || (!isBun && Number(process.versions.node.split(".")[0]) < 24) }, + async t => { + const accepted = Promise.withResolvers(); + const listener = net.createServer({ allowHalfOpen: true }, accepted.resolve) as Listening; + t.after(() => listener.close()); + await new Promise(resolve => listener.listen(0, "127.0.0.1", resolve)); + const rawClient = net.connect({ port: listener.address().port, host: "127.0.0.1", allowHalfOpen: true }); + t.after(() => rawClient.destroy()); + const [rawServer] = await Promise.all([accepted.promise, once(rawClient, "connect")]); + t.after(() => rawServer.destroy()); + const errors: Error[] = []; + const failed = Promise.withResolvers(); + void failed.promise.catch(() => {}); + const guard = (pending: Promise) => Promise.race([pending, failed.promise]); + const onError = (error: Error) => { + errors.push(error); + failed.reject(error); + }; + for (const socket of [rawClient, rawServer]) socket.on("error", onError); + const [writer, peer] = side === "client" ? [rawClient, rawServer] : [rawServer, rawClient]; + await turn(); // Leave both native connect dispatches before issuing the plain write. + const preamble = Buffer.alloc(64 * 1024 * 1024, 0x61); + let callbacks = 0; + const written = Promise.withResolvers(); + writer.write(preamble, error => { + callbacks++; + if (error) written.reject(error); + else written.resolve(); + }); + // This must precede the constructor synchronously: no event-loop turn can drain the OS write. + assert.notStrictEqual(writer.writableLength, 0); + assert.strictEqual(callbacks, 0); + const first = + side === "client" + ? tls.connect({ socket: writer, rejectUnauthorized: false }) + : new tls.TLSSocket(writer, { isServer: true, key, cert }); + t.after(() => first.destroy()); + first.on("error", onError); + first.once("close", () => { + if (!first.readableEnded || !first.writableFinished) failed.reject(new Error("TLS closed before finishing")); + }); + + // Consume only the known plaintext prefix. TLS bytes stay buffered for the new listener. + let received = 0; + const prefixEnded = () => { + if (received < preamble.length) failed.reject(new Error("plaintext closed before the known prefix")); + }; + peer.once("end", prefixEnded); + peer.once("close", prefixEnded); + while (received < preamble.length) { + const chunk = peer.read(Math.min(preamble.length - received, peer.readableLength)) as Buffer | null; + if (chunk === null) { + assert.strictEqual(peer.readableEnded || peer.destroyed, false, "plaintext ended before the known prefix"); + await guard(once(peer, "readable")); + continue; + } + assert.ok(chunk.equals(preamble.subarray(received, received + chunk.length))); + received += chunk.length; + } + const second = + side === "client" + ? new tls.TLSSocket(peer, { isServer: true, key, cert }) + : tls.connect({ socket: peer, rejectUnauthorized: false }); + t.after(() => second.destroy()); + second.on("error", onError); + second.once("close", () => { + if (!second.readableEnded || !second.writableFinished) failed.reject(new Error("TLS closed before finishing")); + }); + const [client, server] = side === "client" ? [first, second] : [second, first]; + await guard(Promise.all([once(client, "secureConnect"), once(server, "secure"), written.promise])); + const closed = [once(client, "close"), once(server, "close")]; + let request = ""; + let response = ""; + let tlsCallbacks = 0; + server.on("data", chunk => (request += chunk)); + client.on("data", chunk => (response += chunk)); + const replied = Promise.withResolvers(); + server.once("end", () => + server.end("reply", () => { + tlsCallbacks++; + replied.resolve(); + }), + ); + const ended = once(client, "end"); + const requested = Promise.withResolvers(); + client.end("request", () => { + tlsCallbacks++; + requested.resolve(); + }); + await guard(Promise.all([ended, replied.promise, requested.promise, ...closed])); + assert.strictEqual(received, preamble.length); + assert.strictEqual(callbacks, 1); + assert.strictEqual(tlsCallbacks, 2); + assert.strictEqual(request, "request"); + assert.strictEqual(response, "reply"); + assert.deepStrictEqual(errors, []); + }, + ); +} + +test("native nested TLS pauses and resumes reads without calling the lower Socket methods", async () => { + const { clients, servers, close } = await nestedSessions(2); + const lower = clients[0]; + const client = clients[1]; + const server = servers[1]; + const errors: Error[] = []; + const failed = Promise.withResolvers(); + void failed.promise.catch(() => {}); + const guard = (pending: Promise) => Promise.race([pending, failed.promise]); + for (const socket of [...clients, ...servers]) + socket.on("error", error => { + errors.push(error); + failed.reject(error); + }); + client.once("close", () => failed.reject(new Error("paused TLS closed before its payload completed"))); + try { + // A retained Socket (including an HTTP/2 socket proxy) can restrict its public stream methods. + // The upper TLSWrap owns native flow control once it has taken the handle. + for (const method of ["pause", "resume"]) + Object.defineProperty(lower, method, { + configurable: true, + get() { + throw new Error(`lower public ${method} was used`); + }, + }); + client.pause(); + const payload = Buffer.alloc(512 * 1024, 0x6e); + const readable = once(client, "readable"); + const written = Promise.withResolvers(); + server.write(payload, error => (error ? written.reject(error) : written.resolve())); + await guard(readable); + await turn(); + assert.ok(client.readableLength >= client.readableHighWaterMark); + const received = Promise.withResolvers(); + const chunks: Buffer[] = []; + let length = 0; + client.on("data", chunk => { + chunks.push(chunk); + length += chunk.length; + if (length === payload.length) received.resolve(Buffer.concat(chunks)); + }); + client.resume(); + assert.ok((await guard(received.promise)).equals(payload)); + await guard(written.promise); + assert.deepStrictEqual(errors, []); + } finally { + Reflect.deleteProperty(lower, "pause"); + Reflect.deleteProperty(lower, "resume"); + close(); + } +}); + +for (const native of [false, true]) { + test( + `native TLS over a generic lower transport drains ${native ? "native-handle" : "public Writable"} reentry`, + { skip: native && !isBun }, + async () => { + let reenter: (() => void) | undefined; + const left: Duplex = new Duplex({ + allowHalfOpen: true, + read() {}, + write(chunk, _encoding, callback) { + right.push(chunk); + const action = reenter; + reenter = undefined; + action?.(); + callback(); + }, + final(callback) { + right.push(null); + callback(); + }, + }); + const right: Duplex = new Duplex({ + allowHalfOpen: true, + read() {}, + write(chunk, _encoding, callback) { + left.push(chunk); + callback(); + }, + final(callback) { + left.push(null); + callback(); + }, + }); + const outerServer = new tls.TLSSocket(right, { isServer: true, key, cert }); + const outerClient = tls.connect({ socket: left, rejectUnauthorized: false }); + const all: Duplex[] = [left, right, outerClient, outerServer]; + const errors: Error[] = []; + const failed = Promise.withResolvers(); + void failed.promise.catch(() => {}); + const guard = (pending: Promise) => Promise.race([pending, failed.promise]); + const onError = (error: Error) => { + errors.push(error); + failed.reject(error); + }; + for (const socket of all) socket.on("error", onError); + try { + await guard(Promise.all([once(outerClient, "secureConnect"), once(outerServer, "secure")])); + const server = new tls.TLSSocket(outerServer, { isServer: true, key, cert }); + const client = tls.connect({ socket: outerClient, rejectUnauthorized: false }); + all.push(client, server); + for (const socket of [client, server]) { + socket.on("error", onError); + socket.once("close", () => failed.reject(new Error("reentrant TLS closed before completing"))); + } + await guard(Promise.all([once(client, "secureConnect"), once(server, "secure")])); + const received = Promise.withResolvers(); + let body = ""; + server.on("data", chunk => { + body += chunk; + if (body.length === "firstreentered".length) received.resolve(body); + }); + const callbacks: string[] = []; + const first = Promise.withResolvers(); + const second = Promise.withResolvers(); + let reentries = 0; + reenter = () => { + reentries++; + if (native) { + // Existing Bun native write API bypasses the public Writable queue: this must append during flush_native. + const data = Buffer.from("reentered"); + assert.strictEqual((client as any)._handle.write(data), data.length); + second.resolve(); + return; + } + client.write("reentered", error => { + callbacks.push("reentered"); + if (error) second.reject(error); + else second.resolve(); + }); + }; + client.write("first", error => { + callbacks.push("first"); + if (error) first.reject(error); + else first.resolve(); + }); + assert.strictEqual(await guard(received.promise), "firstreentered"); + await guard(Promise.all([first.promise, second.promise])); + assert.strictEqual(reentries, 1); + assert.deepStrictEqual(callbacks, native ? ["first"] : ["first", "reentered"]); + assert.deepStrictEqual(errors, []); + } finally { + for (const socket of all.reverse()) socket.destroy(); + } + }, + ); +} From 1dff3f9036fa7a73856e091c3faf72baa5caa1f3 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Mon, 26 Oct 2026 19:03:11 +0800 Subject: [PATCH 02/74] style(socket): format native TLS ownership paths --- src/runtime/api/bun/h2_frame_parser.rs | 5 +++-- src/runtime/socket/socket_body.rs | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/runtime/api/bun/h2_frame_parser.rs b/src/runtime/api/bun/h2_frame_parser.rs index 24e549640bfd..1a2ab085fc26 100644 --- a/src/runtime/api/bun/h2_frame_parser.rs +++ b/src/runtime/api/bun/h2_frame_parser.rs @@ -110,8 +110,9 @@ impl NativeSocket { } fn detach(&self) { - let owner = - crate::socket::socket_body::NativeCallbackId::H2(self.owner.replace(core::ptr::null_mut())); + let owner = crate::socket::socket_body::NativeCallbackId::H2( + self.owner.replace(core::ptr::null_mut()), + ); match self.socket.replace(BunSocket::None) { BunSocket::Tcp(socket) => socket.remove_native_callback(owner), BunSocket::Tls(socket) => socket.remove_native_callback(owner), diff --git a/src/runtime/socket/socket_body.rs b/src/runtime/socket/socket_body.rs index 8efae27fb8c9..da9b1c3cb504 100644 --- a/src/runtime/socket/socket_body.rs +++ b/src/runtime/socket/socket_body.rs @@ -5611,7 +5611,9 @@ pub(crate) fn upgrade_duplex_to_tls<'a>( // No task has been queued. Consume the connection without queueing its free, // as release_unrun does; any reentrant entry pin still delays destruction. if duplex_context_ref.take_connection_ref() { - duplex_context_ref.ref_count.set(duplex_context_ref.ref_count.get() - 1); + duplex_context_ref + .ref_count + .set(duplex_context_ref.ref_count.get() - 1); } if duplex_context_ref.ref_count.get() == 0 { unsafe { DuplexUpgradeContext::deinit(duplex_context) }; From eb84ae249678e9336e20334ee41c2e98f5bf4835 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Mon, 26 Oct 2026 21:38:05 +0800 Subject: [PATCH 03/74] test(ci): qualify TLS with native release artifacts --- .github/OPENCLAW_RELEASE.md | 18 +++ .github/workflows/openclaw-release.yml | 34 +++++- CHANGELOG.md | 2 + scripts/openclaw-release/release.ts | 111 +++++++++++++++++- .../source-lints/openclaw-release.test.ts | 47 ++++++++ 5 files changed, 207 insertions(+), 5 deletions(-) diff --git a/.github/OPENCLAW_RELEASE.md b/.github/OPENCLAW_RELEASE.md index 7551eb8710dc..01a31e5ad009 100644 --- a/.github/OPENCLAW_RELEASE.md +++ b/.github/OPENCLAW_RELEASE.md @@ -47,6 +47,24 @@ Rebuilding a commit that already has a release (a toolchain or pipeline change, not a source change) gets a new tag with `--rebuild 2` (`…-r2`). Published assets are never replaced: packagers pin their checksums. +### TLS qualification dry runs + +Set `qualify-tls` on a manual dry run to test the built artifacts with the +socket-retention, duplex-end, duplex-close, raw-end, wrapped-close and HTTP/2 +upgrade suites. Leave `tag` empty and `publish` false. The default target set +runs all six native platforms; musl containers are rejected because their Node.js +reference runtime is not qualified. + +Each smoke job checks out the exact source commit, verifies genuine Node.js +26.12.1 for its platform and architecture, and runs the six files with the +artifact's executable and the checked-in test configuration. No dependency +installation or test timeout override is added. Each file must execute passing +cases with no failures; declared platform and Debug/ASAN-only skips remain +visible in the six JUnit reports uploaded as `tls-qualification-`. +The opted-in smoke job has a 30-minute limit and its test step a 20-minute limit. +Normal smoke jobs retain their 15-minute limit. These dry runs do not publish +artifacts or establish sanitizer coverage for release builds. + ## Tag scheme ```text diff --git a/.github/workflows/openclaw-release.yml b/.github/workflows/openclaw-release.yml index 3f0d4ed84d2d..c06b5bef96b9 100644 --- a/.github/workflows/openclaw-release.yml +++ b/.github/workflows/openclaw-release.yml @@ -25,6 +25,10 @@ on: description: "Comma-separated targets; empty = enabled release targets (all six in dry runs). Also: linux-x64-musl, linux-arm64-musl" type: string default: "" + qualify-tls: + description: "Run the TLS qualification suites on native dry-run artifacts (no tag or musl targets)" + type: boolean + default: false permissions: contents: read @@ -47,6 +51,7 @@ jobs: tag: ${{ steps.plan.outputs.tag }} webkit: ${{ steps.plan.outputs.webkit }} publish: ${{ steps.plan.outputs.publish }} + qualify-tls: ${{ steps.plan.outputs.qualify-tls }} matrix: ${{ steps.plan.outputs.matrix }} darwin: ${{ steps.plan.outputs.darwin }} windows: ${{ steps.plan.outputs.windows }} @@ -80,12 +85,14 @@ jobs: env: TARGETS: ${{ inputs.targets }} WINDOWS_SIGNED: ${{ vars.OPENCLAW_RELEASE_WINDOWS_SIGNED }} + QUALIFY_TLS: ${{ inputs.qualify-tls == true }} run: | args=(--source source --main origin/main) [ -n "$RELEASE_TAG" ] && args+=(--tag "$RELEASE_TAG") [ "$PUBLISH" = true ] && args+=(--publish) [ "$WINDOWS_SIGNED" = true ] && args+=(--windows-signed) [ -n "$TARGETS" ] && args+=(--targets "$TARGETS") + [ "$QUALIFY_TLS" = true ] && args+=(--qualify-tls) bun pipeline/scripts/openclaw-release/release.ts plan "${args[@]}" build: @@ -355,7 +362,7 @@ jobs: needs: [plan, build, sign-macos, sign-windows, windows-test-only] if: ${{ !cancelled() && needs.build.result == 'success' && needs.sign-macos.result != 'failure' && (needs.plan.outputs.windows != 'true' || (needs.plan.outputs.publish == 'true' && needs.sign-windows.result == 'success') || (needs.plan.outputs.publish != 'true' && needs.windows-test-only.result == 'success')) }} runs-on: ${{ matrix.runner }} - timeout-minutes: 15 + timeout-minutes: ${{ needs.plan.outputs.qualify-tls == 'true' && 30 || 15 }} strategy: fail-fast: false matrix: @@ -371,7 +378,20 @@ jobs: with: name: ${{ matrix.artifact }} path: dist + - name: Checkout qualification source + if: needs.plan.outputs.qualify-tls == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: source + ref: ${{ needs.plan.outputs.commit }} + persist-credentials: false + - name: Setup Node.js reference + if: needs.plan.outputs.qualify-tls == 'true' + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "26.12.1" - name: Smoke test + timeout-minutes: ${{ needs.plan.outputs.qualify-tls == 'true' && 20 || 15 }} shell: bash env: TARGET: ${{ matrix.target }} @@ -379,12 +399,16 @@ jobs: CONTAINER: ${{ matrix.container }} COMMIT: ${{ needs.plan.outputs.commit }} WEBKIT: ${{ needs.plan.outputs.webkit }} + QUALIFY_TLS: ${{ needs.plan.outputs.qualify-tls }} run: | # The release's own executable runs the checks, so the runner needs no other Bun. unzip -q "dist/$TRIPLET.zip" -d unpacked exe="unpacked/$TRIPLET/bun" [ -f "$exe.exe" ] && exe="$exe.exe" smoke=("$exe" pipeline/scripts/openclaw-release/release.ts smoke --zip "dist/$TRIPLET.zip" --target "$TARGET" --commit "$COMMIT" --webkit "$WEBKIT") + if [ "$QUALIFY_TLS" = true ]; then + smoke+=(--qualify-tls --source source --out tls-results) + fi if [ -n "$CONTAINER" ]; then docker run --rm -v "$PWD:/work" -w /work "$CONTAINER" sh -c 'apk add --no-cache libgcc libstdc++ unzip > /dev/null && "$@"' sh "${smoke[@]}" else @@ -395,6 +419,14 @@ jobs: codesign --verify --strict --verbose=2 "$exe" codesign -dv "$exe" 2>&1 | grep -E '^(Authority|TeamIdentifier|CodeDirectory)' || true fi + - name: Upload TLS qualification reports + if: ${{ always() && needs.plan.outputs.qualify-tls == 'true' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: tls-qualification-${{ matrix.target }} + path: tls-results/*.xml + if-no-files-found: error + retention-days: 14 windows-compatibility: name: Compatibility ${{ matrix.target }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 606554d8e5a8..0b13f86de06b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Add opt-in native TLS qualification to release dry runs, checking the exact artifact and source with a genuine Node.js reference and per-file JUnit results. + - Preserve native socket ownership when nesting TLS, including pending writes, read backpressure, shutdown completion and buried HTTP/2 listeners. Keep generic Duplex write completions and queued TLS callbacks alive through reentrant teardown and garbage collection. - Publish `node:sqlite` query diagnostics with expanded SQL, database identity and duration, including safe synchronous nested queries; support statement close/disposal and the current constructor aliases. diff --git a/scripts/openclaw-release/release.ts b/scripts/openclaw-release/release.ts index 2cb466464523..69a84ba58b23 100644 --- a/scripts/openclaw-release/release.ts +++ b/scripts/openclaw-release/release.ts @@ -6,10 +6,10 @@ * `.github/workflows/openclaw-release.yml`. * * bun scripts/openclaw-release/release.ts tag [--commit ] [--rebuild ] - * bun scripts/openclaw-release/release.ts plan [--tag ] [--publish] [--windows-signed] + * bun scripts/openclaw-release/release.ts plan [--tag ] [--publish] [--windows-signed] [--qualify-tls] * bun scripts/openclaw-release/release.ts provision --source --target ... * bun scripts/openclaw-release/release.ts build --source --target --out - * bun scripts/openclaw-release/release.ts smoke --zip --target --commit --webkit + * bun scripts/openclaw-release/release.ts smoke --zip --target --commit --webkit [--qualify-tls --source --out ] * bun scripts/openclaw-release/release.ts manifest --dist --tag --commit --out * * Every command takes `--source `, the checkout of the commit being @@ -28,6 +28,7 @@ import { mkdirSync, mkdtempSync, readFileSync, + realpathSync, readdirSync, rmSync, statSync, @@ -406,8 +407,96 @@ export function zip(path: string, dir: string, files: string[], mtime: number): // ─────────────────────────────────────────────────────────────── smoke test +export const tlsQualificationFiles = [ + "test/js/bun/net/socket-retention.test.ts", + "test/js/node/tls/node-tls-duplex-end-verify.test.ts", + "test/js/node/tls/node-tls-duplex-close-throw-uaf.test.ts", + "test/js/node/tls/node-tls-raw-end.test.ts", + "test/js/node/tls/node-tls-wrapped-socket-close.test.ts", + "test/js/node/http2/node-http2-upgrade.test.mts", +] as const; + +export function validateTLSQualification(names: readonly TargetName[], publish = false, tag = ""): void { + if (publish || tag) throw new Error("TLS qualification requires a dry run without a release tag"); + if (names.some(name => targets[name].abi === "musl")) { + throw new Error("TLS qualification requires native runners; musl containers have no qualified Node.js reference"); + } +} + +/** Checks the single-file report emitted by Bun, including declared skips. This is not a general XML reader. */ +export function tlsQualificationResult(xml: string): { passed: number; skipped: number } { + const root = /^\s*(?:<\?xml[^?]*\?>\s*)?]*)>/.exec(xml); + const count = (name: string) => { + const value = root && new RegExp(`\\s${name}="(\\d+)"`).exec(root[1]!)?.[1]; + if (value === null || value === undefined) throw new Error(`TLS report is missing a valid ${name} count`); + const result = Number(value); + if (!Number.isSafeInteger(result)) throw new Error(`TLS report has an invalid ${name} count`); + return result; + }; + const tests = count("tests"); + const skipped = count("skipped"); + const failures = count("failures"); + if (!/<\/testsuites>\s*$/.test(xml) || (xml.match(/ { rebuild: { type: "string" }, tag: { type: "string" }, publish: { type: "boolean", default: false }, + "qualify-tls": { type: "boolean", default: false }, "windows-signed": { type: "boolean", default: false }, main: { type: "string", default: "origin/main" }, target: { type: "string", multiple: true }, @@ -724,6 +815,9 @@ async function main(argv: string[]): Promise { }, }); const source = resolve(values.source!); + if (values["qualify-tls"] && command !== "plan" && command !== "smoke") { + throw new Error("--qualify-tls is supported only by plan and smoke"); + } const names = (): TargetName[] => { const list = [...(values.target ?? []), ...(values.targets ? [values.targets] : [])].join(","); const parsed = parseTargets(list); @@ -752,6 +846,7 @@ async function main(argv: string[]): Promise { } } const built = planTargets(values.targets, values.publish, values["windows-signed"]); + if (values["qualify-tls"]) validateTLSQualification(built, values.publish, tag); output("matrix", JSON.stringify(matrix(built, values.publish))); output("darwin", String(built.some(name => targets[name].os === "darwin"))); output("windows", String(built.some(name => targets[name].os === "windows"))); @@ -770,6 +865,7 @@ async function main(argv: string[]): Promise { output("version", facts.version); output("webkit", facts.webkitVersion); output("publish", String(values.publish)); + output("qualify-tls", String(values["qualify-tls"])); output("windows-signed", String(values["windows-signed"])); return; } @@ -784,7 +880,14 @@ async function main(argv: string[]): Promise { } case "smoke": { const name = names()[0]!; - smoke(resolve(values.zip!), name, values.commit!, values.webkit!); + if (values["qualify-tls"]) validateTLSQualification([name], values.publish, values.tag); + smoke( + resolve(values.zip!), + name, + values.commit!, + values.webkit!, + values["qualify-tls"] ? { source, reports: resolve(values.out ?? "tls-results") } : undefined, + ); return; } case "manifest": { diff --git a/test/internal/source-lints/openclaw-release.test.ts b/test/internal/source-lints/openclaw-release.test.ts index a59d95d85a89..580276349d2c 100644 --- a/test/internal/source-lints/openclaw-release.test.ts +++ b/test/internal/source-lints/openclaw-release.test.ts @@ -26,6 +26,9 @@ import { tagFor, tagMismatches, targets, + tlsQualificationFiles, + tlsQualificationResult, + validateTLSQualification, webkitTagPart, windowsSignerSubject, zip, @@ -41,6 +44,50 @@ const facts: SourceFacts = { webkitVersion: "35e8970dfd926abf6661c9d356f5d60d8f611c9b", }; +test("TLS qualification is opt-in for native untagged dry runs", () => { + expect(() => validateTLSQualification(planTargets())).not.toThrow(); + expect(() => validateTLSQualification(planTargets(), true)).toThrow("dry run without a release tag"); + expect(() => validateTLSQualification(planTargets(), false, tagFor(facts))).toThrow("dry run without a release tag"); + for (const name of ["linux-x64-musl", "linux-arm64-musl"] as const) { + expect(() => validateTLSQualification([name])).toThrow("musl containers have no qualified Node.js reference"); + } + expect(tlsQualificationFiles).toHaveLength(6); + expect(new Set(tlsQualificationFiles).size).toBe(6); + const workflow = readFileSync(join(import.meta.dir, "../../../.github/workflows/openclaw-release.yml"), "utf8"); + expect(workflow).toContain("qualify-tls:\n description:"); + expect(workflow).toContain("smoke+=(--qualify-tls --source source --out tls-results)"); + expect(workflow).toContain("ref: ${{ needs.plan.outputs.commit }}"); + expect(workflow).toContain('node-version: "26.12.1"'); +}); + +test("TLS reports require executed passing cases and preserve declared skips", () => { + const report = + '' + + ''; + expect(tlsQualificationResult(report)).toEqual({ passed: 1, skipped: 1 }); + for (const xml of [ + "", + '', + report.replace('tests="2"', 'tests="bad"'), + ]) { + expect(() => tlsQualificationResult(xml)).toThrow("missing a valid tests count"); + } + expect(() => tlsQualificationResult(report.replace("", ""))).toThrow( + "incomplete or contains multiple runs", + ); + expect(() => tlsQualificationResult(report + report)).toThrow("incomplete or contains multiple runs"); + expect(() => tlsQualificationResult(report.replace('tests="2"', 'tests="3"'))).toThrow("counts do not match"); + expect(() => tlsQualificationResult(report.replace('skipped="1"', 'skipped="0"'))).toThrow("counts do not match"); + for (const xml of [ + '', + '', + report.replace('failures="0"', 'failures="1"'), + report.replace('failures="0"', 'failures="0" errors="1"'), + report.replace('', ""), + ]) + expect(() => tlsQualificationResult(xml)).toThrow("passing cases and no failures or errors"); +}); + test("a tag names the version, commit date, commit and WebKit revision, and parses back", () => { const tag = tagFor(facts); expect(tag).toBe("openclaw-v1.4.3-20260925-9e3414b3f7-webkit-35e8970dfd"); From 9f54e7902b3aeff6ef1ed5f47bee37739e9f16c8 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Mon, 26 Oct 2026 22:50:25 +0800 Subject: [PATCH 04/74] style: organize TLS qualification imports --- scripts/openclaw-release/release.ts | 2 +- test/js/node/tls/node-tls-duplex-end-verify.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/openclaw-release/release.ts b/scripts/openclaw-release/release.ts index 2bd375b63173..2ce38e9b5d92 100644 --- a/scripts/openclaw-release/release.ts +++ b/scripts/openclaw-release/release.ts @@ -27,9 +27,9 @@ import { lstatSync, mkdirSync, mkdtempSync, + readdirSync, readFileSync, realpathSync, - readdirSync, rmSync, statSync, utimesSync, diff --git a/test/js/node/tls/node-tls-duplex-end-verify.test.ts b/test/js/node/tls/node-tls-duplex-end-verify.test.ts index 17e175102702..767b90955be8 100644 --- a/test/js/node/tls/node-tls-duplex-end-verify.test.ts +++ b/test/js/node/tls/node-tls-duplex-end-verify.test.ts @@ -1,8 +1,8 @@ // A TLS handshake that finishes after this side shut its write direction down keeps the certificate verdict. // Runs under node:test, so the same file runs on node (`node --test`) and on bun (`bun test`). import assert from "node:assert"; -import fs from "node:fs"; import { once } from "node:events"; +import fs from "node:fs"; import net from "node:net"; import { Duplex, duplexPair } from "node:stream"; import { test } from "node:test"; From c55262dc1ef4e1f4b398e61730a4dc6e26f58dd5 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Mon, 26 Oct 2026 23:25:27 +0800 Subject: [PATCH 05/74] fix(tls): preserve allocation roots in native dispatch --- src/js/node/net.ts | 3 ++- src/runtime/socket/socket_body.rs | 25 ++++++++++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/js/node/net.ts b/src/js/node/net.ts index a891c8e4772c..36e3b88f4636 100644 --- a/src/js/node/net.ts +++ b/src/js/node/net.ts @@ -597,7 +597,8 @@ function attachTLSEngine(self, connection, options) { connection.once(kReadError, onUpgradedReadError.bind(null, self)); let initialData; if (connection instanceof Socket) { - if (connection.readableLength) initialData = connection.read(connection.readableLength) || undefined; + const readableLength = connection.readableLength; + if (readableLength) initialData = connection.read(readableLength) || undefined; if ( self.destroyed || self[kNativeTLSTransport] !== token || diff --git a/src/runtime/socket/socket_body.rs b/src/runtime/socket/socket_body.rs index da9b1c3cb504..ad03a068696f 100644 --- a/src/runtime/socket/socket_body.rs +++ b/src/runtime/socket/socket_body.rs @@ -632,11 +632,13 @@ impl NewSocket { } if SSL { if let Some(raw) = socket.socket.get() { - // Same fatal-error producer as write_check_error; these are read-only observations. + // SAFETY: the open Connected socket owns this live C socket. This read-only + // observer cannot invoke JS or reenter and close it. let fatal = unsafe { us_internal_ssl_fatal_write_error(raw) }; if fatal != 0 { return native_write_errno(fatal); } + // SAFETY: the same open C socket remains live; neither observer reenters JS. if unsafe { us_socket_ssl_spill_dropped(raw) } != 0 { return sys::SystemErrno::EPIPE as i32; } @@ -651,7 +653,8 @@ impl NewSocket { } let spill = if SSL { self.socket.get().socket.get().map_or(0, |socket| { - // The live socket is pinned by NativeTransport. This observes, never changes, fd policy. + // SAFETY: native_transport_error established an open Connected socket, and + // nothing can reenter before this read-only spill observation. unsafe { us_socket_ssl_spill_pending(socket) } }) } else { @@ -3351,7 +3354,8 @@ impl NewSocket { fn transport_idle(&self) -> bool { match self.socket.get().socket { uws::InternalSocket::UpgradedDuplex(pointer) => { - // Installed by DuplexUpgradeContext::duplex_socket and live until detach. + // SAFETY: duplex_socket installs the aligned, stable context.upgrade member. + // The attached tag keeps it live for this shared borrow until detach. unsafe { &*pointer.cast::() }.transport_idle() } _ => true, @@ -4599,10 +4603,7 @@ impl DuplexUpgradeContext { self.native_failed.set(true); let error = sys::Error::from_code_int(errno, sys::Tag::write).to_js(&self.upgrade.global.unwrap()); - Self::on_error( - unsafe { bun_ptr::ThisPtr::new(ptr::from_ref(self).cast_mut()) }, - error, - ); + (self.upgrade.handlers.on_error)(self.upgrade.handlers.ctx, error); self.finish_native_shutdown(error, false); } @@ -4716,13 +4717,13 @@ impl DuplexUpgradeContext { .clone() .is_some_and(|transport| !transport.previous_write_pending(self.native_id())) { - Self::start_tls(unsafe { bun_ptr::ThisPtr::new(ptr::from_ref(self).cast_mut()) }); + Self::start_tls(_pin.this_ptr()); } return; } self.flush_native(); if self.native_idle() { - Self::on_writable(unsafe { bun_ptr::ThisPtr::new(ptr::from_ref(self).cast_mut()) }); + Self::on_writable(_pin.this_ptr()); } } @@ -5232,6 +5233,8 @@ pub(crate) fn attach_native_tls_transport( let uws::InternalSocket::UpgradedDuplex(pointer) = socket.socket.get().socket else { return Ok(false); }; + // SAFETY: the attached tag holds the aligned context.upgrade member; close detaches + // it before releasing the context. Take a keep-alive before any reentrant work. let upgrade = unsafe { &*pointer.cast::() }; let context = upgrade.keep_alive(); let transport = match transport { @@ -5261,6 +5264,8 @@ pub(crate) fn attach_native_tls_transport( pub(crate) fn native_tls_previous_writes_done(_: &mut JsScope<'_>, socket: &TLSSocket) { if let uws::InternalSocket::UpgradedDuplex(pointer) = socket.socket.get().socket { + // SAFETY: the attached tag holds the aligned context.upgrade member; close detaches + // it before releasing the context. Take a keep-alive before any reentrant work. let upgrade = unsafe { &*pointer.cast::() }; let context = upgrade.keep_alive(); if context.native_previous_js_write.replace(false) { @@ -5280,6 +5285,8 @@ pub(crate) fn shutdown_native_tls_transport( let uws::InternalSocket::UpgradedDuplex(pointer) = socket.socket.get().socket else { return false; }; + // SAFETY: the attached tag holds the aligned context.upgrade member; close detaches + // it before releasing the context. Take a keep-alive before any reentrant work. let upgrade = unsafe { &*pointer.cast::() }; let context = upgrade.keep_alive(); if !context.uses_native_transport() { From 8f174699e37827fbe4f0fda8dfcd06532ce91907 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 27 Oct 2026 00:06:17 +0800 Subject: [PATCH 06/74] fix(tls): narrow wrapped socket types --- src/js/node/net.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/js/node/net.ts b/src/js/node/net.ts index 36e3b88f4636..e6270b9e63f5 100644 --- a/src/js/node/net.ts +++ b/src/js/node/net.ts @@ -87,6 +87,12 @@ interface SocketInternals { } type SocketInstance = Omit & SocketInternals; +// $toClass installs Socket's Duplex prototype at runtime, which the constructor's +// TypeScript instance type does not reflect. +function isNodeSocket(value: unknown): value is SocketInstance { + return value instanceof Socket; +} + interface TLSSocketInternals { servername: string | false | undefined; alpnProtocol: string | false | null; @@ -596,7 +602,7 @@ function attachTLSEngine(self, connection, options) { // Native attachment also needs the JS owner's original transport read error. connection.once(kReadError, onUpgradedReadError.bind(null, self)); let initialData; - if (connection instanceof Socket) { + if (isNodeSocket(connection)) { const readableLength = connection.readableLength; if (readableLength) initialData = connection.read(readableLength) || undefined; if ( @@ -4702,7 +4708,7 @@ function closeTransportAndJoin(self, handle, isException, token, onClose) { !self._closeAfterHandlingError && !self.resetAndClosing; const transport = finishedNative ? token.connection : transportToDestroy(self); - const join = transport instanceof Socket && (finishedNative ? !token.closeEmitted : !handle); + const join = isNodeSocket(transport) && (finishedNative ? !token.closeEmitted : !handle); // The existing observer also completes a join requested by an earlier close listener. // A new once('close') here would miss that in-progress emission. if (join) { From 20b21a6c45a9e0cc18712ff30470db44de24aef7 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 27 Oct 2026 12:11:01 +0800 Subject: [PATCH 07/74] test(tls): deliver peer FIN and expose failed close traces --- test/js/node/tls/node-tls-connect.test.ts | 16 ++++++++-------- .../tls/tls-half-close-transport-fixture.cjs | 1 + 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/test/js/node/tls/node-tls-connect.test.ts b/test/js/node/tls/node-tls-connect.test.ts index 6b8796d6477a..ce04d7d57ca0 100644 --- a/test/js/node/tls/node-tls-connect.test.ts +++ b/test/js/node/tls/node-tls-connect.test.ts @@ -2269,7 +2269,7 @@ describe("a TLS socket over a Duplex transport follows that transport's teardown await once(server.listen(0, "127.0.0.1"), "listening"); const transport = net.connect({ port: (server.address() as AddressInfo).port, host: "127.0.0.1" }); await once(transport, "connect"); - // A held write sends the wrap to the engine that runs over the stream. + // Keep a plain write queued across the TLS wrap. transport.cork(); transport.write("x"); const client = tls.connect({ socket: transport, rejectUnauthorized: false }); @@ -2443,13 +2443,10 @@ describe("a TLS socket over a Duplex transport follows that transport's teardown }); it.each([true, false])( - "a connected net.Socket with allowHalfOpen: %p is closed with the socket", + "a peer FIN closes a wrapped net.Socket with allowHalfOpen: %p", async allowHalfOpen => { - // A half-open socket does not close itself after its EOF, and the TLS - // socket does not destroy a net.Socket. So the close of the TLS socket - // has to end it, also when no engine exists yet. A FIN over loopback - // arrives after the engine started, so the EOF is pushed by hand. const peerSawFin = Promise.withResolvers(); + const acceptedPeer = Promise.withResolvers(); await using server = net.createServer({ allowHalfOpen: true }, peer => { peer.on("error", () => {}); peer.resume(); @@ -2457,6 +2454,7 @@ describe("a TLS socket over a Duplex transport follows that transport's teardown peerSawFin.resolve(); peer.end(); }); + acceptedPeer.resolve(peer); }); await once(server.listen(0, "127.0.0.1"), "listening"); const transport = net.connect({ @@ -2465,13 +2463,15 @@ describe("a TLS socket over a Duplex transport follows that transport's teardown allowHalfOpen, }); await once(transport, "connect"); - // A held write sends the wrap to the engine that runs over the stream. + const peer = await acceptedPeer.promise; + // Keep a plain write queued across the TLS wrap. The EOF must come from + // the peer: push(null) only ends the transport's JavaScript readable. transport.cork(); transport.write("x"); const client = tls.connect({ socket: transport, rejectUnauthorized: false }); const lifecycle = recordLifecycle(client); const transportClosed = once(transport, "close"); - transport.push(null); + peer.end(); expect(await lifecycle).toEqual(["end", "error:ECONNRESET", "close:true"]); await Promise.all([transportClosed, peerSawFin.promise]); expect(transport.destroyed).toBe(true); diff --git a/test/js/node/tls/tls-half-close-transport-fixture.cjs b/test/js/node/tls/tls-half-close-transport-fixture.cjs index fa230cb9d74a..49acf96347b4 100644 --- a/test/js/node/tls/tls-half-close-transport-fixture.cjs +++ b/test/js/node/tls/tls-half-close-transport-fixture.cjs @@ -58,6 +58,7 @@ async function child(name) { complete, failure: failure ?? mainFailure, failedChecks: Object.keys(checks).filter(key => !checks[key]), + ...(pass ? {} : { trace: trace.slice(-64), omittedTraceEvents: Math.max(0, trace.length - 64) }), }); process.exitCode = pass ? 0 : 1; }); From 62a0f8811660641181a6269a128f8b0ecee1d564 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 27 Oct 2026 16:04:20 +0800 Subject: [PATCH 08/74] chore(tls): document unsafe lifetimes and format peer FIN test --- src/runtime/socket/UpgradedDuplex.rs | 8 +-- src/runtime/socket/socket_body.rs | 2 + test/js/node/tls/node-tls-connect.test.ts | 65 +++++++++++------------ 3 files changed, 38 insertions(+), 37 deletions(-) diff --git a/src/runtime/socket/UpgradedDuplex.rs b/src/runtime/socket/UpgradedDuplex.rs index 4d2dade24997..3e66d903b4fc 100644 --- a/src/runtime/socket/UpgradedDuplex.rs +++ b/src/runtime/socket/UpgradedDuplex.rs @@ -900,6 +900,8 @@ impl UpgradedDuplex { pub(super) fn teardown(&self) { bun_output::scoped_log!(UpgradedDuplex, "deinit"); if !self.handlers.ctx.is_null() { + // SAFETY: ctx points to the containing DuplexUpgradeContext, + // which outlives this field through Drop. let context = unsafe { &*self.handlers.ctx.cast::() }; context.teardown_native(self.origin.get().is_empty()); } @@ -1047,7 +1049,7 @@ pub(crate) fn on_write_complete<'a>( ) -> JsResult<()> { let global = cx.global(); if let Some(pointer) = host_fn::get_function_data(callee.to_js()) { - // The bound callback roots its owner; live teardown neuters this target. + // SAFETY: The bound callback roots its owner; live teardown neuters this target. let this = unsafe { &*pointer.cast::() }; let _context = this.keep_alive(); let error = error.to_js(); @@ -1089,7 +1091,7 @@ pub(crate) fn dispatch_write_complete(cx: &mut JsScope<'_>, callee: JsLocal<'_>, let Some(pointer) = host_fn::get_function_data(callee.to_js()) else { return; }; - // The queued bound dispatcher roots the owner until the numeric completion is delivered. + // SAFETY: The queued bound dispatcher roots the owner until the numeric completion is delivered. let this = unsafe { &*pointer.cast::() }; let _context = this.keep_alive(); if this.pending_writes.get() == 0 { @@ -1136,7 +1138,7 @@ pub(crate) fn on_close_js(cx: &mut JsScope<'_>, callee: JsLocal<'_>) -> JsResult pub(crate) fn dispatch_close(_: &mut JsScope<'_>, callee: JsLocal<'_>) { if let Some(pointer) = host_fn::get_function_data(callee.to_js()) { - // Live teardown neuters the target; the bound dispatcher keeps its owner traced. + // SAFETY: Live teardown neuters the target; the bound dispatcher keeps its owner traced. let this = unsafe { &*pointer.cast::() }; let _context = this.keep_alive(); this.close(); diff --git a/src/runtime/socket/socket_body.rs b/src/runtime/socket/socket_body.rs index ad03a068696f..2752cbc89415 100644 --- a/src/runtime/socket/socket_body.rs +++ b/src/runtime/socket/socket_body.rs @@ -5623,6 +5623,8 @@ pub(crate) fn upgrade_duplex_to_tls<'a>( .set(duplex_context_ref.ref_count.get() - 1); } if duplex_context_ref.ref_count.get() == 0 { + // SAFETY: No task was queued and all entry/connection pins are gone; + // this is the original allocation pointer. unsafe { DuplexUpgradeContext::deinit(duplex_context) }; } return Err(error); diff --git a/test/js/node/tls/node-tls-connect.test.ts b/test/js/node/tls/node-tls-connect.test.ts index ce04d7d57ca0..724937e873bd 100644 --- a/test/js/node/tls/node-tls-connect.test.ts +++ b/test/js/node/tls/node-tls-connect.test.ts @@ -2442,41 +2442,38 @@ describe("a TLS socket over a Duplex transport follows that transport's teardown expect(await lifecycle).toEqual(["transport close", "close:false"]); }); - it.each([true, false])( - "a peer FIN closes a wrapped net.Socket with allowHalfOpen: %p", - async allowHalfOpen => { - const peerSawFin = Promise.withResolvers(); - const acceptedPeer = Promise.withResolvers(); - await using server = net.createServer({ allowHalfOpen: true }, peer => { - peer.on("error", () => {}); - peer.resume(); - peer.on("end", () => { - peerSawFin.resolve(); - peer.end(); - }); - acceptedPeer.resolve(peer); - }); - await once(server.listen(0, "127.0.0.1"), "listening"); - const transport = net.connect({ - port: (server.address() as AddressInfo).port, - host: "127.0.0.1", - allowHalfOpen, + it.each([true, false])("a peer FIN closes a wrapped net.Socket with allowHalfOpen: %p", async allowHalfOpen => { + const peerSawFin = Promise.withResolvers(); + const acceptedPeer = Promise.withResolvers(); + await using server = net.createServer({ allowHalfOpen: true }, peer => { + peer.on("error", () => {}); + peer.resume(); + peer.on("end", () => { + peerSawFin.resolve(); + peer.end(); }); - await once(transport, "connect"); - const peer = await acceptedPeer.promise; - // Keep a plain write queued across the TLS wrap. The EOF must come from - // the peer: push(null) only ends the transport's JavaScript readable. - transport.cork(); - transport.write("x"); - const client = tls.connect({ socket: transport, rejectUnauthorized: false }); - const lifecycle = recordLifecycle(client); - const transportClosed = once(transport, "close"); - peer.end(); - expect(await lifecycle).toEqual(["end", "error:ECONNRESET", "close:true"]); - await Promise.all([transportClosed, peerSawFin.promise]); - expect(transport.destroyed).toBe(true); - }, - ); + acceptedPeer.resolve(peer); + }); + await once(server.listen(0, "127.0.0.1"), "listening"); + const transport = net.connect({ + port: (server.address() as AddressInfo).port, + host: "127.0.0.1", + allowHalfOpen, + }); + await once(transport, "connect"); + const peer = await acceptedPeer.promise; + // Keep a plain write queued across the TLS wrap. The EOF must come from + // the peer: push(null) only ends the transport's JavaScript readable. + transport.cork(); + transport.write("x"); + const client = tls.connect({ socket: transport, rejectUnauthorized: false }); + const lifecycle = recordLifecycle(client); + const transportClosed = once(transport, "close"); + peer.end(); + expect(await lifecycle).toEqual(["end", "error:ECONNRESET", "close:true"]); + await Promise.all([transportClosed, peerSawFin.promise]); + expect(transport.destroyed).toBe(true); + }); }); }); From 86589d60d41db0bc8d86c4b191bf87b7a81c75fa Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 27 Oct 2026 16:21:37 +0800 Subject: [PATCH 09/74] fix(tls): wait for generic ciphertext before transport shutdown --- src/runtime/socket/UpgradedDuplex.rs | 15 +- test/js/node/tls/node-tls-connect.test.ts | 63 +++++++ .../tls-shutdown-before-handshake-fixture.mjs | 162 +++++++++++++++++- 3 files changed, 235 insertions(+), 5 deletions(-) diff --git a/src/runtime/socket/UpgradedDuplex.rs b/src/runtime/socket/UpgradedDuplex.rs index 3e66d903b4fc..44b2bbba8e7f 100644 --- a/src/runtime/socket/UpgradedDuplex.rs +++ b/src/runtime/socket/UpgradedDuplex.rs @@ -71,7 +71,7 @@ pub(crate) struct UpgradedDuplex { /// The transport closed before the TLS engine existed (same window as /// [`Self::pending_data`]). Consumed by the queued `StartTLS` task. pub pending_close: Cell, - /// [`Self::shutdown`] arrived before the engine existed. [`Self::drain_pending`] replays it. + /// Shutdown is waiting for the engine or the generic transport's ciphertext completion. pub pending_shutdown: Cell, /// `pause_stream` paused the wrapped stream and `resume_stream` has not run since. paused_origin: Cell, @@ -793,10 +793,16 @@ impl UpgradedDuplex { _keep_alive.shutdown_native(); return; } - if self.origin.get().is_empty() || self.shutdown_queued.replace(true) { + if self.origin.get().is_empty() || self.write_failed.get() || self.shutdown_queued.get() { + return; + } + if self.pending_writes.get() > 0 { + self.pending_shutdown.set(true); return; } let Some(global) = self.global else { return }; + self.pending_shutdown.set(false); + self.shutdown_queued.set(true); self.pending_writes.set(self.pending_writes.get() + 1); let callback = bun_jsc::JSFunction::create( &global, @@ -1112,7 +1118,10 @@ pub(crate) fn dispatch_write_complete(cx: &mut JsScope<'_>, callee: JsLocal<'_>, (this.handlers.on_error)(this.handlers.ctx, error); } } else if this.transport_idle() && !this.origin.get().is_empty() { - (this.handlers.on_writable)(this.handlers.ctx); + this.drain_pending_shutdown(); + if !this.origin.get().is_empty() && !this.write_failed.get() && this.transport_idle() { + (this.handlers.on_writable)(this.handlers.ctx); + } } } diff --git a/test/js/node/tls/node-tls-connect.test.ts b/test/js/node/tls/node-tls-connect.test.ts index 724937e873bd..a20511b6b44f 100644 --- a/test/js/node/tls/node-tls-connect.test.ts +++ b/test/js/node/tls/node-tls-connect.test.ts @@ -4523,6 +4523,69 @@ describe.each([ destroyed: false, }); }); + + for (const outcome of ["success", "error", "destroy"] as const) { + it.skipIf(!exe)(`joins the held close_notify write completion: ${outcome}`, async () => { + const result = await runShutdownFixture(exe!, "duplex-end-completion", outcome); + const whileHeld = { + endCalls: 0, + transportFinishes: 0, + transportCloses: 0, + transportErrors: 0, + clientFinishes: 0, + clientCloses: 0, + clientErrors: 0, + endCallbacks: 0, + writableEnded: false, + }; + const common = { + log: [], + whileHeld, + clientErrorCode: null, + transportErrorCode: null, + endCallbackCode: null, + }; + if (outcome === "success") { + expect(result).toEqual({ + ...common, + completionTurn: whileHeld, + afterRelease: { + ...whileHeld, + endCalls: 1, + transportFinishes: 1, + clientFinishes: 1, + endCallbacks: 1, + writableEnded: true, + }, + }); + } else if (outcome === "error") { + // Node finishes the upper writable after shutdown has started, but the + // transport error must still close both streams without ending the lower. + expect(result).toEqual({ + ...common, + clientErrorCode: "EPIPE", + transportErrorCode: "EPIPE", + afterRelease: { + ...whileHeld, + transportCloses: 1, + transportErrors: 1, + clientFinishes: 1, + clientCloses: 1, + clientErrors: 1, + endCallbacks: 1, + }, + }); + } else { + const closed = { ...whileHeld, clientCloses: 1, transportCloses: 1, endCallbacks: 1 }; + expect(result).toEqual({ + ...common, + beforeRelease: closed, + afterRelease: closed, + endCallbackCode: "ERR_STREAM_DESTROYED", + }); + } + }); + } }); // Runs report(mode, version) of a fixture module in node, for every row at diff --git a/test/js/node/tls/tls-shutdown-before-handshake-fixture.mjs b/test/js/node/tls/tls-shutdown-before-handshake-fixture.mjs index 56fbd4f5951e..71f339767a99 100644 --- a/test/js/node/tls/tls-shutdown-before-handshake-fixture.mjs +++ b/test/js/node/tls/tls-shutdown-before-handshake-fixture.mjs @@ -1,7 +1,7 @@ // Shutdown shapes issued before a TLS handshake completes, as one report per // mode. Runs on bun and on node, so node-tls-connect.test.ts can assert the -// same report for both. "duplex-end-established" is the one shape issued after -// the handshake: it shares the stream transport below. +// same report for both. The established shutdown modes share the stream +// transport below and issue end() after the handshake. // // The readable side's 'end' is not part of these shapes, so the log does not // subscribe to it: bun emits an extra one after destroy() (pre-existing, it @@ -345,6 +345,164 @@ if (mode === "end" || mode === "end-strict" || mode === "destroySoon") { raw.destroy(); server.close(); report({ transportFinalCalled: transport.finalCalled, server: outcome, destroyed }); +} else if (mode === "duplex-end-completion") { + const outcome = process.argv[3]; + if (!["success", "error", "destroy"].includes(outcome)) throw new Error("unknown completion outcome"); + const failure = Promise.withResolvers(); + failure.promise.catch(() => {}); + const wait = promise => Promise.race([promise, failure.promise]); + let phase = "setup"; + const unexpectedClose = source => { + if (phase !== "error" && phase !== "destroy" && phase !== "cleanup") { + failure.reject(new Error(`${source} closed before the expected completion`)); + } + }; + const unexpectedError = (source, error) => { + if (phase !== "error" && phase !== "cleanup") { + failure.reject(new Error(`${source} failed: ${error.code ?? error.name}`)); + } + }; + const held = Promise.withResolvers(); + const clientFinished = Promise.withResolvers(); + const clientClosed = Promise.withResolvers(); + const transportFinished = Promise.withResolvers(); + const transportClosed = Promise.withResolvers(); + const peerSawCloseNotify = Promise.withResolvers(); + const counts = { + endCalls: 0, + transportFinishes: 0, + transportCloses: 0, + transportErrors: 0, + clientFinishes: 0, + clientCloses: 0, + clientErrors: 0, + endCallbacks: 0, + }; + let clientErrorCode = null; + let transportErrorCode = null; + let endCallbackCode = null; + let peer; + let raw; + let transport; + let client; + let release; + let result; + const server = tls.createServer( + { key: process.env.TLS_KEY, cert: process.env.TLS_CERT, allowHalfOpen: true, maxVersion: "TLSv1.2" }, + socket => { + peer = socket; + socket.on("error", error => unexpectedError("peer", error)); + socket.on("close", () => unexpectedClose("peer")); + socket.on("end", () => peerSawCloseNotify.resolve()); + socket.resume(); + }, + ); + // Capture the accepted socket before the handshake too, so setup failures are owned. + let accepted; + server.on("connection", socket => (accepted = socket)); + server.on("error", error => failure.reject(error)); + server.on("tlsClientError", error => unexpectedError("server handshake", error)); + try { + await wait(new Promise(resolve => server.listen(0, "127.0.0.1", resolve))); + raw = net.connect(server.address().port, "127.0.0.1"); + raw.on("error", error => unexpectedError("raw transport", error)); + raw.on("close", () => unexpectedClose("raw transport")); + await wait(once(raw, "connect")); + transport = transportOver(raw); + client = tls.connect({ socket: transport.stream, rejectUnauthorized: false, allowHalfOpen: true }); + client.on("error", error => { + counts.clientErrors++; + clientErrorCode = error.code ?? null; + unexpectedError("client", error); + }); + client.on("finish", () => { + counts.clientFinishes++; + clientFinished.resolve(); + }); + client.on("close", () => { + counts.clientCloses++; + clientClosed.resolve(); + unexpectedClose("client"); + }); + transport.stream.on("error", error => { + counts.transportErrors++; + transportErrorCode = error.code ?? null; + unexpectedError("transport", error); + }); + transport.stream.on("finish", () => { + counts.transportFinishes++; + transportFinished.resolve(); + }); + transport.stream.on("close", () => { + counts.transportCloses++; + transportClosed.resolve(); + unexpectedClose("transport"); + }); + await wait(once(client, "secureConnect")); + + const write = transport.stream._write; + transport.stream._write = function (chunk, encoding, callback) { + if (chunk[0] !== 0x15) return write.call(this, chunk, encoding, callback); + // Deliver the real TLS 1.2 alert, but leave its transport write outstanding. + write.call(this, chunk, encoding, error => { + if (error) return held.reject(error); + release = callback; + held.resolve(); + }); + }; + const end = transport.stream.end; + transport.stream.end = function (...args) { + counts.endCalls++; + return end.apply(this, args); + }; + const snapshot = () => ({ ...counts, writableEnded: transport.stream.writableEnded }); + client.end(error => { + counts.endCallbacks++; + endCallbackCode = error?.code ?? null; + }); + await wait(held.promise); + await wait(new Promise(resolve => setImmediate(resolve))); + const whileHeld = snapshot(); + let beforeRelease; + let completionTurn; + if (outcome === "destroy") { + phase = "destroy"; + client.destroy(); + await wait(Promise.all([clientClosed.promise, transportClosed.promise])); + beforeRelease = snapshot(); + } + if (outcome === "success") { + // Queue this before releasing the write. JSStreamSocket completes it in an + // Immediate; calling end() synchronously from the callback is too early. + const observer = new Promise(resolve => setImmediate(() => resolve(snapshot()))); + release(); + completionTurn = await wait(observer); + await wait(Promise.all([clientFinished.promise, transportFinished.promise, peerSawCloseNotify.promise])); + } else { + phase = outcome; + release(outcome === "error" ? Object.assign(new Error("held transport write failed"), { code: "EPIPE" }) : undefined); + if (outcome === "error") await wait(Promise.all([clientClosed.promise, transportClosed.promise])); + await wait(new Promise(resolve => setImmediate(resolve))); + } + result = { + whileHeld, + beforeRelease, + completionTurn, + afterRelease: snapshot(), + clientErrorCode, + transportErrorCode, + endCallbackCode, + }; + } finally { + phase = "cleanup"; + client?.destroy(); + transport?.stream.destroy(); + raw?.destroy(); + peer?.destroy(); + accepted?.destroy(); + await new Promise(resolve => server.close(resolve)); + } + report(result); } else if (mode === "duplex-end-established") { // After the handshake, against a server that reads the close_notify and // does not answer it (allowHalfOpen). The transport must be ended anyway. From c03aa0cefb2fbc6ef5aa9beece778933a8c3f985 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 27 Oct 2026 16:23:44 +0800 Subject: [PATCH 10/74] fix(tls): preserve verifier callbacks when checking resumed sessions --- packages/bun-usockets/src/crypto/openssl.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/bun-usockets/src/crypto/openssl.c b/packages/bun-usockets/src/crypto/openssl.c index 0a0ca2ff1c43..e824707281bc 100644 --- a/packages/bun-usockets/src/crypto/openssl.c +++ b/packages/bun-usockets/src/crypto/openssl.c @@ -1970,12 +1970,17 @@ const char *us_X509_error_code(long err) { } static int us_x509_session_chain_error(X509_STORE *store, X509_VERIFY_PARAM *param, - STACK_OF(X509) *chain, int purpose, int allow_untrusted) { + STACK_OF(X509) *chain, int purpose, int allow_untrusted, SSL *ssl) { int error = X509_V_ERR_UNSPECIFIED; X509_STORE_CTX *ctx = X509_STORE_CTX_new(); + /* Keep the full-handshake final error: its verify callback may continue the + * chain walk and needs the current SSL owner in the store context's ex_data. */ if (ctx && X509_STORE_CTX_init(ctx, store, sk_X509_value(chain, 0), chain) == 1 && + (!ssl || X509_STORE_CTX_set_ex_data(ctx, SSL_get_ex_data_X509_STORE_CTX_idx(), ssl) == 1) && X509_STORE_CTX_set_default(ctx, purpose == X509_PURPOSE_SSL_CLIENT ? "ssl_client" : "ssl_server") == 1 && X509_VERIFY_PARAM_set1(X509_STORE_CTX_get0_param(ctx), param) == 1) { + int (*verify_callback)(int, X509_STORE_CTX *) = ssl ? SSL_get_verify_callback(ssl) : NULL; + if (verify_callback) X509_STORE_CTX_set_verify_cb(ctx, verify_callback); int ok = us_x509_verify_cert(ctx, purpose); error = allow_untrusted ? us_x509_store_ctx_blocked_key_error(ctx) : X509_STORE_CTX_get_error(ctx); if (!allow_untrusted && !ok && error == X509_V_OK) error = X509_V_ERR_UNSPECIFIED; @@ -1994,7 +1999,7 @@ long us_ssl_verify_result(SSL *ssl) { if (!chain || sk_X509_num(chain) == 0) return result; ERR_set_mark(); int error = us_x509_session_chain_error(SSL_CTX_get_cert_store(SSL_get_SSL_CTX(ssl)), - SSL_get0_param(ssl), chain, SSL_is_server(ssl) ? X509_PURPOSE_SSL_CLIENT : X509_PURPOSE_SSL_SERVER, 0); + SSL_get0_param(ssl), chain, SSL_is_server(ssl) ? X509_PURPOSE_SSL_CLIENT : X509_PURPOSE_SSL_SERVER, 0, ssl); ERR_pop_to_mark(); return error == X509_V_OK ? result : error; } @@ -2023,7 +2028,7 @@ int us_ssl_ctx_session_is_trusted(SSL_CTX *ctx, const unsigned char *der, size_t } } trusted = us_x509_session_chain_error(SSL_CTX_get_cert_store(ctx), SSL_CTX_get0_param(ctx), - chain, X509_PURPOSE_SSL_SERVER, allow_untrusted) == X509_V_OK; + chain, X509_PURPOSE_SSL_SERVER, allow_untrusted, NULL) == X509_V_OK; cleanup: sk_X509_pop_free(chain, X509_free); SSL_SESSION_free(session); From 62068143e76423a8bcec4edab21becf6f7029477 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 27 Oct 2026 17:26:07 +0800 Subject: [PATCH 11/74] test(tls): qualify destruction scenarios independently --- test/js/node/tls/node-tls-connect.test.ts | 24 ++++---- .../tls/tls-destroy-transport-fixture.cjs | 57 +------------------ 2 files changed, 15 insertions(+), 66 deletions(-) diff --git a/test/js/node/tls/node-tls-connect.test.ts b/test/js/node/tls/node-tls-connect.test.ts index a20511b6b44f..d68d9728975b 100644 --- a/test/js/node/tls/node-tls-connect.test.ts +++ b/test/js/node/tls/node-tls-connect.test.ts @@ -928,8 +928,7 @@ describe("a fatal post-handshake SSL error over a Duplex", () => { expect(code).toBe(ALERT_BAD_RECORD_MAC); }); - // Known inherited gap: a fatal SSL error closes the Duplex before Node does; see the separate close-timing follow-up. - it.failing("is reported on a server socket", async () => { + it("is reported on a server socket", async () => { const result = await withFaultProxy({ connect: tlsConnect, serverOverDuplex: true }, peers => { const { client, server, toServer } = peers; const events = faultEvents(server, client); @@ -2671,11 +2670,17 @@ describe("a TLS socket over a Duplex transport reports that transport's error", expect(events).toEqual(["error:transport failed", "close destroyed=true"]); }); - it("destroying TLS destroys its transport without ending it", async () => { + it.each([ + ["isolated-immediate", 1], + ["isolated-after-check", 1], + ["consecutive-immediate-after-check", 2], + ["consecutive-after-check-immediate", 2], + ] as const)("destroying TLS destroys its transport without ending it: %s", async (scenario, expectedCases) => { // Node 24 destroys the transport without accessing end, even if destroy is a no-op. // Observe natural quiescence before cleanup so a delayed end getter cannot escape the assertion. + // Each process gets its own test budget; consecutive operations stay in the same child. await using proc = Bun.spawn({ - cmd: [bunExe(), join(import.meta.dir, "tls-destroy-transport-fixture.cjs")], + cmd: [bunExe(), join(import.meta.dir, "tls-destroy-transport-fixture.cjs"), "--scenario", scenario], env: bunEnv, stdout: "pipe", stderr: "pipe", @@ -2686,13 +2691,8 @@ describe("a TLS socket over a Duplex transport reports that transport's error", .split("\n") .map(line => JSON.parse(line)); const scenarios = rows.filter(row => row.type === "scenario-result"); - expect(scenarios.map(row => row.scenario)).toEqual([ - "isolated-immediate", - "isolated-after-check", - "consecutive-immediate-after-check", - "consecutive-after-check-immediate", - ]); - expect(scenarios.flatMap(row => row.cases)).toHaveLength(6); + expect(scenarios.map(row => row.scenario)).toEqual([scenario]); + expect(scenarios[0].cases).toHaveLength(expectedCases); for (const scenario of scenarios) { expect(scenario).toMatchObject({ pass: true, @@ -2714,7 +2714,7 @@ describe("a TLS socket over a Duplex transport reports that transport's error", }); } } - expect(rows.at(-1)).toMatchObject({ type: "parent-result", pass: true, timedOut: false }); + expect(rows.at(-1)).toMatchObject({ type: "scenario-result", scenario, pass: true }); expect(stderr).toBe(""); expect(exitCode).toBe(0); }); diff --git a/test/js/node/tls/tls-destroy-transport-fixture.cjs b/test/js/node/tls/tls-destroy-transport-fixture.cjs index b331e922cca3..27ae1d8d361b 100644 --- a/test/js/node/tls/tls-destroy-transport-fixture.cjs +++ b/test/js/node/tls/tls-destroy-transport-fixture.cjs @@ -1,6 +1,5 @@ "use strict"; -const { spawn } = require("node:child_process"); const { writeSync } = require("node:fs"); const { Duplex } = require("node:stream"); const tls = require("node:tls"); @@ -25,52 +24,6 @@ function deferred() { return { promise, resolve }; } -async function parent() { - const results = []; - let active; - let timedOut = false; - output({ - type: "runtime", - version: process.version, - versions: process.versions, - platform: process.platform, - arch: process.arch, - }); - const watchdog = setTimeout(() => { - timedOut = true; - output({ type: "watchdog-failure", milliseconds: 5000 }); - active?.kill("SIGKILL"); - }, 5000); - watchdog.unref(); - for (const scenario of Object.keys(scenarios)) { - if (timedOut) break; - results.push( - await new Promise(resolve => { - let stderrBytes = 0; - let spawnError = null; - active = spawn(process.execPath, [__filename, "--scenario", scenario], { stdio: ["ignore", "pipe", "pipe"] }); - active.stdout.on("data", chunk => writeSync(1, chunk)); - active.stderr.on("data", chunk => { - stderrBytes += chunk.length; - writeSync(2, chunk); - }); - active.on("error", error => { - spawnError = error.message; - }); - active.on("close", (code, signal) => resolve({ scenario, code, signal, stderrBytes, spawnError })); - }), - ); - active = undefined; - } - clearTimeout(watchdog); - const pass = - !timedOut && - results.length === Object.keys(scenarios).length && - results.every(result => result.code === 0 && !result.signal && !result.stderrBytes && !result.spawnError); - output({ type: "parent-result", pass, timedOut, results }); - process.exitCode = pass ? 0 : 1; -} - function child(scenario) { const records = []; const errorOwners = new Map(); @@ -310,11 +263,7 @@ function child(scenario) { }); } -if (process.argv[2] === "--scenario" && Object.hasOwn(scenarios, process.argv[3])) { - child(process.argv[3]); -} else { - parent().catch(error => { - writeSync(2, String(error) + "\n"); - process.exitCode = 1; - }); +if (process.argv.length !== 4 || process.argv[2] !== "--scenario" || !Object.hasOwn(scenarios, process.argv[3])) { + throw new Error("Expected --scenario with one known scenario name"); } +child(process.argv[3]); From 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 28 Oct 2026 12:42:51 +0800 Subject: [PATCH 12/74] test(tls): collect completion owners before module re-entry --- test/js/bun/net/socket-retention.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/js/bun/net/socket-retention.test.ts b/test/js/bun/net/socket-retention.test.ts index 930e10adc343..d88e3fa76237 100644 --- a/test/js/bun/net/socket-retention.test.ts +++ b/test/js/bun/net/socket-retention.test.ts @@ -602,7 +602,8 @@ for (const destroyBeforeCompletion of [false, true]) { Bun.gc(true); const queuedRoot = !!state.weak.deref(); await turn(); - for (let i = 0; i < 30; i++) { await turn(); Bun.gc(true); } + // Collect before module re-entry to avoid retaining the owner through stale native spill slots. + for (let i = 0; i < 30; i++) { await new Promise(resolve => setImmediate(() => { Bun.gc(true); resolve(); })); } console.log(JSON.stringify({ callbackRoot, queuedRoot, collected: !state.weak.deref(), settled: state.settled })); `, ], From 12e500e1ede2f9e95649937fad99fbee388161ca Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 31 Oct 2026 14:04:02 +0800 Subject: [PATCH 13/74] ci(tls): inspect symbols from failing qualification artifacts --- .github/workflows/openclaw-tls-symbols.yml | 170 +++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 .github/workflows/openclaw-tls-symbols.yml diff --git a/.github/workflows/openclaw-tls-symbols.yml b/.github/workflows/openclaw-tls-symbols.yml new file mode 100644 index 000000000000..dc78c9b46713 --- /dev/null +++ b/.github/workflows/openclaw-tls-symbols.yml @@ -0,0 +1,170 @@ +name: OpenClaw TLS symbol inspection + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-symbols.yml + workflow_dispatch: + +permissions: + actions: read + contents: read + +concurrency: + group: openclaw-tls-symbols-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +# Inspect the exact failing PR #215 artifacts without rebuilding or running them. +env: + SOURCE_COMMIT: 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e + SOURCE_RUN: "40897692970" + +jobs: + inspect: + name: Inspect ${{ matrix.target }} symbols + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - target: linux-arm64 + runner: ubuntu-24.04-arm + triplet: bun-linux-aarch64 + artifact: "12908842070" + sha256: 65f1e6de6618b9fc2dd75c688d49fff27bcedc61fd03105f1f92c9df49e4f5c4 + - target: darwin-x64 + runner: macos-15-intel + triplet: bun-darwin-x64 + artifact: "12909560910" + sha256: 99eb0c3444b61fd6756b4879dfa5698c4a9e69706a9a1ff7f1cff685a741d1d9 + env: + TARGET: ${{ matrix.target }} + TRIPLET: ${{ matrix.triplet }} + ARTIFACT_ID: ${{ matrix.artifact }} + ARTIFACT_SHA256: ${{ matrix.sha256 }} + steps: + - name: Fetch and verify frozen artifact + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir diagnostics + gh api "repos/openclaw/bun/actions/artifacts/$ARTIFACT_ID" > diagnostics/artifact.json + gh api "repos/openclaw/bun/actions/artifacts/$ARTIFACT_ID/zip" > artifact.zip + python3 - <<'PY' + import json, os + from pathlib import Path + + artifact = json.loads(Path("diagnostics/artifact.json").read_text()) + assert artifact["id"] == int(os.environ["ARTIFACT_ID"]), artifact + assert artifact["name"] == "build-" + os.environ["TARGET"], artifact + assert not artifact["expired"], artifact + assert artifact["workflow_run"]["id"] == int(os.environ["SOURCE_RUN"]), artifact + assert artifact["workflow_run"]["head_sha"] == os.environ["SOURCE_COMMIT"], artifact + expected = os.environ["ARTIFACT_SHA256"] + assert artifact["digest"] == "sha256:" + expected, artifact + PY + actual=$(shasum -a 256 artifact.zip | awk '{print $1}') + [ "$actual" = "$ARTIFACT_SHA256" ] + unzip -q artifact.zip -d downloaded + unzip -q "downloaded/$TRIPLET.zip" -d unpacked + unzip -q "downloaded/$TRIPLET-profile.zip" -d unpacked + shasum -a 256 "unpacked/$TRIPLET/bun" \ + "unpacked/$TRIPLET-profile/bun-profile" > diagnostics/executables.sha256 + + - name: Install Linux debugger + if: runner.os == 'Linux' + shell: bash + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends gdb binutils + + - name: Inspect Linux build identity and types + if: runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + [ "$(uname -m)" = aarch64 ] + exe="unpacked/$TRIPLET/bun" + profile="unpacked/$TRIPLET-profile/bun-profile" + readelf -n "$exe" > diagnostics/executable-notes.txt + readelf -n "$profile" > diagnostics/profile-notes.txt + id=$(sed -n 's/.*Build ID: //p' diagnostics/executable-notes.txt) + profile_id=$(sed -n 's/.*Build ID: //p' diagnostics/profile-notes.txt) + [ -n "$id" ] && [ "$id" = "$profile_id" ] + readelf -SW "$exe" > diagnostics/executable-sections.txt + readelf -SW "$profile" > diagnostics/profile-sections.txt + gdb --batch --nx --init-eval-command='set auto-load off' \ + -ex "file $profile" -ex "exec-file $exe" \ + -ex 'info functions JSC::Collector::runEndPhase' \ + -ex 'info functions JSC::MachineThreads::gatherConservativeRoots' \ + > diagnostics/capture-functions.txt 2>&1 + for type in JSCell Heap SlotVisitor ConservativeRoots; do + if gdb --batch --nx --init-eval-command='set auto-load off' \ + -ex "file $exe" -ex "symbol-file $profile" -ex "ptype /o JSC::$type" \ + > "diagnostics/type-$type.txt" 2>&1; then + echo "$type: available" >> diagnostics/types.txt + else + echo "$type: unavailable" >> diagnostics/types.txt + fi + done + + - name: Inspect Darwin UUID and types + if: runner.os == 'macOS' + shell: bash + run: | + set -euo pipefail + [ "$(uname -m)" = x86_64 ] + exe="unpacked/$TRIPLET/bun" + profile="unpacked/$TRIPLET-profile/bun-profile" + dsym="$profile.dSYM" + xcrun dwarfdump --uuid "$exe" > diagnostics/executable-uuid.txt + xcrun dwarfdump --uuid "$profile" > diagnostics/profile-uuid.txt + xcrun dwarfdump --uuid "$dsym" > diagnostics/dsym-uuid.txt + id=$(awk '{print $2}' diagnostics/executable-uuid.txt) + [ -n "$id" ] + [ "$id" = "$(awk '{print $2}' diagnostics/profile-uuid.txt)" ] + [ "$id" = "$(awk '{print $2}' diagnostics/dsym-uuid.txt)" ] + otool -l "$exe" > diagnostics/executable-load-commands.txt + xcrun lldb --no-lldbinit --batch \ + -o 'settings set target.load-script-from-symbol-file false' \ + -o "target create $exe" -o "target symbols add $dsym" \ + -o "image lookup -rn 'JSC::Collector::runEndPhase|JSC::MachineThreads::gatherConservativeRoots'" \ + > diagnostics/capture-functions.txt 2>&1 + for type in JSCell Heap SlotVisitor ConservativeRoots; do + if xcrun lldb --no-lldbinit --batch \ + -o 'settings set target.load-script-from-symbol-file false' \ + -o "target create $exe" -o "target symbols add $dsym" \ + -o "image lookup -t JSC::$type" > "diagnostics/type-$type.txt" 2>&1; then + echo "$type: available" >> diagnostics/types.txt + else + echo "$type: unavailable" >> diagnostics/types.txt + fi + done + + - name: Report inspection scope + if: always() + shell: bash + run: | + { + echo "### TLS symbol inspection: $TARGET" + echo "Source: $SOURCE_COMMIT; build run: $SOURCE_RUN; artifact: $ARTIFACT_ID." + echo 'This inspects saved symbols without running the executable or collecting a heap.' + echo 'Type availability is a prerequisite, not proof of usable root or stack capture.' + if [ -f diagnostics/types.txt ]; then + echo '```text' + cat diagnostics/types.txt + echo '```' + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Retain symbol inspection + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: tls-symbols-${{ matrix.target }} + path: diagnostics + if-no-files-found: error + retention-days: 7 From 9d7208df26efc2e0201ba3221f87a3049bd3ca49 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 31 Oct 2026 14:13:16 +0800 Subject: [PATCH 14/74] fix(ci): complete Darwin inspection when symbol lookups are empty --- .github/workflows/openclaw-tls-symbols.yml | 35 ++++++++++++++-------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/.github/workflows/openclaw-tls-symbols.yml b/.github/workflows/openclaw-tls-symbols.yml index dc78c9b46713..3a5dfe8111a1 100644 --- a/.github/workflows/openclaw-tls-symbols.yml +++ b/.github/workflows/openclaw-tls-symbols.yml @@ -128,21 +128,32 @@ jobs: [ "$id" = "$(awk '{print $2}' diagnostics/profile-uuid.txt)" ] [ "$id" = "$(awk '{print $2}' diagnostics/dsym-uuid.txt)" ] otool -l "$exe" > diagnostics/executable-load-commands.txt + cat > diagnostics/inspect-symbols.py <<'PY' + import lldb + + result = lldb.SBCommandReturnObject() + lldb.debugger.GetCommandInterpreter().HandleCommand( + "image lookup -rn 'JSC::Collector::runEndPhase|JSC::MachineThreads::gatherConservativeRoots'", result) + output = result.GetOutput() or "" + error = result.GetError() or "" + with open("diagnostics/capture-functions.txt", "w") as report: + report.write(output + error or "no matching functions\n") + report.write(f"lookup succeeded: {result.Succeeded()}\n") + if error: + raise RuntimeError(error) + target = lldb.debugger.GetSelectedTarget() + with open("diagnostics/types.txt", "w") as report: + for name in ("JSCell", "Heap", "SlotVisitor", "ConservativeRoots"): + kind = target.FindFirstType("JSC::" + name) + complete = kind.IsValid() and kind.IsTypeComplete() + report.write(f"{name}: complete={complete}, bytes={kind.GetByteSize()}, fields={kind.GetNumberOfFields()}\n") + PY xcrun lldb --no-lldbinit --batch \ -o 'settings set target.load-script-from-symbol-file false' \ -o "target create $exe" -o "target symbols add $dsym" \ - -o "image lookup -rn 'JSC::Collector::runEndPhase|JSC::MachineThreads::gatherConservativeRoots'" \ - > diagnostics/capture-functions.txt 2>&1 - for type in JSCell Heap SlotVisitor ConservativeRoots; do - if xcrun lldb --no-lldbinit --batch \ - -o 'settings set target.load-script-from-symbol-file false' \ - -o "target create $exe" -o "target symbols add $dsym" \ - -o "image lookup -t JSC::$type" > "diagnostics/type-$type.txt" 2>&1; then - echo "$type: available" >> diagnostics/types.txt - else - echo "$type: unavailable" >> diagnostics/types.txt - fi - done + -o 'script exec(open("diagnostics/inspect-symbols.py").read())' \ + > diagnostics/debugger.txt 2>&1 + [ "$(wc -l < diagnostics/types.txt)" -eq 4 ] - name: Report inspection scope if: always() From a2e411a57754f36bf2d040b2773c6a71cd09e54c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 31 Oct 2026 14:27:24 +0800 Subject: [PATCH 15/74] test(tls): compare dropped cycles against published baseline --- .../openclaw-tls-retention-baseline.yml | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 .github/workflows/openclaw-tls-retention-baseline.yml diff --git a/.github/workflows/openclaw-tls-retention-baseline.yml b/.github/workflows/openclaw-tls-retention-baseline.yml new file mode 100644 index 000000000000..4c6848ae0ebd --- /dev/null +++ b/.github/workflows/openclaw-tls-retention-baseline.yml @@ -0,0 +1,124 @@ +name: OpenClaw TLS retention baseline + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-retention-baseline.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: openclaw-tls-retention-baseline-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + baseline: + name: Baseline ${{ matrix.target }} dropped cycle + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - target: darwin-x64 + runner: macos-15-intel + asset: "679970945" + - target: windows-x64 + runner: windows-2025 + asset: "679971471" + - target: windows-arm64 + runner: windows-11-arm + asset: "679971443" + env: + TARGET: ${{ matrix.target }} + ASSET_ID: ${{ matrix.asset }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e + persist-credentials: false + + - name: Download published baseline + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir diagnostics + gh api repos/openclaw/bun/releases/assets/679971474 \ + -H 'Accept: application/octet-stream' > diagnostics/manifest.json + gh api "repos/openclaw/bun/releases/assets/$ASSET_ID" \ + -H 'Accept: application/octet-stream' > baseline.zip + + - name: Verify baseline and run the exact dropped-cycle case + shell: bash + run: | + set -euo pipefail + python=python3 + command -v python3 >/dev/null || python=python + "$python" - <<'PY' + import hashlib, json, os, subprocess, sys, zipfile + from pathlib import Path + from xml.etree import ElementTree + + def sha256(path): + digest = hashlib.sha256() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + report = Path("diagnostics").resolve() + manifest_path = report / "manifest.json" + assert sha256(manifest_path) == "8c3a9d87f37832fa07173635af52d53b813e29470afa4afdb49a83033fed039e" + manifest = json.loads(manifest_path.read_text()) + commit = "191755ec115ece3dd642eba3cb0143cecb5e5d7a" + webkit = "b82bc917e99110b61c3a21202d6def7457aa194c" + assert manifest["bun"]["commit"] == commit + assert manifest["webkit"]["version"] == webkit + matches = [asset for asset in manifest["assets"] if asset["target"] == os.environ["TARGET"]] + assert len(matches) == 1 + asset = matches[0] + archive = Path("baseline.zip") + assert sha256(archive) == asset["sha256"] + with zipfile.ZipFile(archive) as bundle: + bundle.extractall("baseline") + exe = (Path("baseline") / asset["executable"]["path"]).resolve() + assert sha256(exe) == asset["executable"]["sha256"] + exe.chmod(exe.stat().st_mode | 0o111) + identity = subprocess.run([str(exe), "-e", "console.log(JSON.stringify({commit:Bun.revision,webkit:process.versions.webkit,platform:process.platform,arch:process.arch}))"], + text=True, capture_output=True, check=True) + facts = json.loads(identity.stdout) + (report / "identity.json").write_text(json.dumps(facts, indent=2) + "\n") + platform, arch = os.environ["TARGET"].split("-") + assert facts == {"commit": commit, "webkit": webkit, + "platform": "win32" if platform == "windows" else platform, "arch": arch}, facts + fixture = Path("test/js/bun/net/socket-retention.test.ts") + assert sha256(fixture) == "d4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2" + name = "TLS Duplex cycles collect after startup while a retained origin keeps its wrapper alive" + with (report / "stdout.txt").open("w") as stdout, (report / "stderr.txt").open("w") as stderr: + result = subprocess.run([str(exe), "test", str(fixture), "--test-name-pattern", "^" + name + "$", + "--reporter=junit", "--reporter-outfile=" + str(report / "result.xml")], stdout=stdout, stderr=stderr) + tree = ElementTree.parse(report / "result.xml") + selected = [case for case in tree.iter("testcase") if case.get("name") == name] + assert len(selected) == 1 and selected[0].find("skipped") is None, "selected case did not execute" + case = selected[0] + outcome = {"exitCode": result.returncode, "name": name, + "failure": case.find("failure") is not None, "error": case.find("error") is not None} + (report / "outcome.json").write_text(json.dumps(outcome, indent=2) + "\n") + print(json.dumps(outcome)) + if result.returncode == 0: + assert not outcome["failure"] and not outcome["error"], outcome + sys.exit(result.returncode) + PY + + - name: Retain baseline evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: tls-retention-baseline-${{ matrix.target }} + path: diagnostics + if-no-files-found: error + retention-days: 7 From dc683a462eb90a5c5b22abb1a85dd25639cfc5b6 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 31 Oct 2026 14:35:37 +0800 Subject: [PATCH 16/74] ci(tls): reproduce ARM64 completion retention with full symbols --- .../workflows/openclaw-tls-fullsymbols.yml | 188 ++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 .github/workflows/openclaw-tls-fullsymbols.yml diff --git a/.github/workflows/openclaw-tls-fullsymbols.yml b/.github/workflows/openclaw-tls-fullsymbols.yml new file mode 100644 index 000000000000..c9c3ce5df98b --- /dev/null +++ b/.github/workflows/openclaw-tls-fullsymbols.yml @@ -0,0 +1,188 @@ +name: OpenClaw TLS full-symbol reproduction + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-fullsymbols.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: openclaw-tls-fullsymbols-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + linux-arm64: + runs-on: ubuntu-24.04-arm + timeout-minutes: 340 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e + path: source + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: openclaw/WebKit + ref: b82bc917e99110b61c3a21202d6def7457aa194c + path: webkit + persist-credentials: false + - name: Prepare native diagnostic build + shell: bash + run: | + set -euo pipefail + test "$(uname -m)" = aarch64 + test "$(git -C source rev-parse HEAD)" = 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e + test "$(git -C webkit rev-parse HEAD)" = b82bc917e99110b61c3a21202d6def7457aa194c + echo 'd4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2 source/test/js/bun/net/socket-retention.test.ts' | sha256sum --check + mkdir diagnostics + printf '%s\n' 'Bun=454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e' \ + 'WebKit=b82bc917e99110b61c3a21202d6def7457aa194c' \ + 'Diagnostic variance: non-LTO C++/JSC; Debian 13 system ICU; Rust-only release LTO may remain enabled.' > diagnostics/inputs.txt + cat > diagnostics/probe.py <<'PY' + import gdb, json + gdb.execute("set breakpoint pending off") + types = {} + for name in ("JSC::JSCell", "JSC::Heap", "JSC::SlotVisitor", "JSC::ConservativeRoots"): + value = gdb.lookup_type(name).strip_typedefs() + fields = value.fields() + assert value.sizeof > 0 and fields, "incomplete type: " + name + types[name] = {"bytes": value.sizeof, "fields": len(fields)} + functions = {} + for name in ("JSC::Collector::runEndPhase", "JSC::MachineThreads::gatherConservativeRoots"): + point = gdb.Breakpoint(name) + addresses = [location.address for location in point.locations] + assert addresses and all(addresses), "unresolved function: " + name + functions[name] = addresses + print(json.dumps({"types": types, "functions": functions}, indent=2)) + PY + # Only this disposable hosted runner is affected; leave Docker images alone. + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup /usr/local/share/boost /usr/share/swift + df -h / + - uses: ./source/.github/actions/setup-bun + with: + bun-version: "1.4.4" + - name: Generate common toolchain provisioning + id: provision + shell: bash + run: | + set -euo pipefail + # No target sysroot: local WebKit uses system ICU in this container. + script=$(bun -e 'import { resolve } from "node:path"; import { provision } from "./source/scripts/openclaw-release/release.ts"; console.log(provision(resolve("source"), []));' | tail -1) + echo "script=${script#"$GITHUB_WORKSPACE"/}" >> "$GITHUB_OUTPUT" + - name: Provision native ARM64 container + shell: bash + env: + SCRIPT: ${{ steps.provision.outputs.script }} + run: | + set -euo pipefail + for attempt in 1 2 3 4; do + docker rm --force tls-symbols > /dev/null 2>&1 || true + docker run --detach --name tls-symbols -e CI=true -v "$GITHUB_WORKSPACE:/work" -w /work \ + docker.io/library/debian:13@sha256:9cc080028c43b27d2074d63a5f9caf7166d731494965616c1a6d2827a004585c \ + sleep infinity > /dev/null + if docker exec -e SCRIPT tls-symbols sh -euc \ + 'sh "$SCRIPT" 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e openclaw-tls-diagnostic && apt-get update && apt-get install --yes --no-install-recommends libicu-dev libjson-pp-perl gdb binutils zstd'; then + exit 0 + fi + echo "::warning::Provisioning attempt $attempt failed" + [ "$attempt" = 4 ] && exit 1 + sleep $((attempt * 120)) + done + - name: Build release with full C++ and JSC symbols + shell: bash + run: | + set -euo pipefail + docker exec tls-symbols bash -euo pipefail -c ' + . /etc/profile.d/bun-ci.sh + git config --global --add safe.directory /work/source + git config --global --add safe.directory /work/webkit + { bun --version; node --version; clang --version; cmake --version; rustc --version; gdb --version; dpkg-query -W libicu-dev; } > /work/diagnostics/tools.txt + cd /work/source + export BUN_WEBKIT_PATH=/work/webkit CMAKE_BUILD_PARALLEL_LEVEL=2 + bun run build:release --ci=on --webkit=local --lto=off --os=linux --arch=aarch64 --abi=gnu --build-dir=build/tls-arm64-fullsymbols -j2 + ' + - name: Verify matched symbols and capture capability + shell: bash + run: | + set -euo pipefail + docker exec tls-symbols bash -euo pipefail -c ' + cd /work/source + build=build/tls-arm64-fullsymbols + readelf -n "$build/bun" > /work/diagnostics/bun-build-id.txt + readelf -n "$build/bun-profile" > /work/diagnostics/profile-build-id.txt + python3 - <<'"'"'PY'"'"' + import re + from pathlib import Path + report = Path("/work/diagnostics") + ids = [re.findall(r"Build ID: (\w+)", (report / name).read_text()) for name in ("bun-build-id.txt", "profile-build-id.txt")] + assert len(ids[0]) == 1 and ids[0] == ids[1], ids + PY + gdb --batch --nx --init-eval-command="set auto-load off" -ex "file $build/bun-profile" -ex "python exec(compile(open(\"/work/diagnostics/probe.py\").read(), \"probe.py\", \"exec\"))" > /work/diagnostics/symbol-probe.txt 2>&1 + ' + - name: Run the unchanged destroyed-completion case + shell: bash + run: | + set -euo pipefail + docker exec tls-symbols bash -euo pipefail -c ' + cd /work/source + python3 - <<'"'"'PY'"'"' + import hashlib, json, subprocess, sys + from pathlib import Path + from xml.etree import ElementTree + report = Path("/work/diagnostics") + fixture = Path("test/js/bun/net/socket-retention.test.ts") + assert hashlib.sha256(fixture.read_bytes()).hexdigest() == "d4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2" + exe = Path("build/tls-arm64-fullsymbols/bun").resolve() + identity = subprocess.run([str(exe), "-e", "console.log(JSON.stringify({commit:Bun.revision,webkit:process.versions.webkit,platform:process.platform,arch:process.arch}))"], text=True, capture_output=True, check=True) + facts = json.loads(identity.stdout) + (report / "identity.json").write_text(json.dumps(facts, indent=2) + "\n") + assert facts == {"commit": "454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e", "webkit": "b82bc917e99110b61c3a21202d6def7457aa194c", "platform": "linux", "arch": "arm64"}, facts + name = "a TLS Duplex completion retains its owner and is inert after destroy" + with (report / "stdout.txt").open("w") as stdout, (report / "stderr.txt").open("w") as stderr: + result = subprocess.run([str(exe), "test", str(fixture), "--test-name-pattern", "^" + name + "$", "--reporter=junit", "--reporter-outfile=" + str(report / "result.xml")], stdout=stdout, stderr=stderr) + cases = [case for case in ElementTree.parse(report / "result.xml").iter("testcase") if case.get("name") == name] + assert len(cases) == 1 and cases[0].find("skipped") is None, "selected case did not execute" + outcome = {"exitCode": result.returncode, "name": name, "failure": cases[0].find("failure") is not None, "error": cases[0].find("error") is not None} + (report / "outcome.json").write_text(json.dumps(outcome, indent=2) + "\n") + print(json.dumps(outcome)) + if result.returncode == 0: + assert not outcome["failure"] and not outcome["error"], outcome + sys.exit(result.returncode) + PY + ' + - name: Retain matched binary and split DWARF inputs + if: always() + shell: bash + run: | + set -euo pipefail + sudo chown -R "$(id -u):$(id -g)" diagnostics + if [ -f source/build/tls-arm64-fullsymbols/bun-profile ]; then + python3 - <<'PY' + from pathlib import Path + import subprocess + root = Path("source/build/tls-arm64-fullsymbols") + files = [root / "bun", root / "bun-profile", *root.rglob("*.dwo")] + files = [p for p in files if p.is_file()] + assert len(files) <= 20000 and sum(p.stat().st_size for p in files) <= 8 * 1024**3, "artifact-budget-exceeded" + with Path("diagnostics/symbol-inputs.txt").open("w") as out: + out.write("\n".join(str(p) for p in files) + "\n") + subprocess.run(["tar", "--zstd", "-cf", "diagnostics/matched-symbols.tar.zst", "-T", "diagnostics/symbol-inputs.txt"], check=True) + PY + find source/build/tls-arm64-fullsymbols -name CMakeCache.txt -exec sh -c 'for cache; do printf "\n%s\n" "$cache"; cat "$cache"; done' sh {} + > diagnostics/cmake-caches.txt + sha256sum source/build/tls-arm64-fullsymbols/bun source/build/tls-arm64-fullsymbols/bun-profile diagnostics/matched-symbols.tar.zst > diagnostics/SHA256SUMS + fi + - name: Upload diagnostic evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: tls-linux-arm64-fullsymbols + path: diagnostics + if-no-files-found: error + retention-days: 7 + compression-level: 0 + - name: Stop diagnostic container + if: always() + run: docker rm --force tls-symbols > /dev/null From f1040ae65d4c9990b74912933c643513438542ea Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 31 Oct 2026 15:48:55 +0800 Subject: [PATCH 17/74] ci(tls): reuse retained ARM64 symbols for native diagnosis * ci(tls): reuse matched symbols from the retained ARM64 build * ci(tls): inspect retained symbols with LLVM 23 --- .../workflows/openclaw-tls-fullsymbols.yml | 232 +++++++++++------- 1 file changed, 147 insertions(+), 85 deletions(-) diff --git a/.github/workflows/openclaw-tls-fullsymbols.yml b/.github/workflows/openclaw-tls-fullsymbols.yml index c9c3ce5df98b..5bd669451075 100644 --- a/.github/workflows/openclaw-tls-fullsymbols.yml +++ b/.github/workflows/openclaw-tls-fullsymbols.yml @@ -1,4 +1,4 @@ -name: OpenClaw TLS full-symbol reproduction +name: OpenClaw TLS saved-symbol reproduction on: pull_request: @@ -7,103 +7,167 @@ on: workflow_dispatch: permissions: + actions: read contents: read concurrency: - group: openclaw-tls-fullsymbols-${{ github.head_ref || github.ref }} + group: openclaw-tls-saved-symbols-${{ github.head_ref || github.ref }} cancel-in-progress: true jobs: linux-arm64: runs-on: ubuntu-24.04-arm - timeout-minutes: 340 + timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e path: source persist-credentials: false - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: openclaw/WebKit - ref: b82bc917e99110b61c3a21202d6def7457aa194c - path: webkit - persist-credentials: false - - name: Prepare native diagnostic build + - name: Prepare saved diagnostic inputs shell: bash run: | set -euo pipefail test "$(uname -m)" = aarch64 test "$(git -C source rev-parse HEAD)" = 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e - test "$(git -C webkit rev-parse HEAD)" = b82bc917e99110b61c3a21202d6def7457aa194c echo 'd4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2 source/test/js/bun/net/socket-retention.test.ts' | sha256sum --check mkdir diagnostics printf '%s\n' 'Bun=454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e' \ 'WebKit=b82bc917e99110b61c3a21202d6def7457aa194c' \ 'Diagnostic variance: non-LTO C++/JSC; Debian 13 system ICU; Rust-only release LTO may remain enabled.' > diagnostics/inputs.txt cat > diagnostics/probe.py <<'PY' - import gdb, json - gdb.execute("set breakpoint pending off") + import json, lldb, re + from pathlib import Path + target = lldb.debugger.GetSelectedTarget() + assert target.IsValid(), "missing diagnostic target" + module = target.FindModule(target.GetExecutable()) + assert module.IsValid(), "missing executable module" + uuid = re.sub("[^0-9a-fA-F]", "", module.GetUUIDString() or "").lower() + expected = re.findall(r"Build ID: (\w+)", Path("/work/diagnostics/bun-build-id.txt").read_text()) + assert len(expected) == 1 and uuid == expected[0], "LLDB module identity mismatch" types = {} for name in ("JSC::JSCell", "JSC::Heap", "JSC::SlotVisitor", "JSC::ConservativeRoots"): - value = gdb.lookup_type(name).strip_typedefs() - fields = value.fields() - assert value.sizeof > 0 and fields, "incomplete type: " + name - types[name] = {"bytes": value.sizeof, "fields": len(fields)} + candidates = target.FindTypes(name) + complete = [] + for i in range(candidates.GetSize()): + value = candidates.GetTypeAtIndex(i).GetCanonicalType() + fields = value.GetNumberOfFields() + if value.IsTypeComplete() and value.GetByteSize() > 0 and fields > 0: + layout = [{"name": value.GetFieldAtIndex(j).GetName(), "offset_bits": value.GetFieldAtIndex(j).GetOffsetInBits()} for j in range(fields)] + complete.append({"bytes": value.GetByteSize(), "fields": fields, "layout": layout}) + assert complete, "incomplete type: " + name + types[name] = complete functions = {} for name in ("JSC::Collector::runEndPhase", "JSC::MachineThreads::gatherConservativeRoots"): - point = gdb.Breakpoint(name) - addresses = [location.address for location in point.locations] - assert addresses and all(addresses), "unresolved function: " + name - functions[name] = addresses - print(json.dumps({"types": types, "functions": functions}, indent=2)) + contexts = module.FindFunctions(name, lldb.eFunctionNameTypeAny) + functions[name] = [] + for i in range(contexts.GetSize()): + context = contexts.GetContextAtIndex(i) + function = context.GetFunction() + unit = context.GetCompileUnit() + if not function.IsValid() or not unit.IsValid(): + continue + address = function.GetStartAddress().GetFileAddress() + if address not in (0, lldb.LLDB_INVALID_ADDRESS): + functions[name].append({"address": address, "name": function.GetName(), "compile_unit": str(unit.GetFileSpec())}) + assert functions[name], "missing debug function: " + name + Path("/work/diagnostics/symbol-capability.json").write_text(json.dumps({"module_uuid": uuid, "types": types, "functions": functions}, indent=2) + "\n") PY - # Only this disposable hosted runner is affected; leave Docker images alone. - sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup /usr/local/share/boost /usr/share/swift - df -h / - - uses: ./source/.github/actions/setup-bun - with: - bun-version: "1.4.4" - - name: Generate common toolchain provisioning - id: provision - shell: bash - run: | - set -euo pipefail - # No target sysroot: local WebKit uses system ICU in this container. - script=$(bun -e 'import { resolve } from "node:path"; import { provision } from "./source/scripts/openclaw-release/release.ts"; console.log(provision(resolve("source"), []));' | tail -1) - echo "script=${script#"$GITHUB_WORKSPACE"/}" >> "$GITHUB_OUTPUT" - - name: Provision native ARM64 container + - name: Fetch the frozen failed build shell: bash env: - SCRIPT: ${{ steps.provision.outputs.script }} + GH_TOKEN: ${{ github.token }} run: | set -euo pipefail - for attempt in 1 2 3 4; do - docker rm --force tls-symbols > /dev/null 2>&1 || true - docker run --detach --name tls-symbols -e CI=true -v "$GITHUB_WORKSPACE:/work" -w /work \ - docker.io/library/debian:13@sha256:9cc080028c43b27d2074d63a5f9caf7166d731494965616c1a6d2827a004585c \ - sleep infinity > /dev/null - if docker exec -e SCRIPT tls-symbols sh -euc \ - 'sh "$SCRIPT" 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e openclaw-tls-diagnostic && apt-get update && apt-get install --yes --no-install-recommends libicu-dev libjson-pp-perl gdb binutils zstd'; then - exit 0 - fi - echo "::warning::Provisioning attempt $attempt failed" - [ "$attempt" = 4 ] && exit 1 - sleep $((attempt * 120)) - done - - name: Build release with full C++ and JSC symbols + gh api repos/openclaw/bun/actions/runs/41177263900 > diagnostics/producer-run.json + gh api repos/openclaw/bun/actions/artifacts/13034855054 > diagnostics/producer-artifact.json + gh api repos/openclaw/bun/actions/artifacts/13034855054/zip > artifact.zip + python3 - <<'PYCODE' + import hashlib, json, stat, zipfile + from pathlib import Path, PurePosixPath + artifact = json.loads(Path("diagnostics/producer-artifact.json").read_text()) + run = json.loads(Path("diagnostics/producer-run.json").read_text()) + assert run["id"] == 41177263900 and run["conclusion"] == "failure" + assert run["head_sha"] == "dc683a462eb90a5c5b22abb1a85dd25639cfc5b6" + assert artifact["id"] == 13034855054 and not artifact["expired"] + assert artifact["name"] == "tls-linux-arm64-fullsymbols" and artifact["size_in_bytes"] == 319038169 + assert artifact["workflow_run"]["id"] == run["id"] and artifact["workflow_run"]["head_sha"] == run["head_sha"] + digest = "d4ebf132092bb9c3a6de65fd848523200aaff74e36cd09ecf60f1a5fc69abbb3" + assert artifact["digest"] == "sha256:" + digest + with Path("artifact.zip").open("rb") as archive: + assert hashlib.file_digest(archive, "sha256").hexdigest() == digest + with zipfile.ZipFile("artifact.zip") as archive: + entries = archive.infolist() + assert len(entries) <= 20 and sum(entry.file_size for entry in entries) <= 8 * 1024**3 + seen = set() + for entry in entries: + name = PurePosixPath(entry.filename) + assert not name.is_absolute() and len(name.parts) == 1 and name.name not in ("", ".", "..") + assert not entry.is_dir() and not stat.S_ISLNK(entry.external_attr >> 16) and name.name not in seen + seen.add(name.name) + target = Path("producer") / name.name + target.parent.mkdir(exist_ok=True) + with target.open("xb") as output, archive.open(entry) as content: + import shutil + shutil.copyfileobj(content, output) + sums = dict(line.split(None, 1)[::-1] for line in Path("producer/SHA256SUMS").read_text().splitlines()) + assert set(sums) == {"source/build/tls-arm64-fullsymbols/bun", "source/build/tls-arm64-fullsymbols/bun-profile", "diagnostics/matched-symbols.tar.zst"} + with Path("producer/matched-symbols.tar.zst").open("rb") as archive: + assert hashlib.file_digest(archive, "sha256").hexdigest() == sums["diagnostics/matched-symbols.tar.zst"] + Path("diagnostics/producer-policy.txt").write_text("Producer binary policy FAILED: extra static initializer init_have_sme. These saved non-LTO/system-ICU inputs are diagnostic only; no shipping gate is waived.\n") + PYCODE + - name: Restore matched inputs in the original container layout shell: bash run: | set -euo pipefail - docker exec tls-symbols bash -euo pipefail -c ' - . /etc/profile.d/bun-ci.sh - git config --global --add safe.directory /work/source - git config --global --add safe.directory /work/webkit - { bun --version; node --version; clang --version; cmake --version; rustc --version; gdb --version; dpkg-query -W libicu-dev; } > /work/diagnostics/tools.txt - cd /work/source - export BUN_WEBKIT_PATH=/work/webkit CMAKE_BUILD_PARALLEL_LEVEL=2 - bun run build:release --ci=on --webkit=local --lto=off --os=linux --arch=aarch64 --abi=gnu --build-dir=build/tls-arm64-fullsymbols -j2 + docker run --detach --name tls-symbols -e CI=true -v "$GITHUB_WORKSPACE:/work" -w /work \ + docker.io/library/debian:13@sha256:9cc080028c43b27d2074d63a5f9caf7166d731494965616c1a6d2827a004585c \ + sleep infinity > /dev/null + docker exec tls-symbols sh -euc ' + apt-get update + apt-get install --yes --no-install-recommends ca-certificates curl python3 binutils zstd libicu76 + curl -fsSL --retry 5 https://apt.llvm.org/llvm-snapshot.gpg.key > /etc/apt/trusted.gpg.d/apt.llvm.org.asc + echo "deb https://apt.llvm.org/trixie/ llvm-toolchain-trixie-23 main" > /etc/apt/sources.list.d/llvm.list + apt-get update + apt-get install --yes --no-install-recommends llvm-23 lldb-23 python3-lldb-23 ' + docker exec -i tls-symbols python3 - <<'PYCODE' + import hashlib, json, re, shutil, subprocess, tarfile + from pathlib import Path, PurePosixPath + expected_icu = re.search(r"^libicu-dev(?::arm64)?\s+(\S+)$", Path("producer/tools.txt").read_text(), re.M) + assert expected_icu, "producer ICU identity missing" + actual_icu = subprocess.check_output(["dpkg-query", "-W", "-f=${Version}", "libicu76"], text=True) + assert actual_icu == expected_icu[1], (actual_icu, expected_icu[1]) + count = size = 0 + seen = set() + child = subprocess.Popen(["zstd", "--decompress", "--stdout", "producer/matched-symbols.tar.zst"], stdout=subprocess.PIPE) + try: + with tarfile.open(fileobj=child.stdout, mode="r|") as archive: + for member in archive: + name = PurePosixPath(member.name) + assert member.isfile() and not name.is_absolute() and ".." not in name.parts + assert str(name).startswith("source/build/tls-arm64-fullsymbols/") and str(name) not in seen + assert name.name in ("bun", "bun-profile") or name.suffix == ".dwo" + seen.add(str(name)); count += 1; size += member.size + assert count <= 20000 and size <= 8 * 1024**3 + target = Path(str(name)); target.parent.mkdir(parents=True, exist_ok=True) + with target.open("xb") as output, archive.extractfile(member) as content: + shutil.copyfileobj(content, output) + target.chmod(0o755 if name.name in ("bun", "bun-profile") else 0o644) + assert child.wait(timeout=30) == 0 + finally: + child.stdout.close() + if child.poll() is None: + child.kill(); child.wait() + sums = dict(line.split(None, 1)[::-1] for line in Path("producer/SHA256SUMS").read_text().splitlines()) + for name in ("source/build/tls-arm64-fullsymbols/bun", "source/build/tls-arm64-fullsymbols/bun-profile"): + with Path(name).open("rb") as binary: + assert hashlib.file_digest(binary, "sha256").hexdigest() == sums[name] + Path("diagnostics/restored-inputs.json").write_text(json.dumps({"files": count, "bytes": size, "icu": actual_icu}, indent=2) + "\n") + PYCODE + docker exec tls-symbols sh -euc 'ldd source/build/tls-arm64-fullsymbols/bun' > diagnostics/runtime-libraries.txt + ! grep -q 'not found' diagnostics/runtime-libraries.txt + docker exec tls-symbols sh -euc 'lldb-23 --version; llvm-dwarfdump-23 --version; dpkg-query -W libicu76 python3 llvm-23 lldb-23 python3-lldb-23 binutils zstd' > diagnostics/runtime-tools.txt - name: Verify matched symbols and capture capability shell: bash run: | @@ -120,7 +184,27 @@ jobs: ids = [re.findall(r"Build ID: (\w+)", (report / name).read_text()) for name in ("bun-build-id.txt", "profile-build-id.txt")] assert len(ids[0]) == 1 and ids[0] == ids[1], ids PY - gdb --batch --nx --init-eval-command="set auto-load off" -ex "file $build/bun-profile" -ex "python exec(compile(open(\"/work/diagnostics/probe.py\").read(), \"probe.py\", \"exec\"))" > /work/diagnostics/symbol-probe.txt 2>&1 + python3 - <<'"'"'PYCODE'"'"' + import json, subprocess + from pathlib import Path + with Path("/work/diagnostics/dwarf-reference.txt").open("w") as output: + result = subprocess.run(["llvm-dwarfdump-23", "--show-parents", "--show-form", "--show-children", "--recurse-depth=2", "--debug-info=0x2cad422", "build/tls-arm64-fullsymbols/bun-profile"], stdout=output, stderr=subprocess.STDOUT, timeout=120) + Path("/work/diagnostics/dwarf-inspection.json").write_text(json.dumps({"status": result.returncode, "prior_gdb_error": "DWARF reference to DIE 0 from DIE 0x2cad422", "prior_run": 41180631285}) + "\n") + PYCODE + lldb-23 --no-lldbinit --batch \ + -o "settings set target.load-script-from-symbol-file false" \ + -o "target create $build/bun" \ + -o "target symbols add $build/bun-profile" \ + -o "script exec(open(\"/work/diagnostics/probe.py\").read())" > /work/diagnostics/symbol-probe.txt 2>&1 + python3 - <<'"'"'PYCODE'"'"' + import json + from pathlib import Path + report = json.loads(Path("/work/diagnostics/symbol-capability.json").read_text()) + assert set(report["types"]) == {"JSC::JSCell", "JSC::Heap", "JSC::SlotVisitor", "JSC::ConservativeRoots"} + assert all(values and all(value["bytes"] > 0 and value["fields"] > 0 for value in values) for values in report["types"].values()) + assert set(report["functions"]) == {"JSC::Collector::runEndPhase", "JSC::MachineThreads::gatherConservativeRoots"} + assert all(values and all(value["address"] > 0 and value["compile_unit"] and value["name"] for value in values) for values in report["functions"].values()) + PYCODE ' - name: Run the unchanged destroyed-completion case shell: bash @@ -142,7 +226,7 @@ jobs: assert facts == {"commit": "454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e", "webkit": "b82bc917e99110b61c3a21202d6def7457aa194c", "platform": "linux", "arch": "arm64"}, facts name = "a TLS Duplex completion retains its owner and is inert after destroy" with (report / "stdout.txt").open("w") as stdout, (report / "stderr.txt").open("w") as stderr: - result = subprocess.run([str(exe), "test", str(fixture), "--test-name-pattern", "^" + name + "$", "--reporter=junit", "--reporter-outfile=" + str(report / "result.xml")], stdout=stdout, stderr=stderr) + result = subprocess.run([str(exe), "test", str(fixture), "--test-name-pattern", "^" + name + "$", "--reporter=junit", "--reporter-outfile=" + str(report / "result.xml")], stdout=stdout, stderr=stderr, timeout=120) cases = [case for case in ElementTree.parse(report / "result.xml").iter("testcase") if case.get("name") == name] assert len(cases) == 1 and cases[0].find("skipped") is None, "selected case did not execute" outcome = {"exitCode": result.returncode, "name": name, "failure": cases[0].find("failure") is not None, "error": cases[0].find("error") is not None} @@ -153,36 +237,14 @@ jobs: sys.exit(result.returncode) PY ' - - name: Retain matched binary and split DWARF inputs - if: always() - shell: bash - run: | - set -euo pipefail - sudo chown -R "$(id -u):$(id -g)" diagnostics - if [ -f source/build/tls-arm64-fullsymbols/bun-profile ]; then - python3 - <<'PY' - from pathlib import Path - import subprocess - root = Path("source/build/tls-arm64-fullsymbols") - files = [root / "bun", root / "bun-profile", *root.rglob("*.dwo")] - files = [p for p in files if p.is_file()] - assert len(files) <= 20000 and sum(p.stat().st_size for p in files) <= 8 * 1024**3, "artifact-budget-exceeded" - with Path("diagnostics/symbol-inputs.txt").open("w") as out: - out.write("\n".join(str(p) for p in files) + "\n") - subprocess.run(["tar", "--zstd", "-cf", "diagnostics/matched-symbols.tar.zst", "-T", "diagnostics/symbol-inputs.txt"], check=True) - PY - find source/build/tls-arm64-fullsymbols -name CMakeCache.txt -exec sh -c 'for cache; do printf "\n%s\n" "$cache"; cat "$cache"; done' sh {} + > diagnostics/cmake-caches.txt - sha256sum source/build/tls-arm64-fullsymbols/bun source/build/tls-arm64-fullsymbols/bun-profile diagnostics/matched-symbols.tar.zst > diagnostics/SHA256SUMS - fi - name: Upload diagnostic evidence if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: tls-linux-arm64-fullsymbols + name: tls-linux-arm64-saved-symbols path: diagnostics if-no-files-found: error retention-days: 7 - compression-level: 0 - name: Stop diagnostic container if: always() run: docker rm --force tls-symbols > /dev/null From 2e13ab5cbb267dd0d5c2bb81ddd505f6ec41fe1a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 31 Oct 2026 15:53:45 +0800 Subject: [PATCH 18/74] ci(tls): qualify original shipping symbols with LLVM 23 --- .github/workflows/openclaw-tls-symbols.yml | 207 ++++++++++----------- 1 file changed, 97 insertions(+), 110 deletions(-) diff --git a/.github/workflows/openclaw-tls-symbols.yml b/.github/workflows/openclaw-tls-symbols.yml index 3a5dfe8111a1..634fe6f6228e 100644 --- a/.github/workflows/openclaw-tls-symbols.yml +++ b/.github/workflows/openclaw-tls-symbols.yml @@ -1,4 +1,4 @@ -name: OpenClaw TLS symbol inspection +name: OpenClaw TLS shipping-symbol capability on: pull_request: @@ -11,38 +11,23 @@ permissions: contents: read concurrency: - group: openclaw-tls-symbols-${{ github.head_ref || github.ref }} + group: openclaw-tls-shipping-symbols-${{ github.head_ref || github.ref }} cancel-in-progress: true -# Inspect the exact failing PR #215 artifacts without rebuilding or running them. +# Inspect the original failing PR #215 bytes; never execute or rebuild them. env: SOURCE_COMMIT: 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e SOURCE_RUN: "40897692970" + TARGET: linux-arm64 + TRIPLET: bun-linux-aarch64 + ARTIFACT_ID: "12908842070" + ARTIFACT_SHA256: 65f1e6de6618b9fc2dd75c688d49fff27bcedc61fd03105f1f92c9df49e4f5c4 jobs: inspect: - name: Inspect ${{ matrix.target }} symbols - runs-on: ${{ matrix.runner }} + name: Inspect shipping Linux ARM64 symbols + runs-on: ubuntu-24.04-arm timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - include: - - target: linux-arm64 - runner: ubuntu-24.04-arm - triplet: bun-linux-aarch64 - artifact: "12908842070" - sha256: 65f1e6de6618b9fc2dd75c688d49fff27bcedc61fd03105f1f92c9df49e4f5c4 - - target: darwin-x64 - runner: macos-15-intel - triplet: bun-darwin-x64 - artifact: "12909560910" - sha256: 99eb0c3444b61fd6756b4879dfa5698c4a9e69706a9a1ff7f1cff685a741d1d9 - env: - TARGET: ${{ matrix.target }} - TRIPLET: ${{ matrix.triplet }} - ARTIFACT_ID: ${{ matrix.artifact }} - ARTIFACT_SHA256: ${{ matrix.sha256 }} steps: - name: Fetch and verify frozen artifact shell: bash @@ -74,108 +59,110 @@ jobs: shasum -a 256 "unpacked/$TRIPLET/bun" \ "unpacked/$TRIPLET-profile/bun-profile" > diagnostics/executables.sha256 - - name: Install Linux debugger - if: runner.os == 'Linux' - shell: bash - run: | - sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends gdb binutils - - name: Inspect Linux build identity and types - if: runner.os == 'Linux' + - name: Prepare LLVM capability probe shell: bash run: | set -euo pipefail - [ "$(uname -m)" = aarch64 ] - exe="unpacked/$TRIPLET/bun" - profile="unpacked/$TRIPLET-profile/bun-profile" - readelf -n "$exe" > diagnostics/executable-notes.txt - readelf -n "$profile" > diagnostics/profile-notes.txt - id=$(sed -n 's/.*Build ID: //p' diagnostics/executable-notes.txt) - profile_id=$(sed -n 's/.*Build ID: //p' diagnostics/profile-notes.txt) - [ -n "$id" ] && [ "$id" = "$profile_id" ] - readelf -SW "$exe" > diagnostics/executable-sections.txt - readelf -SW "$profile" > diagnostics/profile-sections.txt - gdb --batch --nx --init-eval-command='set auto-load off' \ - -ex "file $profile" -ex "exec-file $exe" \ - -ex 'info functions JSC::Collector::runEndPhase' \ - -ex 'info functions JSC::MachineThreads::gatherConservativeRoots' \ - > diagnostics/capture-functions.txt 2>&1 - for type in JSCell Heap SlotVisitor ConservativeRoots; do - if gdb --batch --nx --init-eval-command='set auto-load off' \ - -ex "file $exe" -ex "symbol-file $profile" -ex "ptype /o JSC::$type" \ - > "diagnostics/type-$type.txt" 2>&1; then - echo "$type: available" >> diagnostics/types.txt - else - echo "$type: unavailable" >> diagnostics/types.txt - fi - done + test "$(uname -m)" = aarch64 + cat > diagnostics/probe.py <<'PY' + import json, lldb, re + from pathlib import Path - - name: Inspect Darwin UUID and types - if: runner.os == 'macOS' + target = lldb.debugger.GetSelectedTarget() + assert target.IsValid(), "missing shipping target" + module = target.FindModule(target.GetExecutable()) + assert module.IsValid(), "missing executable module" + uuid = re.sub("[^0-9a-fA-F]", "", module.GetUUIDString() or "").lower() + expected = re.findall(r"Build ID: (\w+)", Path("/work/diagnostics/executable-notes.txt").read_text()) + assert len(expected) == 1 and uuid == expected[0], "shipping module identity mismatch" + types = {} + for name in ("JSC::JSCell", "JSC::Heap", "JSC::SlotVisitor", "JSC::ConservativeRoots"): + candidates = target.FindTypes(name) + complete = [] + for i in range(candidates.GetSize()): + value = candidates.GetTypeAtIndex(i).GetCanonicalType() + fields = value.GetNumberOfFields() + if value.IsTypeComplete() and value.GetByteSize() > 0 and fields > 0: + complete.append({"bytes": value.GetByteSize(), "fields": fields}) + types[name] = {"candidates": candidates.GetSize(), "complete": complete} + functions = {} + for name in ("JSC::Collector::runEndPhase", "JSC::MachineThreads::gatherConservativeRoots"): + contexts = module.FindFunctions(name, lldb.eFunctionNameTypeAny) + debug = [] + for i in range(contexts.GetSize()): + context = contexts.GetContextAtIndex(i) + function = context.GetFunction() + unit = context.GetCompileUnit() + if not function.IsValid() or not unit.IsValid(): + continue + address = function.GetStartAddress().GetFileAddress() + if address not in (0, lldb.LLDB_INVALID_ADDRESS): + debug.append({"address": address, "name": function.GetName(), "compile_unit": str(unit.GetFileSpec())}) + functions[name] = {"debug": debug, "symbols": []} + for i in range(module.GetNumSymbols()): + symbol = module.GetSymbolAtIndex(i) + name = symbol.GetName() or "" + for owner, entries in functions.items(): + if name == owner or name.startswith(owner + "("): + address = symbol.GetStartAddress().GetFileAddress() + if address not in (0, lldb.LLDB_INVALID_ADDRESS): + entries["symbols"].append({"name": name, "address": address}) + complete = all(value["complete"] for value in types.values()) and all(value["debug"] for value in functions.values()) + Path("/work/diagnostics/shipping-capability.json").write_text(json.dumps({"module_uuid": uuid, "complete": complete, "types": types, "functions": functions}, indent=2) + "\n") + PY + docker run --detach --name tls-shipping-symbols -v "$GITHUB_WORKSPACE:/work" -w /work \ + docker.io/library/debian:13@sha256:9cc080028c43b27d2074d63a5f9caf7166d731494965616c1a6d2827a004585c \ + sleep infinity > /dev/null + docker exec tls-shipping-symbols sh -euc ' + apt-get update + apt-get install --yes --no-install-recommends ca-certificates curl python3 binutils + curl -fsSL --retry 5 https://apt.llvm.org/llvm-snapshot.gpg.key > /etc/apt/trusted.gpg.d/apt.llvm.org.asc + echo "deb https://apt.llvm.org/trixie/ llvm-toolchain-trixie-23 main" > /etc/apt/sources.list.d/llvm.list + apt-get update + apt-get install --yes --no-install-recommends llvm-23 lldb-23 python3-lldb-23 + ' + docker exec tls-shipping-symbols sh -euc 'lldb-23 --version; dpkg-query -W llvm-23 lldb-23 python3-lldb-23' > diagnostics/runtime-tools.txt + + - name: Require shipping type and function capability shell: bash run: | set -euo pipefail - [ "$(uname -m)" = x86_64 ] exe="unpacked/$TRIPLET/bun" profile="unpacked/$TRIPLET-profile/bun-profile" - dsym="$profile.dSYM" - xcrun dwarfdump --uuid "$exe" > diagnostics/executable-uuid.txt - xcrun dwarfdump --uuid "$profile" > diagnostics/profile-uuid.txt - xcrun dwarfdump --uuid "$dsym" > diagnostics/dsym-uuid.txt - id=$(awk '{print $2}' diagnostics/executable-uuid.txt) - [ -n "$id" ] - [ "$id" = "$(awk '{print $2}' diagnostics/profile-uuid.txt)" ] - [ "$id" = "$(awk '{print $2}' diagnostics/dsym-uuid.txt)" ] - otool -l "$exe" > diagnostics/executable-load-commands.txt - cat > diagnostics/inspect-symbols.py <<'PY' - import lldb - - result = lldb.SBCommandReturnObject() - lldb.debugger.GetCommandInterpreter().HandleCommand( - "image lookup -rn 'JSC::Collector::runEndPhase|JSC::MachineThreads::gatherConservativeRoots'", result) - output = result.GetOutput() or "" - error = result.GetError() or "" - with open("diagnostics/capture-functions.txt", "w") as report: - report.write(output + error or "no matching functions\n") - report.write(f"lookup succeeded: {result.Succeeded()}\n") - if error: - raise RuntimeError(error) - target = lldb.debugger.GetSelectedTarget() - with open("diagnostics/types.txt", "w") as report: - for name in ("JSCell", "Heap", "SlotVisitor", "ConservativeRoots"): - kind = target.FindFirstType("JSC::" + name) - complete = kind.IsValid() and kind.IsTypeComplete() - report.write(f"{name}: complete={complete}, bytes={kind.GetByteSize()}, fields={kind.GetNumberOfFields()}\n") + readelf -n "$exe" > diagnostics/executable-notes.txt + readelf -n "$profile" > diagnostics/profile-notes.txt + python3 - <<'PY' + import re + from pathlib import Path + ids = [re.findall(r"Build ID: (\w+)", Path("diagnostics", name).read_text()) for name in ("executable-notes.txt", "profile-notes.txt")] + assert len(ids[0]) == 1 and ids[0] == ids[1], ids PY - xcrun lldb --no-lldbinit --batch \ + docker exec tls-shipping-symbols lldb-23 --no-lldbinit --batch \ -o 'settings set target.load-script-from-symbol-file false' \ - -o "target create $exe" -o "target symbols add $dsym" \ - -o 'script exec(open("diagnostics/inspect-symbols.py").read())' \ - > diagnostics/debugger.txt 2>&1 - [ "$(wc -l < diagnostics/types.txt)" -eq 4 ] - - - name: Report inspection scope - if: always() - shell: bash - run: | - { - echo "### TLS symbol inspection: $TARGET" - echo "Source: $SOURCE_COMMIT; build run: $SOURCE_RUN; artifact: $ARTIFACT_ID." - echo 'This inspects saved symbols without running the executable or collecting a heap.' - echo 'Type availability is a prerequisite, not proof of usable root or stack capture.' - if [ -f diagnostics/types.txt ]; then - echo '```text' - cat diagnostics/types.txt - echo '```' - fi - } >> "$GITHUB_STEP_SUMMARY" + -o "target create /work/$exe" -o "target symbols add /work/$profile" \ + -o 'script exec(open("/work/diagnostics/probe.py").read())' > diagnostics/debugger.txt 2>&1 + python3 - <<'PY' + import json + from pathlib import Path + report = json.loads(Path("diagnostics/shipping-capability.json").read_text()) + assert set(report["types"]) == {"JSC::JSCell", "JSC::Heap", "JSC::SlotVisitor", "JSC::ConservativeRoots"} + assert all(value["complete"] and all(item["bytes"] > 0 and item["fields"] > 0 for item in value["complete"]) for value in report["types"].values()), "shipping GC types unavailable" + assert set(report["functions"]) == {"JSC::Collector::runEndPhase", "JSC::MachineThreads::gatherConservativeRoots"} + assert all(value["debug"] and all(item["address"] > 0 and item["name"] and item["compile_unit"] for item in value["debug"]) for value in report["functions"].values()), "shipping debug functions unavailable" + assert report["complete"] is True + PY - - name: Retain symbol inspection + - name: Retain shipping capability evidence if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: tls-symbols-${{ matrix.target }} + name: tls-shipping-symbols-linux-arm64 path: diagnostics if-no-files-found: error retention-days: 7 + + - name: Stop diagnostic container + if: always() + shell: bash + run: docker rm --force tls-shipping-symbols > /dev/null 2>&1 || true From 13430a57b67eb4b7930170117aa2eb763fb9e962 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 31 Oct 2026 16:37:08 +0800 Subject: [PATCH 19/74] test(tls): retain shipping optimizations in native debug builds --- .github/openclaw/tls-shipping-debug.py | 227 ++++++++++++++++++ .github/openclaw/tls-symbol-capability.py | 36 +++ .../workflows/openclaw-tls-fullsymbols.yml | 220 ++++++++++++++--- 3 files changed, 444 insertions(+), 39 deletions(-) create mode 100644 .github/openclaw/tls-shipping-debug.py create mode 100644 .github/openclaw/tls-symbol-capability.py diff --git a/.github/openclaw/tls-shipping-debug.py b/.github/openclaw/tls-shipping-debug.py new file mode 100644 index 000000000000..fa4d95c8b25c --- /dev/null +++ b/.github/openclaw/tls-shipping-debug.py @@ -0,0 +1,227 @@ +"""Build typed TLS diagnostic bytes with the frozen shipping optimization and ICU inputs.""" +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tarfile +import urllib.request + +ROOT = Path('/work') +SOURCE = ROOT / 'source' +ENGINE = ROOT / 'engine' +OUT = ROOT / 'diagnostics' +BUN = '454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e' +WEBKIT = 'b82bc917e99110b61c3a21202d6def7457aa194c' +FIXTURE = 'test/js/bun/net/socket-retention.test.ts' +FIXTURE_SHA = 'd4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2' +LANE = 'bun-webkit-linux-arm64-lto.tar.gz' +BUILD = SOURCE / 'build/tls-arm64-shipping-debug' +MANIFEST = SOURCE / 'scripts/build/deps/webkit-artifacts.json' +LIBS = ['libJavaScriptCore.a', 'libWTF.a', 'libbmalloc.a'] +ENV = {key: value for key, value in os.environ.items() + if key not in ('GH_TOKEN', 'GITHUB_TOKEN', 'ACTIONS_RUNTIME_TOKEN', 'ACTIONS_ID_TOKEN_REQUEST_TOKEN')} +ENV.update(CI='true', CARGO_BUILD_JOBS='2') + + +def sha(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def write(name, value): + with (OUT / name).open('x') as stream: + stream.write(json.dumps(value, indent=2) + '\n') + + +def run(argv, name, cwd=SOURCE, env=ENV, timeout=120): + with (OUT / name).open('x') as stream: + subprocess.run(argv, cwd=cwd, env=env, stdout=stream, stderr=subprocess.STDOUT, + check=True, timeout=timeout) + + +def git(repo, *args): + return subprocess.check_output(['git', '-C', str(repo), *args], text=True).strip() + + +def fetch(url, path, digest): + assert not path.exists() + with urllib.request.urlopen(url, timeout=120) as response, path.open('xb') as output: + shutil.copyfileobj(response, output) + assert sha(path) == digest, 'download checksum mismatch' + + +def inventory(root): + result = {} + for path in sorted(root.rglob('*')): + assert not path.is_symlink(), 'unexpected bundle symlink' + if path.is_file(): + result[str(path.relative_to(root))] = sha(path) + return result + + +assert os.uname().machine == 'aarch64' +assert git(SOURCE, 'rev-parse', 'HEAD') == BUN and not git(SOURCE, 'diff', '--name-only', 'HEAD') +assert git(ENGINE, 'rev-parse', 'HEAD') == WEBKIT and not git(ENGINE, 'diff', '--name-only', 'HEAD') +assert sha(SOURCE / FIXTURE) == FIXTURE_SHA +manifest_bytes = MANIFEST.read_bytes() +original_manifest = json.loads(manifest_bytes) +assert original_manifest['version'] == WEBKIT +artifact = original_manifest['artifacts'][LANE] +assert artifact['url'] == f'https://github.com/openclaw/WebKit/releases/download/autobuild-{WEBKIT}/{LANE}' +write('inputs.json', {'bun': BUN, 'webkit': WEBKIT, 'fixture_sha256': FIXTURE_SHA, + 'shipping_archive': artifact, 'variance': 'full debug information; native build host', + 'build_timeout_seconds': {'engine': 7200, 'bun': 7200}, + 'optimization': 'Release O3, ThinLTO, armv8-a+crc, ampere1, original allocator/assertion settings', + 'icu': 'original bundled libraries, headers and filtered/repacked data; never system ICU'}) +original = ROOT / LANE +fetch(artifact['url'], original, artifact['sha256']) +bundle_root = ROOT / 'shipping-bundle' +bundle_root.mkdir() +with tarfile.open(original) as archive: + archive.extractall(bundle_root, filter='data') +bundle = bundle_root / 'bun-webkit' +assert bundle.is_dir() and all((bundle / 'lib' / name).is_file() for name in LIBS) +before = inventory(bundle) +assert all('lib/' + name in before for name in ['libicudata.a', 'libicui18n.a', 'libicuuc.a']) +assert any(name.endswith('/uvernum.h') for name in before) +write('shipping-bundle.json', before) + +# The shell's decoder source matches the frozen engine recipe. ICU itself is not rebuilt. +zstd = ROOT / 'zstd-1.5.7.tar.gz' +fetch('https://github.com/meta/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz', zstd, + 'eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3') +with tarfile.open(zstd) as archive: + archive.extractall(ROOT / 'zstd-source', filter='data') +sysroot = Path('/opt/linux-sysroot-glibc-arm64') +assert (sysroot / 'usr/lib/gcc/aarch64-linux-gnu/13/libstdc++.a').is_file() +compiler = subprocess.check_output(['clang', '--version'], text=True, env=ENV) +assert '23.1.1' in compiler, compiler +write('toolchain.json', {'compiler': compiler, 'sysroot': str(sysroot), + 'libstdc++': sha(sysroot / 'usr/lib/gcc/aarch64-linux-gnu/13/libstdc++.a'), + 'profile': sha(Path('/etc/profile.d/bun-ci.sh'))}) +base_flags = (f'--sysroot={sysroot} --target=aarch64-unknown-linux-gnu -march=armv8-a+crc -mtune=ampere1 ' + '-mno-omit-leaf-frame-pointer -g -fno-omit-frame-pointer -ffunction-sections -fdata-sections ' + '-faddrsig -fno-unwind-tables -fno-asynchronous-unwind-tables -DU_STATIC_IMPLEMENTATION=1 ' + '-stdlib=libstdc++ -g -flto=thin -fno-split-lto-unit -fwhole-program-vtables -fforce-emit-vtables ' + '-ffile-prefix-map=/work/engine/Source=vendor/WebKit/Source -ffile-prefix-map=/work/jsc-build/=. ') +engine_env = ENV.copy() +for name in ('C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH', 'LIBRARY_PATH', 'LD_LIBRARY_PATH', 'CFLAGS', 'CXXFLAGS'): + engine_env.pop(name, None) +engine_env.update(CC='clang', CXX='clang++', LDFLAGS='-fuse-ld=lld') +engine_build = ROOT / 'jsc-build' +argv = ['cmake', '-S', str(ENGINE), '-B', str(engine_build), '-G', 'Ninja', + '-DCMAKE_SYSTEM_NAME=Linux', '-DCMAKE_SYSTEM_PROCESSOR=aarch64', f'-DCMAKE_SYSROOT={sysroot}', + '-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER', '-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH', + '-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=BOTH', '-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH', + '-DPORT=JSCOnly', '-DENABLE_STATIC_JSC=ON', '-DENABLE_BUN_SKIP_FAILING_ASSERTIONS=ON', + '-DCMAKE_BUILD_TYPE=Release', '-DUSE_THIN_ARCHIVES=OFF', '-DUSE_BUN_JSC_ADDITIONS=ON', + '-DUSE_BUN_EVENT_LOOP=ON', '-DENABLE_FTL_JIT=ON', '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', + '-DALLOW_LINE_AND_COLUMN_NUMBER_IN_BUILTINS=ON', '-DENABLE_REMOTE_INSPECTOR=ON', + '-DCMAKE_EXE_LINKER_FLAGS=-fuse-ld=lld', '-DCMAKE_AR=' + shutil.which('llvm-ar'), + '-DCMAKE_RANLIB=' + shutil.which('llvm-ranlib'), '-DCMAKE_C_FLAGS=' + base_flags, + '-DCMAKE_CXX_FLAGS=' + base_flags + ' -fno-c++-static-destructors', + '-DCMAKE_C_FLAGS_RELEASE=-O3 -DNDEBUG=1', '-DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG=1', + '-DICU_ROOT=' + str(bundle), '-DBUN_ICU_ZSTD_SOURCE_DIR=/work/zstd-source/zstd-1.5.7/lib', + '-DICU_INCLUDE_DIR=' + str(bundle / 'include'), + '-DICU_UC_LIBRARY_RELEASE=' + str(bundle / 'lib/libicuuc.a'), + '-DICU_I18N_LIBRARY_RELEASE=' + str(bundle / 'lib/libicui18n.a'), + '-DICU_DATA_LIBRARY_RELEASE=' + str(bundle / 'lib/libicudata.a'), + '-DENABLE_SANITIZERS=', '-DENABLE_ASSERTS=AUTO', '-DENABLE_FUZZILLI=OFF', + '-DUSE_MIMALLOC=ON', '-DUSE_EXTERNAL_MIMALLOC=ON'] +write('engine-build-command.json', {'argv': argv, 'compiler': 'clang/clang++ 23.1.1', + 'linker_flags': '-fuse-ld=lld'}) +run(argv, 'engine-configure.log', cwd=ROOT, env=engine_env, timeout=600) +try: + run(['cmake', '--build', str(engine_build), '--config', 'Release', '--parallel', '2', + '--target', 'jsc', '--target', 'testFFI'], 'engine-build.log', cwd=ROOT, env=engine_env, timeout=7200) +finally: + for name in ('CMakeCache.txt', 'compile_commands.json'): + if (engine_build / name).is_file(): + shutil.copyfile(engine_build / name, OUT / name) +run(['python3', str(ENGINE / 'Tools/Scripts/check-classinfo-uniqueness.py'), str(engine_build / 'bin/jsc')], + 'engine-classinfo.log', cwd=ROOT) +elf = subprocess.check_output(['readelf', '-h', str(engine_build / 'bin/jsc')], text=True) +dynamic = subprocess.check_output(['readelf', '-d', str(engine_build / 'bin/jsc')], text=True) +versions = subprocess.check_output(['readelf', '--version-info', str(engine_build / 'bin/jsc')], text=True) +glibc = [tuple(map(int, value.split('.'))) for value in re.findall(r'GLIBC_([0-9]+\.[0-9]+)', versions)] +assert 'AArch64' in elf and 'libm.so.6' in dynamic and glibc and max(glibc) <= (2, 31) +assert not re.search(r'NEEDED.*libicu', dynamic) +write('engine-binary.json', {'ELF': elf, 'dynamic': dynamic, 'max_glibc': max(glibc)}) +run([str(engine_build / 'bin/jsc'), '-e', 'if(1+1!==2)throw Error("startup");print(new Intl.DateTimeFormat("ja-JP-u-ca-japanese").format(new Date(0)))'], + 'engine-startup.log', cwd=ROOT) + +# The prebuilt consumer must use the same headers as these rebuilt core libraries. +# Match the frozen Dockerfile's flattening order, retaining its ICU headers untouched. +headers = {Path('include') / path.name: path for path in engine_build.glob('*.h')} +for directory in ('JavaScriptCore/DerivedSources', 'JavaScriptCore/Headers/JavaScriptCore', + 'JavaScriptCore/PrivateHeaders/JavaScriptCore'): + parent = engine_build / directory + assert parent.is_dir(), 'missing core header directory: ' + directory + for path in sorted(parent.rglob('*.h')): + headers[Path('include/JavaScriptCore') / path.name] = path +for directory, destination in [('WTF/Headers/wtf', 'include/wtf'), ('bmalloc/Headers/bmalloc', 'include/bmalloc')]: + parent = engine_build / directory + assert parent.is_dir(), 'missing core header directory: ' + directory + for path in sorted(parent.rglob('*')): + if path.is_file(): + headers[Path(destination) / path.relative_to(parent)] = path +assert headers +missing = [str(name) for name in headers if str(name) not in before] +mismatched = {str(name): {'shipping': before[str(name)], 'rebuilt': sha(path)} + for name, path in headers.items() if str(name) in before and sha(path) != before[str(name)]} +write('core-header-parity.json', {'files': len(headers), 'missing': missing, 'mismatched': mismatched}) +assert not missing and not mismatched, 'rebuilt core headers differ from the shipping consumer headers' +for name in LIBS: + shutil.copyfile(engine_build / 'lib' / name, bundle / 'lib' / name) +after = inventory(bundle) +assert before.keys() == after.keys() +assert {name for name in before if before[name] != after[name]} == {'lib/' + name for name in LIBS} +write('adapted-bundle.json', after) + +# Give these diagnostic bytes their own digest and task-owned prefetch/cache. Never reuse the shipping checksum. +rebuilt = ROOT / 'debug-webkit.tar.gz' +with tarfile.open(rebuilt, 'w:gz') as archive: + archive.add(bundle, arcname='bun-webkit') +digest = sha(rebuilt) +prefetch = ROOT / 'debug-prefetch/by-sha256' +prefetch.mkdir(parents=True) +rebuilt.rename(prefetch / digest) +flags = SOURCE / 'scripts/build/flags.ts' +text = flags.read_text() +flags_bytes = flags.read_bytes() +assert text.count('flag: "-g1",') == 1 +flags.write_text(text.replace('flag: "-g1",', 'flag: "-g",')) +manifest = json.loads(MANIFEST.read_text()) +manifest['artifacts'][LANE]['sha256'] = digest +MANIFEST.write_text(json.dumps(manifest, indent=2) + '\n') +write('diagnostic-overlay.json', {'archive_sha256': digest, 'manifest_sha256': sha(MANIFEST), + 'flags_sha256': sha(flags), 'shipping_archive_sha256': artifact['sha256']}) +patch = subprocess.check_output(['git', '-C', str(SOURCE), 'diff', '--binary', 'HEAD']) +(OUT / 'diagnostic-build.patch').write_bytes(patch) +env = ENV.copy() +env.update(BUN_BUILD_PREFETCH_DIR='/work/debug-prefetch', BUN_BUILD_CACHE_DIR='/work/debug-cache') +argv = ['bun', 'scripts/build.ts', '--profile=release', '--ci=on', '--os=linux', '--arch=aarch64', + '--abi=gnu', '--lto=on', '--webkit=prebuilt', '--build-dir=' + str(BUILD), '-j2', '--timings'] +write('bun-build-command.json', {'argv': argv}) +try: + run(argv, 'bun-build.log', env=env, timeout=7200) +finally: + assert sha(SOURCE / FIXTURE) == FIXTURE_SHA + assert git(SOURCE, 'diff', '--name-only', 'HEAD').splitlines() == [ + 'scripts/build/deps/webkit-artifacts.json', 'scripts/build/flags.ts'] + assert sha(MANIFEST) == json.loads((OUT / 'diagnostic-overlay.json').read_text())['manifest_sha256'] + assert sha(flags) == json.loads((OUT / 'diagnostic-overlay.json').read_text())['flags_sha256'] + MANIFEST.write_bytes(manifest_bytes) + flags.write_bytes(flags_bytes) + assert not git(SOURCE, 'diff', '--name-only', 'HEAD') +binary, profile = BUILD / 'bun', BUILD / 'bun-profile' +for path in (binary, profile): + assert path.is_file() + run(['readelf', '-n', str(path)], path.name + '-build-id.txt', cwd=ROOT) +write('executables.json', {str(path.relative_to(ROOT)): {'sha256': sha(path), 'bytes': path.stat().st_size} + for path in (binary, profile)}) +run([str(binary), '-e', 'console.log(JSON.stringify({bun:Bun.revision,webkit:process.versions.webkit,platform:process.platform,arch:process.arch}))'], 'runtime.json') +assert json.loads((OUT / 'runtime.json').read_text()) == {'bun': BUN, 'webkit': WEBKIT, 'platform': 'linux', 'arch': 'arm64'} diff --git a/.github/openclaw/tls-symbol-capability.py b/.github/openclaw/tls-symbol-capability.py new file mode 100644 index 000000000000..472d5ec99ed2 --- /dev/null +++ b/.github/openclaw/tls-symbol-capability.py @@ -0,0 +1,36 @@ +import json, lldb, re +from pathlib import Path +target = lldb.debugger.GetSelectedTarget() +assert target.IsValid(), "missing diagnostic target" +module = target.FindModule(target.GetExecutable()) +assert module.IsValid(), "missing executable module" +uuid = re.sub("[^0-9a-fA-F]", "", module.GetUUIDString() or "").lower() +expected = re.findall(r"Build ID: (\w+)", Path("/work/diagnostics/bun-build-id.txt").read_text()) +assert len(expected) == 1 and uuid == expected[0], "LLDB module identity mismatch" +types = {} +for name in ("JSC::JSCell", "JSC::Heap", "JSC::SlotVisitor", "JSC::ConservativeRoots"): + candidates = target.FindTypes(name) + complete = [] + for i in range(candidates.GetSize()): + value = candidates.GetTypeAtIndex(i).GetCanonicalType() + fields = value.GetNumberOfFields() + if value.IsTypeComplete() and value.GetByteSize() > 0 and fields > 0: + layout = [{"name": value.GetFieldAtIndex(j).GetName(), "offset_bits": value.GetFieldAtIndex(j).GetOffsetInBits()} for j in range(fields)] + complete.append({"bytes": value.GetByteSize(), "fields": fields, "layout": layout}) + assert complete, "incomplete type: " + name + types[name] = complete +functions = {} +for name in ("JSC::Collector::runEndPhase", "JSC::MachineThreads::gatherConservativeRoots"): + contexts = module.FindFunctions(name, lldb.eFunctionNameTypeAny) + functions[name] = [] + for i in range(contexts.GetSize()): + context = contexts.GetContextAtIndex(i) + function = context.GetFunction() + unit = context.GetCompileUnit() + if not function.IsValid() or not unit.IsValid(): + continue + address = function.GetStartAddress().GetFileAddress() + if address not in (0, lldb.LLDB_INVALID_ADDRESS): + functions[name].append({"address": address, "name": function.GetName(), "compile_unit": str(unit.GetFileSpec())}) + assert functions[name], "missing debug function: " + name +Path("/work/diagnostics/symbol-capability.json").write_text(json.dumps({"module_uuid": uuid, "types": types, "functions": functions}, indent=2) + "\n") diff --git a/.github/workflows/openclaw-tls-fullsymbols.yml b/.github/workflows/openclaw-tls-fullsymbols.yml index 5bd669451075..b5dcefb614f8 100644 --- a/.github/workflows/openclaw-tls-fullsymbols.yml +++ b/.github/workflows/openclaw-tls-fullsymbols.yml @@ -1,9 +1,11 @@ -name: OpenClaw TLS saved-symbol reproduction +name: OpenClaw TLS native debug proof on: pull_request: paths: - .github/workflows/openclaw-tls-fullsymbols.yml + - .github/openclaw/tls-symbol-capability.py + - .github/openclaw/tls-shipping-debug.py workflow_dispatch: permissions: @@ -19,6 +21,12 @@ jobs: runs-on: ubuntu-24.04-arm timeout-minutes: 30 steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: pipeline + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: .github/openclaw - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e @@ -35,44 +43,7 @@ jobs: printf '%s\n' 'Bun=454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e' \ 'WebKit=b82bc917e99110b61c3a21202d6def7457aa194c' \ 'Diagnostic variance: non-LTO C++/JSC; Debian 13 system ICU; Rust-only release LTO may remain enabled.' > diagnostics/inputs.txt - cat > diagnostics/probe.py <<'PY' - import json, lldb, re - from pathlib import Path - target = lldb.debugger.GetSelectedTarget() - assert target.IsValid(), "missing diagnostic target" - module = target.FindModule(target.GetExecutable()) - assert module.IsValid(), "missing executable module" - uuid = re.sub("[^0-9a-fA-F]", "", module.GetUUIDString() or "").lower() - expected = re.findall(r"Build ID: (\w+)", Path("/work/diagnostics/bun-build-id.txt").read_text()) - assert len(expected) == 1 and uuid == expected[0], "LLDB module identity mismatch" - types = {} - for name in ("JSC::JSCell", "JSC::Heap", "JSC::SlotVisitor", "JSC::ConservativeRoots"): - candidates = target.FindTypes(name) - complete = [] - for i in range(candidates.GetSize()): - value = candidates.GetTypeAtIndex(i).GetCanonicalType() - fields = value.GetNumberOfFields() - if value.IsTypeComplete() and value.GetByteSize() > 0 and fields > 0: - layout = [{"name": value.GetFieldAtIndex(j).GetName(), "offset_bits": value.GetFieldAtIndex(j).GetOffsetInBits()} for j in range(fields)] - complete.append({"bytes": value.GetByteSize(), "fields": fields, "layout": layout}) - assert complete, "incomplete type: " + name - types[name] = complete - functions = {} - for name in ("JSC::Collector::runEndPhase", "JSC::MachineThreads::gatherConservativeRoots"): - contexts = module.FindFunctions(name, lldb.eFunctionNameTypeAny) - functions[name] = [] - for i in range(contexts.GetSize()): - context = contexts.GetContextAtIndex(i) - function = context.GetFunction() - unit = context.GetCompileUnit() - if not function.IsValid() or not unit.IsValid(): - continue - address = function.GetStartAddress().GetFileAddress() - if address not in (0, lldb.LLDB_INVALID_ADDRESS): - functions[name].append({"address": address, "name": function.GetName(), "compile_unit": str(unit.GetFileSpec())}) - assert functions[name], "missing debug function: " + name - Path("/work/diagnostics/symbol-capability.json").write_text(json.dumps({"module_uuid": uuid, "types": types, "functions": functions}, indent=2) + "\n") - PY + cp pipeline/.github/openclaw/tls-symbol-capability.py diagnostics/probe.py - name: Fetch the frozen failed build shell: bash env: @@ -248,3 +219,174 @@ jobs: - name: Stop diagnostic container if: always() run: docker rm --force tls-symbols > /dev/null + + shipping-debug-linux-arm64: + if: >- + github.repository == 'openclaw/bun' && + (github.head_ref == 'claude/webkit-tls-artifact-diagnostics-20261031' || + github.ref == 'refs/heads/claude/webkit-tls-artifact-diagnostics-20261031') + runs-on: ubuntu-24.04-arm + timeout-minutes: 340 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: pipeline + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: .github/openclaw + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: openclaw/bun + ref: 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e + path: source + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: openclaw/WebKit + ref: b82bc917e99110b61c3a21202d6def7457aa194c + path: engine + persist-credentials: false + - name: Free disposable runner space + if: runner.environment == 'github-hosted' + run: sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup /opt/hostedtoolcache /usr/local/share/boost /usr/share/swift /usr/local/julia* /opt/microsoft + - uses: ./source/.github/actions/setup-bun + with: + bun-version: '1.4.4' + - name: Generate the frozen shipping provisioning script + id: provision + shell: bash + run: | + set -euo pipefail + mkdir diagnostics + script=$(bun source/scripts/openclaw-release/release.ts provision --source source --target linux-arm64 | tail -1) + test -f "$script" + echo "script=${script#"$GITHUB_WORKSPACE"/}" >> "$GITHUB_OUTPUT" + cp pipeline/.github/openclaw/tls-symbol-capability.py diagnostics/probe.py + - name: Provision the original shipping toolchain and sysroot + id: shipping-container + shell: bash + env: + SCRIPT: ${{ steps.provision.outputs.script }} + run: | + set -euo pipefail + test "$(uname -m)" = aarch64 + container_id=$(docker run --detach --name tls-shipping-debug -e CI=true -v "$GITHUB_WORKSPACE:/work" -w /work \ + docker.io/library/debian:13@sha256:9cc080028c43b27d2074d63a5f9caf7166d731494965616c1a6d2827a004585c \ + sleep infinity) + echo "id=$container_id" >> "$GITHUB_OUTPUT" + docker exec -e SCRIPT tls-shipping-debug sh -euc 'sh "$SCRIPT" 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e openclaw-release' + docker exec tls-shipping-debug sh -euc ' + apt-get update + apt-get install --yes --no-install-recommends python3 binutils zstd ruby bison gawk perl libxml2-dev pkg-config + curl -fsSL --retry 5 https://apt.llvm.org/llvm-snapshot.gpg.key > /etc/apt/trusted.gpg.d/apt.llvm.org.asc + echo "deb https://apt.llvm.org/trixie/ llvm-toolchain-trixie-23 main" > /etc/apt/sources.list.d/llvm.list + apt-get update + apt-get install --yes --no-install-recommends llvm-23 lldb-23 python3-lldb-23 + ' + - name: Build full debug information with shipping optimizations and bundled ICU + shell: bash + run: | + set -euo pipefail + docker exec tls-shipping-debug bash -e -lc '. /etc/profile.d/bun-ci.sh; git config --global --add safe.directory /work/source; git config --global --add safe.directory /work/engine; python3 pipeline/.github/openclaw/tls-shipping-debug.py' + - name: Require matched executable IDs and complete GC types + shell: bash + run: | + set -euo pipefail + python3 - <<'PY' + import re + from pathlib import Path + def identity(name): + values = re.findall(r"Build ID: (\w+)", Path("diagnostics/" + name + "-build-id.txt").read_text()) + assert len(values) == 1 + return values[0] + assert identity("bun") == identity("bun-profile") + PY + docker exec tls-shipping-debug timeout 600 lldb-23 --no-lldbinit --batch \ + -o 'target create /work/source/build/tls-arm64-shipping-debug/bun-profile' \ + -o 'command script import /work/diagnostics/probe.py' > diagnostics/lldb-capability.log 2>&1 + python3 - <<'PY' + import json, re + from pathlib import Path + result = json.loads(Path("diagnostics/symbol-capability.json").read_text()) + assert result["module_uuid"] == re.findall(r"Build ID: (\w+)", Path("diagnostics/bun-build-id.txt").read_text())[0] + assert set(result["types"]) == {"JSC::JSCell", "JSC::Heap", "JSC::SlotVisitor", "JSC::ConservativeRoots"} + assert all(result["types"].values()) + assert set(result["functions"]) == {"JSC::Collector::runEndPhase", "JSC::MachineThreads::gatherConservativeRoots"} + assert all(result["functions"].values()) + PY + - name: Run the unchanged destroyed-completion case once + shell: bash + run: | + set -euo pipefail + docker exec tls-shipping-debug bash -lc '. /etc/profile.d/bun-ci.sh; cd /work/source; python3 - <<'"'"'PY'"'"' + import hashlib, json, subprocess, sys + from pathlib import Path + from xml.etree import ElementTree + fixture = Path("test/js/bun/net/socket-retention.test.ts") + assert hashlib.sha256(fixture.read_bytes()).hexdigest() == "d4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2" + name = "a TLS Duplex completion retains its owner and is inert after destroy" + out = Path("/work/diagnostics") + binary = Path("build/tls-arm64-shipping-debug/bun").resolve() + with (out / "fixture.stdout").open("x") as stdout, (out / "fixture.stderr").open("x") as stderr: + result = subprocess.run([str(binary), "test", str(fixture), "--test-name-pattern", "^" + name + "$", "--reporter=junit", "--reporter-outfile=" + str(out / "fixture.xml")], stdout=stdout, stderr=stderr, timeout=120) + cases = list(ElementTree.parse(out / "fixture.xml").iter("testcase")) + selected = [case for case in cases if case.get("name") == name] + assert len(selected) == 1 and selected[0].find("skipped") is None + outcome = {"exitCode": result.returncode, "name": name, "failure": selected[0].find("failure") is not None, "error": selected[0].find("error") is not None} + (out / "fixture-outcome.json").write_text(json.dumps(outcome, indent=2) + "\n") + if result.returncode == 0: + assert not outcome["failure"] and not outcome["error"] + sys.exit(result.returncode) + PY + ' + - name: Retain matched bytes and bounded diagnostic evidence + if: always() + shell: bash + env: + CONTAINER_ID: ${{ steps.shipping-container.outputs.id }} + run: | + set -euo pipefail + if test -n "$CONTAINER_ID"; then + [[ "$CONTAINER_ID" =~ ^[0-9a-f]{64}$ ]] + docker stop --time 30 "$CONTAINER_ID" > /dev/null + test "$(docker inspect --format '{{.State.Running}}' "$CONTAINER_ID")" = false + fi + python3 - <<'PY' + import hashlib, json, stat, subprocess + from pathlib import Path + files = sorted(path for path in Path("diagnostics").rglob("*") if path.is_file()) + files += [path for path in (Path("source/build/tls-arm64-shipping-debug") / name for name in ("bun", "bun-profile")) if path.is_file()] + assert files and len(files) <= 64 and all(stat.S_ISREG(path.lstat().st_mode) for path in files) + assert sum(path.stat().st_size for path in files) <= 8 * 1024**3 + def digest(path): + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + manifest = Path("diagnostics/archive-manifest.json") + manifest.write_text(json.dumps({str(path): {"bytes": path.stat().st_size, "sha256": digest(path)} for path in files}, indent=2) + "\n") + files.append(manifest) + output = Path("matched-shipping-debug.tar.zst.partial") + with output.open("xb") as stream: + subprocess.run(["tar", "--zstd", "-cf", "-", "--", *map(str, files)], stdout=stream, check=True) + assert output.stat().st_size <= 4 * 1024**3 + output.rename("matched-shipping-debug.tar.zst") + Path("shipping-debug-SHA256SUMS").write_text(digest(Path("matched-shipping-debug.tar.zst")) + " matched-shipping-debug.tar.zst\n") + PY + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: tls-linux-arm64-shipping-debug + path: | + matched-shipping-debug.tar.zst + shipping-debug-SHA256SUMS + if-no-files-found: error + retention-days: 7 + compression-level: 0 + - name: Stop shipping diagnostic container + if: always() + env: + CONTAINER_ID: ${{ steps.shipping-container.outputs.id }} + run: | + if test -n "$CONTAINER_ID"; then + [[ "$CONTAINER_ID" =~ ^[0-9a-f]{64}$ ]] + docker rm --force "$CONTAINER_ID" > /dev/null + fi From 1791c83c84194dcaaa05cdc2ffbd37671de8835a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 31 Oct 2026 22:38:11 +0800 Subject: [PATCH 20/74] fix(ci): capture shipping header drift before core compilation --- .github/openclaw/tls-shipping-debug.py | 66 ++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/.github/openclaw/tls-shipping-debug.py b/.github/openclaw/tls-shipping-debug.py index fa4d95c8b25c..ed355af4bbc1 100644 --- a/.github/openclaw/tls-shipping-debug.py +++ b/.github/openclaw/tls-shipping-debug.py @@ -1,4 +1,5 @@ """Build typed TLS diagnostic bytes with the frozen shipping optimization and ICU inputs.""" +import difflib import hashlib import json import os @@ -133,8 +134,67 @@ def inventory(root): '-DUSE_MIMALLOC=ON', '-DUSE_EXTERNAL_MIMALLOC=ON'] write('engine-build-command.json', {'argv': argv, 'compiler': 'clang/clang++ 23.1.1', 'linker_flags': '-fuse-ld=lld'}) +def check_headers(headers, phase): + missing = [str(name) for name in headers if str(name) not in before] + mismatched = {str(name): {'shipping': before[str(name)], 'rebuilt': sha(path)} + for name, path in headers.items() if str(name) in before and sha(path) != before[str(name)]} + evidence = {} + evidence_bytes = 0 + for name in sorted(mismatched)[:4]: + paths = {'shipping': bundle / name, 'rebuilt': headers[Path(name)]} + size = sum(path.stat().st_size for path in paths.values()) + if evidence_bytes + size > 128 * 1024**2: + evidence[name] = {'omitted': 'header evidence exceeds 128 MiB'} + continue + evidence[name] = {} + for kind, path in paths.items(): + destination = OUT / 'header-evidence' / phase / (name + '.' + kind) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, destination) + evidence[name][kind] = {'path': str(destination.relative_to(OUT)), + 'bytes': path.stat().st_size, 'sha256': sha(destination)} + evidence_bytes += size + # Retain large raw headers, but bound the in-memory line comparison separately. + if size > 8 * 1024**2: + evidence[name]['diff'] = {'omitted': 'diff inputs exceed 8 MiB'} + continue + destination = OUT / 'header-evidence' / phase / (name + '.diff') + difference_bytes = 0 + with destination.open('xb') as stream: + for line in difflib.unified_diff( + paths['shipping'].read_text().splitlines(keepends=True), + paths['rebuilt'].read_text().splitlines(keepends=True), + fromfile='shipping/' + name, tofile='rebuilt/' + name): + content = line.encode() + if evidence_bytes + difference_bytes + len(content) > 128 * 1024**2: + break + stream.write(content) + difference_bytes += len(content) + else: + stream.flush() + evidence[name]['diff'] = {'path': str(destination.relative_to(OUT)), + 'bytes': difference_bytes, 'sha256': sha(destination)} + if 'diff' not in evidence[name]: + destination.unlink() + evidence[name]['diff'] = {'omitted': 'diff exceeds remaining evidence budget'} + else: + evidence_bytes += difference_bytes + write(phase + '-header-parity.json', {'files': len(headers), 'missing': missing, + 'mismatched': mismatched, 'evidence': evidence}) + assert not missing and not mismatched, 'rebuilt core headers differ from the shipping consumer headers' + + run(argv, 'engine-configure.log', cwd=ROOT, env=engine_env, timeout=600) try: + # These generated headers can reject an incompatible rebuild before compiling the core libraries. + run(['cmake', '--build', str(engine_build), '--config', 'Release', '--parallel', '2', + '--target', 'JavaScriptCore/DerivedSources/LLIntAssembly.h'], + 'engine-header-build.log', cwd=ROOT, env=engine_env, timeout=1200) + early_headers = {Path('include/cmakeconfig.h'): engine_build / 'cmakeconfig.h'} + for name in ('LLIntAssembly.h', 'LLIntDesiredOffsets.h', 'LLIntDesiredSettings.h'): + early_headers[Path('include/JavaScriptCore') / name] = engine_build / 'JavaScriptCore/DerivedSources' / name + assert all(path.is_file() for path in early_headers.values()), 'generated header preflight incomplete' + check_headers(early_headers, 'generated') run(['cmake', '--build', str(engine_build), '--config', 'Release', '--parallel', '2', '--target', 'jsc', '--target', 'testFFI'], 'engine-build.log', cwd=ROOT, env=engine_env, timeout=7200) finally: @@ -169,11 +229,7 @@ def inventory(root): if path.is_file(): headers[Path(destination) / path.relative_to(parent)] = path assert headers -missing = [str(name) for name in headers if str(name) not in before] -mismatched = {str(name): {'shipping': before[str(name)], 'rebuilt': sha(path)} - for name, path in headers.items() if str(name) in before and sha(path) != before[str(name)]} -write('core-header-parity.json', {'files': len(headers), 'missing': missing, 'mismatched': mismatched}) -assert not missing and not mismatched, 'rebuilt core headers differ from the shipping consumer headers' +check_headers(headers, 'core') for name in LIBS: shutil.copyfile(engine_build / 'lib' / name, bundle / 'lib' / name) after = inventory(bundle) From 0a1544a4b4c6a4bdfbda96983e0734fc66e0320f Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 31 Oct 2026 22:54:54 +0800 Subject: [PATCH 21/74] fix(ci): compare shipping headers without known provenance stamps --- .github/openclaw/tls-shipping-debug.py | 41 ++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/.github/openclaw/tls-shipping-debug.py b/.github/openclaw/tls-shipping-debug.py index ed355af4bbc1..34376bec58dd 100644 --- a/.github/openclaw/tls-shipping-debug.py +++ b/.github/openclaw/tls-shipping-debug.py @@ -134,13 +134,49 @@ def inventory(root): '-DUSE_MIMALLOC=ON', '-DUSE_EXTERNAL_MIMALLOC=ON'] write('engine-build-command.json', {'argv': argv, 'compiler': 'clang/clang++ 23.1.1', 'linker_flags': '-fuse-ld=lld'}) +def header_comparison_bytes(name, path, shipping): + # The pinned packager adds a version after compilation; offlineasm embeds its Ruby-source hash. + if path.stat().st_size > 16 * 1024**2: + return None + data = path.read_bytes() + if name == 'include/cmakeconfig.h': + if shipping: + stamp = ('\n#define BUN_WEBKIT_VERSION "' + WEBKIT + '"\n').encode() + return data[:-len(stamp)] if data.endswith(stamp) else None + return data + provenance = { + 'include/JavaScriptCore/LLIntDesiredSettings.h': (0, b'// SettingsExtractor input hash: '), + 'include/JavaScriptCore/LLIntDesiredOffsets.h': (0, b'// OffsetExtractor input hash: '), + 'include/JavaScriptCore/LLIntAssembly.h': (-1, b'// offlineasm input hash: '), + } + if name not in provenance: + return None + index, prefix = provenance[name] + lines = data.splitlines(keepends=True) + observed = (b'23ac489dfebf7c9e22c7bc44e6826b164320e3d9' if shipping + else b'0c288d42d6b2ddc755f5b6e880f0cc180650c33f') + token = b' ' + observed + b' ' + if not lines or not lines[index].startswith(prefix) or lines[index].count(token) != 1: + return None + lines[index] = lines[index].replace(token, b' ', 1) + return b''.join(lines) + + def check_headers(headers, phase): missing = [str(name) for name in headers if str(name) not in before] mismatched = {str(name): {'shipping': before[str(name)], 'rebuilt': sha(path)} for name, path in headers.items() if str(name) in before and sha(path) != before[str(name)]} + raw_mismatched = dict(mismatched) + provenance_only = {} + for name in raw_mismatched: + original = header_comparison_bytes(name, bundle / name, True) + rebuilt = header_comparison_bytes(name, headers[Path(name)], False) + if original is not None and rebuilt is not None and original == rebuilt: + provenance_only[name] = {**mismatched.pop(name), + 'comparison_sha256': hashlib.sha256(original).hexdigest()} evidence = {} evidence_bytes = 0 - for name in sorted(mismatched)[:4]: + for name in sorted(raw_mismatched)[:4]: paths = {'shipping': bundle / name, 'rebuilt': headers[Path(name)]} size = sum(path.stat().st_size for path in paths.values()) if evidence_bytes + size > 128 * 1024**2: @@ -180,7 +216,8 @@ def check_headers(headers, phase): else: evidence_bytes += difference_bytes write(phase + '-header-parity.json', {'files': len(headers), 'missing': missing, - 'mismatched': mismatched, 'evidence': evidence}) + 'mismatched': mismatched, 'provenance_only': provenance_only, + 'raw_mismatched': raw_mismatched, 'evidence': evidence}) assert not missing and not mismatched, 'rebuilt core headers differ from the shipping consumer headers' From bd7da8a7caf63246f34042c26fd2467a231e0f02 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 00:33:28 +0800 Subject: [PATCH 22/74] test(tls): capture the destroyed completion owner after marking Reuse the immutable shipping-debug artifact, capture the actual unchanged fixture child and trace its matching weak target through the late callback and final explicit collection. Dump verifier marker data only after completed marking; keep raw path interpretation separate from runtime fixes. --- .github/openclaw/tls-retainer-capture.py | 398 ++++++++++++++++++++ .github/workflows/openclaw-tls-retainer.yml | 128 +++++++ 2 files changed, 526 insertions(+) create mode 100644 .github/openclaw/tls-retainer-capture.py create mode 100644 .github/workflows/openclaw-tls-retainer.yml diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py new file mode 100644 index 000000000000..a7d316c8d7cc --- /dev/null +++ b/.github/openclaw/tls-retainer-capture.py @@ -0,0 +1,398 @@ +"""Trace the frozen destroyed-completion owner with the saved shipping-debug bytes.""" +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import signal +import stat +import subprocess +import sys +import tarfile +import time +import zipfile +from xml.etree import ElementTree + +ROOT = Path('/work') +OUT = ROOT / 'capture' +SOURCE = ROOT / 'source' +BUILD = SOURCE / 'build/tls-arm64-shipping-debug' +BINARY = BUILD / 'bun' +PROFILE = BUILD / 'bun-profile' +FIXTURE = 'test/js/bun/net/socket-retention.test.ts' +CASE = 'a TLS Duplex completion retains its owner and is inert after destroy' +UUID = '487687ca47a8a631b87f701a12a351b33dab611a' +BUN = '454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e' +FIXTURE_SHA = 'd4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2' +ARCHIVE_SHA = 'a71acab37ebb674baae960ebe04d554f44f7e69284450855cf90f1d783180f0e' +BINARIES = { + 'bun': '6c1276f536bd52c19aee39ab3eea81b6ddf8c3e0b4e92aa879720a38c4f38cd3', + 'bun-profile': '2be841e8d061ea1a9a7a971a215ca9f1671088717881dded961360ad012d949a', +} + + +def sha(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def write(name, value): + (OUT / name).write_text(json.dumps(value, indent=2) + '\n') + + +def admit_zip(): + root = Path.cwd() + run = json.loads((root / 'producer-run.json').read_text()) + artifact = json.loads((root / 'producer-artifact.json').read_text()) + assert run['id'] == 41212340227 and run['conclusion'] == 'failure' + assert run['head_sha'] == '0a1544a4b4c6a4bdfbda96983e0734fc66e0320f' + assert artifact['id'] == 13048432420 and not artifact['expired'] + assert artifact['name'] == 'tls-linux-arm64-shipping-debug' + assert artifact['workflow_run']['id'] == run['id'] + assert artifact['workflow_run']['head_sha'] == run['head_sha'] + expected = '073a481c6576f8bf640be3e4da37aa1eb869ee037bdebd6f3eebfbdc6b01cb43' + assert artifact['digest'] == 'sha256:' + expected + assert (root / 'producer.zip').stat().st_size == artifact['size_in_bytes'] == 717328810 + assert sha(root / 'producer.zip') == expected + with zipfile.ZipFile(root / 'producer.zip') as archive: + entries = archive.infolist() + assert len(entries) == 2 and sum(e.file_size for e in entries) < 1024**3 + assert {e.filename for e in entries} == {'matched-shipping-debug.tar.zst', 'shipping-debug-SHA256SUMS'} + dest = root / 'producer' + dest.mkdir() + for entry in entries: + assert not entry.is_dir() and not stat.S_ISLNK(entry.external_attr >> 16) + with (dest / entry.filename).open('xb') as output, archive.open(entry) as content: + shutil.copyfileobj(content, output) + assert sha(dest / 'matched-shipping-debug.tar.zst') == ARCHIVE_SHA + assert (dest / 'shipping-debug-SHA256SUMS').read_text() == ARCHIVE_SHA + ' matched-shipping-debug.tar.zst\n' + + +def restore(): + assert os.uname().machine == 'aarch64' + assert subprocess.check_output(['git', '-c', 'safe.directory=' + str(SOURCE), '-C', str(SOURCE), + 'rev-parse', 'HEAD'], text=True).strip() == BUN + assert sha(SOURCE / FIXTURE) == FIXTURE_SHA + archive = ROOT / 'producer/matched-shipping-debug.tar.zst' + assert sha(archive) == ARCHIVE_SHA + entries = {} + manifest = None + total = 0 + child = subprocess.Popen(['zstd', '-d', '-c', str(archive)], stdout=subprocess.PIPE) + try: + with tarfile.open(fileobj=child.stdout, mode='r|') as stream: + for member in stream: + name = PurePosixPath(member.name) + assert member.isfile() and not name.is_absolute() and '..' not in name.parts + assert str(name) not in entries and len(entries) < 64 + total += member.size + assert total <= 8 * 1024**3 + if str(name).startswith('diagnostics/'): + target = ROOT / 'producer-evidence' / str(name) + else: + assert str(name) in {'source/build/tls-arm64-shipping-debug/' + n for n in BINARIES} + target = ROOT / str(name) + target.parent.mkdir(parents=True, exist_ok=True) + with target.open('xb') as output, stream.extractfile(member) as content: + shutil.copyfileobj(content, output) + target.chmod(0o755 if name.name in BINARIES else 0o644) + entries[str(name)] = {'bytes': target.stat().st_size, 'sha256': sha(target)} + if str(name) == 'diagnostics/archive-manifest.json': + manifest = json.loads(target.read_text()) + assert child.wait(timeout=30) == 0 + finally: + child.stdout.close() + if child.poll() is None: + child.kill() + child.wait() + assert len(entries) == 55 and len(manifest) == 54 + del entries['diagnostics/archive-manifest.json'] + assert entries == manifest + assert all(sha(BUILD / name) == digest for name, digest in BINARIES.items()) + assert OUT.is_dir() and not any(OUT.iterdir()) + write('inputs.json', {'producer_run': 41212340227, 'artifact': 13048432420, + 'archive_sha256': ARCHIVE_SHA, 'manifest_entries': len(manifest), + 'bun': BUN, 'fixture_sha256': FIXTURE_SHA, 'binaries': BINARIES, + 'claim': 'diagnostic child replay; no runtime fix or retainer classification'}) + + +def admit_failure(cases, producer): + selected = [case for case in cases if case.get('name') == CASE] + expected = [case for case in producer if case.get('name') == CASE] + assert len(selected) == len(expected) == 1 + assert all(case.find('skipped') is not None and case.find('failure') is None and + case.find('error') is None for case in cases if case is not selected[0]) + actual, baseline = selected[0], expected[0] + assert actual.get('file') == baseline.get('file') == FIXTURE + assert actual.get('line') == baseline.get('line') + assert actual.find('skipped') is None and actual.find('error') is None + failures, baseline_failures = actual.findall('failure'), baseline.findall('failure') + assert len(failures) == len(baseline_failures) == 1 + assert failures[0].get('type') == baseline_failures[0].get('type') == 'AssertionError' + normalize = lambda text: text.replace('\r\n', '\n').strip() + assert normalize(failures[0].get('message', '')) == normalize(baseline_failures[0].get('message', '')) + + +def debugger_stage(): + import lldb + debugger = lldb.debugger + debugger.SetAsync(False) + target = debugger.GetSelectedTarget() + module = target.FindModule(target.GetExecutable()) + assert re.sub('[^0-9a-fA-F]', '', module.GetUUIDString() or '').lower() == UUID + stage = os.environ['TLS_CAPTURE_STAGE'] + report = {'stage': stage, 'passed': False, 'events': []} + process = None + + def event(kind, **values): + report['events'].append({'kind': kind, **values}) + write(stage + '.json', report) + + def entry(name): + contexts = module.FindFunctions(name, lldb.eFunctionNameTypeAny) + addresses = {} + for i in range(contexts.GetSize()): + context = contexts.GetContextAtIndex(i) + function = context.GetFunction() + address = function.GetStartAddress() if function.IsValid() else context.GetSymbol().GetStartAddress() + if address.IsValid(): + addresses[address.GetFileAddress()] = address + assert len(addresses) == 1, 'missing/ambiguous function entry: ' + name + address = next(iter(addresses.values())) + bp = target.BreakpointCreateBySBAddress(address) + assert bp.GetNumLocations() == 1, name + return bp, address + + def reg(frame, name): + value = frame.FindRegister(name) + assert value.IsValid() and value.GetError().Success(), name + return value.GetValueAsUnsigned() + + def word(address, size=8): + error = lldb.SBError() + value = process.ReadUnsignedFromMemory(address, size, error) + assert error.Success(), str(error) + return value + + def text(address, limit=1024**2): + error = lldb.SBError() + value = process.ReadCStringFromMemory(address, limit, error) + assert error.Success() and value is not None and len(value.encode()) < limit - 1 + return value + + def typed(address, name): + values = target.FindTypes(name) + types = [values.GetTypeAtIndex(i).GetCanonicalType() for i in range(values.GetSize())] + types = [t for t in types if t.IsTypeComplete() and t.GetNumberOfFields() > 0] + assert types and len({t.GetByteSize() for t in types}) == 1, name + return target.CreateValueFromAddress(name, target.ResolveLoadAddress(address), types[0]).GetNonSyntheticValue() + + def field(value, name): + value = value.GetNonSyntheticValue().GetChildMemberWithName(name, lldb.eNoDynamicValues) + assert value.IsValid() and value.GetError().Success(), name + return value.GetNonSyntheticValue() + + def number(value): + error = lldb.SBError() + result = value.GetValueAsUnsigned(error) + assert error.Success(), str(error) + return result + + def pointer(value, pointee, depth=0): + assert depth <= 8 + value = value.GetNonSyntheticValue() + t = value.GetType().GetCanonicalType() + if t.IsPointerType() and t.GetPointeeType().GetCanonicalType().GetName() == pointee: + return [number(value)] + assert value.GetNumChildren() < 32 + result = [] + for i in range(value.GetNumChildren()): + child = value.GetChildAtIndex(i, lldb.eNoDynamicValues, False) + if not child.GetType().IsPointerType() or child.GetType().GetPointeeType().GetCanonicalType().GetName() == pointee: + result += pointer(child, pointee, depth + 1) + return result + + try: + environment = {'PATH': '/usr/bin:/bin', 'CI': '1', 'BUN_GARBAGE_COLLECTOR_LEVEL': '0'} + if stage == 'capture': + spawn, _ = entry('posix_spawn_bun') + args = ['test', FIXTURE, '--test-name-pattern', '^' + CASE + '$', + '--reporter=junit', '--reporter-outfile=' + str(OUT / 'fixture.xml')] + else: + child = json.loads((OUT / 'child.json').read_text()) + args = child['argv'][1:] + assert child['argv'][0] == str(BINARY) and len(args) == 2 and args[0] == '-e' + assert hashlib.sha256(args[1].encode()).hexdigest() == child['script_sha256'] + # These are the frozen harness's effective bunEnv additions; no host environment is inherited. + environment.update(GITHUB_ACTIONS='false', BUN_DEBUG_QUIET_LOGS='1', NO_COLOR='1', TZ='Etc/UTC', + BUN_RUNTIME_TRANSPILER_CACHE_PATH='0', BUN_FEATURE_FLAG_INTERNAL_FOR_TESTING='1', + BUN_DISABLE_SLOW_FILESYSTEM_WARNING='1', BUN_CONFIG_HTTP_RETRY_BACKOFF='1', + BUN_INTERNAL_INTERACTIVE_ASSUME_TTY='1', BUN_FEATURE_FLAG_EXPERIMENTAL_BAKE='1', + WANTS_LOUD='0', AGENT='false', BUN_JSC_verboseVerifyGC='1') + weak_bp, _ = entry('JSC::JSWeakObjectRef::finishCreation') + completion_bp, completion_address = entry('Bun__UpgradedDuplex__onWriteComplete') + gc_bp, _ = entry('JSC__VM__runGC') + mark_bp = target.BreakpointCreateByLocation('Collector.cpp', 1068) + assert mark_bp.GetNumLocations() > 0, 'missing post-mark verifier boundary' + mark_bp.SetEnabled(False) + _, dump_address = entry('JSC::VerifierSlotVisitor::dumpMarkerData') + target.GetBreakpointAtIndex(target.GetNumBreakpoints() - 1).SetEnabled(False) + weak = [] + owner = None + returned = False + return_bp = None + gc_count = 0 + info = lldb.SBLaunchInfo(args) + info.SetWorkingDirectory(str(SOURCE)) + info.SetEnvironmentEntries([key + '=' + value for key, value in environment.items()], False) + info.SetLaunchFlags(0) + assert info.AddOpenFileAction(0, '/dev/null', True, False) + assert info.AddOpenFileAction(1, str(OUT / (stage + '.stdout')), False, True) + assert info.AddOpenFileAction(2, str(OUT / (stage + '.stderr')), False, True) + error = lldb.SBError() + process = target.Launch(info, error) + assert error.Success(), str(error) + for _ in range(256): + if process.GetState() == lldb.eStateExited: + break + assert process.GetState() == lldb.eStateStopped, str(process.GetState()) + threads = [t for t in process if t.GetStopReason() == lldb.eStopReasonBreakpoint] + assert len(threads) == 1, 'unexpected stop or concurrent breakpoint' + thread = threads[0] + frame = thread.GetFrameAtIndex(0) + ids = {thread.GetStopReasonDataAtIndex(i) for i in range(0, thread.GetStopReasonDataCount(), 2)} + if stage == 'capture': + assert ids == {spawn.GetID()} + argv = reg(frame, 'x3') + values = [] + for i in range(64): + address = word(argv + i * 8) + if not address: + break + values.append(text(address)) + else: + raise RuntimeError('unterminated child argv') + if len(values) == 3 and values[:2] == [str(BINARY), '-e']: + assert not (OUT / 'child.json').exists(), 'multiple selected children' + write('child.json', {'argv': values, 'script_sha256': hashlib.sha256(values[2].encode()).hexdigest()}) + event('captured-actual-fixture-child') + elif weak_bp.GetID() in ids: + weak.append({'weak': reg(frame, 'x0'), 'vm': reg(frame, 'x1'), 'owner': reg(frame, 'x2')}) + assert len(weak) <= 16 + elif completion_bp.GetID() in ids: + call = reg(frame, 'x1') + # Frozen CallFrame.h: callee, argumentCountIncludingThis, then this, in 8-byte registers. + callee, count, this = word(call + 24), word(call + 32, 4), word(call + 40) + candidates = [w for w in weak if w['owner'] == this] + if candidates: + function = typed(callee, 'Zig::JSFFIFunction') + native = number(field(function, 'm_function')) + assert native == completion_address.GetLoadAddress(target) + if number(field(function, 'dataPtr')) == 0: + assert owner is None and len(candidates) == 1 + owner = dict(candidates[0], callee=callee, argument_count_including_this=count) + assert count == 1, 'fixture late callback unexpectedly has arguments' + return_bp = target.BreakpointCreateByAddress(reg(frame, 'x30')) + return_bp.SetThreadID(thread.GetThreadID()) + return_bp.SetOneShot(True) + event('destroyed-late-callback', **owner) + elif return_bp is not None and return_bp.GetID() in ids: + returned = True + event('late-callback-returned') + elif gc_bp.GetID() in ids: + if returned and reg(frame, 'x0') == owner['vm']: + assert reg(frame, 'w1') == 1 + gc_count += 1 + assert gc_count <= 31 + # The fixture collects once for queuedRoot, then thirty times before its final deref. + if gc_count == 31: + mark_bp.SetEnabled(True) + elif mark_bp.GetID() in ids: + assert returned and gc_count == 31 and 'Collector::verifyGC' in (frame.GetFunctionName() or '') + collector = frame.FindVariable('this', lldb.eNoDynamicValues) + vm = typed(owner['vm'], 'JSC::VM') + assert number(collector) == number(field(field(vm, 'heap'), 'm_collector')) + visitor_ptrs = pointer(field(collector.Dereference(), 'm_verifierSlotVisitor'), 'JSC::VerifierSlotVisitor') + assert len(set(visitor_ptrs)) == 1 and visitor_ptrs[0] + visitor = typed(visitor_ptrs[0], 'JSC::VerifierSlotVisitor') + assert number(field(visitor, 'm_doneMarking')) == 1 + w = typed(owner['weak'], 'JSC::JSWeakObjectRef') + targets = pointer(field(w, 'm_value'), 'JSC::JSCell') + assert targets == [owner['owner']], 'the observed weak target no longer matches' + event('post-mark-owner', gc_count=gc_count, verifier=visitor_ptrs[0], + weak_last_access=number(field(w, 'm_lastAccessVersion')), + vm_weak_version=number(field(vm, 'm_currentWeakRefVersion'))) + target.DisableAllBreakpoints() + options = lldb.SBExpressionOptions() + options.SetLanguage(lldb.eLanguageTypeC_plus_plus) + options.SetTimeoutInMicroSeconds(10_000_000) + options.SetIgnoreBreakpoints(True) + expression = '((void (*)(void*, void*))0x%x)((void*)0x%x, (void*)0x%x)' % ( + dump_address.GetLoadAddress(target), visitor_ptrs[0], owner['owner']) + result = frame.EvaluateExpression(expression, options) + assert result.GetError().Success(), str(result.GetError()) + event('marker-chain-dumped-after-mark') + else: + raise RuntimeError('unrecognized breakpoint') + assert process.Continue().Success() + assert process.GetState() == lldb.eStateExited, 'stop budget exhausted' + report['exit_code'] = process.GetExitStatus() + if stage == 'capture': + assert report['exit_code'] == 1 and (OUT / 'child.json').is_file() + cases = list(ElementTree.parse(OUT / 'fixture.xml').iter('testcase')) + producer = list(ElementTree.parse(ROOT / 'producer-evidence/diagnostics/fixture.xml').iter('testcase')) + admit_failure(cases, producer) + else: + assert report['exit_code'] == 0 and gc_count == 31 + facts = json.loads((OUT / 'trace.stdout').read_text().strip()) + assert facts['callbackRoot'] is True and facts['collected'] is False and facts['settled'] == ['ECANCELED'] + assert any(e['kind'] == 'marker-chain-dumped-after-mark' for e in report['events']) + marker = (OUT / 'trace.stderr').read_text() + assert ('cell 0x%x was marked' % owner['owner']) in marker + assert 'In the verifier GC, cell' in marker and 'Marker data is not available' not in marker + report['outcome'] = facts + report['classification'] = 'raw marker path captured; requires maintainer interpretation' + report['passed'] = True + except BaseException as error: + report['gap'] = str(error) + raise + finally: + if process is not None and process.IsValid() and process.GetState() != lldb.eStateExited: + process.Kill() + write(stage + '.json', report) + + +def capture(): + assert all(sha(BUILD / name) == digest for name, digest in BINARIES.items()) + assert sha(SOURCE / FIXTURE) == FIXTURE_SHA + for stage in ['capture', 'trace']: + env = {'PATH': '/usr/bin:/bin', 'TLS_CAPTURE_STAGE': stage} + command = ['lldb-23', '--no-lldbinit', '--batch', + '-o', 'settings set target.load-script-from-symbol-file false', + '-o', 'settings set target.inherit-env false', + '-o', 'target create ' + str(BINARY), '-o', 'target symbols add ' + str(PROFILE), + '-o', 'script import runpy; runpy.run_path(' + repr(str(Path(__file__).resolve())) + ', run_name="__lldb__")'] + with (OUT / (stage + '-debugger.log')).open('xb') as log: + proc = subprocess.Popen(command, cwd=SOURCE, env=env, stdin=subprocess.DEVNULL, + stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + started = time.monotonic() + try: + while proc.poll() is None: + assert time.monotonic() - started < 240, stage + ' debugger timeout' + assert sum(p.stat().st_size for p in OUT.iterdir() if p.is_file()) <= 32 * 1024**2 + time.sleep(0.25) + assert proc.returncode == 0 + finally: + if proc.poll() is None: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + assert json.loads((OUT / (stage + '.json')).read_text())['passed'], stage + ' diagnostic gap' + assert all(sha(BUILD / name) == digest for name, digest in BINARIES.items()) + assert sha(SOURCE / FIXTURE) == FIXTURE_SHA + + +if __name__ == '__lldb__': + debugger_stage() +elif __name__ == '__main__': + {'admit-zip': admit_zip, 'restore': restore, 'capture': capture}[sys.argv[1]]() diff --git a/.github/workflows/openclaw-tls-retainer.yml b/.github/workflows/openclaw-tls-retainer.yml new file mode 100644 index 000000000000..c49502efa453 --- /dev/null +++ b/.github/workflows/openclaw-tls-retainer.yml @@ -0,0 +1,128 @@ +name: OpenClaw TLS retainer capture + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-retainer.yml + - .github/openclaw/tls-retainer-capture.py + workflow_dispatch: + +permissions: + actions: read + contents: read + +concurrency: + group: openclaw-tls-retainer-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + linux-arm64: + runs-on: ubuntu-24.04-arm + timeout-minutes: 40 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: pipeline + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: .github/openclaw + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: source + ref: 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e + persist-credentials: false + - name: Admit the immutable shipping-debug artifact + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + test "$(uname -m)" = aarch64 + gh api repos/openclaw/bun/actions/runs/41212340227 > producer-run.json + gh api repos/openclaw/bun/actions/artifacts/13048432420 > producer-artifact.json + gh api repos/openclaw/bun/actions/artifacts/13048432420/zip > producer.zip + python3 pipeline/.github/openclaw/tls-retainer-capture.py admit-zip + - name: Restore matched bytes in a secretless debugger container + id: debugger + shell: bash + run: | + set -euo pipefail + mkdir capture + id=$(docker run --detach --cap-add=SYS_PTRACE --security-opt=seccomp=unconfined \ + -v "$GITHUB_WORKSPACE:/work" -w /work \ + docker.io/library/debian:13@sha256:9cc080028c43b27d2074d63a5f9caf7166d731494965616c1a6d2827a004585c \ + sleep infinity) + [[ "$id" =~ ^[0-9a-f]{64}$ ]] + echo "id=$id" >> "$GITHUB_OUTPUT" + docker exec "$id" sh -euc ' + apt-get update + apt-get install --yes --no-install-recommends ca-certificates curl python3 binutils zstd git + curl -fsSL --retry 5 https://apt.llvm.org/llvm-snapshot.gpg.key > /etc/apt/trusted.gpg.d/apt.llvm.org.asc + echo "deb https://apt.llvm.org/trixie/ llvm-toolchain-trixie-23 main" > /etc/apt/sources.list.d/llvm.list + apt-get update + apt-get install --yes --no-install-recommends llvm-23 lldb-23 python3-lldb-23 + ' + docker exec "$id" python3 pipeline/.github/openclaw/tls-retainer-capture.py restore + cp producer-run.json producer-artifact.json capture/ + docker exec "$id" lldb-23 --version > capture/debugger-version.txt + docker exec "$id" ldd source/build/tls-arm64-shipping-debug/bun > capture/runtime-libraries.txt + ! grep -q 'not found' capture/runtime-libraries.txt + - name: Capture the actual fixture child and its post-mark owner path + env: + CONTAINER_ID: ${{ steps.debugger.outputs.id }} + shell: bash + run: | + set -euo pipefail + [[ "$CONTAINER_ID" =~ ^[0-9a-f]{64}$ ]] + docker exec "$CONTAINER_ID" python3 pipeline/.github/openclaw/tls-retainer-capture.py capture + - name: Freeze and package bounded capture evidence + if: always() + env: + CONTAINER_ID: ${{ steps.debugger.outputs.id }} + shell: bash + run: | + set -euo pipefail + if test -n "$CONTAINER_ID"; then + [[ "$CONTAINER_ID" =~ ^[0-9a-f]{64}$ ]] + docker stop --time 15 "$CONTAINER_ID" > /dev/null + test "$(docker inspect --format '{{.State.Running}}' "$CONTAINER_ID")" = false + fi + mkdir -p capture + cp pipeline/.github/openclaw/tls-retainer-capture.py capture/probe.py + python3 - <<'PY' + import hashlib, json, stat, tarfile + from pathlib import Path + files = sorted(Path('capture').iterdir()) + assert files and len(files) <= 24 + assert all(stat.S_ISREG(p.lstat().st_mode) for p in files) + assert sum(p.stat().st_size for p in files) <= 32 * 1024**2 + def sha(p): + with p.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + manifest = Path('capture/archive-manifest.json') + manifest.write_text(json.dumps({p.name: {'bytes': p.stat().st_size, 'sha256': sha(p)} for p in files}, indent=2) + '\n') + files.append(manifest) + with tarfile.open('tls-retainer-proof.tar.gz', 'x:gz') as archive: + for p in files: + archive.add(p, arcname=p.name, recursive=False) + Path('tls-retainer-SHA256SUMS').write_text(sha(Path('tls-retainer-proof.tar.gz')) + ' tls-retainer-proof.tar.gz\n') + PY + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: tls-linux-arm64-retainer + path: | + tls-retainer-proof.tar.gz + tls-retainer-SHA256SUMS + if-no-files-found: error + retention-days: 7 + - name: Remove this job's debugger container + if: always() + env: + CONTAINER_ID: ${{ steps.debugger.outputs.id }} + shell: bash + run: | + if test -n "$CONTAINER_ID"; then + [[ "$CONTAINER_ID" =~ ^[0-9a-f]{64}$ ]] + docker rm --force "$CONTAINER_ID" > /dev/null + fi From c2f61af74c8375d63254ece3e41bab3416cf4a7d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 00:39:20 +0800 Subject: [PATCH 23/74] test(tls): record native capture breakpoint observations Retain entry identities, raw hit counts and final owner/GC correlation on diagnostic failure without changing the capture gates. --- .github/openclaw/tls-retainer-capture.py | 27 ++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py index a7d316c8d7cc..2a249fe606ce 100644 --- a/.github/openclaw/tls-retainer-capture.py +++ b/.github/openclaw/tls-retainer-capture.py @@ -142,8 +142,12 @@ def debugger_stage(): module = target.FindModule(target.GetExecutable()) assert re.sub('[^0-9a-fA-F]', '', module.GetUUIDString() or '').lower() == UUID stage = os.environ['TLS_CAPTURE_STAGE'] - report = {'stage': stage, 'passed': False, 'events': []} + report = {'stage': stage, 'passed': False, 'events': [], 'entries': [], 'breakpoint_hits': {}} process = None + weak = [] + owner = None + returned = False + gc_count = 0 def event(kind, **values): report['events'].append({'kind': kind, **values}) @@ -162,6 +166,8 @@ def entry(name): address = next(iter(addresses.values())) bp = target.BreakpointCreateBySBAddress(address) assert bp.GetNumLocations() == 1, name + report['entries'].append({'name': name, 'breakpoint': bp.GetID(), + 'file_address': address.GetFileAddress()}) return bp, address def reg(frame, name): @@ -238,11 +244,7 @@ def pointer(value, pointee, depth=0): mark_bp.SetEnabled(False) _, dump_address = entry('JSC::VerifierSlotVisitor::dumpMarkerData') target.GetBreakpointAtIndex(target.GetNumBreakpoints() - 1).SetEnabled(False) - weak = [] - owner = None - returned = False return_bp = None - gc_count = 0 info = lldb.SBLaunchInfo(args) info.SetWorkingDirectory(str(SOURCE)) info.SetEnvironmentEntries([key + '=' + value for key, value in environment.items()], False) @@ -262,6 +264,11 @@ def pointer(value, pointee, depth=0): thread = threads[0] frame = thread.GetFrameAtIndex(0) ids = {thread.GetStopReasonDataAtIndex(i) for i in range(0, thread.GetStopReasonDataCount(), 2)} + for bp_id in ids: + key = str(bp_id) + report['breakpoint_hits'][key] = report['breakpoint_hits'].get(key, 0) + 1 + report['last_stop'] = {'pc': frame.GetPC(), 'function': frame.GetFunctionName(), + 'breakpoints': sorted(ids)} if stage == 'capture': assert ids == {spawn.GetID()} argv = reg(frame, 'x3') @@ -280,11 +287,14 @@ def pointer(value, pointee, depth=0): elif weak_bp.GetID() in ids: weak.append({'weak': reg(frame, 'x0'), 'vm': reg(frame, 'x1'), 'owner': reg(frame, 'x2')}) assert len(weak) <= 16 + event('weak-ref-entry', **weak[-1]) elif completion_bp.GetID() in ids: call = reg(frame, 'x1') # Frozen CallFrame.h: callee, argumentCountIncludingThis, then this, in 8-byte registers. callee, count, this = word(call + 24), word(call + 32, 4), word(call + 40) candidates = [w for w in weak if w['owner'] == this] + event('completion-entry', callee=callee, this=this, argument_count_including_this=count, + matching_weak_targets=len(candidates)) if candidates: function = typed(callee, 'Zig::JSFFIFunction') native = number(field(function, 'm_function')) @@ -301,6 +311,8 @@ def pointer(value, pointee, depth=0): returned = True event('late-callback-returned') elif gc_bp.GetID() in ids: + event('explicit-gc-entry', vm=reg(frame, 'x0'), sync=reg(frame, 'w1'), + after_late_callback=returned) if returned and reg(frame, 'x0') == owner['vm']: assert reg(frame, 'w1') == 1 gc_count += 1 @@ -344,7 +356,7 @@ def pointer(value, pointee, depth=0): producer = list(ElementTree.parse(ROOT / 'producer-evidence/diagnostics/fixture.xml').iter('testcase')) admit_failure(cases, producer) else: - assert report['exit_code'] == 0 and gc_count == 31 + assert report['exit_code'] == 0 and gc_count == 31, '31 post-callback explicit collections not observed' facts = json.loads((OUT / 'trace.stdout').read_text().strip()) assert facts['callbackRoot'] is True and facts['collected'] is False and facts['settled'] == ['ECANCELED'] assert any(e['kind'] == 'marker-chain-dumped-after-mark' for e in report['events']) @@ -358,6 +370,9 @@ def pointer(value, pointee, depth=0): report['gap'] = str(error) raise finally: + if stage == 'trace': + report['observation'] = {'weak_targets': weak, 'owner': owner, 'returned': returned, + 'gc_count': gc_count} if process is not None and process.IsValid() and process.GetState() != lldb.eStateExited: process.Kill() write(stage + '.json', report) From 378e5eea1fe40dd0500c375ef29db996d3978a34 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 00:47:56 +0800 Subject: [PATCH 24/74] test(tls): bind WeakRef observation to its physical constructor Reject enclosing inline-function lookup results before reading ABI arguments, and capture the constructor target plus actual returned WeakRef passively. Preserve exact entry-PC, callback, post-mark and outcome checks. --- .github/openclaw/tls-retainer-capture.py | 44 ++++++++++++++++++++---- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py index 2a249fe606ce..db7f73ef13a1 100644 --- a/.github/openclaw/tls-retainer-capture.py +++ b/.github/openclaw/tls-retainer-capture.py @@ -142,9 +142,11 @@ def debugger_stage(): module = target.FindModule(target.GetExecutable()) assert re.sub('[^0-9a-fA-F]', '', module.GetUUIDString() or '').lower() == UUID stage = os.environ['TLS_CAPTURE_STAGE'] - report = {'stage': stage, 'passed': False, 'events': [], 'entries': [], 'breakpoint_hits': {}} + report = {'stage': stage, 'passed': False, 'events': [], 'entries': [], 'resolutions': {}, 'breakpoint_hits': {}} process = None + entry_addresses = {} weak = [] + weak_creation = None owner = None returned = False gc_count = 0 @@ -155,17 +157,25 @@ def event(kind, **values): def entry(name): contexts = module.FindFunctions(name, lldb.eFunctionNameTypeAny) + assert contexts.GetSize() <= 64, name addresses = {} + report['resolutions'][name] = [] for i in range(contexts.GetSize()): context = contexts.GetContextAtIndex(i) function = context.GetFunction() + resolved = function.GetName() if function.IsValid() else context.GetSymbol().GetName() address = function.GetStartAddress() if function.IsValid() else context.GetSymbol().GetStartAddress() + report['resolutions'][name].append({'function': resolved, 'file_address': address.GetFileAddress()}) + # FindFunctions includes inline scopes whose GetFunction is the enclosing physical function. + if not resolved or resolved.split('(', 1)[0].strip().removeprefix('::') != name: + continue if address.IsValid(): addresses[address.GetFileAddress()] = address assert len(addresses) == 1, 'missing/ambiguous function entry: ' + name address = next(iter(addresses.values())) bp = target.BreakpointCreateBySBAddress(address) assert bp.GetNumLocations() == 1, name + entry_addresses[bp.GetID()] = address report['entries'].append({'name': name, 'breakpoint': bp.GetID(), 'file_address': address.GetFileAddress()}) return bp, address @@ -236,7 +246,8 @@ def pointer(value, pointee, depth=0): BUN_DISABLE_SLOW_FILESYSTEM_WARNING='1', BUN_CONFIG_HTTP_RETRY_BACKOFF='1', BUN_INTERNAL_INTERACTIVE_ASSUME_TTY='1', BUN_FEATURE_FLAG_EXPERIMENTAL_BAKE='1', WANTS_LOUD='0', AGENT='false', BUN_JSC_verboseVerifyGC='1') - weak_bp, _ = entry('JSC::JSWeakObjectRef::finishCreation') + weak_bp, _ = entry('JSC::constructWeakRef') + weak_return_bp = None completion_bp, completion_address = entry('Bun__UpgradedDuplex__onWriteComplete') gc_bp, _ = entry('JSC__VM__runGC') mark_bp = target.BreakpointCreateByLocation('Collector.cpp', 1068) @@ -264,11 +275,14 @@ def pointer(value, pointee, depth=0): thread = threads[0] frame = thread.GetFrameAtIndex(0) ids = {thread.GetStopReasonDataAtIndex(i) for i in range(0, thread.GetStopReasonDataCount(), 2)} + report['last_stop'] = {'pc': frame.GetPC(), 'function': frame.GetFunctionName(), + 'breakpoints': sorted(ids), 'expected_entry_pcs': { + str(i): entry_addresses[i].GetLoadAddress(target) for i in ids if i in entry_addresses}} for bp_id in ids: key = str(bp_id) report['breakpoint_hits'][key] = report['breakpoint_hits'].get(key, 0) + 1 - report['last_stop'] = {'pc': frame.GetPC(), 'function': frame.GetFunctionName(), - 'breakpoints': sorted(ids)} + if bp_id in entry_addresses: + assert frame.GetPC() == entry_addresses[bp_id].GetLoadAddress(target), 'stop is not at function entry' if stage == 'capture': assert ids == {spawn.GetID()} argv = reg(frame, 'x3') @@ -285,9 +299,25 @@ def pointer(value, pointee, depth=0): write('child.json', {'argv': values, 'script_sha256': hashlib.sha256(values[2].encode()).hexdigest()}) event('captured-actual-fixture-child') elif weak_bp.GetID() in ids: - weak.append({'weak': reg(frame, 'x0'), 'vm': reg(frame, 'x1'), 'owner': reg(frame, 'x2')}) + assert weak_creation is None + global_object, call = reg(frame, 'x0'), reg(frame, 'x1') + assert word(call + 32, 4) == 2, 'fixture WeakRef constructor arguments changed' + weak_creation = {'global': global_object, + 'vm': number(field(typed(global_object, 'JSC::JSGlobalObject'), 'm_vm')), + 'owner': word(call + 48)} + weak_return_bp = target.BreakpointCreateByAddress(reg(frame, 'x30')) + weak_return_bp.SetThreadID(thread.GetThreadID()) + weak_return_bp.SetOneShot(True) + event('weak-ref-constructor-entry', **weak_creation) + elif weak_return_bp is not None and weak_return_bp.GetID() in ids: + assert weak_creation is not None + created = reg(frame, 'x0') + actual_target = pointer(field(typed(created, 'JSC::JSWeakObjectRef'), 'm_value'), 'JSC::JSCell') + assert actual_target == [weak_creation['owner']], 'constructed weak target does not match' + weak.append(dict(weak_creation, weak=created)) assert len(weak) <= 16 - event('weak-ref-entry', **weak[-1]) + weak_creation = None + event('weak-ref-created', **weak[-1]) elif completion_bp.GetID() in ids: call = reg(frame, 'x1') # Frozen CallFrame.h: callee, argumentCountIncludingThis, then this, in 8-byte registers. @@ -372,7 +402,7 @@ def pointer(value, pointee, depth=0): finally: if stage == 'trace': report['observation'] = {'weak_targets': weak, 'owner': owner, 'returned': returned, - 'gc_count': gc_count} + 'gc_count': gc_count, 'pending_weak_constructor': weak_creation} if process is not None and process.IsValid() and process.GetState() != lldb.eStateExited: process.Kill() write(stage + '.json', report) From 1f159fd68469b538941e91daa76aac3ab72b4ed4 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 01:04:03 +0800 Subject: [PATCH 25/74] test(tls): read verifier marker stacks without inferior calls --- .github/openclaw/tls-retainer-capture.py | 86 +++++++++++++++++++----- 1 file changed, 68 insertions(+), 18 deletions(-) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py index db7f73ef13a1..6fa28c10b5d1 100644 --- a/.github/openclaw/tls-retainer-capture.py +++ b/.github/openclaw/tls-retainer-capture.py @@ -197,12 +197,15 @@ def text(address, limit=1024**2): assert error.Success() and value is not None and len(value.encode()) < limit - 1 return value - def typed(address, name): + def concrete_type(name): values = target.FindTypes(name) types = [values.GetTypeAtIndex(i).GetCanonicalType() for i in range(values.GetSize())] types = [t for t in types if t.IsTypeComplete() and t.GetNumberOfFields() > 0] assert types and len({t.GetByteSize() for t in types}) == 1, name - return target.CreateValueFromAddress(name, target.ResolveLoadAddress(address), types[0]).GetNonSyntheticValue() + return types[0] + + def typed(address, name): + return target.CreateValueFromAddress(name, target.ResolveLoadAddress(address), concrete_type(name)).GetNonSyntheticValue() def field(value, name): value = value.GetNonSyntheticValue().GetChildMemberWithName(name, lldb.eNoDynamicValues) @@ -229,6 +232,65 @@ def pointer(value, pointee, depth=0): result += pointer(child, pointee, depth + 1) return result + def marker_snapshot(visitor, cell): + # This fixture's owner is precise-allocated. Use frozen HashTable metadata and DWARF field offsets; + # target-side marker printers symbolize through native libraries and faulted on these saved bytes. + assert target.GetAddressByteSize() == 8 and target.GetByteOrder() == lldb.eByteOrderLittle + assert cell & 8, 'owner is not the observed precise-allocation kind' + header = concrete_type('JSC::PreciseAllocation').GetByteSize() + allocation = cell - ((header + 15) & ~15) + marked = field(field(typed(allocation, 'JSC::PreciseAllocation'), 'm_isMarked'), 'value') + assert marked.GetByteSize() == 1 and word(marked.GetLoadAddress(), 1) == 1, 'owner was not marked by the real GC' + table_value = field(field(field(visitor, 'm_preciseAllocationMap'), 'm_impl'), 'm_table') + table = number(table_value) + assert table + capacity = word(table - 4, 4) + bucket_type = table_value.GetType().GetCanonicalType().GetPointeeType().GetCanonicalType() + stride = bucket_type.GetByteSize() + members = {bucket_type.GetFieldAtIndex(i).GetName(): bucket_type.GetFieldAtIndex(i) + for i in range(bucket_type.GetNumberOfFields())} + assert {'key', 'value'} <= members.keys() + key = members['key'] + assert key.GetType().GetCanonicalType().IsPointerType() and key.GetType().GetByteSize() == 8 + offset = key.GetOffsetInBytes() + assert 0 < capacity <= 2**20 and capacity & (capacity - 1) == 0 + assert 8 <= stride <= 256 and offset + 8 <= stride and capacity * stride <= 16 * 1024**2 + error = lldb.SBError() + buckets = process.ReadMemory(table, capacity * stride, error) + assert error.Success() and len(buckets) == capacity * stride + matches = [i for i in range(capacity) + if int.from_bytes(buckets[i * stride + offset:i * stride + offset + 8], 'little') == allocation] + assert len(matches) == 1, 'actual precise allocation missing/duplicated in verifier map' + bucket = target.CreateValueFromAddress('owner-marker-bucket', + target.ResolveLoadAddress(table + matches[0] * stride), bucket_type).GetNonSyntheticValue() + data = pointer(field(bucket, 'value'), 'JSC::VerifierSlotVisitor::PreciseAllocationData') + assert len(data) == 1 and data[0] + record = typed(data[0], 'JSC::VerifierSlotVisitor::PreciseAllocationData') + assert number(field(record, 'm_allocation')) == allocation + marker = field(record, 'm_marker') + referrer = number(field(field(marker, 'm_referrer'), 'm_bits')) + assert referrer == 6, 'marker is not the observed direct ConservativeScan root' + stacks = pointer(field(marker, 'm_stack'), 'WTF::StackTrace') + assert len(stacks) == 1 and stacks[0] + stack = typed(stacks[0], 'WTF::StackTrace') + count, initial = number(field(stack, 'm_size')), number(field(stack, 'm_initialFrame')) + assert 0 < count <= 100 and initial == 2 + start = field(stack, 'm_stack').GetLoadAddress() + initial * 8 + frames = [] + for i in range(count): + pc = word(start + i * 8) + assert pc + address = target.ResolveLoadAddress(pc) + function, symbol = address.GetFunction(), address.GetSymbol() + name = function.GetName() if function.IsValid() else symbol.GetName() if symbol.IsValid() else None + line = address.GetLineEntry() + frames.append({'pc': pc, 'function': name, 'file_address': address.GetFileAddress(), + 'line': line.GetLine() if line.IsValid() else None}) + assert any(frame['function'] for frame in frames), 'no marker frame resolved' + event('passive-marker-snapshot', owner=cell, allocation=allocation, real_gc_marked=True, referrer_bits=referrer, + root_reason='ConservativeScan', stack=stacks[0], frames=frames, + unresolved_frames=sum(frame['function'] is None for frame in frames)) + try: environment = {'PATH': '/usr/bin:/bin', 'CI': '1', 'BUN_GARBAGE_COLLECTOR_LEVEL': '0'} if stage == 'capture': @@ -253,8 +315,6 @@ def pointer(value, pointee, depth=0): mark_bp = target.BreakpointCreateByLocation('Collector.cpp', 1068) assert mark_bp.GetNumLocations() > 0, 'missing post-mark verifier boundary' mark_bp.SetEnabled(False) - _, dump_address = entry('JSC::VerifierSlotVisitor::dumpMarkerData') - target.GetBreakpointAtIndex(target.GetNumBreakpoints() - 1).SetEnabled(False) return_bp = None info = lldb.SBLaunchInfo(args) info.SetWorkingDirectory(str(SOURCE)) @@ -366,15 +426,7 @@ def pointer(value, pointee, depth=0): weak_last_access=number(field(w, 'm_lastAccessVersion')), vm_weak_version=number(field(vm, 'm_currentWeakRefVersion'))) target.DisableAllBreakpoints() - options = lldb.SBExpressionOptions() - options.SetLanguage(lldb.eLanguageTypeC_plus_plus) - options.SetTimeoutInMicroSeconds(10_000_000) - options.SetIgnoreBreakpoints(True) - expression = '((void (*)(void*, void*))0x%x)((void*)0x%x, (void*)0x%x)' % ( - dump_address.GetLoadAddress(target), visitor_ptrs[0], owner['owner']) - result = frame.EvaluateExpression(expression, options) - assert result.GetError().Success(), str(result.GetError()) - event('marker-chain-dumped-after-mark') + marker_snapshot(visitor, owner['owner']) else: raise RuntimeError('unrecognized breakpoint') assert process.Continue().Success() @@ -389,12 +441,10 @@ def pointer(value, pointee, depth=0): assert report['exit_code'] == 0 and gc_count == 31, '31 post-callback explicit collections not observed' facts = json.loads((OUT / 'trace.stdout').read_text().strip()) assert facts['callbackRoot'] is True and facts['collected'] is False and facts['settled'] == ['ECANCELED'] - assert any(e['kind'] == 'marker-chain-dumped-after-mark' for e in report['events']) - marker = (OUT / 'trace.stderr').read_text() - assert ('cell 0x%x was marked' % owner['owner']) in marker - assert 'In the verifier GC, cell' in marker and 'Marker data is not available' not in marker + snapshots = [e for e in report['events'] if e['kind'] == 'passive-marker-snapshot'] + assert len(snapshots) == 1 and snapshots[0]['owner'] == owner['owner'] report['outcome'] = facts - report['classification'] = 'raw marker path captured; requires maintainer interpretation' + report['classification'] = 'stored ConservativeScan marker stack captured; originating word remains unproven' report['passed'] = True except BaseException as error: report['gap'] = str(error) From a4e984c27883f04283060c4455e25b5ad201ac87 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 01:10:01 +0800 Subject: [PATCH 26/74] test(tls): record marker layout before stack admission --- .github/openclaw/tls-retainer-capture.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py index 6fa28c10b5d1..014d3436d523 100644 --- a/.github/openclaw/tls-retainer-capture.py +++ b/.github/openclaw/tls-retainer-capture.py @@ -274,6 +274,26 @@ def marker_snapshot(visitor, cell): assert len(stacks) == 1 and stacks[0] stack = typed(stacks[0], 'WTF::StackTrace') count, initial = number(field(stack, 'm_size')), number(field(stack, 'm_initialFrame')) + layout = stack.GetType().GetCanonicalType() + event('passive-marker-layout', owner=cell, allocation=allocation, allocation_header_bytes=header, + marker=marker.GetLoadAddress(), referrer_bits=referrer, stack=stacks[0], count=count, initial=initial, + raw_header_words=[word(stacks[0]), word(stacks[0] + 8)], type_bytes=layout.GetByteSize(), + fields={layout.GetFieldAtIndex(i).GetName(): layout.GetFieldAtIndex(i).GetOffsetInBytes() + for i in range(layout.GetNumberOfFields())}) + contexts = {} + for name in ['WTF::StackTrace::captureStackTrace', 'JSC::VerifierSlotVisitor::testAndSetMarked']: + found = module.FindFunctions(name, lldb.eFunctionNameTypeAny) + assert found.GetSize() <= 64 + contexts[name] = [] + for i in range(found.GetSize()): + context = found.GetContextAtIndex(i) + function, block = context.GetFunction(), context.GetBlock() + start = function.GetStartAddress().GetLineEntry() + contexts[name].append({'function': function.GetName(), + 'compile_unit': str(context.GetCompileUnit().GetFileSpec()), + 'source': str(start.GetFileSpec()), 'line': start.GetLine(), + 'inline': block.IsInlined(), 'inline_name': block.GetInlinedName()}) + event('marker-source-contexts', contexts=contexts) assert 0 < count <= 100 and initial == 2 start = field(stack, 'm_stack').GetLoadAddress() + initial * 8 frames = [] From d694f609b234b73715ed2f88c1648ac4ca89e254 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 01:18:29 +0800 Subject: [PATCH 27/74] test(tls): capture the first precise mark through frame pointers --- .github/openclaw/tls-retainer-capture.py | 99 +++++++++++++++++++----- 1 file changed, 80 insertions(+), 19 deletions(-) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py index 014d3436d523..212cc5aa5866 100644 --- a/.github/openclaw/tls-retainer-capture.py +++ b/.github/openclaw/tls-retainer-capture.py @@ -150,12 +150,13 @@ def debugger_stage(): owner = None returned = False gc_count = 0 + live_marker = None def event(kind, **values): report['events'].append({'kind': kind, **values}) write(stage + '.json', report) - def entry(name): + def entry(name, signature=None): contexts = module.FindFunctions(name, lldb.eFunctionNameTypeAny) assert contexts.GetSize() <= 64, name addresses = {} @@ -169,6 +170,8 @@ def entry(name): # FindFunctions includes inline scopes whose GetFunction is the enclosing physical function. if not resolved or resolved.split('(', 1)[0].strip().removeprefix('::') != name: continue + if signature is not None and resolved.removeprefix('::') != signature: + continue if address.IsValid(): addresses[address.GetFileAddress()] = address assert len(addresses) == 1, 'missing/ambiguous function entry: ' + name @@ -232,18 +235,16 @@ def pointer(value, pointee, depth=0): result += pointer(child, pointee, depth + 1) return result - def marker_snapshot(visitor, cell): - # This fixture's owner is precise-allocated. Use frozen HashTable metadata and DWARF field offsets; - # target-side marker printers symbolize through native libraries and faulted on these saved bytes. + def precise_marker(visitor, cell): + # Use frozen HashTable metadata and DWARF field offsets, without inferior find/get calls. assert target.GetAddressByteSize() == 8 and target.GetByteOrder() == lldb.eByteOrderLittle assert cell & 8, 'owner is not the observed precise-allocation kind' header = concrete_type('JSC::PreciseAllocation').GetByteSize() allocation = cell - ((header + 15) & ~15) - marked = field(field(typed(allocation, 'JSC::PreciseAllocation'), 'm_isMarked'), 'value') - assert marked.GetByteSize() == 1 and word(marked.GetLoadAddress(), 1) == 1, 'owner was not marked by the real GC' table_value = field(field(field(visitor, 'm_preciseAllocationMap'), 'm_impl'), 'm_table') table = number(table_value) - assert table + if not table: + return header, allocation, None capacity = word(table - 4, 4) bucket_type = table_value.GetType().GetCanonicalType().GetPointeeType().GetCanonicalType() stride = bucket_type.GetByteSize() @@ -260,14 +261,61 @@ def marker_snapshot(visitor, cell): assert error.Success() and len(buckets) == capacity * stride matches = [i for i in range(capacity) if int.from_bytes(buckets[i * stride + offset:i * stride + offset + 8], 'little') == allocation] - assert len(matches) == 1, 'actual precise allocation missing/duplicated in verifier map' + assert len(matches) <= 1, 'actual precise allocation duplicated in verifier map' + if not matches: + return header, allocation, None bucket = target.CreateValueFromAddress('owner-marker-bucket', target.ResolveLoadAddress(table + matches[0] * stride), bucket_type).GetNonSyntheticValue() data = pointer(field(bucket, 'value'), 'JSC::VerifierSlotVisitor::PreciseAllocationData') assert len(data) == 1 and data[0] record = typed(data[0], 'JSC::VerifierSlotVisitor::PreciseAllocationData') assert number(field(record, 'm_allocation')) == allocation - marker = field(record, 'm_marker') + return header, allocation, field(record, 'm_marker') + + def symbolize(pc, is_return=False): + address = target.ResolveLoadAddress(pc - 1 if is_return else pc) + function, symbol = address.GetFunction(), address.GetSymbol() + name = function.GetName() if function.IsValid() else symbol.GetName() if symbol.IsValid() else None + line = address.GetLineEntry() + return {'pc': pc, 'function': name, 'file_address': address.GetFileAddress(), + 'source_file': line.GetFileSpec().GetFilename() if line.IsValid() else None, + 'line': line.GetLine() if line.IsValid() else None} + + def frame_pointer_snapshot(frame, thread): + # Shipping keeps frame pointers but removes CFI tables; libc backtrace stored an empty marker stack. + pc, lr, sp, fp = frame.GetPC(), reg(frame, 'x30'), reg(frame, 'sp'), reg(frame, 'x29') + region = lldb.SBMemoryRegionInfo() + assert process.GetMemoryRegionInfo(sp, region).Success() and region.IsReadable() + ceiling = min(region.GetRegionEnd(), sp + 16 * 1024**2) + frames = [symbolize(pc), symbolize(lr, True)] + links = [] + current = fp + for _ in range(128): + if not current: + termination = 'null-frame-pointer' + break + if current % 16 or not sp <= current or current + 16 > ceiling: + termination = 'outside-readable-stack' + break + previous, saved_lr = word(current), word(current + 8) + links.append({'fp': current, 'previous_fp': previous, 'saved_lr': saved_lr}) + if saved_lr: + frames.append(symbolize(saved_lr, True)) + if previous and previous <= current: + termination = 'non-increasing-frame-pointer' + break + current = previous + else: + termination = 'frame-limit' + assert len(links) >= 2 and sum(bool(f['function']) for f in frames) >= 3, 'live frame-pointer path unavailable' + return {'thread': thread.GetThreadID(), 'pc': pc, 'lr': lr, 'sp': sp, 'fp': fp, + 'links': links, 'frames': frames, 'termination': termination} + + def marker_snapshot(visitor, cell): + header, allocation, marker = precise_marker(visitor, cell) + assert marker is not None, 'actual precise allocation missing in completed verifier map' + marked = field(field(typed(allocation, 'JSC::PreciseAllocation'), 'm_isMarked'), 'value') + assert marked.GetByteSize() == 1 and word(marked.GetLoadAddress(), 1) == 1, 'owner was not marked by the real GC' referrer = number(field(field(marker, 'm_referrer'), 'm_bits')) assert referrer == 6, 'marker is not the observed direct ConservativeScan root' stacks = pointer(field(marker, 'm_stack'), 'WTF::StackTrace') @@ -294,19 +342,13 @@ def marker_snapshot(visitor, cell): 'source': str(start.GetFileSpec()), 'line': start.GetLine(), 'inline': block.IsInlined(), 'inline_name': block.GetInlinedName()}) event('marker-source-contexts', contexts=contexts) - assert 0 < count <= 100 and initial == 2 + assert (count == 0 and initial == 0) or (0 < count <= 100 and initial == 2) start = field(stack, 'm_stack').GetLoadAddress() + initial * 8 frames = [] for i in range(count): pc = word(start + i * 8) assert pc - address = target.ResolveLoadAddress(pc) - function, symbol = address.GetFunction(), address.GetSymbol() - name = function.GetName() if function.IsValid() else symbol.GetName() if symbol.IsValid() else None - line = address.GetLineEntry() - frames.append({'pc': pc, 'function': name, 'file_address': address.GetFileAddress(), - 'line': line.GetLine() if line.IsValid() else None}) - assert any(frame['function'] for frame in frames), 'no marker frame resolved' + frames.append(symbolize(pc, True)) event('passive-marker-snapshot', owner=cell, allocation=allocation, real_gc_marked=True, referrer_bits=referrer, root_reason='ConservativeScan', stack=stacks[0], frames=frames, unresolved_frames=sum(frame['function'] is None for frame in frames)) @@ -332,6 +374,9 @@ def marker_snapshot(visitor, cell): weak_return_bp = None completion_bp, completion_address = entry('Bun__UpgradedDuplex__onWriteComplete') gc_bp, _ = entry('JSC__VM__runGC') + live_mark_bp, _ = entry('JSC::VerifierSlotVisitor::testAndSetMarked', + 'JSC::VerifierSlotVisitor::testAndSetMarked(JSC::PreciseAllocation&)') + live_mark_bp.SetEnabled(False) mark_bp = target.BreakpointCreateByLocation('Collector.cpp', 1068) assert mark_bp.GetNumLocations() > 0, 'missing post-mark verifier boundary' mark_bp.SetEnabled(False) @@ -346,7 +391,7 @@ def marker_snapshot(visitor, cell): error = lldb.SBError() process = target.Launch(info, error) assert error.Success(), str(error) - for _ in range(256): + for _ in range(16384 if stage == 'trace' else 256): if process.GetState() == lldb.eStateExited: break assert process.GetState() == lldb.eStateStopped, str(process.GetState()) @@ -429,7 +474,21 @@ def marker_snapshot(visitor, cell): assert gc_count <= 31 # The fixture collects once for queuedRoot, then thirty times before its final deref. if gc_count == 31: + live_mark_bp.SetEnabled(True) mark_bp.SetEnabled(True) + elif live_mark_bp.GetID() in ids: + assert returned and gc_count == 31 + allocation = owner['owner'] - ((concrete_type('JSC::PreciseAllocation').GetByteSize() + 15) & ~15) + if reg(frame, 'x1') == allocation: + visitor_address = reg(frame, 'x0') + visitor = typed(visitor_address, 'JSC::VerifierSlotVisitor') + _, actual_allocation, existing = precise_marker(visitor, owner['owner']) + assert actual_allocation == allocation and existing is None, 'first precise mark entry was missed' + assert live_marker is None and number(field(visitor, 'm_doneMarking')) == 0 + live_marker = dict(visitor=visitor_address, allocation=allocation, + frame_pointer_path=frame_pointer_snapshot(frame, thread)) + event('live-first-precise-mark', **live_marker) + live_mark_bp.SetEnabled(False) elif mark_bp.GetID() in ids: assert returned and gc_count == 31 and 'Collector::verifyGC' in (frame.GetFunctionName() or '') collector = frame.FindVariable('this', lldb.eNoDynamicValues) @@ -437,6 +496,7 @@ def marker_snapshot(visitor, cell): assert number(collector) == number(field(field(vm, 'heap'), 'm_collector')) visitor_ptrs = pointer(field(collector.Dereference(), 'm_verifierSlotVisitor'), 'JSC::VerifierSlotVisitor') assert len(set(visitor_ptrs)) == 1 and visitor_ptrs[0] + assert live_marker is not None and live_marker['visitor'] == visitor_ptrs[0] visitor = typed(visitor_ptrs[0], 'JSC::VerifierSlotVisitor') assert number(field(visitor, 'm_doneMarking')) == 1 w = typed(owner['weak'], 'JSC::JSWeakObjectRef') @@ -463,8 +523,9 @@ def marker_snapshot(visitor, cell): assert facts['callbackRoot'] is True and facts['collected'] is False and facts['settled'] == ['ECANCELED'] snapshots = [e for e in report['events'] if e['kind'] == 'passive-marker-snapshot'] assert len(snapshots) == 1 and snapshots[0]['owner'] == owner['owner'] + assert len([e for e in report['events'] if e['kind'] == 'live-first-precise-mark']) == 1 report['outcome'] = facts - report['classification'] = 'stored ConservativeScan marker stack captured; originating word remains unproven' + report['classification'] = 'first-mark frame-pointer path captured; originating conservative word remains unproven' report['passed'] = True except BaseException as error: report['gap'] = str(error) From 2021c19a8a43b9d69223e43b1cc75447595302c3 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 01:30:54 +0800 Subject: [PATCH 28/74] fix(ci): observe the TLS owner real-mark writer --- .github/openclaw/tls-retainer-capture.py | 82 ++++++++++++++++-------- 1 file changed, 55 insertions(+), 27 deletions(-) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py index 212cc5aa5866..e8ae853f3bf0 100644 --- a/.github/openclaw/tls-retainer-capture.py +++ b/.github/openclaw/tls-retainer-capture.py @@ -150,7 +150,8 @@ def debugger_stage(): owner = None returned = False gc_count = 0 - live_marker = None + mark_watch = None + mark_observation = None def event(kind, **values): report['events'].append({'kind': kind, **values}) @@ -235,12 +236,16 @@ def pointer(value, pointee, depth=0): result += pointer(child, pointee, depth + 1) return result - def precise_marker(visitor, cell): - # Use frozen HashTable metadata and DWARF field offsets, without inferior find/get calls. + def precise_allocation(cell): assert target.GetAddressByteSize() == 8 and target.GetByteOrder() == lldb.eByteOrderLittle assert cell & 8, 'owner is not the observed precise-allocation kind' header = concrete_type('JSC::PreciseAllocation').GetByteSize() allocation = cell - ((header + 15) & ~15) + return header, allocation + + def precise_marker(visitor, cell): + # Use frozen HashTable metadata and DWARF field offsets, without inferior find/get calls. + header, allocation = precise_allocation(cell) table_value = field(field(field(visitor, 'm_preciseAllocationMap'), 'm_impl'), 'm_table') table = number(table_value) if not table: @@ -287,7 +292,8 @@ def frame_pointer_snapshot(frame, thread): region = lldb.SBMemoryRegionInfo() assert process.GetMemoryRegionInfo(sp, region).Success() and region.IsReadable() ceiling = min(region.GetRegionEnd(), sp + 16 * 1024**2) - frames = [symbolize(pc), symbolize(lr, True)] + # A watchpoint may stop after the store; x30 can be a nested call's return, not this frame's caller. + frames = [symbolize(pc)] links = [] current = fp for _ in range(128): @@ -374,9 +380,6 @@ def marker_snapshot(visitor, cell): weak_return_bp = None completion_bp, completion_address = entry('Bun__UpgradedDuplex__onWriteComplete') gc_bp, _ = entry('JSC__VM__runGC') - live_mark_bp, _ = entry('JSC::VerifierSlotVisitor::testAndSetMarked', - 'JSC::VerifierSlotVisitor::testAndSetMarked(JSC::PreciseAllocation&)') - live_mark_bp.SetEnabled(False) mark_bp = target.BreakpointCreateByLocation('Collector.cpp', 1068) assert mark_bp.GetNumLocations() > 0, 'missing post-mark verifier boundary' mark_bp.SetEnabled(False) @@ -391,14 +394,34 @@ def marker_snapshot(visitor, cell): error = lldb.SBError() process = target.Launch(info, error) assert error.Success(), str(error) - for _ in range(16384 if stage == 'trace' else 256): + for _ in range(256): if process.GetState() == lldb.eStateExited: break assert process.GetState() == lldb.eStateStopped, str(process.GetState()) - threads = [t for t in process if t.GetStopReason() == lldb.eStopReasonBreakpoint] - assert len(threads) == 1, 'unexpected stop or concurrent breakpoint' + threads = [t for t in process if t.GetStopReason() in + (lldb.eStopReasonBreakpoint, lldb.eStopReasonWatchpoint)] + assert len(threads) == 1, 'unexpected stop or concurrent debugger event' thread = threads[0] frame = thread.GetFrameAtIndex(0) + if thread.GetStopReason() == lldb.eStopReasonWatchpoint: + assert stage == 'trace' and mark_watch is not None and mark_watch.IsEnabled() + assert thread.GetStopReasonDataCount() == 1 + assert thread.GetStopReasonDataAtIndex(0) == mark_watch.GetID() + value = word(mark_observation['address'], 1) + event('real-mark-write', watchpoint=mark_watch.GetID(), value=value, + stopped_pc=frame.GetPC(), thread=thread.GetThreadID()) + assert value in (0, 1) and value != mark_observation['last_value'] + mark_observation['last_value'] = value + if value == 0: + mark_observation['observed_zero'] = True + else: + assert mark_observation['observed_zero'] and 'frame_pointer_path' not in mark_observation + mark_watch.SetEnabled(False) + assert not mark_watch.IsEnabled() + mark_observation['frame_pointer_path'] = frame_pointer_snapshot(frame, thread) + event('live-real-mark-writer', **mark_observation) + assert process.Continue().Success() + continue ids = {thread.GetStopReasonDataAtIndex(i) for i in range(0, thread.GetStopReasonDataCount(), 2)} report['last_stop'] = {'pc': frame.GetPC(), 'function': frame.GetFunctionName(), 'breakpoints': sorted(ids), 'expected_entry_pcs': { @@ -474,21 +497,23 @@ def marker_snapshot(visitor, cell): assert gc_count <= 31 # The fixture collects once for queuedRoot, then thirty times before its final deref. if gc_count == 31: - live_mark_bp.SetEnabled(True) + _, allocation = precise_allocation(owner['owner']) + marked = field(field(typed(allocation, 'JSC::PreciseAllocation'), 'm_isMarked'), 'value') + assert marked.GetByteSize() == 1 and mark_watch is None + address = marked.GetLoadAddress() + initial = word(address, 1) + assert initial in (0, 1) + error = lldb.SBError() + mark_watch = target.WatchAddress(address, 1, False, True, error) + event('real-mark-watchpoint-admission', address=address, initial=initial, error=str(error), + valid=mark_watch.IsValid()) + assert error.Success() and mark_watch.IsValid() and mark_watch.IsEnabled() + assert mark_watch.GetWatchAddress() == address and mark_watch.GetWatchSize() == 1 + assert mark_watch.GetID() > 0 + mark_observation = dict(allocation=allocation, address=address, initial_value=initial, + last_value=initial, observed_zero=initial == 0, + watchpoint=mark_watch.GetID()) mark_bp.SetEnabled(True) - elif live_mark_bp.GetID() in ids: - assert returned and gc_count == 31 - allocation = owner['owner'] - ((concrete_type('JSC::PreciseAllocation').GetByteSize() + 15) & ~15) - if reg(frame, 'x1') == allocation: - visitor_address = reg(frame, 'x0') - visitor = typed(visitor_address, 'JSC::VerifierSlotVisitor') - _, actual_allocation, existing = precise_marker(visitor, owner['owner']) - assert actual_allocation == allocation and existing is None, 'first precise mark entry was missed' - assert live_marker is None and number(field(visitor, 'm_doneMarking')) == 0 - live_marker = dict(visitor=visitor_address, allocation=allocation, - frame_pointer_path=frame_pointer_snapshot(frame, thread)) - event('live-first-precise-mark', **live_marker) - live_mark_bp.SetEnabled(False) elif mark_bp.GetID() in ids: assert returned and gc_count == 31 and 'Collector::verifyGC' in (frame.GetFunctionName() or '') collector = frame.FindVariable('this', lldb.eNoDynamicValues) @@ -496,7 +521,10 @@ def marker_snapshot(visitor, cell): assert number(collector) == number(field(field(vm, 'heap'), 'm_collector')) visitor_ptrs = pointer(field(collector.Dereference(), 'm_verifierSlotVisitor'), 'JSC::VerifierSlotVisitor') assert len(set(visitor_ptrs)) == 1 and visitor_ptrs[0] - assert live_marker is not None and live_marker['visitor'] == visitor_ptrs[0] + assert mark_observation is not None and 'frame_pointer_path' in mark_observation + assert mark_observation['last_value'] == 1 and not mark_watch.IsEnabled() + _, allocation = precise_allocation(owner['owner']) + assert allocation == mark_observation['allocation'] and word(mark_observation['address'], 1) == 1 visitor = typed(visitor_ptrs[0], 'JSC::VerifierSlotVisitor') assert number(field(visitor, 'm_doneMarking')) == 1 w = typed(owner['weak'], 'JSC::JSWeakObjectRef') @@ -523,9 +551,9 @@ def marker_snapshot(visitor, cell): assert facts['callbackRoot'] is True and facts['collected'] is False and facts['settled'] == ['ECANCELED'] snapshots = [e for e in report['events'] if e['kind'] == 'passive-marker-snapshot'] assert len(snapshots) == 1 and snapshots[0]['owner'] == owner['owner'] - assert len([e for e in report['events'] if e['kind'] == 'live-first-precise-mark']) == 1 + assert len([e for e in report['events'] if e['kind'] == 'live-real-mark-writer']) == 1 report['outcome'] = facts - report['classification'] = 'first-mark frame-pointer path captured; originating conservative word remains unproven' + report['classification'] = 'real-mark writer path captured; verifier first-root path and originating word remain unproven' report['passed'] = True except BaseException as error: report['gap'] = str(error) From 5ffae657708625bb2a0eec215fc2ac6de6a7e188 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 01:54:24 +0800 Subject: [PATCH 29/74] test(tls): trace accepted conservative root spans Classify stack and VM root-list prefixes and correlate ConservativeRoots add entry/return spans with the exact retained owner in the final fixture collection. Keep exact/interior/one-past words labelled as candidates until the scanner iteration is observed. Preserve the existing shipping-byte admission, passive hardware mark observation and completed-verifier gates. Static independent review and Python syntax checks passed; native qualification remains pending. --- .github/openclaw/tls-retainer-capture.py | 127 ++++++++++++++++++++++- 1 file changed, 125 insertions(+), 2 deletions(-) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py index e8ae853f3bf0..fe0a835f5acb 100644 --- a/.github/openclaw/tls-retainer-capture.py +++ b/.github/openclaw/tls-retainer-capture.py @@ -152,6 +152,10 @@ def debugger_stage(): gc_count = 0 mark_watch = None mark_observation = None + root_collections = [] + root_collection = None + scan_returns = {} + vm_roots_return = None def event(kind, **values): report['events'].append({'kind': kind, **values}) @@ -317,6 +321,49 @@ def frame_pointer_snapshot(frame, thread): return {'thread': thread.GetThreadID(), 'pc': pc, 'lr': lr, 'sp': sp, 'fp': fp, 'links': links, 'frames': frames, 'termination': termination} + def roots_snapshot(address): + roots = typed(address, 'JSC::ConservativeRoots') + count = number(field(roots, 'm_size')) + capacity = number(field(roots, 'm_capacity')) + data = number(field(roots, 'm_roots')) + assert 0 <= count <= capacity <= 1024**2 and data + error = lldb.SBError() + raw = process.ReadMemory(data, count * 8, error) if count else b'' + assert not count or (error.Success() and len(raw) == count * 8) + indices = [i for i in range(count) if int.from_bytes(raw[i * 8:i * 8 + 8], 'little') == owner['owner']] + assert len(indices) <= 64, 'accepted-root duplicate bound exceeded' + return dict(roots=address, data=data, count=count, capacity=capacity, + owner_indices=indices, sha256=hashlib.sha256(raw).hexdigest()) + + def accepted_span(scan, after): + assert after['roots'] == scan['roots'] and after['count'] >= scan['before_count'] + accepted = [i for i in after['owner_indices'] if i >= scan['before_count']] + if not accepted: + return + begin, end = sorted((scan['begin'], scan['end'])) + assert begin % 8 == end % 8 == 0 and 0 <= end - begin <= 16 * 1024**2 + error = lldb.SBError() + raw = process.ReadMemory(begin, end - begin, error) if begin != end else b'' + assert begin == end or (error.Success() and len(raw) == end - begin) + _, allocation = precise_allocation(owner['owner']) + cell_size = number(field(typed(allocation, 'JSC::PreciseAllocation'), 'm_cellSize')) + assert 0 < cell_size <= 16 * 1024**2 + candidates = [] + for offset in range(0, len(raw), 8): + value = int.from_bytes(raw[offset:offset + 8], 'little') + if owner['owner'] <= value <= owner['owner'] + cell_size: + candidates.append(dict(address=begin + offset, value=value, + offset_into_cell=value - owner['owner'])) + assert len(candidates) <= 64, 'candidate-word bound exceeded' + receipt = dict(collection=scan['collection'], phase=scan['phase'], roots=scan['roots'], + thread=scan['thread'], begin=begin, end=end, before_count=scan['before_count'], + after_count=after['count'], accepted_owner_indices=accepted, + cell_size=cell_size, candidate_words=candidates, + candidate_interpretation='unmodified exact/interior/one-past pointer candidates; accepted scanner iteration not yet identified', + span_sha256=hashlib.sha256(raw).hexdigest()) + root_collections[scan['collection']]['accepted_spans'].append(receipt) + event('accepted-owner-span', **receipt) + def marker_snapshot(visitor, cell): header, allocation, marker = precise_marker(visitor, cell) assert marker is not None, 'actual precise allocation missing in completed verifier map' @@ -384,6 +431,17 @@ def marker_snapshot(visitor, cell): assert mark_bp.GetNumLocations() > 0, 'missing post-mark verifier boundary' mark_bp.SetEnabled(False) return_bp = None + stack_roots_bp, _ = entry('JSC::Heap::gatherStackRoots') + vm_roots_bp, _ = entry('JSC::Heap::gatherVMRoots') + scan_bps = [] + for signature in [ + 'JSC::ConservativeRoots::add(void*, void*)', + 'JSC::ConservativeRoots::add(void*, void*, JSC::JITStubRoutineSet&, JSC::CodeBlockSet&)']: + bp, _ = entry('JSC::ConservativeRoots::add', signature) + scan_bps.append(bp) + root_bps = [stack_roots_bp, vm_roots_bp, *scan_bps] + for bp in root_bps: + bp.SetEnabled(False) info = lldb.SBLaunchInfo(args) info.SetWorkingDirectory(str(SOURCE)) info.SetEnvironmentEntries([key + '=' + value for key, value in environment.items()], False) @@ -419,6 +477,16 @@ def marker_snapshot(visitor, cell): mark_watch.SetEnabled(False) assert not mark_watch.IsEnabled() mark_observation['frame_pointer_path'] = frame_pointer_snapshot(frame, thread) + assert root_collection is not None and root_collection['phase'] == 'complete' + current_roots = roots_snapshot(root_collection['roots']) + assert current_roots == root_collection['after_vm'], 'accepted roots changed before the owner mark' + assert current_roots['owner_indices'] and root_collection['accepted_spans'] + mark_observation['root_collection'] = root_collection['index'] + event('accepted-root-list-at-owner-mark', **current_roots, + collection=root_collection['index']) + for bp in root_bps: + bp.SetEnabled(False) + assert not scan_returns and vm_roots_return is None event('live-real-mark-writer', **mark_observation) assert process.Continue().Success() continue @@ -446,6 +514,56 @@ def marker_snapshot(visitor, cell): assert not (OUT / 'child.json').exists(), 'multiple selected children' write('child.json', {'argv': values, 'script_sha256': hashlib.sha256(values[2].encode()).hexdigest()}) event('captured-actual-fixture-child') + elif stack_roots_bp.GetID() in ids: + assert returned and gc_count == 31 and owner is not None + vm_heap = field(typed(owner['vm'], 'JSC::VM'), 'heap').GetLoadAddress() + assert reg(frame, 'x0') == vm_heap + assert root_collection is None or root_collection['phase'] == 'complete' + assert len(root_collections) < 8 and not scan_returns + roots = reg(frame, 'x1') + initial_roots = roots_snapshot(roots) + assert initial_roots['count'] == 0 + root_collection = dict(index=len(root_collections), roots=roots, phase='stack', + accepted_spans=[], initial=initial_roots) + root_collections.append(root_collection) + event('stack-root-gather-entry', collection=root_collection['index'], **initial_roots) + elif vm_roots_bp.GetID() in ids: + assert root_collection is not None and root_collection['phase'] == 'stack' + assert not scan_returns and reg(frame, 'x1') == root_collection['roots'] + vm_heap = field(typed(owner['vm'], 'JSC::VM'), 'heap').GetLoadAddress() + assert reg(frame, 'x0') == vm_heap and vm_roots_return is None + root_collection['after_stack'] = roots_snapshot(root_collection['roots']) + event('stack-root-gather-complete', collection=root_collection['index'], + **root_collection['after_stack']) + root_collection['phase'] = 'vm' + vm_roots_return = target.BreakpointCreateByAddress(reg(frame, 'x30')) + assert vm_roots_return.GetNumLocations() == 1 + vm_roots_return.SetThreadID(thread.GetThreadID()) + vm_roots_return.SetOneShot(True) + elif vm_roots_return is not None and vm_roots_return.GetID() in ids: + assert root_collection is not None and root_collection['phase'] == 'vm' and not scan_returns + after = roots_snapshot(root_collection['roots']) + assert after['count'] >= root_collection['after_stack']['count'] + root_collection.update(phase='complete', after_vm=after) + event('vm-root-gather-complete', collection=root_collection['index'], **after) + vm_roots_return = None + elif any(bp.GetID() in ids for bp in scan_bps): + assert root_collection is not None and root_collection['phase'] in ('stack', 'vm') + roots = reg(frame, 'x0') + assert roots == root_collection['roots'] and not scan_returns + before = roots_snapshot(roots) + bp = target.BreakpointCreateByAddress(reg(frame, 'x30')) + assert bp.GetNumLocations() == 1 + bp.SetThreadID(thread.GetThreadID()) + bp.SetOneShot(True) + scan_returns[bp.GetID()] = dict(collection=root_collection['index'], + phase=root_collection['phase'], roots=roots, before_count=before['count'], + begin=reg(frame, 'x1'), end=reg(frame, 'x2'), thread=thread.GetThreadID()) + elif ids.intersection(scan_returns): + assert len(ids.intersection(scan_returns)) == 1 + scan = scan_returns.pop(next(iter(ids.intersection(scan_returns)))) + assert scan['thread'] == thread.GetThreadID() + accepted_span(scan, roots_snapshot(scan['roots'])) elif weak_bp.GetID() in ids: assert weak_creation is None global_object, call = reg(frame, 'x0'), reg(frame, 'x1') @@ -514,6 +632,8 @@ def marker_snapshot(visitor, cell): last_value=initial, observed_zero=initial == 0, watchpoint=mark_watch.GetID()) mark_bp.SetEnabled(True) + for bp in root_bps: + bp.SetEnabled(True) elif mark_bp.GetID() in ids: assert returned and gc_count == 31 and 'Collector::verifyGC' in (frame.GetFunctionName() or '') collector = frame.FindVariable('this', lldb.eNoDynamicValues) @@ -553,7 +673,9 @@ def marker_snapshot(visitor, cell): assert len(snapshots) == 1 and snapshots[0]['owner'] == owner['owner'] assert len([e for e in report['events'] if e['kind'] == 'live-real-mark-writer']) == 1 report['outcome'] = facts - report['classification'] = 'real-mark writer path captured; verifier first-root path and originating word remain unproven' + assert root_collections and any(c['accepted_spans'] for c in root_collections) + report['root_collections'] = root_collections + report['classification'] = 'accepted stack/VM scan spans and real-mark writer captured; exact originating scanner word and source frame remain unproven' report['passed'] = True except BaseException as error: report['gap'] = str(error) @@ -561,7 +683,8 @@ def marker_snapshot(visitor, cell): finally: if stage == 'trace': report['observation'] = {'weak_targets': weak, 'owner': owner, 'returned': returned, - 'gc_count': gc_count, 'pending_weak_constructor': weak_creation} + 'gc_count': gc_count, 'pending_weak_constructor': weak_creation, + 'root_collections': root_collections, 'pending_scan_returns': scan_returns} if process is not None and process.IsValid() and process.GetState() != lldb.eStateExited: process.Kill() write(stage + '.json', report) From 0acdc3e03cdea7dc22b8b5fdeeee973283718c12 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 02:02:12 +0800 Subject: [PATCH 30/74] test(tls): observe the surviving machine-stack root boundary ThinLTO has only inline Heap gather contexts. Admit the surviving MachineThreads physical entry instead, match it to the exact VM, snapshot its accepted-root prefix on return, and distinguish later additions without claiming they all come from VM buffers. Resolve all observer entries before failing capability admission. Independent static review, Python syntax and diff checks passed; native replay remains required. --- .github/openclaw/tls-retainer-capture.py | 82 +++++++++++++----------- 1 file changed, 43 insertions(+), 39 deletions(-) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py index fe0a835f5acb..5b4cce311fec 100644 --- a/.github/openclaw/tls-retainer-capture.py +++ b/.github/openclaw/tls-retainer-capture.py @@ -155,23 +155,24 @@ def debugger_stage(): root_collections = [] root_collection = None scan_returns = {} - vm_roots_return = None + stack_roots_return = None def event(kind, **values): report['events'].append({'kind': kind, **values}) write(stage + '.json', report) - def entry(name, signature=None): + def entry(name, signature=None, required=True): contexts = module.FindFunctions(name, lldb.eFunctionNameTypeAny) assert contexts.GetSize() <= 64, name addresses = {} - report['resolutions'][name] = [] + resolution_key = signature or name + report['resolutions'][resolution_key] = [] for i in range(contexts.GetSize()): context = contexts.GetContextAtIndex(i) function = context.GetFunction() resolved = function.GetName() if function.IsValid() else context.GetSymbol().GetName() address = function.GetStartAddress() if function.IsValid() else context.GetSymbol().GetStartAddress() - report['resolutions'][name].append({'function': resolved, 'file_address': address.GetFileAddress()}) + report['resolutions'][resolution_key].append({'function': resolved, 'file_address': address.GetFileAddress()}) # FindFunctions includes inline scopes whose GetFunction is the enclosing physical function. if not resolved or resolved.split('(', 1)[0].strip().removeprefix('::') != name: continue @@ -179,6 +180,9 @@ def entry(name, signature=None): continue if address.IsValid(): addresses[address.GetFileAddress()] = address + if not required and len(addresses) != 1: + event('observer-entry-unavailable', name=name, signature=signature, physical_entries=len(addresses)) + return None, None assert len(addresses) == 1, 'missing/ambiguous function entry: ' + name address = next(iter(addresses.values())) bp = target.BreakpointCreateBySBAddress(address) @@ -431,15 +435,15 @@ def marker_snapshot(visitor, cell): assert mark_bp.GetNumLocations() > 0, 'missing post-mark verifier boundary' mark_bp.SetEnabled(False) return_bp = None - stack_roots_bp, _ = entry('JSC::Heap::gatherStackRoots') - vm_roots_bp, _ = entry('JSC::Heap::gatherVMRoots') + stack_roots_bp, _ = entry('JSC::MachineThreads::gatherConservativeRoots', required=False) scan_bps = [] for signature in [ 'JSC::ConservativeRoots::add(void*, void*)', 'JSC::ConservativeRoots::add(void*, void*, JSC::JITStubRoutineSet&, JSC::CodeBlockSet&)']: - bp, _ = entry('JSC::ConservativeRoots::add', signature) + bp, _ = entry('JSC::ConservativeRoots::add', signature, required=False) scan_bps.append(bp) - root_bps = [stack_roots_bp, vm_roots_bp, *scan_bps] + root_bps = [stack_roots_bp, *scan_bps] + assert all(bp is not None for bp in root_bps), 'required root-span observer entries unavailable' for bp in root_bps: bp.SetEnabled(False) info = lldb.SBLaunchInfo(args) @@ -477,16 +481,24 @@ def marker_snapshot(visitor, cell): mark_watch.SetEnabled(False) assert not mark_watch.IsEnabled() mark_observation['frame_pointer_path'] = frame_pointer_snapshot(frame, thread) - assert root_collection is not None and root_collection['phase'] == 'complete' + assert root_collection is not None and root_collection['phase'] == 'after-machine-stack' current_roots = roots_snapshot(root_collection['roots']) - assert current_roots == root_collection['after_vm'], 'accepted roots changed before the owner mark' + after_stack = root_collection['after_stack'] + assert current_roots['count'] >= after_stack['count'] assert current_roots['owner_indices'] and root_collection['accepted_spans'] + stack_indices = [i for i in current_roots['owner_indices'] if i < after_stack['count']] + assert stack_indices == after_stack['owner_indices'], 'stack-root prefix changed' + subsequent_indices = [i for i in current_roots['owner_indices'] if i >= after_stack['count']] + root_collection.update(phase='complete', at_mark=current_roots, + machine_stack_indices=stack_indices, + subsequent_indices=subsequent_indices) mark_observation['root_collection'] = root_collection['index'] event('accepted-root-list-at-owner-mark', **current_roots, - collection=root_collection['index']) + collection=root_collection['index'], machine_stack_indices=stack_indices, + subsequent_indices=subsequent_indices) for bp in root_bps: bp.SetEnabled(False) - assert not scan_returns and vm_roots_return is None + assert not scan_returns and stack_roots_return is None event('live-real-mark-writer', **mark_observation) assert process.Continue().Success() continue @@ -516,39 +528,31 @@ def marker_snapshot(visitor, cell): event('captured-actual-fixture-child') elif stack_roots_bp.GetID() in ids: assert returned and gc_count == 31 and owner is not None - vm_heap = field(typed(owner['vm'], 'JSC::VM'), 'heap').GetLoadAddress() - assert reg(frame, 'x0') == vm_heap - assert root_collection is None or root_collection['phase'] == 'complete' - assert len(root_collections) < 8 and not scan_returns + heap = field(typed(owner['vm'], 'JSC::VM'), 'heap') + machines = pointer(field(heap, 'm_machineThreads'), 'JSC::MachineThreads') + assert machines == [reg(frame, 'x0')], 'machine-stack scanner belongs to another VM' + assert root_collection is None or root_collection['phase'] == 'after-machine-stack' + assert len(root_collections) < 8 and not scan_returns and stack_roots_return is None roots = reg(frame, 'x1') initial_roots = roots_snapshot(roots) assert initial_roots['count'] == 0 - root_collection = dict(index=len(root_collections), roots=roots, phase='stack', + root_collection = dict(index=len(root_collections), roots=roots, phase='machine-stack', accepted_spans=[], initial=initial_roots) root_collections.append(root_collection) - event('stack-root-gather-entry', collection=root_collection['index'], **initial_roots) - elif vm_roots_bp.GetID() in ids: - assert root_collection is not None and root_collection['phase'] == 'stack' - assert not scan_returns and reg(frame, 'x1') == root_collection['roots'] - vm_heap = field(typed(owner['vm'], 'JSC::VM'), 'heap').GetLoadAddress() - assert reg(frame, 'x0') == vm_heap and vm_roots_return is None - root_collection['after_stack'] = roots_snapshot(root_collection['roots']) - event('stack-root-gather-complete', collection=root_collection['index'], - **root_collection['after_stack']) - root_collection['phase'] = 'vm' - vm_roots_return = target.BreakpointCreateByAddress(reg(frame, 'x30')) - assert vm_roots_return.GetNumLocations() == 1 - vm_roots_return.SetThreadID(thread.GetThreadID()) - vm_roots_return.SetOneShot(True) - elif vm_roots_return is not None and vm_roots_return.GetID() in ids: - assert root_collection is not None and root_collection['phase'] == 'vm' and not scan_returns + event('machine-stack-gather-entry', collection=root_collection['index'], **initial_roots) + stack_roots_return = target.BreakpointCreateByAddress(reg(frame, 'x30')) + assert stack_roots_return.GetNumLocations() == 1 + stack_roots_return.SetThreadID(thread.GetThreadID()) + stack_roots_return.SetOneShot(True) + elif stack_roots_return is not None and stack_roots_return.GetID() in ids: + assert root_collection is not None and root_collection['phase'] == 'machine-stack' + assert not scan_returns after = roots_snapshot(root_collection['roots']) - assert after['count'] >= root_collection['after_stack']['count'] - root_collection.update(phase='complete', after_vm=after) - event('vm-root-gather-complete', collection=root_collection['index'], **after) - vm_roots_return = None + root_collection.update(phase='after-machine-stack', after_stack=after) + event('machine-stack-gather-complete', collection=root_collection['index'], **after) + stack_roots_return = None elif any(bp.GetID() in ids for bp in scan_bps): - assert root_collection is not None and root_collection['phase'] in ('stack', 'vm') + assert root_collection is not None and root_collection['phase'] in ('machine-stack', 'after-machine-stack') roots = reg(frame, 'x0') assert roots == root_collection['roots'] and not scan_returns before = roots_snapshot(roots) @@ -675,7 +679,7 @@ def marker_snapshot(visitor, cell): report['outcome'] = facts assert root_collections and any(c['accepted_spans'] for c in root_collections) report['root_collections'] = root_collections - report['classification'] = 'accepted stack/VM scan spans and real-mark writer captured; exact originating scanner word and source frame remain unproven' + report['classification'] = 'accepted machine-stack/subsequent scan spans and real-mark writer captured; exact originating scanner word and source frame remain unproven' report['passed'] = True except BaseException as error: report['gap'] = str(error) From 1240e1a216d56e2a5b853fc754edde202dcaeafe Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 02:13:07 +0800 Subject: [PATCH 31/74] test(tls): inventory inline conservative-root scanner sites --- .github/openclaw/tls-retainer-capture.py | 69 ++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py index 5b4cce311fec..73ef44e22556 100644 --- a/.github/openclaw/tls-retainer-capture.py +++ b/.github/openclaw/tls-retainer-capture.py @@ -192,6 +192,73 @@ def entry(name, signature=None, required=True): 'file_address': address.GetFileAddress()}) return bp, address + def inventory_inline_roots(): + # Inspect debug scopes and immutable instructions before launching an inferior. + # Inline add() contexts are not physical ABI entries; their registers cannot + # be interpreted as x0/x1/x2 arguments of add(). + parents = {} + for name in ['JSC::MachineThreads::gatherConservativeRoots', + 'JSC::ConservativeRoots::add', + 'JSC::ConservativeRoots::genericAddSpan', + 'JSC::ConservativeRoots::genericAddPointer']: + contexts = module.FindFunctions(name, lldb.eFunctionNameTypeAny) + assert contexts.GetSize() <= 64, 'root inventory context budget' + for i in range(contexts.GetSize()): + function = contexts.GetContextAtIndex(i).GetFunction() + if function.IsValid(): + parents[function.GetStartAddress().GetFileAddress()] = function + assert 0 < len(parents) <= 16, 'root inventory physical-parent budget' + total_instructions = 0 + for start, function in sorted(parents.items()): + scopes = [] + pending = [function.GetBlock()] + visited = 0 + while pending: + block = pending.pop() + if not block.IsValid(): + continue + visited += 1 + assert visited <= 4096, 'root inventory block budget' + sibling, child = block.GetSibling(), block.GetFirstChild() + if sibling.IsValid(): + pending.append(sibling) + if child.IsValid(): + pending.append(child) + name = block.GetInlinedName() or '' + if 'ConservativeRoots::' not in name: + continue + assert block.GetNumRanges() <= 128, 'root inventory range budget' + ranges = [(block.GetRangeStartAddress(i).GetFileAddress(), + block.GetRangeEndAddress(i).GetFileAddress()) + for i in range(block.GetNumRanges())] + variables = block.GetVariables(target, True, True, False) + assert variables.GetSize() <= 256, 'root inventory variable budget' + declarations = [] + for i in range(variables.GetSize()): + value = variables.GetValueAtIndex(i) + declaration = value.GetDeclaration() + declarations.append({'name': value.GetName(), 'type': value.GetTypeName(), + 'file': declaration.GetFileSpec().GetFilename(), 'line': declaration.GetLine()}) + scopes.append({'name': name, 'ranges': ranges, 'variables': declarations}) + instructions = function.GetInstructions(target) + assert instructions.GetSize() <= 32768, 'root inventory disassembly budget' + rows = [] + for i in range(instructions.GetSize()): + instruction = instructions.GetInstructionAtIndex(i) + address = instruction.GetAddress() + pc = address.GetFileAddress() + line = address.GetLineEntry() + file = line.GetFileSpec().GetFilename() + in_scope = any(a <= pc < b for scope in scopes for a, b in scope['ranges']) + if not in_scope and file != 'ConservativeRoots.cpp': + continue + total_instructions += 1 + assert total_instructions <= 8192, 'root inventory output budget' + rows.append({'file_address': pc, 'file': file, 'line': line.GetLine(), + 'mnemonic': instruction.GetMnemonic(target), 'operands': instruction.GetOperands(target)}) + event('inline-root-inventory', function=function.GetName(), file_address=start, + scopes=scopes, instructions=rows, blocks_visited=visited) + def reg(frame, name): value = frame.FindRegister(name) assert value.IsValid() and value.GetError().Success(), name @@ -443,6 +510,8 @@ def marker_snapshot(visitor, cell): bp, _ = entry('JSC::ConservativeRoots::add', signature, required=False) scan_bps.append(bp) root_bps = [stack_roots_bp, *scan_bps] + if any(bp is None for bp in root_bps): + inventory_inline_roots() assert all(bp is not None for bp in root_bps), 'required root-span observer entries unavailable' for bp in root_bps: bp.SetEnabled(False) From a814c0549ca754b84ad5746dfbc04f9d06a378b8 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 02:27:20 +0800 Subject: [PATCH 32/74] test(tls): observe physical conservative-root span scanners --- .github/openclaw/tls-retainer-capture.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py index 73ef44e22556..804047d005ba 100644 --- a/.github/openclaw/tls-retainer-capture.py +++ b/.github/openclaw/tls-retainer-capture.py @@ -174,9 +174,10 @@ def entry(name, signature=None, required=True): address = function.GetStartAddress() if function.IsValid() else context.GetSymbol().GetStartAddress() report['resolutions'][resolution_key].append({'function': resolved, 'file_address': address.GetFileAddress()}) # FindFunctions includes inline scopes whose GetFunction is the enclosing physical function. - if not resolved or resolved.split('(', 1)[0].strip().removeprefix('::') != name: - continue - if signature is not None and resolved.removeprefix('::') != signature: + if signature is not None: + if not resolved or resolved.removeprefix('::') != signature: + continue + elif not resolved or resolved.split('(', 1)[0].strip().removeprefix('::') != name: continue if address.IsValid(): addresses[address.GetFileAddress()] = address @@ -505,9 +506,9 @@ def marker_snapshot(visitor, cell): stack_roots_bp, _ = entry('JSC::MachineThreads::gatherConservativeRoots', required=False) scan_bps = [] for signature in [ - 'JSC::ConservativeRoots::add(void*, void*)', - 'JSC::ConservativeRoots::add(void*, void*, JSC::JITStubRoutineSet&, JSC::CodeBlockSet&)']: - bp, _ = entry('JSC::ConservativeRoots::add', signature, required=False) + 'void JSC::ConservativeRoots::genericAddSpan(void*, void*, JSC::DummyMarkHook&)', + 'void JSC::ConservativeRoots::genericAddSpan(void*, void*, JSC::CompositeMarkHook&)']: + bp, _ = entry('JSC::ConservativeRoots::genericAddSpan', signature, required=False) scan_bps.append(bp) root_bps = [stack_roots_bp, *scan_bps] if any(bp is None for bp in root_bps): From 07538ea23d1dd351d85f8544fc6f219c070d0e84 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 02:37:12 +0800 Subject: [PATCH 33/74] test(tls): trace the exact accepted conservative scanner word --- .github/openclaw/tls-retainer-capture.py | 145 ++++++++++++++++++++--- 1 file changed, 129 insertions(+), 16 deletions(-) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py index 804047d005ba..98cca3ded7a5 100644 --- a/.github/openclaw/tls-retainer-capture.py +++ b/.github/openclaw/tls-retainer-capture.py @@ -156,6 +156,11 @@ def debugger_stage(): root_collection = None scan_returns = {} stack_roots_return = None + word_watch = None + word_pending = None + word_witness = None + pointer_call_bp = None + pointer_return_bp = None def event(kind, **values): report['events'].append({'kind': kind, **values}) @@ -393,7 +398,7 @@ def frame_pointer_snapshot(frame, thread): return {'thread': thread.GetThreadID(), 'pc': pc, 'lr': lr, 'sp': sp, 'fp': fp, 'links': links, 'frames': frames, 'termination': termination} - def roots_snapshot(address): + def roots_snapshot(address, prefix_count=None): roots = typed(address, 'JSC::ConservativeRoots') count = number(field(roots, 'm_size')) capacity = number(field(roots, 'm_capacity')) @@ -404,15 +409,15 @@ def roots_snapshot(address): assert not count or (error.Success() and len(raw) == count * 8) indices = [i for i in range(count) if int.from_bytes(raw[i * 8:i * 8 + 8], 'little') == owner['owner']] assert len(indices) <= 64, 'accepted-root duplicate bound exceeded' - return dict(roots=address, data=data, count=count, capacity=capacity, - owner_indices=indices, sha256=hashlib.sha256(raw).hexdigest()) + result = dict(roots=address, data=data, count=count, capacity=capacity, + owner_indices=indices, sha256=hashlib.sha256(raw).hexdigest()) + if prefix_count is not None: + assert 0 <= prefix_count <= count + result['prefix_sha256'] = hashlib.sha256(raw[:prefix_count * 8]).hexdigest() + return result - def accepted_span(scan, after): - assert after['roots'] == scan['roots'] and after['count'] >= scan['before_count'] - accepted = [i for i in after['owner_indices'] if i >= scan['before_count']] - if not accepted: - return - begin, end = sorted((scan['begin'], scan['end'])) + def span_candidates(begin, end): + begin, end = sorted((begin, end)) assert begin % 8 == end % 8 == 0 and 0 <= end - begin <= 16 * 1024**2 error = lldb.SBError() raw = process.ReadMemory(begin, end - begin, error) if begin != end else b'' @@ -427,13 +432,24 @@ def accepted_span(scan, after): candidates.append(dict(address=begin + offset, value=value, offset_into_cell=value - owner['owner'])) assert len(candidates) <= 64, 'candidate-word bound exceeded' + return begin, end, cell_size, candidates, hashlib.sha256(raw).hexdigest() + + def accepted_span(scan, after): + assert after['roots'] == scan['roots'] and after['count'] >= scan['before_count'] + accepted = [i for i in after['owner_indices'] if i >= scan['before_count']] + if not accepted: + return + begin, end, cell_size, candidates, span_sha = span_candidates(scan['begin'], scan['end']) receipt = dict(collection=scan['collection'], phase=scan['phase'], roots=scan['roots'], thread=scan['thread'], begin=begin, end=end, before_count=scan['before_count'], after_count=after['count'], accepted_owner_indices=accepted, cell_size=cell_size, candidate_words=candidates, candidate_interpretation='unmodified exact/interior/one-past pointer candidates; accepted scanner iteration not yet identified', - span_sha256=hashlib.sha256(raw).hexdigest()) + span_sha256=span_sha, call_site=scan['call_site'], scanner_sp=scan['scanner_sp']) root_collections[scan['collection']]['accepted_spans'].append(receipt) + if word_witness is not None and word_witness['span_return'] == scan['return_breakpoint']: + assert set(word_witness['accepted_owner_indices']) <= set(accepted) + receipt['scanner_word'] = word_witness event('accepted-owner-span', **receipt) def marker_snapshot(visitor, cell): @@ -505,11 +521,37 @@ def marker_snapshot(visitor, cell): return_bp = None stack_roots_bp, _ = entry('JSC::MachineThreads::gatherConservativeRoots', required=False) scan_bps = [] + scanner_loops = [] for signature in [ 'void JSC::ConservativeRoots::genericAddSpan(void*, void*, JSC::DummyMarkHook&)', 'void JSC::ConservativeRoots::genericAddSpan(void*, void*, JSC::CompositeMarkHook&)']: - bp, _ = entry('JSC::ConservativeRoots::genericAddSpan', signature, required=False) + bp, address = entry('JSC::ConservativeRoots::genericAddSpan', signature, required=False) scan_bps.append(bp) + if address is not None: + # Locked shipping Build ID: admit the actual scanner load, argument moves and call. + sites = {87227072: [(87227200, 87241536), (87227264, 87243712)], + 87227392: [(87227520, 87246848), (87227584, 87248576)]} + assert address.GetFileAddress() in sites + instructions = address.GetFunction().GetInstructions(target) + assert 0 < instructions.GetSize() <= 4096 + rows = {instructions.GetInstructionAtIndex(i).GetAddress().GetFileAddress(): + instructions.GetInstructionAtIndex(i) for i in range(instructions.GetSize())} + for load_pc, callee_pc in sites[address.GetFileAddress()]: + expected = [('ldr', 'x1, [x26], #0x8'), ('mov', 'x0, x20'), + ('mov', 'w2, w22'), ('mov', 'w3, w23'), ('mov', 'x4, x21'), + ('mov', 'x5, x24' if load_pc in (87227200, 87227520) else 'x5, xzr'), + ('mov', 'x6, x19')] + for i, (mnemonic, operands) in enumerate(expected): + instruction = rows[load_pc + i * 4] + assert (instruction.GetMnemonic(target), instruction.GetOperands(target)) == (mnemonic, operands) + instruction = rows[load_pc + 28] + assert instruction.GetMnemonic(target) == 'bl' + assert int(instruction.GetOperands(target), 16) == callee_pc + callee = module.ResolveFileAddress(callee_pc).GetFunction().GetName() + assert callee and 'ConservativeRoots::genericAddPointer<' in callee + scanner_loops.append(dict(span_breakpoint=bp.GetID(), read_stop=load_pc + 4, + call=load_pc + 28, returned=load_pc + 32, callee=callee)) + report['scanner_loops'] = scanner_loops root_bps = [stack_roots_bp, *scan_bps] if any(bp is None for bp in root_bps): inventory_inline_roots() @@ -536,7 +578,29 @@ def marker_snapshot(visitor, cell): thread = threads[0] frame = thread.GetFrameAtIndex(0) if thread.GetStopReason() == lldb.eStopReasonWatchpoint: - assert stage == 'trace' and mark_watch is not None and mark_watch.IsEnabled() + assert stage == 'trace' and thread.GetStopReasonDataCount() == 1 + watch_id = thread.GetStopReasonDataAtIndex(0) + if word_watch is not None and watch_id == word_watch.GetID(): + assert word_watch.IsEnabled() and word_pending is not None + word_watch.SetEnabled(False) + assert not word_watch.IsEnabled() + assert word_pending['phase'] == 'armed' and word_pending['thread'] == thread.GetThreadID() + loops = [row for row in scanner_loops if row['span_breakpoint'] == word_pending['span_breakpoint'] + and module.ResolveFileAddress(row['read_stop']).GetLoadAddress(target) == frame.GetPC()] + # LLDB23 exposes ARM read watches after its transparent single-step of the load. + assert len(loops) == 1, 'candidate was not read by the admitted scanner load' + assert reg(frame, 'x26') == word_pending['address'] + 8 + assert reg(frame, 'x1') == word_pending['value'] == word(word_pending['address']) + assert reg(frame, 'x20') == word_pending['roots'] + word_pending.update(phase='read', loop=loops[0], read_pc=frame.GetPC()) + pointer_call_bp = target.BreakpointCreateBySBAddress(module.ResolveFileAddress(loops[0]['call'])) + assert pointer_call_bp.GetNumLocations() == 1 + pointer_call_bp.SetThreadID(thread.GetThreadID()) + pointer_call_bp.SetOneShot(True) + event('scanner-word-read', **word_pending) + assert process.Continue().Success() + continue + assert mark_watch is not None and mark_watch.IsEnabled() assert thread.GetStopReasonDataCount() == 1 assert thread.GetStopReasonDataAtIndex(0) == mark_watch.GetID() value = word(mark_observation['address'], 1) @@ -556,6 +620,9 @@ def marker_snapshot(visitor, cell): after_stack = root_collection['after_stack'] assert current_roots['count'] >= after_stack['count'] assert current_roots['owner_indices'] and root_collection['accepted_spans'] + assert word_witness is not None and word_pending is None + assert set(word_witness['accepted_owner_indices']) <= set(current_roots['owner_indices']) + assert any(span.get('scanner_word') == word_witness for span in root_collection['accepted_spans']) stack_indices = [i for i in current_roots['owner_indices'] if i < after_stack['count']] assert stack_indices == after_stack['owner_indices'], 'stack-root prefix changed' subsequent_indices = [i for i in current_roots['owner_indices'] if i >= after_stack['count']] @@ -630,14 +697,57 @@ def marker_snapshot(visitor, cell): assert bp.GetNumLocations() == 1 bp.SetThreadID(thread.GetThreadID()) bp.SetOneShot(True) - scan_returns[bp.GetID()] = dict(collection=root_collection['index'], + scan = dict(collection=root_collection['index'], phase=root_collection['phase'], roots=roots, before_count=before['count'], - begin=reg(frame, 'x1'), end=reg(frame, 'x2'), thread=thread.GetThreadID()) + begin=reg(frame, 'x1'), end=reg(frame, 'x2'), thread=thread.GetThreadID(), + return_breakpoint=bp.GetID(), call_site=symbolize(reg(frame, 'x30'), True), + scanner_sp=reg(frame, 'sp')) + scan_returns[bp.GetID()] = scan + if word_witness is None: + _, _, _, candidates, _ = span_candidates(scan['begin'], scan['end']) + if candidates: + assert len(candidates) == 1 and word_pending is None, 'ambiguous scanner-word candidates' + candidate = candidates[0] + error = lldb.SBError() + word_watch = target.WatchAddress(candidate['address'], 8, True, False, error) + assert error.Success() and word_watch.IsValid() and word_watch.IsEnabled() + assert word_watch.GetWatchAddress() == candidate['address'] and word_watch.GetWatchSize() == 8 + assert word_watch.GetID() > 0 and word_watch.GetID() != mark_watch.GetID() + word_pending = dict(candidate, phase='armed', roots=roots, thread=thread.GetThreadID(), + span_return=bp.GetID(), span_breakpoint=next(i for i in ids if i in {b.GetID() for b in scan_bps})) + event('scanner-word-watchpoint-admission', watchpoint=word_watch.GetID(), **word_pending) elif ids.intersection(scan_returns): assert len(ids.intersection(scan_returns)) == 1 scan = scan_returns.pop(next(iter(ids.intersection(scan_returns)))) assert scan['thread'] == thread.GetThreadID() + assert word_pending is None, 'scanner span returned before the word witness completed' accepted_span(scan, roots_snapshot(scan['roots'])) + elif pointer_call_bp is not None and pointer_call_bp.GetID() in ids: + assert word_pending is not None and word_pending['phase'] == 'read' + assert thread.GetThreadID() == word_pending['thread'] + loop = word_pending['loop'] + assert frame.GetPC() == module.ResolveFileAddress(loop['call']).GetLoadAddress(target) + assert reg(frame, 'x0') == word_pending['roots'] and reg(frame, 'x1') == word_pending['value'] + assert reg(frame, 'x26') == word_pending['address'] + 8 + word_pending.update(phase='called', before=roots_snapshot(word_pending['roots'])) + pointer_call_bp = None + pointer_return_bp = target.BreakpointCreateBySBAddress(module.ResolveFileAddress(loop['returned'])) + assert pointer_return_bp.GetNumLocations() == 1 + pointer_return_bp.SetThreadID(thread.GetThreadID()) + pointer_return_bp.SetOneShot(True) + elif pointer_return_bp is not None and pointer_return_bp.GetID() in ids: + assert word_pending is not None and word_pending['phase'] == 'called' + assert thread.GetThreadID() == word_pending['thread'] + assert frame.GetPC() == module.ResolveFileAddress(word_pending['loop']['returned']).GetLoadAddress(target) + before = word_pending['before'] + after = roots_snapshot(word_pending['roots'], before['count']) + assert after['prefix_sha256'] == before['sha256'], 'root prefix changed across the scanner call' + accepted = [i for i in after['owner_indices'] if i >= before['count']] + assert accepted and word_witness is None, 'observed scanner word did not append the owner' + word_witness = dict(word_pending, phase='accepted', after=after, accepted_owner_indices=accepted) + word_pending = None + pointer_return_bp = None + event('accepted-scanner-word', **word_witness) elif weak_bp.GetID() in ids: assert weak_creation is None global_object, call = reg(frame, 'x0'), reg(frame, 'x1') @@ -749,7 +859,9 @@ def marker_snapshot(visitor, cell): report['outcome'] = facts assert root_collections and any(c['accepted_spans'] for c in root_collections) report['root_collections'] = root_collections - report['classification'] = 'accepted machine-stack/subsequent scan spans and real-mark writer captured; exact originating scanner word and source frame remain unproven' + assert word_witness is not None and word_pending is None + report['scanner_word'] = word_witness + report['classification'] = 'exact scanner word appended the owner to accepted roots and real-mark writer captured; originating source frame remains unproven' report['passed'] = True except BaseException as error: report['gap'] = str(error) @@ -758,7 +870,8 @@ def marker_snapshot(visitor, cell): if stage == 'trace': report['observation'] = {'weak_targets': weak, 'owner': owner, 'returned': returned, 'gc_count': gc_count, 'pending_weak_constructor': weak_creation, - 'root_collections': root_collections, 'pending_scan_returns': scan_returns} + 'root_collections': root_collections, 'pending_scan_returns': scan_returns, + 'scanner_word': word_witness, 'pending_word': word_pending} if process is not None and process.IsValid() and process.GetState() != lldb.eStateExited: process.Kill() write(stage + '.json', report) From 8664f99cfc7dda1783052c8b11c24822754bbdf0 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 02:44:52 +0800 Subject: [PATCH 34/74] test(tls): bind the scanner word to conducting stack provenance --- .github/openclaw/tls-retainer-capture.py | 67 +++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py index 98cca3ded7a5..e0388e53e9aa 100644 --- a/.github/openclaw/tls-retainer-capture.py +++ b/.github/openclaw/tls-retainer-capture.py @@ -434,6 +434,54 @@ def span_candidates(begin, end): assert len(candidates) <= 64, 'candidate-word bound exceeded' return begin, end, cell_size, candidates, hashlib.sha256(raw).hexdigest() + def conducting_state(frame, thread, heap): + collector = typed(number(field(heap, 'm_collector')), 'JSC::Collector') + state_address, conductor = reg(frame, 'x4'), reg(frame, 'x5') + assert state_address and conductor, 'no original conducting-mutator state' + assert number(field(collector, 'm_conductingMutatorState')) == state_address + assert number(field(collector, 'm_conductorThread')) == conductor + state = typed(state_address, 'JSC::CurrentThreadState') + top, origin = number(field(state, 'stackTop')), number(field(state, 'stackOrigin')) + registers = number(field(state, 'registerState')) + assert registers and 0 < origin - top <= 16 * 1024**2 + saved = typed(registers, 'JSC::RegisterState') + fp = number(field(saved, 'x29')) + links = [] + for _ in range(128): + if not fp: + break + assert fp % 16 == 0 and top <= fp and fp + 16 <= origin, 'saved FP outside conducting stack' + previous, saved_lr = word(fp), word(fp + 8) + links.append(dict(fp=fp, previous_fp=previous, saved_lr=saved_lr, + caller=symbolize(saved_lr, True) if saved_lr else None)) + assert not previous or previous > fp, 'conducting FP chain does not increase' + fp = previous + else: + raise RuntimeError('conducting frame-pointer budget exhausted') + assert len(links) >= 2, 'conducting frame-pointer chain unavailable' + return dict(address=state_address, conductor=conductor, debugger_thread=thread.GetThreadID(), + scanner_sp=reg(frame, 'sp'), stack_top=top, stack_origin=origin, + registers=registers, register_bytes=saved.GetByteSize(), links=links) + + def word_provenance(scan, witness): + state = root_collections[scan['collection']]['conducting_state'] + begin, end = sorted((scan['begin'], scan['end'])) + address = witness['address'] + if (begin, end) == (state['registers'], state['registers'] + state['register_bytes']): + offset = address - begin + assert offset % 8 == 0 and 0 <= offset < 88 + return dict(kind='saved-conductor-register', register='x' + str(19 + offset // 8), + storage_address=address, conducting_state=state['address']) + assert (begin, end) == (state['stack_top'], state['stack_origin']), 'witnessed span is not an admitted original conductor span' + intervals = [] + for lower, upper in zip(state['links'], state['links'][1:]): + if lower['fp'] + 16 <= address and address + 8 <= upper['fp']: + intervals.append(dict(lower=lower, upper=upper, offset_from_lower_fp=address - lower['fp'])) + assert len(intervals) == 1, 'scanner word lacks a bounded adjacent-frame interval' + return dict(kind='original-conductor-stack', storage_address=address, + conducting_state=state['address'], interval=intervals[0], + interpretation='original stack word bounded by adjacent saved FP records; source variable and originating writer remain unproven') + def accepted_span(scan, after): assert after['roots'] == scan['roots'] and after['count'] >= scan['before_count'] accepted = [i for i in after['owner_indices'] if i >= scan['before_count']] @@ -450,6 +498,8 @@ def accepted_span(scan, after): if word_witness is not None and word_witness['span_return'] == scan['return_breakpoint']: assert set(word_witness['accepted_owner_indices']) <= set(accepted) receipt['scanner_word'] = word_witness + receipt['candidate_interpretation'] = 'exact scanner word proved by watched read and call-return insertion' + receipt['word_provenance'] = word_provenance(scan, word_witness) event('accepted-owner-span', **receipt) def marker_snapshot(visitor, cell): @@ -552,6 +602,16 @@ def marker_snapshot(visitor, cell): scanner_loops.append(dict(span_breakpoint=bp.GetID(), read_stop=load_pc + 4, call=load_pc + 28, returned=load_pc + 32, callee=callee)) report['scanner_loops'] = scanner_loops + # Frozen Linux ARM64 layouts, admitted from DWARF before any inferior reads. + for name, size, fields in [ + ('JSC::CurrentThreadState', 24, ['stackOrigin', 'stackTop', 'registerState']), + ('JSC::RegisterState', 88, ['x' + str(i) for i in range(19, 30)])]: + layout = concrete_type(name) + assert layout.GetByteSize() == size and layout.GetNumberOfFields() == len(fields) + for i, expected in enumerate(fields): + member = layout.GetFieldAtIndex(i) + assert member.GetName() == expected and member.GetOffsetInBytes() == i * 8 + assert member.GetType().GetByteSize() == 8 root_bps = [stack_roots_bp, *scan_bps] if any(bp is None for bp in root_bps): inventory_inline_roots() @@ -675,6 +735,7 @@ def marker_snapshot(visitor, cell): assert initial_roots['count'] == 0 root_collection = dict(index=len(root_collections), roots=roots, phase='machine-stack', accepted_spans=[], initial=initial_roots) + root_collection['conducting_state'] = conducting_state(frame, thread, heap) root_collections.append(root_collection) event('machine-stack-gather-entry', collection=root_collection['index'], **initial_roots) stack_roots_return = target.BreakpointCreateByAddress(reg(frame, 'x30')) @@ -861,7 +922,11 @@ def marker_snapshot(visitor, cell): report['root_collections'] = root_collections assert word_witness is not None and word_pending is None report['scanner_word'] = word_witness - report['classification'] = 'exact scanner word appended the owner to accepted roots and real-mark writer captured; originating source frame remains unproven' + provenance = [span['word_provenance'] for collection in root_collections + for span in collection['accepted_spans'] if 'word_provenance' in span] + assert len(provenance) == 1 + report['word_provenance'] = provenance[0] + report['classification'] = 'exact accepted scanner word and original conductor provenance captured; source variable and originating writer remain unproven' report['passed'] = True except BaseException as error: report['gap'] = str(error) From 8167dca3940655da0c09673f9393fa7ed737494e Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 02:53:40 +0800 Subject: [PATCH 35/74] test(tls): inventory the conducting interpreter call frame --- .github/openclaw/tls-retainer-capture.py | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py index e0388e53e9aa..901770ad187c 100644 --- a/.github/openclaw/tls-retainer-capture.py +++ b/.github/openclaw/tls-retainer-capture.py @@ -265,6 +265,39 @@ def inventory_inline_roots(): event('inline-root-inventory', function=function.GetName(), file_address=start, scopes=scopes, instructions=rows, blocks_visited=visited) + def inventory_conductor_call(): + # This saved return address bounds the witnessed word, but does not name a local. + # Inspect the physical frame before designing a watch of its writer. + call_pc = 88384104 + function = module.ResolveFileAddress(call_pc).GetFunction() + assert function.IsValid() and function.GetName().startswith('JSC::Interpreter::executeCall(') + instructions = function.GetInstructions(target) + assert 0 < instructions.GetSize() <= 8192, 'conductor call disassembly budget' + rows = [] + for i in range(instructions.GetSize()): + instruction = instructions.GetInstructionAtIndex(i) + address = instruction.GetAddress() + pc = address.GetFileAddress() + if i >= 80 and not call_pc - 64 <= pc <= call_pc + 32: + continue + line = address.GetLineEntry() + rows.append(dict(file_address=pc, mnemonic=instruction.GetMnemonic(target), + operands=instruction.GetOperands(target), + file=line.GetFileSpec().GetFilename(), line=line.GetLine())) + call = [row for row in rows if row['file_address'] == call_pc] + assert len(call) == 1 and call[0]['mnemonic'] == 'bl', 'conductor entry-call admission' + layout = concrete_type('JSC::ProtoCallFrame') + assert 0 < layout.GetByteSize() <= 1024 and layout.GetNumberOfFields() <= 32 + fields = [] + for i in range(layout.GetNumberOfFields()): + member = layout.GetFieldAtIndex(i) + fields.append(dict(name=member.GetName(), offset=member.GetOffsetInBytes(), + bytes=member.GetType().GetByteSize(), type=member.GetType().GetName())) + event('conductor-call-inventory', function=function.GetName(), + file_address=function.GetStartAddress().GetFileAddress(), call_pc=call_pc, + instructions=rows, proto_call_frame=dict(bytes=layout.GetByteSize(), fields=fields), + interpretation='immutable frame inventory; no slot-to-local or writer attribution') + def reg(frame, name): value = frame.FindRegister(name) assert value.IsValid() and value.GetError().Success(), name @@ -612,6 +645,7 @@ def marker_snapshot(visitor, cell): member = layout.GetFieldAtIndex(i) assert member.GetName() == expected and member.GetOffsetInBytes() == i * 8 assert member.GetType().GetByteSize() == 8 + inventory_conductor_call() root_bps = [stack_roots_bp, *scan_bps] if any(bp is None for bp in root_bps): inventory_inline_roots() From 555dd508ccd745e753bd66f3999f0ec3cda0208b Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 03:01:42 +0800 Subject: [PATCH 36/74] test(tls): watch the accepted stack word before frame initialization --- .github/openclaw/tls-retainer-capture.py | 101 ++++++++++++++++++++++- 1 file changed, 99 insertions(+), 2 deletions(-) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py index 901770ad187c..97a54f082b82 100644 --- a/.github/openclaw/tls-retainer-capture.py +++ b/.github/openclaw/tls-retainer-capture.py @@ -161,6 +161,9 @@ def debugger_stage(): word_witness = None pointer_call_bp = None pointer_return_bp = None + writer_entry_bp = None + writer_watch = None + writer_state = None def event(kind, **values): report['events'].append({'kind': kind, **values}) @@ -467,6 +470,44 @@ def span_candidates(begin, end): assert len(candidates) <= 64, 'candidate-word bound exceeded' return begin, end, cell_size, candidates, hashlib.sha256(raw).hexdigest() + def stack_word_write(frame, thread): + assert writer_state is not None and thread.GetThreadID() == writer_state['thread'] + assert len(writer_state['writes']) < 32, 'stack-writer stop budget exhausted' + pc = frame.GetPC() + instructions = target.ReadInstructions(target.ResolveLoadAddress(pc - 8), 3) + assert instructions.GetSize() == 3, 'stack-writer instruction window unavailable' + rows = [] + for i in range(3): + instruction = instructions.GetInstructionAtIndex(i) + rows.append(dict(pc=instruction.GetAddress().GetLoadAddress(target), + mnemonic=instruction.GetMnemonic(target), operands=instruction.GetOperands(target))) + event('candidate-stack-writer-stop', pc=pc, slot_value=word(writer_state['address']), + sp=reg(frame, 'sp'), fp=reg(frame, 'x29'), instructions=rows) + # LLDB ARM exposes a write watch after stepping the store. Admit its effective + # destination; an Always watch may otherwise report an adjacent access. + instruction = rows[1] + assert instruction['pc'] == pc - 4 and instruction['mnemonic'] in ('str', 'stur', 'stp'), 'unsupported stack-writer instruction' + match = re.fullmatch(r'([xwqds]\d+|[xw]zr)(?:, ([xwqds]\d+|[xw]zr))?, \[(sp|x29)(?:, #(-?0x[0-9a-f]+))?\](!)?(?:, #(-?0x[0-9a-f]+))?', instruction['operands']) + assert match is not None, 'unsupported stack-writer addressing mode' + first, second, base, offset, pre, post = match.groups() + assert (second is not None) == (instruction['mnemonic'] == 'stp') + width = {'x': 8, 'w': 4, 'q': 16, 'd': 8, 's': 4}[first[0]] + assert second is None or second[0] == first[0] + base_value = reg(frame, base) + address = base_value if pre else base_value + int(offset or '0', 16) + if post: + assert offset is None and pre is None + address = base_value - int(post, 16) + size = width * (2 if second else 1) + slot = writer_state['address'] + assert max(address, slot) < min(address + size, slot + 8), 'watchpoint was an adjacent write' + receipt = dict(pc=pc, store_pc=pc - 4, source=symbolize(pc - 4), + instruction=instruction, destination=address, bytes=size, + registers={name: frame.FindRegister(name).GetValue() for name in [first, second] if name}, + slot_value=word(slot), sp=reg(frame, 'sp'), fp=reg(frame, 'x29')) + writer_state['writes'].append(receipt) + event('stack-word-write', **receipt) + def conducting_state(frame, thread, heap): collector = typed(number(field(heap, 'm_collector')), 'JSC::Collector') state_address, conductor = reg(frame, 'x4'), reg(frame, 'x5') @@ -598,6 +639,18 @@ def marker_snapshot(visitor, cell): weak_return_bp = None completion_bp, completion_address = entry('Bun__UpgradedDuplex__onWriteComplete') gc_bp, _ = entry('JSC__VM__runGC') + writer_entry_bp, writer_entry_address = entry('JSC::Interpreter::executeCall') + assert writer_entry_address.GetFileAddress() == 88383168 + prologue = writer_entry_address.GetFunction().GetInstructions(target) + expected = [('sub', 'sp, sp, #0x180'), ('stp', 'x29, x30, [sp, #0x120]'), + ('stp', 'x28, x27, [sp, #0x130]'), ('stp', 'x26, x25, [sp, #0x140]'), + ('stp', 'x24, x23, [sp, #0x150]'), ('stp', 'x22, x21, [sp, #0x160]'), + ('stp', 'x20, x19, [sp, #0x170]'), ('add', 'x29, sp, #0x120')] + for i, expected_instruction in enumerate(expected): + instruction = prologue.GetInstructionAtIndex(i) + assert instruction.GetAddress().GetFileAddress() == 88383168 + i * 4 + assert (instruction.GetMnemonic(target), instruction.GetOperands(target)) == expected_instruction + writer_entry_bp.SetEnabled(False) mark_bp = target.BreakpointCreateByLocation('Collector.cpp', 1068) assert mark_bp.GetNumLocations() > 0, 'missing post-mark verifier boundary' mark_bp.SetEnabled(False) @@ -674,6 +727,11 @@ def marker_snapshot(visitor, cell): if thread.GetStopReason() == lldb.eStopReasonWatchpoint: assert stage == 'trace' and thread.GetStopReasonDataCount() == 1 watch_id = thread.GetStopReasonDataAtIndex(0) + if writer_watch is not None and watch_id == writer_watch.GetID(): + assert writer_watch.IsEnabled() + stack_word_write(frame, thread) + assert process.Continue().Success() + continue if word_watch is not None and watch_id == word_watch.GetID(): assert word_watch.IsEnabled() and word_pending is not None word_watch.SetEnabled(False) @@ -757,6 +815,31 @@ def marker_snapshot(visitor, cell): assert not (OUT / 'child.json').exists(), 'multiple selected children' write('child.json', {'argv': values, 'script_sha256': hashlib.sha256(values[2].encode()).hexdigest()}) event('captured-actual-fixture-child') + elif writer_entry_bp is not None and writer_entry_bp.GetID() in ids: + assert returned and gc_count == 30 and owner is not None + interpreter = field(typed(owner['vm'], 'JSC::VM'), 'interpreter') + timer_return = module.ResolveFileAddress(27509448).GetLoadAddress(target) + if reg(frame, 'x0') == interpreter.GetLoadAddress() and reg(frame, 'x30') == timer_return: + assert writer_state is None and writer_watch is None + # Admitted prologue: FP=entrySP-0x180+0x120. The previous proof + # placed the accepted word at FP-0x40; this run must rediscover it. + entry_sp = reg(frame, 'sp') + prospective_fp = entry_sp - 0x60 + address = prospective_fp - 0x40 + assert address % 8 == 0 + options = lldb.SBWatchpointOptions() + options.SetWatchpointTypeRead(False) + options.SetWatchpointTypeWrite(lldb.eWatchpointWriteTypeAlways) + assert options.GetWatchpointTypeWrite() == lldb.eWatchpointWriteTypeAlways + error = lldb.SBError() + writer_watch = target.WatchpointCreateByAddress(address, 8, options, error) + assert error.Success() and writer_watch.IsValid() and writer_watch.IsEnabled() + assert writer_watch.GetWatchAddress() == address and writer_watch.GetWatchSize() == 8 + writer_state = dict(address=address, thread=thread.GetThreadID(), entry_sp=entry_sp, + prospective_fp=prospective_fp, initial_value=word(address), writes=[], + incoming={name: reg(frame, name) for name in ['x' + str(i) for i in range(6)]}) + writer_entry_bp.SetEnabled(False) + event('stack-writer-watchpoint-admission', watchpoint=writer_watch.GetID(), **writer_state) elif stack_roots_bp.GetID() in ids: assert returned and gc_count == 31 and owner is not None heap = field(typed(owner['vm'], 'JSC::VM'), 'heap') @@ -803,6 +886,11 @@ def marker_snapshot(visitor, cell): if candidates: assert len(candidates) == 1 and word_pending is None, 'ambiguous scanner-word candidates' candidate = candidates[0] + assert writer_state is not None and writer_watch.IsEnabled(), 'selected timer frame was not watched' + assert writer_state['thread'] == thread.GetThreadID() and candidate['address'] == writer_state['address'], 'accepted word differs from watched frame slot' + assert writer_state['writes'] and writer_state['writes'][-1]['slot_value'] == candidate['value'], 'no admitted write of the accepted owner in this frame' + writer_watch.SetEnabled(False) + assert not writer_watch.IsEnabled() error = lldb.SBError() word_watch = target.WatchAddress(candidate['address'], 8, True, False, error) assert error.Success() and word_watch.IsValid() and word_watch.IsEnabled() @@ -893,6 +981,8 @@ def marker_snapshot(visitor, cell): gc_count += 1 assert gc_count <= 31 # The fixture collects once for queuedRoot, then thirty times before its final deref. + if gc_count == 30: + writer_entry_bp.SetEnabled(True) if gc_count == 31: _, allocation = precise_allocation(owner['owner']) marked = field(field(typed(allocation, 'JSC::PreciseAllocation'), 'm_isMarked'), 'value') @@ -960,7 +1050,13 @@ def marker_snapshot(visitor, cell): for span in collection['accepted_spans'] if 'word_provenance' in span] assert len(provenance) == 1 report['word_provenance'] = provenance[0] - report['classification'] = 'exact accepted scanner word and original conductor provenance captured; source variable and originating writer remain unproven' + assert writer_state is not None and provenance[0]['kind'] == 'original-conductor-stack' + assert provenance[0]['storage_address'] == writer_state['address'] + upper = provenance[0]['interval']['upper'] + assert upper['fp'] == writer_state['prospective_fp'] + assert upper['saved_lr'] == module.ResolveFileAddress(27509448).GetLoadAddress(target) + report['stack_word_writer'] = writer_state + report['classification'] = 'exact accepted scanner word tied to an admitted physical stack write; source variable and earlier register origin remain unproven' report['passed'] = True except BaseException as error: report['gap'] = str(error) @@ -970,7 +1066,8 @@ def marker_snapshot(visitor, cell): report['observation'] = {'weak_targets': weak, 'owner': owner, 'returned': returned, 'gc_count': gc_count, 'pending_weak_constructor': weak_creation, 'root_collections': root_collections, 'pending_scan_returns': scan_returns, - 'scanner_word': word_witness, 'pending_word': word_pending} + 'scanner_word': word_witness, 'pending_word': word_pending, + 'stack_word_writer': writer_state} if process is not None and process.IsValid() and process.GetState() != lldb.eStateExited: process.Kill() write(stage + '.json', report) From c375b7d9caac77d724d045785774e83910c51b05 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 03:08:53 +0800 Subject: [PATCH 37/74] test(tls): observe retained stack storage from the first timer --- .github/openclaw/tls-retainer-capture.py | 32 ++++++++++++++++++------ 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py index 97a54f082b82..d81e966308c8 100644 --- a/.github/openclaw/tls-retainer-capture.py +++ b/.github/openclaw/tls-retainer-capture.py @@ -650,7 +650,19 @@ def marker_snapshot(visitor, cell): instruction = prologue.GetInstructionAtIndex(i) assert instruction.GetAddress().GetFileAddress() == 88383168 + i * 4 assert (instruction.GetMnemonic(target), instruction.GetOperands(target)) == expected_instruction - writer_entry_bp.SetEnabled(False) + timer_call = target.ReadInstructions(module.ResolveFileAddress(27509444), 1) + assert timer_call.GetSize() == 1 + instruction = timer_call.GetInstructionAtIndex(0) + assert instruction.GetMnemonic(target) == 'bl' + assert int(instruction.GetOperands(target), 16) == 88383168 + vm_layout = concrete_type('JSC::VM') + interpreter_fields = [vm_layout.GetFieldAtIndex(i) for i in range(vm_layout.GetNumberOfFields()) + if vm_layout.GetFieldAtIndex(i).GetName() == 'interpreter'] + assert len(interpreter_fields) == 1 + interpreter_offset = interpreter_fields[0].GetOffsetInBytes() + assert interpreter_offset == 0x2d468 + event('early-stack-writer-layout', interpreter_offset=interpreter_offset, + timer_call=27509444, timer_return=27509448) mark_bp = target.BreakpointCreateByLocation('Collector.cpp', 1068) assert mark_bp.GetNumLocations() > 0, 'missing post-mark verifier boundary' mark_bp.SetEnabled(False) @@ -816,10 +828,8 @@ def marker_snapshot(visitor, cell): write('child.json', {'argv': values, 'script_sha256': hashlib.sha256(values[2].encode()).hexdigest()}) event('captured-actual-fixture-child') elif writer_entry_bp is not None and writer_entry_bp.GetID() in ids: - assert returned and gc_count == 30 and owner is not None - interpreter = field(typed(owner['vm'], 'JSC::VM'), 'interpreter') timer_return = module.ResolveFileAddress(27509448).GetLoadAddress(target) - if reg(frame, 'x0') == interpreter.GetLoadAddress() and reg(frame, 'x30') == timer_return: + if reg(frame, 'x30') == timer_return: assert writer_state is None and writer_watch is None # Admitted prologue: FP=entrySP-0x180+0x120. The previous proof # placed the accepted word at FP-0x40; this run must rediscover it. @@ -836,6 +846,7 @@ def marker_snapshot(visitor, cell): assert error.Success() and writer_watch.IsValid() and writer_watch.IsEnabled() assert writer_watch.GetWatchAddress() == address and writer_watch.GetWatchSize() == 8 writer_state = dict(address=address, thread=thread.GetThreadID(), entry_sp=entry_sp, + provisional_vm=reg(frame, 'x0') - interpreter_offset, prospective_fp=prospective_fp, initial_value=word(address), writes=[], incoming={name: reg(frame, name) for name in ['x' + str(i) for i in range(6)]}) writer_entry_bp.SetEnabled(False) @@ -888,7 +899,8 @@ def marker_snapshot(visitor, cell): candidate = candidates[0] assert writer_state is not None and writer_watch.IsEnabled(), 'selected timer frame was not watched' assert writer_state['thread'] == thread.GetThreadID() and candidate['address'] == writer_state['address'], 'accepted word differs from watched frame slot' - assert writer_state['writes'] and writer_state['writes'][-1]['slot_value'] == candidate['value'], 'no admitted write of the accepted owner in this frame' + assert writer_state['provisional_vm'] == owner['vm'], 'first timer belongs to another VM' + assert writer_state['writes'] and writer_state['writes'][-1]['slot_value'] == candidate['value'], 'no admitted write of the accepted owner in watched stack storage' writer_watch.SetEnabled(False) assert not writer_watch.IsEnabled() error = lldb.SBError() @@ -965,6 +977,7 @@ def marker_snapshot(visitor, cell): if number(field(function, 'dataPtr')) == 0: assert owner is None and len(candidates) == 1 owner = dict(candidates[0], callee=callee, argument_count_including_this=count) + assert writer_state is not None and writer_state['provisional_vm'] == owner['vm'], 'late callback VM differs from first timer' assert count == 1, 'fixture late callback unexpectedly has arguments' return_bp = target.BreakpointCreateByAddress(reg(frame, 'x30')) return_bp.SetThreadID(thread.GetThreadID()) @@ -981,8 +994,6 @@ def marker_snapshot(visitor, cell): gc_count += 1 assert gc_count <= 31 # The fixture collects once for queuedRoot, then thirty times before its final deref. - if gc_count == 30: - writer_entry_bp.SetEnabled(True) if gc_count == 31: _, allocation = precise_allocation(owner['owner']) marked = field(field(typed(allocation, 'JSC::PreciseAllocation'), 'm_isMarked'), 'value') @@ -1056,7 +1067,12 @@ def marker_snapshot(visitor, cell): assert upper['fp'] == writer_state['prospective_fp'] assert upper['saved_lr'] == module.ResolveFileAddress(27509448).GetLoadAddress(target) report['stack_word_writer'] = writer_state - report['classification'] = 'exact accepted scanner word tied to an admitted physical stack write; source variable and earlier register origin remain unproven' + last_write = writer_state['writes'][-1] + slot = writer_state['address'] + complete = last_write['destination'] <= slot and slot + 8 <= last_write['destination'] + last_write['bytes'] + report['classification'] = ('exact accepted scanner word tied to an admitted physical stack write covering all eight bytes' + if complete else 'exact accepted scanner word tied to an admitted partial stack write; remaining-byte origin is unproven') + report['classification'] += '; source variable and earlier register origin remain unproven' report['passed'] = True except BaseException as error: report['gap'] = str(error) From 96aa4e718ae734253276a79a7d947be3eb16c43b Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 03:16:06 +0800 Subject: [PATCH 38/74] test(tls): admit the profiled timer tail call --- .github/openclaw/tls-retainer-capture.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py index d81e966308c8..84e1f5e72ca5 100644 --- a/.github/openclaw/tls-retainer-capture.py +++ b/.github/openclaw/tls-retainer-capture.py @@ -654,6 +654,13 @@ def marker_snapshot(visitor, cell): assert timer_call.GetSize() == 1 instruction = timer_call.GetInstructionAtIndex(0) assert instruction.GetMnemonic(target) == 'bl' + assert int(instruction.GetOperands(target), 16) == 91664704 + profiled = module.ResolveFileAddress(91664704).GetFunction() + assert profiled.IsValid() and profiled.GetName().startswith('JSC::profiledCall(') + tail_call = target.ReadInstructions(module.ResolveFileAddress(91664844), 1) + assert tail_call.GetSize() == 1 + instruction = tail_call.GetInstructionAtIndex(0) + assert instruction.GetMnemonic(target) == 'b' assert int(instruction.GetOperands(target), 16) == 88383168 vm_layout = concrete_type('JSC::VM') interpreter_fields = [vm_layout.GetFieldAtIndex(i) for i in range(vm_layout.GetNumberOfFields()) @@ -662,7 +669,8 @@ def marker_snapshot(visitor, cell): interpreter_offset = interpreter_fields[0].GetOffsetInBytes() assert interpreter_offset == 0x2d468 event('early-stack-writer-layout', interpreter_offset=interpreter_offset, - timer_call=27509444, timer_return=27509448) + timer_call=27509444, profiled_call=91664704, interpreter_tail_call=91664844, + timer_return=27509448) mark_bp = target.BreakpointCreateByLocation('Collector.cpp', 1068) assert mark_bp.GetNumLocations() > 0, 'missing post-mark verifier boundary' mark_bp.SetEnabled(False) From aa9d95589ef4fe2e04f04cb1d0ba89194d06e6cb Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 03:22:27 +0800 Subject: [PATCH 39/74] test(tls): admit observed byte stores into retained stack storage --- .github/openclaw/tls-retainer-capture.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py index 84e1f5e72ca5..ecf1c48f9a06 100644 --- a/.github/openclaw/tls-retainer-capture.py +++ b/.github/openclaw/tls-retainer-capture.py @@ -486,12 +486,15 @@ def stack_word_write(frame, thread): # LLDB ARM exposes a write watch after stepping the store. Admit its effective # destination; an Always watch may otherwise report an adjacent access. instruction = rows[1] - assert instruction['pc'] == pc - 4 and instruction['mnemonic'] in ('str', 'stur', 'stp'), 'unsupported stack-writer instruction' + assert instruction['pc'] == pc - 4 and instruction['mnemonic'] in ('str', 'stur', 'stp', 'strb'), 'unsupported stack-writer instruction' match = re.fullmatch(r'([xwqds]\d+|[xw]zr)(?:, ([xwqds]\d+|[xw]zr))?, \[(sp|x29)(?:, #(-?0x[0-9a-f]+))?\](!)?(?:, #(-?0x[0-9a-f]+))?', instruction['operands']) assert match is not None, 'unsupported stack-writer addressing mode' first, second, base, offset, pre, post = match.groups() assert (second is not None) == (instruction['mnemonic'] == 'stp') width = {'x': 8, 'w': 4, 'q': 16, 'd': 8, 's': 4}[first[0]] + if instruction['mnemonic'] == 'strb': + assert first[0] == 'w' and second is None + width = 1 assert second is None or second[0] == first[0] base_value = reg(frame, base) address = base_value if pre else base_value + int(offset or '0', 16) From 3d7b13fc4c79de5e25059e6ae730d8495d3fddd2 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 03:34:59 +0800 Subject: [PATCH 40/74] test(tls): identify the absent config stack copy --- .github/openclaw/tls-retainer-capture.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py index ecf1c48f9a06..23cdbb11af53 100644 --- a/.github/openclaw/tls-retainer-capture.py +++ b/.github/openclaw/tls-retainer-capture.py @@ -508,6 +508,26 @@ def stack_word_write(frame, thread): instruction=instruction, destination=address, bytes=size, registers={name: frame.FindRegister(name).GetValue() for name in [first, second] if name}, slot_value=word(slot), sp=reg(frame, 'sp'), fp=reg(frame, 'x29')) + if receipt['source']['file_address'] == 38748200: + symbol = target.ResolveLoadAddress(pc - 4).GetSymbol() + assert symbol.GetMangledName() == '_RNvMsd_NtNtCsgTvE0GkrhKz_11bun_runtime6socket11socket_bodyNtB5_20DuplexUpgradeContext6deinit' + assert symbol.GetStartAddress().GetFileAddress() == 38747968 + loads = target.ReadInstructions(module.ResolveFileAddress(38748192), 3) + assert loads.GetSize() == 3 + assert [(loads.GetInstructionAtIndex(i).GetMnemonic(target), + loads.GetInstructionAtIndex(i).GetOperands(target)) for i in range(3)] == [ + ('ldr', 'q0, [x19]'), ('ldr', 'q1, [x19, #0x10]'), ('str', 'q0, [sp]')] + assert address == slot == reg(frame, 'sp') and size == 16 + source = reg(frame, 'x19') + assert source and source % 8 == 0 + # The frozen clear copies 192 bytes before testing the saved Option tag at +191. + old_tag, current_tag = word(address + 191, 1), word(source + 191, 1) + assert old_tag == current_tag == reg(frame, 'w8') == 2, 'old config was not the absent variant' + assert word(source) == receipt['slot_value'] and word(source + 8) == word(address + 8) + receipt['absent_config_copy'] = dict(source_context=source, source_address=source, + destination=address, copied_payload_bytes=192, selected_bytes=16, + source_word=word(source), second_word=word(source + 8), old_tag=old_tag, + current_tag=current_tag, tag_offset=191, physical_symbol=symbol.GetMangledName()) writer_state['writes'].append(receipt) event('stack-word-write', **receipt) @@ -1083,7 +1103,9 @@ def marker_snapshot(visitor, cell): complete = last_write['destination'] <= slot and slot + 8 <= last_write['destination'] + last_write['bytes'] report['classification'] = ('exact accepted scanner word tied to an admitted physical stack write covering all eight bytes' if complete else 'exact accepted scanner word tied to an admitted partial stack write; remaining-byte origin is unproven') - report['classification'] += '; source variable and earlier register origin remain unproven' + config_copy = last_write.get('absent_config_copy') + assert config_copy is not None and config_copy['source_word'] == owner['owner'], 'absent config source copy not observed' + report['classification'] += '; copied from the old absent SSLConfig storage while clearing the context; earlier heap-payload writer remains unproven' report['passed'] = True except BaseException as error: report['gap'] = str(error) From 8b02717fe70cda4ec875a9ae93899a39ab4bc6af Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 03:48:47 +0800 Subject: [PATCH 41/74] test(tls): qualify the empty config cleanup repair --- .github/openclaw/tls-config-repair.patch | 27 ++++ .github/openclaw/tls-config-repair.py | 136 ++++++++++++++++++ .../workflows/openclaw-tls-config-repair.yml | 128 +++++++++++++++++ 3 files changed, 291 insertions(+) create mode 100644 .github/openclaw/tls-config-repair.patch create mode 100644 .github/openclaw/tls-config-repair.py create mode 100644 .github/workflows/openclaw-tls-config-repair.yml diff --git a/.github/openclaw/tls-config-repair.patch b/.github/openclaw/tls-config-repair.patch new file mode 100644 index 000000000000..8b73567aefc3 --- /dev/null +++ b/.github/openclaw/tls-config-repair.patch @@ -0,0 +1,27 @@ +diff --git a/src/runtime/socket/socket_body.rs b/src/runtime/socket/socket_body.rs +index 2752cbc894159817f1557df3e7d0f518895a3a18..5fa5d59042a200a50106415325b8f6d1b4255f83 100644 +--- a/src/runtime/socket/socket_body.rs ++++ b/src/runtime/socket/socket_body.rs +@@ -5118,7 +5118,9 @@ impl DuplexUpgradeContext { + Self::release_connection(this); + return; + } +- this.ssl_config.set(None); // Drop frees. ++ if this.ssl_config.get().is_some() { ++ this.ssl_config.set(None); // Drop frees. ++ } + // Bytes the peer sent while this task was still queued were + // staged by `UpgradedDuplex::on_internal_receive_data` (the + // engine did not exist yet to receive them). Feed them now +@@ -5189,7 +5191,10 @@ impl DuplexUpgradeContext { + debug_assert!(!ctx.native_shutdown_callback.get()); + ctx.tls.set(None); + // Close raced ahead of StartTLS — drop the unconsumed config / ctx. +- ctx.ssl_config.set(None); ++ // An absent config has an inactive payload that must not spill onto the JS stack. ++ if ctx.ssl_config.get().is_some() { ++ ctx.ssl_config.set(None); ++ } + ctx.owned_ctx.set(None); + } + // `UpgradedDuplex` cleanup diff --git a/.github/openclaw/tls-config-repair.py b/.github/openclaw/tls-config-repair.py new file mode 100644 index 000000000000..9c21ed3972a5 --- /dev/null +++ b/.github/openclaw/tls-config-repair.py @@ -0,0 +1,136 @@ +"""Qualify the reviewed empty-config repair without publishing the runtime branch.""" +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import subprocess +import time +from xml.etree import ElementTree + +SOURCE = Path('/work/source') +OUT = Path('/work/repair-proof') +INPUTS = Path(__file__).resolve().parent +BUN = '454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e' +WEBKIT = 'b82bc917e99110b61c3a21202d6def7457aa194c' +OWNER = 'src/runtime/socket/socket_body.rs' +BEFORE = '66121e98f4e8537520f976ff572ef538cc7baedf3e59c8ab2fdfce5fe4bc2afc' +AFTER = '2a40a7df44cfbc9344eb92ced128da282ade137fe2a08f5319d85b23c297dfb0' +PATCH = 'a0fbcc59fae14006b39c60d5cd71d0b71f80f151c6b0eac7d689db5fcbcfc25b' +FIXTURE = 'test/js/bun/net/socket-retention.test.ts' +FIXTURE_SHA = 'd4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2' +BUILD = SOURCE / 'build/tls-empty-config' +ENV = {k: v for k, v in os.environ.items() if not any(x in k.upper() for x in ('TOKEN', 'SECRET', 'PASSWORD'))} +ENV.update(CI='true', CARGO_BUILD_JOBS='2') +DEADLINE = time.monotonic() + 240 * 60 +STEPS = [] + + +def sha(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def git(*args): + return subprocess.check_output(['git', '-C', str(SOURCE), *args], timeout=120, text=True).strip() + + +def save(**facts): + (OUT / 'proof.json').write_text(json.dumps(dict(passed=False, bun=BUN, webkit=WEBKIT, + patch_sha256=PATCH, source_before=BEFORE, source_after=AFTER, steps=STEPS, **facts), indent=2) + '\n') + + +def run(name, argv, seconds): + log = OUT / (name + '.log') + step = dict(name=name, argv=argv, exit_code=None, timed_out=False) + STEPS.append(step) + save() + with log.open('xb') as stream: + process = subprocess.Popen(argv, cwd=SOURCE, env=ENV, stdin=subprocess.DEVNULL, + stdout=stream, stderr=subprocess.STDOUT, start_new_session=True) + started = time.monotonic() + try: + while process.poll() is None: + if time.monotonic() >= min(DEADLINE, started + seconds): + step['timed_out'] = True + raise TimeoutError(name) + assert sum(p.stat().st_size for p in OUT.iterdir() if p.is_file()) <= 64 * 1024**2, 'proof exceeds 64 MiB' + time.sleep(1) + finally: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + stream.flush() + step['exit_code'] = process.returncode + save() + assert process.returncode == 0 and not step['timed_out'], name + return log.read_text(errors='replace') + + +def source_state(adapted): + assert git('rev-parse', 'HEAD') == BUN + assert git('write-tree') == original_tree + assert git('diff', '--no-renames', '--name-only', 'HEAD').splitlines() == ([OWNER] if adapted else []) + assert not git('ls-files', '--others', '--exclude-standard') + assert sha(SOURCE / OWNER) == (AFTER if adapted else BEFORE) + assert sha(SOURCE / FIXTURE) == FIXTURE_SHA + + +OUT.mkdir(exist_ok=True) +original_tree = git('rev-parse', 'HEAD^{tree}') +source_state(False) +assert os.uname().machine == 'aarch64' +assert sha(INPUTS / 'tls-config-repair.patch') == PATCH +save() +run('apply-check', ['git', 'apply', '--check', str(INPUTS / 'tls-config-repair.patch')], 60) +run('apply', ['git', 'apply', str(INPUTS / 'tls-config-repair.patch')], 60) +source_state(True) +run('rustfmt', ['rustup', 'run', 'nightly-2026-09-15', 'rustfmt', '--edition', '2024', + '--config-path', 'rustfmt.toml', '--config', 'skip_children=true', '--check', OWNER], 120) +source_state(True) +run('build', ['bun', 'scripts/build.ts', '--profile=release', '--ci=on', '--os=linux', + '--arch=aarch64', '--abi=gnu', '--build-dir=build/tls-empty-config', '-j2'], 13800) +source_state(True) +binary, profile = BUILD / 'bun', BUILD / 'bun-profile' +assert binary.is_file() and profile.is_file() +identity = json.loads(run('identity', [str(binary), '-e', + 'console.log(JSON.stringify({commit:Bun.revision,webkit:process.versions.webkit,platform:process.platform,arch:process.arch}))'], 60)) +assert identity == dict(commit=BUN, webkit=WEBKIT, platform='linux', arch='arm64'), identity +node = json.loads(run('node', ['node', '-e', + 'console.log(JSON.stringify({node:process.versions.node,bun:process.versions.bun??null,platform:process.platform,arch:process.arch}))'], 60)) +assert node == dict(node='26.12.1', bun=None, platform='linux', arch='arm64'), node +symbols = run('symbols', ['llvm-nm', '--defined-only', '--format=posix', str(profile)], 120) +names = [line.split()[0] for line in symbols.splitlines() if re.fullmatch( + r'_RNv\w*11bun_runtime6socket11socket_body\w*20DuplexUpgradeContext6deinit', line.split()[0])] +assert len(names) == 1, 'repair deinit symbol missing or ambiguous' +run('deinit-disassembly', ['llvm-objdump', '--disassemble-symbols=' + names[0], '--demangle', str(profile)], 120) +files = json.loads(run('test-files', ['bun', '-e', + 'import {tlsQualificationFiles} from "./scripts/openclaw-release/release.ts";console.log(JSON.stringify(tlsQualificationFiles))'], 60)) +assert len(files) == 6 and len(set(files)) == 6 and FIXTURE in files +fixture_hashes = {name: sha(SOURCE / name) for name in files} +cases = [] +for index, name in enumerate(files): + xml = OUT / ('tls-' + str(index) + '.xml') + run('tls-' + str(index), [str(binary), 'test', name, '--reporter=junit', '--reporter-outfile=' + str(xml)], 600) + selected = list(ElementTree.parse(xml).iter('testcase')) + assert selected and all(c.find('failure') is None and c.find('error') is None for c in selected), name + assert any(c.find('skipped') is None for c in selected), name + cases += selected + source_state(True) +for name in ['a TLS Duplex completion retains its owner and is inert after destroy', + 'TLS Duplex cycles collect after startup while a retained origin keeps its wrapper alive']: + selected = [case for case in cases if case.get('name') == name] + assert len(selected) == 1 and selected[0].find('skipped') is None, name +assert len(cases) == 291 and sum(c.find('skipped') is not None for c in cases) == 8 +assert fixture_hashes == {name: sha(SOURCE / name) for name in files} +identities = dict(binary=sha(binary), profile=sha(profile), build_ninja=sha(BUILD / 'build.ninja')) +run('restore', ['git', 'restore', '--source=' + BUN, '--worktree', '--', OWNER], 60) +source_state(False) +save(completed=True, identity=identity, binary_sha256=identities, fixture_sha256=fixture_hashes, + passed_cases=283, skipped_cases=8, source_restored=True) +result = json.loads((OUT / 'proof.json').read_text()) +result['passed'] = True +(OUT / 'proof.json').write_text(json.dumps(result, indent=2) + '\n') diff --git a/.github/workflows/openclaw-tls-config-repair.yml b/.github/workflows/openclaw-tls-config-repair.yml new file mode 100644 index 000000000000..d4d1442de3ab --- /dev/null +++ b/.github/workflows/openclaw-tls-config-repair.yml @@ -0,0 +1,128 @@ +name: OpenClaw TLS empty-config repair proof + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-config-repair.yml + - .github/openclaw/tls-config-repair.py + - .github/openclaw/tls-config-repair.patch + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: openclaw-tls-config-repair-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + linux-arm64: + runs-on: ubuntu-24.04-arm + timeout-minutes: 300 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: .github/openclaw + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: source + ref: 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: pipeline + ref: 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e + persist-credentials: false + - uses: ./pipeline/.github/actions/setup-bun + with: + bun-version: '1.4.4' + - name: Generate the frozen native provisioner + id: provision + shell: bash + run: | + set -euo pipefail + test "$(uname -m)" = aarch64 + echo "script=$(bun pipeline/scripts/openclaw-release/release.ts provision --source source --target linux-arm64 | tail -1)" >> "$GITHUB_OUTPUT" + - name: Provision a secretless native container + id: container + env: + SCRIPT: ${{ steps.provision.outputs.script }} + shell: bash + run: | + set -euo pipefail + mkdir repair-proof + id=$(docker run --detach -e CI=true -v "$GITHUB_WORKSPACE:/work" -w /work \ + docker.io/library/debian:13@sha256:9cc080028c43b27d2074d63a5f9caf7166d731494965616c1a6d2827a004585c sleep infinity) + [[ "$id" =~ ^[0-9a-f]{64}$ ]] + echo "id=$id" >> "$GITHUB_OUTPUT" + script="${SCRIPT#"$GITHUB_WORKSPACE"/}" + docker exec -e SCRIPT="$script" "$id" sh -euc \ + 'sh "$SCRIPT" 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e openclaw-tls-repair' + docker exec "$id" bash -e -lc '. /etc/profile.d/bun-ci.sh; rustup component add rustfmt --toolchain nightly-2026-09-15' + - name: Build and exercise the unchanged TLS suites + env: + CONTAINER_ID: ${{ steps.container.outputs.id }} + shell: bash + run: | + set -euo pipefail + [[ "$CONTAINER_ID" =~ ^[0-9a-f]{64}$ ]] + tail -n +1 -F repair-proof/build.log & + tail_pid=$! + trap 'kill "$tail_pid" 2>/dev/null || true' EXIT + docker exec "$CONTAINER_ID" bash -e -lc ' + . /etc/profile.d/bun-ci.sh + for repo in source pipeline controls; do git config --global --add safe.directory "/work/$repo"; done + python3 controls/.github/openclaw/tls-config-repair.py + ' + - name: Freeze bounded evidence + if: always() + env: + CONTAINER_ID: ${{ steps.container.outputs.id }} + shell: bash + run: | + set -euo pipefail + if test -n "$CONTAINER_ID"; then + [[ "$CONTAINER_ID" =~ ^[0-9a-f]{64}$ ]] + docker stop --time 30 "$CONTAINER_ID" > /dev/null + test "$(docker inspect --format '{{.State.Running}}' "$CONTAINER_ID")" = false + fi + python3 - <<'PY' + import hashlib, json, stat, tarfile + from pathlib import Path + out = Path('repair-proof') + out.mkdir(exist_ok=True) + files = sorted(out.iterdir()) + [Path('controls/.github/openclaw') / name for name in ['tls-config-repair.py', 'tls-config-repair.patch']] + assert len(files) <= 32 and all(stat.S_ISREG(p.lstat().st_mode) for p in files) + assert sum(p.stat().st_size for p in files) <= 64 * 1024**2 + def sha(path): + with path.open('rb') as stream: return hashlib.file_digest(stream, 'sha256').hexdigest() + manifest = out / 'archive-manifest.json' + manifest.write_text(json.dumps({str(p): dict(bytes=p.stat().st_size, sha256=sha(p)) for p in files}, indent=2) + '\n') + files.append(manifest) + with tarfile.open('tls-config-repair-proof.tar.gz', 'x:gz') as archive: + for path in files: archive.add(path, arcname=str(path), recursive=False) + assert Path('tls-config-repair-proof.tar.gz').stat().st_size <= 16 * 1024**2 + Path('tls-config-repair-SHA256SUMS').write_text(sha(Path('tls-config-repair-proof.tar.gz')) + ' tls-config-repair-proof.tar.gz\n') + PY + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: tls-empty-config-linux-arm64-proof + path: | + tls-config-repair-proof.tar.gz + tls-config-repair-SHA256SUMS + if-no-files-found: error + retention-days: 7 + - name: Remove this job's container + if: always() + env: + CONTAINER_ID: ${{ steps.container.outputs.id }} + shell: bash + run: | + if test -n "$CONTAINER_ID"; then + [[ "$CONTAINER_ID" =~ ^[0-9a-f]{64}$ ]] + docker rm --force "$CONTAINER_ID" > /dev/null + fi From e0972ab8f7bd7a0f3f9192d6de52584e9d57d4e8 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 04:38:33 +0800 Subject: [PATCH 42/74] test(tls): inspect the retained shipping deinit instructions Bind static LLVM inspection to the exact successful Linux ARM64 build artifact. Reject warning-only or empty disassembly and retain bounded instructions for control-flow review without executing Bun or rebuilding it. --- .github/openclaw/tls-config-disassembly.py | 131 ++++++++++++++++++ .../openclaw-tls-config-disassembly.yml | 89 ++++++++++++ 2 files changed, 220 insertions(+) create mode 100644 .github/openclaw/tls-config-disassembly.py create mode 100644 .github/workflows/openclaw-tls-config-disassembly.yml diff --git a/.github/openclaw/tls-config-disassembly.py b/.github/openclaw/tls-config-disassembly.py new file mode 100644 index 000000000000..28488d1223a0 --- /dev/null +++ b/.github/openclaw/tls-config-disassembly.py @@ -0,0 +1,131 @@ +"""Inspect the exact retained shipping profile without executing it.""" +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import stat +import struct +import subprocess +import zipfile + +RUN = 41238111110 +COMMIT = '510fef558b464c58de1209298f638fa476d03eef' +ARTIFACT = 13056617461 +SIZE = 222623871 +DIGEST = 'c54dbe7ea29a336c76db1dd5ba491bbb117a6c211d938892c62f2f5e74251fcd' +INPUT = Path(os.environ['RUNNER_TEMP']) / 'tls-disassembly-input' +OUT = Path('tls-disassembly-proof') +OUT.mkdir(exist_ok=True) + + +def sha(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def selected(archive, wanted, maximum): + entries = archive.infolist() + assert len(entries) <= 32 + names = set() + for entry in entries: + path = PurePosixPath(entry.filename) + assert entry.filename and not path.is_absolute() and '..' not in path.parts + assert '\\' not in entry.filename and '\x00' not in entry.filename + assert str(path) == entry.filename.rstrip('/') + assert str(path) not in names + names.add(str(path)) + mode = entry.external_attr >> 16 + assert stat.S_IFMT(mode) in (0, stat.S_IFREG, stat.S_IFDIR) + assert not entry.flag_bits & 1 + matches = [entry for entry in entries if entry.filename == wanted] + assert len(matches) == 1 + entry = matches[0] + assert not entry.is_dir() and 0 < entry.file_size <= maximum + return entry + + +def extract(archive_path, member, destination, maximum): + with zipfile.ZipFile(archive_path) as archive: + entry = selected(archive, member, maximum) + with archive.open(entry) as source, destination.open('xb') as target: + count = 0 + while chunk := source.read(1024**2): + count += len(chunk) + assert count <= maximum + target.write(chunk) + assert count == entry.file_size + destination.chmod(0o600) + + +def run(name, arguments): + result = subprocess.run(arguments, stdin=subprocess.DEVNULL, + capture_output=True, timeout=120, check=False) + assert len(result.stdout) <= 32 * 1024**2 and len(result.stderr) <= 1024**2 + (OUT / (name + '.stderr')).write_bytes(result.stderr) + assert result.returncode == 0 and not result.stderr, name + return result.stdout.decode('utf-8') + + +archive = INPUT / 'artifact.zip' +assert archive.stat().st_size == SIZE and sha(archive) == DIGEST +metadata = json.loads((INPUT / 'metadata.json').read_text()) +assert metadata['id'] == ARTIFACT and metadata['name'] == 'build-linux-arm64' +assert metadata['size_in_bytes'] == SIZE and metadata['digest'] == 'sha256:' + DIGEST +assert not metadata['expired'] +assert metadata['workflow_run']['id'] == RUN and metadata['workflow_run']['head_sha'] == COMMIT +profile_zip = INPUT / 'profile.zip' +profile = INPUT / 'bun-profile.elf' +extract(archive, 'bun-linux-aarch64-profile.zip', profile_zip, 1024**3) +extract(profile_zip, 'bun-linux-aarch64-profile/bun-profile', profile, 2 * 1024**3) +with profile.open('rb') as stream: + header = stream.read(64) +assert header[:7] == b'\x7fELF\x02\x01\x01' +assert struct.unpack_from(' 0 and 0 < length <= 65536 +assembly = run('objdump', ['llvm-objdump-23', '--disassemble', '--start-address=' + hex(start), + '--stop-address=' + hex(start + length), str(profile)]) +assert re.search(r'^0*' + format(start, 'x') + r' <' + re.escape(name) + r'>:$', assembly, re.M) +instructions = [] +for line in assembly.splitlines(): + match = re.match(r'^\s*([0-9a-f]+):\s+([0-9a-f]{8})\s+(\S+)', line) + if match: + address = int(match[1], 16) + assert start <= address < start + length + assert match[3] != '' + instructions.append(address) +assert instructions and instructions[0] == start +assert instructions == sorted(set(instructions)) +notes = run('build-id', ['llvm-readelf-23', '--notes', str(profile)]) +build_ids = re.findall(r'Build ID: ([0-9a-f]+)', notes) +assert len(build_ids) == 1 +assert sha(archive) == DIGEST +receipt = dict(captured=True, control_flow_review_pending=True, producer_run=RUN, + commit=COMMIT, artifact_id=ARTIFACT, artifact_sha256=DIGEST, + profile_zip_sha256=sha(profile_zip), profile_sha256=sha(profile), + elf_machine='AArch64', build_id=build_ids[0], symbol=rows[0], + instruction_count=len(instructions), llvm_version=version) +(OUT / 'deinit.asm').write_text(assembly.replace(str(profile), 'bun-profile.elf')) +(OUT / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') +(OUT / 'symbol.txt').write_text(' '.join(rows[0]) + '\n') +(OUT / 'build-id.txt').write_text(notes) +files = sorted(OUT.iterdir()) +assert len(files) <= 12 and sum(path.stat().st_size for path in files) <= 2 * 1024**2 +(OUT / 'manifest.json').write_text(json.dumps({ + path.name: dict(bytes=path.stat().st_size, sha256=sha(path)) for path in files +}, indent=2) + '\n') +print(json.dumps(receipt)) diff --git a/.github/workflows/openclaw-tls-config-disassembly.yml b/.github/workflows/openclaw-tls-config-disassembly.yml new file mode 100644 index 000000000000..d0776b22ef90 --- /dev/null +++ b/.github/workflows/openclaw-tls-config-disassembly.yml @@ -0,0 +1,89 @@ +name: OpenClaw TLS optimized config inspection + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-config-disassembly.yml + - .github/openclaw/tls-config-disassembly.py + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: openclaw-tls-config-disassembly-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + inspect: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: | + .github/openclaw + .github/workflows + - name: Install static LLVM tools + shell: bash + run: | + set -euo pipefail + curl --fail --silent --show-error https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null + echo "deb https://apt.llvm.org/noble/ llvm-toolchain-noble-23 main" | sudo tee /etc/apt/sources.list.d/llvm.list > /dev/null + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends --no-install-suggests llvm-23 + - name: Admit the exact completed Linux ARM64 build + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + input="$RUNNER_TEMP/tls-disassembly-input" + mkdir "$input" + gh api repos/openclaw/bun/actions/runs/41238111110 > "$input/run.json" + gh api repos/openclaw/bun/actions/runs/41238111110/jobs > "$input/jobs.json" + gh api repos/openclaw/bun/actions/artifacts/13056617461 > "$input/metadata.json" + python3 - <<'PY' + import json, os + from pathlib import Path + root = Path(os.environ['RUNNER_TEMP']) / 'tls-disassembly-input' + run = json.loads((root / 'run.json').read_text()) + assert run['id'] == 41238111110 + assert run['head_sha'] == '510fef558b464c58de1209298f638fa476d03eef' + assert run['event'] == 'workflow_dispatch' + assert run['path'] == '.github/workflows/openclaw-release.yml' + assert run['head_branch'] == 'claude/fix-tls-native-transport-20261019' + jobs = json.loads((root / 'jobs.json').read_text())['jobs'] + build = [job for job in jobs if job['id'] == 124646206308 and job['name'] == 'Build linux-arm64'] + assert len(build) == 1 and build[0]['conclusion'] == 'success' + artifact = json.loads((root / 'metadata.json').read_text()) + assert artifact['id'] == 13056617461 and artifact['name'] == 'build-linux-arm64' + assert artifact['size_in_bytes'] == 222623871 and not artifact['expired'] + assert artifact['digest'] == 'sha256:c54dbe7ea29a336c76db1dd5ba491bbb117a6c211d938892c62f2f5e74251fcd' + assert artifact['workflow_run']['id'] == run['id'] + assert artifact['workflow_run']['head_sha'] == run['head_sha'] + PY + curl --location --fail --silent --show-error --proto '=https' --proto-redir '=https' \ + --max-time 300 --max-filesize 222623871 \ + -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/openclaw/bun/actions/artifacts/13056617461/zip \ + --output "$input/artifact.zip" + - name: Capture physical instructions without running Bun + shell: bash + run: | + set -euo pipefail + mkdir tls-disassembly-proof + cp controls/.github/workflows/openclaw-tls-config-disassembly.yml tls-disassembly-proof/workflow.yml + cp controls/.github/openclaw/tls-config-disassembly.py tls-disassembly-proof/controller.py + python3 controls/.github/openclaw/tls-config-disassembly.py + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: tls-config-disassembly + path: tls-disassembly-proof/ + if-no-files-found: error + retention-days: 7 From ddac89a53ab33759726f498c342e81d09df8e689 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 04:56:08 +0800 Subject: [PATCH 43/74] ci(tls): inspect retained Darwin debugger capabilities --- .github/openclaw/tls-darwin-capability.py | 177 ++++++++++++++++++ .../openclaw-tls-darwin-capability.yml | 69 +++++++ 2 files changed, 246 insertions(+) create mode 100644 .github/openclaw/tls-darwin-capability.py create mode 100644 .github/workflows/openclaw-tls-darwin-capability.yml diff --git a/.github/openclaw/tls-darwin-capability.py b/.github/openclaw/tls-darwin-capability.py new file mode 100644 index 000000000000..f3a5e7be02a0 --- /dev/null +++ b/.github/openclaw/tls-darwin-capability.py @@ -0,0 +1,177 @@ +"""Admit the consumed Darwin profile and inventory debugger capabilities without launching it.""" +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shlex +import stat +import subprocess +import sys +import zipfile + +COMMIT = '510fef558b464c58de1209298f638fa476d03eef' +RUN = 41238111110 +ARTIFACT = 13057470428 +SIZE = 232201333 +DIGEST = 'a4d8b786d7bb87d166f757e089ac1705f4ac0701504038ebbecd438c9d6e06d6' +ROOT = Path(os.environ['RUNNER_TEMP']) / 'tls-darwin-input' +OUT = Path('tls-darwin-proof') + + +def sha(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def extract(source, name, dest, maximum): + with zipfile.ZipFile(source) as archive: + entries = archive.infolist() + assert len(entries) <= 32 + names = set() + for entry in entries: + path = PurePosixPath(entry.filename) + assert entry.filename and not path.is_absolute() and '..' not in path.parts + assert '\\' not in entry.filename and '\x00' not in entry.filename + assert str(path) == entry.filename.rstrip('/') and str(path) not in names + names.add(str(path)) + assert stat.S_IFMT(entry.external_attr >> 16) in (0, stat.S_IFREG, stat.S_IFDIR) + assert not entry.flag_bits & 1 + matches = [entry for entry in entries if entry.filename == name] + assert len(matches) == 1 and not matches[0].is_dir() + entry = matches[0] + assert 0 < entry.file_size <= maximum + with archive.open(entry) as incoming, dest.open('xb') as output: + count = 0 + while chunk := incoming.read(1024**2): + count += len(chunk) + assert count <= maximum + output.write(chunk) + assert count == entry.file_size + dest.chmod(0o600) + + +def tool(name, args): + result = subprocess.run(args, stdin=subprocess.DEVNULL, capture_output=True, timeout=180) + assert len(result.stdout) <= 8 * 1024**2 and len(result.stderr) <= 1024**2 + (OUT / (name + '.stderr')).write_bytes(result.stderr) + assert result.returncode == 0, (name, result.returncode) + (OUT / (name + '.txt')).write_bytes(result.stdout) + return result.stdout.decode() + + +def prepare(): + assert os.uname().sysname == 'Darwin' and os.uname().machine == 'x86_64' + run = json.loads((ROOT / 'run.json').read_text()) + meta = json.loads((ROOT / 'metadata.json').read_text()) + assert run['id'] == RUN and run['head_sha'] == COMMIT + assert run['path'] == '.github/workflows/openclaw-release.yml' + assert run['head_branch'] == 'claude/fix-tls-native-transport-20261019' + assert run['event'] == 'workflow_dispatch' and run['status'] == 'completed' + assert run['conclusion'] == 'failure' + assert meta['id'] == ARTIFACT and meta['name'] == 'signed-darwin-x64' + assert meta['size_in_bytes'] == SIZE and meta['digest'] == 'sha256:' + DIGEST + assert not meta['expired'] and meta['workflow_run']['id'] == RUN + assert meta['workflow_run']['head_sha'] == COMMIT + archive = ROOT / 'artifact.zip' + assert archive.stat().st_size == SIZE and sha(archive) == DIGEST + profile_zip = ROOT / 'profile.zip' + extract(archive, 'bun-darwin-x64-profile.zip', profile_zip, 1024**3) + for name in ('bun-profile', 'bun-profile.dSYM'): + extract(profile_zip, 'bun-darwin-x64-profile/' + name, ROOT / name, 2 * 1024**3) + identities = {name: sha(ROOT / name) for name in ('profile.zip', 'bun-profile', 'bun-profile.dSYM')} + uuids = [] + for name in ('bun-profile', 'bun-profile.dSYM'): + text = tool(name + '-uuid', ['xcrun', 'dwarfdump', '--uuid', str(ROOT / name)]) + rows = re.findall(r'^UUID: ([0-9A-Fa-f-]{36}) \(x86_64\) ', text, re.M) + assert len(rows) == 1 and len(text.splitlines()) == 1, text + uuids.append(rows[0].lower()) + assert uuids[0] == uuids[1] + tool('lldb-version', ['xcrun', 'lldb', '--version']) + (OUT / 'inputs.json').write_text(json.dumps(dict(commit=COMMIT, run=RUN, artifact=ARTIFACT, + artifact_sha256=DIGEST, identities=identities, uuid=uuids[0], + claim='static capability inventory only; no inferior launch or GC-root proof'), indent=2) + '\n') + + +def inspect(debugger): + import lldb + debugger.SetAsync(False) + debugger.SetUseColor(False) + command = lldb.SBCommandReturnObject() + debugger.GetCommandInterpreter().HandleCommand('settings set target.load-script-from-symbol-file false', command) + assert command.Succeeded() + target = debugger.CreateTarget(str(ROOT / 'bun-profile')) + assert target.IsValid() and target.GetTriple().startswith('x86_64-apple-') + command = lldb.SBCommandReturnObject() + debugger.GetCommandInterpreter().HandleCommand('target symbols add ' + shlex.quote(str(ROOT / 'bun-profile.dSYM')), command) + assert command.Succeeded(), command.GetError() + module = target.FindModule(target.GetExecutable()) + inputs = json.loads((OUT / 'inputs.json').read_text()) + assert module.GetUUIDString().lower() == inputs['uuid'] + layouts = {} + for name in ('JSC::VM', 'JSC::Heap', 'JSC::Collector', 'JSC::PreciseAllocation', + 'JSC::VerifierSlotVisitor', 'JSC::VerifierSlotVisitor::PreciseAllocationData', + 'JSC::ConservativeRoots', 'JSC::CurrentThreadState', 'JSC::RegisterState', + 'JSC::JSGlobalObject', 'JSC::JSWeakObjectRef', 'JSC::ProtoCallFrame', + 'WTF::StackTrace', 'Zig::JSFFIFunction', + 'bun_runtime::socket::socket_body::DuplexUpgradeContext', + 'bun_runtime::socket::upgraded_duplex::UpgradedDuplex'): + found = target.FindTypes(name) + assert found.GetSize() <= 128, name + values = [] + for i in range(found.GetSize()): + value = found.GetTypeAtIndex(i).GetCanonicalType() + assert value.GetNumberOfFields() <= 512, name + fields = [value.GetFieldAtIndex(j) for j in range(value.GetNumberOfFields())] + values.append(dict(name=value.GetName(), complete=value.IsTypeComplete(), bytes=value.GetByteSize(), + fields=[dict(name=f.GetName(), offset=f.GetOffsetInBytes(), type=f.GetType().GetName(), + bytes=f.GetType().GetByteSize()) for f in fields])) + layouts[name] = values + functions = {} + for name in ('JSC::constructWeakRef', 'JSC__VM__runGC', 'posix_spawn_bun', + 'JSC::MachineThreads::gatherConservativeRoots', 'JSC::ConservativeRoots::genericAddSpan', + 'JSC::ConservativeRoots::genericAddPointer', 'JSC::Interpreter::executeCall', + 'Bun__UpgradedDuplex__onWriteComplete', + 'bun_runtime::socket::socket_body::DuplexUpgradeContext::run_event', + 'bun_runtime::socket::socket_body::DuplexUpgradeContext::deinit'): + contexts = module.FindFunctions(name, lldb.eFunctionNameTypeAny) + assert contexts.GetSize() <= 128, name + parents = {} + for i in range(contexts.GetSize()): + context = contexts.GetContextAtIndex(i) + function = context.GetFunction() + address = function.GetStartAddress() if function.IsValid() else context.GetSymbol().GetStartAddress() + if not address.IsValid(): + continue + resolved = function.GetName() if function.IsValid() else context.GetSymbol().GetName() + key = address.GetFileAddress() + if key in parents: + continue + instructions = function.GetInstructions(target) if function.IsValid() else context.GetSymbol().GetInstructions(target) + assert instructions.GetSize() <= 16384, name + rows = [] + for j in range(instructions.GetSize()): + insn = instructions.GetInstructionAtIndex(j) + rows.append(dict(address=insn.GetAddress().GetFileAddress(), mnemonic=insn.GetMnemonic(target), + operands=insn.GetOperands(target), comment=insn.GetComment(target))) + parents[key] = dict(name=resolved, address=key, instructions=rows) + functions[name] = list(parents.values()) + assert not target.GetProcess().IsValid(), 'static inventory launched an inferior' + (OUT / 'capabilities.json').write_text(json.dumps(dict(layouts=layouts, functions=functions, + inferior_launched=False, root_capture_ready=False), indent=2) + '\n') + + +if __name__ == '__lldb__': + import lldb + inspect(lldb.debugger) +elif __name__ == '__main__': + OUT.mkdir(exist_ok=True) + if sys.argv[1] == 'prepare': + prepare() + else: + assert sys.argv[1] == 'seal' and (OUT / 'capabilities.json').is_file() + files = sorted(OUT.iterdir()) + assert len(files) <= 24 and sum(p.stat().st_size for p in files) <= 32 * 1024**2 + assert all(stat.S_ISREG(p.lstat().st_mode) for p in files) + (OUT / 'manifest.json').write_text(json.dumps({p.name: dict(bytes=p.stat().st_size, + sha256=sha(p)) for p in files}, indent=2) + '\n') diff --git a/.github/workflows/openclaw-tls-darwin-capability.yml b/.github/workflows/openclaw-tls-darwin-capability.yml new file mode 100644 index 000000000000..160d01b108a6 --- /dev/null +++ b/.github/workflows/openclaw-tls-darwin-capability.yml @@ -0,0 +1,69 @@ +name: OpenClaw TLS Darwin debugger capability + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-darwin-capability.yml + - .github/openclaw/tls-darwin-capability.py + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: openclaw-tls-darwin-capability-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + inspect: + runs-on: macos-15-intel + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: | + .github/openclaw + .github/workflows + - name: Download the exact consumed signed artifact + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + input="$RUNNER_TEMP/tls-darwin-input" + mkdir "$input" + gh api repos/openclaw/bun/actions/runs/41238111110 > "$input/run.json" + gh api repos/openclaw/bun/actions/artifacts/13057470428 > "$input/metadata.json" + curl --location --fail --silent --show-error --proto '=https' --proto-redir '=https' \ + --max-time 300 --max-filesize 232201333 \ + -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/openclaw/bun/actions/artifacts/13057470428/zip \ + --output "$input/artifact.zip" + - name: Admit matched Mach-O and dSYM identities + shell: bash + run: | + set -euo pipefail + mkdir tls-darwin-proof + cp controls/.github/openclaw/tls-darwin-capability.py tls-darwin-proof/controller.py + cp controls/.github/workflows/openclaw-tls-darwin-capability.yml tls-darwin-proof/workflow.yml + python3 controls/.github/openclaw/tls-darwin-capability.py prepare + - name: Inventory static debugger types and physical scanner functions + shell: bash + run: | + set -euo pipefail + xcrun lldb --batch --no-lldbinit --no-use-colors \ + -o 'settings set target.load-script-from-symbol-file false' \ + -o 'script import runpy; runpy.run_path("controls/.github/openclaw/tls-darwin-capability.py", run_name="__lldb__")' \ + > tls-darwin-proof/debugger.log 2>&1 + python3 controls/.github/openclaw/tls-darwin-capability.py seal + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: tls-darwin-debugger-capability + path: tls-darwin-proof/ + if-no-files-found: error + retention-days: 7 From 313fefee3e3dc3be5e27a79726649c9f48e60b74 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 05:06:38 +0800 Subject: [PATCH 44/74] ci(tls): reconstruct Darwin type layouts with frozen compiler --- .github/openclaw/tls-darwin-capability.py | 35 ++-- .github/openclaw/tls-darwin-types.py | 153 +++++++++++++++++ .../workflows/openclaw-tls-darwin-types.yml | 161 ++++++++++++++++++ 3 files changed, 334 insertions(+), 15 deletions(-) create mode 100644 .github/openclaw/tls-darwin-types.py create mode 100644 .github/workflows/openclaw-tls-darwin-types.yml diff --git a/.github/openclaw/tls-darwin-capability.py b/.github/openclaw/tls-darwin-capability.py index f3a5e7be02a0..c80215b53c64 100644 --- a/.github/openclaw/tls-darwin-capability.py +++ b/.github/openclaw/tls-darwin-capability.py @@ -93,21 +93,7 @@ def prepare(): claim='static capability inventory only; no inferior launch or GC-root proof'), indent=2) + '\n') -def inspect(debugger): - import lldb - debugger.SetAsync(False) - debugger.SetUseColor(False) - command = lldb.SBCommandReturnObject() - debugger.GetCommandInterpreter().HandleCommand('settings set target.load-script-from-symbol-file false', command) - assert command.Succeeded() - target = debugger.CreateTarget(str(ROOT / 'bun-profile')) - assert target.IsValid() and target.GetTriple().startswith('x86_64-apple-') - command = lldb.SBCommandReturnObject() - debugger.GetCommandInterpreter().HandleCommand('target symbols add ' + shlex.quote(str(ROOT / 'bun-profile.dSYM')), command) - assert command.Succeeded(), command.GetError() - module = target.FindModule(target.GetExecutable()) - inputs = json.loads((OUT / 'inputs.json').read_text()) - assert module.GetUUIDString().lower() == inputs['uuid'] +def type_layouts(target): layouts = {} for name in ('JSC::VM', 'JSC::Heap', 'JSC::Collector', 'JSC::PreciseAllocation', 'JSC::VerifierSlotVisitor', 'JSC::VerifierSlotVisitor::PreciseAllocationData', @@ -127,6 +113,25 @@ def inspect(debugger): fields=[dict(name=f.GetName(), offset=f.GetOffsetInBytes(), type=f.GetType().GetName(), bytes=f.GetType().GetByteSize()) for f in fields])) layouts[name] = values + return layouts + + +def inspect(debugger): + import lldb + debugger.SetAsync(False) + debugger.SetUseColor(False) + command = lldb.SBCommandReturnObject() + debugger.GetCommandInterpreter().HandleCommand('settings set target.load-script-from-symbol-file false', command) + assert command.Succeeded() + target = debugger.CreateTarget(str(ROOT / 'bun-profile')) + assert target.IsValid() and target.GetTriple().startswith('x86_64-apple-') + command = lldb.SBCommandReturnObject() + debugger.GetCommandInterpreter().HandleCommand('target symbols add ' + shlex.quote(str(ROOT / 'bun-profile.dSYM')), command) + assert command.Succeeded(), command.GetError() + module = target.FindModule(target.GetExecutable()) + inputs = json.loads((OUT / 'inputs.json').read_text()) + assert module.GetUUIDString().lower() == inputs['uuid'] + layouts = type_layouts(target) functions = {} for name in ('JSC::constructWeakRef', 'JSC__VM__runGC', 'posix_spawn_bun', 'JSC::MachineThreads::gatherConservativeRoots', 'JSC::ConservativeRoots::genericAddSpan', diff --git a/.github/openclaw/tls-darwin-types.py b/.github/openclaw/tls-darwin-types.py new file mode 100644 index 000000000000..77afec8b59fa --- /dev/null +++ b/.github/openclaw/tls-darwin-types.py @@ -0,0 +1,153 @@ +"""Reconstruct shipping header layouts through the frozen compiler owner; never link or run them.""" +import hashlib +import json +import os +from pathlib import Path +import runpy +import struct +import subprocess +import sys + +SOURCE = Path('/work/source') +BUILD = SOURCE / 'build/openclaw-darwin-x64' +OUT = Path('/work/tls-darwin-types') +COMMIT = '510fef558b464c58de1209298f638fa476d03eef' +ENGINE = 'b82bc917e99110b61c3a21202d6def7457aa194c' +ENGINE_SHA = 'd59f456e79045c98420a0fc24ca624b33654f7d5fc8133e740b78eaec160b0c5' +REQUIRED = ('JSC::VM', 'JSC::Heap', 'JSC::PreciseAllocation', 'JSC::VerifierSlotVisitor', + 'JSC::ConservativeRoots', 'JSC::CurrentThreadState', 'JSC::RegisterState', + 'JSC::JSGlobalObject', 'JSC::JSWeakObjectRef', 'JSC::ProtoCallFrame') + + +def sha(path): + digest = hashlib.sha256() + with path.open('rb') as stream: + while chunk := stream.read(1024**2): + digest.update(chunk) + return digest.hexdigest() + + +def write(name, value): + (OUT / name).write_text(json.dumps(value, indent=2) + '\n') + + +def run(name, args, cwd=SOURCE, timeout=600): + with (OUT / (name + '.log')).open('x') as stream: + subprocess.run(args, cwd=cwd, stdin=subprocess.DEVNULL, stdout=stream, + stderr=subprocess.STDOUT, check=True, timeout=timeout) + assert (OUT / (name + '.log')).stat().st_size <= 8 * 1024**2 + + +def build(): + assert os.uname().machine == 'x86_64' and os.uname().sysname == 'Linux' + assert subprocess.check_output(['git', '-C', str(SOURCE), 'rev-parse', 'HEAD'], text=True).strip() == COMMIT + assert not subprocess.check_output(['git', '-C', str(SOURCE), 'diff', '--name-only', 'HEAD'], text=True) + manifest = SOURCE / 'scripts/build/deps/webkit-artifacts.json' + spec = json.loads(manifest.read_text()) + artifact = spec['artifacts']['bun-webkit-macos-amd64-lto.tar.gz'] + assert spec['version'] == ENGINE and artifact['sha256'] == ENGINE_SHA + assert artifact['url'] == f'https://github.com/openclaw/WebKit/releases/download/autobuild-{ENGINE}/bun-webkit-macos-amd64-lto.tar.gz' + compiler = subprocess.check_output(['clang++', '--version'], text=True) + assert '23.1.1' in compiler + argv = ['bun', 'scripts/build.ts', '--profile=release', '--ci=on', '--os=darwin', + '--arch=x64', '--build-dir=' + str(BUILD), '--configure-only'] + run('configure', argv) + records = json.loads(subprocess.check_output(['ninja', '-C', str(BUILD), '-t', 'compdb', 'pch'], text=True)) + assert len(records) == 1 + pch = records[0] + assert Path(pch['output']).suffix == '.pch' + pch_path = (Path(pch['directory']) / pch['output']).resolve() + assert pch_path.parent == BUILD / 'pch' + run('prepare-headers', ['ninja', '-C', str(BUILD), '-j2', pch['output']], timeout=1800) + database = BUILD / 'compile_commands.json' + records = json.loads(database.read_text()) + chosen = [record for record in records if record['file'] == str(SOURCE / 'src/jsc/bindings/BunObject.cpp')] + assert len(chosen) == 1 + original = chosen[0]['arguments'] + assert '-g1' in original and '-O3' in original and '-flto=thin' in original + assert original[-4:-1] == ['-c', chosen[0]['file'], '-o'] + assert original.count('-include-pch') == 1 + pch_index = original.index('-include-pch') + assert (Path(chosen[0]['directory']) / original[pch_index + 1]).resolve() == pch_path + flags = original[:pch_index] + original[pch_index + 2:-4] + assert original.count('-isysroot') == 1 + sdk = Path(original[original.index('-isysroot') + 1]) + sdk_settings = sdk / 'SDKSettings.json' + assert json.loads(sdk_settings.read_text())['Version'] == '26.5' + includes = [Path(arg[2:]) for arg in original if arg.startswith('-I')] + roots = [p for p in includes if p.name == 'include' and ENGINE_SHA in str(p)] + assert len(roots) == 1 + headers = roots[0] + config = headers / 'cmakeconfig.h' + assert ENGINE in config.read_text() + required_headers = ['VerifierSlotVisitor.h', 'ConservativeRoots.h', 'MachineStackMarker.h', + 'RegisterState.h', 'PreciseAllocation.h', 'JSWeakObjectRef.h', 'ProtoCallFrame.h', 'VM.h'] + assert all((headers / 'JavaScriptCore' / name).is_file() for name in required_headers) + witness = OUT / 'types.cpp' + wrapper = pch_path.with_suffix('') + assert wrapper.is_file() and 'root-pch.h' in wrapper.name + witness.write_text('#include "' + str(wrapper) + '"\n' + ''.join( + '#include \n' for name in required_headers) + + '\nstatic_assert(sizeof(JSC::VM) > 0);\nstatic_assert(sizeof(JSC::ConservativeRoots) > 0);\n') + output = OUT / 'types.o' + # This object supplies layouts only. No runtime code or shipping profile is replaced. + replacement = {'-g1': '-g', '-flto=thin': '-fno-lto', + '-fwhole-program-vtables': '-fno-whole-program-vtables'} + adapted = [replacement.get(arg, arg) for arg in flags] + adapted += ['-fstandalone-debug', '-fno-eliminate-unused-debug-types', '-c', str(witness), '-o', str(output)] + write('compile-command.json', dict(directory=chosen[0]['directory'], original=original, witness=adapted, + variance='full standalone type metadata, no LTO or whole-program vtables, original PCH wrapper included as text, synthetic include-only translation unit; never linked or executed')) + run('compile-types', adapted, cwd=Path(chosen[0]['directory']), timeout=600) + assert 0 < output.stat().st_size <= 64 * 1024**2 + with output.open('rb') as stream: + header = stream.read(32) + assert struct.unpack_from(' 0 and item['fields'] for item in layouts[name])] + assert not target.GetProcess().IsValid() + (out / 'layouts.json').write_text(json.dumps(dict(layouts=layouts, missing=missing, + inferior_launched=False, root_capture_ready=False), indent=2) + '\n') + assert not missing, missing + + +if __name__ == '__lldb__': + import lldb + inspect(lldb.debugger) +elif __name__ == '__main__': + OUT.mkdir(exist_ok=True) + assert sys.argv[1] == 'build' + build() diff --git a/.github/workflows/openclaw-tls-darwin-types.yml b/.github/workflows/openclaw-tls-darwin-types.yml new file mode 100644 index 000000000000..274bb2f5a55e --- /dev/null +++ b/.github/workflows/openclaw-tls-darwin-types.yml @@ -0,0 +1,161 @@ +name: OpenClaw TLS Darwin type witness + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-darwin-types.yml + - .github/openclaw/tls-darwin-types.py + - .github/openclaw/tls-darwin-capability.py + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: openclaw-tls-darwin-types-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + compile: + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: | + .github/openclaw + .github/workflows + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: source + ref: 510fef558b464c58de1209298f638fa476d03eef + persist-credentials: false + - uses: ./source/.github/actions/setup-bun + with: + bun-version: "1.4.4" + - name: Provision the frozen shipping compiler and SDK + shell: bash + run: | + set -euo pipefail + mkdir tls-darwin-types + script=$(bun source/scripts/openclaw-release/release.ts provision --source source --target darwin-x64 | tail -1) + script=${script#"$GITHUB_WORKSPACE"/} + test -f "$script" + id=$(docker run --detach -e CI=true -v "$GITHUB_WORKSPACE:/work" -w /work \ + docker.io/library/debian:13@sha256:9cc080028c43b27d2074d63a5f9caf7166d731494965616c1a6d2827a004585c \ + sleep infinity) + [[ "$id" =~ ^[0-9a-f]{64}$ ]] + echo "$id" > "$RUNNER_TEMP/tls-types-container" + docker exec "$id" sh "$script" 510fef558b464c58de1209298f638fa476d03eef openclaw-release + - name: Emit a separate type object using the native compile owner + shell: bash + run: | + set -euo pipefail + id=$(cat "$RUNNER_TEMP/tls-types-container") + [[ "$id" =~ ^[0-9a-f]{64}$ ]] + docker exec "$id" bash -euo pipefail -c ' + . /etc/profile.d/bun-ci.sh + git config --global --add safe.directory /work/source + python3 controls/.github/openclaw/tls-darwin-types.py build + ' + - name: Seal bounded type metadata and build evidence + if: always() + shell: bash + run: | + set -euo pipefail + if test -f "$RUNNER_TEMP/tls-types-container"; then + id=$(cat "$RUNNER_TEMP/tls-types-container") + [[ "$id" =~ ^[0-9a-f]{64}$ ]] + docker stop --time 15 "$id" > /dev/null + test "$(docker inspect --format '{{.State.Running}}' "$id")" = false + fi + sudo chown -R "$(id -u):$(id -g)" tls-darwin-types + cp controls/.github/openclaw/tls-darwin-types.py tls-darwin-types/controller.py + cp controls/.github/openclaw/tls-darwin-capability.py tls-darwin-types/inventory.py + cp controls/.github/workflows/openclaw-tls-darwin-types.yml tls-darwin-types/workflow.yml + python3 - <<'PY' + import hashlib, json, stat + from pathlib import Path + root = Path('tls-darwin-types') + files = sorted(root.iterdir()) + assert len(files) <= 24 and all(stat.S_ISREG(p.lstat().st_mode) for p in files) + assert sum(p.stat().st_size for p in files) <= 96 * 1024**2 + (root / 'manifest.json').write_text(json.dumps({p.name: dict(bytes=p.stat().st_size, + sha256=hashlib.sha256(p.read_bytes()).hexdigest()) for p in files}, indent=2) + '\n') + PY + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() && hashFiles('tls-darwin-types/manifest.json') != '' + with: + name: tls-darwin-type-object + path: tls-darwin-types/ + if-no-files-found: error + retention-days: 7 + - name: Remove this job's compiler container + if: always() + shell: bash + run: | + if test -f "$RUNNER_TEMP/tls-types-container"; then + id=$(cat "$RUNNER_TEMP/tls-types-container") + [[ "$id" =~ ^[0-9a-f]{64}$ ]] && docker rm --force "$id" > /dev/null + fi + + inspect: + needs: compile + runs-on: macos-15-intel + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: .github/openclaw + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: tls-darwin-type-object + path: tls-darwin-types + - name: Admit every type-object artifact entry + shell: bash + run: | + set -euo pipefail + python3 - <<'PY' + import hashlib, json, stat + from pathlib import Path + root = Path('tls-darwin-types') + manifest = json.loads((root / 'manifest.json').read_text()) + assert set(manifest) == {p.name for p in root.iterdir()} - {'manifest.json'} + for name, entry in manifest.items(): + p = root / name + assert p.name == name and stat.S_ISREG(p.lstat().st_mode) + assert p.stat().st_size == entry['bytes'] and hashlib.sha256(p.read_bytes()).hexdigest() == entry['sha256'] + assert (root / 'controller.py').read_bytes() == Path('controls/.github/openclaw/tls-darwin-types.py').read_bytes() + assert (root / 'inventory.py').read_bytes() == Path('controls/.github/openclaw/tls-darwin-capability.py').read_bytes() + PY + - name: Admit complete layouts without launching an inferior + shell: bash + run: | + set -euo pipefail + xcrun lldb --batch --no-lldbinit --no-use-colors \ + -o 'settings set target.load-script-from-symbol-file false' \ + -o 'script import runpy; _ = runpy.run_path("controls/.github/openclaw/tls-darwin-types.py", run_name="__lldb__")' \ + > tls-darwin-types/debugger.log 2>&1 + test -f tls-darwin-types/layouts.json + python3 - <<'PY' + import json + from pathlib import Path + result = json.loads(Path('tls-darwin-types/layouts.json').read_text()) + assert result['missing'] == [] and result['inferior_launched'] is False + PY + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: tls-darwin-type-layouts + path: | + tls-darwin-types/layouts.json + tls-darwin-types/debugger.log + if-no-files-found: error + retention-days: 7 From 38478f2d32a7496a6f53fec84e7c948ff4093b1a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 05:11:45 +0800 Subject: [PATCH 45/74] fix(ci): match Darwin type probe to shipping build host --- .github/openclaw/tls-darwin-types.py | 2 +- .github/workflows/openclaw-tls-darwin-types.yml | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/openclaw/tls-darwin-types.py b/.github/openclaw/tls-darwin-types.py index 77afec8b59fa..c06944c42aa2 100644 --- a/.github/openclaw/tls-darwin-types.py +++ b/.github/openclaw/tls-darwin-types.py @@ -39,7 +39,7 @@ def run(name, args, cwd=SOURCE, timeout=600): def build(): - assert os.uname().machine == 'x86_64' and os.uname().sysname == 'Linux' + assert os.uname().machine == 'aarch64' and os.uname().sysname == 'Linux' assert subprocess.check_output(['git', '-C', str(SOURCE), 'rev-parse', 'HEAD'], text=True).strip() == COMMIT assert not subprocess.check_output(['git', '-C', str(SOURCE), 'diff', '--name-only', 'HEAD'], text=True) manifest = SOURCE / 'scripts/build/deps/webkit-artifacts.json' diff --git a/.github/workflows/openclaw-tls-darwin-types.yml b/.github/workflows/openclaw-tls-darwin-types.yml index 274bb2f5a55e..324f3b7e0e4c 100644 --- a/.github/workflows/openclaw-tls-darwin-types.yml +++ b/.github/workflows/openclaw-tls-darwin-types.yml @@ -18,7 +18,7 @@ concurrency: jobs: compile: - runs-on: ubuntu-24.04 + runs-on: ubuntu-24.04-arm timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -50,7 +50,8 @@ jobs: sleep infinity) [[ "$id" =~ ^[0-9a-f]{64}$ ]] echo "$id" > "$RUNNER_TEMP/tls-types-container" - docker exec "$id" sh "$script" 510fef558b464c58de1209298f638fa476d03eef openclaw-release + docker exec "$id" sh "$script" 510fef558b464c58de1209298f638fa476d03eef openclaw-release \ + 2>&1 | tee tls-darwin-types/provision.log - name: Emit a separate type object using the native compile owner shell: bash run: | From c7afff80e2edb22871aaea8893fe94868c4c4a95 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 05:17:16 +0800 Subject: [PATCH 46/74] fix(ci): retain native macro arguments in Darwin type probe --- .github/openclaw/tls-darwin-types.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/openclaw/tls-darwin-types.py b/.github/openclaw/tls-darwin-types.py index c06944c42aa2..a736731f8066 100644 --- a/.github/openclaw/tls-darwin-types.py +++ b/.github/openclaw/tls-darwin-types.py @@ -4,6 +4,7 @@ import os from pathlib import Path import runpy +import shlex import struct import subprocess import sys @@ -69,7 +70,15 @@ def build(): assert original.count('-include-pch') == 1 pch_index = original.index('-include-pch') assert (Path(chosen[0]['directory']) / original[pch_index + 1]).resolve() == pch_path - flags = original[:pch_index] + original[pch_index + 2:-4] + assert original[pch_index + 2:-4] == [] + # The compilation database retains shell escapes in macro definitions. + # Admit the actual successful Ninja PCH argv, which has the same shared flags. + pch_argv = shlex.split(pch['command']) + flags = pch_argv[:pch_argv.index('-Winvalid-pch')] + assert len(flags) == pch_index + for recorded, actual in zip(original[:pch_index], flags): + decoded = shlex.split(recorded) if recorded.startswith('-D') else [recorded] + assert decoded == [actual], (recorded, actual) assert original.count('-isysroot') == 1 sdk = Path(original[original.index('-isysroot') + 1]) sdk_settings = sdk / 'SDKSettings.json' @@ -95,7 +104,8 @@ def build(): '-fwhole-program-vtables': '-fno-whole-program-vtables'} adapted = [replacement.get(arg, arg) for arg in flags] adapted += ['-fstandalone-debug', '-fno-eliminate-unused-debug-types', '-c', str(witness), '-o', str(output)] - write('compile-command.json', dict(directory=chosen[0]['directory'], original=original, witness=adapted, + write('compile-command.json', dict(directory=chosen[0]['directory'], original=original, + successful_pch_command=pch['command'], canonical_flags=flags, witness=adapted, variance='full standalone type metadata, no LTO or whole-program vtables, original PCH wrapper included as text, synthetic include-only translation unit; never linked or executed')) run('compile-types', adapted, cwd=Path(chosen[0]['directory']), timeout=600) assert 0 < output.stat().st_size <= 64 * 1024**2 From ccfd29c02bbde1b3b2126510e439c3e948e24ccf Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 05:32:02 +0800 Subject: [PATCH 47/74] ci(tls): inspect physical Darwin guards and retain verifier layout --- .github/openclaw/tls-darwin-disassembly.py | 168 ++++++++++++++++++ .github/openclaw/tls-darwin-types.py | 6 +- .../openclaw-tls-darwin-disassembly.yml | 69 +++++++ 3 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 .github/openclaw/tls-darwin-disassembly.py create mode 100644 .github/workflows/openclaw-tls-darwin-disassembly.yml diff --git a/.github/openclaw/tls-darwin-disassembly.py b/.github/openclaw/tls-darwin-disassembly.py new file mode 100644 index 000000000000..1458e4f1ed8d --- /dev/null +++ b/.github/openclaw/tls-darwin-disassembly.py @@ -0,0 +1,168 @@ +"""Capture bounded physical functions from the exact consumed Darwin profile; never run it.""" +import json +import os +from pathlib import Path +import re +import runpy +import stat +import struct +import subprocess +import uuid + +ROOT = Path(os.environ['RUNNER_TEMP']) / 'tls-darwin-input' +OUT = Path('tls-darwin-disassembly') +LIBRARY = runpy.run_path(str(Path(__file__).with_name('tls-darwin-capability.py')), + run_name='archive_library') +COMMIT = LIBRARY['COMMIT'] +RUN = LIBRARY['RUN'] +ARTIFACT = LIBRARY['ARTIFACT'] +SIZE = LIBRARY['SIZE'] +DIGEST = LIBRARY['DIGEST'] +sha = LIBRARY['sha'] +extract = LIBRARY['extract'] + + +def macho(path, filetype): + with path.open('rb') as stream: + header = stream.read(32) + magic, cpu, subtype, kind, count, size, flags, reserved = struct.unpack('<8I', header) + assert (magic, cpu, subtype, kind) == (0xfeedfacf, 0x01000007, 3, filetype) + assert 0 < count <= 1024 and 0 < size <= 1024**2 + data = stream.read(size) + assert len(data) == size + offset, identities, sections = 0, [], [] + for _ in range(count): + assert offset + 8 <= size + command, length = struct.unpack_from('= 8 and length % 8 == 0 and offset + length <= size + if command == 0x1b: # LC_UUID + assert length == 24 + identities.append(str(uuid.UUID(bytes=data[offset + 8:offset + 24]))) + if command == 0x19: # LC_SEGMENT_64, section_64 records follow its 72-byte header. + assert length >= 72 + segment = data[offset + 8:offset + 24].rstrip(b'\0') + nsects = struct.unpack_from(':$', text, re.M) + cursor, count = start, 0 + for line in text.splitlines(): + if not re.match(r'^\s*[0-9a-f]+:', line): + continue + match = re.fullmatch(r'\s*([0-9a-f]+):\s+((?:[0-9a-f]{2}\s+)+)(\S+)(.*)', line) + assert match, line + address = int(match[1], 16) + width = len(match[2].split()) + assert address == cursor and 0 < width <= 15 and address + width <= end, line + assert match[3] not in ('', '.byte', '.long', '.short'), line + cursor += width + count += 1 + assert count and cursor == end + return count + + +def main(): + OUT.mkdir(exist_ok=True) + run = json.loads((ROOT / 'run.json').read_text()) + meta = json.loads((ROOT / 'metadata.json').read_text()) + assert run['id'] == RUN and run['head_sha'] == COMMIT + assert run['path'] == '.github/workflows/openclaw-release.yml' + assert run['head_branch'] == 'claude/fix-tls-native-transport-20261019' + assert run['event'] == 'workflow_dispatch' and run['status'] == 'completed' + assert run['conclusion'] == 'failure' + assert meta['id'] == ARTIFACT and meta['name'] == 'signed-darwin-x64' + assert meta['size_in_bytes'] == SIZE and meta['digest'] == 'sha256:' + DIGEST + assert not meta['expired'] and meta['workflow_run']['id'] == RUN + assert meta['workflow_run']['head_sha'] == COMMIT + archive = ROOT / 'artifact.zip' + assert archive.stat().st_size == SIZE and sha(archive) == DIGEST + profile_zip = ROOT / 'profile.zip' + profile, dsym = ROOT / 'bun-profile', ROOT / 'bun-profile.dSYM' + extract(archive, 'bun-darwin-x64-profile.zip', profile_zip, 1024**3) + for path in (profile, dsym): + extract(profile_zip, 'bun-darwin-x64-profile/' + path.name, path, 2 * 1024**3) + identity, sections = macho(profile, 2) + assert identity == macho(dsym, 10)[0] == '4c4c4432-5555-3144-a125-d2b0a4297913' + assert len(sections) == 1 + version = tool('llvm-version', ['llvm-objdump-23', '--version']) + assert re.search(r'LLVM version 23\.', version) + nm_args = ['llvm-nm-23', '--defined-only', '--format=posix', '--radix=x', '--no-demangle'] + symbols = select_symbols(tool('nm-profile', nm_args + [str(profile)])) + symbol_owner = 'bun-profile' + if set(symbols) != {'deinit', 'run_event'}: + symbols = select_symbols(tool('nm-dsym', nm_args + [str(dsym)])) + symbol_owner = 'bun-profile.dSYM' + assert set(symbols) == {'deinit', 'run_event'}, 'required physical function missing' + starts_text = tool('function-starts', ['llvm-objdump-23', '--macho', '--no-leading-headers', + '--function-starts=addrs', str(profile)]) + lines = starts_text.splitlines() + assert lines and all(re.fullmatch(r'[0-9a-f]{16}', line) for line in lines) + starts = [int(line, 16) for line in lines] + assert starts == sorted(set(starts)) + for name, item in symbols.items(): + start = item['start'] + assert start in starts + index = starts.index(start) + assert index + 1 < len(starts) + end = starts[index + 1] + assert sections[0][0] <= start < end <= sections[0][1] and end - start <= 65536 + # The generic disassembler honors numeric bounds. The --macho disassembler does not. + assembly = tool(name, ['llvm-objdump-23', '--disassemble', '--disassemble-zeroes', '--show-all-symbols', + '--start-address=' + hex(start), '--stop-address=' + hex(end), str(profile)]) + item.update(end=end, bytes=end - start, + instruction_count=instructions(assembly, item['symbol'], start, end)) + path = OUT / (name + '.asm') + path.write_text(assembly.replace(str(profile), 'bun-profile')) + item['assembly_sha256'] = sha(path) + receipt = dict(commit=COMMIT, producer_run=RUN, artifact_id=ARTIFACT, artifact_sha256=DIGEST, + identities={p.name: sha(p) for p in (profile_zip, profile, dsym)}, uuid=identity, + sections=sections, symbol_owner=symbol_owner, functions=symbols, llvm_version=version, + inferior_launched=False, control_flow_review_pending=True, gc_root_proven=False) + (OUT / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + files = sorted(OUT.iterdir()) + assert len(files) <= 24 and all(stat.S_ISREG(p.lstat().st_mode) for p in files) + assert sum(p.stat().st_size for p in files) <= 32 * 1024**2 + (OUT / 'manifest.json').write_text(json.dumps({p.name: dict(bytes=p.stat().st_size, sha256=sha(p)) + for p in files}, indent=2) + '\n') + print(json.dumps(receipt)) + + +if __name__ == '__main__': + main() diff --git a/.github/openclaw/tls-darwin-types.py b/.github/openclaw/tls-darwin-types.py index a736731f8066..668658d1743f 100644 --- a/.github/openclaw/tls-darwin-types.py +++ b/.github/openclaw/tls-darwin-types.py @@ -97,7 +97,11 @@ def build(): assert wrapper.is_file() and 'root-pch.h' in wrapper.name witness.write_text('#include "' + str(wrapper) + '"\n' + ''.join( '#include \n' for name in required_headers) + - '\nstatic_assert(sizeof(JSC::VM) > 0);\nstatic_assert(sizeof(JSC::ConservativeRoots) > 0);\n') + '\nstatic_assert(sizeof(JSC::VM) > 0);\nstatic_assert(sizeof(JSC::ConservativeRoots) > 0);\n' + 'static_assert(sizeof(JSC::VerifierSlotVisitor) > 0);\n' + # An external vtable leaves only a declaration unless this object witnesses the layout. + '[[gnu::used, gnu::noinline]] static void retainVerifierLayout(JSC::VerifierSlotVisitor& visitor)\n' + '{ __asm__ __volatile__("" : : "m"(visitor) : "memory"); }\n') output = OUT / 'types.o' # This object supplies layouts only. No runtime code or shipping profile is replaced. replacement = {'-g1': '-g', '-flto=thin': '-fno-lto', diff --git a/.github/workflows/openclaw-tls-darwin-disassembly.yml b/.github/workflows/openclaw-tls-darwin-disassembly.yml new file mode 100644 index 000000000000..504923e93f31 --- /dev/null +++ b/.github/workflows/openclaw-tls-darwin-disassembly.yml @@ -0,0 +1,69 @@ +name: OpenClaw TLS Darwin physical config inspection + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-darwin-disassembly.yml + - .github/openclaw/tls-darwin-disassembly.py + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: openclaw-tls-darwin-disassembly-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + inspect: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: | + .github/openclaw + .github/workflows + - name: Install static LLVM tools + shell: bash + run: | + set -euo pipefail + curl --fail --silent --show-error https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null + echo "deb https://apt.llvm.org/noble/ llvm-toolchain-noble-23 main" | sudo tee /etc/apt/sources.list.d/llvm.list > /dev/null + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends --no-install-suggests llvm-23 + - name: Download the exact consumed signed Darwin artifact + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + input="$RUNNER_TEMP/tls-darwin-input" + mkdir "$input" + gh api repos/openclaw/bun/actions/runs/41238111110 > "$input/run.json" + gh api repos/openclaw/bun/actions/artifacts/13057470428 > "$input/metadata.json" + curl --location --fail --silent --show-error --proto '=https' --proto-redir '=https' \ + --max-time 300 --max-filesize 232201333 \ + -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/openclaw/bun/actions/artifacts/13057470428/zip \ + --output "$input/artifact.zip" + - name: Capture bounded functions without launching Bun + shell: bash + run: | + set -euo pipefail + mkdir tls-darwin-disassembly + cp controls/.github/workflows/openclaw-tls-darwin-disassembly.yml tls-darwin-disassembly/workflow.yml + cp controls/.github/openclaw/tls-darwin-disassembly.py tls-darwin-disassembly/controller.py + cp controls/.github/openclaw/tls-darwin-capability.py tls-darwin-disassembly/archive.py + python3 controls/.github/openclaw/tls-darwin-disassembly.py + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: tls-darwin-disassembly + path: tls-darwin-disassembly/ + if-no-files-found: error + retention-days: 7 From b13a1fd5150d800f428add21819b1b9804b3347a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 05:35:31 +0800 Subject: [PATCH 48/74] fix(ci): admit Mach-O x64 library capability bit --- .github/openclaw/tls-darwin-disassembly.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/openclaw/tls-darwin-disassembly.py b/.github/openclaw/tls-darwin-disassembly.py index 1458e4f1ed8d..6af1a0fc4260 100644 --- a/.github/openclaw/tls-darwin-disassembly.py +++ b/.github/openclaw/tls-darwin-disassembly.py @@ -26,7 +26,9 @@ def macho(path, filetype): with path.open('rb') as stream: header = stream.read(32) magic, cpu, subtype, kind, count, size, flags, reserved = struct.unpack('<8I', header) - assert (magic, cpu, subtype, kind) == (0xfeedfacf, 0x01000007, 3, filetype) + assert (magic, cpu, kind) == (0xfeedfacf, 0x01000007, filetype), tuple(map(hex, (magic, cpu, subtype, kind))) + # CPU_SUBTYPE_LIB64 is a capability bit, not a different x86_64 architecture. + assert subtype in (3, 0x80000003), hex(subtype) assert 0 < count <= 1024 and 0 < size <= 1024**2 data = stream.read(size) assert len(data) == size @@ -53,7 +55,7 @@ def macho(path, filetype): sections.append((address, address + extent)) offset += length assert offset == size and len(identities) == 1 - return identities[0], sections + return identities[0], sections, subtype def tool(name, args): @@ -118,7 +120,7 @@ def main(): extract(archive, 'bun-darwin-x64-profile.zip', profile_zip, 1024**3) for path in (profile, dsym): extract(profile_zip, 'bun-darwin-x64-profile/' + path.name, path, 2 * 1024**3) - identity, sections = macho(profile, 2) + identity, sections, subtype = macho(profile, 2) assert identity == macho(dsym, 10)[0] == '4c4c4432-5555-3144-a125-d2b0a4297913' assert len(sections) == 1 version = tool('llvm-version', ['llvm-objdump-23', '--version']) @@ -153,7 +155,7 @@ def main(): item['assembly_sha256'] = sha(path) receipt = dict(commit=COMMIT, producer_run=RUN, artifact_id=ARTIFACT, artifact_sha256=DIGEST, identities={p.name: sha(p) for p in (profile_zip, profile, dsym)}, uuid=identity, - sections=sections, symbol_owner=symbol_owner, functions=symbols, llvm_version=version, + sections=sections, cpu_subtype=hex(subtype), symbol_owner=symbol_owner, functions=symbols, llvm_version=version, inferior_launched=False, control_flow_review_pending=True, gc_root_proven=False) (OUT / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') files = sorted(OUT.iterdir()) From e6afcbd938bfc0ec15b7458dc546661dd9aa29be Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 05:42:49 +0800 Subject: [PATCH 49/74] ci(tls): compare collection checkpoints on shipping Darwin --- .github/openclaw/tls-cycle-checkpoint.py | 136 ++++++++++++++++++ .../openclaw-tls-cycle-checkpoint.yml | 69 +++++++++ 2 files changed, 205 insertions(+) create mode 100644 .github/openclaw/tls-cycle-checkpoint.py create mode 100644 .github/workflows/openclaw-tls-cycle-checkpoint.yml diff --git a/.github/openclaw/tls-cycle-checkpoint.py b/.github/openclaw/tls-cycle-checkpoint.py new file mode 100644 index 000000000000..e92de45b395d --- /dev/null +++ b/.github/openclaw/tls-cycle-checkpoint.py @@ -0,0 +1,136 @@ +"""Compare two GC checkpoints with identical retained shipping bytes and assertions.""" +import difflib +import hashlib +import json +import os +from pathlib import Path +import re +import runpy +import subprocess +import xml.etree.ElementTree as ET + +ROOT = Path(os.environ['RUNNER_TEMP']) / 'tls-darwin-input' +OUT = Path('tls-cycle-checkpoint') +SOURCE = Path('source') +COMMIT = '510fef558b464c58de1209298f638fa476d03eef' +ENGINE = 'b82bc917e99110b61c3a21202d6def7457aa194c' +ARTIFACT = 13057470428 +DIGEST = 'a4d8b786d7bb87d166f757e089ac1705f4ac0701504038ebbecd438c9d6e06d6' +FIXTURE = 'test/js/bun/net/socket-retention.test.ts' +FIXTURE_SHA = 'd4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2' +CASE = 'TLS Duplex cycles collect after startup while a retained origin keeps its wrapper alive' +ORIGINAL = ''' const dropped = await makePair(false); + for (let i = 0; i < 30; i++) { await turn(); Bun.gc(true); }''' +CANDIDATE = ''' const dropped = await makePair(false); + for (let i = 0; i < 30; i++) { + await new Promise(resolve => setImmediate(() => { Bun.gc(true); resolve(); })); + }''' + + +def sha(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def run(args, **kwargs): + return subprocess.run(args, stdin=subprocess.DEVNULL, capture_output=True, timeout=120, **kwargs) + + +def main(): + OUT.mkdir(exist_ok=True) + assert os.uname().sysname == 'Darwin' and os.uname().machine == 'x86_64' + assert not any(os.environ.get(k) for k in ('GH_TOKEN', 'GITHUB_TOKEN', 'NODE_OPTIONS', 'BUN_OPTIONS')) + assert subprocess.check_output(['git', '-C', str(SOURCE), 'rev-parse', 'HEAD'], text=True).strip() == COMMIT + assert not subprocess.check_output(['git', '-C', str(SOURCE), 'diff', '--name-only', 'HEAD'], text=True) + fixture = SOURCE / FIXTURE + original = fixture.read_bytes() + assert hashlib.sha256(original).hexdigest() == FIXTURE_SHA + assert original.count(ORIGINAL.encode()) == 1 + candidate = original.replace(ORIGINAL.encode(), CANDIDATE.encode(), 1) + assert candidate.count(b'Bun.gc(true)') == original.count(b'Bun.gc(true)') + (OUT / 'fixture.patch').write_text(''.join(difflib.unified_diff(original.decode().splitlines(True), + candidate.decode().splitlines(True), fromfile=FIXTURE, tofile=FIXTURE))) + run_meta = json.loads((ROOT / 'run.json').read_text()) + meta = json.loads((ROOT / 'metadata.json').read_text()) + assert run_meta['id'] == 41238111110 and run_meta['head_sha'] == COMMIT + assert run_meta['path'] == '.github/workflows/openclaw-release.yml' + assert run_meta['event'] == 'workflow_dispatch' and run_meta['status'] == 'completed' + assert run_meta['conclusion'] == 'failure' + assert meta['id'] == ARTIFACT and meta['name'] == 'signed-darwin-x64' + assert meta['size_in_bytes'] == 232201333 and not meta['expired'] + assert meta['digest'] == 'sha256:' + DIGEST and meta['workflow_run']['id'] == 41238111110 + assert meta['workflow_run']['head_sha'] == COMMIT + archive = ROOT / 'artifact.zip' + assert archive.stat().st_size == 232201333 and sha(archive) == DIGEST + library = runpy.run_path(str(Path(__file__).with_name('tls-darwin-capability.py')), run_name='archive_library') + product_zip, exe = ROOT / 'product.zip', ROOT / 'bun' + library['extract'](archive, 'bun-darwin-x64.zip', product_zip, 1024**3) + library['extract'](product_zip, 'bun-darwin-x64/bun', exe, 1024**3) + exe.chmod(0o700) + exe = exe.resolve() + identity = run([str(exe), '-e', 'console.log(JSON.stringify({revision:Bun.revision,webkit:process.versions.webkit,platform:process.platform,arch:process.arch}))'], check=True, text=True) + assert not identity.stderr + facts = json.loads(identity.stdout) + assert facts == dict(revision=COMMIT, webkit=ENGINE, platform='darwin', arch='x64') + signature = run(['codesign', '--verify', '--strict', str(exe)]) + assert signature.returncode == 0 + receipt = dict(commit=COMMIT, engine=ENGINE, artifact_id=ARTIFACT, artifact_sha256=DIGEST, + product_zip_sha256=sha(product_zip), executable_sha256=sha(exe), facts=facts, + fixture_sha256=FIXTURE_SHA, candidate_sha256=hashlib.sha256(candidate).hexdigest(), + explicit_collections_first_phase=30, assertions_changed=False, runs=[], + claim='paired GC frame/job-phase sensitivity only; no retaining owner or runtime fix established') + try: + for repetition in (1, 2): + for arm, contents in (('baseline', original), ('candidate', candidate)): + fixture.write_bytes(contents) + name = f'{arm}-{repetition}' + report = (OUT / (name + '.xml')).resolve() + assert not report.exists() + argv = [str(exe), 'test', FIXTURE, '--test-name-pattern', '^' + re.escape(CASE) + '$', + '--reporter=junit', '--reporter-outfile=' + str(report)] + result = run(argv, cwd=SOURCE) + assert len(result.stdout) + len(result.stderr) <= 4 * 1024**2 + (OUT / (name + '.stdout')).write_bytes(result.stdout) + (OUT / (name + '.stderr')).write_bytes(result.stderr) + root = ET.parse(report).getroot() + cases = list(root.iter('testcase')) + chosen = [case for case in cases if case.get('name') == CASE] + assert len(chosen) == 1 and not list(root.iter('error')) + case = chosen[0] + assert case.get('file') == FIXTURE and case.find('skipped') is None + assert all(item.find('skipped') is not None and item.find('failure') is None + for item in cases if item is not case) + failures = case.findall('failure') + received = None + if failures: + assert len(failures) == 1 and failures[0].get('type') == 'AssertionError' + message = failures[0].get('message', '') + match = re.search(r'Received: "(\{[^\n]+\})"', message) + assert match, message + received = json.loads(match[1]) + receipt['runs'].append(dict(arm=arm, repetition=repetition, exit_code=result.returncode, + failures=len(failures), received=received, assertions='unchanged')) + (OUT / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + assert sha(exe) == receipt['executable_sha256'] + receipt['repeatable_checkpoint_sensitivity'] = all( + row['exit_code'] == (1 if row['arm'] == 'baseline' else 0) and + row['failures'] == (1 if row['arm'] == 'baseline' else 0) and + (row['arm'] == 'candidate' or row['received'] == dict( + retainedAlive=True, droppedCollected=False, releasedCollected=True)) + for row in receipt['runs']) + finally: + fixture.write_bytes(original) + assert sha(fixture) == FIXTURE_SHA + assert not subprocess.check_output(['git', '-C', str(SOURCE), 'diff', '--name-only', 'HEAD'], text=True) + receipt['source_restored'] = True + (OUT / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + files = sorted(OUT.iterdir()) + assert len(files) <= 24 and sum(p.stat().st_size for p in files) <= 32 * 1024**2 + (OUT / 'manifest.json').write_text(json.dumps({p.name: dict(bytes=p.stat().st_size, sha256=sha(p)) + for p in files}, indent=2) + '\n') + print(json.dumps(receipt)) + assert receipt['repeatable_checkpoint_sensitivity'], 'experiment inconclusive: inspect all arms without changing expectations' + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/openclaw-tls-cycle-checkpoint.yml b/.github/workflows/openclaw-tls-cycle-checkpoint.yml new file mode 100644 index 000000000000..e5a0ba32bbfd --- /dev/null +++ b/.github/workflows/openclaw-tls-cycle-checkpoint.yml @@ -0,0 +1,69 @@ +name: OpenClaw TLS cycle GC checkpoint experiment + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-cycle-checkpoint.yml + - .github/openclaw/tls-cycle-checkpoint.py + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: openclaw-tls-cycle-checkpoint-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + compare: + runs-on: macos-15-intel + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: | + .github/openclaw + .github/workflows + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: source + ref: 510fef558b464c58de1209298f638fa476d03eef + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: "26.12.1" + - name: Download the exact consumed signed Darwin artifact + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + input="$RUNNER_TEMP/tls-darwin-input" + mkdir "$input" + gh api repos/openclaw/bun/actions/runs/41238111110 > "$input/run.json" + gh api repos/openclaw/bun/actions/artifacts/13057470428 > "$input/metadata.json" + curl --location --fail --silent --show-error --proto '=https' --proto-redir '=https' \ + --max-time 300 --max-filesize 232201333 \ + -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/openclaw/bun/actions/artifacts/13057470428/zip \ + --output "$input/artifact.zip" + - name: Compare unchanged assertions at two GC checkpoints + shell: bash + run: | + set -euo pipefail + mkdir tls-cycle-checkpoint + cp controls/.github/workflows/openclaw-tls-cycle-checkpoint.yml tls-cycle-checkpoint/workflow.yml + cp controls/.github/openclaw/tls-cycle-checkpoint.py tls-cycle-checkpoint/controller.py + cp controls/.github/openclaw/tls-darwin-capability.py tls-cycle-checkpoint/archive.py + python3 controls/.github/openclaw/tls-cycle-checkpoint.py + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: tls-cycle-checkpoint + path: tls-cycle-checkpoint/ + if-no-files-found: error + retention-days: 7 From ef6edc1cec48457e5d79098928d7aeca2103a3f3 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 05:49:03 +0800 Subject: [PATCH 50/74] fix(ci): match TLS replay to dry-run signing contract --- .github/openclaw/tls-cycle-checkpoint.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/openclaw/tls-cycle-checkpoint.py b/.github/openclaw/tls-cycle-checkpoint.py index e92de45b395d..a9be5af85fbd 100644 --- a/.github/openclaw/tls-cycle-checkpoint.py +++ b/.github/openclaw/tls-cycle-checkpoint.py @@ -8,6 +8,7 @@ import runpy import subprocess import xml.etree.ElementTree as ET +import zipfile ROOT = Path(os.environ['RUNNER_TEMP']) / 'tls-darwin-input' OUT = Path('tls-cycle-checkpoint') @@ -73,9 +74,22 @@ def main(): facts = json.loads(identity.stdout) assert facts == dict(revision=COMMIT, webkit=ENGINE, platform='darwin', arch='x64') signature = run(['codesign', '--verify', '--strict', str(exe)]) - assert signature.returncode == 0 + with zipfile.ZipFile(archive) as files: + signature_required = 'darwin-x64.signing.json' in files.namelist() + signature_record = dict(required_by_release_contract=signature_required, + exit_code=signature.returncode, + stdout=signature.stdout.decode().replace(str(exe), 'bun'), + stderr=signature.stderr.decode().replace(str(exe), 'bun')) + if signature_required: + signing = ROOT / 'signing.json' + library['extract'](archive, 'darwin-x64.signing.json', signing, 1024**2) + signature_record['manifest_sha256'] = sha(signing) + (OUT / 'signature.json').write_text(json.dumps(signature_record, indent=2) + '\n') + # The release owner requires this for x64 only when it has a signing manifest. + assert not signature_required or signature.returncode == 0 receipt = dict(commit=COMMIT, engine=ENGINE, artifact_id=ARTIFACT, artifact_sha256=DIGEST, product_zip_sha256=sha(product_zip), executable_sha256=sha(exe), facts=facts, + signature=signature_record, fixture_sha256=FIXTURE_SHA, candidate_sha256=hashlib.sha256(candidate).hexdigest(), explicit_collections_first_phase=30, assertions_changed=False, runs=[], claim='paired GC frame/job-phase sensitivity only; no retaining owner or runtime fix established') From a9c7eecb44e67632902c71f0e5789808d40456cc Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 05:49:03 +0800 Subject: [PATCH 51/74] ci(tls): preserve active frozen layout reconstruction --- .github/workflows/openclaw-tls-darwin-types.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/openclaw-tls-darwin-types.yml b/.github/workflows/openclaw-tls-darwin-types.yml index 324f3b7e0e4c..9dcfcc53ad32 100644 --- a/.github/workflows/openclaw-tls-darwin-types.yml +++ b/.github/workflows/openclaw-tls-darwin-types.yml @@ -14,7 +14,8 @@ permissions: concurrency: group: openclaw-tls-darwin-types-${{ github.head_ref || github.ref }} - cancel-in-progress: true + # Preserve the active frozen-input reconstruction across unrelated diagnostic pushes. + cancel-in-progress: false jobs: compile: From 84ebe5c2f36776620abd1c56783a8ae2b3321656 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 05:57:28 +0800 Subject: [PATCH 52/74] ci(tls): qualify fixture correction on retained native artifacts --- .github/openclaw/tls-fixture-qualification.py | 150 ++++++++++++++++++ .../openclaw-tls-fixture-qualification.yml | 98 ++++++++++++ 2 files changed, 248 insertions(+) create mode 100644 .github/openclaw/tls-fixture-qualification.py create mode 100644 .github/workflows/openclaw-tls-fixture-qualification.yml diff --git a/.github/openclaw/tls-fixture-qualification.py b/.github/openclaw/tls-fixture-qualification.py new file mode 100644 index 000000000000..63792b70ad13 --- /dev/null +++ b/.github/openclaw/tls-fixture-qualification.py @@ -0,0 +1,150 @@ +"""Qualify a test-only source delta on the exact retained native runtime artifacts.""" +import hashlib +import json +import os +from pathlib import Path +import re +import runpy +import shutil +import stat +import subprocess +import sys +import time +import xml.etree.ElementTree as ET +import zipfile + +LIBRARY = runpy.run_path(str(Path(__file__).with_name('tls-cycle-checkpoint.py')), run_name='checkpoint_contract') +COMMIT, ENGINE, FIXTURE = (LIBRARY[key] for key in ('COMMIT', 'ENGINE', 'FIXTURE')) +TEST_SOURCE = '8918ddaed89d97ae1605dbd1e1149121f2433d04' +FIXTURE_SHA = '5cca33c82b9120f350f8c51cfaba49109caaa4697272628b48e97a9221b0ec04' +ROOT = Path(os.environ['RUNNER_TEMP']) / 'tls-fixture-input' +OUT = Path('tls-fixture-qualification') +SOURCE = Path('source') +SPECS = { + 'darwin-x64': (13057470428, 'signed-darwin-x64', 232201333, 'a4d8b786d7bb87d166f757e089ac1705f4ac0701504038ebbecd438c9d6e06d6'), + 'darwin-arm64': (13057590237, 'signed-darwin-arm64', 216129450, 'a93fdf3cf995662876b87348f74c1135a62980910e7423a9750260cdde30080f'), + 'linux-x64': (13056872046, 'build-linux-x64', 222954227, '7ef34a5999d54ec04afd44b8f85041bc33e78a54b8e5dec16c908920a49cb55d'), + 'linux-arm64': (13056617461, 'build-linux-arm64', 222623871, 'c54dbe7ea29a336c76db1dd5ba491bbb117a6c211d938892c62f2f5e74251fcd'), + 'windows-x64': (13056907950, 'test-only-windows', 596111335, '5c81e59c6e6ee0b1a21a9f03b61eda6ff8616d3a1fab631c7e1c1d4899a58bb6'), + 'windows-arm64': (13056907950, 'test-only-windows', 596111335, '5c81e59c6e6ee0b1a21a9f03b61eda6ff8616d3a1fab631c7e1c1d4899a58bb6'), +} + + +def sha(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def clean(): + subprocess.run(['git', '-C', str(SOURCE), 'diff-index', '--quiet', 'HEAD', '--'], check=True) + + +def main(target): + OUT.mkdir(exist_ok=True) + artifact, name, size, digest = SPECS[target] + platform, arch = target.split('-') + platform = 'win32' if platform == 'windows' else platform + assert not any(os.environ.get(k) for k in ('GH_TOKEN', 'GITHUB_TOKEN', 'NODE_OPTIONS', 'BUN_OPTIONS')) + head = subprocess.check_output(['git', '-C', str(SOURCE), 'rev-parse', 'HEAD'], text=True).strip() + assert head == TEST_SOURCE + assert subprocess.check_output(['git', '-C', str(SOURCE), 'show', '-s', '--format=%P', 'HEAD'], text=True).strip() == COMMIT + assert subprocess.check_output(['git', '-C', str(SOURCE), 'diff', '--name-only', COMMIT, 'HEAD'], text=True).splitlines() == [FIXTURE] + clean() + original = subprocess.check_output(['git', '-C', str(SOURCE), 'show', COMMIT + ':' + FIXTURE]) + assert hashlib.sha256(original).hexdigest() == LIBRARY['FIXTURE_SHA'] + assert original.count(LIBRARY['ORIGINAL'].encode()) == 1 + candidate = original.replace(LIBRARY['ORIGINAL'].encode(), LIBRARY['CANDIDATE'].encode(), 1) + assert (SOURCE / FIXTURE).read_bytes() == candidate and sha(SOURCE / FIXTURE) == FIXTURE_SHA + owner = (SOURCE / 'scripts/openclaw-release/release.ts').read_text() + block = re.search(r'export const tlsQualificationFiles = \[(.*?)\] as const;', owner, re.S) + assert block + files = json.loads('[' + block[1].rstrip().removesuffix(',') + ']') + assert len(files) == len(set(files)) == 6 and files[0] == FIXTURE + assert len({Path(file).name for file in files}) == 6 + assert all((SOURCE / file).is_file() and not (SOURCE / file).is_symlink() for file in files) + run = json.loads((ROOT / 'run.json').read_text()) + meta = json.loads((ROOT / 'metadata.json').read_text()) + assert run['id'] == 41238111110 and run['head_sha'] == COMMIT + assert run['status'] == 'completed' and run['conclusion'] == 'failure' + assert run['event'] == 'workflow_dispatch' and run['path'] == '.github/workflows/openclaw-release.yml' + assert meta['id'] == artifact and meta['name'] == name and not meta['expired'] + assert meta['size_in_bytes'] == size and meta['digest'] == 'sha256:' + digest + assert meta['workflow_run']['id'] == run['id'] and meta['workflow_run']['head_sha'] == COMMIT + archive = ROOT / 'artifact.zip' + assert archive.stat().st_size == size and sha(archive) == digest + archive_library = runpy.run_path(str(Path(__file__).with_name('tls-darwin-capability.py')), run_name='archive_library') + triplet = 'bun-' + ('windows' if platform == 'win32' else platform) + '-' + ('aarch64' if arch == 'arm64' else arch) + product, exe = ROOT / 'product.zip', ROOT / ('bun.exe' if platform == 'win32' else 'bun') + archive_library['extract'](archive, triplet + '.zip', product, 1024**3) + archive_library['extract'](product, triplet + '/' + exe.name, exe, 1024**3) + exe.chmod(0o700) + exe = exe.resolve() + identity = subprocess.run([str(exe), '-e', 'console.log(JSON.stringify({revision:Bun.revision,webkit:process.versions.webkit,platform:process.platform,arch:process.arch}))'], + capture_output=True, check=True, text=True, timeout=120) + assert not identity.stderr + facts = json.loads(identity.stdout) + assert facts == dict(revision=COMMIT, webkit=ENGINE, platform=platform, arch=arch) + node = shutil.which('node') + assert node + reference = subprocess.run([node, '-e', 'console.log(JSON.stringify({name:process.release.name,version:process.versions.node,bun:process.versions.bun??null,platform:process.platform,arch:process.arch,exe:process.execPath}))'], + capture_output=True, check=True, text=True, timeout=120) + ref = json.loads(reference.stdout) + assert ref == dict(name='node', version='26.12.1', bun=None, platform=platform, arch=arch, exe=ref['exe']) + assert os.path.normcase(os.path.realpath(ref['exe'])) == os.path.normcase(os.path.realpath(node)) + signature = None + if platform == 'darwin': + with zipfile.ZipFile(archive) as contents: + required = arch == 'arm64' or target + '.signing.json' in contents.namelist() + result = subprocess.run(['codesign', '--verify', '--strict', str(exe)], capture_output=True, timeout=120) + signature = dict(required_by_release_contract=required, exit_code=result.returncode, + stderr=result.stderr.decode().replace(str(exe), exe.name)) + assert not required or result.returncode == 0 + receipt = dict(target=target, runtime_source=COMMIT, test_source=TEST_SOURCE, engine=ENGINE, + artifact_id=artifact, artifact_sha256=digest, product_zip_sha256=sha(product), + executable_sha256=sha(exe), facts=facts, fixture_sha256=FIXTURE_SHA, + runtime_diff_from_product_source=[], sole_source_diff=FIXTURE, node_version='26.12.1', + signature=signature, suites=[], claim='unchanged runtime artifacts, exact test-only source correction; no release publication') + deadline = time.monotonic() + 20 * 60 + try: + for file in files: + basename = Path(file).name + report = (OUT / (basename + '.xml')).resolve() + assert not report.exists() + argv = [str(exe), 'test', file, '--reporter=junit', '--reporter-outfile=' + str(report)] + result = subprocess.run(argv, cwd=SOURCE, stdin=subprocess.DEVNULL, capture_output=True, + timeout=max(1, deadline - time.monotonic())) + assert len(result.stdout) + len(result.stderr) <= 4 * 1024**2 + (OUT / (basename + '.stdout')).write_bytes(result.stdout) + (OUT / (basename + '.stderr')).write_bytes(result.stderr) + doc = ET.parse(report).getroot() + cases = list(doc.iter('testcase')) + skipped = len(list(doc.iter('skipped'))) + failures, errors = len(list(doc.iter('failure'))), len(list(doc.iter('error'))) + assert doc.tag == 'testsuites' and int(doc.get('tests')) == len(cases) + assert int(doc.get('skipped')) == skipped and int(doc.get('failures')) == failures + row = dict(file=file, exit_code=result.returncode, tests=len(cases), skipped=skipped, failures=failures, errors=errors) + receipt['suites'].append(row) + assert result.returncode == 0 and failures == errors == 0 and len(cases) > skipped, row + assert sha(exe) == receipt['executable_sha256'] + receipt['passed'] = True + finally: + clean() + assert sha(SOURCE / FIXTURE) == FIXTURE_SHA + receipt['source_and_index_unchanged'] = True + (OUT / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + paths = sorted(OUT.iterdir()) + assert len(paths) <= 23 and all(stat.S_ISREG(p.lstat().st_mode) for p in paths) + assert sum(p.stat().st_size for p in paths) <= 32 * 1024**2 + (OUT / 'manifest.json').write_text(json.dumps({p.name: dict(bytes=p.stat().st_size, sha256=sha(p)) + for p in paths}, indent=2) + '\n') + print(json.dumps(receipt)) + + +if __name__ == '__main__': + action, target = sys.argv[1:] + if action == 'plan': + artifact, name, size, digest = SPECS[target] + print(f'artifact={artifact}\nsize={size}') + else: + assert action == 'qualify' + main(target) diff --git a/.github/workflows/openclaw-tls-fixture-qualification.yml b/.github/workflows/openclaw-tls-fixture-qualification.yml new file mode 100644 index 000000000000..2e833eaeeb42 --- /dev/null +++ b/.github/workflows/openclaw-tls-fixture-qualification.yml @@ -0,0 +1,98 @@ +name: OpenClaw TLS native fixture qualification + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-fixture-qualification.yml + - .github/openclaw/tls-fixture-qualification.py + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: openclaw-tls-fixture-qualification-${{ github.head_ref || github.ref }} + cancel-in-progress: false + +jobs: + qualify: + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + max-parallel: 2 + matrix: + include: + - { target: linux-x64, runner: ubuntu-24.04 } + - { target: linux-arm64, runner: ubuntu-24.04-arm } + - { target: darwin-x64, runner: macos-15-intel } + - { target: darwin-arm64, runner: macos-15 } + - { target: windows-x64, runner: windows-2025 } + - { target: windows-arm64, runner: windows-11-arm } + steps: + - name: Preserve canonical source bytes on every host + shell: bash + run: git config --global core.autocrlf false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: | + .github/openclaw + .github/workflows + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: source + ref: 8918ddaed89d97ae1605dbd1e1149121f2433d04 + fetch-depth: 2 + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: "26.12.1" + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: "3.14.8" + - name: Plan the exact retained product artifact + id: plan + shell: bash + env: + TARGET: ${{ matrix.target }} + run: python controls/.github/openclaw/tls-fixture-qualification.py plan "$TARGET" >> "$GITHUB_OUTPUT" + - name: Download retained artifact metadata and bytes + env: + GH_TOKEN: ${{ github.token }} + ARTIFACT: ${{ steps.plan.outputs.artifact }} + SIZE: ${{ steps.plan.outputs.size }} + shell: bash + run: | + set -euo pipefail + input="$RUNNER_TEMP/tls-fixture-input" + mkdir "$input" + gh api repos/openclaw/bun/actions/runs/41238111110 > "$input/run.json" + gh api "repos/openclaw/bun/actions/artifacts/$ARTIFACT" > "$input/metadata.json" + curl --location --fail --silent --show-error --proto '=https' --proto-redir '=https' \ + --max-time 300 --max-filesize "$SIZE" \ + -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/openclaw/bun/actions/artifacts/$ARTIFACT/zip" \ + --output "$input/artifact.zip" + - name: Qualify all six native TLS suites + shell: bash + env: + TARGET: ${{ matrix.target }} + run: | + set -euo pipefail + mkdir tls-fixture-qualification + cp controls/.github/workflows/openclaw-tls-fixture-qualification.yml tls-fixture-qualification/workflow.yml + cp controls/.github/openclaw/tls-fixture-qualification.py tls-fixture-qualification/controller.py + cp controls/.github/openclaw/tls-cycle-checkpoint.py tls-fixture-qualification/checkpoint.py + cp controls/.github/openclaw/tls-darwin-capability.py tls-fixture-qualification/archive.py + python controls/.github/openclaw/tls-fixture-qualification.py qualify "$TARGET" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: tls-fixture-qualification-${{ matrix.target }} + path: tls-fixture-qualification/ + if-no-files-found: error + retention-days: 7 From 10f0792e55f8e23d4cca56262b6d22378bcab1c0 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 06:11:05 +0800 Subject: [PATCH 53/74] ci(tls): capture verifier layout from compiler AST --- .github/openclaw/tls-darwin-types.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/openclaw/tls-darwin-types.py b/.github/openclaw/tls-darwin-types.py index 668658d1743f..343d0e4192be 100644 --- a/.github/openclaw/tls-darwin-types.py +++ b/.github/openclaw/tls-darwin-types.py @@ -4,6 +4,7 @@ import os from pathlib import Path import runpy +import re import shlex import struct import subprocess @@ -98,10 +99,7 @@ def build(): witness.write_text('#include "' + str(wrapper) + '"\n' + ''.join( '#include \n' for name in required_headers) + '\nstatic_assert(sizeof(JSC::VM) > 0);\nstatic_assert(sizeof(JSC::ConservativeRoots) > 0);\n' - 'static_assert(sizeof(JSC::VerifierSlotVisitor) > 0);\n' - # An external vtable leaves only a declaration unless this object witnesses the layout. - '[[gnu::used, gnu::noinline]] static void retainVerifierLayout(JSC::VerifierSlotVisitor& visitor)\n' - '{ __asm__ __volatile__("" : : "m"(visitor) : "memory"); }\n') + 'static_assert(sizeof(JSC::VerifierSlotVisitor) > 0);\n') output = OUT / 'types.o' # This object supplies layouts only. No runtime code or shipping profile is replaced. replacement = {'-g1': '-g', '-flto=thin': '-fno-lto', @@ -111,6 +109,20 @@ def build(): write('compile-command.json', dict(directory=chosen[0]['directory'], original=original, successful_pch_command=pch['command'], canonical_flags=flags, witness=adapted, variance='full standalone type metadata, no LTO or whole-program vtables, original PCH wrapper included as text, synthetic include-only translation unit; never linked or executed')) + layout_argv = flags + ['-fsyntax-only', '-Xclang', '-fdump-record-layouts', str(witness)] + run('ast-record-layouts', layout_argv, cwd=Path(chosen[0]['directory']), timeout=600) + blocks = (OUT / 'ast-record-layouts.log').read_text().split('*** Dumping AST Record Layout') + verifier = [block.strip() for block in blocks if block.strip() and + block.strip().splitlines()[0].partition('|')[2].strip() == 'class JSC::VerifierSlotVisitor'] + assert len(verifier) == 1, 'missing or ambiguous VerifierSlotVisitor AST record layout' + layout = verifier[0] + size = re.search(r'\[sizeof=(\d+),.*?align=(\d+),', layout, re.S) + assert size and int(size[1]) > 0 and int(size[2]) > 0 + assert all(field in layout for field in ('m_preciseAllocationMap', 'm_doneMarking')) + write('ast-layouts.json', dict(type='JSC::VerifierSlotVisitor', bytes=int(size[1]), + alignment=int(size[2]), raw_layout=layout, argv=layout_argv, + complete_ast_layout=True, root_capture_ready=False, linked=False, executed=False, + claim='compiler AST layout only; nested map/bucket and physical root admission remain separate')) run('compile-types', adapted, cwd=Path(chosen[0]['directory']), timeout=600) assert 0 < output.stat().st_size <= 64 * 1024**2 with output.open('rb') as stream: @@ -133,6 +145,7 @@ def build(): cmakeconfig_sha256=sha(config), header_manifest_sha256=sha(OUT / 'header-manifest.json'), generated_headers_sha256=sha(OUT / 'generated-headers.json'), witness_sha256=sha(output), witness_bytes=output.stat().st_size, + ast_layouts_sha256=sha(OUT / 'ast-layouts.json'), ast_dump_sha256=sha(OUT / 'ast-record-layouts.log'), configured_inputs_reconstructed=True, original_shipping_compile_database_available=False, profile_replaced=False, linked=False, executed=False, claim='reconstructed matching-header layouts only; no optimized local, instruction or GC-root proof')) From 922d5a81aeaf1542c79d06562a8188b754832abd Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 06:23:02 +0800 Subject: [PATCH 54/74] ci(tls): capture production cycle heap roots --- .github/openclaw/tls-cycle-heap-test.py | 61 +++++ .github/openclaw/tls-cycle-heap.py | 230 ++++++++++++++++++ .github/workflows/openclaw-tls-cycle-heap.yml | 76 ++++++ 3 files changed, 367 insertions(+) create mode 100644 .github/openclaw/tls-cycle-heap-test.py create mode 100644 .github/openclaw/tls-cycle-heap.py create mode 100644 .github/workflows/openclaw-tls-cycle-heap.yml diff --git a/.github/openclaw/tls-cycle-heap-test.py b/.github/openclaw/tls-cycle-heap-test.py new file mode 100644 index 000000000000..1609ac6ed1dc --- /dev/null +++ b/.github/openclaw/tls-cycle-heap-test.py @@ -0,0 +1,61 @@ +"""Graph evidence must retain roots and distinguish incoming edges from SCC ownership.""" +import copy +from pathlib import Path +import runpy +import unittest + +LIBRARY = runpy.run_path(str(Path(__file__).with_name('tls-cycle-heap.py')), run_name='graph_contract') +PROJECT, TAGS = LIBRARY['project_graph'], LIBRARY['TAGS'] + + +def fixture(): + nodes = [] + for ident in range(10): + nodes.extend((ident, 16, 0, 0, 0, hex(ident * 16), '0x0')) + return dict(version=3, type='GCDebugging', nodes=nodes, + nodeClassNames=['Object'], edgeTypes=['Internal', 'Property', 'Index', 'Variable'], + edgeNames=list(TAGS), labels=['', 'DOMGCOutput'], roots=[1, 0, 0, 9, 1, 0], + edges=[1, 5, 1, 0, 2, 6, 1, 1, 3, 7, 1, 2, 4, 8, 1, 3, + 0, 1, 0, 0, 1, 2, 0, 0, 3, 4, 0, 0, 4, 3, 0, 0, 9, 3, 0, 0]) + + +class GraphContract(unittest.TestCase): + def test_all_roots_and_scc_boundary_are_preserved(self): + result = PROJECT(fixture()) + target = result['targets'][TAGS[2]] + self.assertEqual(result['roots'], [dict(id=1, reason='', reachability=''), + dict(id=9, reason='DOMGCOutput', reachability='')]) + self.assertEqual({node['id'] for node in target['scc']}, {3, 4}) + self.assertEqual({edge['source'] for edge in target['incoming']}, {4, 9}) + self.assertEqual([edge['source'] for edge in target['scc_external_incoming']], [9]) + self.assertEqual(target['root_paths'][0]['root']['reason'], 'DOMGCOutput') + self.assertEqual([(edge['source'], edge['dest']) for edge in target['root_paths'][0]['path']], [(9, 3)]) + self.assertIsNone(target['synthetic_root_path']) + self.assertEqual([(edge['source'], edge['dest']) for edge in + result['targets'][TAGS[1]]['synthetic_root_path']], [(0, 1), (1, 2)]) + + def test_absent_dropped_tag_is_inconclusive(self): + graph = fixture() + del graph['edges'][8:12] + target = PROJECT(graph)['targets'][TAGS[2]] + self.assertFalse(target['present']) + self.assertEqual(target['interpretation'], 'collected or otherwise absent during this diagnostic') + + def test_repeated_edges_do_not_create_a_second_owner(self): + graph = fixture() + graph['edges'].extend([1, 5, 1, 0]) + self.assertEqual(PROJECT(graph)['targets'][TAGS[0]]['node']['id'], 1) + + def test_ambiguous_owner_and_invalid_root_are_rejected(self): + graph = fixture() + other = copy.deepcopy(graph) + other['edges'].extend([2, 5, 1, 0]) + with self.assertRaises(AssertionError): + PROJECT(other) + graph['roots'][0] = 99 + with self.assertRaises(AssertionError): + PROJECT(graph) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/openclaw/tls-cycle-heap.py b/.github/openclaw/tls-cycle-heap.py new file mode 100644 index 000000000000..0370dd4b5953 --- /dev/null +++ b/.github/openclaw/tls-cycle-heap.py @@ -0,0 +1,230 @@ +"""Capture the retained production cycle graph, with snapshot GC kept explicit.""" +from collections import defaultdict, deque +import difflib +import hashlib +import json +import os +from pathlib import Path +import re +import runpy +import subprocess +import xml.etree.ElementTree as ET +import zipfile + +OUT = Path('tls-cycle-heap') +TAGS = tuple('__tls_cycle_' + name for name in + ('control_origin', 'control_handle', 'dropped_origin', 'dropped_handle')) + + +def project_graph(graph): + assert graph['version'] == 3 and graph['type'] == 'GCDebugging' + assert graph['edgeTypes'] == ['Internal', 'Property', 'Index', 'Variable'] + assert len(graph['nodes']) % 7 == len(graph['edges']) % 4 == len(graph['roots']) % 3 == 0 + classes, labels, names = (graph[key] for key in ('nodeClassNames', 'labels', 'edgeNames')) + assert all(isinstance(s, str) for table in (classes, labels, names) for s in table) + nodes = {} + for i in range(0, len(graph['nodes']), 7): + ident, size, cls, flags, label, cell, wrapped = graph['nodes'][i:i + 7] + assert all(type(n) is int and n >= 0 for n in (ident, size, cls, flags, label)) + assert ident not in nodes and cls < len(classes) and label < len(labels) + assert all(isinstance(a, str) and re.fullmatch(r'0x[0-9a-fA-F]+', a) for a in (cell, wrapped)) + nodes[ident] = dict(id=ident, bytes=size, type=classes[cls], label=labels[label], + flags=flags, cell=cell, wrapped=wrapped) + assert 0 in nodes + incoming, outgoing, tagged = defaultdict(list), defaultdict(list), defaultdict(set) + edges = [] + for i in range(0, len(graph['edges']), 4): + source, dest, kind, data = graph['edges'][i:i + 4] + assert all(type(n) is int and n >= 0 for n in (source, dest, kind, data)) + assert source in nodes and dest in nodes and kind < 4 + if kind in (1, 3): + assert data < len(names) + edge = dict(source=source, dest=dest, type=graph['edgeTypes'][kind], + name=names[data] if kind in (1, 3) else data) + edges.append(edge) + incoming[dest].append(edge) + outgoing[source].append(edge) + if kind == 1 and edge['name'] in TAGS: + tagged[edge['name']].add(source) + roots = [] + for i in range(0, len(graph['roots']), 3): + ident, reason, reachability = graph['roots'][i:i + 3] + assert all(type(n) is int and n >= 0 for n in (ident, reason, reachability)) + assert ident in nodes and reason < len(labels) and reachability < len(labels) + roots.append(dict(id=ident, reason=labels[reason], reachability=labels[reachability])) + assert all(len(tagged[name]) == 1 for name in TAGS[:2]), 'missing or ambiguous live-control tags' + assert all(len(tagged[name]) <= 1 for name in TAGS), 'ambiguous diagnostic tag' + + def reachable(start, adjacency, field): + seen, queue, toward = {start}, deque([start]), {} + while queue: + node = queue.popleft() + for edge in adjacency[node]: + other = edge[field] + if other not in seen: + seen.add(other) + toward[other] = edge + queue.append(other) + return seen, toward + + def path(start, target, toward): + result = [] + while start != target: + edge = toward[start] + result.append(edge) + start = edge['dest'] + return result + + targets = {} + for tag in TAGS: + if not tagged[tag]: + targets[tag] = dict(present=False, interpretation='collected or otherwise absent during this diagnostic') + continue + target = next(iter(tagged[tag])) + ancestors, toward = reachable(target, incoming, 'source') + descendants, _ = reachable(target, outgoing, 'dest') + component = ancestors & descendants + targets[tag] = dict(present=True, node=nodes[target], incoming=incoming[target], + direct_roots=[root for root in roots if root['id'] == target], + root_paths=[dict(root=root, path=path(root['id'], target, toward)) + for root in roots if root['id'] in ancestors], + synthetic_root_path=path(0, target, toward) if 0 in ancestors else None, + scc=[nodes[ident] for ident in sorted(component)], + scc_external_incoming=[edge for ident in sorted(component) for edge in incoming[ident] + if edge['source'] not in component]) + return dict(nodes=len(nodes), edges=len(edges), roots=roots, targets=targets, + claim='snapshot-GC graph with all recorded roots; no same-process scanner or writer proof') + + +def main(): + library = runpy.run_path(str(Path(__file__).with_name('tls-cycle-checkpoint.py')), run_name='checkpoint_contract') + source, root, commit, engine, fixture_name, case_name = ( + library[name] for name in ('SOURCE', 'ROOT', 'COMMIT', 'ENGINE', 'FIXTURE', 'CASE')) + sha, run = library['sha'], library['run'] + assert os.uname().sysname == 'Darwin' and os.uname().machine == 'x86_64' + assert not any(os.environ.get(key) for key in ('GH_TOKEN', 'GITHUB_TOKEN', 'NODE_OPTIONS', 'BUN_OPTIONS')) + assert subprocess.check_output(['git', '-C', str(source), 'rev-parse', 'HEAD'], text=True).strip() == commit + assert not subprocess.check_output(['git', '-C', str(source), 'diff', '--name-only', 'HEAD'], text=True) + fixture = source / fixture_name + original = fixture.read_bytes() + assert hashlib.sha256(original).hexdigest() == library['FIXTURE_SHA'] + producer, artifact = (json.loads((root / name).read_text()) for name in ('run.json', 'metadata.json')) + assert producer['id'] == 41238111110 and producer['head_sha'] == commit + assert producer['path'] == '.github/workflows/openclaw-release.yml' and producer['event'] == 'workflow_dispatch' + assert producer['status'] == 'completed' and producer['conclusion'] == 'failure' + assert artifact['id'] == library['ARTIFACT'] and artifact['name'] == 'signed-darwin-x64' and not artifact['expired'] + assert artifact['size_in_bytes'] == 232201333 and artifact['digest'] == 'sha256:' + library['DIGEST'] + assert artifact['workflow_run']['id'] == producer['id'] and artifact['workflow_run']['head_sha'] == commit + archive = root / 'artifact.zip' + assert archive.stat().st_size == artifact['size_in_bytes'] and sha(archive) == library['DIGEST'] + extractor = runpy.run_path(str(Path(__file__).with_name('tls-darwin-capability.py')), run_name='archive_library')['extract'] + product, exe = root / 'product.zip', root / 'bun' + extractor(archive, 'bun-darwin-x64.zip', product, 1024**3) + extractor(product, 'bun-darwin-x64/bun', exe, 1024**3) + exe.chmod(0o700) + exe = exe.resolve() + executable_sha = '3b8fd7a18caf0dd8365b1672483d6651f4cb2b6d02bb6b5f704fb6888f886310' + assert sha(exe) == executable_sha + for name in ('home', 'tmp'): + (root / name).mkdir() + execution_env = dict(PATH=os.environ['PATH'], HOME=str(root / 'home'), + TMPDIR=str(root / 'tmp') + '/', CI='true', LANG='en_US.UTF-8') + identity = run([str(exe), '-e', 'console.log(JSON.stringify({revision:Bun.revision,webkit:process.versions.webkit,platform:process.platform,arch:process.arch}))'], check=True, text=True, env=execution_env) + assert not identity.stderr + facts = json.loads(identity.stdout) + assert facts == dict(revision=commit, webkit=engine, platform='darwin', arch='x64') + signature = run(['codesign', '--verify', '--strict', str(exe)]) + with zipfile.ZipFile(archive) as files: + required = 'darwin-x64.signing.json' in files.namelist() + assert not required or signature.returncode == 0 + signature_record = dict(required_by_release_contract=required, exit_code=signature.returncode, + stderr=signature.stderr.decode().replace(str(exe), 'bun')) + (OUT / 'signature.json').write_text(json.dumps(signature_record, indent=2) + '\n') + start = original.index(('test("' + case_name + '"').encode()) + end = original.index(b'\nfor (const destroyBeforeCompletion', start) + chunk = original[start:end].decode() + preload = ' const tls = require("node:tls");' + imports = ' const { writeFileSync } = require("node:fs");\n const { generateHeapSnapshotForDebugging } = require("bun:jsc");\n' + tags = ''' Object.defineProperty(clientSide, retain ? "__tls_cycle_control_origin" : "__tls_cycle_dropped_origin", { value: Object.create(null) }); + Object.defineProperty(client._handle, retain ? "__tls_cycle_control_handle" : "__tls_cycle_dropped_handle", { value: Object.create(null) }); +''' + before_return = ' return { weak: new WeakRef(client._handle), origin: retain ? clientSide : null };' + checkpoint = ' for (let i = 0; i < 30; i++) { await turn(); Bun.gc(true); }' + assert chunk.count(preload) == chunk.count(before_return) == 1 and chunk.count(checkpoint) == 2 + tagged = chunk.replace(preload, imports + preload).replace(before_return, tags + before_return) + snapshot = (OUT / 'snapshot.json').resolve() + capture = ('\n const snapshotText = JSON.stringify(generateHeapSnapshotForDebugging());\n' + ' if (Buffer.byteLength(snapshotText) > 24 * 1024**2) throw new Error("snapshot evidence exceeds 24 MiB");\n' + ' writeFileSync(' + json.dumps(os.path.relpath(snapshot, source.resolve())) + ', snapshotText);') + captured = tagged.replace(checkpoint, checkpoint + capture, 1) + arms = [('baseline', original), ('tagged-control', original[:start] + tagged.encode() + original[end:]), + ('snapshot', original[:start] + captured.encode() + original[end:])] + receipt = dict(runtime_source=commit, engine=engine, executable_sha256=executable_sha, + artifact_id=artifact['id'], artifact_sha256=library['DIGEST'], product_zip_sha256=sha(product), + facts=facts, fixture_sha256=library['FIXTURE_SHA'], signature=signature_record, runs=[], + observer_effects=['tags change allocation and object shapes', 'snapshot API performs an additional full GC', + 'diagnostic imports and snapshot serialization add work'], + execution_environment_keys=sorted(execution_env), execution_home='empty task-owned directory', + unchanged_checkpoint_claim=False, scanner_cause_proven=False) + try: + for arm, contents in arms: + fixture.write_bytes(contents) + if arm != 'baseline': + (OUT / (arm + '.patch')).write_text(''.join(difflib.unified_diff( + original.decode().splitlines(True), contents.decode().splitlines(True), + fromfile=fixture_name, tofile=fixture_name))) + report = (OUT / (arm + '.xml')).resolve() + assert not report.exists() + result = run([str(exe), 'test', fixture_name, '--test-name-pattern', '^' + re.escape(case_name) + '$', + '--reporter=junit', '--reporter-outfile=' + str(report)], cwd=source, env=execution_env) + assert len(result.stdout) + len(result.stderr) <= 4 * 1024**2 + (OUT / (arm + '.stdout')).write_bytes(result.stdout) + (OUT / (arm + '.stderr')).write_bytes(result.stderr) + doc = ET.parse(report).getroot() + cases = list(doc.iter('testcase')) + assert doc.tag == 'testsuites' and int(doc.get('tests')) == len(cases) + assert int(doc.get('failures')) == len(list(doc.iter('failure'))) + assert int(doc.get('skipped')) == len(list(doc.iter('skipped'))) + selected = [case for case in cases if case.get('name') == case_name] + assert len(selected) == 1 and not list(doc.iter('error')) + selected = selected[0] + assert selected.get('file') == fixture_name and selected.find('skipped') is None + assert all(case.find('skipped') is not None and case.find('failure') is None + for case in cases if case is not selected) + failures = selected.findall('failure') + received = None + if failures: + assert len(failures) == 1 and failures[0].get('type') == 'AssertionError' + match = re.search(r'Received: "(\{[^\n]+\})"', failures[0].get('message', '')) + assert match + received = json.loads(match[1]) + row = dict(arm=arm, exit_code=result.returncode, failures=len(failures), received=received, + fixture_sha256=hashlib.sha256(contents).hexdigest(), assertions='unchanged') + receipt['runs'].append(row) + assert sha(exe) == executable_sha + if arm != 'snapshot': + assert result.returncode == 1 and len(failures) == 1 and received == dict( + retainedAlive=True, droppedCollected=False, releasedCollected=True), row + else: + assert result.returncode == (1 if failures else 0) + assert snapshot.is_file() and snapshot.stat().st_size <= 24 * 1024**2 + graph = project_graph(json.loads(snapshot.read_text())) + (OUT / 'graph.json').write_text(json.dumps(graph, indent=2) + '\n') + receipt['snapshot_sha256'] = sha(snapshot) + receipt['graph_captured'] = True + finally: + fixture.write_bytes(original) + assert sha(fixture) == library['FIXTURE_SHA'] + assert not subprocess.check_output(['git', '-C', str(source), 'diff', '--name-only', 'HEAD'], text=True) + receipt['source_restored'] = True + (OUT / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + files = sorted(OUT.iterdir()) + assert len(files) <= 24 and all(p.is_file() and not p.is_symlink() for p in files) + assert sum(p.stat().st_size for p in files) <= 32 * 1024**2 + (OUT / 'manifest.json').write_text(json.dumps({p.name: dict(bytes=p.stat().st_size, sha256=sha(p)) + for p in files}, indent=2) + '\n') + print(json.dumps(receipt)) + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/openclaw-tls-cycle-heap.yml b/.github/workflows/openclaw-tls-cycle-heap.yml new file mode 100644 index 000000000000..3817032f6898 --- /dev/null +++ b/.github/workflows/openclaw-tls-cycle-heap.yml @@ -0,0 +1,76 @@ +name: OpenClaw TLS cycle production heap snapshot + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-cycle-heap.yml + - .github/openclaw/tls-cycle-heap.py + - .github/openclaw/tls-cycle-heap-test.py + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: openclaw-tls-cycle-heap-${{ github.head_ref || github.ref }} + cancel-in-progress: false + +jobs: + capture: + runs-on: macos-15-intel + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: | + .github/openclaw + .github/workflows + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: source + ref: 510fef558b464c58de1209298f638fa476d03eef + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: "26.12.1" + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: "3.14.8" + - name: Download the exact consumed Darwin artifact + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + input="$RUNNER_TEMP/tls-darwin-input" + mkdir "$input" + gh api repos/openclaw/bun/actions/runs/41238111110 > "$input/run.json" + gh api repos/openclaw/bun/actions/artifacts/13057470428 > "$input/metadata.json" + curl --location --fail --silent --show-error --proto '=https' --proto-redir '=https' \ + --max-time 300 --max-filesize 232201333 \ + -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/openclaw/bun/actions/artifacts/13057470428/zip \ + --output "$input/artifact.zip" + - name: Reproduce the original failure and capture a separate snapshot collection + shell: bash + run: | + set -euo pipefail + mkdir tls-cycle-heap + cp controls/.github/workflows/openclaw-tls-cycle-heap.yml tls-cycle-heap/workflow.yml + cp controls/.github/openclaw/tls-cycle-heap.py tls-cycle-heap/controller.py + cp controls/.github/openclaw/tls-cycle-heap-test.py tls-cycle-heap/graph-test.py + cp controls/.github/openclaw/tls-cycle-checkpoint.py tls-cycle-heap/checkpoint.py + cp controls/.github/openclaw/tls-darwin-capability.py tls-cycle-heap/archive.py + python3 controls/.github/openclaw/tls-cycle-heap-test.py + python3 controls/.github/openclaw/tls-cycle-heap.py + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: tls-cycle-production-heap + path: tls-cycle-heap/ + if-no-files-found: error + retention-days: 7 From 7b50b0a5c42adce387e27fdc92ad2f447188424f Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 06:33:58 +0800 Subject: [PATCH 55/74] ci(tls): inventory async frame root boundaries --- .github/openclaw/tls-darwin-capability.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/openclaw/tls-darwin-capability.py b/.github/openclaw/tls-darwin-capability.py index c80215b53c64..bd6470e1e25c 100644 --- a/.github/openclaw/tls-darwin-capability.py +++ b/.github/openclaw/tls-darwin-capability.py @@ -98,6 +98,7 @@ def type_layouts(target): for name in ('JSC::VM', 'JSC::Heap', 'JSC::Collector', 'JSC::PreciseAllocation', 'JSC::VerifierSlotVisitor', 'JSC::VerifierSlotVisitor::PreciseAllocationData', 'JSC::ConservativeRoots', 'JSC::CurrentThreadState', 'JSC::RegisterState', + 'JSC::JSLexicalEnvironment', 'JSC::JSGenerator', 'JSC::JSGlobalObject', 'JSC::JSWeakObjectRef', 'JSC::ProtoCallFrame', 'WTF::StackTrace', 'Zig::JSFFIFunction', 'bun_runtime::socket::socket_body::DuplexUpgradeContext', @@ -133,7 +134,10 @@ def inspect(debugger): assert module.GetUUIDString().lower() == inputs['uuid'] layouts = type_layouts(target) functions = {} - for name in ('JSC::constructWeakRef', 'JSC__VM__runGC', 'posix_spawn_bun', + for name in ('JSC::constructWeakRef', 'JSC::JSWeakObjectRef::finishCreation', + 'JSC::JSWeakObjectRef::deref', 'JSC__VM__runGC', 'posix_spawn_bun', + 'JSC::SlotVisitor::appendConservativeRoots', 'JSC::SlotVisitor::appendJSCellOrAuxiliary', + 'JSC::Heap::testAndSetMarked', 'JSC::sanitizeStackForVM', 'JSC::MachineThreads::gatherConservativeRoots', 'JSC::ConservativeRoots::genericAddSpan', 'JSC::ConservativeRoots::genericAddPointer', 'JSC::Interpreter::executeCall', 'Bun__UpgradedDuplex__onWriteComplete', From 4cbd185049a07f5c43ed7999d618604319e8e0cd Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 06:45:11 +0800 Subject: [PATCH 56/74] ci(tls): trace cycle frame roots at the original checkpoint --- .github/openclaw/tls-cycle-root.py | 390 ++++++++++++++++++ .github/workflows/openclaw-tls-cycle-root.yml | 87 ++++ 2 files changed, 477 insertions(+) create mode 100644 .github/openclaw/tls-cycle-root.py create mode 100644 .github/workflows/openclaw-tls-cycle-root.yml diff --git a/.github/openclaw/tls-cycle-root.py b/.github/openclaw/tls-cycle-root.py new file mode 100644 index 000000000000..cb730437e227 --- /dev/null +++ b/.github/openclaw/tls-cycle-root.py @@ -0,0 +1,390 @@ +"""Join one original checkpoint's physical observations to its later heap graph.""" +import difflib +import hashlib +import json +import os +from pathlib import Path +import re +import runpy +import shlex +import struct +import subprocess +import sys +import time +from xml.etree import ElementTree + +OUT = Path('tls-cycle-root') +SOURCE = Path('source') +ROOT = Path(os.environ['RUNNER_TEMP']) / 'tls-darwin-input' +FIXTURE = 'test/js/bun/net/socket-retention.test.ts' +CASE = 'TLS Duplex cycles collect after startup while a retained origin keeps its wrapper alive' +SOURCE_SHA = '510fef558b464c58de1209298f638fa476d03eef' +FIXTURE_SHA = 'd4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2' +PROFILE_SHA = '52f2048920d8b31ef4471e121cc06ad7b4828761a2c3278349f66ed9b4acaf2b' +DSYM_SHA = '45aa427a18e5f97dda7a9fb23641ee066ecc5084de26d121b54530f62c41b0c6' +UUID = '4c4c4432-5555-3144-a125-d2b0a4297913' +EXPECTED = dict(retainedAlive=True, droppedCollected=False, releasedCollected=True) + + +def sha(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def write(name, value): + (OUT / name).write_text(json.dumps(value, indent=2) + '\n') + + +def environment(): + return dict(PATH=os.environ['PATH'], HOME=str(ROOT / 'empty-home'), + TMPDIR=str(ROOT / 'child-tmp'), CI='true', LANG='en_US.UTF-8') + + +def run(args, name, **kwargs): + result = subprocess.run(args, stdin=subprocess.DEVNULL, capture_output=True, timeout=120, + cwd=SOURCE, env=environment(), **kwargs) + assert len(result.stdout) + len(result.stderr) <= 4 * 1024**2 + (OUT / (name + '.stdout')).write_bytes(result.stdout) + (OUT / (name + '.stderr')).write_bytes(result.stderr) + return result + + +def prepare(): + assert os.uname().sysname == 'Darwin' and os.uname().machine == 'x86_64' + assert not any(os.environ.get(k) for k in ('GH_TOKEN', 'GITHUB_TOKEN', 'NODE_OPTIONS', 'BUN_OPTIONS')) + library = runpy.run_path(str(Path(__file__).with_name('tls-darwin-capability.py'))) + library['prepare']() + write('inputs.json', json.loads(Path('tls-darwin-proof/inputs.json').read_text())) + assert sha(ROOT / 'bun-profile') == PROFILE_SHA and sha(ROOT / 'bun-profile.dSYM') == DSYM_SHA + exe = (ROOT / 'bun-profile').resolve() + exe.chmod(0o700) + for name in ('empty-home', 'child-tmp'): + (ROOT / name).mkdir() + identity = run([str(exe), '-e', 'console.log(JSON.stringify({revision:Bun.revision,webkit:process.versions.webkit,platform:process.platform,arch:process.arch}))'], 'identity') + assert identity.returncode == 0 and not identity.stderr + assert json.loads(identity.stdout) == dict(revision=SOURCE_SHA, + webkit='b82bc917e99110b61c3a21202d6def7457aa194c', platform='darwin', arch='x64') + assert subprocess.check_output(['git', '-C', str(SOURCE), 'rev-parse', 'HEAD'], text=True).strip() == SOURCE_SHA + assert not subprocess.check_output(['git', '-C', str(SOURCE), 'diff', '--name-only', 'HEAD'], text=True) + fixture = SOURCE / FIXTURE + original = fixture.read_bytes() + assert hashlib.sha256(original).hexdigest() == FIXTURE_SHA + receipt = dict(source=SOURCE_SHA, fixture_sha256=FIXTURE_SHA, profile_sha256=PROFILE_SHA, + dsym_sha256=DSYM_SHA, uuid=UUID, scanner_cause_proven=False, retaining_writer_proven=False, + claim='original GC32 observations joined to a later snapshot GC; profile is not the production binary', + observer_effects=['profile executable', 'outgoing object tags', 'diagnostic imports', + 'snapshot performs an additional full GC', 'debugger stops'], runs=[]) + try: + for name, contents in [('baseline', original), ('materialize', None)]: + report = (OUT / (name + '.xml')).resolve() + if contents is None: + text = original.decode() + start = text.index('test(' + json.dumps(CASE)) + begin = text.index(' `', start) + end = text.index(' `,', begin + 7) + 6 + child = (OUT / 'original-child.js').resolve() + prefix = ' ((value) => { require("node:fs").writeFileSync(' + json.dumps(str(child)) + ', value); return value; })(`' + contents = (text[:begin] + prefix + text[begin + 7:end] + '`),' + text[end + 2:]).encode() + (OUT / 'materialize.patch').write_text(''.join(difflib.unified_diff( + original.decode().splitlines(True), contents.decode().splitlines(True), + fromfile=FIXTURE, tofile=FIXTURE))) + fixture.write_bytes(contents) + result = run([str(exe), 'test', FIXTURE, '--test-name-pattern', '^' + re.escape(CASE) + '$', + '--reporter=junit', '--reporter-outfile=' + str(report)], name) + doc = ElementTree.parse(report).getroot() + selected = [c for c in doc.iter('testcase') if c.get('name') == CASE] + assert len(selected) == 1 and not list(doc.iter('error')) + case = selected[0] + assert case.get('file') == FIXTURE and case.find('skipped') is None + assert all(c.find('skipped') is not None and c.find('failure') is None + for c in doc.iter('testcase') if c is not case) + failures = case.findall('failure') + assert result.returncode == 1 and len(failures) == 1 + match = re.search(r'Received: "(\{[^\n]+\})"', failures[0].get('message', '')) + assert match and json.loads(match[1]) == EXPECTED + receipt['runs'].append(dict(arm=name, exit_code=1, received=EXPECTED)) + child = (OUT / 'original-child.js').read_text() + template = original.decode()[begin + 7:end] + before, after = template.split('${JSON.stringify(tlsCert)}') + assert child.startswith(before) and child.endswith(after) + cert = json.loads(child[len(before):len(child) - len(after)]) + harness = (SOURCE / 'test/harness.ts').read_text() + cert_source = harness.split('export const tls = Object.freeze({', 1)[1].split('\n});', 1)[0] + pairs = re.findall(r'^ (\w+): ("(?:[^"\\]|\\.)*"),?$', cert_source, re.M) + assert dict((k, json.loads(v)) for k, v in pairs) == cert and set(cert) == {'cert', 'key'} + preload = ' const tls = require("node:tls");' + before_return = ' return { weak: new WeakRef(client._handle), origin: retain ? clientSide : null };' + checkpoint = ' for (let i = 0; i < 30; i++) { await turn(); Bun.gc(true); }' + tags = ''' Object.defineProperty(clientSide, retain ? "__tls_cycle_control_origin" : "__tls_cycle_dropped_origin", { value: Object.create(null) }); + Object.defineProperty(client._handle, retain ? "__tls_cycle_control_handle" : "__tls_cycle_dropped_handle", { value: Object.create(null) }); +''' + assert child.count(preload) == child.count(before_return) == 1 and child.count(checkpoint) == 2 + tagged = child.replace(preload, ' const { generateHeapSnapshotForDebugging } = require("bun:jsc");\n const { writeFileSync } = require("node:fs");\n' + preload) + tagged = tagged.replace(before_return, tags + before_return) + for name, code in [('direct-baseline', child), ('tagged-control', tagged)]: + result = run([str(exe), '-e', code], name) + assert result.returncode == 0 and json.loads(result.stdout) == EXPECTED + receipt['runs'].append(dict(arm=name, exit_code=0, received=EXPECTED)) + snapshot = (OUT / 'snapshot.json').resolve() + insertion = '\n const snapshotText = JSON.stringify(generateHeapSnapshotForDebugging());\n if (Buffer.byteLength(snapshotText) > 24 * 1024**2) throw new Error("snapshot too large");\n writeFileSync(' + json.dumps(str(snapshot)) + ', snapshotText);' + captured = tagged.replace(checkpoint, checkpoint + insertion, 1) + assert captured.count('Bun.gc(true)') == child.count('Bun.gc(true)') + (OUT / 'debugger-child.js').write_text(captured) + receipt['original_child_sha256'] = hashlib.sha256(child.encode()).hexdigest() + receipt['debugger_child_sha256'] = hashlib.sha256(captured.encode()).hexdigest() + finally: + fixture.write_bytes(original) + assert sha(fixture) == FIXTURE_SHA + assert not subprocess.check_output(['git', '-C', str(SOURCE), 'diff', '--name-only', 'HEAD'], text=True) + receipt['source_restored'] = True + write('receipt.json', receipt) + + +def inspect(debugger): + import lldb + debugger.SetAsync(True) + command = lldb.SBCommandReturnObject() + debugger.GetCommandInterpreter().HandleCommand('settings set target.load-script-from-symbol-file false', command) + assert command.Succeeded() + target = debugger.CreateTarget(str((ROOT / 'bun-profile').resolve())) + assert target.IsValid() and target.GetTriple().startswith('x86_64-apple-') + debugger.GetCommandInterpreter().HandleCommand('target symbols add ' + shlex.quote(str((ROOT / 'bun-profile.dSYM').resolve())), command) + assert command.Succeeded(), command.GetError() + module = target.FindModule(target.GetExecutable()) + assert module.GetUUIDString().lower() == UUID + report = dict(gc_count=0, gc32_completed=False, spans=[], marks=[], thread_states=[], + scanner_cause_proven=False, retaining_writer_proven=False, inferior_launched=False) + process = None + pending_spans, pending_marks = {}, {} + raw = bytearray() + active = False + points = {} + dynamic = {} + + def save(): + write('capture.json', report) + (OUT / 'spans.bin').write_bytes(raw) + + def memory(address, size): + assert address > 0 and 0 <= size <= 4 * 1024**2 + error = lldb.SBError() + data = process.ReadMemory(address, size, error) + assert error.Success() and len(data) == size, (hex(address), size, str(error)) + return data + + def word(address): + return struct.unpack(' "$input/run.json" + gh api repos/openclaw/bun/actions/artifacts/13057470428 > "$input/metadata.json" + curl --location --fail --silent --show-error --proto '=https' --proto-redir '=https' \ + --max-time 300 --max-filesize 232201333 \ + -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/openclaw/bun/actions/artifacts/13057470428/zip \ + --output "$input/artifact.zip" + - name: Admit the profile and reproduce the unchanged checkpoint + shell: bash + run: | + set -euo pipefail + mkdir tls-cycle-root tls-darwin-proof + cp controls/.github/workflows/openclaw-tls-cycle-root.yml tls-cycle-root/workflow.yml + cp controls/.github/openclaw/tls-cycle-root.py tls-cycle-root/controller.py + cp controls/.github/openclaw/tls-cycle-heap.py tls-cycle-root/graph-control.py + cp controls/.github/openclaw/tls-darwin-capability.py tls-cycle-root/archive.py + python3 controls/.github/openclaw/tls-cycle-root.py prepare + - name: Observe original GC32 scan spans and physical mark transitions + shell: bash + run: | + set -euo pipefail + xcrun lldb --batch --no-lldbinit --no-use-colors \ + -o 'settings set target.load-script-from-symbol-file false' \ + -o 'script import runpy; runpy.run_path("controls/.github/openclaw/tls-cycle-root.py", run_name="__lldb__")' \ + > tls-cycle-root/debugger.log 2>&1 + - name: Join the snapshot and seal bounded evidence + if: always() + shell: bash + run: python3 controls/.github/openclaw/tls-cycle-root.py seal + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: tls-cycle-physical-root + path: tls-cycle-root/ + if-no-files-found: error + retention-days: 7 From e4a502d7cde1b7349d0ee412a8f597e8f0702efa Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 06:55:45 +0800 Subject: [PATCH 57/74] ci(tls): preserve debugger stop and exception details --- .github/openclaw/tls-cycle-root.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/openclaw/tls-cycle-root.py b/.github/openclaw/tls-cycle-root.py index cb730437e227..4199aa885bfe 100644 --- a/.github/openclaw/tls-cycle-root.py +++ b/.github/openclaw/tls-cycle-root.py @@ -11,6 +11,7 @@ import subprocess import sys import time +import traceback from xml.etree import ElementTree OUT = Path('tls-cycle-root') @@ -232,6 +233,10 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): continue stops += 1 assert stops <= 30000 + report['last_stop'] = dict(id=process.GetStopID(), threads=[dict( + thread=t.GetThreadID(), reason=t.GetStopReason(), + data=[t.GetStopReasonDataAtIndex(i) for i in range(min(t.GetStopReasonDataCount(), 32))], + function=t.GetFrameAtIndex(0).GetDisplayFunctionName()) for t in process]) handled = False for thread in process: if thread.GetStopReason() != lldb.eStopReasonBreakpoint: @@ -323,6 +328,11 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): assert report['gc32_completed'] and report['exit_status'] == 0 assert json.loads((OUT / 'debugger.stdout').read_text()) == EXPECTED report['original_failure_preserved'] = True + except BaseException as exc: + report['error'] = dict(type=type(exc).__name__, message=str(exc)[:2000], + frames=[dict(file=Path(f.filename).name, line=f.lineno, function=f.name) + for f in traceback.extract_tb(exc.__traceback__)]) + raise finally: if process is not None and process.IsValid() and process.GetState() not in (lldb.eStateExited, lldb.eStateDetached): report['task_process_killed'] = process.Kill().Success() From c3700717fcb107b9a83e0a2c4eb19cb9bd1f74bc Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 07:00:59 +0800 Subject: [PATCH 58/74] ci(tls): synchronize debugger stops before counting GC --- .github/openclaw/tls-cycle-root.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/openclaw/tls-cycle-root.py b/.github/openclaw/tls-cycle-root.py index 4199aa885bfe..9a30b4c8dad8 100644 --- a/.github/openclaw/tls-cycle-root.py +++ b/.github/openclaw/tls-cycle-root.py @@ -143,7 +143,7 @@ def prepare(): def inspect(debugger): import lldb - debugger.SetAsync(True) + debugger.SetAsync(False) command = lldb.SBCommandReturnObject() debugger.GetCommandInterpreter().HandleCommand('settings set target.load-script-from-symbol-file false', command) assert command.Succeeded() @@ -212,25 +212,25 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): launch = lldb.SBLaunchInfo(['-e', (OUT / 'debugger-child.js').read_text()]) launch.SetWorkingDirectory(str(SOURCE.resolve())) launch.SetEnvironmentEntries([k + '=' + v for k, v in environment().items()], False) + launch.SetLaunchFlags(0) + report['launch_flags'] = launch.GetLaunchFlags() assert launch.AddOpenFileAction(0, '/dev/null', True, False) assert launch.AddOpenFileAction(1, str((OUT / 'debugger.stdout').resolve()), False, True) assert launch.AddOpenFileAction(2, str((OUT / 'debugger.stderr').resolve()), False, True) launch.SetDetachOnError(False) - listener = debugger.GetListener() - launch.SetListener(listener) error = lldb.SBError() try: process = target.Launch(launch, error) assert error.Success() and process.IsValid(), str(error) report['inferior_launched'] = True report['pid'] = process.GetProcessID() - deadline, stops = time.monotonic() + 240, 0 + deadline, stops, previous_stop = time.monotonic() + 240, 0, 0 while process.GetState() not in (lldb.eStateExited, lldb.eStateCrashed, lldb.eStateDetached): assert time.monotonic() < deadline, 'debugger capture deadline' - if process.GetState() != lldb.eStateStopped: - event = lldb.SBEvent() - listener.WaitForEvent(1, event) - continue + assert process.GetState() == lldb.eStateStopped, 'synchronous debugger did not stop' + stop_id = process.GetStopID() + assert stop_id > previous_stop, 'duplicate debugger stop would corrupt GC numbering' + previous_stop = stop_id stops += 1 assert stops <= 30000 report['last_stop'] = dict(id=process.GetStopID(), threads=[dict( From d8abd7aa122ad36845872807fab26b92721c5764 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 07:05:30 +0800 Subject: [PATCH 59/74] ci(tls): handle empty conservative root spans --- .github/openclaw/tls-cycle-root.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/openclaw/tls-cycle-root.py b/.github/openclaw/tls-cycle-root.py index 9a30b4c8dad8..870e62ac5340 100644 --- a/.github/openclaw/tls-cycle-root.py +++ b/.github/openclaw/tls-cycle-root.py @@ -168,6 +168,8 @@ def save(): def memory(address, size): assert address > 0 and 0 <= size <= 4 * 1024**2 + if size == 0: + return b'' error = lldb.SBError() data = process.ReadMemory(address, size, error) assert error.Success() and len(data) == size, (hex(address), size, str(error)) From 9feaffcb79cf2ff35ebe7afef2eae5d5b15cb656 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 07:19:32 +0800 Subject: [PATCH 60/74] test(tls): map module-entry stack instructions --- .github/openclaw/tls-darwin-capability.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/openclaw/tls-darwin-capability.py b/.github/openclaw/tls-darwin-capability.py index bd6470e1e25c..1c5e3c367996 100644 --- a/.github/openclaw/tls-darwin-capability.py +++ b/.github/openclaw/tls-darwin-capability.py @@ -140,6 +140,7 @@ def inspect(debugger): 'JSC::Heap::testAndSetMarked', 'JSC::sanitizeStackForVM', 'JSC::MachineThreads::gatherConservativeRoots', 'JSC::ConservativeRoots::genericAddSpan', 'JSC::ConservativeRoots::genericAddPointer', 'JSC::Interpreter::executeCall', + 'JSC::Interpreter::executeModuleProgram', 'vmEntryToJavaScript', 'Bun__UpgradedDuplex__onWriteComplete', 'bun_runtime::socket::socket_body::DuplexUpgradeContext::run_event', 'bun_runtime::socket::socket_body::DuplexUpgradeContext::deinit'): @@ -161,8 +162,13 @@ def inspect(debugger): rows = [] for j in range(instructions.GetSize()): insn = instructions.GetInstructionAtIndex(j) - rows.append(dict(address=insn.GetAddress().GetFileAddress(), mnemonic=insn.GetMnemonic(target), - operands=insn.GetOperands(target), comment=insn.GetComment(target))) + entry = insn.GetAddress().GetLineEntry() + data = bytes(insn.GetData(target).uint8s) + assert len(data) == insn.GetByteSize() + rows.append(dict(address=insn.GetAddress().GetFileAddress(), bytes=data.hex(), + mnemonic=insn.GetMnemonic(target), operands=insn.GetOperands(target), + comment=insn.GetComment(target), file=entry.GetFileSpec().GetFilename(), + line=entry.GetLine())) parents[key] = dict(name=resolved, address=key, instructions=rows) functions[name] = list(parents.values()) assert not target.GetProcess().IsValid(), 'static inventory launched an inferior' From 5d7e4ec098be87fef5d5513592a10eb52c27ca93 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 07:26:38 +0800 Subject: [PATCH 61/74] test(tls): trace writes to the retained stack word --- .github/openclaw/tls-cycle-root.py | 83 +++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) diff --git a/.github/openclaw/tls-cycle-root.py b/.github/openclaw/tls-cycle-root.py index 870e62ac5340..f06a03a72b0c 100644 --- a/.github/openclaw/tls-cycle-root.py +++ b/.github/openclaw/tls-cycle-root.py @@ -161,6 +161,7 @@ def inspect(debugger): active = False points = {} dynamic = {} + watch = None def save(): write('capture.json', report) @@ -189,6 +190,31 @@ def frames(thread): file=f.GetLineEntry().GetFileSpec().GetFilename(), line=f.GetLineEntry().GetLine()) for f in [thread.GetFrameAtIndex(i) for i in range(thread.GetNumFrames())]] + def instructions(rows, begin=0, end=None): + result = [] + for i in range(begin, rows.GetSize() if end is None else end): + insn = rows.GetInstructionAtIndex(i) + address = insn.GetAddress() + entry = address.GetLineEntry() + data = bytes(insn.GetData(target).uint8s) + assert len(data) == insn.GetByteSize() + result.append(dict(address=address.GetFileAddress(), bytes=data.hex(), + mnemonic=insn.GetMnemonic(target), operands=insn.GetOperands(target), + comment=insn.GetComment(target), file=entry.GetFileSpec().GetFilename(), line=entry.GetLine())) + return result + + module_rows = target.ReadInstructions(target.ResolveFileAddress(0x103cf8280), 244) + assert module_rows.GetSize() == 244 + report['module_entry_instructions'] = instructions(module_rows) + call = next(r for r in report['module_entry_instructions'] if r['address'] == 0x103cf8527) + assert call['bytes'] == 'e8dc8c5d01' and call['mnemonic'] == 'callq' + + def module_frame(thread): + selected = [f for f in frames(thread) if f['file'] == 'Interpreter.cpp' and f['line'] == 1839 + and target.ResolveLoadAddress(f['pc']).GetFileAddress() == 0x103cf852c] + assert len(selected) == 1 and selected[0]['fp'] - selected[0]['sp'] == 224 + return selected[0] + def point(name, address, mnemonic, operands=None, enabled=True, thread=None): address = target.ResolveFileAddress(address) if isinstance(address, int) else address instructions = target.ReadInstructions(address, 1) @@ -241,6 +267,35 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): function=t.GetFrameAtIndex(0).GetDisplayFunctionName()) for t in process]) handled = False for thread in process: + if thread.GetStopReason() == lldb.eStopReasonWatchpoint: + assert watch is not None and watch.IsEnabled() + assert thread.GetStopReasonDataCount() == 1 + assert thread.GetStopReasonDataAtIndex(0) == watch.GetID() + row = report['word_watch'] + assert len(row['changes']) < 2048 + frame = thread.GetFrameAtIndex(0) + pc = frame.GetPC() + value = word(row['address']) + function, symbol = frame.GetFunction(), frame.GetSymbol() + decoded = function.GetInstructions(target) if function.IsValid() else symbol.GetInstructions(target) + window, preceding = [], [] + if 0 < decoded.GetSize() <= 16384: + for i in range(decoded.GetSize()): + insn = decoded.GetInstructionAtIndex(i) + end = insn.GetAddress().GetLoadAddress(target) + insn.GetByteSize() + if end == pc: + window = instructions(decoded, max(0, i - 5), min(decoded.GetSize(), i + 3)) + preceding = instructions(decoded, i, i + 1) + break + row['changes'].append(dict(thread=thread.GetThreadID(), gc_count=report['gc_count'], + before=row['last_value'], after=value, stop_pc=pc, + stop_file_pc=target.ResolveLoadAddress(pc).GetFileAddress(), + registers={name: reg(frame, name) for name in + ('rax', 'rbx', 'rcx', 'rdx', 'rsi', 'rdi', 'rbp', 'rsp', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15', 'rflags')}, + frames=frames(thread), preceding_instruction=preceding, instruction_window=window)) + row['last_value'] = value + handled = True + continue if thread.GetStopReason() != lldb.eStopReasonBreakpoint: continue frame, tid = thread.GetFrameAtIndex(0), thread.GetThreadID() @@ -251,7 +306,22 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): handled = True if name == 'gc': report['gc_count'] += 1 + if report['gc_count'] == 1: + owner = module_frame(thread) + address = owner['sp'] + error = lldb.SBError() + watch = target.WatchAddress(address, 8, False, True, error) + assert error.Success() and watch.IsValid(), str(error) + assert watch.GetWatchAddress() == address and watch.GetWatchSize() == 8 + initial = word(address) + report['word_watch'] = dict(address=address, initial_value=initial, last_value=initial, + initial_owner=owner, changes=[], + claim='one physical address; its frame may exit or storage may be reused; equal-value stores may be unreported') if report['gc_count'] == 32: + owner = module_frame(thread) + assert owner['sp'] == report['word_watch']['address'], 'GC32 uses a different stack slot' + report['word_watch']['gc32_owner'] = owner + assert word(owner['sp']) == report['word_watch']['last_value'] active = True report['gc32_entry_frames'] = frames(thread) ret = target.ResolveLoadAddress(word(frame.GetSP())) @@ -268,6 +338,8 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): for b in scans + [gather, pre, post, gc]: b.SetEnabled(False) target.BreakpointDelete(ident) + report['word_watch']['gc32_return_value'] = word(report['word_watch']['address']) + assert target.DeleteWatchpoint(watch.GetID()) elif name == 'gather': assert active state = reg(frame, 'r8') @@ -378,8 +450,15 @@ def seal(): matches.append(dict(address=address, span_begin=span['begin'], span_end=span['end'], accepted=cell in span['accepted_roots'], physical_frames=owners)) marks = [m for m in capture['marks'] if m['cell'] == cell] + watched = capture['word_watch'] + changes = [row for row in watched['changes'] if row['after'] == cell] + watch_join = dict(address=watched['address'], initial_value_matches=watched['initial_value'] == cell, + gc32_return_value_matches=watched['gc32_return_value'] == cell, + accepted_root_matches=any(m['accepted'] and m['address'] == watched['address'] for m in matches), + observed_changes_to_owner=changes, + claim='observed value-changing stores only; decode the preceding instruction and intended lifetime before proposing a repair') write('join.json', dict(owner_id=owner[0], owner_cell=owner[5], gc32_word_matches=matches, - gc32_mark_events=marks, scanner_cause_proven=False, retaining_writer_proven=False, + gc32_mark_events=marks, word_watch=watch_join, scanner_cause_proven=False, retaining_writer_proven=False, claim='physical observations; identify the frame writer before proposing a runtime repair')) receipt.update(snapshot_sha256=sha(snapshot), gc32_completed=capture['gc32_completed'], word_matches=len(matches), new_marks=sum(m['new_mark'] for m in marks), @@ -387,6 +466,8 @@ def seal(): write('receipt.json', receipt) assert capture['gc32_completed'] and capture['original_failure_preserved'] assert any(m['accepted'] for m in matches) and any(m['new_mark'] for m in marks), 'root not established; no causal claim' + assert watch_join['accepted_root_matches'] and watch_join['gc32_return_value_matches'] + assert changes, 'owner value was not observed being written; earlier writer remains unknown' finally: files = sorted(OUT.iterdir()) assert len(files) <= 32 and all(p.is_file() and not p.is_symlink() for p in files) From 0bffad66cc6b7873923fb92e74d910ddefc24f77 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 07:39:34 +0800 Subject: [PATCH 62/74] test(tls): bound stack-word capture overhead --- .github/openclaw/tls-cycle-root.py | 49 +++++++++++++++++++----------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/.github/openclaw/tls-cycle-root.py b/.github/openclaw/tls-cycle-root.py index f06a03a72b0c..c9166c80c6f2 100644 --- a/.github/openclaw/tls-cycle-root.py +++ b/.github/openclaw/tls-cycle-root.py @@ -162,6 +162,7 @@ def inspect(debugger): points = {} dynamic = {} watch = None + decoded_functions = {} def save(): write('capture.json', report) @@ -272,27 +273,37 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): assert thread.GetStopReasonDataCount() == 1 assert thread.GetStopReasonDataAtIndex(0) == watch.GetID() row = report['word_watch'] - assert len(row['changes']) < 2048 + assert len(row['changes']) < 12000 frame = thread.GetFrameAtIndex(0) pc = frame.GetPC() value = word(row['address']) - function, symbol = frame.GetFunction(), frame.GetSymbol() - decoded = function.GetInstructions(target) if function.IsValid() else symbol.GetInstructions(target) - window, preceding = [], [] - if 0 < decoded.GetSize() <= 16384: - for i in range(decoded.GetSize()): - insn = decoded.GetInstructionAtIndex(i) - end = insn.GetAddress().GetLoadAddress(target) + insn.GetByteSize() - if end == pc: - window = instructions(decoded, max(0, i - 5), min(decoded.GetSize(), i + 3)) - preceding = instructions(decoded, i, i + 1) - break - row['changes'].append(dict(thread=thread.GetThreadID(), gc_count=report['gc_count'], + event = dict(thread=thread.GetThreadID(), gc_count=report['gc_count'], before=row['last_value'], after=value, stop_pc=pc, - stop_file_pc=target.ResolveLoadAddress(pc).GetFileAddress(), - registers={name: reg(frame, name) for name in - ('rax', 'rbx', 'rcx', 'rdx', 'rsi', 'rdi', 'rbp', 'rsp', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15', 'rflags')}, - frames=frames(thread), preceding_instruction=preceding, instruction_window=window)) + stop_file_pc=target.ResolveLoadAddress(pc).GetFileAddress()) + # Capture every value change; expensive context is useful only for potential cell pointers. + if 0x10000 <= value < 2**44 and value % 16 == 0: + function, symbol = frame.GetFunction(), frame.GetSymbol() + start = function.GetStartAddress() if function.IsValid() else symbol.GetStartAddress() + image = start.GetModule() + key = (image.GetUUIDString(), start.GetFileAddress()) if image.IsValid() and image.GetUUIDString() else None + cached = decoded_functions.get(key) if key is not None else None + if cached is None: + decoded = function.GetInstructions(target) if function.IsValid() else symbol.GetInstructions(target) + ends = {} + if 0 < decoded.GetSize() <= 16384: + ends = {decoded.GetInstructionAtIndex(i).GetAddress().GetLoadAddress(target) + + decoded.GetInstructionAtIndex(i).GetByteSize(): i + for i in range(decoded.GetSize())} + cached = (decoded, ends) + if key is not None: + decoded_functions[key] = cached + decoded, ends = cached + index = ends.get(pc) + event.update(registers={name: reg(frame, name) for name in + ('rax', 'rbx', 'rcx', 'rdx', 'rsi', 'rdi', 'rbp', 'rsp', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15', 'rflags')}, + frames=frames(thread), preceding_instruction=[] if index is None else instructions(decoded, index, index + 1), + instruction_window=[] if index is None else instructions(decoded, max(0, index - 5), min(decoded.GetSize(), index + 3))) + row['changes'].append(event) row['last_value'] = value handled = True continue @@ -316,11 +327,12 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): initial = word(address) report['word_watch'] = dict(address=address, initial_value=initial, last_value=initial, initial_owner=owner, changes=[], + context_filter='0x10000 <= value < 2**44 and value % 16 == 0; all changes retain values and PC', claim='one physical address; its frame may exit or storage may be reused; equal-value stores may be unreported') if report['gc_count'] == 32: owner = module_frame(thread) - assert owner['sp'] == report['word_watch']['address'], 'GC32 uses a different stack slot' report['word_watch']['gc32_owner'] = owner + assert owner['sp'] == report['word_watch']['address'], 'GC32 uses a different stack slot' assert word(owner['sp']) == report['word_watch']['last_value'] active = True report['gc32_entry_frames'] = frames(thread) @@ -431,6 +443,7 @@ def seal(): owner = next(nodes[i:i + 7] for i in range(0, len(nodes), 7) if nodes[i] in external) assert raw_graph['nodeClassNames'][owner[2]] == 'JSLexicalEnvironment' cell = int(owner[5], 16) + assert 0x10000 <= cell < 2**44 and cell % 16 == 0, 'owner falls outside the detailed-context filter' edges = raw_graph['edges'] assert all(edges[i + 1] != owner[0] for i in range(0, len(edges), 4)) assert owner[0] not in raw_graph['roots'][::3] From 7e718d305357fd9dc4fb1efcbba147701668815c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 07:47:53 +0800 Subject: [PATCH 63/74] ci(tls): capture all watched writes and diagnose unexpected stops --- .github/openclaw/tls-cycle-root.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/.github/openclaw/tls-cycle-root.py b/.github/openclaw/tls-cycle-root.py index c9166c80c6f2..c6b8a40525cf 100644 --- a/.github/openclaw/tls-cycle-root.py +++ b/.github/openclaw/tls-cycle-root.py @@ -265,6 +265,8 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): report['last_stop'] = dict(id=process.GetStopID(), threads=[dict( thread=t.GetThreadID(), reason=t.GetStopReason(), data=[t.GetStopReasonDataAtIndex(i) for i in range(min(t.GetStopReasonDataCount(), 32))], + description=t.GetStopDescription(512), pc=t.GetFrameAtIndex(0).GetPC(), + sp=t.GetFrameAtIndex(0).GetSP(), fp=t.GetFrameAtIndex(0).GetFP(), function=t.GetFrameAtIndex(0).GetDisplayFunctionName()) for t in process]) handled = False for thread in process: @@ -273,14 +275,14 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): assert thread.GetStopReasonDataCount() == 1 assert thread.GetStopReasonDataAtIndex(0) == watch.GetID() row = report['word_watch'] - assert len(row['changes']) < 12000 + assert len(row['writes']) < 12000 frame = thread.GetFrameAtIndex(0) pc = frame.GetPC() value = word(row['address']) event = dict(thread=thread.GetThreadID(), gc_count=report['gc_count'], before=row['last_value'], after=value, stop_pc=pc, stop_file_pc=target.ResolveLoadAddress(pc).GetFileAddress()) - # Capture every value change; expensive context is useful only for potential cell pointers. + # Capture every write; expensive context is useful only for potential cell pointers. if 0x10000 <= value < 2**44 and value % 16 == 0: function, symbol = frame.GetFunction(), frame.GetSymbol() start = function.GetStartAddress() if function.IsValid() else symbol.GetStartAddress() @@ -303,7 +305,7 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): ('rax', 'rbx', 'rcx', 'rdx', 'rsi', 'rdi', 'rbp', 'rsp', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15', 'rflags')}, frames=frames(thread), preceding_instruction=[] if index is None else instructions(decoded, index, index + 1), instruction_window=[] if index is None else instructions(decoded, max(0, index - 5), min(decoded.GetSize(), index + 3))) - row['changes'].append(event) + row['writes'].append(event) row['last_value'] = value handled = True continue @@ -321,14 +323,18 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): owner = module_frame(thread) address = owner['sp'] error = lldb.SBError() - watch = target.WatchAddress(address, 8, False, True, error) + options = lldb.SBWatchpointOptions() + options.SetWatchpointTypeRead(False) + options.SetWatchpointTypeWrite(lldb.eWatchpointWriteTypeAlways) + assert options.GetWatchpointTypeWrite() == lldb.eWatchpointWriteTypeAlways + watch = target.WatchpointCreateByAddress(address, 8, options, error) assert error.Success() and watch.IsValid(), str(error) assert watch.GetWatchAddress() == address and watch.GetWatchSize() == 8 initial = word(address) report['word_watch'] = dict(address=address, initial_value=initial, last_value=initial, - initial_owner=owner, changes=[], - context_filter='0x10000 <= value < 2**44 and value % 16 == 0; all changes retain values and PC', - claim='one physical address; its frame may exit or storage may be reused; equal-value stores may be unreported') + initial_owner=owner, writes=[], write_mode='always', + context_filter='0x10000 <= value < 2**44 and value % 16 == 0; all writes retain values and PC', + claim='one physical address; its frame may exit or storage may be reused; unexpected stops remain failures') if report['gc_count'] == 32: owner = module_frame(thread) report['word_watch']['gc32_owner'] = owner @@ -407,6 +413,8 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): if row['new_mark']: assert not row['before'] & (1 << row['bit']) report['marks'].append(row) + if not handled and watch is not None: + report['last_stop']['watched_value'] = word(report['word_watch']['address']) assert handled, 'unexpected debugger stop' assert process.Continue().Success() report['exit_status'] = process.GetExitStatus() @@ -464,12 +472,12 @@ def seal(): accepted=cell in span['accepted_roots'], physical_frames=owners)) marks = [m for m in capture['marks'] if m['cell'] == cell] watched = capture['word_watch'] - changes = [row for row in watched['changes'] if row['after'] == cell] + owner_writes = [row for row in watched['writes'] if row['after'] == cell] watch_join = dict(address=watched['address'], initial_value_matches=watched['initial_value'] == cell, gc32_return_value_matches=watched['gc32_return_value'] == cell, accepted_root_matches=any(m['accepted'] and m['address'] == watched['address'] for m in matches), - observed_changes_to_owner=changes, - claim='observed value-changing stores only; decode the preceding instruction and intended lifetime before proposing a repair') + observed_writes_of_owner=owner_writes, + claim='observed write accesses; decode the preceding instruction and intended lifetime before proposing a repair') write('join.json', dict(owner_id=owner[0], owner_cell=owner[5], gc32_word_matches=matches, gc32_mark_events=marks, word_watch=watch_join, scanner_cause_proven=False, retaining_writer_proven=False, claim='physical observations; identify the frame writer before proposing a runtime repair')) @@ -480,7 +488,7 @@ def seal(): assert capture['gc32_completed'] and capture['original_failure_preserved'] assert any(m['accepted'] for m in matches) and any(m['new_mark'] for m in marks), 'root not established; no causal claim' assert watch_join['accepted_root_matches'] and watch_join['gc32_return_value_matches'] - assert changes, 'owner value was not observed being written; earlier writer remains unknown' + assert owner_writes, 'owner value was not observed being written; earlier writer remains unknown' finally: files = sorted(OUT.iterdir()) assert len(files) <= 32 and all(p.is_file() and not p.is_symlink() for p in files) From 18f68a3d1c88f39cd173d8698067059f670f05e8 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 07:53:06 +0800 Subject: [PATCH 64/74] ci(tls): reject unexpected stops on every debugger thread --- .github/openclaw/tls-cycle-root.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/openclaw/tls-cycle-root.py b/.github/openclaw/tls-cycle-root.py index c6b8a40525cf..e3289e96bfe0 100644 --- a/.github/openclaw/tls-cycle-root.py +++ b/.github/openclaw/tls-cycle-root.py @@ -268,6 +268,13 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): description=t.GetStopDescription(512), pc=t.GetFrameAtIndex(0).GetPC(), sp=t.GetFrameAtIndex(0).GetSP(), fp=t.GetFrameAtIndex(0).GetFP(), function=t.GetFrameAtIndex(0).GetDisplayFunctionName()) for t in process]) + unexpected = [t.GetThreadID() for t in process if t.GetStopReason() not in + (lldb.eStopReasonInvalid, lldb.eStopReasonNone, + lldb.eStopReasonWatchpoint, lldb.eStopReasonBreakpoint)] + if unexpected: + if watch is not None: + report['last_stop']['watched_value'] = word(report['word_watch']['address']) + raise AssertionError('unexpected debugger stop on threads ' + str(unexpected)) handled = False for thread in process: if thread.GetStopReason() == lldb.eStopReasonWatchpoint: @@ -312,7 +319,7 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): if thread.GetStopReason() != lldb.eStopReasonBreakpoint: continue frame, tid = thread.GetFrameAtIndex(0), thread.GetThreadID() - assert thread.GetStopReasonDataCount() % 2 == 0 + assert thread.GetStopReasonDataCount() > 0 and thread.GetStopReasonDataCount() % 2 == 0 for i in range(0, thread.GetStopReasonDataCount(), 2): ident = thread.GetStopReasonDataAtIndex(i) name = points[ident] From 988374576502970a9f87e49994c2fe85e4c2c73f Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 07:57:08 +0800 Subject: [PATCH 65/74] ci(tls): retain bounded debugger step and remote-stop diagnostics --- .github/openclaw/tls-cycle-root.py | 31 +++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/openclaw/tls-cycle-root.py b/.github/openclaw/tls-cycle-root.py index e3289e96bfe0..0f6a72bd3c43 100644 --- a/.github/openclaw/tls-cycle-root.py +++ b/.github/openclaw/tls-cycle-root.py @@ -74,7 +74,7 @@ def prepare(): dsym_sha256=DSYM_SHA, uuid=UUID, scanner_cause_proven=False, retaining_writer_proven=False, claim='original GC32 observations joined to a later snapshot GC; profile is not the production binary', observer_effects=['profile executable', 'outgoing object tags', 'diagnostic imports', - 'snapshot performs an additional full GC', 'debugger stops'], runs=[]) + 'snapshot performs an additional full GC', 'debugger stops', 'circular debugger logging'], runs=[]) try: for name, contents in [('baseline', original), ('materialize', None)]: report = (OUT / (name + '.xml')).resolve() @@ -164,6 +164,17 @@ def inspect(debugger): watch = None decoded_functions = {} + def debugger_command(text): + result = lldb.SBCommandReturnObject() + debugger.GetCommandInterpreter().HandleCommand(text, result) + output, error = result.GetOutput() or '', result.GetError() or '' + assert len(output.encode()) + len(error.encode()) <= 4 * 1024**2, text + return dict(command=text, succeeded=result.Succeeded(), output=output, error=error) + + def debugger_state(): + return [debugger_command(text) for text in + ('thread plan list -i -v all', 'breakpoint list -v', 'watchpoint list -v')] + def save(): write('capture.json', report) (OUT / 'spans.bin').write_bytes(raw) @@ -249,6 +260,10 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): launch.SetDetachOnError(False) error = lldb.SBError() try: + report['log_setup'] = [debugger_command(text) for text in + ('log enable -h circular -b 4096 -s -T -p lldb process thread step break watch', + 'log enable -h circular -b 4096 -s -T -p gdb-remote packets')] + assert all(row['succeeded'] for row in report['log_setup']), 'circular debugger logs unavailable' process = target.Launch(launch, error) assert error.Success() and process.IsValid(), str(error) report['inferior_launched'] = True @@ -342,6 +357,8 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): initial_owner=owner, writes=[], write_mode='always', context_filter='0x10000 <= value < 2**44 and value % 16 == 0; all writes retain values and PC', claim='one physical address; its frame may exit or storage may be reused; unexpected stops remain failures') + report['watch_admission'] = debugger_state() + assert all(row['succeeded'] for row in report['watch_admission']) if report['gc_count'] == 32: owner = module_frame(thread) report['word_watch']['gc32_owner'] = owner @@ -433,6 +450,18 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): report['error'] = dict(type=type(exc).__name__, message=str(exc)[:2000], frames=[dict(file=Path(f.filename).name, line=f.lineno, function=f.name) for f in traceback.extract_tb(exc.__traceback__)]) + try: + # Dump history before inspection commands overwrite the circular packet buffer. + report['failure_logs'] = [debugger_command('log dump ' + channel) + for channel in ('lldb', 'gdb-remote')] + report['failure_debugger_state'] = debugger_state() + if process is not None and process.IsValid() and process.GetState() == lldb.eStateStopped: + report['failure_threads'] = [dict(thread=t.GetThreadID(), frames=frames(t), + stop_instructions=instructions(target.ReadInstructions(t.GetFrameAtIndex(0).GetPCAddress(), 4))) + for t in process if t.GetStopReason() not in (lldb.eStopReasonInvalid, lldb.eStopReasonNone)] + except BaseException as diagnostic_error: + report['failure_diagnostics_error'] = dict(type=type(diagnostic_error).__name__, + message=str(diagnostic_error)[:2000]) raise finally: if process is not None and process.IsValid() and process.GetState() not in (lldb.eStateExited, lldb.eStateDetached): From 1c8fc4bbed1aeb7dac655afc83d0746d047b2fdb Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 08:05:50 +0800 Subject: [PATCH 66/74] ci(tls): bound physical breakpoint sites to the captured collection --- .github/openclaw/tls-cycle-root.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/openclaw/tls-cycle-root.py b/.github/openclaw/tls-cycle-root.py index 0f6a72bd3c43..e9beac28bf06 100644 --- a/.github/openclaw/tls-cycle-root.py +++ b/.github/openclaw/tls-cycle-root.py @@ -227,7 +227,7 @@ def module_frame(thread): assert len(selected) == 1 and selected[0]['fp'] - selected[0]['sp'] == 224 return selected[0] - def point(name, address, mnemonic, operands=None, enabled=True, thread=None): + def point(name, address, mnemonic, operands=None): address = target.ResolveFileAddress(address) if isinstance(address, int) else address instructions = target.ReadInstructions(address, 1) assert instructions.GetSize() == 1 @@ -237,17 +237,10 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): assert instruction.GetOperands(target) == operands bp = target.BreakpointCreateBySBAddress(address) assert bp.GetNumLocations() == 1 - bp.SetEnabled(enabled) - if thread is not None: - bp.SetThreadID(thread) points[bp.GetID()] = name return bp gc = point('gc', 0x10055a780, 'pushq', '%rbp') - scans = [point('span', a, 'pushq', '%rbp', False) for a in (0x103bcf0a0, 0x103bcf1d0)] - gather = point('gather', 0x103c368c0, 'pushq', '%rbp', False) - pre = point('mark-before', 0x103c4b37d, 'lock', enabled=False) - post = point('mark-after', 0x103c4b384, 'jb', enabled=False) report['physical_points'] = {str(k): v for k, v in points.items()} launch = lldb.SBLaunchInfo(['-e', (OUT / 'debugger-child.js').read_text()]) launch.SetWorkingDirectory(str(SOURCE.resolve())) @@ -371,15 +364,19 @@ def point(name, address, mnemonic, operands=None, enabled=True, thread=None): bp.SetThreadID(tid) assert bp.GetNumLocations() == 1 points[bp.GetID()] = 'gc-return' - for b in scans + [gather, pre, post]: - b.SetEnabled(True) + # LLDB1700 stepped through disabled sites before GC32; create them only when needed. + scans = [point('span', a, 'pushq', '%rbp') for a in (0x103bcf0a0, 0x103bcf1d0)] + gather = point('gather', 0x103c368c0, 'pushq', '%rbp') + pre = point('mark-before', 0x103c4b37d, 'lock') + post = point('mark-after', 0x103c4b384, 'jb') + report['physical_points'] = {str(k): v for k, v in points.items()} elif name == 'gc-return': assert active and not pending_spans and not pending_marks report['gc32_completed'] = True active = False for b in scans + [gather, pre, post, gc]: - b.SetEnabled(False) - target.BreakpointDelete(ident) + assert target.BreakpointDelete(b.GetID()) + assert target.BreakpointDelete(ident) report['word_watch']['gc32_return_value'] = word(report['word_watch']['address']) assert target.DeleteWatchpoint(watch.GetID()) elif name == 'gather': From 36c0f839049ff82f4af34f7ef19fff54456beb9c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 08:15:32 +0800 Subject: [PATCH 67/74] ci(tls): bound root history to value-changing stores --- .github/openclaw/tls-cycle-root.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/openclaw/tls-cycle-root.py b/.github/openclaw/tls-cycle-root.py index e9beac28bf06..cc87017f1312 100644 --- a/.github/openclaw/tls-cycle-root.py +++ b/.github/openclaw/tls-cycle-root.py @@ -290,14 +290,14 @@ def point(name, address, mnemonic, operands=None): assert thread.GetStopReasonDataCount() == 1 assert thread.GetStopReasonDataAtIndex(0) == watch.GetID() row = report['word_watch'] - assert len(row['writes']) < 12000 + assert len(row['writes']) < 12000, 'value-change observation limit' frame = thread.GetFrameAtIndex(0) pc = frame.GetPC() value = word(row['address']) event = dict(thread=thread.GetThreadID(), gc_count=report['gc_count'], before=row['last_value'], after=value, stop_pc=pc, stop_file_pc=target.ResolveLoadAddress(pc).GetFileAddress()) - # Capture every write; expensive context is useful only for potential cell pointers. + # Capture every value change; expensive context is useful only for potential cell pointers. if 0x10000 <= value < 2**44 and value % 16 == 0: function, symbol = frame.GetFunction(), frame.GetSymbol() start = function.GetStartAddress() if function.IsValid() else symbol.GetStartAddress() @@ -340,16 +340,16 @@ def point(name, address, mnemonic, operands=None): error = lldb.SBError() options = lldb.SBWatchpointOptions() options.SetWatchpointTypeRead(False) - options.SetWatchpointTypeWrite(lldb.eWatchpointWriteTypeAlways) - assert options.GetWatchpointTypeWrite() == lldb.eWatchpointWriteTypeAlways + options.SetWatchpointTypeWrite(lldb.eWatchpointWriteTypeOnModify) + assert options.GetWatchpointTypeWrite() == lldb.eWatchpointWriteTypeOnModify watch = target.WatchpointCreateByAddress(address, 8, options, error) assert error.Success() and watch.IsValid(), str(error) assert watch.GetWatchAddress() == address and watch.GetWatchSize() == 8 initial = word(address) report['word_watch'] = dict(address=address, initial_value=initial, last_value=initial, - initial_owner=owner, writes=[], write_mode='always', - context_filter='0x10000 <= value < 2**44 and value % 16 == 0; all writes retain values and PC', - claim='one physical address; its frame may exit or storage may be reused; unexpected stops remain failures') + initial_owner=owner, writes=[], write_mode='on_modify', + context_filter='0x10000 <= value < 2**44 and value % 16 == 0; all value changes retain values and PC', + claim='value-changing stores only; equal-value stores may be unreported; one address across frame/storage reuse') report['watch_admission'] = debugger_state() assert all(row['succeeded'] for row in report['watch_admission']) if report['gc_count'] == 32: @@ -510,7 +510,7 @@ def seal(): gc32_return_value_matches=watched['gc32_return_value'] == cell, accepted_root_matches=any(m['accepted'] and m['address'] == watched['address'] for m in matches), observed_writes_of_owner=owner_writes, - claim='observed write accesses; decode the preceding instruction and intended lifetime before proposing a repair') + claim='observed value-changing stores only; equal-value/final writers and intended lifetime need separate proof before a repair') write('join.json', dict(owner_id=owner[0], owner_cell=owner[5], gc32_word_matches=matches, gc32_mark_events=marks, word_watch=watch_join, scanner_cause_proven=False, retaining_writer_proven=False, claim='physical observations; identify the frame writer before proposing a runtime repair')) @@ -521,7 +521,7 @@ def seal(): assert capture['gc32_completed'] and capture['original_failure_preserved'] assert any(m['accepted'] for m in matches) and any(m['new_mark'] for m in marks), 'root not established; no causal claim' assert watch_join['accepted_root_matches'] and watch_join['gc32_return_value_matches'] - assert owner_writes, 'owner value was not observed being written; earlier writer remains unknown' + assert owner_writes, 'owner value was not observed changing; earlier or equal-value writer remains unknown' finally: files = sorted(OUT.iterdir()) assert len(files) <= 32 and all(p.is_file() and not p.is_symlink() for p in files) From 3e0bc52bc860da6f9cdac3dc744d0b6f8a43de58 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 08:26:33 +0800 Subject: [PATCH 68/74] ci(tls): separate raw root history from writer symbol inspection --- .github/openclaw/tls-cycle-root.py | 40 +++++++----------------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/.github/openclaw/tls-cycle-root.py b/.github/openclaw/tls-cycle-root.py index cc87017f1312..6037c6293736 100644 --- a/.github/openclaw/tls-cycle-root.py +++ b/.github/openclaw/tls-cycle-root.py @@ -162,7 +162,6 @@ def inspect(debugger): points = {} dynamic = {} watch = None - decoded_functions = {} def debugger_command(text): result = lldb.SBCommandReturnObject() @@ -273,9 +272,9 @@ def point(name, address, mnemonic, operands=None): report['last_stop'] = dict(id=process.GetStopID(), threads=[dict( thread=t.GetThreadID(), reason=t.GetStopReason(), data=[t.GetStopReasonDataAtIndex(i) for i in range(min(t.GetStopReasonDataCount(), 32))], - description=t.GetStopDescription(512), pc=t.GetFrameAtIndex(0).GetPC(), + pc=t.GetFrameAtIndex(0).GetPC(), sp=t.GetFrameAtIndex(0).GetSP(), fp=t.GetFrameAtIndex(0).GetFP(), - function=t.GetFrameAtIndex(0).GetDisplayFunctionName()) for t in process]) + ) for t in process]) unexpected = [t.GetThreadID() for t in process if t.GetStopReason() not in (lldb.eStopReasonInvalid, lldb.eStopReasonNone, lldb.eStopReasonWatchpoint, lldb.eStopReasonBreakpoint)] @@ -296,30 +295,9 @@ def point(name, address, mnemonic, operands=None): value = word(row['address']) event = dict(thread=thread.GetThreadID(), gc_count=report['gc_count'], before=row['last_value'], after=value, stop_pc=pc, - stop_file_pc=target.ResolveLoadAddress(pc).GetFileAddress()) - # Capture every value change; expensive context is useful only for potential cell pointers. - if 0x10000 <= value < 2**44 and value % 16 == 0: - function, symbol = frame.GetFunction(), frame.GetSymbol() - start = function.GetStartAddress() if function.IsValid() else symbol.GetStartAddress() - image = start.GetModule() - key = (image.GetUUIDString(), start.GetFileAddress()) if image.IsValid() and image.GetUUIDString() else None - cached = decoded_functions.get(key) if key is not None else None - if cached is None: - decoded = function.GetInstructions(target) if function.IsValid() else symbol.GetInstructions(target) - ends = {} - if 0 < decoded.GetSize() <= 16384: - ends = {decoded.GetInstructionAtIndex(i).GetAddress().GetLoadAddress(target) - + decoded.GetInstructionAtIndex(i).GetByteSize(): i - for i in range(decoded.GetSize())} - cached = (decoded, ends) - if key is not None: - decoded_functions[key] = cached - decoded, ends = cached - index = ends.get(pc) - event.update(registers={name: reg(frame, name) for name in - ('rax', 'rbx', 'rcx', 'rdx', 'rsi', 'rdi', 'rbp', 'rsp', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15', 'rflags')}, - frames=frames(thread), preceding_instruction=[] if index is None else instructions(decoded, index, index + 1), - instruction_window=[] if index is None else instructions(decoded, max(0, index - 5), min(decoded.GetSize(), index + 3))) + stop_file_pc=target.ResolveLoadAddress(pc).GetFileAddress(), + registers={name: reg(frame, name) for name in + ('rax', 'rbx', 'rcx', 'rdx', 'rsi', 'rdi', 'rbp', 'rsp', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15', 'rflags')}) row['writes'].append(event) row['last_value'] = value handled = True @@ -348,7 +326,7 @@ def point(name, address, mnemonic, operands=None): initial = word(address) report['word_watch'] = dict(address=address, initial_value=initial, last_value=initial, initial_owner=owner, writes=[], write_mode='on_modify', - context_filter='0x10000 <= value < 2**44 and value % 16 == 0; all value changes retain values and PC', + context='all value changes retain PCs and post-store general registers; decoding and writer-stack capture are separate', claim='value-changing stores only; equal-value stores may be unreported; one address across frame/storage reuse') report['watch_admission'] = debugger_state() assert all(row['succeeded'] for row in report['watch_admission']) @@ -453,7 +431,7 @@ def point(name, address, mnemonic, operands=None): for channel in ('lldb', 'gdb-remote')] report['failure_debugger_state'] = debugger_state() if process is not None and process.IsValid() and process.GetState() == lldb.eStateStopped: - report['failure_threads'] = [dict(thread=t.GetThreadID(), frames=frames(t), + report['failure_threads'] = [dict(thread=t.GetThreadID(), description=t.GetStopDescription(512), frames=frames(t), stop_instructions=instructions(target.ReadInstructions(t.GetFrameAtIndex(0).GetPCAddress(), 4))) for t in process if t.GetStopReason() not in (lldb.eStopReasonInvalid, lldb.eStopReasonNone)] except BaseException as diagnostic_error: @@ -484,7 +462,7 @@ def seal(): owner = next(nodes[i:i + 7] for i in range(0, len(nodes), 7) if nodes[i] in external) assert raw_graph['nodeClassNames'][owner[2]] == 'JSLexicalEnvironment' cell = int(owner[5], 16) - assert 0x10000 <= cell < 2**44 and cell % 16 == 0, 'owner falls outside the detailed-context filter' + assert cell > 0 and cell % 16 == 0 edges = raw_graph['edges'] assert all(edges[i + 1] != owner[0] for i in range(0, len(edges), 4)) assert owner[0] not in raw_graph['roots'][::3] @@ -505,7 +483,7 @@ def seal(): accepted=cell in span['accepted_roots'], physical_frames=owners)) marks = [m for m in capture['marks'] if m['cell'] == cell] watched = capture['word_watch'] - owner_writes = [row for row in watched['writes'] if row['after'] == cell] + owner_writes = [row for row in watched['writes'] if row['after'] == cell and row['before'] != cell] watch_join = dict(address=watched['address'], initial_value_matches=watched['initial_value'] == cell, gc32_return_value_matches=watched['gc32_return_value'] == cell, accepted_root_matches=any(m['accepted'] and m['address'] == watched['address'] for m in matches), From 4a864ccc3f33c7f9ddd12fd2591e69a311f07798 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 09:04:53 +0800 Subject: [PATCH 69/74] test(tls): bound asynchronous physical root capture --- .github/openclaw/tls-cycle-root.py | 111 ++++++++++++++++-- .github/workflows/openclaw-tls-cycle-root.yml | 5 +- 2 files changed, 104 insertions(+), 12 deletions(-) diff --git a/.github/openclaw/tls-cycle-root.py b/.github/openclaw/tls-cycle-root.py index 6037c6293736..2d3e4b6801d3 100644 --- a/.github/openclaw/tls-cycle-root.py +++ b/.github/openclaw/tls-cycle-root.py @@ -7,6 +7,7 @@ import re import runpy import shlex +import signal import struct import subprocess import sys @@ -33,7 +34,10 @@ def sha(path): def write(name, value): - (OUT / name).write_text(json.dumps(value, indent=2) + '\n') + path = OUT / name + partial = path.with_name(path.name + '.partial') + partial.write_text(json.dumps(value, indent=2) + '\n') + partial.replace(path) def environment(): @@ -143,7 +147,7 @@ def prepare(): def inspect(debugger): import lldb - debugger.SetAsync(False) + debugger.SetAsync(True) command = lldb.SBCommandReturnObject() debugger.GetCommandInterpreter().HandleCommand('settings set target.load-script-from-symbol-file false', command) assert command.Succeeded() @@ -174,9 +178,24 @@ def debugger_state(): return [debugger_command(text) for text in ('thread plan list -i -v all', 'breakpoint list -v', 'watchpoint list -v')] + last_saved = time.monotonic() + def save(): + nonlocal last_saved + partial = OUT / 'spans.bin.partial' + partial.write_bytes(raw) + partial.replace(OUT / 'spans.bin') write('capture.json', report) - (OUT / 'spans.bin').write_bytes(raw) + last_saved = time.monotonic() + + def checkpoint(phase, full=False): + report['phase'] = phase + write('progress.json', dict(phase=phase, debugger_pid=os.getpid(), + inferior_pid=report.get('pid'), gc_count=report['gc_count'], + stop_id=report.get('last_stop', {}).get('id'), + writes=len(report.get('word_watch', {}).get('writes', [])))) + if full or time.monotonic() - last_saved >= 5: + save() def memory(address, size): assert address > 0 and 0 <= size <= 4 * 1024**2 @@ -250,20 +269,42 @@ def point(name, address, mnemonic, operands=None): assert launch.AddOpenFileAction(1, str((OUT / 'debugger.stdout').resolve()), False, True) assert launch.AddOpenFileAction(2, str((OUT / 'debugger.stderr').resolve()), False, True) launch.SetDetachOnError(False) + listener = lldb.SBListener('tls-cycle-root') + assert listener.IsValid() + launch.SetListener(listener) + assert launch.GetListener().IsValid() error = lldb.SBError() try: report['log_setup'] = [debugger_command(text) for text in ('log enable -h circular -b 4096 -s -T -p lldb process thread step break watch', 'log enable -h circular -b 4096 -s -T -p gdb-remote packets')] assert all(row['succeeded'] for row in report['log_setup']), 'circular debugger logs unavailable' + deadline, stops, previous_stop = time.monotonic() + 240, 0, 0 + checkpoint('launch', full=True) process = target.Launch(launch, error) assert error.Success() and process.IsValid(), str(error) report['inferior_launched'] = True report['pid'] = process.GetProcessID() - deadline, stops, previous_stop = time.monotonic() + 240, 0, 0 - while process.GetState() not in (lldb.eStateExited, lldb.eStateCrashed, lldb.eStateDetached): + header = module.GetObjectFileHeaderAddress() + report['image_header'] = dict(file=header.GetFileAddress(), load=header.GetLoadAddress(target)) + checkpoint('launched', full=True) + while True: assert time.monotonic() < deadline, 'debugger capture deadline' - assert process.GetState() == lldb.eStateStopped, 'synchronous debugger did not stop' + checkpoint('wait-event') + event = lldb.SBEvent() + if not listener.WaitForEvent(1, event): + continue + if not lldb.SBProcess.EventIsProcessEvent(event): + continue + owner = lldb.SBProcess.GetProcessFromEvent(event) + assert owner.GetUniqueID() == process.GetUniqueID(), 'event belongs to another process' + state = lldb.SBProcess.GetStateFromEvent(event) + if state in (lldb.eStateExited, lldb.eStateCrashed, lldb.eStateDetached): + break + if state != lldb.eStateStopped or lldb.SBProcess.GetRestartedFromEvent(event): + continue + checkpoint('inspect-stop') + assert process.GetState() == lldb.eStateStopped, 'stale stopped event' stop_id = process.GetStopID() assert stop_id > previous_stop, 'duplicate debugger stop would corrupt GC numbering' previous_stop = stop_id @@ -295,7 +336,6 @@ def point(name, address, mnemonic, operands=None): value = word(row['address']) event = dict(thread=thread.GetThreadID(), gc_count=report['gc_count'], before=row['last_value'], after=value, stop_pc=pc, - stop_file_pc=target.ResolveLoadAddress(pc).GetFileAddress(), registers={name: reg(frame, name) for name in ('rax', 'rbx', 'rcx', 'rdx', 'rsi', 'rdi', 'rbp', 'rsp', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15', 'rflags')}) row['writes'].append(event) @@ -415,7 +455,9 @@ def point(name, address, mnemonic, operands=None): if not handled and watch is not None: report['last_stop']['watched_value'] = word(report['word_watch']['address']) assert handled, 'unexpected debugger stop' + checkpoint('resume') assert process.Continue().Success() + checkpoint('terminal', full=True) report['exit_status'] = process.GetExitStatus() report['terminal_state'] = lldb.SBDebugger.StateAsCString(process.GetState()) assert report['gc32_completed'] and report['exit_status'] == 0 @@ -425,6 +467,7 @@ def point(name, address, mnemonic, operands=None): report['error'] = dict(type=type(exc).__name__, message=str(exc)[:2000], frames=[dict(file=Path(f.filename).name, line=f.lineno, function=f.name) for f in traceback.extract_tb(exc.__traceback__)]) + checkpoint('failure', full=True) try: # Dump history before inspection commands overwrite the circular packet buffer. report['failure_logs'] = [debugger_command('log dump ' + channel) @@ -439,15 +482,67 @@ def point(name, address, mnemonic, operands=None): message=str(diagnostic_error)[:2000]) raise finally: + checkpoint('cleanup', full=True) if process is not None and process.IsValid() and process.GetState() not in (lldb.eStateExited, lldb.eStateDetached): report['task_process_killed'] = process.Kill().Success() + report['cleanup_completed'] = True save() +def observe(): + command = ['xcrun', 'lldb', '--batch', '--no-lldbinit', '--no-use-colors', + '-o', 'settings set target.load-script-from-symbol-file false', + '-o', 'script import runpy; runpy.run_path(' + repr(str(Path(__file__))) + ', run_name="__lldb__")'] + receipt = dict(timed_out=False, inferior_cleanup='unproven') + with (OUT / 'debugger.log').open('xb') as log: + process = subprocess.Popen(command, stdin=subprocess.DEVNULL, stdout=log, + stderr=subprocess.STDOUT, start_new_session=True) + receipt['debugger_pid'] = process.pid + write('supervisor.json', receipt) + try: + process.wait(timeout=330) + except subprocess.TimeoutExpired: + receipt['timed_out'] = True + finally: + if process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=10) + receipt['exit_status'] = process.returncode + progress = json.loads((OUT / 'progress.json').read_text()) if (OUT / 'progress.json').is_file() else {} + pid = progress.get('inferior_pid') + if progress.get('debugger_pid') == process.pid and isinstance(pid, int) and pid > 1: + receipt['inferior_pid'] = pid + identity = subprocess.run(['ps', '-p', str(pid), '-o', 'comm='], + text=True, capture_output=True, timeout=5, check=False) + if identity.returncode == 1 and not identity.stdout.strip(): + receipt['inferior_cleanup'] = 'absent' + elif identity.returncode == 0 and identity.stdout.strip() == str((ROOT / 'bun-profile').resolve()): + # Only the admitted inferior created by this debugger can be killed here. + os.kill(pid, signal.SIGKILL) + limit = time.monotonic() + 5 + while time.monotonic() < limit: + try: + os.kill(pid, 0) + except ProcessLookupError: + receipt['inferior_cleanup'] = 'killed' + break + time.sleep(0.1) + write('supervisor.json', receipt) + assert not receipt['timed_out'], 'debugger supervisor deadline; checkpoint retained, result inconclusive' + assert receipt['exit_status'] == 0 and receipt['inferior_cleanup'] in ('absent', 'killed'), receipt + + def seal(): try: receipt = json.loads((OUT / 'receipt.json').read_text()) capture = json.loads((OUT / 'capture.json').read_text()) + supervisor = json.loads((OUT / 'supervisor.json').read_text()) + assert not supervisor['timed_out'] and supervisor['exit_status'] == 0 + assert supervisor['inferior_cleanup'] in ('absent', 'killed') and capture['cleanup_completed'] snapshot = OUT / 'snapshot.json' assert snapshot.is_file() and snapshot.stat().st_size <= 24 * 1024**2 library = runpy.run_path(str(Path(__file__).with_name('tls-cycle-heap.py'))) @@ -512,4 +607,4 @@ def seal(): import lldb inspect(lldb.debugger) elif __name__ == '__main__': - {'prepare': prepare, 'seal': seal}[sys.argv[1]]() + {'prepare': prepare, 'observe': observe, 'seal': seal}[sys.argv[1]]() diff --git a/.github/workflows/openclaw-tls-cycle-root.yml b/.github/workflows/openclaw-tls-cycle-root.yml index 613323712998..600456abf206 100644 --- a/.github/workflows/openclaw-tls-cycle-root.yml +++ b/.github/workflows/openclaw-tls-cycle-root.yml @@ -70,10 +70,7 @@ jobs: shell: bash run: | set -euo pipefail - xcrun lldb --batch --no-lldbinit --no-use-colors \ - -o 'settings set target.load-script-from-symbol-file false' \ - -o 'script import runpy; runpy.run_path("controls/.github/openclaw/tls-cycle-root.py", run_name="__lldb__")' \ - > tls-cycle-root/debugger.log 2>&1 + python3 controls/.github/openclaw/tls-cycle-root.py observe - name: Join the snapshot and seal bounded evidence if: always() shell: bash From 056f35af435a32f842446d4284a921843ee03a27 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 09:16:07 +0800 Subject: [PATCH 70/74] test(tls): serialize bounded physical root capture --- .github/openclaw/tls-cycle-root.py | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/.github/openclaw/tls-cycle-root.py b/.github/openclaw/tls-cycle-root.py index 2d3e4b6801d3..9642d49a5692 100644 --- a/.github/openclaw/tls-cycle-root.py +++ b/.github/openclaw/tls-cycle-root.py @@ -147,7 +147,7 @@ def prepare(): def inspect(debugger): import lldb - debugger.SetAsync(True) + debugger.SetAsync(False) command = lldb.SBCommandReturnObject() debugger.GetCommandInterpreter().HandleCommand('settings set target.load-script-from-symbol-file false', command) assert command.Succeeded() @@ -269,16 +269,13 @@ def point(name, address, mnemonic, operands=None): assert launch.AddOpenFileAction(1, str((OUT / 'debugger.stdout').resolve()), False, True) assert launch.AddOpenFileAction(2, str((OUT / 'debugger.stderr').resolve()), False, True) launch.SetDetachOnError(False) - listener = lldb.SBListener('tls-cycle-root') - assert listener.IsValid() - launch.SetListener(listener) - assert launch.GetListener().IsValid() error = lldb.SBError() try: report['log_setup'] = [debugger_command(text) for text in ('log enable -h circular -b 4096 -s -T -p lldb process thread step break watch', 'log enable -h circular -b 4096 -s -T -p gdb-remote packets')] assert all(row['succeeded'] for row in report['log_setup']), 'circular debugger logs unavailable' + # Cooperative deadline; observe() also bounds blocking debugger calls. deadline, stops, previous_stop = time.monotonic() + 240, 0, 0 checkpoint('launch', full=True) process = target.Launch(launch, error) @@ -288,23 +285,10 @@ def point(name, address, mnemonic, operands=None): header = module.GetObjectFileHeaderAddress() report['image_header'] = dict(file=header.GetFileAddress(), load=header.GetLoadAddress(target)) checkpoint('launched', full=True) - while True: + while process.GetState() not in (lldb.eStateExited, lldb.eStateCrashed, lldb.eStateDetached): assert time.monotonic() < deadline, 'debugger capture deadline' - checkpoint('wait-event') - event = lldb.SBEvent() - if not listener.WaitForEvent(1, event): - continue - if not lldb.SBProcess.EventIsProcessEvent(event): - continue - owner = lldb.SBProcess.GetProcessFromEvent(event) - assert owner.GetUniqueID() == process.GetUniqueID(), 'event belongs to another process' - state = lldb.SBProcess.GetStateFromEvent(event) - if state in (lldb.eStateExited, lldb.eStateCrashed, lldb.eStateDetached): - break - if state != lldb.eStateStopped or lldb.SBProcess.GetRestartedFromEvent(event): - continue checkpoint('inspect-stop') - assert process.GetState() == lldb.eStateStopped, 'stale stopped event' + assert process.GetState() == lldb.eStateStopped, 'synchronous debugger did not stop' stop_id = process.GetStopID() assert stop_id > previous_stop, 'duplicate debugger stop would corrupt GC numbering' previous_stop = stop_id From 16faa52ec209105e8cdd8172c32666bc206f2c20 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 09:27:11 +0800 Subject: [PATCH 71/74] test(tls): join late word writes to accepted scan order --- .github/openclaw/tls-cycle-root.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/openclaw/tls-cycle-root.py b/.github/openclaw/tls-cycle-root.py index 9642d49a5692..0f8530f98a1b 100644 --- a/.github/openclaw/tls-cycle-root.py +++ b/.github/openclaw/tls-cycle-root.py @@ -318,7 +318,7 @@ def point(name, address, mnemonic, operands=None): frame = thread.GetFrameAtIndex(0) pc = frame.GetPC() value = word(row['address']) - event = dict(thread=thread.GetThreadID(), gc_count=report['gc_count'], + event = dict(thread=thread.GetThreadID(), gc_count=report['gc_count'], stop_id=stop_id, before=row['last_value'], after=value, stop_pc=pc, registers={name: reg(frame, name) for name in ('rax', 'rbx', 'rcx', 'rdx', 'rsi', 'rdi', 'rbp', 'rsp', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15', 'rflags')}) @@ -336,7 +336,7 @@ def point(name, address, mnemonic, operands=None): handled = True if name == 'gc': report['gc_count'] += 1 - if report['gc_count'] == 1: + if report['gc_count'] == 30: owner = module_frame(thread) address = owner['sp'] error = lldb.SBError() @@ -349,9 +349,9 @@ def point(name, address, mnemonic, operands=None): assert watch.GetWatchAddress() == address and watch.GetWatchSize() == 8 initial = word(address) report['word_watch'] = dict(address=address, initial_value=initial, last_value=initial, - initial_owner=owner, writes=[], write_mode='on_modify', + initial_owner=owner, start_gc=report['gc_count'], writes=[], write_mode='on_modify', context='all value changes retain PCs and post-store general registers; decoding and writer-stack capture are separate', - claim='value-changing stores only; equal-value stores may be unreported; one address across frame/storage reuse') + claim='GC30 through GC32 only; earlier and equal-value writers and intended lifetime remain unknown') report['watch_admission'] = debugger_state() assert all(row['succeeded'] for row in report['watch_admission']) if report['gc_count'] == 32: @@ -397,7 +397,7 @@ def point(name, address, mnemonic, operands=None): assert end - begin <= 1024**2 and len(raw) + end - begin <= 4 * 1024**2 assert (end - begin) % 8 == 0 # Exact x64 type witness 41247026590: m_roots at 0, m_size at 8. - row = dict(thread=tid, begin=begin, end=end, raw_offset=len(raw), + row = dict(thread=tid, begin=begin, end=end, raw_offset=len(raw), stop_id=stop_id, roots=roots, count_before=word(roots + 8), threads=[dict(id=t.GetThreadID(), frames=frames(t)) for t in process]) raw.extend(memory(begin, end - begin)) @@ -559,14 +559,24 @@ def seal(): if 0 < frame['sp'] <= address < caller['sp'] < 2**64 - 1: owners.append(dict(thread=thread['id'], frame=frame, caller=caller)) matches.append(dict(address=address, span_begin=span['begin'], span_end=span['end'], + span_stop_id=span['stop_id'], accepted=cell in span['accepted_roots'], physical_frames=owners)) marks = [m for m in capture['marks'] if m['cell'] == cell] watched = capture['word_watch'] owner_writes = [row for row in watched['writes'] if row['after'] == cell and row['before'] != cell] - watch_join = dict(address=watched['address'], initial_value_matches=watched['initial_value'] == cell, + accepted_scan_writes = [] + for match in matches: + if not match['accepted'] or match['address'] != watched['address']: + continue + preceding = [row for row in watched['writes'] if row['stop_id'] < match['span_stop_id']] + last = preceding[-1] if preceding else None + accepted_scan_writes.append(dict(span_stop_id=match['span_stop_id'], last_value_change=last)) + watch_join = dict(address=watched['address'], start_gc=watched['start_gc'], + initial_value_matches=watched['initial_value'] == cell, gc32_return_value_matches=watched['gc32_return_value'] == cell, accepted_root_matches=any(m['accepted'] and m['address'] == watched['address'] for m in matches), observed_writes_of_owner=owner_writes, + accepted_scan_writes=accepted_scan_writes, claim='observed value-changing stores only; equal-value/final writers and intended lifetime need separate proof before a repair') write('join.json', dict(owner_id=owner[0], owner_cell=owner[5], gc32_word_matches=matches, gc32_mark_events=marks, word_watch=watch_join, scanner_cause_proven=False, retaining_writer_proven=False, @@ -579,6 +589,9 @@ def seal(): assert any(m['accepted'] for m in matches) and any(m['new_mark'] for m in marks), 'root not established; no causal claim' assert watch_join['accepted_root_matches'] and watch_join['gc32_return_value_matches'] assert owner_writes, 'owner value was not observed changing; earlier or equal-value writer remains unknown' + assert accepted_scan_writes and all(row['last_value_change'] is not None + and row['last_value_change']['after'] == cell and row['last_value_change']['before'] != cell + for row in accepted_scan_writes), 'last observed change before the accepted scan does not establish the owner value' finally: files = sorted(OUT.iterdir()) assert len(files) <= 32 and all(p.is_file() and not p.is_symlink() for p in files) From 97b715d73c1349b302394206a426bc3e63412a3e Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 09:36:49 +0800 Subject: [PATCH 72/74] test(tls): capture bounded context for the first word change --- .github/openclaw/tls-cycle-root.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/openclaw/tls-cycle-root.py b/.github/openclaw/tls-cycle-root.py index 0f8530f98a1b..91c653425034 100644 --- a/.github/openclaw/tls-cycle-root.py +++ b/.github/openclaw/tls-cycle-root.py @@ -322,6 +322,17 @@ def point(name, address, mnemonic, operands=None): before=row['last_value'], after=value, stop_pc=pc, registers={name: reg(frame, name) for name in ('rax', 'rbx', 'rcx', 'rdx', 'rsi', 'rdi', 'rbp', 'rsp', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15', 'rflags')}) + if not row['writes']: + writer_module = frame.GetModule() + symbol = frame.GetSymbol() + begin = symbol.GetStartAddress() if symbol.IsValid() else lldb.SBAddress() + if not begin.IsValid() or not 0 <= pc - begin.GetLoadAddress(target) <= 4096: + begin = frame.GetPCAddress() + event['first_change_context'] = dict( + module=writer_module.GetFileSpec().GetFilename(), uuid=writer_module.GetUUIDString(), + file_pc=frame.GetPCAddress().GetFileAddress(), symbol=symbol.GetName(), + instructions=instructions(target.ReadInstructions(begin, 64)), frames=frames(thread), + claim='first observed value-changing writer only; no original-retainer or lifetime attribution') row['writes'].append(event) row['last_value'] = value handled = True From 8ebb643978cb5e1f4fc71c8ab77c7a66174272f5 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 11:13:01 +0800 Subject: [PATCH 73/74] test(tls): collect all suites before reporting failure --- .github/openclaw/tls-fixture-qualification.py | 36 ++++++++----------- .../openclaw-tls-fixture-qualification.yml | 7 +--- 2 files changed, 16 insertions(+), 27 deletions(-) diff --git a/.github/openclaw/tls-fixture-qualification.py b/.github/openclaw/tls-fixture-qualification.py index 63792b70ad13..7eb66036e343 100644 --- a/.github/openclaw/tls-fixture-qualification.py +++ b/.github/openclaw/tls-fixture-qualification.py @@ -1,4 +1,4 @@ -"""Qualify a test-only source delta on the exact retained native runtime artifacts.""" +"""Collect all canonical TLS suites on the two retained failing native targets.""" import hashlib import json import os @@ -15,17 +15,12 @@ LIBRARY = runpy.run_path(str(Path(__file__).with_name('tls-cycle-checkpoint.py')), run_name='checkpoint_contract') COMMIT, ENGINE, FIXTURE = (LIBRARY[key] for key in ('COMMIT', 'ENGINE', 'FIXTURE')) -TEST_SOURCE = '8918ddaed89d97ae1605dbd1e1149121f2433d04' -FIXTURE_SHA = '5cca33c82b9120f350f8c51cfaba49109caaa4697272628b48e97a9221b0ec04' +FIXTURE_SHA = LIBRARY['FIXTURE_SHA'] ROOT = Path(os.environ['RUNNER_TEMP']) / 'tls-fixture-input' OUT = Path('tls-fixture-qualification') SOURCE = Path('source') SPECS = { 'darwin-x64': (13057470428, 'signed-darwin-x64', 232201333, 'a4d8b786d7bb87d166f757e089ac1705f4ac0701504038ebbecd438c9d6e06d6'), - 'darwin-arm64': (13057590237, 'signed-darwin-arm64', 216129450, 'a93fdf3cf995662876b87348f74c1135a62980910e7423a9750260cdde30080f'), - 'linux-x64': (13056872046, 'build-linux-x64', 222954227, '7ef34a5999d54ec04afd44b8f85041bc33e78a54b8e5dec16c908920a49cb55d'), - 'linux-arm64': (13056617461, 'build-linux-arm64', 222623871, 'c54dbe7ea29a336c76db1dd5ba491bbb117a6c211d938892c62f2f5e74251fcd'), - 'windows-x64': (13056907950, 'test-only-windows', 596111335, '5c81e59c6e6ee0b1a21a9f03b61eda6ff8616d3a1fab631c7e1c1d4899a58bb6'), 'windows-arm64': (13056907950, 'test-only-windows', 596111335, '5c81e59c6e6ee0b1a21a9f03b61eda6ff8616d3a1fab631c7e1c1d4899a58bb6'), } @@ -36,7 +31,10 @@ def sha(path): def clean(): - subprocess.run(['git', '-C', str(SOURCE), 'diff-index', '--quiet', 'HEAD', '--'], check=True) + result = subprocess.run(['git', '-C', str(SOURCE), 'diff-index', '--quiet', 'HEAD', '--']) + if result.returncode: + subprocess.run(['git', '-C', str(SOURCE), 'diff', '--name-status', 'HEAD', '--'], check=True) + result.check_returncode() def main(target): @@ -46,15 +44,9 @@ def main(target): platform = 'win32' if platform == 'windows' else platform assert not any(os.environ.get(k) for k in ('GH_TOKEN', 'GITHUB_TOKEN', 'NODE_OPTIONS', 'BUN_OPTIONS')) head = subprocess.check_output(['git', '-C', str(SOURCE), 'rev-parse', 'HEAD'], text=True).strip() - assert head == TEST_SOURCE - assert subprocess.check_output(['git', '-C', str(SOURCE), 'show', '-s', '--format=%P', 'HEAD'], text=True).strip() == COMMIT - assert subprocess.check_output(['git', '-C', str(SOURCE), 'diff', '--name-only', COMMIT, 'HEAD'], text=True).splitlines() == [FIXTURE] + assert head == COMMIT clean() - original = subprocess.check_output(['git', '-C', str(SOURCE), 'show', COMMIT + ':' + FIXTURE]) - assert hashlib.sha256(original).hexdigest() == LIBRARY['FIXTURE_SHA'] - assert original.count(LIBRARY['ORIGINAL'].encode()) == 1 - candidate = original.replace(LIBRARY['ORIGINAL'].encode(), LIBRARY['CANDIDATE'].encode(), 1) - assert (SOURCE / FIXTURE).read_bytes() == candidate and sha(SOURCE / FIXTURE) == FIXTURE_SHA + assert sha(SOURCE / FIXTURE) == FIXTURE_SHA owner = (SOURCE / 'scripts/openclaw-release/release.ts').read_text() block = re.search(r'export const tlsQualificationFiles = \[(.*?)\] as const;', owner, re.S) assert block @@ -99,11 +91,12 @@ def main(target): signature = dict(required_by_release_contract=required, exit_code=result.returncode, stderr=result.stderr.decode().replace(str(exe), exe.name)) assert not required or result.returncode == 0 - receipt = dict(target=target, runtime_source=COMMIT, test_source=TEST_SOURCE, engine=ENGINE, + receipt = dict(target=target, runtime_source=COMMIT, test_source=COMMIT, engine=ENGINE, artifact_id=artifact, artifact_sha256=digest, product_zip_sha256=sha(product), executable_sha256=sha(exe), facts=facts, fixture_sha256=FIXTURE_SHA, - runtime_diff_from_product_source=[], sole_source_diff=FIXTURE, node_version='26.12.1', - signature=signature, suites=[], claim='unchanged runtime artifacts, exact test-only source correction; no release publication') + runtime_diff_from_product_source=[], source_diff_from_product_source=[], node_version='26.12.1', + signature=signature, suites=[], passed=False, + claim='unchanged source and runtime artifacts; all six suites recorded, failures retained; no release publication') deadline = time.monotonic() + 20 * 60 try: for file in files: @@ -124,9 +117,10 @@ def main(target): assert int(doc.get('skipped')) == skipped and int(doc.get('failures')) == failures row = dict(file=file, exit_code=result.returncode, tests=len(cases), skipped=skipped, failures=failures, errors=errors) receipt['suites'].append(row) - assert result.returncode == 0 and failures == errors == 0 and len(cases) > skipped, row + assert len(cases) > skipped, row assert sha(exe) == receipt['executable_sha256'] - receipt['passed'] = True + receipt['passed'] = all(row['exit_code'] == row['failures'] == row['errors'] == 0 for row in receipt['suites']) + assert receipt['passed'], receipt['suites'] finally: clean() assert sha(SOURCE / FIXTURE) == FIXTURE_SHA diff --git a/.github/workflows/openclaw-tls-fixture-qualification.yml b/.github/workflows/openclaw-tls-fixture-qualification.yml index 2e833eaeeb42..44f419272e30 100644 --- a/.github/workflows/openclaw-tls-fixture-qualification.yml +++ b/.github/workflows/openclaw-tls-fixture-qualification.yml @@ -24,11 +24,7 @@ jobs: max-parallel: 2 matrix: include: - - { target: linux-x64, runner: ubuntu-24.04 } - - { target: linux-arm64, runner: ubuntu-24.04-arm } - { target: darwin-x64, runner: macos-15-intel } - - { target: darwin-arm64, runner: macos-15 } - - { target: windows-x64, runner: windows-2025 } - { target: windows-arm64, runner: windows-11-arm } steps: - name: Preserve canonical source bytes on every host @@ -45,8 +41,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: path: source - ref: 8918ddaed89d97ae1605dbd1e1149121f2433d04 - fetch-depth: 2 + ref: 510fef558b464c58de1209298f638fa476d03eef persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 with: From d8cb2722419e67e46d77e6f7c9d3ea0214441cf1 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 11:23:19 +0800 Subject: [PATCH 74/74] fix(ci): refresh checkout metadata before TLS admission --- .github/openclaw/tls-fixture-qualification.py | 2 ++ .github/workflows/openclaw-tls-fixture-qualification.yml | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/openclaw/tls-fixture-qualification.py b/.github/openclaw/tls-fixture-qualification.py index 7eb66036e343..cec906c47d99 100644 --- a/.github/openclaw/tls-fixture-qualification.py +++ b/.github/openclaw/tls-fixture-qualification.py @@ -31,6 +31,8 @@ def sha(path): def clean(): + # Refresh checkout stat metadata without staging mode/content changes. + subprocess.run(['git', '-C', str(SOURCE), 'update-index', '--refresh'], check=True) result = subprocess.run(['git', '-C', str(SOURCE), 'diff-index', '--quiet', 'HEAD', '--']) if result.returncode: subprocess.run(['git', '-C', str(SOURCE), 'diff', '--name-status', 'HEAD', '--'], check=True) diff --git a/.github/workflows/openclaw-tls-fixture-qualification.yml b/.github/workflows/openclaw-tls-fixture-qualification.yml index 44f419272e30..6e0657b1e06b 100644 --- a/.github/workflows/openclaw-tls-fixture-qualification.yml +++ b/.github/workflows/openclaw-tls-fixture-qualification.yml @@ -24,7 +24,6 @@ jobs: max-parallel: 2 matrix: include: - - { target: darwin-x64, runner: macos-15-intel } - { target: windows-arm64, runner: windows-11-arm } steps: - name: Preserve canonical source bytes on every host