diff --git a/.github/openclaw/tls-config-disassembly.py b/.github/openclaw/tls-config-disassembly.py new file mode 100644 index 000000000000..28488d1223a0 --- /dev/null +++ b/.github/openclaw/tls-config-disassembly.py @@ -0,0 +1,131 @@ +"""Inspect the exact retained shipping profile without executing it.""" +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import stat +import struct +import subprocess +import zipfile + +RUN = 41238111110 +COMMIT = '510fef558b464c58de1209298f638fa476d03eef' +ARTIFACT = 13056617461 +SIZE = 222623871 +DIGEST = 'c54dbe7ea29a336c76db1dd5ba491bbb117a6c211d938892c62f2f5e74251fcd' +INPUT = Path(os.environ['RUNNER_TEMP']) / 'tls-disassembly-input' +OUT = Path('tls-disassembly-proof') +OUT.mkdir(exist_ok=True) + + +def sha(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def selected(archive, wanted, maximum): + entries = archive.infolist() + assert len(entries) <= 32 + names = set() + for entry in entries: + path = PurePosixPath(entry.filename) + assert entry.filename and not path.is_absolute() and '..' not in path.parts + assert '\\' not in entry.filename and '\x00' not in entry.filename + assert str(path) == entry.filename.rstrip('/') + assert str(path) not in names + names.add(str(path)) + mode = entry.external_attr >> 16 + assert stat.S_IFMT(mode) in (0, stat.S_IFREG, stat.S_IFDIR) + assert not entry.flag_bits & 1 + matches = [entry for entry in entries if entry.filename == wanted] + assert len(matches) == 1 + entry = matches[0] + assert not entry.is_dir() and 0 < entry.file_size <= maximum + return entry + + +def extract(archive_path, member, destination, maximum): + with zipfile.ZipFile(archive_path) as archive: + entry = selected(archive, member, maximum) + with archive.open(entry) as source, destination.open('xb') as target: + count = 0 + while chunk := source.read(1024**2): + count += len(chunk) + assert count <= maximum + target.write(chunk) + assert count == entry.file_size + destination.chmod(0o600) + + +def run(name, arguments): + result = subprocess.run(arguments, stdin=subprocess.DEVNULL, + capture_output=True, timeout=120, check=False) + assert len(result.stdout) <= 32 * 1024**2 and len(result.stderr) <= 1024**2 + (OUT / (name + '.stderr')).write_bytes(result.stderr) + assert result.returncode == 0 and not result.stderr, name + return result.stdout.decode('utf-8') + + +archive = INPUT / 'artifact.zip' +assert archive.stat().st_size == SIZE and sha(archive) == DIGEST +metadata = json.loads((INPUT / 'metadata.json').read_text()) +assert metadata['id'] == ARTIFACT and metadata['name'] == 'build-linux-arm64' +assert metadata['size_in_bytes'] == SIZE and metadata['digest'] == 'sha256:' + DIGEST +assert not metadata['expired'] +assert metadata['workflow_run']['id'] == RUN and metadata['workflow_run']['head_sha'] == COMMIT +profile_zip = INPUT / 'profile.zip' +profile = INPUT / 'bun-profile.elf' +extract(archive, 'bun-linux-aarch64-profile.zip', profile_zip, 1024**3) +extract(profile_zip, 'bun-linux-aarch64-profile/bun-profile', profile, 2 * 1024**3) +with profile.open('rb') as stream: + header = stream.read(64) +assert header[:7] == b'\x7fELF\x02\x01\x01' +assert struct.unpack_from(' 0 and 0 < length <= 65536 +assembly = run('objdump', ['llvm-objdump-23', '--disassemble', '--start-address=' + hex(start), + '--stop-address=' + hex(start + length), str(profile)]) +assert re.search(r'^0*' + format(start, 'x') + r' <' + re.escape(name) + r'>:$', assembly, re.M) +instructions = [] +for line in assembly.splitlines(): + match = re.match(r'^\s*([0-9a-f]+):\s+([0-9a-f]{8})\s+(\S+)', line) + if match: + address = int(match[1], 16) + assert start <= address < start + length + assert match[3] != '' + instructions.append(address) +assert instructions and instructions[0] == start +assert instructions == sorted(set(instructions)) +notes = run('build-id', ['llvm-readelf-23', '--notes', str(profile)]) +build_ids = re.findall(r'Build ID: ([0-9a-f]+)', notes) +assert len(build_ids) == 1 +assert sha(archive) == DIGEST +receipt = dict(captured=True, control_flow_review_pending=True, producer_run=RUN, + commit=COMMIT, artifact_id=ARTIFACT, artifact_sha256=DIGEST, + profile_zip_sha256=sha(profile_zip), profile_sha256=sha(profile), + elf_machine='AArch64', build_id=build_ids[0], symbol=rows[0], + instruction_count=len(instructions), llvm_version=version) +(OUT / 'deinit.asm').write_text(assembly.replace(str(profile), 'bun-profile.elf')) +(OUT / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') +(OUT / 'symbol.txt').write_text(' '.join(rows[0]) + '\n') +(OUT / 'build-id.txt').write_text(notes) +files = sorted(OUT.iterdir()) +assert len(files) <= 12 and sum(path.stat().st_size for path in files) <= 2 * 1024**2 +(OUT / 'manifest.json').write_text(json.dumps({ + path.name: dict(bytes=path.stat().st_size, sha256=sha(path)) for path in files +}, indent=2) + '\n') +print(json.dumps(receipt)) diff --git a/.github/openclaw/tls-config-repair.patch b/.github/openclaw/tls-config-repair.patch new file mode 100644 index 000000000000..8b73567aefc3 --- /dev/null +++ b/.github/openclaw/tls-config-repair.patch @@ -0,0 +1,27 @@ +diff --git a/src/runtime/socket/socket_body.rs b/src/runtime/socket/socket_body.rs +index 2752cbc894159817f1557df3e7d0f518895a3a18..5fa5d59042a200a50106415325b8f6d1b4255f83 100644 +--- a/src/runtime/socket/socket_body.rs ++++ b/src/runtime/socket/socket_body.rs +@@ -5118,7 +5118,9 @@ impl DuplexUpgradeContext { + Self::release_connection(this); + return; + } +- this.ssl_config.set(None); // Drop frees. ++ if this.ssl_config.get().is_some() { ++ this.ssl_config.set(None); // Drop frees. ++ } + // Bytes the peer sent while this task was still queued were + // staged by `UpgradedDuplex::on_internal_receive_data` (the + // engine did not exist yet to receive them). Feed them now +@@ -5189,7 +5191,10 @@ impl DuplexUpgradeContext { + debug_assert!(!ctx.native_shutdown_callback.get()); + ctx.tls.set(None); + // Close raced ahead of StartTLS — drop the unconsumed config / ctx. +- ctx.ssl_config.set(None); ++ // An absent config has an inactive payload that must not spill onto the JS stack. ++ if ctx.ssl_config.get().is_some() { ++ ctx.ssl_config.set(None); ++ } + ctx.owned_ctx.set(None); + } + // `UpgradedDuplex` cleanup diff --git a/.github/openclaw/tls-config-repair.py b/.github/openclaw/tls-config-repair.py new file mode 100644 index 000000000000..9c21ed3972a5 --- /dev/null +++ b/.github/openclaw/tls-config-repair.py @@ -0,0 +1,136 @@ +"""Qualify the reviewed empty-config repair without publishing the runtime branch.""" +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import subprocess +import time +from xml.etree import ElementTree + +SOURCE = Path('/work/source') +OUT = Path('/work/repair-proof') +INPUTS = Path(__file__).resolve().parent +BUN = '454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e' +WEBKIT = 'b82bc917e99110b61c3a21202d6def7457aa194c' +OWNER = 'src/runtime/socket/socket_body.rs' +BEFORE = '66121e98f4e8537520f976ff572ef538cc7baedf3e59c8ab2fdfce5fe4bc2afc' +AFTER = '2a40a7df44cfbc9344eb92ced128da282ade137fe2a08f5319d85b23c297dfb0' +PATCH = 'a0fbcc59fae14006b39c60d5cd71d0b71f80f151c6b0eac7d689db5fcbcfc25b' +FIXTURE = 'test/js/bun/net/socket-retention.test.ts' +FIXTURE_SHA = 'd4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2' +BUILD = SOURCE / 'build/tls-empty-config' +ENV = {k: v for k, v in os.environ.items() if not any(x in k.upper() for x in ('TOKEN', 'SECRET', 'PASSWORD'))} +ENV.update(CI='true', CARGO_BUILD_JOBS='2') +DEADLINE = time.monotonic() + 240 * 60 +STEPS = [] + + +def sha(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def git(*args): + return subprocess.check_output(['git', '-C', str(SOURCE), *args], timeout=120, text=True).strip() + + +def save(**facts): + (OUT / 'proof.json').write_text(json.dumps(dict(passed=False, bun=BUN, webkit=WEBKIT, + patch_sha256=PATCH, source_before=BEFORE, source_after=AFTER, steps=STEPS, **facts), indent=2) + '\n') + + +def run(name, argv, seconds): + log = OUT / (name + '.log') + step = dict(name=name, argv=argv, exit_code=None, timed_out=False) + STEPS.append(step) + save() + with log.open('xb') as stream: + process = subprocess.Popen(argv, cwd=SOURCE, env=ENV, stdin=subprocess.DEVNULL, + stdout=stream, stderr=subprocess.STDOUT, start_new_session=True) + started = time.monotonic() + try: + while process.poll() is None: + if time.monotonic() >= min(DEADLINE, started + seconds): + step['timed_out'] = True + raise TimeoutError(name) + assert sum(p.stat().st_size for p in OUT.iterdir() if p.is_file()) <= 64 * 1024**2, 'proof exceeds 64 MiB' + time.sleep(1) + finally: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + stream.flush() + step['exit_code'] = process.returncode + save() + assert process.returncode == 0 and not step['timed_out'], name + return log.read_text(errors='replace') + + +def source_state(adapted): + assert git('rev-parse', 'HEAD') == BUN + assert git('write-tree') == original_tree + assert git('diff', '--no-renames', '--name-only', 'HEAD').splitlines() == ([OWNER] if adapted else []) + assert not git('ls-files', '--others', '--exclude-standard') + assert sha(SOURCE / OWNER) == (AFTER if adapted else BEFORE) + assert sha(SOURCE / FIXTURE) == FIXTURE_SHA + + +OUT.mkdir(exist_ok=True) +original_tree = git('rev-parse', 'HEAD^{tree}') +source_state(False) +assert os.uname().machine == 'aarch64' +assert sha(INPUTS / 'tls-config-repair.patch') == PATCH +save() +run('apply-check', ['git', 'apply', '--check', str(INPUTS / 'tls-config-repair.patch')], 60) +run('apply', ['git', 'apply', str(INPUTS / 'tls-config-repair.patch')], 60) +source_state(True) +run('rustfmt', ['rustup', 'run', 'nightly-2026-09-15', 'rustfmt', '--edition', '2024', + '--config-path', 'rustfmt.toml', '--config', 'skip_children=true', '--check', OWNER], 120) +source_state(True) +run('build', ['bun', 'scripts/build.ts', '--profile=release', '--ci=on', '--os=linux', + '--arch=aarch64', '--abi=gnu', '--build-dir=build/tls-empty-config', '-j2'], 13800) +source_state(True) +binary, profile = BUILD / 'bun', BUILD / 'bun-profile' +assert binary.is_file() and profile.is_file() +identity = json.loads(run('identity', [str(binary), '-e', + 'console.log(JSON.stringify({commit:Bun.revision,webkit:process.versions.webkit,platform:process.platform,arch:process.arch}))'], 60)) +assert identity == dict(commit=BUN, webkit=WEBKIT, platform='linux', arch='arm64'), identity +node = json.loads(run('node', ['node', '-e', + 'console.log(JSON.stringify({node:process.versions.node,bun:process.versions.bun??null,platform:process.platform,arch:process.arch}))'], 60)) +assert node == dict(node='26.12.1', bun=None, platform='linux', arch='arm64'), node +symbols = run('symbols', ['llvm-nm', '--defined-only', '--format=posix', str(profile)], 120) +names = [line.split()[0] for line in symbols.splitlines() if re.fullmatch( + r'_RNv\w*11bun_runtime6socket11socket_body\w*20DuplexUpgradeContext6deinit', line.split()[0])] +assert len(names) == 1, 'repair deinit symbol missing or ambiguous' +run('deinit-disassembly', ['llvm-objdump', '--disassemble-symbols=' + names[0], '--demangle', str(profile)], 120) +files = json.loads(run('test-files', ['bun', '-e', + 'import {tlsQualificationFiles} from "./scripts/openclaw-release/release.ts";console.log(JSON.stringify(tlsQualificationFiles))'], 60)) +assert len(files) == 6 and len(set(files)) == 6 and FIXTURE in files +fixture_hashes = {name: sha(SOURCE / name) for name in files} +cases = [] +for index, name in enumerate(files): + xml = OUT / ('tls-' + str(index) + '.xml') + run('tls-' + str(index), [str(binary), 'test', name, '--reporter=junit', '--reporter-outfile=' + str(xml)], 600) + selected = list(ElementTree.parse(xml).iter('testcase')) + assert selected and all(c.find('failure') is None and c.find('error') is None for c in selected), name + assert any(c.find('skipped') is None for c in selected), name + cases += selected + source_state(True) +for name in ['a TLS Duplex completion retains its owner and is inert after destroy', + 'TLS Duplex cycles collect after startup while a retained origin keeps its wrapper alive']: + selected = [case for case in cases if case.get('name') == name] + assert len(selected) == 1 and selected[0].find('skipped') is None, name +assert len(cases) == 291 and sum(c.find('skipped') is not None for c in cases) == 8 +assert fixture_hashes == {name: sha(SOURCE / name) for name in files} +identities = dict(binary=sha(binary), profile=sha(profile), build_ninja=sha(BUILD / 'build.ninja')) +run('restore', ['git', 'restore', '--source=' + BUN, '--worktree', '--', OWNER], 60) +source_state(False) +save(completed=True, identity=identity, binary_sha256=identities, fixture_sha256=fixture_hashes, + passed_cases=283, skipped_cases=8, source_restored=True) +result = json.loads((OUT / 'proof.json').read_text()) +result['passed'] = True +(OUT / 'proof.json').write_text(json.dumps(result, indent=2) + '\n') diff --git a/.github/openclaw/tls-cycle-checkpoint.py b/.github/openclaw/tls-cycle-checkpoint.py new file mode 100644 index 000000000000..a9be5af85fbd --- /dev/null +++ b/.github/openclaw/tls-cycle-checkpoint.py @@ -0,0 +1,150 @@ +"""Compare two GC checkpoints with identical retained shipping bytes and assertions.""" +import difflib +import hashlib +import json +import os +from pathlib import Path +import re +import runpy +import subprocess +import xml.etree.ElementTree as ET +import zipfile + +ROOT = Path(os.environ['RUNNER_TEMP']) / 'tls-darwin-input' +OUT = Path('tls-cycle-checkpoint') +SOURCE = Path('source') +COMMIT = '510fef558b464c58de1209298f638fa476d03eef' +ENGINE = 'b82bc917e99110b61c3a21202d6def7457aa194c' +ARTIFACT = 13057470428 +DIGEST = 'a4d8b786d7bb87d166f757e089ac1705f4ac0701504038ebbecd438c9d6e06d6' +FIXTURE = 'test/js/bun/net/socket-retention.test.ts' +FIXTURE_SHA = 'd4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2' +CASE = 'TLS Duplex cycles collect after startup while a retained origin keeps its wrapper alive' +ORIGINAL = ''' const dropped = await makePair(false); + for (let i = 0; i < 30; i++) { await turn(); Bun.gc(true); }''' +CANDIDATE = ''' const dropped = await makePair(false); + for (let i = 0; i < 30; i++) { + await new Promise(resolve => setImmediate(() => { Bun.gc(true); resolve(); })); + }''' + + +def sha(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def run(args, **kwargs): + return subprocess.run(args, stdin=subprocess.DEVNULL, capture_output=True, timeout=120, **kwargs) + + +def main(): + OUT.mkdir(exist_ok=True) + assert os.uname().sysname == 'Darwin' and os.uname().machine == 'x86_64' + assert not any(os.environ.get(k) for k in ('GH_TOKEN', 'GITHUB_TOKEN', 'NODE_OPTIONS', 'BUN_OPTIONS')) + assert subprocess.check_output(['git', '-C', str(SOURCE), 'rev-parse', 'HEAD'], text=True).strip() == COMMIT + assert not subprocess.check_output(['git', '-C', str(SOURCE), 'diff', '--name-only', 'HEAD'], text=True) + fixture = SOURCE / FIXTURE + original = fixture.read_bytes() + assert hashlib.sha256(original).hexdigest() == FIXTURE_SHA + assert original.count(ORIGINAL.encode()) == 1 + candidate = original.replace(ORIGINAL.encode(), CANDIDATE.encode(), 1) + assert candidate.count(b'Bun.gc(true)') == original.count(b'Bun.gc(true)') + (OUT / 'fixture.patch').write_text(''.join(difflib.unified_diff(original.decode().splitlines(True), + candidate.decode().splitlines(True), fromfile=FIXTURE, tofile=FIXTURE))) + run_meta = json.loads((ROOT / 'run.json').read_text()) + meta = json.loads((ROOT / 'metadata.json').read_text()) + assert run_meta['id'] == 41238111110 and run_meta['head_sha'] == COMMIT + assert run_meta['path'] == '.github/workflows/openclaw-release.yml' + assert run_meta['event'] == 'workflow_dispatch' and run_meta['status'] == 'completed' + assert run_meta['conclusion'] == 'failure' + assert meta['id'] == ARTIFACT and meta['name'] == 'signed-darwin-x64' + assert meta['size_in_bytes'] == 232201333 and not meta['expired'] + assert meta['digest'] == 'sha256:' + DIGEST and meta['workflow_run']['id'] == 41238111110 + assert meta['workflow_run']['head_sha'] == COMMIT + archive = ROOT / 'artifact.zip' + assert archive.stat().st_size == 232201333 and sha(archive) == DIGEST + library = runpy.run_path(str(Path(__file__).with_name('tls-darwin-capability.py')), run_name='archive_library') + product_zip, exe = ROOT / 'product.zip', ROOT / 'bun' + library['extract'](archive, 'bun-darwin-x64.zip', product_zip, 1024**3) + library['extract'](product_zip, 'bun-darwin-x64/bun', exe, 1024**3) + exe.chmod(0o700) + exe = exe.resolve() + identity = run([str(exe), '-e', 'console.log(JSON.stringify({revision:Bun.revision,webkit:process.versions.webkit,platform:process.platform,arch:process.arch}))'], check=True, text=True) + assert not identity.stderr + facts = json.loads(identity.stdout) + assert facts == dict(revision=COMMIT, webkit=ENGINE, platform='darwin', arch='x64') + signature = run(['codesign', '--verify', '--strict', str(exe)]) + with zipfile.ZipFile(archive) as files: + signature_required = 'darwin-x64.signing.json' in files.namelist() + signature_record = dict(required_by_release_contract=signature_required, + exit_code=signature.returncode, + stdout=signature.stdout.decode().replace(str(exe), 'bun'), + stderr=signature.stderr.decode().replace(str(exe), 'bun')) + if signature_required: + signing = ROOT / 'signing.json' + library['extract'](archive, 'darwin-x64.signing.json', signing, 1024**2) + signature_record['manifest_sha256'] = sha(signing) + (OUT / 'signature.json').write_text(json.dumps(signature_record, indent=2) + '\n') + # The release owner requires this for x64 only when it has a signing manifest. + assert not signature_required or signature.returncode == 0 + receipt = dict(commit=COMMIT, engine=ENGINE, artifact_id=ARTIFACT, artifact_sha256=DIGEST, + product_zip_sha256=sha(product_zip), executable_sha256=sha(exe), facts=facts, + signature=signature_record, + fixture_sha256=FIXTURE_SHA, candidate_sha256=hashlib.sha256(candidate).hexdigest(), + explicit_collections_first_phase=30, assertions_changed=False, runs=[], + claim='paired GC frame/job-phase sensitivity only; no retaining owner or runtime fix established') + try: + for repetition in (1, 2): + for arm, contents in (('baseline', original), ('candidate', candidate)): + fixture.write_bytes(contents) + name = f'{arm}-{repetition}' + report = (OUT / (name + '.xml')).resolve() + assert not report.exists() + argv = [str(exe), 'test', FIXTURE, '--test-name-pattern', '^' + re.escape(CASE) + '$', + '--reporter=junit', '--reporter-outfile=' + str(report)] + result = run(argv, cwd=SOURCE) + assert len(result.stdout) + len(result.stderr) <= 4 * 1024**2 + (OUT / (name + '.stdout')).write_bytes(result.stdout) + (OUT / (name + '.stderr')).write_bytes(result.stderr) + root = ET.parse(report).getroot() + cases = list(root.iter('testcase')) + chosen = [case for case in cases if case.get('name') == CASE] + assert len(chosen) == 1 and not list(root.iter('error')) + case = chosen[0] + assert case.get('file') == FIXTURE and case.find('skipped') is None + assert all(item.find('skipped') is not None and item.find('failure') is None + for item in cases if item is not case) + failures = case.findall('failure') + received = None + if failures: + assert len(failures) == 1 and failures[0].get('type') == 'AssertionError' + message = failures[0].get('message', '') + match = re.search(r'Received: "(\{[^\n]+\})"', message) + assert match, message + received = json.loads(match[1]) + receipt['runs'].append(dict(arm=arm, repetition=repetition, exit_code=result.returncode, + failures=len(failures), received=received, assertions='unchanged')) + (OUT / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + assert sha(exe) == receipt['executable_sha256'] + receipt['repeatable_checkpoint_sensitivity'] = all( + row['exit_code'] == (1 if row['arm'] == 'baseline' else 0) and + row['failures'] == (1 if row['arm'] == 'baseline' else 0) and + (row['arm'] == 'candidate' or row['received'] == dict( + retainedAlive=True, droppedCollected=False, releasedCollected=True)) + for row in receipt['runs']) + finally: + fixture.write_bytes(original) + assert sha(fixture) == FIXTURE_SHA + assert not subprocess.check_output(['git', '-C', str(SOURCE), 'diff', '--name-only', 'HEAD'], text=True) + receipt['source_restored'] = True + (OUT / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + files = sorted(OUT.iterdir()) + assert len(files) <= 24 and sum(p.stat().st_size for p in files) <= 32 * 1024**2 + (OUT / 'manifest.json').write_text(json.dumps({p.name: dict(bytes=p.stat().st_size, sha256=sha(p)) + for p in files}, indent=2) + '\n') + print(json.dumps(receipt)) + assert receipt['repeatable_checkpoint_sensitivity'], 'experiment inconclusive: inspect all arms without changing expectations' + + +if __name__ == '__main__': + main() diff --git a/.github/openclaw/tls-cycle-heap-test.py b/.github/openclaw/tls-cycle-heap-test.py new file mode 100644 index 000000000000..1609ac6ed1dc --- /dev/null +++ b/.github/openclaw/tls-cycle-heap-test.py @@ -0,0 +1,61 @@ +"""Graph evidence must retain roots and distinguish incoming edges from SCC ownership.""" +import copy +from pathlib import Path +import runpy +import unittest + +LIBRARY = runpy.run_path(str(Path(__file__).with_name('tls-cycle-heap.py')), run_name='graph_contract') +PROJECT, TAGS = LIBRARY['project_graph'], LIBRARY['TAGS'] + + +def fixture(): + nodes = [] + for ident in range(10): + nodes.extend((ident, 16, 0, 0, 0, hex(ident * 16), '0x0')) + return dict(version=3, type='GCDebugging', nodes=nodes, + nodeClassNames=['Object'], edgeTypes=['Internal', 'Property', 'Index', 'Variable'], + edgeNames=list(TAGS), labels=['', 'DOMGCOutput'], roots=[1, 0, 0, 9, 1, 0], + edges=[1, 5, 1, 0, 2, 6, 1, 1, 3, 7, 1, 2, 4, 8, 1, 3, + 0, 1, 0, 0, 1, 2, 0, 0, 3, 4, 0, 0, 4, 3, 0, 0, 9, 3, 0, 0]) + + +class GraphContract(unittest.TestCase): + def test_all_roots_and_scc_boundary_are_preserved(self): + result = PROJECT(fixture()) + target = result['targets'][TAGS[2]] + self.assertEqual(result['roots'], [dict(id=1, reason='', reachability=''), + dict(id=9, reason='DOMGCOutput', reachability='')]) + self.assertEqual({node['id'] for node in target['scc']}, {3, 4}) + self.assertEqual({edge['source'] for edge in target['incoming']}, {4, 9}) + self.assertEqual([edge['source'] for edge in target['scc_external_incoming']], [9]) + self.assertEqual(target['root_paths'][0]['root']['reason'], 'DOMGCOutput') + self.assertEqual([(edge['source'], edge['dest']) for edge in target['root_paths'][0]['path']], [(9, 3)]) + self.assertIsNone(target['synthetic_root_path']) + self.assertEqual([(edge['source'], edge['dest']) for edge in + result['targets'][TAGS[1]]['synthetic_root_path']], [(0, 1), (1, 2)]) + + def test_absent_dropped_tag_is_inconclusive(self): + graph = fixture() + del graph['edges'][8:12] + target = PROJECT(graph)['targets'][TAGS[2]] + self.assertFalse(target['present']) + self.assertEqual(target['interpretation'], 'collected or otherwise absent during this diagnostic') + + def test_repeated_edges_do_not_create_a_second_owner(self): + graph = fixture() + graph['edges'].extend([1, 5, 1, 0]) + self.assertEqual(PROJECT(graph)['targets'][TAGS[0]]['node']['id'], 1) + + def test_ambiguous_owner_and_invalid_root_are_rejected(self): + graph = fixture() + other = copy.deepcopy(graph) + other['edges'].extend([2, 5, 1, 0]) + with self.assertRaises(AssertionError): + PROJECT(other) + graph['roots'][0] = 99 + with self.assertRaises(AssertionError): + PROJECT(graph) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/openclaw/tls-cycle-heap.py b/.github/openclaw/tls-cycle-heap.py new file mode 100644 index 000000000000..0370dd4b5953 --- /dev/null +++ b/.github/openclaw/tls-cycle-heap.py @@ -0,0 +1,230 @@ +"""Capture the retained production cycle graph, with snapshot GC kept explicit.""" +from collections import defaultdict, deque +import difflib +import hashlib +import json +import os +from pathlib import Path +import re +import runpy +import subprocess +import xml.etree.ElementTree as ET +import zipfile + +OUT = Path('tls-cycle-heap') +TAGS = tuple('__tls_cycle_' + name for name in + ('control_origin', 'control_handle', 'dropped_origin', 'dropped_handle')) + + +def project_graph(graph): + assert graph['version'] == 3 and graph['type'] == 'GCDebugging' + assert graph['edgeTypes'] == ['Internal', 'Property', 'Index', 'Variable'] + assert len(graph['nodes']) % 7 == len(graph['edges']) % 4 == len(graph['roots']) % 3 == 0 + classes, labels, names = (graph[key] for key in ('nodeClassNames', 'labels', 'edgeNames')) + assert all(isinstance(s, str) for table in (classes, labels, names) for s in table) + nodes = {} + for i in range(0, len(graph['nodes']), 7): + ident, size, cls, flags, label, cell, wrapped = graph['nodes'][i:i + 7] + assert all(type(n) is int and n >= 0 for n in (ident, size, cls, flags, label)) + assert ident not in nodes and cls < len(classes) and label < len(labels) + assert all(isinstance(a, str) and re.fullmatch(r'0x[0-9a-fA-F]+', a) for a in (cell, wrapped)) + nodes[ident] = dict(id=ident, bytes=size, type=classes[cls], label=labels[label], + flags=flags, cell=cell, wrapped=wrapped) + assert 0 in nodes + incoming, outgoing, tagged = defaultdict(list), defaultdict(list), defaultdict(set) + edges = [] + for i in range(0, len(graph['edges']), 4): + source, dest, kind, data = graph['edges'][i:i + 4] + assert all(type(n) is int and n >= 0 for n in (source, dest, kind, data)) + assert source in nodes and dest in nodes and kind < 4 + if kind in (1, 3): + assert data < len(names) + edge = dict(source=source, dest=dest, type=graph['edgeTypes'][kind], + name=names[data] if kind in (1, 3) else data) + edges.append(edge) + incoming[dest].append(edge) + outgoing[source].append(edge) + if kind == 1 and edge['name'] in TAGS: + tagged[edge['name']].add(source) + roots = [] + for i in range(0, len(graph['roots']), 3): + ident, reason, reachability = graph['roots'][i:i + 3] + assert all(type(n) is int and n >= 0 for n in (ident, reason, reachability)) + assert ident in nodes and reason < len(labels) and reachability < len(labels) + roots.append(dict(id=ident, reason=labels[reason], reachability=labels[reachability])) + assert all(len(tagged[name]) == 1 for name in TAGS[:2]), 'missing or ambiguous live-control tags' + assert all(len(tagged[name]) <= 1 for name in TAGS), 'ambiguous diagnostic tag' + + def reachable(start, adjacency, field): + seen, queue, toward = {start}, deque([start]), {} + while queue: + node = queue.popleft() + for edge in adjacency[node]: + other = edge[field] + if other not in seen: + seen.add(other) + toward[other] = edge + queue.append(other) + return seen, toward + + def path(start, target, toward): + result = [] + while start != target: + edge = toward[start] + result.append(edge) + start = edge['dest'] + return result + + targets = {} + for tag in TAGS: + if not tagged[tag]: + targets[tag] = dict(present=False, interpretation='collected or otherwise absent during this diagnostic') + continue + target = next(iter(tagged[tag])) + ancestors, toward = reachable(target, incoming, 'source') + descendants, _ = reachable(target, outgoing, 'dest') + component = ancestors & descendants + targets[tag] = dict(present=True, node=nodes[target], incoming=incoming[target], + direct_roots=[root for root in roots if root['id'] == target], + root_paths=[dict(root=root, path=path(root['id'], target, toward)) + for root in roots if root['id'] in ancestors], + synthetic_root_path=path(0, target, toward) if 0 in ancestors else None, + scc=[nodes[ident] for ident in sorted(component)], + scc_external_incoming=[edge for ident in sorted(component) for edge in incoming[ident] + if edge['source'] not in component]) + return dict(nodes=len(nodes), edges=len(edges), roots=roots, targets=targets, + claim='snapshot-GC graph with all recorded roots; no same-process scanner or writer proof') + + +def main(): + library = runpy.run_path(str(Path(__file__).with_name('tls-cycle-checkpoint.py')), run_name='checkpoint_contract') + source, root, commit, engine, fixture_name, case_name = ( + library[name] for name in ('SOURCE', 'ROOT', 'COMMIT', 'ENGINE', 'FIXTURE', 'CASE')) + sha, run = library['sha'], library['run'] + assert os.uname().sysname == 'Darwin' and os.uname().machine == 'x86_64' + assert not any(os.environ.get(key) for key in ('GH_TOKEN', 'GITHUB_TOKEN', 'NODE_OPTIONS', 'BUN_OPTIONS')) + assert subprocess.check_output(['git', '-C', str(source), 'rev-parse', 'HEAD'], text=True).strip() == commit + assert not subprocess.check_output(['git', '-C', str(source), 'diff', '--name-only', 'HEAD'], text=True) + fixture = source / fixture_name + original = fixture.read_bytes() + assert hashlib.sha256(original).hexdigest() == library['FIXTURE_SHA'] + producer, artifact = (json.loads((root / name).read_text()) for name in ('run.json', 'metadata.json')) + assert producer['id'] == 41238111110 and producer['head_sha'] == commit + assert producer['path'] == '.github/workflows/openclaw-release.yml' and producer['event'] == 'workflow_dispatch' + assert producer['status'] == 'completed' and producer['conclusion'] == 'failure' + assert artifact['id'] == library['ARTIFACT'] and artifact['name'] == 'signed-darwin-x64' and not artifact['expired'] + assert artifact['size_in_bytes'] == 232201333 and artifact['digest'] == 'sha256:' + library['DIGEST'] + assert artifact['workflow_run']['id'] == producer['id'] and artifact['workflow_run']['head_sha'] == commit + archive = root / 'artifact.zip' + assert archive.stat().st_size == artifact['size_in_bytes'] and sha(archive) == library['DIGEST'] + extractor = runpy.run_path(str(Path(__file__).with_name('tls-darwin-capability.py')), run_name='archive_library')['extract'] + product, exe = root / 'product.zip', root / 'bun' + extractor(archive, 'bun-darwin-x64.zip', product, 1024**3) + extractor(product, 'bun-darwin-x64/bun', exe, 1024**3) + exe.chmod(0o700) + exe = exe.resolve() + executable_sha = '3b8fd7a18caf0dd8365b1672483d6651f4cb2b6d02bb6b5f704fb6888f886310' + assert sha(exe) == executable_sha + for name in ('home', 'tmp'): + (root / name).mkdir() + execution_env = dict(PATH=os.environ['PATH'], HOME=str(root / 'home'), + TMPDIR=str(root / 'tmp') + '/', CI='true', LANG='en_US.UTF-8') + identity = run([str(exe), '-e', 'console.log(JSON.stringify({revision:Bun.revision,webkit:process.versions.webkit,platform:process.platform,arch:process.arch}))'], check=True, text=True, env=execution_env) + assert not identity.stderr + facts = json.loads(identity.stdout) + assert facts == dict(revision=commit, webkit=engine, platform='darwin', arch='x64') + signature = run(['codesign', '--verify', '--strict', str(exe)]) + with zipfile.ZipFile(archive) as files: + required = 'darwin-x64.signing.json' in files.namelist() + assert not required or signature.returncode == 0 + signature_record = dict(required_by_release_contract=required, exit_code=signature.returncode, + stderr=signature.stderr.decode().replace(str(exe), 'bun')) + (OUT / 'signature.json').write_text(json.dumps(signature_record, indent=2) + '\n') + start = original.index(('test("' + case_name + '"').encode()) + end = original.index(b'\nfor (const destroyBeforeCompletion', start) + chunk = original[start:end].decode() + preload = ' const tls = require("node:tls");' + imports = ' const { writeFileSync } = require("node:fs");\n const { generateHeapSnapshotForDebugging } = require("bun:jsc");\n' + tags = ''' Object.defineProperty(clientSide, retain ? "__tls_cycle_control_origin" : "__tls_cycle_dropped_origin", { value: Object.create(null) }); + Object.defineProperty(client._handle, retain ? "__tls_cycle_control_handle" : "__tls_cycle_dropped_handle", { value: Object.create(null) }); +''' + before_return = ' return { weak: new WeakRef(client._handle), origin: retain ? clientSide : null };' + checkpoint = ' for (let i = 0; i < 30; i++) { await turn(); Bun.gc(true); }' + assert chunk.count(preload) == chunk.count(before_return) == 1 and chunk.count(checkpoint) == 2 + tagged = chunk.replace(preload, imports + preload).replace(before_return, tags + before_return) + snapshot = (OUT / 'snapshot.json').resolve() + capture = ('\n const snapshotText = JSON.stringify(generateHeapSnapshotForDebugging());\n' + ' if (Buffer.byteLength(snapshotText) > 24 * 1024**2) throw new Error("snapshot evidence exceeds 24 MiB");\n' + ' writeFileSync(' + json.dumps(os.path.relpath(snapshot, source.resolve())) + ', snapshotText);') + captured = tagged.replace(checkpoint, checkpoint + capture, 1) + arms = [('baseline', original), ('tagged-control', original[:start] + tagged.encode() + original[end:]), + ('snapshot', original[:start] + captured.encode() + original[end:])] + receipt = dict(runtime_source=commit, engine=engine, executable_sha256=executable_sha, + artifact_id=artifact['id'], artifact_sha256=library['DIGEST'], product_zip_sha256=sha(product), + facts=facts, fixture_sha256=library['FIXTURE_SHA'], signature=signature_record, runs=[], + observer_effects=['tags change allocation and object shapes', 'snapshot API performs an additional full GC', + 'diagnostic imports and snapshot serialization add work'], + execution_environment_keys=sorted(execution_env), execution_home='empty task-owned directory', + unchanged_checkpoint_claim=False, scanner_cause_proven=False) + try: + for arm, contents in arms: + fixture.write_bytes(contents) + if arm != 'baseline': + (OUT / (arm + '.patch')).write_text(''.join(difflib.unified_diff( + original.decode().splitlines(True), contents.decode().splitlines(True), + fromfile=fixture_name, tofile=fixture_name))) + report = (OUT / (arm + '.xml')).resolve() + assert not report.exists() + result = run([str(exe), 'test', fixture_name, '--test-name-pattern', '^' + re.escape(case_name) + '$', + '--reporter=junit', '--reporter-outfile=' + str(report)], cwd=source, env=execution_env) + assert len(result.stdout) + len(result.stderr) <= 4 * 1024**2 + (OUT / (arm + '.stdout')).write_bytes(result.stdout) + (OUT / (arm + '.stderr')).write_bytes(result.stderr) + doc = ET.parse(report).getroot() + cases = list(doc.iter('testcase')) + assert doc.tag == 'testsuites' and int(doc.get('tests')) == len(cases) + assert int(doc.get('failures')) == len(list(doc.iter('failure'))) + assert int(doc.get('skipped')) == len(list(doc.iter('skipped'))) + selected = [case for case in cases if case.get('name') == case_name] + assert len(selected) == 1 and not list(doc.iter('error')) + selected = selected[0] + assert selected.get('file') == fixture_name and selected.find('skipped') is None + assert all(case.find('skipped') is not None and case.find('failure') is None + for case in cases if case is not selected) + failures = selected.findall('failure') + received = None + if failures: + assert len(failures) == 1 and failures[0].get('type') == 'AssertionError' + match = re.search(r'Received: "(\{[^\n]+\})"', failures[0].get('message', '')) + assert match + received = json.loads(match[1]) + row = dict(arm=arm, exit_code=result.returncode, failures=len(failures), received=received, + fixture_sha256=hashlib.sha256(contents).hexdigest(), assertions='unchanged') + receipt['runs'].append(row) + assert sha(exe) == executable_sha + if arm != 'snapshot': + assert result.returncode == 1 and len(failures) == 1 and received == dict( + retainedAlive=True, droppedCollected=False, releasedCollected=True), row + else: + assert result.returncode == (1 if failures else 0) + assert snapshot.is_file() and snapshot.stat().st_size <= 24 * 1024**2 + graph = project_graph(json.loads(snapshot.read_text())) + (OUT / 'graph.json').write_text(json.dumps(graph, indent=2) + '\n') + receipt['snapshot_sha256'] = sha(snapshot) + receipt['graph_captured'] = True + finally: + fixture.write_bytes(original) + assert sha(fixture) == library['FIXTURE_SHA'] + assert not subprocess.check_output(['git', '-C', str(source), 'diff', '--name-only', 'HEAD'], text=True) + receipt['source_restored'] = True + (OUT / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + files = sorted(OUT.iterdir()) + assert len(files) <= 24 and all(p.is_file() and not p.is_symlink() for p in files) + assert sum(p.stat().st_size for p in files) <= 32 * 1024**2 + (OUT / 'manifest.json').write_text(json.dumps({p.name: dict(bytes=p.stat().st_size, sha256=sha(p)) + for p in files}, indent=2) + '\n') + print(json.dumps(receipt)) + + +if __name__ == '__main__': + main() diff --git a/.github/openclaw/tls-cycle-root.py b/.github/openclaw/tls-cycle-root.py new file mode 100644 index 000000000000..91c653425034 --- /dev/null +++ b/.github/openclaw/tls-cycle-root.py @@ -0,0 +1,618 @@ +"""Join one original checkpoint's physical observations to its later heap graph.""" +import difflib +import hashlib +import json +import os +from pathlib import Path +import re +import runpy +import shlex +import signal +import struct +import subprocess +import sys +import time +import traceback +from xml.etree import ElementTree + +OUT = Path('tls-cycle-root') +SOURCE = Path('source') +ROOT = Path(os.environ['RUNNER_TEMP']) / 'tls-darwin-input' +FIXTURE = 'test/js/bun/net/socket-retention.test.ts' +CASE = 'TLS Duplex cycles collect after startup while a retained origin keeps its wrapper alive' +SOURCE_SHA = '510fef558b464c58de1209298f638fa476d03eef' +FIXTURE_SHA = 'd4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2' +PROFILE_SHA = '52f2048920d8b31ef4471e121cc06ad7b4828761a2c3278349f66ed9b4acaf2b' +DSYM_SHA = '45aa427a18e5f97dda7a9fb23641ee066ecc5084de26d121b54530f62c41b0c6' +UUID = '4c4c4432-5555-3144-a125-d2b0a4297913' +EXPECTED = dict(retainedAlive=True, droppedCollected=False, releasedCollected=True) + + +def sha(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def write(name, value): + path = OUT / name + partial = path.with_name(path.name + '.partial') + partial.write_text(json.dumps(value, indent=2) + '\n') + partial.replace(path) + + +def environment(): + return dict(PATH=os.environ['PATH'], HOME=str(ROOT / 'empty-home'), + TMPDIR=str(ROOT / 'child-tmp'), CI='true', LANG='en_US.UTF-8') + + +def run(args, name, **kwargs): + result = subprocess.run(args, stdin=subprocess.DEVNULL, capture_output=True, timeout=120, + cwd=SOURCE, env=environment(), **kwargs) + assert len(result.stdout) + len(result.stderr) <= 4 * 1024**2 + (OUT / (name + '.stdout')).write_bytes(result.stdout) + (OUT / (name + '.stderr')).write_bytes(result.stderr) + return result + + +def prepare(): + assert os.uname().sysname == 'Darwin' and os.uname().machine == 'x86_64' + assert not any(os.environ.get(k) for k in ('GH_TOKEN', 'GITHUB_TOKEN', 'NODE_OPTIONS', 'BUN_OPTIONS')) + library = runpy.run_path(str(Path(__file__).with_name('tls-darwin-capability.py'))) + library['prepare']() + write('inputs.json', json.loads(Path('tls-darwin-proof/inputs.json').read_text())) + assert sha(ROOT / 'bun-profile') == PROFILE_SHA and sha(ROOT / 'bun-profile.dSYM') == DSYM_SHA + exe = (ROOT / 'bun-profile').resolve() + exe.chmod(0o700) + for name in ('empty-home', 'child-tmp'): + (ROOT / name).mkdir() + identity = run([str(exe), '-e', 'console.log(JSON.stringify({revision:Bun.revision,webkit:process.versions.webkit,platform:process.platform,arch:process.arch}))'], 'identity') + assert identity.returncode == 0 and not identity.stderr + assert json.loads(identity.stdout) == dict(revision=SOURCE_SHA, + webkit='b82bc917e99110b61c3a21202d6def7457aa194c', platform='darwin', arch='x64') + assert subprocess.check_output(['git', '-C', str(SOURCE), 'rev-parse', 'HEAD'], text=True).strip() == SOURCE_SHA + assert not subprocess.check_output(['git', '-C', str(SOURCE), 'diff', '--name-only', 'HEAD'], text=True) + fixture = SOURCE / FIXTURE + original = fixture.read_bytes() + assert hashlib.sha256(original).hexdigest() == FIXTURE_SHA + receipt = dict(source=SOURCE_SHA, fixture_sha256=FIXTURE_SHA, profile_sha256=PROFILE_SHA, + dsym_sha256=DSYM_SHA, uuid=UUID, scanner_cause_proven=False, retaining_writer_proven=False, + claim='original GC32 observations joined to a later snapshot GC; profile is not the production binary', + observer_effects=['profile executable', 'outgoing object tags', 'diagnostic imports', + 'snapshot performs an additional full GC', 'debugger stops', 'circular debugger logging'], runs=[]) + try: + for name, contents in [('baseline', original), ('materialize', None)]: + report = (OUT / (name + '.xml')).resolve() + if contents is None: + text = original.decode() + start = text.index('test(' + json.dumps(CASE)) + begin = text.index(' `', start) + end = text.index(' `,', begin + 7) + 6 + child = (OUT / 'original-child.js').resolve() + prefix = ' ((value) => { require("node:fs").writeFileSync(' + json.dumps(str(child)) + ', value); return value; })(`' + contents = (text[:begin] + prefix + text[begin + 7:end] + '`),' + text[end + 2:]).encode() + (OUT / 'materialize.patch').write_text(''.join(difflib.unified_diff( + original.decode().splitlines(True), contents.decode().splitlines(True), + fromfile=FIXTURE, tofile=FIXTURE))) + fixture.write_bytes(contents) + result = run([str(exe), 'test', FIXTURE, '--test-name-pattern', '^' + re.escape(CASE) + '$', + '--reporter=junit', '--reporter-outfile=' + str(report)], name) + doc = ElementTree.parse(report).getroot() + selected = [c for c in doc.iter('testcase') if c.get('name') == CASE] + assert len(selected) == 1 and not list(doc.iter('error')) + case = selected[0] + assert case.get('file') == FIXTURE and case.find('skipped') is None + assert all(c.find('skipped') is not None and c.find('failure') is None + for c in doc.iter('testcase') if c is not case) + failures = case.findall('failure') + assert result.returncode == 1 and len(failures) == 1 + match = re.search(r'Received: "(\{[^\n]+\})"', failures[0].get('message', '')) + assert match and json.loads(match[1]) == EXPECTED + receipt['runs'].append(dict(arm=name, exit_code=1, received=EXPECTED)) + child = (OUT / 'original-child.js').read_text() + template = original.decode()[begin + 7:end] + before, after = template.split('${JSON.stringify(tlsCert)}') + assert child.startswith(before) and child.endswith(after) + cert = json.loads(child[len(before):len(child) - len(after)]) + harness = (SOURCE / 'test/harness.ts').read_text() + cert_source = harness.split('export const tls = Object.freeze({', 1)[1].split('\n});', 1)[0] + pairs = re.findall(r'^ (\w+): ("(?:[^"\\]|\\.)*"),?$', cert_source, re.M) + assert dict((k, json.loads(v)) for k, v in pairs) == cert and set(cert) == {'cert', 'key'} + preload = ' const tls = require("node:tls");' + before_return = ' return { weak: new WeakRef(client._handle), origin: retain ? clientSide : null };' + checkpoint = ' for (let i = 0; i < 30; i++) { await turn(); Bun.gc(true); }' + tags = ''' Object.defineProperty(clientSide, retain ? "__tls_cycle_control_origin" : "__tls_cycle_dropped_origin", { value: Object.create(null) }); + Object.defineProperty(client._handle, retain ? "__tls_cycle_control_handle" : "__tls_cycle_dropped_handle", { value: Object.create(null) }); +''' + assert child.count(preload) == child.count(before_return) == 1 and child.count(checkpoint) == 2 + tagged = child.replace(preload, ' const { generateHeapSnapshotForDebugging } = require("bun:jsc");\n const { writeFileSync } = require("node:fs");\n' + preload) + tagged = tagged.replace(before_return, tags + before_return) + for name, code in [('direct-baseline', child), ('tagged-control', tagged)]: + result = run([str(exe), '-e', code], name) + assert result.returncode == 0 and json.loads(result.stdout) == EXPECTED + receipt['runs'].append(dict(arm=name, exit_code=0, received=EXPECTED)) + snapshot = (OUT / 'snapshot.json').resolve() + insertion = '\n const snapshotText = JSON.stringify(generateHeapSnapshotForDebugging());\n if (Buffer.byteLength(snapshotText) > 24 * 1024**2) throw new Error("snapshot too large");\n writeFileSync(' + json.dumps(str(snapshot)) + ', snapshotText);' + captured = tagged.replace(checkpoint, checkpoint + insertion, 1) + assert captured.count('Bun.gc(true)') == child.count('Bun.gc(true)') + (OUT / 'debugger-child.js').write_text(captured) + receipt['original_child_sha256'] = hashlib.sha256(child.encode()).hexdigest() + receipt['debugger_child_sha256'] = hashlib.sha256(captured.encode()).hexdigest() + finally: + fixture.write_bytes(original) + assert sha(fixture) == FIXTURE_SHA + assert not subprocess.check_output(['git', '-C', str(SOURCE), 'diff', '--name-only', 'HEAD'], text=True) + receipt['source_restored'] = True + write('receipt.json', receipt) + + +def inspect(debugger): + import lldb + debugger.SetAsync(False) + command = lldb.SBCommandReturnObject() + debugger.GetCommandInterpreter().HandleCommand('settings set target.load-script-from-symbol-file false', command) + assert command.Succeeded() + target = debugger.CreateTarget(str((ROOT / 'bun-profile').resolve())) + assert target.IsValid() and target.GetTriple().startswith('x86_64-apple-') + debugger.GetCommandInterpreter().HandleCommand('target symbols add ' + shlex.quote(str((ROOT / 'bun-profile.dSYM').resolve())), command) + assert command.Succeeded(), command.GetError() + module = target.FindModule(target.GetExecutable()) + assert module.GetUUIDString().lower() == UUID + report = dict(gc_count=0, gc32_completed=False, spans=[], marks=[], thread_states=[], + scanner_cause_proven=False, retaining_writer_proven=False, inferior_launched=False) + process = None + pending_spans, pending_marks = {}, {} + raw = bytearray() + active = False + points = {} + dynamic = {} + watch = None + + def debugger_command(text): + result = lldb.SBCommandReturnObject() + debugger.GetCommandInterpreter().HandleCommand(text, result) + output, error = result.GetOutput() or '', result.GetError() or '' + assert len(output.encode()) + len(error.encode()) <= 4 * 1024**2, text + return dict(command=text, succeeded=result.Succeeded(), output=output, error=error) + + def debugger_state(): + return [debugger_command(text) for text in + ('thread plan list -i -v all', 'breakpoint list -v', 'watchpoint list -v')] + + last_saved = time.monotonic() + + def save(): + nonlocal last_saved + partial = OUT / 'spans.bin.partial' + partial.write_bytes(raw) + partial.replace(OUT / 'spans.bin') + write('capture.json', report) + last_saved = time.monotonic() + + def checkpoint(phase, full=False): + report['phase'] = phase + write('progress.json', dict(phase=phase, debugger_pid=os.getpid(), + inferior_pid=report.get('pid'), gc_count=report['gc_count'], + stop_id=report.get('last_stop', {}).get('id'), + writes=len(report.get('word_watch', {}).get('writes', [])))) + if full or time.monotonic() - last_saved >= 5: + save() + + def memory(address, size): + assert address > 0 and 0 <= size <= 4 * 1024**2 + if size == 0: + return b'' + error = lldb.SBError() + data = process.ReadMemory(address, size, error) + assert error.Success() and len(data) == size, (hex(address), size, str(error)) + return data + + def word(address): + return struct.unpack(' previous_stop, 'duplicate debugger stop would corrupt GC numbering' + previous_stop = stop_id + stops += 1 + assert stops <= 30000 + report['last_stop'] = dict(id=process.GetStopID(), threads=[dict( + thread=t.GetThreadID(), reason=t.GetStopReason(), + data=[t.GetStopReasonDataAtIndex(i) for i in range(min(t.GetStopReasonDataCount(), 32))], + pc=t.GetFrameAtIndex(0).GetPC(), + sp=t.GetFrameAtIndex(0).GetSP(), fp=t.GetFrameAtIndex(0).GetFP(), + ) for t in process]) + unexpected = [t.GetThreadID() for t in process if t.GetStopReason() not in + (lldb.eStopReasonInvalid, lldb.eStopReasonNone, + lldb.eStopReasonWatchpoint, lldb.eStopReasonBreakpoint)] + if unexpected: + if watch is not None: + report['last_stop']['watched_value'] = word(report['word_watch']['address']) + raise AssertionError('unexpected debugger stop on threads ' + str(unexpected)) + handled = False + for thread in process: + if thread.GetStopReason() == lldb.eStopReasonWatchpoint: + assert watch is not None and watch.IsEnabled() + assert thread.GetStopReasonDataCount() == 1 + assert thread.GetStopReasonDataAtIndex(0) == watch.GetID() + row = report['word_watch'] + assert len(row['writes']) < 12000, 'value-change observation limit' + frame = thread.GetFrameAtIndex(0) + pc = frame.GetPC() + value = word(row['address']) + event = dict(thread=thread.GetThreadID(), gc_count=report['gc_count'], stop_id=stop_id, + before=row['last_value'], after=value, stop_pc=pc, + registers={name: reg(frame, name) for name in + ('rax', 'rbx', 'rcx', 'rdx', 'rsi', 'rdi', 'rbp', 'rsp', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15', 'rflags')}) + if not row['writes']: + writer_module = frame.GetModule() + symbol = frame.GetSymbol() + begin = symbol.GetStartAddress() if symbol.IsValid() else lldb.SBAddress() + if not begin.IsValid() or not 0 <= pc - begin.GetLoadAddress(target) <= 4096: + begin = frame.GetPCAddress() + event['first_change_context'] = dict( + module=writer_module.GetFileSpec().GetFilename(), uuid=writer_module.GetUUIDString(), + file_pc=frame.GetPCAddress().GetFileAddress(), symbol=symbol.GetName(), + instructions=instructions(target.ReadInstructions(begin, 64)), frames=frames(thread), + claim='first observed value-changing writer only; no original-retainer or lifetime attribution') + row['writes'].append(event) + row['last_value'] = value + handled = True + continue + if thread.GetStopReason() != lldb.eStopReasonBreakpoint: + continue + frame, tid = thread.GetFrameAtIndex(0), thread.GetThreadID() + assert thread.GetStopReasonDataCount() > 0 and thread.GetStopReasonDataCount() % 2 == 0 + for i in range(0, thread.GetStopReasonDataCount(), 2): + ident = thread.GetStopReasonDataAtIndex(i) + name = points[ident] + handled = True + if name == 'gc': + report['gc_count'] += 1 + if report['gc_count'] == 30: + owner = module_frame(thread) + address = owner['sp'] + error = lldb.SBError() + options = lldb.SBWatchpointOptions() + options.SetWatchpointTypeRead(False) + options.SetWatchpointTypeWrite(lldb.eWatchpointWriteTypeOnModify) + assert options.GetWatchpointTypeWrite() == lldb.eWatchpointWriteTypeOnModify + watch = target.WatchpointCreateByAddress(address, 8, options, error) + assert error.Success() and watch.IsValid(), str(error) + assert watch.GetWatchAddress() == address and watch.GetWatchSize() == 8 + initial = word(address) + report['word_watch'] = dict(address=address, initial_value=initial, last_value=initial, + initial_owner=owner, start_gc=report['gc_count'], writes=[], write_mode='on_modify', + context='all value changes retain PCs and post-store general registers; decoding and writer-stack capture are separate', + claim='GC30 through GC32 only; earlier and equal-value writers and intended lifetime remain unknown') + report['watch_admission'] = debugger_state() + assert all(row['succeeded'] for row in report['watch_admission']) + if report['gc_count'] == 32: + owner = module_frame(thread) + report['word_watch']['gc32_owner'] = owner + assert owner['sp'] == report['word_watch']['address'], 'GC32 uses a different stack slot' + assert word(owner['sp']) == report['word_watch']['last_value'] + active = True + report['gc32_entry_frames'] = frames(thread) + ret = target.ResolveLoadAddress(word(frame.GetSP())) + bp = target.BreakpointCreateBySBAddress(ret) + bp.SetThreadID(tid) + assert bp.GetNumLocations() == 1 + points[bp.GetID()] = 'gc-return' + # LLDB1700 stepped through disabled sites before GC32; create them only when needed. + scans = [point('span', a, 'pushq', '%rbp') for a in (0x103bcf0a0, 0x103bcf1d0)] + gather = point('gather', 0x103c368c0, 'pushq', '%rbp') + pre = point('mark-before', 0x103c4b37d, 'lock') + post = point('mark-after', 0x103c4b384, 'jb') + report['physical_points'] = {str(k): v for k, v in points.items()} + elif name == 'gc-return': + assert active and not pending_spans and not pending_marks + report['gc32_completed'] = True + active = False + for b in scans + [gather, pre, post, gc]: + assert target.BreakpointDelete(b.GetID()) + assert target.BreakpointDelete(ident) + report['word_watch']['gc32_return_value'] = word(report['word_watch']['address']) + assert target.DeleteWatchpoint(watch.GetID()) + elif name == 'gather': + assert active + state = reg(frame, 'r8') + row = dict(thread=tid, state=state) + if state: + origin, top, registers = struct.unpack(' 1: + receipt['inferior_pid'] = pid + identity = subprocess.run(['ps', '-p', str(pid), '-o', 'comm='], + text=True, capture_output=True, timeout=5, check=False) + if identity.returncode == 1 and not identity.stdout.strip(): + receipt['inferior_cleanup'] = 'absent' + elif identity.returncode == 0 and identity.stdout.strip() == str((ROOT / 'bun-profile').resolve()): + # Only the admitted inferior created by this debugger can be killed here. + os.kill(pid, signal.SIGKILL) + limit = time.monotonic() + 5 + while time.monotonic() < limit: + try: + os.kill(pid, 0) + except ProcessLookupError: + receipt['inferior_cleanup'] = 'killed' + break + time.sleep(0.1) + write('supervisor.json', receipt) + assert not receipt['timed_out'], 'debugger supervisor deadline; checkpoint retained, result inconclusive' + assert receipt['exit_status'] == 0 and receipt['inferior_cleanup'] in ('absent', 'killed'), receipt + + +def seal(): + try: + receipt = json.loads((OUT / 'receipt.json').read_text()) + capture = json.loads((OUT / 'capture.json').read_text()) + supervisor = json.loads((OUT / 'supervisor.json').read_text()) + assert not supervisor['timed_out'] and supervisor['exit_status'] == 0 + assert supervisor['inferior_cleanup'] in ('absent', 'killed') and capture['cleanup_completed'] + snapshot = OUT / 'snapshot.json' + assert snapshot.is_file() and snapshot.stat().st_size <= 24 * 1024**2 + library = runpy.run_path(str(Path(__file__).with_name('tls-cycle-heap.py'))) + graph = library['project_graph'](json.loads(snapshot.read_text())) + write('graph.json', graph) + target = graph['targets']['__tls_cycle_dropped_handle'] + assert target['present'] and not target['root_paths'] and target['synthetic_root_path'] is None + external = {edge['source'] for edge in target['scc_external_incoming']} + assert len(external) == 1, 'external owner changed; inspect raw evidence' + raw_graph = json.loads(snapshot.read_text()) + nodes = raw_graph['nodes'] + owner = next(nodes[i:i + 7] for i in range(0, len(nodes), 7) if nodes[i] in external) + assert raw_graph['nodeClassNames'][owner[2]] == 'JSLexicalEnvironment' + cell = int(owner[5], 16) + assert cell > 0 and cell % 16 == 0 + edges = raw_graph['edges'] + assert all(edges[i + 1] != owner[0] for i in range(0, len(edges), 4)) + assert owner[0] not in raw_graph['roots'][::3] + spans = (OUT / 'spans.bin').read_bytes() + matches = [] + for span in capture['spans']: + data = spans[span['raw_offset']:span['raw_offset'] + span['end'] - span['begin']] + for offset in range(0, len(data), 8): + if struct.unpack_from('> 16) in (0, stat.S_IFREG, stat.S_IFDIR) + assert not entry.flag_bits & 1 + matches = [entry for entry in entries if entry.filename == name] + assert len(matches) == 1 and not matches[0].is_dir() + entry = matches[0] + assert 0 < entry.file_size <= maximum + with archive.open(entry) as incoming, dest.open('xb') as output: + count = 0 + while chunk := incoming.read(1024**2): + count += len(chunk) + assert count <= maximum + output.write(chunk) + assert count == entry.file_size + dest.chmod(0o600) + + +def tool(name, args): + result = subprocess.run(args, stdin=subprocess.DEVNULL, capture_output=True, timeout=180) + assert len(result.stdout) <= 8 * 1024**2 and len(result.stderr) <= 1024**2 + (OUT / (name + '.stderr')).write_bytes(result.stderr) + assert result.returncode == 0, (name, result.returncode) + (OUT / (name + '.txt')).write_bytes(result.stdout) + return result.stdout.decode() + + +def prepare(): + assert os.uname().sysname == 'Darwin' and os.uname().machine == 'x86_64' + run = json.loads((ROOT / 'run.json').read_text()) + meta = json.loads((ROOT / 'metadata.json').read_text()) + assert run['id'] == RUN and run['head_sha'] == COMMIT + assert run['path'] == '.github/workflows/openclaw-release.yml' + assert run['head_branch'] == 'claude/fix-tls-native-transport-20261019' + assert run['event'] == 'workflow_dispatch' and run['status'] == 'completed' + assert run['conclusion'] == 'failure' + assert meta['id'] == ARTIFACT and meta['name'] == 'signed-darwin-x64' + assert meta['size_in_bytes'] == SIZE and meta['digest'] == 'sha256:' + DIGEST + assert not meta['expired'] and meta['workflow_run']['id'] == RUN + assert meta['workflow_run']['head_sha'] == COMMIT + archive = ROOT / 'artifact.zip' + assert archive.stat().st_size == SIZE and sha(archive) == DIGEST + profile_zip = ROOT / 'profile.zip' + extract(archive, 'bun-darwin-x64-profile.zip', profile_zip, 1024**3) + for name in ('bun-profile', 'bun-profile.dSYM'): + extract(profile_zip, 'bun-darwin-x64-profile/' + name, ROOT / name, 2 * 1024**3) + identities = {name: sha(ROOT / name) for name in ('profile.zip', 'bun-profile', 'bun-profile.dSYM')} + uuids = [] + for name in ('bun-profile', 'bun-profile.dSYM'): + text = tool(name + '-uuid', ['xcrun', 'dwarfdump', '--uuid', str(ROOT / name)]) + rows = re.findall(r'^UUID: ([0-9A-Fa-f-]{36}) \(x86_64\) ', text, re.M) + assert len(rows) == 1 and len(text.splitlines()) == 1, text + uuids.append(rows[0].lower()) + assert uuids[0] == uuids[1] + tool('lldb-version', ['xcrun', 'lldb', '--version']) + (OUT / 'inputs.json').write_text(json.dumps(dict(commit=COMMIT, run=RUN, artifact=ARTIFACT, + artifact_sha256=DIGEST, identities=identities, uuid=uuids[0], + claim='static capability inventory only; no inferior launch or GC-root proof'), indent=2) + '\n') + + +def type_layouts(target): + layouts = {} + for name in ('JSC::VM', 'JSC::Heap', 'JSC::Collector', 'JSC::PreciseAllocation', + 'JSC::VerifierSlotVisitor', 'JSC::VerifierSlotVisitor::PreciseAllocationData', + 'JSC::ConservativeRoots', 'JSC::CurrentThreadState', 'JSC::RegisterState', + 'JSC::JSLexicalEnvironment', 'JSC::JSGenerator', + 'JSC::JSGlobalObject', 'JSC::JSWeakObjectRef', 'JSC::ProtoCallFrame', + 'WTF::StackTrace', 'Zig::JSFFIFunction', + 'bun_runtime::socket::socket_body::DuplexUpgradeContext', + 'bun_runtime::socket::upgraded_duplex::UpgradedDuplex'): + found = target.FindTypes(name) + assert found.GetSize() <= 128, name + values = [] + for i in range(found.GetSize()): + value = found.GetTypeAtIndex(i).GetCanonicalType() + assert value.GetNumberOfFields() <= 512, name + fields = [value.GetFieldAtIndex(j) for j in range(value.GetNumberOfFields())] + values.append(dict(name=value.GetName(), complete=value.IsTypeComplete(), bytes=value.GetByteSize(), + fields=[dict(name=f.GetName(), offset=f.GetOffsetInBytes(), type=f.GetType().GetName(), + bytes=f.GetType().GetByteSize()) for f in fields])) + layouts[name] = values + return layouts + + +def inspect(debugger): + import lldb + debugger.SetAsync(False) + debugger.SetUseColor(False) + command = lldb.SBCommandReturnObject() + debugger.GetCommandInterpreter().HandleCommand('settings set target.load-script-from-symbol-file false', command) + assert command.Succeeded() + target = debugger.CreateTarget(str(ROOT / 'bun-profile')) + assert target.IsValid() and target.GetTriple().startswith('x86_64-apple-') + command = lldb.SBCommandReturnObject() + debugger.GetCommandInterpreter().HandleCommand('target symbols add ' + shlex.quote(str(ROOT / 'bun-profile.dSYM')), command) + assert command.Succeeded(), command.GetError() + module = target.FindModule(target.GetExecutable()) + inputs = json.loads((OUT / 'inputs.json').read_text()) + assert module.GetUUIDString().lower() == inputs['uuid'] + layouts = type_layouts(target) + functions = {} + for name in ('JSC::constructWeakRef', 'JSC::JSWeakObjectRef::finishCreation', + 'JSC::JSWeakObjectRef::deref', 'JSC__VM__runGC', 'posix_spawn_bun', + 'JSC::SlotVisitor::appendConservativeRoots', 'JSC::SlotVisitor::appendJSCellOrAuxiliary', + 'JSC::Heap::testAndSetMarked', 'JSC::sanitizeStackForVM', + 'JSC::MachineThreads::gatherConservativeRoots', 'JSC::ConservativeRoots::genericAddSpan', + 'JSC::ConservativeRoots::genericAddPointer', 'JSC::Interpreter::executeCall', + 'JSC::Interpreter::executeModuleProgram', 'vmEntryToJavaScript', + 'Bun__UpgradedDuplex__onWriteComplete', + 'bun_runtime::socket::socket_body::DuplexUpgradeContext::run_event', + 'bun_runtime::socket::socket_body::DuplexUpgradeContext::deinit'): + contexts = module.FindFunctions(name, lldb.eFunctionNameTypeAny) + assert contexts.GetSize() <= 128, name + parents = {} + for i in range(contexts.GetSize()): + context = contexts.GetContextAtIndex(i) + function = context.GetFunction() + address = function.GetStartAddress() if function.IsValid() else context.GetSymbol().GetStartAddress() + if not address.IsValid(): + continue + resolved = function.GetName() if function.IsValid() else context.GetSymbol().GetName() + key = address.GetFileAddress() + if key in parents: + continue + instructions = function.GetInstructions(target) if function.IsValid() else context.GetSymbol().GetInstructions(target) + assert instructions.GetSize() <= 16384, name + rows = [] + for j in range(instructions.GetSize()): + insn = instructions.GetInstructionAtIndex(j) + entry = insn.GetAddress().GetLineEntry() + data = bytes(insn.GetData(target).uint8s) + assert len(data) == insn.GetByteSize() + rows.append(dict(address=insn.GetAddress().GetFileAddress(), bytes=data.hex(), + mnemonic=insn.GetMnemonic(target), operands=insn.GetOperands(target), + comment=insn.GetComment(target), file=entry.GetFileSpec().GetFilename(), + line=entry.GetLine())) + parents[key] = dict(name=resolved, address=key, instructions=rows) + functions[name] = list(parents.values()) + assert not target.GetProcess().IsValid(), 'static inventory launched an inferior' + (OUT / 'capabilities.json').write_text(json.dumps(dict(layouts=layouts, functions=functions, + inferior_launched=False, root_capture_ready=False), indent=2) + '\n') + + +if __name__ == '__lldb__': + import lldb + inspect(lldb.debugger) +elif __name__ == '__main__': + OUT.mkdir(exist_ok=True) + if sys.argv[1] == 'prepare': + prepare() + else: + assert sys.argv[1] == 'seal' and (OUT / 'capabilities.json').is_file() + files = sorted(OUT.iterdir()) + assert len(files) <= 24 and sum(p.stat().st_size for p in files) <= 32 * 1024**2 + assert all(stat.S_ISREG(p.lstat().st_mode) for p in files) + (OUT / 'manifest.json').write_text(json.dumps({p.name: dict(bytes=p.stat().st_size, + sha256=sha(p)) for p in files}, indent=2) + '\n') diff --git a/.github/openclaw/tls-darwin-disassembly.py b/.github/openclaw/tls-darwin-disassembly.py new file mode 100644 index 000000000000..6af1a0fc4260 --- /dev/null +++ b/.github/openclaw/tls-darwin-disassembly.py @@ -0,0 +1,170 @@ +"""Capture bounded physical functions from the exact consumed Darwin profile; never run it.""" +import json +import os +from pathlib import Path +import re +import runpy +import stat +import struct +import subprocess +import uuid + +ROOT = Path(os.environ['RUNNER_TEMP']) / 'tls-darwin-input' +OUT = Path('tls-darwin-disassembly') +LIBRARY = runpy.run_path(str(Path(__file__).with_name('tls-darwin-capability.py')), + run_name='archive_library') +COMMIT = LIBRARY['COMMIT'] +RUN = LIBRARY['RUN'] +ARTIFACT = LIBRARY['ARTIFACT'] +SIZE = LIBRARY['SIZE'] +DIGEST = LIBRARY['DIGEST'] +sha = LIBRARY['sha'] +extract = LIBRARY['extract'] + + +def macho(path, filetype): + with path.open('rb') as stream: + header = stream.read(32) + magic, cpu, subtype, kind, count, size, flags, reserved = struct.unpack('<8I', header) + assert (magic, cpu, kind) == (0xfeedfacf, 0x01000007, filetype), tuple(map(hex, (magic, cpu, subtype, kind))) + # CPU_SUBTYPE_LIB64 is a capability bit, not a different x86_64 architecture. + assert subtype in (3, 0x80000003), hex(subtype) + assert 0 < count <= 1024 and 0 < size <= 1024**2 + data = stream.read(size) + assert len(data) == size + offset, identities, sections = 0, [], [] + for _ in range(count): + assert offset + 8 <= size + command, length = struct.unpack_from('= 8 and length % 8 == 0 and offset + length <= size + if command == 0x1b: # LC_UUID + assert length == 24 + identities.append(str(uuid.UUID(bytes=data[offset + 8:offset + 24]))) + if command == 0x19: # LC_SEGMENT_64, section_64 records follow its 72-byte header. + assert length >= 72 + segment = data[offset + 8:offset + 24].rstrip(b'\0') + nsects = struct.unpack_from(':$', text, re.M) + cursor, count = start, 0 + for line in text.splitlines(): + if not re.match(r'^\s*[0-9a-f]+:', line): + continue + match = re.fullmatch(r'\s*([0-9a-f]+):\s+((?:[0-9a-f]{2}\s+)+)(\S+)(.*)', line) + assert match, line + address = int(match[1], 16) + width = len(match[2].split()) + assert address == cursor and 0 < width <= 15 and address + width <= end, line + assert match[3] not in ('', '.byte', '.long', '.short'), line + cursor += width + count += 1 + assert count and cursor == end + return count + + +def main(): + OUT.mkdir(exist_ok=True) + run = json.loads((ROOT / 'run.json').read_text()) + meta = json.loads((ROOT / 'metadata.json').read_text()) + assert run['id'] == RUN and run['head_sha'] == COMMIT + assert run['path'] == '.github/workflows/openclaw-release.yml' + assert run['head_branch'] == 'claude/fix-tls-native-transport-20261019' + assert run['event'] == 'workflow_dispatch' and run['status'] == 'completed' + assert run['conclusion'] == 'failure' + assert meta['id'] == ARTIFACT and meta['name'] == 'signed-darwin-x64' + assert meta['size_in_bytes'] == SIZE and meta['digest'] == 'sha256:' + DIGEST + assert not meta['expired'] and meta['workflow_run']['id'] == RUN + assert meta['workflow_run']['head_sha'] == COMMIT + archive = ROOT / 'artifact.zip' + assert archive.stat().st_size == SIZE and sha(archive) == DIGEST + profile_zip = ROOT / 'profile.zip' + profile, dsym = ROOT / 'bun-profile', ROOT / 'bun-profile.dSYM' + extract(archive, 'bun-darwin-x64-profile.zip', profile_zip, 1024**3) + for path in (profile, dsym): + extract(profile_zip, 'bun-darwin-x64-profile/' + path.name, path, 2 * 1024**3) + identity, sections, subtype = macho(profile, 2) + assert identity == macho(dsym, 10)[0] == '4c4c4432-5555-3144-a125-d2b0a4297913' + assert len(sections) == 1 + version = tool('llvm-version', ['llvm-objdump-23', '--version']) + assert re.search(r'LLVM version 23\.', version) + nm_args = ['llvm-nm-23', '--defined-only', '--format=posix', '--radix=x', '--no-demangle'] + symbols = select_symbols(tool('nm-profile', nm_args + [str(profile)])) + symbol_owner = 'bun-profile' + if set(symbols) != {'deinit', 'run_event'}: + symbols = select_symbols(tool('nm-dsym', nm_args + [str(dsym)])) + symbol_owner = 'bun-profile.dSYM' + assert set(symbols) == {'deinit', 'run_event'}, 'required physical function missing' + starts_text = tool('function-starts', ['llvm-objdump-23', '--macho', '--no-leading-headers', + '--function-starts=addrs', str(profile)]) + lines = starts_text.splitlines() + assert lines and all(re.fullmatch(r'[0-9a-f]{16}', line) for line in lines) + starts = [int(line, 16) for line in lines] + assert starts == sorted(set(starts)) + for name, item in symbols.items(): + start = item['start'] + assert start in starts + index = starts.index(start) + assert index + 1 < len(starts) + end = starts[index + 1] + assert sections[0][0] <= start < end <= sections[0][1] and end - start <= 65536 + # The generic disassembler honors numeric bounds. The --macho disassembler does not. + assembly = tool(name, ['llvm-objdump-23', '--disassemble', '--disassemble-zeroes', '--show-all-symbols', + '--start-address=' + hex(start), '--stop-address=' + hex(end), str(profile)]) + item.update(end=end, bytes=end - start, + instruction_count=instructions(assembly, item['symbol'], start, end)) + path = OUT / (name + '.asm') + path.write_text(assembly.replace(str(profile), 'bun-profile')) + item['assembly_sha256'] = sha(path) + receipt = dict(commit=COMMIT, producer_run=RUN, artifact_id=ARTIFACT, artifact_sha256=DIGEST, + identities={p.name: sha(p) for p in (profile_zip, profile, dsym)}, uuid=identity, + sections=sections, cpu_subtype=hex(subtype), symbol_owner=symbol_owner, functions=symbols, llvm_version=version, + inferior_launched=False, control_flow_review_pending=True, gc_root_proven=False) + (OUT / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + files = sorted(OUT.iterdir()) + assert len(files) <= 24 and all(stat.S_ISREG(p.lstat().st_mode) for p in files) + assert sum(p.stat().st_size for p in files) <= 32 * 1024**2 + (OUT / 'manifest.json').write_text(json.dumps({p.name: dict(bytes=p.stat().st_size, sha256=sha(p)) + for p in files}, indent=2) + '\n') + print(json.dumps(receipt)) + + +if __name__ == '__main__': + main() diff --git a/.github/openclaw/tls-darwin-types.py b/.github/openclaw/tls-darwin-types.py new file mode 100644 index 000000000000..343d0e4192be --- /dev/null +++ b/.github/openclaw/tls-darwin-types.py @@ -0,0 +1,180 @@ +"""Reconstruct shipping header layouts through the frozen compiler owner; never link or run them.""" +import hashlib +import json +import os +from pathlib import Path +import runpy +import re +import shlex +import struct +import subprocess +import sys + +SOURCE = Path('/work/source') +BUILD = SOURCE / 'build/openclaw-darwin-x64' +OUT = Path('/work/tls-darwin-types') +COMMIT = '510fef558b464c58de1209298f638fa476d03eef' +ENGINE = 'b82bc917e99110b61c3a21202d6def7457aa194c' +ENGINE_SHA = 'd59f456e79045c98420a0fc24ca624b33654f7d5fc8133e740b78eaec160b0c5' +REQUIRED = ('JSC::VM', 'JSC::Heap', 'JSC::PreciseAllocation', 'JSC::VerifierSlotVisitor', + 'JSC::ConservativeRoots', 'JSC::CurrentThreadState', 'JSC::RegisterState', + 'JSC::JSGlobalObject', 'JSC::JSWeakObjectRef', 'JSC::ProtoCallFrame') + + +def sha(path): + digest = hashlib.sha256() + with path.open('rb') as stream: + while chunk := stream.read(1024**2): + digest.update(chunk) + return digest.hexdigest() + + +def write(name, value): + (OUT / name).write_text(json.dumps(value, indent=2) + '\n') + + +def run(name, args, cwd=SOURCE, timeout=600): + with (OUT / (name + '.log')).open('x') as stream: + subprocess.run(args, cwd=cwd, stdin=subprocess.DEVNULL, stdout=stream, + stderr=subprocess.STDOUT, check=True, timeout=timeout) + assert (OUT / (name + '.log')).stat().st_size <= 8 * 1024**2 + + +def build(): + assert os.uname().machine == 'aarch64' and os.uname().sysname == 'Linux' + assert subprocess.check_output(['git', '-C', str(SOURCE), 'rev-parse', 'HEAD'], text=True).strip() == COMMIT + assert not subprocess.check_output(['git', '-C', str(SOURCE), 'diff', '--name-only', 'HEAD'], text=True) + manifest = SOURCE / 'scripts/build/deps/webkit-artifacts.json' + spec = json.loads(manifest.read_text()) + artifact = spec['artifacts']['bun-webkit-macos-amd64-lto.tar.gz'] + assert spec['version'] == ENGINE and artifact['sha256'] == ENGINE_SHA + assert artifact['url'] == f'https://github.com/openclaw/WebKit/releases/download/autobuild-{ENGINE}/bun-webkit-macos-amd64-lto.tar.gz' + compiler = subprocess.check_output(['clang++', '--version'], text=True) + assert '23.1.1' in compiler + argv = ['bun', 'scripts/build.ts', '--profile=release', '--ci=on', '--os=darwin', + '--arch=x64', '--build-dir=' + str(BUILD), '--configure-only'] + run('configure', argv) + records = json.loads(subprocess.check_output(['ninja', '-C', str(BUILD), '-t', 'compdb', 'pch'], text=True)) + assert len(records) == 1 + pch = records[0] + assert Path(pch['output']).suffix == '.pch' + pch_path = (Path(pch['directory']) / pch['output']).resolve() + assert pch_path.parent == BUILD / 'pch' + run('prepare-headers', ['ninja', '-C', str(BUILD), '-j2', pch['output']], timeout=1800) + database = BUILD / 'compile_commands.json' + records = json.loads(database.read_text()) + chosen = [record for record in records if record['file'] == str(SOURCE / 'src/jsc/bindings/BunObject.cpp')] + assert len(chosen) == 1 + original = chosen[0]['arguments'] + assert '-g1' in original and '-O3' in original and '-flto=thin' in original + assert original[-4:-1] == ['-c', chosen[0]['file'], '-o'] + assert original.count('-include-pch') == 1 + pch_index = original.index('-include-pch') + assert (Path(chosen[0]['directory']) / original[pch_index + 1]).resolve() == pch_path + assert original[pch_index + 2:-4] == [] + # The compilation database retains shell escapes in macro definitions. + # Admit the actual successful Ninja PCH argv, which has the same shared flags. + pch_argv = shlex.split(pch['command']) + flags = pch_argv[:pch_argv.index('-Winvalid-pch')] + assert len(flags) == pch_index + for recorded, actual in zip(original[:pch_index], flags): + decoded = shlex.split(recorded) if recorded.startswith('-D') else [recorded] + assert decoded == [actual], (recorded, actual) + assert original.count('-isysroot') == 1 + sdk = Path(original[original.index('-isysroot') + 1]) + sdk_settings = sdk / 'SDKSettings.json' + assert json.loads(sdk_settings.read_text())['Version'] == '26.5' + includes = [Path(arg[2:]) for arg in original if arg.startswith('-I')] + roots = [p for p in includes if p.name == 'include' and ENGINE_SHA in str(p)] + assert len(roots) == 1 + headers = roots[0] + config = headers / 'cmakeconfig.h' + assert ENGINE in config.read_text() + required_headers = ['VerifierSlotVisitor.h', 'ConservativeRoots.h', 'MachineStackMarker.h', + 'RegisterState.h', 'PreciseAllocation.h', 'JSWeakObjectRef.h', 'ProtoCallFrame.h', 'VM.h'] + assert all((headers / 'JavaScriptCore' / name).is_file() for name in required_headers) + witness = OUT / 'types.cpp' + wrapper = pch_path.with_suffix('') + assert wrapper.is_file() and 'root-pch.h' in wrapper.name + witness.write_text('#include "' + str(wrapper) + '"\n' + ''.join( + '#include \n' for name in required_headers) + + '\nstatic_assert(sizeof(JSC::VM) > 0);\nstatic_assert(sizeof(JSC::ConservativeRoots) > 0);\n' + 'static_assert(sizeof(JSC::VerifierSlotVisitor) > 0);\n') + output = OUT / 'types.o' + # This object supplies layouts only. No runtime code or shipping profile is replaced. + replacement = {'-g1': '-g', '-flto=thin': '-fno-lto', + '-fwhole-program-vtables': '-fno-whole-program-vtables'} + adapted = [replacement.get(arg, arg) for arg in flags] + adapted += ['-fstandalone-debug', '-fno-eliminate-unused-debug-types', '-c', str(witness), '-o', str(output)] + write('compile-command.json', dict(directory=chosen[0]['directory'], original=original, + successful_pch_command=pch['command'], canonical_flags=flags, witness=adapted, + variance='full standalone type metadata, no LTO or whole-program vtables, original PCH wrapper included as text, synthetic include-only translation unit; never linked or executed')) + layout_argv = flags + ['-fsyntax-only', '-Xclang', '-fdump-record-layouts', str(witness)] + run('ast-record-layouts', layout_argv, cwd=Path(chosen[0]['directory']), timeout=600) + blocks = (OUT / 'ast-record-layouts.log').read_text().split('*** Dumping AST Record Layout') + verifier = [block.strip() for block in blocks if block.strip() and + block.strip().splitlines()[0].partition('|')[2].strip() == 'class JSC::VerifierSlotVisitor'] + assert len(verifier) == 1, 'missing or ambiguous VerifierSlotVisitor AST record layout' + layout = verifier[0] + size = re.search(r'\[sizeof=(\d+),.*?align=(\d+),', layout, re.S) + assert size and int(size[1]) > 0 and int(size[2]) > 0 + assert all(field in layout for field in ('m_preciseAllocationMap', 'm_doneMarking')) + write('ast-layouts.json', dict(type='JSC::VerifierSlotVisitor', bytes=int(size[1]), + alignment=int(size[2]), raw_layout=layout, argv=layout_argv, + complete_ast_layout=True, root_capture_ready=False, linked=False, executed=False, + claim='compiler AST layout only; nested map/bucket and physical root admission remain separate')) + run('compile-types', adapted, cwd=Path(chosen[0]['directory']), timeout=600) + assert 0 < output.stat().st_size <= 64 * 1024**2 + with output.open('rb') as stream: + header = stream.read(32) + assert struct.unpack_from(' 0 and item['fields'] for item in layouts[name])] + assert not target.GetProcess().IsValid() + (out / 'layouts.json').write_text(json.dumps(dict(layouts=layouts, missing=missing, + inferior_launched=False, root_capture_ready=False), indent=2) + '\n') + assert not missing, missing + + +if __name__ == '__lldb__': + import lldb + inspect(lldb.debugger) +elif __name__ == '__main__': + OUT.mkdir(exist_ok=True) + assert sys.argv[1] == 'build' + build() diff --git a/.github/openclaw/tls-fixture-qualification.py b/.github/openclaw/tls-fixture-qualification.py new file mode 100644 index 000000000000..cec906c47d99 --- /dev/null +++ b/.github/openclaw/tls-fixture-qualification.py @@ -0,0 +1,146 @@ +"""Collect all canonical TLS suites on the two retained failing native targets.""" +import hashlib +import json +import os +from pathlib import Path +import re +import runpy +import shutil +import stat +import subprocess +import sys +import time +import xml.etree.ElementTree as ET +import zipfile + +LIBRARY = runpy.run_path(str(Path(__file__).with_name('tls-cycle-checkpoint.py')), run_name='checkpoint_contract') +COMMIT, ENGINE, FIXTURE = (LIBRARY[key] for key in ('COMMIT', 'ENGINE', 'FIXTURE')) +FIXTURE_SHA = LIBRARY['FIXTURE_SHA'] +ROOT = Path(os.environ['RUNNER_TEMP']) / 'tls-fixture-input' +OUT = Path('tls-fixture-qualification') +SOURCE = Path('source') +SPECS = { + 'darwin-x64': (13057470428, 'signed-darwin-x64', 232201333, 'a4d8b786d7bb87d166f757e089ac1705f4ac0701504038ebbecd438c9d6e06d6'), + 'windows-arm64': (13056907950, 'test-only-windows', 596111335, '5c81e59c6e6ee0b1a21a9f03b61eda6ff8616d3a1fab631c7e1c1d4899a58bb6'), +} + + +def sha(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def clean(): + # Refresh checkout stat metadata without staging mode/content changes. + subprocess.run(['git', '-C', str(SOURCE), 'update-index', '--refresh'], check=True) + result = subprocess.run(['git', '-C', str(SOURCE), 'diff-index', '--quiet', 'HEAD', '--']) + if result.returncode: + subprocess.run(['git', '-C', str(SOURCE), 'diff', '--name-status', 'HEAD', '--'], check=True) + result.check_returncode() + + +def main(target): + OUT.mkdir(exist_ok=True) + artifact, name, size, digest = SPECS[target] + platform, arch = target.split('-') + platform = 'win32' if platform == 'windows' else platform + assert not any(os.environ.get(k) for k in ('GH_TOKEN', 'GITHUB_TOKEN', 'NODE_OPTIONS', 'BUN_OPTIONS')) + head = subprocess.check_output(['git', '-C', str(SOURCE), 'rev-parse', 'HEAD'], text=True).strip() + assert head == COMMIT + clean() + assert sha(SOURCE / FIXTURE) == FIXTURE_SHA + owner = (SOURCE / 'scripts/openclaw-release/release.ts').read_text() + block = re.search(r'export const tlsQualificationFiles = \[(.*?)\] as const;', owner, re.S) + assert block + files = json.loads('[' + block[1].rstrip().removesuffix(',') + ']') + assert len(files) == len(set(files)) == 6 and files[0] == FIXTURE + assert len({Path(file).name for file in files}) == 6 + assert all((SOURCE / file).is_file() and not (SOURCE / file).is_symlink() for file in files) + run = json.loads((ROOT / 'run.json').read_text()) + meta = json.loads((ROOT / 'metadata.json').read_text()) + assert run['id'] == 41238111110 and run['head_sha'] == COMMIT + assert run['status'] == 'completed' and run['conclusion'] == 'failure' + assert run['event'] == 'workflow_dispatch' and run['path'] == '.github/workflows/openclaw-release.yml' + assert meta['id'] == artifact and meta['name'] == name and not meta['expired'] + assert meta['size_in_bytes'] == size and meta['digest'] == 'sha256:' + digest + assert meta['workflow_run']['id'] == run['id'] and meta['workflow_run']['head_sha'] == COMMIT + archive = ROOT / 'artifact.zip' + assert archive.stat().st_size == size and sha(archive) == digest + archive_library = runpy.run_path(str(Path(__file__).with_name('tls-darwin-capability.py')), run_name='archive_library') + triplet = 'bun-' + ('windows' if platform == 'win32' else platform) + '-' + ('aarch64' if arch == 'arm64' else arch) + product, exe = ROOT / 'product.zip', ROOT / ('bun.exe' if platform == 'win32' else 'bun') + archive_library['extract'](archive, triplet + '.zip', product, 1024**3) + archive_library['extract'](product, triplet + '/' + exe.name, exe, 1024**3) + exe.chmod(0o700) + exe = exe.resolve() + identity = subprocess.run([str(exe), '-e', 'console.log(JSON.stringify({revision:Bun.revision,webkit:process.versions.webkit,platform:process.platform,arch:process.arch}))'], + capture_output=True, check=True, text=True, timeout=120) + assert not identity.stderr + facts = json.loads(identity.stdout) + assert facts == dict(revision=COMMIT, webkit=ENGINE, platform=platform, arch=arch) + node = shutil.which('node') + assert node + reference = subprocess.run([node, '-e', 'console.log(JSON.stringify({name:process.release.name,version:process.versions.node,bun:process.versions.bun??null,platform:process.platform,arch:process.arch,exe:process.execPath}))'], + capture_output=True, check=True, text=True, timeout=120) + ref = json.loads(reference.stdout) + assert ref == dict(name='node', version='26.12.1', bun=None, platform=platform, arch=arch, exe=ref['exe']) + assert os.path.normcase(os.path.realpath(ref['exe'])) == os.path.normcase(os.path.realpath(node)) + signature = None + if platform == 'darwin': + with zipfile.ZipFile(archive) as contents: + required = arch == 'arm64' or target + '.signing.json' in contents.namelist() + result = subprocess.run(['codesign', '--verify', '--strict', str(exe)], capture_output=True, timeout=120) + signature = dict(required_by_release_contract=required, exit_code=result.returncode, + stderr=result.stderr.decode().replace(str(exe), exe.name)) + assert not required or result.returncode == 0 + receipt = dict(target=target, runtime_source=COMMIT, test_source=COMMIT, engine=ENGINE, + artifact_id=artifact, artifact_sha256=digest, product_zip_sha256=sha(product), + executable_sha256=sha(exe), facts=facts, fixture_sha256=FIXTURE_SHA, + runtime_diff_from_product_source=[], source_diff_from_product_source=[], node_version='26.12.1', + signature=signature, suites=[], passed=False, + claim='unchanged source and runtime artifacts; all six suites recorded, failures retained; no release publication') + deadline = time.monotonic() + 20 * 60 + try: + for file in files: + basename = Path(file).name + report = (OUT / (basename + '.xml')).resolve() + assert not report.exists() + argv = [str(exe), 'test', file, '--reporter=junit', '--reporter-outfile=' + str(report)] + result = subprocess.run(argv, cwd=SOURCE, stdin=subprocess.DEVNULL, capture_output=True, + timeout=max(1, deadline - time.monotonic())) + assert len(result.stdout) + len(result.stderr) <= 4 * 1024**2 + (OUT / (basename + '.stdout')).write_bytes(result.stdout) + (OUT / (basename + '.stderr')).write_bytes(result.stderr) + doc = ET.parse(report).getroot() + cases = list(doc.iter('testcase')) + skipped = len(list(doc.iter('skipped'))) + failures, errors = len(list(doc.iter('failure'))), len(list(doc.iter('error'))) + assert doc.tag == 'testsuites' and int(doc.get('tests')) == len(cases) + assert int(doc.get('skipped')) == skipped and int(doc.get('failures')) == failures + row = dict(file=file, exit_code=result.returncode, tests=len(cases), skipped=skipped, failures=failures, errors=errors) + receipt['suites'].append(row) + assert len(cases) > skipped, row + assert sha(exe) == receipt['executable_sha256'] + receipt['passed'] = all(row['exit_code'] == row['failures'] == row['errors'] == 0 for row in receipt['suites']) + assert receipt['passed'], receipt['suites'] + finally: + clean() + assert sha(SOURCE / FIXTURE) == FIXTURE_SHA + receipt['source_and_index_unchanged'] = True + (OUT / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + paths = sorted(OUT.iterdir()) + assert len(paths) <= 23 and all(stat.S_ISREG(p.lstat().st_mode) for p in paths) + assert sum(p.stat().st_size for p in paths) <= 32 * 1024**2 + (OUT / 'manifest.json').write_text(json.dumps({p.name: dict(bytes=p.stat().st_size, sha256=sha(p)) + for p in paths}, indent=2) + '\n') + print(json.dumps(receipt)) + + +if __name__ == '__main__': + action, target = sys.argv[1:] + if action == 'plan': + artifact, name, size, digest = SPECS[target] + print(f'artifact={artifact}\nsize={size}') + else: + assert action == 'qualify' + main(target) diff --git a/.github/openclaw/tls-retainer-capture.py b/.github/openclaw/tls-retainer-capture.py new file mode 100644 index 000000000000..23cdbb11af53 --- /dev/null +++ b/.github/openclaw/tls-retainer-capture.py @@ -0,0 +1,1157 @@ +"""Trace the frozen destroyed-completion owner with the saved shipping-debug bytes.""" +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import signal +import stat +import subprocess +import sys +import tarfile +import time +import zipfile +from xml.etree import ElementTree + +ROOT = Path('/work') +OUT = ROOT / 'capture' +SOURCE = ROOT / 'source' +BUILD = SOURCE / 'build/tls-arm64-shipping-debug' +BINARY = BUILD / 'bun' +PROFILE = BUILD / 'bun-profile' +FIXTURE = 'test/js/bun/net/socket-retention.test.ts' +CASE = 'a TLS Duplex completion retains its owner and is inert after destroy' +UUID = '487687ca47a8a631b87f701a12a351b33dab611a' +BUN = '454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e' +FIXTURE_SHA = 'd4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2' +ARCHIVE_SHA = 'a71acab37ebb674baae960ebe04d554f44f7e69284450855cf90f1d783180f0e' +BINARIES = { + 'bun': '6c1276f536bd52c19aee39ab3eea81b6ddf8c3e0b4e92aa879720a38c4f38cd3', + 'bun-profile': '2be841e8d061ea1a9a7a971a215ca9f1671088717881dded961360ad012d949a', +} + + +def sha(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def write(name, value): + (OUT / name).write_text(json.dumps(value, indent=2) + '\n') + + +def admit_zip(): + root = Path.cwd() + run = json.loads((root / 'producer-run.json').read_text()) + artifact = json.loads((root / 'producer-artifact.json').read_text()) + assert run['id'] == 41212340227 and run['conclusion'] == 'failure' + assert run['head_sha'] == '0a1544a4b4c6a4bdfbda96983e0734fc66e0320f' + assert artifact['id'] == 13048432420 and not artifact['expired'] + assert artifact['name'] == 'tls-linux-arm64-shipping-debug' + assert artifact['workflow_run']['id'] == run['id'] + assert artifact['workflow_run']['head_sha'] == run['head_sha'] + expected = '073a481c6576f8bf640be3e4da37aa1eb869ee037bdebd6f3eebfbdc6b01cb43' + assert artifact['digest'] == 'sha256:' + expected + assert (root / 'producer.zip').stat().st_size == artifact['size_in_bytes'] == 717328810 + assert sha(root / 'producer.zip') == expected + with zipfile.ZipFile(root / 'producer.zip') as archive: + entries = archive.infolist() + assert len(entries) == 2 and sum(e.file_size for e in entries) < 1024**3 + assert {e.filename for e in entries} == {'matched-shipping-debug.tar.zst', 'shipping-debug-SHA256SUMS'} + dest = root / 'producer' + dest.mkdir() + for entry in entries: + assert not entry.is_dir() and not stat.S_ISLNK(entry.external_attr >> 16) + with (dest / entry.filename).open('xb') as output, archive.open(entry) as content: + shutil.copyfileobj(content, output) + assert sha(dest / 'matched-shipping-debug.tar.zst') == ARCHIVE_SHA + assert (dest / 'shipping-debug-SHA256SUMS').read_text() == ARCHIVE_SHA + ' matched-shipping-debug.tar.zst\n' + + +def restore(): + assert os.uname().machine == 'aarch64' + assert subprocess.check_output(['git', '-c', 'safe.directory=' + str(SOURCE), '-C', str(SOURCE), + 'rev-parse', 'HEAD'], text=True).strip() == BUN + assert sha(SOURCE / FIXTURE) == FIXTURE_SHA + archive = ROOT / 'producer/matched-shipping-debug.tar.zst' + assert sha(archive) == ARCHIVE_SHA + entries = {} + manifest = None + total = 0 + child = subprocess.Popen(['zstd', '-d', '-c', str(archive)], stdout=subprocess.PIPE) + try: + with tarfile.open(fileobj=child.stdout, mode='r|') as stream: + for member in stream: + name = PurePosixPath(member.name) + assert member.isfile() and not name.is_absolute() and '..' not in name.parts + assert str(name) not in entries and len(entries) < 64 + total += member.size + assert total <= 8 * 1024**3 + if str(name).startswith('diagnostics/'): + target = ROOT / 'producer-evidence' / str(name) + else: + assert str(name) in {'source/build/tls-arm64-shipping-debug/' + n for n in BINARIES} + target = ROOT / str(name) + target.parent.mkdir(parents=True, exist_ok=True) + with target.open('xb') as output, stream.extractfile(member) as content: + shutil.copyfileobj(content, output) + target.chmod(0o755 if name.name in BINARIES else 0o644) + entries[str(name)] = {'bytes': target.stat().st_size, 'sha256': sha(target)} + if str(name) == 'diagnostics/archive-manifest.json': + manifest = json.loads(target.read_text()) + assert child.wait(timeout=30) == 0 + finally: + child.stdout.close() + if child.poll() is None: + child.kill() + child.wait() + assert len(entries) == 55 and len(manifest) == 54 + del entries['diagnostics/archive-manifest.json'] + assert entries == manifest + assert all(sha(BUILD / name) == digest for name, digest in BINARIES.items()) + assert OUT.is_dir() and not any(OUT.iterdir()) + write('inputs.json', {'producer_run': 41212340227, 'artifact': 13048432420, + 'archive_sha256': ARCHIVE_SHA, 'manifest_entries': len(manifest), + 'bun': BUN, 'fixture_sha256': FIXTURE_SHA, 'binaries': BINARIES, + 'claim': 'diagnostic child replay; no runtime fix or retainer classification'}) + + +def admit_failure(cases, producer): + selected = [case for case in cases if case.get('name') == CASE] + expected = [case for case in producer if case.get('name') == CASE] + assert len(selected) == len(expected) == 1 + assert all(case.find('skipped') is not None and case.find('failure') is None and + case.find('error') is None for case in cases if case is not selected[0]) + actual, baseline = selected[0], expected[0] + assert actual.get('file') == baseline.get('file') == FIXTURE + assert actual.get('line') == baseline.get('line') + assert actual.find('skipped') is None and actual.find('error') is None + failures, baseline_failures = actual.findall('failure'), baseline.findall('failure') + assert len(failures) == len(baseline_failures) == 1 + assert failures[0].get('type') == baseline_failures[0].get('type') == 'AssertionError' + normalize = lambda text: text.replace('\r\n', '\n').strip() + assert normalize(failures[0].get('message', '')) == normalize(baseline_failures[0].get('message', '')) + + +def debugger_stage(): + import lldb + debugger = lldb.debugger + debugger.SetAsync(False) + target = debugger.GetSelectedTarget() + module = target.FindModule(target.GetExecutable()) + assert re.sub('[^0-9a-fA-F]', '', module.GetUUIDString() or '').lower() == UUID + stage = os.environ['TLS_CAPTURE_STAGE'] + report = {'stage': stage, 'passed': False, 'events': [], 'entries': [], 'resolutions': {}, 'breakpoint_hits': {}} + process = None + entry_addresses = {} + weak = [] + weak_creation = None + owner = None + returned = False + gc_count = 0 + mark_watch = None + mark_observation = None + root_collections = [] + root_collection = None + scan_returns = {} + stack_roots_return = None + word_watch = None + word_pending = None + word_witness = None + pointer_call_bp = None + pointer_return_bp = None + writer_entry_bp = None + writer_watch = None + writer_state = None + + def event(kind, **values): + report['events'].append({'kind': kind, **values}) + write(stage + '.json', report) + + def entry(name, signature=None, required=True): + contexts = module.FindFunctions(name, lldb.eFunctionNameTypeAny) + assert contexts.GetSize() <= 64, name + addresses = {} + resolution_key = signature or name + report['resolutions'][resolution_key] = [] + for i in range(contexts.GetSize()): + context = contexts.GetContextAtIndex(i) + function = context.GetFunction() + resolved = function.GetName() if function.IsValid() else context.GetSymbol().GetName() + address = function.GetStartAddress() if function.IsValid() else context.GetSymbol().GetStartAddress() + report['resolutions'][resolution_key].append({'function': resolved, 'file_address': address.GetFileAddress()}) + # FindFunctions includes inline scopes whose GetFunction is the enclosing physical function. + if signature is not None: + if not resolved or resolved.removeprefix('::') != signature: + continue + elif not resolved or resolved.split('(', 1)[0].strip().removeprefix('::') != name: + continue + if address.IsValid(): + addresses[address.GetFileAddress()] = address + if not required and len(addresses) != 1: + event('observer-entry-unavailable', name=name, signature=signature, physical_entries=len(addresses)) + return None, None + assert len(addresses) == 1, 'missing/ambiguous function entry: ' + name + address = next(iter(addresses.values())) + bp = target.BreakpointCreateBySBAddress(address) + assert bp.GetNumLocations() == 1, name + entry_addresses[bp.GetID()] = address + report['entries'].append({'name': name, 'breakpoint': bp.GetID(), + 'file_address': address.GetFileAddress()}) + return bp, address + + def inventory_inline_roots(): + # Inspect debug scopes and immutable instructions before launching an inferior. + # Inline add() contexts are not physical ABI entries; their registers cannot + # be interpreted as x0/x1/x2 arguments of add(). + parents = {} + for name in ['JSC::MachineThreads::gatherConservativeRoots', + 'JSC::ConservativeRoots::add', + 'JSC::ConservativeRoots::genericAddSpan', + 'JSC::ConservativeRoots::genericAddPointer']: + contexts = module.FindFunctions(name, lldb.eFunctionNameTypeAny) + assert contexts.GetSize() <= 64, 'root inventory context budget' + for i in range(contexts.GetSize()): + function = contexts.GetContextAtIndex(i).GetFunction() + if function.IsValid(): + parents[function.GetStartAddress().GetFileAddress()] = function + assert 0 < len(parents) <= 16, 'root inventory physical-parent budget' + total_instructions = 0 + for start, function in sorted(parents.items()): + scopes = [] + pending = [function.GetBlock()] + visited = 0 + while pending: + block = pending.pop() + if not block.IsValid(): + continue + visited += 1 + assert visited <= 4096, 'root inventory block budget' + sibling, child = block.GetSibling(), block.GetFirstChild() + if sibling.IsValid(): + pending.append(sibling) + if child.IsValid(): + pending.append(child) + name = block.GetInlinedName() or '' + if 'ConservativeRoots::' not in name: + continue + assert block.GetNumRanges() <= 128, 'root inventory range budget' + ranges = [(block.GetRangeStartAddress(i).GetFileAddress(), + block.GetRangeEndAddress(i).GetFileAddress()) + for i in range(block.GetNumRanges())] + variables = block.GetVariables(target, True, True, False) + assert variables.GetSize() <= 256, 'root inventory variable budget' + declarations = [] + for i in range(variables.GetSize()): + value = variables.GetValueAtIndex(i) + declaration = value.GetDeclaration() + declarations.append({'name': value.GetName(), 'type': value.GetTypeName(), + 'file': declaration.GetFileSpec().GetFilename(), 'line': declaration.GetLine()}) + scopes.append({'name': name, 'ranges': ranges, 'variables': declarations}) + instructions = function.GetInstructions(target) + assert instructions.GetSize() <= 32768, 'root inventory disassembly budget' + rows = [] + for i in range(instructions.GetSize()): + instruction = instructions.GetInstructionAtIndex(i) + address = instruction.GetAddress() + pc = address.GetFileAddress() + line = address.GetLineEntry() + file = line.GetFileSpec().GetFilename() + in_scope = any(a <= pc < b for scope in scopes for a, b in scope['ranges']) + if not in_scope and file != 'ConservativeRoots.cpp': + continue + total_instructions += 1 + assert total_instructions <= 8192, 'root inventory output budget' + rows.append({'file_address': pc, 'file': file, 'line': line.GetLine(), + 'mnemonic': instruction.GetMnemonic(target), 'operands': instruction.GetOperands(target)}) + event('inline-root-inventory', function=function.GetName(), file_address=start, + scopes=scopes, instructions=rows, blocks_visited=visited) + + def inventory_conductor_call(): + # This saved return address bounds the witnessed word, but does not name a local. + # Inspect the physical frame before designing a watch of its writer. + call_pc = 88384104 + function = module.ResolveFileAddress(call_pc).GetFunction() + assert function.IsValid() and function.GetName().startswith('JSC::Interpreter::executeCall(') + instructions = function.GetInstructions(target) + assert 0 < instructions.GetSize() <= 8192, 'conductor call disassembly budget' + rows = [] + for i in range(instructions.GetSize()): + instruction = instructions.GetInstructionAtIndex(i) + address = instruction.GetAddress() + pc = address.GetFileAddress() + if i >= 80 and not call_pc - 64 <= pc <= call_pc + 32: + continue + line = address.GetLineEntry() + rows.append(dict(file_address=pc, mnemonic=instruction.GetMnemonic(target), + operands=instruction.GetOperands(target), + file=line.GetFileSpec().GetFilename(), line=line.GetLine())) + call = [row for row in rows if row['file_address'] == call_pc] + assert len(call) == 1 and call[0]['mnemonic'] == 'bl', 'conductor entry-call admission' + layout = concrete_type('JSC::ProtoCallFrame') + assert 0 < layout.GetByteSize() <= 1024 and layout.GetNumberOfFields() <= 32 + fields = [] + for i in range(layout.GetNumberOfFields()): + member = layout.GetFieldAtIndex(i) + fields.append(dict(name=member.GetName(), offset=member.GetOffsetInBytes(), + bytes=member.GetType().GetByteSize(), type=member.GetType().GetName())) + event('conductor-call-inventory', function=function.GetName(), + file_address=function.GetStartAddress().GetFileAddress(), call_pc=call_pc, + instructions=rows, proto_call_frame=dict(bytes=layout.GetByteSize(), fields=fields), + interpretation='immutable frame inventory; no slot-to-local or writer attribution') + + def reg(frame, name): + value = frame.FindRegister(name) + assert value.IsValid() and value.GetError().Success(), name + return value.GetValueAsUnsigned() + + def word(address, size=8): + error = lldb.SBError() + value = process.ReadUnsignedFromMemory(address, size, error) + assert error.Success(), str(error) + return value + + def text(address, limit=1024**2): + error = lldb.SBError() + value = process.ReadCStringFromMemory(address, limit, error) + assert error.Success() and value is not None and len(value.encode()) < limit - 1 + return value + + def concrete_type(name): + values = target.FindTypes(name) + types = [values.GetTypeAtIndex(i).GetCanonicalType() for i in range(values.GetSize())] + types = [t for t in types if t.IsTypeComplete() and t.GetNumberOfFields() > 0] + assert types and len({t.GetByteSize() for t in types}) == 1, name + return types[0] + + def typed(address, name): + return target.CreateValueFromAddress(name, target.ResolveLoadAddress(address), concrete_type(name)).GetNonSyntheticValue() + + def field(value, name): + value = value.GetNonSyntheticValue().GetChildMemberWithName(name, lldb.eNoDynamicValues) + assert value.IsValid() and value.GetError().Success(), name + return value.GetNonSyntheticValue() + + def number(value): + error = lldb.SBError() + result = value.GetValueAsUnsigned(error) + assert error.Success(), str(error) + return result + + def pointer(value, pointee, depth=0): + assert depth <= 8 + value = value.GetNonSyntheticValue() + t = value.GetType().GetCanonicalType() + if t.IsPointerType() and t.GetPointeeType().GetCanonicalType().GetName() == pointee: + return [number(value)] + assert value.GetNumChildren() < 32 + result = [] + for i in range(value.GetNumChildren()): + child = value.GetChildAtIndex(i, lldb.eNoDynamicValues, False) + if not child.GetType().IsPointerType() or child.GetType().GetPointeeType().GetCanonicalType().GetName() == pointee: + result += pointer(child, pointee, depth + 1) + return result + + def precise_allocation(cell): + assert target.GetAddressByteSize() == 8 and target.GetByteOrder() == lldb.eByteOrderLittle + assert cell & 8, 'owner is not the observed precise-allocation kind' + header = concrete_type('JSC::PreciseAllocation').GetByteSize() + allocation = cell - ((header + 15) & ~15) + return header, allocation + + def precise_marker(visitor, cell): + # Use frozen HashTable metadata and DWARF field offsets, without inferior find/get calls. + header, allocation = precise_allocation(cell) + table_value = field(field(field(visitor, 'm_preciseAllocationMap'), 'm_impl'), 'm_table') + table = number(table_value) + if not table: + return header, allocation, None + capacity = word(table - 4, 4) + bucket_type = table_value.GetType().GetCanonicalType().GetPointeeType().GetCanonicalType() + stride = bucket_type.GetByteSize() + members = {bucket_type.GetFieldAtIndex(i).GetName(): bucket_type.GetFieldAtIndex(i) + for i in range(bucket_type.GetNumberOfFields())} + assert {'key', 'value'} <= members.keys() + key = members['key'] + assert key.GetType().GetCanonicalType().IsPointerType() and key.GetType().GetByteSize() == 8 + offset = key.GetOffsetInBytes() + assert 0 < capacity <= 2**20 and capacity & (capacity - 1) == 0 + assert 8 <= stride <= 256 and offset + 8 <= stride and capacity * stride <= 16 * 1024**2 + error = lldb.SBError() + buckets = process.ReadMemory(table, capacity * stride, error) + assert error.Success() and len(buckets) == capacity * stride + matches = [i for i in range(capacity) + if int.from_bytes(buckets[i * stride + offset:i * stride + offset + 8], 'little') == allocation] + assert len(matches) <= 1, 'actual precise allocation duplicated in verifier map' + if not matches: + return header, allocation, None + bucket = target.CreateValueFromAddress('owner-marker-bucket', + target.ResolveLoadAddress(table + matches[0] * stride), bucket_type).GetNonSyntheticValue() + data = pointer(field(bucket, 'value'), 'JSC::VerifierSlotVisitor::PreciseAllocationData') + assert len(data) == 1 and data[0] + record = typed(data[0], 'JSC::VerifierSlotVisitor::PreciseAllocationData') + assert number(field(record, 'm_allocation')) == allocation + return header, allocation, field(record, 'm_marker') + + def symbolize(pc, is_return=False): + address = target.ResolveLoadAddress(pc - 1 if is_return else pc) + function, symbol = address.GetFunction(), address.GetSymbol() + name = function.GetName() if function.IsValid() else symbol.GetName() if symbol.IsValid() else None + line = address.GetLineEntry() + return {'pc': pc, 'function': name, 'file_address': address.GetFileAddress(), + 'source_file': line.GetFileSpec().GetFilename() if line.IsValid() else None, + 'line': line.GetLine() if line.IsValid() else None} + + def frame_pointer_snapshot(frame, thread): + # Shipping keeps frame pointers but removes CFI tables; libc backtrace stored an empty marker stack. + pc, lr, sp, fp = frame.GetPC(), reg(frame, 'x30'), reg(frame, 'sp'), reg(frame, 'x29') + region = lldb.SBMemoryRegionInfo() + assert process.GetMemoryRegionInfo(sp, region).Success() and region.IsReadable() + ceiling = min(region.GetRegionEnd(), sp + 16 * 1024**2) + # A watchpoint may stop after the store; x30 can be a nested call's return, not this frame's caller. + frames = [symbolize(pc)] + links = [] + current = fp + for _ in range(128): + if not current: + termination = 'null-frame-pointer' + break + if current % 16 or not sp <= current or current + 16 > ceiling: + termination = 'outside-readable-stack' + break + previous, saved_lr = word(current), word(current + 8) + links.append({'fp': current, 'previous_fp': previous, 'saved_lr': saved_lr}) + if saved_lr: + frames.append(symbolize(saved_lr, True)) + if previous and previous <= current: + termination = 'non-increasing-frame-pointer' + break + current = previous + else: + termination = 'frame-limit' + assert len(links) >= 2 and sum(bool(f['function']) for f in frames) >= 3, 'live frame-pointer path unavailable' + return {'thread': thread.GetThreadID(), 'pc': pc, 'lr': lr, 'sp': sp, 'fp': fp, + 'links': links, 'frames': frames, 'termination': termination} + + def roots_snapshot(address, prefix_count=None): + roots = typed(address, 'JSC::ConservativeRoots') + count = number(field(roots, 'm_size')) + capacity = number(field(roots, 'm_capacity')) + data = number(field(roots, 'm_roots')) + assert 0 <= count <= capacity <= 1024**2 and data + error = lldb.SBError() + raw = process.ReadMemory(data, count * 8, error) if count else b'' + assert not count or (error.Success() and len(raw) == count * 8) + indices = [i for i in range(count) if int.from_bytes(raw[i * 8:i * 8 + 8], 'little') == owner['owner']] + assert len(indices) <= 64, 'accepted-root duplicate bound exceeded' + result = dict(roots=address, data=data, count=count, capacity=capacity, + owner_indices=indices, sha256=hashlib.sha256(raw).hexdigest()) + if prefix_count is not None: + assert 0 <= prefix_count <= count + result['prefix_sha256'] = hashlib.sha256(raw[:prefix_count * 8]).hexdigest() + return result + + def span_candidates(begin, end): + begin, end = sorted((begin, end)) + assert begin % 8 == end % 8 == 0 and 0 <= end - begin <= 16 * 1024**2 + error = lldb.SBError() + raw = process.ReadMemory(begin, end - begin, error) if begin != end else b'' + assert begin == end or (error.Success() and len(raw) == end - begin) + _, allocation = precise_allocation(owner['owner']) + cell_size = number(field(typed(allocation, 'JSC::PreciseAllocation'), 'm_cellSize')) + assert 0 < cell_size <= 16 * 1024**2 + candidates = [] + for offset in range(0, len(raw), 8): + value = int.from_bytes(raw[offset:offset + 8], 'little') + if owner['owner'] <= value <= owner['owner'] + cell_size: + candidates.append(dict(address=begin + offset, value=value, + offset_into_cell=value - owner['owner'])) + assert len(candidates) <= 64, 'candidate-word bound exceeded' + return begin, end, cell_size, candidates, hashlib.sha256(raw).hexdigest() + + def stack_word_write(frame, thread): + assert writer_state is not None and thread.GetThreadID() == writer_state['thread'] + assert len(writer_state['writes']) < 32, 'stack-writer stop budget exhausted' + pc = frame.GetPC() + instructions = target.ReadInstructions(target.ResolveLoadAddress(pc - 8), 3) + assert instructions.GetSize() == 3, 'stack-writer instruction window unavailable' + rows = [] + for i in range(3): + instruction = instructions.GetInstructionAtIndex(i) + rows.append(dict(pc=instruction.GetAddress().GetLoadAddress(target), + mnemonic=instruction.GetMnemonic(target), operands=instruction.GetOperands(target))) + event('candidate-stack-writer-stop', pc=pc, slot_value=word(writer_state['address']), + sp=reg(frame, 'sp'), fp=reg(frame, 'x29'), instructions=rows) + # LLDB ARM exposes a write watch after stepping the store. Admit its effective + # destination; an Always watch may otherwise report an adjacent access. + instruction = rows[1] + assert instruction['pc'] == pc - 4 and instruction['mnemonic'] in ('str', 'stur', 'stp', 'strb'), 'unsupported stack-writer instruction' + match = re.fullmatch(r'([xwqds]\d+|[xw]zr)(?:, ([xwqds]\d+|[xw]zr))?, \[(sp|x29)(?:, #(-?0x[0-9a-f]+))?\](!)?(?:, #(-?0x[0-9a-f]+))?', instruction['operands']) + assert match is not None, 'unsupported stack-writer addressing mode' + first, second, base, offset, pre, post = match.groups() + assert (second is not None) == (instruction['mnemonic'] == 'stp') + width = {'x': 8, 'w': 4, 'q': 16, 'd': 8, 's': 4}[first[0]] + if instruction['mnemonic'] == 'strb': + assert first[0] == 'w' and second is None + width = 1 + assert second is None or second[0] == first[0] + base_value = reg(frame, base) + address = base_value if pre else base_value + int(offset or '0', 16) + if post: + assert offset is None and pre is None + address = base_value - int(post, 16) + size = width * (2 if second else 1) + slot = writer_state['address'] + assert max(address, slot) < min(address + size, slot + 8), 'watchpoint was an adjacent write' + receipt = dict(pc=pc, store_pc=pc - 4, source=symbolize(pc - 4), + instruction=instruction, destination=address, bytes=size, + registers={name: frame.FindRegister(name).GetValue() for name in [first, second] if name}, + slot_value=word(slot), sp=reg(frame, 'sp'), fp=reg(frame, 'x29')) + if receipt['source']['file_address'] == 38748200: + symbol = target.ResolveLoadAddress(pc - 4).GetSymbol() + assert symbol.GetMangledName() == '_RNvMsd_NtNtCsgTvE0GkrhKz_11bun_runtime6socket11socket_bodyNtB5_20DuplexUpgradeContext6deinit' + assert symbol.GetStartAddress().GetFileAddress() == 38747968 + loads = target.ReadInstructions(module.ResolveFileAddress(38748192), 3) + assert loads.GetSize() == 3 + assert [(loads.GetInstructionAtIndex(i).GetMnemonic(target), + loads.GetInstructionAtIndex(i).GetOperands(target)) for i in range(3)] == [ + ('ldr', 'q0, [x19]'), ('ldr', 'q1, [x19, #0x10]'), ('str', 'q0, [sp]')] + assert address == slot == reg(frame, 'sp') and size == 16 + source = reg(frame, 'x19') + assert source and source % 8 == 0 + # The frozen clear copies 192 bytes before testing the saved Option tag at +191. + old_tag, current_tag = word(address + 191, 1), word(source + 191, 1) + assert old_tag == current_tag == reg(frame, 'w8') == 2, 'old config was not the absent variant' + assert word(source) == receipt['slot_value'] and word(source + 8) == word(address + 8) + receipt['absent_config_copy'] = dict(source_context=source, source_address=source, + destination=address, copied_payload_bytes=192, selected_bytes=16, + source_word=word(source), second_word=word(source + 8), old_tag=old_tag, + current_tag=current_tag, tag_offset=191, physical_symbol=symbol.GetMangledName()) + writer_state['writes'].append(receipt) + event('stack-word-write', **receipt) + + def conducting_state(frame, thread, heap): + collector = typed(number(field(heap, 'm_collector')), 'JSC::Collector') + state_address, conductor = reg(frame, 'x4'), reg(frame, 'x5') + assert state_address and conductor, 'no original conducting-mutator state' + assert number(field(collector, 'm_conductingMutatorState')) == state_address + assert number(field(collector, 'm_conductorThread')) == conductor + state = typed(state_address, 'JSC::CurrentThreadState') + top, origin = number(field(state, 'stackTop')), number(field(state, 'stackOrigin')) + registers = number(field(state, 'registerState')) + assert registers and 0 < origin - top <= 16 * 1024**2 + saved = typed(registers, 'JSC::RegisterState') + fp = number(field(saved, 'x29')) + links = [] + for _ in range(128): + if not fp: + break + assert fp % 16 == 0 and top <= fp and fp + 16 <= origin, 'saved FP outside conducting stack' + previous, saved_lr = word(fp), word(fp + 8) + links.append(dict(fp=fp, previous_fp=previous, saved_lr=saved_lr, + caller=symbolize(saved_lr, True) if saved_lr else None)) + assert not previous or previous > fp, 'conducting FP chain does not increase' + fp = previous + else: + raise RuntimeError('conducting frame-pointer budget exhausted') + assert len(links) >= 2, 'conducting frame-pointer chain unavailable' + return dict(address=state_address, conductor=conductor, debugger_thread=thread.GetThreadID(), + scanner_sp=reg(frame, 'sp'), stack_top=top, stack_origin=origin, + registers=registers, register_bytes=saved.GetByteSize(), links=links) + + def word_provenance(scan, witness): + state = root_collections[scan['collection']]['conducting_state'] + begin, end = sorted((scan['begin'], scan['end'])) + address = witness['address'] + if (begin, end) == (state['registers'], state['registers'] + state['register_bytes']): + offset = address - begin + assert offset % 8 == 0 and 0 <= offset < 88 + return dict(kind='saved-conductor-register', register='x' + str(19 + offset // 8), + storage_address=address, conducting_state=state['address']) + assert (begin, end) == (state['stack_top'], state['stack_origin']), 'witnessed span is not an admitted original conductor span' + intervals = [] + for lower, upper in zip(state['links'], state['links'][1:]): + if lower['fp'] + 16 <= address and address + 8 <= upper['fp']: + intervals.append(dict(lower=lower, upper=upper, offset_from_lower_fp=address - lower['fp'])) + assert len(intervals) == 1, 'scanner word lacks a bounded adjacent-frame interval' + return dict(kind='original-conductor-stack', storage_address=address, + conducting_state=state['address'], interval=intervals[0], + interpretation='original stack word bounded by adjacent saved FP records; source variable and originating writer remain unproven') + + def accepted_span(scan, after): + assert after['roots'] == scan['roots'] and after['count'] >= scan['before_count'] + accepted = [i for i in after['owner_indices'] if i >= scan['before_count']] + if not accepted: + return + begin, end, cell_size, candidates, span_sha = span_candidates(scan['begin'], scan['end']) + receipt = dict(collection=scan['collection'], phase=scan['phase'], roots=scan['roots'], + thread=scan['thread'], begin=begin, end=end, before_count=scan['before_count'], + after_count=after['count'], accepted_owner_indices=accepted, + cell_size=cell_size, candidate_words=candidates, + candidate_interpretation='unmodified exact/interior/one-past pointer candidates; accepted scanner iteration not yet identified', + span_sha256=span_sha, call_site=scan['call_site'], scanner_sp=scan['scanner_sp']) + root_collections[scan['collection']]['accepted_spans'].append(receipt) + if word_witness is not None and word_witness['span_return'] == scan['return_breakpoint']: + assert set(word_witness['accepted_owner_indices']) <= set(accepted) + receipt['scanner_word'] = word_witness + receipt['candidate_interpretation'] = 'exact scanner word proved by watched read and call-return insertion' + receipt['word_provenance'] = word_provenance(scan, word_witness) + event('accepted-owner-span', **receipt) + + def marker_snapshot(visitor, cell): + header, allocation, marker = precise_marker(visitor, cell) + assert marker is not None, 'actual precise allocation missing in completed verifier map' + marked = field(field(typed(allocation, 'JSC::PreciseAllocation'), 'm_isMarked'), 'value') + assert marked.GetByteSize() == 1 and word(marked.GetLoadAddress(), 1) == 1, 'owner was not marked by the real GC' + referrer = number(field(field(marker, 'm_referrer'), 'm_bits')) + assert referrer == 6, 'marker is not the observed direct ConservativeScan root' + stacks = pointer(field(marker, 'm_stack'), 'WTF::StackTrace') + assert len(stacks) == 1 and stacks[0] + stack = typed(stacks[0], 'WTF::StackTrace') + count, initial = number(field(stack, 'm_size')), number(field(stack, 'm_initialFrame')) + layout = stack.GetType().GetCanonicalType() + event('passive-marker-layout', owner=cell, allocation=allocation, allocation_header_bytes=header, + marker=marker.GetLoadAddress(), referrer_bits=referrer, stack=stacks[0], count=count, initial=initial, + raw_header_words=[word(stacks[0]), word(stacks[0] + 8)], type_bytes=layout.GetByteSize(), + fields={layout.GetFieldAtIndex(i).GetName(): layout.GetFieldAtIndex(i).GetOffsetInBytes() + for i in range(layout.GetNumberOfFields())}) + contexts = {} + for name in ['WTF::StackTrace::captureStackTrace', 'JSC::VerifierSlotVisitor::testAndSetMarked']: + found = module.FindFunctions(name, lldb.eFunctionNameTypeAny) + assert found.GetSize() <= 64 + contexts[name] = [] + for i in range(found.GetSize()): + context = found.GetContextAtIndex(i) + function, block = context.GetFunction(), context.GetBlock() + start = function.GetStartAddress().GetLineEntry() + contexts[name].append({'function': function.GetName(), + 'compile_unit': str(context.GetCompileUnit().GetFileSpec()), + 'source': str(start.GetFileSpec()), 'line': start.GetLine(), + 'inline': block.IsInlined(), 'inline_name': block.GetInlinedName()}) + event('marker-source-contexts', contexts=contexts) + assert (count == 0 and initial == 0) or (0 < count <= 100 and initial == 2) + start = field(stack, 'm_stack').GetLoadAddress() + initial * 8 + frames = [] + for i in range(count): + pc = word(start + i * 8) + assert pc + frames.append(symbolize(pc, True)) + event('passive-marker-snapshot', owner=cell, allocation=allocation, real_gc_marked=True, referrer_bits=referrer, + root_reason='ConservativeScan', stack=stacks[0], frames=frames, + unresolved_frames=sum(frame['function'] is None for frame in frames)) + + try: + environment = {'PATH': '/usr/bin:/bin', 'CI': '1', 'BUN_GARBAGE_COLLECTOR_LEVEL': '0'} + if stage == 'capture': + spawn, _ = entry('posix_spawn_bun') + args = ['test', FIXTURE, '--test-name-pattern', '^' + CASE + '$', + '--reporter=junit', '--reporter-outfile=' + str(OUT / 'fixture.xml')] + else: + child = json.loads((OUT / 'child.json').read_text()) + args = child['argv'][1:] + assert child['argv'][0] == str(BINARY) and len(args) == 2 and args[0] == '-e' + assert hashlib.sha256(args[1].encode()).hexdigest() == child['script_sha256'] + # These are the frozen harness's effective bunEnv additions; no host environment is inherited. + environment.update(GITHUB_ACTIONS='false', BUN_DEBUG_QUIET_LOGS='1', NO_COLOR='1', TZ='Etc/UTC', + BUN_RUNTIME_TRANSPILER_CACHE_PATH='0', BUN_FEATURE_FLAG_INTERNAL_FOR_TESTING='1', + BUN_DISABLE_SLOW_FILESYSTEM_WARNING='1', BUN_CONFIG_HTTP_RETRY_BACKOFF='1', + BUN_INTERNAL_INTERACTIVE_ASSUME_TTY='1', BUN_FEATURE_FLAG_EXPERIMENTAL_BAKE='1', + WANTS_LOUD='0', AGENT='false', BUN_JSC_verboseVerifyGC='1') + weak_bp, _ = entry('JSC::constructWeakRef') + weak_return_bp = None + completion_bp, completion_address = entry('Bun__UpgradedDuplex__onWriteComplete') + gc_bp, _ = entry('JSC__VM__runGC') + writer_entry_bp, writer_entry_address = entry('JSC::Interpreter::executeCall') + assert writer_entry_address.GetFileAddress() == 88383168 + prologue = writer_entry_address.GetFunction().GetInstructions(target) + expected = [('sub', 'sp, sp, #0x180'), ('stp', 'x29, x30, [sp, #0x120]'), + ('stp', 'x28, x27, [sp, #0x130]'), ('stp', 'x26, x25, [sp, #0x140]'), + ('stp', 'x24, x23, [sp, #0x150]'), ('stp', 'x22, x21, [sp, #0x160]'), + ('stp', 'x20, x19, [sp, #0x170]'), ('add', 'x29, sp, #0x120')] + for i, expected_instruction in enumerate(expected): + instruction = prologue.GetInstructionAtIndex(i) + assert instruction.GetAddress().GetFileAddress() == 88383168 + i * 4 + assert (instruction.GetMnemonic(target), instruction.GetOperands(target)) == expected_instruction + timer_call = target.ReadInstructions(module.ResolveFileAddress(27509444), 1) + assert timer_call.GetSize() == 1 + instruction = timer_call.GetInstructionAtIndex(0) + assert instruction.GetMnemonic(target) == 'bl' + assert int(instruction.GetOperands(target), 16) == 91664704 + profiled = module.ResolveFileAddress(91664704).GetFunction() + assert profiled.IsValid() and profiled.GetName().startswith('JSC::profiledCall(') + tail_call = target.ReadInstructions(module.ResolveFileAddress(91664844), 1) + assert tail_call.GetSize() == 1 + instruction = tail_call.GetInstructionAtIndex(0) + assert instruction.GetMnemonic(target) == 'b' + assert int(instruction.GetOperands(target), 16) == 88383168 + vm_layout = concrete_type('JSC::VM') + interpreter_fields = [vm_layout.GetFieldAtIndex(i) for i in range(vm_layout.GetNumberOfFields()) + if vm_layout.GetFieldAtIndex(i).GetName() == 'interpreter'] + assert len(interpreter_fields) == 1 + interpreter_offset = interpreter_fields[0].GetOffsetInBytes() + assert interpreter_offset == 0x2d468 + event('early-stack-writer-layout', interpreter_offset=interpreter_offset, + timer_call=27509444, profiled_call=91664704, interpreter_tail_call=91664844, + timer_return=27509448) + mark_bp = target.BreakpointCreateByLocation('Collector.cpp', 1068) + assert mark_bp.GetNumLocations() > 0, 'missing post-mark verifier boundary' + mark_bp.SetEnabled(False) + return_bp = None + stack_roots_bp, _ = entry('JSC::MachineThreads::gatherConservativeRoots', required=False) + scan_bps = [] + scanner_loops = [] + for signature in [ + 'void JSC::ConservativeRoots::genericAddSpan(void*, void*, JSC::DummyMarkHook&)', + 'void JSC::ConservativeRoots::genericAddSpan(void*, void*, JSC::CompositeMarkHook&)']: + bp, address = entry('JSC::ConservativeRoots::genericAddSpan', signature, required=False) + scan_bps.append(bp) + if address is not None: + # Locked shipping Build ID: admit the actual scanner load, argument moves and call. + sites = {87227072: [(87227200, 87241536), (87227264, 87243712)], + 87227392: [(87227520, 87246848), (87227584, 87248576)]} + assert address.GetFileAddress() in sites + instructions = address.GetFunction().GetInstructions(target) + assert 0 < instructions.GetSize() <= 4096 + rows = {instructions.GetInstructionAtIndex(i).GetAddress().GetFileAddress(): + instructions.GetInstructionAtIndex(i) for i in range(instructions.GetSize())} + for load_pc, callee_pc in sites[address.GetFileAddress()]: + expected = [('ldr', 'x1, [x26], #0x8'), ('mov', 'x0, x20'), + ('mov', 'w2, w22'), ('mov', 'w3, w23'), ('mov', 'x4, x21'), + ('mov', 'x5, x24' if load_pc in (87227200, 87227520) else 'x5, xzr'), + ('mov', 'x6, x19')] + for i, (mnemonic, operands) in enumerate(expected): + instruction = rows[load_pc + i * 4] + assert (instruction.GetMnemonic(target), instruction.GetOperands(target)) == (mnemonic, operands) + instruction = rows[load_pc + 28] + assert instruction.GetMnemonic(target) == 'bl' + assert int(instruction.GetOperands(target), 16) == callee_pc + callee = module.ResolveFileAddress(callee_pc).GetFunction().GetName() + assert callee and 'ConservativeRoots::genericAddPointer<' in callee + scanner_loops.append(dict(span_breakpoint=bp.GetID(), read_stop=load_pc + 4, + call=load_pc + 28, returned=load_pc + 32, callee=callee)) + report['scanner_loops'] = scanner_loops + # Frozen Linux ARM64 layouts, admitted from DWARF before any inferior reads. + for name, size, fields in [ + ('JSC::CurrentThreadState', 24, ['stackOrigin', 'stackTop', 'registerState']), + ('JSC::RegisterState', 88, ['x' + str(i) for i in range(19, 30)])]: + layout = concrete_type(name) + assert layout.GetByteSize() == size and layout.GetNumberOfFields() == len(fields) + for i, expected in enumerate(fields): + member = layout.GetFieldAtIndex(i) + assert member.GetName() == expected and member.GetOffsetInBytes() == i * 8 + assert member.GetType().GetByteSize() == 8 + inventory_conductor_call() + root_bps = [stack_roots_bp, *scan_bps] + if any(bp is None for bp in root_bps): + inventory_inline_roots() + assert all(bp is not None for bp in root_bps), 'required root-span observer entries unavailable' + for bp in root_bps: + bp.SetEnabled(False) + info = lldb.SBLaunchInfo(args) + info.SetWorkingDirectory(str(SOURCE)) + info.SetEnvironmentEntries([key + '=' + value for key, value in environment.items()], False) + info.SetLaunchFlags(0) + assert info.AddOpenFileAction(0, '/dev/null', True, False) + assert info.AddOpenFileAction(1, str(OUT / (stage + '.stdout')), False, True) + assert info.AddOpenFileAction(2, str(OUT / (stage + '.stderr')), False, True) + error = lldb.SBError() + process = target.Launch(info, error) + assert error.Success(), str(error) + for _ in range(256): + if process.GetState() == lldb.eStateExited: + break + assert process.GetState() == lldb.eStateStopped, str(process.GetState()) + threads = [t for t in process if t.GetStopReason() in + (lldb.eStopReasonBreakpoint, lldb.eStopReasonWatchpoint)] + assert len(threads) == 1, 'unexpected stop or concurrent debugger event' + thread = threads[0] + frame = thread.GetFrameAtIndex(0) + if thread.GetStopReason() == lldb.eStopReasonWatchpoint: + assert stage == 'trace' and thread.GetStopReasonDataCount() == 1 + watch_id = thread.GetStopReasonDataAtIndex(0) + if writer_watch is not None and watch_id == writer_watch.GetID(): + assert writer_watch.IsEnabled() + stack_word_write(frame, thread) + assert process.Continue().Success() + continue + if word_watch is not None and watch_id == word_watch.GetID(): + assert word_watch.IsEnabled() and word_pending is not None + word_watch.SetEnabled(False) + assert not word_watch.IsEnabled() + assert word_pending['phase'] == 'armed' and word_pending['thread'] == thread.GetThreadID() + loops = [row for row in scanner_loops if row['span_breakpoint'] == word_pending['span_breakpoint'] + and module.ResolveFileAddress(row['read_stop']).GetLoadAddress(target) == frame.GetPC()] + # LLDB23 exposes ARM read watches after its transparent single-step of the load. + assert len(loops) == 1, 'candidate was not read by the admitted scanner load' + assert reg(frame, 'x26') == word_pending['address'] + 8 + assert reg(frame, 'x1') == word_pending['value'] == word(word_pending['address']) + assert reg(frame, 'x20') == word_pending['roots'] + word_pending.update(phase='read', loop=loops[0], read_pc=frame.GetPC()) + pointer_call_bp = target.BreakpointCreateBySBAddress(module.ResolveFileAddress(loops[0]['call'])) + assert pointer_call_bp.GetNumLocations() == 1 + pointer_call_bp.SetThreadID(thread.GetThreadID()) + pointer_call_bp.SetOneShot(True) + event('scanner-word-read', **word_pending) + assert process.Continue().Success() + continue + assert mark_watch is not None and mark_watch.IsEnabled() + assert thread.GetStopReasonDataCount() == 1 + assert thread.GetStopReasonDataAtIndex(0) == mark_watch.GetID() + value = word(mark_observation['address'], 1) + event('real-mark-write', watchpoint=mark_watch.GetID(), value=value, + stopped_pc=frame.GetPC(), thread=thread.GetThreadID()) + assert value in (0, 1) and value != mark_observation['last_value'] + mark_observation['last_value'] = value + if value == 0: + mark_observation['observed_zero'] = True + else: + assert mark_observation['observed_zero'] and 'frame_pointer_path' not in mark_observation + mark_watch.SetEnabled(False) + assert not mark_watch.IsEnabled() + mark_observation['frame_pointer_path'] = frame_pointer_snapshot(frame, thread) + assert root_collection is not None and root_collection['phase'] == 'after-machine-stack' + current_roots = roots_snapshot(root_collection['roots']) + after_stack = root_collection['after_stack'] + assert current_roots['count'] >= after_stack['count'] + assert current_roots['owner_indices'] and root_collection['accepted_spans'] + assert word_witness is not None and word_pending is None + assert set(word_witness['accepted_owner_indices']) <= set(current_roots['owner_indices']) + assert any(span.get('scanner_word') == word_witness for span in root_collection['accepted_spans']) + stack_indices = [i for i in current_roots['owner_indices'] if i < after_stack['count']] + assert stack_indices == after_stack['owner_indices'], 'stack-root prefix changed' + subsequent_indices = [i for i in current_roots['owner_indices'] if i >= after_stack['count']] + root_collection.update(phase='complete', at_mark=current_roots, + machine_stack_indices=stack_indices, + subsequent_indices=subsequent_indices) + mark_observation['root_collection'] = root_collection['index'] + event('accepted-root-list-at-owner-mark', **current_roots, + collection=root_collection['index'], machine_stack_indices=stack_indices, + subsequent_indices=subsequent_indices) + for bp in root_bps: + bp.SetEnabled(False) + assert not scan_returns and stack_roots_return is None + event('live-real-mark-writer', **mark_observation) + assert process.Continue().Success() + continue + ids = {thread.GetStopReasonDataAtIndex(i) for i in range(0, thread.GetStopReasonDataCount(), 2)} + report['last_stop'] = {'pc': frame.GetPC(), 'function': frame.GetFunctionName(), + 'breakpoints': sorted(ids), 'expected_entry_pcs': { + str(i): entry_addresses[i].GetLoadAddress(target) for i in ids if i in entry_addresses}} + for bp_id in ids: + key = str(bp_id) + report['breakpoint_hits'][key] = report['breakpoint_hits'].get(key, 0) + 1 + if bp_id in entry_addresses: + assert frame.GetPC() == entry_addresses[bp_id].GetLoadAddress(target), 'stop is not at function entry' + if stage == 'capture': + assert ids == {spawn.GetID()} + argv = reg(frame, 'x3') + values = [] + for i in range(64): + address = word(argv + i * 8) + if not address: + break + values.append(text(address)) + else: + raise RuntimeError('unterminated child argv') + if len(values) == 3 and values[:2] == [str(BINARY), '-e']: + assert not (OUT / 'child.json').exists(), 'multiple selected children' + write('child.json', {'argv': values, 'script_sha256': hashlib.sha256(values[2].encode()).hexdigest()}) + event('captured-actual-fixture-child') + elif writer_entry_bp is not None and writer_entry_bp.GetID() in ids: + timer_return = module.ResolveFileAddress(27509448).GetLoadAddress(target) + if reg(frame, 'x30') == timer_return: + assert writer_state is None and writer_watch is None + # Admitted prologue: FP=entrySP-0x180+0x120. The previous proof + # placed the accepted word at FP-0x40; this run must rediscover it. + entry_sp = reg(frame, 'sp') + prospective_fp = entry_sp - 0x60 + address = prospective_fp - 0x40 + assert address % 8 == 0 + options = lldb.SBWatchpointOptions() + options.SetWatchpointTypeRead(False) + options.SetWatchpointTypeWrite(lldb.eWatchpointWriteTypeAlways) + assert options.GetWatchpointTypeWrite() == lldb.eWatchpointWriteTypeAlways + error = lldb.SBError() + writer_watch = target.WatchpointCreateByAddress(address, 8, options, error) + assert error.Success() and writer_watch.IsValid() and writer_watch.IsEnabled() + assert writer_watch.GetWatchAddress() == address and writer_watch.GetWatchSize() == 8 + writer_state = dict(address=address, thread=thread.GetThreadID(), entry_sp=entry_sp, + provisional_vm=reg(frame, 'x0') - interpreter_offset, + prospective_fp=prospective_fp, initial_value=word(address), writes=[], + incoming={name: reg(frame, name) for name in ['x' + str(i) for i in range(6)]}) + writer_entry_bp.SetEnabled(False) + event('stack-writer-watchpoint-admission', watchpoint=writer_watch.GetID(), **writer_state) + elif stack_roots_bp.GetID() in ids: + assert returned and gc_count == 31 and owner is not None + heap = field(typed(owner['vm'], 'JSC::VM'), 'heap') + machines = pointer(field(heap, 'm_machineThreads'), 'JSC::MachineThreads') + assert machines == [reg(frame, 'x0')], 'machine-stack scanner belongs to another VM' + assert root_collection is None or root_collection['phase'] == 'after-machine-stack' + assert len(root_collections) < 8 and not scan_returns and stack_roots_return is None + roots = reg(frame, 'x1') + initial_roots = roots_snapshot(roots) + assert initial_roots['count'] == 0 + root_collection = dict(index=len(root_collections), roots=roots, phase='machine-stack', + accepted_spans=[], initial=initial_roots) + root_collection['conducting_state'] = conducting_state(frame, thread, heap) + root_collections.append(root_collection) + event('machine-stack-gather-entry', collection=root_collection['index'], **initial_roots) + stack_roots_return = target.BreakpointCreateByAddress(reg(frame, 'x30')) + assert stack_roots_return.GetNumLocations() == 1 + stack_roots_return.SetThreadID(thread.GetThreadID()) + stack_roots_return.SetOneShot(True) + elif stack_roots_return is not None and stack_roots_return.GetID() in ids: + assert root_collection is not None and root_collection['phase'] == 'machine-stack' + assert not scan_returns + after = roots_snapshot(root_collection['roots']) + root_collection.update(phase='after-machine-stack', after_stack=after) + event('machine-stack-gather-complete', collection=root_collection['index'], **after) + stack_roots_return = None + elif any(bp.GetID() in ids for bp in scan_bps): + assert root_collection is not None and root_collection['phase'] in ('machine-stack', 'after-machine-stack') + roots = reg(frame, 'x0') + assert roots == root_collection['roots'] and not scan_returns + before = roots_snapshot(roots) + bp = target.BreakpointCreateByAddress(reg(frame, 'x30')) + assert bp.GetNumLocations() == 1 + bp.SetThreadID(thread.GetThreadID()) + bp.SetOneShot(True) + scan = dict(collection=root_collection['index'], + phase=root_collection['phase'], roots=roots, before_count=before['count'], + begin=reg(frame, 'x1'), end=reg(frame, 'x2'), thread=thread.GetThreadID(), + return_breakpoint=bp.GetID(), call_site=symbolize(reg(frame, 'x30'), True), + scanner_sp=reg(frame, 'sp')) + scan_returns[bp.GetID()] = scan + if word_witness is None: + _, _, _, candidates, _ = span_candidates(scan['begin'], scan['end']) + if candidates: + assert len(candidates) == 1 and word_pending is None, 'ambiguous scanner-word candidates' + candidate = candidates[0] + assert writer_state is not None and writer_watch.IsEnabled(), 'selected timer frame was not watched' + assert writer_state['thread'] == thread.GetThreadID() and candidate['address'] == writer_state['address'], 'accepted word differs from watched frame slot' + assert writer_state['provisional_vm'] == owner['vm'], 'first timer belongs to another VM' + assert writer_state['writes'] and writer_state['writes'][-1]['slot_value'] == candidate['value'], 'no admitted write of the accepted owner in watched stack storage' + writer_watch.SetEnabled(False) + assert not writer_watch.IsEnabled() + error = lldb.SBError() + word_watch = target.WatchAddress(candidate['address'], 8, True, False, error) + assert error.Success() and word_watch.IsValid() and word_watch.IsEnabled() + assert word_watch.GetWatchAddress() == candidate['address'] and word_watch.GetWatchSize() == 8 + assert word_watch.GetID() > 0 and word_watch.GetID() != mark_watch.GetID() + word_pending = dict(candidate, phase='armed', roots=roots, thread=thread.GetThreadID(), + span_return=bp.GetID(), span_breakpoint=next(i for i in ids if i in {b.GetID() for b in scan_bps})) + event('scanner-word-watchpoint-admission', watchpoint=word_watch.GetID(), **word_pending) + elif ids.intersection(scan_returns): + assert len(ids.intersection(scan_returns)) == 1 + scan = scan_returns.pop(next(iter(ids.intersection(scan_returns)))) + assert scan['thread'] == thread.GetThreadID() + assert word_pending is None, 'scanner span returned before the word witness completed' + accepted_span(scan, roots_snapshot(scan['roots'])) + elif pointer_call_bp is not None and pointer_call_bp.GetID() in ids: + assert word_pending is not None and word_pending['phase'] == 'read' + assert thread.GetThreadID() == word_pending['thread'] + loop = word_pending['loop'] + assert frame.GetPC() == module.ResolveFileAddress(loop['call']).GetLoadAddress(target) + assert reg(frame, 'x0') == word_pending['roots'] and reg(frame, 'x1') == word_pending['value'] + assert reg(frame, 'x26') == word_pending['address'] + 8 + word_pending.update(phase='called', before=roots_snapshot(word_pending['roots'])) + pointer_call_bp = None + pointer_return_bp = target.BreakpointCreateBySBAddress(module.ResolveFileAddress(loop['returned'])) + assert pointer_return_bp.GetNumLocations() == 1 + pointer_return_bp.SetThreadID(thread.GetThreadID()) + pointer_return_bp.SetOneShot(True) + elif pointer_return_bp is not None and pointer_return_bp.GetID() in ids: + assert word_pending is not None and word_pending['phase'] == 'called' + assert thread.GetThreadID() == word_pending['thread'] + assert frame.GetPC() == module.ResolveFileAddress(word_pending['loop']['returned']).GetLoadAddress(target) + before = word_pending['before'] + after = roots_snapshot(word_pending['roots'], before['count']) + assert after['prefix_sha256'] == before['sha256'], 'root prefix changed across the scanner call' + accepted = [i for i in after['owner_indices'] if i >= before['count']] + assert accepted and word_witness is None, 'observed scanner word did not append the owner' + word_witness = dict(word_pending, phase='accepted', after=after, accepted_owner_indices=accepted) + word_pending = None + pointer_return_bp = None + event('accepted-scanner-word', **word_witness) + elif weak_bp.GetID() in ids: + assert weak_creation is None + global_object, call = reg(frame, 'x0'), reg(frame, 'x1') + assert word(call + 32, 4) == 2, 'fixture WeakRef constructor arguments changed' + weak_creation = {'global': global_object, + 'vm': number(field(typed(global_object, 'JSC::JSGlobalObject'), 'm_vm')), + 'owner': word(call + 48)} + weak_return_bp = target.BreakpointCreateByAddress(reg(frame, 'x30')) + weak_return_bp.SetThreadID(thread.GetThreadID()) + weak_return_bp.SetOneShot(True) + event('weak-ref-constructor-entry', **weak_creation) + elif weak_return_bp is not None and weak_return_bp.GetID() in ids: + assert weak_creation is not None + created = reg(frame, 'x0') + actual_target = pointer(field(typed(created, 'JSC::JSWeakObjectRef'), 'm_value'), 'JSC::JSCell') + assert actual_target == [weak_creation['owner']], 'constructed weak target does not match' + weak.append(dict(weak_creation, weak=created)) + assert len(weak) <= 16 + weak_creation = None + event('weak-ref-created', **weak[-1]) + elif completion_bp.GetID() in ids: + call = reg(frame, 'x1') + # Frozen CallFrame.h: callee, argumentCountIncludingThis, then this, in 8-byte registers. + callee, count, this = word(call + 24), word(call + 32, 4), word(call + 40) + candidates = [w for w in weak if w['owner'] == this] + event('completion-entry', callee=callee, this=this, argument_count_including_this=count, + matching_weak_targets=len(candidates)) + if candidates: + function = typed(callee, 'Zig::JSFFIFunction') + native = number(field(function, 'm_function')) + assert native == completion_address.GetLoadAddress(target) + if number(field(function, 'dataPtr')) == 0: + assert owner is None and len(candidates) == 1 + owner = dict(candidates[0], callee=callee, argument_count_including_this=count) + assert writer_state is not None and writer_state['provisional_vm'] == owner['vm'], 'late callback VM differs from first timer' + assert count == 1, 'fixture late callback unexpectedly has arguments' + return_bp = target.BreakpointCreateByAddress(reg(frame, 'x30')) + return_bp.SetThreadID(thread.GetThreadID()) + return_bp.SetOneShot(True) + event('destroyed-late-callback', **owner) + elif return_bp is not None and return_bp.GetID() in ids: + returned = True + event('late-callback-returned') + elif gc_bp.GetID() in ids: + event('explicit-gc-entry', vm=reg(frame, 'x0'), sync=reg(frame, 'w1'), + after_late_callback=returned) + if returned and reg(frame, 'x0') == owner['vm']: + assert reg(frame, 'w1') == 1 + gc_count += 1 + assert gc_count <= 31 + # The fixture collects once for queuedRoot, then thirty times before its final deref. + if gc_count == 31: + _, allocation = precise_allocation(owner['owner']) + marked = field(field(typed(allocation, 'JSC::PreciseAllocation'), 'm_isMarked'), 'value') + assert marked.GetByteSize() == 1 and mark_watch is None + address = marked.GetLoadAddress() + initial = word(address, 1) + assert initial in (0, 1) + error = lldb.SBError() + mark_watch = target.WatchAddress(address, 1, False, True, error) + event('real-mark-watchpoint-admission', address=address, initial=initial, error=str(error), + valid=mark_watch.IsValid()) + assert error.Success() and mark_watch.IsValid() and mark_watch.IsEnabled() + assert mark_watch.GetWatchAddress() == address and mark_watch.GetWatchSize() == 1 + assert mark_watch.GetID() > 0 + mark_observation = dict(allocation=allocation, address=address, initial_value=initial, + last_value=initial, observed_zero=initial == 0, + watchpoint=mark_watch.GetID()) + mark_bp.SetEnabled(True) + for bp in root_bps: + bp.SetEnabled(True) + elif mark_bp.GetID() in ids: + assert returned and gc_count == 31 and 'Collector::verifyGC' in (frame.GetFunctionName() or '') + collector = frame.FindVariable('this', lldb.eNoDynamicValues) + vm = typed(owner['vm'], 'JSC::VM') + assert number(collector) == number(field(field(vm, 'heap'), 'm_collector')) + visitor_ptrs = pointer(field(collector.Dereference(), 'm_verifierSlotVisitor'), 'JSC::VerifierSlotVisitor') + assert len(set(visitor_ptrs)) == 1 and visitor_ptrs[0] + assert mark_observation is not None and 'frame_pointer_path' in mark_observation + assert mark_observation['last_value'] == 1 and not mark_watch.IsEnabled() + _, allocation = precise_allocation(owner['owner']) + assert allocation == mark_observation['allocation'] and word(mark_observation['address'], 1) == 1 + visitor = typed(visitor_ptrs[0], 'JSC::VerifierSlotVisitor') + assert number(field(visitor, 'm_doneMarking')) == 1 + w = typed(owner['weak'], 'JSC::JSWeakObjectRef') + targets = pointer(field(w, 'm_value'), 'JSC::JSCell') + assert targets == [owner['owner']], 'the observed weak target no longer matches' + event('post-mark-owner', gc_count=gc_count, verifier=visitor_ptrs[0], + weak_last_access=number(field(w, 'm_lastAccessVersion')), + vm_weak_version=number(field(vm, 'm_currentWeakRefVersion'))) + target.DisableAllBreakpoints() + marker_snapshot(visitor, owner['owner']) + else: + raise RuntimeError('unrecognized breakpoint') + assert process.Continue().Success() + assert process.GetState() == lldb.eStateExited, 'stop budget exhausted' + report['exit_code'] = process.GetExitStatus() + if stage == 'capture': + assert report['exit_code'] == 1 and (OUT / 'child.json').is_file() + cases = list(ElementTree.parse(OUT / 'fixture.xml').iter('testcase')) + producer = list(ElementTree.parse(ROOT / 'producer-evidence/diagnostics/fixture.xml').iter('testcase')) + admit_failure(cases, producer) + else: + assert report['exit_code'] == 0 and gc_count == 31, '31 post-callback explicit collections not observed' + facts = json.loads((OUT / 'trace.stdout').read_text().strip()) + assert facts['callbackRoot'] is True and facts['collected'] is False and facts['settled'] == ['ECANCELED'] + snapshots = [e for e in report['events'] if e['kind'] == 'passive-marker-snapshot'] + assert len(snapshots) == 1 and snapshots[0]['owner'] == owner['owner'] + assert len([e for e in report['events'] if e['kind'] == 'live-real-mark-writer']) == 1 + report['outcome'] = facts + assert root_collections and any(c['accepted_spans'] for c in root_collections) + report['root_collections'] = root_collections + assert word_witness is not None and word_pending is None + report['scanner_word'] = word_witness + provenance = [span['word_provenance'] for collection in root_collections + for span in collection['accepted_spans'] if 'word_provenance' in span] + assert len(provenance) == 1 + report['word_provenance'] = provenance[0] + assert writer_state is not None and provenance[0]['kind'] == 'original-conductor-stack' + assert provenance[0]['storage_address'] == writer_state['address'] + upper = provenance[0]['interval']['upper'] + assert upper['fp'] == writer_state['prospective_fp'] + assert upper['saved_lr'] == module.ResolveFileAddress(27509448).GetLoadAddress(target) + report['stack_word_writer'] = writer_state + last_write = writer_state['writes'][-1] + slot = writer_state['address'] + complete = last_write['destination'] <= slot and slot + 8 <= last_write['destination'] + last_write['bytes'] + report['classification'] = ('exact accepted scanner word tied to an admitted physical stack write covering all eight bytes' + if complete else 'exact accepted scanner word tied to an admitted partial stack write; remaining-byte origin is unproven') + config_copy = last_write.get('absent_config_copy') + assert config_copy is not None and config_copy['source_word'] == owner['owner'], 'absent config source copy not observed' + report['classification'] += '; copied from the old absent SSLConfig storage while clearing the context; earlier heap-payload writer remains unproven' + report['passed'] = True + except BaseException as error: + report['gap'] = str(error) + raise + finally: + if stage == 'trace': + report['observation'] = {'weak_targets': weak, 'owner': owner, 'returned': returned, + 'gc_count': gc_count, 'pending_weak_constructor': weak_creation, + 'root_collections': root_collections, 'pending_scan_returns': scan_returns, + 'scanner_word': word_witness, 'pending_word': word_pending, + 'stack_word_writer': writer_state} + if process is not None and process.IsValid() and process.GetState() != lldb.eStateExited: + process.Kill() + write(stage + '.json', report) + + +def capture(): + assert all(sha(BUILD / name) == digest for name, digest in BINARIES.items()) + assert sha(SOURCE / FIXTURE) == FIXTURE_SHA + for stage in ['capture', 'trace']: + env = {'PATH': '/usr/bin:/bin', 'TLS_CAPTURE_STAGE': stage} + command = ['lldb-23', '--no-lldbinit', '--batch', + '-o', 'settings set target.load-script-from-symbol-file false', + '-o', 'settings set target.inherit-env false', + '-o', 'target create ' + str(BINARY), '-o', 'target symbols add ' + str(PROFILE), + '-o', 'script import runpy; runpy.run_path(' + repr(str(Path(__file__).resolve())) + ', run_name="__lldb__")'] + with (OUT / (stage + '-debugger.log')).open('xb') as log: + proc = subprocess.Popen(command, cwd=SOURCE, env=env, stdin=subprocess.DEVNULL, + stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + started = time.monotonic() + try: + while proc.poll() is None: + assert time.monotonic() - started < 240, stage + ' debugger timeout' + assert sum(p.stat().st_size for p in OUT.iterdir() if p.is_file()) <= 32 * 1024**2 + time.sleep(0.25) + assert proc.returncode == 0 + finally: + if proc.poll() is None: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + assert json.loads((OUT / (stage + '.json')).read_text())['passed'], stage + ' diagnostic gap' + assert all(sha(BUILD / name) == digest for name, digest in BINARIES.items()) + assert sha(SOURCE / FIXTURE) == FIXTURE_SHA + + +if __name__ == '__lldb__': + debugger_stage() +elif __name__ == '__main__': + {'admit-zip': admit_zip, 'restore': restore, 'capture': capture}[sys.argv[1]]() diff --git a/.github/openclaw/tls-shipping-debug.py b/.github/openclaw/tls-shipping-debug.py new file mode 100644 index 000000000000..34376bec58dd --- /dev/null +++ b/.github/openclaw/tls-shipping-debug.py @@ -0,0 +1,320 @@ +"""Build typed TLS diagnostic bytes with the frozen shipping optimization and ICU inputs.""" +import difflib +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tarfile +import urllib.request + +ROOT = Path('/work') +SOURCE = ROOT / 'source' +ENGINE = ROOT / 'engine' +OUT = ROOT / 'diagnostics' +BUN = '454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e' +WEBKIT = 'b82bc917e99110b61c3a21202d6def7457aa194c' +FIXTURE = 'test/js/bun/net/socket-retention.test.ts' +FIXTURE_SHA = 'd4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2' +LANE = 'bun-webkit-linux-arm64-lto.tar.gz' +BUILD = SOURCE / 'build/tls-arm64-shipping-debug' +MANIFEST = SOURCE / 'scripts/build/deps/webkit-artifacts.json' +LIBS = ['libJavaScriptCore.a', 'libWTF.a', 'libbmalloc.a'] +ENV = {key: value for key, value in os.environ.items() + if key not in ('GH_TOKEN', 'GITHUB_TOKEN', 'ACTIONS_RUNTIME_TOKEN', 'ACTIONS_ID_TOKEN_REQUEST_TOKEN')} +ENV.update(CI='true', CARGO_BUILD_JOBS='2') + + +def sha(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def write(name, value): + with (OUT / name).open('x') as stream: + stream.write(json.dumps(value, indent=2) + '\n') + + +def run(argv, name, cwd=SOURCE, env=ENV, timeout=120): + with (OUT / name).open('x') as stream: + subprocess.run(argv, cwd=cwd, env=env, stdout=stream, stderr=subprocess.STDOUT, + check=True, timeout=timeout) + + +def git(repo, *args): + return subprocess.check_output(['git', '-C', str(repo), *args], text=True).strip() + + +def fetch(url, path, digest): + assert not path.exists() + with urllib.request.urlopen(url, timeout=120) as response, path.open('xb') as output: + shutil.copyfileobj(response, output) + assert sha(path) == digest, 'download checksum mismatch' + + +def inventory(root): + result = {} + for path in sorted(root.rglob('*')): + assert not path.is_symlink(), 'unexpected bundle symlink' + if path.is_file(): + result[str(path.relative_to(root))] = sha(path) + return result + + +assert os.uname().machine == 'aarch64' +assert git(SOURCE, 'rev-parse', 'HEAD') == BUN and not git(SOURCE, 'diff', '--name-only', 'HEAD') +assert git(ENGINE, 'rev-parse', 'HEAD') == WEBKIT and not git(ENGINE, 'diff', '--name-only', 'HEAD') +assert sha(SOURCE / FIXTURE) == FIXTURE_SHA +manifest_bytes = MANIFEST.read_bytes() +original_manifest = json.loads(manifest_bytes) +assert original_manifest['version'] == WEBKIT +artifact = original_manifest['artifacts'][LANE] +assert artifact['url'] == f'https://github.com/openclaw/WebKit/releases/download/autobuild-{WEBKIT}/{LANE}' +write('inputs.json', {'bun': BUN, 'webkit': WEBKIT, 'fixture_sha256': FIXTURE_SHA, + 'shipping_archive': artifact, 'variance': 'full debug information; native build host', + 'build_timeout_seconds': {'engine': 7200, 'bun': 7200}, + 'optimization': 'Release O3, ThinLTO, armv8-a+crc, ampere1, original allocator/assertion settings', + 'icu': 'original bundled libraries, headers and filtered/repacked data; never system ICU'}) +original = ROOT / LANE +fetch(artifact['url'], original, artifact['sha256']) +bundle_root = ROOT / 'shipping-bundle' +bundle_root.mkdir() +with tarfile.open(original) as archive: + archive.extractall(bundle_root, filter='data') +bundle = bundle_root / 'bun-webkit' +assert bundle.is_dir() and all((bundle / 'lib' / name).is_file() for name in LIBS) +before = inventory(bundle) +assert all('lib/' + name in before for name in ['libicudata.a', 'libicui18n.a', 'libicuuc.a']) +assert any(name.endswith('/uvernum.h') for name in before) +write('shipping-bundle.json', before) + +# The shell's decoder source matches the frozen engine recipe. ICU itself is not rebuilt. +zstd = ROOT / 'zstd-1.5.7.tar.gz' +fetch('https://github.com/meta/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz', zstd, + 'eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3') +with tarfile.open(zstd) as archive: + archive.extractall(ROOT / 'zstd-source', filter='data') +sysroot = Path('/opt/linux-sysroot-glibc-arm64') +assert (sysroot / 'usr/lib/gcc/aarch64-linux-gnu/13/libstdc++.a').is_file() +compiler = subprocess.check_output(['clang', '--version'], text=True, env=ENV) +assert '23.1.1' in compiler, compiler +write('toolchain.json', {'compiler': compiler, 'sysroot': str(sysroot), + 'libstdc++': sha(sysroot / 'usr/lib/gcc/aarch64-linux-gnu/13/libstdc++.a'), + 'profile': sha(Path('/etc/profile.d/bun-ci.sh'))}) +base_flags = (f'--sysroot={sysroot} --target=aarch64-unknown-linux-gnu -march=armv8-a+crc -mtune=ampere1 ' + '-mno-omit-leaf-frame-pointer -g -fno-omit-frame-pointer -ffunction-sections -fdata-sections ' + '-faddrsig -fno-unwind-tables -fno-asynchronous-unwind-tables -DU_STATIC_IMPLEMENTATION=1 ' + '-stdlib=libstdc++ -g -flto=thin -fno-split-lto-unit -fwhole-program-vtables -fforce-emit-vtables ' + '-ffile-prefix-map=/work/engine/Source=vendor/WebKit/Source -ffile-prefix-map=/work/jsc-build/=. ') +engine_env = ENV.copy() +for name in ('C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH', 'LIBRARY_PATH', 'LD_LIBRARY_PATH', 'CFLAGS', 'CXXFLAGS'): + engine_env.pop(name, None) +engine_env.update(CC='clang', CXX='clang++', LDFLAGS='-fuse-ld=lld') +engine_build = ROOT / 'jsc-build' +argv = ['cmake', '-S', str(ENGINE), '-B', str(engine_build), '-G', 'Ninja', + '-DCMAKE_SYSTEM_NAME=Linux', '-DCMAKE_SYSTEM_PROCESSOR=aarch64', f'-DCMAKE_SYSROOT={sysroot}', + '-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER', '-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH', + '-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=BOTH', '-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH', + '-DPORT=JSCOnly', '-DENABLE_STATIC_JSC=ON', '-DENABLE_BUN_SKIP_FAILING_ASSERTIONS=ON', + '-DCMAKE_BUILD_TYPE=Release', '-DUSE_THIN_ARCHIVES=OFF', '-DUSE_BUN_JSC_ADDITIONS=ON', + '-DUSE_BUN_EVENT_LOOP=ON', '-DENABLE_FTL_JIT=ON', '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', + '-DALLOW_LINE_AND_COLUMN_NUMBER_IN_BUILTINS=ON', '-DENABLE_REMOTE_INSPECTOR=ON', + '-DCMAKE_EXE_LINKER_FLAGS=-fuse-ld=lld', '-DCMAKE_AR=' + shutil.which('llvm-ar'), + '-DCMAKE_RANLIB=' + shutil.which('llvm-ranlib'), '-DCMAKE_C_FLAGS=' + base_flags, + '-DCMAKE_CXX_FLAGS=' + base_flags + ' -fno-c++-static-destructors', + '-DCMAKE_C_FLAGS_RELEASE=-O3 -DNDEBUG=1', '-DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG=1', + '-DICU_ROOT=' + str(bundle), '-DBUN_ICU_ZSTD_SOURCE_DIR=/work/zstd-source/zstd-1.5.7/lib', + '-DICU_INCLUDE_DIR=' + str(bundle / 'include'), + '-DICU_UC_LIBRARY_RELEASE=' + str(bundle / 'lib/libicuuc.a'), + '-DICU_I18N_LIBRARY_RELEASE=' + str(bundle / 'lib/libicui18n.a'), + '-DICU_DATA_LIBRARY_RELEASE=' + str(bundle / 'lib/libicudata.a'), + '-DENABLE_SANITIZERS=', '-DENABLE_ASSERTS=AUTO', '-DENABLE_FUZZILLI=OFF', + '-DUSE_MIMALLOC=ON', '-DUSE_EXTERNAL_MIMALLOC=ON'] +write('engine-build-command.json', {'argv': argv, 'compiler': 'clang/clang++ 23.1.1', + 'linker_flags': '-fuse-ld=lld'}) +def header_comparison_bytes(name, path, shipping): + # The pinned packager adds a version after compilation; offlineasm embeds its Ruby-source hash. + if path.stat().st_size > 16 * 1024**2: + return None + data = path.read_bytes() + if name == 'include/cmakeconfig.h': + if shipping: + stamp = ('\n#define BUN_WEBKIT_VERSION "' + WEBKIT + '"\n').encode() + return data[:-len(stamp)] if data.endswith(stamp) else None + return data + provenance = { + 'include/JavaScriptCore/LLIntDesiredSettings.h': (0, b'// SettingsExtractor input hash: '), + 'include/JavaScriptCore/LLIntDesiredOffsets.h': (0, b'// OffsetExtractor input hash: '), + 'include/JavaScriptCore/LLIntAssembly.h': (-1, b'// offlineasm input hash: '), + } + if name not in provenance: + return None + index, prefix = provenance[name] + lines = data.splitlines(keepends=True) + observed = (b'23ac489dfebf7c9e22c7bc44e6826b164320e3d9' if shipping + else b'0c288d42d6b2ddc755f5b6e880f0cc180650c33f') + token = b' ' + observed + b' ' + if not lines or not lines[index].startswith(prefix) or lines[index].count(token) != 1: + return None + lines[index] = lines[index].replace(token, b' ', 1) + return b''.join(lines) + + +def check_headers(headers, phase): + missing = [str(name) for name in headers if str(name) not in before] + mismatched = {str(name): {'shipping': before[str(name)], 'rebuilt': sha(path)} + for name, path in headers.items() if str(name) in before and sha(path) != before[str(name)]} + raw_mismatched = dict(mismatched) + provenance_only = {} + for name in raw_mismatched: + original = header_comparison_bytes(name, bundle / name, True) + rebuilt = header_comparison_bytes(name, headers[Path(name)], False) + if original is not None and rebuilt is not None and original == rebuilt: + provenance_only[name] = {**mismatched.pop(name), + 'comparison_sha256': hashlib.sha256(original).hexdigest()} + evidence = {} + evidence_bytes = 0 + for name in sorted(raw_mismatched)[:4]: + paths = {'shipping': bundle / name, 'rebuilt': headers[Path(name)]} + size = sum(path.stat().st_size for path in paths.values()) + if evidence_bytes + size > 128 * 1024**2: + evidence[name] = {'omitted': 'header evidence exceeds 128 MiB'} + continue + evidence[name] = {} + for kind, path in paths.items(): + destination = OUT / 'header-evidence' / phase / (name + '.' + kind) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, destination) + evidence[name][kind] = {'path': str(destination.relative_to(OUT)), + 'bytes': path.stat().st_size, 'sha256': sha(destination)} + evidence_bytes += size + # Retain large raw headers, but bound the in-memory line comparison separately. + if size > 8 * 1024**2: + evidence[name]['diff'] = {'omitted': 'diff inputs exceed 8 MiB'} + continue + destination = OUT / 'header-evidence' / phase / (name + '.diff') + difference_bytes = 0 + with destination.open('xb') as stream: + for line in difflib.unified_diff( + paths['shipping'].read_text().splitlines(keepends=True), + paths['rebuilt'].read_text().splitlines(keepends=True), + fromfile='shipping/' + name, tofile='rebuilt/' + name): + content = line.encode() + if evidence_bytes + difference_bytes + len(content) > 128 * 1024**2: + break + stream.write(content) + difference_bytes += len(content) + else: + stream.flush() + evidence[name]['diff'] = {'path': str(destination.relative_to(OUT)), + 'bytes': difference_bytes, 'sha256': sha(destination)} + if 'diff' not in evidence[name]: + destination.unlink() + evidence[name]['diff'] = {'omitted': 'diff exceeds remaining evidence budget'} + else: + evidence_bytes += difference_bytes + write(phase + '-header-parity.json', {'files': len(headers), 'missing': missing, + 'mismatched': mismatched, 'provenance_only': provenance_only, + 'raw_mismatched': raw_mismatched, 'evidence': evidence}) + assert not missing and not mismatched, 'rebuilt core headers differ from the shipping consumer headers' + + +run(argv, 'engine-configure.log', cwd=ROOT, env=engine_env, timeout=600) +try: + # These generated headers can reject an incompatible rebuild before compiling the core libraries. + run(['cmake', '--build', str(engine_build), '--config', 'Release', '--parallel', '2', + '--target', 'JavaScriptCore/DerivedSources/LLIntAssembly.h'], + 'engine-header-build.log', cwd=ROOT, env=engine_env, timeout=1200) + early_headers = {Path('include/cmakeconfig.h'): engine_build / 'cmakeconfig.h'} + for name in ('LLIntAssembly.h', 'LLIntDesiredOffsets.h', 'LLIntDesiredSettings.h'): + early_headers[Path('include/JavaScriptCore') / name] = engine_build / 'JavaScriptCore/DerivedSources' / name + assert all(path.is_file() for path in early_headers.values()), 'generated header preflight incomplete' + check_headers(early_headers, 'generated') + run(['cmake', '--build', str(engine_build), '--config', 'Release', '--parallel', '2', + '--target', 'jsc', '--target', 'testFFI'], 'engine-build.log', cwd=ROOT, env=engine_env, timeout=7200) +finally: + for name in ('CMakeCache.txt', 'compile_commands.json'): + if (engine_build / name).is_file(): + shutil.copyfile(engine_build / name, OUT / name) +run(['python3', str(ENGINE / 'Tools/Scripts/check-classinfo-uniqueness.py'), str(engine_build / 'bin/jsc')], + 'engine-classinfo.log', cwd=ROOT) +elf = subprocess.check_output(['readelf', '-h', str(engine_build / 'bin/jsc')], text=True) +dynamic = subprocess.check_output(['readelf', '-d', str(engine_build / 'bin/jsc')], text=True) +versions = subprocess.check_output(['readelf', '--version-info', str(engine_build / 'bin/jsc')], text=True) +glibc = [tuple(map(int, value.split('.'))) for value in re.findall(r'GLIBC_([0-9]+\.[0-9]+)', versions)] +assert 'AArch64' in elf and 'libm.so.6' in dynamic and glibc and max(glibc) <= (2, 31) +assert not re.search(r'NEEDED.*libicu', dynamic) +write('engine-binary.json', {'ELF': elf, 'dynamic': dynamic, 'max_glibc': max(glibc)}) +run([str(engine_build / 'bin/jsc'), '-e', 'if(1+1!==2)throw Error("startup");print(new Intl.DateTimeFormat("ja-JP-u-ca-japanese").format(new Date(0)))'], + 'engine-startup.log', cwd=ROOT) + +# The prebuilt consumer must use the same headers as these rebuilt core libraries. +# Match the frozen Dockerfile's flattening order, retaining its ICU headers untouched. +headers = {Path('include') / path.name: path for path in engine_build.glob('*.h')} +for directory in ('JavaScriptCore/DerivedSources', 'JavaScriptCore/Headers/JavaScriptCore', + 'JavaScriptCore/PrivateHeaders/JavaScriptCore'): + parent = engine_build / directory + assert parent.is_dir(), 'missing core header directory: ' + directory + for path in sorted(parent.rglob('*.h')): + headers[Path('include/JavaScriptCore') / path.name] = path +for directory, destination in [('WTF/Headers/wtf', 'include/wtf'), ('bmalloc/Headers/bmalloc', 'include/bmalloc')]: + parent = engine_build / directory + assert parent.is_dir(), 'missing core header directory: ' + directory + for path in sorted(parent.rglob('*')): + if path.is_file(): + headers[Path(destination) / path.relative_to(parent)] = path +assert headers +check_headers(headers, 'core') +for name in LIBS: + shutil.copyfile(engine_build / 'lib' / name, bundle / 'lib' / name) +after = inventory(bundle) +assert before.keys() == after.keys() +assert {name for name in before if before[name] != after[name]} == {'lib/' + name for name in LIBS} +write('adapted-bundle.json', after) + +# Give these diagnostic bytes their own digest and task-owned prefetch/cache. Never reuse the shipping checksum. +rebuilt = ROOT / 'debug-webkit.tar.gz' +with tarfile.open(rebuilt, 'w:gz') as archive: + archive.add(bundle, arcname='bun-webkit') +digest = sha(rebuilt) +prefetch = ROOT / 'debug-prefetch/by-sha256' +prefetch.mkdir(parents=True) +rebuilt.rename(prefetch / digest) +flags = SOURCE / 'scripts/build/flags.ts' +text = flags.read_text() +flags_bytes = flags.read_bytes() +assert text.count('flag: "-g1",') == 1 +flags.write_text(text.replace('flag: "-g1",', 'flag: "-g",')) +manifest = json.loads(MANIFEST.read_text()) +manifest['artifacts'][LANE]['sha256'] = digest +MANIFEST.write_text(json.dumps(manifest, indent=2) + '\n') +write('diagnostic-overlay.json', {'archive_sha256': digest, 'manifest_sha256': sha(MANIFEST), + 'flags_sha256': sha(flags), 'shipping_archive_sha256': artifact['sha256']}) +patch = subprocess.check_output(['git', '-C', str(SOURCE), 'diff', '--binary', 'HEAD']) +(OUT / 'diagnostic-build.patch').write_bytes(patch) +env = ENV.copy() +env.update(BUN_BUILD_PREFETCH_DIR='/work/debug-prefetch', BUN_BUILD_CACHE_DIR='/work/debug-cache') +argv = ['bun', 'scripts/build.ts', '--profile=release', '--ci=on', '--os=linux', '--arch=aarch64', + '--abi=gnu', '--lto=on', '--webkit=prebuilt', '--build-dir=' + str(BUILD), '-j2', '--timings'] +write('bun-build-command.json', {'argv': argv}) +try: + run(argv, 'bun-build.log', env=env, timeout=7200) +finally: + assert sha(SOURCE / FIXTURE) == FIXTURE_SHA + assert git(SOURCE, 'diff', '--name-only', 'HEAD').splitlines() == [ + 'scripts/build/deps/webkit-artifacts.json', 'scripts/build/flags.ts'] + assert sha(MANIFEST) == json.loads((OUT / 'diagnostic-overlay.json').read_text())['manifest_sha256'] + assert sha(flags) == json.loads((OUT / 'diagnostic-overlay.json').read_text())['flags_sha256'] + MANIFEST.write_bytes(manifest_bytes) + flags.write_bytes(flags_bytes) + assert not git(SOURCE, 'diff', '--name-only', 'HEAD') +binary, profile = BUILD / 'bun', BUILD / 'bun-profile' +for path in (binary, profile): + assert path.is_file() + run(['readelf', '-n', str(path)], path.name + '-build-id.txt', cwd=ROOT) +write('executables.json', {str(path.relative_to(ROOT)): {'sha256': sha(path), 'bytes': path.stat().st_size} + for path in (binary, profile)}) +run([str(binary), '-e', 'console.log(JSON.stringify({bun:Bun.revision,webkit:process.versions.webkit,platform:process.platform,arch:process.arch}))'], 'runtime.json') +assert json.loads((OUT / 'runtime.json').read_text()) == {'bun': BUN, 'webkit': WEBKIT, 'platform': 'linux', 'arch': 'arm64'} diff --git a/.github/openclaw/tls-symbol-capability.py b/.github/openclaw/tls-symbol-capability.py new file mode 100644 index 000000000000..472d5ec99ed2 --- /dev/null +++ b/.github/openclaw/tls-symbol-capability.py @@ -0,0 +1,36 @@ +import json, lldb, re +from pathlib import Path +target = lldb.debugger.GetSelectedTarget() +assert target.IsValid(), "missing diagnostic target" +module = target.FindModule(target.GetExecutable()) +assert module.IsValid(), "missing executable module" +uuid = re.sub("[^0-9a-fA-F]", "", module.GetUUIDString() or "").lower() +expected = re.findall(r"Build ID: (\w+)", Path("/work/diagnostics/bun-build-id.txt").read_text()) +assert len(expected) == 1 and uuid == expected[0], "LLDB module identity mismatch" +types = {} +for name in ("JSC::JSCell", "JSC::Heap", "JSC::SlotVisitor", "JSC::ConservativeRoots"): + candidates = target.FindTypes(name) + complete = [] + for i in range(candidates.GetSize()): + value = candidates.GetTypeAtIndex(i).GetCanonicalType() + fields = value.GetNumberOfFields() + if value.IsTypeComplete() and value.GetByteSize() > 0 and fields > 0: + layout = [{"name": value.GetFieldAtIndex(j).GetName(), "offset_bits": value.GetFieldAtIndex(j).GetOffsetInBits()} for j in range(fields)] + complete.append({"bytes": value.GetByteSize(), "fields": fields, "layout": layout}) + assert complete, "incomplete type: " + name + types[name] = complete +functions = {} +for name in ("JSC::Collector::runEndPhase", "JSC::MachineThreads::gatherConservativeRoots"): + contexts = module.FindFunctions(name, lldb.eFunctionNameTypeAny) + functions[name] = [] + for i in range(contexts.GetSize()): + context = contexts.GetContextAtIndex(i) + function = context.GetFunction() + unit = context.GetCompileUnit() + if not function.IsValid() or not unit.IsValid(): + continue + address = function.GetStartAddress().GetFileAddress() + if address not in (0, lldb.LLDB_INVALID_ADDRESS): + functions[name].append({"address": address, "name": function.GetName(), "compile_unit": str(unit.GetFileSpec())}) + assert functions[name], "missing debug function: " + name +Path("/work/diagnostics/symbol-capability.json").write_text(json.dumps({"module_uuid": uuid, "types": types, "functions": functions}, indent=2) + "\n") diff --git a/.github/workflows/openclaw-tls-config-disassembly.yml b/.github/workflows/openclaw-tls-config-disassembly.yml new file mode 100644 index 000000000000..d0776b22ef90 --- /dev/null +++ b/.github/workflows/openclaw-tls-config-disassembly.yml @@ -0,0 +1,89 @@ +name: OpenClaw TLS optimized config inspection + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-config-disassembly.yml + - .github/openclaw/tls-config-disassembly.py + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: openclaw-tls-config-disassembly-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + inspect: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: | + .github/openclaw + .github/workflows + - name: Install static LLVM tools + shell: bash + run: | + set -euo pipefail + curl --fail --silent --show-error https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null + echo "deb https://apt.llvm.org/noble/ llvm-toolchain-noble-23 main" | sudo tee /etc/apt/sources.list.d/llvm.list > /dev/null + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends --no-install-suggests llvm-23 + - name: Admit the exact completed Linux ARM64 build + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + input="$RUNNER_TEMP/tls-disassembly-input" + mkdir "$input" + gh api repos/openclaw/bun/actions/runs/41238111110 > "$input/run.json" + gh api repos/openclaw/bun/actions/runs/41238111110/jobs > "$input/jobs.json" + gh api repos/openclaw/bun/actions/artifacts/13056617461 > "$input/metadata.json" + python3 - <<'PY' + import json, os + from pathlib import Path + root = Path(os.environ['RUNNER_TEMP']) / 'tls-disassembly-input' + run = json.loads((root / 'run.json').read_text()) + assert run['id'] == 41238111110 + assert run['head_sha'] == '510fef558b464c58de1209298f638fa476d03eef' + assert run['event'] == 'workflow_dispatch' + assert run['path'] == '.github/workflows/openclaw-release.yml' + assert run['head_branch'] == 'claude/fix-tls-native-transport-20261019' + jobs = json.loads((root / 'jobs.json').read_text())['jobs'] + build = [job for job in jobs if job['id'] == 124646206308 and job['name'] == 'Build linux-arm64'] + assert len(build) == 1 and build[0]['conclusion'] == 'success' + artifact = json.loads((root / 'metadata.json').read_text()) + assert artifact['id'] == 13056617461 and artifact['name'] == 'build-linux-arm64' + assert artifact['size_in_bytes'] == 222623871 and not artifact['expired'] + assert artifact['digest'] == 'sha256:c54dbe7ea29a336c76db1dd5ba491bbb117a6c211d938892c62f2f5e74251fcd' + assert artifact['workflow_run']['id'] == run['id'] + assert artifact['workflow_run']['head_sha'] == run['head_sha'] + PY + curl --location --fail --silent --show-error --proto '=https' --proto-redir '=https' \ + --max-time 300 --max-filesize 222623871 \ + -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/openclaw/bun/actions/artifacts/13056617461/zip \ + --output "$input/artifact.zip" + - name: Capture physical instructions without running Bun + shell: bash + run: | + set -euo pipefail + mkdir tls-disassembly-proof + cp controls/.github/workflows/openclaw-tls-config-disassembly.yml tls-disassembly-proof/workflow.yml + cp controls/.github/openclaw/tls-config-disassembly.py tls-disassembly-proof/controller.py + python3 controls/.github/openclaw/tls-config-disassembly.py + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: tls-config-disassembly + path: tls-disassembly-proof/ + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/openclaw-tls-config-repair.yml b/.github/workflows/openclaw-tls-config-repair.yml new file mode 100644 index 000000000000..d4d1442de3ab --- /dev/null +++ b/.github/workflows/openclaw-tls-config-repair.yml @@ -0,0 +1,128 @@ +name: OpenClaw TLS empty-config repair proof + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-config-repair.yml + - .github/openclaw/tls-config-repair.py + - .github/openclaw/tls-config-repair.patch + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: openclaw-tls-config-repair-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + linux-arm64: + runs-on: ubuntu-24.04-arm + timeout-minutes: 300 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: .github/openclaw + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: source + ref: 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: pipeline + ref: 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e + persist-credentials: false + - uses: ./pipeline/.github/actions/setup-bun + with: + bun-version: '1.4.4' + - name: Generate the frozen native provisioner + id: provision + shell: bash + run: | + set -euo pipefail + test "$(uname -m)" = aarch64 + echo "script=$(bun pipeline/scripts/openclaw-release/release.ts provision --source source --target linux-arm64 | tail -1)" >> "$GITHUB_OUTPUT" + - name: Provision a secretless native container + id: container + env: + SCRIPT: ${{ steps.provision.outputs.script }} + shell: bash + run: | + set -euo pipefail + mkdir repair-proof + id=$(docker run --detach -e CI=true -v "$GITHUB_WORKSPACE:/work" -w /work \ + docker.io/library/debian:13@sha256:9cc080028c43b27d2074d63a5f9caf7166d731494965616c1a6d2827a004585c sleep infinity) + [[ "$id" =~ ^[0-9a-f]{64}$ ]] + echo "id=$id" >> "$GITHUB_OUTPUT" + script="${SCRIPT#"$GITHUB_WORKSPACE"/}" + docker exec -e SCRIPT="$script" "$id" sh -euc \ + 'sh "$SCRIPT" 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e openclaw-tls-repair' + docker exec "$id" bash -e -lc '. /etc/profile.d/bun-ci.sh; rustup component add rustfmt --toolchain nightly-2026-09-15' + - name: Build and exercise the unchanged TLS suites + env: + CONTAINER_ID: ${{ steps.container.outputs.id }} + shell: bash + run: | + set -euo pipefail + [[ "$CONTAINER_ID" =~ ^[0-9a-f]{64}$ ]] + tail -n +1 -F repair-proof/build.log & + tail_pid=$! + trap 'kill "$tail_pid" 2>/dev/null || true' EXIT + docker exec "$CONTAINER_ID" bash -e -lc ' + . /etc/profile.d/bun-ci.sh + for repo in source pipeline controls; do git config --global --add safe.directory "/work/$repo"; done + python3 controls/.github/openclaw/tls-config-repair.py + ' + - name: Freeze bounded evidence + if: always() + env: + CONTAINER_ID: ${{ steps.container.outputs.id }} + shell: bash + run: | + set -euo pipefail + if test -n "$CONTAINER_ID"; then + [[ "$CONTAINER_ID" =~ ^[0-9a-f]{64}$ ]] + docker stop --time 30 "$CONTAINER_ID" > /dev/null + test "$(docker inspect --format '{{.State.Running}}' "$CONTAINER_ID")" = false + fi + python3 - <<'PY' + import hashlib, json, stat, tarfile + from pathlib import Path + out = Path('repair-proof') + out.mkdir(exist_ok=True) + files = sorted(out.iterdir()) + [Path('controls/.github/openclaw') / name for name in ['tls-config-repair.py', 'tls-config-repair.patch']] + assert len(files) <= 32 and all(stat.S_ISREG(p.lstat().st_mode) for p in files) + assert sum(p.stat().st_size for p in files) <= 64 * 1024**2 + def sha(path): + with path.open('rb') as stream: return hashlib.file_digest(stream, 'sha256').hexdigest() + manifest = out / 'archive-manifest.json' + manifest.write_text(json.dumps({str(p): dict(bytes=p.stat().st_size, sha256=sha(p)) for p in files}, indent=2) + '\n') + files.append(manifest) + with tarfile.open('tls-config-repair-proof.tar.gz', 'x:gz') as archive: + for path in files: archive.add(path, arcname=str(path), recursive=False) + assert Path('tls-config-repair-proof.tar.gz').stat().st_size <= 16 * 1024**2 + Path('tls-config-repair-SHA256SUMS').write_text(sha(Path('tls-config-repair-proof.tar.gz')) + ' tls-config-repair-proof.tar.gz\n') + PY + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: tls-empty-config-linux-arm64-proof + path: | + tls-config-repair-proof.tar.gz + tls-config-repair-SHA256SUMS + if-no-files-found: error + retention-days: 7 + - name: Remove this job's container + if: always() + env: + CONTAINER_ID: ${{ steps.container.outputs.id }} + shell: bash + run: | + if test -n "$CONTAINER_ID"; then + [[ "$CONTAINER_ID" =~ ^[0-9a-f]{64}$ ]] + docker rm --force "$CONTAINER_ID" > /dev/null + fi diff --git a/.github/workflows/openclaw-tls-cycle-checkpoint.yml b/.github/workflows/openclaw-tls-cycle-checkpoint.yml new file mode 100644 index 000000000000..e5a0ba32bbfd --- /dev/null +++ b/.github/workflows/openclaw-tls-cycle-checkpoint.yml @@ -0,0 +1,69 @@ +name: OpenClaw TLS cycle GC checkpoint experiment + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-cycle-checkpoint.yml + - .github/openclaw/tls-cycle-checkpoint.py + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: openclaw-tls-cycle-checkpoint-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + compare: + runs-on: macos-15-intel + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: | + .github/openclaw + .github/workflows + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: source + ref: 510fef558b464c58de1209298f638fa476d03eef + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: "26.12.1" + - name: Download the exact consumed signed Darwin artifact + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + input="$RUNNER_TEMP/tls-darwin-input" + mkdir "$input" + gh api repos/openclaw/bun/actions/runs/41238111110 > "$input/run.json" + gh api repos/openclaw/bun/actions/artifacts/13057470428 > "$input/metadata.json" + curl --location --fail --silent --show-error --proto '=https' --proto-redir '=https' \ + --max-time 300 --max-filesize 232201333 \ + -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/openclaw/bun/actions/artifacts/13057470428/zip \ + --output "$input/artifact.zip" + - name: Compare unchanged assertions at two GC checkpoints + shell: bash + run: | + set -euo pipefail + mkdir tls-cycle-checkpoint + cp controls/.github/workflows/openclaw-tls-cycle-checkpoint.yml tls-cycle-checkpoint/workflow.yml + cp controls/.github/openclaw/tls-cycle-checkpoint.py tls-cycle-checkpoint/controller.py + cp controls/.github/openclaw/tls-darwin-capability.py tls-cycle-checkpoint/archive.py + python3 controls/.github/openclaw/tls-cycle-checkpoint.py + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: tls-cycle-checkpoint + path: tls-cycle-checkpoint/ + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/openclaw-tls-cycle-heap.yml b/.github/workflows/openclaw-tls-cycle-heap.yml new file mode 100644 index 000000000000..3817032f6898 --- /dev/null +++ b/.github/workflows/openclaw-tls-cycle-heap.yml @@ -0,0 +1,76 @@ +name: OpenClaw TLS cycle production heap snapshot + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-cycle-heap.yml + - .github/openclaw/tls-cycle-heap.py + - .github/openclaw/tls-cycle-heap-test.py + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: openclaw-tls-cycle-heap-${{ github.head_ref || github.ref }} + cancel-in-progress: false + +jobs: + capture: + runs-on: macos-15-intel + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: | + .github/openclaw + .github/workflows + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: source + ref: 510fef558b464c58de1209298f638fa476d03eef + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: "26.12.1" + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: "3.14.8" + - name: Download the exact consumed Darwin artifact + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + input="$RUNNER_TEMP/tls-darwin-input" + mkdir "$input" + gh api repos/openclaw/bun/actions/runs/41238111110 > "$input/run.json" + gh api repos/openclaw/bun/actions/artifacts/13057470428 > "$input/metadata.json" + curl --location --fail --silent --show-error --proto '=https' --proto-redir '=https' \ + --max-time 300 --max-filesize 232201333 \ + -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/openclaw/bun/actions/artifacts/13057470428/zip \ + --output "$input/artifact.zip" + - name: Reproduce the original failure and capture a separate snapshot collection + shell: bash + run: | + set -euo pipefail + mkdir tls-cycle-heap + cp controls/.github/workflows/openclaw-tls-cycle-heap.yml tls-cycle-heap/workflow.yml + cp controls/.github/openclaw/tls-cycle-heap.py tls-cycle-heap/controller.py + cp controls/.github/openclaw/tls-cycle-heap-test.py tls-cycle-heap/graph-test.py + cp controls/.github/openclaw/tls-cycle-checkpoint.py tls-cycle-heap/checkpoint.py + cp controls/.github/openclaw/tls-darwin-capability.py tls-cycle-heap/archive.py + python3 controls/.github/openclaw/tls-cycle-heap-test.py + python3 controls/.github/openclaw/tls-cycle-heap.py + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: tls-cycle-production-heap + path: tls-cycle-heap/ + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/openclaw-tls-cycle-root.yml b/.github/workflows/openclaw-tls-cycle-root.yml new file mode 100644 index 000000000000..600456abf206 --- /dev/null +++ b/.github/workflows/openclaw-tls-cycle-root.yml @@ -0,0 +1,84 @@ +name: OpenClaw TLS cycle physical root capture + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-cycle-root.yml + - .github/openclaw/tls-cycle-root.py + - .github/openclaw/tls-cycle-heap.py + - .github/openclaw/tls-darwin-capability.py + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: openclaw-tls-cycle-root-${{ github.head_ref || github.ref }} + cancel-in-progress: false + +jobs: + capture: + runs-on: macos-15-intel + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: | + .github/openclaw + .github/workflows + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: source + ref: 510fef558b464c58de1209298f638fa476d03eef + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: "26.12.1" + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: "3.14.8" + - name: Download the exact consumed Darwin artifact + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + input="$RUNNER_TEMP/tls-darwin-input" + mkdir "$input" + gh api repos/openclaw/bun/actions/runs/41238111110 > "$input/run.json" + gh api repos/openclaw/bun/actions/artifacts/13057470428 > "$input/metadata.json" + curl --location --fail --silent --show-error --proto '=https' --proto-redir '=https' \ + --max-time 300 --max-filesize 232201333 \ + -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/openclaw/bun/actions/artifacts/13057470428/zip \ + --output "$input/artifact.zip" + - name: Admit the profile and reproduce the unchanged checkpoint + shell: bash + run: | + set -euo pipefail + mkdir tls-cycle-root tls-darwin-proof + cp controls/.github/workflows/openclaw-tls-cycle-root.yml tls-cycle-root/workflow.yml + cp controls/.github/openclaw/tls-cycle-root.py tls-cycle-root/controller.py + cp controls/.github/openclaw/tls-cycle-heap.py tls-cycle-root/graph-control.py + cp controls/.github/openclaw/tls-darwin-capability.py tls-cycle-root/archive.py + python3 controls/.github/openclaw/tls-cycle-root.py prepare + - name: Observe original GC32 scan spans and physical mark transitions + shell: bash + run: | + set -euo pipefail + python3 controls/.github/openclaw/tls-cycle-root.py observe + - name: Join the snapshot and seal bounded evidence + if: always() + shell: bash + run: python3 controls/.github/openclaw/tls-cycle-root.py seal + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: tls-cycle-physical-root + path: tls-cycle-root/ + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/openclaw-tls-darwin-capability.yml b/.github/workflows/openclaw-tls-darwin-capability.yml new file mode 100644 index 000000000000..160d01b108a6 --- /dev/null +++ b/.github/workflows/openclaw-tls-darwin-capability.yml @@ -0,0 +1,69 @@ +name: OpenClaw TLS Darwin debugger capability + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-darwin-capability.yml + - .github/openclaw/tls-darwin-capability.py + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: openclaw-tls-darwin-capability-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + inspect: + runs-on: macos-15-intel + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: | + .github/openclaw + .github/workflows + - name: Download the exact consumed signed artifact + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + input="$RUNNER_TEMP/tls-darwin-input" + mkdir "$input" + gh api repos/openclaw/bun/actions/runs/41238111110 > "$input/run.json" + gh api repos/openclaw/bun/actions/artifacts/13057470428 > "$input/metadata.json" + curl --location --fail --silent --show-error --proto '=https' --proto-redir '=https' \ + --max-time 300 --max-filesize 232201333 \ + -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/openclaw/bun/actions/artifacts/13057470428/zip \ + --output "$input/artifact.zip" + - name: Admit matched Mach-O and dSYM identities + shell: bash + run: | + set -euo pipefail + mkdir tls-darwin-proof + cp controls/.github/openclaw/tls-darwin-capability.py tls-darwin-proof/controller.py + cp controls/.github/workflows/openclaw-tls-darwin-capability.yml tls-darwin-proof/workflow.yml + python3 controls/.github/openclaw/tls-darwin-capability.py prepare + - name: Inventory static debugger types and physical scanner functions + shell: bash + run: | + set -euo pipefail + xcrun lldb --batch --no-lldbinit --no-use-colors \ + -o 'settings set target.load-script-from-symbol-file false' \ + -o 'script import runpy; runpy.run_path("controls/.github/openclaw/tls-darwin-capability.py", run_name="__lldb__")' \ + > tls-darwin-proof/debugger.log 2>&1 + python3 controls/.github/openclaw/tls-darwin-capability.py seal + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: tls-darwin-debugger-capability + path: tls-darwin-proof/ + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/openclaw-tls-darwin-disassembly.yml b/.github/workflows/openclaw-tls-darwin-disassembly.yml new file mode 100644 index 000000000000..504923e93f31 --- /dev/null +++ b/.github/workflows/openclaw-tls-darwin-disassembly.yml @@ -0,0 +1,69 @@ +name: OpenClaw TLS Darwin physical config inspection + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-darwin-disassembly.yml + - .github/openclaw/tls-darwin-disassembly.py + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: openclaw-tls-darwin-disassembly-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + inspect: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: | + .github/openclaw + .github/workflows + - name: Install static LLVM tools + shell: bash + run: | + set -euo pipefail + curl --fail --silent --show-error https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null + echo "deb https://apt.llvm.org/noble/ llvm-toolchain-noble-23 main" | sudo tee /etc/apt/sources.list.d/llvm.list > /dev/null + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends --no-install-suggests llvm-23 + - name: Download the exact consumed signed Darwin artifact + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + input="$RUNNER_TEMP/tls-darwin-input" + mkdir "$input" + gh api repos/openclaw/bun/actions/runs/41238111110 > "$input/run.json" + gh api repos/openclaw/bun/actions/artifacts/13057470428 > "$input/metadata.json" + curl --location --fail --silent --show-error --proto '=https' --proto-redir '=https' \ + --max-time 300 --max-filesize 232201333 \ + -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/openclaw/bun/actions/artifacts/13057470428/zip \ + --output "$input/artifact.zip" + - name: Capture bounded functions without launching Bun + shell: bash + run: | + set -euo pipefail + mkdir tls-darwin-disassembly + cp controls/.github/workflows/openclaw-tls-darwin-disassembly.yml tls-darwin-disassembly/workflow.yml + cp controls/.github/openclaw/tls-darwin-disassembly.py tls-darwin-disassembly/controller.py + cp controls/.github/openclaw/tls-darwin-capability.py tls-darwin-disassembly/archive.py + python3 controls/.github/openclaw/tls-darwin-disassembly.py + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: tls-darwin-disassembly + path: tls-darwin-disassembly/ + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/openclaw-tls-darwin-types.yml b/.github/workflows/openclaw-tls-darwin-types.yml new file mode 100644 index 000000000000..9dcfcc53ad32 --- /dev/null +++ b/.github/workflows/openclaw-tls-darwin-types.yml @@ -0,0 +1,163 @@ +name: OpenClaw TLS Darwin type witness + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-darwin-types.yml + - .github/openclaw/tls-darwin-types.py + - .github/openclaw/tls-darwin-capability.py + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: openclaw-tls-darwin-types-${{ github.head_ref || github.ref }} + # Preserve the active frozen-input reconstruction across unrelated diagnostic pushes. + cancel-in-progress: false + +jobs: + compile: + runs-on: ubuntu-24.04-arm + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: | + .github/openclaw + .github/workflows + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: source + ref: 510fef558b464c58de1209298f638fa476d03eef + persist-credentials: false + - uses: ./source/.github/actions/setup-bun + with: + bun-version: "1.4.4" + - name: Provision the frozen shipping compiler and SDK + shell: bash + run: | + set -euo pipefail + mkdir tls-darwin-types + script=$(bun source/scripts/openclaw-release/release.ts provision --source source --target darwin-x64 | tail -1) + script=${script#"$GITHUB_WORKSPACE"/} + test -f "$script" + id=$(docker run --detach -e CI=true -v "$GITHUB_WORKSPACE:/work" -w /work \ + docker.io/library/debian:13@sha256:9cc080028c43b27d2074d63a5f9caf7166d731494965616c1a6d2827a004585c \ + sleep infinity) + [[ "$id" =~ ^[0-9a-f]{64}$ ]] + echo "$id" > "$RUNNER_TEMP/tls-types-container" + docker exec "$id" sh "$script" 510fef558b464c58de1209298f638fa476d03eef openclaw-release \ + 2>&1 | tee tls-darwin-types/provision.log + - name: Emit a separate type object using the native compile owner + shell: bash + run: | + set -euo pipefail + id=$(cat "$RUNNER_TEMP/tls-types-container") + [[ "$id" =~ ^[0-9a-f]{64}$ ]] + docker exec "$id" bash -euo pipefail -c ' + . /etc/profile.d/bun-ci.sh + git config --global --add safe.directory /work/source + python3 controls/.github/openclaw/tls-darwin-types.py build + ' + - name: Seal bounded type metadata and build evidence + if: always() + shell: bash + run: | + set -euo pipefail + if test -f "$RUNNER_TEMP/tls-types-container"; then + id=$(cat "$RUNNER_TEMP/tls-types-container") + [[ "$id" =~ ^[0-9a-f]{64}$ ]] + docker stop --time 15 "$id" > /dev/null + test "$(docker inspect --format '{{.State.Running}}' "$id")" = false + fi + sudo chown -R "$(id -u):$(id -g)" tls-darwin-types + cp controls/.github/openclaw/tls-darwin-types.py tls-darwin-types/controller.py + cp controls/.github/openclaw/tls-darwin-capability.py tls-darwin-types/inventory.py + cp controls/.github/workflows/openclaw-tls-darwin-types.yml tls-darwin-types/workflow.yml + python3 - <<'PY' + import hashlib, json, stat + from pathlib import Path + root = Path('tls-darwin-types') + files = sorted(root.iterdir()) + assert len(files) <= 24 and all(stat.S_ISREG(p.lstat().st_mode) for p in files) + assert sum(p.stat().st_size for p in files) <= 96 * 1024**2 + (root / 'manifest.json').write_text(json.dumps({p.name: dict(bytes=p.stat().st_size, + sha256=hashlib.sha256(p.read_bytes()).hexdigest()) for p in files}, indent=2) + '\n') + PY + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() && hashFiles('tls-darwin-types/manifest.json') != '' + with: + name: tls-darwin-type-object + path: tls-darwin-types/ + if-no-files-found: error + retention-days: 7 + - name: Remove this job's compiler container + if: always() + shell: bash + run: | + if test -f "$RUNNER_TEMP/tls-types-container"; then + id=$(cat "$RUNNER_TEMP/tls-types-container") + [[ "$id" =~ ^[0-9a-f]{64}$ ]] && docker rm --force "$id" > /dev/null + fi + + inspect: + needs: compile + runs-on: macos-15-intel + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: .github/openclaw + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: tls-darwin-type-object + path: tls-darwin-types + - name: Admit every type-object artifact entry + shell: bash + run: | + set -euo pipefail + python3 - <<'PY' + import hashlib, json, stat + from pathlib import Path + root = Path('tls-darwin-types') + manifest = json.loads((root / 'manifest.json').read_text()) + assert set(manifest) == {p.name for p in root.iterdir()} - {'manifest.json'} + for name, entry in manifest.items(): + p = root / name + assert p.name == name and stat.S_ISREG(p.lstat().st_mode) + assert p.stat().st_size == entry['bytes'] and hashlib.sha256(p.read_bytes()).hexdigest() == entry['sha256'] + assert (root / 'controller.py').read_bytes() == Path('controls/.github/openclaw/tls-darwin-types.py').read_bytes() + assert (root / 'inventory.py').read_bytes() == Path('controls/.github/openclaw/tls-darwin-capability.py').read_bytes() + PY + - name: Admit complete layouts without launching an inferior + shell: bash + run: | + set -euo pipefail + xcrun lldb --batch --no-lldbinit --no-use-colors \ + -o 'settings set target.load-script-from-symbol-file false' \ + -o 'script import runpy; _ = runpy.run_path("controls/.github/openclaw/tls-darwin-types.py", run_name="__lldb__")' \ + > tls-darwin-types/debugger.log 2>&1 + test -f tls-darwin-types/layouts.json + python3 - <<'PY' + import json + from pathlib import Path + result = json.loads(Path('tls-darwin-types/layouts.json').read_text()) + assert result['missing'] == [] and result['inferior_launched'] is False + PY + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: tls-darwin-type-layouts + path: | + tls-darwin-types/layouts.json + tls-darwin-types/debugger.log + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/openclaw-tls-fixture-qualification.yml b/.github/workflows/openclaw-tls-fixture-qualification.yml new file mode 100644 index 000000000000..6e0657b1e06b --- /dev/null +++ b/.github/workflows/openclaw-tls-fixture-qualification.yml @@ -0,0 +1,92 @@ +name: OpenClaw TLS native fixture qualification + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-fixture-qualification.yml + - .github/openclaw/tls-fixture-qualification.py + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: openclaw-tls-fixture-qualification-${{ github.head_ref || github.ref }} + cancel-in-progress: false + +jobs: + qualify: + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + max-parallel: 2 + matrix: + include: + - { target: windows-arm64, runner: windows-11-arm } + steps: + - name: Preserve canonical source bytes on every host + shell: bash + run: git config --global core.autocrlf false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: controls + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: | + .github/openclaw + .github/workflows + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: source + ref: 510fef558b464c58de1209298f638fa476d03eef + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: "26.12.1" + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: "3.14.8" + - name: Plan the exact retained product artifact + id: plan + shell: bash + env: + TARGET: ${{ matrix.target }} + run: python controls/.github/openclaw/tls-fixture-qualification.py plan "$TARGET" >> "$GITHUB_OUTPUT" + - name: Download retained artifact metadata and bytes + env: + GH_TOKEN: ${{ github.token }} + ARTIFACT: ${{ steps.plan.outputs.artifact }} + SIZE: ${{ steps.plan.outputs.size }} + shell: bash + run: | + set -euo pipefail + input="$RUNNER_TEMP/tls-fixture-input" + mkdir "$input" + gh api repos/openclaw/bun/actions/runs/41238111110 > "$input/run.json" + gh api "repos/openclaw/bun/actions/artifacts/$ARTIFACT" > "$input/metadata.json" + curl --location --fail --silent --show-error --proto '=https' --proto-redir '=https' \ + --max-time 300 --max-filesize "$SIZE" \ + -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/openclaw/bun/actions/artifacts/$ARTIFACT/zip" \ + --output "$input/artifact.zip" + - name: Qualify all six native TLS suites + shell: bash + env: + TARGET: ${{ matrix.target }} + run: | + set -euo pipefail + mkdir tls-fixture-qualification + cp controls/.github/workflows/openclaw-tls-fixture-qualification.yml tls-fixture-qualification/workflow.yml + cp controls/.github/openclaw/tls-fixture-qualification.py tls-fixture-qualification/controller.py + cp controls/.github/openclaw/tls-cycle-checkpoint.py tls-fixture-qualification/checkpoint.py + cp controls/.github/openclaw/tls-darwin-capability.py tls-fixture-qualification/archive.py + python controls/.github/openclaw/tls-fixture-qualification.py qualify "$TARGET" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: tls-fixture-qualification-${{ matrix.target }} + path: tls-fixture-qualification/ + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/openclaw-tls-fullsymbols.yml b/.github/workflows/openclaw-tls-fullsymbols.yml new file mode 100644 index 000000000000..b5dcefb614f8 --- /dev/null +++ b/.github/workflows/openclaw-tls-fullsymbols.yml @@ -0,0 +1,392 @@ +name: OpenClaw TLS native debug proof + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-fullsymbols.yml + - .github/openclaw/tls-symbol-capability.py + - .github/openclaw/tls-shipping-debug.py + workflow_dispatch: + +permissions: + actions: read + contents: read + +concurrency: + group: openclaw-tls-saved-symbols-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + linux-arm64: + runs-on: ubuntu-24.04-arm + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: pipeline + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: .github/openclaw + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e + path: source + persist-credentials: false + - name: Prepare saved diagnostic inputs + shell: bash + run: | + set -euo pipefail + test "$(uname -m)" = aarch64 + test "$(git -C source rev-parse HEAD)" = 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e + echo 'd4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2 source/test/js/bun/net/socket-retention.test.ts' | sha256sum --check + mkdir diagnostics + printf '%s\n' 'Bun=454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e' \ + 'WebKit=b82bc917e99110b61c3a21202d6def7457aa194c' \ + 'Diagnostic variance: non-LTO C++/JSC; Debian 13 system ICU; Rust-only release LTO may remain enabled.' > diagnostics/inputs.txt + cp pipeline/.github/openclaw/tls-symbol-capability.py diagnostics/probe.py + - name: Fetch the frozen failed build + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + gh api repos/openclaw/bun/actions/runs/41177263900 > diagnostics/producer-run.json + gh api repos/openclaw/bun/actions/artifacts/13034855054 > diagnostics/producer-artifact.json + gh api repos/openclaw/bun/actions/artifacts/13034855054/zip > artifact.zip + python3 - <<'PYCODE' + import hashlib, json, stat, zipfile + from pathlib import Path, PurePosixPath + artifact = json.loads(Path("diagnostics/producer-artifact.json").read_text()) + run = json.loads(Path("diagnostics/producer-run.json").read_text()) + assert run["id"] == 41177263900 and run["conclusion"] == "failure" + assert run["head_sha"] == "dc683a462eb90a5c5b22abb1a85dd25639cfc5b6" + assert artifact["id"] == 13034855054 and not artifact["expired"] + assert artifact["name"] == "tls-linux-arm64-fullsymbols" and artifact["size_in_bytes"] == 319038169 + assert artifact["workflow_run"]["id"] == run["id"] and artifact["workflow_run"]["head_sha"] == run["head_sha"] + digest = "d4ebf132092bb9c3a6de65fd848523200aaff74e36cd09ecf60f1a5fc69abbb3" + assert artifact["digest"] == "sha256:" + digest + with Path("artifact.zip").open("rb") as archive: + assert hashlib.file_digest(archive, "sha256").hexdigest() == digest + with zipfile.ZipFile("artifact.zip") as archive: + entries = archive.infolist() + assert len(entries) <= 20 and sum(entry.file_size for entry in entries) <= 8 * 1024**3 + seen = set() + for entry in entries: + name = PurePosixPath(entry.filename) + assert not name.is_absolute() and len(name.parts) == 1 and name.name not in ("", ".", "..") + assert not entry.is_dir() and not stat.S_ISLNK(entry.external_attr >> 16) and name.name not in seen + seen.add(name.name) + target = Path("producer") / name.name + target.parent.mkdir(exist_ok=True) + with target.open("xb") as output, archive.open(entry) as content: + import shutil + shutil.copyfileobj(content, output) + sums = dict(line.split(None, 1)[::-1] for line in Path("producer/SHA256SUMS").read_text().splitlines()) + assert set(sums) == {"source/build/tls-arm64-fullsymbols/bun", "source/build/tls-arm64-fullsymbols/bun-profile", "diagnostics/matched-symbols.tar.zst"} + with Path("producer/matched-symbols.tar.zst").open("rb") as archive: + assert hashlib.file_digest(archive, "sha256").hexdigest() == sums["diagnostics/matched-symbols.tar.zst"] + Path("diagnostics/producer-policy.txt").write_text("Producer binary policy FAILED: extra static initializer init_have_sme. These saved non-LTO/system-ICU inputs are diagnostic only; no shipping gate is waived.\n") + PYCODE + - name: Restore matched inputs in the original container layout + shell: bash + run: | + set -euo pipefail + docker run --detach --name tls-symbols -e CI=true -v "$GITHUB_WORKSPACE:/work" -w /work \ + docker.io/library/debian:13@sha256:9cc080028c43b27d2074d63a5f9caf7166d731494965616c1a6d2827a004585c \ + sleep infinity > /dev/null + docker exec tls-symbols sh -euc ' + apt-get update + apt-get install --yes --no-install-recommends ca-certificates curl python3 binutils zstd libicu76 + curl -fsSL --retry 5 https://apt.llvm.org/llvm-snapshot.gpg.key > /etc/apt/trusted.gpg.d/apt.llvm.org.asc + echo "deb https://apt.llvm.org/trixie/ llvm-toolchain-trixie-23 main" > /etc/apt/sources.list.d/llvm.list + apt-get update + apt-get install --yes --no-install-recommends llvm-23 lldb-23 python3-lldb-23 + ' + docker exec -i tls-symbols python3 - <<'PYCODE' + import hashlib, json, re, shutil, subprocess, tarfile + from pathlib import Path, PurePosixPath + expected_icu = re.search(r"^libicu-dev(?::arm64)?\s+(\S+)$", Path("producer/tools.txt").read_text(), re.M) + assert expected_icu, "producer ICU identity missing" + actual_icu = subprocess.check_output(["dpkg-query", "-W", "-f=${Version}", "libicu76"], text=True) + assert actual_icu == expected_icu[1], (actual_icu, expected_icu[1]) + count = size = 0 + seen = set() + child = subprocess.Popen(["zstd", "--decompress", "--stdout", "producer/matched-symbols.tar.zst"], stdout=subprocess.PIPE) + try: + with tarfile.open(fileobj=child.stdout, mode="r|") as archive: + for member in archive: + name = PurePosixPath(member.name) + assert member.isfile() and not name.is_absolute() and ".." not in name.parts + assert str(name).startswith("source/build/tls-arm64-fullsymbols/") and str(name) not in seen + assert name.name in ("bun", "bun-profile") or name.suffix == ".dwo" + seen.add(str(name)); count += 1; size += member.size + assert count <= 20000 and size <= 8 * 1024**3 + target = Path(str(name)); target.parent.mkdir(parents=True, exist_ok=True) + with target.open("xb") as output, archive.extractfile(member) as content: + shutil.copyfileobj(content, output) + target.chmod(0o755 if name.name in ("bun", "bun-profile") else 0o644) + assert child.wait(timeout=30) == 0 + finally: + child.stdout.close() + if child.poll() is None: + child.kill(); child.wait() + sums = dict(line.split(None, 1)[::-1] for line in Path("producer/SHA256SUMS").read_text().splitlines()) + for name in ("source/build/tls-arm64-fullsymbols/bun", "source/build/tls-arm64-fullsymbols/bun-profile"): + with Path(name).open("rb") as binary: + assert hashlib.file_digest(binary, "sha256").hexdigest() == sums[name] + Path("diagnostics/restored-inputs.json").write_text(json.dumps({"files": count, "bytes": size, "icu": actual_icu}, indent=2) + "\n") + PYCODE + docker exec tls-symbols sh -euc 'ldd source/build/tls-arm64-fullsymbols/bun' > diagnostics/runtime-libraries.txt + ! grep -q 'not found' diagnostics/runtime-libraries.txt + docker exec tls-symbols sh -euc 'lldb-23 --version; llvm-dwarfdump-23 --version; dpkg-query -W libicu76 python3 llvm-23 lldb-23 python3-lldb-23 binutils zstd' > diagnostics/runtime-tools.txt + - name: Verify matched symbols and capture capability + shell: bash + run: | + set -euo pipefail + docker exec tls-symbols bash -euo pipefail -c ' + cd /work/source + build=build/tls-arm64-fullsymbols + readelf -n "$build/bun" > /work/diagnostics/bun-build-id.txt + readelf -n "$build/bun-profile" > /work/diagnostics/profile-build-id.txt + python3 - <<'"'"'PY'"'"' + import re + from pathlib import Path + report = Path("/work/diagnostics") + ids = [re.findall(r"Build ID: (\w+)", (report / name).read_text()) for name in ("bun-build-id.txt", "profile-build-id.txt")] + assert len(ids[0]) == 1 and ids[0] == ids[1], ids + PY + python3 - <<'"'"'PYCODE'"'"' + import json, subprocess + from pathlib import Path + with Path("/work/diagnostics/dwarf-reference.txt").open("w") as output: + result = subprocess.run(["llvm-dwarfdump-23", "--show-parents", "--show-form", "--show-children", "--recurse-depth=2", "--debug-info=0x2cad422", "build/tls-arm64-fullsymbols/bun-profile"], stdout=output, stderr=subprocess.STDOUT, timeout=120) + Path("/work/diagnostics/dwarf-inspection.json").write_text(json.dumps({"status": result.returncode, "prior_gdb_error": "DWARF reference to DIE 0 from DIE 0x2cad422", "prior_run": 41180631285}) + "\n") + PYCODE + lldb-23 --no-lldbinit --batch \ + -o "settings set target.load-script-from-symbol-file false" \ + -o "target create $build/bun" \ + -o "target symbols add $build/bun-profile" \ + -o "script exec(open(\"/work/diagnostics/probe.py\").read())" > /work/diagnostics/symbol-probe.txt 2>&1 + python3 - <<'"'"'PYCODE'"'"' + import json + from pathlib import Path + report = json.loads(Path("/work/diagnostics/symbol-capability.json").read_text()) + assert set(report["types"]) == {"JSC::JSCell", "JSC::Heap", "JSC::SlotVisitor", "JSC::ConservativeRoots"} + assert all(values and all(value["bytes"] > 0 and value["fields"] > 0 for value in values) for values in report["types"].values()) + assert set(report["functions"]) == {"JSC::Collector::runEndPhase", "JSC::MachineThreads::gatherConservativeRoots"} + assert all(values and all(value["address"] > 0 and value["compile_unit"] and value["name"] for value in values) for values in report["functions"].values()) + PYCODE + ' + - name: Run the unchanged destroyed-completion case + shell: bash + run: | + set -euo pipefail + docker exec tls-symbols bash -euo pipefail -c ' + cd /work/source + python3 - <<'"'"'PY'"'"' + import hashlib, json, subprocess, sys + from pathlib import Path + from xml.etree import ElementTree + report = Path("/work/diagnostics") + fixture = Path("test/js/bun/net/socket-retention.test.ts") + assert hashlib.sha256(fixture.read_bytes()).hexdigest() == "d4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2" + exe = Path("build/tls-arm64-fullsymbols/bun").resolve() + identity = subprocess.run([str(exe), "-e", "console.log(JSON.stringify({commit:Bun.revision,webkit:process.versions.webkit,platform:process.platform,arch:process.arch}))"], text=True, capture_output=True, check=True) + facts = json.loads(identity.stdout) + (report / "identity.json").write_text(json.dumps(facts, indent=2) + "\n") + assert facts == {"commit": "454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e", "webkit": "b82bc917e99110b61c3a21202d6def7457aa194c", "platform": "linux", "arch": "arm64"}, facts + name = "a TLS Duplex completion retains its owner and is inert after destroy" + with (report / "stdout.txt").open("w") as stdout, (report / "stderr.txt").open("w") as stderr: + result = subprocess.run([str(exe), "test", str(fixture), "--test-name-pattern", "^" + name + "$", "--reporter=junit", "--reporter-outfile=" + str(report / "result.xml")], stdout=stdout, stderr=stderr, timeout=120) + cases = [case for case in ElementTree.parse(report / "result.xml").iter("testcase") if case.get("name") == name] + assert len(cases) == 1 and cases[0].find("skipped") is None, "selected case did not execute" + outcome = {"exitCode": result.returncode, "name": name, "failure": cases[0].find("failure") is not None, "error": cases[0].find("error") is not None} + (report / "outcome.json").write_text(json.dumps(outcome, indent=2) + "\n") + print(json.dumps(outcome)) + if result.returncode == 0: + assert not outcome["failure"] and not outcome["error"], outcome + sys.exit(result.returncode) + PY + ' + - name: Upload diagnostic evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: tls-linux-arm64-saved-symbols + path: diagnostics + if-no-files-found: error + retention-days: 7 + - name: Stop diagnostic container + if: always() + run: docker rm --force tls-symbols > /dev/null + + shipping-debug-linux-arm64: + if: >- + github.repository == 'openclaw/bun' && + (github.head_ref == 'claude/webkit-tls-artifact-diagnostics-20261031' || + github.ref == 'refs/heads/claude/webkit-tls-artifact-diagnostics-20261031') + runs-on: ubuntu-24.04-arm + timeout-minutes: 340 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: pipeline + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: .github/openclaw + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: openclaw/bun + ref: 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e + path: source + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: openclaw/WebKit + ref: b82bc917e99110b61c3a21202d6def7457aa194c + path: engine + persist-credentials: false + - name: Free disposable runner space + if: runner.environment == 'github-hosted' + run: sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup /opt/hostedtoolcache /usr/local/share/boost /usr/share/swift /usr/local/julia* /opt/microsoft + - uses: ./source/.github/actions/setup-bun + with: + bun-version: '1.4.4' + - name: Generate the frozen shipping provisioning script + id: provision + shell: bash + run: | + set -euo pipefail + mkdir diagnostics + script=$(bun source/scripts/openclaw-release/release.ts provision --source source --target linux-arm64 | tail -1) + test -f "$script" + echo "script=${script#"$GITHUB_WORKSPACE"/}" >> "$GITHUB_OUTPUT" + cp pipeline/.github/openclaw/tls-symbol-capability.py diagnostics/probe.py + - name: Provision the original shipping toolchain and sysroot + id: shipping-container + shell: bash + env: + SCRIPT: ${{ steps.provision.outputs.script }} + run: | + set -euo pipefail + test "$(uname -m)" = aarch64 + container_id=$(docker run --detach --name tls-shipping-debug -e CI=true -v "$GITHUB_WORKSPACE:/work" -w /work \ + docker.io/library/debian:13@sha256:9cc080028c43b27d2074d63a5f9caf7166d731494965616c1a6d2827a004585c \ + sleep infinity) + echo "id=$container_id" >> "$GITHUB_OUTPUT" + docker exec -e SCRIPT tls-shipping-debug sh -euc 'sh "$SCRIPT" 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e openclaw-release' + docker exec tls-shipping-debug sh -euc ' + apt-get update + apt-get install --yes --no-install-recommends python3 binutils zstd ruby bison gawk perl libxml2-dev pkg-config + curl -fsSL --retry 5 https://apt.llvm.org/llvm-snapshot.gpg.key > /etc/apt/trusted.gpg.d/apt.llvm.org.asc + echo "deb https://apt.llvm.org/trixie/ llvm-toolchain-trixie-23 main" > /etc/apt/sources.list.d/llvm.list + apt-get update + apt-get install --yes --no-install-recommends llvm-23 lldb-23 python3-lldb-23 + ' + - name: Build full debug information with shipping optimizations and bundled ICU + shell: bash + run: | + set -euo pipefail + docker exec tls-shipping-debug bash -e -lc '. /etc/profile.d/bun-ci.sh; git config --global --add safe.directory /work/source; git config --global --add safe.directory /work/engine; python3 pipeline/.github/openclaw/tls-shipping-debug.py' + - name: Require matched executable IDs and complete GC types + shell: bash + run: | + set -euo pipefail + python3 - <<'PY' + import re + from pathlib import Path + def identity(name): + values = re.findall(r"Build ID: (\w+)", Path("diagnostics/" + name + "-build-id.txt").read_text()) + assert len(values) == 1 + return values[0] + assert identity("bun") == identity("bun-profile") + PY + docker exec tls-shipping-debug timeout 600 lldb-23 --no-lldbinit --batch \ + -o 'target create /work/source/build/tls-arm64-shipping-debug/bun-profile' \ + -o 'command script import /work/diagnostics/probe.py' > diagnostics/lldb-capability.log 2>&1 + python3 - <<'PY' + import json, re + from pathlib import Path + result = json.loads(Path("diagnostics/symbol-capability.json").read_text()) + assert result["module_uuid"] == re.findall(r"Build ID: (\w+)", Path("diagnostics/bun-build-id.txt").read_text())[0] + assert set(result["types"]) == {"JSC::JSCell", "JSC::Heap", "JSC::SlotVisitor", "JSC::ConservativeRoots"} + assert all(result["types"].values()) + assert set(result["functions"]) == {"JSC::Collector::runEndPhase", "JSC::MachineThreads::gatherConservativeRoots"} + assert all(result["functions"].values()) + PY + - name: Run the unchanged destroyed-completion case once + shell: bash + run: | + set -euo pipefail + docker exec tls-shipping-debug bash -lc '. /etc/profile.d/bun-ci.sh; cd /work/source; python3 - <<'"'"'PY'"'"' + import hashlib, json, subprocess, sys + from pathlib import Path + from xml.etree import ElementTree + fixture = Path("test/js/bun/net/socket-retention.test.ts") + assert hashlib.sha256(fixture.read_bytes()).hexdigest() == "d4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2" + name = "a TLS Duplex completion retains its owner and is inert after destroy" + out = Path("/work/diagnostics") + binary = Path("build/tls-arm64-shipping-debug/bun").resolve() + with (out / "fixture.stdout").open("x") as stdout, (out / "fixture.stderr").open("x") as stderr: + result = subprocess.run([str(binary), "test", str(fixture), "--test-name-pattern", "^" + name + "$", "--reporter=junit", "--reporter-outfile=" + str(out / "fixture.xml")], stdout=stdout, stderr=stderr, timeout=120) + cases = list(ElementTree.parse(out / "fixture.xml").iter("testcase")) + selected = [case for case in cases if case.get("name") == name] + assert len(selected) == 1 and selected[0].find("skipped") is None + outcome = {"exitCode": result.returncode, "name": name, "failure": selected[0].find("failure") is not None, "error": selected[0].find("error") is not None} + (out / "fixture-outcome.json").write_text(json.dumps(outcome, indent=2) + "\n") + if result.returncode == 0: + assert not outcome["failure"] and not outcome["error"] + sys.exit(result.returncode) + PY + ' + - name: Retain matched bytes and bounded diagnostic evidence + if: always() + shell: bash + env: + CONTAINER_ID: ${{ steps.shipping-container.outputs.id }} + run: | + set -euo pipefail + if test -n "$CONTAINER_ID"; then + [[ "$CONTAINER_ID" =~ ^[0-9a-f]{64}$ ]] + docker stop --time 30 "$CONTAINER_ID" > /dev/null + test "$(docker inspect --format '{{.State.Running}}' "$CONTAINER_ID")" = false + fi + python3 - <<'PY' + import hashlib, json, stat, subprocess + from pathlib import Path + files = sorted(path for path in Path("diagnostics").rglob("*") if path.is_file()) + files += [path for path in (Path("source/build/tls-arm64-shipping-debug") / name for name in ("bun", "bun-profile")) if path.is_file()] + assert files and len(files) <= 64 and all(stat.S_ISREG(path.lstat().st_mode) for path in files) + assert sum(path.stat().st_size for path in files) <= 8 * 1024**3 + def digest(path): + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + manifest = Path("diagnostics/archive-manifest.json") + manifest.write_text(json.dumps({str(path): {"bytes": path.stat().st_size, "sha256": digest(path)} for path in files}, indent=2) + "\n") + files.append(manifest) + output = Path("matched-shipping-debug.tar.zst.partial") + with output.open("xb") as stream: + subprocess.run(["tar", "--zstd", "-cf", "-", "--", *map(str, files)], stdout=stream, check=True) + assert output.stat().st_size <= 4 * 1024**3 + output.rename("matched-shipping-debug.tar.zst") + Path("shipping-debug-SHA256SUMS").write_text(digest(Path("matched-shipping-debug.tar.zst")) + " matched-shipping-debug.tar.zst\n") + PY + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: tls-linux-arm64-shipping-debug + path: | + matched-shipping-debug.tar.zst + shipping-debug-SHA256SUMS + if-no-files-found: error + retention-days: 7 + compression-level: 0 + - name: Stop shipping diagnostic container + if: always() + env: + CONTAINER_ID: ${{ steps.shipping-container.outputs.id }} + run: | + if test -n "$CONTAINER_ID"; then + [[ "$CONTAINER_ID" =~ ^[0-9a-f]{64}$ ]] + docker rm --force "$CONTAINER_ID" > /dev/null + fi diff --git a/.github/workflows/openclaw-tls-retainer.yml b/.github/workflows/openclaw-tls-retainer.yml new file mode 100644 index 000000000000..c49502efa453 --- /dev/null +++ b/.github/workflows/openclaw-tls-retainer.yml @@ -0,0 +1,128 @@ +name: OpenClaw TLS retainer capture + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-retainer.yml + - .github/openclaw/tls-retainer-capture.py + workflow_dispatch: + +permissions: + actions: read + contents: read + +concurrency: + group: openclaw-tls-retainer-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + linux-arm64: + runs-on: ubuntu-24.04-arm + timeout-minutes: 40 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: pipeline + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout: .github/openclaw + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: source + ref: 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e + persist-credentials: false + - name: Admit the immutable shipping-debug artifact + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + test "$(uname -m)" = aarch64 + gh api repos/openclaw/bun/actions/runs/41212340227 > producer-run.json + gh api repos/openclaw/bun/actions/artifacts/13048432420 > producer-artifact.json + gh api repos/openclaw/bun/actions/artifacts/13048432420/zip > producer.zip + python3 pipeline/.github/openclaw/tls-retainer-capture.py admit-zip + - name: Restore matched bytes in a secretless debugger container + id: debugger + shell: bash + run: | + set -euo pipefail + mkdir capture + id=$(docker run --detach --cap-add=SYS_PTRACE --security-opt=seccomp=unconfined \ + -v "$GITHUB_WORKSPACE:/work" -w /work \ + docker.io/library/debian:13@sha256:9cc080028c43b27d2074d63a5f9caf7166d731494965616c1a6d2827a004585c \ + sleep infinity) + [[ "$id" =~ ^[0-9a-f]{64}$ ]] + echo "id=$id" >> "$GITHUB_OUTPUT" + docker exec "$id" sh -euc ' + apt-get update + apt-get install --yes --no-install-recommends ca-certificates curl python3 binutils zstd git + curl -fsSL --retry 5 https://apt.llvm.org/llvm-snapshot.gpg.key > /etc/apt/trusted.gpg.d/apt.llvm.org.asc + echo "deb https://apt.llvm.org/trixie/ llvm-toolchain-trixie-23 main" > /etc/apt/sources.list.d/llvm.list + apt-get update + apt-get install --yes --no-install-recommends llvm-23 lldb-23 python3-lldb-23 + ' + docker exec "$id" python3 pipeline/.github/openclaw/tls-retainer-capture.py restore + cp producer-run.json producer-artifact.json capture/ + docker exec "$id" lldb-23 --version > capture/debugger-version.txt + docker exec "$id" ldd source/build/tls-arm64-shipping-debug/bun > capture/runtime-libraries.txt + ! grep -q 'not found' capture/runtime-libraries.txt + - name: Capture the actual fixture child and its post-mark owner path + env: + CONTAINER_ID: ${{ steps.debugger.outputs.id }} + shell: bash + run: | + set -euo pipefail + [[ "$CONTAINER_ID" =~ ^[0-9a-f]{64}$ ]] + docker exec "$CONTAINER_ID" python3 pipeline/.github/openclaw/tls-retainer-capture.py capture + - name: Freeze and package bounded capture evidence + if: always() + env: + CONTAINER_ID: ${{ steps.debugger.outputs.id }} + shell: bash + run: | + set -euo pipefail + if test -n "$CONTAINER_ID"; then + [[ "$CONTAINER_ID" =~ ^[0-9a-f]{64}$ ]] + docker stop --time 15 "$CONTAINER_ID" > /dev/null + test "$(docker inspect --format '{{.State.Running}}' "$CONTAINER_ID")" = false + fi + mkdir -p capture + cp pipeline/.github/openclaw/tls-retainer-capture.py capture/probe.py + python3 - <<'PY' + import hashlib, json, stat, tarfile + from pathlib import Path + files = sorted(Path('capture').iterdir()) + assert files and len(files) <= 24 + assert all(stat.S_ISREG(p.lstat().st_mode) for p in files) + assert sum(p.stat().st_size for p in files) <= 32 * 1024**2 + def sha(p): + with p.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + manifest = Path('capture/archive-manifest.json') + manifest.write_text(json.dumps({p.name: {'bytes': p.stat().st_size, 'sha256': sha(p)} for p in files}, indent=2) + '\n') + files.append(manifest) + with tarfile.open('tls-retainer-proof.tar.gz', 'x:gz') as archive: + for p in files: + archive.add(p, arcname=p.name, recursive=False) + Path('tls-retainer-SHA256SUMS').write_text(sha(Path('tls-retainer-proof.tar.gz')) + ' tls-retainer-proof.tar.gz\n') + PY + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: tls-linux-arm64-retainer + path: | + tls-retainer-proof.tar.gz + tls-retainer-SHA256SUMS + if-no-files-found: error + retention-days: 7 + - name: Remove this job's debugger container + if: always() + env: + CONTAINER_ID: ${{ steps.debugger.outputs.id }} + shell: bash + run: | + if test -n "$CONTAINER_ID"; then + [[ "$CONTAINER_ID" =~ ^[0-9a-f]{64}$ ]] + docker rm --force "$CONTAINER_ID" > /dev/null + fi diff --git a/.github/workflows/openclaw-tls-retention-baseline.yml b/.github/workflows/openclaw-tls-retention-baseline.yml new file mode 100644 index 000000000000..4c6848ae0ebd --- /dev/null +++ b/.github/workflows/openclaw-tls-retention-baseline.yml @@ -0,0 +1,124 @@ +name: OpenClaw TLS retention baseline + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-retention-baseline.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: openclaw-tls-retention-baseline-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + baseline: + name: Baseline ${{ matrix.target }} dropped cycle + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - target: darwin-x64 + runner: macos-15-intel + asset: "679970945" + - target: windows-x64 + runner: windows-2025 + asset: "679971471" + - target: windows-arm64 + runner: windows-11-arm + asset: "679971443" + env: + TARGET: ${{ matrix.target }} + ASSET_ID: ${{ matrix.asset }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e + persist-credentials: false + + - name: Download published baseline + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir diagnostics + gh api repos/openclaw/bun/releases/assets/679971474 \ + -H 'Accept: application/octet-stream' > diagnostics/manifest.json + gh api "repos/openclaw/bun/releases/assets/$ASSET_ID" \ + -H 'Accept: application/octet-stream' > baseline.zip + + - name: Verify baseline and run the exact dropped-cycle case + shell: bash + run: | + set -euo pipefail + python=python3 + command -v python3 >/dev/null || python=python + "$python" - <<'PY' + import hashlib, json, os, subprocess, sys, zipfile + from pathlib import Path + from xml.etree import ElementTree + + def sha256(path): + digest = hashlib.sha256() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + report = Path("diagnostics").resolve() + manifest_path = report / "manifest.json" + assert sha256(manifest_path) == "8c3a9d87f37832fa07173635af52d53b813e29470afa4afdb49a83033fed039e" + manifest = json.loads(manifest_path.read_text()) + commit = "191755ec115ece3dd642eba3cb0143cecb5e5d7a" + webkit = "b82bc917e99110b61c3a21202d6def7457aa194c" + assert manifest["bun"]["commit"] == commit + assert manifest["webkit"]["version"] == webkit + matches = [asset for asset in manifest["assets"] if asset["target"] == os.environ["TARGET"]] + assert len(matches) == 1 + asset = matches[0] + archive = Path("baseline.zip") + assert sha256(archive) == asset["sha256"] + with zipfile.ZipFile(archive) as bundle: + bundle.extractall("baseline") + exe = (Path("baseline") / asset["executable"]["path"]).resolve() + assert sha256(exe) == asset["executable"]["sha256"] + exe.chmod(exe.stat().st_mode | 0o111) + identity = subprocess.run([str(exe), "-e", "console.log(JSON.stringify({commit:Bun.revision,webkit:process.versions.webkit,platform:process.platform,arch:process.arch}))"], + text=True, capture_output=True, check=True) + facts = json.loads(identity.stdout) + (report / "identity.json").write_text(json.dumps(facts, indent=2) + "\n") + platform, arch = os.environ["TARGET"].split("-") + assert facts == {"commit": commit, "webkit": webkit, + "platform": "win32" if platform == "windows" else platform, "arch": arch}, facts + fixture = Path("test/js/bun/net/socket-retention.test.ts") + assert sha256(fixture) == "d4c04a72b89bc7d33c292a16a57c38f5da540815dc2e141adfd58c1bd9973af2" + name = "TLS Duplex cycles collect after startup while a retained origin keeps its wrapper alive" + with (report / "stdout.txt").open("w") as stdout, (report / "stderr.txt").open("w") as stderr: + result = subprocess.run([str(exe), "test", str(fixture), "--test-name-pattern", "^" + name + "$", + "--reporter=junit", "--reporter-outfile=" + str(report / "result.xml")], stdout=stdout, stderr=stderr) + tree = ElementTree.parse(report / "result.xml") + selected = [case for case in tree.iter("testcase") if case.get("name") == name] + assert len(selected) == 1 and selected[0].find("skipped") is None, "selected case did not execute" + case = selected[0] + outcome = {"exitCode": result.returncode, "name": name, + "failure": case.find("failure") is not None, "error": case.find("error") is not None} + (report / "outcome.json").write_text(json.dumps(outcome, indent=2) + "\n") + print(json.dumps(outcome)) + if result.returncode == 0: + assert not outcome["failure"] and not outcome["error"], outcome + sys.exit(result.returncode) + PY + + - name: Retain baseline evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: tls-retention-baseline-${{ matrix.target }} + path: diagnostics + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/openclaw-tls-symbols.yml b/.github/workflows/openclaw-tls-symbols.yml new file mode 100644 index 000000000000..634fe6f6228e --- /dev/null +++ b/.github/workflows/openclaw-tls-symbols.yml @@ -0,0 +1,168 @@ +name: OpenClaw TLS shipping-symbol capability + +on: + pull_request: + paths: + - .github/workflows/openclaw-tls-symbols.yml + workflow_dispatch: + +permissions: + actions: read + contents: read + +concurrency: + group: openclaw-tls-shipping-symbols-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +# Inspect the original failing PR #215 bytes; never execute or rebuild them. +env: + SOURCE_COMMIT: 454a78a4d3b0202ecdbedf7dd051fc84ffc59e9e + SOURCE_RUN: "40897692970" + TARGET: linux-arm64 + TRIPLET: bun-linux-aarch64 + ARTIFACT_ID: "12908842070" + ARTIFACT_SHA256: 65f1e6de6618b9fc2dd75c688d49fff27bcedc61fd03105f1f92c9df49e4f5c4 + +jobs: + inspect: + name: Inspect shipping Linux ARM64 symbols + runs-on: ubuntu-24.04-arm + timeout-minutes: 15 + steps: + - name: Fetch and verify frozen artifact + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir diagnostics + gh api "repos/openclaw/bun/actions/artifacts/$ARTIFACT_ID" > diagnostics/artifact.json + gh api "repos/openclaw/bun/actions/artifacts/$ARTIFACT_ID/zip" > artifact.zip + python3 - <<'PY' + import json, os + from pathlib import Path + + artifact = json.loads(Path("diagnostics/artifact.json").read_text()) + assert artifact["id"] == int(os.environ["ARTIFACT_ID"]), artifact + assert artifact["name"] == "build-" + os.environ["TARGET"], artifact + assert not artifact["expired"], artifact + assert artifact["workflow_run"]["id"] == int(os.environ["SOURCE_RUN"]), artifact + assert artifact["workflow_run"]["head_sha"] == os.environ["SOURCE_COMMIT"], artifact + expected = os.environ["ARTIFACT_SHA256"] + assert artifact["digest"] == "sha256:" + expected, artifact + PY + actual=$(shasum -a 256 artifact.zip | awk '{print $1}') + [ "$actual" = "$ARTIFACT_SHA256" ] + unzip -q artifact.zip -d downloaded + unzip -q "downloaded/$TRIPLET.zip" -d unpacked + unzip -q "downloaded/$TRIPLET-profile.zip" -d unpacked + shasum -a 256 "unpacked/$TRIPLET/bun" \ + "unpacked/$TRIPLET-profile/bun-profile" > diagnostics/executables.sha256 + + + - name: Prepare LLVM capability probe + shell: bash + run: | + set -euo pipefail + test "$(uname -m)" = aarch64 + cat > diagnostics/probe.py <<'PY' + import json, lldb, re + from pathlib import Path + + target = lldb.debugger.GetSelectedTarget() + assert target.IsValid(), "missing shipping target" + module = target.FindModule(target.GetExecutable()) + assert module.IsValid(), "missing executable module" + uuid = re.sub("[^0-9a-fA-F]", "", module.GetUUIDString() or "").lower() + expected = re.findall(r"Build ID: (\w+)", Path("/work/diagnostics/executable-notes.txt").read_text()) + assert len(expected) == 1 and uuid == expected[0], "shipping module identity mismatch" + types = {} + for name in ("JSC::JSCell", "JSC::Heap", "JSC::SlotVisitor", "JSC::ConservativeRoots"): + candidates = target.FindTypes(name) + complete = [] + for i in range(candidates.GetSize()): + value = candidates.GetTypeAtIndex(i).GetCanonicalType() + fields = value.GetNumberOfFields() + if value.IsTypeComplete() and value.GetByteSize() > 0 and fields > 0: + complete.append({"bytes": value.GetByteSize(), "fields": fields}) + types[name] = {"candidates": candidates.GetSize(), "complete": complete} + functions = {} + for name in ("JSC::Collector::runEndPhase", "JSC::MachineThreads::gatherConservativeRoots"): + contexts = module.FindFunctions(name, lldb.eFunctionNameTypeAny) + debug = [] + for i in range(contexts.GetSize()): + context = contexts.GetContextAtIndex(i) + function = context.GetFunction() + unit = context.GetCompileUnit() + if not function.IsValid() or not unit.IsValid(): + continue + address = function.GetStartAddress().GetFileAddress() + if address not in (0, lldb.LLDB_INVALID_ADDRESS): + debug.append({"address": address, "name": function.GetName(), "compile_unit": str(unit.GetFileSpec())}) + functions[name] = {"debug": debug, "symbols": []} + for i in range(module.GetNumSymbols()): + symbol = module.GetSymbolAtIndex(i) + name = symbol.GetName() or "" + for owner, entries in functions.items(): + if name == owner or name.startswith(owner + "("): + address = symbol.GetStartAddress().GetFileAddress() + if address not in (0, lldb.LLDB_INVALID_ADDRESS): + entries["symbols"].append({"name": name, "address": address}) + complete = all(value["complete"] for value in types.values()) and all(value["debug"] for value in functions.values()) + Path("/work/diagnostics/shipping-capability.json").write_text(json.dumps({"module_uuid": uuid, "complete": complete, "types": types, "functions": functions}, indent=2) + "\n") + PY + docker run --detach --name tls-shipping-symbols -v "$GITHUB_WORKSPACE:/work" -w /work \ + docker.io/library/debian:13@sha256:9cc080028c43b27d2074d63a5f9caf7166d731494965616c1a6d2827a004585c \ + sleep infinity > /dev/null + docker exec tls-shipping-symbols sh -euc ' + apt-get update + apt-get install --yes --no-install-recommends ca-certificates curl python3 binutils + curl -fsSL --retry 5 https://apt.llvm.org/llvm-snapshot.gpg.key > /etc/apt/trusted.gpg.d/apt.llvm.org.asc + echo "deb https://apt.llvm.org/trixie/ llvm-toolchain-trixie-23 main" > /etc/apt/sources.list.d/llvm.list + apt-get update + apt-get install --yes --no-install-recommends llvm-23 lldb-23 python3-lldb-23 + ' + docker exec tls-shipping-symbols sh -euc 'lldb-23 --version; dpkg-query -W llvm-23 lldb-23 python3-lldb-23' > diagnostics/runtime-tools.txt + + - name: Require shipping type and function capability + shell: bash + run: | + set -euo pipefail + exe="unpacked/$TRIPLET/bun" + profile="unpacked/$TRIPLET-profile/bun-profile" + readelf -n "$exe" > diagnostics/executable-notes.txt + readelf -n "$profile" > diagnostics/profile-notes.txt + python3 - <<'PY' + import re + from pathlib import Path + ids = [re.findall(r"Build ID: (\w+)", Path("diagnostics", name).read_text()) for name in ("executable-notes.txt", "profile-notes.txt")] + assert len(ids[0]) == 1 and ids[0] == ids[1], ids + PY + docker exec tls-shipping-symbols lldb-23 --no-lldbinit --batch \ + -o 'settings set target.load-script-from-symbol-file false' \ + -o "target create /work/$exe" -o "target symbols add /work/$profile" \ + -o 'script exec(open("/work/diagnostics/probe.py").read())' > diagnostics/debugger.txt 2>&1 + python3 - <<'PY' + import json + from pathlib import Path + report = json.loads(Path("diagnostics/shipping-capability.json").read_text()) + assert set(report["types"]) == {"JSC::JSCell", "JSC::Heap", "JSC::SlotVisitor", "JSC::ConservativeRoots"} + assert all(value["complete"] and all(item["bytes"] > 0 and item["fields"] > 0 for item in value["complete"]) for value in report["types"].values()), "shipping GC types unavailable" + assert set(report["functions"]) == {"JSC::Collector::runEndPhase", "JSC::MachineThreads::gatherConservativeRoots"} + assert all(value["debug"] and all(item["address"] > 0 and item["name"] and item["compile_unit"] for item in value["debug"]) for value in report["functions"].values()), "shipping debug functions unavailable" + assert report["complete"] is True + PY + + - name: Retain shipping capability evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: tls-shipping-symbols-linux-arm64 + path: diagnostics + if-no-files-found: error + retention-days: 7 + + - name: Stop diagnostic container + if: always() + shell: bash + run: docker rm --force tls-shipping-symbols > /dev/null 2>&1 || true