From 2fdbc321a31bfeb184f95ec0fa1799de4621a2f5 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 1 Nov 2026 06:00:27 +0000 Subject: [PATCH] fix(ci): include built AOT sources in artifacts --- .github/openclaw/artifacts.py | 23 ++++++++-- .github/openclaw/release-notes.md | 2 + .github/openclaw/release.py | 24 +++++----- .github/openclaw/test_artifacts.py | 71 ++++++++++++++++++++++++++++++ .github/openclaw/test_release.py | 70 +++++++++++++++++++++++++++++ 5 files changed, 176 insertions(+), 14 deletions(-) diff --git a/.github/openclaw/artifacts.py b/.github/openclaw/artifacts.py index 81a63ee912213..01cb85c776bdb 100644 --- a/.github/openclaw/artifacts.py +++ b/.github/openclaw/artifacts.py @@ -77,6 +77,13 @@ def write_build_defines(artifact, lane, sha): def lane_definitions(): return json.loads(capture('node', '.github/scripts/lanes.mjs', '--json')) +def archive_names(lane): + label = lane['label'] + sources = 'bun-webkit-aot-sources' if label == 'bun-webkit-linux-amd64-lto-aot' else '' + if lane.get('sources', '') != sources: + raise ValueError('invalid AOT source producer metadata') + return [label + '.tar.gz'] + ([sources + '.tar.gz'] if sources else []) + def preflight(): source_record = source() sha = source_record['source'] @@ -102,6 +109,7 @@ def build(label, destination): if label not in [l['label'] for l in CONFIG['lanes']]: raise ValueError('lane is outside the approved consumer matrix') lane = next(l for l in lane_definitions() if l['label'] == label) + names = archive_names(lane) destination = destination.resolve() destination.mkdir(parents=True, exist_ok=False) artifact = destination / 'bun-webkit' @@ -127,18 +135,25 @@ def build(label, destination): for path in (artifact/'lib').glob('*.so*'): if path.is_file() or path.is_symlink(): path.unlink() - archive = destination/(label+'.tar.gz') - run('tar', '-czf', str(archive), '-C', str(destination), 'bun-webkit') + for name in names: + archive = destination / name + members = ['bun-webkit'] if name == label + '.tar.gz' else [ + '--exclude=bun-webkit/Source/JavaScriptCore/aot/tests', + 'bun-webkit/Source/JavaScriptCore/aot', + ] + run('tar', '-czf', str(archive), '-C', str(destination), *members) + files = {name: {'sha256': digest(destination / name), 'size': (destination / name).stat().st_size} + for name in names} paths = capture('git', 'ls-files', '--', *RECIPE).splitlines() manifest = {'schema_version':1,**source_record,'lane':lane, 'recipe_sha256':{p:digest(ROOT/p) for p in paths},'toolchain_image_id':image['Id'], 'commands':{'base':base_command,'artifact':build_command}, 'tool_versions':tool_versions,'docker_version':capture('docker','version'),'buildx_version':capture('docker','buildx','version'), 'workflow_run':os.environ.get('GITHUB_RUN_ID'),'run_attempt':os.environ.get('GITHUB_RUN_ATTEMPT'), - 'elapsed_seconds':time.time()-started,'files':{archive.name:{'sha256':digest(archive),'size':archive.stat().st_size}}, + 'elapsed_seconds':time.time()-started,'files':files, 'provenance_files':{p.name:digest(p) for p in provenance.iterdir() if p.is_file()}} write(destination/(label+'.manifest.json'), manifest) - (destination/'SHA256SUMS').write_text(digest(archive)+' '+archive.name+'\n') + (destination/'SHA256SUMS').write_text(''.join(files[name]['sha256']+' '+name+'\n' for name in names)) if label == 'bun-webkit-linux-amd64': run(str(artifact/'bin/jsc'), '-e', 'print("OpenClaw WebKit ready")') run(str(artifact/'bin/testFFI')) diff --git a/.github/openclaw/release-notes.md b/.github/openclaw/release-notes.md index 18a385a9adc77..c913a68727ea5 100644 --- a/.github/openclaw/release-notes.md +++ b/.github/openclaw/release-notes.md @@ -1,3 +1,5 @@ +When the AOT source-producing lane is selected, packaging retains its generator sources and built engine declarations in a separate checksum-verified archive for Bun's generator-only jobs. Release assembly checks and includes that archive alongside the native lane output. + Static JSC API tests now link the same AOT program callbacks and shared builtin-table object as the standalone tools. The table object belongs to the engine build rather than the optional shell, so API tests can configure with the shell disabled. This update retains the main and published fork histories through normal merges and integrates upstream WebKit `e66916298091dd12469076fd854e135b03fb16ff`. Worker limits and passive collector progress move to the upstream Collector owner. ArrayBuffer ownership remains separate from other extra memory while using the upstream allocation cadence. Namespace facades, ALS, allocation sampling, trap delivery, initialized VM entry storage and ten artifact lanes remain covered by their existing gates. Parser and cache integration preserves syntax-selected positions, including parenthesized optional calls and tags. The ten existing artifact lanes disable AOT and record the matching lane ABI definitions in their packaged headers. diff --git a/.github/openclaw/release.py b/.github/openclaw/release.py index b76f234e51f1b..0715a6afb327b 100644 --- a/.github/openclaw/release.py +++ b/.github/openclaw/release.py @@ -86,21 +86,25 @@ def assemble(inputs,out,source): artifacts={}; lanes=[] expected_patches=None for lane in config['lanes']: - label=lane['label']; name=label+'.tar.gz'; directory=inputs/label + label=lane['label']; directory=inputs/label manifest=json.loads((directory/(label+'.manifest.json')).read_text()) if manifest['source']!=source or manifest['lane']['label']!=label: raise ValueError('mixed source or lane') patches=lane_patch_records(manifest,source) if expected_patches is None:expected_patches=patches elif patches!=expected_patches:raise ValueError('lane patch provenance mismatch') - f=directory/name - if sha(f)!=manifest['files'][name]['sha256'] or f.stat().st_size!=manifest['files'][name]['size']: - raise ValueError('lane digest mismatch') - if label==consumer_identity.LANE: - measured=consumer_identity.engine_input(f,directory/(label+'.manifest.json'),source) - consumer_identity.validate_evidence(identity,source,measured,binaries,proof) - shutil.copyfile(f,out/name) - artifacts[name]={'url':f'https://github.com/{REPO}/releases/download/{tag}/{name}','sha256':sha(f),'size':f.stat().st_size} + for name in artifact_source.archive_names(manifest['lane']): + if name in artifacts:raise ValueError('duplicate release artifact: '+name) + entry=manifest['files'].get(name) + if entry is None:raise ValueError('missing lane artifact: '+name) + f=directory/name + if sha(f)!=entry['sha256'] or f.stat().st_size!=entry['size']: + raise ValueError('lane digest mismatch') + if name==label+'.tar.gz' and label==consumer_identity.LANE: + measured=consumer_identity.engine_input(f,directory/(label+'.manifest.json'),source) + consumer_identity.validate_evidence(identity,source,measured,binaries,proof) + shutil.copyfile(f,out/name) + artifacts[name]={'url':f'https://github.com/{REPO}/releases/download/{tag}/{name}','sha256':sha(f),'size':f.stat().st_size} lanes.append(manifest) windows=json.loads((inputs/'windows-arm64-qualification/gate.json').read_text()) if (windows.get('passed') is not True or windows.get('source')!=source @@ -181,7 +185,7 @@ def publish(directory,source): raise ValueError('publishing requires reviewed source on an explicitly permitted owned branch') # Tag creation fails atomically if the name exists. Neither existing drafts nor releases are resumed. api('-X','POST',f'repos/{REPO}/git/refs','-f','ref=refs/tags/'+tag,'-f','sha='+source) - body=f'OpenClaw-built WebKit at `{source}`. All {len(manifest["artifacts"])} lanes, Linux qualification and native Windows ARM64 engine probes passed.\n\n'+(ROOT/'.github/openclaw/release-notes.md').read_text() + body=f'OpenClaw-built WebKit at `{source}`. All {len(manifest["lanes"])} lanes, Linux qualification and native Windows ARM64 engine probes passed.\n\n'+(ROOT/'.github/openclaw/release-notes.md').read_text() draft=api('-X','POST',f'repos/{REPO}/releases','-f','tag_name='+tag,'-f','name='+tag,'-f','body='+body,'-F','draft=true') release_id=draft['id'] execute('gh','release','upload',tag,'-R',REPO,*[str(p) for p in sorted(directory.iterdir())]) diff --git a/.github/openclaw/test_artifacts.py b/.github/openclaw/test_artifacts.py index ff5f0ae99c70f..4337997ebef10 100644 --- a/.github/openclaw/test_artifacts.py +++ b/.github/openclaw/test_artifacts.py @@ -1,4 +1,6 @@ import hashlib +import json +import tarfile import os from pathlib import Path import subprocess @@ -82,6 +84,75 @@ def rejects_before_lane(self, repository, base, floor, error=ValueError): lanes.assert_not_called() self.assertFalse(destination.exists()) + + def test_build_packs_aot_sources_from_built_output(self): + repository, commits = self.graph('source-package') + label = 'bun-webkit-linux-amd64-lto-aot' + lane = {'label': label, 'sources': 'bun-webkit-aot-sources', 'aot': 1, + 'image': 'fixture-toolchain', 'package_os': 'linux', 'package_cpu': 'x64', 'defines': ''} + destination = self.root / 'built-output' + artifact = destination / 'bun-webkit' + generated = b'{"origin":"built-engine-declarations"}\n' + members = { + 'include/cmakeconfig.h': b'#pragma once\n', + 'lib/libJavaScriptCore.a': b'native-library', + 'Source/JavaScriptCore/aot/ir/AOTSchema.def': b'built-schema\n', + 'Source/JavaScriptCore/aot/ir/generator/index.ts': b'// built-generator\n', + 'Source/JavaScriptCore/aot/builtins/generated/AOTBuiltinEngineDeclarations.json': generated, + 'Source/JavaScriptCore/aot/tests/example.ir': b'test-only\n', + } + real_run, real_capture = artifacts.run, artifacts.capture + + def run(*args, **kwargs): + if args[0] != 'docker': + return real_run(*args, **kwargs) + if '--target=artifact' in args: + for name, data in members.items(): + path = artifact / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + def capture(*args): + if args[0] == 'node': + if args[-1] == '--json': + return json.dumps([lane]) + return json.dumps(['docker', 'buildx', 'build', '--target=artifact', '.']) + if args[:3] == ('docker', 'image', 'inspect'): + return json.dumps([{'Id': 'fixture-image'}]) + if args[0] == 'docker': + return 'fixture-tool-versions' + if args[:2] == ('git', 'ls-files'): + return self.capture(repository, *args[1:]) + return real_capture(*args) + + with patch.dict(os.environ, self.env, clear=True), patch.multiple( + artifacts, ROOT=repository, BASE=commits['B'], HISTORY_FLOOR=commits['F'], + RECIPE=['Dockerfile'], CONFIG={'lanes': [{'label': label}]}), patch.object( + artifacts, 'run', side_effect=run), patch.object(artifacts, 'capture', side_effect=capture): + artifacts.build(label, destination) + + native = destination / (label + '.tar.gz') + sources = destination / 'bun-webkit-aot-sources.tar.gz' + declaration = 'bun-webkit/Source/JavaScriptCore/aot/builtins/generated/AOTBuiltinEngineDeclarations.json' + with tarfile.open(sources) as archive: + self.assertEqual(archive.extractfile(declaration).read(), generated) + names = archive.getnames() + self.assertTrue(all(name.startswith('bun-webkit/Source/JavaScriptCore/aot') for name in names)) + self.assertFalse(any('/tests' in name for name in names)) + self.assertFalse(any(name.startswith('bun-webkit/lib/') for name in names)) + with tarfile.open(native) as archive: + self.assertEqual(archive.extractfile('bun-webkit/lib/libJavaScriptCore.a').read(), b'native-library') + self.assertEqual(archive.extractfile(declaration).read(), generated) + manifest = json.loads((destination / (label + '.manifest.json')).read_text()) + expected = self.oracle(repository, commits, ['P', 'H']) + self.assertEqual({key: manifest[key] for key in expected}, expected) + files = {path.name: {'sha256': hashlib.sha256(path.read_bytes()).hexdigest(), 'size': path.stat().st_size} + for path in (native, sources)} + self.assertEqual(manifest['files'], files) + self.assertEqual((destination / 'SHA256SUMS').read_text(), + ''.join(entry['sha256'] + ' ' + name + '\n' for name, entry in files.items())) + self.assertFalse(artifact.exists()) + def test_complete_history_and_raw_patch_hashes(self): repository, commits = self.graph('complete') self.assertEqual(self.source(repository, commits['B'], commits['F']), diff --git a/.github/openclaw/test_release.py b/.github/openclaw/test_release.py index a491905fc3655..929acfde771eb 100644 --- a/.github/openclaw/test_release.py +++ b/.github/openclaw/test_release.py @@ -1,5 +1,7 @@ import importlib.util import json +import io +import tarfile import base64 import hashlib import xml.etree.ElementTree as ET @@ -137,6 +139,74 @@ def test_assembly_rechecks_raw_facts_after_plausible_redigested_summaries(self): def test_complete_set_roundtrip(self): self.assemble();self.assertEqual(len(release.verify(self.out,SOURCE)['artifacts']),10) + + def test_aot_source_producer_roundtrip_and_integrity(self): + config=json.loads((release.ROOT/'.github/openclaw/variants.json').read_text()) + label='bun-webkit-linux-amd64-lto-aot' + source_name='bun-webkit-aot-sources.tar.gz' + config['lanes'].append({'label':label}) + root=self.root/'source' + config_path=root/'.github/openclaw/variants.json' + config_path.parent.mkdir(parents=True) + config_path.write_text(json.dumps(config)) + directory=self.inputs/label + directory.mkdir();(directory/'provenance').mkdir() + declaration='bun-webkit/Source/JavaScriptCore/aot/builtins/generated/AOTBuiltinEngineDeclarations.json' + generated=b'{"origin":"built-engine-declarations"}\n' + for name,members in [ + (label+'.tar.gz',{'bun-webkit/lib/libJavaScriptCore.a':b'native-library',declaration:generated}), + (source_name,{declaration:generated}), + ]: + with tarfile.open(directory/name,'w:gz') as archive: + for member,data in members.items(): + info=tarfile.TarInfo(member);info.size=len(data) + archive.addfile(info,io.BytesIO(data)) + manifest={'source':SOURCE,'base':release.artifact_source.BASE, + 'history_floor':release.artifact_source.HISTORY_FLOOR,'patches':PATCHES, + 'lane':{'label':label,'sources':'bun-webkit-aot-sources'}, + 'files':{name:{'sha256':release.sha(directory/name),'size':(directory/name).stat().st_size} + for name in (label+'.tar.gz',source_name)}} + manifest_path=directory/(label+'.manifest.json') + manifest_path.write_text(json.dumps(manifest)) + source_path=directory/source_name + original=source_path.read_bytes() + + with patch.object(release,'ROOT',root): + release.assemble(self.inputs,self.out,SOURCE) + result=release.verify(self.out,SOURCE) + self.assertEqual(len(result['lanes']),len(config['lanes'])) + self.assertEqual(len(result['artifacts']),len(config['lanes'])+1) + self.assertEqual(result['artifacts'][source_name],{ + 'url':f'https://github.com/openclaw/WebKit/releases/download/autobuild-{SOURCE}/{source_name}', + 'sha256':hashlib.sha256(original).hexdigest(),'size':len(original)}) + self.assertEqual((self.out/source_name).read_bytes(),original) + with tarfile.open(self.out/source_name) as archive: + self.assertEqual(archive.extractfile(declaration).read(),generated) + with tarfile.open(self.out/'provenance.tar.gz') as archive: + recorded=json.loads(archive.extractfile(label+'/'+label+'.manifest.json').read()) + self.assertEqual(recorded,manifest) + for index,case in enumerate(('missing-file','missing-entry','changed-bytes','wrong-size','missing-sources','duplicate')): + with self.subTest(case=case): + source_path.write_bytes(original) + changed=json.loads(json.dumps(manifest)) + expected=ValueError + if case=='missing-file': + source_path.unlink();expected=FileNotFoundError + elif case=='missing-entry': + del changed['files'][source_name] + elif case=='changed-bytes': + source_path.write_bytes(bytes([original[0]^1])+original[1:]) + elif case=='wrong-size': + changed['files'][source_name]['size']+=1 + elif case=='missing-sources': + del changed['lane']['sources'] + elif case=='duplicate': + config['lanes'].append({'label':label}) + config_path.write_text(json.dumps(config)) + manifest_path.write_text(json.dumps(changed)) + with self.assertRaises(expected): + release.assemble(self.inputs,self.root/('invalid-source-'+str(index)),SOURCE) + def test_changed_lane_bytes_rejected(self): next(self.inputs.glob('*/*.tar.gz')).write_bytes(b'tampered') with self.assertRaises(ValueError):self.assemble()