Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions .github/openclaw/artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,13 @@ def write_build_defines(artifact, lane, sha):
def lane_definitions():
return json.loads(capture('node', '.github/scripts/lanes.mjs', '--json'))

def archive_names(lane):
label = lane['label']
sources = 'bun-webkit-aot-sources' if label == 'bun-webkit-linux-amd64-lto-aot' else ''
if lane.get('sources', '') != sources:
raise ValueError('invalid AOT source producer metadata')
return [label + '.tar.gz'] + ([sources + '.tar.gz'] if sources else [])

def preflight():
source_record = source()
sha = source_record['source']
Expand All @@ -102,6 +109,7 @@ def build(label, destination):
if label not in [l['label'] for l in CONFIG['lanes']]:
raise ValueError('lane is outside the approved consumer matrix')
lane = next(l for l in lane_definitions() if l['label'] == label)
names = archive_names(lane)
destination = destination.resolve()
destination.mkdir(parents=True, exist_ok=False)
artifact = destination / 'bun-webkit'
Expand All @@ -127,18 +135,25 @@ def build(label, destination):
for path in (artifact/'lib').glob('*.so*'):
if path.is_file() or path.is_symlink():
path.unlink()
archive = destination/(label+'.tar.gz')
run('tar', '-czf', str(archive), '-C', str(destination), 'bun-webkit')
for name in names:
archive = destination / name
members = ['bun-webkit'] if name == label + '.tar.gz' else [
'--exclude=bun-webkit/Source/JavaScriptCore/aot/tests',
'bun-webkit/Source/JavaScriptCore/aot',
]
run('tar', '-czf', str(archive), '-C', str(destination), *members)
files = {name: {'sha256': digest(destination / name), 'size': (destination / name).stat().st_size}
for name in names}
paths = capture('git', 'ls-files', '--', *RECIPE).splitlines()
manifest = {'schema_version':1,**source_record,'lane':lane,
'recipe_sha256':{p:digest(ROOT/p) for p in paths},'toolchain_image_id':image['Id'],
'commands':{'base':base_command,'artifact':build_command},
'tool_versions':tool_versions,'docker_version':capture('docker','version'),'buildx_version':capture('docker','buildx','version'),
'workflow_run':os.environ.get('GITHUB_RUN_ID'),'run_attempt':os.environ.get('GITHUB_RUN_ATTEMPT'),
'elapsed_seconds':time.time()-started,'files':{archive.name:{'sha256':digest(archive),'size':archive.stat().st_size}},
'elapsed_seconds':time.time()-started,'files':files,
'provenance_files':{p.name:digest(p) for p in provenance.iterdir() if p.is_file()}}
write(destination/(label+'.manifest.json'), manifest)
(destination/'SHA256SUMS').write_text(digest(archive)+' '+archive.name+'\n')
(destination/'SHA256SUMS').write_text(''.join(files[name]['sha256']+' '+name+'\n' for name in names))
if label == 'bun-webkit-linux-amd64':
run(str(artifact/'bin/jsc'), '-e', 'print("OpenClaw WebKit ready")')
run(str(artifact/'bin/testFFI'))
Expand Down
2 changes: 2 additions & 0 deletions .github/openclaw/release-notes.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
When the AOT source-producing lane is selected, packaging retains its generator sources and built engine declarations in a separate checksum-verified archive for Bun's generator-only jobs. Release assembly checks and includes that archive alongside the native lane output.

Static JSC API tests now link the same AOT program callbacks and shared builtin-table object as the standalone tools. The table object belongs to the engine build rather than the optional shell, so API tests can configure with the shell disabled.

This update retains the main and published fork histories through normal merges and integrates upstream WebKit `e66916298091dd12469076fd854e135b03fb16ff`. Worker limits and passive collector progress move to the upstream Collector owner. ArrayBuffer ownership remains separate from other extra memory while using the upstream allocation cadence. Namespace facades, ALS, allocation sampling, trap delivery, initialized VM entry storage and ten artifact lanes remain covered by their existing gates. Parser and cache integration preserves syntax-selected positions, including parenthesized optional calls and tags. The ten existing artifact lanes disable AOT and record the matching lane ABI definitions in their packaged headers.
Expand Down
24 changes: 14 additions & 10 deletions .github/openclaw/release.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,21 +86,25 @@ def assemble(inputs,out,source):
artifacts={}; lanes=[]
expected_patches=None
for lane in config['lanes']:
label=lane['label']; name=label+'.tar.gz'; directory=inputs/label
label=lane['label']; directory=inputs/label
manifest=json.loads((directory/(label+'.manifest.json')).read_text())
if manifest['source']!=source or manifest['lane']['label']!=label:
raise ValueError('mixed source or lane')
patches=lane_patch_records(manifest,source)
if expected_patches is None:expected_patches=patches
elif patches!=expected_patches:raise ValueError('lane patch provenance mismatch')
f=directory/name
if sha(f)!=manifest['files'][name]['sha256'] or f.stat().st_size!=manifest['files'][name]['size']:
raise ValueError('lane digest mismatch')
if label==consumer_identity.LANE:
measured=consumer_identity.engine_input(f,directory/(label+'.manifest.json'),source)
consumer_identity.validate_evidence(identity,source,measured,binaries,proof)
shutil.copyfile(f,out/name)
artifacts[name]={'url':f'https://github.com/{REPO}/releases/download/{tag}/{name}','sha256':sha(f),'size':f.stat().st_size}
for name in artifact_source.archive_names(manifest['lane']):
if name in artifacts:raise ValueError('duplicate release artifact: '+name)
entry=manifest['files'].get(name)
if entry is None:raise ValueError('missing lane artifact: '+name)
f=directory/name
if sha(f)!=entry['sha256'] or f.stat().st_size!=entry['size']:
raise ValueError('lane digest mismatch')
if name==label+'.tar.gz' and label==consumer_identity.LANE:
measured=consumer_identity.engine_input(f,directory/(label+'.manifest.json'),source)
consumer_identity.validate_evidence(identity,source,measured,binaries,proof)
shutil.copyfile(f,out/name)
artifacts[name]={'url':f'https://github.com/{REPO}/releases/download/{tag}/{name}','sha256':sha(f),'size':f.stat().st_size}
lanes.append(manifest)
windows=json.loads((inputs/'windows-arm64-qualification/gate.json').read_text())
if (windows.get('passed') is not True or windows.get('source')!=source
Expand Down Expand Up @@ -181,7 +185,7 @@ def publish(directory,source):
raise ValueError('publishing requires reviewed source on an explicitly permitted owned branch')
# Tag creation fails atomically if the name exists. Neither existing drafts nor releases are resumed.
api('-X','POST',f'repos/{REPO}/git/refs','-f','ref=refs/tags/'+tag,'-f','sha='+source)
body=f'OpenClaw-built WebKit at `{source}`. All {len(manifest["artifacts"])} lanes, Linux qualification and native Windows ARM64 engine probes passed.\n\n'+(ROOT/'.github/openclaw/release-notes.md').read_text()
body=f'OpenClaw-built WebKit at `{source}`. All {len(manifest["lanes"])} lanes, Linux qualification and native Windows ARM64 engine probes passed.\n\n'+(ROOT/'.github/openclaw/release-notes.md').read_text()
draft=api('-X','POST',f'repos/{REPO}/releases','-f','tag_name='+tag,'-f','name='+tag,'-f','body='+body,'-F','draft=true')
release_id=draft['id']
execute('gh','release','upload',tag,'-R',REPO,*[str(p) for p in sorted(directory.iterdir())])
Expand Down
71 changes: 71 additions & 0 deletions .github/openclaw/test_artifacts.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import hashlib
import json
import tarfile
import os
from pathlib import Path
import subprocess
Expand Down Expand Up @@ -82,6 +84,75 @@ def rejects_before_lane(self, repository, base, floor, error=ValueError):
lanes.assert_not_called()
self.assertFalse(destination.exists())


def test_build_packs_aot_sources_from_built_output(self):
repository, commits = self.graph('source-package')
label = 'bun-webkit-linux-amd64-lto-aot'
lane = {'label': label, 'sources': 'bun-webkit-aot-sources', 'aot': 1,
'image': 'fixture-toolchain', 'package_os': 'linux', 'package_cpu': 'x64', 'defines': ''}
destination = self.root / 'built-output'
artifact = destination / 'bun-webkit'
generated = b'{"origin":"built-engine-declarations"}\n'
members = {
'include/cmakeconfig.h': b'#pragma once\n',
'lib/libJavaScriptCore.a': b'native-library',
'Source/JavaScriptCore/aot/ir/AOTSchema.def': b'built-schema\n',
'Source/JavaScriptCore/aot/ir/generator/index.ts': b'// built-generator\n',
'Source/JavaScriptCore/aot/builtins/generated/AOTBuiltinEngineDeclarations.json': generated,
'Source/JavaScriptCore/aot/tests/example.ir': b'test-only\n',
}
real_run, real_capture = artifacts.run, artifacts.capture

def run(*args, **kwargs):
if args[0] != 'docker':
return real_run(*args, **kwargs)
if '--target=artifact' in args:
for name, data in members.items():
path = artifact / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)

def capture(*args):
if args[0] == 'node':
if args[-1] == '--json':
return json.dumps([lane])
return json.dumps(['docker', 'buildx', 'build', '--target=artifact', '.'])
if args[:3] == ('docker', 'image', 'inspect'):
return json.dumps([{'Id': 'fixture-image'}])
if args[0] == 'docker':
return 'fixture-tool-versions'
if args[:2] == ('git', 'ls-files'):
return self.capture(repository, *args[1:])
return real_capture(*args)

with patch.dict(os.environ, self.env, clear=True), patch.multiple(
artifacts, ROOT=repository, BASE=commits['B'], HISTORY_FLOOR=commits['F'],
RECIPE=['Dockerfile'], CONFIG={'lanes': [{'label': label}]}), patch.object(
artifacts, 'run', side_effect=run), patch.object(artifacts, 'capture', side_effect=capture):
artifacts.build(label, destination)

native = destination / (label + '.tar.gz')
sources = destination / 'bun-webkit-aot-sources.tar.gz'
declaration = 'bun-webkit/Source/JavaScriptCore/aot/builtins/generated/AOTBuiltinEngineDeclarations.json'
with tarfile.open(sources) as archive:
self.assertEqual(archive.extractfile(declaration).read(), generated)
names = archive.getnames()
self.assertTrue(all(name.startswith('bun-webkit/Source/JavaScriptCore/aot') for name in names))
self.assertFalse(any('/tests' in name for name in names))
self.assertFalse(any(name.startswith('bun-webkit/lib/') for name in names))
with tarfile.open(native) as archive:
self.assertEqual(archive.extractfile('bun-webkit/lib/libJavaScriptCore.a').read(), b'native-library')
self.assertEqual(archive.extractfile(declaration).read(), generated)
manifest = json.loads((destination / (label + '.manifest.json')).read_text())
expected = self.oracle(repository, commits, ['P', 'H'])
self.assertEqual({key: manifest[key] for key in expected}, expected)
files = {path.name: {'sha256': hashlib.sha256(path.read_bytes()).hexdigest(), 'size': path.stat().st_size}
for path in (native, sources)}
self.assertEqual(manifest['files'], files)
self.assertEqual((destination / 'SHA256SUMS').read_text(),
''.join(entry['sha256'] + ' ' + name + '\n' for name, entry in files.items()))
self.assertFalse(artifact.exists())

def test_complete_history_and_raw_patch_hashes(self):
repository, commits = self.graph('complete')
self.assertEqual(self.source(repository, commits['B'], commits['F']),
Expand Down
70 changes: 70 additions & 0 deletions .github/openclaw/test_release.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import importlib.util
import json
import io
import tarfile
import base64
import hashlib
import xml.etree.ElementTree as ET
Expand Down Expand Up @@ -137,6 +139,74 @@ def test_assembly_rechecks_raw_facts_after_plausible_redigested_summaries(self):

def test_complete_set_roundtrip(self):
self.assemble();self.assertEqual(len(release.verify(self.out,SOURCE)['artifacts']),10)

def test_aot_source_producer_roundtrip_and_integrity(self):
config=json.loads((release.ROOT/'.github/openclaw/variants.json').read_text())
label='bun-webkit-linux-amd64-lto-aot'
source_name='bun-webkit-aot-sources.tar.gz'
config['lanes'].append({'label':label})
root=self.root/'source'
config_path=root/'.github/openclaw/variants.json'
config_path.parent.mkdir(parents=True)
config_path.write_text(json.dumps(config))
directory=self.inputs/label
directory.mkdir();(directory/'provenance').mkdir()
declaration='bun-webkit/Source/JavaScriptCore/aot/builtins/generated/AOTBuiltinEngineDeclarations.json'
generated=b'{"origin":"built-engine-declarations"}\n'
for name,members in [
(label+'.tar.gz',{'bun-webkit/lib/libJavaScriptCore.a':b'native-library',declaration:generated}),
(source_name,{declaration:generated}),
]:
with tarfile.open(directory/name,'w:gz') as archive:
for member,data in members.items():
info=tarfile.TarInfo(member);info.size=len(data)
archive.addfile(info,io.BytesIO(data))
manifest={'source':SOURCE,'base':release.artifact_source.BASE,
'history_floor':release.artifact_source.HISTORY_FLOOR,'patches':PATCHES,
'lane':{'label':label,'sources':'bun-webkit-aot-sources'},
'files':{name:{'sha256':release.sha(directory/name),'size':(directory/name).stat().st_size}
for name in (label+'.tar.gz',source_name)}}
manifest_path=directory/(label+'.manifest.json')
manifest_path.write_text(json.dumps(manifest))
source_path=directory/source_name
original=source_path.read_bytes()

with patch.object(release,'ROOT',root):
release.assemble(self.inputs,self.out,SOURCE)
result=release.verify(self.out,SOURCE)
self.assertEqual(len(result['lanes']),len(config['lanes']))
self.assertEqual(len(result['artifacts']),len(config['lanes'])+1)
self.assertEqual(result['artifacts'][source_name],{
'url':f'https://github.com/openclaw/WebKit/releases/download/autobuild-{SOURCE}/{source_name}',
'sha256':hashlib.sha256(original).hexdigest(),'size':len(original)})
self.assertEqual((self.out/source_name).read_bytes(),original)
with tarfile.open(self.out/source_name) as archive:
self.assertEqual(archive.extractfile(declaration).read(),generated)
with tarfile.open(self.out/'provenance.tar.gz') as archive:
recorded=json.loads(archive.extractfile(label+'/'+label+'.manifest.json').read())
self.assertEqual(recorded,manifest)
for index,case in enumerate(('missing-file','missing-entry','changed-bytes','wrong-size','missing-sources','duplicate')):
with self.subTest(case=case):
source_path.write_bytes(original)
changed=json.loads(json.dumps(manifest))
expected=ValueError
if case=='missing-file':
source_path.unlink();expected=FileNotFoundError
elif case=='missing-entry':
del changed['files'][source_name]
elif case=='changed-bytes':
source_path.write_bytes(bytes([original[0]^1])+original[1:])
elif case=='wrong-size':
changed['files'][source_name]['size']+=1
elif case=='missing-sources':
del changed['lane']['sources']
elif case=='duplicate':
config['lanes'].append({'label':label})
config_path.write_text(json.dumps(config))
manifest_path.write_text(json.dumps(changed))
with self.assertRaises(expected):
release.assemble(self.inputs,self.root/('invalid-source-'+str(index)),SOURCE)

def test_changed_lane_bytes_rejected(self):
next(self.inputs.glob('*/*.tar.gz')).write_bytes(b'tampered')
with self.assertRaises(ValueError):self.assemble()
Expand Down