From a2eec1813a48051cd9978dadc29ec8cde289b75b Mon Sep 17 00:00:00 2001 From: davidmaamoaix Date: Mon, 26 Oct 2026 21:06:07 -0400 Subject: [PATCH 01/10] [stack] Type NativeStack.range as NativeStackSegment --- src/engine/native/NativeStack.v3 | 2 +- test/unittest/x86-64-linux/NativeStackManagerTest.v3 | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/engine/native/NativeStack.v3 b/src/engine/native/NativeStack.v3 index 46535641b..1cc55e8d7 100644 --- a/src/engine/native/NativeStack.v3 +++ b/src/engine/native/NativeStack.v3 @@ -6,7 +6,7 @@ // See diagram at the end of the file for mapping layout. class NativeStack extends WasmStack { def valuerep = Target.tagging; - var range: MemoryRange; + var range: NativeStackSegment; var vsp: Pointer; var rsp: Pointer; var func: Function; // TODO: move into "enter-func-stub" diff --git a/test/unittest/x86-64-linux/NativeStackManagerTest.v3 b/test/unittest/x86-64-linux/NativeStackManagerTest.v3 index fa8e7c109..2a9b6ba04 100644 --- a/test/unittest/x86-64-linux/NativeStackManagerTest.v3 +++ b/test/unittest/x86-64-linux/NativeStackManagerTest.v3 @@ -115,7 +115,7 @@ private class NativeStackManagerTester extends ExeTester { if (limit_total >= 0 && M.total > limit_total) t.fail3("%s: expected at most %d segments, got %d", where, limit_total, M.total); // Every stack in the cache owns its segment. for (s = M.cache; s != null; s = s.next_stack) { - var seg = NativeStackSegment.!(s.range); + var seg = s.range; if (seg.stack.get() != s) t.fail1("%s: cached stack does not own its segment", where); } } @@ -183,7 +183,7 @@ def EMPTY_STRUCT = StructDecl.new(false, ValueTypes.NO_HEAPTYPES, []); def test_fresh_empty(t: NativeStackManagerTester) { var s = t.fresh(); t.t.assert_eq(u64.view(EngineOptions.STACK_SIZE.get()), s.range.size()); - var seg = NativeStackSegment.!(s.range); + var seg = s.range; if (seg.stack.get() != s) t.t.fail("expected segment to point back at its stack"); t.checkInvariants("fresh"); } @@ -226,7 +226,7 @@ def test_weak_ref_survives_gc(t: NativeStackManagerTester) { Target.forceGC(); Target.forceGC(); for (s in stacks) { - var seg = NativeStackSegment.!(s.range); + var seg = s.range; if (seg.stack.get() != s) t.t.fail("expected weak reference to live stack to survive GC"); } t.checkInvariants("after GC"); @@ -265,7 +265,7 @@ def test_reclaim_skips_live(t: NativeStackManagerTester) { t.assertAllIn(stacks, dropped); t.assertNoneIn(stacks, live_starts); for (s in live) { - if (NativeStackSegment.!(s.range).stack.get() != s) t.t.fail("expected live stack to keep its segment"); + if (s.range.stack.get() != s) t.t.fail("expected live stack to keep its segment"); } t.checkInvariants("skips live"); } From 4e5a92ab12d1b7713c6b701486bb2ec92a753cbe Mon Sep 17 00:00:00 2001 From: davidmaamoaix Date: Mon, 26 Oct 2026 21:07:25 -0400 Subject: [PATCH 02/10] [stack] Add attach and detach for moving segments between stacks --- src/engine/native/NativeStack.v3 | 8 +-- src/engine/native/NativeStackManager.v3 | 31 +++++++--- src/engine/x86-64/V3Offsets.v3 | 2 +- .../x86-64-linux/NativeStackManagerTest.v3 | 56 +++++++++++++++++++ 4 files changed, 83 insertions(+), 14 deletions(-) diff --git a/src/engine/native/NativeStack.v3 b/src/engine/native/NativeStack.v3 index 1cc55e8d7..00dc6df7d 100644 --- a/src/engine/native/NativeStack.v3 +++ b/src/engine/native/NativeStack.v3 @@ -6,7 +6,7 @@ // See diagram at the end of the file for mapping layout. class NativeStack extends WasmStack { def valuerep = Target.tagging; - var range: NativeStackSegment; + var range: NativeStackSegment; // attached by {NativeStackManager.attach}, or null var vsp: Pointer; var rsp: Pointer; var func: Function; // TODO: move into "enter-func-stub" @@ -19,11 +19,8 @@ class NativeStack extends WasmStack { var handler_tier: HandlerTier; var next_stack: NativeStack; - new(range) { + new() { if (valuerep.tagged) RiGc.registerScanner(this, NativeStack.scan); - if (range == null) return; // allocated without a mapping, no need to initialize further - clear(); - if (Trace.stack && Debug.stack) Trace.OUT.put2("newStack start=0x%x, end=0x%x", range.start - Pointer.NULL, range.end - Pointer.NULL).ln(); } // Gets the state of this stack. def state() -> StackState { @@ -511,6 +508,7 @@ class NativeStack extends WasmStack { } // GC callback to scan (tagged) values on this stack def scan() { + if (range == null) return; // no segment attached if (Trace.gc) { Trace.OUT.put1("scanStack obj=0x%x", u64.view(Pointer.atObject(this) - Pointer.NULL)).ln(); } diff --git a/src/engine/native/NativeStackManager.v3 b/src/engine/native/NativeStackManager.v3 index 842f00d09..d6c7f8b3e 100644 --- a/src/engine/native/NativeStackManager.v3 +++ b/src/engine/native/NativeStackManager.v3 @@ -23,6 +23,27 @@ component NativeStackManager { result.next_stack = null; return result.clear(); } + // Makes {stack} the empty owner of {segment}, which no other stack may own. + def attach(stack: NativeStack, segment: NativeStackSegment) { + stack.range = segment; + segment.stack.set(stack); + stack.clear(); + if (Trace.stack && Debug.stack) Trace.OUT.put2("attachSegment start=0x%x, end=0x%x", segment.start - Pointer.NULL, segment.end - Pointer.NULL).ln(); + } + // Takes the segment away from {stack}, which must not be used until another is attached. + def detach(stack: NativeStack) -> NativeStackSegment { + var segment = stack.range; + segment.stack.set(null); + stack.range = null; + return segment; + } + // Wraps {segment} in a new cached stack that owns it. + private def addToCache(segment: NativeStackSegment) { + var stack = NativeStack.new(); + attach(stack, segment); + stack.next_stack = cache; + cache = stack; + } private def tryAllocatingNewStacks() -> bool { var size = EngineOptions.STACK_SIZE.get(); var batch = Ints.min(StackTuning.stackBatchSize, StackTuning.maxStacks - total); @@ -40,10 +61,7 @@ component NativeStackManager { // insert redzone roughly in the middle of the mapping. var rzStart = t.0 + ((size >> 1) & ~(PAGE_SIZE - 1u)), rzEnd = rzStart + PAGE_SIZE; if (!RedZones.addRedZone(rzStart, rzEnd)) fatal("could not protect value stack red zone"); - var curr = NativeStack.new(segment); - segment.stack.set(curr); - curr.next_stack = cache; - cache = curr; + addToCache(segment); total++; } return cache != null; // OK if succeeded in allocating at least one @@ -52,10 +70,7 @@ component NativeStackManager { var ok = false; for (p = all; p != null; p = p.next) { if (p.stack.get() == null) { // weak reference was cleared by GC, can reclaim - var s = NativeStack.new(p); - p.stack.set(s); - s.next_stack = cache; - cache = s; + addToCache(p); ok = true; } } diff --git a/src/engine/x86-64/V3Offsets.v3 b/src/engine/x86-64/V3Offsets.v3 index a32a2cbd4..983123e29 100644 --- a/src/engine/x86-64/V3Offsets.v3 +++ b/src/engine/x86-64/V3Offsets.v3 @@ -15,7 +15,7 @@ class V3Offsets { private def t = Table.new(ValueTypes.FUNCREF, TableDecl.new(ValueTypes.FUNCREF, SizeConstraint(false, 0, Max.Set(1)))); private def wf = WasmFunction.new(i, decl); private def mem = NativeWasmMemory.new(null); - private def vs = NativeStack.new(null); + private def vs = NativeStack.new(); private def acc = NativeFrameAccessor.new(vs, NativeFrameHandle.Interpreter(Pointer.NULL), decl); private def cnt = CountProbe.new(); private def metric = Metric.new("", "", ""); diff --git a/test/unittest/x86-64-linux/NativeStackManagerTest.v3 b/test/unittest/x86-64-linux/NativeStackManagerTest.v3 index 2a9b6ba04..f5fc22a51 100644 --- a/test/unittest/x86-64-linux/NativeStackManagerTest.v3 +++ b/test/unittest/x86-64-linux/NativeStackManagerTest.v3 @@ -30,6 +30,9 @@ def X_ = void( T("run_on_fresh_stack_gc", test_run_on_fresh_stack_gc), T("out_of_stacks_host", test_out_of_stacks_host), T("out_of_stacks_cont_new", test_out_of_stacks_cont_new), + T("attach_detach", test_attach_detach), + T("moved_segment_kept", test_moved_segment_kept), + T("reattach_runs", test_reattach_runs), () ); @@ -95,6 +98,25 @@ private class NativeStackManagerTester extends ExeTester { } return r; } + // Moves a fresh stack's segment to a new stack and drops the fresh stack, returning the new owner. + def moveSegmentToNewStack() -> NativeStack #no-inline { + var a = fresh(); + var b = NativeStack.new(); + M.attach(b, M.detach(a)); + if (a.range != null) t.fail("expected detached stack to have no segment"); + return b; + } + // Checks that a moved segment is kept while its new owner lives, then drops the new owner. + def checkMovedSegment() -> NativeStackSegment #no-inline { + var b = moveSegmentToNewStack(); + var seg = b.range; + Target.forceGC(); + Target.forceGC(); + if (seg.stack.get() != b) t.fail("expected moved segment to be owned by the new stack"); + if (M.getFreshStack() != null) t.fail("expected no segment to be reclaimable while the new owner lives"); + if (b.range != seg || seg.stack.get() != b) t.fail("expected the new owner to keep the segment"); + return seg; + } // Checks global invariants of the manager. def checkInvariants(where: string) { var count = 0, size = EngineOptions.STACK_SIZE.get(); @@ -391,6 +413,40 @@ def test_out_of_stacks_host(t: NativeStackManagerTester) { t.assertResult(Result.Value([Values.box_i(7)]), M.runOnFreshStack(hf, [])); } +def test_attach_detach(t: NativeStackManagerTester) { + var s = t.fresh(); + var seg = s.range; + if (M.detach(s) != seg) t.t.fail("expected detach to return the attached segment"); + if (s.range != null) t.t.fail("expected detached stack to have no segment"); + if (seg.stack.get() != null) t.t.fail("expected detached segment to have no owner"); + M.attach(s, seg); + if (s.range != seg) t.t.fail("expected attach to set the segment"); + if (seg.stack.get() != s) t.t.fail("expected attach to set the owner"); + if (s.state() != StackState.EMPTY || s.vsp != seg.start || s.rsp != seg.end) t.t.fail("expected attach to leave the stack empty"); + t.checkInvariants("attach"); +} + +def test_moved_segment_kept(t: NativeStackManagerTester) { + // A segment moved to another stack is reclaimed only when the new owner dies. + t.limit(1); + var seg = t.checkMovedSegment(); + Target.forceGC(); + if (seg.stack.get() != null) t.t.fail("expected the moved segment to be unowned after its new owner died"); + var s = t.fresh(); + if (s.range != seg) t.t.fail("expected the moved segment to be reclaimed"); + t.checkInvariants("moved"); +} + +def test_reattach_runs(t: NativeStackManagerTester) { + var f = t.subFunc(); + var s = t.fresh(); + var seg = s.range; + for (i < 3) M.attach(s, M.detach(s)); + if (s.range != seg || seg.stack.get() != s) t.t.fail("expected reattached stack to own its segment"); + t.assertResult(Result.Value([Values.box_i(2)]), s.reset(f).bind([Values.box_i(7), Values.box_i(5)]).resume()); + t.checkInvariants("reattach"); +} + def test_out_of_stacks_cont_new(t: NativeStackManagerTester) { StackTuning.enableCompression = false; var make = t.makeFunc(); From 5ea06df6c7b9e1b277d317d6db7780776de4b0d0 Mon Sep 17 00:00:00 2001 From: davidmaamoaix Date: Mon, 26 Oct 2026 21:53:35 -0400 Subject: [PATCH 03/10] [stack] Add compression and decompression of suspended native stacks --- src/engine/WasmStack.v3 | 1 + src/engine/native/NativeStack.v3 | 13 +- src/engine/native/NativeStackCompression.v3 | 110 +++ src/engine/native/NativeStackManager.v3 | 11 + src/engine/x86-64/X86_64Runtime.v3 | 2 + .../NativeStackCompressionTest.v3 | 624 ++++++++++++++++++ 6 files changed, 760 insertions(+), 1 deletion(-) create mode 100644 src/engine/native/NativeStackCompression.v3 create mode 100644 test/unittest/x86-64-linux/NativeStackCompressionTest.v3 diff --git a/src/engine/WasmStack.v3 b/src/engine/WasmStack.v3 index ad1e6bfbd..d5bcbfb20 100644 --- a/src/engine/WasmStack.v3 +++ b/src/engine/WasmStack.v3 @@ -98,6 +98,7 @@ enum StackState { SUSPENDED, // contains one or more frames, waiting for inputs CALL_CHILD, // waiting for child stack to return RUNNING, // currently running Wasm or host code + COMPRESSED, // suspended, with its frames or bound values held in the heap } // Provides access to the state of an executing frame. diff --git a/src/engine/native/NativeStack.v3 b/src/engine/native/NativeStack.v3 index 00dc6df7d..5927705a4 100644 --- a/src/engine/native/NativeStack.v3 +++ b/src/engine/native/NativeStack.v3 @@ -13,6 +13,8 @@ class NativeStack extends WasmStack { var parent_rsp_ptr: Pointer; var return_results: Array; var state_: StackState = StackState.EMPTY; + var compressed: CompressedStack; // frames or bound entry values while {COMPRESSED} + var saved_parent_rsp: Pointer; // contents of the parent rsp slot while {COMPRESSED} // Handler table (for inline suspend/switch). var handlers: Array; @@ -44,10 +46,16 @@ class NativeStack extends WasmStack { def resetForFrameWrites(func: Function) -> this { reset0(func, 0); } + // Returns {true} if this stack has no frames yet and will enter {func} when resumed. + def isAtFuncEntry() -> bool { + return range != null && rsp < range.end && rsp.load() == STACK_ENTER_FUNC_STUB.getEntry() + NOP_ENTRY; + } // Requires {state == SUSPENDED}. // Pushes {args} incrementally onto the value stack. def bind(args: Range) -> this { if (args.length == 0) return; + if (NativeStackCompression.bindEntry(this, args)) return; + if (!NativeStackCompression.decompress(this)) fatal("out of stacks while binding a compressed stack"); checkState("bind()", StackState.SUSPENDED); for (v in args) push(v); } @@ -55,6 +63,7 @@ class NativeStack extends WasmStack { // Resumes running the Wasm or host code in this stack until that code either returns, throws, // or suspends itself. def resume() -> Result { + if (!NativeStackCompression.decompress(this)) return Result.Throw(Trap.new(TrapReason.OUT_OF_STACKS, null, null)); checkState("resume()", StackState.SUSPENDED); if (Trace.stack) traceFrames("stack.resume", rsp + RETADDR_SIZE); var bottom = NativeStack.!(this.bottom); @@ -550,6 +559,7 @@ class NativeStack extends WasmStack { BpTypeCode.ARRAYREF.code, BpTypeCode.FUNCREF.code, BpTypeCode.EXNREF.code, + BpTypeCode.NULLEXNREF.code, BpTypeCode.NULLFUNCREF.code => return Value.Ref(vp.load()); BpTypeCode.I32.code => return Value.I32(vp.load()); @@ -557,7 +567,8 @@ class NativeStack extends WasmStack { BpTypeCode.F32.code => return Value.F32(vp.load()); BpTypeCode.F64.code => return Value.F64(vp.load()); BpTypeCode.V128.code => return Value.V128(vp.load(), (vp + 8).load()); - BpTypeCode.CONTREF.code => { + BpTypeCode.CONTREF.code, + BpTypeCode.NULLCONTREF.code => { var stack = vp.load(); var version = (vp + 8).load(); return Value.Cont(Continuation(stack, version)); diff --git a/src/engine/native/NativeStackCompression.v3 b/src/engine/native/NativeStackCompression.v3 new file mode 100644 index 000000000..057e8d758 --- /dev/null +++ b/src/engine/native/NativeStackCompression.v3 @@ -0,0 +1,110 @@ +// Copyright 2026 Wizard authors. All rights reserved. +// See LICENSE for details of Apache 2.0 license. + +// Values bound to the function of a {COMPRESSED} stack that has not been entered. +class EntryValues(vals: Array) extends CompressedStack { + def name() -> string { return "Entry Values"; } + def size() -> u64 { return 32uL + 24uL * u64.view(vals.length); } +} + +// Compresses suspended stacks into the heap to free their segments, and decompresses them before use. +component NativeStackCompression { + var numCompressed: int; + var numDecompressed: int; + + // Returns {true} if {stack} is suspended at a function entry or with only interpreter and SPC frames. + def canCompress(stack: NativeStack) -> bool { + if (stack.state_ != StackState.SUSPENDED) return false; + var range = stack.range; + if (range == null) return false; + var end = range.end; + // [end - 8] holds the parent's rsp, [end - 16] the return-parent stub. + if (stack.parent_rsp_ptr != end + -Pointer.SIZE) return false; + if (stack.isAtFuncEntry()) return stack.rsp == end + -3 * Pointer.SIZE; + var srp = stack.rsp, count = 0, bottom_sfp = Pointer.NULL; + while (srp >= range.start && srp < end) { + var retip = srp.load(); + var code = RiRuntime.findUserCode(retip); + match (code) { + x: X86_64InterpreterCode, x: X86_64SpcModuleCode => { + count++; + bottom_sfp = srp + Pointer.SIZE; + srp = code.nextFrame(retip, srp).1; + } + x: X86_64ReturnParentStub => { + if (count == 0 || srp != end + -2 * Pointer.SIZE) return false; + // Values below the bottom frame would be lost. + return NativeFrameHandle.Interpreter(bottom_sfp).vfp() == range.start; + } + _ => return false; // inlined frames, stubs, or host frames + } + } + return false; + } + // Compresses {stack}, which must satisfy {canCompress}, and returns its now unowned segment. + def compress(stack: NativeStack) -> NativeStackSegment { + var parent_rsp = stack.parent_rsp_ptr.load(); + var c: CompressedStack; + if (stack.isAtFuncEntry()) { + var n = int.!((stack.vsp - stack.range.start) / stack.valuerep.slot_size); + if (n > 0) { + var vals = Array.new(n); + for (i < n) vals[i] = stack.readValue(stack.range.start, i); + c = EntryValues.new(vals); + } + } else { + c = StackCompression.compressStack(stack); + } + var segment = NativeStackManager.detach(stack); + if (Trace.stack) Trace.OUT.put2("compress stack @ 0x%x, freeing segment 0x%x", + Pointer.atObject(stack) - Pointer.NULL, segment.start - Pointer.NULL).ln(); + stack.vsp = stack.rsp = Pointer.NULL; + stack.parent_rsp_ptr = Pointer.NULL; + stack.saved_parent_rsp = parent_rsp; + stack.compressed = c; + stack.state_ = StackState.COMPRESSED; + numCompressed++; + return segment; + } + // Binds {args} to a {COMPRESSED} stack that has not been entered, without decompressing it. + def bindEntry(stack: NativeStack, args: Range) -> bool { + if (stack.state_ != StackState.COMPRESSED) return false; + var vals: Array; + match (stack.compressed) { + null => vals = Ranges.dup(args); + x: EntryValues => vals = Arrays.concat(x.vals, Ranges.dup(args)); + _ => return false; + } + stack.compressed = EntryValues.new(vals); + return true; + } + // Decompresses {stack}, if compressed, onto a fresh segment; returns {false} if none is available. + def decompress(stack: NativeStack) -> bool { + if (stack.state_ != StackState.COMPRESSED) return true; + var segment = NativeStackManager.acquireSegment(); + if (segment == null) return false; + var c = stack.compressed, parent_rsp = stack.saved_parent_rsp; + var func = stack.func, results = stack.return_results, parent = stack.parent, bottom = stack.bottom; + if (Trace.stack) Trace.OUT.put2("decompress stack @ 0x%x onto segment 0x%x", + Pointer.atObject(stack) - Pointer.NULL, segment.start - Pointer.NULL).ln(); + NativeStackManager.attach(stack, segment); + match (c) { + null => stack.reset(func); + x: EntryValues => { + stack.reset(func); + for (v in x.vals) stack.push(v); + } + _ => Target.writeFramesToStack(stack, StackCompression.decompressFrames(c)); + } + stack.func = func; + stack.return_results = results; + stack.parent = parent; + stack.bottom = bottom; + stack.parent_rsp_ptr.store(parent_rsp); + stack.saved_parent_rsp = Pointer.NULL; + stack.compressed = null; + stack.state_ = StackState.SUSPENDED; + numDecompressed++; + return true; + } +} diff --git a/src/engine/native/NativeStackManager.v3 b/src/engine/native/NativeStackManager.v3 index d6c7f8b3e..cd312c33b 100644 --- a/src/engine/native/NativeStackManager.v3 +++ b/src/engine/native/NativeStackManager.v3 @@ -37,6 +37,17 @@ component NativeStackManager { stack.range = null; return segment; } + // Compresses {stack} if possible, adding its segment to the cache. + def compressStack(stack: NativeStack) -> bool { + if (!NativeStackCompression.canCompress(stack)) return false; + addToCache(NativeStackCompression.compress(stack)); + return true; + } + // Gets an unowned segment, which must be attached before the next refill, or {null} if out of stacks. + def acquireSegment() -> NativeStackSegment { + var s = getFreshStack(); + return if(s == null, null, detach(s)); // {s} becomes garbage + } // Wraps {segment} in a new cached stack that owns it. private def addToCache(segment: NativeStackSegment) { var stack = NativeStack.new(); diff --git a/src/engine/x86-64/X86_64Runtime.v3 b/src/engine/x86-64/X86_64Runtime.v3 index 5610a9832..09c615011 100644 --- a/src/engine/x86-64/X86_64Runtime.v3 +++ b/src/engine/x86-64/X86_64Runtime.v3 @@ -129,6 +129,7 @@ component X86_64Runtime { .put2("resume_throw %q @ contStack[addr=0x%x] ", ex.render, Pointer.atObject(contStack) - Pointer.NULL) .put1("on stack[rsp=0x%x]", stack.rsp - Pointer.NULL).ln(); + if (!NativeStackCompression.decompress(contStack)) return stack.trap(TrapReason.OUT_OF_STACKS); // search for handler in continuation var result = contStack.throw(ex); match (result) { @@ -209,6 +210,7 @@ component X86_64Runtime { if (Continuations.isNull(target_cont)) return stack.trap(TrapReason.NULL_DEREF); if (Continuations.isUsed(target_cont)) return stack.trap(TrapReason.USED_CONTINUATION); var target_stack = target_cont.stack; + if (!NativeStackCompression.decompress(NativeStack.!(target_stack))) return stack.trap(TrapReason.OUT_OF_STACKS); Continuations.setUsed(target_cont); var this_cont = stack.unwindChain(tag, WasmStack.tryHandleSwitch); diff --git a/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 b/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 new file mode 100644 index 000000000..9ac251206 --- /dev/null +++ b/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 @@ -0,0 +1,624 @@ +// Copyright 2026 Wizard authors. All rights reserved. +// See LICENSE for details of Apache 2.0 license. + +// Tests for compressing suspended native stacks and resuming them from host code in every tier. + +def M: NativeStackManager; +def C: NativeStackCompression; + +def T(name: string, f: StackCompressionTester -> void) { + UnitTests.registerT("x86stackcomp:", name, StackCompressionTester.new(_, X86_64InterpreterOnlyStrategy.new()), runTest(f, _)); +} +def TT(name: string, f: StackCompressionTester -> void) { + for (l = Target.getTestTiers(); l != null; l = l.tail) { + UnitTests.registerT(Strings.format1("x86stackcomp:%s", l.head.0), name, StackCompressionTester.new(_, l.head.1), runTest(f, _)); + } +} +def X_ = void( + T("can_compress_states", test_can_compress_states), + T("roundtrip_entry", test_roundtrip_entry), + T("roundtrip_entry_bind", test_roundtrip_entry_bind), + T("roundtrip_refs_gc", test_roundtrip_refs_gc), + T("no_frames_not_compressible", test_no_frames_not_compressible), + T("running_not_compressible", test_running_not_compressible), + T("entry_host_func", test_entry_host_func), + T("repeat_roundtrip", test_repeat_roundtrip), + T("acquire_segment", test_acquire_segment), + T("decompress_out_of_stacks", test_decompress_out_of_stacks), + T("version_kept", test_version_kept), + TT("mid_host_resume", test_mid_host_resume), + TT("mid_host_bind", test_mid_host_bind), + TT("mid_twice", test_mid_twice), + TT("mid_gc", test_mid_gc), + TT("gen_frames", test_gen_frames), + TT("deep_frames", test_deep_frames), + TT("chain_top_roundtrip", test_chain_top_roundtrip), + TT("trap_after_decompress", test_trap_after_decompress), + TT("throw_after_decompress", test_throw_after_decompress), + () +); + +// The module below, compiled with the stack-switching reference interpreter. +// (module +// (type $fv (func)) +// (type $cv (cont $fv)) +// (type $fi (func (param i32))) +// (type $ci (cont $fi)) +// (type $f_i (func (result i32))) +// (type $c_i (cont $f_i)) +// (type $fi_i (func (param i32) (result i32))) +// (type $ci_i (cont $fi_i)) +// (type $box (struct (field i32))) +// +// (tag $yield (param i32)) +// (tag $ask (result i32)) +// +// (global $last (mut i32) (i32.const -1)) +// +// ;; Recurses {n} frames deep, each holding a v128 and a struct, then yields {v}. +// (func $deep (param $n i32) (param $v i32) +// (local $x v128) (local $b (ref null $box)) +// (local.set $x (i32x4.splat (local.get $v))) +// (local.set $b (struct.new $box (local.get $v))) +// (if (i32.eqz (local.get $n)) +// (then (suspend $yield (local.get $v))) +// (else (call $deep (i32.sub (local.get $n) (i32.const 1)) (local.get $v)))) +// (if (i32.ne (i32x4.extract_lane 3 (local.get $x)) (local.get $v)) (then (unreachable))) +// (if (i32.ne (struct.get $box 0 (local.get $b)) (local.get $v)) (then (unreachable))) +// ) +// ;; Yields {start}, {start+1}, ..., {start+9} from 3 frames deep, checking i64 and f64 locals. +// (func $gen (param $start i32) +// (local $i i32) (local $l i64) (local $d f64) (local $v i32) +// (loop $loop +// (local.set $v (i32.add (local.get $start) (local.get $i))) +// (call $deep (i32.const 3) (local.get $v)) +// (local.set $l (i64.add (local.get $l) (i64.extend_i32_s (local.get $v)))) +// (local.set $d (f64.add (local.get $d) (f64.convert_i32_s (local.get $v)))) +// (if (i64.ne (local.get $l) (i64.trunc_f64_s (local.get $d))) (then (unreachable))) +// (local.set $i (i32.add (local.get $i) (i32.const 1))) +// (br_if $loop (i32.lt_u (local.get $i) (i32.const 10))) +// ) +// ) +// ;; Yields once, then returns {x + 1000}. +// (func $once (param $x i32) (result i32) +// (suspend $yield (local.get $x)) +// (i32.add (local.get $x) (i32.const 1000)) +// ) +// ;; Asks for a value, then returns {x * 10 + answer}. +// (func $asker (param $x i32) (result i32) +// (i32.add (i32.mul (local.get $x) (i32.const 10)) (suspend $ask)) +// ) +// (elem declare func $gen $once $asker) +// +// ;; Resumes {k} until it yields (stores the value in $last and returns the new continuation) or returns (null). +// (func $step (param $k (ref $cv)) (result (ref null $cv)) +// (local $next (ref null $cv)) +// (block $h (result i32 (ref $cv)) +// (resume $cv (on $yield $h) (local.get $k)) +// (return (ref.null $cv)) +// ) +// (local.set $next) +// (global.set $last) +// (local.get $next) +// ) +// (func (export "make") (param $start i32) (result (ref null $cv)) +// (call $step (cont.bind $ci $cv (local.get $start) (cont.new $ci (ref.func $gen)))) +// ) +// (func (export "next") (param $k (ref null $cv)) (result (ref null $cv)) +// (call $step (ref.as_non_null (local.get $k))) +// ) +// (func (export "last") (result i32) (global.get $last)) +// +// ;; Returns a continuation of {$once} suspended at its yield. +// (func (export "make_once") (param $x i32) (result (ref null $c_i)) +// (local $k (ref null $c_i)) +// (block $h (result i32 (ref $c_i)) +// (drop (resume $ci_i (on $yield $h) (local.get $x) (cont.new $ci_i (ref.func $once)))) +// (return (ref.null $c_i)) +// ) +// (local.set $k) +// (drop) +// (local.get $k) +// ) +// ;; Returns a continuation of {$asker} suspended at its question. +// (func (export "make_asker") (param $x i32) (result (ref null $ci_i)) +// (block $h (result (ref $ci_i)) +// (drop (resume $ci_i (on $ask $h) (local.get $x) (cont.new $ci_i (ref.func $asker)))) +// (return (ref.null $ci_i)) +// ) +// ) +// ;; Answers the question of {k} with {v} via cont.bind, then resumes it. +// (func (export "answer") (param $k (ref null $ci_i)) (param $v i32) (result i32) +// (resume $c_i (cont.bind $ci_i $c_i (local.get $v) (local.get $k))) +// ) +// ;; --- Chains, deep frames, traps, and exceptions --- +// (tag $other) +// (tag $exn (param i32)) +// ;; Suspends with {yield}, which its parent {mid} does not handle, then returns {2 * x}. +// (func $inner (param $x i32) (result i32) +// (suspend $yield (local.get $x)) +// (i32.mul (local.get $x) (i32.const 2)) +// ) +// ;; Resumes {inner} with a handler for an unrelated tag, then returns its result + 1. +// (func $mid (param $x i32) (result i32) +// (block $h (result (ref $c_i)) +// (return (i32.add (i32.const 1) +// (resume $ci_i (on $other $h) (local.get $x) (cont.new $ci_i (ref.func $inner))))) +// ) +// (unreachable) +// ) +// ;; Recurses {d} frames deep, each with a local, then yields {x}, then returns the sum of locals. +// (func $deep_sum (param $d i32) (param $x i32) (result i32) +// (local $mine i32) +// (local.set $mine (i32.add (local.get $d) (local.get $x))) +// (if (result i32) (i32.eqz (local.get $d)) +// (then (suspend $yield (local.get $x)) (local.get $mine)) +// (else (i32.add (local.get $mine) (call $deep_sum (i32.sub (local.get $d) (i32.const 1)) (local.get $x))))) +// ) +// (func $deep_entry (param $x i32) (result i32) +// (call $deep_sum (i32.const 100) (local.get $x)) +// ) +// ;; Yields, then traps. +// (func $trapper (param $x i32) (result i32) +// (suspend $yield (local.get $x)) +// (unreachable) +// ) +// ;; Yields, then throws {x}. +// (func $thrower (param $x i32) (result i32) +// (suspend $yield (local.get $x)) +// (throw $exn (local.get $x)) +// ) +// (elem declare func $inner $mid $deep_entry $trapper $thrower) +// +// ;; Runs {f(x)} until it yields, returning the suspended continuation. +// (func $start (param $f (ref $ci_i)) (param $x i32) (result (ref null $c_i)) +// (local $k (ref null $c_i)) +// (block $h (result i32 (ref $c_i)) +// (drop (resume $ci_i (on $yield $h) (local.get $x) (local.get $f))) +// (return (ref.null $c_i)) +// ) +// (local.set $k) +// (global.set $last) +// (local.get $k) +// ) +// ;; Returns a continuation whose top stack is {inner} and whose bottom stack is {mid}. +// (func (export "make_chain") (param $x i32) (result (ref null $c_i)) +// (call $start (cont.new $ci_i (ref.func $mid)) (local.get $x)) +// ) +// (func (export "make_deep") (param $x i32) (result (ref null $c_i)) +// (call $start (cont.new $ci_i (ref.func $deep_entry)) (local.get $x)) +// ) +// (func (export "make_trapper") (param $x i32) (result (ref null $c_i)) +// (call $start (cont.new $ci_i (ref.func $trapper)) (local.get $x)) +// ) +// (func (export "make_thrower") (param $x i32) (result (ref null $c_i)) +// (call $start (cont.new $ci_i (ref.func $thrower)) (local.get $x)) +// ) +// ;; Resumes {k} to completion. +// (func (export "finish") (param $k (ref null $c_i)) (result i32) +// (resume $c_i (local.get $k)) +// ) +// ;; Resumes {k} to completion, returning the payload of an exception it throws. +// (func (export "catch") (param $k (ref null $c_i)) (result i32) +// (block $c (result i32) +// (try_table (catch $exn $c) (drop (resume $c_i (local.get $k)))) +// (i32.const -1) +// ) +// ) +// (func (export "resume_null") (result i32) +// (resume $c_i (ref.null $c_i)) +// ) +// ) +def GEN_MODULE: Array = [ + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0xe9, 0x80, 0x80, 0x80, 0x00, 0x15, 0x60, + 0x00, 0x00, 0x5d, 0x00, 0x60, 0x01, 0x7f, 0x00, 0x5d, 0x02, 0x60, 0x00, 0x01, 0x7f, 0x5d, 0x04, + 0x60, 0x01, 0x7f, 0x01, 0x7f, 0x5d, 0x06, 0x5f, 0x01, 0x7f, 0x00, 0x60, 0x02, 0x7f, 0x7f, 0x00, + 0x60, 0x01, 0x64, 0x01, 0x01, 0x63, 0x01, 0x60, 0x00, 0x02, 0x7f, 0x64, 0x01, 0x60, 0x01, 0x7f, + 0x01, 0x63, 0x01, 0x60, 0x01, 0x63, 0x01, 0x01, 0x63, 0x01, 0x60, 0x01, 0x7f, 0x01, 0x63, 0x05, + 0x60, 0x00, 0x02, 0x7f, 0x64, 0x05, 0x60, 0x01, 0x7f, 0x01, 0x63, 0x07, 0x60, 0x02, 0x63, 0x07, + 0x7f, 0x01, 0x7f, 0x60, 0x02, 0x7f, 0x7f, 0x01, 0x7f, 0x60, 0x02, 0x64, 0x07, 0x7f, 0x01, 0x63, + 0x05, 0x60, 0x01, 0x63, 0x05, 0x01, 0x7f, 0x03, 0x9a, 0x80, 0x80, 0x80, 0x00, 0x19, 0x09, 0x02, + 0x06, 0x06, 0x0a, 0x0c, 0x0d, 0x04, 0x0e, 0x10, 0x11, 0x06, 0x06, 0x12, 0x06, 0x06, 0x06, 0x13, + 0x0e, 0x0e, 0x0e, 0x0e, 0x14, 0x14, 0x04, 0x0d, 0x89, 0x80, 0x80, 0x80, 0x00, 0x04, 0x00, 0x02, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x06, 0x86, 0x80, 0x80, 0x80, 0x00, 0x01, 0x7f, 0x01, 0x41, + 0x7f, 0x0b, 0x07, 0x8e, 0x81, 0x80, 0x80, 0x00, 0x0d, 0x04, 0x6d, 0x61, 0x6b, 0x65, 0x00, 0x05, + 0x04, 0x6e, 0x65, 0x78, 0x74, 0x00, 0x06, 0x04, 0x6c, 0x61, 0x73, 0x74, 0x00, 0x07, 0x09, 0x6d, + 0x61, 0x6b, 0x65, 0x5f, 0x6f, 0x6e, 0x63, 0x65, 0x00, 0x08, 0x0a, 0x6d, 0x61, 0x6b, 0x65, 0x5f, + 0x61, 0x73, 0x6b, 0x65, 0x72, 0x00, 0x09, 0x06, 0x61, 0x6e, 0x73, 0x77, 0x65, 0x72, 0x00, 0x0a, + 0x0a, 0x6d, 0x61, 0x6b, 0x65, 0x5f, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x00, 0x12, 0x09, 0x6d, 0x61, + 0x6b, 0x65, 0x5f, 0x64, 0x65, 0x65, 0x70, 0x00, 0x13, 0x0c, 0x6d, 0x61, 0x6b, 0x65, 0x5f, 0x74, + 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x00, 0x14, 0x0c, 0x6d, 0x61, 0x6b, 0x65, 0x5f, 0x74, 0x68, + 0x72, 0x6f, 0x77, 0x65, 0x72, 0x00, 0x15, 0x06, 0x66, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x00, 0x16, + 0x05, 0x63, 0x61, 0x74, 0x63, 0x68, 0x00, 0x17, 0x0b, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x5f, + 0x6e, 0x75, 0x6c, 0x6c, 0x00, 0x18, 0x09, 0x8f, 0x80, 0x80, 0x80, 0x00, 0x02, 0x03, 0x00, 0x03, + 0x01, 0x02, 0x03, 0x03, 0x00, 0x05, 0x0b, 0x0c, 0x0e, 0x0f, 0x10, 0x0a, 0xcd, 0x84, 0x80, 0x80, + 0x00, 0x19, 0xc1, 0x80, 0x80, 0x80, 0x00, 0x02, 0x01, 0x7b, 0x01, 0x63, 0x08, 0x20, 0x01, 0xfd, + 0x11, 0x21, 0x02, 0x20, 0x01, 0xfb, 0x00, 0x08, 0x21, 0x03, 0x20, 0x00, 0x45, 0x04, 0x40, 0x20, + 0x01, 0xe2, 0x00, 0x05, 0x20, 0x00, 0x41, 0x01, 0x6b, 0x20, 0x01, 0x10, 0x00, 0x0b, 0x20, 0x02, + 0xfd, 0x1b, 0x03, 0x20, 0x01, 0x47, 0x04, 0x40, 0x00, 0x0b, 0x20, 0x03, 0xfb, 0x02, 0x08, 0x00, + 0x20, 0x01, 0x47, 0x04, 0x40, 0x00, 0x0b, 0x0b, 0xc2, 0x80, 0x80, 0x80, 0x00, 0x04, 0x01, 0x7f, + 0x01, 0x7e, 0x01, 0x7c, 0x01, 0x7f, 0x03, 0x40, 0x20, 0x00, 0x20, 0x01, 0x6a, 0x21, 0x04, 0x41, + 0x03, 0x20, 0x04, 0x10, 0x00, 0x20, 0x02, 0x20, 0x04, 0xac, 0x7c, 0x21, 0x02, 0x20, 0x03, 0x20, + 0x04, 0xb7, 0xa0, 0x21, 0x03, 0x20, 0x02, 0x20, 0x03, 0xb0, 0x52, 0x04, 0x40, 0x00, 0x0b, 0x20, + 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x20, 0x01, 0x41, 0x0a, 0x49, 0x0d, 0x00, 0x0b, 0x0b, 0x8c, + 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x00, 0xe2, 0x00, 0x20, 0x00, 0x41, 0xe8, 0x07, 0x6a, 0x0b, + 0x8a, 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x00, 0x41, 0x0a, 0x6c, 0xe2, 0x01, 0x6a, 0x0b, 0x99, + 0x80, 0x80, 0x80, 0x00, 0x01, 0x01, 0x63, 0x01, 0x02, 0x0b, 0x20, 0x00, 0xe3, 0x01, 0x01, 0x00, + 0x00, 0x00, 0xd0, 0x01, 0x0f, 0x0b, 0x21, 0x01, 0x24, 0x00, 0x20, 0x01, 0x0b, 0x8d, 0x80, 0x80, + 0x80, 0x00, 0x00, 0x20, 0x00, 0xd2, 0x01, 0xe0, 0x03, 0xe1, 0x03, 0x01, 0x10, 0x04, 0x0b, 0x87, + 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x00, 0xd4, 0x10, 0x04, 0x0b, 0x84, 0x80, 0x80, 0x80, 0x00, + 0x00, 0x23, 0x00, 0x0b, 0x9d, 0x80, 0x80, 0x80, 0x00, 0x01, 0x01, 0x63, 0x05, 0x02, 0x0f, 0x20, + 0x00, 0xd2, 0x02, 0xe0, 0x07, 0xe3, 0x07, 0x01, 0x00, 0x00, 0x00, 0x1a, 0xd0, 0x05, 0x0f, 0x0b, + 0x21, 0x01, 0x1a, 0x20, 0x01, 0x0b, 0x96, 0x80, 0x80, 0x80, 0x00, 0x00, 0x02, 0x64, 0x07, 0x20, + 0x00, 0xd2, 0x03, 0xe0, 0x07, 0xe3, 0x07, 0x01, 0x00, 0x01, 0x00, 0x1a, 0xd0, 0x07, 0x0f, 0x0b, + 0x0b, 0x8c, 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x01, 0x20, 0x00, 0xe1, 0x07, 0x05, 0xe3, 0x05, + 0x00, 0x0b, 0x8b, 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x00, 0xe2, 0x00, 0x20, 0x00, 0x41, 0x02, + 0x6c, 0x0b, 0x97, 0x80, 0x80, 0x80, 0x00, 0x00, 0x02, 0x64, 0x05, 0x41, 0x01, 0x20, 0x00, 0xd2, + 0x0b, 0xe0, 0x07, 0xe3, 0x07, 0x01, 0x00, 0x02, 0x00, 0x6a, 0x0f, 0x0b, 0x00, 0x0b, 0xa4, 0x80, + 0x80, 0x80, 0x00, 0x01, 0x01, 0x7f, 0x20, 0x00, 0x20, 0x01, 0x6a, 0x21, 0x02, 0x20, 0x00, 0x45, + 0x04, 0x7f, 0x20, 0x01, 0xe2, 0x00, 0x20, 0x02, 0x05, 0x20, 0x02, 0x20, 0x00, 0x41, 0x01, 0x6b, + 0x20, 0x01, 0x10, 0x0d, 0x6a, 0x0b, 0x0b, 0x89, 0x80, 0x80, 0x80, 0x00, 0x00, 0x41, 0xe4, 0x00, + 0x20, 0x00, 0x10, 0x0d, 0x0b, 0x87, 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x00, 0xe2, 0x00, 0x00, + 0x0b, 0x8a, 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x00, 0xe2, 0x00, 0x20, 0x00, 0x08, 0x03, 0x0b, + 0x9c, 0x80, 0x80, 0x80, 0x00, 0x01, 0x01, 0x63, 0x05, 0x02, 0x0f, 0x20, 0x01, 0x20, 0x00, 0xe3, + 0x07, 0x01, 0x00, 0x00, 0x00, 0x1a, 0xd0, 0x05, 0x0f, 0x0b, 0x21, 0x02, 0x24, 0x00, 0x20, 0x02, + 0x0b, 0x8a, 0x80, 0x80, 0x80, 0x00, 0x00, 0xd2, 0x0c, 0xe0, 0x07, 0x20, 0x00, 0x10, 0x11, 0x0b, + 0x8a, 0x80, 0x80, 0x80, 0x00, 0x00, 0xd2, 0x0e, 0xe0, 0x07, 0x20, 0x00, 0x10, 0x11, 0x0b, 0x8a, + 0x80, 0x80, 0x80, 0x00, 0x00, 0xd2, 0x0f, 0xe0, 0x07, 0x20, 0x00, 0x10, 0x11, 0x0b, 0x8a, 0x80, + 0x80, 0x80, 0x00, 0x00, 0xd2, 0x10, 0xe0, 0x07, 0x20, 0x00, 0x10, 0x11, 0x0b, 0x87, 0x80, 0x80, + 0x80, 0x00, 0x00, 0x20, 0x00, 0xe3, 0x05, 0x00, 0x0b, 0x94, 0x80, 0x80, 0x80, 0x00, 0x00, 0x02, + 0x7f, 0x1f, 0x40, 0x01, 0x00, 0x03, 0x00, 0x20, 0x00, 0xe3, 0x05, 0x00, 0x1a, 0x0b, 0x41, 0x7f, + 0x0b, 0x0b, 0x87, 0x80, 0x80, 0x80, 0x00, 0x00, 0xd0, 0x05, 0xe3, 0x05, 0x00, 0x0b, + +]; + +private class StackCompressionTester(t: Tester, tiering: ExecutionStrategy) { + def saved_max = StackTuning.maxStacks; + var held: Array; // stacks pinned by {limit} + var instance: Instance; + var limit_total = -1; + + new() { Execute.tiering = tiering; } + + def restore() { + StackTuning.maxStacks = saved_max; + limit_total = -1; + } + // Pins all existing segments, then allows at most {extra} more. + def limit(extra: int) { + StackTuning.maxStacks = M.total; + M.cache = null; + Target.forceGC(); + var free = 0; + for (seg = M.all; seg != null; seg = seg.next) if (seg.stack.get() == null) free++; + held = Array.new(free); + for (i < free) held[i] = M.getFreshStack(); + StackTuning.maxStacks = M.total + extra; + limit_total = StackTuning.maxStacks; + } + def checkInvariants(where: string) { + var count = 0; + for (seg = M.all; seg != null; seg = seg.next) { + count++; + var s = seg.stack.get(); + if (s == null) continue; + if (s.range != seg) t.fail1("%s: owning stack does not point back at its segment", where); + if (s.state() == StackState.COMPRESSED) t.fail1("%s: compressed stack owns a segment", where); + } + if (count != M.total) t.fail1("%s: segment count mismatch", where); + if (limit_total >= 0 && M.total > limit_total) t.fail3("%s: expected at most %d segments, got %d", where, limit_total, M.total); + } + def assertState(expected: StackState, s: NativeStack) { + var got = s.state(); + if (got != expected) t.fail2("expected state %s, got %s", expected.name, got.name); + var compressed = got == StackState.COMPRESSED; + if (compressed != (s.range == null)) t.fail("expected a stack to have a segment iff not compressed"); + if (!compressed && s.compressed != null) t.fail("expected no compressed data if not compressed"); + if (!compressed && s.saved_parent_rsp != Pointer.NULL) t.fail("expected no saved parent rsp if not compressed"); + } + def assertResult(expected: Result, got: Result) { + if (!expected.equals(got)) t.fail2("expected %q, got %q", expected.render, got.render); + } + def assertCompress(s: NativeStack) { + if (s == null) return t.fail("expected a stack, got null"); + if (!M.compressStack(s)) return t.fail1("expected stack in state %s to be compressible", s.state().name); + assertState(StackState.COMPRESSED, s); + } + def assertDecompress(s: NativeStack) { + if (!C.decompress(s)) return t.fail("expected decompression to succeed"); + assertState(StackState.SUSPENDED, s); + } + def assertTrap(reason: TrapReason, r: Result) { + match (r) { + Throw(thrown) => if (Trap.?(thrown) && Trap.!(thrown).reason == reason) return; + _ => ; + } + t.fail2("expected trap %s, got %q", reason.name, r.render); + } + // A Wasm function [i32, i32] -> [i32] that subtracts its arguments. + def subFunc() -> WasmFunction { + var b = ModuleBuilder.new(); + b.sig(SigCache.ii_i).code([ + u8.!(Opcode.LOCAL_GET.code), 0, + u8.!(Opcode.LOCAL_GET.code), 1, + u8.!(Opcode.I32_SUB.code) + ]); + var err = ErrorGen.new("subFunc"); + var inst = Instantiator.new(Extension.set.all, b.module, [], err).run(); + return WasmFunction.!(inst.functions[0]); + } + // Instantiates {GEN_MODULE} for this tester's tier. + def gen() -> this { + if (instance != null) return; + var err = ErrorGen.new("GEN_MODULE"); + var limits = Limits.new().set(Extension.set.all); + var bp = BinParser.new(Extension.set.all, limits, err, "GEN_MODULE"); + bp.tiering = tiering; + var module = bp.push(GEN_MODULE, 0, GEN_MODULE.length).finish(); + if (!err.ok()) return t.fail(err.error_msg); + instance = Instantiator.new(Extension.set.all, module, [], err).run(); + if (!err.ok()) return t.fail(err.error_msg); + tiering.onTestModule(module); + for (f in instance.functions) { + if (WasmFunction.?(f)) tiering.onTestRun(WasmFunction.!(f), err); + } + if (!err.ok()) return t.fail(err.error_msg); + } + def call(name: string, args: Array) -> Result { + gen(); + for (i < instance.module.exports.length) { + if (Strings.equal(name, instance.module.exports[i].0)) { + return Execute.call(Function.!(instance.exports[i]), args); + } + } + t.fail1("export %s not found", name); + return Result.Value([]); + } + def callCont(name: string, args: Array) -> Continuation { + var r = call(name, args); + match (r) { + Value(vals) => if (vals.length == 1 && Value.Cont.?(vals[0])) return Value.Cont.!(vals[0]).val; + _ => ; + } + t.fail2("expected %s to return a continuation, got %q", name, r.render); + return Continuations.NULL; + } + def stackOf(k: Continuation) -> NativeStack { + return NativeStack.!(k.stack); + } + // Runs the exported Wasm function {name} on {x} until it suspends, returning its stack. + def suspended(name: string, x: int) -> NativeStack { + var k = callCont(name, [Values.box_i(x)]); + if (Continuations.isNull(k)) { t.fail1("expected %s to return a continuation", name); return null; } + var s = stackOf(k); + assertState(StackState.SUSPENDED, s); + return s; + } +} + +// Runs a test, then restores global settings and checks invariants. +def runTest(f: StackCompressionTester -> void, t: StackCompressionTester) { + f(t); + t.checkInvariants("end of test"); + t.restore(); +} + +def EMPTY_STRUCT = StructDecl.new(false, ValueTypes.NO_HEAPTYPES, []); +def obj() -> Value { return Value.Ref(ObjectModel.newStruct(EMPTY_STRUCT)); } +def DEEP_X = 3; +def DEEP_SUM = 5050 + 101 * DEEP_X; // sum of (d + x) for d in 0..100 + +def test_can_compress_states(t: StackCompressionTester) { + var f = t.subFunc(); + var s = M.getFreshStack(); + t.assertState(StackState.EMPTY, s); + if (C.canCompress(s)) t.t.fail("expected EMPTY stack not to be compressible"); + s.reset(f); + t.assertState(StackState.SUSPENDED, s); + if (!C.canCompress(s)) t.t.fail("expected SUSPENDED stack to be compressible"); + s.bind([Values.box_i(9)]); + if (!C.canCompress(s)) t.t.fail("expected SUSPENDED stack with bound values to be compressible"); + t.assertCompress(s); + if (C.canCompress(s)) t.t.fail("expected COMPRESSED stack not to be compressible"); + if (M.compressStack(s)) t.t.fail("expected compressing a COMPRESSED stack to fail"); + t.assertDecompress(s); + if (!C.canCompress(s)) t.t.fail("expected decompressed stack to be compressible again"); + t.assertResult(Result.Value([Values.box_i(5)]), s.bind([Values.box_i(4)]).resume()); + t.assertState(StackState.EMPTY, s); +} + +def test_roundtrip_entry(t: StackCompressionTester) { + var f = t.subFunc(); + var s = M.getFreshStack().reset(f).bind([Values.box_i(100), Values.box_i(1)]); + var before = (C.numCompressed, C.numDecompressed); + t.assertCompress(s); + t.t.assert_eq(before.0 + 1, C.numCompressed); + t.assertResult(Result.Value([Values.box_i(99)]), s.resume()); // decompresses + t.t.assert_eq(before.1 + 1, C.numDecompressed); +} + +def test_roundtrip_entry_bind(t: StackCompressionTester) { + var f = t.subFunc(); + var s = M.getFreshStack().reset(f).bind([Values.box_i(50)]); + t.assertCompress(s); + s.bind([Values.box_i(8)]); // binds without decompressing + t.assertState(StackState.COMPRESSED, s); + t.assertResult(Result.Value([Values.box_i(42)]), s.resume()); +} + +def test_roundtrip_refs_gc(t: StackCompressionTester) { + var b = ModuleBuilder.new(); + var sig = b.newSig([ValueTypes.ANYREF, ValueTypes.ANYREF], [ValueTypes.ANYREF, ValueTypes.ANYREF]); + b.sig(sig).code([u8.!(Opcode.LOCAL_GET.code), 1, u8.!(Opcode.LOCAL_GET.code), 0]); + var err = ErrorGen.new("swap"); + var f = Instantiator.new(Extension.set.all, b.module, [], err).run().functions[0]; + var a = obj(), c = obj(); + var s = M.getFreshStack().reset(f).bind([a, c]); + t.assertCompress(s); + for (i < 3) Target.forceGC(); // the compressed values must keep the objects alive and be updated + t.assertResult(Result.Value([c, a]), s.resume()); +} + +def test_no_frames_not_compressible(t: StackCompressionTester) { + var s = M.getFreshStack().resetForFrameWrites(t.subFunc()); + t.assertState(StackState.SUSPENDED, s); + if (C.canCompress(s)) t.t.fail("expected a stack with no frames and no entry not to be compressible"); + s.clear(); +} + +def test_running_not_compressible(t: StackCompressionTester) { + // Inside a host call, the running stack cannot be compressed. + var hf = HostFunction.new("check", SigCache.v_i, checkRunning(t, _)); + t.assertResult(Result.Value([Values.box_i(7)]), M.runOnFreshStack(hf, [])); +} +def checkRunning(t: StackCompressionTester, args: Range) -> HostResult { + var cur = X86_64Runtime.curStack; + if (C.canCompress(cur)) t.t.fail1("expected %s stack not to be compressible", cur.state().name); + if (M.compressStack(cur)) t.t.fail("expected compressing the running stack to fail"); + return HostResult.Value1(Values.box_i(7)); +} + +def test_entry_host_func(t: StackCompressionTester) { + var hf = HostFunction.new("add", SigCache.ii_i, fun args => HostResult.Value1(Values.box_i(Values.unbox_i(args[0]) + Values.unbox_i(args[1])))); + var s = M.getFreshStack().reset(hf).bind([Values.box_i(30)]); + t.assertCompress(s); + t.assertResult(Result.Value([Values.box_i(42)]), s.bind([Values.box_i(12)]).resume()); +} + +def test_repeat_roundtrip(t: StackCompressionTester) { + var f = t.subFunc(); + var s = M.getFreshStack().reset(f).bind([Values.box_i(1000)]); + var before = (C.numCompressed, C.numDecompressed); + for (i < 50) { + t.assertCompress(s); + t.assertDecompress(s); + if ((i % 10) == 0) Target.forceGC(); + } + t.t.assert_eq(before.0 + 50, C.numCompressed); + t.t.assert_eq(before.1 + 50, C.numDecompressed); + t.checkInvariants("roundtrip"); + t.assertResult(Result.Value([Values.box_i(999)]), s.bind([Values.box_i(1)]).resume()); +} + +def test_acquire_segment(t: StackCompressionTester) { + t.limit(1); + var seg = M.acquireSegment(); + if (seg == null) return t.t.fail("expected a segment"); + if (seg.stack.get() != null) t.t.fail("expected an acquired segment to have no owner"); + // An unowned segment looks free to the reclaim sweep, so attach it before requesting another. + var s = NativeStack.new(); + M.attach(s, seg); + if (M.acquireSegment() != null) t.t.fail("expected no second segment under the limit"); + t.assertTrap(TrapReason.OUT_OF_STACKS, M.runOnFreshStack(HostFunction.new("seven", SigCache.v_i, fun args => HostResult.Value1(Values.box_i(7))), [])); + if (seg.stack.get() != s) t.t.fail("expected the attached segment to keep its owner"); + M.recycleStack(s); + if (M.getFreshStack() != s) t.t.fail("expected the returned segment's stack to be reused"); +} + +def test_decompress_out_of_stacks(t: StackCompressionTester) { + var f = t.subFunc(); + t.limit(1); + var s = M.getFreshStack().reset(f).bind([Values.box_i(10), Values.box_i(3)]); + t.assertCompress(s); + var hog = M.getFreshStack(); // takes the only segment + if (hog == null) return t.t.fail("expected the freed segment to be available"); + t.assertTrap(TrapReason.OUT_OF_STACKS, s.resume()); + t.assertState(StackState.COMPRESSED, s); + M.recycleStack(hog); + t.assertResult(Result.Value([Values.box_i(7)]), s.resume()); +} + +def test_version_kept(t: StackCompressionTester) { + var f = t.subFunc(); + var s = M.getFreshStack().reset(f).bind([Values.box_i(10), Values.box_i(3)]); + var k = Continuations.makeContinuation(s); + t.assertCompress(s); + if (Continuations.isUsed(k)) t.t.fail("expected compression not to consume the continuation"); + t.assertDecompress(s); + if (Continuations.isUsed(k)) t.t.fail("expected decompression not to consume the continuation"); + t.assertResult(Result.Value([Values.box_i(7)]), s.resume()); +} + +def test_mid_host_resume(t: StackCompressionTester) { + var s = t.suspended("make_once", 5); + t.assertCompress(s); + t.assertResult(Result.Value([Values.box_i(1005)]), s.resume()); +} + +def test_mid_host_bind(t: StackCompressionTester) { + var s = t.suspended("make_asker", 4); + t.assertCompress(s); + s.bind([Values.box_i(2)]); // decompresses, since the stack has frames + t.assertState(StackState.SUSPENDED, s); + t.assertResult(Result.Value([Values.box_i(42)]), s.resume()); +} + +def test_mid_twice(t: StackCompressionTester) { + var s = t.suspended("make_once", 7); + t.assertCompress(s); + t.assertDecompress(s); + t.assertCompress(s); + for (i < 3) Target.forceGC(); + t.assertResult(Result.Value([Values.box_i(1007)]), s.resume()); +} + +def test_mid_gc(t: StackCompressionTester) { + var stacks = Array.new(8); + for (i < stacks.length) { + stacks[i] = t.suspended("make_once", i); + t.assertCompress(stacks[i]); + } + for (i < 3) Target.forceGC(); + for (i < stacks.length) t.assertResult(Result.Value([Values.box_i(1000 + i)]), stacks[i].resume()); +} + +def test_gen_frames(t: StackCompressionTester) { + // Three frames deep with v128 and struct locals; resumed from the host, it checks them and yields again. + var s = t.suspended("make", 100); + t.assertCompress(s); + for (i < 3) Target.forceGC(); + t.assertDecompress(s); + t.assertCompress(s); + t.assertTrap(TrapReason.INVALID_SUSPEND, s.resume()); // the host handles no tags +} + +def test_deep_frames(t: StackCompressionTester) { + var s = t.suspended("make_deep", DEEP_X); + t.assertCompress(s); + for (i < 3) Target.forceGC(); + t.assertResult(Result.Value([Values.box_i(DEEP_SUM)]), s.resume()); +} + +def test_chain_top_roundtrip(t: StackCompressionTester) { + // The continuation's top stack runs {inner}, whose parent {mid} waits for it. + var top = t.suspended("make_chain", 5); + var mid = NativeStack.!(top.parent); + if (mid == null) return t.t.fail("expected the continuation to be a chain of stacks"); + t.assertState(StackState.CALL_CHILD, mid); + if (M.compressStack(mid)) t.t.fail("expected the waiting parent not to be compressible"); + var parent_rsp = top.parent_rsp_ptr.load(); + if (parent_rsp != mid.rsp) t.t.fail("expected the top stack's parent slot to hold the parent's rsp"); + t.assertCompress(top); + if (top.saved_parent_rsp != parent_rsp) t.t.fail("expected compression to save the parent rsp"); + for (i < 3) Target.forceGC(); + t.assertDecompress(top); + if (top.parent != mid || top.bottom != mid) t.t.fail("expected decompression to keep the chain links"); + if (top.parent_rsp_ptr.load() != parent_rsp) t.t.fail("expected decompression to restore the parent rsp"); + t.assertState(StackState.CALL_CHILD, mid); +} + +def test_trap_after_decompress(t: StackCompressionTester) { + var s = t.suspended("make_trapper", 1); + t.assertCompress(s); + t.assertTrap(TrapReason.UNREACHABLE, s.resume()); +} + +def test_throw_after_decompress(t: StackCompressionTester) { + var s = t.suspended("make_thrower", 9); + t.assertCompress(s); + var r = s.resume(); + match (r) { + Throw(thrown) => if (!Exception.?(thrown)) t.t.fail1("expected an exception, got %q", thrown.render); + _ => t.t.fail1("expected an exception, got %q", r.render); + } +} From 665ac90ccd64a39f6bb7acd95c8331d972bd0620 Mon Sep 17 00:00:00 2001 From: davidmaamoaix Date: Mon, 26 Oct 2026 22:44:15 -0400 Subject: [PATCH 04/10] [stack] Decompress compressed stacks at resume sites --- src/engine/compiler/MacroAssembler.v3 | 3 + src/engine/compiler/SinglePassCompiler.v3 | 23 ++ src/engine/x86-64/X86_64Interpreter.v3 | 14 ++ src/engine/x86-64/X86_64MacroAssembler.v3 | 7 + src/engine/x86-64/X86_64Runtime.v3 | 8 + .../NativeStackCompressionTest.v3 | 213 +++++++++++++++++- 6 files changed, 263 insertions(+), 5 deletions(-) diff --git a/src/engine/compiler/MacroAssembler.v3 b/src/engine/compiler/MacroAssembler.v3 index a12885746..8062a2899 100644 --- a/src/engine/compiler/MacroAssembler.v3 +++ b/src/engine/compiler/MacroAssembler.v3 @@ -356,6 +356,7 @@ class MacroAssembler(valuerep: Tagging, regConfig: RegConfig) { def emit_call_runtime_SUSPEND(); def emit_call_runtime_SWITCH(); def emit_call_runtime_RESUME_THROW(); + def emit_call_runtime_DECOMPRESS_CONT(); def emit_call_runtime_RESUME_THROW_REF(); def emit_jump_to_trap_at(reason: TrapReason); def emit_call_runtime_op(op: Opcode); @@ -379,6 +380,8 @@ class MacroAssembler(valuerep: Tagging, regConfig: RegConfig) { // stk.state_ = state; def emit_set_stack_state(stk: Reg, state: StackState); + // Branches to {label} if the state of the stack in {stk} is not {state}. + def emit_br_stack_state_ne(stk: Reg, state: StackState, label: MasmLabel); // Publishes the handler table. def emit_publish_handlers(r_stack: Reg, r_table: Reg, tier: HandlerTier) { var offsets = getOffsets(); diff --git a/src/engine/compiler/SinglePassCompiler.v3 b/src/engine/compiler/SinglePassCompiler.v3 index 0a45c4c13..22bd359c4 100644 --- a/src/engine/compiler/SinglePassCompiler.v3 +++ b/src/engine/compiler/SinglePassCompiler.v3 @@ -1454,6 +1454,7 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl buildResumeHandlerDests(it.pc, handlers); var cont_decl = ContDecl.!(module.heaptypes[cont_id]); if (checkForConstNull(state.peek())) return; + emit_decompress_top_cont(); var n = cont_decl.sig.params.length, slot_size = masm.valuerep.slot_size, offsets = masm.getOffsets(); // Spill the value stack and consume the continuation on top of it. @@ -2425,6 +2426,28 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl masm.emit_mov_r_r(ValueKind.REF, dst, regs.vfp); // XXX: use 3-addr adjustment of VSP (i.e. lea) if (slots > 0) masm.emit_addw_r_i(dst, int.view(slots) * masm.valuerep.slot_size); } + // Decompresses the stack of the continuation on top of the value stack, if compressed. + def emit_decompress_top_cont() { + state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS); // spilled values can be updated by GC + emit_compute_vsp(regs.vsp, state.sp); + emit_spill_vsp(regs.vsp); + masm.emit_store_curstack_vsp(regs.vsp); + var done = masm.newLabel(it.pc), tmp = regs.scratch, valuerep = masm.valuerep; + masm.emit_mov_r_m(ValueKind.REF, tmp, MasmAddr(regs.vsp, valuerep.tag_size - valuerep.slot_size)); + masm.emit_br_r(tmp, MasmBrCond.REF_NULL, done); // null continuation traps later + masm.emit_br_stack_state_ne(tmp, StackState.COMPRESSED, done); + def emit = fun { + masm.emit_get_curstack(regs.runtime_arg0); + masm.emit_v3_set_NativeStack_rsp_r_r(regs.runtime_arg0, regs.sp); + masm.emit_push_NativeStack_rsp_r_r(regs.runtime_arg0); + masm.emit_call_runtime_DECOMPRESS_CONT(); + }; + withReconstructedInlinedFrames(emit); // the call can trap + masm.emit_get_curstack(regs.scratch); + masm.emit_pop_NativeStack_rsp_r_r(regs.scratch); + emit_reload_regs(); + masm.bindLabel(done); + } def emit_spill_vsp(reg: Reg) { masm.emit_mov_m_r(ValueKind.REF, frame.vsp_slot, reg); } diff --git a/src/engine/x86-64/X86_64Interpreter.v3 b/src/engine/x86-64/X86_64Interpreter.v3 index 60339a989..4c85a4a79 100644 --- a/src/engine/x86-64/X86_64Interpreter.v3 +++ b/src/engine/x86-64/X86_64Interpreter.v3 @@ -2496,6 +2496,7 @@ class X86_64InterpreterGen(ic: X86_64InterpreterCode, w: DataWriter) { bindHandler(Opcode.RESUME); { var cont_id = xenv.tmp0; saveResumeIVars(); + genDecompressTopCont(); genReadUleb32(G(cont_id)); // read signature index var contStack = xenv.tmp1; @@ -4157,6 +4158,19 @@ class X86_64InterpreterGen(ic: X86_64InterpreterCode, w: DataWriter) { asm.movq_r_m(t, t.plus(offsets.FuncHandlerInfo_resume_sites)); masm.emit_v3_Array_elem_r_rr(ValueKind.REF, r_tmp, r_tmp, xenv.curpc); } + // Decompresses the stack of the continuation on top of the value stack, if compressed. + private def genDecompressTopCont() { + var done = X86_64Label.new(); + asm.movq_r_m(r_tmp1, vsph[-1].value); // %tmp1 = cont.stack + asm.q.cmp_r_i(r_tmp1, 0); + asm.jc_rel_far(X86_64Conds.Z, done); // null continuation traps later + asm.cmpb_m_i(r_tmp1.plus(offsets.NativeStack_state), StackState.COMPRESSED.tag); + asm.jc_rel_far(X86_64Conds.NZ, done); + masm.emit_get_curstack(xenv.tmp1); + callRuntime(refRuntimeCall(X86_64RT.runtime_DECOMPRESS_CONT), [r_tmp1], true); + restoreCallerIVars(); + asm.bind(done); + } // Parks the current stack, publishes the handler table, chains {contStack} under it and switches. def genResumeChain(curStack: Reg, contStack: Reg, skip_tag: bool) { masm.emit_get_curstack(curStack); diff --git a/src/engine/x86-64/X86_64MacroAssembler.v3 b/src/engine/x86-64/X86_64MacroAssembler.v3 index 4efbb7e58..813bc3ade 100644 --- a/src/engine/x86-64/X86_64MacroAssembler.v3 +++ b/src/engine/x86-64/X86_64MacroAssembler.v3 @@ -882,6 +882,9 @@ class X86_64MacroAssembler extends MacroAssembler { def emit_call_runtime_RESUME_THROW() { emit_call_runtime(X86_64RT.runtime_handle_resume_throw); } + def emit_call_runtime_DECOMPRESS_CONT() { + emit_call_runtime(X86_64RT.runtime_DECOMPRESS_CONT); + } def emit_call_runtime_RESUME_THROW_REF() { emit_call_runtime(X86_64RT.runtime_resume_throw_ref); } @@ -1679,6 +1682,10 @@ class X86_64MacroAssembler extends MacroAssembler { def emit_set_stack_state(stk: Reg, state: StackState) { emit_mov_m_b(MasmAddr(stk, offsets.NativeStack_state), byte.view(state.tag)); } + def emit_br_stack_state_ne(stk: Reg, state: StackState, label: MasmLabel) { + asm.cmpb_m_i(G(stk).plus(offsets.NativeStack_state), state.tag); + asm.jc_rel_far(X86_64Conds.NZ, X86_64MasmLabel.!(label).label); + } // Moves values from the current stack to %cont.top. def emit_cont_mv(from_vsp: Reg, contStack: Reg, n_vals: Reg, tmp1: Reg, tmp2: Reg, xmm0: Reg) { diff --git a/src/engine/x86-64/X86_64Runtime.v3 b/src/engine/x86-64/X86_64Runtime.v3 index 09c615011..66151b01e 100644 --- a/src/engine/x86-64/X86_64Runtime.v3 +++ b/src/engine/x86-64/X86_64Runtime.v3 @@ -113,6 +113,14 @@ component X86_64Runtime { Unhandled(t) => return t; } } + // Decompresses the continuation on top of {stack}'s value stack, where GC can update it. + def runtime_DECOMPRESS_CONT(stack: NativeStack) -> Throwable { + var ptr = stack.vsp + (stack.valuerep.tag_size - stack.valuerep.slot_size); + var cont_stack = NativeStack.!(ptr.load()); + if (cont_stack == null) return null; + if (!NativeStackCompression.decompress(cont_stack)) return stack.trap(TrapReason.OUT_OF_STACKS); + return null; + } def runtime_CONT_NEW(stack: NativeStack, instance: Instance, cont_index: u31) -> Throwable { return Runtime.CONT_NEW(stack, instance, cont_index); } diff --git a/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 b/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 index 9ac251206..43e6d9585 100644 --- a/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 +++ b/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 @@ -1,7 +1,7 @@ // Copyright 2026 Wizard authors. All rights reserved. // See LICENSE for details of Apache 2.0 license. -// Tests for compressing suspended native stacks and resuming them from host code in every tier. +// Tests for compressing suspended native stacks and resuming them from host and Wasm code in every tier. def M: NativeStackManager; def C: NativeStackCompression; @@ -35,6 +35,20 @@ def X_ = void( TT("chain_top_roundtrip", test_chain_top_roundtrip), TT("trap_after_decompress", test_trap_after_decompress), TT("throw_after_decompress", test_throw_after_decompress), + TT("mid_explicit", test_mid_explicit), + TT("mid_wasm_twice", test_mid_wasm_twice), + TT("mid_wasm_bind", test_mid_wasm_bind), + TT("mid_gc_wasm", test_mid_gc_wasm), + TT("chain_top", test_chain_top), + TT("deep_frames_wasm", test_deep_frames_wasm), + TT("used_after_compress", test_used_after_compress), + TT("trap_after_decompress_wasm", test_trap_after_decompress_wasm), + TT("throw_after_decompress_wasm", test_throw_after_decompress_wasm), + TT("resume_null", test_resume_null), + TT("entry_cont_arg", test_entry_cont_arg), + TT("compress_from_host_call", test_compress_from_host_call), + TT("decompress_traps_in_wasm", test_decompress_traps_in_wasm), + TT("decompress_via_gc_wasm", test_decompress_via_gc_wasm), () ); @@ -386,6 +400,48 @@ private class StackCompressionTester(t: Tester, tiering: ExecutionStrategy) { def stackOf(k: Continuation) -> NativeStack { return NativeStack.!(k.stack); } + def exported(name: string) -> Function { + gen(); + for (i < instance.module.exports.length) { + if (Strings.equal(name, instance.module.exports[i].0)) return Function.!(instance.exports[i]); + } + t.fail1("export %s not found", name); + return null; + } + def callInt(name: string, args: Array) -> int { + var r = call(name, args); + match (r) { + Value(vals) => if (vals.length == 1 && Value.I32.?(vals[0])) return int.view(Value.I32.!(vals[0]).val); + _ => ; + } + t.fail2("expected %s to return an i32, got %q", name, r.render); + return -1; + } + // Makes a generator yielding {start} .. {start + 9}, suspended at its first yield. + def make(start: int) -> Continuation { + var k = callCont("make", [Values.box_i(start)]); + if (!Continuations.isNull(k)) assertLast(start); + return k; + } + // Steps a generator, which must yield {expected}, or finish if {expected < 0}. + def next(k: Continuation, expected: int) -> Continuation { + var n = callCont("next", [Value.Cont(k)]); + if (expected < 0) { + if (!Continuations.isNull(n)) t.fail("expected generator to finish"); + } else { + if (Continuations.isNull(n)) t.fail1("expected generator to yield %d, but it finished", expected); + else assertLast(expected); + } + return n; + } + def assertLast(expected: int) { + var got = callInt("last", []); + if (got != expected) t.fail2("expected generator to yield %d, got %d", expected, got); + } + // Takes a fresh stack and drops it, so that its segment is reclaimable only after a GC. + def dropFresh() #no-inline { + if (M.getFreshStack() == null) t.fail("expected a fresh stack to drop"); + } // Runs the exported Wasm function {name} on {x} until it suspends, returning its stack. def suspended(name: string, x: int) -> NativeStack { var k = callCont(name, [Values.box_i(x)]); @@ -405,8 +461,8 @@ def runTest(f: StackCompressionTester -> void, t: StackCompressionTester) { def EMPTY_STRUCT = StructDecl.new(false, ValueTypes.NO_HEAPTYPES, []); def obj() -> Value { return Value.Ref(ObjectModel.newStruct(EMPTY_STRUCT)); } -def DEEP_X = 3; -def DEEP_SUM = 5050 + 101 * DEEP_X; // sum of (d + x) for d in 0..100 +// The result of {make_deep}: the sum of (d + x) for d in 0..100. +def deepSum(x: int) -> int { return 5050 + 101 * x; } def test_can_compress_states(t: StackCompressionTester) { var f = t.subFunc(); @@ -583,10 +639,10 @@ def test_gen_frames(t: StackCompressionTester) { } def test_deep_frames(t: StackCompressionTester) { - var s = t.suspended("make_deep", DEEP_X); + var s = t.suspended("make_deep", 3); t.assertCompress(s); for (i < 3) Target.forceGC(); - t.assertResult(Result.Value([Values.box_i(DEEP_SUM)]), s.resume()); + t.assertResult(Result.Value([Values.box_i(deepSum(3))]), s.resume()); } def test_chain_top_roundtrip(t: StackCompressionTester) { @@ -622,3 +678,150 @@ def test_throw_after_decompress(t: StackCompressionTester) { _ => t.t.fail1("expected an exception, got %q", r.render); } } + +def test_mid_explicit(t: StackCompressionTester) { + var k = t.make(100); + var s = t.stackOf(k); + t.assertState(StackState.SUSPENDED, s); + t.assertCompress(s); // three frames deep, with v128 and struct locals + k = t.next(k, 101); // the Wasm resume decompresses + if (t.stackOf(k) != s) t.t.fail("expected the continuation to keep its stack"); + t.assertState(StackState.SUSPENDED, s); + for (i = 2; i < 10; i++) { + t.assertCompress(t.stackOf(k)); + k = t.next(k, 100 + i); + } + t.assertCompress(t.stackOf(k)); + t.next(k, -1); +} + +def test_mid_wasm_twice(t: StackCompressionTester) { + var k = t.make(7); + var s = t.stackOf(k); + t.assertCompress(s); + t.assertDecompress(s); + t.assertCompress(s); + for (i < 3) Target.forceGC(); + k = t.next(k, 8); + k = t.next(k, 9); +} + +def test_mid_wasm_bind(t: StackCompressionTester) { + var k = t.callCont("make_asker", [Values.box_i(3)]); + t.assertCompress(t.stackOf(k)); + t.t.assert_eq(37, t.callInt("answer", [Value.Cont(k), Values.box_i(7)])); // cont.bind decompresses +} + +def test_mid_gc_wasm(t: StackCompressionTester) { + var gens = Array.new(8); + for (i < gens.length) { + gens[i] = t.make(i * 10); + t.assertCompress(t.stackOf(gens[i])); + } + for (i < 3) Target.forceGC(); + for (j = 1; j < 10; j++) { + for (i < gens.length) { + gens[i] = t.next(gens[i], i * 10 + j); + if ((i + j) % 2 == 0) t.assertCompress(t.stackOf(gens[i])); + } + Target.forceGC(); + } + for (i < gens.length) t.next(gens[i], -1); +} + +def test_chain_top(t: StackCompressionTester) { + // The continuation's top stack runs {inner}, whose parent {mid} waits for it. + var k = t.callCont("make_chain", [Values.box_i(5)]); + var top = t.stackOf(k); + var mid = NativeStack.!(top.parent); + if (mid == null) return t.t.fail("expected the continuation to be a chain of stacks"); + t.assertState(StackState.CALL_CHILD, mid); + t.assertCompress(top); + for (i < 3) Target.forceGC(); + t.t.assert_eq(11, t.callInt("finish", [Value.Cont(k)])); +} + +def test_deep_frames_wasm(t: StackCompressionTester) { + var k = t.callCont("make_deep", [Values.box_i(7)]); + t.assertLast(7); + t.assertCompress(t.stackOf(k)); + Target.forceGC(); + t.t.assert_eq(deepSum(7), t.callInt("finish", [Value.Cont(k)])); +} + +def test_used_after_compress(t: StackCompressionTester) { + var k = t.callCont("make_deep", [Values.box_i(1)]); + t.assertCompress(t.stackOf(k)); + t.t.assert_eq(deepSum(1), t.callInt("finish", [Value.Cont(k)])); + t.assertTrap(TrapReason.USED_CONTINUATION, t.call("finish", [Value.Cont(k)])); + // A continuation consumed by {cont.bind} after compression stays unusable. + var a = t.callCont("make_asker", [Values.box_i(2)]); + t.assertCompress(t.stackOf(a)); + t.t.assert_eq(25, t.callInt("answer", [Value.Cont(a), Values.box_i(5)])); + t.assertTrap(TrapReason.USED_CONTINUATION, t.call("answer", [Value.Cont(a), Values.box_i(5)])); +} + +def test_trap_after_decompress_wasm(t: StackCompressionTester) { + var k = t.callCont("make_trapper", [Values.box_i(3)]); + t.assertCompress(t.stackOf(k)); + t.assertTrap(TrapReason.UNREACHABLE, t.call("finish", [Value.Cont(k)])); +} + +def test_throw_after_decompress_wasm(t: StackCompressionTester) { + var k = t.callCont("make_thrower", [Values.box_i(9)]); + t.assertCompress(t.stackOf(k)); + t.t.assert_eq(9, t.callInt("catch", [Value.Cont(k)])); +} + +def test_resume_null(t: StackCompressionTester) { + t.assertTrap(TrapReason.NULL_DEREF, t.call("resume_null", [])); +} + +def test_entry_cont_arg(t: StackCompressionTester) { + // A compressed entry stack whose bound argument is the continuation of another compressed stack. + var k = t.callCont("make_deep", [Values.box_i(2)]); + var s = M.getFreshStack().reset(t.exported("finish")).bind([Value.Cont(k)]); + t.assertCompress(t.stackOf(k)); + t.assertCompress(s); + for (i < 3) Target.forceGC(); + t.assertResult(Result.Value([Values.box_i(deepSum(2))]), s.resume()); +} + +def test_compress_from_host_call(t: StackCompressionTester) { + // Stacks are compressed while Wasm frames are live on other stacks. + var gens = Array.new(5); + for (i < gens.length) gens[i] = t.make(i * 10); + var hf = HostFunction.new("compress_all", SigCache.v_i, compressAll(t, gens, _)); + t.assertResult(Result.Value([Values.box_i(gens.length)]), M.runOnFreshStack(hf, [])); + for (j = 1; j < 10; j++) { + for (i < gens.length) gens[i] = t.next(gens[i], i * 10 + j); + } + for (k in gens) t.next(k, -1); +} +def compressAll(t: StackCompressionTester, gens: Array, args: Range) -> HostResult { + var count = 0; + for (k in gens) if (M.compressStack(t.stackOf(k))) count++; + return HostResult.Value1(Values.box_i(count)); +} + +def test_decompress_traps_in_wasm(t: StackCompressionTester) { + t.gen(); + t.limit(2); + var k = t.callCont("make_once", [Values.box_i(5)]); + t.assertCompress(t.stackOf(k)); + var hog = M.getFreshStack(); // leaves one segment, which the call itself takes + if (hog == null) return t.t.fail("expected a segment to hold"); + t.assertTrap(TrapReason.OUT_OF_STACKS, t.call("finish", [Value.Cont(k)])); + t.assertState(StackState.COMPRESSED, t.stackOf(k)); // a failed resume leaves the continuation intact + M.recycleStack(hog); + t.t.assert_eq(1005, t.callInt("finish", [Value.Cont(k)])); +} + +def test_decompress_via_gc_wasm(t: StackCompressionTester) { + t.gen(); + t.limit(2); + var k = t.callCont("make_once", [Values.box_i(5)]); + t.assertCompress(t.stackOf(k)); + t.dropFresh(); // decompression must force a GC to reclaim this segment + t.t.assert_eq(1005, t.callInt("finish", [Value.Cont(k)])); +} From eee7cf490d0577ff7aa13c91915c798c60d20442 Mon Sep 17 00:00:00 2001 From: davidmaamoaix Date: Mon, 26 Oct 2026 22:55:37 -0400 Subject: [PATCH 05/10] [stack] Compress suspended stacks when out of segments --- src/engine/compiler/SinglePassCompiler.v3 | 1 + src/engine/native/NativeStackManager.v3 | 56 +++++- test/regress/stacks/compression.failures | 15 -- .../NativeStackCompressionTest.v3 | 178 ++++++++++++++++++ 4 files changed, 225 insertions(+), 25 deletions(-) diff --git a/src/engine/compiler/SinglePassCompiler.v3 b/src/engine/compiler/SinglePassCompiler.v3 index 22bd359c4..75c847dd4 100644 --- a/src/engine/compiler/SinglePassCompiler.v3 +++ b/src/engine/compiler/SinglePassCompiler.v3 @@ -1579,6 +1579,7 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl var target_top = target_decl.sig.params[target_decl.sig.params.length - 1]; var this_decl = ContDecl.!(HeapType.Cont.!(ValueType.Ref.!(target_top).heap).cont); var n = target_decl.sig.params.length - 1, slot_size = masm.valuerep.slot_size, offsets = masm.getOffsets(); + emit_decompress_top_cont(); state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS); var vsp = allocTmp(ValueKind.REF), cur = allocTmp(ValueKind.REF), target = allocTmp(ValueKind.REF); var prev = allocTmp(ValueKind.REF), parent = allocTmp(ValueKind.REF), tag = allocTmp(ValueKind.REF); diff --git a/src/engine/native/NativeStackManager.v3 b/src/engine/native/NativeStackManager.v3 index cd312c33b..48874af99 100644 --- a/src/engine/native/NativeStackManager.v3 +++ b/src/engine/native/NativeStackManager.v3 @@ -9,15 +9,15 @@ component NativeStackManager { var all: NativeStackSegment; // all segments ever mapped; never unmapped var total: int; // number of segments in {all} var cache: NativeStack; // used in fast-path allocation in generated code + private var sweep: NativeStackSegment; // clock hand for compression victims + private var allocsSinceGC: int; // stacks requested since the last forced GC + private var gcInterval: int; // requests to wait before collecting ahead of compressing // Gets a stack with a segment, or {null} if none is available. def getFreshStack() -> NativeStack { if (Trace.stack) Trace.OUT.puts("Requested fresh x86_stack...\n"); - if (cache == null) { - var ok = tryAllocatingNewStacks() || tryReclaimingSegments() || tryCollectingAndReclaiming() || tryCompressingStacks(); - if (!ok) return null; - } - + allocsSinceGC++; + if (cache == null && !refill()) return null; var result = cache; cache = cache.next_stack; result.next_stack = null; @@ -48,6 +48,21 @@ component NativeStackManager { var s = getFreshStack(); return if(s == null, null, detach(s)); // {s} becomes garbage } + // Refills the cache with new segments, segments of dead stacks, or segments of compressed stacks. + private def refill() -> bool { + if (tryAllocatingNewStacks()) return true; + if (tryReclaimingSegments() > 0) return true; + var collected = false; + if (StackTuning.enableCompression) { + // Collect before compressing if collecting has been productive lately. + if (allocsSinceGC >= gcInterval) { + collected = true; + if (tryCollectingAndReclaiming()) return true; + } + if (tryCompressingStacks()) return true; + } + return !collected && tryCollectingAndReclaiming(); + } // Wraps {segment} in a new cached stack that owns it. private def addToCache(segment: NativeStackSegment) { var stack = NativeStack.new(); @@ -77,23 +92,42 @@ component NativeStackManager { } return cache != null; // OK if succeeded in allocating at least one } - private def tryReclaimingSegments() -> bool { - var ok = false; + // Returns the number of segments reclaimed from stacks that the GC found dead. + private def tryReclaimingSegments() -> int { + var count = 0; for (p = all; p != null; p = p.next) { if (p.stack.get() == null) { // weak reference was cleared by GC, can reclaim addToCache(p); - ok = true; + count++; } } - return ok; + return count; } // Weak references are only cleared by a GC, so force one to find dead stacks. private def tryCollectingAndReclaiming() -> bool { if (all == null) return false; Target.forceGC(); - return tryReclaimingSegments(); + allocsSinceGC = 0; + var count = tryReclaimingSegments(); + // Back off exponentially while collecting is unproductive. + gcInterval = if(count * 4 >= total, total, Ints.min(gcInterval * 2 + 1, 1 << 24)); + return count > 0; } + // Compresses one suspended stack, sweeping round-robin over all segments. private def tryCompressingStacks() -> bool { + for (i < total) { + if (sweep == null) sweep = all; + var segment = sweep; + sweep = segment.next; + var s = segment.stack.get(); + if (s == null) { // its stack died since the last reclamation + addToCache(segment); + return true; + } + if (!NativeStackCompression.canCompress(s)) continue; + addToCache(NativeStackCompression.compress(s)); + return true; + } return false; } // Checks whether a fault at {addr} hit the guard page above a stack segment, which indicates @@ -106,7 +140,9 @@ component NativeStackManager { } } } + // Returns {stack} to the cache, clearing it so that it is never chosen for compression. def recycleStack(stack: NativeStack) { + stack.clear(); stack.next_stack = cache; cache = stack; } diff --git a/test/regress/stacks/compression.failures b/test/regress/stacks/compression.failures index fb7a22735..6920a6c85 100644 --- a/test/regress/stacks/compression.failures +++ b/test/regress/stacks/compression.failures @@ -1,17 +1,2 @@ -test/regress/ext:stack-switching/many_stacks_dyn.bin.wast -test/regress/ext:stack-switching/many_stacks.bin.wast test/regress/ext:stack-switching/resume_chain3.bin.wast test/regress/ext:stack-switching/suspend15.bin.wast -test/regress/stacks/bind_partial0.bin.wast -test/regress/stacks/bind0.bin.wast -test/regress/stacks/chains0.bin.wast -test/regress/stacks/cont_in_frames0.bin.wast -test/regress/stacks/deep0.bin.wast -test/regress/stacks/exnref0.bin.wast -test/regress/stacks/live_conts0.bin.wast -test/regress/stacks/resume_throw_ref0.bin.wast -test/regress/stacks/resume_throw0.bin.wast -test/regress/stacks/scheduler0.bin.wast -test/regress/stacks/switch_ring0.bin.wast -test/regress/stacks/trap0.bin.wast -test/regress/stacks/value_kinds0.bin.wast diff --git a/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 b/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 index 43e6d9585..a6b066b17 100644 --- a/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 +++ b/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 @@ -49,6 +49,17 @@ def X_ = void( TT("compress_from_host_call", test_compress_from_host_call), TT("decompress_traps_in_wasm", test_decompress_traps_in_wasm), TT("decompress_via_gc_wasm", test_decompress_via_gc_wasm), + T("victim_suspended_only", test_victim_suspended_only), + T("victim_round_robin", test_victim_round_robin), + T("victim_not_recycled", test_victim_not_recycled), + T("decompress_compresses_other", test_decompress_compresses_other), + T("disabled_no_compress", test_disabled_no_compress), + TT("gens_round_robin", test_gens_round_robin), + TT("gens_many", test_gens_many), + TT("gens_limit_one", test_gens_limit_one), + TT("gens_random_order", test_gens_random_order), + TT("chains_under_limit", test_chains_under_limit), + TT("deep_frames_under_limit", test_deep_frames_under_limit), () ); @@ -288,6 +299,7 @@ def GEN_MODULE: Array = [ private class StackCompressionTester(t: Tester, tiering: ExecutionStrategy) { def saved_max = StackTuning.maxStacks; + def saved_compression = StackTuning.enableCompression; var held: Array; // stacks pinned by {limit} var instance: Instance; var limit_total = -1; @@ -296,8 +308,14 @@ private class StackCompressionTester(t: Tester, tiering: ExecutionStrategy) { def restore() { StackTuning.maxStacks = saved_max; + StackTuning.enableCompression = saved_compression; limit_total = -1; } + // Enables compression under a limit of {extra} more segments. + def compressing(extra: int) { + StackTuning.enableCompression = true; + limit(extra); + } // Pins all existing segments, then allows at most {extra} more. def limit(extra: int) { StackTuning.maxStacks = M.total; @@ -825,3 +843,163 @@ def test_decompress_via_gc_wasm(t: StackCompressionTester) { t.dropFresh(); // decompression must force a GC to reclaim this segment t.t.assert_eq(1005, t.callInt("finish", [Value.Cont(k)])); } + +def test_victim_suspended_only(t: StackCompressionTester) { + var f = t.subFunc(); + t.compressing(4); + var empty = M.getFreshStack(); + var s1 = M.getFreshStack().reset(f).bind([Values.box_i(10), Values.box_i(1)]); + var s2 = M.getFreshStack().reset(f).bind([Values.box_i(20), Values.box_i(2)]); + var s3 = M.getFreshStack().reset(f).bind([Values.box_i(30), Values.box_i(3)]); + // All segments are now owned; the next allocation must compress a SUSPENDED stack. + var fresh = M.getFreshStack(); + t.assertState(StackState.EMPTY, empty); + t.assertState(StackState.EMPTY, fresh); + var compressed = 0; + for (s in [s1, s2, s3]) if (s.state() == StackState.COMPRESSED) compressed++; + t.t.assert_eq(1, compressed); + t.checkInvariants("victim"); + t.assertResult(Result.Value([Values.box_i(9)]), s1.resume()); + t.assertResult(Result.Value([Values.box_i(18)]), s2.resume()); + t.assertResult(Result.Value([Values.box_i(27)]), s3.resume()); +} + +def test_victim_round_robin(t: StackCompressionTester) { + var f = t.subFunc(); + t.compressing(4); + var stacks = Array.new(4); + for (i < stacks.length) stacks[i] = M.getFreshStack().reset(f).bind([Values.box_i(i), Values.box_i(0)]); + // Each further allocation compresses a different stack. + var fresh = Array.new(4); + for (i < fresh.length) { + fresh[i] = M.getFreshStack(); + var compressed = 0; + for (s in stacks) if (s.state() == StackState.COMPRESSED) compressed++; + t.t.assert_eq(i + 1, compressed); + } + for (i < stacks.length) t.assertResult(Result.Value([Values.box_i(i)]), stacks[i].resume()); +} + +def test_victim_not_recycled(t: StackCompressionTester) { + var f = t.subFunc(); + t.compressing(2); + var a = M.getFreshStack().reset(f).bind([Values.box_i(7), Values.box_i(2)]); + var b = M.getFreshStack().reset(f); + M.recycleStack(b); // a recycled stack is cleared, so never compressed + t.assertState(StackState.EMPTY, b); + var c = M.getFreshStack(); + if (c != b) t.t.fail("expected the recycled stack to be reused"); + t.assertState(StackState.SUSPENDED, a); + var d = M.getFreshStack(); // must compress {a}, the only SUSPENDED stack + t.assertState(StackState.COMPRESSED, a); + t.assertState(StackState.EMPTY, d); + if (d == a || d == c) t.t.fail("expected a new stack for the compressed segment"); + t.assertResult(Result.Value([Values.box_i(5)]), a.resume()); +} + +def test_decompress_compresses_other(t: StackCompressionTester) { + var f = t.subFunc(); + t.compressing(2); + var a = M.getFreshStack().reset(f).bind([Values.box_i(100), Values.box_i(1)]); + var b = M.getFreshStack().reset(f).bind([Values.box_i(200), Values.box_i(2)]); + t.assertCompress(a); + var hold = M.getFreshStack(); // takes the segment freed by {a} + // Decompressing {a} requires a segment, which can only come from compressing {b}. + t.assertDecompress(a); + t.assertState(StackState.COMPRESSED, b); + t.assertState(StackState.EMPTY, hold); + t.checkInvariants("ping"); + t.assertResult(Result.Value([Values.box_i(99)]), a.resume()); + t.assertResult(Result.Value([Values.box_i(198)]), b.resume()); +} + +def test_disabled_no_compress(t: StackCompressionTester) { + var f = t.subFunc(); + StackTuning.enableCompression = false; + t.limit(3); + var a = M.getFreshStack().reset(f); + var b = M.getFreshStack().reset(f); + var dropped = allocAndDrop(f); + var c = M.getFreshStack(); // must come from the dead stack, never compression + if (c.range.start != dropped) t.t.fail("expected the dead stack's segment to be reclaimed"); + t.assertState(StackState.SUSPENDED, a); + t.assertState(StackState.SUSPENDED, b); + t.assertState(StackState.EMPTY, c); +} +def allocAndDrop(f: Function) -> Pointer #no-inline { + return M.getFreshStack().reset(f).range.start; +} + +// Steps {n} generators round-robin to completion under a limit of {extra} segments. +def roundRobin(t: StackCompressionTester, n: int, extra: int) { + t.gen(); + t.compressing(extra); + var before = (C.numCompressed, C.numDecompressed); + var gens = Array.new(n); + for (i < n) gens[i] = t.make(i * 100); + for (j = 1; j < 10; j++) { + for (i < n) gens[i] = t.next(gens[i], i * 100 + j); + } + for (i < n) t.next(gens[i], -1); + if (C.numCompressed <= before.0) t.t.fail("expected stacks to be compressed"); + if (C.numDecompressed <= before.1) t.t.fail("expected stacks to be decompressed"); + t.checkInvariants("round robin"); +} + +def test_gens_round_robin(t: StackCompressionTester) { + roundRobin(t, 20, 4); +} + +def test_gens_many(t: StackCompressionTester) { + roundRobin(t, 500, 8); +} + +def test_gens_limit_one(t: StackCompressionTester) { + // Each call runs on its own stack and the generator needs another; one more suffices for decompression. + roundRobin(t, 10, 3); +} + +def test_gens_random_order(t: StackCompressionTester) { + t.gen(); + t.compressing(6); + var n = 30; + var gens = Array.new(n), steps = Array.new(n); + for (i < n) gens[i] = t.make(i * 100); + var live = n, seed = 12345; + while (live > 0) { + seed = (seed * 1103515245 + 12345) & 0x7FFFFFFF; + var i = (seed >> 8) % n; + if (Continuations.isNull(gens[i])) continue; + var j = ++steps[i]; + gens[i] = t.next(gens[i], if(j < 10, i * 100 + j, -1)); + if (j >= 10) live--; + } + t.checkInvariants("random"); +} + +def test_chains_under_limit(t: StackCompressionTester) { + t.gen(); + var n = 10; + // The waiting parents cannot be compressed, so leave room for them. + t.compressing(n + 4); + var before = C.numCompressed; + var chains = Array.new(n); + for (i < n) chains[i] = t.callCont("make_chain", [Values.box_i(i)]); + for (i < n) { // some tops may already have been compressed to make room + var top = t.stackOf(chains[i]); + if (top.state() == StackState.SUSPENDED) t.assertCompress(top); + t.assertState(StackState.COMPRESSED, top); + t.assertState(StackState.CALL_CHILD, NativeStack.!(top.parent)); + } + for (i = n - 1; i >= 0; i--) t.t.assert_eq(2 * i + 1, t.callInt("finish", [Value.Cont(chains[i])])); + if (C.numCompressed < before + n) t.t.fail("expected chains to be compressed"); +} + +def test_deep_frames_under_limit(t: StackCompressionTester) { + t.gen(); + t.compressing(4); + var n = 40; + var ks = Array.new(n); + for (i < n) ks[i] = t.callCont("make_deep", [Values.box_i(i)]); + for (i < n) t.t.assert_eq(deepSum(i), t.callInt("finish", [Value.Cont(ks[i])])); +} From eb0a630c514c13704839e9be181e8273479bb534 Mon Sep 17 00:00:00 2001 From: davidmaamoaix Date: Wed, 28 Oct 2026 09:59:01 -0400 Subject: [PATCH 06/10] cleanup --- src/engine/compiler/MacroAssembler.v3 | 2 +- src/engine/compiler/SinglePassCompiler.v3 | 5 +- src/engine/native/NativeStack.v3 | 5 +- src/engine/native/NativeStackCompression.v3 | 2 +- src/engine/native/NativeStackManager.v3 | 12 +- src/engine/x86-64/X86_64Interpreter.v3 | 1 - .../NativeStackCompressionTest.v3 | 174 +----------------- .../x86-64-linux/NativeStackManagerTest.v3 | 3 +- 8 files changed, 11 insertions(+), 193 deletions(-) diff --git a/src/engine/compiler/MacroAssembler.v3 b/src/engine/compiler/MacroAssembler.v3 index 8062a2899..e1f8312b7 100644 --- a/src/engine/compiler/MacroAssembler.v3 +++ b/src/engine/compiler/MacroAssembler.v3 @@ -380,7 +380,7 @@ class MacroAssembler(valuerep: Tagging, regConfig: RegConfig) { // stk.state_ = state; def emit_set_stack_state(stk: Reg, state: StackState); - // Branches to {label} if the state of the stack in {stk} is not {state}. + // if (stk.state_ != state) goto label; def emit_br_stack_state_ne(stk: Reg, state: StackState, label: MasmLabel); // Publishes the handler table. def emit_publish_handlers(r_stack: Reg, r_table: Reg, tier: HandlerTier) { diff --git a/src/engine/compiler/SinglePassCompiler.v3 b/src/engine/compiler/SinglePassCompiler.v3 index 75c847dd4..aa66b78e1 100644 --- a/src/engine/compiler/SinglePassCompiler.v3 +++ b/src/engine/compiler/SinglePassCompiler.v3 @@ -2427,9 +2427,8 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl masm.emit_mov_r_r(ValueKind.REF, dst, regs.vfp); // XXX: use 3-addr adjustment of VSP (i.e. lea) if (slots > 0) masm.emit_addw_r_i(dst, int.view(slots) * masm.valuerep.slot_size); } - // Decompresses the stack of the continuation on top of the value stack, if compressed. def emit_decompress_top_cont() { - state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS); // spilled values can be updated by GC + state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS); emit_compute_vsp(regs.vsp, state.sp); emit_spill_vsp(regs.vsp); masm.emit_store_curstack_vsp(regs.vsp); @@ -2443,7 +2442,7 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl masm.emit_push_NativeStack_rsp_r_r(regs.runtime_arg0); masm.emit_call_runtime_DECOMPRESS_CONT(); }; - withReconstructedInlinedFrames(emit); // the call can trap + withReconstructedInlinedFrames(emit); masm.emit_get_curstack(regs.scratch); masm.emit_pop_NativeStack_rsp_r_r(regs.scratch); emit_reload_regs(); diff --git a/src/engine/native/NativeStack.v3 b/src/engine/native/NativeStack.v3 index 5927705a4..df32b07be 100644 --- a/src/engine/native/NativeStack.v3 +++ b/src/engine/native/NativeStack.v3 @@ -6,7 +6,7 @@ // See diagram at the end of the file for mapping layout. class NativeStack extends WasmStack { def valuerep = Target.tagging; - var range: NativeStackSegment; // attached by {NativeStackManager.attach}, or null + var range: NativeStackSegment; var vsp: Pointer; var rsp: Pointer; var func: Function; // TODO: move into "enter-func-stub" @@ -46,7 +46,6 @@ class NativeStack extends WasmStack { def resetForFrameWrites(func: Function) -> this { reset0(func, 0); } - // Returns {true} if this stack has no frames yet and will enter {func} when resumed. def isAtFuncEntry() -> bool { return range != null && rsp < range.end && rsp.load() == STACK_ENTER_FUNC_STUB.getEntry() + NOP_ENTRY; } @@ -517,7 +516,7 @@ class NativeStack extends WasmStack { } // GC callback to scan (tagged) values on this stack def scan() { - if (range == null) return; // no segment attached + if (range == null) return; if (Trace.gc) { Trace.OUT.put1("scanStack obj=0x%x", u64.view(Pointer.atObject(this) - Pointer.NULL)).ln(); } diff --git a/src/engine/native/NativeStackCompression.v3 b/src/engine/native/NativeStackCompression.v3 index 057e8d758..c03beab36 100644 --- a/src/engine/native/NativeStackCompression.v3 +++ b/src/engine/native/NativeStackCompression.v3 @@ -41,7 +41,7 @@ component NativeStackCompression { } return false; } - // Compresses {stack}, which must satisfy {canCompress}, and returns its now unowned segment. + // Requires {canCompress(stack)}; returns the detached segment. def compress(stack: NativeStack) -> NativeStackSegment { var parent_rsp = stack.parent_rsp_ptr.load(); var c: CompressedStack; diff --git a/src/engine/native/NativeStackManager.v3 b/src/engine/native/NativeStackManager.v3 index 48874af99..4862caa3c 100644 --- a/src/engine/native/NativeStackManager.v3 +++ b/src/engine/native/NativeStackManager.v3 @@ -23,21 +23,19 @@ component NativeStackManager { result.next_stack = null; return result.clear(); } - // Makes {stack} the empty owner of {segment}, which no other stack may own. + // Requires {segment} to be unowned; leaves {stack} empty. def attach(stack: NativeStack, segment: NativeStackSegment) { stack.range = segment; segment.stack.set(stack); stack.clear(); if (Trace.stack && Debug.stack) Trace.OUT.put2("attachSegment start=0x%x, end=0x%x", segment.start - Pointer.NULL, segment.end - Pointer.NULL).ln(); } - // Takes the segment away from {stack}, which must not be used until another is attached. def detach(stack: NativeStack) -> NativeStackSegment { var segment = stack.range; segment.stack.set(null); stack.range = null; return segment; } - // Compresses {stack} if possible, adding its segment to the cache. def compressStack(stack: NativeStack) -> bool { if (!NativeStackCompression.canCompress(stack)) return false; addToCache(NativeStackCompression.compress(stack)); @@ -46,9 +44,8 @@ component NativeStackManager { // Gets an unowned segment, which must be attached before the next refill, or {null} if out of stacks. def acquireSegment() -> NativeStackSegment { var s = getFreshStack(); - return if(s == null, null, detach(s)); // {s} becomes garbage + return if(s == null, null, detach(s)); } - // Refills the cache with new segments, segments of dead stacks, or segments of compressed stacks. private def refill() -> bool { if (tryAllocatingNewStacks()) return true; if (tryReclaimingSegments() > 0) return true; @@ -63,7 +60,6 @@ component NativeStackManager { } return !collected && tryCollectingAndReclaiming(); } - // Wraps {segment} in a new cached stack that owns it. private def addToCache(segment: NativeStackSegment) { var stack = NativeStack.new(); attach(stack, segment); @@ -92,7 +88,6 @@ component NativeStackManager { } return cache != null; // OK if succeeded in allocating at least one } - // Returns the number of segments reclaimed from stacks that the GC found dead. private def tryReclaimingSegments() -> int { var count = 0; for (p = all; p != null; p = p.next) { @@ -140,9 +135,8 @@ component NativeStackManager { } } } - // Returns {stack} to the cache, clearing it so that it is never chosen for compression. def recycleStack(stack: NativeStack) { - stack.clear(); + stack.clear(); // so that it is never chosen for compression stack.next_stack = cache; cache = stack; } diff --git a/src/engine/x86-64/X86_64Interpreter.v3 b/src/engine/x86-64/X86_64Interpreter.v3 index 4c85a4a79..2f2c7d949 100644 --- a/src/engine/x86-64/X86_64Interpreter.v3 +++ b/src/engine/x86-64/X86_64Interpreter.v3 @@ -4158,7 +4158,6 @@ class X86_64InterpreterGen(ic: X86_64InterpreterCode, w: DataWriter) { asm.movq_r_m(t, t.plus(offsets.FuncHandlerInfo_resume_sites)); masm.emit_v3_Array_elem_r_rr(ValueKind.REF, r_tmp, r_tmp, xenv.curpc); } - // Decompresses the stack of the continuation on top of the value stack, if compressed. private def genDecompressTopCont() { var done = X86_64Label.new(); asm.movq_r_m(r_tmp1, vsph[-1].value); // %tmp1 = cont.stack diff --git a/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 b/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 index a6b066b17..29915d951 100644 --- a/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 +++ b/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 @@ -63,177 +63,7 @@ def X_ = void( () ); -// The module below, compiled with the stack-switching reference interpreter. -// (module -// (type $fv (func)) -// (type $cv (cont $fv)) -// (type $fi (func (param i32))) -// (type $ci (cont $fi)) -// (type $f_i (func (result i32))) -// (type $c_i (cont $f_i)) -// (type $fi_i (func (param i32) (result i32))) -// (type $ci_i (cont $fi_i)) -// (type $box (struct (field i32))) -// -// (tag $yield (param i32)) -// (tag $ask (result i32)) -// -// (global $last (mut i32) (i32.const -1)) -// -// ;; Recurses {n} frames deep, each holding a v128 and a struct, then yields {v}. -// (func $deep (param $n i32) (param $v i32) -// (local $x v128) (local $b (ref null $box)) -// (local.set $x (i32x4.splat (local.get $v))) -// (local.set $b (struct.new $box (local.get $v))) -// (if (i32.eqz (local.get $n)) -// (then (suspend $yield (local.get $v))) -// (else (call $deep (i32.sub (local.get $n) (i32.const 1)) (local.get $v)))) -// (if (i32.ne (i32x4.extract_lane 3 (local.get $x)) (local.get $v)) (then (unreachable))) -// (if (i32.ne (struct.get $box 0 (local.get $b)) (local.get $v)) (then (unreachable))) -// ) -// ;; Yields {start}, {start+1}, ..., {start+9} from 3 frames deep, checking i64 and f64 locals. -// (func $gen (param $start i32) -// (local $i i32) (local $l i64) (local $d f64) (local $v i32) -// (loop $loop -// (local.set $v (i32.add (local.get $start) (local.get $i))) -// (call $deep (i32.const 3) (local.get $v)) -// (local.set $l (i64.add (local.get $l) (i64.extend_i32_s (local.get $v)))) -// (local.set $d (f64.add (local.get $d) (f64.convert_i32_s (local.get $v)))) -// (if (i64.ne (local.get $l) (i64.trunc_f64_s (local.get $d))) (then (unreachable))) -// (local.set $i (i32.add (local.get $i) (i32.const 1))) -// (br_if $loop (i32.lt_u (local.get $i) (i32.const 10))) -// ) -// ) -// ;; Yields once, then returns {x + 1000}. -// (func $once (param $x i32) (result i32) -// (suspend $yield (local.get $x)) -// (i32.add (local.get $x) (i32.const 1000)) -// ) -// ;; Asks for a value, then returns {x * 10 + answer}. -// (func $asker (param $x i32) (result i32) -// (i32.add (i32.mul (local.get $x) (i32.const 10)) (suspend $ask)) -// ) -// (elem declare func $gen $once $asker) -// -// ;; Resumes {k} until it yields (stores the value in $last and returns the new continuation) or returns (null). -// (func $step (param $k (ref $cv)) (result (ref null $cv)) -// (local $next (ref null $cv)) -// (block $h (result i32 (ref $cv)) -// (resume $cv (on $yield $h) (local.get $k)) -// (return (ref.null $cv)) -// ) -// (local.set $next) -// (global.set $last) -// (local.get $next) -// ) -// (func (export "make") (param $start i32) (result (ref null $cv)) -// (call $step (cont.bind $ci $cv (local.get $start) (cont.new $ci (ref.func $gen)))) -// ) -// (func (export "next") (param $k (ref null $cv)) (result (ref null $cv)) -// (call $step (ref.as_non_null (local.get $k))) -// ) -// (func (export "last") (result i32) (global.get $last)) -// -// ;; Returns a continuation of {$once} suspended at its yield. -// (func (export "make_once") (param $x i32) (result (ref null $c_i)) -// (local $k (ref null $c_i)) -// (block $h (result i32 (ref $c_i)) -// (drop (resume $ci_i (on $yield $h) (local.get $x) (cont.new $ci_i (ref.func $once)))) -// (return (ref.null $c_i)) -// ) -// (local.set $k) -// (drop) -// (local.get $k) -// ) -// ;; Returns a continuation of {$asker} suspended at its question. -// (func (export "make_asker") (param $x i32) (result (ref null $ci_i)) -// (block $h (result (ref $ci_i)) -// (drop (resume $ci_i (on $ask $h) (local.get $x) (cont.new $ci_i (ref.func $asker)))) -// (return (ref.null $ci_i)) -// ) -// ) -// ;; Answers the question of {k} with {v} via cont.bind, then resumes it. -// (func (export "answer") (param $k (ref null $ci_i)) (param $v i32) (result i32) -// (resume $c_i (cont.bind $ci_i $c_i (local.get $v) (local.get $k))) -// ) -// ;; --- Chains, deep frames, traps, and exceptions --- -// (tag $other) -// (tag $exn (param i32)) -// ;; Suspends with {yield}, which its parent {mid} does not handle, then returns {2 * x}. -// (func $inner (param $x i32) (result i32) -// (suspend $yield (local.get $x)) -// (i32.mul (local.get $x) (i32.const 2)) -// ) -// ;; Resumes {inner} with a handler for an unrelated tag, then returns its result + 1. -// (func $mid (param $x i32) (result i32) -// (block $h (result (ref $c_i)) -// (return (i32.add (i32.const 1) -// (resume $ci_i (on $other $h) (local.get $x) (cont.new $ci_i (ref.func $inner))))) -// ) -// (unreachable) -// ) -// ;; Recurses {d} frames deep, each with a local, then yields {x}, then returns the sum of locals. -// (func $deep_sum (param $d i32) (param $x i32) (result i32) -// (local $mine i32) -// (local.set $mine (i32.add (local.get $d) (local.get $x))) -// (if (result i32) (i32.eqz (local.get $d)) -// (then (suspend $yield (local.get $x)) (local.get $mine)) -// (else (i32.add (local.get $mine) (call $deep_sum (i32.sub (local.get $d) (i32.const 1)) (local.get $x))))) -// ) -// (func $deep_entry (param $x i32) (result i32) -// (call $deep_sum (i32.const 100) (local.get $x)) -// ) -// ;; Yields, then traps. -// (func $trapper (param $x i32) (result i32) -// (suspend $yield (local.get $x)) -// (unreachable) -// ) -// ;; Yields, then throws {x}. -// (func $thrower (param $x i32) (result i32) -// (suspend $yield (local.get $x)) -// (throw $exn (local.get $x)) -// ) -// (elem declare func $inner $mid $deep_entry $trapper $thrower) -// -// ;; Runs {f(x)} until it yields, returning the suspended continuation. -// (func $start (param $f (ref $ci_i)) (param $x i32) (result (ref null $c_i)) -// (local $k (ref null $c_i)) -// (block $h (result i32 (ref $c_i)) -// (drop (resume $ci_i (on $yield $h) (local.get $x) (local.get $f))) -// (return (ref.null $c_i)) -// ) -// (local.set $k) -// (global.set $last) -// (local.get $k) -// ) -// ;; Returns a continuation whose top stack is {inner} and whose bottom stack is {mid}. -// (func (export "make_chain") (param $x i32) (result (ref null $c_i)) -// (call $start (cont.new $ci_i (ref.func $mid)) (local.get $x)) -// ) -// (func (export "make_deep") (param $x i32) (result (ref null $c_i)) -// (call $start (cont.new $ci_i (ref.func $deep_entry)) (local.get $x)) -// ) -// (func (export "make_trapper") (param $x i32) (result (ref null $c_i)) -// (call $start (cont.new $ci_i (ref.func $trapper)) (local.get $x)) -// ) -// (func (export "make_thrower") (param $x i32) (result (ref null $c_i)) -// (call $start (cont.new $ci_i (ref.func $thrower)) (local.get $x)) -// ) -// ;; Resumes {k} to completion. -// (func (export "finish") (param $k (ref null $c_i)) (result i32) -// (resume $c_i (local.get $k)) -// ) -// ;; Resumes {k} to completion, returning the payload of an exception it throws. -// (func (export "catch") (param $k (ref null $c_i)) (result i32) -// (block $c (result i32) -// (try_table (catch $exn $c) (drop (resume $c_i (local.get $k)))) -// (i32.const -1) -// ) -// ) -// (func (export "resume_null") (result i32) -// (resume $c_i (ref.null $c_i)) -// ) -// ) +// Generators, chains, deep frames, a trapper and a thrower, driven through the exports below. def GEN_MODULE: Array = [ 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0xe9, 0x80, 0x80, 0x80, 0x00, 0x15, 0x60, 0x00, 0x00, 0x5d, 0x00, 0x60, 0x01, 0x7f, 0x00, 0x5d, 0x02, 0x60, 0x00, 0x01, 0x7f, 0x5d, 0x04, @@ -379,7 +209,6 @@ private class StackCompressionTester(t: Tester, tiering: ExecutionStrategy) { var inst = Instantiator.new(Extension.set.all, b.module, [], err).run(); return WasmFunction.!(inst.functions[0]); } - // Instantiates {GEN_MODULE} for this tester's tier. def gen() -> this { if (instance != null) return; var err = ErrorGen.new("GEN_MODULE"); @@ -470,7 +299,6 @@ private class StackCompressionTester(t: Tester, tiering: ExecutionStrategy) { } } -// Runs a test, then restores global settings and checks invariants. def runTest(f: StackCompressionTester -> void, t: StackCompressionTester) { f(t); t.checkInvariants("end of test"); diff --git a/test/unittest/x86-64-linux/NativeStackManagerTest.v3 b/test/unittest/x86-64-linux/NativeStackManagerTest.v3 index f5fc22a51..ad9ec5973 100644 --- a/test/unittest/x86-64-linux/NativeStackManagerTest.v3 +++ b/test/unittest/x86-64-linux/NativeStackManagerTest.v3 @@ -98,7 +98,7 @@ private class NativeStackManagerTester extends ExeTester { } return r; } - // Moves a fresh stack's segment to a new stack and drops the fresh stack, returning the new owner. + // Not inlined, so that the fresh stack is dead once this returns. def moveSegmentToNewStack() -> NativeStack #no-inline { var a = fresh(); var b = NativeStack.new(); @@ -106,7 +106,6 @@ private class NativeStackManagerTester extends ExeTester { if (a.range != null) t.fail("expected detached stack to have no segment"); return b; } - // Checks that a moved segment is kept while its new owner lives, then drops the new owner. def checkMovedSegment() -> NativeStackSegment #no-inline { var b = moveSegmentToNewStack(); var seg = b.range; From 2f7f1cb19c511adfe3176a6238459e2acf17dfe6 Mon Sep 17 00:00:00 2001 From: davidmaamoaix Date: Wed, 28 Oct 2026 10:36:04 -0400 Subject: [PATCH 07/10] more cleanup --- src/engine/WasmStack.v3 | 2 +- src/engine/native/NativeStack.v3 | 6 +- src/engine/native/NativeStackCompression.v3 | 44 +----- test/regress/stacks/compression.failures | 3 + .../NativeStackCompressionTest.v3 | 145 ++++++------------ 5 files changed, 57 insertions(+), 143 deletions(-) diff --git a/src/engine/WasmStack.v3 b/src/engine/WasmStack.v3 index d5bcbfb20..92782ba66 100644 --- a/src/engine/WasmStack.v3 +++ b/src/engine/WasmStack.v3 @@ -98,7 +98,7 @@ enum StackState { SUSPENDED, // contains one or more frames, waiting for inputs CALL_CHILD, // waiting for child stack to return RUNNING, // currently running Wasm or host code - COMPRESSED, // suspended, with its frames or bound values held in the heap + COMPRESSED, // suspended, with its frames held in the heap } // Provides access to the state of an executing frame. diff --git a/src/engine/native/NativeStack.v3 b/src/engine/native/NativeStack.v3 index df32b07be..5a49aa046 100644 --- a/src/engine/native/NativeStack.v3 +++ b/src/engine/native/NativeStack.v3 @@ -13,7 +13,7 @@ class NativeStack extends WasmStack { var parent_rsp_ptr: Pointer; var return_results: Array; var state_: StackState = StackState.EMPTY; - var compressed: CompressedStack; // frames or bound entry values while {COMPRESSED} + var compressed: CompressedStack; // frames while {COMPRESSED} var saved_parent_rsp: Pointer; // contents of the parent rsp slot while {COMPRESSED} // Handler table (for inline suspend/switch). @@ -46,14 +46,10 @@ class NativeStack extends WasmStack { def resetForFrameWrites(func: Function) -> this { reset0(func, 0); } - def isAtFuncEntry() -> bool { - return range != null && rsp < range.end && rsp.load() == STACK_ENTER_FUNC_STUB.getEntry() + NOP_ENTRY; - } // Requires {state == SUSPENDED}. // Pushes {args} incrementally onto the value stack. def bind(args: Range) -> this { if (args.length == 0) return; - if (NativeStackCompression.bindEntry(this, args)) return; if (!NativeStackCompression.decompress(this)) fatal("out of stacks while binding a compressed stack"); checkState("bind()", StackState.SUSPENDED); for (v in args) push(v); diff --git a/src/engine/native/NativeStackCompression.v3 b/src/engine/native/NativeStackCompression.v3 index c03beab36..47d943c5c 100644 --- a/src/engine/native/NativeStackCompression.v3 +++ b/src/engine/native/NativeStackCompression.v3 @@ -1,18 +1,12 @@ // Copyright 2026 Wizard authors. All rights reserved. // See LICENSE for details of Apache 2.0 license. -// Values bound to the function of a {COMPRESSED} stack that has not been entered. -class EntryValues(vals: Array) extends CompressedStack { - def name() -> string { return "Entry Values"; } - def size() -> u64 { return 32uL + 24uL * u64.view(vals.length); } -} - // Compresses suspended stacks into the heap to free their segments, and decompresses them before use. component NativeStackCompression { var numCompressed: int; var numDecompressed: int; - // Returns {true} if {stack} is suspended at a function entry or with only interpreter and SPC frames. + // Returns {true} if {stack} is suspended with only interpreter and SPC frames. def canCompress(stack: NativeStack) -> bool { if (stack.state_ != StackState.SUSPENDED) return false; var range = stack.range; @@ -20,7 +14,6 @@ component NativeStackCompression { var end = range.end; // [end - 8] holds the parent's rsp, [end - 16] the return-parent stub. if (stack.parent_rsp_ptr != end + -Pointer.SIZE) return false; - if (stack.isAtFuncEntry()) return stack.rsp == end + -3 * Pointer.SIZE; var srp = stack.rsp, count = 0, bottom_sfp = Pointer.NULL; while (srp >= range.start && srp < end) { var retip = srp.load(); @@ -36,7 +29,7 @@ component NativeStackCompression { // Values below the bottom frame would be lost. return NativeFrameHandle.Interpreter(bottom_sfp).vfp() == range.start; } - _ => return false; // inlined frames, stubs, or host frames + _ => return false; // the enter-func stub, inlined frames, trap stubs, or host frames } } return false; @@ -44,17 +37,7 @@ component NativeStackCompression { // Requires {canCompress(stack)}; returns the detached segment. def compress(stack: NativeStack) -> NativeStackSegment { var parent_rsp = stack.parent_rsp_ptr.load(); - var c: CompressedStack; - if (stack.isAtFuncEntry()) { - var n = int.!((stack.vsp - stack.range.start) / stack.valuerep.slot_size); - if (n > 0) { - var vals = Array.new(n); - for (i < n) vals[i] = stack.readValue(stack.range.start, i); - c = EntryValues.new(vals); - } - } else { - c = StackCompression.compressStack(stack); - } + var c = StackCompression.compressStack(stack); var segment = NativeStackManager.detach(stack); if (Trace.stack) Trace.OUT.put2("compress stack @ 0x%x, freeing segment 0x%x", Pointer.atObject(stack) - Pointer.NULL, segment.start - Pointer.NULL).ln(); @@ -66,18 +49,6 @@ component NativeStackCompression { numCompressed++; return segment; } - // Binds {args} to a {COMPRESSED} stack that has not been entered, without decompressing it. - def bindEntry(stack: NativeStack, args: Range) -> bool { - if (stack.state_ != StackState.COMPRESSED) return false; - var vals: Array; - match (stack.compressed) { - null => vals = Ranges.dup(args); - x: EntryValues => vals = Arrays.concat(x.vals, Ranges.dup(args)); - _ => return false; - } - stack.compressed = EntryValues.new(vals); - return true; - } // Decompresses {stack}, if compressed, onto a fresh segment; returns {false} if none is available. def decompress(stack: NativeStack) -> bool { if (stack.state_ != StackState.COMPRESSED) return true; @@ -88,14 +59,7 @@ component NativeStackCompression { if (Trace.stack) Trace.OUT.put2("decompress stack @ 0x%x onto segment 0x%x", Pointer.atObject(stack) - Pointer.NULL, segment.start - Pointer.NULL).ln(); NativeStackManager.attach(stack, segment); - match (c) { - null => stack.reset(func); - x: EntryValues => { - stack.reset(func); - for (v in x.vals) stack.push(v); - } - _ => Target.writeFramesToStack(stack, StackCompression.decompressFrames(c)); - } + Target.writeFramesToStack(stack, StackCompression.decompressFrames(c)); stack.func = func; stack.return_results = results; stack.parent = parent; diff --git a/test/regress/stacks/compression.failures b/test/regress/stacks/compression.failures index 6920a6c85..b0a96c73d 100644 --- a/test/regress/stacks/compression.failures +++ b/test/regress/stacks/compression.failures @@ -1,2 +1,5 @@ test/regress/ext:stack-switching/resume_chain3.bin.wast test/regress/ext:stack-switching/suspend15.bin.wast +test/regress/stacks/scheduler0.bin.wast +test/regress/stacks/switch_ring0.bin.wast +test/regress/stacks/value_kinds0.bin.wast diff --git a/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 b/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 index 29915d951..0deecb566 100644 --- a/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 +++ b/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 @@ -16,12 +16,9 @@ def TT(name: string, f: StackCompressionTester -> void) { } def X_ = void( T("can_compress_states", test_can_compress_states), - T("roundtrip_entry", test_roundtrip_entry), - T("roundtrip_entry_bind", test_roundtrip_entry_bind), - T("roundtrip_refs_gc", test_roundtrip_refs_gc), + T("roundtrip", test_roundtrip), T("no_frames_not_compressible", test_no_frames_not_compressible), T("running_not_compressible", test_running_not_compressible), - T("entry_host_func", test_entry_host_func), T("repeat_roundtrip", test_repeat_roundtrip), T("acquire_segment", test_acquire_segment), T("decompress_out_of_stacks", test_decompress_out_of_stacks), @@ -45,7 +42,6 @@ def X_ = void( TT("trap_after_decompress_wasm", test_trap_after_decompress_wasm), TT("throw_after_decompress_wasm", test_throw_after_decompress_wasm), TT("resume_null", test_resume_null), - TT("entry_cont_arg", test_entry_cont_arg), TT("compress_from_host_call", test_compress_from_host_call), TT("decompress_traps_in_wasm", test_decompress_traps_in_wasm), TT("decompress_via_gc_wasm", test_decompress_via_gc_wasm), @@ -305,66 +301,44 @@ def runTest(f: StackCompressionTester -> void, t: StackCompressionTester) { t.restore(); } -def EMPTY_STRUCT = StructDecl.new(false, ValueTypes.NO_HEAPTYPES, []); -def obj() -> Value { return Value.Ref(ObjectModel.newStruct(EMPTY_STRUCT)); } // The result of {make_deep}: the sum of (d + x) for d in 0..100. def deepSum(x: int) -> int { return 5050 + 101 * x; } def test_can_compress_states(t: StackCompressionTester) { - var f = t.subFunc(); var s = M.getFreshStack(); t.assertState(StackState.EMPTY, s); if (C.canCompress(s)) t.t.fail("expected EMPTY stack not to be compressible"); - s.reset(f); + s.reset(t.subFunc()).bind([Values.box_i(9)]); t.assertState(StackState.SUSPENDED, s); - if (!C.canCompress(s)) t.t.fail("expected SUSPENDED stack to be compressible"); - s.bind([Values.box_i(9)]); - if (!C.canCompress(s)) t.t.fail("expected SUSPENDED stack with bound values to be compressible"); - t.assertCompress(s); - if (C.canCompress(s)) t.t.fail("expected COMPRESSED stack not to be compressible"); - if (M.compressStack(s)) t.t.fail("expected compressing a COMPRESSED stack to fail"); - t.assertDecompress(s); - if (!C.canCompress(s)) t.t.fail("expected decompressed stack to be compressible again"); + if (C.canCompress(s)) t.t.fail("expected a stack at function entry not to be compressible"); + if (M.compressStack(s)) t.t.fail("expected compressing a stack at function entry to fail"); t.assertResult(Result.Value([Values.box_i(5)]), s.bind([Values.box_i(4)]).resume()); t.assertState(StackState.EMPTY, s); -} - -def test_roundtrip_entry(t: StackCompressionTester) { - var f = t.subFunc(); - var s = M.getFreshStack().reset(f).bind([Values.box_i(100), Values.box_i(1)]); + var w = t.suspended("make_once", 5); + if (!C.canCompress(w)) t.t.fail("expected a SUSPENDED stack with frames to be compressible"); + t.assertCompress(w); + if (C.canCompress(w)) t.t.fail("expected COMPRESSED stack not to be compressible"); + if (M.compressStack(w)) t.t.fail("expected compressing a COMPRESSED stack to fail"); + t.assertDecompress(w); + if (!C.canCompress(w)) t.t.fail("expected decompressed stack to be compressible again"); + t.assertResult(Result.Value([Values.box_i(1005)]), w.resume()); +} + +def test_roundtrip(t: StackCompressionTester) { + var s = t.suspended("make_once", 100); var before = (C.numCompressed, C.numDecompressed); t.assertCompress(s); t.t.assert_eq(before.0 + 1, C.numCompressed); - t.assertResult(Result.Value([Values.box_i(99)]), s.resume()); // decompresses + t.assertResult(Result.Value([Values.box_i(1100)]), s.resume()); // decompresses t.t.assert_eq(before.1 + 1, C.numDecompressed); } -def test_roundtrip_entry_bind(t: StackCompressionTester) { - var f = t.subFunc(); - var s = M.getFreshStack().reset(f).bind([Values.box_i(50)]); - t.assertCompress(s); - s.bind([Values.box_i(8)]); // binds without decompressing - t.assertState(StackState.COMPRESSED, s); - t.assertResult(Result.Value([Values.box_i(42)]), s.resume()); -} -def test_roundtrip_refs_gc(t: StackCompressionTester) { - var b = ModuleBuilder.new(); - var sig = b.newSig([ValueTypes.ANYREF, ValueTypes.ANYREF], [ValueTypes.ANYREF, ValueTypes.ANYREF]); - b.sig(sig).code([u8.!(Opcode.LOCAL_GET.code), 1, u8.!(Opcode.LOCAL_GET.code), 0]); - var err = ErrorGen.new("swap"); - var f = Instantiator.new(Extension.set.all, b.module, [], err).run().functions[0]; - var a = obj(), c = obj(); - var s = M.getFreshStack().reset(f).bind([a, c]); - t.assertCompress(s); - for (i < 3) Target.forceGC(); // the compressed values must keep the objects alive and be updated - t.assertResult(Result.Value([c, a]), s.resume()); -} def test_no_frames_not_compressible(t: StackCompressionTester) { var s = M.getFreshStack().resetForFrameWrites(t.subFunc()); t.assertState(StackState.SUSPENDED, s); - if (C.canCompress(s)) t.t.fail("expected a stack with no frames and no entry not to be compressible"); + if (C.canCompress(s)) t.t.fail("expected a stack with no frames not to be compressible"); s.clear(); } @@ -380,16 +354,9 @@ def checkRunning(t: StackCompressionTester, args: Range) -> HostResult { return HostResult.Value1(Values.box_i(7)); } -def test_entry_host_func(t: StackCompressionTester) { - var hf = HostFunction.new("add", SigCache.ii_i, fun args => HostResult.Value1(Values.box_i(Values.unbox_i(args[0]) + Values.unbox_i(args[1])))); - var s = M.getFreshStack().reset(hf).bind([Values.box_i(30)]); - t.assertCompress(s); - t.assertResult(Result.Value([Values.box_i(42)]), s.bind([Values.box_i(12)]).resume()); -} def test_repeat_roundtrip(t: StackCompressionTester) { - var f = t.subFunc(); - var s = M.getFreshStack().reset(f).bind([Values.box_i(1000)]); + var s = t.suspended("make_once", 1000); var before = (C.numCompressed, C.numDecompressed); for (i < 50) { t.assertCompress(s); @@ -399,7 +366,7 @@ def test_repeat_roundtrip(t: StackCompressionTester) { t.t.assert_eq(before.0 + 50, C.numCompressed); t.t.assert_eq(before.1 + 50, C.numDecompressed); t.checkInvariants("roundtrip"); - t.assertResult(Result.Value([Values.box_i(999)]), s.bind([Values.box_i(1)]).resume()); + t.assertResult(Result.Value([Values.box_i(2000)]), s.resume()); } def test_acquire_segment(t: StackCompressionTester) { @@ -418,27 +385,25 @@ def test_acquire_segment(t: StackCompressionTester) { } def test_decompress_out_of_stacks(t: StackCompressionTester) { - var f = t.subFunc(); - t.limit(1); - var s = M.getFreshStack().reset(f).bind([Values.box_i(10), Values.box_i(3)]); + var s = t.suspended("make_once", 10); + t.limit(0); t.assertCompress(s); var hog = M.getFreshStack(); // takes the only segment if (hog == null) return t.t.fail("expected the freed segment to be available"); t.assertTrap(TrapReason.OUT_OF_STACKS, s.resume()); t.assertState(StackState.COMPRESSED, s); M.recycleStack(hog); - t.assertResult(Result.Value([Values.box_i(7)]), s.resume()); + t.assertResult(Result.Value([Values.box_i(1010)]), s.resume()); } def test_version_kept(t: StackCompressionTester) { - var f = t.subFunc(); - var s = M.getFreshStack().reset(f).bind([Values.box_i(10), Values.box_i(3)]); + var s = t.suspended("make_once", 10); var k = Continuations.makeContinuation(s); t.assertCompress(s); if (Continuations.isUsed(k)) t.t.fail("expected compression not to consume the continuation"); t.assertDecompress(s); if (Continuations.isUsed(k)) t.t.fail("expected decompression not to consume the continuation"); - t.assertResult(Result.Value([Values.box_i(7)]), s.resume()); + t.assertResult(Result.Value([Values.box_i(1010)]), s.resume()); } def test_mid_host_resume(t: StackCompressionTester) { @@ -623,15 +588,6 @@ def test_resume_null(t: StackCompressionTester) { t.assertTrap(TrapReason.NULL_DEREF, t.call("resume_null", [])); } -def test_entry_cont_arg(t: StackCompressionTester) { - // A compressed entry stack whose bound argument is the continuation of another compressed stack. - var k = t.callCont("make_deep", [Values.box_i(2)]); - var s = M.getFreshStack().reset(t.exported("finish")).bind([Value.Cont(k)]); - t.assertCompress(t.stackOf(k)); - t.assertCompress(s); - for (i < 3) Target.forceGC(); - t.assertResult(Result.Value([Values.box_i(deepSum(2))]), s.resume()); -} def test_compress_from_host_call(t: StackCompressionTester) { // Stacks are compressed while Wasm frames are live on other stacks. @@ -673,12 +629,9 @@ def test_decompress_via_gc_wasm(t: StackCompressionTester) { } def test_victim_suspended_only(t: StackCompressionTester) { - var f = t.subFunc(); - t.compressing(4); + var s1 = t.suspended("make_once", 10), s2 = t.suspended("make_once", 20), s3 = t.suspended("make_once", 30); + t.compressing(1); var empty = M.getFreshStack(); - var s1 = M.getFreshStack().reset(f).bind([Values.box_i(10), Values.box_i(1)]); - var s2 = M.getFreshStack().reset(f).bind([Values.box_i(20), Values.box_i(2)]); - var s3 = M.getFreshStack().reset(f).bind([Values.box_i(30), Values.box_i(3)]); // All segments are now owned; the next allocation must compress a SUSPENDED stack. var fresh = M.getFreshStack(); t.assertState(StackState.EMPTY, empty); @@ -687,16 +640,17 @@ def test_victim_suspended_only(t: StackCompressionTester) { for (s in [s1, s2, s3]) if (s.state() == StackState.COMPRESSED) compressed++; t.t.assert_eq(1, compressed); t.checkInvariants("victim"); - t.assertResult(Result.Value([Values.box_i(9)]), s1.resume()); - t.assertResult(Result.Value([Values.box_i(18)]), s2.resume()); - t.assertResult(Result.Value([Values.box_i(27)]), s3.resume()); + M.recycleStack(empty); + M.recycleStack(fresh); + t.assertResult(Result.Value([Values.box_i(1010)]), s1.resume()); + t.assertResult(Result.Value([Values.box_i(1020)]), s2.resume()); + t.assertResult(Result.Value([Values.box_i(1030)]), s3.resume()); } def test_victim_round_robin(t: StackCompressionTester) { - var f = t.subFunc(); - t.compressing(4); var stacks = Array.new(4); - for (i < stacks.length) stacks[i] = M.getFreshStack().reset(f).bind([Values.box_i(i), Values.box_i(0)]); + for (i < stacks.length) stacks[i] = t.suspended("make_once", i); + t.compressing(0); // Each further allocation compresses a different stack. var fresh = Array.new(4); for (i < fresh.length) { @@ -705,14 +659,13 @@ def test_victim_round_robin(t: StackCompressionTester) { for (s in stacks) if (s.state() == StackState.COMPRESSED) compressed++; t.t.assert_eq(i + 1, compressed); } - for (i < stacks.length) t.assertResult(Result.Value([Values.box_i(i)]), stacks[i].resume()); + for (s in fresh) M.recycleStack(s); + for (i < stacks.length) t.assertResult(Result.Value([Values.box_i(1000 + i)]), stacks[i].resume()); } def test_victim_not_recycled(t: StackCompressionTester) { - var f = t.subFunc(); - t.compressing(2); - var a = M.getFreshStack().reset(f).bind([Values.box_i(7), Values.box_i(2)]); - var b = M.getFreshStack().reset(f); + var a = t.suspended("make_once", 7), b = t.suspended("make_once", 1); + t.compressing(0); M.recycleStack(b); // a recycled stack is cleared, so never compressed t.assertState(StackState.EMPTY, b); var c = M.getFreshStack(); @@ -722,14 +675,14 @@ def test_victim_not_recycled(t: StackCompressionTester) { t.assertState(StackState.COMPRESSED, a); t.assertState(StackState.EMPTY, d); if (d == a || d == c) t.t.fail("expected a new stack for the compressed segment"); - t.assertResult(Result.Value([Values.box_i(5)]), a.resume()); + M.recycleStack(c); + M.recycleStack(d); + t.assertResult(Result.Value([Values.box_i(1007)]), a.resume()); } def test_decompress_compresses_other(t: StackCompressionTester) { - var f = t.subFunc(); - t.compressing(2); - var a = M.getFreshStack().reset(f).bind([Values.box_i(100), Values.box_i(1)]); - var b = M.getFreshStack().reset(f).bind([Values.box_i(200), Values.box_i(2)]); + var a = t.suspended("make_once", 100), b = t.suspended("make_once", 200); + t.compressing(0); t.assertCompress(a); var hold = M.getFreshStack(); // takes the segment freed by {a} // Decompressing {a} requires a segment, which can only come from compressing {b}. @@ -737,17 +690,15 @@ def test_decompress_compresses_other(t: StackCompressionTester) { t.assertState(StackState.COMPRESSED, b); t.assertState(StackState.EMPTY, hold); t.checkInvariants("ping"); - t.assertResult(Result.Value([Values.box_i(99)]), a.resume()); - t.assertResult(Result.Value([Values.box_i(198)]), b.resume()); + t.assertResult(Result.Value([Values.box_i(1100)]), a.resume()); + t.assertResult(Result.Value([Values.box_i(1200)]), b.resume()); } def test_disabled_no_compress(t: StackCompressionTester) { - var f = t.subFunc(); + var a = t.suspended("make_once", 1), b = t.suspended("make_once", 2); StackTuning.enableCompression = false; - t.limit(3); - var a = M.getFreshStack().reset(f); - var b = M.getFreshStack().reset(f); - var dropped = allocAndDrop(f); + t.limit(1); + var dropped = allocAndDrop(t.subFunc()); var c = M.getFreshStack(); // must come from the dead stack, never compression if (c.range.start != dropped) t.t.fail("expected the dead stack's segment to be reclaimed"); t.assertState(StackState.SUSPENDED, a); From 1eb7e983982c33e6fbf277a5688003258cd7c182 Mon Sep 17 00:00:00 2001 From: davidmaamoaix Date: Wed, 28 Oct 2026 13:46:03 -0400 Subject: [PATCH 08/10] [stack] Reclaim dead segments and compress stacks in a single sweep --- src/engine/native/NativeStackManager.v3 | 59 ++++--------------- .../NativeStackCompressionTest.v3 | 42 ++++++------- 2 files changed, 34 insertions(+), 67 deletions(-) diff --git a/src/engine/native/NativeStackManager.v3 b/src/engine/native/NativeStackManager.v3 index 4862caa3c..bbcfd65bc 100644 --- a/src/engine/native/NativeStackManager.v3 +++ b/src/engine/native/NativeStackManager.v3 @@ -9,14 +9,10 @@ component NativeStackManager { var all: NativeStackSegment; // all segments ever mapped; never unmapped var total: int; // number of segments in {all} var cache: NativeStack; // used in fast-path allocation in generated code - private var sweep: NativeStackSegment; // clock hand for compression victims - private var allocsSinceGC: int; // stacks requested since the last forced GC - private var gcInterval: int; // requests to wait before collecting ahead of compressing // Gets a stack with a segment, or {null} if none is available. def getFreshStack() -> NativeStack { if (Trace.stack) Trace.OUT.puts("Requested fresh x86_stack...\n"); - allocsSinceGC++; if (cache == null && !refill()) return null; var result = cache; cache = cache.next_stack; @@ -46,19 +42,13 @@ component NativeStackManager { var s = getFreshStack(); return if(s == null, null, detach(s)); } + // Refills the cache from new segments or a sweep, forcing a GC if the first sweep finds nothing. private def refill() -> bool { if (tryAllocatingNewStacks()) return true; - if (tryReclaimingSegments() > 0) return true; - var collected = false; - if (StackTuning.enableCompression) { - // Collect before compressing if collecting has been productive lately. - if (allocsSinceGC >= gcInterval) { - collected = true; - if (tryCollectingAndReclaiming()) return true; - } - if (tryCompressingStacks()) return true; - } - return !collected && tryCollectingAndReclaiming(); + if (sweep() > 0) return true; + if (all == null) return false; + Target.forceGC(); // weak references are only cleared by a GC + return sweep() > 0; } private def addToCache(segment: NativeStackSegment) { var stack = NativeStack.new(); @@ -88,43 +78,18 @@ component NativeStackManager { } return cache != null; // OK if succeeded in allocating at least one } - private def tryReclaimingSegments() -> int { + // Caches the segments of dead stacks and, with compression enabled, of every compressible stack. + private def sweep() -> int { var count = 0; for (p = all; p != null; p = p.next) { - if (p.stack.get() == null) { // weak reference was cleared by GC, can reclaim - addToCache(p); - count++; - } + var s = p.stack.get(); + if (s == null) addToCache(p); + else if (StackTuning.enableCompression && NativeStackCompression.canCompress(s)) addToCache(NativeStackCompression.compress(s)); + else continue; + count++; } return count; } - // Weak references are only cleared by a GC, so force one to find dead stacks. - private def tryCollectingAndReclaiming() -> bool { - if (all == null) return false; - Target.forceGC(); - allocsSinceGC = 0; - var count = tryReclaimingSegments(); - // Back off exponentially while collecting is unproductive. - gcInterval = if(count * 4 >= total, total, Ints.min(gcInterval * 2 + 1, 1 << 24)); - return count > 0; - } - // Compresses one suspended stack, sweeping round-robin over all segments. - private def tryCompressingStacks() -> bool { - for (i < total) { - if (sweep == null) sweep = all; - var segment = sweep; - sweep = segment.next; - var s = segment.stack.get(); - if (s == null) { // its stack died since the last reclamation - addToCache(segment); - return true; - } - if (!NativeStackCompression.canCompress(s)) continue; - addToCache(NativeStackCompression.compress(s)); - return true; - } - return false; - } // Checks whether a fault at {addr} hit the guard page above a stack segment, which indicates // an engine bug, such as reading a frame that is not there. If so, fails with an error. def checkGuardPageFault(addr: Pointer, ip: Pointer) { diff --git a/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 b/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 index 0deecb566..c257a967e 100644 --- a/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 +++ b/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 @@ -45,9 +45,9 @@ def X_ = void( TT("compress_from_host_call", test_compress_from_host_call), TT("decompress_traps_in_wasm", test_decompress_traps_in_wasm), TT("decompress_via_gc_wasm", test_decompress_via_gc_wasm), - T("victim_suspended_only", test_victim_suspended_only), - T("victim_round_robin", test_victim_round_robin), - T("victim_not_recycled", test_victim_not_recycled), + T("sweep_compresses_suspended", test_sweep_compresses_suspended), + T("sweep_batches", test_sweep_batches), + T("recycled_not_compressed", test_recycled_not_compressed), T("decompress_compresses_other", test_decompress_compresses_other), T("disabled_no_compress", test_disabled_no_compress), TT("gens_round_robin", test_gens_round_robin), @@ -137,10 +137,10 @@ private class StackCompressionTester(t: Tester, tiering: ExecutionStrategy) { StackTuning.enableCompression = saved_compression; limit_total = -1; } - // Enables compression under a limit of {extra} more segments. + // Enables compression under a limit of {extra} more segments; pinning must not sweep. def compressing(extra: int) { - StackTuning.enableCompression = true; limit(extra); + StackTuning.enableCompression = true; } // Pins all existing segments, then allows at most {extra} more. def limit(extra: int) { @@ -628,18 +628,16 @@ def test_decompress_via_gc_wasm(t: StackCompressionTester) { t.t.assert_eq(1005, t.callInt("finish", [Value.Cont(k)])); } -def test_victim_suspended_only(t: StackCompressionTester) { +def test_sweep_compresses_suspended(t: StackCompressionTester) { var s1 = t.suspended("make_once", 10), s2 = t.suspended("make_once", 20), s3 = t.suspended("make_once", 30); t.compressing(1); var empty = M.getFreshStack(); - // All segments are now owned; the next allocation must compress a SUSPENDED stack. + // All segments are now owned; the next allocation sweeps, compressing every SUSPENDED stack. var fresh = M.getFreshStack(); t.assertState(StackState.EMPTY, empty); t.assertState(StackState.EMPTY, fresh); - var compressed = 0; - for (s in [s1, s2, s3]) if (s.state() == StackState.COMPRESSED) compressed++; - t.t.assert_eq(1, compressed); - t.checkInvariants("victim"); + for (s in [s1, s2, s3]) t.assertState(StackState.COMPRESSED, s); + t.checkInvariants("sweep"); M.recycleStack(empty); M.recycleStack(fresh); t.assertResult(Result.Value([Values.box_i(1010)]), s1.resume()); @@ -647,23 +645,27 @@ def test_victim_suspended_only(t: StackCompressionTester) { t.assertResult(Result.Value([Values.box_i(1030)]), s3.resume()); } -def test_victim_round_robin(t: StackCompressionTester) { +def test_sweep_batches(t: StackCompressionTester) { var stacks = Array.new(4); for (i < stacks.length) stacks[i] = t.suspended("make_once", i); t.compressing(0); - // Each further allocation compresses a different stack. + var before = C.numCompressed; var fresh = Array.new(4); - for (i < fresh.length) { - fresh[i] = M.getFreshStack(); - var compressed = 0; - for (s in stacks) if (s.state() == StackState.COMPRESSED) compressed++; - t.t.assert_eq(i + 1, compressed); - } + fresh[0] = M.getFreshStack(); // one sweep compresses all four stacks + for (s in stacks) t.assertState(StackState.COMPRESSED, s); + t.t.assert_eq(before + 4, C.numCompressed); + // The remaining allocations are served from the cache without another sweep. + for (i = 1; i < fresh.length; i++) fresh[i] = M.getFreshStack(); + for (s in fresh) { + if (s == null) return t.t.fail("expected a cached stack"); + t.assertState(StackState.EMPTY, s); + } + t.t.assert_eq(before + 4, C.numCompressed); for (s in fresh) M.recycleStack(s); for (i < stacks.length) t.assertResult(Result.Value([Values.box_i(1000 + i)]), stacks[i].resume()); } -def test_victim_not_recycled(t: StackCompressionTester) { +def test_recycled_not_compressed(t: StackCompressionTester) { var a = t.suspended("make_once", 7), b = t.suspended("make_once", 1); t.compressing(0); M.recycleStack(b); // a recycled stack is cleared, so never compressed From 0dd340211812791a2e71e6fc3d359ce593037b73 Mon Sep 17 00:00:00 2001 From: davidmaamoaix Date: Wed, 28 Oct 2026 13:50:13 -0400 Subject: [PATCH 09/10] cleanup --- src/engine/native/NativeStack.v3 | 3 ++- src/engine/native/NativeStackManager.v3 | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/engine/native/NativeStack.v3 b/src/engine/native/NativeStack.v3 index 5a49aa046..89467a7e3 100644 --- a/src/engine/native/NativeStack.v3 +++ b/src/engine/native/NativeStack.v3 @@ -14,7 +14,8 @@ class NativeStack extends WasmStack { var return_results: Array; var state_: StackState = StackState.EMPTY; var compressed: CompressedStack; // frames while {COMPRESSED} - var saved_parent_rsp: Pointer; // contents of the parent rsp slot while {COMPRESSED} + // XXX: merge into compressed? + var saved_parent_rsp: Pointer; // contents of the parent rsp slot while {COMPRESSED} // Handler table (for inline suspend/switch). var handlers: Array; diff --git a/src/engine/native/NativeStackManager.v3 b/src/engine/native/NativeStackManager.v3 index bbcfd65bc..286a13878 100644 --- a/src/engine/native/NativeStackManager.v3 +++ b/src/engine/native/NativeStackManager.v3 @@ -5,6 +5,7 @@ def fatal = System.error("NativeStackError", _); def PAGE_SIZE = 4096u; // Manages native memory mappings for native stacks. +// TODO[rv]: clean this up; remove the residual structure from master's stack management component NativeStackManager { var all: NativeStackSegment; // all segments ever mapped; never unmapped var total: int; // number of segments in {all} From 8cf9d61ec36e499799f93437df074d59ed6b1a01 Mon Sep 17 00:00:00 2001 From: davidmaamoaix Date: Sun, 1 Nov 2026 10:27:29 -0500 Subject: [PATCH 10/10] [stack] Manage free segments directly instead of through cached stacks Unowned segments now sit on their own free list in the manager, so a segment that is mapped, reclaimed by a sweep, or freed by compression is no longer wrapped in a throwaway NativeStack, and decompression no longer pops and discards one. Acquiring and attaching a segment is a single call, which removes the requirement that a detached segment be attached before the next refill. The cache keeps its meaning as the list of finished stacks pushed by the return-parent stub. --- src/engine/native/NativeStackCompression.v3 | 15 +-- src/engine/native/NativeStackManager.v3 | 111 +++++++++++------- .../NativeStackCompressionTest.v3 | 18 +-- .../x86-64-linux/NativeStackManagerTest.v3 | 58 +++++++-- 4 files changed, 130 insertions(+), 72 deletions(-) diff --git a/src/engine/native/NativeStackCompression.v3 b/src/engine/native/NativeStackCompression.v3 index 47d943c5c..4994c0fb9 100644 --- a/src/engine/native/NativeStackCompression.v3 +++ b/src/engine/native/NativeStackCompression.v3 @@ -34,31 +34,28 @@ component NativeStackCompression { } return false; } - // Requires {canCompress(stack)}; returns the detached segment. - def compress(stack: NativeStack) -> NativeStackSegment { + // Requires {canCompress(stack)}. + def compress(stack: NativeStack) { var parent_rsp = stack.parent_rsp_ptr.load(); var c = StackCompression.compressStack(stack); - var segment = NativeStackManager.detach(stack); if (Trace.stack) Trace.OUT.put2("compress stack @ 0x%x, freeing segment 0x%x", - Pointer.atObject(stack) - Pointer.NULL, segment.start - Pointer.NULL).ln(); + Pointer.atObject(stack) - Pointer.NULL, stack.range.start - Pointer.NULL).ln(); + NativeStackManager.releaseSegment(stack); stack.vsp = stack.rsp = Pointer.NULL; stack.parent_rsp_ptr = Pointer.NULL; stack.saved_parent_rsp = parent_rsp; stack.compressed = c; stack.state_ = StackState.COMPRESSED; numCompressed++; - return segment; } // Decompresses {stack}, if compressed, onto a fresh segment; returns {false} if none is available. def decompress(stack: NativeStack) -> bool { if (stack.state_ != StackState.COMPRESSED) return true; - var segment = NativeStackManager.acquireSegment(); - if (segment == null) return false; var c = stack.compressed, parent_rsp = stack.saved_parent_rsp; var func = stack.func, results = stack.return_results, parent = stack.parent, bottom = stack.bottom; + if (!NativeStackManager.attachSegment(stack)) return false; if (Trace.stack) Trace.OUT.put2("decompress stack @ 0x%x onto segment 0x%x", - Pointer.atObject(stack) - Pointer.NULL, segment.start - Pointer.NULL).ln(); - NativeStackManager.attach(stack, segment); + Pointer.atObject(stack) - Pointer.NULL, stack.range.start - Pointer.NULL).ln(); Target.writeFramesToStack(stack, StackCompression.decompressFrames(c)); stack.func = func; stack.return_results = results; diff --git a/src/engine/native/NativeStackManager.v3 b/src/engine/native/NativeStackManager.v3 index 286a13878..c7fe2207b 100644 --- a/src/engine/native/NativeStackManager.v3 +++ b/src/engine/native/NativeStackManager.v3 @@ -5,63 +5,89 @@ def fatal = System.error("NativeStackError", _); def PAGE_SIZE = 4096u; // Manages native memory mappings for native stacks. -// TODO[rv]: clean this up; remove the residual structure from master's stack management component NativeStackManager { var all: NativeStackSegment; // all segments ever mapped; never unmapped var total: int; // number of segments in {all} - var cache: NativeStack; // used in fast-path allocation in generated code + var free: NativeStackSegment; // unowned segments, linked by {next_free} + var cache: NativeStack; // finished stacks that still own a segment; pushed by the return-parent stub - // Gets a stack with a segment, or {null} if none is available. + // Gets an empty stack with a segment, or {null} if none is available. def getFreshStack() -> NativeStack { if (Trace.stack) Trace.OUT.puts("Requested fresh x86_stack...\n"); - if (cache == null && !refill()) return null; + if (cache != null) return popCache(); + var segment = acquire(); + if (segment == null) return null; + var stack = NativeStack.new(); + attach(stack, segment); + return stack; + } + // Gives {stack}, which must have no segment, a segment; returns {false} if none is available. + def attachSegment(stack: NativeStack) -> bool { + var segment = acquire(); + if (segment == null) return false; + attach(stack, segment); + return true; + } + // Frees the segment of {stack}, whose frames must already be gone. + def releaseSegment(stack: NativeStack) { + push(detach(stack)); + } + def recycleStack(stack: NativeStack) { + stack.clear(); // so that it is never chosen for compression + stack.next_stack = cache; + cache = stack; + } + def compressStack(stack: NativeStack) -> bool { + if (!NativeStackCompression.canCompress(stack)) return false; + NativeStackCompression.compress(stack); + return true; + } + // Gets an unowned segment from {free}, a cached stack, a new mapping or a sweep, or {null}. + private def acquire() -> NativeStackSegment { + if (free == null && cache != null) return detach(popCache()); + if (free == null && !refill()) return null; + var segment = free; + free = segment.next_free; + segment.next_free = null; + return segment; + } + // Refills {free} from new mappings or a sweep, forcing a GC if the first sweep finds nothing. + private def refill() -> bool { + if (mapSegments() > 0) return true; + if (sweep() > 0) return true; + if (total == 0) return false; + Target.forceGC(); // weak references are only cleared by a GC + return sweep() > 0; + } + private def popCache() -> NativeStack { var result = cache; - cache = cache.next_stack; + cache = result.next_stack; result.next_stack = null; return result.clear(); } + private def push(segment: NativeStackSegment) { + segment.next_free = free; + free = segment; + } // Requires {segment} to be unowned; leaves {stack} empty. - def attach(stack: NativeStack, segment: NativeStackSegment) { + private def attach(stack: NativeStack, segment: NativeStackSegment) { stack.range = segment; segment.stack.set(stack); stack.clear(); if (Trace.stack && Debug.stack) Trace.OUT.put2("attachSegment start=0x%x, end=0x%x", segment.start - Pointer.NULL, segment.end - Pointer.NULL).ln(); } - def detach(stack: NativeStack) -> NativeStackSegment { + private def detach(stack: NativeStack) -> NativeStackSegment { var segment = stack.range; segment.stack.set(null); stack.range = null; return segment; } - def compressStack(stack: NativeStack) -> bool { - if (!NativeStackCompression.canCompress(stack)) return false; - addToCache(NativeStackCompression.compress(stack)); - return true; - } - // Gets an unowned segment, which must be attached before the next refill, or {null} if out of stacks. - def acquireSegment() -> NativeStackSegment { - var s = getFreshStack(); - return if(s == null, null, detach(s)); - } - // Refills the cache from new segments or a sweep, forcing a GC if the first sweep finds nothing. - private def refill() -> bool { - if (tryAllocatingNewStacks()) return true; - if (sweep() > 0) return true; - if (all == null) return false; - Target.forceGC(); // weak references are only cleared by a GC - return sweep() > 0; - } - private def addToCache(segment: NativeStackSegment) { - var stack = NativeStack.new(); - attach(stack, segment); - stack.next_stack = cache; - cache = stack; - } - private def tryAllocatingNewStacks() -> bool { + // Maps up to a batch of new segments onto {free}, within {StackTuning.maxStacks}. + private def mapSegments() -> int { var size = EngineOptions.STACK_SIZE.get(); var batch = Ints.min(StackTuning.stackBatchSize, StackTuning.maxStacks - total); if (Trace.stack) Trace.OUT.put1("Batch allocating %d stacks", batch).ln(); - + var count = 0; for (i < batch) { // Map a guard page above the segment, so that accesses beyond its end always fault. var t = Mmap.reserveRaw(size + PAGE_SIZE, Mmap.PROT_READ | Mmap.PROT_WRITE, Mmap.MAP_PRIVATE | Mmap.MAP_ANONYMOUS); @@ -74,18 +100,19 @@ component NativeStackManager { // insert redzone roughly in the middle of the mapping. var rzStart = t.0 + ((size >> 1) & ~(PAGE_SIZE - 1u)), rzEnd = rzStart + PAGE_SIZE; if (!RedZones.addRedZone(rzStart, rzEnd)) fatal("could not protect value stack red zone"); - addToCache(segment); + push(segment); total++; + count++; } - return cache != null; // OK if succeeded in allocating at least one + return count; } - // Caches the segments of dead stacks and, with compression enabled, of every compressible stack. + // Frees dead segments and, with compression on, compressible stacks' segments; requires {free} empty. private def sweep() -> int { var count = 0; for (p = all; p != null; p = p.next) { var s = p.stack.get(); - if (s == null) addToCache(p); - else if (StackTuning.enableCompression && NativeStackCompression.canCompress(s)) addToCache(NativeStackCompression.compress(s)); + if (s == null) push(p); + else if (StackTuning.enableCompression && NativeStackCompression.canCompress(s)) NativeStackCompression.compress(s); else continue; count++; } @@ -101,11 +128,6 @@ component NativeStackManager { } } } - def recycleStack(stack: NativeStack) { - stack.clear(); // so that it is never chosen for compression - stack.next_stack = cache; - cache = stack; - } def runOnFreshStack(f: Function, args: Range) -> Result { // Always run functions on a separate, fresh stack. var prev = X86_64Runtime.curStack; // handle reentrancy @@ -119,7 +141,8 @@ component NativeStackManager { // Native stack segments are individually mmap'd with a guard page in the middle class NativeStackSegment extends MemoryRange { - var next: NativeStackSegment; + var next: NativeStackSegment; // link in {NativeStackManager.all} + var next_free: NativeStackSegment; // link in {NativeStackManager.free}, while unowned def stack = WeakRef.new(); // owner, if any; cleared by GC new(start: Pointer, end: Pointer) super(start, end) { } diff --git a/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 b/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 index c257a967e..0145cf4aa 100644 --- a/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 +++ b/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 @@ -20,7 +20,7 @@ def X_ = void( T("no_frames_not_compressible", test_no_frames_not_compressible), T("running_not_compressible", test_running_not_compressible), T("repeat_roundtrip", test_repeat_roundtrip), - T("acquire_segment", test_acquire_segment), + T("attach_segment", test_attach_segment), T("decompress_out_of_stacks", test_decompress_out_of_stacks), T("version_kept", test_version_kept), TT("mid_host_resume", test_mid_host_resume), @@ -146,6 +146,7 @@ private class StackCompressionTester(t: Tester, tiering: ExecutionStrategy) { def limit(extra: int) { StackTuning.maxStacks = M.total; M.cache = null; + M.free = null; Target.forceGC(); var free = 0; for (seg = M.all; seg != null; seg = seg.next) if (seg.stack.get() == null) free++; @@ -163,6 +164,9 @@ private class StackCompressionTester(t: Tester, tiering: ExecutionStrategy) { if (s.range != seg) t.fail1("%s: owning stack does not point back at its segment", where); if (s.state() == StackState.COMPRESSED) t.fail1("%s: compressed stack owns a segment", where); } + for (seg = M.free; seg != null; seg = seg.next_free) { + if (seg.stack.get() != null) t.fail1("%s: free segment has an owner", where); + } if (count != M.total) t.fail1("%s: segment count mismatch", where); if (limit_total >= 0 && M.total > limit_total) t.fail3("%s: expected at most %d segments, got %d", where, limit_total, M.total); } @@ -369,15 +373,13 @@ def test_repeat_roundtrip(t: StackCompressionTester) { t.assertResult(Result.Value([Values.box_i(2000)]), s.resume()); } -def test_acquire_segment(t: StackCompressionTester) { +def test_attach_segment(t: StackCompressionTester) { t.limit(1); - var seg = M.acquireSegment(); - if (seg == null) return t.t.fail("expected a segment"); - if (seg.stack.get() != null) t.t.fail("expected an acquired segment to have no owner"); - // An unowned segment looks free to the reclaim sweep, so attach it before requesting another. var s = NativeStack.new(); - M.attach(s, seg); - if (M.acquireSegment() != null) t.t.fail("expected no second segment under the limit"); + if (!M.attachSegment(s)) return t.t.fail("expected a segment"); + var seg = s.range; + if (seg == null || seg.stack.get() != s) return t.t.fail("expected the attached segment to be owned"); + if (M.attachSegment(NativeStack.new())) t.t.fail("expected no second segment under the limit"); t.assertTrap(TrapReason.OUT_OF_STACKS, M.runOnFreshStack(HostFunction.new("seven", SigCache.v_i, fun args => HostResult.Value1(Values.box_i(7))), [])); if (seg.stack.get() != s) t.t.fail("expected the attached segment to keep its owner"); M.recycleStack(s); diff --git a/test/unittest/x86-64-linux/NativeStackManagerTest.v3 b/test/unittest/x86-64-linux/NativeStackManagerTest.v3 index ad9ec5973..e548b9b4a 100644 --- a/test/unittest/x86-64-linux/NativeStackManagerTest.v3 +++ b/test/unittest/x86-64-linux/NativeStackManagerTest.v3 @@ -30,7 +30,9 @@ def X_ = void( T("run_on_fresh_stack_gc", test_run_on_fresh_stack_gc), T("out_of_stacks_host", test_out_of_stacks_host), T("out_of_stacks_cont_new", test_out_of_stacks_cont_new), - T("attach_detach", test_attach_detach), + T("release_attach", test_release_attach), + T("free_before_map", test_free_before_map), + T("attach_takes_cached", test_attach_takes_cached), T("moved_segment_kept", test_moved_segment_kept), T("reattach_runs", test_reattach_runs), () @@ -50,12 +52,13 @@ private class NativeStackManagerTester extends ExeTester { def limit(extra: int) { StackTuning.maxStacks = M.total; M.cache = null; + M.free = null; Target.forceGC(); var free = 0; for (seg = M.all; seg != null; seg = seg.next) if (seg.stack.get() == null) free++; held = Array.new(free); for (i < free) held[i] = fresh(); - if (M.cache != null) t.fail("expected cache to be empty after pinning"); + if (M.cache != null || M.free != null) t.fail("expected no free segments after pinning"); for (seg = M.all; seg != null; seg = seg.next) { if (seg.stack.get() == null) t.fail("expected all segments to be owned after pinning"); } @@ -102,8 +105,9 @@ private class NativeStackManagerTester extends ExeTester { def moveSegmentToNewStack() -> NativeStack #no-inline { var a = fresh(); var b = NativeStack.new(); - M.attach(b, M.detach(a)); - if (a.range != null) t.fail("expected detached stack to have no segment"); + M.releaseSegment(a); + if (!M.attachSegment(b)) t.fail("expected the released segment to be reattached"); + if (a.range != null) t.fail("expected released stack to have no segment"); return b; } def checkMovedSegment() -> NativeStackSegment #no-inline { @@ -139,6 +143,9 @@ private class NativeStackManagerTester extends ExeTester { var seg = s.range; if (seg.stack.get() != s) t.fail1("%s: cached stack does not own its segment", where); } + for (seg = M.free; seg != null; seg = seg.next_free) { + if (seg.stack.get() != null) t.fail1("%s: free segment has an owner", where); + } } def assertDisjoint(stacks: Array) { for (i < stacks.length) { @@ -412,19 +419,45 @@ def test_out_of_stacks_host(t: NativeStackManagerTester) { t.assertResult(Result.Value([Values.box_i(7)]), M.runOnFreshStack(hf, [])); } -def test_attach_detach(t: NativeStackManagerTester) { +def test_release_attach(t: NativeStackManagerTester) { var s = t.fresh(); var seg = s.range; - if (M.detach(s) != seg) t.t.fail("expected detach to return the attached segment"); - if (s.range != null) t.t.fail("expected detached stack to have no segment"); - if (seg.stack.get() != null) t.t.fail("expected detached segment to have no owner"); - M.attach(s, seg); - if (s.range != seg) t.t.fail("expected attach to set the segment"); + M.releaseSegment(s); + if (s.range != null) t.t.fail("expected released stack to have no segment"); + if (seg.stack.get() != null) t.t.fail("expected released segment to have no owner"); + if (M.free != seg) t.t.fail("expected released segment to be first in the free list"); + if (!M.attachSegment(s)) return t.t.fail("expected a segment"); + if (s.range != seg) t.t.fail("expected the released segment to be reattached"); if (seg.stack.get() != s) t.t.fail("expected attach to set the owner"); if (s.state() != StackState.EMPTY || s.vsp != seg.start || s.rsp != seg.end) t.t.fail("expected attach to leave the stack empty"); t.checkInvariants("attach"); } +def test_free_before_map(t: NativeStackManagerTester) { + t.limit(2); + var s = t.fresh(); + var seg = s.range, total = M.total; + M.releaseSegment(s); + var s2 = t.fresh(); + if (s2.range != seg) t.t.fail("expected the free segment to be reused before mapping a new one"); + if (s2 == s) t.t.fail("expected a new stack for the free segment"); + t.t.assert_eq(total, M.total); + t.checkInvariants("free"); +} + +def test_attach_takes_cached(t: NativeStackManagerTester) { + t.limit(1); + var a = t.fresh(); + var seg = a.range, total = M.total; + M.recycleStack(a); + var b = NativeStack.new(); + if (!M.attachSegment(b)) return t.t.fail("expected the cached stack's segment"); + if (b.range != seg || seg.stack.get() != b) t.t.fail("expected the cached stack's segment to move"); + if (a.range != null || M.cache != null) t.t.fail("expected the cached stack to be dropped"); + t.t.assert_eq(total, M.total); + t.checkInvariants("cached"); +} + def test_moved_segment_kept(t: NativeStackManagerTester) { // A segment moved to another stack is reclaimed only when the new owner dies. t.limit(1); @@ -440,7 +473,10 @@ def test_reattach_runs(t: NativeStackManagerTester) { var f = t.subFunc(); var s = t.fresh(); var seg = s.range; - for (i < 3) M.attach(s, M.detach(s)); + for (i < 3) { + M.releaseSegment(s); + if (!M.attachSegment(s)) return t.t.fail("expected a segment"); + } if (s.range != seg || seg.stack.get() != s) t.t.fail("expected reattached stack to own its segment"); t.assertResult(Result.Value([Values.box_i(2)]), s.reset(f).bind([Values.box_i(7), Values.box_i(5)]).resume()); t.checkInvariants("reattach");