Stack compression hook

src/engine/WasmStack.v3

@@ -98,6 +98,7 @@ enum StackState {
 	SUSPENDED,		// contains one or more frames, waiting for inputs
 	CALL_CHILD,		// waiting for child stack to return
 	RUNNING,		// currently running Wasm or host code
+	COMPRESSED,		// suspended, with its frames held in the heap
 }
 
 // Provides access to the state of an executing frame.

src/engine/compiler/MacroAssembler.v3

@@ -356,6 +356,7 @@ class MacroAssembler(valuerep: Tagging, regConfig: RegConfig) {
 	def emit_call_runtime_SUSPEND();
 	def emit_call_runtime_SWITCH();
 	def emit_call_runtime_RESUME_THROW();
+	def emit_call_runtime_DECOMPRESS_CONT();
 	def emit_call_runtime_RESUME_THROW_REF();
 	def emit_jump_to_trap_at(reason: TrapReason);
 	def emit_call_runtime_op(op: Opcode);
@@ -379,6 +380,8 @@ class MacroAssembler(valuerep: Tagging, regConfig: RegConfig) {
 
 	// stk.state_ = state;
 	def emit_set_stack_state(stk: Reg, state: StackState);
+	// if (stk.state_ != state) goto label;
+	def emit_br_stack_state_ne(stk: Reg, state: StackState, label: MasmLabel);
 	// Publishes the handler table.
 	def emit_publish_handlers(r_stack: Reg, r_table: Reg, tier: HandlerTier) {
 		var offsets = getOffsets();

src/engine/compiler/SinglePassCompiler.v3

@@ -1454,6 +1454,7 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl
 		buildResumeHandlerDests(it.pc, handlers);
 		var cont_decl = ContDecl.!(module.heaptypes[cont_id]);
 		if (checkForConstNull(state.peek())) return;
+		emit_decompress_top_cont();
 		var n = cont_decl.sig.params.length, slot_size = masm.valuerep.slot_size, offsets = masm.getOffsets();
 
 		// Spill the value stack and consume the continuation on top of it.
@@ -1578,6 +1579,7 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl
 		var target_top = target_decl.sig.params[target_decl.sig.params.length - 1];
 		var this_decl = ContDecl.!(HeapType.Cont.!(ValueType.Ref.!(target_top).heap).cont);
 		var n = target_decl.sig.params.length - 1, slot_size = masm.valuerep.slot_size, offsets = masm.getOffsets();
+		emit_decompress_top_cont();
 		state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS);
 		var vsp = allocTmp(ValueKind.REF), cur = allocTmp(ValueKind.REF), target = allocTmp(ValueKind.REF);
 		var prev = allocTmp(ValueKind.REF), parent = allocTmp(ValueKind.REF), tag = allocTmp(ValueKind.REF);
@@ -2425,6 +2427,27 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl
 		masm.emit_mov_r_r(ValueKind.REF, dst, regs.vfp); // XXX: use 3-addr adjustment of VSP (i.e. lea)
 		if (slots > 0) masm.emit_addw_r_i(dst, int.view(slots) * masm.valuerep.slot_size);
 	}
+	def emit_decompress_top_cont() {
+		state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS);
+		emit_compute_vsp(regs.vsp, state.sp);
+		emit_spill_vsp(regs.vsp);
+		masm.emit_store_curstack_vsp(regs.vsp);
+		var done = masm.newLabel(it.pc), tmp = regs.scratch, valuerep = masm.valuerep;
+		masm.emit_mov_r_m(ValueKind.REF, tmp, MasmAddr(regs.vsp, valuerep.tag_size - valuerep.slot_size));
+		masm.emit_br_r(tmp, MasmBrCond.REF_NULL, done); // null continuation traps later
+		masm.emit_br_stack_state_ne(tmp, StackState.COMPRESSED, done);
+		def emit = fun {
+			masm.emit_get_curstack(regs.runtime_arg0);
+			masm.emit_v3_set_NativeStack_rsp_r_r(regs.runtime_arg0, regs.sp);
+			masm.emit_push_NativeStack_rsp_r_r(regs.runtime_arg0);
+			masm.emit_call_runtime_DECOMPRESS_CONT();
+		};
+		withReconstructedInlinedFrames(emit);
+		masm.emit_get_curstack(regs.scratch);
+		masm.emit_pop_NativeStack_rsp_r_r(regs.scratch);
+		emit_reload_regs();
+		masm.bindLabel(done);
+	}
 	def emit_spill_vsp(reg: Reg) {
 		masm.emit_mov_m_r(ValueKind.REF, frame.vsp_slot, reg);
 	}

src/engine/native/NativeStack.v3

@@ -6,24 +6,24 @@
 // See diagram at the end of the file for mapping layout.
 class NativeStack extends WasmStack {
 	def valuerep = Target.tagging;
-	var range: MemoryRange;
+	var range: NativeStackSegment;
 	var vsp: Pointer;
 	var rsp: Pointer;
 	var func: Function; // TODO: move into "enter-func-stub"
 	var parent_rsp_ptr: Pointer;
 	var return_results: Array<ValueType>;
 	var state_: StackState = StackState.EMPTY;
+	var compressed: CompressedStack;	// frames while {COMPRESSED}
+	// XXX: merge into compressed?
+	var saved_parent_rsp: Pointer; // contents of the parent rsp slot while {COMPRESSED}
 
 	// Handler table (for inline suspend/switch).
 	var handlers: Array<byte>;
 	var handler_tier: HandlerTier;
 	var next_stack: NativeStack;
 
-	new(range) {
+	new() {
 		if (valuerep.tagged) RiGc.registerScanner(this, NativeStack.scan);
-		if (range == null) return; // allocated without a mapping, no need to initialize further
-		clear();
-		if (Trace.stack && Debug.stack) Trace.OUT.put2("newStack start=0x%x, end=0x%x", range.start - Pointer.NULL, range.end - Pointer.NULL).ln();
 	}
 	// Gets the state of this stack.
 	def state() -> StackState {
@@ -51,13 +51,15 @@ class NativeStack extends WasmStack {
 	// Pushes {args} incrementally onto the value stack.
 	def bind(args: Range<Value>) -> this {
 		if (args.length == 0) return;
+		if (!NativeStackCompression.decompress(this)) fatal("out of stacks while binding a compressed stack");
 		checkState("bind()", StackState.SUSPENDED);
 		for (v in args) push(v);
 	}
 	// Requires {state == SUSPENDED}.
 	// Resumes running the Wasm or host code in this stack until that code either returns, throws,
 	// or suspends itself.
 	def resume() -> Result {
+		if (!NativeStackCompression.decompress(this)) return Result.Throw(Trap.new(TrapReason.OUT_OF_STACKS, null, null));
 		checkState("resume()", StackState.SUSPENDED);
 		if (Trace.stack) traceFrames("stack.resume", rsp + RETADDR_SIZE);
 		var bottom = NativeStack.!(this.bottom);
@@ -511,6 +513,7 @@ class NativeStack extends WasmStack {
 	}
 	// GC callback to scan (tagged) values on this stack
 	def scan() {
+		if (range == null) return;
 		if (Trace.gc) {
 			Trace.OUT.put1("scanStack obj=0x%x", u64.view(Pointer.atObject(this) - Pointer.NULL)).ln();
 		}
@@ -552,14 +555,16 @@ class NativeStack extends WasmStack {
 			BpTypeCode.ARRAYREF.code,
 			BpTypeCode.FUNCREF.code,
 			BpTypeCode.EXNREF.code,
+			BpTypeCode.NULLEXNREF.code,
 			BpTypeCode.NULLFUNCREF.code => return Value.Ref(vp.load<Object>());
 
 			BpTypeCode.I32.code => return Value.I32(vp.load<u32>());
 			BpTypeCode.I64.code => return Value.I64(vp.load<u64>());
 			BpTypeCode.F32.code => return Value.F32(vp.load<u32>());
 			BpTypeCode.F64.code => return Value.F64(vp.load<u64>());
 			BpTypeCode.V128.code => return Value.V128(vp.load<u64>(), (vp + 8).load<u64>());
-			BpTypeCode.CONTREF.code => {
+			BpTypeCode.CONTREF.code,
+			BpTypeCode.NULLCONTREF.code => {
 				var stack = vp.load<WasmStack>();
 				var version = (vp + 8).load<u64>();
 				return Value.Cont(Continuation(stack, version));

src/engine/native/NativeStackCompression.v3

@@ -0,0 +1,71 @@
+// Copyright 2026 Wizard authors. All rights reserved.
+// See LICENSE for details of Apache 2.0 license.
+
+// Compresses suspended stacks into the heap to free their segments, and decompresses them before use.
+component NativeStackCompression {
+	var numCompressed: int;
+	var numDecompressed: int;
+
+	// Returns {true} if {stack} is suspended with only interpreter and SPC frames.
+	def canCompress(stack: NativeStack) -> bool {
+		if (stack.state_ != StackState.SUSPENDED) return false;
+		var range = stack.range;
+		if (range == null) return false;
+		var end = range.end;
+		// [end - 8] holds the parent's rsp, [end - 16] the return-parent stub.
+		if (stack.parent_rsp_ptr != end + -Pointer.SIZE) return false;
+		var srp = stack.rsp, count = 0, bottom_sfp = Pointer.NULL;
+		while (srp >= range.start && srp < end) {
+			var retip = srp.load<Pointer>();
+			var code = RiRuntime.findUserCode(retip);
+			match (code) {
+				x: X86_64InterpreterCode, x: X86_64SpcModuleCode => {
+					count++;
+					bottom_sfp = srp + Pointer.SIZE;
+					srp = code.nextFrame(retip, srp).1;
+				}
+				x: X86_64ReturnParentStub => {
+					if (count == 0 || srp != end + -2 * Pointer.SIZE) return false;
+					// Values below the bottom frame would be lost.
+					return NativeFrameHandle.Interpreter(bottom_sfp).vfp() == range.start;
+				}
+				_ => return false; // the enter-func stub, inlined frames, trap stubs, or host frames
+			}
+		}
+		return false;
+	}
+	// Requires {canCompress(stack)}.
+	def compress(stack: NativeStack) {
+		var parent_rsp = stack.parent_rsp_ptr.load<Pointer>();
+		var c = StackCompression.compressStack(stack);
+		if (Trace.stack) Trace.OUT.put2("compress stack @ 0x%x, freeing segment 0x%x",
+			Pointer.atObject(stack) - Pointer.NULL, stack.range.start - Pointer.NULL).ln();
+		NativeStackManager.releaseSegment(stack);
+		stack.vsp = stack.rsp = Pointer.NULL;
+		stack.parent_rsp_ptr = Pointer.NULL;
+		stack.saved_parent_rsp = parent_rsp;
+		stack.compressed = c;
+		stack.state_ = StackState.COMPRESSED;
+		numCompressed++;
+	}
+	// Decompresses {stack}, if compressed, onto a fresh segment; returns {false} if none is available.
+	def decompress(stack: NativeStack) -> bool {
+		if (stack.state_ != StackState.COMPRESSED) return true;
+		var c = stack.compressed, parent_rsp = stack.saved_parent_rsp;
+		var func = stack.func, results = stack.return_results, parent = stack.parent, bottom = stack.bottom;
+		if (!NativeStackManager.attachSegment(stack)) return false;
+		if (Trace.stack) Trace.OUT.put2("decompress stack @ 0x%x onto segment 0x%x",
+			Pointer.atObject(stack) - Pointer.NULL, stack.range.start - Pointer.NULL).ln();
+		Target.writeFramesToStack(stack, StackCompression.decompressFrames(c));
+		stack.func = func;
+		stack.return_results = results;
+		stack.parent = parent;
+		stack.bottom = bottom;
+		stack.parent_rsp_ptr.store<Pointer>(parent_rsp);
+		stack.saved_parent_rsp = Pointer.NULL;
+		stack.compressed = null;
+		stack.state_ = StackState.SUSPENDED;
+		numDecompressed++;
+		return true;
+	}
+}

src/engine/native/NativeStackManager.v3

@@ -8,26 +8,86 @@ def PAGE_SIZE = 4096u;
 component NativeStackManager {
 	var all: NativeStackSegment;	// all segments ever mapped; never unmapped
 	var total: int;			// number of segments in {all}
-	var cache: NativeStack;		// used in fast-path allocation in generated code
+	var free: NativeStackSegment;	// unowned segments, linked by {next_free}
+	var cache: NativeStack;		// finished stacks that still own a segment; pushed by the return-parent stub
 
-	// Gets a stack with a segment, or {null} if none is available.
+	// Gets an empty stack with a segment, or {null} if none is available.
 	def getFreshStack() -> NativeStack {
 		if (Trace.stack) Trace.OUT.puts("Requested fresh x86_stack...\n");
-		if (cache == null) {
-			var ok = tryAllocatingNewStacks() || tryReclaimingSegments() || tryCollectingAndReclaiming() || tryCompressingStacks();
-			if (!ok) return null;
-		}
-
+		if (cache != null) return popCache();
+		var segment = acquire();
+		if (segment == null) return null;
+		var stack = NativeStack.new();
+		attach(stack, segment);
+		return stack;
+	}
+	// Gives {stack}, which must have no segment, a segment; returns {false} if none is available.
+	def attachSegment(stack: NativeStack) -> bool {
+		var segment = acquire();
+		if (segment == null) return false;
+		attach(stack, segment);
+		return true;
+	}
+	// Frees the segment of {stack}, whose frames must already be gone.
+	def releaseSegment(stack: NativeStack) {
+		push(detach(stack));
+	}
+	def recycleStack(stack: NativeStack) {
+		stack.clear(); // so that it is never chosen for compression
+		stack.next_stack = cache;
+		cache = stack;
+	}
+	def compressStack(stack: NativeStack) -> bool {
+		if (!NativeStackCompression.canCompress(stack)) return false;
+		NativeStackCompression.compress(stack);
+		return true;
+	}
+	// Gets an unowned segment from {free}, a cached stack, a new mapping or a sweep, or {null}.
+	private def acquire() -> NativeStackSegment {
+		if (free == null && cache != null) return detach(popCache());
+		if (free == null && !refill()) return null;
+		var segment = free;
+		free = segment.next_free;
+		segment.next_free = null;
+		return segment;
+	}
+	// Refills {free} from new mappings or a sweep, forcing a GC if the first sweep finds nothing.
+	private def refill() -> bool {
+		if (mapSegments() > 0) return true;
+		if (sweep() > 0) return true;
+		if (total == 0) return false;
+		Target.forceGC(); // weak references are only cleared by a GC
+		return sweep() > 0;
+	}
+	private def popCache() -> NativeStack {
 		var result = cache;
-		cache = cache.next_stack;
+		cache = result.next_stack;
 		result.next_stack = null;
 		return result.clear();
 	}
-	private def tryAllocatingNewStacks() -> bool {
+	private def push(segment: NativeStackSegment) {
+		segment.next_free = free;
+		free = segment;
+	}
+	// Requires {segment} to be unowned; leaves {stack} empty.
+	private def attach(stack: NativeStack, segment: NativeStackSegment) {
+		stack.range = segment;
+		segment.stack.set(stack);
+		stack.clear();
+		if (Trace.stack && Debug.stack) Trace.OUT.put2("attachSegment start=0x%x, end=0x%x", segment.start - Pointer.NULL, segment.end - Pointer.NULL).ln();
+	}
+	private def detach(stack: NativeStack) -> NativeStackSegment {
+		var segment = stack.range;
+		segment.stack.set(null);
+		stack.range = null;
+		return segment;
+	}
+	// Maps up to a batch of new segments onto {free}, within {StackTuning.maxStacks}.
+	private def mapSegments() -> int {
 		var size = EngineOptions.STACK_SIZE.get();
 		var batch = Ints.min(StackTuning.stackBatchSize, StackTuning.maxStacks - total);
 		if (Trace.stack) Trace.OUT.put1("Batch allocating %d stacks", batch).ln();
-
+		var count = 0;
 		for (i < batch) {
 			// Map a guard page above the segment, so that accesses beyond its end always fault.
 			var t = Mmap.reserveRaw(size + PAGE_SIZE, Mmap.PROT_READ | Mmap.PROT_WRITE, Mmap.MAP_PRIVATE | Mmap.MAP_ANONYMOUS);
@@ -40,35 +100,23 @@ component NativeStackManager {
 			// insert redzone roughly in the middle of the mapping.
 			var rzStart = t.0 + ((size >> 1) & ~(PAGE_SIZE - 1u)), rzEnd = rzStart + PAGE_SIZE;
 			if (!RedZones.addRedZone(rzStart, rzEnd)) fatal("could not protect value stack red zone");
-			var curr = NativeStack.new(segment);
-			segment.stack.set(curr);
-			curr.next_stack = cache;
-			cache = curr;
+			push(segment);
 			total++;
+			count++;
 		}
-		return cache != null; // OK if succeeded in allocating at least one
+		return count;
 	}
-	private def tryReclaimingSegments() -> bool {
-		var ok = false;
+	// Frees dead segments and, with compression on, compressible stacks' segments; requires {free} empty.
+	private def sweep() -> int {
+		var count = 0;
 		for (p = all; p != null; p = p.next) {
-			if (p.stack.get() == null) { // weak reference was cleared by GC, can reclaim
-				var s = NativeStack.new(p);
-				p.stack.set(s);
-				s.next_stack = cache;
-				cache = s;
-				ok = true;
-			}
+			var s = p.stack.get();
+			if (s == null) push(p);
+			else if (StackTuning.enableCompression && NativeStackCompression.canCompress(s)) NativeStackCompression.compress(s);
+			else continue;
+			count++;
 		}
-		return ok;
-	}
-	// Weak references are only cleared by a GC, so force one to find dead stacks.
-	private def tryCollectingAndReclaiming() -> bool {
-		if (all == null) return false;
-		Target.forceGC();
-		return tryReclaimingSegments();
-	}
-	private def tryCompressingStacks() -> bool {
-		return false;
+		return count;
 	}
 	// Checks whether a fault at {addr} hit the guard page above a stack segment, which indicates
 	// an engine bug, such as reading a frame that is not there. If so, fails with an error.
@@ -80,10 +128,6 @@ component NativeStackManager {
 			}
 		}
 	}
-	def recycleStack(stack: NativeStack) {
-		stack.next_stack = cache;
-		cache = stack;
-	}
 	def runOnFreshStack(f: Function, args: Range<Value>) -> Result {
 		// Always run functions on a separate, fresh stack.
 		var prev = X86_64Runtime.curStack; // handle reentrancy
@@ -97,7 +141,8 @@ component NativeStackManager {
 
 // Native stack segments are individually mmap'd with a guard page in the middle
 class NativeStackSegment extends MemoryRange {
-	var next: NativeStackSegment;
+	var next: NativeStackSegment;		// link in {NativeStackManager.all}
+	var next_free: NativeStackSegment;	// link in {NativeStackManager.free}, while unowned
 	def stack = WeakRef<NativeStack>.new();	// owner, if any; cleared by GC
 
 	new(start: Pointer, end: Pointer) super(start, end) { }

src/engine/x86-64/V3Offsets.v3

@@ -15,7 +15,7 @@ class V3Offsets {
 	private def t = Table.new(ValueTypes.FUNCREF, TableDecl.new(ValueTypes.FUNCREF, SizeConstraint(false, 0, Max.Set(1))));
 	private def wf = WasmFunction.new(i, decl);
 	private def mem = NativeWasmMemory.new(null);
-	private def vs = NativeStack.new(null);
+	private def vs = NativeStack.new();
 	private def acc = NativeFrameAccessor.new(vs, NativeFrameHandle.Interpreter(Pointer.NULL), decl);
 	private def cnt = CountProbe.new();
 	private def metric = Metric.new("", "", "");