From 23b26972528d87e69c93de928e7a4524cf86fda0 Mon Sep 17 00:00:00 2001 From: "Ben L. Titzer" Date: Wed, 7 Oct 2026 14:58:57 -0400 Subject: [PATCH] [exception-handling] Don't push exception values for catch_all handlers (#705) A catch_all (or catch_all_ref) handler receives no payload, but both the V3Interpreter and NativeStack unconditionally pushed the exception's tag values onto the value stack when transferring to a handler. This left extra values on the stack below the handler's expected operands, corrupting the results of code following the try_table (or legacy try/catch_all). Add a push_vals field to ExHandler, set by FuncDecl.findHandler to false for EX_TAG_ALL entries, and only push ex.vals when it is set. This covers try_table catch_all/catch_all_ref and legacy catch_all. Adds regress_705{,b,c} reproducers and removes them from the jvm, v3i, and x86-linux expected-failure lists. Co-Authored-By: Claude Opus 5.5 --- src/engine/EngineOptions.v3 | 3 + src/engine/Metrics.v3 | 28 + src/engine/Runtime.v3 | 3 +- src/engine/Tuning.v3 | 1 + src/engine/WasmStack.v3 | 1 + src/engine/compiler/MacroAssembler.v3 | 1 + src/engine/compiler/SinglePassCompiler.v3 | 23 + .../compression/NaiveCompressionStrategy.v3 | 5 + .../compression/PackedCompressionStrategy.v3 | 4 + src/engine/native/NativeStack.v3 | 38 +- src/engine/native/NativeStackCompression.v3 | 120 +++ src/engine/native/NativeStackManager.v3 | 95 +- src/engine/v3/V3Target.v3 | 2 + src/engine/x86-64/X86_64Interpreter.v3 | 15 + src/engine/x86-64/X86_64MacroAssembler.v3 | 3 + src/engine/x86-64/X86_64Runtime.v3 | 8 + src/engine/x86-64/X86_64Target.v3 | 13 + src/modules/wizeng/WizengModule.v3 | 12 + test/regress/stacks/compression.failures | 15 - .../NativeStackCompressionTest.v3 | 932 ++++++++++++++++++ 20 files changed, 1288 insertions(+), 34 deletions(-) create mode 100644 src/engine/native/NativeStackCompression.v3 create mode 100644 test/unittest/x86-64-linux/NativeStackCompressionTest.v3 diff --git a/src/engine/EngineOptions.v3 b/src/engine/EngineOptions.v3 index 545d68d6a..4836cda09 100644 --- a/src/engine/EngineOptions.v3 +++ b/src/engine/EngineOptions.v3 @@ -9,6 +9,9 @@ component EngineOptions { .onSet(fun v => void(StackTuning.defaultStackSize = v)); def MAX_STACKS = group.newIntOption("max-stacks", StackTuning.maxStacks, "Maximum number of Wasm execution stacks.") .onSet(fun v => void(StackTuning.maxStacks = v)); + def LAZY_ENTRY_STACKS = group.newIntOption("lazy-entry-stacks", StackTuning.lazyEntryStacks, + "Give new continuations a stack segment when first resumed: 0 never, 1 near capacity, 2 always.") + .onSet(fun v => void(StackTuning.lazyEntryStacks = v)); def STACK_COMPRESSION = group.newBoolOption("stack-compression", StackTuning.enableCompression, "Enable stack compression.") .onSet(fun v => void(StackTuning.enableCompression = v)); def FAST_FUNCTIONS = group.newBoolOption("fast-functions", false, "Treat functions exported with `fast:` in the name as fast functions.") diff --git a/src/engine/Metrics.v3 b/src/engine/Metrics.v3 index 31ebf81af..e2e74bc08 100644 --- a/src/engine/Metrics.v3 +++ b/src/engine/Metrics.v3 @@ -50,6 +50,23 @@ component Metrics { def whamm_bind_time_us = t("whamm:bind_time_us", "Time taken to bind Whamm monitors, including static predicates."); def whamm_report_time_us = t("whamm:report_time_us", "Time taken to report Whamm monitors."); + // Metrics from native stacks and stack compression. + def stack_segments = m("stack:segments", "segments", "Number of native stack segments mapped"); + def stack_forced_gcs = m("stack:forced_gcs", "GCs", "Number of GCs forced to reclaim stack segments"); + def stack_lazy_entries = m("stack:lazy_entries", "stacks", "Number of new continuations given a stack without a segment"); + def stack_compressed = m("stack:compressed", "stacks", "Number of stacks compressed"); + def stack_compressed_bytes = m("stack:compressed_bytes", bytes, "Total estimated size of compressed stacks"); + def stack_compress_cycles = c("stack:compress_cycles", "Cycles taken to compress stacks"); + def stack_decompressed = m("stack:decompressed", "stacks", "Number of stacks decompressed"); + def stack_decompress_cycles = c("stack:decompress_cycles", "Cycles taken to decompress stacks"); + + // Metrics from the garbage collector of the runtime, if any. + def gc_count = register(GcMetric.new("gc:count", "GCs", "Number of garbage collections", Target.gcStat(_, 0))); + def gc_time_us = register(GcMetric.new("gc:time_us", us, "Total time spent in garbage collection", Target.gcStat(_, 1))); + def gc_max_time_us = register(GcMetric.new("gc:max_time_us", us, "Longest garbage collection", Target.gcStat(_, 2))); + def gc_max_live_bytes = register(GcMetric.new("gc:max_live_bytes", bytes, "Most bytes surviving a garbage collection", Target.gcStat(_, 3))); + def gc_heap_bytes = register(GcMetric.new("gc:heap_bytes", bytes, "Usable heap size", Target.gcStat(_, 4))); + // Keep the metrics in the above-declared order. private def unused__ = all = Lists.reverse(all); @@ -166,6 +183,17 @@ class CyclesMetric extends Metric { } } +// A metric whose value is read from the runtime when reported. +class GcMetric extends Metric { + def get: void -> u64; + + new(name: string, units: string, help: string, get) super(name, units, help) { } + + def report(out: StringBuilder) -> StringBuilder { + return out.putd(get()); + } +} + // A metric which is the ratio of two other metrics. class RatioMetric extends Metric { def n: Metric; diff --git a/src/engine/Runtime.v3 b/src/engine/Runtime.v3 index c59c9b222..70affeabc 100644 --- a/src/engine/Runtime.v3 +++ b/src/engine/Runtime.v3 @@ -363,9 +363,8 @@ component Runtime { var func = Function.!(Value.Ref.!(func_ref).val); if (func == null) return stack.trap(TrapReason.NULL_DEREF); - var new_stack = Target.newWasmStack(); + var new_stack = Target.newWasmStackFor(func); if (new_stack == null) return stack.trap(TrapReason.OUT_OF_STACKS); - new_stack.reset(func); new_stack.bottom = new_stack; var cont = Continuations.makeContinuation(new_stack); stack.push(Value.Cont(cont)); diff --git a/src/engine/Tuning.v3 b/src/engine/Tuning.v3 index c797c2e16..d1a2069d6 100644 --- a/src/engine/Tuning.v3 +++ b/src/engine/Tuning.v3 @@ -77,4 +77,5 @@ component StackTuning { var defaultStackSize = 512u * 1024u; var maxStacks = 2048; var enableCompression = false; + var lazyEntryStacks = 1; // 0: never, 1: near capacity, 2: always give new continuations a segment lazily } diff --git a/src/engine/WasmStack.v3 b/src/engine/WasmStack.v3 index ad1e6bfbd..22f32b8f9 100644 --- a/src/engine/WasmStack.v3 +++ b/src/engine/WasmStack.v3 @@ -98,6 +98,7 @@ enum StackState { SUSPENDED, // contains one or more frames, waiting for inputs CALL_CHILD, // waiting for child stack to return RUNNING, // currently running Wasm or host code + COMPRESSED, // suspended, with frames compressed into the heap } // Provides access to the state of an executing frame. diff --git a/src/engine/compiler/MacroAssembler.v3 b/src/engine/compiler/MacroAssembler.v3 index f00fbbeea..4d6834e25 100644 --- a/src/engine/compiler/MacroAssembler.v3 +++ b/src/engine/compiler/MacroAssembler.v3 @@ -338,6 +338,7 @@ class MacroAssembler(valuerep: Tagging, regConfig: RegConfig) { def emit_call_runtime_SUSPEND(); def emit_call_runtime_SWITCH(); def emit_call_runtime_RESUME_THROW(); + def emit_call_runtime_DECOMPRESS_CONT(); def emit_call_runtime_RESUME_THROW_REF(); def emit_jump_to_trap_at(reason: TrapReason); def emit_call_runtime_op(op: Opcode); diff --git a/src/engine/compiler/SinglePassCompiler.v3 b/src/engine/compiler/SinglePassCompiler.v3 index 14980cf5f..45b840cb3 100644 --- a/src/engine/compiler/SinglePassCompiler.v3 +++ b/src/engine/compiler/SinglePassCompiler.v3 @@ -1462,6 +1462,7 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl var stub_resume = masm.newLabel(it.pc), end = masm.newLabel(it.pc); var cont_decl = ContDecl.!(module.heaptypes[cont_id]); if (checkForConstNull(state.peek())) return; + emit_decompress_top_cont(); var vsp = allocTmpFixed(ValueKind.REF, regs.vsp); var top = popReg(); @@ -2360,6 +2361,28 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl masm.emit_mov_r_r(ValueKind.REF, dst, regs.vfp); // XXX: use 3-addr adjustment of VSP (i.e. lea) if (slots > 0) masm.emit_addw_r_i(dst, int.view(slots) * masm.valuerep.slot_size); } + // Decompresses the stack of the continuation on top of the value stack, if compressed. + // All values are spilled, so that GC can update them during the runtime call. + def emit_decompress_top_cont() { + state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS); + emit_compute_vsp(regs.vsp, state.sp); + emit_spill_vsp(regs.vsp); + masm.emit_store_curstack_vsp(regs.vsp); + var done = masm.newLabel(it.pc), tmp = regs.scratch; + var valuerep = masm.valuerep; + masm.emit_mov_r_m(ValueKind.REF, tmp, MasmAddr(regs.vsp, valuerep.tag_size - valuerep.slot_size)); + masm.emit_br_r(tmp, MasmBrCond.REF_NULL, done); // null continuation traps later + masm.emit_mov_r_m(ValueKind.I32, tmp, MasmAddr(tmp, masm.getOffsets().NativeStack_state)); + masm.emit_brne_r_i(tmp, StackState.COMPRESSED.tag, done); + masm.emit_get_curstack(regs.runtime_arg0); + masm.emit_v3_set_NativeStack_rsp_r_r(regs.runtime_arg0, regs.sp); + masm.emit_push_NativeStack_rsp_r_r(regs.runtime_arg0); + masm.emit_call_runtime_DECOMPRESS_CONT(); + masm.emit_get_curstack(regs.scratch); + masm.emit_pop_NativeStack_rsp_r_r(regs.scratch); + emit_reload_regs(); + masm.bindLabel(done); + } def emit_spill_vsp(reg: Reg) { masm.emit_mov_m_r(ValueKind.REF, frame.vsp_slot, reg); } diff --git a/src/engine/compression/NaiveCompressionStrategy.v3 b/src/engine/compression/NaiveCompressionStrategy.v3 index a0722d3d1..d5335992b 100644 --- a/src/engine/compression/NaiveCompressionStrategy.v3 +++ b/src/engine/compression/NaiveCompressionStrategy.v3 @@ -16,4 +16,9 @@ class NaiveCompressedStack extends CompressedStack { var frames: Array; def name() -> string { return "Naive Compressed Stack"; } + def size() -> u64 { + var s = 32uL + 16uL * u64.view(frames.length); + for (f in frames) s += 64uL + 24uL * u64.view(f.data.vals.length); + return s; + } } diff --git a/src/engine/compression/PackedCompressionStrategy.v3 b/src/engine/compression/PackedCompressionStrategy.v3 index 99e3d5022..4b57f5923 100644 --- a/src/engine/compression/PackedCompressionStrategy.v3 +++ b/src/engine/compression/PackedCompressionStrategy.v3 @@ -83,4 +83,8 @@ class PackedCompressedStack extends CompressedStack { var refs: Array; def name() -> string { return "Packed Compressed Stack"; } + // Estimates the size of this object and its arrays in bytes. + def size() -> u64 { + return 32uL + 16uL * 3 + 40uL * u64.view(frames.length) + u64.view(packed.length) + 8uL * u64.view(refs.length); + } } diff --git a/src/engine/native/NativeStack.v3 b/src/engine/native/NativeStack.v3 index 5b00cf6b4..34d0a41fe 100644 --- a/src/engine/native/NativeStack.v3 +++ b/src/engine/native/NativeStack.v3 @@ -14,10 +14,12 @@ class NativeStack extends WasmStack { var return_results: Array; var state_: StackState = StackState.EMPTY; var next_stack: NativeStack; + var compressed: CompressedStack; // frames or bound values, if {COMPRESSED}; see {NativeStackCompression} + var scanned: bool; // GC scanner is registered new(range) { - if (valuerep.tagged) RiGc.registerScanner(this, NativeStack.scan); - if (range == null) return; // allocated without a mapping, no need to initialize further + if (range == null) return; // allocated without a segment, no need to initialize further + registerScanner(); clear(); if (Trace.stack && Debug.stack) Trace.OUT.put2("newStack start=0x%x, end=0x%x", range.start - Pointer.NULL, range.end - Pointer.NULL).ln(); } @@ -35,7 +37,17 @@ class NativeStack extends WasmStack { pushRspPointer(STACK_RETURN_PARENT_STUB.getEntry() + NOP_ENTRY); } // Requires {state == EMPTY}. + // A stack without a segment becomes {COMPRESSED}, getting a segment when resumed. def reset(func: Function) -> this { + if (range == null) { + checkState("reset()", StackState.EMPTY); + this.func = func; + return_results = func.sig.results; + parent = null; + bottom = this; + state_ = StackState.COMPRESSED; + return; + } reset0(func, func.sig.params.length); pushRspPointer(STACK_ENTER_FUNC_STUB.getEntry() + NOP_ENTRY); } @@ -43,10 +55,26 @@ class NativeStack extends WasmStack { def resetForFrameWrites(func: Function) -> this { reset0(func, 0); } + // Attaches {segment} to this stack, which has none. + def attach(segment: MemoryRange) { + range = segment; + registerScanner(); + } + private def registerScanner() { + if (scanned || !valuerep.tagged) return; + RiGc.registerScanner(this, NativeStack.scan); + scanned = true; + } + // Returns {true} if this stack has no frames yet and will enter {func} when resumed. + def isAtFuncEntry() -> bool { + return range != null && rsp < range.end && rsp.load() == STACK_ENTER_FUNC_STUB.getEntry() + NOP_ENTRY; + } // Requires {state == SUSPENDED}. // Pushes {args} incrementally onto the value stack. def bind(args: Range) -> this { if (args.length == 0) return; + if (NativeStackCompression.bindEntry(this, args)) return; + NativeStackCompression.decompress(this); checkState("bind()", StackState.SUSPENDED); for (v in args) push(v); } @@ -54,6 +82,7 @@ class NativeStack extends WasmStack { // Resumes running the Wasm or host code in this stack until that code either returns, throws, // or suspends itself. def resume() -> Result { + NativeStackCompression.decompress(this); checkState("resume()", StackState.SUSPENDED); if (Trace.stack) traceFrames("stack.resume", rsp + RETADDR_SIZE); var bottom = NativeStack.!(this.bottom); @@ -464,6 +493,7 @@ class NativeStack extends WasmStack { } // GC callback to scan (tagged) values on this stack def scan() { + if (range == null) return; // compressed, or allocated without a mapping if (Trace.gc) { Trace.OUT.put1("scanStack obj=0x%x", u64.view(Pointer.atObject(this) - Pointer.NULL)).ln(); } @@ -505,6 +535,7 @@ class NativeStack extends WasmStack { BpTypeCode.ARRAYREF.code, BpTypeCode.FUNCREF.code, BpTypeCode.EXNREF.code, + BpTypeCode.NULLEXNREF.code, BpTypeCode.NULLFUNCREF.code => return Value.Ref(vp.load()); BpTypeCode.I32.code => return Value.I32(vp.load()); @@ -512,7 +543,8 @@ class NativeStack extends WasmStack { BpTypeCode.F32.code => return Value.F32(vp.load()); BpTypeCode.F64.code => return Value.F64(vp.load()); BpTypeCode.V128.code => return Value.V128(vp.load(), (vp + 8).load()); - BpTypeCode.CONTREF.code => { + BpTypeCode.CONTREF.code, + BpTypeCode.NULLCONTREF.code => { var stack = vp.load(); var version = (vp + 8).load(); return Value.Cont(Continuation(stack, version)); diff --git a/src/engine/native/NativeStackCompression.v3 b/src/engine/native/NativeStackCompression.v3 new file mode 100644 index 000000000..1f5707462 --- /dev/null +++ b/src/engine/native/NativeStackCompression.v3 @@ -0,0 +1,120 @@ +// Copyright 2026 Wizard authors. All rights reserved. +// See LICENSE for details of Apache 2.0 license. + +// Values bound to the function of a {COMPRESSED} stack that has not been entered. +class EntryValues(vals: Array) extends CompressedStack { + def name() -> string { return "Entry Values"; } + def size() -> u64 { return 32uL + 24uL * u64.view(vals.length); } +} + +// Compresses suspended stacks to free their segments, and decompresses them before use. +// A {COMPRESSED} stack has no segment; its {compressed} field holds its frames, the values bound +// to its function ({EntryValues}), or {null} if neither, and its {parent_rsp_ptr} field holds +// the saved contents of the parent rsp slot. +component NativeStackCompression { + var numCompressed: int; + var numDecompressed: int; + + // Returns {true} if {stack} is suspended at a function entry or with only interpreter and SPC frames. + def canCompress(stack: NativeStack) -> bool { + if (stack.state_ != StackState.SUSPENDED) return false; + var range = stack.range; + if (range == null) return false; + var end = range.end; + // [end - 8] holds the parent's rsp, [end - 16] the return-parent stub. + if (stack.parent_rsp_ptr != end + -Pointer.SIZE) return false; + if (stack.isAtFuncEntry()) return stack.rsp == end + -3 * Pointer.SIZE; + var srp = stack.rsp, count = 0, bottom_sfp = Pointer.NULL; + while (srp >= range.start && srp < end) { + var retip = srp.load(); + var code = RiRuntime.findUserCode(retip); + match (code) { + x: X86_64InterpreterCode, x: X86_64SpcModuleCode => { + count++; + bottom_sfp = srp + Pointer.SIZE; + srp = code.nextFrame(retip, srp).1; + } + x: X86_64ReturnParentStub => { + if (count == 0 || srp != end + -2 * Pointer.SIZE) return false; + // Values below the bottom frame would be lost. + return NativeFrameHandle.Interpreter(bottom_sfp).vfp() == range.start; + } + _ => return false; // inlined frames, stubs, or host frames + } + } + return false; + } + // Compresses {stack}, which must satisfy {canCompress}, and returns its now unowned segment. + def compress(stack: NativeStack) -> NativeStackSegment { + var before = Metrics.stack_compress_cycles.ticks(); + var parent_rsp = stack.parent_rsp_ptr.load(); + var c: CompressedStack; + if (stack.isAtFuncEntry()) { + var n = int.!((stack.vsp - stack.range.start) / stack.valuerep.slot_size); + if (n > 0) { + var vals = Array.new(n); + for (i < n) vals[i] = stack.readValue(stack.range.start, i); + c = EntryValues.new(vals); + } + } else { + c = StackCompression.compressStack(stack); + } + var segment = NativeStackSegment.!(stack.range); + if (Trace.stack) Trace.OUT.put2("compress stack @ 0x%x, freeing segment 0x%x", + Pointer.atObject(stack) - Pointer.NULL, segment.start - Pointer.NULL).ln(); + segment.stack.set(null); + stack.range = null; + stack.vsp = stack.rsp = Pointer.NULL; + stack.parent_rsp_ptr = parent_rsp; + stack.compressed = c; + stack.state_ = StackState.COMPRESSED; + numCompressed++; + Metrics.stack_compressed.val++; + if (c != null) Metrics.stack_compressed_bytes.val += c.size(); + Metrics.stack_compress_cycles.updateDiff(before); + return segment; + } + // Binds {args} to a {COMPRESSED} stack that has not been entered, without decompressing it. + def bindEntry(stack: NativeStack, args: Range) -> bool { + if (stack.state_ != StackState.COMPRESSED) return false; + var vals: Array; + match (stack.compressed) { + null => vals = Ranges.dup(args); + x: EntryValues => vals = Arrays.concat(x.vals, Ranges.dup(args)); + _ => return false; + } + stack.compressed = EntryValues.new(vals); + return true; + } + // Decompresses {stack}, if compressed, onto a fresh segment. + def decompress(stack: NativeStack) { + if (stack.state_ != StackState.COMPRESSED) return; + var before = Metrics.stack_decompress_cycles.ticks(); + var segment = NativeStackManager.allocSegment(); + var c = stack.compressed, parent_rsp = stack.parent_rsp_ptr; + var func = stack.func, results = stack.return_results, parent = stack.parent, bottom = stack.bottom; + if (Trace.stack) Trace.OUT.put2("decompress stack @ 0x%x onto segment 0x%x", + Pointer.atObject(stack) - Pointer.NULL, segment.start - Pointer.NULL).ln(); + stack.attach(segment); + segment.stack.set(stack); + stack.clear(); + match (c) { + null => stack.reset(func); + x: EntryValues => { + stack.reset(func); + for (v in x.vals) stack.push(v); + } + _ => Target.writeFramesToStack(stack, StackCompression.decompressFrames(c)); + } + stack.func = func; + stack.return_results = results; + stack.parent = parent; + stack.bottom = bottom; + stack.parent_rsp_ptr.store(parent_rsp); + stack.compressed = null; + stack.state_ = StackState.SUSPENDED; + numDecompressed++; + Metrics.stack_decompressed.val++; + Metrics.stack_decompress_cycles.updateDiff(before); + } +} diff --git a/src/engine/native/NativeStackManager.v3 b/src/engine/native/NativeStackManager.v3 index 842f00d09..8706b6d34 100644 --- a/src/engine/native/NativeStackManager.v3 +++ b/src/engine/native/NativeStackManager.v3 @@ -9,20 +9,61 @@ component NativeStackManager { var all: NativeStackSegment; // all segments ever mapped; never unmapped var total: int; // number of segments in {all} var cache: NativeStack; // used in fast-path allocation in generated code + private var sweep: NativeStackSegment; // clock hand for compression victims + private var allocsSinceGC: int; + private var gcInterval: int; // minimum {allocsSinceGC} to collect before compressing // Gets a stack with a segment, or {null} if none is available. def getFreshStack() -> NativeStack { if (Trace.stack) Trace.OUT.puts("Requested fresh x86_stack...\n"); - if (cache == null) { - var ok = tryAllocatingNewStacks() || tryReclaimingSegments() || tryCollectingAndReclaiming() || tryCompressingStacks(); - if (!ok) return null; - } - + allocsSinceGC++; + if (cache == null && !refill()) return null; var result = cache; cache = cache.next_stack; result.next_stack = null; return result.clear(); } + // Gets a stack that will enter {func}. When no segment is available without compressing another + // stack, the new stack gets one lazily, when it is first resumed. Returns {null} if out of stacks. + def newStackFor(func: Function) -> NativeStack { + var lazy = StackTuning.lazyEntryStacks; + if (StackTuning.enableCompression && lazy > 0) { + var nearCapacity = cache == null && total >= StackTuning.maxStacks && tryReclaimingSegments() == 0; + if (lazy > 1 || nearCapacity) { + Metrics.stack_lazy_entries.val++; + return NativeStack.new(null).reset(func); + } + } + var s = getFreshStack(); + return if(s == null, null, s.reset(func)); + } + // Compresses {stack} if possible, adding its segment to the cache. + def compressStack(stack: NativeStack) -> bool { + if (!NativeStackCompression.canCompress(stack)) return false; + addToCache(NativeStackCompression.compress(stack)); + return true; + } + // Gets a segment that does not belong to any stack, e.g. to decompress a stack onto. + def allocSegment() -> NativeStackSegment { + var s = getFreshStack(); + if (s == null) fatal("out of stacks"); // TODO: trap instead + var segment = NativeStackSegment.!(s.range); + s.range = null; // detach; {s} is garbage + segment.stack.set(null); + return segment; + } + // Refills the cache with new segments, segments of dead stacks, or segments of compressed stacks. + // Returns {false} if no segment is available. + private def refill() -> bool { + if (tryAllocatingNewStacks()) return true; + if (tryReclaimingSegments() > 0) return true; + if (StackTuning.enableCompression) { + // Collect before compressing if collecting has been productive lately. + if (allocsSinceGC >= gcInterval && tryCollectingAndReclaiming()) return true; + if (tryCompressingStacks()) return true; + } + return tryCollectingAndReclaiming(); + } private def tryAllocatingNewStacks() -> bool { var size = EngineOptions.STACK_SIZE.get(); var batch = Ints.min(StackTuning.stackBatchSize, StackTuning.maxStacks - total); @@ -37,6 +78,7 @@ component NativeStackManager { var segment = NativeStackSegment.new(t.0, end); segment.next = all; all = segment; + Metrics.stack_segments.val++; // insert redzone roughly in the middle of the mapping. var rzStart = t.0 + ((size >> 1) & ~(PAGE_SIZE - 1u)), rzEnd = rzStart + PAGE_SIZE; if (!RedZones.addRedZone(rzStart, rzEnd)) fatal("could not protect value stack red zone"); @@ -48,26 +90,43 @@ component NativeStackManager { } return cache != null; // OK if succeeded in allocating at least one } - private def tryReclaimingSegments() -> bool { - var ok = false; + // Returns the number of segments reclaimed. + private def tryReclaimingSegments() -> int { + var count = 0; for (p = all; p != null; p = p.next) { if (p.stack.get() == null) { // weak reference was cleared by GC, can reclaim - var s = NativeStack.new(p); - p.stack.set(s); - s.next_stack = cache; - cache = s; - ok = true; + addToCache(p); + count++; } } - return ok; + return count; } // Weak references are only cleared by a GC, so force one to find dead stacks. private def tryCollectingAndReclaiming() -> bool { if (all == null) return false; Target.forceGC(); - return tryReclaimingSegments(); + Metrics.stack_forced_gcs.val++; + allocsSinceGC = 0; + var count = tryReclaimingSegments(); + // Back off exponentially if unproductive. + gcInterval = if(count * 4 >= total, total, Ints.min(gcInterval * 2 + 1, 1 << 24)); + return count > 0; } + // Compresses one suspended stack, sweeping round-robin over all segments. private def tryCompressingStacks() -> bool { + for (i < total) { + if (sweep == null) sweep = all; + var segment = sweep; + sweep = segment.next; + var s = segment.stack.get(); + if (s == null) { // collected since the last reclamation + addToCache(segment); + return true; + } + if (!NativeStackCompression.canCompress(s)) continue; + addToCache(NativeStackCompression.compress(s)); + return true; + } return false; } // Checks whether a fault at {addr} hit the guard page above a stack segment, which indicates @@ -80,7 +139,15 @@ component NativeStackManager { } } } + private def addToCache(segment: NativeStackSegment) { + var s = NativeStack.new(segment); + segment.stack.set(s); + s.next_stack = cache; + cache = s; + } + // Returns {stack} to the cache, clearing it so that it cannot be compressed. def recycleStack(stack: NativeStack) { + stack.clear(); stack.next_stack = cache; cache = stack; } diff --git a/src/engine/v3/V3Target.v3 b/src/engine/v3/V3Target.v3 index e692bf004..5f1a14913 100644 --- a/src/engine/v3/V3Target.v3 +++ b/src/engine/v3/V3Target.v3 @@ -10,9 +10,11 @@ component Target { def newMemory = V3Memory.new; def newWasmStack = V3Interpreter.new; + def newWasmStackFor(func: Function) -> WasmStack { return V3Interpreter.new().reset(func); } var unused_ = ExecuteOptions.registerDefaultMode("v3-int", V3InterpreterOnlyStrategy.new(), "slow interpreter only"); def forceGC() { } // nop + def gcStat(v: void, which: int) -> u64 { return 0; } // no GC statistics available def ticksNs() => u64.view(u32.view(System.ticksNs())); def rdtsc() => u64.view(u32.view(System.ticksNs())); def getRdtsc() => rdtsc; diff --git a/src/engine/x86-64/X86_64Interpreter.v3 b/src/engine/x86-64/X86_64Interpreter.v3 index 991faa1dd..2ee190256 100644 --- a/src/engine/x86-64/X86_64Interpreter.v3 +++ b/src/engine/x86-64/X86_64Interpreter.v3 @@ -2498,6 +2498,7 @@ class X86_64InterpreterGen(ic: X86_64InterpreterCode, w: DataWriter) { computeCurIpForTrap(-1); computePcFromCurIp(); saveCallerIVars(); + genDecompressTopCont(); genReadUleb32(G(cont_id)); // read signature index var cont = xenv.xmm0; // (stack, version) in low/high 8 bytes @@ -4267,6 +4268,20 @@ class X86_64InterpreterGen(ic: X86_64InterpreterCode, w: DataWriter) { decrementVsp(); asm.movq_r_m(reg, vsph[0].value); } + // Decompresses the stack of the continuation on top of the value stack, if compressed. + // Requires the caller's ivars to be saved. + private def genDecompressTopCont() { + var done = X86_64Label.new(); + asm.movq_r_m(r_tmp1, vsph[-1].value); // %tmp1 = cont.stack + asm.q.cmp_r_i(r_tmp1, 0); + asm.jc_rel_far(C.Z, done); // null continuation traps later + asm.d.cmp_m_i(r_tmp1.plus(offsets.NativeStack_state), StackState.COMPRESSED.tag); + asm.jc_rel_far(C.NZ, done); + masm.emit_get_curstack(xenv.tmp1); + callRuntime(refRuntimeCall(X86_64RT.runtime_DECOMPRESS_CONT), [r_tmp1], false); + restoreCallerIVars(); + asm.bind(done); + } // Pops a continuation value from the value stack into the XMM register {reg}, // which receives the 16-byte (stack, version) pair. private def genPopCont(reg: Reg) { diff --git a/src/engine/x86-64/X86_64MacroAssembler.v3 b/src/engine/x86-64/X86_64MacroAssembler.v3 index b4043ae6a..87c124621 100644 --- a/src/engine/x86-64/X86_64MacroAssembler.v3 +++ b/src/engine/x86-64/X86_64MacroAssembler.v3 @@ -872,6 +872,9 @@ class X86_64MacroAssembler extends MacroAssembler { def emit_call_runtime_RESUME_THROW() { emit_call_runtime(X86_64RT.runtime_handle_resume_throw); } + def emit_call_runtime_DECOMPRESS_CONT() { + emit_call_runtime(X86_64RT.runtime_DECOMPRESS_CONT); + } def emit_call_runtime_RESUME_THROW_REF() { emit_call_runtime(X86_64RT.runtime_resume_throw_ref); } diff --git a/src/engine/x86-64/X86_64Runtime.v3 b/src/engine/x86-64/X86_64Runtime.v3 index f214040d8..4434db7b7 100644 --- a/src/engine/x86-64/X86_64Runtime.v3 +++ b/src/engine/x86-64/X86_64Runtime.v3 @@ -113,6 +113,12 @@ component X86_64Runtime { Unhandled(t) => return t; } } + // Decompresses the continuation on top of {stack}'s value stack, where GC can update it. + def runtime_DECOMPRESS_CONT(stack: NativeStack) { + var ptr = stack.vsp + (stack.valuerep.tag_size - stack.valuerep.slot_size); + var cont_stack = ptr.load(); + if (cont_stack != null) NativeStackCompression.decompress(NativeStack.!(cont_stack)); + } def runtime_CONT_NEW(stack: NativeStack, instance: Instance, cont_index: u31) -> Throwable { return Runtime.CONT_NEW(stack, instance, cont_index); } @@ -129,6 +135,7 @@ component X86_64Runtime { .put2("resume_throw %q @ contStack[addr=0x%x] ", ex.render, Pointer.atObject(contStack) - Pointer.NULL) .put1("on stack[rsp=0x%x]", stack.rsp - Pointer.NULL).ln(); + NativeStackCompression.decompress(contStack); // search for handler in continuation var result = contStack.throw(ex); match (result) { @@ -208,6 +215,7 @@ component X86_64Runtime { if (Continuations.isUsed(target_cont)) return stack.trap(TrapReason.USED_CONTINUATION); var target_stack = target_cont.stack; Continuations.setUsed(target_cont); + NativeStackCompression.decompress(NativeStack.!(target_stack)); var this_cont = stack.unwindChain(tag, WasmStack.tryHandleSwitch); if (Continuations.isNull(this_cont)) return stack.trap(TrapReason.INVALID_SUSPEND); diff --git a/src/engine/x86-64/X86_64Target.v3 b/src/engine/x86-64/X86_64Target.v3 index 33af71f0a..caa66951d 100644 --- a/src/engine/x86-64/X86_64Target.v3 +++ b/src/engine/x86-64/X86_64Target.v3 @@ -16,7 +16,20 @@ component Target { def newMemory = NativeWasmMemory.new; def forceGC = RiGc.forceGC; + // Gets GC statistics: count, time, max time, max live bytes, heap bytes, last live bytes. + def gcStat(v: void, which: int) -> u64 { + match (which) { + 0 => return u64.view(GcStats.gc_count); + 1 => return u64.view(GcStats.collection_us); + 2 => return u64.view(GcStats.max_collection_us); + 3 => return u64.view(GcStats.max_live_bytes); + 4 => return u64.view(GcStats.heap_bytes); + 5 => return u64.view(GcStats.last_live_bytes); + } + return 0; + } def newWasmStack = NativeStackManager.getFreshStack; + def newWasmStackFor = NativeStackManager.newStackFor; def recycleWasmStack = NativeStackManager.recycleStack; def tagging = Tagging.new(!FeatureDisable.valueTags, !FeatureDisable.simd); diff --git a/src/modules/wizeng/WizengModule.v3 b/src/modules/wizeng/WizengModule.v3 index 9e9c36532..b7e63cf2b 100644 --- a/src/modules/wizeng/WizengModule.v3 +++ b/src/modules/wizeng/WizengModule.v3 @@ -11,6 +11,8 @@ enum WizengExports(sig: SigDecl, invoke: (WizengModule, Range) -> HostRes puti(SigCache.i_v, WizengModule.puti), puts(SigCache.ii_v, WizengModule.puts), rdtsc(SigCache.v_l, WizengModule.rdtsc), + heap_live(SigCache.v_l, WizengModule.heap_live), + putl(SigCache.l_v, WizengModule.putl), // XXX: wizeng module // .new_module() } @@ -157,6 +159,16 @@ class WizengModule extends HostModule("wizeng") { def rdtsc(args: Range) -> HostResult { return HostResult.Value1(Value.I64(Target.rdtsc())); } + // Exported: force a GC and return the number of live heap bytes, or 0 if unknown. + def heap_live(args: Range) -> HostResult { + Target.forceGC(); + return HostResult.Value1(Value.I64(Target.gcStat((), 5))); + } + // Exported: output a 64-bit integer to stdout. + def putl(args: Range) -> HostResult { + System.puts(Strings.format1("%d", Values.unbox_l(args[0]))); + return HostResult.Value0; + } // Exported: output a single byte to stdout. def putc(args: Range) -> HostResult { var v = Values.unbox_i(args[0]); diff --git a/test/regress/stacks/compression.failures b/test/regress/stacks/compression.failures index fb7a22735..6920a6c85 100644 --- a/test/regress/stacks/compression.failures +++ b/test/regress/stacks/compression.failures @@ -1,17 +1,2 @@ -test/regress/ext:stack-switching/many_stacks_dyn.bin.wast -test/regress/ext:stack-switching/many_stacks.bin.wast test/regress/ext:stack-switching/resume_chain3.bin.wast test/regress/ext:stack-switching/suspend15.bin.wast -test/regress/stacks/bind_partial0.bin.wast -test/regress/stacks/bind0.bin.wast -test/regress/stacks/chains0.bin.wast -test/regress/stacks/cont_in_frames0.bin.wast -test/regress/stacks/deep0.bin.wast -test/regress/stacks/exnref0.bin.wast -test/regress/stacks/live_conts0.bin.wast -test/regress/stacks/resume_throw_ref0.bin.wast -test/regress/stacks/resume_throw0.bin.wast -test/regress/stacks/scheduler0.bin.wast -test/regress/stacks/switch_ring0.bin.wast -test/regress/stacks/trap0.bin.wast -test/regress/stacks/value_kinds0.bin.wast diff --git a/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 b/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 new file mode 100644 index 000000000..e03809cd2 --- /dev/null +++ b/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 @@ -0,0 +1,932 @@ +// Copyright 2026 Wizard authors. All rights reserved. +// See LICENSE for details of Apache 2.0 license. + +// Tests for compressing suspended native stacks and resuming them from host and Wasm code in every tier. + +def M: NativeStackManager; +def C: NativeStackCompression; + +def T(name: string, f: StackCompressionTester -> void) { + UnitTests.registerT("x86stackcomp:", name, StackCompressionTester.new(_, X86_64InterpreterOnlyStrategy.new()), runTest(f, _)); +} +def TT(name: string, f: StackCompressionTester -> void) { + for (l = Target.getTestTiers(); l != null; l = l.tail) { + UnitTests.registerT(Strings.format1("x86stackcomp:%s", l.head.0), name, StackCompressionTester.new(_, l.head.1), runTest(f, _)); + } +} +def X_ = void( + T("can_compress_states", test_can_compress_states), + T("roundtrip_entry", test_roundtrip_entry), + T("roundtrip_entry_bind", test_roundtrip_entry_bind), + T("roundtrip_refs_gc", test_roundtrip_refs_gc), + T("victim_suspended_only", test_victim_suspended_only), + T("victim_round_robin", test_victim_round_robin), + T("victim_not_recycled", test_victim_not_recycled), + T("decompress_compresses_other", test_decompress_compresses_other), + T("disabled_no_compress", test_disabled_no_compress), + TT("mid_explicit", test_mid_explicit), + TT("mid_twice", test_mid_twice), + TT("mid_host_resume", test_mid_host_resume), + TT("mid_host_bind", test_mid_host_bind), + TT("mid_wasm_bind", test_mid_wasm_bind), + TT("mid_gc", test_mid_gc), + TT("gens_round_robin", test_gens_round_robin), + TT("gens_many", test_gens_many), + TT("gens_limit_one", test_gens_limit_one), + T("no_frames_not_compressible", test_no_frames_not_compressible), + T("running_not_compressible", test_running_not_compressible), + T("entry_host_func", test_entry_host_func), + T("repeat_roundtrip", test_repeat_roundtrip), + TT("chain_top", test_chain_top), + TT("chains_under_limit", test_chains_under_limit), + TT("deep_frames", test_deep_frames), + TT("deep_frames_under_limit", test_deep_frames_under_limit), + TT("used_after_compress", test_used_after_compress), + TT("trap_after_decompress", test_trap_after_decompress), + TT("throw_after_decompress", test_throw_after_decompress), + TT("resume_null", test_resume_null), + TT("entry_cont_arg", test_entry_cont_arg), + TT("gens_random_order", test_gens_random_order), + TT("compress_from_host_call", test_compress_from_host_call), + T("lazy_entry_near_capacity", test_lazy_entry_near_capacity), + T("lazy_entry_always", test_lazy_entry_always), + T("lazy_entry_refs_gc", test_lazy_entry_refs_gc), + TT("lazy_gens_round_robin", test_lazy_gens_round_robin), + TT("lazy_mid_wasm_bind", test_lazy_mid_wasm_bind), + TT("lazy_chain_top", test_lazy_chain_top), + () +); + +// The module below, compiled with the stack-switching reference interpreter. +// (module +// (type $fv (func)) +// (type $cv (cont $fv)) +// (type $fi (func (param i32))) +// (type $ci (cont $fi)) +// (type $f_i (func (result i32))) +// (type $c_i (cont $f_i)) +// (type $fi_i (func (param i32) (result i32))) +// (type $ci_i (cont $fi_i)) +// (type $box (struct (field i32))) +// +// (tag $yield (param i32)) +// (tag $ask (result i32)) +// +// (global $last (mut i32) (i32.const -1)) +// +// ;; Recurses {n} frames deep, each holding a v128 and a struct, then yields {v}. +// (func $deep (param $n i32) (param $v i32) +// (local $x v128) (local $b (ref null $box)) +// (local.set $x (i32x4.splat (local.get $v))) +// (local.set $b (struct.new $box (local.get $v))) +// (if (i32.eqz (local.get $n)) +// (then (suspend $yield (local.get $v))) +// (else (call $deep (i32.sub (local.get $n) (i32.const 1)) (local.get $v)))) +// (if (i32.ne (i32x4.extract_lane 3 (local.get $x)) (local.get $v)) (then (unreachable))) +// (if (i32.ne (struct.get $box 0 (local.get $b)) (local.get $v)) (then (unreachable))) +// ) +// ;; Yields {start}, {start+1}, ..., {start+9} from 3 frames deep, checking i64 and f64 locals. +// (func $gen (param $start i32) +// (local $i i32) (local $l i64) (local $d f64) (local $v i32) +// (loop $loop +// (local.set $v (i32.add (local.get $start) (local.get $i))) +// (call $deep (i32.const 3) (local.get $v)) +// (local.set $l (i64.add (local.get $l) (i64.extend_i32_s (local.get $v)))) +// (local.set $d (f64.add (local.get $d) (f64.convert_i32_s (local.get $v)))) +// (if (i64.ne (local.get $l) (i64.trunc_f64_s (local.get $d))) (then (unreachable))) +// (local.set $i (i32.add (local.get $i) (i32.const 1))) +// (br_if $loop (i32.lt_u (local.get $i) (i32.const 10))) +// ) +// ) +// ;; Yields once, then returns {x + 1000}. +// (func $once (param $x i32) (result i32) +// (suspend $yield (local.get $x)) +// (i32.add (local.get $x) (i32.const 1000)) +// ) +// ;; Asks for a value, then returns {x * 10 + answer}. +// (func $asker (param $x i32) (result i32) +// (i32.add (i32.mul (local.get $x) (i32.const 10)) (suspend $ask)) +// ) +// (elem declare func $gen $once $asker) +// +// ;; Resumes {k} until it yields (stores the value in $last and returns the new continuation) or returns (null). +// (func $step (param $k (ref $cv)) (result (ref null $cv)) +// (local $next (ref null $cv)) +// (block $h (result i32 (ref $cv)) +// (resume $cv (on $yield $h) (local.get $k)) +// (return (ref.null $cv)) +// ) +// (local.set $next) +// (global.set $last) +// (local.get $next) +// ) +// (func (export "make") (param $start i32) (result (ref null $cv)) +// (call $step (cont.bind $ci $cv (local.get $start) (cont.new $ci (ref.func $gen)))) +// ) +// (func (export "next") (param $k (ref null $cv)) (result (ref null $cv)) +// (call $step (ref.as_non_null (local.get $k))) +// ) +// (func (export "last") (result i32) (global.get $last)) +// +// ;; Returns a continuation of {$once} suspended at its yield. +// (func (export "make_once") (param $x i32) (result (ref null $c_i)) +// (local $k (ref null $c_i)) +// (block $h (result i32 (ref $c_i)) +// (drop (resume $ci_i (on $yield $h) (local.get $x) (cont.new $ci_i (ref.func $once)))) +// (return (ref.null $c_i)) +// ) +// (local.set $k) +// (drop) +// (local.get $k) +// ) +// ;; Returns a continuation of {$asker} suspended at its question. +// (func (export "make_asker") (param $x i32) (result (ref null $ci_i)) +// (block $h (result (ref $ci_i)) +// (drop (resume $ci_i (on $ask $h) (local.get $x) (cont.new $ci_i (ref.func $asker)))) +// (return (ref.null $ci_i)) +// ) +// ) +// ;; Answers the question of {k} with {v} via cont.bind, then resumes it. +// (func (export "answer") (param $k (ref null $ci_i)) (param $v i32) (result i32) +// (resume $c_i (cont.bind $ci_i $c_i (local.get $v) (local.get $k))) +// ) +// ;; --- Chains, deep frames, traps, and exceptions --- +// (tag $other) +// (tag $exn (param i32)) +// ;; Suspends with {yield}, which its parent {mid} does not handle, then returns {2 * x}. +// (func $inner (param $x i32) (result i32) +// (suspend $yield (local.get $x)) +// (i32.mul (local.get $x) (i32.const 2)) +// ) +// ;; Resumes {inner} with a handler for an unrelated tag, then returns its result + 1. +// (func $mid (param $x i32) (result i32) +// (block $h (result (ref $c_i)) +// (return (i32.add (i32.const 1) +// (resume $ci_i (on $other $h) (local.get $x) (cont.new $ci_i (ref.func $inner))))) +// ) +// (unreachable) +// ) +// ;; Recurses {d} frames deep, each with a local, then yields {x}, then returns the sum of locals. +// (func $deep_sum (param $d i32) (param $x i32) (result i32) +// (local $mine i32) +// (local.set $mine (i32.add (local.get $d) (local.get $x))) +// (if (result i32) (i32.eqz (local.get $d)) +// (then (suspend $yield (local.get $x)) (local.get $mine)) +// (else (i32.add (local.get $mine) (call $deep_sum (i32.sub (local.get $d) (i32.const 1)) (local.get $x))))) +// ) +// (func $deep_entry (param $x i32) (result i32) +// (call $deep_sum (i32.const 100) (local.get $x)) +// ) +// ;; Yields, then traps. +// (func $trapper (param $x i32) (result i32) +// (suspend $yield (local.get $x)) +// (unreachable) +// ) +// ;; Yields, then throws {x}. +// (func $thrower (param $x i32) (result i32) +// (suspend $yield (local.get $x)) +// (throw $exn (local.get $x)) +// ) +// (elem declare func $inner $mid $deep_entry $trapper $thrower) +// +// ;; Runs {f(x)} until it yields, returning the suspended continuation. +// (func $start (param $f (ref $ci_i)) (param $x i32) (result (ref null $c_i)) +// (local $k (ref null $c_i)) +// (block $h (result i32 (ref $c_i)) +// (drop (resume $ci_i (on $yield $h) (local.get $x) (local.get $f))) +// (return (ref.null $c_i)) +// ) +// (local.set $k) +// (global.set $last) +// (local.get $k) +// ) +// ;; Returns a continuation whose top stack is {inner} and whose bottom stack is {mid}. +// (func (export "make_chain") (param $x i32) (result (ref null $c_i)) +// (call $start (cont.new $ci_i (ref.func $mid)) (local.get $x)) +// ) +// (func (export "make_deep") (param $x i32) (result (ref null $c_i)) +// (call $start (cont.new $ci_i (ref.func $deep_entry)) (local.get $x)) +// ) +// (func (export "make_trapper") (param $x i32) (result (ref null $c_i)) +// (call $start (cont.new $ci_i (ref.func $trapper)) (local.get $x)) +// ) +// (func (export "make_thrower") (param $x i32) (result (ref null $c_i)) +// (call $start (cont.new $ci_i (ref.func $thrower)) (local.get $x)) +// ) +// ;; Resumes {k} to completion. +// (func (export "finish") (param $k (ref null $c_i)) (result i32) +// (resume $c_i (local.get $k)) +// ) +// ;; Resumes {k} to completion, returning the payload of an exception it throws. +// (func (export "catch") (param $k (ref null $c_i)) (result i32) +// (block $c (result i32) +// (try_table (catch $exn $c) (drop (resume $c_i (local.get $k)))) +// (i32.const -1) +// ) +// ) +// (func (export "resume_null") (result i32) +// (resume $c_i (ref.null $c_i)) +// ) +// ) +def GEN_MODULE: Array = [ + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0xe9, 0x80, 0x80, 0x80, 0x00, 0x15, 0x60, + 0x00, 0x00, 0x5d, 0x00, 0x60, 0x01, 0x7f, 0x00, 0x5d, 0x02, 0x60, 0x00, 0x01, 0x7f, 0x5d, 0x04, + 0x60, 0x01, 0x7f, 0x01, 0x7f, 0x5d, 0x06, 0x5f, 0x01, 0x7f, 0x00, 0x60, 0x02, 0x7f, 0x7f, 0x00, + 0x60, 0x01, 0x64, 0x01, 0x01, 0x63, 0x01, 0x60, 0x00, 0x02, 0x7f, 0x64, 0x01, 0x60, 0x01, 0x7f, + 0x01, 0x63, 0x01, 0x60, 0x01, 0x63, 0x01, 0x01, 0x63, 0x01, 0x60, 0x01, 0x7f, 0x01, 0x63, 0x05, + 0x60, 0x00, 0x02, 0x7f, 0x64, 0x05, 0x60, 0x01, 0x7f, 0x01, 0x63, 0x07, 0x60, 0x02, 0x63, 0x07, + 0x7f, 0x01, 0x7f, 0x60, 0x02, 0x7f, 0x7f, 0x01, 0x7f, 0x60, 0x02, 0x64, 0x07, 0x7f, 0x01, 0x63, + 0x05, 0x60, 0x01, 0x63, 0x05, 0x01, 0x7f, 0x03, 0x9a, 0x80, 0x80, 0x80, 0x00, 0x19, 0x09, 0x02, + 0x06, 0x06, 0x0a, 0x0c, 0x0d, 0x04, 0x0e, 0x10, 0x11, 0x06, 0x06, 0x12, 0x06, 0x06, 0x06, 0x13, + 0x0e, 0x0e, 0x0e, 0x0e, 0x14, 0x14, 0x04, 0x0d, 0x89, 0x80, 0x80, 0x80, 0x00, 0x04, 0x00, 0x02, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x06, 0x86, 0x80, 0x80, 0x80, 0x00, 0x01, 0x7f, 0x01, 0x41, + 0x7f, 0x0b, 0x07, 0x8e, 0x81, 0x80, 0x80, 0x00, 0x0d, 0x04, 0x6d, 0x61, 0x6b, 0x65, 0x00, 0x05, + 0x04, 0x6e, 0x65, 0x78, 0x74, 0x00, 0x06, 0x04, 0x6c, 0x61, 0x73, 0x74, 0x00, 0x07, 0x09, 0x6d, + 0x61, 0x6b, 0x65, 0x5f, 0x6f, 0x6e, 0x63, 0x65, 0x00, 0x08, 0x0a, 0x6d, 0x61, 0x6b, 0x65, 0x5f, + 0x61, 0x73, 0x6b, 0x65, 0x72, 0x00, 0x09, 0x06, 0x61, 0x6e, 0x73, 0x77, 0x65, 0x72, 0x00, 0x0a, + 0x0a, 0x6d, 0x61, 0x6b, 0x65, 0x5f, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x00, 0x12, 0x09, 0x6d, 0x61, + 0x6b, 0x65, 0x5f, 0x64, 0x65, 0x65, 0x70, 0x00, 0x13, 0x0c, 0x6d, 0x61, 0x6b, 0x65, 0x5f, 0x74, + 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x00, 0x14, 0x0c, 0x6d, 0x61, 0x6b, 0x65, 0x5f, 0x74, 0x68, + 0x72, 0x6f, 0x77, 0x65, 0x72, 0x00, 0x15, 0x06, 0x66, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x00, 0x16, + 0x05, 0x63, 0x61, 0x74, 0x63, 0x68, 0x00, 0x17, 0x0b, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x5f, + 0x6e, 0x75, 0x6c, 0x6c, 0x00, 0x18, 0x09, 0x8f, 0x80, 0x80, 0x80, 0x00, 0x02, 0x03, 0x00, 0x03, + 0x01, 0x02, 0x03, 0x03, 0x00, 0x05, 0x0b, 0x0c, 0x0e, 0x0f, 0x10, 0x0a, 0xcd, 0x84, 0x80, 0x80, + 0x00, 0x19, 0xc1, 0x80, 0x80, 0x80, 0x00, 0x02, 0x01, 0x7b, 0x01, 0x63, 0x08, 0x20, 0x01, 0xfd, + 0x11, 0x21, 0x02, 0x20, 0x01, 0xfb, 0x00, 0x08, 0x21, 0x03, 0x20, 0x00, 0x45, 0x04, 0x40, 0x20, + 0x01, 0xe2, 0x00, 0x05, 0x20, 0x00, 0x41, 0x01, 0x6b, 0x20, 0x01, 0x10, 0x00, 0x0b, 0x20, 0x02, + 0xfd, 0x1b, 0x03, 0x20, 0x01, 0x47, 0x04, 0x40, 0x00, 0x0b, 0x20, 0x03, 0xfb, 0x02, 0x08, 0x00, + 0x20, 0x01, 0x47, 0x04, 0x40, 0x00, 0x0b, 0x0b, 0xc2, 0x80, 0x80, 0x80, 0x00, 0x04, 0x01, 0x7f, + 0x01, 0x7e, 0x01, 0x7c, 0x01, 0x7f, 0x03, 0x40, 0x20, 0x00, 0x20, 0x01, 0x6a, 0x21, 0x04, 0x41, + 0x03, 0x20, 0x04, 0x10, 0x00, 0x20, 0x02, 0x20, 0x04, 0xac, 0x7c, 0x21, 0x02, 0x20, 0x03, 0x20, + 0x04, 0xb7, 0xa0, 0x21, 0x03, 0x20, 0x02, 0x20, 0x03, 0xb0, 0x52, 0x04, 0x40, 0x00, 0x0b, 0x20, + 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x20, 0x01, 0x41, 0x0a, 0x49, 0x0d, 0x00, 0x0b, 0x0b, 0x8c, + 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x00, 0xe2, 0x00, 0x20, 0x00, 0x41, 0xe8, 0x07, 0x6a, 0x0b, + 0x8a, 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x00, 0x41, 0x0a, 0x6c, 0xe2, 0x01, 0x6a, 0x0b, 0x99, + 0x80, 0x80, 0x80, 0x00, 0x01, 0x01, 0x63, 0x01, 0x02, 0x0b, 0x20, 0x00, 0xe3, 0x01, 0x01, 0x00, + 0x00, 0x00, 0xd0, 0x01, 0x0f, 0x0b, 0x21, 0x01, 0x24, 0x00, 0x20, 0x01, 0x0b, 0x8d, 0x80, 0x80, + 0x80, 0x00, 0x00, 0x20, 0x00, 0xd2, 0x01, 0xe0, 0x03, 0xe1, 0x03, 0x01, 0x10, 0x04, 0x0b, 0x87, + 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x00, 0xd4, 0x10, 0x04, 0x0b, 0x84, 0x80, 0x80, 0x80, 0x00, + 0x00, 0x23, 0x00, 0x0b, 0x9d, 0x80, 0x80, 0x80, 0x00, 0x01, 0x01, 0x63, 0x05, 0x02, 0x0f, 0x20, + 0x00, 0xd2, 0x02, 0xe0, 0x07, 0xe3, 0x07, 0x01, 0x00, 0x00, 0x00, 0x1a, 0xd0, 0x05, 0x0f, 0x0b, + 0x21, 0x01, 0x1a, 0x20, 0x01, 0x0b, 0x96, 0x80, 0x80, 0x80, 0x00, 0x00, 0x02, 0x64, 0x07, 0x20, + 0x00, 0xd2, 0x03, 0xe0, 0x07, 0xe3, 0x07, 0x01, 0x00, 0x01, 0x00, 0x1a, 0xd0, 0x07, 0x0f, 0x0b, + 0x0b, 0x8c, 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x01, 0x20, 0x00, 0xe1, 0x07, 0x05, 0xe3, 0x05, + 0x00, 0x0b, 0x8b, 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x00, 0xe2, 0x00, 0x20, 0x00, 0x41, 0x02, + 0x6c, 0x0b, 0x97, 0x80, 0x80, 0x80, 0x00, 0x00, 0x02, 0x64, 0x05, 0x41, 0x01, 0x20, 0x00, 0xd2, + 0x0b, 0xe0, 0x07, 0xe3, 0x07, 0x01, 0x00, 0x02, 0x00, 0x6a, 0x0f, 0x0b, 0x00, 0x0b, 0xa4, 0x80, + 0x80, 0x80, 0x00, 0x01, 0x01, 0x7f, 0x20, 0x00, 0x20, 0x01, 0x6a, 0x21, 0x02, 0x20, 0x00, 0x45, + 0x04, 0x7f, 0x20, 0x01, 0xe2, 0x00, 0x20, 0x02, 0x05, 0x20, 0x02, 0x20, 0x00, 0x41, 0x01, 0x6b, + 0x20, 0x01, 0x10, 0x0d, 0x6a, 0x0b, 0x0b, 0x89, 0x80, 0x80, 0x80, 0x00, 0x00, 0x41, 0xe4, 0x00, + 0x20, 0x00, 0x10, 0x0d, 0x0b, 0x87, 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x00, 0xe2, 0x00, 0x00, + 0x0b, 0x8a, 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x00, 0xe2, 0x00, 0x20, 0x00, 0x08, 0x03, 0x0b, + 0x9c, 0x80, 0x80, 0x80, 0x00, 0x01, 0x01, 0x63, 0x05, 0x02, 0x0f, 0x20, 0x01, 0x20, 0x00, 0xe3, + 0x07, 0x01, 0x00, 0x00, 0x00, 0x1a, 0xd0, 0x05, 0x0f, 0x0b, 0x21, 0x02, 0x24, 0x00, 0x20, 0x02, + 0x0b, 0x8a, 0x80, 0x80, 0x80, 0x00, 0x00, 0xd2, 0x0c, 0xe0, 0x07, 0x20, 0x00, 0x10, 0x11, 0x0b, + 0x8a, 0x80, 0x80, 0x80, 0x00, 0x00, 0xd2, 0x0e, 0xe0, 0x07, 0x20, 0x00, 0x10, 0x11, 0x0b, 0x8a, + 0x80, 0x80, 0x80, 0x00, 0x00, 0xd2, 0x0f, 0xe0, 0x07, 0x20, 0x00, 0x10, 0x11, 0x0b, 0x8a, 0x80, + 0x80, 0x80, 0x00, 0x00, 0xd2, 0x10, 0xe0, 0x07, 0x20, 0x00, 0x10, 0x11, 0x0b, 0x87, 0x80, 0x80, + 0x80, 0x00, 0x00, 0x20, 0x00, 0xe3, 0x05, 0x00, 0x0b, 0x94, 0x80, 0x80, 0x80, 0x00, 0x00, 0x02, + 0x7f, 0x1f, 0x40, 0x01, 0x00, 0x03, 0x00, 0x20, 0x00, 0xe3, 0x05, 0x00, 0x1a, 0x0b, 0x41, 0x7f, + 0x0b, 0x0b, 0x87, 0x80, 0x80, 0x80, 0x00, 0x00, 0xd0, 0x05, 0xe3, 0x05, 0x00, 0x0b, + +]; + +private class StackCompressionTester(t: Tester, tiering: ExecutionStrategy) { + def saved_max = StackTuning.maxStacks; + def saved_compression = StackTuning.enableCompression; + def saved_lazy = StackTuning.lazyEntryStacks; + var held: Array; // stacks pinned by {limit} + var instance: Instance; + var limit_total = -1; + + new() { Execute.tiering = tiering; } + + def restore() { + StackTuning.maxStacks = saved_max; + StackTuning.enableCompression = saved_compression; + StackTuning.lazyEntryStacks = saved_lazy; + limit_total = -1; + } + // Pins all existing segments, then allows at most {extra} more, so that only stacks + // created by the test are candidates for reclamation or compression. + def limit(extra: int) { + StackTuning.maxStacks = M.total; + M.cache = null; + Target.forceGC(); + var free = 0; + for (seg = M.all; seg != null; seg = seg.next) if (seg.stack.get() == null) free++; + held = Array.new(free); + for (i < free) held[i] = M.getFreshStack(); + StackTuning.maxStacks = M.total + extra; + limit_total = StackTuning.maxStacks; + } + def compressing(extra: int) { + StackTuning.enableCompression = true; + limit(extra); + } + def checkInvariants(where: string) { + var count = 0; + for (seg = M.all; seg != null; seg = seg.next) { + count++; + var s = seg.stack.get(); + if (s == null) continue; + if (s.range != seg) t.fail1("%s: owning stack does not point back at its segment", where); + if (s.state() == StackState.COMPRESSED) t.fail1("%s: compressed stack owns a segment", where); + } + if (count != M.total) t.fail1("%s: segment count mismatch", where); + if (limit_total >= 0 && M.total > limit_total) t.fail3("%s: expected at most %d segments, got %d", where, limit_total, M.total); + } + def assertState(expected: StackState, s: NativeStack) { + var got = s.state(); + if (got != expected) t.fail2("expected state %s, got %s", expected.name, got.name); + var compressed = got == StackState.COMPRESSED; + if (compressed != (s.range == null)) t.fail("expected a stack to have a segment iff not compressed"); + if (!compressed && s.compressed != null) t.fail("expected no compressed data if not compressed"); + } + def assertResult(expected: Result, got: Result) { + if (!expected.equals(got)) t.fail2("expected %q, got %q", expected.render, got.render); + } + def assertCompress(s: NativeStack) { + if (s == null) return t.fail("expected a stack, got null"); + if (!M.compressStack(s)) return t.fail1("expected stack in state %s to be compressible", s.state().name); + assertState(StackState.COMPRESSED, s); + } + // A Wasm function [i32, i32] -> [i32] that subtracts its arguments. + def subFunc() -> WasmFunction { + var b = ModuleBuilder.new(); + b.sig(SigCache.ii_i).code([ + u8.!(Opcode.LOCAL_GET.code), 0, + u8.!(Opcode.LOCAL_GET.code), 1, + u8.!(Opcode.I32_SUB.code) + ]); + var err = ErrorGen.new("subFunc"); + var inst = Instantiator.new(Extension.set.all, b.module, [], err).run(); + return WasmFunction.!(inst.functions[0]); + } + // Instantiates {GEN_MODULE} for this tester's tier. + def gen() -> this { + if (instance != null) return; + var err = ErrorGen.new("GEN_MODULE"); + var limits = Limits.new().set(Extension.set.all); + var bp = BinParser.new(Extension.set.all, limits, err, "GEN_MODULE"); + bp.tiering = tiering; + var module = bp.push(GEN_MODULE, 0, GEN_MODULE.length).finish(); + if (!err.ok()) return t.fail(err.error_msg); + instance = Instantiator.new(Extension.set.all, module, [], err).run(); + if (!err.ok()) return t.fail(err.error_msg); + tiering.onTestModule(module); + for (f in instance.functions) { + if (WasmFunction.?(f)) tiering.onTestRun(WasmFunction.!(f), err); + } + if (!err.ok()) return t.fail(err.error_msg); + } + def call(name: string, args: Array) -> Result { + gen(); + for (i < instance.module.exports.length) { + if (Strings.equal(name, instance.module.exports[i].0)) { + return Execute.call(Function.!(instance.exports[i]), args); + } + } + t.fail1("export %s not found", name); + return Result.Value([]); + } + def callCont(name: string, args: Array) -> Continuation { + var r = call(name, args); + match (r) { + Value(vals) => if (vals.length == 1 && Value.Cont.?(vals[0])) return Value.Cont.!(vals[0]).val; + _ => ; + } + t.fail2("expected %s to return a continuation, got %q", name, r.render); + return Continuations.NULL; + } + def exported(name: string) -> Function { + gen(); + for (i < instance.module.exports.length) { + if (Strings.equal(name, instance.module.exports[i].0)) return Function.!(instance.exports[i]); + } + t.fail1("export %s not found", name); + return null; + } + def assertTrap(reason: TrapReason, r: Result) { + match (r) { + Throw(thrown) => if (Trap.?(thrown) && Trap.!(thrown).reason == reason) return; + _ => ; + } + t.fail2("expected trap %s, got %q", reason.name, r.render); + } + def callInt(name: string, args: Array) -> int { + var r = call(name, args); + match (r) { + Value(vals) => if (vals.length == 1 && Value.I32.?(vals[0])) return int.view(Value.I32.!(vals[0]).val); + _ => ; + } + t.fail2("expected %s to return an i32, got %q", name, r.render); + return -1; + } + // Makes a generator yielding {start} .. {start + 9}, suspended at its first yield. + def make(start: int) -> Continuation { + var k = callCont("make", [Values.box_i(start)]); + if (!Continuations.isNull(k)) assertLast(start); + return k; + } + // Steps a generator, which must yield {expected}, or finish if {expected < 0}. + def next(k: Continuation, expected: int) -> Continuation { + var n = callCont("next", [Value.Cont(k)]); + if (expected < 0) { + if (!Continuations.isNull(n)) t.fail("expected generator to finish"); + } else { + if (Continuations.isNull(n)) t.fail1("expected generator to yield %d, but it finished", expected); + else assertLast(expected); + } + return n; + } + def assertLast(expected: int) { + var got = callInt("last", []); + if (got != expected) t.fail2("expected generator to yield %d, got %d", expected, got); + } + def stackOf(k: Continuation) -> NativeStack { + return NativeStack.!(k.stack); + } +} + +// Runs a test, then restores global settings and checks invariants. +def runTest(f: StackCompressionTester -> void, t: StackCompressionTester) { + f(t); + t.checkInvariants("end of test"); + t.restore(); +} + +def EMPTY_STRUCT = StructDecl.new(false, ValueTypes.NO_HEAPTYPES, []); +def obj() -> Value { return Value.Ref(ObjectModel.newStruct(EMPTY_STRUCT)); } + +def test_can_compress_states(t: StackCompressionTester) { + var f = t.subFunc(); + var s = M.getFreshStack(); + t.assertState(StackState.EMPTY, s); + if (C.canCompress(s)) t.t.fail("expected EMPTY stack not to be compressible"); + s.reset(f); + t.assertState(StackState.SUSPENDED, s); + if (!C.canCompress(s)) t.t.fail("expected SUSPENDED stack to be compressible"); + s.bind([Values.box_i(9)]); + if (!C.canCompress(s)) t.t.fail("expected SUSPENDED stack with bound values to be compressible"); + t.assertCompress(s); + if (C.canCompress(s)) t.t.fail("expected COMPRESSED stack not to be compressible"); + if (M.compressStack(s)) t.t.fail("expected compressing a COMPRESSED stack to fail"); + C.decompress(s); + t.assertState(StackState.SUSPENDED, s); + if (!C.canCompress(s)) t.t.fail("expected decompressed stack to be compressible again"); + t.assertResult(Result.Value([Values.box_i(5)]), s.bind([Values.box_i(4)]).resume()); + t.assertState(StackState.EMPTY, s); +} + +def test_roundtrip_entry(t: StackCompressionTester) { + var f = t.subFunc(); + var s = M.getFreshStack().reset(f).bind([Values.box_i(100), Values.box_i(1)]); + var before = (C.numCompressed, C.numDecompressed); + t.assertCompress(s); + t.t.assert_eq(before.0 + 1, C.numCompressed); + t.assertResult(Result.Value([Values.box_i(99)]), s.resume()); // decompresses + t.t.assert_eq(before.1 + 1, C.numDecompressed); +} + +def test_roundtrip_entry_bind(t: StackCompressionTester) { + var f = t.subFunc(); + var s = M.getFreshStack().reset(f).bind([Values.box_i(50)]); + t.assertCompress(s); + s.bind([Values.box_i(8)]); // binds without decompressing + t.assertState(StackState.COMPRESSED, s); + t.assertResult(Result.Value([Values.box_i(42)]), s.resume()); +} + +def test_roundtrip_refs_gc(t: StackCompressionTester) { + var b = ModuleBuilder.new(); + var sig = b.newSig([ValueTypes.ANYREF, ValueTypes.ANYREF], [ValueTypes.ANYREF, ValueTypes.ANYREF]); + b.sig(sig).code([u8.!(Opcode.LOCAL_GET.code), 1, u8.!(Opcode.LOCAL_GET.code), 0]); + var err = ErrorGen.new("swap"); + var f = Instantiator.new(Extension.set.all, b.module, [], err).run().functions[0]; + var a = obj(), c = obj(); + var s = M.getFreshStack().reset(f).bind([a, c]); + t.assertCompress(s); + for (i < 3) Target.forceGC(); // the compressed values must keep the objects alive (and be updated) + t.assertResult(Result.Value([c, a]), s.resume()); +} + +def test_victim_suspended_only(t: StackCompressionTester) { + var f = t.subFunc(); + t.compressing(4); + var empty = M.getFreshStack(); + var s1 = M.getFreshStack().reset(f).bind([Values.box_i(10), Values.box_i(1)]); + var s2 = M.getFreshStack().reset(f).bind([Values.box_i(20), Values.box_i(2)]); + var s3 = M.getFreshStack().reset(f).bind([Values.box_i(30), Values.box_i(3)]); + // All segments are now owned; the next allocation must compress a SUSPENDED stack. + var fresh = M.getFreshStack(); + t.assertState(StackState.EMPTY, empty); + t.assertState(StackState.EMPTY, fresh); + var compressed = 0; + for (s in [s1, s2, s3]) if (s.state() == StackState.COMPRESSED) compressed++; + t.t.assert_eq(1, compressed); + t.checkInvariants("victim"); + t.assertResult(Result.Value([Values.box_i(9)]), s1.resume()); + t.assertResult(Result.Value([Values.box_i(18)]), s2.resume()); + t.assertResult(Result.Value([Values.box_i(27)]), s3.resume()); +} + +def test_victim_round_robin(t: StackCompressionTester) { + var f = t.subFunc(); + t.compressing(4); + var stacks = Array.new(4); + for (i < stacks.length) stacks[i] = M.getFreshStack().reset(f).bind([Values.box_i(i), Values.box_i(0)]); + // Each further allocation compresses a different stack. + var fresh = Array.new(4); + for (i < fresh.length) { + fresh[i] = M.getFreshStack(); + var compressed = 0; + for (s in stacks) if (s.state() == StackState.COMPRESSED) compressed++; + t.t.assert_eq(i + 1, compressed); + } + for (i < stacks.length) t.assertResult(Result.Value([Values.box_i(i)]), stacks[i].resume()); +} + +def test_victim_not_recycled(t: StackCompressionTester) { + var f = t.subFunc(); + t.compressing(2); + var a = M.getFreshStack().reset(f).bind([Values.box_i(7), Values.box_i(2)]); + var b = M.getFreshStack().reset(f); + M.recycleStack(b); // a recycled stack is cleared, so never compressed + t.assertState(StackState.EMPTY, b); + var c = M.getFreshStack(); + if (c != b) t.t.fail("expected the recycled stack to be reused"); + t.assertState(StackState.SUSPENDED, a); + var d = M.getFreshStack(); // must compress {a}, the only SUSPENDED stack + t.assertState(StackState.COMPRESSED, a); + t.assertState(StackState.EMPTY, d); + if (d == a || d == c) t.t.fail("expected a new stack for the compressed segment"); + t.assertResult(Result.Value([Values.box_i(5)]), a.resume()); +} + +def test_decompress_compresses_other(t: StackCompressionTester) { + var f = t.subFunc(); + t.compressing(2); + var a = M.getFreshStack().reset(f).bind([Values.box_i(100), Values.box_i(1)]); + var b = M.getFreshStack().reset(f).bind([Values.box_i(200), Values.box_i(2)]); + t.assertCompress(a); + var hold = M.getFreshStack(); // takes the segment freed by {a} + // Decompressing {a} requires a segment, which can only come from compressing {b}. + C.decompress(a); + t.assertState(StackState.SUSPENDED, a); + t.assertState(StackState.COMPRESSED, b); + t.assertState(StackState.EMPTY, hold); + t.checkInvariants("ping"); + t.assertResult(Result.Value([Values.box_i(99)]), a.resume()); + t.assertResult(Result.Value([Values.box_i(198)]), b.resume()); +} + +def test_disabled_no_compress(t: StackCompressionTester) { + var f = t.subFunc(); + StackTuning.enableCompression = false; + t.limit(3); + var a = M.getFreshStack().reset(f); + var b = M.getFreshStack().reset(f); + var dropped = allocAndDrop(f); + var c = M.getFreshStack(); // must come from the dead stack, never compression + if (c.range.start != dropped) t.t.fail("expected the dead stack's segment to be reclaimed"); + t.assertState(StackState.SUSPENDED, a); + t.assertState(StackState.SUSPENDED, b); + t.assertState(StackState.EMPTY, c); +} + +def allocAndDrop(f: Function) -> Pointer #no-inline { + return M.getFreshStack().reset(f).range.start; +} + +def test_mid_explicit(t: StackCompressionTester) { + var k = t.make(100); + var s = t.stackOf(k); + t.assertState(StackState.SUSPENDED, s); + t.assertCompress(s); // three frames deep, with v128 and struct locals + k = t.next(k, 101); // Wasm resume decompresses + if (t.stackOf(k) != s) t.t.fail("expected the continuation to keep its stack"); + t.assertState(StackState.SUSPENDED, s); + for (i = 2; i < 10; i++) { + t.assertCompress(t.stackOf(k)); + k = t.next(k, 100 + i); + } + t.assertCompress(t.stackOf(k)); + t.next(k, -1); +} + +def test_mid_twice(t: StackCompressionTester) { + var k = t.make(7); + var s = t.stackOf(k); + t.assertCompress(s); + C.decompress(s); + t.assertCompress(s); + for (i < 3) Target.forceGC(); + k = t.next(k, 8); + k = t.next(k, 9); +} + +def test_mid_host_resume(t: StackCompressionTester) { + var k = t.callCont("make_once", [Values.box_i(5)]); + var s = t.stackOf(k); + t.assertCompress(s); + t.assertResult(Result.Value([Values.box_i(1005)]), s.resume()); +} + +def test_mid_host_bind(t: StackCompressionTester) { + var k = t.callCont("make_asker", [Values.box_i(4)]); + var s = t.stackOf(k); + t.assertCompress(s); + s.bind([Values.box_i(2)]); + t.assertResult(Result.Value([Values.box_i(42)]), s.resume()); +} + +def test_mid_wasm_bind(t: StackCompressionTester) { + var k = t.callCont("make_asker", [Values.box_i(3)]); + t.assertCompress(t.stackOf(k)); + t.t.assert_eq(37, t.callInt("answer", [Value.Cont(k), Values.box_i(7)])); // cont.bind decompresses +} + +def test_mid_gc(t: StackCompressionTester) { + var gens = Array.new(8); + for (i < gens.length) { + gens[i] = t.make(i * 10); + t.assertCompress(t.stackOf(gens[i])); + } + for (i < 3) Target.forceGC(); + for (j = 1; j < 10; j++) { + for (i < gens.length) { + gens[i] = t.next(gens[i], i * 10 + j); + if ((i + j) % 2 == 0) t.assertCompress(t.stackOf(gens[i])); + } + Target.forceGC(); + } + for (i < gens.length) t.next(gens[i], -1); +} + +// Steps {n} generators round-robin to completion under a limit of {extra} segments. +def roundRobin(t: StackCompressionTester, n: int, extra: int) { + t.gen(); + t.compressing(extra); + var before = (C.numCompressed, C.numDecompressed); + var gens = Array.new(n); + for (i < n) gens[i] = t.make(i * 100); + for (j = 1; j < 10; j++) { + for (i < n) gens[i] = t.next(gens[i], i * 100 + j); + } + for (i < n) t.next(gens[i], -1); + if (C.numCompressed <= before.0) t.t.fail("expected stacks to be compressed"); + if (C.numDecompressed <= before.1) t.t.fail("expected stacks to be decompressed"); + t.checkInvariants("round robin"); +} + +def test_gens_round_robin(t: StackCompressionTester) { + roundRobin(t, 20, 4); +} + +def test_gens_many(t: StackCompressionTester) { + roundRobin(t, 500, 8); +} + +def test_gens_limit_one(t: StackCompressionTester) { + // Each call runs on its own stack and the generator needs another; one more suffices for decompression. + roundRobin(t, 10, 3); +} + +def test_no_frames_not_compressible(t: StackCompressionTester) { + var s = M.getFreshStack().resetForFrameWrites(t.subFunc()); + t.assertState(StackState.SUSPENDED, s); + if (C.canCompress(s)) t.t.fail("expected a stack with no frames and no entry not to be compressible"); + s.clear(); +} + +def test_running_not_compressible(t: StackCompressionTester) { + // Inside a host call, the running stack cannot be compressed. + var hf = HostFunction.new("check", SigCache.v_i, checkRunning(t, _)); + t.assertResult(Result.Value([Values.box_i(7)]), M.runOnFreshStack(hf, [])); +} +def checkRunning(t: StackCompressionTester, args: Range) -> HostResult { + var cur = X86_64Runtime.curStack; + if (C.canCompress(cur)) t.t.fail1("expected %s stack not to be compressible", cur.state().name); + if (M.compressStack(cur)) t.t.fail("expected compressing the running stack to fail"); + return HostResult.Value1(Values.box_i(7)); +} + +def test_entry_host_func(t: StackCompressionTester) { + var hf = HostFunction.new("add", SigCache.ii_i, fun args => HostResult.Value1(Values.box_i(Values.unbox_i(args[0]) + Values.unbox_i(args[1])))); + var s = M.getFreshStack().reset(hf).bind([Values.box_i(30)]); + t.assertCompress(s); + t.assertResult(Result.Value([Values.box_i(42)]), s.bind([Values.box_i(12)]).resume()); +} + +def test_repeat_roundtrip(t: StackCompressionTester) { + var f = t.subFunc(); + var s = M.getFreshStack().reset(f).bind([Values.box_i(1000)]); + var before = (C.numCompressed, C.numDecompressed); + for (i < 50) { + t.assertCompress(s); + C.decompress(s); + t.assertState(StackState.SUSPENDED, s); + if ((i % 10) == 0) Target.forceGC(); + } + t.t.assert_eq(before.0 + 50, C.numCompressed); + t.t.assert_eq(before.1 + 50, C.numDecompressed); + t.checkInvariants("roundtrip"); + t.assertResult(Result.Value([Values.box_i(999)]), s.bind([Values.box_i(1)]).resume()); +} + +def test_chain_top(t: StackCompressionTester) { + // The continuation's top stack runs {inner}, whose parent {mid} waits for it. + var k = t.callCont("make_chain", [Values.box_i(5)]); + var top = t.stackOf(k); + var mid = NativeStack.!(top.parent); + if (mid == null) return t.t.fail("expected the continuation to be a chain of stacks"); + t.assertState(StackState.CALL_CHILD, mid); + if (M.compressStack(mid)) t.t.fail("expected the waiting parent not to be compressible"); + t.assertCompress(top); + for (i < 3) Target.forceGC(); + t.t.assert_eq(11, t.callInt("finish", [Value.Cont(k)])); +} + +def test_chains_under_limit(t: StackCompressionTester) { + t.gen(); + var n = 10; + // The waiting parents cannot be compressed, so leave room for them. + t.compressing(n + 4); + var before = C.numCompressed; + var chains = Array.new(n); + for (i < n) chains[i] = t.callCont("make_chain", [Values.box_i(i)]); + for (i < n) { // some tops may already have been compressed to make room + var top = t.stackOf(chains[i]); + if (top.state() == StackState.SUSPENDED) t.assertCompress(top); + t.assertState(StackState.COMPRESSED, top); + t.assertState(StackState.CALL_CHILD, NativeStack.!(top.parent)); + } + for (i = n - 1; i >= 0; i--) t.t.assert_eq(2 * i + 1, t.callInt("finish", [Value.Cont(chains[i])])); + if (C.numCompressed < before + n) t.t.fail("expected chains to be compressed"); +} + +def DEEP_SUM = 5050; // sum of 0..100 +def test_deep_frames(t: StackCompressionTester) { + var k = t.callCont("make_deep", [Values.box_i(7)]); + t.assertLast(7); + t.assertCompress(t.stackOf(k)); + Target.forceGC(); + t.t.assert_eq(DEEP_SUM + 101 * 7, t.callInt("finish", [Value.Cont(k)])); +} + +def test_deep_frames_under_limit(t: StackCompressionTester) { + t.gen(); + t.compressing(4); + var n = 40; + var ks = Array.new(n); + for (i < n) ks[i] = t.callCont("make_deep", [Values.box_i(i)]); + for (i < n) t.t.assert_eq(DEEP_SUM + 101 * i, t.callInt("finish", [Value.Cont(ks[i])])); +} + +def test_used_after_compress(t: StackCompressionTester) { + var k = t.callCont("make_deep", [Values.box_i(1)]); + t.assertCompress(t.stackOf(k)); + t.t.assert_eq(DEEP_SUM + 101, t.callInt("finish", [Value.Cont(k)])); + t.assertTrap(TrapReason.USED_CONTINUATION, t.call("finish", [Value.Cont(k)])); + // A continuation made unusable by {cont.bind} stays unusable after compression. + var a = t.callCont("make_asker", [Values.box_i(2)]); + t.t.assert_eq(25, t.callInt("answer", [Value.Cont(a), Values.box_i(5)])); + t.assertTrap(TrapReason.USED_CONTINUATION, t.call("answer", [Value.Cont(a), Values.box_i(5)])); +} + +def test_trap_after_decompress(t: StackCompressionTester) { + var k = t.callCont("make_trapper", [Values.box_i(3)]); + t.assertCompress(t.stackOf(k)); + t.assertTrap(TrapReason.UNREACHABLE, t.call("finish", [Value.Cont(k)])); +} + +def test_throw_after_decompress(t: StackCompressionTester) { + var k = t.callCont("make_thrower", [Values.box_i(9)]); + t.assertCompress(t.stackOf(k)); + t.t.assert_eq(9, t.callInt("catch", [Value.Cont(k)])); +} + +def test_resume_null(t: StackCompressionTester) { + t.gen(); + t.compressing(4); + t.assertTrap(TrapReason.NULL_DEREF, t.call("resume_null", [])); +} + +def test_entry_cont_arg(t: StackCompressionTester) { + // A compressed stack whose bound argument is a continuation of another compressed stack. + var k = t.callCont("make_deep", [Values.box_i(2)]); + var s = M.getFreshStack().reset(t.exported("finish")).bind([Value.Cont(k)]); + t.assertCompress(t.stackOf(k)); + t.assertCompress(s); + for (i < 3) Target.forceGC(); + t.assertResult(Result.Value([Values.box_i(DEEP_SUM + 202)]), s.resume()); +} + +def test_gens_random_order(t: StackCompressionTester) { + t.gen(); + t.compressing(6); + var n = 30; + var gens = Array.new(n), steps = Array.new(n); + for (i < n) gens[i] = t.make(i * 100); + var live = n, seed = 12345; + while (live > 0) { + seed = (seed * 1103515245 + 12345) & 0x7FFFFFFF; + var i = (seed >> 8) % n; + if (Continuations.isNull(gens[i])) continue; + var j = ++steps[i]; + gens[i] = t.next(gens[i], if(j < 10, i * 100 + j, -1)); + if (j >= 10) live--; + } + t.checkInvariants("random"); +} + +def test_compress_from_host_call(t: StackCompressionTester) { + // Stacks are compressed while Wasm frames are live on other stacks. + var gens = Array.new(5); + for (i < gens.length) gens[i] = t.make(i * 10); + var hf = HostFunction.new("compress_all", SigCache.v_i, compressAll(t, gens, _)); + t.assertResult(Result.Value([Values.box_i(gens.length)]), M.runOnFreshStack(hf, [])); + for (j = 1; j < 10; j++) { + for (i < gens.length) gens[i] = t.next(gens[i], i * 10 + j); + } + for (k in gens) t.next(k, -1); +} +def compressAll(t: StackCompressionTester, gens: Array, args: Range) -> HostResult { + var count = 0; + for (k in gens) if (M.compressStack(t.stackOf(k))) count++; + return HostResult.Value1(Values.box_i(count)); +} + +def test_lazy_entry_near_capacity(t: StackCompressionTester) { + var f = t.subFunc(); + t.compressing(2); + var a = M.getFreshStack().reset(f).bind([Values.box_i(10)]); + var b = M.getFreshStack().reset(f).bind([Values.box_i(20)]); + var before = (C.numCompressed, C.numDecompressed); + // No segment is free, so the new stack gets one only when resumed, without compressing another. + var s = M.newStackFor(f); + t.assertState(StackState.COMPRESSED, s); + t.t.assert_eq(before.0, C.numCompressed); + s.bind([Values.box_i(50)]); + s.bind([Values.box_i(8)]); + t.assertState(StackState.COMPRESSED, s); + t.t.assert_eq(before.1, C.numDecompressed); + t.assertResult(Result.Value([Values.box_i(42)]), s.resume()); + t.t.assert_eq(before.1 + 1, C.numDecompressed); + t.assertResult(Result.Value([Values.box_i(9)]), a.bind([Values.box_i(1)]).resume()); + t.assertResult(Result.Value([Values.box_i(18)]), b.bind([Values.box_i(2)]).resume()); +} + +def test_lazy_entry_always(t: StackCompressionTester) { + var f = t.subFunc(); + StackTuning.enableCompression = true; + StackTuning.lazyEntryStacks = 2; + var s = M.newStackFor(f); + t.assertState(StackState.COMPRESSED, s); + if (C.canCompress(s)) t.t.fail("expected a stack without a segment not to be compressible"); + t.assertResult(Result.Value([Values.box_i(7)]), s.bind([Values.box_i(10), Values.box_i(3)]).resume()); + // A stack with no bound values is entered directly. + var hf = HostFunction.new("seven", SigCache.v_i, fun args => HostResult.Value1(Values.box_i(7))); + t.assertResult(Result.Value([Values.box_i(7)]), M.newStackFor(hf).resume()); +} + +def test_lazy_entry_refs_gc(t: StackCompressionTester) { + var b = ModuleBuilder.new(); + var sig = b.newSig([ValueTypes.ANYREF, ValueTypes.ANYREF], [ValueTypes.ANYREF, ValueTypes.ANYREF]); + b.sig(sig).code([u8.!(Opcode.LOCAL_GET.code), 1, u8.!(Opcode.LOCAL_GET.code), 0]); + var err = ErrorGen.new("swap"); + var f = Instantiator.new(Extension.set.all, b.module, [], err).run().functions[0]; + StackTuning.enableCompression = true; + StackTuning.lazyEntryStacks = 2; + var stacks = Array.new(1000), objs = Array.new(1000); + for (i < stacks.length) { + objs[i] = obj(); + stacks[i] = M.newStackFor(f).bind([objs[i]]); + } + for (i < 3) Target.forceGC(); + for (i < stacks.length) { + var other = obj(); + t.assertResult(Result.Value([other, objs[i]]), stacks[i].bind([other]).resume()); + } +} + +def test_lazy_gens_round_robin(t: StackCompressionTester) { + t.gen(); + StackTuning.lazyEntryStacks = 2; + roundRobin(t, 20, 4); +} + +def test_lazy_mid_wasm_bind(t: StackCompressionTester) { + t.gen(); + StackTuning.enableCompression = true; + StackTuning.lazyEntryStacks = 2; + t.t.assert_eq(37, t.callInt("answer", [Value.Cont(t.callCont("make_asker", [Values.box_i(3)])), Values.box_i(7)])); +} + +def test_lazy_chain_top(t: StackCompressionTester) { + t.gen(); + StackTuning.enableCompression = true; + StackTuning.lazyEntryStacks = 2; + t.t.assert_eq(11, t.callInt("finish", [Value.Cont(t.callCont("make_chain", [Values.box_i(5)]))])); +}