diff --git a/src/engine/WasmStack.v3 b/src/engine/WasmStack.v3 index ad1e6bfbd..92782ba66 100644 --- a/src/engine/WasmStack.v3 +++ b/src/engine/WasmStack.v3 @@ -98,6 +98,7 @@ enum StackState { SUSPENDED, // contains one or more frames, waiting for inputs CALL_CHILD, // waiting for child stack to return RUNNING, // currently running Wasm or host code + COMPRESSED, // suspended, with its frames held in the heap } // Provides access to the state of an executing frame. diff --git a/src/engine/compiler/MacroAssembler.v3 b/src/engine/compiler/MacroAssembler.v3 index a12885746..e1f8312b7 100644 --- a/src/engine/compiler/MacroAssembler.v3 +++ b/src/engine/compiler/MacroAssembler.v3 @@ -356,6 +356,7 @@ class MacroAssembler(valuerep: Tagging, regConfig: RegConfig) { def emit_call_runtime_SUSPEND(); def emit_call_runtime_SWITCH(); def emit_call_runtime_RESUME_THROW(); + def emit_call_runtime_DECOMPRESS_CONT(); def emit_call_runtime_RESUME_THROW_REF(); def emit_jump_to_trap_at(reason: TrapReason); def emit_call_runtime_op(op: Opcode); @@ -379,6 +380,8 @@ class MacroAssembler(valuerep: Tagging, regConfig: RegConfig) { // stk.state_ = state; def emit_set_stack_state(stk: Reg, state: StackState); + // if (stk.state_ != state) goto label; + def emit_br_stack_state_ne(stk: Reg, state: StackState, label: MasmLabel); // Publishes the handler table. def emit_publish_handlers(r_stack: Reg, r_table: Reg, tier: HandlerTier) { var offsets = getOffsets(); diff --git a/src/engine/compiler/SinglePassCompiler.v3 b/src/engine/compiler/SinglePassCompiler.v3 index 0a45c4c13..aa66b78e1 100644 --- a/src/engine/compiler/SinglePassCompiler.v3 +++ b/src/engine/compiler/SinglePassCompiler.v3 @@ -1454,6 +1454,7 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl buildResumeHandlerDests(it.pc, handlers); var cont_decl = ContDecl.!(module.heaptypes[cont_id]); if (checkForConstNull(state.peek())) return; + emit_decompress_top_cont(); var n = cont_decl.sig.params.length, slot_size = masm.valuerep.slot_size, offsets = masm.getOffsets(); // Spill the value stack and consume the continuation on top of it. @@ -1578,6 +1579,7 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl var target_top = target_decl.sig.params[target_decl.sig.params.length - 1]; var this_decl = ContDecl.!(HeapType.Cont.!(ValueType.Ref.!(target_top).heap).cont); var n = target_decl.sig.params.length - 1, slot_size = masm.valuerep.slot_size, offsets = masm.getOffsets(); + emit_decompress_top_cont(); state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS); var vsp = allocTmp(ValueKind.REF), cur = allocTmp(ValueKind.REF), target = allocTmp(ValueKind.REF); var prev = allocTmp(ValueKind.REF), parent = allocTmp(ValueKind.REF), tag = allocTmp(ValueKind.REF); @@ -2425,6 +2427,27 @@ class SinglePassCompiler(xenv: SpcExecEnv, masm: MacroAssembler, regAlloc: RegAl masm.emit_mov_r_r(ValueKind.REF, dst, regs.vfp); // XXX: use 3-addr adjustment of VSP (i.e. lea) if (slots > 0) masm.emit_addw_r_i(dst, int.view(slots) * masm.valuerep.slot_size); } + def emit_decompress_top_cont() { + state.emitSaveAll(resolver, SpillMode.SAVE_AND_FREE_REGS); + emit_compute_vsp(regs.vsp, state.sp); + emit_spill_vsp(regs.vsp); + masm.emit_store_curstack_vsp(regs.vsp); + var done = masm.newLabel(it.pc), tmp = regs.scratch, valuerep = masm.valuerep; + masm.emit_mov_r_m(ValueKind.REF, tmp, MasmAddr(regs.vsp, valuerep.tag_size - valuerep.slot_size)); + masm.emit_br_r(tmp, MasmBrCond.REF_NULL, done); // null continuation traps later + masm.emit_br_stack_state_ne(tmp, StackState.COMPRESSED, done); + def emit = fun { + masm.emit_get_curstack(regs.runtime_arg0); + masm.emit_v3_set_NativeStack_rsp_r_r(regs.runtime_arg0, regs.sp); + masm.emit_push_NativeStack_rsp_r_r(regs.runtime_arg0); + masm.emit_call_runtime_DECOMPRESS_CONT(); + }; + withReconstructedInlinedFrames(emit); + masm.emit_get_curstack(regs.scratch); + masm.emit_pop_NativeStack_rsp_r_r(regs.scratch); + emit_reload_regs(); + masm.bindLabel(done); + } def emit_spill_vsp(reg: Reg) { masm.emit_mov_m_r(ValueKind.REF, frame.vsp_slot, reg); } diff --git a/src/engine/native/NativeStack.v3 b/src/engine/native/NativeStack.v3 index 46535641b..89467a7e3 100644 --- a/src/engine/native/NativeStack.v3 +++ b/src/engine/native/NativeStack.v3 @@ -6,24 +6,24 @@ // See diagram at the end of the file for mapping layout. class NativeStack extends WasmStack { def valuerep = Target.tagging; - var range: MemoryRange; + var range: NativeStackSegment; var vsp: Pointer; var rsp: Pointer; var func: Function; // TODO: move into "enter-func-stub" var parent_rsp_ptr: Pointer; var return_results: Array; var state_: StackState = StackState.EMPTY; + var compressed: CompressedStack; // frames while {COMPRESSED} + // XXX: merge into compressed? + var saved_parent_rsp: Pointer; // contents of the parent rsp slot while {COMPRESSED} // Handler table (for inline suspend/switch). var handlers: Array; var handler_tier: HandlerTier; var next_stack: NativeStack; - new(range) { + new() { if (valuerep.tagged) RiGc.registerScanner(this, NativeStack.scan); - if (range == null) return; // allocated without a mapping, no need to initialize further - clear(); - if (Trace.stack && Debug.stack) Trace.OUT.put2("newStack start=0x%x, end=0x%x", range.start - Pointer.NULL, range.end - Pointer.NULL).ln(); } // Gets the state of this stack. def state() -> StackState { @@ -51,6 +51,7 @@ class NativeStack extends WasmStack { // Pushes {args} incrementally onto the value stack. def bind(args: Range) -> this { if (args.length == 0) return; + if (!NativeStackCompression.decompress(this)) fatal("out of stacks while binding a compressed stack"); checkState("bind()", StackState.SUSPENDED); for (v in args) push(v); } @@ -58,6 +59,7 @@ class NativeStack extends WasmStack { // Resumes running the Wasm or host code in this stack until that code either returns, throws, // or suspends itself. def resume() -> Result { + if (!NativeStackCompression.decompress(this)) return Result.Throw(Trap.new(TrapReason.OUT_OF_STACKS, null, null)); checkState("resume()", StackState.SUSPENDED); if (Trace.stack) traceFrames("stack.resume", rsp + RETADDR_SIZE); var bottom = NativeStack.!(this.bottom); @@ -511,6 +513,7 @@ class NativeStack extends WasmStack { } // GC callback to scan (tagged) values on this stack def scan() { + if (range == null) return; if (Trace.gc) { Trace.OUT.put1("scanStack obj=0x%x", u64.view(Pointer.atObject(this) - Pointer.NULL)).ln(); } @@ -552,6 +555,7 @@ class NativeStack extends WasmStack { BpTypeCode.ARRAYREF.code, BpTypeCode.FUNCREF.code, BpTypeCode.EXNREF.code, + BpTypeCode.NULLEXNREF.code, BpTypeCode.NULLFUNCREF.code => return Value.Ref(vp.load()); BpTypeCode.I32.code => return Value.I32(vp.load()); @@ -559,7 +563,8 @@ class NativeStack extends WasmStack { BpTypeCode.F32.code => return Value.F32(vp.load()); BpTypeCode.F64.code => return Value.F64(vp.load()); BpTypeCode.V128.code => return Value.V128(vp.load(), (vp + 8).load()); - BpTypeCode.CONTREF.code => { + BpTypeCode.CONTREF.code, + BpTypeCode.NULLCONTREF.code => { var stack = vp.load(); var version = (vp + 8).load(); return Value.Cont(Continuation(stack, version)); diff --git a/src/engine/native/NativeStackCompression.v3 b/src/engine/native/NativeStackCompression.v3 new file mode 100644 index 000000000..4994c0fb9 --- /dev/null +++ b/src/engine/native/NativeStackCompression.v3 @@ -0,0 +1,71 @@ +// Copyright 2026 Wizard authors. All rights reserved. +// See LICENSE for details of Apache 2.0 license. + +// Compresses suspended stacks into the heap to free their segments, and decompresses them before use. +component NativeStackCompression { + var numCompressed: int; + var numDecompressed: int; + + // Returns {true} if {stack} is suspended with only interpreter and SPC frames. + def canCompress(stack: NativeStack) -> bool { + if (stack.state_ != StackState.SUSPENDED) return false; + var range = stack.range; + if (range == null) return false; + var end = range.end; + // [end - 8] holds the parent's rsp, [end - 16] the return-parent stub. + if (stack.parent_rsp_ptr != end + -Pointer.SIZE) return false; + var srp = stack.rsp, count = 0, bottom_sfp = Pointer.NULL; + while (srp >= range.start && srp < end) { + var retip = srp.load(); + var code = RiRuntime.findUserCode(retip); + match (code) { + x: X86_64InterpreterCode, x: X86_64SpcModuleCode => { + count++; + bottom_sfp = srp + Pointer.SIZE; + srp = code.nextFrame(retip, srp).1; + } + x: X86_64ReturnParentStub => { + if (count == 0 || srp != end + -2 * Pointer.SIZE) return false; + // Values below the bottom frame would be lost. + return NativeFrameHandle.Interpreter(bottom_sfp).vfp() == range.start; + } + _ => return false; // the enter-func stub, inlined frames, trap stubs, or host frames + } + } + return false; + } + // Requires {canCompress(stack)}. + def compress(stack: NativeStack) { + var parent_rsp = stack.parent_rsp_ptr.load(); + var c = StackCompression.compressStack(stack); + if (Trace.stack) Trace.OUT.put2("compress stack @ 0x%x, freeing segment 0x%x", + Pointer.atObject(stack) - Pointer.NULL, stack.range.start - Pointer.NULL).ln(); + NativeStackManager.releaseSegment(stack); + stack.vsp = stack.rsp = Pointer.NULL; + stack.parent_rsp_ptr = Pointer.NULL; + stack.saved_parent_rsp = parent_rsp; + stack.compressed = c; + stack.state_ = StackState.COMPRESSED; + numCompressed++; + } + // Decompresses {stack}, if compressed, onto a fresh segment; returns {false} if none is available. + def decompress(stack: NativeStack) -> bool { + if (stack.state_ != StackState.COMPRESSED) return true; + var c = stack.compressed, parent_rsp = stack.saved_parent_rsp; + var func = stack.func, results = stack.return_results, parent = stack.parent, bottom = stack.bottom; + if (!NativeStackManager.attachSegment(stack)) return false; + if (Trace.stack) Trace.OUT.put2("decompress stack @ 0x%x onto segment 0x%x", + Pointer.atObject(stack) - Pointer.NULL, stack.range.start - Pointer.NULL).ln(); + Target.writeFramesToStack(stack, StackCompression.decompressFrames(c)); + stack.func = func; + stack.return_results = results; + stack.parent = parent; + stack.bottom = bottom; + stack.parent_rsp_ptr.store(parent_rsp); + stack.saved_parent_rsp = Pointer.NULL; + stack.compressed = null; + stack.state_ = StackState.SUSPENDED; + numDecompressed++; + return true; + } +} diff --git a/src/engine/native/NativeStackManager.v3 b/src/engine/native/NativeStackManager.v3 index 842f00d09..c7fe2207b 100644 --- a/src/engine/native/NativeStackManager.v3 +++ b/src/engine/native/NativeStackManager.v3 @@ -8,26 +8,86 @@ def PAGE_SIZE = 4096u; component NativeStackManager { var all: NativeStackSegment; // all segments ever mapped; never unmapped var total: int; // number of segments in {all} - var cache: NativeStack; // used in fast-path allocation in generated code + var free: NativeStackSegment; // unowned segments, linked by {next_free} + var cache: NativeStack; // finished stacks that still own a segment; pushed by the return-parent stub - // Gets a stack with a segment, or {null} if none is available. + // Gets an empty stack with a segment, or {null} if none is available. def getFreshStack() -> NativeStack { if (Trace.stack) Trace.OUT.puts("Requested fresh x86_stack...\n"); - if (cache == null) { - var ok = tryAllocatingNewStacks() || tryReclaimingSegments() || tryCollectingAndReclaiming() || tryCompressingStacks(); - if (!ok) return null; - } - + if (cache != null) return popCache(); + var segment = acquire(); + if (segment == null) return null; + var stack = NativeStack.new(); + attach(stack, segment); + return stack; + } + // Gives {stack}, which must have no segment, a segment; returns {false} if none is available. + def attachSegment(stack: NativeStack) -> bool { + var segment = acquire(); + if (segment == null) return false; + attach(stack, segment); + return true; + } + // Frees the segment of {stack}, whose frames must already be gone. + def releaseSegment(stack: NativeStack) { + push(detach(stack)); + } + def recycleStack(stack: NativeStack) { + stack.clear(); // so that it is never chosen for compression + stack.next_stack = cache; + cache = stack; + } + def compressStack(stack: NativeStack) -> bool { + if (!NativeStackCompression.canCompress(stack)) return false; + NativeStackCompression.compress(stack); + return true; + } + // Gets an unowned segment from {free}, a cached stack, a new mapping or a sweep, or {null}. + private def acquire() -> NativeStackSegment { + if (free == null && cache != null) return detach(popCache()); + if (free == null && !refill()) return null; + var segment = free; + free = segment.next_free; + segment.next_free = null; + return segment; + } + // Refills {free} from new mappings or a sweep, forcing a GC if the first sweep finds nothing. + private def refill() -> bool { + if (mapSegments() > 0) return true; + if (sweep() > 0) return true; + if (total == 0) return false; + Target.forceGC(); // weak references are only cleared by a GC + return sweep() > 0; + } + private def popCache() -> NativeStack { var result = cache; - cache = cache.next_stack; + cache = result.next_stack; result.next_stack = null; return result.clear(); } - private def tryAllocatingNewStacks() -> bool { + private def push(segment: NativeStackSegment) { + segment.next_free = free; + free = segment; + } + // Requires {segment} to be unowned; leaves {stack} empty. + private def attach(stack: NativeStack, segment: NativeStackSegment) { + stack.range = segment; + segment.stack.set(stack); + stack.clear(); + if (Trace.stack && Debug.stack) Trace.OUT.put2("attachSegment start=0x%x, end=0x%x", segment.start - Pointer.NULL, segment.end - Pointer.NULL).ln(); + } + private def detach(stack: NativeStack) -> NativeStackSegment { + var segment = stack.range; + segment.stack.set(null); + stack.range = null; + return segment; + } + // Maps up to a batch of new segments onto {free}, within {StackTuning.maxStacks}. + private def mapSegments() -> int { var size = EngineOptions.STACK_SIZE.get(); var batch = Ints.min(StackTuning.stackBatchSize, StackTuning.maxStacks - total); if (Trace.stack) Trace.OUT.put1("Batch allocating %d stacks", batch).ln(); - + var count = 0; for (i < batch) { // Map a guard page above the segment, so that accesses beyond its end always fault. var t = Mmap.reserveRaw(size + PAGE_SIZE, Mmap.PROT_READ | Mmap.PROT_WRITE, Mmap.MAP_PRIVATE | Mmap.MAP_ANONYMOUS); @@ -40,35 +100,23 @@ component NativeStackManager { // insert redzone roughly in the middle of the mapping. var rzStart = t.0 + ((size >> 1) & ~(PAGE_SIZE - 1u)), rzEnd = rzStart + PAGE_SIZE; if (!RedZones.addRedZone(rzStart, rzEnd)) fatal("could not protect value stack red zone"); - var curr = NativeStack.new(segment); - segment.stack.set(curr); - curr.next_stack = cache; - cache = curr; + push(segment); total++; + count++; } - return cache != null; // OK if succeeded in allocating at least one + return count; } - private def tryReclaimingSegments() -> bool { - var ok = false; + // Frees dead segments and, with compression on, compressible stacks' segments; requires {free} empty. + private def sweep() -> int { + var count = 0; for (p = all; p != null; p = p.next) { - if (p.stack.get() == null) { // weak reference was cleared by GC, can reclaim - var s = NativeStack.new(p); - p.stack.set(s); - s.next_stack = cache; - cache = s; - ok = true; - } + var s = p.stack.get(); + if (s == null) push(p); + else if (StackTuning.enableCompression && NativeStackCompression.canCompress(s)) NativeStackCompression.compress(s); + else continue; + count++; } - return ok; - } - // Weak references are only cleared by a GC, so force one to find dead stacks. - private def tryCollectingAndReclaiming() -> bool { - if (all == null) return false; - Target.forceGC(); - return tryReclaimingSegments(); - } - private def tryCompressingStacks() -> bool { - return false; + return count; } // Checks whether a fault at {addr} hit the guard page above a stack segment, which indicates // an engine bug, such as reading a frame that is not there. If so, fails with an error. @@ -80,10 +128,6 @@ component NativeStackManager { } } } - def recycleStack(stack: NativeStack) { - stack.next_stack = cache; - cache = stack; - } def runOnFreshStack(f: Function, args: Range) -> Result { // Always run functions on a separate, fresh stack. var prev = X86_64Runtime.curStack; // handle reentrancy @@ -97,7 +141,8 @@ component NativeStackManager { // Native stack segments are individually mmap'd with a guard page in the middle class NativeStackSegment extends MemoryRange { - var next: NativeStackSegment; + var next: NativeStackSegment; // link in {NativeStackManager.all} + var next_free: NativeStackSegment; // link in {NativeStackManager.free}, while unowned def stack = WeakRef.new(); // owner, if any; cleared by GC new(start: Pointer, end: Pointer) super(start, end) { } diff --git a/src/engine/x86-64/V3Offsets.v3 b/src/engine/x86-64/V3Offsets.v3 index a32a2cbd4..983123e29 100644 --- a/src/engine/x86-64/V3Offsets.v3 +++ b/src/engine/x86-64/V3Offsets.v3 @@ -15,7 +15,7 @@ class V3Offsets { private def t = Table.new(ValueTypes.FUNCREF, TableDecl.new(ValueTypes.FUNCREF, SizeConstraint(false, 0, Max.Set(1)))); private def wf = WasmFunction.new(i, decl); private def mem = NativeWasmMemory.new(null); - private def vs = NativeStack.new(null); + private def vs = NativeStack.new(); private def acc = NativeFrameAccessor.new(vs, NativeFrameHandle.Interpreter(Pointer.NULL), decl); private def cnt = CountProbe.new(); private def metric = Metric.new("", "", ""); diff --git a/src/engine/x86-64/X86_64Interpreter.v3 b/src/engine/x86-64/X86_64Interpreter.v3 index 60339a989..2f2c7d949 100644 --- a/src/engine/x86-64/X86_64Interpreter.v3 +++ b/src/engine/x86-64/X86_64Interpreter.v3 @@ -2496,6 +2496,7 @@ class X86_64InterpreterGen(ic: X86_64InterpreterCode, w: DataWriter) { bindHandler(Opcode.RESUME); { var cont_id = xenv.tmp0; saveResumeIVars(); + genDecompressTopCont(); genReadUleb32(G(cont_id)); // read signature index var contStack = xenv.tmp1; @@ -4157,6 +4158,18 @@ class X86_64InterpreterGen(ic: X86_64InterpreterCode, w: DataWriter) { asm.movq_r_m(t, t.plus(offsets.FuncHandlerInfo_resume_sites)); masm.emit_v3_Array_elem_r_rr(ValueKind.REF, r_tmp, r_tmp, xenv.curpc); } + private def genDecompressTopCont() { + var done = X86_64Label.new(); + asm.movq_r_m(r_tmp1, vsph[-1].value); // %tmp1 = cont.stack + asm.q.cmp_r_i(r_tmp1, 0); + asm.jc_rel_far(X86_64Conds.Z, done); // null continuation traps later + asm.cmpb_m_i(r_tmp1.plus(offsets.NativeStack_state), StackState.COMPRESSED.tag); + asm.jc_rel_far(X86_64Conds.NZ, done); + masm.emit_get_curstack(xenv.tmp1); + callRuntime(refRuntimeCall(X86_64RT.runtime_DECOMPRESS_CONT), [r_tmp1], true); + restoreCallerIVars(); + asm.bind(done); + } // Parks the current stack, publishes the handler table, chains {contStack} under it and switches. def genResumeChain(curStack: Reg, contStack: Reg, skip_tag: bool) { masm.emit_get_curstack(curStack); diff --git a/src/engine/x86-64/X86_64MacroAssembler.v3 b/src/engine/x86-64/X86_64MacroAssembler.v3 index 4efbb7e58..813bc3ade 100644 --- a/src/engine/x86-64/X86_64MacroAssembler.v3 +++ b/src/engine/x86-64/X86_64MacroAssembler.v3 @@ -882,6 +882,9 @@ class X86_64MacroAssembler extends MacroAssembler { def emit_call_runtime_RESUME_THROW() { emit_call_runtime(X86_64RT.runtime_handle_resume_throw); } + def emit_call_runtime_DECOMPRESS_CONT() { + emit_call_runtime(X86_64RT.runtime_DECOMPRESS_CONT); + } def emit_call_runtime_RESUME_THROW_REF() { emit_call_runtime(X86_64RT.runtime_resume_throw_ref); } @@ -1679,6 +1682,10 @@ class X86_64MacroAssembler extends MacroAssembler { def emit_set_stack_state(stk: Reg, state: StackState) { emit_mov_m_b(MasmAddr(stk, offsets.NativeStack_state), byte.view(state.tag)); } + def emit_br_stack_state_ne(stk: Reg, state: StackState, label: MasmLabel) { + asm.cmpb_m_i(G(stk).plus(offsets.NativeStack_state), state.tag); + asm.jc_rel_far(X86_64Conds.NZ, X86_64MasmLabel.!(label).label); + } // Moves values from the current stack to %cont.top. def emit_cont_mv(from_vsp: Reg, contStack: Reg, n_vals: Reg, tmp1: Reg, tmp2: Reg, xmm0: Reg) { diff --git a/src/engine/x86-64/X86_64Runtime.v3 b/src/engine/x86-64/X86_64Runtime.v3 index 5610a9832..66151b01e 100644 --- a/src/engine/x86-64/X86_64Runtime.v3 +++ b/src/engine/x86-64/X86_64Runtime.v3 @@ -113,6 +113,14 @@ component X86_64Runtime { Unhandled(t) => return t; } } + // Decompresses the continuation on top of {stack}'s value stack, where GC can update it. + def runtime_DECOMPRESS_CONT(stack: NativeStack) -> Throwable { + var ptr = stack.vsp + (stack.valuerep.tag_size - stack.valuerep.slot_size); + var cont_stack = NativeStack.!(ptr.load()); + if (cont_stack == null) return null; + if (!NativeStackCompression.decompress(cont_stack)) return stack.trap(TrapReason.OUT_OF_STACKS); + return null; + } def runtime_CONT_NEW(stack: NativeStack, instance: Instance, cont_index: u31) -> Throwable { return Runtime.CONT_NEW(stack, instance, cont_index); } @@ -129,6 +137,7 @@ component X86_64Runtime { .put2("resume_throw %q @ contStack[addr=0x%x] ", ex.render, Pointer.atObject(contStack) - Pointer.NULL) .put1("on stack[rsp=0x%x]", stack.rsp - Pointer.NULL).ln(); + if (!NativeStackCompression.decompress(contStack)) return stack.trap(TrapReason.OUT_OF_STACKS); // search for handler in continuation var result = contStack.throw(ex); match (result) { @@ -209,6 +218,7 @@ component X86_64Runtime { if (Continuations.isNull(target_cont)) return stack.trap(TrapReason.NULL_DEREF); if (Continuations.isUsed(target_cont)) return stack.trap(TrapReason.USED_CONTINUATION); var target_stack = target_cont.stack; + if (!NativeStackCompression.decompress(NativeStack.!(target_stack))) return stack.trap(TrapReason.OUT_OF_STACKS); Continuations.setUsed(target_cont); var this_cont = stack.unwindChain(tag, WasmStack.tryHandleSwitch); diff --git a/test/regress/stacks/compression.failures b/test/regress/stacks/compression.failures index fb7a22735..b0a96c73d 100644 --- a/test/regress/stacks/compression.failures +++ b/test/regress/stacks/compression.failures @@ -1,17 +1,5 @@ -test/regress/ext:stack-switching/many_stacks_dyn.bin.wast -test/regress/ext:stack-switching/many_stacks.bin.wast test/regress/ext:stack-switching/resume_chain3.bin.wast test/regress/ext:stack-switching/suspend15.bin.wast -test/regress/stacks/bind_partial0.bin.wast -test/regress/stacks/bind0.bin.wast -test/regress/stacks/chains0.bin.wast -test/regress/stacks/cont_in_frames0.bin.wast -test/regress/stacks/deep0.bin.wast -test/regress/stacks/exnref0.bin.wast -test/regress/stacks/live_conts0.bin.wast -test/regress/stacks/resume_throw_ref0.bin.wast -test/regress/stacks/resume_throw0.bin.wast test/regress/stacks/scheduler0.bin.wast test/regress/stacks/switch_ring0.bin.wast -test/regress/stacks/trap0.bin.wast test/regress/stacks/value_kinds0.bin.wast diff --git a/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 b/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 new file mode 100644 index 000000000..0145cf4aa --- /dev/null +++ b/test/unittest/x86-64-linux/NativeStackCompressionTest.v3 @@ -0,0 +1,788 @@ +// Copyright 2026 Wizard authors. All rights reserved. +// See LICENSE for details of Apache 2.0 license. + +// Tests for compressing suspended native stacks and resuming them from host and Wasm code in every tier. + +def M: NativeStackManager; +def C: NativeStackCompression; + +def T(name: string, f: StackCompressionTester -> void) { + UnitTests.registerT("x86stackcomp:", name, StackCompressionTester.new(_, X86_64InterpreterOnlyStrategy.new()), runTest(f, _)); +} +def TT(name: string, f: StackCompressionTester -> void) { + for (l = Target.getTestTiers(); l != null; l = l.tail) { + UnitTests.registerT(Strings.format1("x86stackcomp:%s", l.head.0), name, StackCompressionTester.new(_, l.head.1), runTest(f, _)); + } +} +def X_ = void( + T("can_compress_states", test_can_compress_states), + T("roundtrip", test_roundtrip), + T("no_frames_not_compressible", test_no_frames_not_compressible), + T("running_not_compressible", test_running_not_compressible), + T("repeat_roundtrip", test_repeat_roundtrip), + T("attach_segment", test_attach_segment), + T("decompress_out_of_stacks", test_decompress_out_of_stacks), + T("version_kept", test_version_kept), + TT("mid_host_resume", test_mid_host_resume), + TT("mid_host_bind", test_mid_host_bind), + TT("mid_twice", test_mid_twice), + TT("mid_gc", test_mid_gc), + TT("gen_frames", test_gen_frames), + TT("deep_frames", test_deep_frames), + TT("chain_top_roundtrip", test_chain_top_roundtrip), + TT("trap_after_decompress", test_trap_after_decompress), + TT("throw_after_decompress", test_throw_after_decompress), + TT("mid_explicit", test_mid_explicit), + TT("mid_wasm_twice", test_mid_wasm_twice), + TT("mid_wasm_bind", test_mid_wasm_bind), + TT("mid_gc_wasm", test_mid_gc_wasm), + TT("chain_top", test_chain_top), + TT("deep_frames_wasm", test_deep_frames_wasm), + TT("used_after_compress", test_used_after_compress), + TT("trap_after_decompress_wasm", test_trap_after_decompress_wasm), + TT("throw_after_decompress_wasm", test_throw_after_decompress_wasm), + TT("resume_null", test_resume_null), + TT("compress_from_host_call", test_compress_from_host_call), + TT("decompress_traps_in_wasm", test_decompress_traps_in_wasm), + TT("decompress_via_gc_wasm", test_decompress_via_gc_wasm), + T("sweep_compresses_suspended", test_sweep_compresses_suspended), + T("sweep_batches", test_sweep_batches), + T("recycled_not_compressed", test_recycled_not_compressed), + T("decompress_compresses_other", test_decompress_compresses_other), + T("disabled_no_compress", test_disabled_no_compress), + TT("gens_round_robin", test_gens_round_robin), + TT("gens_many", test_gens_many), + TT("gens_limit_one", test_gens_limit_one), + TT("gens_random_order", test_gens_random_order), + TT("chains_under_limit", test_chains_under_limit), + TT("deep_frames_under_limit", test_deep_frames_under_limit), + () +); + +// Generators, chains, deep frames, a trapper and a thrower, driven through the exports below. +def GEN_MODULE: Array = [ + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0xe9, 0x80, 0x80, 0x80, 0x00, 0x15, 0x60, + 0x00, 0x00, 0x5d, 0x00, 0x60, 0x01, 0x7f, 0x00, 0x5d, 0x02, 0x60, 0x00, 0x01, 0x7f, 0x5d, 0x04, + 0x60, 0x01, 0x7f, 0x01, 0x7f, 0x5d, 0x06, 0x5f, 0x01, 0x7f, 0x00, 0x60, 0x02, 0x7f, 0x7f, 0x00, + 0x60, 0x01, 0x64, 0x01, 0x01, 0x63, 0x01, 0x60, 0x00, 0x02, 0x7f, 0x64, 0x01, 0x60, 0x01, 0x7f, + 0x01, 0x63, 0x01, 0x60, 0x01, 0x63, 0x01, 0x01, 0x63, 0x01, 0x60, 0x01, 0x7f, 0x01, 0x63, 0x05, + 0x60, 0x00, 0x02, 0x7f, 0x64, 0x05, 0x60, 0x01, 0x7f, 0x01, 0x63, 0x07, 0x60, 0x02, 0x63, 0x07, + 0x7f, 0x01, 0x7f, 0x60, 0x02, 0x7f, 0x7f, 0x01, 0x7f, 0x60, 0x02, 0x64, 0x07, 0x7f, 0x01, 0x63, + 0x05, 0x60, 0x01, 0x63, 0x05, 0x01, 0x7f, 0x03, 0x9a, 0x80, 0x80, 0x80, 0x00, 0x19, 0x09, 0x02, + 0x06, 0x06, 0x0a, 0x0c, 0x0d, 0x04, 0x0e, 0x10, 0x11, 0x06, 0x06, 0x12, 0x06, 0x06, 0x06, 0x13, + 0x0e, 0x0e, 0x0e, 0x0e, 0x14, 0x14, 0x04, 0x0d, 0x89, 0x80, 0x80, 0x80, 0x00, 0x04, 0x00, 0x02, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x06, 0x86, 0x80, 0x80, 0x80, 0x00, 0x01, 0x7f, 0x01, 0x41, + 0x7f, 0x0b, 0x07, 0x8e, 0x81, 0x80, 0x80, 0x00, 0x0d, 0x04, 0x6d, 0x61, 0x6b, 0x65, 0x00, 0x05, + 0x04, 0x6e, 0x65, 0x78, 0x74, 0x00, 0x06, 0x04, 0x6c, 0x61, 0x73, 0x74, 0x00, 0x07, 0x09, 0x6d, + 0x61, 0x6b, 0x65, 0x5f, 0x6f, 0x6e, 0x63, 0x65, 0x00, 0x08, 0x0a, 0x6d, 0x61, 0x6b, 0x65, 0x5f, + 0x61, 0x73, 0x6b, 0x65, 0x72, 0x00, 0x09, 0x06, 0x61, 0x6e, 0x73, 0x77, 0x65, 0x72, 0x00, 0x0a, + 0x0a, 0x6d, 0x61, 0x6b, 0x65, 0x5f, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x00, 0x12, 0x09, 0x6d, 0x61, + 0x6b, 0x65, 0x5f, 0x64, 0x65, 0x65, 0x70, 0x00, 0x13, 0x0c, 0x6d, 0x61, 0x6b, 0x65, 0x5f, 0x74, + 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x00, 0x14, 0x0c, 0x6d, 0x61, 0x6b, 0x65, 0x5f, 0x74, 0x68, + 0x72, 0x6f, 0x77, 0x65, 0x72, 0x00, 0x15, 0x06, 0x66, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x00, 0x16, + 0x05, 0x63, 0x61, 0x74, 0x63, 0x68, 0x00, 0x17, 0x0b, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x5f, + 0x6e, 0x75, 0x6c, 0x6c, 0x00, 0x18, 0x09, 0x8f, 0x80, 0x80, 0x80, 0x00, 0x02, 0x03, 0x00, 0x03, + 0x01, 0x02, 0x03, 0x03, 0x00, 0x05, 0x0b, 0x0c, 0x0e, 0x0f, 0x10, 0x0a, 0xcd, 0x84, 0x80, 0x80, + 0x00, 0x19, 0xc1, 0x80, 0x80, 0x80, 0x00, 0x02, 0x01, 0x7b, 0x01, 0x63, 0x08, 0x20, 0x01, 0xfd, + 0x11, 0x21, 0x02, 0x20, 0x01, 0xfb, 0x00, 0x08, 0x21, 0x03, 0x20, 0x00, 0x45, 0x04, 0x40, 0x20, + 0x01, 0xe2, 0x00, 0x05, 0x20, 0x00, 0x41, 0x01, 0x6b, 0x20, 0x01, 0x10, 0x00, 0x0b, 0x20, 0x02, + 0xfd, 0x1b, 0x03, 0x20, 0x01, 0x47, 0x04, 0x40, 0x00, 0x0b, 0x20, 0x03, 0xfb, 0x02, 0x08, 0x00, + 0x20, 0x01, 0x47, 0x04, 0x40, 0x00, 0x0b, 0x0b, 0xc2, 0x80, 0x80, 0x80, 0x00, 0x04, 0x01, 0x7f, + 0x01, 0x7e, 0x01, 0x7c, 0x01, 0x7f, 0x03, 0x40, 0x20, 0x00, 0x20, 0x01, 0x6a, 0x21, 0x04, 0x41, + 0x03, 0x20, 0x04, 0x10, 0x00, 0x20, 0x02, 0x20, 0x04, 0xac, 0x7c, 0x21, 0x02, 0x20, 0x03, 0x20, + 0x04, 0xb7, 0xa0, 0x21, 0x03, 0x20, 0x02, 0x20, 0x03, 0xb0, 0x52, 0x04, 0x40, 0x00, 0x0b, 0x20, + 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x20, 0x01, 0x41, 0x0a, 0x49, 0x0d, 0x00, 0x0b, 0x0b, 0x8c, + 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x00, 0xe2, 0x00, 0x20, 0x00, 0x41, 0xe8, 0x07, 0x6a, 0x0b, + 0x8a, 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x00, 0x41, 0x0a, 0x6c, 0xe2, 0x01, 0x6a, 0x0b, 0x99, + 0x80, 0x80, 0x80, 0x00, 0x01, 0x01, 0x63, 0x01, 0x02, 0x0b, 0x20, 0x00, 0xe3, 0x01, 0x01, 0x00, + 0x00, 0x00, 0xd0, 0x01, 0x0f, 0x0b, 0x21, 0x01, 0x24, 0x00, 0x20, 0x01, 0x0b, 0x8d, 0x80, 0x80, + 0x80, 0x00, 0x00, 0x20, 0x00, 0xd2, 0x01, 0xe0, 0x03, 0xe1, 0x03, 0x01, 0x10, 0x04, 0x0b, 0x87, + 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x00, 0xd4, 0x10, 0x04, 0x0b, 0x84, 0x80, 0x80, 0x80, 0x00, + 0x00, 0x23, 0x00, 0x0b, 0x9d, 0x80, 0x80, 0x80, 0x00, 0x01, 0x01, 0x63, 0x05, 0x02, 0x0f, 0x20, + 0x00, 0xd2, 0x02, 0xe0, 0x07, 0xe3, 0x07, 0x01, 0x00, 0x00, 0x00, 0x1a, 0xd0, 0x05, 0x0f, 0x0b, + 0x21, 0x01, 0x1a, 0x20, 0x01, 0x0b, 0x96, 0x80, 0x80, 0x80, 0x00, 0x00, 0x02, 0x64, 0x07, 0x20, + 0x00, 0xd2, 0x03, 0xe0, 0x07, 0xe3, 0x07, 0x01, 0x00, 0x01, 0x00, 0x1a, 0xd0, 0x07, 0x0f, 0x0b, + 0x0b, 0x8c, 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x01, 0x20, 0x00, 0xe1, 0x07, 0x05, 0xe3, 0x05, + 0x00, 0x0b, 0x8b, 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x00, 0xe2, 0x00, 0x20, 0x00, 0x41, 0x02, + 0x6c, 0x0b, 0x97, 0x80, 0x80, 0x80, 0x00, 0x00, 0x02, 0x64, 0x05, 0x41, 0x01, 0x20, 0x00, 0xd2, + 0x0b, 0xe0, 0x07, 0xe3, 0x07, 0x01, 0x00, 0x02, 0x00, 0x6a, 0x0f, 0x0b, 0x00, 0x0b, 0xa4, 0x80, + 0x80, 0x80, 0x00, 0x01, 0x01, 0x7f, 0x20, 0x00, 0x20, 0x01, 0x6a, 0x21, 0x02, 0x20, 0x00, 0x45, + 0x04, 0x7f, 0x20, 0x01, 0xe2, 0x00, 0x20, 0x02, 0x05, 0x20, 0x02, 0x20, 0x00, 0x41, 0x01, 0x6b, + 0x20, 0x01, 0x10, 0x0d, 0x6a, 0x0b, 0x0b, 0x89, 0x80, 0x80, 0x80, 0x00, 0x00, 0x41, 0xe4, 0x00, + 0x20, 0x00, 0x10, 0x0d, 0x0b, 0x87, 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x00, 0xe2, 0x00, 0x00, + 0x0b, 0x8a, 0x80, 0x80, 0x80, 0x00, 0x00, 0x20, 0x00, 0xe2, 0x00, 0x20, 0x00, 0x08, 0x03, 0x0b, + 0x9c, 0x80, 0x80, 0x80, 0x00, 0x01, 0x01, 0x63, 0x05, 0x02, 0x0f, 0x20, 0x01, 0x20, 0x00, 0xe3, + 0x07, 0x01, 0x00, 0x00, 0x00, 0x1a, 0xd0, 0x05, 0x0f, 0x0b, 0x21, 0x02, 0x24, 0x00, 0x20, 0x02, + 0x0b, 0x8a, 0x80, 0x80, 0x80, 0x00, 0x00, 0xd2, 0x0c, 0xe0, 0x07, 0x20, 0x00, 0x10, 0x11, 0x0b, + 0x8a, 0x80, 0x80, 0x80, 0x00, 0x00, 0xd2, 0x0e, 0xe0, 0x07, 0x20, 0x00, 0x10, 0x11, 0x0b, 0x8a, + 0x80, 0x80, 0x80, 0x00, 0x00, 0xd2, 0x0f, 0xe0, 0x07, 0x20, 0x00, 0x10, 0x11, 0x0b, 0x8a, 0x80, + 0x80, 0x80, 0x00, 0x00, 0xd2, 0x10, 0xe0, 0x07, 0x20, 0x00, 0x10, 0x11, 0x0b, 0x87, 0x80, 0x80, + 0x80, 0x00, 0x00, 0x20, 0x00, 0xe3, 0x05, 0x00, 0x0b, 0x94, 0x80, 0x80, 0x80, 0x00, 0x00, 0x02, + 0x7f, 0x1f, 0x40, 0x01, 0x00, 0x03, 0x00, 0x20, 0x00, 0xe3, 0x05, 0x00, 0x1a, 0x0b, 0x41, 0x7f, + 0x0b, 0x0b, 0x87, 0x80, 0x80, 0x80, 0x00, 0x00, 0xd0, 0x05, 0xe3, 0x05, 0x00, 0x0b, + +]; + +private class StackCompressionTester(t: Tester, tiering: ExecutionStrategy) { + def saved_max = StackTuning.maxStacks; + def saved_compression = StackTuning.enableCompression; + var held: Array; // stacks pinned by {limit} + var instance: Instance; + var limit_total = -1; + + new() { Execute.tiering = tiering; } + + def restore() { + StackTuning.maxStacks = saved_max; + StackTuning.enableCompression = saved_compression; + limit_total = -1; + } + // Enables compression under a limit of {extra} more segments; pinning must not sweep. + def compressing(extra: int) { + limit(extra); + StackTuning.enableCompression = true; + } + // Pins all existing segments, then allows at most {extra} more. + def limit(extra: int) { + StackTuning.maxStacks = M.total; + M.cache = null; + M.free = null; + Target.forceGC(); + var free = 0; + for (seg = M.all; seg != null; seg = seg.next) if (seg.stack.get() == null) free++; + held = Array.new(free); + for (i < free) held[i] = M.getFreshStack(); + StackTuning.maxStacks = M.total + extra; + limit_total = StackTuning.maxStacks; + } + def checkInvariants(where: string) { + var count = 0; + for (seg = M.all; seg != null; seg = seg.next) { + count++; + var s = seg.stack.get(); + if (s == null) continue; + if (s.range != seg) t.fail1("%s: owning stack does not point back at its segment", where); + if (s.state() == StackState.COMPRESSED) t.fail1("%s: compressed stack owns a segment", where); + } + for (seg = M.free; seg != null; seg = seg.next_free) { + if (seg.stack.get() != null) t.fail1("%s: free segment has an owner", where); + } + if (count != M.total) t.fail1("%s: segment count mismatch", where); + if (limit_total >= 0 && M.total > limit_total) t.fail3("%s: expected at most %d segments, got %d", where, limit_total, M.total); + } + def assertState(expected: StackState, s: NativeStack) { + var got = s.state(); + if (got != expected) t.fail2("expected state %s, got %s", expected.name, got.name); + var compressed = got == StackState.COMPRESSED; + if (compressed != (s.range == null)) t.fail("expected a stack to have a segment iff not compressed"); + if (!compressed && s.compressed != null) t.fail("expected no compressed data if not compressed"); + if (!compressed && s.saved_parent_rsp != Pointer.NULL) t.fail("expected no saved parent rsp if not compressed"); + } + def assertResult(expected: Result, got: Result) { + if (!expected.equals(got)) t.fail2("expected %q, got %q", expected.render, got.render); + } + def assertCompress(s: NativeStack) { + if (s == null) return t.fail("expected a stack, got null"); + if (!M.compressStack(s)) return t.fail1("expected stack in state %s to be compressible", s.state().name); + assertState(StackState.COMPRESSED, s); + } + def assertDecompress(s: NativeStack) { + if (!C.decompress(s)) return t.fail("expected decompression to succeed"); + assertState(StackState.SUSPENDED, s); + } + def assertTrap(reason: TrapReason, r: Result) { + match (r) { + Throw(thrown) => if (Trap.?(thrown) && Trap.!(thrown).reason == reason) return; + _ => ; + } + t.fail2("expected trap %s, got %q", reason.name, r.render); + } + // A Wasm function [i32, i32] -> [i32] that subtracts its arguments. + def subFunc() -> WasmFunction { + var b = ModuleBuilder.new(); + b.sig(SigCache.ii_i).code([ + u8.!(Opcode.LOCAL_GET.code), 0, + u8.!(Opcode.LOCAL_GET.code), 1, + u8.!(Opcode.I32_SUB.code) + ]); + var err = ErrorGen.new("subFunc"); + var inst = Instantiator.new(Extension.set.all, b.module, [], err).run(); + return WasmFunction.!(inst.functions[0]); + } + def gen() -> this { + if (instance != null) return; + var err = ErrorGen.new("GEN_MODULE"); + var limits = Limits.new().set(Extension.set.all); + var bp = BinParser.new(Extension.set.all, limits, err, "GEN_MODULE"); + bp.tiering = tiering; + var module = bp.push(GEN_MODULE, 0, GEN_MODULE.length).finish(); + if (!err.ok()) return t.fail(err.error_msg); + instance = Instantiator.new(Extension.set.all, module, [], err).run(); + if (!err.ok()) return t.fail(err.error_msg); + tiering.onTestModule(module); + for (f in instance.functions) { + if (WasmFunction.?(f)) tiering.onTestRun(WasmFunction.!(f), err); + } + if (!err.ok()) return t.fail(err.error_msg); + } + def call(name: string, args: Array) -> Result { + gen(); + for (i < instance.module.exports.length) { + if (Strings.equal(name, instance.module.exports[i].0)) { + return Execute.call(Function.!(instance.exports[i]), args); + } + } + t.fail1("export %s not found", name); + return Result.Value([]); + } + def callCont(name: string, args: Array) -> Continuation { + var r = call(name, args); + match (r) { + Value(vals) => if (vals.length == 1 && Value.Cont.?(vals[0])) return Value.Cont.!(vals[0]).val; + _ => ; + } + t.fail2("expected %s to return a continuation, got %q", name, r.render); + return Continuations.NULL; + } + def stackOf(k: Continuation) -> NativeStack { + return NativeStack.!(k.stack); + } + def exported(name: string) -> Function { + gen(); + for (i < instance.module.exports.length) { + if (Strings.equal(name, instance.module.exports[i].0)) return Function.!(instance.exports[i]); + } + t.fail1("export %s not found", name); + return null; + } + def callInt(name: string, args: Array) -> int { + var r = call(name, args); + match (r) { + Value(vals) => if (vals.length == 1 && Value.I32.?(vals[0])) return int.view(Value.I32.!(vals[0]).val); + _ => ; + } + t.fail2("expected %s to return an i32, got %q", name, r.render); + return -1; + } + // Makes a generator yielding {start} .. {start + 9}, suspended at its first yield. + def make(start: int) -> Continuation { + var k = callCont("make", [Values.box_i(start)]); + if (!Continuations.isNull(k)) assertLast(start); + return k; + } + // Steps a generator, which must yield {expected}, or finish if {expected < 0}. + def next(k: Continuation, expected: int) -> Continuation { + var n = callCont("next", [Value.Cont(k)]); + if (expected < 0) { + if (!Continuations.isNull(n)) t.fail("expected generator to finish"); + } else { + if (Continuations.isNull(n)) t.fail1("expected generator to yield %d, but it finished", expected); + else assertLast(expected); + } + return n; + } + def assertLast(expected: int) { + var got = callInt("last", []); + if (got != expected) t.fail2("expected generator to yield %d, got %d", expected, got); + } + // Takes a fresh stack and drops it, so that its segment is reclaimable only after a GC. + def dropFresh() #no-inline { + if (M.getFreshStack() == null) t.fail("expected a fresh stack to drop"); + } + // Runs the exported Wasm function {name} on {x} until it suspends, returning its stack. + def suspended(name: string, x: int) -> NativeStack { + var k = callCont(name, [Values.box_i(x)]); + if (Continuations.isNull(k)) { t.fail1("expected %s to return a continuation", name); return null; } + var s = stackOf(k); + assertState(StackState.SUSPENDED, s); + return s; + } +} + +def runTest(f: StackCompressionTester -> void, t: StackCompressionTester) { + f(t); + t.checkInvariants("end of test"); + t.restore(); +} + +// The result of {make_deep}: the sum of (d + x) for d in 0..100. +def deepSum(x: int) -> int { return 5050 + 101 * x; } + +def test_can_compress_states(t: StackCompressionTester) { + var s = M.getFreshStack(); + t.assertState(StackState.EMPTY, s); + if (C.canCompress(s)) t.t.fail("expected EMPTY stack not to be compressible"); + s.reset(t.subFunc()).bind([Values.box_i(9)]); + t.assertState(StackState.SUSPENDED, s); + if (C.canCompress(s)) t.t.fail("expected a stack at function entry not to be compressible"); + if (M.compressStack(s)) t.t.fail("expected compressing a stack at function entry to fail"); + t.assertResult(Result.Value([Values.box_i(5)]), s.bind([Values.box_i(4)]).resume()); + t.assertState(StackState.EMPTY, s); + var w = t.suspended("make_once", 5); + if (!C.canCompress(w)) t.t.fail("expected a SUSPENDED stack with frames to be compressible"); + t.assertCompress(w); + if (C.canCompress(w)) t.t.fail("expected COMPRESSED stack not to be compressible"); + if (M.compressStack(w)) t.t.fail("expected compressing a COMPRESSED stack to fail"); + t.assertDecompress(w); + if (!C.canCompress(w)) t.t.fail("expected decompressed stack to be compressible again"); + t.assertResult(Result.Value([Values.box_i(1005)]), w.resume()); +} + +def test_roundtrip(t: StackCompressionTester) { + var s = t.suspended("make_once", 100); + var before = (C.numCompressed, C.numDecompressed); + t.assertCompress(s); + t.t.assert_eq(before.0 + 1, C.numCompressed); + t.assertResult(Result.Value([Values.box_i(1100)]), s.resume()); // decompresses + t.t.assert_eq(before.1 + 1, C.numDecompressed); +} + + + +def test_no_frames_not_compressible(t: StackCompressionTester) { + var s = M.getFreshStack().resetForFrameWrites(t.subFunc()); + t.assertState(StackState.SUSPENDED, s); + if (C.canCompress(s)) t.t.fail("expected a stack with no frames not to be compressible"); + s.clear(); +} + +def test_running_not_compressible(t: StackCompressionTester) { + // Inside a host call, the running stack cannot be compressed. + var hf = HostFunction.new("check", SigCache.v_i, checkRunning(t, _)); + t.assertResult(Result.Value([Values.box_i(7)]), M.runOnFreshStack(hf, [])); +} +def checkRunning(t: StackCompressionTester, args: Range) -> HostResult { + var cur = X86_64Runtime.curStack; + if (C.canCompress(cur)) t.t.fail1("expected %s stack not to be compressible", cur.state().name); + if (M.compressStack(cur)) t.t.fail("expected compressing the running stack to fail"); + return HostResult.Value1(Values.box_i(7)); +} + + +def test_repeat_roundtrip(t: StackCompressionTester) { + var s = t.suspended("make_once", 1000); + var before = (C.numCompressed, C.numDecompressed); + for (i < 50) { + t.assertCompress(s); + t.assertDecompress(s); + if ((i % 10) == 0) Target.forceGC(); + } + t.t.assert_eq(before.0 + 50, C.numCompressed); + t.t.assert_eq(before.1 + 50, C.numDecompressed); + t.checkInvariants("roundtrip"); + t.assertResult(Result.Value([Values.box_i(2000)]), s.resume()); +} + +def test_attach_segment(t: StackCompressionTester) { + t.limit(1); + var s = NativeStack.new(); + if (!M.attachSegment(s)) return t.t.fail("expected a segment"); + var seg = s.range; + if (seg == null || seg.stack.get() != s) return t.t.fail("expected the attached segment to be owned"); + if (M.attachSegment(NativeStack.new())) t.t.fail("expected no second segment under the limit"); + t.assertTrap(TrapReason.OUT_OF_STACKS, M.runOnFreshStack(HostFunction.new("seven", SigCache.v_i, fun args => HostResult.Value1(Values.box_i(7))), [])); + if (seg.stack.get() != s) t.t.fail("expected the attached segment to keep its owner"); + M.recycleStack(s); + if (M.getFreshStack() != s) t.t.fail("expected the returned segment's stack to be reused"); +} + +def test_decompress_out_of_stacks(t: StackCompressionTester) { + var s = t.suspended("make_once", 10); + t.limit(0); + t.assertCompress(s); + var hog = M.getFreshStack(); // takes the only segment + if (hog == null) return t.t.fail("expected the freed segment to be available"); + t.assertTrap(TrapReason.OUT_OF_STACKS, s.resume()); + t.assertState(StackState.COMPRESSED, s); + M.recycleStack(hog); + t.assertResult(Result.Value([Values.box_i(1010)]), s.resume()); +} + +def test_version_kept(t: StackCompressionTester) { + var s = t.suspended("make_once", 10); + var k = Continuations.makeContinuation(s); + t.assertCompress(s); + if (Continuations.isUsed(k)) t.t.fail("expected compression not to consume the continuation"); + t.assertDecompress(s); + if (Continuations.isUsed(k)) t.t.fail("expected decompression not to consume the continuation"); + t.assertResult(Result.Value([Values.box_i(1010)]), s.resume()); +} + +def test_mid_host_resume(t: StackCompressionTester) { + var s = t.suspended("make_once", 5); + t.assertCompress(s); + t.assertResult(Result.Value([Values.box_i(1005)]), s.resume()); +} + +def test_mid_host_bind(t: StackCompressionTester) { + var s = t.suspended("make_asker", 4); + t.assertCompress(s); + s.bind([Values.box_i(2)]); // decompresses, since the stack has frames + t.assertState(StackState.SUSPENDED, s); + t.assertResult(Result.Value([Values.box_i(42)]), s.resume()); +} + +def test_mid_twice(t: StackCompressionTester) { + var s = t.suspended("make_once", 7); + t.assertCompress(s); + t.assertDecompress(s); + t.assertCompress(s); + for (i < 3) Target.forceGC(); + t.assertResult(Result.Value([Values.box_i(1007)]), s.resume()); +} + +def test_mid_gc(t: StackCompressionTester) { + var stacks = Array.new(8); + for (i < stacks.length) { + stacks[i] = t.suspended("make_once", i); + t.assertCompress(stacks[i]); + } + for (i < 3) Target.forceGC(); + for (i < stacks.length) t.assertResult(Result.Value([Values.box_i(1000 + i)]), stacks[i].resume()); +} + +def test_gen_frames(t: StackCompressionTester) { + // Three frames deep with v128 and struct locals; resumed from the host, it checks them and yields again. + var s = t.suspended("make", 100); + t.assertCompress(s); + for (i < 3) Target.forceGC(); + t.assertDecompress(s); + t.assertCompress(s); + t.assertTrap(TrapReason.INVALID_SUSPEND, s.resume()); // the host handles no tags +} + +def test_deep_frames(t: StackCompressionTester) { + var s = t.suspended("make_deep", 3); + t.assertCompress(s); + for (i < 3) Target.forceGC(); + t.assertResult(Result.Value([Values.box_i(deepSum(3))]), s.resume()); +} + +def test_chain_top_roundtrip(t: StackCompressionTester) { + // The continuation's top stack runs {inner}, whose parent {mid} waits for it. + var top = t.suspended("make_chain", 5); + var mid = NativeStack.!(top.parent); + if (mid == null) return t.t.fail("expected the continuation to be a chain of stacks"); + t.assertState(StackState.CALL_CHILD, mid); + if (M.compressStack(mid)) t.t.fail("expected the waiting parent not to be compressible"); + var parent_rsp = top.parent_rsp_ptr.load(); + if (parent_rsp != mid.rsp) t.t.fail("expected the top stack's parent slot to hold the parent's rsp"); + t.assertCompress(top); + if (top.saved_parent_rsp != parent_rsp) t.t.fail("expected compression to save the parent rsp"); + for (i < 3) Target.forceGC(); + t.assertDecompress(top); + if (top.parent != mid || top.bottom != mid) t.t.fail("expected decompression to keep the chain links"); + if (top.parent_rsp_ptr.load() != parent_rsp) t.t.fail("expected decompression to restore the parent rsp"); + t.assertState(StackState.CALL_CHILD, mid); +} + +def test_trap_after_decompress(t: StackCompressionTester) { + var s = t.suspended("make_trapper", 1); + t.assertCompress(s); + t.assertTrap(TrapReason.UNREACHABLE, s.resume()); +} + +def test_throw_after_decompress(t: StackCompressionTester) { + var s = t.suspended("make_thrower", 9); + t.assertCompress(s); + var r = s.resume(); + match (r) { + Throw(thrown) => if (!Exception.?(thrown)) t.t.fail1("expected an exception, got %q", thrown.render); + _ => t.t.fail1("expected an exception, got %q", r.render); + } +} + +def test_mid_explicit(t: StackCompressionTester) { + var k = t.make(100); + var s = t.stackOf(k); + t.assertState(StackState.SUSPENDED, s); + t.assertCompress(s); // three frames deep, with v128 and struct locals + k = t.next(k, 101); // the Wasm resume decompresses + if (t.stackOf(k) != s) t.t.fail("expected the continuation to keep its stack"); + t.assertState(StackState.SUSPENDED, s); + for (i = 2; i < 10; i++) { + t.assertCompress(t.stackOf(k)); + k = t.next(k, 100 + i); + } + t.assertCompress(t.stackOf(k)); + t.next(k, -1); +} + +def test_mid_wasm_twice(t: StackCompressionTester) { + var k = t.make(7); + var s = t.stackOf(k); + t.assertCompress(s); + t.assertDecompress(s); + t.assertCompress(s); + for (i < 3) Target.forceGC(); + k = t.next(k, 8); + k = t.next(k, 9); +} + +def test_mid_wasm_bind(t: StackCompressionTester) { + var k = t.callCont("make_asker", [Values.box_i(3)]); + t.assertCompress(t.stackOf(k)); + t.t.assert_eq(37, t.callInt("answer", [Value.Cont(k), Values.box_i(7)])); // cont.bind decompresses +} + +def test_mid_gc_wasm(t: StackCompressionTester) { + var gens = Array.new(8); + for (i < gens.length) { + gens[i] = t.make(i * 10); + t.assertCompress(t.stackOf(gens[i])); + } + for (i < 3) Target.forceGC(); + for (j = 1; j < 10; j++) { + for (i < gens.length) { + gens[i] = t.next(gens[i], i * 10 + j); + if ((i + j) % 2 == 0) t.assertCompress(t.stackOf(gens[i])); + } + Target.forceGC(); + } + for (i < gens.length) t.next(gens[i], -1); +} + +def test_chain_top(t: StackCompressionTester) { + // The continuation's top stack runs {inner}, whose parent {mid} waits for it. + var k = t.callCont("make_chain", [Values.box_i(5)]); + var top = t.stackOf(k); + var mid = NativeStack.!(top.parent); + if (mid == null) return t.t.fail("expected the continuation to be a chain of stacks"); + t.assertState(StackState.CALL_CHILD, mid); + t.assertCompress(top); + for (i < 3) Target.forceGC(); + t.t.assert_eq(11, t.callInt("finish", [Value.Cont(k)])); +} + +def test_deep_frames_wasm(t: StackCompressionTester) { + var k = t.callCont("make_deep", [Values.box_i(7)]); + t.assertLast(7); + t.assertCompress(t.stackOf(k)); + Target.forceGC(); + t.t.assert_eq(deepSum(7), t.callInt("finish", [Value.Cont(k)])); +} + +def test_used_after_compress(t: StackCompressionTester) { + var k = t.callCont("make_deep", [Values.box_i(1)]); + t.assertCompress(t.stackOf(k)); + t.t.assert_eq(deepSum(1), t.callInt("finish", [Value.Cont(k)])); + t.assertTrap(TrapReason.USED_CONTINUATION, t.call("finish", [Value.Cont(k)])); + // A continuation consumed by {cont.bind} after compression stays unusable. + var a = t.callCont("make_asker", [Values.box_i(2)]); + t.assertCompress(t.stackOf(a)); + t.t.assert_eq(25, t.callInt("answer", [Value.Cont(a), Values.box_i(5)])); + t.assertTrap(TrapReason.USED_CONTINUATION, t.call("answer", [Value.Cont(a), Values.box_i(5)])); +} + +def test_trap_after_decompress_wasm(t: StackCompressionTester) { + var k = t.callCont("make_trapper", [Values.box_i(3)]); + t.assertCompress(t.stackOf(k)); + t.assertTrap(TrapReason.UNREACHABLE, t.call("finish", [Value.Cont(k)])); +} + +def test_throw_after_decompress_wasm(t: StackCompressionTester) { + var k = t.callCont("make_thrower", [Values.box_i(9)]); + t.assertCompress(t.stackOf(k)); + t.t.assert_eq(9, t.callInt("catch", [Value.Cont(k)])); +} + +def test_resume_null(t: StackCompressionTester) { + t.assertTrap(TrapReason.NULL_DEREF, t.call("resume_null", [])); +} + + +def test_compress_from_host_call(t: StackCompressionTester) { + // Stacks are compressed while Wasm frames are live on other stacks. + var gens = Array.new(5); + for (i < gens.length) gens[i] = t.make(i * 10); + var hf = HostFunction.new("compress_all", SigCache.v_i, compressAll(t, gens, _)); + t.assertResult(Result.Value([Values.box_i(gens.length)]), M.runOnFreshStack(hf, [])); + for (j = 1; j < 10; j++) { + for (i < gens.length) gens[i] = t.next(gens[i], i * 10 + j); + } + for (k in gens) t.next(k, -1); +} +def compressAll(t: StackCompressionTester, gens: Array, args: Range) -> HostResult { + var count = 0; + for (k in gens) if (M.compressStack(t.stackOf(k))) count++; + return HostResult.Value1(Values.box_i(count)); +} + +def test_decompress_traps_in_wasm(t: StackCompressionTester) { + t.gen(); + t.limit(2); + var k = t.callCont("make_once", [Values.box_i(5)]); + t.assertCompress(t.stackOf(k)); + var hog = M.getFreshStack(); // leaves one segment, which the call itself takes + if (hog == null) return t.t.fail("expected a segment to hold"); + t.assertTrap(TrapReason.OUT_OF_STACKS, t.call("finish", [Value.Cont(k)])); + t.assertState(StackState.COMPRESSED, t.stackOf(k)); // a failed resume leaves the continuation intact + M.recycleStack(hog); + t.t.assert_eq(1005, t.callInt("finish", [Value.Cont(k)])); +} + +def test_decompress_via_gc_wasm(t: StackCompressionTester) { + t.gen(); + t.limit(2); + var k = t.callCont("make_once", [Values.box_i(5)]); + t.assertCompress(t.stackOf(k)); + t.dropFresh(); // decompression must force a GC to reclaim this segment + t.t.assert_eq(1005, t.callInt("finish", [Value.Cont(k)])); +} + +def test_sweep_compresses_suspended(t: StackCompressionTester) { + var s1 = t.suspended("make_once", 10), s2 = t.suspended("make_once", 20), s3 = t.suspended("make_once", 30); + t.compressing(1); + var empty = M.getFreshStack(); + // All segments are now owned; the next allocation sweeps, compressing every SUSPENDED stack. + var fresh = M.getFreshStack(); + t.assertState(StackState.EMPTY, empty); + t.assertState(StackState.EMPTY, fresh); + for (s in [s1, s2, s3]) t.assertState(StackState.COMPRESSED, s); + t.checkInvariants("sweep"); + M.recycleStack(empty); + M.recycleStack(fresh); + t.assertResult(Result.Value([Values.box_i(1010)]), s1.resume()); + t.assertResult(Result.Value([Values.box_i(1020)]), s2.resume()); + t.assertResult(Result.Value([Values.box_i(1030)]), s3.resume()); +} + +def test_sweep_batches(t: StackCompressionTester) { + var stacks = Array.new(4); + for (i < stacks.length) stacks[i] = t.suspended("make_once", i); + t.compressing(0); + var before = C.numCompressed; + var fresh = Array.new(4); + fresh[0] = M.getFreshStack(); // one sweep compresses all four stacks + for (s in stacks) t.assertState(StackState.COMPRESSED, s); + t.t.assert_eq(before + 4, C.numCompressed); + // The remaining allocations are served from the cache without another sweep. + for (i = 1; i < fresh.length; i++) fresh[i] = M.getFreshStack(); + for (s in fresh) { + if (s == null) return t.t.fail("expected a cached stack"); + t.assertState(StackState.EMPTY, s); + } + t.t.assert_eq(before + 4, C.numCompressed); + for (s in fresh) M.recycleStack(s); + for (i < stacks.length) t.assertResult(Result.Value([Values.box_i(1000 + i)]), stacks[i].resume()); +} + +def test_recycled_not_compressed(t: StackCompressionTester) { + var a = t.suspended("make_once", 7), b = t.suspended("make_once", 1); + t.compressing(0); + M.recycleStack(b); // a recycled stack is cleared, so never compressed + t.assertState(StackState.EMPTY, b); + var c = M.getFreshStack(); + if (c != b) t.t.fail("expected the recycled stack to be reused"); + t.assertState(StackState.SUSPENDED, a); + var d = M.getFreshStack(); // must compress {a}, the only SUSPENDED stack + t.assertState(StackState.COMPRESSED, a); + t.assertState(StackState.EMPTY, d); + if (d == a || d == c) t.t.fail("expected a new stack for the compressed segment"); + M.recycleStack(c); + M.recycleStack(d); + t.assertResult(Result.Value([Values.box_i(1007)]), a.resume()); +} + +def test_decompress_compresses_other(t: StackCompressionTester) { + var a = t.suspended("make_once", 100), b = t.suspended("make_once", 200); + t.compressing(0); + t.assertCompress(a); + var hold = M.getFreshStack(); // takes the segment freed by {a} + // Decompressing {a} requires a segment, which can only come from compressing {b}. + t.assertDecompress(a); + t.assertState(StackState.COMPRESSED, b); + t.assertState(StackState.EMPTY, hold); + t.checkInvariants("ping"); + t.assertResult(Result.Value([Values.box_i(1100)]), a.resume()); + t.assertResult(Result.Value([Values.box_i(1200)]), b.resume()); +} + +def test_disabled_no_compress(t: StackCompressionTester) { + var a = t.suspended("make_once", 1), b = t.suspended("make_once", 2); + StackTuning.enableCompression = false; + t.limit(1); + var dropped = allocAndDrop(t.subFunc()); + var c = M.getFreshStack(); // must come from the dead stack, never compression + if (c.range.start != dropped) t.t.fail("expected the dead stack's segment to be reclaimed"); + t.assertState(StackState.SUSPENDED, a); + t.assertState(StackState.SUSPENDED, b); + t.assertState(StackState.EMPTY, c); +} +def allocAndDrop(f: Function) -> Pointer #no-inline { + return M.getFreshStack().reset(f).range.start; +} + +// Steps {n} generators round-robin to completion under a limit of {extra} segments. +def roundRobin(t: StackCompressionTester, n: int, extra: int) { + t.gen(); + t.compressing(extra); + var before = (C.numCompressed, C.numDecompressed); + var gens = Array.new(n); + for (i < n) gens[i] = t.make(i * 100); + for (j = 1; j < 10; j++) { + for (i < n) gens[i] = t.next(gens[i], i * 100 + j); + } + for (i < n) t.next(gens[i], -1); + if (C.numCompressed <= before.0) t.t.fail("expected stacks to be compressed"); + if (C.numDecompressed <= before.1) t.t.fail("expected stacks to be decompressed"); + t.checkInvariants("round robin"); +} + +def test_gens_round_robin(t: StackCompressionTester) { + roundRobin(t, 20, 4); +} + +def test_gens_many(t: StackCompressionTester) { + roundRobin(t, 500, 8); +} + +def test_gens_limit_one(t: StackCompressionTester) { + // Each call runs on its own stack and the generator needs another; one more suffices for decompression. + roundRobin(t, 10, 3); +} + +def test_gens_random_order(t: StackCompressionTester) { + t.gen(); + t.compressing(6); + var n = 30; + var gens = Array.new(n), steps = Array.new(n); + for (i < n) gens[i] = t.make(i * 100); + var live = n, seed = 12345; + while (live > 0) { + seed = (seed * 1103515245 + 12345) & 0x7FFFFFFF; + var i = (seed >> 8) % n; + if (Continuations.isNull(gens[i])) continue; + var j = ++steps[i]; + gens[i] = t.next(gens[i], if(j < 10, i * 100 + j, -1)); + if (j >= 10) live--; + } + t.checkInvariants("random"); +} + +def test_chains_under_limit(t: StackCompressionTester) { + t.gen(); + var n = 10; + // The waiting parents cannot be compressed, so leave room for them. + t.compressing(n + 4); + var before = C.numCompressed; + var chains = Array.new(n); + for (i < n) chains[i] = t.callCont("make_chain", [Values.box_i(i)]); + for (i < n) { // some tops may already have been compressed to make room + var top = t.stackOf(chains[i]); + if (top.state() == StackState.SUSPENDED) t.assertCompress(top); + t.assertState(StackState.COMPRESSED, top); + t.assertState(StackState.CALL_CHILD, NativeStack.!(top.parent)); + } + for (i = n - 1; i >= 0; i--) t.t.assert_eq(2 * i + 1, t.callInt("finish", [Value.Cont(chains[i])])); + if (C.numCompressed < before + n) t.t.fail("expected chains to be compressed"); +} + +def test_deep_frames_under_limit(t: StackCompressionTester) { + t.gen(); + t.compressing(4); + var n = 40; + var ks = Array.new(n); + for (i < n) ks[i] = t.callCont("make_deep", [Values.box_i(i)]); + for (i < n) t.t.assert_eq(deepSum(i), t.callInt("finish", [Value.Cont(ks[i])])); +} diff --git a/test/unittest/x86-64-linux/NativeStackManagerTest.v3 b/test/unittest/x86-64-linux/NativeStackManagerTest.v3 index fa8e7c109..e548b9b4a 100644 --- a/test/unittest/x86-64-linux/NativeStackManagerTest.v3 +++ b/test/unittest/x86-64-linux/NativeStackManagerTest.v3 @@ -30,6 +30,11 @@ def X_ = void( T("run_on_fresh_stack_gc", test_run_on_fresh_stack_gc), T("out_of_stacks_host", test_out_of_stacks_host), T("out_of_stacks_cont_new", test_out_of_stacks_cont_new), + T("release_attach", test_release_attach), + T("free_before_map", test_free_before_map), + T("attach_takes_cached", test_attach_takes_cached), + T("moved_segment_kept", test_moved_segment_kept), + T("reattach_runs", test_reattach_runs), () ); @@ -47,12 +52,13 @@ private class NativeStackManagerTester extends ExeTester { def limit(extra: int) { StackTuning.maxStacks = M.total; M.cache = null; + M.free = null; Target.forceGC(); var free = 0; for (seg = M.all; seg != null; seg = seg.next) if (seg.stack.get() == null) free++; held = Array.new(free); for (i < free) held[i] = fresh(); - if (M.cache != null) t.fail("expected cache to be empty after pinning"); + if (M.cache != null || M.free != null) t.fail("expected no free segments after pinning"); for (seg = M.all; seg != null; seg = seg.next) { if (seg.stack.get() == null) t.fail("expected all segments to be owned after pinning"); } @@ -95,6 +101,25 @@ private class NativeStackManagerTester extends ExeTester { } return r; } + // Not inlined, so that the fresh stack is dead once this returns. + def moveSegmentToNewStack() -> NativeStack #no-inline { + var a = fresh(); + var b = NativeStack.new(); + M.releaseSegment(a); + if (!M.attachSegment(b)) t.fail("expected the released segment to be reattached"); + if (a.range != null) t.fail("expected released stack to have no segment"); + return b; + } + def checkMovedSegment() -> NativeStackSegment #no-inline { + var b = moveSegmentToNewStack(); + var seg = b.range; + Target.forceGC(); + Target.forceGC(); + if (seg.stack.get() != b) t.fail("expected moved segment to be owned by the new stack"); + if (M.getFreshStack() != null) t.fail("expected no segment to be reclaimable while the new owner lives"); + if (b.range != seg || seg.stack.get() != b) t.fail("expected the new owner to keep the segment"); + return seg; + } // Checks global invariants of the manager. def checkInvariants(where: string) { var count = 0, size = EngineOptions.STACK_SIZE.get(); @@ -115,9 +140,12 @@ private class NativeStackManagerTester extends ExeTester { if (limit_total >= 0 && M.total > limit_total) t.fail3("%s: expected at most %d segments, got %d", where, limit_total, M.total); // Every stack in the cache owns its segment. for (s = M.cache; s != null; s = s.next_stack) { - var seg = NativeStackSegment.!(s.range); + var seg = s.range; if (seg.stack.get() != s) t.fail1("%s: cached stack does not own its segment", where); } + for (seg = M.free; seg != null; seg = seg.next_free) { + if (seg.stack.get() != null) t.fail1("%s: free segment has an owner", where); + } } def assertDisjoint(stacks: Array) { for (i < stacks.length) { @@ -183,7 +211,7 @@ def EMPTY_STRUCT = StructDecl.new(false, ValueTypes.NO_HEAPTYPES, []); def test_fresh_empty(t: NativeStackManagerTester) { var s = t.fresh(); t.t.assert_eq(u64.view(EngineOptions.STACK_SIZE.get()), s.range.size()); - var seg = NativeStackSegment.!(s.range); + var seg = s.range; if (seg.stack.get() != s) t.t.fail("expected segment to point back at its stack"); t.checkInvariants("fresh"); } @@ -226,7 +254,7 @@ def test_weak_ref_survives_gc(t: NativeStackManagerTester) { Target.forceGC(); Target.forceGC(); for (s in stacks) { - var seg = NativeStackSegment.!(s.range); + var seg = s.range; if (seg.stack.get() != s) t.t.fail("expected weak reference to live stack to survive GC"); } t.checkInvariants("after GC"); @@ -265,7 +293,7 @@ def test_reclaim_skips_live(t: NativeStackManagerTester) { t.assertAllIn(stacks, dropped); t.assertNoneIn(stacks, live_starts); for (s in live) { - if (NativeStackSegment.!(s.range).stack.get() != s) t.t.fail("expected live stack to keep its segment"); + if (s.range.stack.get() != s) t.t.fail("expected live stack to keep its segment"); } t.checkInvariants("skips live"); } @@ -391,6 +419,69 @@ def test_out_of_stacks_host(t: NativeStackManagerTester) { t.assertResult(Result.Value([Values.box_i(7)]), M.runOnFreshStack(hf, [])); } +def test_release_attach(t: NativeStackManagerTester) { + var s = t.fresh(); + var seg = s.range; + M.releaseSegment(s); + if (s.range != null) t.t.fail("expected released stack to have no segment"); + if (seg.stack.get() != null) t.t.fail("expected released segment to have no owner"); + if (M.free != seg) t.t.fail("expected released segment to be first in the free list"); + if (!M.attachSegment(s)) return t.t.fail("expected a segment"); + if (s.range != seg) t.t.fail("expected the released segment to be reattached"); + if (seg.stack.get() != s) t.t.fail("expected attach to set the owner"); + if (s.state() != StackState.EMPTY || s.vsp != seg.start || s.rsp != seg.end) t.t.fail("expected attach to leave the stack empty"); + t.checkInvariants("attach"); +} + +def test_free_before_map(t: NativeStackManagerTester) { + t.limit(2); + var s = t.fresh(); + var seg = s.range, total = M.total; + M.releaseSegment(s); + var s2 = t.fresh(); + if (s2.range != seg) t.t.fail("expected the free segment to be reused before mapping a new one"); + if (s2 == s) t.t.fail("expected a new stack for the free segment"); + t.t.assert_eq(total, M.total); + t.checkInvariants("free"); +} + +def test_attach_takes_cached(t: NativeStackManagerTester) { + t.limit(1); + var a = t.fresh(); + var seg = a.range, total = M.total; + M.recycleStack(a); + var b = NativeStack.new(); + if (!M.attachSegment(b)) return t.t.fail("expected the cached stack's segment"); + if (b.range != seg || seg.stack.get() != b) t.t.fail("expected the cached stack's segment to move"); + if (a.range != null || M.cache != null) t.t.fail("expected the cached stack to be dropped"); + t.t.assert_eq(total, M.total); + t.checkInvariants("cached"); +} + +def test_moved_segment_kept(t: NativeStackManagerTester) { + // A segment moved to another stack is reclaimed only when the new owner dies. + t.limit(1); + var seg = t.checkMovedSegment(); + Target.forceGC(); + if (seg.stack.get() != null) t.t.fail("expected the moved segment to be unowned after its new owner died"); + var s = t.fresh(); + if (s.range != seg) t.t.fail("expected the moved segment to be reclaimed"); + t.checkInvariants("moved"); +} + +def test_reattach_runs(t: NativeStackManagerTester) { + var f = t.subFunc(); + var s = t.fresh(); + var seg = s.range; + for (i < 3) { + M.releaseSegment(s); + if (!M.attachSegment(s)) return t.t.fail("expected a segment"); + } + if (s.range != seg || seg.stack.get() != s) t.t.fail("expected reattached stack to own its segment"); + t.assertResult(Result.Value([Values.box_i(2)]), s.reset(f).bind([Values.box_i(7), Values.box_i(5)]).resume()); + t.checkInvariants("reattach"); +} + def test_out_of_stacks_cont_new(t: NativeStackManagerTester) { StackTuning.enableCompression = false; var make = t.makeFunc();