From 1662aa08771984088e6f6b5e891117410a586531 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Wed, 7 Oct 2026 18:23:58 -0700 Subject: [PATCH 1/4] fix(resume): an unusable memory image recovers by disk-only cold boot, not Dead A memory snapshot taken before the ADR 0112 phase 2b roll has no swap manifest, so every host refuses to restore it. The resume verb counted each refusal as a terminal failure and set the session Dead after five, although its disk was intact. A host-agent that adopts a running guest from an older host-agent writes snapshots of the same shape, and those can reference swap pages that no snapshot holds. PG cannot tell the two kinds apart. - engram-core: new SandboxError::MemoryImageUnusable. The refusal is deterministic, and the disk is unaffected. - host-agent: the three swap restore refusals return it. It crosses gRPC as a failed_precondition marker (the HarnessSpawn precedent), so WIRE_VERSION does not change. - coordinator: resume_from_fc_snapshot catches it and does a disk-only cold boot on the newer of the live and snapshot disk lineages. The guest loses its processes and keeps its disk. A new counter, engram_session_resume_memory_image_fallback_total, counts each case. - engram-dst: a sim host can refuse memory images and records each create's root disk. The new memory_image_fallback test fails without the coordinator change. - ADR 0112: a dated addendum line records the gap and the fallback. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/engram-coordinator/src/api/snapshot.rs | 42 +++++- crates/engram-coordinator/src/metrics.rs | 6 + crates/engram-core/src/error.rs | 10 ++ crates/engram-dst/src/world.rs | 24 +++- .../engram-dst/tests/memory_image_fallback.rs | 129 ++++++++++++++++++ crates/engram-host-agent/src/grpc_server.rs | 17 +++ .../engram-host-agent/src/pooled_backend.rs | 6 +- crates/engram-protocol/src/grpc_client.rs | 17 +++ crates/engram-protocol/src/wire.rs | 48 +++++++ docs/adr/0112-ephemeral-guest-swap.md | 14 ++ 10 files changed, 302 insertions(+), 11 deletions(-) create mode 100644 crates/engram-dst/tests/memory_image_fallback.rs diff --git a/crates/engram-coordinator/src/api/snapshot.rs b/crates/engram-coordinator/src/api/snapshot.rs index da74b9b38..ca6a947cc 100644 --- a/crates/engram-coordinator/src/api/snapshot.rs +++ b/crates/engram-coordinator/src/api/snapshot.rs @@ -1106,8 +1106,8 @@ pub(crate) async fn resume_from_idle( // instead of declaring the session dead. This is also the documented // recovery path for `ColdBootUnavailable` Idle fallbacks: re-enable // the image, then /resume lands here. - if session.live_disk_manifest.is_some() { - return resume_disk_only_cold_boot(ctx, session).await; + if let Some(rootfs) = session.live_disk_manifest { + return resume_disk_only_cold_boot(ctx, session, rootfs).await; } tracing::warn!( session_id = %id, @@ -1146,13 +1146,16 @@ pub(crate) async fn destroy_retained_sandbox( } /// ADR 0028 Fix B — manual-resume flavor of the disk-only cold boot: -/// fresh kernel boot mounting the session's `live_disk_manifest` on -/// whichever host can take it, fresh harness. On-disk work survives; -/// in-RAM context does not (this path only exists because no coherent -/// memory snapshot was ever recorded). +/// fresh kernel boot mounting `rootfs` on whichever host can take it, +/// fresh harness. On-disk work survives; in-RAM context does not. This +/// path runs when no coherent memory snapshot was recorded (`rootfs` is +/// the session's `live_disk_manifest`), or when the host refused the +/// memory image as unusable (`rootfs` is the newest disk of the live +/// and snapshot lineages). async fn resume_disk_only_cold_boot( ctx: &crate::session_ops::OpCtx<'_>, session: Session, + rootfs: engram_core::types::manifest::ManifestRef, ) -> Result { let state = ctx.state; let id = session.id; @@ -1162,7 +1165,7 @@ async fn resume_disk_only_cold_boot( let mut spec = crate::boot_materializer::materialize_cold_boot(state, &session) .await? .ok_or_else(|| ApiError::Conflict("disk-only recovery requires an enabled image".into()))?; - spec.rootfs_manifest = session.live_disk_manifest; + spec.rootfs_manifest = Some(rootfs); let (repo, tag) = engram_core::types::session::split_image_ref(&session.image); let context = crate::placement::ScheduleContext { nbd_slot_need: 1 + u32::from(spec.swap_mib.unwrap_or(0) > 0), @@ -1949,6 +1952,31 @@ async fn resume_from_fc_snapshot( .await { Ok(v) => v, + // The host refused the memory image (for example, one that can + // reference swap pages no snapshot holds). Every host refuses + // it the same way, so a retry cannot help, but the disk is + // intact: boot a fresh kernel on the newest disk. The memory + // pairing rule above does not apply, because no memory is + // restored, so the live lineage wins when it is newer. + Err(SandboxError::MemoryImageUnusable(reason)) => { + let Some(rootfs) = + effective_resume_disk_manifest(session.live_disk_manifest, record.disk_manifest) + else { + return Err(ApiError::Internal(format!( + "memory image unusable and the session has no disk manifest: {reason}" + ))); + }; + ::metrics::counter!(crate::metrics::SESSION_RESUME_MEMORY_IMAGE_FALLBACK_TOTAL) + .increment(1); + tracing::warn!( + session_id = %id, + snapshot_id = %record.id, + rootfs = ?rootfs, + %reason, + "resume: host refused the memory image; recovering with a disk-only cold boot", + ); + return resume_disk_only_cold_boot(op_ctx, session, rootfs).await; + } Err(SandboxError::Snapshot(msg)) => { // Lost local artifacts on every viable host — chunked // restore couldn't rehydrate from the manifest either. diff --git a/crates/engram-coordinator/src/metrics.rs b/crates/engram-coordinator/src/metrics.rs index 3575b44a5..25f6695a9 100644 --- a/crates/engram-coordinator/src/metrics.rs +++ b/crates/engram-coordinator/src/metrics.rs @@ -388,6 +388,12 @@ pub const SESSION_OP_RESUME_BUDGET_EXHAUSTED_TOTAL: &str = /// Should be ~0; each increment is one session honestly declared /// unresumable instead of churning the outbox shim forever. pub const SESSION_UNRESUMABLE_DEMOTED_TOTAL: &str = "engram_session_unresumable_demoted_total"; +/// Resumes whose memory image the host refused as unusable (for +/// example, a memory image captured before swap was a chunked disk), +/// recovered with a disk-only cold boot. Each increment is one session +/// that lost its in-RAM context but kept its disk. +pub const SESSION_RESUME_MEMORY_IMAGE_FALLBACK_TOTAL: &str = + "engram_session_resume_memory_image_fallback_total"; /// ADR 0079 (review finding #5): orphaned Pending sessions (placed but /// no active create_boot op) re-enqueued by the reclaim sweep backstop. pub const SESSION_OP_PENDING_ORPHANS_RECOVERED_TOTAL: &str = diff --git a/crates/engram-core/src/error.rs b/crates/engram-core/src/error.rs index bbd751d06..3c3e0921c 100644 --- a/crates/engram-core/src/error.rs +++ b/crates/engram-core/src/error.rs @@ -172,6 +172,15 @@ pub enum SandboxError { kind: String, message: String, }, + /// The host refused a snapshot restore because the memory image + /// cannot be restored exactly on this fleet, for example a memory + /// image that can reference swap pages no snapshot holds. The + /// refusal is deterministic: every host gives the same answer, so + /// a retry cannot succeed. The disk is unaffected, and the resume + /// recovers with a disk-only cold boot. Crosses the host→coord + /// gRPC boundary as a `failed_precondition` with a marker message + /// (the `HarnessSpawn` precedent). + MemoryImageUnusable(String), } /// ADR 0116 B-D4: is a harness-spawn failure of this `kind` @@ -214,6 +223,7 @@ impl fmt::Display for SandboxError { Self::HarnessSpawn { kind, message } => { write!(f, "harness_spawn: kind={kind} {message}") } + Self::MemoryImageUnusable(msg) => write!(f, "memory_image_unusable: {msg}"), } } } diff --git a/crates/engram-dst/src/world.rs b/crates/engram-dst/src/world.rs index 4cb6e1655..a8bd9ea45 100644 --- a/crates/engram-dst/src/world.rs +++ b/crates/engram-dst/src/world.rs @@ -36,6 +36,14 @@ pub struct SimHostState { pub rpc_hang: Option, /// sandbox -> owning session (as told to us via create's spec). pub sandboxes: BTreeMap>, + /// Refuse every snapshot `restore` as `MemoryImageUnusable`, the + /// answer a real host gives for a memory image it cannot restore + /// exactly (for example, one captured before swap was a chunked + /// disk). Off by default, so seeds are unchanged. + pub refuse_memory_images: bool, + /// The root disk manifest each `create` was asked to mount, in call + /// order. A disk-only cold boot carries `Some`. + pub created_rootfs: Vec>, } /// A mutating host-verb's WORLD-side effect (ADR 0098 R2). Every @@ -863,13 +871,16 @@ impl SimHostClient { #[async_trait] impl HostClient for SimHostClient { - async fn create(&self, _spec: SandboxSpec) -> Result { + async fn create(&self, spec: SandboxSpec) -> Result { self.maybe_hang().await; // Draw the id BEFORE the liveness gate so the entropy stream is // identical to the pre-effect-queue world even on a down-host // failure (Calm determinism). let id = SandboxId::from(self.entropy.uuid()); self.world.require_up(self.host_id)?; + if let Some(h) = self.world.hosts.lock().get_mut(&self.host_id) { + h.created_rootfs.push(spec.rootfs_manifest); + } // Ownership is learned at bind_session time (the spec is a // template, not a binding — see sandbox.rs's type docs). self.world.record_effect( @@ -1003,6 +1014,17 @@ impl HostClient for SimHostClient { self.maybe_hang().await; let id = SandboxId::from(self.entropy.uuid()); self.world.require_up(self.host_id)?; + if self + .world + .hosts + .lock() + .get(&self.host_id) + .is_some_and(|h| h.refuse_memory_images) + { + return Err(SandboxError::MemoryImageUnusable( + "sim: memory image refused".into(), + )); + } // ADR 0108 E: a snapshot restore resumes a captured-warm harness // (ADR 0037) which re-dials on the vsock epoch bump. The dial is // scheduled by the effect's APPLICATION (the harness lives inside diff --git a/crates/engram-dst/tests/memory_image_fallback.rs b/crates/engram-dst/tests/memory_image_fallback.rs new file mode 100644 index 000000000..73a2fb83e --- /dev/null +++ b/crates/engram-dst/tests/memory_image_fallback.rs @@ -0,0 +1,129 @@ +//! A resume whose memory image the host refuses must recover the +//! session from its disk, never declare it dead. +//! +//! The production case: a memory snapshot captured before the swap +//! drive was a chunked disk carries no swap manifest, and every host +//! refuses to restore it. The disk is intact, so the resume verb must +//! fall back to a disk-only cold boot on the newest disk lineage (the +//! live manifest when it is ahead of the snapshot's disk). In-RAM +//! context is lost; the session is not. + +use engram_core::traits::BlobStorage as _; +use engram_core::types::manifest::ManifestRef; +use engram_core::types::session::SessionState; +use engram_dst::{Profile, Sim, Step}; + +fn rt() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .expect("current-thread runtime") +} + +#[test] +fn refused_memory_image_resumes_by_disk_only_cold_boot() { + rt().block_on(async { + tokio::time::pause(); + let mut sim = Sim::new(23, Profile::Calm).with_faithful_hosts(); + + sim.execute(Step::HostHeartbeats).await; + sim.execute(Step::CreateSession).await; + let sid = sim + .world + .meta + .with_db(|db| { + db.sessions + .values() + .find(|r| r.session.status == SessionState::Active) + .map(|r| r.session.id) + }) + .expect("one Active session after create"); + for h in 0..sim.world.host_ids.len() { + sim.execute(Step::HostCheckpoint(h)).await; + } + // The sim checkpoint records a manifest-less row. Give the newest + // one the shape of a production memory snapshot, backed by real + // blobs, so the resume's artifact checks pass and the host's + // refusal is the only obstacle. + let snapshot_disk = ManifestRef { + manifest_id: uuid::Uuid::from_u128(0xd15c), + version: 7, + }; + let memory = ManifestRef { + manifest_id: uuid::Uuid::from_u128(0x3e3), + version: 1, + }; + let snapshot_id = sim + .world + .meta + .with_db_mut(|db| { + let snap = db + .snapshots + .values_mut() + .filter(|s| s.session_id == Some(sid)) + .max_by_key(|s| s.created_at)?; + snap.disk_manifest = Some(snapshot_disk); + snap.memory_manifest = Some(memory); + Some(snap.id) + }) + .expect("the checkpoint recorded a snapshot row"); + let blob = sim.world.host_world.blob(); + for key in [ + snapshot_disk.storage_key(), + memory.storage_key(), + engram_chunk_store::snapshot_blob::state_blob_key(snapshot_id), + engram_chunk_store::snapshot_blob::sidecar_blob_key(snapshot_id), + ] { + blob.put(&key, bytes::Bytes::from_static(b"sim")) + .await + .expect("blob put"); + } + + // Rest to Idle the surgical way, with the continuously flushed + // disk one version ahead of the snapshot's disk. + let mut live = snapshot_disk; + live.version += 1; + sim.world.meta.with_db_mut(|db| { + let row = db.sessions.get_mut(&sid).expect("session row"); + row.session.status = SessionState::Idle; + row.session.sandbox_id = None; + row.session.host_id = None; + row.session.live_disk_manifest = Some(live); + }); + + // Every host refuses the memory image, as a real fleet does. + { + let mut hosts = sim.world.host_world.hosts.lock(); + for h in hosts.values_mut() { + h.refuse_memory_images = true; + h.created_rootfs.clear(); + } + } + + sim.execute(Step::ResumeSession).await; + + let status = sim + .world + .meta + .with_db(|db| db.sessions.get(&sid).map(|r| r.session.status)) + .expect("session row"); + assert_eq!( + status, + SessionState::Active, + "a refused memory image must recover from disk, not kill the session", + ); + let created: Vec<_> = sim + .world + .host_world + .hosts + .lock() + .values() + .flat_map(|h| h.created_rootfs.clone()) + .collect(); + assert_eq!( + created, + vec![Some(live)], + "exactly one cold boot, on the live disk (newer than the snapshot's)", + ); + }); +} diff --git a/crates/engram-host-agent/src/grpc_server.rs b/crates/engram-host-agent/src/grpc_server.rs index 51760e094..3ecd1d7be 100644 --- a/crates/engram-host-agent/src/grpc_server.rs +++ b/crates/engram-host-agent/src/grpc_server.rs @@ -1793,6 +1793,11 @@ fn sandbox_to_status(err: SandboxError) -> Status { SandboxError::HarnessSpawn { kind, message } => Status::failed_precondition( engram_protocol::wire::harness_spawn_message(&kind, &message), ), + // Same marker shape: the coordinator reads the typed variant and + // recovers the session from its disk instead of retrying. + SandboxError::MemoryImageUnusable(message) => Status::failed_precondition( + engram_protocol::wire::memory_image_unusable_message(&message), + ), } } @@ -2041,4 +2046,16 @@ mod wire_version_tests { )), ); } + + #[test] + fn memory_image_unusable_maps_to_failed_precondition_marker() { + let status = sandbox_to_status(SandboxError::MemoryImageUnusable( + "swap restore has no manifest".into(), + )); + assert_eq!(status.code(), tonic::Code::FailedPrecondition); + assert_eq!( + engram_protocol::wire::parse_memory_image_unusable_message(status.message()), + Some("swap restore has no manifest".to_string()), + ); + } } diff --git a/crates/engram-host-agent/src/pooled_backend.rs b/crates/engram-host-agent/src/pooled_backend.rs index f24adbaa7..55693b59f 100644 --- a/crates/engram-host-agent/src/pooled_backend.rs +++ b/crates/engram-host-agent/src/pooled_backend.rs @@ -1869,7 +1869,7 @@ impl PooledBackend { .await .is_some_and(|path| path.starts_with("/dev/nbd")) { - return Err(SandboxError::Snapshot( + return Err(SandboxError::MemoryImageUnusable( "swap restore would reopen a stale literal NBD device".into(), )); } @@ -5351,7 +5351,7 @@ impl PooledBackend { let size = spec.get("swap_mib").and_then(|v| v.as_u64()).unwrap_or(0); if size == 0 { if metadata.swap_manifest.is_some() { - return Err(SandboxError::Snapshot( + return Err(SandboxError::MemoryImageUnusable( "swap manifest has no device geometry".into(), )); } @@ -5363,7 +5363,7 @@ impl PooledBackend { .and_then(|v| v.as_str()) == Some("1"); if !fresh && !base && metadata.swap_manifest.is_none() { - return Err(SandboxError::Snapshot( + return Err(SandboxError::MemoryImageUnusable( "swap restore has no manifest; refusing a stale device".into(), )); } diff --git a/crates/engram-protocol/src/grpc_client.rs b/crates/engram-protocol/src/grpc_client.rs index af9544668..ab2685375 100644 --- a/crates/engram-protocol/src/grpc_client.rs +++ b/crates/engram-protocol/src/grpc_client.rs @@ -1856,6 +1856,10 @@ fn grpc_to_sandbox_err(status: tonic::Status) -> SandboxError { crate::wire::parse_harness_spawn_message(status.message()) { SandboxError::HarnessSpawn { kind, message } + } else if let Some(message) = + crate::wire::parse_memory_image_unusable_message(status.message()) + { + SandboxError::MemoryImageUnusable(message) } else { SandboxError::Vm(format!("grpc {}: {}", status.code(), status.message()).into()) } @@ -1959,4 +1963,17 @@ mod grpc_err_tests { other => panic!("expected HarnessSpawn, got {other:?}"), } } + + #[test] + fn memory_image_unusable_failed_precondition_maps_to_typed_variant() { + let status = tonic::Status::failed_precondition( + crate::wire::memory_image_unusable_message("swap restore has no manifest"), + ); + match grpc_to_sandbox_err(status) { + SandboxError::MemoryImageUnusable(message) => { + assert_eq!(message, "swap restore has no manifest"); + } + other => panic!("expected MemoryImageUnusable, got {other:?}"), + } + } } diff --git a/crates/engram-protocol/src/wire.rs b/crates/engram-protocol/src/wire.rs index eec1e9807..0b8492059 100644 --- a/crates/engram-protocol/src/wire.rs +++ b/crates/engram-protocol/src/wire.rs @@ -232,6 +232,30 @@ pub fn parse_harness_spawn_message(msg: &str) -> Option<(String, String)> { Some((kind.to_string(), message.trim().to_string())) } +/// Marker prefix for the host-agent's "memory image unusable" restore +/// refusal ([`engram_core::SandboxError::MemoryImageUnusable`], the +/// [`HARNESS_SPAWN_STATUS_PREFIX`] precedent). The host returns a +/// `failed_precondition` status whose message starts with this prefix; +/// the coord-side client maps it back to the typed variant, so the resume +/// verb can fall back to a disk-only cold boot. A peer that predates the +/// marker sees an ordinary `failed_precondition` string — no +/// `WIRE_VERSION` bump. +pub const MEMORY_IMAGE_UNUSABLE_STATUS_PREFIX: &str = "memory_image_unusable:"; + +/// Render the host-agent's memory-image refusal message. +pub fn memory_image_unusable_message(message: &str) -> String { + format!("{MEMORY_IMAGE_UNUSABLE_STATUS_PREFIX} {message}") +} + +/// Parse the reason out of a [`memory_image_unusable_message`]. Returns +/// `None` if `msg` isn't that marker. +pub fn parse_memory_image_unusable_message(msg: &str) -> Option { + let rest = msg + .strip_prefix(MEMORY_IMAGE_UNUSABLE_STATUS_PREFIX)? + .trim(); + (!rest.is_empty()).then(|| rest.to_string()) +} + /// Wire-friendly mirror of [`engram_core::types::sandbox::ExecRequest`]. /// /// Defined here so the gRPC payload bincode roundtrips cleanly via @@ -356,6 +380,30 @@ mod tests { ); } + #[test] + fn memory_image_unusable_message_round_trips() { + let msg = memory_image_unusable_message("swap restore has no manifest"); + assert_eq!( + parse_memory_image_unusable_message(&msg), + Some("swap restore has no manifest".to_string()) + ); + // The core Display is prefix-compatible with the marker. + let err = engram_core::SandboxError::MemoryImageUnusable("x y".into()); + assert_eq!( + parse_memory_image_unusable_message(&err.to_string()), + Some("x y".to_string()) + ); + assert_eq!(parse_memory_image_unusable_message("image not ready"), None); + assert_eq!( + parse_memory_image_unusable_message("memory_image_unusable:"), + None + ); + assert_eq!( + parse_memory_image_unusable_message("harness_spawn: kind=NotFound x"), + None + ); + } + #[test] fn harness_spawn_message_round_trips() { // ADR 0116 B-D4: the host renders the spawn-rejection marker; the diff --git a/docs/adr/0112-ephemeral-guest-swap.md b/docs/adr/0112-ephemeral-guest-swap.md index 3643b169f..7774ffb54 100644 --- a/docs/adr/0112-ephemeral-guest-swap.md +++ b/docs/adr/0112-ephemeral-guest-swap.md @@ -750,3 +750,17 @@ S7's base-capture rule lands with phase 2b, once the base must be swap-free for 128 total and 128 warm slots, for two devices per swap-enabled guest. Changes to the module parameter require node recreation. Swap-in latency and capture-upload measurements on the dev VM remain an operator step. + +- **2026-10-08, resume fallback:** The phase 2b rollout did not cover a + memory snapshot taken before phase 2b. Such a snapshot has no swap + manifest, so every host refuses to restore it, and the coordinator set + the session `Dead` after five refusals. A host-agent that adopts a running + guest from an older host-agent also writes snapshots of this kind, because + the guest still uses a raw swap file and the new capture runs no `swapoff`. + The two kinds look the same in PG, so neither a migration nor a host + rule can tell a safe one from an unsafe one. The host now refuses these + memory images with the typed `MemoryImageUnusable` error (a marker on + `failed_precondition`; no wire version change). The resume verb then does + a disk-only cold boot on the newer of the live and snapshot disk lineages. + The guest loses its processes and keeps its disk. Teleport already rolls + back on a destination refusal. From dbec1f7b8383e777af3c2bab4d3752d1b82a4a2e Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Wed, 7 Oct 2026 18:51:06 -0700 Subject: [PATCH 2/4] fix(protocol): bump WIRE_VERSION to 35 to fence untyped memory image refusals Before this branch, a host refused a pre-roll memory snapshot with an untyped Snapshot error. The new coordinator reads that as a generic failure and counts it toward the five-failure Dead budget, and snapshot affinity keeps sending the resume to that same host. The exact-match wire gate now fences such hosts out of placement during the roll, so resumes wait for upgraded hosts instead of failing. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/engram-protocol/src/wire.rs | 12 ++++++++---- crates/engram-protocol/tests/wire_golden.rs | 4 +++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/crates/engram-protocol/src/wire.rs b/crates/engram-protocol/src/wire.rs index 0b8492059..86a415c44 100644 --- a/crates/engram-protocol/src/wire.rs +++ b/crates/engram-protocol/src/wire.rs @@ -165,7 +165,11 @@ use serde::{Deserialize, Serialize}; // v32 appends SnapshotMetadata.swap_manifest; coordinator and hosts roll together. // v33 appends the chunked swap source and manifest to SandboxSpec. // v34 carries the swap base lineage in live migration presetup. -pub const WIRE_VERSION: u32 = 34; +// v35 types the memory image refusal (`memory_image_unusable:` marker). The +// payloads do not change. The bump fences mixed fleets: an older host refuses +// the same snapshot with an untyped error, which the resume verb counts toward +// the Dead budget instead of recovering from disk. +pub const WIRE_VERSION: u32 = 35; /// gRPC metadata (header) key carrying the caller's [`WIRE_VERSION`] on /// every coord→host request (issue #229). ASCII, lowercase — tonic @@ -237,9 +241,9 @@ pub fn parse_harness_spawn_message(msg: &str) -> Option<(String, String)> { /// [`HARNESS_SPAWN_STATUS_PREFIX`] precedent). The host returns a /// `failed_precondition` status whose message starts with this prefix; /// the coord-side client maps it back to the typed variant, so the resume -/// verb can fall back to a disk-only cold boot. A peer that predates the -/// marker sees an ordinary `failed_precondition` string — no -/// `WIRE_VERSION` bump. +/// verb can fall back to a disk-only cold boot. Unlike the harness-spawn +/// marker, it came with a `WIRE_VERSION` bump (v35): an older host refuses +/// the same snapshot untyped, so the exact-match gate fences it out. pub const MEMORY_IMAGE_UNUSABLE_STATUS_PREFIX: &str = "memory_image_unusable:"; /// Render the host-agent's memory-image refusal message. diff --git a/crates/engram-protocol/tests/wire_golden.rs b/crates/engram-protocol/tests/wire_golden.rs index 225048c73..675f76bfb 100644 --- a/crates/engram-protocol/tests/wire_golden.rs +++ b/crates/engram-protocol/tests/wire_golden.rs @@ -697,8 +697,10 @@ fn wire_version_pinned() { // HostUtilization also drops the committed-swap reservation. // 33 -> 34: MigrationPresetup advertises the swap base lineage. // Existing bincode goldens do not embed presetup; their bytes do not change. + // 34 -> 35: the memory image refusal is typed. No payload changes; the + // bump fences old hosts that refuse untyped. assert_eq!( - WIRE_VERSION, 34, + WIRE_VERSION, 35, "WIRE_VERSION changed — confirm payload goldens were regenerated too" ); } From 27d2a11f72bc5ef14299366897a780685c13a52a Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Wed, 7 Oct 2026 18:51:06 -0700 Subject: [PATCH 3/4] fix(teleport): a live move whose destination refuses the memory image rolls back finish_live_restore sent every untyped error through three retries and then fail_move, which destroys the source guest. A memory image refusal is deterministic, so a retry cannot help. Handle it like postcopy-never-loaded: abort the export and roll back, so the source keeps running. The new scenario fails without this branch. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/engram-coordinator/src/teleport.rs | 9 ++++++- .../tests/support/teleport.rs | 8 ++++++ .../tests/support/teleport_scenarios.rs | 27 +++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/crates/engram-coordinator/src/teleport.rs b/crates/engram-coordinator/src/teleport.rs index d0bd04408..41a65e46a 100644 --- a/crates/engram-coordinator/src/teleport.rs +++ b/crates/engram-coordinator/src/teleport.rs @@ -525,7 +525,14 @@ mod steps { Err(SandboxError::NotFound | SandboxError::AlreadyExists) => { rollback_begin(ctx, row, "live_export_lost".into()).await } - Err(e) if e.to_string().contains("postcopy-never-loaded") => { + // The destination refused the guest's memory image (for + // example, a guest still on a raw swap file). Every host + // refuses it the same way, so a retry cannot help; abort the + // export and keep the source running. + Err(e) + if matches!(e, SandboxError::MemoryImageUnusable(_)) + || e.to_string().contains("postcopy-never-loaded") => + { ctx.state .host_registry .backend_for(row.source_host_id) diff --git a/crates/engram-coordinator/tests/support/teleport.rs b/crates/engram-coordinator/tests/support/teleport.rs index 6240d124a..f806537f1 100644 --- a/crates/engram-coordinator/tests/support/teleport.rs +++ b/crates/engram-coordinator/tests/support/teleport.rs @@ -21,6 +21,8 @@ pub struct ScriptedHost { pub clock: Arc, pub capture_fails: AtomicBool, pub restore_fails: AtomicBool, + /// `restore` refuses the memory image (`MemoryImageUnusable`). + pub memory_refused: AtomicBool, pub resume_fails: AtomicBool, /// `resume` answers NotFound: the source sandbox no longer exists. pub resume_not_found: AtomicBool, @@ -50,6 +52,7 @@ impl ScriptedHost { clock, capture_fails: AtomicBool::new(false), restore_fails: AtomicBool::new(false), + memory_refused: AtomicBool::new(false), resume_fails: AtomicBool::new(false), resume_not_found: AtomicBool::new(false), abort_not_found: AtomicBool::new(false), @@ -161,6 +164,11 @@ impl HostClient for ScriptedHost { _fence: engram_core::traits::SessionFence, ) -> Result { self.restores.fetch_add(1, Ordering::SeqCst); + if self.memory_refused.load(Ordering::SeqCst) { + return Err(SandboxError::MemoryImageUnusable( + "swap restore has no manifest".into(), + )); + } if self.restore_fails.load(Ordering::SeqCst) { return Err(SandboxError::Snapshot("restore failed".into())); } diff --git a/crates/engram-coordinator/tests/support/teleport_scenarios.rs b/crates/engram-coordinator/tests/support/teleport_scenarios.rs index 51adf4278..05122302a 100644 --- a/crates/engram-coordinator/tests/support/teleport_scenarios.rs +++ b/crates/engram-coordinator/tests/support/teleport_scenarios.rs @@ -442,6 +442,33 @@ async fn lost_live_export_rolls_back_with_durable_error(rig: Rig) { } scenario!(lost_live_export_rolls_back_with_durable_error); +/// A destination that refuses the guest's memory image refuses it on +/// every attempt. The move rolls back at once and the source keeps +/// running; it never retries into `fail_move`, which would destroy it. +async fn refused_memory_image_rolls_back_live_move(rig: Rig) { + rig.make_live().await; + rig.dest.memory_refused.store(true, Ordering::SeqCst); + // One drive settles the move: before the typed arm, the refusal + // returned Retry here and went to `fail_move` after three attempts. + assert!(matches!(rig.drive().await, OpOutcome::Done)); + assert_eq!(rig.dest.restores.load(Ordering::SeqCst), 1, "no retry"); + assert!( + rig.source.aborts.load(Ordering::SeqCst) >= 1, + "the live export is released through migration_abort", + ); + assert_eq!( + rig.source.destroys.load(Ordering::SeqCst), + 0, + "the source guest is never destroyed", + ); + assert_eq!(rig.phase().await, None); + assert_eq!( + meta_session(&rig).await.status, + engram_core::types::SessionState::Active + ); +} +scenario!(refused_memory_image_rolls_back_live_move); + /// A consumed export (an earlier abort passed its point of no return) is /// not a resumed guest: the rollback still needs the resume ack before it /// declares Active. From 8375a4ed3c6cdb5fdde6ef81aa4e3301a3fead89 Mon Sep 17 00:00:00 2001 From: Nikhil Unni Date: Wed, 7 Oct 2026 18:51:07 -0700 Subject: [PATCH 4/4] fix(resume): the disk fallback checks capacity and tells the user processes were lost - resume_disk_only_cold_boot passes the image's memory and CPU budgets to placement. Before, it passed none, so a cold boot could land on a host without room. - After a successful fallback, the coordinator appends a fenced resumed_from_disk event. The web renders it as a marker that says the files are intact and running processes stopped. The orchestrator frame taxonomy lists the new kind. - The fallback counter increments only after the cold boot succeeds. - The sim host counts refusals, and the DST test asserts one refusal and one resumed_from_disk event. The cold boot can no longer come from the resume skipping the snapshot. - ADR 0112: the addendum line covers the wire bump, the live teleport rollback, and why the refused row stays recoverable (demoting it would unpin the disk the cold boot can mount). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/engram-coordinator/src/api/snapshot.rs | 28 +++++++++++++--- crates/engram-coordinator/src/metrics.rs | 7 ++-- crates/engram-coordinator/src/state.rs | 12 +++++++ crates/engram-dst/src/world.rs | 9 +++-- .../engram-dst/tests/memory_image_fallback.rs | 23 +++++++++++++ docs/adr/0112-ephemeral-guest-swap.md | 15 ++++++--- .../__tests__/frame-taxonomy.test.ts | 1 + .../session-thread/SystemMessage.tsx | 33 +++++++++++++++++++ .../session-thread/buildMessages.test.ts | 18 ++++++++++ .../session-thread/buildMessages.ts | 15 +++++++++ web/src/events.ts | 9 +++++ web/src/sse.ts | 2 ++ 12 files changed, 156 insertions(+), 16 deletions(-) diff --git a/crates/engram-coordinator/src/api/snapshot.rs b/crates/engram-coordinator/src/api/snapshot.rs index ca6a947cc..f2c0f6b21 100644 --- a/crates/engram-coordinator/src/api/snapshot.rs +++ b/crates/engram-coordinator/src/api/snapshot.rs @@ -1167,13 +1167,17 @@ async fn resume_disk_only_cold_boot( .ok_or_else(|| ApiError::Conflict("disk-only recovery requires an enabled image".into()))?; spec.rootfs_manifest = Some(rootfs); let (repo, tag) = engram_core::types::session::split_image_ref(&session.image); + // The same budgets the memory restore was admitted with: a cold boot + // needs the RAM and CPU too, so placement must check that they fit. + let budget = + crate::boot_materializer::resolve_resume_budget(&state.services.meta, &session).await; let context = crate::placement::ScheduleContext { nbd_slot_need: 1 + u32::from(spec.swap_mib.unwrap_or(0) > 0), repo, image_version: tag, snapshot_host: None, - memory_mib: None, - cpu_budget_vcpus: None, + memory_mib: budget.map(|(mib, _)| mib), + cpu_budget_vcpus: budget.map(|(_, vcpus)| vcpus), required_image_digest: None, exclude_host: None, prefer_host: session.host_id, @@ -1966,8 +1970,6 @@ async fn resume_from_fc_snapshot( "memory image unusable and the session has no disk manifest: {reason}" ))); }; - ::metrics::counter!(crate::metrics::SESSION_RESUME_MEMORY_IMAGE_FALLBACK_TOTAL) - .increment(1); tracing::warn!( session_id = %id, snapshot_id = %record.id, @@ -1975,7 +1977,23 @@ async fn resume_from_fc_snapshot( %reason, "resume: host refused the memory image; recovering with a disk-only cold boot", ); - return resume_disk_only_cold_boot(op_ctx, session, rootfs).await; + let response = resume_disk_only_cold_boot(op_ctx, session, rootfs).await?; + ::metrics::counter!(crate::metrics::SESSION_RESUME_MEMORY_IMAGE_FALLBACK_TOTAL) + .increment(1); + // Tell the user why their processes are gone. Best-effort: the + // session is already Active, and a failed emit must not undo that. + let _ = state + .emit_fenced( + id, + op_ctx.fence(), + SessionEvent::ResumedFromDisk { + disk_manifest: rootfs, + reason, + at: state.services.clock.now_utc(), + }, + ) + .await; + return Ok(response); } Err(SandboxError::Snapshot(msg)) => { // Lost local artifacts on every viable host — chunked diff --git a/crates/engram-coordinator/src/metrics.rs b/crates/engram-coordinator/src/metrics.rs index 25f6695a9..dabac6608 100644 --- a/crates/engram-coordinator/src/metrics.rs +++ b/crates/engram-coordinator/src/metrics.rs @@ -389,9 +389,10 @@ pub const SESSION_OP_RESUME_BUDGET_EXHAUSTED_TOTAL: &str = /// unresumable instead of churning the outbox shim forever. pub const SESSION_UNRESUMABLE_DEMOTED_TOTAL: &str = "engram_session_unresumable_demoted_total"; /// Resumes whose memory image the host refused as unusable (for -/// example, a memory image captured before swap was a chunked disk), -/// recovered with a disk-only cold boot. Each increment is one session -/// that lost its in-RAM context but kept its disk. +/// example, a memory image captured before swap was a chunked disk) +/// that then completed with a disk-only cold boot. Counted only on +/// success: each increment is one session that lost its in-RAM context +/// but kept its disk. pub const SESSION_RESUME_MEMORY_IMAGE_FALLBACK_TOTAL: &str = "engram_session_resume_memory_image_fallback_total"; /// ADR 0079 (review finding #5): orphaned Pending sessions (placed but diff --git a/crates/engram-coordinator/src/state.rs b/crates/engram-coordinator/src/state.rs index f8df4be4f..e5d00fbea 100644 --- a/crates/engram-coordinator/src/state.rs +++ b/crates/engram-coordinator/src/state.rs @@ -420,6 +420,17 @@ pub enum SessionEvent { reason: String, at: DateTime, }, + /// A resume recovered with a disk-only cold boot because the host + /// refused the session's memory image. The disk and every file on it + /// are kept; running processes, shells and in-memory state are gone. + /// Coordinator-authoritative, so a rewind keeps it. + ResumedFromDisk { + /// The disk the fresh kernel booted on. + disk_manifest: engram_core::types::manifest::ManifestRef, + /// Why the host refused the memory image. + reason: String, + at: DateTime, + }, /// ADR 0107: a validated session-mode directive rode a prompt (e.g. /// `plan`). Coordinator-authoritative — the user genuinely selected the /// mode — so `rewind_session_to_cursor` excludes this kind from its @@ -549,6 +560,7 @@ impl SessionEvent { Self::FileShared { .. } => "file_shared", Self::RecoveredFromCheckpoint { .. } => "recovered_from_checkpoint", Self::DurabilityRollback { .. } => "durability_rollback", + Self::ResumedFromDisk { .. } => "resumed_from_disk", } } diff --git a/crates/engram-dst/src/world.rs b/crates/engram-dst/src/world.rs index a8bd9ea45..4259f18e4 100644 --- a/crates/engram-dst/src/world.rs +++ b/crates/engram-dst/src/world.rs @@ -41,6 +41,8 @@ pub struct SimHostState { /// exactly (for example, one captured before swap was a chunked /// disk). Off by default, so seeds are unchanged. pub refuse_memory_images: bool, + /// How many restores `refuse_memory_images` refused. + pub memory_refusals: u32, /// The root disk manifest each `create` was asked to mount, in call /// order. A disk-only cold boot carries `Some`. pub created_rootfs: Vec>, @@ -1014,13 +1016,14 @@ impl HostClient for SimHostClient { self.maybe_hang().await; let id = SandboxId::from(self.entropy.uuid()); self.world.require_up(self.host_id)?; - if self + if let Some(h) = self .world .hosts .lock() - .get(&self.host_id) - .is_some_and(|h| h.refuse_memory_images) + .get_mut(&self.host_id) + .filter(|h| h.refuse_memory_images) { + h.memory_refusals += 1; return Err(SandboxError::MemoryImageUnusable( "sim: memory image refused".into(), )); diff --git a/crates/engram-dst/tests/memory_image_fallback.rs b/crates/engram-dst/tests/memory_image_fallback.rs index 73a2fb83e..f84b585a3 100644 --- a/crates/engram-dst/tests/memory_image_fallback.rs +++ b/crates/engram-dst/tests/memory_image_fallback.rs @@ -125,5 +125,28 @@ fn refused_memory_image_resumes_by_disk_only_cold_boot() { vec![Some(live)], "exactly one cold boot, on the live disk (newer than the snapshot's)", ); + // The cold boot came from the refusal, not from the resume skipping + // the snapshot (that path boots the same disk). + let refusals: u32 = sim + .world + .host_world + .hosts + .lock() + .values() + .map(|h| h.memory_refusals) + .sum(); + assert_eq!(refusals, 1, "the host refused the memory image once"); + let notices = sim.world.meta.with_db(|db| { + db.session_events + .get(&sid) + .map(|events| { + events + .iter() + .filter(|e| e.kind == "resumed_from_disk") + .count() + }) + .unwrap_or(0) + }); + assert_eq!(notices, 1, "the user is told the processes were lost"); }); } diff --git a/docs/adr/0112-ephemeral-guest-swap.md b/docs/adr/0112-ephemeral-guest-swap.md index 7774ffb54..3d52972d1 100644 --- a/docs/adr/0112-ephemeral-guest-swap.md +++ b/docs/adr/0112-ephemeral-guest-swap.md @@ -751,7 +751,7 @@ S7's base-capture rule lands with phase 2b, once the base must be swap-free for Changes to the module parameter require node recreation. Swap-in latency and capture-upload measurements on the dev VM remain an operator step. -- **2026-10-08, resume fallback:** The phase 2b rollout did not cover a +- **2026-10-07, resume fallback:** The phase 2b rollout did not cover a memory snapshot taken before phase 2b. Such a snapshot has no swap manifest, so every host refuses to restore it, and the coordinator set the session `Dead` after five refusals. A host-agent that adopts a running @@ -760,7 +760,12 @@ S7's base-capture rule lands with phase 2b, once the base must be swap-free for The two kinds look the same in PG, so neither a migration nor a host rule can tell a safe one from an unsafe one. The host now refuses these memory images with the typed `MemoryImageUnusable` error (a marker on - `failed_precondition`; no wire version change). The resume verb then does - a disk-only cold boot on the newer of the live and snapshot disk lineages. - The guest loses its processes and keeps its disk. Teleport already rolls - back on a destination refusal. + `failed_precondition`). Wire version is 35, so an older host that refuses + with an untyped error is fenced out during the roll. The resume verb then + does a disk-only cold boot on the newer of the live and snapshot disk + lineages, with the image's memory and CPU budgets, and appends a + `resumed_from_disk` event. The guest loses its processes and keeps its + disk. A teleport whose destination refuses the memory image rolls back + at once, for both the captured and the live kind, so the source keeps + running. The refused snapshot row stays `recoverable`: demoting it would + unpin the disk the cold boot can mount. diff --git a/orchestrator/src/listeners/__tests__/frame-taxonomy.test.ts b/orchestrator/src/listeners/__tests__/frame-taxonomy.test.ts index 7ec912d9d..4914cecfa 100644 --- a/orchestrator/src/listeners/__tests__/frame-taxonomy.test.ts +++ b/orchestrator/src/listeners/__tests__/frame-taxonomy.test.ts @@ -73,6 +73,7 @@ const WIRE_FRAME_KINDS: Readonly> = { file_shared: "durable", recovered_from_checkpoint: "durable", durability_rollback: "durable", + resumed_from_disk: "durable", user_question: "durable", question_answered: "durable", file_changed: "durable", diff --git a/web/src/components/session-thread/SystemMessage.tsx b/web/src/components/session-thread/SystemMessage.tsx index 16d418c2c..090d34b50 100644 --- a/web/src/components/session-thread/SystemMessage.tsx +++ b/web/src/components/session-thread/SystemMessage.tsx @@ -47,6 +47,8 @@ export function SystemMessage() { return ; case "durability_rollback": return ; + case "resumed_from_disk": + return ; case "user_question": return ; case "plan": @@ -133,6 +135,37 @@ function DurabilityRollback({ ); } +// The host refused the session's memory image, so the resume booted a fresh +// kernel on the newest disk. Files are intact, but anything that was running +// (dev servers, shells, background jobs) stopped. Surfaced so the user knows +// why, never hidden. +function ResumedFromDisk({ + marker, +}: { + marker: Extract; +}) { + return ( + + +
+ + + resumed from disk + + + {hms(marker.at)} + +
+

+ This session could not restore its saved memory, so it started again on its latest disk. + Your files are intact. Processes that were running, such as dev servers and shells, were + stopped; start them again if you need them. +

+ + + ); +} + // ADR 0028 A.log: the honest recovery boundary. Everything above (greyed) // was rolled back by a rung-1 recovery; the thread resumes below. Surviving // outside-world side effects are called out — the platform can't undo them. diff --git a/web/src/components/session-thread/buildMessages.test.ts b/web/src/components/session-thread/buildMessages.test.ts index 965b541df..9d857b87d 100644 --- a/web/src/components/session-thread/buildMessages.test.ts +++ b/web/src/components/session-thread/buildMessages.test.ts @@ -489,6 +489,24 @@ describe("buildMessages — message/part shaping", () => { }); }); + test("resumed_from_disk becomes a resumed_from_disk marker with the reason", () => { + const { messages } = buildMessages( + indexed([ + { + type: "resumed_from_disk", + disk_manifest: { manifest_id: "abc", version: 8 }, + reason: "swap restore has no manifest", + at: AT, + }, + ]), + SID, + ); + expect(customMarker(real(messages)[0]!)).toMatchObject({ + kind: "resumed_from_disk", + reason: "swap restore has no manifest", + }); + }); + test("a system-role agent_message becomes a note system marker", () => { const { messages } = buildMessages( indexed([ diff --git a/web/src/components/session-thread/buildMessages.ts b/web/src/components/session-thread/buildMessages.ts index ff87446bf..7b8ac4363 100644 --- a/web/src/components/session-thread/buildMessages.ts +++ b/web/src/components/session-thread/buildMessages.ts @@ -161,6 +161,12 @@ export type SystemMarker = reason: string; at: string; } + // The resume booted a fresh kernel on the disk: files kept, processes lost. + | { + kind: "resumed_from_disk"; + reason: string; + at: string; + } // ADR 0107: the agent proposed a plan via the deferred `exit_plan_mode` // tool. Rendered as an INTERACTIVE card — the doc + approve/reject while // unresolved, a one-line receipt once decided. `resolution` folds in from @@ -1174,6 +1180,15 @@ export function buildMessages( }); break; + // The host refused the memory image; the resume booted on the disk. + case "resumed_from_disk": + pushSystem(`rfd:${idx}`, "resumed from disk; running processes were stopped", { + kind: "resumed_from_disk", + reason: ev.reason, + at: ev.at, + }); + break; + // ADR 0054: the interactive question card. Ends the active assistant // turn (the run deferred here) and renders below the agent's reasoning. // The answer (if it has landed, in a later run) is folded in from the diff --git a/web/src/events.ts b/web/src/events.ts index 85bef7595..fab199c76 100644 --- a/web/src/events.ts +++ b/web/src/events.ts @@ -301,6 +301,15 @@ export type SessionEvent = rewind_disk_manifest: { manifest_id: string; version: number } | null; reason: string; at: string; + } + // The host refused the session's memory image, so the resume booted a fresh + // kernel on the newest disk. Files are kept; running processes, shells and + // in-memory state are gone. Coordinator-authoritative. + | { + type: "resumed_from_disk"; + disk_manifest: { manifest_id: string; version: number }; + reason: string; + at: string; }; export type SessionEventKind = SessionEvent["type"]; diff --git a/web/src/sse.ts b/web/src/sse.ts index f3dae4753..f98d1a3a8 100644 --- a/web/src/sse.ts +++ b/web/src/sse.ts @@ -64,6 +64,8 @@ export const SESSION_EVENT_KINDS: readonly SessionEventKind[] = [ "recovered_from_checkpoint", // ADR 0090: the durability-rollback warning marker. "durability_rollback", + // A resume booted a fresh kernel on the disk; running processes were lost. + "resumed_from_disk", // ADR 0054: historical interactive AskUserQuestion round-trip. "user_question", "question_answered",