diff --git a/crates/pantograph-embedded-runtime/src/serial_cpu_residency_key_reuse_qualification.rs b/crates/pantograph-embedded-runtime/src/serial_cpu_residency_key_reuse_qualification.rs new file mode 100644 index 000000000..dbc8dc637 --- /dev/null +++ b/crates/pantograph-embedded-runtime/src/serial_cpu_residency_key_reuse_qualification.rs @@ -0,0 +1,1275 @@ +//! One explicitly held native comparison; ordinary tests exercise only its oracle. +//! Public facade methods perform all actions. Private test-support reads observe +//! scalar work counts only and never supply readiness, native output or custody. + +use serde_json::{json, Value}; + +const WORKFLOW: &str = "cpu-m5-key-reuse-pair"; +const NODES: [&str; 2] = ["infer_a", "infer_b"]; +const OUTPUTS: [&str; 2] = ["vectors_a", "vectors_b"]; +const PHASES: [&str; 5] = ["probe", "capture", "fill", "lookup", "key_calculation"]; + +fn require(condition: bool, message: &str) -> Result<(), String> { + condition.then_some(()).ok_or_else(|| message.into()) +} + +fn array<'a>(value: &'a Value, field: &str) -> Result<&'a Vec, String> { + value[field] + .as_array() + .ok_or_else(|| format!("missing {field}")) +} + +fn rows<'a>(arm: &'a Value, phase: &str) -> Result, String> { + Ok(array(arm, "logs")? + .iter() + .filter(|row| row["record"]["phase"] == phase) + .collect()) +} + +fn vector_matches(actual: &Value, expected: &Value) -> bool { + let (Some(actual), Some(expected)) = (actual.as_array(), expected.as_array()) else { + return false; + }; + actual.len() == 8 + && expected.len() == 8 + && actual.iter().zip(expected).all(|(actual, expected)| { + match (actual.as_f64(), expected.as_f64()) { + (Some(actual), Some(expected)) => { + actual.is_finite() && expected.is_finite() && (actual - expected).abs() <= 1e-5 + } + _ => false, + } + }) +} + +fn verify_owner(owner: &Value) -> Result<(), String> { + for (field, length) in [ + ("loaded_instance", 16), + ("loaded_profile", 32), + ("effective_settings", 32), + ] { + require( + owner[field].as_array().is_some_and(|bytes| { + bytes.len() == length + && bytes + .iter() + .all(|byte| byte.as_u64().is_some_and(|byte| byte <= 255)) + }), + "native owner identity/settings missing", + )?; + } + require( + owner["cpu_threads"] == 1 + && owner["generation"] + .as_u64() + .is_some_and(|generation| generation > 0), + "native owner generation/threads invalid", + ) +} + +/// Hand-declared identities/counts are independent of production selection. +fn verify_arm(arm: &Value, enabled: bool, golden: &Value) -> Result<(), String> { + require( + arm["enabled"] == enabled && arm["configuration"]["key_reuse"] == enabled, + "wrong constructor configuration", + )?; + require( + arm["configuration"]["fair"] == true && arm["configuration"]["residency"] == true, + "required per-member modes missing", + )?; + require( + arm["error"].is_null() + && arm["collector_error"].is_null() + && arm["diagnostic_io_error"].is_null() + && arm["event_error"].is_null(), + "execution or collector failed", + )?; + require( + arm["session_close"]["ok"] == true + && arm["graph_closes_ok"] == true + && arm["shutdown"]["ok"] == true + && arm["shutdown"]["timed_out"] == false, + "teardown incomplete or uncertain", + )?; + require( + array(&arm["final_registry"], "reservations")?.is_empty() + && array(&arm["final_registry"], "runtimes")? + .iter() + .all(|runtime| runtime["model_resource_residency"].is_null()), + "final resources remain", + )?; + let diagnostics = &arm["diagnostics"]; + require( + diagnostics["loads"] == 1 + && diagnostics["retains"] == 1 + && diagnostics["forward_drains"] == 2 + && diagnostics["stops"] == 1 + && diagnostics["has_idle_owner"] == false, + "native populations differ", + )?; + require( + arm["work"]["error"].is_null() && arm["work"]["retained"] == 0, + "cache observation unavailable or entries retained", + )?; + require( + arm["work"]["encodings"] == if enabled { 4 } else { 6 } + && arm["work"]["hits"] == if enabled { 4 } else { 0 }, + "fixed mechanism counts differ", + )?; + let run = arm["response"]["workflow_run_id"] + .as_str() + .filter(|run| !run.is_empty()) + .ok_or("missing run")?; + require( + array(arm, "resume_run_ids")?.iter().all(|id| id == run), + "readiness resume changed run", + )?; + let returned = array(&arm["response"], "outputs")?; + require(returned.len() == 2, "public output population differs")?; + for (index, node) in OUTPUTS.iter().enumerate() { + let values: Vec<_> = returned + .iter() + .filter(|output| output["node_id"] == *node && output["port_id"] == "vector") + .collect(); + require( + values.len() == 1 + && vector_matches(&values[0]["value"], &golden["single_vectors"][index]), + "public golden vector differs", + )?; + let retained: Vec<_> = array(arm, "retained")? + .iter() + .filter(|row| row["node_id"] == *node) + .collect(); + require( + !retained.is_empty() + && retained.iter().all(|row| { + row["workflow_run_id"] == run + && row["value"] == values[0]["value"] + && row["read"]["complete"] == true + && row["read"]["artifact_id"] == row["artifact_id"] + && row["read"]["byte_length"] == row["body_bytes"] + && row["artifact_id"].as_str().is_some_and(|id| !id.is_empty()) + }), + "retained artifact missing or differs", + )?; + } + + let grants = rows(arm, "service_opportunity")?; + let drained = rows(arm, "native_drained")?; + let acks = rows(arm, "task_registry_ack")?; + require( + grants.len() == 2 && drained.len() == 2 && acks.len() == 2, + "grant/drain/registry-ACK populations differ", + )?; + let events = array(arm, "events")?; + let lifecycle = |transition: &str| { + events + .iter() + .filter(|event| { + event["payload"]["payload_type"] == "scheduler_task_attempt_lifecycle_changed" + && event["payload"]["execution_class"] == "runtime" + && event["payload"]["transition"] == transition + }) + .collect::>() + }; + let starts = lifecycle("started"); + let completed = lifecycle("completed"); + require( + starts.len() == 2 && completed.len() == 2, + "Start/Completed populations differ", + )?; + require( + grants[0]["record"]["ticket"].as_u64().is_some() + && grants[1]["record"]["ticket"].as_u64().is_some() + && grants[0]["record"]["ticket"] != grants[1]["record"]["ticket"], + "service ticket reused", + )?; + for field in [ + "task_id", + "attempt_id", + "execution_request_id", + "reservation_lease_id", + ] { + require( + !drained[0]["record"][field].is_null() + && drained[0]["record"][field] != drained[1]["record"][field], + "native identity reused", + )?; + } + let mut task_ids = Vec::new(); + for (index, row) in drained.iter().enumerate() { + let native = &row["record"]; + // Candidate identifies the selected runtime alternative, not an attempt. + require( + native["candidate_id"] == "runtime.candle", + "native candidate does not identify the Candle runtime", + )?; + let (previous, current) = (&native["previous_owner"], &native["current_owner"]); + verify_owner(previous)?; + verify_owner(current)?; + for field in [ + "loaded_instance", + "loaded_profile", + "effective_settings", + "cpu_threads", + ] { + require( + previous[field] == current[field], + "native warm identity/settings changed", + )?; + } + require( + previous["generation"] + .as_u64() + .and_then(|generation| generation.checked_add(2)) + == current["generation"].as_u64(), + "native warm generation did not advance by two", + )?; + if index == 1 { + require( + drained[0]["record"]["current_owner"] == *previous, + "native owner continuity broken", + )?; + } + require( + native["workflow_id"] == WORKFLOW + && native["workflow_run_id"] == run + && native["node_id"] == NODES[index], + "native node/run order differs", + )?; + let task = native["task_id"] + .as_str() + .filter(|id| !id.is_empty()) + .ok_or("native task absent")?; + task_ids.push(json!(task)); + let lease = native["reservation_lease_id"] + .as_u64() + .ok_or("native lease absent")?; + let ack_rows: Vec<_> = acks + .iter() + .filter(|ack| ack["record"]["task_id"] == task) + .collect(); + let start_rows: Vec<_> = starts + .iter() + .filter(|event| event["payload"]["scheduler_task_id"] == task) + .collect(); + let complete_rows: Vec<_> = completed + .iter() + .filter(|event| event["payload"]["scheduler_task_id"] == task) + .collect(); + require( + ack_rows.len() == 1 && start_rows.len() == 1 && complete_rows.len() == 1, + "identity joins are not bijective", + )?; + let ack = &ack_rows[0]["record"]; + for field in [ + "workflow_id", + "workflow_run_id", + "node_id", + "task_id", + "attempt_id", + "execution_request_id", + "candidate_id", + "reservation_lease_id", + ] { + require( + !native[field].is_null() && ack[field] == native[field], + "registry ACK identity differs", + )?; + } + require( + ack["registry_task_release_acknowledged"] == true + && ack["scheduler_cleanup_acknowledged"] == false + && ack["lifecycle_event_id"] + .as_str() + .is_some_and(|id| !id.is_empty()), + "registry ACK missing or mislabeled", + )?; + let start = start_rows[0]; + let complete = complete_rows[0]; + require( + start["workflow_run_id"] == run + && complete["workflow_run_id"] == run + && start["node_id"] == native["node_id"] + && complete["node_id"] == native["node_id"] + && start["payload"]["scheduler_attempt_id"] == native["attempt_id"] + && complete["payload"]["scheduler_attempt_id"] == native["attempt_id"] + && complete["payload"]["reservation_id"] == format!("runtime-registry.{lease}") + && complete["payload"]["selected_runtime_id"] == "candle" + && complete["payload"]["selected_runtime_variant_id"] == "candle.cpu" + && complete["payload"]["selected_device_id"] == "cpu" + && start["event_seq"] + .as_u64() + .zip(complete["event_seq"].as_u64()) + .is_some_and(|(a, b)| a < b), + "ledger/native dispatch join differs", + )?; + require( + grants[index]["record"]["workflow_run_id"] == run + && grants[index]["record"]["task_id"] == task, + "service grant belongs to another member", + )?; + require( + grants[index]["sequence"] + .as_u64() + .zip(row["sequence"].as_u64()) + .is_some_and(|(a, b)| a < b) + && row["sequence"] + .as_u64() + .zip(ack_rows[0]["sequence"].as_u64()) + .is_some_and(|(a, b)| a < b), + "grant/native/ACK order differs", + )?; + if index == 0 { + require( + ack_rows[0]["sequence"] + .as_u64() + .zip(grants[1]["sequence"].as_u64()) + .is_some_and(|(a, b)| a < b), + "next grant preceded member registry ACK", + )?; + } + } + let proposals = rows(arm, "proposal")?; + require( + proposals.len() == 1 + && proposals[0]["record"]["workflow_run_id"] == run + && proposals[0]["record"]["task_ids"] == json!(task_ids) + && proposals[0]["sequence"] + .as_u64() + .zip(drained[0]["sequence"].as_u64()) + .is_some_and(|(a, b)| a < b), + "genuine two-Ready pair missing", + )?; + let metrics = rows(arm, "grouping_observations")?; + let forecasts = rows(arm, "warm_service_forecasts")?; + if enabled { + require( + !metrics.is_empty() + && forecasts.len() == 1 + && array(&forecasts[0]["record"], "forecasts")?.len() == 2 + && forecasts[0]["record"]["coverage"] == "captured_compatible_prefix", + "two-member captured prefix missing", + )?; + let snapshot = &metrics.last().ok_or("phase snapshot absent")?["record"]; + require( + snapshot["scope"] + .as_str() + .is_some_and(|scope| !scope.is_empty()) + && snapshot["scope"] == forecasts[0]["record"]["scope"], + "phase scope differs", + )?; + let phases = array(snapshot, "phases")?; + require(phases.len() == 5, "phase population differs")?; + for name in PHASES { + let phase: Vec<_> = phases + .iter() + .filter(|phase| phase["phase"] == name) + .collect(); + require( + phase.len() == 1 + && phase[0]["aggregate"]["state"] == "available" + && phase[0]["aggregate"]["calls"] + .as_u64() + .is_some_and(|calls| calls > 0) + && phase[0]["aggregate"]["elapsed_ns"].as_u64().is_some(), + "phase unavailable or unobserved", + )?; + } + } else { + require( + metrics.is_empty() && forecasts.is_empty(), + "disabled baseline acquired grouping measurements", + )?; + } + Ok(()) +} + +fn verify_comparison(arms: &[Value], golden: &Value) -> Result<(), String> { + require(arms.len() == 2, "two arms required")?; + verify_arm(&arms[0], false, golden)?; + verify_arm(&arms[1], true, golden)?; + require( + arms[0]["response"]["workflow_run_id"] != arms[1]["response"]["workflow_run_id"], + "fresh arms reused run identity", + )?; + for output in OUTPUTS { + let value = |arm: &Value| { + array(&arm["response"], "outputs") + .ok()? + .iter() + .find(|row| row["node_id"] == output) + .map(|row| row["value"].clone()) + }; + require( + value(&arms[0]) == value(&arms[1]), + "baseline/enabled output differs", + )?; + } + Ok(()) +} + +#[cfg(feature = "host-dependency-inventory")] +mod held { + use super::super::{config, seed, MODEL}; + use super::*; + use crate::EmbeddedSerialCpuResidency; + use pantograph_diagnostics_ledger::{DiagnosticsLedgerRepository, SqliteDiagnosticsLedger}; + use pantograph_workflow_service::graph::{ + authored_snapshot_from_descriptor, NodeRegistry, WorkflowGraph, + }; + use pantograph_workflow_service::*; + use std::{ + error::Error, + fs::{File, OpenOptions}, + io::Write, + path::{Path, PathBuf}, + sync::Mutex, + time::{Duration, Instant}, + }; + + type Outcome = Result>; + const PROTOCOL: &str = "m5-native-two-ready-key-reuse-v2"; + const TEXTS: [&str; 2] = ["hello world", "hello"]; + const RUN_BUDGET: Duration = Duration::from_secs(60); + const SHUTDOWN_BUDGET: Duration = Duration::from_secs(30); + struct LogBuffer { + rows: Vec, + bytes: usize, + error: Option<&'static str>, + file: Option, + arm: &'static str, + } + struct Collector(Mutex); + static LOG: Collector = Collector(Mutex::new(LogBuffer { + rows: Vec::new(), + bytes: 0, + error: None, + file: None, + arm: "preflight", + })); + impl log::Log for Collector { + fn enabled(&self, metadata: &log::Metadata<'_>) -> bool { + matches!( + metadata.target(), + "workflow::serial_ready" + | "workflow::serial_observations" + | "embedded::serial_cpu_residency" + ) + } + fn log(&self, record: &log::Record<'_>) { + if !self.enabled(record.metadata()) { + return; + } + let Ok(mut buffer) = self.0.lock() else { + return; + }; + let text = record.args().to_string(); + match serde_json::from_str::(&text) { + Ok(value) => { + let sequence = buffer.rows.len(); + let row = json!({"sequence":sequence,"arm":buffer.arm,"target":record.target(),"record":value}); + let Ok(mut bytes) = serde_json::to_vec(&row) else { + buffer.error = Some("structured row serialization failed"); + return; + }; + bytes.push(b'\n'); + if buffer.rows.len() >= 2048 + || bytes.len() > 64 * 1024 + || buffer.bytes + bytes.len() > 4 * 1024 * 1024 + { + buffer.error = Some("bounded log capacity exceeded"); + return; + } + buffer.bytes += bytes.len(); + buffer.rows.push(row); + if let Some(file) = buffer.file.as_mut() { + if file + .write_all(&bytes) + .and_then(|_| file.sync_all()) + .is_err() + { + buffer.error = Some("native row persistence failed"); + buffer.file = None; + } + } + } + Err(_) => buffer.error = Some("structured evidence parse failed"), + } + } + fn flush(&self) {} + } + fn logs(from: usize) -> (Vec, Option<&'static str>) { + LOG.0 + .lock() + .map_or((Vec::new(), Some("collector lock unavailable")), |buffer| { + (buffer.rows[from..].to_vec(), buffer.error) + }) + } + fn save(root: &Path, name: &str, value: &Value) -> Outcome<()> { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(root.join(name))?; + serde_json::to_writer_pretty(&mut file, value)?; + file.write_all(b"\n")?; + file.sync_all()?; + Ok(()) + } + fn progress(file: &mut File, arm: &str, stage: &str, detail: Value) -> Outcome<()> { + serde_json::to_writer( + &mut *file, + &json!({"arm":arm,"stage":stage,"detail":detail}), + )?; + file.write_all(b"\n")?; + file.sync_all()?; + Ok(()) + } + + async fn author_pair( + facade: &EmbeddedSerialCpuResidency, + edits: &mut Vec, + ) -> Outcome<()> { + let mut nodes = Vec::new(); + let mut edges = Vec::new(); + for suffix in ["a", "b"] { + nodes.extend([ + json!({"id":format!("deps_{suffix}"),"node_type":"dependency-environment","position":{"x":0,"y":0},"data":{"mode":"manual"}}), + json!({"id":format!("prompt_{suffix}"),"node_type":"text-input","position":{"x":0,"y":100},"data":{}}), + json!({"id":format!("infer_{suffix}"),"node_type":"llm-inference","position":{"x":300,"y":100},"data":{ + "task_kind":"embedding","runtime":"candle","device":"cpu", + "runtime_source_context":{"operation_type":"embedding.text","context_shape_key":"embedding.one-text","cancellation_mode":"run_scoped"}, + "pumas_model_ref":{"model_id":MODEL,"selected_artifact_id":"main"}}}), + json!({"id":format!("vectors_{suffix}"),"node_type":"vector-output","position":{"x":600,"y":100},"data":{}}), + ]); + for (prefix, source, source_handle, target, target_handle) in [ + ( + "text", + format!("prompt_{suffix}"), + "text", + format!("infer_{suffix}"), + "text", + ), + ( + "vector", + format!("infer_{suffix}"), + "embedding", + format!("vectors_{suffix}"), + "vector", + ), + ( + "dep", + format!("deps_{suffix}"), + "dependency_environment_sidecar", + format!("infer_{suffix}"), + "dependency_environment_sidecar", + ), + ] { + edges.push(json!({"id":format!("{prefix}_{suffix}"),"source":source,"source_handle":source_handle,"target":target,"target_handle":target_handle})); + } + } + let mut graph: WorkflowGraph = + serde_json::from_value(json!({"nodes":nodes,"edges":edges}))?; + let registry = NodeRegistry::new(); + for node in &mut graph.nodes { + if node.node_type != "llm-inference" { + node.data["definition"] = serde_json::to_value( + registry + .get_definition(&node.node_type) + .ok_or("missing built-in definition")?, + )?; + } + } + let preview = facade + .workflow_graph_create_edit_session(WorkflowGraphEditSessionCreateRequest { + graph: graph.clone(), + workflow_id: None, + }) + .await?; + edits.push(preview.session_id.clone()); + let validation = facade + .workflow_graph_refresh_current_validation_summary( + WorkflowGraphCurrentValidationRefreshRequest { + graph_session_id: preview.session_id.clone(), + graph_revision: preview.graph_revision.parse()?, + }, + ) + .await?; + for id in NODES { + let descriptor = &validation + .node_projections + .iter() + .find(|node| node.node_id.as_str() == id) + .ok_or("missing native descriptor")? + .descriptor; + graph + .nodes + .iter_mut() + .find(|node| node.id == id) + .ok_or("missing infer node")? + .data["inference_interface_snapshot"] = + serde_json::to_value(authored_snapshot_from_descriptor(descriptor)?)?; + } + facade + .workflow_graph_close_edit_session(WorkflowGraphEditSessionCloseRequest { + session_id: preview.session_id, + }) + .await?; + edits.pop(); + facade.workflow_graph_save(WorkflowGraphSaveRequest { + name: WORKFLOW.into(), + graph: graph.clone(), + })?; + let edit = facade + .workflow_graph_create_edit_session(WorkflowGraphEditSessionCreateRequest { + graph, + workflow_id: Some(WORKFLOW.into()), + }) + .await?; + edits.push(edit.session_id.clone()); + let validation = facade + .workflow_graph_refresh_current_validation_summary( + WorkflowGraphCurrentValidationRefreshRequest { + graph_session_id: edit.session_id.clone(), + graph_revision: edit.graph_revision.parse()?, + }, + ) + .await?; + for id in ["deps_a", "deps_b"] { + let resolved = facade.workflow_graph_resolve_dependency_environment_action_intent(pantograph_inference_interface_contracts::DependencyEnvironmentActionIntent { + contract_version: 1, graph_session_id: edit.session_id.parse()?, graph_revision: edit.graph_revision.parse()?, + validation_session_id: validation.summary.validation_session_id.clone(), target_node_id: id.parse()?, + action: pantograph_inference_interface_contracts::DependencyEnvironmentAction::Resolve, + }).await?; + if resolved.status != pantograph_inference_interface_contracts::DependencyEnvironmentActionIntentStatus::RequestReady { return Err("dependency action not ready".into()); } + } + facade + .publish_graph_session_executable_validation_snapshot( + WorkflowGraphSessionExecutableValidationSnapshotPublishRequest { + workflow_id: WORKFLOW.into(), + workflow_semantic_version: "1.0.0".into(), + graph_session_id: edit.session_id, + validation_snapshot_id: None, + validation_session_id: Some( + validation + .summary + .validation_session_id + .ok_or("validation session absent")?, + ), + }, + ) + .await?; + Ok(()) + } + + async fn run_pair( + facade: &EmbeddedSerialCpuResidency, + session: &str, + arm: &mut Value, + ) -> Outcome { + let operation = async { + let mut result = facade + .run_workflow_execution_session(WorkflowExecutionSessionRunRequest { + session_id: session.into(), + workflow_semantic_version: "1.0.0".into(), + inputs: ["a", "b"] + .into_iter() + .zip(TEXTS) + .map(|(suffix, text)| WorkflowPortBinding { + node_id: format!("prompt_{suffix}"), + port_id: "text".into(), + value: json!(text), + }) + .collect(), + output_targets: Some( + OUTPUTS + .into_iter() + .map(|node| WorkflowOutputTarget { + node_id: node.into(), + port_id: "vector".into(), + }) + .collect(), + ), + override_selection: None, + timeout_ms: None, + priority: Some(0), + }) + .await; + let mut resumes = Vec::new(); + let mut polls = 0; + while matches!( + result, + Err(WorkflowServiceError::RuntimeDependencyReadinessPending { .. }) + ) { + arm["readiness_pending"] = json!(result.as_ref().err().map(ToString::to_string)); + polls += 1; + if polls > 1200 { + return Err("readiness poll bound exceeded".into()); + } + tokio::time::sleep(Duration::from_millis(50)).await; + if let Some(request) = facade + .workflow_execution_session_runtime_dependency_readiness_resume_candidates()? + .into_iter() + .find(|request| request.session_id == session) + { + if resumes.len() >= 32 { + return Err("readiness resume bound exceeded".into()); + } + resumes.push(request.workflow_run_id.clone()); + arm["resume_run_ids"] = json!(resumes); + result = facade + .resume_workflow_execution_session_runtime_dependency_readiness(request) + .await; + } + } + Ok::<_, Box>(result?) + }; + tokio::time::timeout(RUN_BUDGET, operation).await? + } + + fn events(root: &Path) -> Outcome> { + let ledger = + SqliteDiagnosticsLedger::open(root.join("app-data/serial-cpu-diagnostics.db"))?; + let events = ledger.diagnostic_events_after(0, 500)?; + if events.len() >= 500 { + return Err("ledger page bound exceeded".into()); + } + events + .into_iter() + .map(|event| { + Ok( + json!({"event_seq":event.event_seq,"workflow_run_id":event.workflow_run_id, + "node_id":event.node_id,"payload":serde_json::from_str::(&event.payload_json)?}), + ) + }) + .collect() + } + + async fn retained( + facade: &EmbeddedSerialCpuResidency, + session: &str, + response: &WorkflowRunResponse, + arm: &mut Value, + ) -> Outcome<()> { + let refresh = facade.workflow_diagnostics_projection_refresh( + WorkflowDiagnosticsProjectionRefreshRequest { + projections: vec![ + WorkflowDiagnosticsProjectionKind::RunDetail, + WorkflowDiagnosticsProjectionKind::NodeStatus, + WorkflowDiagnosticsProjectionKind::IoArtifact, + ], + workflow_run_id: Some(response.workflow_run_id.clone()), + workflow_id: Some(WORKFLOW.into()), + reason: WorkflowDiagnosticsProjectionRefreshReason::ExplicitRefresh, + batch_size: 500, + }, + )?; + arm["projection_refresh"] = json!(refresh); + if !refresh.failed.is_empty() || refresh.advanced.len() != 3 { + return Err("projection refresh incomplete".into()); + } + let inspection = + facade.workflow_run_inspection_query(WorkflowRunInspectionQueryRequest { + workflow_run_id: response.workflow_run_id.clone(), + artifact_limit: Some(128), + projection_batch_size: Some(500), + })?; + let mut values = Vec::new(); + for artifact in &inspection.io_artifacts { + let Some(node) = artifact + .producer_node_id + .as_deref() + .filter(|node| OUTPUTS.contains(node)) + else { + continue; + }; + if artifact.producer_port_id.as_deref() != Some("vector") { + continue; + } + let body = facade.workflow_read_artifact_body(ArtifactReadRequest { + artifact_id: artifact.payload_artifact_id.clone(), + byte_range_start: None, + byte_range_end_exclusive: None, + })?; + values.push(json!({"node_id":node,"workflow_run_id":artifact.workflow_run_id,"artifact_id":artifact.payload_artifact_id, + "read":body.response,"body_bytes":body.body.len(),"value":serde_json::from_slice::(&body.body)?,"body_blake3":blake3::hash(&body.body).to_hex().to_string()})); + } + arm["inspection"] = json!(inspection); + arm["retained"] = json!(values); + arm["task_states"] = json!(facade.workflow_get_scheduler_task_state_read_models( + pantograph_workflow_service::workflow::WorkflowSchedulerTaskStateReadModelQueryRequest { session_id: session.into(), workflow_run_id: response.workflow_run_id.clone() }).await?); + Ok(()) + } + + async fn run_arm(root: &Path, enabled: bool, progress_file: &mut File) -> Outcome { + std::fs::create_dir(root)?; + let name = if enabled { "enabled" } else { "baseline" }; + LOG.0.lock().map_err(|_| "collector lock unavailable")?.arm = name; + progress(progress_file, name, "seed", json!({}))?; + seed(root); + let log_start = logs(0).0.len(); + let constructed = if enabled { + EmbeddedSerialCpuResidency::new_with_bounded_key_reuse(config(root)).await + } else { + EmbeddedSerialCpuResidency::new(config(root)).await + }; + let facade = match constructed { + Ok(facade) => facade, + Err(error) => { + let row = json!({"enabled":enabled,"error":error.to_string(),"stage":"construct","construction_failed":true}); + save(root, "arm.json", &row)?; + return Err(error.into()); + } + }; + let snapshot = facade.scheduler_configuration(); + let mut arm = json!({"enabled":enabled,"configuration":{"key_reuse":snapshot.serial_key_reuse(),"fair":snapshot.serial_fair_opportunities(),"residency":snapshot.serial_residency()}, + "error":null,"collector_error":null,"resume_run_ids":[],"retained":[],"scheduler_cleanup_match":"source_assisted_adapter_validation; registry ACK is distinct"}); + let mut edit_sessions = Vec::new(); + let mut session = None; + let body: Outcome<()> = async { + progress(progress_file, name, "author", json!({}))?; + author_pair(&facade, &mut edit_sessions).await?; + let client = facade.register_attribution_client(ClientRegistrationRequest { + display_name: Some(PROTOCOL.into()), + metadata_json: None, + })?; + let opened = facade.open_client_session(ClientSessionOpenRequest { + credential: client.credential_proof_request(), + takeover: false, + reason: Some(PROTOCOL.into()), + })?; + let created = facade + .create_attributed_workflow_execution_session( + WorkflowExecutionSessionAttributedCreateRequest { + workflow_id: WORKFLOW.into(), + usage_profile: None, + keep_alive: false, + attribution: WorkflowExecutionSessionAttributionRequest { + credential: client.credential_proof_request(), + client_session_id: opened.session.client_session_id.as_str().into(), + bucket_selection: + pantograph_runtime_attribution::BucketSelection::Default, + }, + }, + ) + .await?; + session = Some(created.session_id.clone()); + arm["session"] = json!(created); + progress(progress_file, name, "run", json!({"session_id":session}))?; + let start = Instant::now(); + let response = run_pair( + &facade, + session.as_deref().ok_or("session missing")?, + &mut arm, + ) + .await?; + arm["submit_to_result_elapsed_ns"] = json!(start.elapsed().as_nanos()); + arm["response"] = json!(response); + progress(progress_file, name, "run_returned", arm["response"].clone())?; + arm["work"] = match facade + .runtime + .workflow_service + .try_serial_grouping_work_for_test() + { + Ok((encodings, hits, retained)) => { + json!({"encodings":encodings,"hits":hits,"retained":retained,"error":null}) + } + Err(error) => json!({"error":error.to_string()}), + }; + retained( + &facade, + session.as_deref().ok_or("session missing")?, + &response, + &mut arm, + ) + .await?; + Ok(()) + } + .await; + if let Err(error) = &body { + arm["error"] = json!(error.to_string()); + } + // No fallible diagnostic write may bypass close and the existing owner. + let mut io_error = progress( + progress_file, + name, + "teardown", + json!({"body_error":arm["error"]}), + ) + .err() + .map(|error| error.to_string()); + let close_deadline = tokio::time::Instant::now() + Duration::from_secs(5); + if let Some(session_id) = session { + let close = tokio::time::timeout_at( + close_deadline, + facade.close_workflow_execution_session(WorkflowExecutionSessionCloseRequest { + session_id, + }), + ) + .await; + arm["session_close"] = match close { + Ok(result) => { + json!({"ok":result.is_ok(),"error":result.err().map(|error|error.to_string())}) + } + Err(error) => json!({"ok":false,"timed_out":true,"error":error.to_string()}), + }; + } + let mut graph_closes_ok = true; + let mut graph_closes = Vec::new(); + for session_id in edit_sessions { + let close = tokio::time::timeout_at( + close_deadline, + facade.workflow_graph_close_edit_session(WorkflowGraphEditSessionCloseRequest { + session_id: session_id.clone(), + }), + ) + .await; + let row = match close { + Ok(result) => { + json!({"session_id":session_id,"ok":result.is_ok(),"error":result.err().map(|error|error.to_string())}) + } + Err(error) => { + json!({"session_id":session_id,"ok":false,"timed_out":true,"error":error.to_string()}) + } + }; + graph_closes_ok &= row["ok"] == true; + graph_closes.push(row); + } + arm["graph_closes_ok"] = json!(graph_closes_ok); + arm["graph_closes"] = json!(graph_closes); + arm["before_shutdown_registry"] = json!(facade.registry_snapshot()); + let shutdown = tokio::time::timeout(SHUTDOWN_BUDGET, facade.shutdown()).await; + arm["shutdown"] = match shutdown { + Ok(result) => { + json!({"ok":result.is_ok(),"timed_out":false,"error":result.err().map(|error|error.to_string())}) + } + Err(error) => { + json!({"ok":false,"timed_out":true,"error":error.to_string(),"custody":"supervisor retained; timeout is not release evidence"}) + } + }; + arm["final_registry"] = json!(facade.registry_snapshot()); + arm["diagnostics"] = json!(facade.residency_diagnostics()); + match events(root) { + Ok(events) => arm["events"] = json!(events), + Err(error) => { + arm["event_error"] = json!(error.to_string()); + } + } + let (records, error) = logs(log_start); + arm["logs"] = json!(records); + arm["collector_error"] = json!(error); + if let Err(error) = progress( + progress_file, + name, + "teardown_returned", + arm["shutdown"].clone(), + ) { + io_error.get_or_insert_with(|| error.to_string()); + } + arm["diagnostic_io_error"] = json!(io_error); + // Save raw outcomes before the caller runs any acceptance oracle. + save(root, "arm.json", &arm)?; + drop(facade); + if io_error.is_some() { + return Err("diagnostic output failed; retained prefix preserved".into()); + } + Ok(arm) + } + + #[tokio::test] + #[ignore = "one held native protocol; requires separate exact-source admission and evidence root"] + async fn fixed_two_ready_native_key_reuse_comparison_v2() -> Outcome<()> { + let root = PathBuf::from( + std::env::var_os("PANTOGRAPH_M5_KEY_REUSE_EVIDENCE_ROOT") + .ok_or("mandatory evidence root missing")?, + ); + if !root.is_absolute() { + return Err("absolute evidence root required".into()); + } + if !root.is_dir() { + return Err("outer source/binary freeze must create the evidence root first".into()); + } + let metadata = PathBuf::from( + std::env::var_os("PUMAS_REGISTRY_DB_PATH") + .ok_or("static Pumas metadata path missing")?, + ); + if metadata != root.join("metadata.db") || metadata.exists() { + return Err( + "fresh root/metadata.db must be selected before this process starts".into(), + ); + } + save( + &root, + "protocol.json", + &json!({"protocol":PROTOCOL,"arms":["baseline","enabled"],"texts":TEXTS, + "public_runs_per_arm":1,"native_forwards_per_arm":2,"cpu_threads":1,"declared_ram_bytes":2_u64*1024*1024*1024, + "run_readiness_seconds":60,"teardown_close_seconds_total":5,"shutdown_seconds":30,"retries":0,"speedup_claim":false,"memory_claim":false, + "input_source":"committed BERT8 golden texts[0..2]","scheduler_cleanup_match":"source-assisted, registry ACK distinct", + "pumas_metadata":"one process-wide fresh metadata.db shared by arms; private library/project/RuntimeRegistry roots differ"}), + )?; + let mut progress_file = OpenOptions::new() + .write(true) + .create_new(true) + .open(root.join("progress.jsonl"))?; + LOG.0.lock().map_err(|_| "collector lock unavailable")?.file = Some( + OpenOptions::new() + .write(true) + .create_new(true) + .open(root.join("native-rows.jsonl"))?, + ); + log::set_logger(&LOG) + .map_err(|_| "isolated held process requires exclusive logger ownership")?; + log::set_max_level(log::LevelFilter::Info); + let golden: Value = serde_json::from_slice(include_bytes!( + "../../inference/tests/fixtures/candle_bert/bert-8/golden.json" + ))?; + require( + golden["texts"][0] == TEXTS[0] && golden["texts"][1] == TEXTS[1], + "fixed golden inputs changed", + )?; + let mut arms = Vec::new(); + let result: Outcome<()> = async { + for enabled in [false, true] { + let arm = run_arm( + &root.join(if enabled { "enabled" } else { "baseline" }), + enabled, + &mut progress_file, + ) + .await?; + arms.push(arm); + verify_arm(arms.last().ok_or("arm absent")?, enabled, &golden)?; + } + verify_comparison(&arms, &golden)?; + Ok(()) + } + .await; + save( + &root, + "comparison.json", + &json!({"ok":result.is_ok(),"error":result.as_ref().err().map(ToString::to_string),"arms":arms, + "speedup_claim":false,"complete_memory_claim":false,"native_batching_claim":false}), + )?; + result + } +} + +#[cfg(test)] +mod oracle_tests { + use super::*; + + fn positive(enabled: bool) -> (Value, Value) { + let run = if enabled { + "run-enabled" + } else { + "run-baseline" + }; + let golden = + json!({"single_vectors":[[1.,0.,0.,0.,0.,0.,0.,0.],[0.,1.,0.,0.,0.,0.,0.,0.]]}); + let mut logs = Vec::new(); + let mut events = Vec::new(); + for (index, node) in NODES.iter().enumerate() { + let task = format!("task-{index}"); + let attempt = format!("attempt-{index}"); + let lease = 10 + index; + logs.push(json!({"sequence":index*10,"record":{"phase":"service_opportunity","workflow_run_id":run,"task_id":task,"ticket":index+1}})); + let native = json!({"phase":"native_drained","workflow_id":WORKFLOW,"workflow_run_id":run,"node_id":node,"task_id":task, + "attempt_id":attempt,"execution_request_id":format!("request-{index}"),"candidate_id":"runtime.candle","reservation_lease_id":lease, + "previous_owner":{"loaded_instance":vec![7;16],"loaded_profile":vec![8;32],"effective_settings":vec![9;32],"generation":2+index*2,"cpu_threads":1}, + "current_owner":{"loaded_instance":vec![7;16],"loaded_profile":vec![8;32],"effective_settings":vec![9;32],"generation":4+index*2,"cpu_threads":1}}); + logs.push(json!({"sequence":index*10+4,"record":native})); + let mut ack = native; + ack["phase"] = json!("task_registry_ack"); + ack["registry_task_release_acknowledged"] = json!(true); + ack["scheduler_cleanup_acknowledged"] = json!(false); + ack["lifecycle_event_id"] = json!(format!("lifecycle-{index}")); + logs.push(json!({"sequence":index*10+5,"record":ack})); + for (offset, transition) in ["started", "completed"].iter().enumerate() { + events.push(json!({"event_seq":index*2+offset+1,"workflow_run_id":run,"node_id":node,"payload":{ + "payload_type":"scheduler_task_attempt_lifecycle_changed","execution_class":"runtime","transition":transition, + "scheduler_task_id":task,"scheduler_attempt_id":attempt,"reservation_id":format!("runtime-registry.{lease}"), + "selected_runtime_id":"candle","selected_runtime_variant_id":"candle.cpu","selected_device_id":"cpu"}})); + } + } + logs.push(json!({"sequence":2,"record":{"phase":"proposal","workflow_run_id":run,"task_ids":["task-0","task-1"]}})); + if enabled { + logs.push(json!({"sequence":1,"record":{"phase":"warm_service_forecasts","coverage":"captured_compatible_prefix","scope":"opaque","forecasts":[{},{}]}})); + logs.push(json!({"sequence":30,"record":{"phase":"grouping_observations","scope":"opaque","phases":PHASES.iter().map(|phase|json!({"phase":phase,"aggregate":{"state":"available","calls":1,"elapsed_ns":0}})).collect::>()}})); + } + let arm = json!({"enabled":enabled,"configuration":{"key_reuse":enabled,"fair":true,"residency":true},"error":null,"collector_error":null, + "session_close":{"ok":true},"graph_closes_ok":true,"shutdown":{"ok":true,"timed_out":false},"final_registry":{"reservations":[],"runtimes":[{}]}, + "diagnostics":{"loads":1,"retains":1,"forward_drains":2,"stops":1,"has_idle_owner":false}, + "work":{"encodings":if enabled {4}else{6},"hits":if enabled {4}else{0},"retained":0,"error":null}, + "response":{"workflow_run_id":run,"outputs":OUTPUTS.iter().enumerate().map(|(index,node)|json!({"node_id":node,"port_id":"vector","value":golden["single_vectors"][index]})).collect::>()}, + "retained":OUTPUTS.iter().enumerate().map(|(index,node)|json!({"node_id":node,"workflow_run_id":run,"artifact_id":format!("artifact-{index}"), + "read":{"artifact_id":format!("artifact-{index}"),"complete":true,"byte_length":32},"body_bytes":32,"value":golden["single_vectors"][index]})).collect::>(), + "resume_run_ids":[run],"logs":logs,"events":events}); + (arm, golden) + } + + #[test] + fn hand_declared_pair_oracle_accepts_both_arms_and_zero_resolution_elapsed() { + let (baseline, golden) = positive(false); + let (enabled, _) = positive(true); + assert_eq!(verify_comparison(&[baseline, enabled], &golden), Ok(())); + } + + #[test] + fn mechanism_oracle_refuses_missing_hit_and_unavailable_or_nonempty_cache() { + let (arm, golden) = positive(true); + for work in [ + json!({"encodings":4,"hits":0,"retained":0,"error":null}), + json!({"error":"grouping cache unavailable"}), + json!({"encodings":4,"hits":4,"retained":1,"error":null}), + ] { + let mut changed = arm.clone(); + changed["work"] = work; + assert!(verify_arm(&changed, true, &golden).is_err()); + } + } + + #[test] + fn identity_oracle_refuses_foreign_attempt_lease_missing_start_and_reused_grant() { + let (arm, golden) = positive(true); + for field in ["attempt_id", "reservation_lease_id"] { + let mut changed = arm.clone(); + changed["logs"][2]["record"][field] = json!("foreign"); + assert!(verify_arm(&changed, true, &golden).is_err()); + } + let mut changed = arm.clone(); + changed["events"].as_array_mut().unwrap().remove(0); + assert!(verify_arm(&changed, true, &golden).is_err()); + let mut changed = arm; + changed["logs"][3]["record"]["ticket"] = json!(1); + assert!(verify_arm(&changed, true, &golden).is_err()); + } + + #[test] + fn runtime_candidate_can_repeat_but_coordinated_foreign_candidates_are_refused() { + let (arm, golden) = positive(true); + for native_index in [1, 4] { + assert_eq!( + arm["logs"][native_index]["record"]["candidate_id"], + "runtime.candle" + ); + } + assert_eq!(verify_arm(&arm, true, &golden), Ok(())); + let mut changed = arm; + for row in [1, 2] { + changed["logs"][row]["record"]["candidate_id"] = json!("runtime.foreign"); + } + assert_eq!( + verify_arm(&changed, true, &golden), + Err("native candidate does not identify the Candle runtime".into()) + ); + } + + #[test] + fn runtime_candidate_must_match_its_member_registry_ack() { + let (mut arm, golden) = positive(true); + arm["logs"][2]["record"]["candidate_id"] = json!("runtime.foreign"); + assert_eq!( + verify_arm(&arm, true, &golden), + Err("registry ACK identity differs".into()) + ); + } + + #[test] + fn repeating_an_attempt_custody_identity_still_refuses_before_member_joins() { + let (arm, golden) = positive(true); + for field in [ + "task_id", + "attempt_id", + "execution_request_id", + "reservation_lease_id", + ] { + let mut changed = arm.clone(); + let repeated = arm["logs"][1]["record"][field].clone(); + for row in [4, 5] { + changed["logs"][row]["record"][field] = repeated.clone(); + } + assert_eq!( + verify_arm(&changed, true, &golden), + Err("native identity reused".into()) + ); + } + } + + #[test] + fn phase_oracle_refuses_missing_unavailable_or_unobserved_spans_and_baseline_clocks() { + let (arm, golden) = positive(true); + for aggregate in [ + json!({"state":"unavailable","reason":"lock_unavailable"}), + json!({"state":"available","calls":0,"elapsed_ns":1}), + ] { + let mut changed = arm.clone(); + changed["logs"][8]["record"]["phases"][1]["aggregate"] = aggregate; + assert!(verify_arm(&changed, true, &golden).is_err()); + } + let mut changed = arm.clone(); + changed["logs"][8]["record"]["phases"] + .as_array_mut() + .unwrap() + .pop(); + assert!(verify_arm(&changed, true, &golden).is_err()); + let (mut baseline, _) = positive(false); + baseline["logs"] + .as_array_mut() + .unwrap() + .push(arm["logs"][8].clone()); + assert!(verify_arm(&baseline, false, &golden).is_err()); + } + + #[test] + fn cleanup_and_output_oracle_refuses_uncertain_shutdown_false_ack_and_bad_golden() { + let (arm, golden) = positive(true); + let mut changed = arm.clone(); + changed["shutdown"]["timed_out"] = json!(true); + assert!(verify_arm(&changed, true, &golden).is_err()); + let mut changed = arm.clone(); + changed["logs"][2]["record"]["registry_task_release_acknowledged"] = json!(false); + assert!(verify_arm(&changed, true, &golden).is_err()); + let mut changed = arm.clone(); + changed["response"]["outputs"][0]["value"][0] = json!(0.1); + assert!(verify_arm(&changed, true, &golden).is_err()); + let mut changed = arm; + changed["retained"][0]["workflow_run_id"] = json!("foreign"); + assert!(verify_arm(&changed, true, &golden).is_err()); + } + + #[test] + fn native_owner_oracle_refuses_missing_settings_wrong_threads_and_broken_lineage() { + let (arm, golden) = positive(true); + for owner in [Value::Null, json!({"cpu_threads":2})] { + let mut changed = arm.clone(); + changed["logs"][1]["record"]["previous_owner"] = owner; + assert!(verify_arm(&changed, true, &golden).is_err()); + } + for (field, value) in [ + ("generation", json!(99)), + ("cpu_threads", json!(2)), + ("effective_settings", json!(vec![10; 32])), + ] { + let mut changed = arm.clone(); + changed["logs"][1]["record"]["current_owner"][field] = value; + assert!(verify_arm(&changed, true, &golden).is_err()); + } + let mut changed = arm; + changed["logs"][4]["record"]["previous_owner"]["generation"] = json!(8); + changed["logs"][4]["record"]["current_owner"]["generation"] = json!(10); + assert!(verify_arm(&changed, true, &golden).is_err()); + } + + #[test] + fn retained_read_oracle_refuses_truncated_foreign_or_inconsistent_byte_reads() { + let (arm, golden) = positive(true); + for (field, value) in [ + ("complete", json!(false)), + ("artifact_id", json!("foreign")), + ("byte_length", json!(1)), + ] { + let mut changed = arm.clone(); + changed["retained"][0]["read"][field] = value; + assert!(verify_arm(&changed, true, &golden).is_err()); + } + } +} diff --git a/crates/pantograph-embedded-runtime/src/serial_cpu_residency_tests.rs b/crates/pantograph-embedded-runtime/src/serial_cpu_residency_tests.rs index a1cd3eeb7..c80dc356c 100644 --- a/crates/pantograph-embedded-runtime/src/serial_cpu_residency_tests.rs +++ b/crates/pantograph-embedded-runtime/src/serial_cpu_residency_tests.rs @@ -4,6 +4,9 @@ use pantograph_workflow_service::graph::{ }; use serde_json::json; +#[path = "serial_cpu_residency_key_reuse_qualification.rs"] +mod key_reuse_qualification; + const MODEL: &str = "embedding/qualification/synthetic-bert-8"; pub(crate) fn save_model_free_evidence(case: &str, observations: serde_json::Value) { diff --git a/crates/pantograph-workflow-service/src/workflow/serial_ready/configuration_tests.rs b/crates/pantograph-workflow-service/src/workflow/serial_ready/configuration_tests.rs index 61b584bd7..e83a9b1fa 100644 --- a/crates/pantograph-workflow-service/src/workflow/serial_ready/configuration_tests.rs +++ b/crates/pantograph-workflow-service/src/workflow/serial_ready/configuration_tests.rs @@ -77,6 +77,41 @@ impl SerialRuntimeHostBatchExecutionPort for UnqueriedPort { } struct UnqueriedCandidates; + +#[test] +fn fallible_grouping_observation_refuses_absent_mode_and_poisoned_cache() { + assert!(matches!( + WorkflowService::new().try_serial_grouping_work_for_test(), + Err(crate::workflow::WorkflowServiceError::InvalidRequest(_)) + )); + let service = WorkflowService::new_serial_ready_cpu( + Arc::new(UnqueriedPort), + WorkflowSerialReadyConfig::default() + .with_fair_service_opportunities() + .with_bounded_grouping() + .unwrap(), + ); + assert_eq!( + service.try_serial_grouping_work_for_test().unwrap(), + (0, 0, 0) + ); + let poisoned = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _guard = service + .serial_ready_mode + .as_ref() + .unwrap() + .grouping + .lock() + .unwrap(); + panic!("controlled test-support cache poison"); + })); + assert!(poisoned.is_err()); + assert!(matches!( + service.try_serial_grouping_work_for_test(), + Err(crate::workflow::WorkflowServiceError::Internal(_)) + )); +} + impl WorkflowRuntimeDispatchCandidateProvider for UnqueriedCandidates { fn runtime_dispatch_candidates( &self, diff --git a/crates/pantograph-workflow-service/src/workflow/serial_ready/grouping.rs b/crates/pantograph-workflow-service/src/workflow/serial_ready/grouping.rs index 372443418..bc5567891 100644 --- a/crates/pantograph-workflow-service/src/workflow/serial_ready/grouping.rs +++ b/crates/pantograph-workflow-service/src/workflow/serial_ready/grouping.rs @@ -298,6 +298,31 @@ impl SerialReadyMode { pub(super) mod tests; impl super::WorkflowService { + /// Fallible test-support observation of mechanism work and retained entries. + /// Absent serial mode or poisoned cache is unavailable, never measured zero. + /// These fixed counters confer no execution or cleanup authority. + #[cfg(any(test, feature = "test-support"))] + pub fn try_serial_grouping_work_for_test( + &self, + ) -> Result<(u64, u64, usize), super::WorkflowServiceError> { + use std::sync::atomic::Ordering; + let mode = self.serial_ready_mode.as_ref().ok_or_else(|| { + super::WorkflowServiceError::InvalidRequest( + "serial grouping observation requires serial mode".into(), + ) + })?; + let cache = mode.grouping.lock().map_err(|_| { + super::WorkflowServiceError::Internal( + "serial grouping observation cache unavailable".into(), + ) + })?; + Ok(( + mode.key_encodings.load(Ordering::Relaxed), + mode.key_cache_hits.load(Ordering::Relaxed), + cache.entries.len(), + )) + } + /// Test-only mechanism counts, including independent final-request hashing. /// These counters establish no end-to-end or real-model performance result. #[cfg(any(test, feature = "test-support"))] diff --git a/docs/plans/cpu-scheduler/execution-ledger.md b/docs/plans/cpu-scheduler/execution-ledger.md index 267edbfdf..aff5045be 100644 --- a/docs/plans/cpu-scheduler/execution-ledger.md +++ b/docs/plans/cpu-scheduler/execution-ledger.md @@ -1,6 +1,153 @@ # CPU Scheduler Execution Ledger + +## 2026-10-21 — Single v2 cohort passed narrow qualification + +- Exact staged six-path correction ffc8ddc4 passed independent commit/draft + review, committed/published normally as1f6af8a2/tree0368ee2e on PR91. Recovery + after capacity/environment interruption checked all preserved bytes, completed + build, process absence, provided actor and remote tips before any work retry. + No native launch was pending/uncertain; v2 root/marker were absent. +- Independent exact-manifest/binary review admitted one v2 run: manifest331e55de, + producerc41b0a57, protocol780f42c2 and supervisorcdc2efb9. All15 hashes, exact + discovery, cgroups and affinity matched. No local compilation overlapped. +- Root executed that held v2 test once at07:11:49Z; terminal exit0, reaped, no + timeout/error. Both fresh arms ran two native forwards. Baseline6enc/0hits and + enabled4enc/4hits both finished with zero healthy retained cache entries. + Public closes/shutdown acknowledged, final reservations empty/no idle owner. +- Independent raw review confirmed the frozen criteria, actual artifact bytes/ + golden/cross-arm equality, genuine Ready/captured pair, distinct per-member + custody/ACK joins, fresh loaded instances, one-thread owners4→6→8, all five + enabled final phase observations, and28 persisted rows within25965bytes. + Cold history remained unavailable; no trained ranking or timing win is claimed. +- The [successor v2 result](reports/2026-10-21-m5-native-key-reuse-result-v2.md) + and39-file durable archive retain exact originals plus preparation/admission/ + independent review. V1 source/protocol/archive/result/marker remain unchanged + and failed. No rerun or retrospective acceptance change occurred. +- M5-CPU now has independent narrow v2 evidence. M6-CLOSE awaits final source/ + report/archive review and exact-candidate CI/normal integration/post-merge + gates. Existing source/branches/worktrees and PR84 stay protected. No admission, + default, allocator, security, dependency/lockfile or CI change is selected. + +## 2026-10-21 — Separately authorized prospective v2 correction + +- Fresh user delegation explicitly authorizes a corrected oracle, independent + full source/protocol review and one bounded new baseline/enabled cohort. V1's + failed source, protocol, archive and marker remain unchanged; neither its + baseline nor verdict is reused. Parent's second disconnect report was followed + by successful pwd/Git/metadata recovery, clean source AEA, unchanged v1 archive + e6edef6c/result77a24b38 and provided actor MrScripty. Remote main ecfed47d and + PR91 source AEA match. No write or native launch was pending/uncertain. +- Independent pre-code review admitted removing only candidate_id from across- + member uniqueness, adding exact runtime.candle per-native validation, and + retaining exact within-member candidate ACK tuples and ledger selected-runtime + joins. All task/attempt/execution-request/reservation and grant-ticket custody + distinctions remain. New positive/negative oracles must decide this contract + directly, including coordinated foreign candidates and intended failure cause. +- Existing branch/worktree test/cpu-scheduler-native-key-reuse is retained and + remains the sole publication owner. Root owns canonical docs, new prospective + v2 document/supervisor, compilation, commits/publication, freeze and one run. + review_m3_merge owns only the existing held producer file; review_c2a_m6 owns + independent read-only review. No helper/default/admission/allocator/manifest/ + lockfile/CI or unrelated change is selected. V1 evidence paths are excluded + from this write set. V2 outcome report/archive may be added after the run. +- [V2](reports/2026-10-21-m5-native-key-reuse-protocol-v2.md) retains unchanged + baseline6/0 versus enabled4/4 mechanism targets, vector/genuine Ready/captured + pair/five phase/owner/Start/ACK/cleanup requirements and all execution/log bounds. + Exact cross-arm output equality is stated explicitly to match existing source. + Conservative VRAM claims are not zero reservation or physical GPU evidence. + No v2 physical execution has occurred; full source/protocol/supervisor review, + model-free checks, exact source/binary/fixtures freeze and manifest admission + still gate the single new run. Publication stays draft while unqualified. +- Correction checks passed: ten inventory-enabled model-free oracle cases + (held v2 compiled, never executed), full default embedded library617 with two + existing ignored probes, strict all-feature/all-target own-package Clippy, + warning-deny own-package Rustdoc, formatting and declared pre-push261/one + existing ignore+168. Workflow-service source/helper is byte-unchanged from + reviewed AEA and its previous1048 passes; no unrelated suite is relabeled as + new v2 evidence. Source/protocol/supervisor review approved freezing after + prerequisites. The apparent missing cgroup/affinity check review finding was + withdrawn after confirming the existing pre-marker assertions in exact bytes; + no unnecessary supervisor behavior change was introduced. + +## 2026-10-21 — Single native cohort terminal failure and finite stop + +- PR90 actual main ecfed47d completed all20 checks/four workflows successfully, + preserving tree540618b7, twelve prior branch tips and eight prior clean + worktrees. Integration receipt SHA2569d7e20d34d5a1a55811134dd35058f3795f0f966a0fdf49b0c05b95b84a38daf. +- Independent review approved the exact nine-path staged source for commit/draft, + then exact frozen manifest/binary for one run after that actual-main gate. + Source fb7ba0e3/tree203f7b4a and manifest1aecf82f were unchanged. Root alone ran + the reviewed supervisor once, with no concurrent local compilation. +- Terminal result FAILED, exit101, reaped, no timeout: baseline completed two + forwards,6encodings/0hits/0retained and acknowledged closes/shutdown; frozen + candidate-ID distinctness oracle failed on runtime.candle. Enabled never ran. + Both source-author and independent source/raw review confirmed candidate_id + is stable runtime-alternative identity, not unique member custody. The + pre-run source/fixture review missed that distinction. Do not reinterpret + the old oracle, erase its failure or continue to an enabled-only run. +- The [successor result report](reports/2026-10-21-m5-native-key-reuse-result.md) + records exact provenance, actual baseline facts, limits and durable archived + originals. Distinct actual custody identities, joins, owners4→6→8, numerical + outputs/readbacks and cleanup checked out independently. Conservative VRAM + reservation claims are present; no physical allocation/GPU claim follows. +- Finite stop: INT-89/M5-SW/M6-SW complete within existing limits; M5-CPU failed + v1, M6-CLOSE unaccepted. Publish unchanged producer/oracle/protocol plus + independently reviewed failure disposition as draft. Do not merge a failed + qualification or select another allocator/diagnostic domain. Any evidenced + oracle correction/new physical cohort requires separate prospective replanning. + Missing representative workloads/hardware did not prevent this software work. +- Original once-only evidence root, exclusive marker and all branches/worktrees + remain retained. Historical denied-log evidence and baseline strict inference + lint/Rustdoc limitations remain unretried and unsuppressed. + +## 2026-10-21 — PR90 integration and fixed native comparison admission + +- User explicitly authorized guarded normal merge at c45d103f and the finite + native comparison producer, independent review, prospective freeze and one + retained execution. Fresh remote/actor/exact tree540618b7, successful22checks/ + four PR workflows and empty review/thread arrays were verified. Independent + source reviewer extended approval to merge. Ready transition preserved all + checks; normal merge succeeded as ecfed47d, parentsdc64b895/c45d103f and identical + tree. Actual-main CI is pending independently; all earlier resources retained. +- Producer write set: new serial_cpu_residency_key_reuse_qualification.rs under + existing test module plus its registration; test-support fallible scalar read + in serial_ready/grouping.rs and focused configuration_tests.rs; this canonical + plan/ledger/issues/checklist and the prospective protocol/result report. No + production policy/default/native allocator or dependency/manifest/lock change. +- Two independent pre-code reviews admitted fixed two-arm/two-Ready/four-forward + BERT-8 comparison and strict source-derived counts (baseline6/0, enabled4/4). + Each public member keeps its own grant/Start/drain/registryACK. No native wrapper, + injected readiness/history or redefined scheduler ACK. Resource, termination, + golden/input, evidence availability and no-winner-retry conditions are fixed in + the [protocol](reports/2026-10-21-m5-native-key-reuse-protocol.md). +- Root owns grouping helper/tests, module registration, canonical docs, freeze/ + invocation and serial integration. review_m3_merge owns only the new producer + file. review_c2a_m6 independently reviews source/oracles/protocol/evidence and + never executes the held run. New branchtest/cpu-scheduler-native-key-reuse and + /workspace/pantograph-native-key-reuse are retained-protected for parent review, + target main ecfed47d; no cleanup is authorized. +- The legacy test getter maps poisoned cache to0, so it cannot prove emptiness. + A new fallible companion retains old callers and rejects absent mode/poison. + Its controlled poison/healthy/absent regression passed, with no native work. + Full workflow library1048 and default embedded library614 passed; the latter + retains two existing ignored cost probes. Seven model-free oracle cases passed + with inventory enabled, compiling but never executing the held native body. + All-feature/all-target own-package warning-deny Clippy and warning-deny + own-package Rustdoc passed, as did formatting, staged traceability/local links + and declared pre-push261/one existing ignore+168. Independent source/protocol/ + supervisor review approved freezing; exact manifest/binary review still gates + execution, alongside actual-main CI. No native comparison has executed. +- Retained preparation failures: first default library run569passed/45setup + failures because the supported scratch Pumas metadata override was omitted; + the corrected process-local override run passed614. No HOME/security change. + Initial traceability lacked explicit staged/range selection; corrected staged + gate passed nine paths. First strict Clippy found a redundant Ok/try expression + in the new held helper; a semantic-preserving fix was independently re-reviewed + before the corrected strict gate passed. Initial author misprofile compile + stopped130 before any tests. No failure is erased or relabeled native evidence. + ## 2026-10-21 — PR89 integration and finite M5/M6 consuming path - User authorized normal PR89 integration and required a complete remaining-plan diff --git a/docs/plans/cpu-scheduler/issues.md b/docs/plans/cpu-scheduler/issues.md index a2212badb..aed3ff2e2 100644 --- a/docs/plans/cpu-scheduler/issues.md +++ b/docs/plans/cpu-scheduler/issues.md @@ -47,6 +47,15 @@ are bounded implementation corrections, not default-policy or recovery expansion | CPU-34 | The Ready boundary lacked a provenance-aware historical warm-service producer and inclusive measurements covering capture/cloning, bounds/equality and key work. Encoding counts alone establish no total-overhead result. | The bounded producer merged normally as PR88/dc926783 with exact approved tree88aaf720 after independent review, 22 successful check runs and four successful PR workflows. Local1,047-case evidence and the hosted84-case selection remain distinct. Admission and key-work counts stay unchanged. See [contract/evidence](reports/2026-10-21-provenance-producer.md). | | CPU-35 | Pinned Candle has no inspected complete CPU allocation-lifetime observer: tensor IDs alias storage, while BERT temporaries, GEMM scratch and Rayon attribution escape returned-buffer snapshots. | This is missing software instrumentation, not a hardware-only blocker. The reviewed completed-result foundation covers only original immutable output Vec payload ownership after materialization, with explicit bookkeeping windows. No placeholder memory API or reservation/RSS arithmetic is added. Dependency allocation/drop, shared-storage, scratch and worker attribution remains required for a future complete incremental-memory claim or admission use. It is not required for the cost-free C2a path selected by the finite completion checklist; no further allocator slice is next. Representative model/hardware evidence qualifies future claims later. | -| CPU-36 | Original completed embedding-result payload can remain allocated in an unpolled join or surviving Shared/Arc reader after worker exit/drain; wrapper completion is not final storage retirement. | PR89 merged normally as dc64b895 after independent merge approval and 22 successful source check runs/four PR workflows; post-merge CI is pending. The reviewed safe-Rust foundation attaches one lease inside the blocking worker to the immutable completed result and releases after actual payload drop. Four fixed job-owner counters measure only observed retained Vec capacity. Complete allocator/attempt memory remains unavailable; existing admission and cleanup authority stay unchanged. See [contract](reports/2026-10-21-result-payload-attribution.md). | +| CPU-36 | Original completed embedding-result payload can remain allocated in an unpolled join or surviving Shared/Arc reader after worker exit/drain; wrapper completion is not final storage retirement. | PR89 merged normally as dc64b895 after independent merge approval and 22 successful source check runs/four PR workflows; all20 post-merge checks/four workflows passed. The reviewed safe-Rust foundation attaches one lease inside the blocking worker to the immutable completed result and releases after actual payload drop. Four fixed job-owner counters measure only observed retained Vec capacity. Complete allocator/attempt memory remains unavailable; existing admission and cleanup authority stay unchanged. See [contract](reports/2026-10-21-result-payload-attribution.md). | -| CPU-37 | The protected `EmbeddedSerialCpuResidency` consumer hardcodes fair+residency with key reuse off and exposes no immutable configuration query, leaving the existing C2a opt-in inaccessible through that safe owner. | Add one explicit key-reuse constructor and delegate the existing value query, preserving defaults, config literals, private owners and shutdown. The [finite checklist](reports/2026-10-21-m5-m6-completion-checklist.md) owns remaining consumer and actual CPU gates; complete allocator diagnostics are not a substitute. | +| CPU-37 | The protected `EmbeddedSerialCpuResidency` consumer hardcodes fair+residency with key reuse off and exposes no immutable configuration query, leaving the existing C2a opt-in inaccessible through that safe owner. | PR90 implemented one explicit key-reuse constructor and direct existing value query, preserving defaults, config literals, private owners and shutdown. Normal merge ecfed47d and all20 actual-main checks/four workflows passed. The [finite checklist](reports/2026-10-21-m5-m6-completion-checklist.md) owns remaining consumer and actual CPU gates; complete allocator diagnostics are not a substitute. | + +| CPU-38 | Existing test-support cache length maps poison/absent mode to0, so a qualification oracle could confuse unavailable evidence with an empty healthy cache. | Add a fallible test-only scalar companion, preserve legacy callers, and require explicit failure for absent/poisoned evidence before interpreting zero. Controlled regression passes; no production accounting/admission change. | +| CPU-39 | Actual native baseline-versus-key-reuse evidence remains missing despite controlled-worker and constructor coverage. | User authorized the [prospectively fixed protocol](reports/2026-10-21-m5-native-key-reuse-protocol.md), source/oracle review and one frozen retained execution. Do not run before prerequisites or choose criteria after outcomes. Counts/CPU outputs qualify only the disclosed two-member untrained-fixture envelope; representative model/hardware performance remains separate. | + +| CPU-40 | The frozen qualification oracle confuses stable runtime-alternative candidate identity with distinct member custody identity. | Actual provider correctly uses runtime.candle for both members. One retained v1 cohort failed after the baseline; enabled never started. Independent source/raw review confirms the software-oracle defect. Preserve the frozen source/criteria and [failure evidence](reports/2026-10-21-m5-native-key-reuse-result.md); no rerun or retrospective acceptance. Any correction/new cohort requires separate prospective replanning. This is not a missing external hardware/model input or a production authority-reuse finding. | + +| CPU-41 | A reviewed successor qualification must distinguish stable provider identity from unique custody without dropping candidate receipt matching or changing other acceptance after outcomes. | User separately authorized prospective v2. Require every native candidate runtime.candle plus exact within-member ACK/ledger joins, and all distinct custody IDs/tickets. New root/protocol/test identity and one new fresh baseline/enabled execution; preserve original failed v1 and all other criteria. [V2 protocol](reports/2026-10-21-m5-native-key-reuse-protocol-v2.md) owns prerequisites and stop conditions. No execution yet. | + +| CPU-42 | Corrected native qualification needs actual retained evidence and independent integration gates before finite closure. | One separately frozen v2 cohort passed the [narrow protocol/result](reports/2026-10-21-m5-native-key-reuse-result-v2.md), including baseline6/0 versus enabled4/4 and acknowledged cleanup. Independent raw review confirms criteria; original failed v1 remains unchanged. Final report/archive/source review and exact-candidate CI/integration remain pending. No real-model/hardware or complete-memory/admission claim is inferred. | diff --git a/docs/plans/cpu-scheduler/plan.md b/docs/plans/cpu-scheduler/plan.md index 9dd860f4d..d8914c173 100644 --- a/docs/plans/cpu-scheduler/plan.md +++ b/docs/plans/cpu-scheduler/plan.md @@ -2,9 +2,9 @@ **Plan status:** `Active` — M1/M2/M3, bounded M4, M5 C1/C2a, M6a and the bounded warm-service/grouping diagnostic producer are accepted and merged. Overall M5, broader M6 and real-model qualification remain unaccepted. -**Current phase:** PR89 merged normally as `dc64b895180260ceef8619bedd1ac6d8b756a3ec`, tree `2d59d369f11b89e119c54afce9f0a6df86ada4bd`, preserving main `dc926783` and source `48717570` as parents. Independent exact-source review approved merging; all 22 source check runs and four PR workflows passed. Post-merge CI is pending. CodeRabbit status supplies no independent source-review credit. PR88 and prior integrations remain accepted within their reported limits; all earlier branches, PR84 lineage and unrelated work remain protected. +**Current phase:** PR90 merged normally as `ecfed47da7c51d03b968bed739e2abc06eb4a6de`, exact reviewed tree `540618b75e7ebbde9a5713c8ac14f05e77936f80`, preserving main `dc64b895` and source `c45d103f` as parents. Independent source approval extended to merge and all 22 source checks/four PR workflows passed. All20 actual-main post-merge checks and all four workflows passed. PR89 main `dc64b895` completed all20 post-merge checks/four workflows. CodeRabbit supplies no independent source-review credit. Prior branches, PR84 lineage and unrelated work remain retained-protected. -**Next slice:** Verify and publish the implemented ordinary-member key-reuse consuming path through the protected residency facade, following the [finite M5/M6 checklist](reports/2026-10-21-m5-m6-completion-checklist.md). The additive constructor choice reuses the immutable configuration snapshot; preserve the existing constructor, admission and lifecycle. The integrator selects C2's safe ordinary-member planning-reuse option as this supported envelope. Actual CPU comparison and final consumer qualification remain required before accepting that envelope. Complete allocator instrumentation is separate future work for a complete-memory claim, not a prerequisite for cost-free key reuse. Do not add another partial instrumentation domain instead of closing the five finite checklist rows. +**Current finite task:** The separately authorized [v2 cohort](reports/2026-10-21-m5-native-key-reuse-result-v2.md), frozen at source1f6af8a2/tree0368ee2e, passed the unchanged narrow mechanism/output/ownership/phase/cleanup criteria after correcting provider identity. Independent raw review confirmed actual artifact bytes, distinct custody, genuine two-Ready/captured pair, fresh owners, baseline6/0 versus enabled4/4 with empty healthy caches, and acknowledged teardown. [V1](reports/2026-10-21-m5-native-key-reuse-result.md) remains failed and unchanged. M5-CPU has scoped v2 evidence; M6-CLOSE awaits final source/report/archive review, exact-candidate CI and normal integration/post-merge checks under the [finite checklist](reports/2026-10-21-m5-m6-completion-checklist.md). Do not add another instrumentation/allocator slice or extend admission/defaults. No representative performance, complete-memory, trained-ranking or broader model/hardware qualification follows. **Acceptance status:** M1 merged as PR73 in main `07b23d47`; the original negative cohort and the separately accepted C3 supplement remain unchanged. diff --git a/docs/plans/cpu-scheduler/reports/2026-10-21-m5-m6-completion-checklist.md b/docs/plans/cpu-scheduler/reports/2026-10-21-m5-m6-completion-checklist.md index ddd00cd0c..8b07c50b5 100644 --- a/docs/plans/cpu-scheduler/reports/2026-10-21-m5-m6-completion-checklist.md +++ b/docs/plans/cpu-scheduler/reports/2026-10-21-m5-m6-completion-checklist.md @@ -26,11 +26,11 @@ advisory grouping proposal as executing a group. | ID | Required outcome and deciding evidence | Current disposition / stop condition | | --- | --- | --- | -| INT-89 | PR89 exact head `48717570004be87735c4b693212fdadf154d8738`, tree `2d59d369f11b89e119c54afce9f0a6df86ada4bd`, independently approved for merge; normal merge preserves both parents; actual merge CI terminal. | Source had 22 successful checks/four PR workflows. Merged as `dc64b895180260ceef8619bedd1ac6d8b756a3ec`, parents `dc926783` and `48717570`, identical tree. Post-merge CI is pending; the integration receipt owns final run identities. Stop integration on an actual failed required check and diagnose permitted fresh evidence. | +| INT-89 | PR89 exact head `48717570004be87735c4b693212fdadf154d8738`, tree `2d59d369f11b89e119c54afce9f0a6df86ada4bd`, independently approved for merge; normal merge preserves both parents; actual merge CI terminal. | Source had 22 successful checks/four PR workflows. Merged as `dc64b895180260ceef8619bedd1ac6d8b756a3ec`, parents `dc926783` and `48717570`, identical tree. All20 post-merge checks/four workflows passed at dc64b895; the integration receipt and merged PR89 own final run identities. Stop integration on an actual failed required check and diagnose permitted fresh evidence. | | M5-SW | Bounded C1 library API and C2a ordinary-worker key reuse preserve fresh per-member authority, fallback, bounds and cleanup. | Implemented/merged PR85/86; independently reviewed focused and controlled-worker evidence exists. Do not add another scheduler, forecast store or allocator observer to close this row. | -| M6-SW | One supported owning host can explicitly choose that existing key-reuse mode and read its immutable configuration without exposing private authority. Existing constructor remains unchanged; shutdown/reconstruction is rollback. | Concrete missing production capability after PR89: `EmbeddedSerialCpuResidency::new` fixes fair+residency with reuse off and offers no choice/query. The reviewed implementation adds the explicit constructor and delegates the existing query. Default/inventory consumer tests pass; publication and required exact-head CI remain to decide this row. | -| M5-CPU | One prospectively fixed actual CPU baseline-versus-key-reuse comparison through public sessions verifies exact outputs, member/attempt identities, separate grants/Starts and matched cleanup, with fresh owner/proof checks. Capture actual cache behavior and all five existing phase spans; report elapsed observations descriptively. | Pending software qualification producer/protocol admission, then one retained execution using committed untrained BERT fixtures. Existing controlled 30-versus-16 encoding counts and isolated local times are not native acceptance. Missing instrumentation in this producer is software work, not an external-input blocker. Stop on the fixed protocol's terminal outcome, any missing criterion or infrastructure failure; preserve all outcomes and do not tune/retry for a winner. | -| M6-CLOSE | Independent exact source/consumer/evidence review, required exact-head CI and post-merge checks support the bounded host choice and operator instructions. | Pending M6-SW and M5-CPU. Close only the named ordinary-member envelope; retain other qualifications below. No UI, IPC, persistence, deployment or default activation is implied. | +| M6-SW | One supported owning host can explicitly choose that existing key-reuse mode and read its immutable configuration without exposing private authority. Existing constructor remains unchanged; shutdown/reconstruction is rollback. | Concrete missing production capability after PR89: `EmbeddedSerialCpuResidency::new` fixes fair+residency with reuse off and offers no choice/query. The reviewed implementation adds the explicit constructor and delegates the existing query. PR90 source c45d103f passed all22 checks/four workflows and both new hosted cases (607 library passes/two existing ignores), then merged normally as ecfed47d with identical approved tree. All20 actual-main post-merge checks and all four workflows passed at ecfed47d. This software-consumer row is complete within its existing evidence limits. | +| M5-CPU | One prospectively fixed actual CPU baseline-versus-key-reuse comparison through public sessions verifies exact outputs, member/attempt identities, separate grants/Starts and matched cleanup, with fresh owner/proof checks. Capture actual cache behavior and all five existing phase spans; report elapsed observations descriptively. | **Scoped v2 evidence passed**, independently reviewed against the [unchanged prospective criteria](2026-10-21-m5-native-key-reuse-protocol-v2.md): fresh baseline6encodings/0hits versus enabled4encodings/4hits, zero retained entries, four native forwards, exact outputs/readbacks, genuine two-Ready/captured pair, owners/joins/five phases and acknowledged cleanup. The [v2 result/archive](2026-10-21-m5-native-key-reuse-result-v2.md) owns exact provenance and limits. V1 remains **FAILED**, with its original [result/evidence](2026-10-21-m5-native-key-reuse-result.md) unchanged; no retrospective acceptance. V2 was separately authorized/frozen and executed once after correction of provider identity, preserving distinct member custody. No model/hardware performance claim or trained ranking follows. | +| M6-CLOSE | Independent exact source/consumer/evidence review, required exact-head CI and post-merge checks support the bounded host choice and operator instructions. | **Verifying integration:** M6-SW is complete and M5-CPU v2 passed independent raw review. Final successor report/archive/source review, required exact-candidate CI and normal integration/post-merge checks remain. Close only the named ordinary-member envelope after those gates pass. V1 stays failed; broader qualifications below are not accepted by this result. No UI, IPC, persistence, deployment or default activation is implied. | These five rows are the completion boundary. After they pass, stop implementation of this envelope and mark only these scoped claims accepted. If a row fails, @@ -68,10 +68,10 @@ suitable devices/hosts already excluded by the plan. existing authority. Optional telemetry expansion must name a decision it changes before it delays this finite checklist. -## Next software slice and review boundary +## Accepted consumer slice and next qualification boundary -Write set: `serial_cpu_residency.rs`, its existing tests, the host-options guide, -this checklist and canonical CPU plan/ledger/issues. Add +The accepted consumer write set was `serial_cpu_residency.rs`, its existing tests, the host-options guide, +this checklist and canonical CPU plan/ledger/issues. It added `EmbeddedSerialCpuResidency::new_with_bounded_key_reuse(config)` and `scheduler_configuration()` returning the existing value snapshot. Reuse one private constructor implementation and existing builders; preserve config struct diff --git a/docs/plans/cpu-scheduler/reports/2026-10-21-m5-native-key-reuse-protocol-v2.md b/docs/plans/cpu-scheduler/reports/2026-10-21-m5-native-key-reuse-protocol-v2.md new file mode 100644 index 000000000..61510e0ea --- /dev/null +++ b/docs/plans/cpu-scheduler/reports/2026-10-21-m5-native-key-reuse-protocol-v2.md @@ -0,0 +1,114 @@ +# Fixed native CPU key-reuse comparison v2: prospective contract + +Authority: continue docs/plans/cpu-scheduler/plan.md and finite checklist M5-CPU. +The user separately authorized correcting the reviewed candidate-identity oracle, +independent review, a prospective v2 freeze and one bounded new execution. +[The original v1 cohort](2026-10-21-m5-native-key-reuse-result.md) remains failed; +its source fb7ba0e3, protocol, archive, marker and outcomes are immutable. V2 uses +newly constructed baseline and enabled arms; no v1 baseline is reused. This +prospective document is not an execution receipt. + +One baseline arm then one enabled arm. Each constructs a fresh protected +EmbeddedSerialCpuResidency using the existing baseline or explicit key-reuse +constructor, authors one two-branch graph, opens one authenticated attributed +session and executes exactly one run. Both arms use the committed untrained +BERT-8 fixture and golden texts[0]="hello world", texts[1]="hello", paired with +single_vectors[0] and [1]. Exactly two forwards per arm/four overall; minimum +three history samples prevents trained ranking. No seeded timing/Ready/proofs, +downloaded model, external backend or fake native delegate is admitted. + +Each arm must observe a genuine two-Ready proposal. Enabled capture must contain +two compatible members. Mechanism oracle fixed before execution: baseline six +key encodings/zero hits, enabled four encodings/four hits, both healthy retained +entries zero after completed scope. These are the existing source-derived +n=2 controlled-worker formulas, not elapsed-time or total CPU improvement claims. +Poison/absent mode is unavailable and fails. A changed snapshot that defeats this +specific target is a retained failure, not permission to reinterpret the target. + +Both returned and retained vectors must match their corresponding frozen golden +within fixed absolute tolerance <=1e-5 and identical dimensions. Corresponding +baseline/enabled returned vectors must also be exactly equal, as required by +the unchanged comparison oracle. Require exactly +two distinct member grants/Starts/drains/registry receipt ACKs with bijective +run/task/attempt/lease joins and observed native previous/current owner identity, +settings, one-thread and generation continuity checks; no injected capability. +The reviewed correction distinguishes provider identity from member custody: +production runtime_candidate_draft constructs candidate_id as runtime.. +Each native drain must therefore have candidate_id exactly runtime.candle, and +that field must match its own registry ACK within the full unchanged identity +tuple. Ledger selected runtime candle / variant candle.cpu / device cpu checks +remain required. Candidate IDs may legitimately be shared across members; no +cross-member uniqueness is required for that provider field. Task, attempt, +execution-request, reservation-lease and grant-ticket identities must still be +distinct. Negative oracles must reject coordinated foreign candidates, mismatched +ACK candidates and each repeated custody field for the intended invariant. +Registry ACK is not relabeled +scheduler ACK. Scheduler matching remains explicitly source-assisted through the +existing adapter and successful public completion. Full raw private readiness +proof/owner lineage is not independently observed. Independent source and focused +negative tests retain the final owner/Start/cleanup fence proof. + +Enabled diagnostics must contain available positive-call observations for all +five existing grouping phases and the actual captured pair. Nested spans overlap +and must not be summed; logging/observer overhead remains uncalibrated. Baseline +grouping instrumentation is disabled and reported absent, never measured zero. +No duration threshold or winner is chosen. Counts and correctness decide only +the disclosed two-member synthetic mechanism/ownership envelope. + +Resource/termination bounds: one configured CPU thread, existing explicit +operator declarations (2 GiB RAM budget, 1 GiB VRAM budget, 32 MiB +resident and 600 MiB load envelope), eight graph nodes/six edges per arm, two short +fixed texts, four native forwards total. Declarations are not allocation +measurements. CPU dispatch may still carry conservative nonzero VRAM +reservation claims under existing policy; neither those claims nor declarations +are physical RAM/GPU allocation measurements. Record actual cgroup/affinity and fixture/binary sizes. Logger +capacity2048rows/4MiB; raw structured rows persist incrementally with sync so a +watchdog retains the observed prefix. Parse/overflow/I/O failure vetoes success. Each arm run plus +readiness resume waits at most60seconds, public shutdown at most30seconds. Session/graph closes share a five-second +absolute budget before the independent shutdown attempt. +A180second outer own-process execution watchdog is a hard global ceiling, not a guarantee +that both arms plus setup/teardown fit. At the deadline the supervisor kills only its own new process group and +attempts reaping for at most five more seconds. Watchdog failure/unknown shutdown +cannot be labeled cleanup success. Attempt public shutdown after every post-construction +body/oracle/I/O error; never start the next arm before acknowledged first teardown. + +All actual actions use public facade graph/session/run/resume/readback/artifact/ +close/shutdown methods. Private access is read-only scalar test-support evidence. +Pumas metadata uses one fresh process-local registry override, shared across arms; +each owns distinct local project/library/appdata and native RuntimeRegistry. The child has an8GiB virtual-address-space ceiling and the current observed +cgroup ceilings remain in force; this execution limit is not a footprint +measurement. No HOME, authentication, permission, sandbox or admission setting +changes. + +Before execution: PR90 actual-main post-merge checks/workflows successful; +independent source/oracle/protocol review; ordinary model-free +oracles including shared candidate identity, foreign/mismatched candidates, +repeated custody identities, missing hits/phases/outputs/cleanup and poisoned +counter negatives; exact source commit/tree and clean checkout; compile without +running the held test; copy/hash the exact producer binary; hash every fixture +and protocol; record toolchain, settings, environment allowlist, limits and input +oracle into a create-new manifest/evidence root. Review actual binary discovery +with --list (no native execution). No code/build changes or concurrent local +compilation during the held run. Execute exactly the discovered held test +serial_cpu_residency::tests::key_reuse_qualification::held::fixed_two_ready_native_key_reuse_comparison_v2. +Use protocol ID m5-native-two-ready-key-reuse-v2 and the create-new evidence root +/workspace/scratch/pantograph-recovery/native-key-reuse-cohort-v2. Require the +static metadata override at that root/metadata.db. V1 paths and marker must never +be reused, overwritten or removed. The frozen +supervisor uses an exclusive execution-started marker, no automatic retry, +immutable hashes and the same cgroup/affinity at execution. Launch only the exact ignored v2 held test once, after independent review of +the full source/protocol/supervisor change and exact frozen manifest/binary. +Publish the correction through the existing isolated PR91 branch with normal +commits; keep it draft while qualification is incomplete. Hosted CI runs on +separate runners may overlap; no local compilation may overlap the held run. + +Retain progress, raw rows, body error, teardown outcome and oracle outcome even +when failure occurs. Stop on the single terminal outcome, missing/unknown facts, +overflow, infrastructure failure or uncertain teardown. No rerun, post-outcome +input tuning, population growth, continued failed owners or retrospective +acceptance change. A genuine evidenced implementation defect may be repaired +separately only after re-planning; original cohort remains failed. + +After execution independently review raw evidence against these frozen criteria. +Do not close overall M5/M6, performance, complete memory, native batching, +representative model quality or hardware claims from this narrow comparison. diff --git a/docs/plans/cpu-scheduler/reports/2026-10-21-m5-native-key-reuse-protocol.md b/docs/plans/cpu-scheduler/reports/2026-10-21-m5-native-key-reuse-protocol.md new file mode 100644 index 000000000..6e7575261 --- /dev/null +++ b/docs/plans/cpu-scheduler/reports/2026-10-21-m5-native-key-reuse-protocol.md @@ -0,0 +1,87 @@ +# Fixed native CPU key-reuse comparison: prospective contract + +Authority: continue docs/plans/cpu-scheduler/plan.md and finite checklist M5-CPU. +The user explicitly authorized implementation, independent review, freezing and +one bounded retained execution. This draft is not an execution receipt. + +One baseline arm then one enabled arm. Each constructs a fresh protected +EmbeddedSerialCpuResidency using the existing baseline or explicit key-reuse +constructor, authors one two-branch graph, opens one authenticated attributed +session and executes exactly one run. Both arms use the committed untrained +BERT-8 fixture and golden texts[0]="hello world", texts[1]="hello", paired with +single_vectors[0] and [1]. Exactly two forwards per arm/four overall; minimum +three history samples prevents trained ranking. No seeded timing/Ready/proofs, +downloaded model, external backend or fake native delegate is admitted. + +Each arm must observe a genuine two-Ready proposal. Enabled capture must contain +two compatible members. Mechanism oracle fixed before execution: baseline six +key encodings/zero hits, enabled four encodings/four hits, both healthy retained +entries zero after completed scope. These are the existing source-derived +n=2 controlled-worker formulas, not elapsed-time or total CPU improvement claims. +Poison/absent mode is unavailable and fails. A changed snapshot that defeats this +specific target is a retained failure, not permission to reinterpret the target. + +Both returned and retained vectors must match their corresponding frozen golden +within fixed absolute tolerance <=1e-5 and identical dimensions. Require exactly +two distinct member grants/Starts/drains/registry receipt ACKs with bijective +run/task/attempt/lease joins and observed native previous/current owner identity, +settings, one-thread and generation continuity checks; no injected capability. Registry ACK is not relabeled +scheduler ACK. Scheduler matching remains explicitly source-assisted through the +existing adapter and successful public completion. Full raw private readiness +proof/owner lineage is not independently observed. Independent source and focused +negative tests retain the final owner/Start/cleanup fence proof. + +Enabled diagnostics must contain available positive-call observations for all +five existing grouping phases and the actual captured pair. Nested spans overlap +and must not be summed; logging/observer overhead remains uncalibrated. Baseline +grouping instrumentation is disabled and reported absent, never measured zero. +No duration threshold or winner is chosen. Counts and correctness decide only +the disclosed two-member synthetic mechanism/ownership envelope. + +Resource/termination bounds: one configured CPU thread, existing explicit +operator declarations (2GiB RAM budget,1GiB VRAM budget unused by CPU,32MiB +resident and600MiB load envelope), eight graph nodes/six edges per arm, two short +fixed texts, four native forwards total. Declarations are not allocation +measurements. Record actual cgroup/affinity and fixture/binary sizes. Logger +capacity2048rows/4MiB; raw structured rows persist incrementally with sync so a +watchdog retains the observed prefix. Parse/overflow/I/O failure vetoes success. Each arm run plus +readiness resume waits at most60seconds, public shutdown at most30seconds. Session/graph closes share a five-second +absolute budget before the independent shutdown attempt. +A180second outer own-process execution watchdog is a hard global ceiling, not a guarantee +that both arms plus setup/teardown fit. At the deadline the supervisor kills only its own new process group and +attempts reaping for at most five more seconds. Watchdog failure/unknown shutdown +cannot be labeled cleanup success. Attempt public shutdown after every post-construction +body/oracle/I/O error; never start the next arm before acknowledged first teardown. + +All actual actions use public facade graph/session/run/resume/readback/artifact/ +close/shutdown methods. Private access is read-only scalar test-support evidence. +Pumas metadata uses one fresh process-local registry override, shared across arms; +each owns distinct local project/library/appdata and native RuntimeRegistry. The child has an8GiB virtual-address-space ceiling and the current observed +cgroup ceilings remain in force; this execution limit is not a footprint +measurement. No HOME, authentication, permission, sandbox or admission setting +changes. + +Before execution: PR90 actual-main post-merge checks/workflows successful; +independent source/oracle/protocol review; ordinary model-free +oracles including missing identities/hits/phases/outputs/cleanup and poisoned +counter negatives; exact source commit/tree and clean checkout; compile without +running the held test; copy/hash the exact producer binary; hash every fixture +and protocol; record toolchain, settings, environment allowlist, limits and input +oracle into a create-new manifest/evidence root. Review actual binary discovery +with --list (no native execution). No code/build changes or concurrent local +compilation during the held run. Execute exactly the discovered held test +serial_cpu_residency::tests::key_reuse_qualification::held::fixed_two_ready_native_key_reuse_comparison. +Require the static metadata override at evidence-root/metadata.db. The frozen +supervisor uses an exclusive execution-started marker, no automatic retry, +immutable hashes and the same cgroup/affinity at execution. Launch only the exact ignored held test once. + +Retain progress, raw rows, body error, teardown outcome and oracle outcome even +when failure occurs. Stop on the single terminal outcome, missing/unknown facts, +overflow, infrastructure failure or uncertain teardown. No rerun, post-outcome +input tuning, population growth, continued failed owners or retrospective +acceptance change. A genuine evidenced implementation defect may be repaired +separately only after re-planning; original cohort remains failed. + +After execution independently review raw evidence against these frozen criteria. +Do not close overall M5/M6, performance, complete memory, native batching, +representative model quality or hardware claims from this narrow comparison. diff --git a/docs/plans/cpu-scheduler/reports/2026-10-21-m5-native-key-reuse-result-v2.md b/docs/plans/cpu-scheduler/reports/2026-10-21-m5-native-key-reuse-result-v2.md new file mode 100644 index 000000000..f35f5609e --- /dev/null +++ b/docs/plans/cpu-scheduler/reports/2026-10-21-m5-native-key-reuse-result-v2.md @@ -0,0 +1,139 @@ +# Native CPU key-reuse v2: bounded comparison result + +The single separately authorized v2 cohort passed its prospectively frozen +producer oracle. Both fresh arms completed. Independent raw-evidence review +and exact-head integration checks decide scoped acceptance separately below. +[V1](2026-10-21-m5-native-key-reuse-result.md) remains failed; its original +source, protocol, marker, evidence and archive are unchanged. + +## Prospective correction and provenance + +The reviewed correction requires every native candidate to identify +`runtime.candle` and match its own member's registry ACK. It preserves distinct +task, attempt, execution-request, reservation-lease and grant-ticket custody +identities within each arm, along with all other original criteria. Stable +graph task names and registry-local lease numbers may repeat across fresh runs; +their run/attempt/owner/registry context supplies the namespace. V2 does not +reuse v1's baseline. No acceptance criterion was selected after v2 execution. + +- Frozen source `1f6af8a25440d6a4049a3e86c86785689b71cf70`, tree + `0368ee2e945976ec84930f3a779995b4cc6f473a`, published normally on PR91 before + execution, preserving v1 commits `fb7ba0e3` and `aea54f7f` as ancestors. +- Producer source SHA256 + `f088036bfed607633c3fa9e93d019444c105ba7f6d1380e080fadc9a4fc480c2`; + copied executable SHA256 + `c41b0a573cc834595118545d70c723085ee33afdcd012971b5d6fc96cb88b8f0`. +- [Prospective v2 protocol](2026-10-21-m5-native-key-reuse-protocol-v2.md) + SHA256 `780f42c20dcc69ff210a08bac8208df3b6b30492db65667960b79bbe63327c91`; + supervisor SHA256 + `cdc2efb9e0b3a715ce47b71a7433c23c5776eb0bb0a62b238257fb6dfeaae51f`. +- Frozen manifest SHA256 + `331e55de0b7854484fa45527b57e95dd792baab5f8000a34c8bff310c0a51d53`. + Independent source/protocol/supervisor and exact manifest/binary review + preceded the single run. Discovery listed exactly one v2 held test without + facade startup, metadata or native work. +- Evidence root: + `/workspace/scratch/pantograph-recovery/native-key-reuse-cohort-v2`. + The exclusive execution marker remains. V1 uses its separate retained root. +- Actual execution: 2026-10-21T07:11:49.339705Z to 07:11:55.326203Z; + supervised elapsed 5,986,517,955 ns, exit 0, reaped, no timeout or supervisor + error. The owned process is absent. No second v2 execution was performed. + +The build is a debug/test executable on accessible CPU hardware using the same +committed untrained BERT-8 fixture and fixed two texts as v1. One configured CPU +thread, 8 GiB virtual-address-space ceiling, 180-second outer watchdog and all +per-arm/log/cleanup bounds were fixed before execution. Actual cgroup/affinity, +toolchain, fixture and binary hashes are in the frozen manifest. Resource +declarations and reservation claims are not physical allocation measurements. + +## Observed mechanism, correctness and lifecycle + +| Fixed criterion | Baseline | Enabled | +| --- | --- | --- | +| Explicit key reuse | false | true | +| Genuine Ready proposal | infer_a, infer_b | infer_a, infer_b | +| Encodings / hits / healthy retained entries | 6 / 0 / 0 | 4 / 4 / 0 | +| Loads / retains / native forwards / stops | 1 / 1 / 2 / 1 | 1 / 1 / 2 / 1 | +| Session/graph closes and public shutdown | acknowledged | acknowledged | +| Final reservations / idle owner | empty / false | empty / false | +| Body / collector / event / diagnostic-I/O errors | null | null | +| Native owner generations | 4→6→8, one thread | 4→6→8, one thread | +| Structured rows / ledger events | 12 / 19 | 16 / 19 | + +Both arms used distinct fresh run identities, private runtime registries and +loaded-instance identities. Every member kept its own grant, Start, attempt, +execution request, reservation and matching registry ACK; within each arm the +two lease numbers were 3 and 5. Candidate IDs correctly repeated +`runtime.candle`. Native current/previous owner identity, settings and generation +continuity were retained. The complete vectors were returned and read back +through actual retained artifacts, met the frozen golden tolerance (maximum +absolute error 5.960464477539063e-8), and were +exactly equal between arms. Numerical evidence covers these fixed untrained +fixture outputs only. + +The enabled compatible prefix contains both members. Its forecast diagnostics +report zero eligible samples and `no_fresh_samples`; proposals report +`insufficient_fresh_samples` with sample counts [0,0]. Minimum three samples +remains in force. This is no trained-ranking or forecast-accuracy evidence. + +| Enabled grouping phase | Observed calls | Inclusive elapsed ns | +| --- | --- | --- | +| probe | 2 | 49254 | +| capture | 1 | 598840 | +| fill | 1 | 127322 | +| lookup | 4 | 48874 | +| key_calculation | 4 | 154894 | + +All five aggregates were Available. Baseline grouping observations were absent, +not measured zero. These are service-lifetime, monotonic, inclusive spans with +uncalibrated observer/logging overhead; they overlap and must not be summed. +No timing threshold, latency/throughput gain or representative speedup is +inferred from this one debug-build synthetic comparison. + +Each native drain retained conservative member RAM and VRAM claims of +536895107 bytes, alongside the existing load/resident claims. Those claims +remained charged through drain until member cleanup. Final reservations were +empty after acknowledged shutdown. No reservation arithmetic or RSS was used +as incremental-allocation evidence. Scheduler cleanup matching and final +readiness/Start fence validation remain source-assisted through the existing +adapter; raw registry ACKs are not relabeled independent scheduler ACK proof. + +## Evidence retention and finite disposition + +Original v2 execution result SHA256: +`d09e727c3b1c6cb6315cd20b83e439a3d9064d087a0828c4dd398ee65c197a2c`. +Comparison SHA256: +`0930afc7e8f7c437b08549b764098fa70efc66914051587c8a22a90795266a64`. +Preparation receipt SHA256: +`19e5268c012970dcee68e7b1a649ef412641db15e78c2ad7bbb3a6c380bf81b1`. +It records ten model-free oracle passes, default embedded 617 passes/two +existing ignores, strict own-package Clippy/Rustdoc, formatting, traceability +and declared pre-push checks. The environment/model-capacity interruption was +reconciled before committing: preserved staged bytes and completed no-run build, +no active producer or v2 root/marker, and unchanged v1 hashes. No uncertain +native launch was retried. No credentials, permissions or security settings +changed. + +Independent raw-evidence review confirmed all frozen criteria and found no +material issue. Review receipt SHA256: +`07e9636205e7bdd8bd6f0d41d0988a326b497e30b7f1b2657ce30b11daf0dc56`. +The [retained original archive](m5-native-key-reuse-v2/retained-cohort.tar.gz) +contains 39 files / 131938 bytes, SHA256 +`d7cd6334bcec3b2a72c9f4eee026b0a507d62f7388db9739e611d4c179b98828`. +Its [per-file manifest](m5-native-key-reuse-v2/original-file-sha256.json) and +[archive identity](m5-native-key-reuse-v2/archive-sha256.json) preserve both raw +arms, the actual artifact bodies, journal, prospective protocol/manifest/ +supervisor, source/preparation/review receipts and check logs. All entry bytes +were matched to originals and their hashes without executing them. Executable +binaries, databases and client credentials are excluded; full local originals +remain protected. + +M5-CPU is supported by the narrow v2 evidence and independent review. M6-CLOSE +remains verifying until final source/report/archive review, exact-candidate CI +and actual integration checks pass. Acceptance, if earned, is limited to the finite ordinary-member +bounded pure-key-reuse envelope in the [checklist](2026-10-21-m5-m6-completion-checklist.md). +Broader C1 memory-aware dispatch, complete incremental allocation, native +batching, trained-model quality, forecast usefulness, deployed performance, +GPU/remote and representative hardware qualification remain separate. No +allocator expansion, admission-policy extension or default activation is +selected to close this finite task. diff --git a/docs/plans/cpu-scheduler/reports/2026-10-21-m5-native-key-reuse-result.md b/docs/plans/cpu-scheduler/reports/2026-10-21-m5-native-key-reuse-result.md new file mode 100644 index 000000000..8cf299892 --- /dev/null +++ b/docs/plans/cpu-scheduler/reports/2026-10-21-m5-native-key-reuse-result.md @@ -0,0 +1,133 @@ +# Fixed native CPU comparison v1: retained failure + +The single prospectively admitted cohort **failed**. Its enabled arm never +started. This report does not change the frozen criteria or accept M5-CPU. +No second physical execution, input tuning or post-outcome acceptance change +was performed. + +## Exact provenance and terminal result + +- Frozen source `fb7ba0e37645aaa6627ef6ae1b729c931387a772`, tree + `203f7b4a3d99f637fb96e99cb8fbc1c55b3f8f82`, based on PR90's actual normal + merge `ecfed47da7c51d03b968bed739e2abc06eb4a6de`. +- Producer source SHA256 + `cdd3ec4888a01dd1723616bfabd4099d5368013ba76714cc16b11bb527894b0e`; + copied executable SHA256 + `d465fc543206ba8879550901b415a1f092d995b6bbf4410606da0f2eeb5ac832`. +- [Prospective protocol](2026-10-21-m5-native-key-reuse-protocol.md) SHA256 + `a6648d0785bac9979ea3fc1f951f4c9b31a59d277f55f02b375b3bb7ae9c6216`; + supervisor SHA256 + `83f8598ee2d8f61e26711b64c18ae1e10fc1e596d1c1009b1b54e06d47b38d3c`. +- Independently approved frozen manifest SHA256 + `1aecf82f87d586f9eb28f8f5a4afdb03d721f72358f646804cc3fe58073b8e63`. + Exact held-test discovery listed one case without facade startup or metadata. +- Evidence root retained at + `/workspace/scratch/pantograph-recovery/native-key-reuse-cohort-v1`. + All original binary, fixture hashes, source identity, environment allowlist, + progress, raw rows and outcomes remain there. The exclusive execution marker + remains; the supervisor provides no retry path. +- Actual execution: 2026-10-21T06:15:00.456242Z to 06:15:05.539324Z; + supervised elapsed 5,083,100,620 ns, exit 101, reaped, no watchdog timeout, + supervisor error null. Producer verdict false, error `native identity reused`. +- Actual-main PR90 CI passed all 20 checks/four workflows before launch. The + integration receipt SHA256 is + `9d7e20d34d5a1a55811134dd35058f3795f0f966a0fdf49b0c05b95b84a38daf`. + Independent source/oracle/protocol/supervisor and exact-manifest/binary review + preceded execution. No local compilation ran concurrently. + +The environment was a debug/test binary, untrained committed BERT-8, one +configured CPU thread, CPU cgroup 400000 us/100000 us, memory cgroup 16 GiB, an 8 GiB +virtual-address-space ceiling and host-reported AMD EPYC 9V74. These facts do +not make the case representative of deployed models or hardware. + +## Observed baseline facts and failed criterion + +The baseline returned both requested 8-dimensional outputs, retained four +artifact readbacks, and recorded two native drains. Independent review of actual +artifact bytes confirmed agreement with returned values and the frozen golden; +maximum absolute golden error was 5.960464477539063e-8. Its scalar work observation +was six key encodings, zero hits and zero healthy retained cache entries. +Diagnostics recorded one load, one retain, two forward drains and one stop. +Public session close, graph closes and shutdown acknowledged success; shutdown +did not time out, final reservations were empty and no idle owner remained. +Body, collector, event and diagnostic-I/O errors were all null. The fsynced +journal contains 12 structured rows and the ledger 19 events. + +The frozen oracle requires distinct `task_id`, `attempt_id`, +`execution_request_id`, `candidate_id` and `reservation_lease_id` across the two +native drains. Actual task IDs were `infer_a` and `infer_b`; attempts and requests +differed, and reservation leases were 3 and 5. Both candidate IDs were +`runtime.candle`. This equality triggered the fixed failure. It is not evidence +that the distinct grants or leases were reused. Both producer-author and +independent source/evidence review confirmed that production deliberately builds +candidate IDs as `runtime.` in +[runtime_candidate_draft](../../../../crates/pantograph-embedded-runtime/src/runtime_dispatch_candidate_provider.rs). +Candidate uniqueness applies within one alternatives list, as enforced by +[dispatch selection](../../../../crates/pantograph-scheduler/src/dispatch_selection_policy.rs), +and not across member attempts. The frozen oracle and its hand-declared positive +fixture confused provider identity with custody identity. This is a qualification +software defect missed during pre-run review, rather than a discovered production +reuse-of-authority defect. This report does not retroactively exempt that field +or relabel the arm as passed. + +Independent raw review also confirmed exact joins across two grants, Starts, +Completed events and registry ACKs, with one-thread owner continuity 4→6→8 and +all 12 persisted rows matching the saved arm. Actual records contain conservative +VRAM reservation claims of 536895107 bytes per drain despite the CPU dispatch. +The frozen protocol's CPU/VRAM wording does not establish zero VRAM reservation +claims or physical GPU use. Neither reservation estimates nor declarations are +physical allocation evidence. + +The baseline failure stopped the loop after acknowledged teardown. There is no +enabled-arm directory, native output, key-reuse count, captured-prefix or phase +measurement from this cohort. Only two native forwards occurred, within the +four-forward maximum. No baseline-versus-enabled result exists. Scheduler +cleanup matching remains source-assisted through the existing adapter; the raw +registry ACK is not independent scheduler-ACK proof. + +## Retained evidence identities + +The [original retained cohort archive](m5-native-key-reuse-v1/retained-failed-cohort.tar.gz) +contains 40 files, 131098 bytes, SHA256 +`e6edef6c58518ab809be8fc5b31d1c9e18abd700ddafdb5037c82aa70fbed340`. +Its [per-file hashes](m5-native-key-reuse-v1/original-file-sha256.json) and +[archive identity/scope](m5-native-key-reuse-v1/archive-sha256.json) permit exact +verification without execution. It includes the actual artifact bodies and +review/preparation receipts, excluding executable binaries, databases and client +credentials. Original full local evidence remains preserved as well. + +| File under evidence root | Bytes | SHA256 | +| --- | --- | --- | +| execution-started.json | 145 | edc88952ee485a00fc4328c12c0c6ce8d873ae4ec5ec8375fb9594641565e6bf | +| execution-result.json | 598 | 77a24b38ccf56afccba928988e52d8c4fa99d2968b5e620689a19159b6fc1a91 | +| comparison.json | 150464 | 27cb921f9f4cf86bbd5cb23c224ce2084029c40bc1b2b05276f7e7592a15a0df | +| baseline/arm.json | 133686 | be7f39bc407b869955d282a40bcc22569e28d5952470ea35c225457dd4f2684f | +| native-rows.jsonl | 11369 | 3bfe2b7316a2d3754382e4a815755d560e2d94391827708278805789782d2f57 | +| progress.jsonl | 920 | 8870f3b249d6fc1ddef531cab2d29413ee465fb4a69dad0ef7e7004309b3bf9b | +| producer.stdout.log | 400 | 61139d5816979f711d6f99a11837422df7941fb5c3cd63949b188f01e6fad32c | + +Preparation receipt `native-key-reuse-local-validation.json` SHA256 +`3adf218aacf4b9cdec2bfe0fcdfeeb21a66b5a5f05a8b9c79af9a656939d71b1` +retains successful affected checks and initial failures. A wrong full source SHA +was refused by freeze preflight before creating the evidence root; the unchanged +actual Git HEAD then froze successfully. A PR90 receipt collector initially +referenced its not-yet-created output instead of the prior PR89 receipt; that +read-only collection error was corrected before the successful receipt and run. +Neither preparation error executed the held test. + +## Finite disposition + +PR89 integration, M5 software and the PR90 owning consumer are implemented and +integrated within their existing evidence limits. M5-CPU is failed for v1; +M6-CLOSE remains unaccepted. Stop at this terminal outcome. Preserve the frozen +source/protocol and this failure; any evidenced oracle repair and fresh cohort +must be separately re-planned prospectively. Missing representative model or +hardware measurements did not cause this failure and are not a blocker to +diagnosing the software assumption. No allocator expansion is selected. + +Production admission/defaults are unchanged. This cohort establishes no +key-reuse speedup, complete incremental-memory guarantee, native batching, +representative model quality or hardware qualification. RSS and reservation +estimates were not substituted for allocation measurements. Historical denied +log routes remain unretried, and baseline strict inference lint/Rustdoc +limitations remain unsuppressed. diff --git a/docs/plans/cpu-scheduler/reports/m5-native-key-reuse-v1/archive-sha256.json b/docs/plans/cpu-scheduler/reports/m5-native-key-reuse-v1/archive-sha256.json new file mode 100644 index 000000000..4b9f52bcf --- /dev/null +++ b/docs/plans/cpu-scheduler/reports/m5-native-key-reuse-v1/archive-sha256.json @@ -0,0 +1,9 @@ +{ + "archive": "retained-failed-cohort.tar.gz", + "bytes": 131098, + "sha256": "e6edef6c58518ab809be8fc5b31d1c9e18abd700ddafdb5037c82aa70fbed340", + "member_count": 40, + "scope": "original once-only failed cohort plus pre-run source/binary/fixture manifest, supervisor, actual artifact bytes, review/integration/local-check receipts and preparation logs; no executable/databases/client credentials", + "original_frozen_source": "fb7ba0e37645aaa6627ef6ae1b729c931387a772", + "result": "FAILED; enabled arm absent; no rerun" +} diff --git a/docs/plans/cpu-scheduler/reports/m5-native-key-reuse-v1/original-file-sha256.json b/docs/plans/cpu-scheduler/reports/m5-native-key-reuse-v1/original-file-sha256.json new file mode 100644 index 000000000..3aa7946be --- /dev/null +++ b/docs/plans/cpu-scheduler/reports/m5-native-key-reuse-v1/original-file-sha256.json @@ -0,0 +1,162 @@ +{ + "baseline/app-data/serial-cpu-artifacts/bodies/workflow-io-78821a221fa227fcd3caeb3f2990a121123a395a3a8210c3bf15683e0cef0c76.bin": { + "bytes": 160, + "sha256": "d3692d4df22e61e4e2c965304a642d323d05f42dc7d4f87018126b9672050494" + }, + "baseline/app-data/serial-cpu-artifacts/bodies/workflow-io-9f5e97f7733fcc67ce48b2867e34b3a9753f25e377a46b833b39805febda7470.bin": { + "bytes": 160, + "sha256": "4a688224ef6edeb30360652fa2b8953fa94120f6dfd649c22fe49494021effce" + }, + "baseline/app-data/serial-cpu-artifacts/bodies/workflow-io-c42766c95f453c55a06caf5aae4eca649c5fa8ab436221d53e578a159355075c.bin": { + "bytes": 11, + "sha256": "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + }, + "baseline/app-data/serial-cpu-artifacts/bodies/workflow-io-f96ad5684917e2ee1797911f0e979421274e6e392d6895b82a6012bd050cae96.bin": { + "bytes": 5, + "sha256": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" + }, + "baseline/app-data/serial-cpu-artifacts/manifest.json": { + "bytes": 5012, + "sha256": "546495ec2200f27eade8b21c78667db45b04aa70d5f299f18fadde4dcb04f5a5" + }, + "baseline/arm.json": { + "bytes": 133686, + "sha256": "be7f39bc407b869955d282a40bcc22569e28d5952470ea35c225457dd4f2684f" + }, + "comparison.json": { + "bytes": 150464, + "sha256": "27cb921f9f4cf86bbd5cb23c224ce2084029c40bc1b2b05276f7e7592a15a0df" + }, + "discovery.stderr": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "discovery.stdout": { + "bytes": 132, + "sha256": "18790687fc28f2237596cb20cd11f762831554ff8b2476b6c7f1838eab9d3a22" + }, + "execution-result.json": { + "bytes": 598, + "sha256": "77a24b38ccf56afccba928988e52d8c4fa99d2968b5e620689a19159b6fc1a91" + }, + "execution-started.json": { + "bytes": 145, + "sha256": "edc88952ee485a00fc4328c12c0c6ce8d873ae4ec5ec8375fb9594641565e6bf" + }, + "frozen-manifest.json": { + "bytes": 5619, + "sha256": "1aecf82f87d586f9eb28f8f5a4afdb03d721f72358f646804cc3fe58073b8e63" + }, + "native-rows.jsonl": { + "bytes": 11369, + "sha256": "3bfe2b7316a2d3754382e4a815755d560e2d94391827708278805789782d2f57" + }, + "producer.stdout.log": { + "bytes": 400, + "sha256": "61139d5816979f711d6f99a11837422df7941fb5c3cd63949b188f01e6fad32c" + }, + "progress.jsonl": { + "bytes": 920, + "sha256": "8870f3b249d6fc1ddef531cab2d29413ee465fb4a69dad0ef7e7004309b3bf9b" + }, + "prospective-protocol.md": { + "bytes": 6170, + "sha256": "a6648d0785bac9979ea3fc1f951f4c9b31a59d277f55f02b375b3bb7ae9c6216" + }, + "protocol.json": { + "bytes": 680, + "sha256": "8834b7a8a53f9cee75c86a86ce2c89b87840b499bad8108782520aa20831def2" + }, + "supervisor.py": { + "bytes": 10347, + "sha256": "83f8598ee2d8f61e26711b64c18ae1e10fc1e596d1c1009b1b54e06d47b38d3c" + }, + "support/logs/native-key-reuse-build.log": { + "bytes": 186, + "sha256": "88cc95363ec99ade2504f9d86568c940e4cb0d48262148688a92e08643ad2179" + }, + "support/logs/native-key-reuse-clippy-corrected.log": { + "bytes": 2443, + "sha256": "edc5ec95569a91d97f4acf0094646ba0a0ed647379df80bbd81f04bbf4813f71" + }, + "support/logs/native-key-reuse-clippy.log": { + "bytes": 3205, + "sha256": "29c272d39bbfecb0f7672ee16d1783ba84b7420186685ac666f48a19748f33db" + }, + "support/logs/native-key-reuse-embedded-corrected.log": { + "bytes": 71308, + "sha256": "ddb92e856e6782166a4134c20d0c9d334c231615423d342e0b013d36f91ef330" + }, + "support/logs/native-key-reuse-embedded-full.log": { + "bytes": 101986, + "sha256": "c003d5e17a2001ec7c521aab13432e4579e9108c6c64423ca4dc3ef7a727181c" + }, + "support/logs/native-key-reuse-final-fmt.log": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "support/logs/native-key-reuse-final-oracles.log": { + "bytes": 1206, + "sha256": "c8de7126022de7aecc5de5f2c166c6f72cf541c4e3434129d32bfe04e0efb146" + }, + "support/logs/native-key-reuse-final-traceability.log": { + "bytes": 277, + "sha256": "8a71ef6bd357cc379c4b862e9df626f27b09da4656530681881dfcac75a086f9" + }, + "support/logs/native-key-reuse-lint-staged.log": { + "bytes": 3174, + "sha256": "271494bf62f1e1c00d66b7da89c582e9f708dcad5bfeaadf31c794df788f16e8" + }, + "support/logs/native-key-reuse-lint.log": { + "bytes": 3027, + "sha256": "1a39139815f170b7404be1375d3222de635581eb209f4d5e98acc5cc03ca607a" + }, + "support/logs/native-key-reuse-oracles-misprofile-stopped.log": { + "bytes": 13209, + "sha256": "c265b6793078b6abbf209344aeccfe7d95914dfab23efdc62f8cb35565270f7d" + }, + "support/logs/native-key-reuse-oracles.log": { + "bytes": 3848, + "sha256": "b95836ac6c6a45b69a036de784a5f63a7b35feb3f4359b806b8f23e89bf6325b" + }, + "support/logs/native-key-reuse-prepush.log": { + "bytes": 35127, + "sha256": "33ecebae24571313ea2c000264f2319f153a73641612ba22f1ee4a9f0576ea29" + }, + "support/logs/native-key-reuse-rustdoc-strict.log": { + "bytes": 471, + "sha256": "52ff88ae5092ae9668ab12077f838689005baba275de6bee1cba6b026caa1afa" + }, + "support/logs/native-key-reuse-rustdoc.log": { + "bytes": 2675, + "sha256": "359fa0e69da1187b4bd2b41000ecb7070db2dee718b9063e2196db5a4586de49" + }, + "support/logs/native-key-reuse-workflow-full.log": { + "bytes": 121326, + "sha256": "91cfc383f9418121faaaab54fbffa1ffea5f24f2d554a31cd5fb1d7d4fc5377c" + }, + "support/native-key-reuse-build-artifacts.jsonl": { + "bytes": 444411, + "sha256": "f66d09804b9713ba1a06573c69f799cc3c99c58cd5279062398f555527423e95" + }, + "support/native-key-reuse-freeze-preflight-failure.json": { + "bytes": 459, + "sha256": "29e121a62395fb7c5e18d6ead8997da3759934a49b5edde73b33daf9539699a0" + }, + "support/native-key-reuse-frozen-admission.json": { + "bytes": 904, + "sha256": "7d21d65f38d53080bee8a573afc84d9630aae5969dce24b4987fb595a8298e5a" + }, + "support/native-key-reuse-local-validation.json": { + "bytes": 4869, + "sha256": "3adf218aacf4b9cdec2bfe0fcdfeeb21a66b5a5f05a8b9c79af9a656939d71b1" + }, + "support/native-key-reuse-selected-artifact.json": { + "bytes": 245, + "sha256": "a5ff74c750a43ccfdf7e5243d08b72feb5b16d9153b35a2da993992ba3f8bae3" + }, + "support/pr90-integration-final-receipt.json": { + "bytes": 14133, + "sha256": "9d7e20d34d5a1a55811134dd35058f3795f0f966a0fdf49b0c05b95b84a38daf" + } +} diff --git a/docs/plans/cpu-scheduler/reports/m5-native-key-reuse-v1/retained-failed-cohort.tar.gz b/docs/plans/cpu-scheduler/reports/m5-native-key-reuse-v1/retained-failed-cohort.tar.gz new file mode 100644 index 000000000..298cfd29a Binary files /dev/null and b/docs/plans/cpu-scheduler/reports/m5-native-key-reuse-v1/retained-failed-cohort.tar.gz differ diff --git a/docs/plans/cpu-scheduler/reports/m5-native-key-reuse-v2/archive-sha256.json b/docs/plans/cpu-scheduler/reports/m5-native-key-reuse-v2/archive-sha256.json new file mode 100644 index 000000000..53e701007 --- /dev/null +++ b/docs/plans/cpu-scheduler/reports/m5-native-key-reuse-v2/archive-sha256.json @@ -0,0 +1,10 @@ +{ + "archive": "retained-cohort.tar.gz", + "bytes": 131938, + "sha256": "d7cd6334bcec3b2a72c9f4eee026b0a507d62f7388db9739e611d4c179b98828", + "member_count": 39, + "scope": "original once-only v2 cohort, both raw arms/actual artifact bodies, journal, prospective protocol/supervisor/frozen manifest, pre-run source/review/check receipts and independent outcome review; no binary/databases/client credentials", + "source_head": "1f6af8a25440d6a4049a3e86c86785689b71cf70", + "result": "producer oracle passed; independent narrow raw evidence qualified; integration gates separate", + "v1_remains_failed_and_unchanged": true +} diff --git a/docs/plans/cpu-scheduler/reports/m5-native-key-reuse-v2/original-file-sha256.json b/docs/plans/cpu-scheduler/reports/m5-native-key-reuse-v2/original-file-sha256.json new file mode 100644 index 000000000..27951fb21 --- /dev/null +++ b/docs/plans/cpu-scheduler/reports/m5-native-key-reuse-v2/original-file-sha256.json @@ -0,0 +1,158 @@ +{ + "baseline/app-data/serial-cpu-artifacts/bodies/workflow-io-132d038a0c24c49f2047a0fc13dba14bbe033b798a89a29b80c7f6ab8b1e2846.bin": { + "bytes": 160, + "sha256": "4a688224ef6edeb30360652fa2b8953fa94120f6dfd649c22fe49494021effce" + }, + "baseline/app-data/serial-cpu-artifacts/bodies/workflow-io-1454cceee6b3b7c2d5a2fe70748e50b576a61042e05b4e81ae505e12e9b36eb2.bin": { + "bytes": 11, + "sha256": "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + }, + "baseline/app-data/serial-cpu-artifacts/bodies/workflow-io-5b020b5022215befaab7d0a4bff27e2111f9a01d4cf1c96cbc0fc1e843ad5e8c.bin": { + "bytes": 160, + "sha256": "d3692d4df22e61e4e2c965304a642d323d05f42dc7d4f87018126b9672050494" + }, + "baseline/app-data/serial-cpu-artifacts/bodies/workflow-io-8bd75b6f5175b6e588882eea3536bf4f5ce96514f87df617d8a5b52f211ef87a.bin": { + "bytes": 5, + "sha256": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" + }, + "baseline/app-data/serial-cpu-artifacts/manifest.json": { + "bytes": 5012, + "sha256": "1460adf95832bae5eba519bfd4d872ee299826f9c5feef43baf7a06b9f67d867" + }, + "baseline/arm.json": { + "bytes": 133695, + "sha256": "d8f7ca384c79d2cb5e44e930807f8c14015642e13713a08034e025d77f052dd4" + }, + "comparison.json": { + "bytes": 308564, + "sha256": "0930afc7e8f7c437b08549b764098fa70efc66914051587c8a22a90795266a64" + }, + "discovery.stderr": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "discovery.stdout": { + "bytes": 135, + "sha256": "28a325a30686d65652b0e594cd547ea83f5b783365ddffb9644302705246e38f" + }, + "enabled/app-data/serial-cpu-artifacts/bodies/workflow-io-1c635f453cae2bf86e347d2d25ffabed2a6d28c7fa9d414f72c4c2fcef19f697.bin": { + "bytes": 160, + "sha256": "4a688224ef6edeb30360652fa2b8953fa94120f6dfd649c22fe49494021effce" + }, + "enabled/app-data/serial-cpu-artifacts/bodies/workflow-io-594bcdd18efb41543a33e20d202244bd60ecddfa2a70d9ce719ae4e7298eb9ba.bin": { + "bytes": 11, + "sha256": "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + }, + "enabled/app-data/serial-cpu-artifacts/bodies/workflow-io-83bb444a586d18f85e1b2e15322a58338a05d32d7a405abe8581b1b22f9dd21a.bin": { + "bytes": 160, + "sha256": "d3692d4df22e61e4e2c965304a642d323d05f42dc7d4f87018126b9672050494" + }, + "enabled/app-data/serial-cpu-artifacts/bodies/workflow-io-a5fd5ac272efd46cc3063afb80e6e4f26fa5b3c5616d49ef47c1283247218ec2.bin": { + "bytes": 5, + "sha256": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" + }, + "enabled/app-data/serial-cpu-artifacts/manifest.json": { + "bytes": 5012, + "sha256": "3d27462c9847cc9a8fb4cccbec00163d650141f89262e1d8aaea474f70d7b939" + }, + "enabled/arm.json": { + "bytes": 140403, + "sha256": "72de6049a835a92b64770ce44ae3c5e57655b393b317be190af98fd97de7e3ae" + }, + "execution-result.json": { + "bytes": 574, + "sha256": "d09e727c3b1c6cb6315cd20b83e439a3d9064d087a0828c4dd398ee65c197a2c" + }, + "execution-started.json": { + "bytes": 145, + "sha256": "c83a33d983acfa44b11fc4e2505ee24b1876c9aef9efd1a96a3cd1300f4e1c46" + }, + "frozen-manifest.json": { + "bytes": 6019, + "sha256": "331e55de0b7854484fa45527b57e95dd792baab5f8000a34c8bff310c0a51d53" + }, + "native-rows.jsonl": { + "bytes": 25965, + "sha256": "2647ad1d6eccfc179534fa8512fb2de639f94476421e0f9a8c998f93058402c5" + }, + "producer.stdout.log": { + "bytes": 233, + "sha256": "9d18ca56e458ece847bfb897f9af75ea680b7dee1781cd615b87348f14b4ce39" + }, + "progress.jsonl": { + "bytes": 1834, + "sha256": "f00cfa862bc16c4ea312e187fbb26fc32601787817a13d2fe87115fb1b7e189d" + }, + "prospective-protocol.md": { + "bytes": 8182, + "sha256": "780f42c20dcc69ff210a08bac8208df3b6b30492db65667960b79bbe63327c91" + }, + "protocol.json": { + "bytes": 680, + "sha256": "eb8a8cc87a558e099bd764e149b6e4d908a2d2870efc6ff244778a2679f0e94b" + }, + "supervisor.py": { + "bytes": 10783, + "sha256": "cdc2efb9e0b3a715ce47b71a7433c23c5776eb0bb0a62b238257fb6dfeaae51f" + }, + "support/logs/native-key-reuse-v2-build.log": { + "bytes": 61, + "sha256": "fbab1cf5fbfcfda9e9235ad8750457194c4bc2068ff21d557440977d0d4c8f5f" + }, + "support/logs/native-key-reuse-v2-clippy.log": { + "bytes": 2389, + "sha256": "9d73d6465f9d76fe203de90468e2a65afa3c1907666f1c5f18a47627358fe0d7" + }, + "support/logs/native-key-reuse-v2-embedded.log": { + "bytes": 71819, + "sha256": "9f4a9da63b4c8892eda86bdf7b1973c0b097e9ecabfbd1020931f7c30ceb1dbd" + }, + "support/logs/native-key-reuse-v2-fmt.log": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "support/logs/native-key-reuse-v2-oracles.log": { + "bytes": 2577, + "sha256": "0b05ddef1803ee95097aaadabf0bc86e795644dd103cd4f5e595c5846e73ff1c" + }, + "support/logs/native-key-reuse-v2-prepush.log": { + "bytes": 34017, + "sha256": "c0b0e2aedd80889a98b1b94d9e556e112701b948cbf6646349af59bce4b3c1f4" + }, + "support/logs/native-key-reuse-v2-rustdoc.log": { + "bytes": 400, + "sha256": "6d693761959915e9c43293c657db7ff0e782b8c44911d1e1b7edcfb51fe0a4e7" + }, + "support/logs/native-key-reuse-v2-traceability.log": { + "bytes": 277, + "sha256": "abb3b972432409dd87ae7acef30f66f90b4f85d871bcb9c3ace9de5d918aff32" + }, + "support/native-key-reuse-v2-build-artifacts.jsonl": { + "bytes": 444410, + "sha256": "861a15483a7a735d9cfb501efcd837a290f6386b88f20851db90d774a47dc5be" + }, + "support/native-key-reuse-v2-frozen-admission.json": { + "bytes": 682, + "sha256": "ab40e602b4ae5463686bf532daef59742cb918ced8c8759daf00c9828e3ea567" + }, + "support/native-key-reuse-v2-outcome-review.json": { + "bytes": 2160, + "sha256": "07e9636205e7bdd8bd6f0d41d0988a326b497e30b7f1b2657ce30b11daf0dc56" + }, + "support/native-key-reuse-v2-preparation.json": { + "bytes": 3208, + "sha256": "19e5268c012970dcee68e7b1a649ef412641db15e78c2ad7bbb3a6c380bf81b1" + }, + "support/native-key-reuse-v2-reviewed-source.patch": { + "bytes": 28672, + "sha256": "ffc8ddc4e89d8c3771dce929d265da910b9536b28b921146701bad0df7a68dc3" + }, + "support/native-key-reuse-v2-selected-artifact.json": { + "bytes": 244, + "sha256": "efa6e342c25b3555f7067e4a81ed12a6ad5a86c846433e3ce35861d286cf747e" + }, + "support/pr90-integration-final-receipt.json": { + "bytes": 14133, + "sha256": "9d7e20d34d5a1a55811134dd35058f3795f0f966a0fdf49b0c05b95b84a38daf" + } +} diff --git a/docs/plans/cpu-scheduler/reports/m5-native-key-reuse-v2/retained-cohort.tar.gz b/docs/plans/cpu-scheduler/reports/m5-native-key-reuse-v2/retained-cohort.tar.gz new file mode 100644 index 000000000..76c954992 Binary files /dev/null and b/docs/plans/cpu-scheduler/reports/m5-native-key-reuse-v2/retained-cohort.tar.gz differ