From a8e5b23b599b7a6e3d746307cd91c2238903559c Mon Sep 17 00:00:00 2001 From: ado11231 Date: Fri, 16 Oct 2026 10:41:07 -0400 Subject: [PATCH 1/2] fix: drop a dead shared iroh connection before reusing it After an Agent crash, the shared ssh connection over iroh stayed open until iroh's 30 second idle timeout, and the next command waited with it. Before the first reuse, a forward to a closed port goes through it, which the Agent refuses at once. A connection that does not answer within 3 seconds is closed. Commands back to back over iroh pay about 0.14 seconds. Short network drops still do not end a run. --- crates/slingshot-cli/src/route.rs | 13 ++-- crates/slingshot-cli/src/ssh.rs | 116 +++++++++++++++++++++++++++++- 2 files changed, 124 insertions(+), 5 deletions(-) diff --git a/crates/slingshot-cli/src/route.rs b/crates/slingshot-cli/src/route.rs index 55ee8cc..782caa8 100644 --- a/crates/slingshot-cli/src/route.rs +++ b/crates/slingshot-cli/src/route.rs @@ -83,16 +83,21 @@ pub fn assume(agent: &Agent, token: &str) { *CHOSEN.lock().expect("route lock was poisoned") = Some((agent.name.clone(), route)); } -/// Probing waits on the network for up to a second. Inside the async runtime the wait -/// is handed off, so other tasks on this worker keep running meanwhile. +/// Probing waits on the network for up to a second. fn probe(agent: &Agent) -> Route { + off_runtime(|| probe_now(agent)) +} + +/// Run `work`, which waits on the network. Inside the async runtime the wait is handed +/// off, so other tasks on this worker keep running meanwhile. +pub fn off_runtime(work: impl FnOnce() -> T) -> T { use tokio::runtime::{Handle, RuntimeFlavor}; match Handle::try_current() { Ok(handle) if handle.runtime_flavor() == RuntimeFlavor::MultiThread => { - tokio::task::block_in_place(|| probe_now(agent)) + tokio::task::block_in_place(work) } - _ => probe_now(agent), + _ => work(), } } diff --git a/crates/slingshot-cli/src/ssh.rs b/crates/slingshot-cli/src/ssh.rs index bb03ff8..b0ae12c 100644 --- a/crates/slingshot-cli/src/ssh.rs +++ b/crates/slingshot-cli/src/ssh.rs @@ -8,6 +8,8 @@ use slingshot_core::config; use slingshot_core::presentation::Style; use std::path::PathBuf; use std::process::Stdio; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; use tokio::process::Command; /// The exit code reported when SSH ends without a remote exit status. @@ -22,6 +24,28 @@ const SSH_OPTIONS: &[&str] = &[ "LogLevel=ERROR", ]; +/// How long a call through the shared iroh connection may take before the connection is +/// treated as dead. A live one answers in well under a second. +const SHARE_CHECK: Duration = Duration::from_secs(3); + +/// The shared connection is checked once per process, before its first reuse. +static SHARE_CHECKED: AtomicBool = AtomicBool::new(false); + +/// Whether `child` exits within `limit`, however it ends. One that does not is stopped. +fn exits_within(mut child: std::process::Child, limit: Duration) -> bool { + let started = Instant::now(); + while started.elapsed() < limit { + match child.try_wait() { + Ok(Some(_)) => return true, + Ok(None) => std::thread::sleep(Duration::from_millis(20)), + Err(_) => break, + } + } + let _ = child.kill(); + let _ = child.wait(); + false +} + /// Request a terminal only for interactive use. /// A terminal lets Ctrl C reach remote work but merges stdout and stderr. pub fn wants_terminal() -> bool { @@ -69,7 +93,7 @@ impl RemoteCommand { tunnel::shared_socket(&key), ), }; - RemoteCommand { + let command = RemoteCommand { host, proxy, shared, @@ -83,7 +107,66 @@ impl RemoteCommand { quiet: false, program, args, + }; + if std::env::var_os(route::ROUTE_ENV).is_none() { + command.drop_stale_share(); + } + command + } + + /// A shared iroh connection to an Agent that crashed stays open until iroh gives up on + /// it, about 30 seconds, and every call through it waits that long. So before the first + /// reuse, one quick call goes through it, and a connection that does not answer is + /// closed so ssh opens a new one. The rsync helper skips this, since its parent checked. + fn drop_stale_share(&self) { + let Some(socket) = self.shared.as_ref().filter(|socket| socket.exists()) else { + return; + }; + if SHARE_CHECKED.swap(true, Ordering::SeqCst) { + return; } + let answered = route::off_runtime(|| { + let child = std::process::Command::new("ssh") + .args(self.share_check_args()) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn(); + child.is_ok_and(|child| exits_within(child, SHARE_CHECK)) + }); + if answered { + return; + } + let _ = std::process::Command::new("ssh") + .args(self.share_exit_args()) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + let _ = std::fs::remove_file(socket); + } + + /// A forward to a closed port on the Agent, through the shared connection. The Agent + /// refuses it at once without starting a shell, so any quick exit means the connection + /// is alive. The first value ssh reads for an option wins, so `ControlMaster=no` placed + /// first keeps this call from starting a new shared connection. + fn share_check_args(&self) -> Vec { + let mut argv = vec![ + "-o".to_string(), + "ControlMaster=no".to_string(), + "-W".to_string(), + "127.0.0.1:9".to_string(), + ]; + argv.extend(self.connection_args().into_iter().filter(|arg| arg != "-t")); + argv + } + + /// Ask the process holding the shared connection to end it. + fn share_exit_args(&self) -> Vec { + let mut argv = self.connection_args(); + let destination = argv.pop().unwrap_or_default(); + argv.extend(["-O".to_string(), "exit".to_string(), destination]); + argv } /// Who to log in as and where, in the form ssh expects. @@ -599,6 +682,37 @@ mod tests { assert_eq!(argv[argv.len() - 2], "100.67.90.119"); } + #[test] + fn the_share_check_uses_only_the_existing_connection_and_never_a_terminal() { + let mut command = remote(&["hi"]); + command.shared = Some(PathBuf::from("/tmp/slingshot-me/ab.sock")); + command.tty = true; + let check = command.share_check_args(); + assert_eq!(&check[..4], ["-o", "ControlMaster=no", "-W", "127.0.0.1:9"]); + assert!(!check.contains(&"-t".to_string()), "{check:?}"); + assert_eq!(check.last(), Some(&command.destination())); + let exit = command.share_exit_args(); + assert_eq!( + exit[exit.len() - 3..], + ["-O".to_string(), "exit".to_string(), command.destination()] + ); + } + + #[test] + fn any_quick_exit_is_an_answer_and_a_hang_is_not() { + let quick = std::process::Command::new("true").spawn().unwrap(); + assert!(exits_within(quick, Duration::from_secs(2))); + let refused = std::process::Command::new("false").spawn().unwrap(); + assert!(exits_within(refused, Duration::from_secs(2))); + let hung = std::process::Command::new("sleep") + .arg("5") + .spawn() + .unwrap(); + let started = Instant::now(); + assert!(!exits_within(hung, Duration::from_millis(200))); + assert!(started.elapsed() < Duration::from_secs(2)); + } + #[test] fn a_shared_connection_is_asked_for_only_when_there_is_a_socket() { assert!( From df5a37c3abfd13d19977dd3e5aaf3513aed73164 Mon Sep 17 00:00:00 2001 From: ado11231 Date: Fri, 16 Oct 2026 11:42:50 -0400 Subject: [PATCH 2/2] docs: an Agent crash over iroh The roadmap records the crash test and drops the old 20 second limitation. --- docs/ROADMAP.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 8b91ce9..3123c9e 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -87,7 +87,7 @@ * On Linux, a boot service with nobody logged in cannot block sleep, so an Agent that can sleep may suspend after about 15 minutes idle ([#68](https://github.com/ado11231/ss/issues/68)). * Starting by itself on a macOS Agent is untested. -* After the Agent loses power or crashes, the first iroh connection can wait about 20 seconds for the old one to time out. +* After a power loss, the first iroh connection is untested. A crash is tested below. * Edits in a session whose connection dropped stay on the Agent until `slingshot sync --pull`. * The tools step installs Docker only with `pacman`, `apt`, `dnf`, and `zypper`, and Git, Node, and Python only with package managers it knows. * Untested: file watchers in sessions, two Clients on one Agent account at once, and large Node and Python projects. @@ -186,3 +186,8 @@ 1. `slingshot projects` listed three projects on archbox with their sizes, last sync, and edits waiting. 2. `clean` on Slingshot itself freed 3.9 GiB, and the next `cargo build` made it again in 28 seconds. 3. A throwaway project showed 2 edits waiting. `remove` listed them, kept everything on Enter, and on `y` deleted the copy and its `~/Slingshot` link. + +### Agent Crash Over iroh: October 16, 2026 + +1. With archbox's service killed by `kill -9` while a shared iroh connection was open, the next `run` took 29.4 seconds before the fix and 3.8 seconds after. +2. Commands back to back over iroh went from 0.30 to 0.44 seconds on average. Runs over the local network and tailnet are unchanged.