diff --git a/Cargo.lock b/Cargo.lock index 4951f65..4e7543b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -128,6 +128,12 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bitflags" version = "2.13.2" @@ -279,6 +285,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -348,10 +360,12 @@ dependencies = [ "cfg-if", "cpufeatures", "curve25519-dalek-derive", + "digest", "fiat-crypto", "rand_core", "rustc_version", "subtle", + "zeroize", ] [[package]] @@ -371,6 +385,16 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "digest" version = "0.10.7" @@ -403,6 +427,30 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2", + "subtle", + "zeroize", +] + [[package]] name = "entangle" version = "0.6.0" @@ -415,6 +463,7 @@ dependencies = [ "crossterm", "data-encoding", "dirs", + "ed25519-dalek", "entangle-core", "entangle-link", "entangle-mcp", @@ -937,6 +986,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "poly1305" version = "0.8.0" @@ -1276,6 +1335,15 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core", +] + [[package]] name = "slab" version = "0.4.12" @@ -1347,6 +1415,16 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "strsim" version = "0.11.1" diff --git a/README.md b/README.md index 8d98792..cb2e4e1 100644 --- a/README.md +++ b/README.md @@ -64,8 +64,11 @@ confirmation closes that invite. The inviter should stay online briefly so existing members can learn about newly joined devices through one-hop introductions. Use `entangle group list`, `entangle group members `, and `entangle group leave ` to inspect -or leave groups. A second Entangle server using the same data directory gets -a temporary ID while the first server is running. See +or leave groups. Human admins can run `entangle group remove ` +to remove a member and rotate the key, `entangle group rekey ` to +rotate without changing membership, and `entangle group admin ` +to hand off administration. A second Entangle server using the same data +directory gets a temporary ID while the first server is running. See [docs/SETUP.md](docs/SETUP.md) for group setup, firewall, client configuration, and a two-machine smoke test. @@ -82,8 +85,9 @@ The standalone commands are `entangle serve` (default), `entangle node`, and ## Design and security The crate boundaries, message flow, wire format, and security limitations are -documented in [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md). Protocol version 4 -uses group-bound Noise links; all devices must share a group to communicate. +documented in [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md). Protocol version 5 +uses epoch-bound, group-authenticated Noise links with signed membership +updates; all devices must share a group to communicate. ## License diff --git a/crates/entangle-core/src/lib.rs b/crates/entangle-core/src/lib.rs index 4e92e8a..bde413a 100644 --- a/crates/entangle-core/src/lib.rs +++ b/crates/entangle-core/src/lib.rs @@ -7,7 +7,7 @@ use std::{fmt, path::Path}; pub mod sas; /// Current version of the Entangle control protocol. -pub const PROTOCOL_VERSION: u32 = 4; +pub const PROTOCOL_VERSION: u32 = 5; /// A process-unique particle identity rendered as 32 lowercase hexadecimal digits. #[derive(Clone, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] @@ -20,10 +20,42 @@ pub struct Member { pub particle_id: String, pub name: String, pub static_key: String, + pub sign_key: String, pub added_by: String, pub added_at: String, } +/// A signed group membership and key update. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct GroupUpdate { + pub group_id: String, + pub epoch: u32, + pub key_hex: String, + pub admin: String, + pub admin_sign_key: String, + pub revoked: Vec, + pub kind: UpdateKind, + pub issued_by: String, + pub issued_at: String, + pub signature: String, +} + +/// A member removed from a group, bound to the static key that was revoked. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct Revocation { + pub particle_id: String, + pub static_key: String, +} + +/// The operation represented by a signed group update. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum UpdateKind { + Remove { particle_id: String }, + Rekey, + AdminHandoff { to: String }, +} + impl ParticleId { /// Creates a fresh identity using the OS random number generator; callers may persist it. pub fn generate() -> Self { diff --git a/crates/entangle-link/src/lib.rs b/crates/entangle-link/src/lib.rs index c6b9131..d8c2207 100644 --- a/crates/entangle-link/src/lib.rs +++ b/crates/entangle-link/src/lib.rs @@ -1,7 +1,7 @@ //! Noise-protected control frames and bounded state-sync payload streams. use async_trait::async_trait; -use entangle_core::{Manifest, Member, ParticleId, StateKind}; +use entangle_core::{GroupUpdate, Manifest, Member, ParticleId, StateKind}; use entangle_transport::{ self, KeyRing, SecureStream, SharedKeyRing, TransportError, MAX_PLAINTEXT, }; @@ -60,6 +60,8 @@ pub struct PeerInfo { pub group: [u8; 4], /// Whether this connection arrived through a tunnel. pub relayed: bool, + /// Group key epoch negotiated by the Noise transport. + pub epoch: u32, } /// Control protocol message; each frame is one encrypted Noise message. @@ -81,6 +83,14 @@ pub enum Frame { /// Number of records newly added. added: u32, }, + /// Signed group update history needed to reach this link's key epoch. + GroupUpdates { updates: Vec }, + /// Acknowledges the last applied group update epoch. + GroupUpdatesAck { epoch: u32 }, + /// Requests removal of the authenticated sender from this group. + Leave, + /// Acknowledges a leave notice. + LeaveAck, /// Announces a payload to the receiver. SyncOffer { /// Sender's particle identity. @@ -209,6 +219,9 @@ pub enum LinkError { /// The peer failed an application operation. #[error("{0}")] Handler(String), + /// The update chain was applied; the connection must be redialed at the new epoch. + #[error("group key epoch changed; reload keys and redial")] + Stale, } /// Reader for the exact-size raw payload following an accepted offer. @@ -284,6 +297,47 @@ pub trait LinkHandler: Send + Sync + 'static { ) -> Result { Ok(0) } + + /// Returns this particle's current group epoch. + fn group_epoch(&self, _peer: &PeerInfo) -> u32 { + 0 + } + + /// Enables the signed-update prelude for this link handler. + fn supports_group_updates(&self) -> bool { + false + } + + /// Returns the authenticated current member associated with this peer. + fn peer_particle_id(&self, _peer: &PeerInfo) -> Option { + None + } + + /// Returns signed updates after the given epoch for a current member. + fn updates_since(&self, _peer: &PeerInfo, _epoch: u32) -> Vec { + Vec::new() + } + + /// Applies updates relayed by an authenticated current member. + fn on_updates( + &self, + _peer: &PeerInfo, + _sender: &ParticleId, + _updates: Vec, + ) -> Result { + Ok(self.group_epoch(_peer)) + } + + /// Handles a leave notice from an authenticated current member. + fn on_leave(&self, _peer: &PeerInfo, _sender: &ParticleId) -> Result<(), LinkError> { + Ok(()) + } + + /// Records a peer that acknowledged a signed update epoch. + fn on_updates_ack(&self, _peer: &PeerInfo, _epoch: u32) {} + + /// Records that a peer completed a link at its current signed update epoch. + fn on_link_epoch(&self, _peer: &PeerInfo, _epoch: u32) {} } /// Bound TCP listener for short-lived encrypted control connections. @@ -325,13 +379,17 @@ pub async fn serve_io( handler: Arc, ) -> Result<(), LinkError> { let current = current_keys(&keys)?; - let stream = entangle_transport::accept_io(io, ¤t, HANDSHAKE_TIMEOUT).await?; + let mut stream = entangle_transport::accept_io(io, ¤t, HANDSHAKE_TIMEOUT).await?; let peer = PeerInfo { addr: peer_addr, remote_static: stream.remote_static(), group: stream.group_id(), relayed, + epoch: stream.epoch(), }; + if handler.supports_group_updates() { + synchronize_updates(&mut stream, &peer, handler.as_ref()).await?; + } serve_connection(stream, peer, handler).await } @@ -348,6 +406,36 @@ pub async fn connect_secure( .map_err(|_| LinkError::Timeout)? } +/// Opens a route and exchanges any necessary signed group updates. +pub async fn connect_secure_with_handler( + route: &Route, + keys: SharedKeyRing, + group_id: [u8; 4], + handler: &dyn LinkHandler, + duration: Duration, +) -> Result { + let keys = current_keys(&keys)?; + timeout(duration, async { + let mut stream = connect_route(route, &keys, group_id, duration).await?; + let peer = PeerInfo { + addr: match route { + Route::Direct(addr) => *addr, + Route::Tunnel(_) => SocketAddr::from(([0, 0, 0, 0], 0)), + }, + remote_static: stream.remote_static(), + group: stream.group_id(), + relayed: matches!(route, Route::Tunnel(_)), + epoch: stream.epoch(), + }; + if handler.supports_group_updates() { + synchronize_updates(&mut stream, &peer, handler).await?; + } + Ok(stream) + }) + .await + .map_err(|_| LinkError::Timeout)? +} + /// Connects to a peer, exchanges manifests, and returns its static key. pub async fn handshake( route: &Route, @@ -368,10 +456,46 @@ pub async fn handshake_stream( group_id: [u8; 4], manifest: &Manifest, duration: Duration, +) -> Result<(Manifest, [u8; 32], SecureStream), LinkError> { + handshake_stream_inner(route, keys, group_id, manifest, duration, None).await +} + +/// Connects, catches up signed group updates, and then exchanges manifests. +pub async fn handshake_stream_with_handler( + route: &Route, + keys: SharedKeyRing, + group_id: [u8; 4], + manifest: &Manifest, + handler: &dyn LinkHandler, + duration: Duration, +) -> Result<(Manifest, [u8; 32], SecureStream), LinkError> { + handshake_stream_inner(route, keys, group_id, manifest, duration, Some(handler)).await +} + +async fn handshake_stream_inner( + route: &Route, + keys: SharedKeyRing, + group_id: [u8; 4], + manifest: &Manifest, + duration: Duration, + handler: Option<&dyn LinkHandler>, ) -> Result<(Manifest, [u8; 32], SecureStream), LinkError> { let keys = current_keys(&keys)?; timeout(duration, async { let mut stream = connect_route(route, &keys, group_id, duration).await?; + let peer = PeerInfo { + addr: match route { + Route::Direct(addr) => *addr, + Route::Tunnel(_) => SocketAddr::from(([0, 0, 0, 0], 0)), + }, + remote_static: stream.remote_static(), + group: stream.group_id(), + relayed: matches!(route, Route::Tunnel(_)), + epoch: stream.epoch(), + }; + if let Some(handler) = handler.filter(|handler| handler.supports_group_updates()) { + synchronize_updates(&mut stream, &peer, handler).await?; + } write_frame( &mut stream, &Frame::Hello { @@ -404,6 +528,81 @@ pub async fn handshake_stream( .map_err(|_| LinkError::Timeout)? } +async fn synchronize_updates( + stream: &mut SecureStream, + peer: &PeerInfo, + handler: &dyn LinkHandler, +) -> Result<(), LinkError> { + let local_epoch = handler.group_epoch(peer); + let link_epoch = stream.epoch(); + let updates = if local_epoch > link_epoch { + let updates = handler.updates_since(peer, link_epoch); + if updates.is_empty() { + return Err(LinkError::Stale); + } + oldest_updates_that_fit(updates, link_epoch)? + } else { + Vec::new() + }; + write_frame(stream, &Frame::GroupUpdates { updates }).await?; + match read_frame(stream).await? { + Frame::GroupUpdates { updates } if !updates.is_empty() => { + let sender = handler.peer_particle_id(peer).ok_or_else(|| { + LinkError::Handler("update sender is not a current group member".into()) + })?; + let epoch = handler.on_updates(peer, &sender, updates)?; + write_frame(stream, &Frame::GroupUpdatesAck { epoch }).await?; + Err(LinkError::Stale) + } + Frame::GroupUpdates { updates } if updates.is_empty() && local_epoch > link_epoch => { + match read_frame(stream).await? { + Frame::GroupUpdatesAck { epoch } if epoch > link_epoch && epoch <= local_epoch => { + handler.on_updates_ack(peer, epoch); + Err(LinkError::Stale) + } + frame => Err(LinkError::Unexpected(format!("{frame:?}"))), + } + } + Frame::GroupUpdates { updates } if updates.is_empty() && local_epoch == link_epoch => { + handler.on_link_epoch(peer, link_epoch); + Ok(()) + } + frame => Err(LinkError::Unexpected(format!("{frame:?}"))), + } +} + +fn oldest_updates_that_fit( + updates: Vec, + link_epoch: u32, +) -> Result, LinkError> { + let mut selected = Vec::new(); + let mut expected_epoch = Some( + link_epoch + .checked_add(1) + .ok_or_else(|| LinkError::Handler("group update epoch overflow".into()))?, + ); + for update in updates { + if expected_epoch != Some(update.epoch) { + return Err(LinkError::Handler( + "group update history is not contiguous".into(), + )); + } + let mut candidate = selected.clone(); + candidate.push(update.clone()); + let frame = Frame::GroupUpdates { updates: candidate }; + if serde_json::to_vec(&frame)?.len() > MAX_FRAME_BYTES { + if selected.is_empty() { + return Err(LinkError::Oversize); + } + break; + } + let update_epoch = update.epoch; + selected.push(update); + expected_epoch = update_epoch.checked_add(1); + } + Ok(selected) +} + /// Opens a link, sends a sync offer, streams its declared bytes, and waits for completion. pub async fn offer( route: &Route, @@ -486,6 +685,19 @@ pub async fn introduce_on( .map_err(|_| LinkError::Timeout)? } +/// Sends a leave notice and waits for the authenticated member's acknowledgement. +pub async fn leave_on(mut stream: SecureStream, duration: Duration) -> Result<(), LinkError> { + timeout(duration, async { + write_frame(&mut stream, &Frame::Leave).await?; + match read_frame(&mut stream).await? { + Frame::LeaveAck => Ok(()), + frame => Err(LinkError::Unexpected(format!("{frame:?}"))), + } + }) + .await + .map_err(|_| LinkError::Timeout)? +} + async fn connect_route( route: &Route, keys: &KeyRing, @@ -542,6 +754,13 @@ async fn serve_connection( handler: std::sync::Arc, ) -> Result<(), LinkError> { match read_frame(&mut stream).await? { + Frame::Leave => { + let sender = handler.peer_particle_id(&peer).ok_or_else(|| { + LinkError::Handler("leave sender is not a current group member".into()) + })?; + handler.on_leave(&peer, &sender)?; + write_frame(&mut stream, &Frame::LeaveAck).await + } Frame::Hello { manifest } => { let sender = manifest.particle_id.clone(); let own = handler.on_hello(&peer, manifest).await?; @@ -677,10 +896,11 @@ pub async fn read_frame(stream: &mut SecureStream) -> Result { #[cfg(test)] mod tests { use super::*; - use entangle_core::{ParticleId, PROTOCOL_VERSION}; + use entangle_core::{GroupUpdate, ParticleId, UpdateKind, PROTOCOL_VERSION}; use sha2::{Digest, Sha256}; use std::{ - collections::HashMap, + collections::{BTreeMap, HashMap}, + sync::atomic::{AtomicU32, Ordering}, sync::{Arc, Mutex, RwLock}, }; @@ -703,7 +923,13 @@ mod tests { fn keys() -> KeyRing { KeyRing { static_private: entangle_transport::generate_static(), - groups: HashMap::from([(GROUP, [0x33; 32])]), + groups: HashMap::from([( + GROUP, + entangle_transport::GroupKeys { + current: 0, + keys: BTreeMap::from([(0, [0x33; 32])]), + }, + )]), } } @@ -725,6 +951,105 @@ mod tests { } } + struct CatchUpHandler { + manifest: Manifest, + particle_id: ParticleId, + epoch: AtomicU32, + updates: Vec, + received_updates: Mutex>, + acknowledged: Mutex>, + } + + fn group_update(epoch: u32) -> GroupUpdate { + GroupUpdate { + group_id: "42424242".into(), + epoch, + key_hex: "44".repeat(32), + admin: "admin".into(), + admin_sign_key: "sign-key".into(), + revoked: Vec::new(), + kind: UpdateKind::Rekey, + issued_by: "admin".into(), + issued_at: "2026-10-12T00:00:00Z".into(), + signature: "signature".into(), + } + } + + fn group_update_with_revocations(epoch: u32) -> GroupUpdate { + let mut update = group_update(epoch); + update.revoked = (0..40) + .map(|_| entangle_core::Revocation { + particle_id: ParticleId::generate().to_string(), + static_key: "55".repeat(32), + }) + .collect(); + update + } + + #[async_trait] + impl LinkHandler for CatchUpHandler { + async fn on_hello( + &self, + _peer: &PeerInfo, + _manifest: Manifest, + ) -> Result { + Ok(self.manifest.clone()) + } + + async fn on_offer( + &self, + _peer: &PeerInfo, + _offer: SyncOffer, + ) -> Result { + Ok(SyncDecision::Reject("not accepted".into())) + } + + async fn on_accepted( + &self, + _peer: &PeerInfo, + _offer: SyncOffer, + _payload: &mut PayloadReader<'_>, + ) -> Result { + Err("should not be called".into()) + } + + fn group_epoch(&self, _peer: &PeerInfo) -> u32 { + self.epoch.load(Ordering::SeqCst) + } + + fn supports_group_updates(&self) -> bool { + true + } + + fn peer_particle_id(&self, _peer: &PeerInfo) -> Option { + Some(self.particle_id.clone()) + } + + fn updates_since(&self, _peer: &PeerInfo, epoch: u32) -> Vec { + self.updates + .iter() + .filter(|update| update.epoch > epoch) + .cloned() + .collect() + } + + fn on_updates( + &self, + _peer: &PeerInfo, + _sender: &ParticleId, + updates: Vec, + ) -> Result { + let epoch = updates.iter().map(|update| update.epoch).max().unwrap_or(0); + self.received_updates.lock().unwrap().extend(updates); + self.epoch.store(epoch, Ordering::SeqCst); + Ok(epoch) + } + + fn on_updates_ack(&self, _peer: &PeerInfo, epoch: u32) { + self.acknowledged.lock().unwrap().push(epoch); + } + } + fn payload_hash(payload: &[u8]) -> String { format!("{:x}", Sha256::digest(payload)) } @@ -894,6 +1219,199 @@ mod tests { task.abort(); } + #[tokio::test] + async fn same_epoch_with_handler_completes_the_link() { + let key = keys(); + let expected = manifest(); + let remote = Arc::new(CatchUpHandler { + manifest: expected.clone(), + particle_id: ParticleId::generate(), + epoch: AtomicU32::new(0), + updates: Vec::new(), + received_updates: Mutex::new(Vec::new()), + acknowledged: Mutex::new(Vec::new()), + }); + let local = CatchUpHandler { + manifest: manifest(), + particle_id: ParticleId::generate(), + epoch: AtomicU32::new(0), + updates: Vec::new(), + received_updates: Mutex::new(Vec::new()), + acknowledged: Mutex::new(Vec::new()), + }; + let (address, task) = start_listener(key.clone(), remote as Arc).await; + let (peer, _, _) = handshake_stream_with_handler( + &Route::Direct(address), + key_slot(&key), + GROUP, + &manifest(), + &local, + Duration::from_secs(2), + ) + .await + .unwrap(); + assert_eq!(peer, expected); + task.abort(); + } + + #[tokio::test] + async fn behind_member_applies_updates_and_acknowledges_before_linking() { + let group_key_0 = [0x33; 32]; + let group_key_1 = [0x44; 32]; + let mut responder_keys = keys(); + responder_keys.groups.insert( + GROUP, + entangle_transport::GroupKeys { + current: 1, + keys: BTreeMap::from([(0, group_key_0), (1, group_key_1)]), + }, + ); + let mut initiator_keys = keys(); + initiator_keys.groups.insert( + GROUP, + entangle_transport::GroupKeys { + current: 0, + keys: BTreeMap::from([(0, group_key_0)]), + }, + ); + let update = group_update(1); + let responder = Arc::new(CatchUpHandler { + manifest: manifest(), + particle_id: ParticleId::generate(), + epoch: AtomicU32::new(1), + updates: vec![update.clone()], + received_updates: Mutex::new(Vec::new()), + acknowledged: Mutex::new(Vec::new()), + }); + let initiator = CatchUpHandler { + manifest: manifest(), + particle_id: ParticleId::generate(), + epoch: AtomicU32::new(0), + updates: Vec::new(), + received_updates: Mutex::new(Vec::new()), + acknowledged: Mutex::new(Vec::new()), + }; + let (address, task) = + start_listener(responder_keys, responder.clone() as Arc).await; + let result = connect_secure_with_handler( + &Route::Direct(address), + key_slot(&initiator_keys), + GROUP, + &initiator, + Duration::from_secs(2), + ) + .await; + assert!(matches!(result, Err(LinkError::Stale))); + assert_eq!(*initiator.received_updates.lock().unwrap(), vec![update]); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if responder.acknowledged.lock().unwrap().contains(&1) { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("the ahead member should receive an update acknowledgement"); + task.abort(); + } + + #[tokio::test] + async fn oversized_update_history_catches_up_over_multiple_links() { + let updates = (1..=17) + .map(group_update_with_revocations) + .collect::>(); + assert!( + serde_json::to_vec(&Frame::GroupUpdates { + updates: updates.clone() + }) + .unwrap() + .len() + > MAX_FRAME_BYTES + ); + let mut responder_keys = keys(); + responder_keys.groups.insert( + GROUP, + entangle_transport::GroupKeys { + current: 17, + keys: (0..=17) + .map(|epoch| (epoch, if epoch == 0 { [0x33; 32] } else { [0x44; 32] })) + .collect(), + }, + ); + let mut initiator_keys = keys(); + initiator_keys.groups.insert( + GROUP, + entangle_transport::GroupKeys { + current: 0, + keys: BTreeMap::from([(0, [0x33; 32])]), + }, + ); + let responder = Arc::new(CatchUpHandler { + manifest: manifest(), + particle_id: ParticleId::generate(), + epoch: AtomicU32::new(17), + updates: updates.clone(), + received_updates: Mutex::new(Vec::new()), + acknowledged: Mutex::new(Vec::new()), + }); + let initiator = CatchUpHandler { + manifest: manifest(), + particle_id: ParticleId::generate(), + epoch: AtomicU32::new(0), + updates: Vec::new(), + received_updates: Mutex::new(Vec::new()), + acknowledged: Mutex::new(Vec::new()), + }; + let initiator_keys = key_slot(&initiator_keys); + let (address, task) = + start_listener(responder_keys, responder.clone() as Arc).await; + let mut redials = 0; + while initiator.epoch.load(Ordering::SeqCst) < 17 { + let result = connect_secure_with_handler( + &Route::Direct(address), + Arc::clone(&initiator_keys), + GROUP, + &initiator, + Duration::from_secs(2), + ) + .await; + assert!(matches!(result, Err(LinkError::Stale))); + redials += 1; + let epoch = initiator.epoch.load(Ordering::SeqCst); + let mut keys = initiator_keys.write().unwrap(); + let mut updated = (**keys).clone(); + let group_keys = updated.groups.get_mut(&GROUP).unwrap(); + group_keys.current = epoch; + group_keys.keys.insert(epoch, [0x44; 32]); + *keys = Arc::new(updated); + } + assert!(redials > 1); + assert_eq!(*initiator.received_updates.lock().unwrap(), updates); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if responder.acknowledged.lock().unwrap().len() == redials { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("the ahead member should receive every partial update acknowledgement"); + + let final_link = connect_secure_with_handler( + &Route::Direct(address), + Arc::clone(&initiator_keys), + GROUP, + &initiator, + Duration::from_secs(2), + ) + .await + .unwrap(); + drop(final_link); + task.abort(); + } + #[tokio::test] async fn tunnel_route_handshakes_and_transfers_payload() { let key = keys(); @@ -1077,7 +1595,13 @@ mod tests { async fn wrong_group_key_is_reported_as_a_handshake_error() { let server_key = keys(); let mut client_key = server_key.clone(); - client_key.groups.get_mut(&GROUP).unwrap()[0] ^= 0x80; + client_key + .groups + .get_mut(&GROUP) + .unwrap() + .keys + .get_mut(&0) + .unwrap()[0] ^= 0x80; let (address, task) = start_listener(server_key, Arc::new(RejectingHandler(manifest()))).await; let route = Route::Direct(address); diff --git a/crates/entangle-repeater/src/integration_tests.rs b/crates/entangle-repeater/src/integration_tests.rs index 93b6195..f654cdb 100644 --- a/crates/entangle-repeater/src/integration_tests.rs +++ b/crates/entangle-repeater/src/integration_tests.rs @@ -479,11 +479,27 @@ async fn forwarding_does_not_expose_end_to_end_noise_plaintext() { let group_id = [0xaa; 4]; let keys_a = KeyRing { static_private: entangle_transport::generate_static(), - groups: [(group_id, [47; 32])].into_iter().collect(), + groups: [( + group_id, + entangle_transport::GroupKeys { + current: 0, + keys: std::collections::BTreeMap::from([(0, [47; 32])]), + }, + )] + .into_iter() + .collect(), }; let keys_b = KeyRing { static_private: entangle_transport::generate_static(), - groups: [(group_id, [47; 32])].into_iter().collect(), + groups: [( + group_id, + entangle_transport::GroupKeys { + current: 0, + keys: std::collections::BTreeMap::from([(0, [47; 32])]), + }, + )] + .into_iter() + .collect(), }; let (secure_a, secure_b) = tokio::join!( entangle_transport::connect_io( diff --git a/crates/entangle-resonance/src/lib.rs b/crates/entangle-resonance/src/lib.rs index 5fca0f7..b5d9e86 100644 --- a/crates/entangle-resonance/src/lib.rs +++ b/crates/entangle-resonance/src/lib.rs @@ -587,7 +587,7 @@ mod tests { .find(|(key, _)| *key == "proto") .unwrap() .1, - "4" + "5" ); } diff --git a/crates/entangle-transport/src/lib.rs b/crates/entangle-transport/src/lib.rs index 8cbd17a..e14205b 100644 --- a/crates/entangle-transport/src/lib.rs +++ b/crates/entangle-transport/src/lib.rs @@ -6,7 +6,7 @@ use snow::{ Builder, HandshakeState, TransportState, }; use std::{ - collections::HashMap, + collections::{BTreeMap, HashMap}, fmt, io, net::SocketAddr, sync::{Arc, RwLock}, @@ -24,7 +24,10 @@ pub const NOISE_PATTERN: &str = "Noise_XXpsk3_25519_ChaChaPoly_BLAKE2s"; const MAX_MESSAGE: usize = u16::MAX as usize; const TAG_LENGTH: usize = 16; const PREFACE: &[u8; 4] = b"ENTG"; -const PREFACE_VERSION: u8 = 1; +const PREFACE_VERSION: u8 = 2; +const ACK_CONTINUE: u8 = 0; +const ACK_RETRY_EPOCH: u8 = 1; +const ACK_TOO_FAR_BEHIND: u8 = 2; /// Maximum plaintext bytes carried by one Noise transport message. pub const MAX_PLAINTEXT: usize = MAX_MESSAGE - TAG_LENGTH; @@ -44,8 +47,14 @@ pub type SharedKeyRing = Arc>>; pub struct KeyRing { /// X25519 static private key. pub static_private: [u8; 32], - /// Group-ID-to-Noise-PSK map. - pub groups: HashMap<[u8; 4], [u8; 32]>, + /// Group-ID-to-current-and-retained Noise PSKs. + pub groups: HashMap<[u8; 4], GroupKeys>, +} + +#[derive(Clone, Debug)] +pub struct GroupKeys { + pub current: u32, + pub keys: BTreeMap, } impl fmt::Debug for KeyRing { @@ -91,12 +100,18 @@ pub enum TransportError { /// Noise could not authenticate or complete a handshake. #[error("Noise handshake failed: {0}")] Handshake(String), - /// The nine-byte group preface was malformed. + /// The group preface or epoch acknowledgement was malformed. #[error("invalid group preface")] BadPreface, /// The requested group is not present in this key ring. #[error("unknown group")] UnknownGroup, + /// The responder retains the group, but not this older epoch. + #[error("local group epoch {local} is too far behind; peer is at epoch {current}")] + TooFarBehind { local: u32, current: u32 }, + /// The responder's epoch is not retained by the initiator. + #[error("peer group epoch {peer_epoch} is too far behind")] + PeerTooFarBehind { peer_epoch: u32 }, /// A Noise transport message could not be encrypted or authenticated. #[error("Noise transport failed: {0}")] Noise(String), @@ -114,6 +129,7 @@ pub struct SecureStream { noise: TransportState, remote_static: [u8; 32], group_id: [u8; 4], + epoch: u32, } impl SecureStream { @@ -154,6 +170,11 @@ impl SecureStream { pub fn group_id(&self) -> [u8; 4] { self.group_id } + + /// Returns the group key epoch used by this authenticated session. + pub fn epoch(&self) -> u32 { + self.epoch + } } /// Connects as a Noise initiator and completes the XXpsk3 handshake. @@ -179,16 +200,45 @@ pub async fn connect_io( duration: Duration, ) -> Result { timeout(duration, async { - if !ring.groups.contains_key(&group_id) { - return Err(TransportError::UnknownGroup); - } - write_preface(&mut stream, group_id).await?; - let (noise, remote_static) = initiator_handshake(&mut stream, ring, group_id).await?; + let group_keys = ring + .groups + .get(&group_id) + .ok_or(TransportError::UnknownGroup)?; + let requested_epoch = group_keys.current; + write_preface(&mut stream, group_id, requested_epoch).await?; + let mut ack = [0_u8; 1]; + stream.read_exact(&mut ack).await?; + let epoch = match ack[0] { + ACK_CONTINUE => requested_epoch, + ACK_RETRY_EPOCH => { + let mut epoch_bytes = [0_u8; 4]; + stream.read_exact(&mut epoch_bytes).await?; + let epoch = u32::from_be_bytes(epoch_bytes); + if !group_keys.keys.contains_key(&epoch) { + return Err(TransportError::PeerTooFarBehind { peer_epoch: epoch }); + } + stream.write_all(&epoch.to_be_bytes()).await?; + stream.flush().await?; + epoch + } + ACK_TOO_FAR_BEHIND => { + let mut epoch_bytes = [0_u8; 4]; + stream.read_exact(&mut epoch_bytes).await?; + return Err(TransportError::TooFarBehind { + local: requested_epoch, + current: u32::from_be_bytes(epoch_bytes), + }); + } + _ => return Err(TransportError::BadPreface), + }; + let (noise, remote_static) = + initiator_handshake(&mut stream, ring, group_id, epoch).await?; Ok(SecureStream { stream, noise, remote_static, group_id, + epoch, }) }) .await @@ -211,16 +261,47 @@ pub async fn accept_io( duration: Duration, ) -> Result { timeout(duration, async { - let group_id = read_preface(&mut stream).await?; - if !ring.groups.contains_key(&group_id) { + let (group_id, requested_epoch) = read_preface(&mut stream).await?; + let group_keys = ring + .groups + .get(&group_id) + .ok_or(TransportError::UnknownGroup)?; + let epoch = if group_keys.keys.contains_key(&requested_epoch) { + stream.write_all(&[ACK_CONTINUE]).await?; + stream.flush().await?; + requested_epoch + } else if requested_epoch > group_keys.current { + stream.write_all(&[ACK_RETRY_EPOCH]).await?; + stream.write_all(&group_keys.current.to_be_bytes()).await?; + stream.flush().await?; + let mut retry = [0_u8; 4]; + stream.read_exact(&mut retry).await?; + let retry_epoch = u32::from_be_bytes(retry); + if retry_epoch != group_keys.current || !group_keys.keys.contains_key(&retry_epoch) { + return Err(TransportError::UnknownGroup); + } + retry_epoch + } else if group_keys + .keys + .keys() + .next() + .is_some_and(|oldest| requested_epoch < *oldest) + { + stream.write_all(&[ACK_TOO_FAR_BEHIND]).await?; + stream.write_all(&group_keys.current.to_be_bytes()).await?; + stream.flush().await?; return Err(TransportError::UnknownGroup); - } - let (noise, remote_static) = responder_handshake(&mut stream, ring, group_id).await?; + } else { + return Err(TransportError::UnknownGroup); + }; + let (noise, remote_static) = + responder_handshake(&mut stream, ring, group_id, epoch).await?; Ok(SecureStream { stream, noise, remote_static, group_id, + epoch, }) }) .await @@ -230,30 +311,38 @@ pub async fn accept_io( async fn write_preface( stream: &mut (impl AsyncWrite + Unpin), group_id: [u8; 4], + epoch: u32, ) -> Result<(), TransportError> { - let mut preface = [0_u8; 9]; + let mut preface = [0_u8; 13]; preface[..4].copy_from_slice(PREFACE); preface[4] = PREFACE_VERSION; - preface[5..].copy_from_slice(&group_id); + preface[5..9].copy_from_slice(&group_id); + preface[9..].copy_from_slice(&epoch.to_be_bytes()); stream.write_all(&preface).await?; stream.flush().await?; Ok(()) } -async fn read_preface(stream: &mut (impl AsyncRead + Unpin)) -> Result<[u8; 4], TransportError> { - let mut preface = [0_u8; 9]; +async fn read_preface( + stream: &mut (impl AsyncRead + Unpin), +) -> Result<([u8; 4], u32), TransportError> { + let mut preface = [0_u8; 13]; stream.read_exact(&mut preface).await?; if &preface[..4] != PREFACE || preface[4] != PREFACE_VERSION { return Err(TransportError::BadPreface); } - Ok(preface[5..] - .try_into() - .expect("the preface has four group-ID bytes")) + Ok(( + preface[5..9] + .try_into() + .expect("the preface has four group-ID bytes"), + u32::from_be_bytes(preface[9..].try_into().expect("four epoch bytes")), + )) } fn handshake_state( ring: &KeyRing, group_id: [u8; 4], + epoch: u32, initiator: bool, ) -> Result { let params = NOISE_PATTERN @@ -262,9 +351,11 @@ fn handshake_state( let psk = ring .groups .get(&group_id) + .and_then(|keys| keys.keys.get(&epoch)) .ok_or(TransportError::UnknownGroup)?; - let mut prologue = b"entangle-link-v4".to_vec(); + let mut prologue = b"entangle-link-v5".to_vec(); prologue.extend_from_slice(&group_id); + prologue.extend_from_slice(&epoch.to_be_bytes()); let builder = Builder::new(params) .local_private_key(&ring.static_private) .map_err(|error| TransportError::Handshake(error.to_string()))? @@ -287,8 +378,9 @@ async fn initiator_handshake( stream: &mut (impl AsyncRead + AsyncWrite + Unpin + Send), ring: &KeyRing, group_id: [u8; 4], + epoch: u32, ) -> Result<(TransportState, [u8; 32]), TransportError> { - let mut state = handshake_state(ring, group_id, true)?; + let mut state = handshake_state(ring, group_id, epoch, true)?; let mut message = vec![0_u8; MAX_MESSAGE]; let length = state .write_message(&[], &mut message) @@ -321,8 +413,9 @@ async fn responder_handshake( stream: &mut (impl AsyncRead + AsyncWrite + Unpin + Send), ring: &KeyRing, group_id: [u8; 4], + epoch: u32, ) -> Result<(TransportState, [u8; 32]), TransportError> { - let mut state = handshake_state(ring, group_id, false)?; + let mut state = handshake_state(ring, group_id, epoch, false)?; let incoming = read_message(stream).await.map_err(handshake_io_error)?; let mut payload = vec![0_u8; MAX_MESSAGE]; state @@ -395,9 +488,20 @@ mod tests { const GROUP_B: [u8; 4] = [0xb2; 4]; fn ring(psk_a: [u8; 32], psk_b: [u8; 32]) -> KeyRing { + let keys = |psk| GroupKeys { + current: 0, + keys: BTreeMap::from([(0, psk)]), + }; + KeyRing { + static_private: generate_static(), + groups: HashMap::from([(GROUP_A, keys(psk_a)), (GROUP_B, keys(psk_b))]), + } + } + + fn ring_at(current: u32, keys: BTreeMap) -> KeyRing { KeyRing { static_private: generate_static(), - groups: HashMap::from([(GROUP_A, psk_a), (GROUP_B, psk_b)]), + groups: HashMap::from([(GROUP_A, GroupKeys { current, keys })]), } } @@ -420,6 +524,104 @@ mod tests { (client, accept.await.unwrap()) } + async fn connected_pair_result( + client_ring: KeyRing, + server_ring: KeyRing, + ) -> ( + Result, + Result, + ) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let accept = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + accept(stream, &server_ring, Duration::from_secs(2)).await + }); + let client = connect(address, &client_ring, GROUP_A, Duration::from_secs(2)).await; + (client, accept.await.unwrap()) + } + + #[tokio::test] + async fn retained_epochs_negotiate_both_directions_and_report_epoch() { + let psk = [0x31; 32]; + let (client, server) = + connected_pair(ring(psk, [0x32; 32]), ring(psk, [0x32; 32]), GROUP_A).await; + assert_eq!(client.epoch(), 0); + assert_eq!(server.epoch(), 0); + + let client = ring_at(0, BTreeMap::from([(0, psk)])); + let server = ring_at(1, BTreeMap::from([(0, psk), (1, [0x33; 32])])); + let (client, server) = connected_pair(client, server, GROUP_A).await; + assert_eq!(client.epoch(), 0); + assert_eq!(server.epoch(), 0); + + let client = ring_at(1, BTreeMap::from([(0, psk), (1, [0x33; 32])])); + let server = ring_at(0, BTreeMap::from([(0, psk)])); + let (client, server) = connected_pair(client, server, GROUP_A).await; + assert_eq!(client.epoch(), 0); + assert_eq!(server.epoch(), 0); + } + + #[tokio::test] + async fn epochs_older_than_retained_keys_return_too_far_behind_before_noise() { + let client = ring_at(0, BTreeMap::from([(0, [0; 32])])); + let server = ring_at( + 17, + (1..=17).map(|epoch| (epoch, [epoch as u8; 32])).collect(), + ); + let (client, server) = connected_pair_result(client, server).await; + assert!(matches!( + client, + Err(TransportError::TooFarBehind { + local: 0, + current: 17 + }) + )); + assert!(server.is_err()); + } + + #[tokio::test] + async fn epoch_before_oldest_retained_key_is_too_far_behind_before_noise() { + let client = ring_at(1, BTreeMap::from([(1, [1; 32])])); + let server = ring_at( + 18, + (2..=18).map(|epoch| (epoch, [epoch as u8; 32])).collect(), + ); + let (client, server) = connected_pair_result(client, server).await; + assert!(matches!( + client, + Err(TransportError::TooFarBehind { + local: 1, + current: 18 + }) + )); + assert!(server.is_err()); + } + + #[tokio::test] + async fn initiator_without_responder_epoch_returns_peer_too_far_behind_before_noise() { + let client = ring_at( + 17, + (1..=17).map(|epoch| (epoch, [epoch as u8; 32])).collect(), + ); + let server = ring_at(0, BTreeMap::from([(0, [0; 32])])); + let (client, server) = connected_pair_result(client, server).await; + assert!(matches!( + client, + Err(TransportError::PeerTooFarBehind { peer_epoch: 0 }) + )); + assert!(server.is_err()); + } + + #[tokio::test] + async fn responder_rejects_an_unknown_epoch() { + let client = ring_at(1, BTreeMap::from([(1, [1; 32])])); + let server = ring_at(2, BTreeMap::from([(0, [0; 32]), (2, [2; 32])])); + let (client, server) = connected_pair_result(client, server).await; + assert!(client.is_err()); + assert!(matches!(server, Err(TransportError::UnknownGroup))); + } + #[tokio::test] async fn two_group_ring_roundtrips_and_streams_report_the_selected_group() { let client_ring = ring([0x5a; 32], [0x6b; 32]); @@ -463,7 +665,13 @@ mod tests { let client_ring = ring([0x5a; 32], [0x6b; 32]); let server_ring = ring([0x5a; 32], [0x6b; 32]); let mut server_ring = server_ring; - server_ring.groups.get_mut(&GROUP_A).unwrap()[0] ^= 1; + server_ring + .groups + .get_mut(&GROUP_A) + .unwrap() + .keys + .get_mut(&0) + .unwrap()[0] ^= 1; let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let server = tokio::spawn(async move { @@ -485,7 +693,10 @@ mod tests { accept_io(Box::new(server), &ring, Duration::from_secs(1)).await }); let mut client = client; - client.write_all(b"NOPE\x01\x00\x00\x00\x00").await.unwrap(); + client + .write_all(b"NOPE\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00") + .await + .unwrap(); assert!(matches!( accept.await.unwrap(), Err(TransportError::BadPreface) @@ -510,7 +721,13 @@ mod tests { assert_eq!(key.len(), 32); let keys = KeyRing { static_private: key, - groups: HashMap::from([(GROUP_A, [0xa5; 32])]), + groups: HashMap::from([( + GROUP_A, + GroupKeys { + current: 0, + keys: BTreeMap::from([(0, [0xa5; 32])]), + }, + )]), }; let debug = format!("{keys:?}"); assert!(debug.contains("static_private: \"[REDACTED]\"")); diff --git a/crates/entangle/Cargo.toml b/crates/entangle/Cargo.toml index 4e664d1..3a8c825 100644 --- a/crates/entangle/Cargo.toml +++ b/crates/entangle/Cargo.toml @@ -21,6 +21,7 @@ clap.workspace = true crossterm = "0.28.1" dirs.workspace = true data-encoding.workspace = true +ed25519-dalek = "2" gethostname.workspace = true hex.workspace = true rand.workspace = true diff --git a/crates/entangle/src/groups.rs b/crates/entangle/src/groups.rs index c9525e1..1c2a5fc 100644 --- a/crates/entangle/src/groups.rs +++ b/crates/entangle/src/groups.rs @@ -2,13 +2,14 @@ use crate::{identity, private_fs}; use chrono::{SecondsFormat, Utc}; +use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey}; pub use entangle_core::Member; -use entangle_core::ParticleId; +use entangle_core::{GroupUpdate, ParticleId, Revocation, UpdateKind}; use rand::RngCore; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::{ - collections::HashMap, + collections::{BTreeMap, HashMap}, fs::{self, File, OpenOptions}, io, path::{Path, PathBuf}, @@ -16,12 +17,17 @@ use std::{ use thiserror::Error; pub const MAX_GROUPS: usize = 8; -const GROUP_VERSION: u32 = 1; +const GROUP_VERSION: u32 = 2; const GROUP_DOMAIN: &[u8] = b"entangle-group-id-v1"; const PSK_DOMAIN: &[u8] = b"entangle-group-psk-v1"; +const SIGN_DOMAIN: &[u8] = b"entangle-sign-v1"; +const UPDATE_DOMAIN: &[u8] = b"entangle-group-update-v1\n"; +const RETAINED_LIMIT: usize = 16; +const RETAINED_UPDATE_LIMIT: usize = RETAINED_LIMIT + 1; #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "lowercase")] +#[cfg(test)] pub enum Role { Admin, Member, @@ -31,8 +37,19 @@ pub enum Role { pub struct GroupMeta { pub version: u32, pub name: String, - pub role: Role, pub created_at: String, + pub epoch: u32, + pub admin: String, + pub admin_sign_key: String, + pub revoked: Vec, + #[serde(default)] + pub leaving: bool, +} + +impl GroupMeta { + pub fn is_admin(&self, self_id: &str) -> bool { + self.admin == self_id + } } #[derive(Clone, Debug, Eq, PartialEq)] @@ -41,6 +58,22 @@ pub struct Group { pub key: [u8; 32], pub meta: GroupMeta, pub members: Vec, + pub retained_keys: BTreeMap, + pub updates: Vec, + pub outdated: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ApplyResult { + pub epoch: u32, + pub removed: bool, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct GroupStatus { + pub epoch: u32, + pub op: String, + pub confirmed: Vec, } #[derive(Debug, Error)] @@ -55,6 +88,10 @@ pub enum GroupError { TooMany, #[error("group not found: {0}")] NotFound(String), + #[error("group {name} was created by Entangle 0.6; run `entangle group leave {name}` and create it again")] + Outdated { name: String }, + #[error("{name} changed while joining; run the invite again")] + Conflict { name: String }, #[error("could not generate a unique group ID after 16 key attempts")] IdCollision, #[error("group member key conflicts for particle {0}")] @@ -68,6 +105,8 @@ pub enum GroupError { Malformed { path: PathBuf, reason: String }, #[error("unsafe group storage path: {0}")] UnsafePath(PathBuf), + #[error("invalid signed group update: {0}")] + InvalidUpdate(String), #[error("group storage I/O failed at {path}: {source}")] Io { path: PathBuf, @@ -112,6 +151,13 @@ pub fn group_id_bytes(key: &[u8; 32]) -> [u8; 4] { .expect("four-byte group ID") } +pub fn id_bytes(id: &str) -> Option<[u8; 4]> { + if !is_group_id(id) { + return None; + } + hex::decode(id).ok()?.try_into().ok() +} + pub fn derive_psk(key: &[u8; 32]) -> [u8; 32] { let mut hasher = Sha256::new(); hasher.update(PSK_DOMAIN); @@ -119,6 +165,18 @@ pub fn derive_psk(key: &[u8; 32]) -> [u8; 32] { hasher.finalize().into() } +pub fn signing_key(static_private: &[u8; 32]) -> SigningKey { + let mut hasher = Sha256::new(); + hasher.update(SIGN_DOMAIN); + hasher.update(static_private); + let seed: [u8; 32] = hasher.finalize().into(); + SigningKey::from_bytes(&seed) +} + +pub fn signing_public_key(static_private: &[u8; 32]) -> String { + hex::encode(signing_key(static_private).verifying_key().to_bytes()) +} + pub fn validate_name(name: &str) -> Result<(), GroupError> { let bytes = name.as_bytes(); if bytes.is_empty() @@ -165,6 +223,7 @@ fn create_with_key_source( validate_member(&self_member, root)?; if self_member.particle_id != identity_id.as_str() || self_member.static_key != hex::encode(entangle_transport::public_key(&identity_key)) + || self_member.sign_key != signing_public_key(&identity_key) { return Err(GroupError::malformed( root, @@ -192,10 +251,17 @@ fn create_with_key_source( meta: GroupMeta { version: GROUP_VERSION, name: name.to_owned(), - role: Role::Admin, created_at: now, + epoch: 0, + admin: self_member.particle_id.clone(), + admin_sign_key: self_member.sign_key.clone(), + revoked: Vec::new(), + leaving: false, }, members: vec![self_member], + retained_keys: BTreeMap::new(), + updates: Vec::new(), + outdated: false, }; write_group(root, &group)?; Ok(group) @@ -208,28 +274,122 @@ pub fn insert( meta: GroupMeta, members: Vec, ) -> Result { - validate_meta(&meta, root)?; - validate_members(&members, root)?; + insert_with_id(root, group_id(&key), key, meta, members) +} + +pub fn insert_with_id( + root: &Path, + id: String, + key: [u8; 32], + meta: GroupMeta, + members: Vec, +) -> Result { + insert_group( + root, + Group { + id, + key, + meta, + members, + retained_keys: BTreeMap::new(), + updates: Vec::new(), + outdated: false, + }, + ) +} + +pub fn insert_group(root: &Path, group: Group) -> Result { + validate_meta(&group.meta, root)?; + validate_members(&group.members, root)?; + validate_admin_member(&group.meta, &group.members, root)?; + if !is_group_id(&group.id) { + return Err(GroupError::malformed( + Path::new("group ID"), + "directory name is not an eight-character group ID", + )); + } + if group.retained_keys.len() > RETAINED_LIMIT || group.updates.len() > RETAINED_UPDATE_LIMIT { + return Err(GroupError::InvalidUpdate( + "too many retained group keys or updates".into(), + )); + } let _lock = lock_groups(root)?; let existing = read_groups_unlocked(root)?; - ensure_available(&existing, &meta.name)?; + ensure_available(&existing, &group.meta.name)?; if existing.len() >= MAX_GROUPS { return Err(GroupError::TooMany); } - let id = group_id(&key); - if existing.iter().any(|group| group.id == id) { - return Err(GroupError::AlreadyMember(id)); + if existing.iter().any(|existing| existing.id == group.id) { + return Err(GroupError::AlreadyMember(group.id)); } - let group = Group { - id, - key, - meta, - members, - }; write_group(root, &group)?; Ok(group) } +pub fn replace_existing( + root: &Path, + mut group: Group, + expected_epoch: u32, +) -> Result { + if !is_group_id(&group.id) { + return Err(GroupError::NotFound(group.id)); + } + let _lock = lock_groups(root)?; + cleanup_transaction_dirs(&groups_dir(root))?; + let previous = read_groups_unlocked(root)? + .into_iter() + .find(|existing| existing.id == group.id) + .ok_or_else(|| GroupError::Conflict { + name: group.meta.name.clone(), + })?; + if previous.outdated { + return Err(GroupError::Outdated { + name: previous.meta.name, + }); + } + if previous.meta.epoch != expected_epoch { + return Err(GroupError::Conflict { + name: previous.meta.name, + }); + } + group.meta.name = previous.meta.name.clone(); + write_group(root, &group)?; + Ok(previous) +} + +pub fn restore_existing_if_unchanged( + root: &Path, + previous: Group, + installed: &Group, +) -> Result<(), GroupError> { + let _lock = lock_groups(root)?; + cleanup_transaction_dirs(&groups_dir(root))?; + let current = read_groups_unlocked(root)? + .into_iter() + .find(|existing| existing.id == previous.id) + .ok_or_else(|| GroupError::Conflict { + name: previous.meta.name.clone(), + })?; + let mut current_meta = current.meta.clone(); + current_meta.name.clear(); + let mut installed_meta = installed.meta.clone(); + installed_meta.name.clear(); + if current.outdated + || installed.outdated + || current.id != installed.id + || current.key != installed.key + || current_meta != installed_meta + || current.members != installed.members + || current.retained_keys != installed.retained_keys + || current.updates != installed.updates + { + return Err(GroupError::Conflict { + name: current.meta.name, + }); + } + write_group(root, &previous) +} + #[allow(dead_code)] pub fn add_member(root: &Path, group_id: &str, member: Member) -> Result { validate_member(&member, root)?; @@ -240,6 +400,17 @@ pub fn add_member(root: &Path, group_id: &str, member: Member) -> Result Result Result<(), GroupError> { + if !is_group_id(group_id) { + return Err(GroupError::NotFound(group_id.to_owned())); + } + let _lock = lock_groups(root)?; + cleanup_transaction_dirs(&groups_dir(root))?; + let path = group_path(root, group_id); + let mut group = read_group(&path)?; + group + .members + .retain(|member| member.particle_id != particle_id); + if group.members.is_empty() { + fs::remove_dir_all(&path).map_err(|error| GroupError::io(&path, error))?; + } else { + write_group(root, &group)?; + } + Ok(()) +} + +pub fn set_leaving(root: &Path, group_id: &str, leaving: bool) -> Result { + if !is_group_id(group_id) { + return Err(GroupError::NotFound(group_id.to_owned())); + } + let _lock = lock_groups(root)?; + cleanup_transaction_dirs(&groups_dir(root))?; + let path = group_path(root, group_id); + let mut group = read_group(&path)?; + group.meta.leaving = leaving; + write_group(root, &group)?; + Ok(group) +} + +pub fn issue_update( + root: &Path, + group_id: &str, + kind: UpdateKind, + static_private: &[u8; 32], +) -> Result { + issue_update_with(root, group_id, kind, static_private, None) +} + +pub fn issue_unrevoke_update( + root: &Path, + group_id: &str, + particle_id: &str, + static_private: &[u8; 32], +) -> Result { + issue_update_with( + root, + group_id, + UpdateKind::Rekey, + static_private, + Some(particle_id), + ) +} + +fn issue_update_with( + root: &Path, + group_id: &str, + kind: UpdateKind, + static_private: &[u8; 32], + unrevoke: Option<&str>, +) -> Result { + let Some((self_id, saved_private)) = identity::read_saved(root) + .map_err(|error| GroupError::malformed(root, error.to_string()))? + else { + return Err(GroupError::MissingIdentity(root.to_owned())); + }; + if &saved_private != static_private { + return Err(GroupError::InvalidUpdate( + "signing identity does not match the saved identity".into(), + )); + } + let _lock = lock_groups(root)?; + cleanup_transaction_dirs(&groups_dir(root))?; + let path = group_path(root, group_id); + let mut group = read_group(&path)?; + if !group.meta.is_admin(self_id.as_str()) + || group.meta.admin_sign_key != signing_public_key(static_private) + { + return Err(GroupError::InvalidUpdate(format!( + "only group admin {} may issue updates", + group.meta.admin + ))); + } + let issued_by = self_id.to_string(); + let mut new_key = group.key; + let mut new_admin = group.meta.admin.clone(); + let mut new_admin_sign_key = group.meta.admin_sign_key.clone(); + let mut revoked = group.meta.revoked.clone(); + if let Some(particle_id) = unrevoke { + revoked.retain(|revocation| revocation.particle_id != particle_id); + } + match &kind { + UpdateKind::Remove { particle_id } => { + if particle_id == &issued_by { + return Err(GroupError::InvalidUpdate( + "the admin cannot remove itself".into(), + )); + } + let member = group + .members + .iter() + .find(|member| member.particle_id == *particle_id) + .ok_or_else(|| GroupError::InvalidUpdate(format!("no member {particle_id}")))?; + let revocation = Revocation { + particle_id: member.particle_id.clone(), + static_key: member.static_key.clone(), + }; + if !revoked.iter().any(|existing| existing == &revocation) { + revoked.push(revocation); + } + group + .members + .retain(|member| member.particle_id != *particle_id); + rand::rngs::OsRng.fill_bytes(&mut new_key); + } + UpdateKind::Rekey => { + if unrevoke.is_none() { + rand::rngs::OsRng.fill_bytes(&mut new_key); + } + } + UpdateKind::AdminHandoff { to } => { + if to == &issued_by { + return Err(GroupError::InvalidUpdate( + "the group is already administered by this particle".into(), + )); + } + let member = group + .members + .iter() + .find(|member| member.particle_id == *to) + .ok_or_else(|| GroupError::InvalidUpdate(format!("no member {to}")))?; + new_admin = member.particle_id.clone(); + new_admin_sign_key = member.sign_key.clone(); + } + } + let epoch = group + .meta + .epoch + .checked_add(1) + .ok_or_else(|| GroupError::InvalidUpdate("group epoch overflow".into()))?; + revoked.sort_by(|left, right| left.particle_id.cmp(&right.particle_id)); + let mut update = GroupUpdate { + group_id: group.id.clone(), + epoch, + key_hex: hex::encode(new_key), + admin: new_admin.clone(), + admin_sign_key: new_admin_sign_key.clone(), + revoked: revoked.clone(), + kind, + issued_by, + issued_at: timestamp(), + signature: String::new(), + }; + let message = update_signing_bytes(&update)?; + update.signature = hex::encode(signing_key(static_private).sign(&message).to_bytes()); + group.retained_keys.insert(group.meta.epoch, group.key); + trim_oldest(&mut group.retained_keys); + group.key = new_key; + group.meta.epoch = epoch; + group.meta.admin = new_admin; + group.meta.admin_sign_key = new_admin_sign_key; + group.meta.revoked = revoked; + group.updates.push(update); + if group.updates.len() > RETAINED_UPDATE_LIMIT { + group + .updates + .drain(..group.updates.len() - RETAINED_UPDATE_LIMIT); + } + write_group(root, &group)?; + Ok(group) +} + +pub fn apply_updates( + root: &Path, + group_id: &str, + updates: &[GroupUpdate], + _via: Option<&Member>, +) -> Result { + if !is_group_id(group_id) { + return Err(GroupError::NotFound(group_id.to_owned())); + } + let Some((self_id, _)) = identity::read_saved(root) + .map_err(|error| GroupError::malformed(root, error.to_string()))? + else { + return Err(GroupError::MissingIdentity(root.to_owned())); + }; + let _lock = lock_groups(root)?; + cleanup_transaction_dirs(&groups_dir(root))?; + let path = group_path(root, group_id); + let mut group = read_group(&path)?; + let mut ordered = updates.to_vec(); + ordered.sort_by_key(|update| update.epoch); + for update in ordered { + if update.epoch <= group.meta.epoch { + continue; + } + validate_update(&group, &update)?; + let new_key = decode_key(&update.key_hex, &path.join("updates.json"))?; + group.retained_keys.insert(group.meta.epoch, group.key); + trim_oldest(&mut group.retained_keys); + group.key = new_key; + group.meta.epoch = update.epoch; + group.meta.admin = update.admin.clone(); + group.meta.admin_sign_key = update.admin_sign_key.clone(); + group.meta.revoked = update.revoked.clone(); + if let UpdateKind::Remove { particle_id } = &update.kind { + group + .members + .retain(|member| member.particle_id != *particle_id); + } + group.members.retain(|member| { + !group + .meta + .revoked + .iter() + .any(|revocation| revocation.particle_id == member.particle_id) + }); + group.updates.push(update); + if group.updates.len() > RETAINED_UPDATE_LIMIT { + group + .updates + .drain(..group.updates.len() - RETAINED_UPDATE_LIMIT); + } + let self_revoked = group + .meta + .revoked + .iter() + .any(|revocation| revocation.particle_id == self_id.as_str()); + if self_revoked { + fs::remove_dir_all(&path).map_err(|error| GroupError::io(&path, error))?; + tracing::info!("removed from group {} by its admin", group.meta.name); + return Ok(ApplyResult { + epoch: group.meta.epoch, + removed: true, + }); + } + write_group(root, &group)?; + } + Ok(ApplyResult { + epoch: group.meta.epoch, + removed: false, + }) +} + +fn validate_update(group: &Group, update: &GroupUpdate) -> Result<(), GroupError> { + if update.group_id != group.id { + return Err(GroupError::InvalidUpdate("group ID mismatch".into())); + } + if update.epoch != group.meta.epoch.saturating_add(1) { + return Err(GroupError::InvalidUpdate(format!( + "expected epoch {}, received {}", + group.meta.epoch.saturating_add(1), + update.epoch + ))); + } + if update.issued_by != group.meta.admin { + return Err(GroupError::InvalidUpdate( + "update issuer is not the current admin".into(), + )); + } + if update.admin != group.meta.admin { + let member = group + .members + .iter() + .find(|member| member.particle_id == update.admin) + .ok_or_else(|| GroupError::InvalidUpdate("new admin is not a current member".into()))?; + if update.admin_sign_key != member.sign_key { + return Err(GroupError::InvalidUpdate( + "new admin signing key does not match its member record".into(), + )); + } + } else if update.admin_sign_key != group.meta.admin_sign_key { + return Err(GroupError::InvalidUpdate( + "admin signing key changed without a handoff".into(), + )); + } + verify_update_signature(update, &group.meta.admin_sign_key)?; + decode_key(&update.key_hex, Path::new("updates.json"))?; + Ok(()) +} + +pub(crate) fn validate_join_history( + group_id: &str, + meta: &GroupMeta, + current_key: &[u8; 32], + retained_keys: &[(u32, String)], + updates: &[GroupUpdate], + predecessor: Option<(&str, &str)>, +) -> Result, GroupError> { + if retained_keys.len() > RETAINED_LIMIT { + return Err(GroupError::InvalidUpdate( + "too many retained group keys".into(), + )); + } + if updates.len() > RETAINED_UPDATE_LIMIT { + return Err(GroupError::InvalidUpdate( + "too many retained group updates".into(), + )); + } + + let mut keys = BTreeMap::new(); + for (epoch, key_hex) in retained_keys { + if *epoch >= meta.epoch { + return Err(GroupError::InvalidUpdate( + "retained key epoch is not below the grant epoch".into(), + )); + } + let key = decode_key(key_hex, Path::new("JoinGrant retained key")) + .map_err(|error| GroupError::InvalidUpdate(error.to_string()))?; + if keys.insert(*epoch, key).is_some() { + return Err(GroupError::InvalidUpdate( + "duplicate retained key epoch".into(), + )); + } + } + + if updates.is_empty() { + if meta.epoch != 0 || !keys.is_empty() { + return Err(GroupError::InvalidUpdate( + "join grant has incomplete update history".into(), + )); + } + return Ok(keys); + } + + for (index, update) in updates.iter().enumerate() { + if update.group_id != group_id { + return Err(GroupError::InvalidUpdate("group ID mismatch".into())); + } + if update.epoch == 0 + || (index > 0 && updates[index - 1].epoch.checked_add(1) != Some(update.epoch)) + { + return Err(GroupError::InvalidUpdate( + "join grant updates are not contiguous".into(), + )); + } + decode_key(&update.key_hex, Path::new("JoinGrant update")) + .map_err(|error| GroupError::InvalidUpdate(error.to_string()))?; + let known_predecessor = if index == 0 { + predecessor + } else { + let previous = &updates[index - 1]; + Some((previous.admin.as_str(), previous.admin_sign_key.as_str())) + }; + if let Some((admin, signing_key)) = known_predecessor { + if update.issued_by != admin { + return Err(GroupError::InvalidUpdate( + "update issuer does not match the previous admin".into(), + )); + } + verify_update_signature(update, signing_key)?; + } + } + + let latest = updates.last().expect("nonempty update history"); + let latest_key = decode_key(&latest.key_hex, Path::new("JoinGrant update")) + .map_err(|error| GroupError::InvalidUpdate(error.to_string()))?; + if latest.epoch != meta.epoch + || latest_key != *current_key + || latest.admin != meta.admin + || latest.admin_sign_key != meta.admin_sign_key + || latest.revoked != meta.revoked + { + return Err(GroupError::InvalidUpdate( + "latest update does not match the group grant".into(), + )); + } + Ok(keys) +} + +pub(crate) fn merge_previous_retained_keys( + retained_keys: &mut BTreeMap, + previous: &Group, +) { + for (epoch, key) in &previous.retained_keys { + retained_keys.entry(*epoch).or_insert(*key); + } + retained_keys + .entry(previous.meta.epoch) + .or_insert(previous.key); + trim_oldest(retained_keys); +} + +fn verify_update_signature(update: &GroupUpdate, signing_key: &str) -> Result<(), GroupError> { + let verifying_key_bytes = + hex::decode(signing_key).map_err(|error| GroupError::InvalidUpdate(error.to_string()))?; + let verifying_key_bytes: [u8; 32] = verifying_key_bytes + .try_into() + .map_err(|_| GroupError::InvalidUpdate("invalid admin signing key length".into()))?; + let verifying_key = VerifyingKey::from_bytes(&verifying_key_bytes) + .map_err(|error| GroupError::InvalidUpdate(error.to_string()))?; + let signature_bytes = hex::decode(&update.signature) + .map_err(|error| GroupError::InvalidUpdate(error.to_string()))?; + let signature = Signature::from_slice(&signature_bytes) + .map_err(|error| GroupError::InvalidUpdate(error.to_string()))?; + verifying_key + .verify(&update_signing_bytes(update)?, &signature) + .map_err(|error| GroupError::InvalidUpdate(error.to_string())) +} + +fn update_signing_bytes(update: &GroupUpdate) -> Result, GroupError> { + let mut unsigned = update.clone(); + unsigned.signature.clear(); + let bytes = serde_json::to_vec(&unsigned) + .map_err(|error| GroupError::InvalidUpdate(error.to_string()))?; + let mut message = UPDATE_DOMAIN.to_vec(); + message.extend_from_slice(&bytes); + Ok(message) +} + +fn decode_key(value: &str, path: &Path) -> Result<[u8; 32], GroupError> { + let decoded = + hex::decode(value).map_err(|error| GroupError::malformed(path, error.to_string()))?; + decoded + .try_into() + .map_err(|_| GroupError::malformed(path, "expected a 32-byte key")) +} + +fn trim_oldest(map: &mut BTreeMap) { + while map.len() > RETAINED_LIMIT { + map.pop_first(); + } +} + +pub fn updates_since(group: &Group, epoch: u32) -> Option> { + if epoch >= group.meta.epoch { + return Some(Vec::new()); + } + let updates = group + .updates + .iter() + .filter(|update| update.epoch > epoch) + .cloned() + .collect::>(); + let mut expected = epoch.checked_add(1)?; + for update in &updates { + if update.epoch != expected { + return None; + } + expected = expected.checked_add(1)?; + } + (expected == group.meta.epoch.checked_add(1)?).then_some(updates) +} + +pub fn write_status(root: &Path, group_id: &str, status: &GroupStatus) -> Result<(), GroupError> { + if !is_group_id(group_id) { + return Err(GroupError::NotFound(group_id.to_owned())); + } + let directory = groups_dir(root).join(".status"); + private_fs::create_private_directory(&directory).map_err(GroupError::private_fs)?; + let path = directory.join(format!("{group_id}.json")); + let bytes = serde_json::to_vec_pretty(status) + .map_err(|error| GroupError::malformed(&path, error.to_string()))?; + private_fs::atomic_write(&path, &bytes).map_err(GroupError::private_fs) +} + +pub fn read_status(root: &Path, group_id: &str) -> Result, GroupError> { + if !is_group_id(group_id) { + return Ok(None); + } + let path = groups_dir(root) + .join(".status") + .join(format!("{group_id}.json")); + let Some(bytes) = private_fs::read_regular_file(&path).map_err(GroupError::private_fs)? else { + return Ok(None); + }; + serde_json::from_slice(&bytes) + .map(Some) + .map_err(|error| GroupError::malformed(&path, error.to_string())) +} + pub fn leave(root: &Path, name_or_id: &str) -> Result { let _lock = lock_groups(root)?; let group = read_groups_unlocked(root)? @@ -346,7 +990,13 @@ fn read_groups_unlocked(root: &Path) -> Result, GroupError> { if !metadata.file_type().is_dir() { return Err(GroupError::malformed(&path, "expected a group directory")); } - groups.push(read_group(&path)?); + match read_group(&path) { + Ok(group) => groups.push(group), + Err(GroupError::Outdated { name }) => { + groups.push(read_outdated_group(&path, name)?); + } + Err(error) => return Err(error), + } } groups.sort_by(|left, right| { left.meta @@ -357,6 +1007,44 @@ fn read_groups_unlocked(root: &Path) -> Result, GroupError> { Ok(groups) } +fn read_outdated_group(path: &Path, name: String) -> Result { + let key_path = path.join("key"); + let key_bytes = private_fs::read_regular_file(&key_path) + .map_err(GroupError::private_fs)? + .ok_or_else(|| GroupError::malformed(&key_path, "missing key"))?; + let key_text = std::str::from_utf8(&key_bytes) + .map_err(|error| GroupError::malformed(&key_path, error.to_string()))? + .trim_end_matches(['\r', '\n']); + let decoded = hex::decode(key_text) + .map_err(|error| GroupError::malformed(&key_path, error.to_string()))?; + let key = decoded + .try_into() + .map_err(|_| GroupError::malformed(&key_path, "expected a 32-byte key"))?; + let id = path + .file_name() + .and_then(|name| name.to_str()) + .filter(|id| is_group_id(id)) + .ok_or_else(|| GroupError::malformed(path, "invalid group directory name"))?; + Ok(Group { + id: id.to_owned(), + key, + meta: GroupMeta { + version: 1, + name, + created_at: String::new(), + epoch: 0, + admin: String::new(), + admin_sign_key: String::new(), + revoked: Vec::new(), + leaving: false, + }, + members: Vec::new(), + retained_keys: BTreeMap::new(), + updates: Vec::new(), + outdated: true, + }) +} + fn read_group(path: &Path) -> Result { let metadata = fs::symlink_metadata(path).map_err(|error| { if error.kind() == io::ErrorKind::NotFound { @@ -396,7 +1084,22 @@ fn read_group(path: &Path) -> Result { let meta_bytes = private_fs::read_regular_file(&meta_path) .map_err(GroupError::private_fs)? .ok_or_else(|| GroupError::malformed(&meta_path, "missing metadata"))?; - let meta: GroupMeta = serde_json::from_slice(&meta_bytes) + let meta_value: serde_json::Value = serde_json::from_slice(&meta_bytes) + .map_err(|error| GroupError::malformed(&meta_path, error.to_string()))?; + if meta_value + .get("version") + .and_then(serde_json::Value::as_u64) + == Some(1) + { + let name = meta_value + .get("name") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| GroupError::malformed(&meta_path, "missing group name"))?; + return Err(GroupError::Outdated { + name: name.to_owned(), + }); + } + let meta: GroupMeta = serde_json::from_value(meta_value) .map_err(|error| GroupError::malformed(&meta_path, error.to_string()))?; validate_meta(&meta, &meta_path)?; @@ -407,6 +1110,7 @@ fn read_group(path: &Path) -> Result { let members: Vec = serde_json::from_slice(&members_bytes) .map_err(|error| GroupError::malformed(&members_path, error.to_string()))?; validate_members(&members, &members_path)?; + validate_admin_member(&meta, &members, &members_path)?; if members.is_empty() { return Err(GroupError::malformed(&members_path, "group has no members")); } @@ -415,10 +1119,50 @@ fn read_group(path: &Path) -> Result { .file_name() .and_then(|name| name.to_str()) .ok_or_else(|| GroupError::malformed(path, "invalid group directory name"))?; - if id != group_id(&key) { + if !is_group_id(id) { return Err(GroupError::malformed( path, - "directory name does not match the key's group ID", + "directory name is not an eight-character group ID", + )); + } + let retained_path = path.join("keys.json"); + let retained_bytes = private_fs::read_regular_file(&retained_path) + .map_err(GroupError::private_fs)? + .ok_or_else(|| GroupError::malformed(&retained_path, "missing retained key list"))?; + let retained_hex: BTreeMap = serde_json::from_slice(&retained_bytes) + .map_err(|error| GroupError::malformed(&retained_path, error.to_string()))?; + let mut retained_keys = BTreeMap::new(); + for (epoch, key_hex) in retained_hex { + let bytes = hex::decode(&key_hex) + .map_err(|error| GroupError::malformed(&retained_path, error.to_string()))?; + let retained_key: [u8; 32] = bytes + .try_into() + .map_err(|_| GroupError::malformed(&retained_path, "expected 32-byte keys"))?; + if epoch >= meta.epoch { + return Err(GroupError::malformed( + &retained_path, + "retained key epoch must be older than the current epoch", + )); + } + retained_keys.insert(epoch, retained_key); + } + if retained_keys.len() > RETAINED_LIMIT { + return Err(GroupError::malformed( + &retained_path, + "too many retained group keys", + )); + } + + let updates_path = path.join("updates.json"); + let updates_bytes = private_fs::read_regular_file(&updates_path) + .map_err(GroupError::private_fs)? + .ok_or_else(|| GroupError::malformed(&updates_path, "missing update log"))?; + let updates: Vec = serde_json::from_slice(&updates_bytes) + .map_err(|error| GroupError::malformed(&updates_path, error.to_string()))?; + if updates.len() > RETAINED_UPDATE_LIMIT { + return Err(GroupError::malformed( + &updates_path, + "too many retained group updates", )); } Ok(Group { @@ -426,40 +1170,52 @@ fn read_group(path: &Path) -> Result { key, meta, members, + retained_keys, + updates, + outdated: false, }) } fn write_group(root: &Path, group: &Group) -> Result<(), GroupError> { let path = group_path(root, &group.id); + if group.outdated { + return Err(GroupError::Outdated { + name: group.meta.name.clone(), + }); + } + if !is_group_id(&group.id) { + return Err(GroupError::malformed( + &path, + "directory name is not an eight-character group ID", + )); + } validate_meta(&group.meta, &path.join("meta.json"))?; validate_members(&group.members, &path.join("members.json"))?; + validate_admin_member(&group.meta, &group.members, &path.join("members.json"))?; if group.members.is_empty() { return Err(GroupError::malformed( &path.join("members.json"), "group has no members", )); } - if group.id != group_id(&group.key) { - return Err(GroupError::malformed( - &path.join("key"), - "group ID does not match the key", - )); - } let key = format!("{}\n", hex::encode(group.key)); let meta = serde_json::to_vec_pretty(&group.meta) .map_err(|error| GroupError::malformed(&path.join("meta.json"), error.to_string()))?; let members = serde_json::to_vec_pretty(&group.members) .map_err(|error| GroupError::malformed(&path.join("members.json"), error.to_string()))?; + let retained = group + .retained_keys + .iter() + .map(|(epoch, key)| (*epoch, hex::encode(key))) + .collect::>(); + let retained = serde_json::to_vec_pretty(&retained) + .map_err(|error| GroupError::malformed(&path.join("keys.json"), error.to_string()))?; + let updates = serde_json::to_vec_pretty(&group.updates) + .map_err(|error| GroupError::malformed(&path.join("updates.json"), error.to_string()))?; let replace_existing = match fs::symlink_metadata(&path) { Ok(metadata) if metadata.file_type().is_dir() => { - let existing = read_group(&path)?; - if existing.key != group.key { - return Err(GroupError::malformed( - &path.join("key"), - "refusing to replace a group directory with a different key", - )); - } + read_group(&path)?; true } Ok(_) => return Err(GroupError::UnsafePath(path)), @@ -474,6 +1230,10 @@ fn write_group(root: &Path, group: &Group) -> Result<(), GroupError> { .map_err(GroupError::private_fs)?; private_fs::atomic_write(&staging.join("members.json"), &members) .map_err(GroupError::private_fs)?; + private_fs::atomic_write(&staging.join("keys.json"), &retained) + .map_err(GroupError::private_fs)?; + private_fs::atomic_write(&staging.join("updates.json"), &updates) + .map_err(GroupError::private_fs)?; Ok(()) })(); if let Err(error) = write_result { @@ -632,6 +1392,14 @@ fn validate_meta(meta: &GroupMeta, path: &Path) -> Result<(), GroupError> { validate_name(&meta.name).map_err(|error| GroupError::malformed(path, error.to_string()))?; chrono::DateTime::parse_from_rfc3339(&meta.created_at) .map_err(|error| GroupError::malformed(path, error.to_string()))?; + ParticleId::parse(meta.admin.clone()) + .map_err(|error| GroupError::malformed(path, error.to_string()))?; + validate_hex_key(&meta.admin_sign_key, path, "admin signing key")?; + for revocation in &meta.revoked { + ParticleId::parse(revocation.particle_id.clone()) + .map_err(|error| GroupError::malformed(path, error.to_string()))?; + validate_hex_key(&revocation.static_key, path, "revoked static key")?; + } Ok(()) } @@ -649,24 +1417,47 @@ fn validate_members(members: &[Member], path: &Path) -> Result<(), GroupError> { Ok(()) } +fn validate_admin_member( + meta: &GroupMeta, + members: &[Member], + path: &Path, +) -> Result<(), GroupError> { + let admin = members + .iter() + .find(|member| member.particle_id == meta.admin) + .ok_or_else(|| GroupError::malformed(path, "admin is not a current group member"))?; + if admin.sign_key != meta.admin_sign_key { + return Err(GroupError::malformed( + path, + "admin signing key does not match its member record", + )); + } + Ok(()) +} + fn validate_member(member: &Member, path: &Path) -> Result<(), GroupError> { for id in [&member.particle_id, &member.added_by] { ParticleId::parse(id.clone()) .map_err(|error| GroupError::malformed(path, error.to_string()))?; } - if member.static_key.len() != 64 - || !member - .static_key + validate_hex_key(&member.static_key, path, "member static key")?; + validate_hex_key(&member.sign_key, path, "member signing key")?; + chrono::DateTime::parse_from_rfc3339(&member.added_at) + .map_err(|error| GroupError::malformed(path, error.to_string()))?; + Ok(()) +} + +fn validate_hex_key(value: &str, path: &Path, label: &str) -> Result<(), GroupError> { + if value.len() != 64 + || !value .bytes() .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) { return Err(GroupError::malformed( path, - "member static key must be 64 lowercase hexadecimal characters", + format!("{label} must be 64 lowercase hexadecimal characters"), )); } - chrono::DateTime::parse_from_rfc3339(&member.added_at) - .map_err(|error| GroupError::malformed(path, error.to_string()))?; Ok(()) } @@ -690,6 +1481,7 @@ mod tests { particle_id: identity.id.to_string(), name: "workstation".into(), static_key: hex::encode(entangle_transport::public_key(&identity.static_private)), + sign_key: signing_public_key(&identity.static_private), added_by: identity.id.to_string(), added_at: timestamp(), } @@ -701,11 +1493,25 @@ mod tests { particle_id: id.clone(), name: format!("peer-{index}"), static_key: hex::encode([index; 32]), + sign_key: signing_public_key(&[index; 32]), added_by: id, added_at: timestamp(), } } + fn group_meta(name: &str, admin: &Member) -> GroupMeta { + GroupMeta { + version: GROUP_VERSION, + name: name.into(), + created_at: timestamp(), + epoch: 0, + admin: admin.particle_id.clone(), + admin_sign_key: admin.sign_key.clone(), + revoked: Vec::new(), + leaving: false, + } + } + fn create_named(root: &Path, name: &str) -> Group { groups_create(root, name, saved_identity(root)) } @@ -728,12 +1534,626 @@ mod tests { fn create_list_and_find_by_name_or_id() { let root = tempdir().unwrap(); let group = create_named(root.path(), "work"); + let self_member = &group.members[0]; + assert_eq!(group.meta.epoch, 0); + assert_eq!(group.meta.admin, self_member.particle_id); + assert_eq!(group.meta.admin_sign_key, self_member.sign_key); + assert_eq!( + group.meta.admin_sign_key, + signing_public_key(&identity::read_saved(root.path()).unwrap().unwrap().1) + ); assert_eq!(list(root.path()).unwrap(), vec![group.clone()]); assert_eq!(find(root.path(), "work").unwrap(), Some(group.clone())); assert_eq!(find(root.path(), &group.id).unwrap(), Some(group)); assert_eq!(find(root.path(), "missing").unwrap(), None); } + #[test] + fn version_one_group_is_reported_as_outdated() { + let root = tempdir().unwrap(); + let path = root.path().join("groups").join("deadbeef"); + fs::create_dir_all(&path).unwrap(); + fs::write(path.join("key"), format!("{}\n", hex::encode([0x44; 32]))).unwrap(); + fs::write( + path.join("meta.json"), + br#"{"version":1,"name":"legacy","created_at":"2026-01-01T00:00:00Z","role":"admin"}"#, + ) + .unwrap(); + assert!(matches!( + read_group(&path), + Err(GroupError::Outdated { name }) if name == "legacy" + )); + let listed = list(root.path()).unwrap(); + assert_eq!(listed.len(), 1); + assert!(listed[0].outdated); + assert_eq!(listed[0].meta.name, "legacy"); + } + + #[test] + fn remove_rotates_key_records_revocation_and_retains_the_update() { + let admin_root = tempdir().unwrap(); + let admin = saved_identity(admin_root.path()); + let group = insert( + admin_root.path(), + [0x41; 32], + group_meta("work", &admin), + vec![admin.clone(), member(1)], + ) + .unwrap(); + let updated = issue_update( + admin_root.path(), + &group.id, + UpdateKind::Remove { + particle_id: member(1).particle_id, + }, + &identity::read_saved(admin_root.path()).unwrap().unwrap().1, + ) + .unwrap(); + assert_eq!(updated.meta.epoch, 1); + assert_ne!(updated.key, group.key); + assert_eq!(updated.members.len(), 1); + assert_eq!(updated.meta.revoked.len(), 1); + assert_eq!(updated.retained_keys.get(&0), Some(&group.key)); + assert_eq!(updated.updates.len(), 1); + let stored = find(admin_root.path(), &group.id).unwrap().unwrap(); + assert_eq!(stored, updated); + } + + #[test] + fn re_invitation_unrevokes_without_rotating_the_key() { + let root = tempdir().unwrap(); + let admin = saved_identity(root.path()); + let target = member(3); + let group = insert( + root.path(), + [0x47; 32], + group_meta("work", &admin), + vec![admin.clone(), target.clone()], + ) + .unwrap(); + let private = identity::read_saved(root.path()).unwrap().unwrap().1; + let removed = issue_update( + root.path(), + &group.id, + UpdateKind::Remove { + particle_id: target.particle_id.clone(), + }, + &private, + ) + .unwrap(); + let restored = + issue_unrevoke_update(root.path(), &group.id, &target.particle_id, &private).unwrap(); + assert_eq!(restored.meta.epoch, removed.meta.epoch + 1); + assert_eq!(restored.key, removed.key); + assert!(restored.meta.revoked.is_empty()); + } + + #[test] + fn valid_update_chain_applies_and_replays_are_ignored() { + let admin_root = tempdir().unwrap(); + let peer_root = tempdir().unwrap(); + let admin = saved_identity(admin_root.path()); + let peer = saved_identity(peer_root.path()); + let group = insert( + admin_root.path(), + [0x42; 32], + group_meta("work", &admin), + vec![admin.clone(), peer.clone()], + ) + .unwrap(); + insert_with_id( + peer_root.path(), + group.id.clone(), + group.key, + group.meta.clone(), + group.members.clone(), + ) + .unwrap(); + let private = identity::read_saved(admin_root.path()).unwrap().unwrap().1; + let first = + issue_update(admin_root.path(), &group.id, UpdateKind::Rekey, &private).unwrap(); + let second = + issue_update(admin_root.path(), &group.id, UpdateKind::Rekey, &private).unwrap(); + let updates = vec![first.updates[0].clone(), second.updates[1].clone()]; + let applied = apply_updates(peer_root.path(), &group.id, &updates, Some(&admin)).unwrap(); + assert_eq!(applied.epoch, 2); + assert!(!applied.removed); + let stored = find(peer_root.path(), &group.id).unwrap().unwrap(); + assert_eq!(stored.key, second.key); + assert_eq!(stored.updates.len(), 2); + assert_eq!( + apply_updates(peer_root.path(), &group.id, &updates, None) + .unwrap() + .epoch, + 2 + ); + } + + #[test] + fn replacement_conflicts_with_an_update_after_the_joiner_read() { + let admin_root = tempdir().unwrap(); + let joiner_root = tempdir().unwrap(); + let admin = saved_identity(admin_root.path()); + let joiner = saved_identity(joiner_root.path()); + let group = insert( + admin_root.path(), + [0x48; 32], + group_meta("work", &admin), + vec![admin.clone(), joiner], + ) + .unwrap(); + let mut local_meta = group.meta.clone(); + local_meta.name = "local-work".into(); + insert_with_id( + joiner_root.path(), + group.id.clone(), + group.key, + local_meta, + group.members.clone(), + ) + .unwrap(); + let read_at_start = find(joiner_root.path(), &group.id).unwrap().unwrap(); + let private = identity::read_saved(admin_root.path()).unwrap().unwrap().1; + let updated = + issue_update(admin_root.path(), &group.id, UpdateKind::Rekey, &private).unwrap(); + apply_updates( + joiner_root.path(), + &group.id, + &updated.updates, + Some(&admin), + ) + .unwrap(); + let inviter_group = find(admin_root.path(), &group.id).unwrap().unwrap(); + + assert!(matches!( + replace_existing( + joiner_root.path(), + inviter_group, + read_at_start.meta.epoch + ), + Err(GroupError::Conflict { name }) if name == "local-work" + )); + assert_eq!( + find(joiner_root.path(), &group.id) + .unwrap() + .unwrap() + .meta + .epoch, + updated.meta.epoch + ); + assert_eq!( + find(joiner_root.path(), &group.id).unwrap().unwrap().key, + updated.key + ); + } + + #[test] + fn join_history_checks_epochs_signatures_and_retention_limits() { + let root = tempdir().unwrap(); + let admin = saved_identity(root.path()); + let group = insert( + root.path(), + [0x4a; 32], + group_meta("work", &admin), + vec![admin.clone()], + ) + .unwrap(); + let private = identity::read_saved(root.path()).unwrap().unwrap().1; + issue_update(root.path(), &group.id, UpdateKind::Rekey, &private).unwrap(); + let current = issue_update(root.path(), &group.id, UpdateKind::Rekey, &private).unwrap(); + let retained = current + .retained_keys + .iter() + .map(|(epoch, key)| (*epoch, hex::encode(key))) + .collect::>(); + assert_eq!( + validate_join_history( + ¤t.id, + ¤t.meta, + ¤t.key, + &retained, + ¤t.updates, + Some((&group.meta.admin, &group.meta.admin_sign_key)), + ) + .unwrap(), + current.retained_keys + ); + + let mut unknown_first_signature = current.updates.clone(); + unknown_first_signature[0].signature = "00".repeat(64); + assert!(validate_join_history( + ¤t.id, + ¤t.meta, + ¤t.key, + &retained, + &unknown_first_signature, + None, + ) + .is_ok()); + assert!(validate_join_history( + ¤t.id, + ¤t.meta, + ¤t.key, + &retained, + &unknown_first_signature, + Some((&group.meta.admin, &group.meta.admin_sign_key)), + ) + .is_err()); + + let mut bad_signature = current.updates.clone(); + bad_signature[1].signature = "00".repeat(64); + assert!(validate_join_history( + ¤t.id, + ¤t.meta, + ¤t.key, + &retained, + &bad_signature, + None, + ) + .is_err()); + let mut gap = current.updates.clone(); + gap[1].epoch += 1; + assert!(validate_join_history( + ¤t.id, + ¤t.meta, + ¤t.key, + &retained, + &gap, + None, + ) + .is_err()); + assert!(validate_join_history( + ¤t.id, + ¤t.meta, + ¤t.key, + &retained, + ¤t.updates[..1], + None, + ) + .is_err()); + let mut wrong_group = current.updates.clone(); + wrong_group[1].group_id = "deadbeef".into(); + assert!(validate_join_history( + ¤t.id, + ¤t.meta, + ¤t.key, + &retained, + &wrong_group, + None, + ) + .is_err()); + assert!(validate_join_history( + ¤t.id, + ¤t.meta, + ¤t.key, + &[(current.meta.epoch, hex::encode(current.key))], + ¤t.updates, + None, + ) + .is_err()); + assert!(validate_join_history( + ¤t.id, + ¤t.meta, + ¤t.key, + &vec![(0, hex::encode([0x50; 32])); RETAINED_LIMIT + 1], + ¤t.updates, + None, + ) + .is_err()); + assert!(validate_join_history( + ¤t.id, + ¤t.meta, + ¤t.key, + &retained, + &vec![current.updates[0].clone(); RETAINED_UPDATE_LIMIT + 1], + None, + ) + .is_err()); + } + + #[test] + fn rollback_does_not_overwrite_an_update_after_the_grant_was_installed() { + let admin_root = tempdir().unwrap(); + let joiner_root = tempdir().unwrap(); + let admin = saved_identity(admin_root.path()); + let joiner = saved_identity(joiner_root.path()); + let group = insert( + admin_root.path(), + [0x49; 32], + group_meta("work", &admin), + vec![admin.clone(), joiner], + ) + .unwrap(); + let mut local_meta = group.meta.clone(); + local_meta.name = "local-work".into(); + insert_with_id( + joiner_root.path(), + group.id.clone(), + group.key, + local_meta, + group.members.clone(), + ) + .unwrap(); + let snapshot = find(joiner_root.path(), &group.id).unwrap().unwrap(); + let admin_private = identity::read_saved(admin_root.path()).unwrap().unwrap().1; + let grant = issue_update( + admin_root.path(), + &group.id, + UpdateKind::Rekey, + &admin_private, + ) + .unwrap(); + let previous = + replace_existing(joiner_root.path(), grant.clone(), snapshot.meta.epoch).unwrap(); + assert_eq!(previous.meta.name, "local-work"); + + let concurrent = issue_update( + admin_root.path(), + &group.id, + UpdateKind::Rekey, + &admin_private, + ) + .unwrap(); + apply_updates( + joiner_root.path(), + &group.id, + &[concurrent.updates.last().unwrap().clone()], + Some(&admin), + ) + .unwrap(); + let concurrent_state = find(joiner_root.path(), &group.id).unwrap().unwrap(); + assert!(matches!( + restore_existing_if_unchanged( + joiner_root.path(), + previous, + &grant + ), + Err(GroupError::Conflict { name }) if name == "local-work" + )); + assert_eq!( + find(joiner_root.path(), &group.id).unwrap().unwrap(), + concurrent_state + ); + } + + #[test] + fn rollback_does_not_restore_a_member_who_left_after_grant_installation() { + let admin_root = tempdir().unwrap(); + let joiner_root = tempdir().unwrap(); + let departed_root = tempdir().unwrap(); + let admin = saved_identity(admin_root.path()); + let joiner = saved_identity(joiner_root.path()); + let departed = saved_identity(departed_root.path()); + let group = insert( + admin_root.path(), + [0x4b; 32], + group_meta("work", &admin), + vec![admin.clone(), joiner, departed.clone()], + ) + .unwrap(); + let mut local_meta = group.meta.clone(); + local_meta.name = "local-work".into(); + insert_with_id( + joiner_root.path(), + group.id.clone(), + group.key, + local_meta, + group.members.clone(), + ) + .unwrap(); + let snapshot = find(joiner_root.path(), &group.id).unwrap().unwrap(); + let admin_private = identity::read_saved(admin_root.path()).unwrap().unwrap().1; + let grant = issue_update( + admin_root.path(), + &group.id, + UpdateKind::Rekey, + &admin_private, + ) + .unwrap(); + let mut installed = grant.clone(); + installed.meta.name = "local-work".into(); + let previous = + replace_existing(joiner_root.path(), installed.clone(), snapshot.meta.epoch).unwrap(); + + remove_member(joiner_root.path(), &group.id, &departed.particle_id).unwrap(); + let after_leave = find(joiner_root.path(), &group.id).unwrap().unwrap(); + assert!(!after_leave + .members + .iter() + .any(|member| member.particle_id == departed.particle_id)); + + assert!(matches!( + restore_existing_if_unchanged(joiner_root.path(), previous, &installed), + Err(GroupError::Conflict { name }) if name == "local-work" + )); + assert_eq!( + find(joiner_root.path(), &group.id).unwrap().unwrap(), + after_leave + ); + } + + #[test] + fn update_chain_rejects_invalid_signature_issuer_epoch_and_group() { + let admin_root = tempdir().unwrap(); + let peer_root = tempdir().unwrap(); + let admin = saved_identity(admin_root.path()); + let peer = saved_identity(peer_root.path()); + let group = insert( + admin_root.path(), + [0x43; 32], + group_meta("work", &admin), + vec![admin.clone(), peer], + ) + .unwrap(); + let private = identity::read_saved(admin_root.path()).unwrap().unwrap().1; + let update = issue_update(admin_root.path(), &group.id, UpdateKind::Rekey, &private) + .unwrap() + .updates[0] + .clone(); + let reject = |mutated: GroupUpdate| { + let root = tempdir().unwrap(); + let self_member = saved_identity(root.path()); + insert_with_id( + root.path(), + group.id.clone(), + group.key, + group.meta.clone(), + vec![admin.clone(), self_member], + ) + .unwrap(); + assert!(apply_updates(root.path(), &group.id, &[mutated], None).is_err()); + }; + let mut bad_signature = update.clone(); + bad_signature.signature = "00".repeat(64); + reject(bad_signature); + let mut wrong_issuer = update.clone(); + wrong_issuer.issued_by = member(7).particle_id; + reject(wrong_issuer); + let mut gap = update.clone(); + gap.epoch = 2; + reject(gap); + let mut wrong_group = update; + wrong_group.group_id = "deadbeef".into(); + reject(wrong_group); + } + + #[test] + fn self_revocation_deletes_the_group_and_handoff_changes_the_signer() { + let admin_root = tempdir().unwrap(); + let peer_root = tempdir().unwrap(); + let admin = saved_identity(admin_root.path()); + let peer = saved_identity(peer_root.path()); + let group = insert( + admin_root.path(), + [0x45; 32], + group_meta("work", &admin), + vec![admin.clone(), peer.clone()], + ) + .unwrap(); + insert_with_id( + peer_root.path(), + group.id.clone(), + group.key, + group.meta.clone(), + group.members.clone(), + ) + .unwrap(); + let admin_private = identity::read_saved(admin_root.path()).unwrap().unwrap().1; + let handoff = issue_update( + admin_root.path(), + &group.id, + UpdateKind::AdminHandoff { + to: peer.particle_id.clone(), + }, + &admin_private, + ) + .unwrap(); + apply_updates(peer_root.path(), &group.id, &handoff.updates, Some(&admin)).unwrap(); + let peer_private = identity::read_saved(peer_root.path()).unwrap().unwrap().1; + let peer_update = issue_update( + peer_root.path(), + &group.id, + UpdateKind::Rekey, + &peer_private, + ) + .unwrap(); + assert_eq!(peer_update.meta.admin, peer.particle_id); + let mut old_admin_update = peer_update.updates[1].clone(); + old_admin_update.epoch += 1; + old_admin_update.issued_by = admin.particle_id.clone(); + old_admin_update.signature = hex::encode( + signing_key(&admin_private) + .sign(&update_signing_bytes(&old_admin_update).unwrap()) + .to_bytes(), + ); + assert!(apply_updates( + peer_root.path(), + &group.id, + &[old_admin_update], + Some(&admin) + ) + .is_err()); + + let revoke_admin = issue_update( + peer_root.path(), + &group.id, + UpdateKind::Remove { + particle_id: admin.particle_id.clone(), + }, + &peer_private, + ) + .unwrap(); + let revoked = apply_updates( + admin_root.path(), + &group.id, + &revoke_admin.updates, + Some(&peer), + ) + .unwrap(); + assert!(revoked.removed); + assert!(find(admin_root.path(), &group.id).unwrap().is_none()); + } + + #[test] + fn retained_update_log_covers_every_negotiable_epoch() { + let root = tempdir().unwrap(); + let admin = saved_identity(root.path()); + let group = insert( + root.path(), + [0x46; 32], + group_meta("work", &admin), + vec![admin], + ) + .unwrap(); + let private = identity::read_saved(root.path()).unwrap().unwrap().1; + + let at_epoch_1 = issue_update(root.path(), &group.id, UpdateKind::Rekey, &private).unwrap(); + let at_epoch_2 = issue_update(root.path(), &group.id, UpdateKind::Rekey, &private).unwrap(); + for _ in 0..15 { + issue_update(root.path(), &group.id, UpdateKind::Rekey, &private).unwrap(); + } + + let at_epoch_17 = find(root.path(), &group.id).unwrap().unwrap(); + assert_eq!(at_epoch_17.meta.epoch, 17); + assert_eq!(at_epoch_17.retained_keys.len(), 16); + assert_eq!(at_epoch_17.retained_keys.keys().next(), Some(&1)); + assert_eq!(at_epoch_17.updates.len(), 17); + let updates_from_oldest = updates_since(&at_epoch_17, 1).unwrap(); + assert_eq!(updates_from_oldest.len(), 16); + let stale_at_1 = tempdir().unwrap(); + saved_identity(stale_at_1.path()); + std::fs::create_dir_all(groups_dir(stale_at_1.path())).unwrap(); + write_group(stale_at_1.path(), &at_epoch_1).unwrap(); + let applied = + apply_updates(stale_at_1.path(), &group.id, &updates_from_oldest, None).unwrap(); + assert_eq!(applied.epoch, 17); + assert_eq!( + find(stale_at_1.path(), &group.id).unwrap().unwrap().key, + at_epoch_17.key + ); + + issue_update(root.path(), &group.id, UpdateKind::Rekey, &private).unwrap(); + let at_epoch_18 = find(root.path(), &group.id).unwrap().unwrap(); + assert_eq!(at_epoch_18.retained_keys.len(), 16); + assert_eq!(at_epoch_18.retained_keys.keys().next(), Some(&2)); + assert_eq!(at_epoch_18.updates.len(), 17); + assert_eq!(at_epoch_18.updates.first().unwrap().epoch, 2); + assert_eq!(at_epoch_18.updates.last().unwrap().epoch, 18); + let updates_from_oldest = updates_since(&at_epoch_18, 2).unwrap(); + assert_eq!(updates_from_oldest.len(), 16); + let stale_at_2 = tempdir().unwrap(); + saved_identity(stale_at_2.path()); + std::fs::create_dir_all(groups_dir(stale_at_2.path())).unwrap(); + write_group(stale_at_2.path(), &at_epoch_2).unwrap(); + let applied = + apply_updates(stale_at_2.path(), &group.id, &updates_from_oldest, None).unwrap(); + assert_eq!(applied.epoch, 18); + assert_eq!( + find(stale_at_2.path(), &group.id).unwrap().unwrap().key, + at_epoch_18.key + ); + } + #[test] fn duplicate_name_and_ninth_group_are_rejected() { let root = tempdir().unwrap(); @@ -760,12 +2180,7 @@ mod tests { let existing = insert( root.path(), existing_key, - GroupMeta { - version: GROUP_VERSION, - name: "existing".into(), - role: Role::Admin, - created_at: timestamp(), - }, + group_meta("existing", &self_member), vec![self_member.clone()], ) .unwrap(); @@ -854,7 +2269,7 @@ mod tests { } #[test] - fn group_directory_name_must_match_key_id() { + fn stable_group_id_does_not_change_with_the_current_key() { let root = tempdir().unwrap(); let group = create_named(root.path(), "work"); let original_path = root.path().join("groups").join(&group.id); @@ -867,10 +2282,10 @@ mod tests { assert_eq!(renamed_id.len(), 8); fs::rename(&original_path, &renamed_path).unwrap(); - match list(root.path()).unwrap_err() { - GroupError::Malformed { path, .. } => assert_eq!(path, renamed_path), - error => panic!("expected malformed group data, got {error}"), - } + let mut groups = list(root.path()).unwrap(); + let renamed = groups.remove(0); + assert_eq!(renamed.id, renamed_id); + assert_eq!(renamed.key, group.key); } #[test] diff --git a/crates/entangle/src/identity.rs b/crates/entangle/src/identity.rs index 5e22fed..c1a976c 100644 --- a/crates/entangle/src/identity.rs +++ b/crates/entangle/src/identity.rs @@ -120,6 +120,20 @@ pub fn claim(root: &Path) -> Result { persistent(id, lock, root) } +pub fn is_running(root: &Path) -> bool { + let lock_path = root.join("identity.lock"); + let Ok(lock) = OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(lock_path) + else { + return false; + }; + matches!(lock.try_lock(), Err(std::fs::TryLockError::WouldBlock)) +} + /// Reads a saved identity without acquiring its process lock or creating files. pub fn read_saved(root: &Path) -> Result, IdentityError> { let identity_path = root.join("identity"); diff --git a/crates/entangle/src/invite/protocol.rs b/crates/entangle/src/invite/protocol.rs index f3f09ca..5323362 100644 --- a/crates/entangle/src/invite/protocol.rs +++ b/crates/entangle/src/invite/protocol.rs @@ -1,5 +1,5 @@ use crate::{ - groups::{self, Group, GroupError, GroupMeta, Member, Role}, + groups::{self, Group, GroupError, GroupMeta, Member}, identity, invite::code::InviteCode, }; @@ -26,6 +26,7 @@ use tokio::{ }; const MAX_FRAME_BYTES: usize = 64 * 1024; +const JOIN_GRANT_FRAME_MARGIN: usize = 4 * 1024; const SESSION_TIMEOUT: Duration = Duration::from_secs(30); const SPAKE_TIMEOUT: Duration = Duration::from_secs(5); const MAX_SESSIONS: usize = 8; @@ -203,7 +204,14 @@ impl InviteServer { else { return Err(GroupError::MissingIdentity(root).into()); }; - groups::find(&root, &group_id)?.ok_or_else(|| GroupError::NotFound(group_id.clone()))?; + let group = groups::find(&root, &group_id)? + .ok_or_else(|| GroupError::NotFound(group_id.clone()))?; + if group.outdated { + return Err(GroupError::Outdated { + name: group.meta.name, + } + .into()); + } Ok(Self { root, group_id, @@ -316,8 +324,9 @@ impl InviteServer { .send( stream, &Message::GroupInfo { - name: group.meta.name, - group_id: group.id, + name: group.meta.name.clone(), + group_id: group.id.clone(), + epoch: group.meta.epoch, inviter_name: self.inviter_name.clone(), }, ) @@ -333,10 +342,12 @@ impl InviteServer { particle_id, name, static_key, + sign_key, } => Member { particle_id, name, static_key, + sign_key, added_by: self.inviter_id.clone(), added_at: Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), }, @@ -349,6 +360,43 @@ impl InviteServer { let joined_name = request.name.clone(); self.state.reserve().await?; *reserved = true; + let current = groups::find(&self.root, &self.group_id)? + .ok_or_else(|| GroupError::NotFound(self.group_id.clone()))?; + if current + .meta + .revoked + .iter() + .any(|revocation| revocation.particle_id == request.particle_id) + { + if current.meta.admin != self.inviter_id { + sender + .send( + stream, + &Message::Decline { + reason: DeclineReason::Removed, + }, + ) + .await?; + return Err(InviteError::Declined { + reason: DeclineReason::Removed.as_str().to_owned(), + group_name: Some(current.meta.name), + }); + } + let Some((_, static_private)) = + identity::read_saved(&self.root).map_err(|error| GroupError::Malformed { + path: self.root.clone(), + reason: error.to_string(), + })? + else { + return Err(GroupError::MissingIdentity(self.root.clone()).into()); + }; + groups::issue_unrevoke_update( + &self.root, + &self.group_id, + &request.particle_id, + &static_private, + )?; + } let mut proposed = groups::find(&self.root, &self.group_id)? .ok_or_else(|| GroupError::NotFound(self.group_id.clone()))?; if let Some(existing) = proposed @@ -365,17 +413,8 @@ impl InviteServer { .members .sort_by(|left, right| left.particle_id.cmp(&right.particle_id)); } - sender - .send( - stream, - &Message::JoinGrant { - name: proposed.meta.name, - key_hex: hex::encode(proposed.key), - created_at: proposed.meta.created_at, - members: proposed.members, - }, - ) - .await?; + let grant = bounded_join_grant(&proposed)?; + sender.send(stream, &grant).await?; if receiver.receive::<_, Message>(stream).await? != Message::Done { return Err(InviteError::Message("expected Done".into())); } @@ -398,12 +437,53 @@ impl InviteServer { } } +fn bounded_join_grant(group: &Group) -> Result { + let mut first_update = 0; + loop { + let updates = group.updates[first_update..].to_vec(); + let oldest_key_epoch = updates.first().map(|update| update.epoch.saturating_sub(1)); + let retained_keys = oldest_key_epoch + .into_iter() + .flat_map(|oldest| { + group + .retained_keys + .range(oldest..) + .map(|(epoch, key)| (*epoch, hex::encode(key))) + }) + .collect(); + let grant = Message::JoinGrant { + group_id: group.id.clone(), + name: group.meta.name.clone(), + key_hex: hex::encode(group.key), + created_at: group.meta.created_at.clone(), + epoch: group.meta.epoch, + admin: group.meta.admin.clone(), + admin_sign_key: group.meta.admin_sign_key.clone(), + revoked: group.meta.revoked.clone(), + members: group.members.clone(), + retained_keys, + updates, + }; + let plaintext_len = serde_json::to_vec(&grant) + .map_err(|error| InviteError::Message(error.to_string()))? + .len(); + if plaintext_len + 16 <= MAX_FRAME_BYTES - JOIN_GRANT_FRAME_MARGIN { + return Ok(grant); + } + if first_update == group.updates.len() { + return Err(InviteError::Oversize); + } + first_update += 1; + } +} + #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(tag = "type", rename_all = "snake_case")] enum Message { GroupInfo { name: String, group_id: String, + epoch: u32, inviter_name: String, }, Decline { @@ -413,12 +493,20 @@ enum Message { particle_id: String, name: String, static_key: String, + sign_key: String, }, JoinGrant { + group_id: String, name: String, key_hex: String, created_at: String, + epoch: u32, + admin: String, + admin_sign_key: String, + revoked: Vec, members: Vec, + retained_keys: Vec<(u32, String)>, + updates: Vec, }, Done, Committed, @@ -429,6 +517,7 @@ enum Message { enum DeclineReason { AlreadyMember, NameTaken, + Removed, } impl DeclineReason { @@ -436,6 +525,7 @@ impl DeclineReason { match self { Self::AlreadyMember => "already_member", Self::NameTaken => "name_taken", + Self::Removed => "removed", } } } @@ -444,6 +534,29 @@ impl DeclineReason { pub struct Joined { pub group: Group, pub inviter_name: String, + pub refreshed: bool, +} + +enum JoinMutation { + Inserted(String), + Refreshed { + previous: Box, + installed: Box, + }, +} + +struct GroupGrant { + group_id: String, + name: String, + key_hex: String, + created_at: String, + epoch: u32, + admin: String, + admin_sign_key: String, + revoked: Vec, + members: Vec, + retained_keys: Vec<(u32, String)>, + updates: Vec, } /// Joins an inviter over an arbitrary byte stream. @@ -487,7 +600,7 @@ where else { return Err(GroupError::MissingIdentity(root.to_owned()).into()); }; - let mut inserted_group = None; + let mut mutation = None; let result = timeout( session_timeout, join_inner( @@ -497,7 +610,7 @@ where local_name, particle_name, (id, static_private), - &mut inserted_group, + &mut mutation, ), ) .await; @@ -508,10 +621,19 @@ where match result { Ok(joined) => Ok(joined), Err(error) => { - if let Some(group_id) = inserted_group.take() { - match groups::leave(root, &group_id) { - Ok(_) | Err(GroupError::NotFound(_)) => Err(InviteError::Incomplete), - Err(error) => Err(error.into()), + if let Some(mutation) = mutation.take() { + match mutation { + JoinMutation::Inserted(group_id) => match groups::leave(root, &group_id) { + Ok(_) | Err(GroupError::NotFound(_)) => Err(InviteError::Incomplete), + Err(error) => Err(error.into()), + }, + JoinMutation::Refreshed { + previous, + installed, + } => match groups::restore_existing_if_unchanged(root, *previous, &installed) { + Ok(_) => Err(InviteError::Incomplete), + Err(error) => Err(error.into()), + }, } } else { Err(error) @@ -527,12 +649,13 @@ async fn join_inner( local_name: Option<&str>, particle_name: &str, identity: (entangle_core::ParticleId, [u8; 32]), - inserted_group: &mut Option, + mutation: &mut Option, ) -> Result where S: AsyncRead + AsyncWrite + Unpin, { let (id, static_private) = identity; + let local_groups = groups::list(root)?; let (spake, local_msg) = Spake2::::start_b( &Password::new(code.to_string().as_bytes()), &Identity::new(ID_INVITER), @@ -555,26 +678,37 @@ where let mut sender = SecureJson::new(keys.enc_joiner); let mut receiver = SecureJson::new(keys.enc_inviter); - let (name, group_id, inviter_name) = match receiver.receive::<_, Message>(stream).await? { - Message::GroupInfo { - name, - group_id, - inviter_name, - } => (name, group_id, inviter_name), - message => { - return Err(InviteError::Message(format!( - "expected group info, got {message:?}" - ))) - } - }; - let local_group_name = local_name.unwrap_or(&name); + let (name, group_id, inviter_epoch, inviter_name) = + match receiver.receive::<_, Message>(stream).await? { + Message::GroupInfo { + name, + group_id, + epoch, + inviter_name, + } => (name, group_id, epoch, inviter_name), + message => { + return Err(InviteError::Message(format!( + "expected group info, got {message:?}" + ))) + } + }; + let existing_group = local_groups + .iter() + .find(|group| group.id == group_id) + .cloned(); + let refresh_existing = existing_group + .as_ref() + .is_some_and(|group| !group.outdated && group.meta.epoch < inviter_epoch); + let local_group_name = existing_group + .as_ref() + .map(|group| group.meta.name.as_str()) + .unwrap_or_else(|| local_name.unwrap_or(&name)); groups::validate_name(local_group_name)?; - let local_groups = groups::list(root)?; - let decline = if local_groups.iter().any(|group| group.id == group_id) { + let decline = if existing_group.is_some() && !refresh_existing { Some(DeclineReason::AlreadyMember) } else if local_groups .iter() - .any(|group| group.meta.name == local_group_name) + .any(|group| group.id != group_id && group.meta.name == local_group_name) { Some(DeclineReason::NameTaken) } else { @@ -584,12 +718,14 @@ where let group_name = match reason { DeclineReason::AlreadyMember => &name, DeclineReason::NameTaken => local_group_name, + DeclineReason::Removed => &name, }; sender.send(stream, &Message::Decline { reason }).await?; return Err(InviteError::Declined { reason: match reason { DeclineReason::AlreadyMember => "already_member", DeclineReason::NameTaken => "name_taken", + DeclineReason::Removed => "removed", } .to_owned(), group_name: Some(group_name.to_owned()), @@ -602,60 +738,133 @@ where particle_id: id.to_string(), name: particle_name.to_owned(), static_key: hex::encode(entangle_transport::public_key(&static_private)), + sign_key: groups::signing_public_key(&static_private), }, ) .await?; let grant = match receiver.receive::<_, Message>(stream).await? { + Message::Decline { reason } => { + return Err(InviteError::Declined { + reason: reason.as_str().to_owned(), + group_name: (reason == DeclineReason::Removed).then(|| name.clone()), + }); + } Message::JoinGrant { + group_id, name, key_hex, created_at, + epoch, + admin, + admin_sign_key, + revoked, members, - } => (name, key_hex, created_at, members), + retained_keys, + updates, + } => GroupGrant { + group_id, + name, + key_hex, + created_at, + epoch, + admin, + admin_sign_key, + revoked, + members, + retained_keys, + updates, + }, message => { return Err(InviteError::Message(format!( "expected join grant, got {message:?}" ))) } }; - if grant.0 != name - || grant.1.len() != 64 + if grant.group_id != group_id + || grant.name != name + || grant.key_hex.len() != 64 || !grant - .1 + .key_hex .bytes() .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) { return Err(InviteError::Message("invalid join grant".into())); } - let key: [u8; 32] = hex::decode(&grant.1) + let key: [u8; 32] = hex::decode(&grant.key_hex) .map_err(|_| InviteError::Message("invalid group key".into()))? .try_into() .map_err(|_| InviteError::Message("invalid group key".into()))?; - if groups::group_id(&key) != group_id { - return Err(InviteError::Message( - "group ID does not match the granted key".into(), - )); - } - let group = groups::insert( - root, + let meta = GroupMeta { + version: 2, + name: local_group_name.to_owned(), + created_at: grant.created_at.clone(), + epoch: grant.epoch, + admin: grant.admin.clone(), + admin_sign_key: grant.admin_sign_key.clone(), + revoked: grant.revoked.clone(), + leaving: false, + }; + let known_predecessor = existing_group.as_ref().and_then(|previous| { + let next_epoch = previous.meta.epoch.checked_add(1)?; + grant + .updates + .first() + .filter(|update| update.epoch == next_epoch) + .map(|_| { + ( + previous.meta.admin.as_str(), + previous.meta.admin_sign_key.as_str(), + ) + }) + }); + let mut retained_keys = groups::validate_join_history( + &grant.group_id, + &meta, + &key, + &grant.retained_keys, + &grant.updates, + known_predecessor, + )?; + let group = Group { + id: grant.group_id, key, - GroupMeta { - version: 1, - name: local_group_name.to_owned(), - role: Role::Member, - created_at: grant.2, + meta, + members: grant.members, + retained_keys: { + if let Some(previous) = existing_group.as_ref().filter(|_| refresh_existing) { + groups::merge_previous_retained_keys(&mut retained_keys, previous); + } + retained_keys }, - grant.3, - )?; - *inserted_group = Some(group.id.clone()); + updates: grant.updates, + outdated: false, + }; + let group = if refresh_existing { + let expected_epoch = existing_group + .as_ref() + .expect("refresh requires an existing group") + .meta + .epoch; + let previous = groups::replace_existing(root, group.clone(), expected_epoch)?; + *mutation = Some(JoinMutation::Refreshed { + previous: Box::new(previous), + installed: Box::new(group.clone()), + }); + group + } else { + let inserted = groups::insert_group(root, group.clone())?; + *mutation = Some(JoinMutation::Inserted(inserted.id.clone())); + inserted + }; sender.send(stream, &Message::Done).await?; if receiver.receive::<_, Message>(stream).await? != Message::Committed { return Err(InviteError::Message("expected commit confirmation".into())); } - *inserted_group = None; + *mutation = None; Ok(Joined { group, inviter_name, + refreshed: refresh_existing, }) } @@ -840,6 +1049,7 @@ mod tests { particle_id: id.clone(), name: name.into(), static_key: hex::encode(entangle_transport::public_key(&identity.static_private)), + sign_key: groups::signing_public_key(&identity.static_private), added_by: id, added_at: Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), }; @@ -853,10 +1063,14 @@ mod tests { root, [0x42; 32], GroupMeta { - version: 1, + version: 2, name: "work".into(), - role: Role::Admin, created_at: Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + epoch: 0, + admin: member.particle_id.clone(), + admin_sign_key: member.sign_key.clone(), + revoked: vec![], + leaving: false, }, vec![member], ) @@ -899,6 +1113,101 @@ mod tests { ) } + #[tokio::test] + async fn admin_reinvitation_unrevokes_a_removed_particle_before_granting() { + let inviter_root = tempfile::tempdir().unwrap(); + let group = setup_group(inviter_root.path()); + let joiner_root = tempfile::tempdir().unwrap(); + let removed = saved_member(joiner_root.path(), "removed"); + groups::add_member(inviter_root.path(), &group.id, removed.clone()).unwrap(); + let admin_private = identity::read_saved(inviter_root.path()) + .unwrap() + .unwrap() + .1; + let revoked = groups::issue_update( + inviter_root.path(), + &group.id, + entangle_core::UpdateKind::Remove { + particle_id: removed.particle_id.clone(), + }, + &admin_private, + ) + .unwrap(); + let server = server(inviter_root.path(), &revoked, 1, DEFAULT_TTL); + + let joined = successful_join(&server, joiner_root.path(), "removed") + .await + .unwrap(); + let inviter_group = groups::find(inviter_root.path(), &group.id) + .unwrap() + .unwrap(); + assert_eq!(joined.group.meta.epoch, 2); + assert_eq!(inviter_group.meta.epoch, 2); + assert_eq!(joined.group.key, revoked.key); + assert!(joined.group.meta.revoked.is_empty()); + assert!(inviter_group.meta.revoked.is_empty()); + assert!(joined + .group + .members + .iter() + .any(|member| member.particle_id == removed.particle_id)); + } + + #[tokio::test] + async fn non_admin_inviter_declines_a_revoked_particle() { + let admin_root = tempfile::tempdir().unwrap(); + let group = setup_group(admin_root.path()); + let inviter_root = tempfile::tempdir().unwrap(); + let inviter = saved_member(inviter_root.path(), "member"); + let joiner_root = tempfile::tempdir().unwrap(); + let removed = saved_member(joiner_root.path(), "removed"); + groups::add_member(admin_root.path(), &group.id, inviter.clone()).unwrap(); + groups::add_member(admin_root.path(), &group.id, removed.clone()).unwrap(); + let admin_private = identity::read_saved(admin_root.path()).unwrap().unwrap().1; + let revoked = groups::issue_update( + admin_root.path(), + &group.id, + entangle_core::UpdateKind::Remove { + particle_id: removed.particle_id.clone(), + }, + &admin_private, + ) + .unwrap(); + groups::insert_with_id( + inviter_root.path(), + revoked.id.clone(), + revoked.key, + revoked.meta.clone(), + revoked.members.clone(), + ) + .unwrap(); + let server = InviteServer::new( + inviter_root.path(), + revoked.id.clone(), + inviter.particle_id, + code(), + 1, + DEFAULT_TTL, + ) + .unwrap(); + + let result = successful_join(&server, joiner_root.path(), "removed").await; + assert!( + matches!( + &result, + Err(InviteError::Declined { reason, group_name }) + if reason == "removed" && group_name.as_deref() == Some("work") + ), + "unexpected invitation result: {result:?}" + ); + let unchanged = groups::find(inviter_root.path(), &revoked.id) + .unwrap() + .unwrap(); + assert_eq!(unchanged.meta.epoch, revoked.meta.epoch); + assert_eq!(unchanged.meta.revoked, revoked.meta.revoked); + assert_eq!(server.state().counts().await, (0, 1)); + } + async fn successful_join( server: &InviteServer, root: &Path, @@ -926,7 +1235,12 @@ mod tests { let joined = successful_join(&server, joiner_root.path(), "joiner") .await .unwrap(); - assert_eq!(joined.group.meta.role, Role::Member); + let joiner_id = identity::read_saved(joiner_root.path()) + .unwrap() + .unwrap() + .0 + .to_string(); + assert!(!joined.group.meta.is_admin(&joiner_id)); assert_eq!(joined.group.members.len(), 2); assert!(joined .group @@ -1043,6 +1357,7 @@ mod tests { particle_id: joiner_id.to_string(), name: "joiner".into(), static_key: hex::encode(entangle_transport::public_key(&joiner_private)), + sign_key: groups::signing_public_key(&joiner_private), }, ) .await @@ -1118,6 +1433,246 @@ mod tests { assert_eq!(server.state().counts().await, (1, 1)); } + #[tokio::test] + async fn failed_refresh_confirmation_does_not_restore_a_member_who_left() { + let inviter_root = tempfile::tempdir().unwrap(); + let group = setup_group(inviter_root.path()); + let joiner_root = tempfile::tempdir().unwrap(); + let joiner_member = saved_member(joiner_root.path(), "joiner"); + let departed_root = tempfile::tempdir().unwrap(); + let departed = saved_member(departed_root.path(), "departed"); + groups::add_member(inviter_root.path(), &group.id, joiner_member.clone()).unwrap(); + groups::add_member(inviter_root.path(), &group.id, departed.clone()).unwrap(); + let admin_private = identity::read_saved(inviter_root.path()) + .unwrap() + .unwrap() + .1; + let grant = groups::issue_update( + inviter_root.path(), + &group.id, + entangle_core::UpdateKind::Rekey, + &admin_private, + ) + .unwrap(); + let mut local_meta = group.meta.clone(); + local_meta.name = "local-work".into(); + groups::insert_with_id( + joiner_root.path(), + group.id.clone(), + group.key, + local_meta, + grant.members.clone(), + ) + .unwrap(); + + let (joiner_stream, mut inviter_stream) = duplex(1024 * 1024); + let (installed_tx, installed_rx) = tokio::sync::oneshot::channel(); + let (continue_tx, continue_rx) = tokio::sync::oneshot::channel(); + let invitation_group = grant.clone(); + let inviter_task = tokio::spawn(async move { + let (spake, local_msg) = Spake2::::start_a( + &Password::new(code().to_string().as_bytes()), + &Identity::new(ID_INVITER), + &Identity::new(ID_JOINER), + ); + write_tagged_frame(&mut inviter_stream, FRAME_SPAKE, &local_msg) + .await + .unwrap(); + let peer_frame = read_frame(&mut inviter_stream).await.unwrap(); + let peer_msg = tagged_body(&peer_frame, FRAME_SPAKE).unwrap(); + let shared = spake.finish(peer_msg).unwrap(); + let keys = DerivedKeys::new(&shared); + let confirmation_frame = read_frame(&mut inviter_stream).await.unwrap(); + let confirmation = tagged_body(&confirmation_frame, FRAME_CONFIRM).unwrap(); + assert!(bool::from(keys.confirm_joiner.ct_eq(confirmation))); + write_tagged_frame(&mut inviter_stream, FRAME_CONFIRM, &keys.confirm_inviter) + .await + .unwrap(); + + let mut sender = SecureJson::new(keys.enc_inviter); + let mut receiver = SecureJson::new(keys.enc_joiner); + sender + .send( + &mut inviter_stream, + &Message::GroupInfo { + name: invitation_group.meta.name.clone(), + group_id: invitation_group.id.clone(), + epoch: invitation_group.meta.epoch, + inviter_name: "inviter".into(), + }, + ) + .await + .unwrap(); + assert!(matches!( + receiver + .receive::<_, Message>(&mut inviter_stream) + .await + .unwrap(), + Message::JoinRequest { .. } + )); + sender + .send( + &mut inviter_stream, + &bounded_join_grant(&invitation_group).unwrap(), + ) + .await + .unwrap(); + assert_eq!( + receiver + .receive::<_, Message>(&mut inviter_stream) + .await + .unwrap(), + Message::Done + ); + installed_tx.send(()).unwrap(); + continue_rx.await.unwrap(); + drop(inviter_stream); + }); + let joiner_path = joiner_root.path().to_path_buf(); + let joiner_task = tokio::spawn(async move { + join_with_timeout( + joiner_stream, + &joiner_path, + code(), + None, + "joiner", + Duration::from_secs(5), + ) + .await + }); + + installed_rx.await.unwrap(); + groups::remove_member(joiner_root.path(), &group.id, &departed.particle_id).unwrap(); + continue_tx.send(()).unwrap(); + + assert!(matches!( + joiner_task.await.unwrap(), + Err(InviteError::Group(GroupError::Conflict { name })) if name == "local-work" + )); + let current = groups::find(joiner_root.path(), &group.id) + .unwrap() + .unwrap(); + assert_eq!(current.meta.epoch, grant.meta.epoch); + assert!(!current + .members + .iter() + .any(|member| member.particle_id == departed.particle_id)); + inviter_task.await.unwrap(); + } + + #[tokio::test] + async fn oversized_update_history_is_bounded_and_invitation_succeeds() { + let inviter_root = tempfile::tempdir().unwrap(); + let admin = saved_member(inviter_root.path(), "inviter"); + let revoked = (0..40) + .map(|index| entangle_core::Revocation { + particle_id: entangle_core::ParticleId::generate().to_string(), + static_key: format!("{index:064x}"), + }) + .collect::>(); + let group = groups::insert( + inviter_root.path(), + [0x52; 32], + GroupMeta { + version: 2, + name: "work".into(), + created_at: Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + epoch: 0, + admin: admin.particle_id.clone(), + admin_sign_key: admin.sign_key.clone(), + revoked: revoked.clone(), + leaving: false, + }, + vec![admin], + ) + .unwrap(); + let admin_private = identity::read_saved(inviter_root.path()) + .unwrap() + .unwrap() + .1; + let mut group = group; + for _ in 0..17 { + group = groups::issue_update( + inviter_root.path(), + &group.id, + entangle_core::UpdateKind::Rekey, + &admin_private, + ) + .unwrap(); + } + assert!(group + .updates + .iter() + .all(|update| update.revoked.len() == 40)); + let full_grant = Message::JoinGrant { + group_id: group.id.clone(), + name: group.meta.name.clone(), + key_hex: hex::encode(group.key), + created_at: group.meta.created_at.clone(), + epoch: group.meta.epoch, + admin: group.meta.admin.clone(), + admin_sign_key: group.meta.admin_sign_key.clone(), + revoked: group.meta.revoked.clone(), + members: group.members.clone(), + retained_keys: group + .retained_keys + .iter() + .map(|(epoch, key)| (*epoch, hex::encode(key))) + .collect(), + updates: group.updates.clone(), + }; + assert!( + serde_json::to_vec(&full_grant).unwrap().len() + 16 > MAX_FRAME_BYTES, + "the regression fixture must exceed the invite frame limit" + ); + let bounded = bounded_join_grant(&group).unwrap(); + let Message::JoinGrant { + retained_keys, + updates, + .. + } = bounded + else { + unreachable!() + }; + assert!(updates.len() < group.updates.len()); + assert!( + serde_json::to_vec(&Message::JoinGrant { + group_id: group.id.clone(), + name: group.meta.name.clone(), + key_hex: hex::encode(group.key), + created_at: group.meta.created_at.clone(), + epoch: group.meta.epoch, + admin: group.meta.admin.clone(), + admin_sign_key: group.meta.admin_sign_key.clone(), + revoked: group.meta.revoked.clone(), + members: group.members.clone(), + retained_keys: retained_keys.clone(), + updates: updates.clone(), + }) + .unwrap() + .len() + + 16 + <= MAX_FRAME_BYTES - JOIN_GRANT_FRAME_MARGIN + ); + let oldest_key_epoch = updates.first().unwrap().epoch.saturating_sub(1); + assert!(retained_keys + .iter() + .all(|(epoch, _)| *epoch >= oldest_key_epoch)); + + let invite_server = server(inviter_root.path(), &group, 1, DEFAULT_TTL); + let joiner_root = tempfile::tempdir().unwrap(); + let _joiner = saved_member(joiner_root.path(), "joiner"); + let joined = successful_join(&invite_server, joiner_root.path(), "joiner") + .await + .unwrap(); + assert_eq!(joined.group.meta.epoch, 17); + assert_eq!( + joined.group.updates.first().unwrap().epoch, + updates.first().unwrap().epoch + ); + assert_eq!(joined.group.updates.last().unwrap().epoch, 17); + } + #[tokio::test] async fn ninth_concurrent_session_is_refused_without_closing_invite() { let inviter_root = tempfile::tempdir().unwrap(); @@ -1205,6 +1760,7 @@ mod tests { particle_id: good_id.to_string(), name: "good-joiner".into(), static_key: hex::encode(entangle_transport::public_key(&good_private)), + sign_key: groups::signing_public_key(&good_private), }, ) .await @@ -1275,12 +1831,16 @@ mod tests { already_root.path(), group.key, GroupMeta { - version: 1, + version: 2, name: "work".into(), - role: Role::Member, created_at: group.meta.created_at.clone(), + epoch: 0, + admin: group.meta.admin.clone(), + admin_sign_key: group.meta.admin_sign_key.clone(), + revoked: vec![], + leaving: false, }, - vec![already_member], + vec![group.members[0].clone(), already_member], ) .unwrap(); assert!(matches!( @@ -1296,10 +1856,14 @@ mod tests { taken_root.path(), [0x11; 32], GroupMeta { - version: 1, + version: 2, name: "work".into(), - role: Role::Admin, created_at: Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + epoch: 0, + admin: _taken.particle_id.clone(), + admin_sign_key: _taken.sign_key.clone(), + revoked: vec![], + leaving: false, }, vec![_taken], ) diff --git a/crates/entangle/src/main.rs b/crates/entangle/src/main.rs index 1612d80..f032bf6 100644 --- a/crates/entangle/src/main.rs +++ b/crates/entangle/src/main.rs @@ -16,7 +16,7 @@ use anyhow::Result; use chrono::{SecondsFormat, Utc}; use clap::{Parser, Subcommand}; use config::Options; -use groups::{Member, Role}; +use groups::Member; use invite::{ code::InviteCode, protocol::{self, InviteEnd, InviteError, InviteServer, InviteState}, @@ -121,6 +121,23 @@ enum GroupCommand { #[arg(id = "group_name", value_name = "NAME")] name: String, }, + /// Remove a member and rotate the group's key. + Remove { + #[arg(id = "group_name", value_name = "NAME")] + name: String, + member: String, + }, + /// Rotate the group's key. + Rekey { + #[arg(id = "group_name", value_name = "NAME")] + name: String, + }, + /// Transfer group administration to a member. + Admin { + #[arg(id = "group_name", value_name = "NAME")] + name: String, + member: String, + }, } #[tokio::main] @@ -246,6 +263,9 @@ async fn run_group( | GroupCommand::Invite { .. } | GroupCommand::Join { .. } | GroupCommand::Leave { .. } + | GroupCommand::Remove { .. } + | GroupCommand::Rekey { .. } + | GroupCommand::Admin { .. } ) && !(io::stdin().is_terminal() && io::stdout().is_terminal()) { writeln!( @@ -268,6 +288,7 @@ async fn run_group( particle_id: particle_id.clone(), name: particle_name, static_key: hex::encode(entangle_transport::public_key(&static_private)), + sign_key: groups::signing_public_key(&static_private), added_by: particle_id, added_at: Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), }; @@ -308,14 +329,29 @@ async fn run_group( GroupCommand::List { json } => { let groups = groups::list(&data_root)?; if json { + let self_id = identity::read_saved(&data_root)?.map(|(id, _)| id.to_string()); let rows = groups .iter() .map(|group| { serde_json::json!({ "name": &group.meta.name, "group": &group.id, - "role": role_name(&group.meta.role), + "role": role_name( + !group.outdated + && self_id + .as_deref() + .is_some_and(|id| group.meta.is_admin(id)) + ), "members": group.members.len(), + "admin": group.members.iter() + .find(|member| member.particle_id == group.meta.admin) + .map(|member| member.name.clone()), + "epoch": group.meta.epoch, + "outdated": group.outdated, + "hint": group.outdated.then(|| format!( + "group {} was created by Entangle 0.6; run `entangle group leave {}` and create it again", + group.meta.name, group.meta.name + )), }) }) .collect::>(); @@ -324,15 +360,25 @@ async fn run_group( writeln!(output, "This machine isn't in any group yet. Run `entangle group create ` or `entangle group join `.")?; } else { writeln!(output, "NAME GROUP ROLE MEMBERS")?; + let self_id = identity::read_saved(&data_root)?.map(|(id, _)| id.to_string()); for group in groups { - writeln!( - output, - "{} {} {} {}", - group.meta.name, - group.id, - role_name(&group.meta.role), - group.members.len() - )?; + if group.outdated { + writeln!( + output, + "{} {} outdated -- group {} was created by Entangle 0.6; run `entangle group leave {}` and create it again", + group.meta.name, group.id, group.meta.name, group.meta.name + )?; + } else { + let is_admin = self_id.as_deref().is_some_and(|id| group.meta.is_admin(id)); + writeln!( + output, + "{} {} {} {}", + group.meta.name, + group.id, + role_name(is_admin), + group.members.len() + )?; + } } } } @@ -353,38 +399,218 @@ async fn run_group( "particle_id": &member.particle_id, "fingerprint": fingerprint, "added_by": &member.added_by, + "role": role_name(group.meta.is_admin(&member.particle_id)), "this_machine": is_self, }) }).collect::>(); if json { writeln!(output, "{}", serde_json::to_string_pretty(&rows)?)?; } else { - writeln!(output, "NAME PARTICLE(short) FINGERPRINT ADDED BY")?; + writeln!(output, "NAME PARTICLE(short) FINGERPRINT ADDED BY ROLE")?; for (row, member) in rows.iter().zip(&group.members) { let name = row["name"].as_str().unwrap_or_default(); writeln!( output, - "{} {} {} {}", + "{} {} {} {} {}", name, short_particle_id(&member.particle_id), row["fingerprint"].as_str().unwrap_or_default(), - short_particle_id(row["added_by"].as_str().unwrap_or_default()) + short_particle_id(row["added_by"].as_str().unwrap_or_default()), + row["role"].as_str().unwrap_or_default() )?; } } } + GroupCommand::Remove { name, member } => { + let group = groups::find(&data_root, &name)? + .ok_or_else(|| groups::GroupError::NotFound(name.clone()))?; + let static_private = require_admin(&data_root, &group)?; + let target = resolve_group_member(&group, &member)?; + let self_id = identity::read_saved(&data_root)? + .map(|(id, _)| id.to_string()) + .unwrap_or_default(); + if target.particle_id == self_id { + anyhow::bail!("you cannot remove yourself from {}", group.meta.name); + } + write!( + output, + "Remove {} from {} and rotate the group key? [y/N] ", + target.name, group.meta.name + )?; + output.flush()?; + if !peer::read_confirmation_key()? { + writeln!(output, "Cancelled; still in {}.", group.meta.name)?; + return Ok(0); + } + let next_epoch = group + .meta + .epoch + .checked_add(1) + .ok_or_else(|| anyhow::anyhow!("group epoch exhausted"))?; + write_update_status(&data_root, &group.id, next_epoch)?; + let updated = groups::issue_update( + &data_root, + &group.id, + entangle_core::UpdateKind::Remove { + particle_id: target.particle_id.clone(), + }, + &static_private, + )?; + let line = update_outcome( + &data_root, + &updated, + &format!("Removed {} from {}", target.name, group.meta.name), + "New key", + identity::is_running(&data_root), + updated.members.len().saturating_sub(1), + Duration::from_secs(20), + ) + .await?; + writeln!(output, "{line}")?; + } + GroupCommand::Rekey { name } => { + let group = groups::find(&data_root, &name)? + .ok_or_else(|| groups::GroupError::NotFound(name.clone()))?; + let static_private = require_admin(&data_root, &group)?; + write!(output, "Rotate the key for {}? [y/N] ", group.meta.name)?; + output.flush()?; + if !peer::read_confirmation_key()? { + writeln!(output, "Cancelled; still in {}.", group.meta.name)?; + return Ok(0); + } + let next_epoch = group + .meta + .epoch + .checked_add(1) + .ok_or_else(|| anyhow::anyhow!("group epoch exhausted"))?; + write_update_status(&data_root, &group.id, next_epoch)?; + let updated = groups::issue_update( + &data_root, + &group.id, + entangle_core::UpdateKind::Rekey, + &static_private, + )?; + let line = update_outcome( + &data_root, + &updated, + &format!("Rekeyed {}", group.meta.name), + "New key", + identity::is_running(&data_root), + group.members.len().saturating_sub(1), + Duration::from_secs(20), + ) + .await?; + writeln!(output, "{line}")?; + } + GroupCommand::Admin { name, member } => { + let group = groups::find(&data_root, &name)? + .ok_or_else(|| groups::GroupError::NotFound(name.clone()))?; + let static_private = require_admin(&data_root, &group)?; + let target = resolve_group_member(&group, &member)?; + if target.particle_id == group.meta.admin { + anyhow::bail!( + "{} is already the admin of {}", + target.name, + group.meta.name + ); + } + write!( + output, + "Make {} the admin of {}? [y/N] ", + target.name, group.meta.name + )?; + output.flush()?; + if !peer::read_confirmation_key()? { + writeln!(output, "Cancelled; still in {}.", group.meta.name)?; + return Ok(0); + } + let next_epoch = group + .meta + .epoch + .checked_add(1) + .ok_or_else(|| anyhow::anyhow!("group epoch exhausted"))?; + write_update_status(&data_root, &group.id, next_epoch)?; + let updated = groups::issue_update( + &data_root, + &group.id, + entangle_core::UpdateKind::AdminHandoff { + to: target.particle_id, + }, + &static_private, + )?; + let line = update_outcome( + &data_root, + &updated, + &format!("{} is now the admin of {}", target.name, group.meta.name), + "Notified", + identity::is_running(&data_root), + group.members.len().saturating_sub(1), + Duration::from_secs(20), + ) + .await?; + writeln!(output, "{line}")?; + } GroupCommand::Leave { name } => { let group = groups::find(&data_root, &name)? .ok_or_else(|| groups::GroupError::NotFound(name.clone()))?; + let self_id = identity::read_saved(&data_root)? + .map(|(id, _)| id.to_string()) + .unwrap_or_default(); + if group.meta.is_admin(&self_id) && group.members.len() > 1 { + let member = group + .members + .iter() + .find(|member| member.particle_id != self_id) + .map(|member| member.name.as_str()) + .unwrap_or(""); + anyhow::bail!( + "You're the admin of {}. Hand it off first: entangle group admin {} {}", + group.meta.name, + group.meta.name, + member + ); + } write!( output, - "Leave group {} ({})? This deletes its key from this machine. [y/N] ", + "Leave group {} ({})? [y/N] ", group.meta.name, group.id )?; output.flush()?; if peer::read_confirmation_key()? { - groups::leave(&data_root, &name)?; - writeln!(output, "Left group {}.", group.meta.name)?; + if group.outdated { + groups::leave(&data_root, &name)?; + writeln!(output, "Left {}.", group.meta.name)?; + } else if identity::is_running(&data_root) { + groups::write_status( + &data_root, + &group.id, + &groups::GroupStatus { + epoch: group.meta.epoch, + op: "leave".into(), + confirmed: Vec::new(), + }, + )?; + groups::set_leaving(&data_root, &group.id, true)?; + let status = wait_for_status( + &data_root, + &group.id, + group.meta.epoch, + "leave", + group.members.len().saturating_sub(1), + Duration::from_secs(60), + ) + .await; + writeln!( + output, + "Left {}. Told {} of {} members.", + group.meta.name, + status.confirmed.len(), + group.members.len().saturating_sub(1) + )?; + } else { + groups::leave(&data_root, &name)?; + writeln!(output, "Left {}.", group.meta.name)?; + } } else { writeln!(output, "Cancelled; still in {}.", group.meta.name)?; } @@ -393,6 +619,151 @@ async fn run_group( Ok(0) } +struct GroupMemberTarget { + particle_id: String, + name: String, +} + +fn require_admin(root: &std::path::Path, group: &groups::Group) -> Result<[u8; 32]> { + let saved = identity::read_saved(root)?; + let Some((self_id, private)) = saved else { + return Err(groups::GroupError::MissingIdentity(root.to_owned()).into()); + }; + if !group.meta.is_admin(self_id.as_str()) { + let admin_name = group + .members + .iter() + .find(|member| member.particle_id == group.meta.admin) + .map_or(group.meta.admin.as_str(), |member| member.name.as_str()); + anyhow::bail!( + "Only the admin of {} ({}) can do this.", + group.meta.name, + admin_name + ); + } + Ok(private) +} + +fn resolve_group_member(group: &groups::Group, selector: &str) -> Result { + let matches = group + .members + .iter() + .filter(|member| member.name == selector || member.particle_id.starts_with(selector)) + .collect::>(); + match matches.as_slice() { + [] => anyhow::bail!("no member matches {selector} in {}", group.meta.name), + [member] => Ok(GroupMemberTarget { + particle_id: member.particle_id.clone(), + name: member.name.clone(), + }), + _ => anyhow::bail!("ambiguous member: {selector}"), + } +} + +fn write_update_status(root: &std::path::Path, group_id: &str, epoch: u32) -> Result<()> { + groups::write_status( + root, + group_id, + &groups::GroupStatus { + epoch, + op: "update".into(), + confirmed: Vec::new(), + }, + )?; + Ok(()) +} + +async fn wait_for_status( + root: &std::path::Path, + group_id: &str, + epoch: u32, + op: &str, + expected: usize, + duration: Duration, +) -> groups::GroupStatus { + let deadline = std::time::Instant::now() + duration; + let mut latest = None; + loop { + if let Ok(Some(status)) = groups::read_status(root, group_id) { + if status.epoch >= epoch && status.op == op { + if status.confirmed.len() >= expected { + return status; + } + latest = Some(status); + } + } + if std::time::Instant::now() >= deadline { + break; + } + tokio::time::sleep(Duration::from_millis(200)).await; + } + latest.unwrap_or_else(|| groups::GroupStatus { + epoch, + op: op.into(), + confirmed: Vec::new(), + }) +} + +async fn update_outcome( + root: &std::path::Path, + group: &groups::Group, + action: &str, + delivery_label: &str, + running: bool, + total: usize, + timeout: Duration, +) -> Result { + if !running { + return Ok(format!( + "{action}. Entangle isn't running here, so the change goes out the next time it starts." + )); + } + let status = wait_for_status(root, &group.id, group.meta.epoch, "update", total, timeout).await; + let self_id = identity::read_saved(root) + .ok() + .flatten() + .map(|(id, _)| id.to_string()); + let delivered_members = group + .members + .iter() + .filter(|member| { + self_id.as_deref() != Some(member.particle_id.as_str()) + && status.confirmed.contains(&member.particle_id) + }) + .collect::>(); + let delivered = delivered_members.len().min(total); + if delivered >= total { + if delivery_label == "Notified" { + Ok(format!( + "{action}. Notified {delivered} of {total} other members." + )) + } else { + Ok(format!( + "{action}. {delivery_label} delivered to {delivered} of {total} other members." + )) + } + } else { + let confirmed = delivered_members + .iter() + .map(|member| member.name.as_str()) + .collect::>() + .join(", "); + let missing = group + .members + .iter() + .filter(|member| { + self_id.as_deref() != Some(member.particle_id.as_str()) + && !status.confirmed.contains(&member.particle_id) + }) + .map(|member| member.name.as_str()) + .collect::>() + .join(", "); + Ok(format!( + "{action}. Delivered to {delivered} of {total} members so far ({confirmed}); {missing} gets it the next time it connects to any member." + )) + } +} + async fn run_invite( data_root: &std::path::Path, name: &str, @@ -674,6 +1045,7 @@ fn is_terminal_join_error(error: &InviteError) -> bool { | InviteError::Declined { .. } | InviteError::Incomplete | InviteError::Group(groups::GroupError::MissingIdentity(_)) + | InviteError::Group(groups::GroupError::Conflict { .. }) ) } @@ -696,12 +1068,20 @@ fn write_join_failure(output: &mut dyn Write, error: InviteError) -> Result "A group named {} already exists here; rerun with --as .", group_name.unwrap_or_else(|| "that group".into()) )?, + "removed" => writeln!( + output, + "This device was removed from {}; ask its admin to invite it.", + group_name.unwrap_or_else(|| "that group".into()) + )?, _ => writeln!(output, "The inviter declined this join: {reason}.")?, }, InviteError::Group(groups::GroupError::MissingIdentity(path)) => { let error = groups::GroupError::MissingIdentity(path); writeln!(output, "{error}")?; } + InviteError::Group(groups::GroupError::Conflict { name }) => { + writeln!(output, "{name} changed while joining; run the invite again")?; + } InviteError::Incomplete => writeln!( output, "The join didn't complete; nothing was kept. Run the same command again while the invite is open." @@ -712,14 +1092,22 @@ fn write_join_failure(output: &mut dyn Write, error: InviteError) -> Result } fn write_join_success(output: &mut dyn Write, joined: protocol::Joined) -> Result<()> { - writeln!( - output, - "Joined {} ({}) via {}. {} members.", - joined.group.meta.name, - joined.group.id, - joined.inviter_name, - joined.group.members.len() - )?; + if joined.refreshed { + writeln!( + output, + "Refreshed {}: this device was out of date and now has the current key.", + joined.group.meta.name + )?; + } else { + writeln!( + output, + "Joined {} ({}) via {}. {} members.", + joined.group.meta.name, + joined.group.id, + joined.inviter_name, + joined.group.members.len() + )?; + } Ok(()) } @@ -730,10 +1118,11 @@ fn local_ipv4() -> Option { (!address.is_loopback()).then_some(address) } -fn role_name(role: &Role) -> &'static str { - match role { - Role::Admin => "admin", - Role::Member => "member", +fn role_name(is_admin: bool) -> &'static str { + if is_admin { + "admin" + } else { + "member" } } @@ -838,6 +1227,60 @@ mod tests { )); } + #[test] + fn group_remove_keeps_particle_name_separate_from_group_name() { + let cli = parse_cli_without_name_env(&[ + "entangle", "--name", "pc-a", "group", "remove", "work", "pc-c", + ]); + assert_eq!(cli.options.name.as_deref(), Some("pc-a")); + assert!(matches!( + group_command(cli), + GroupCommand::Remove { name, member } if name == "work" && member == "pc-c" + )); + } + + #[test] + fn group_rekey_keeps_particle_name_separate_from_group_name() { + let cli = + parse_cli_without_name_env(&["entangle", "--name", "pc-a", "group", "rekey", "work"]); + assert_eq!(cli.options.name.as_deref(), Some("pc-a")); + assert!(matches!( + group_command(cli), + GroupCommand::Rekey { name } if name == "work" + )); + } + + #[test] + fn group_admin_keeps_particle_name_separate_from_group_name() { + let cli = parse_cli_without_name_env(&[ + "entangle", "--name", "pc-a", "group", "admin", "work", "pc-b", + ]); + assert_eq!(cli.options.name.as_deref(), Some("pc-a")); + assert!(matches!( + group_command(cli), + GroupCommand::Admin { name, member } if name == "work" && member == "pc-b" + )); + } + + #[test] + fn join_conflict_prints_the_retry_message_without_a_generic_prefix() { + let mut output = Vec::new(); + assert_eq!( + write_join_failure( + &mut output, + InviteError::Group(groups::GroupError::Conflict { + name: "work".into(), + }), + ) + .unwrap(), + 1 + ); + assert_eq!( + String::from_utf8(output).unwrap(), + "work changed while joining; run the invite again\n" + ); + } + #[test] fn detects_broken_pipe_root_cause() { let error = anyhow::Error::new(io::Error::new(io::ErrorKind::BrokenPipe, "closed")); @@ -895,18 +1338,24 @@ mod tests { particle_id: inviter_id.clone(), name: "inviter".into(), static_key: hex::encode(entangle_transport::public_key(&identity.static_private)), - added_by: inviter_id, + sign_key: groups::signing_public_key(&identity.static_private), + added_by: inviter_id.clone(), added_at: "2026-10-12T00:00:00Z".into(), }; + let inviter_sign_key = groups::signing_public_key(&identity.static_private); drop(identity); let group = groups::insert( inviter_root.path(), [0x42; 32], groups::GroupMeta { - version: 1, + version: 2, name: "work".into(), - role: Role::Admin, created_at: "2026-10-12T00:00:00Z".into(), + epoch: 0, + admin: inviter_id.clone(), + admin_sign_key: inviter_sign_key, + revoked: vec![], + leaving: false, }, vec![inviter_member], ) @@ -969,13 +1418,15 @@ mod tests { particle_id: local_id.clone(), name: "local".into(), static_key: hex::encode(entangle_transport::public_key(&local_key)), - added_by: local_id, + sign_key: groups::signing_public_key(&local_key), + added_by: local_id.clone(), added_at: "2026-10-12T00:00:00Z".into(), }, Member { particle_id: remote_id.clone(), name: "remote".into(), static_key: hex::encode(entangle_transport::public_key(&[0x42; 32])), + sign_key: groups::signing_public_key(&[0x42; 32]), added_by: remote_id, added_at: "2026-10-12T00:00:00Z".into(), }, @@ -984,10 +1435,14 @@ mod tests { temp.path(), [0x24; 32], groups::GroupMeta { - version: 1, + version: 2, name: "work".into(), - role: Role::Admin, created_at: "2026-10-12T00:00:00Z".into(), + epoch: 0, + admin: local_id.clone(), + admin_sign_key: groups::signing_public_key(&local_key), + revoked: vec![], + leaving: false, }, members, ) diff --git a/crates/entangle/src/node.rs b/crates/entangle/src/node.rs index 9d4315d..e4eb3b1 100644 --- a/crates/entangle/src/node.rs +++ b/crates/entangle/src/node.rs @@ -55,11 +55,22 @@ fn build_key_ring(static_private: [u8; 32], groups: &[Group]) -> KeyRing { static_private, groups: groups .iter() - .map(|group| { - ( - groups::group_id_bytes(&group.key), - groups::derive_psk(&group.key), - ) + .filter(|group| !group.outdated) + .filter_map(|group| { + let id = groups::id_bytes(&group.id)?; + let mut keys = group + .retained_keys + .iter() + .map(|(epoch, key)| (*epoch, groups::derive_psk(key))) + .collect::>(); + keys.insert(group.meta.epoch, groups::derive_psk(&group.key)); + Some(( + id, + entangle_transport::GroupKeys { + current: group.meta.epoch, + keys, + }, + )) }) .collect(), } @@ -168,6 +179,10 @@ pub struct Particle { inbox: Mutex>, in_flight: StdMutex>, groups_reload_warned: AtomicBool, + too_far_warned: StdMutex>, + peer_too_far_logged: StdMutex>, + peer_backoff: StdMutex>, + leave_started: StdMutex>, notifications: Option>, inbound_limit: Arc, shutdown: tokio::sync::watch::Sender, @@ -210,6 +225,14 @@ impl Particle { info!("ignoring pre-0.6 pairing/approval state; use entangle group create/join"); } let initial_groups = groups::list(&root)?; + for group in initial_groups.iter().filter(|group| group.outdated) { + warn!( + group = %group.meta.name, + "group {} was created by Entangle 0.6; run `entangle group leave {}` and create it again", + group.meta.name, + group.meta.name + ); + } let groups = Arc::new(RwLock::new(initial_groups.clone())); let keys: SharedKeyRing = Arc::new(RwLock::new(Arc::new(build_key_ring( identity.static_private, @@ -291,6 +314,10 @@ impl Particle { inbox: Mutex::new(persisted_inbox), in_flight: StdMutex::new(HashSet::new()), groups_reload_warned: AtomicBool::new(false), + too_far_warned: StdMutex::new(HashSet::new()), + peer_too_far_logged: StdMutex::new(HashSet::new()), + peer_backoff: StdMutex::new(HashMap::new()), + leave_started: StdMutex::new(HashMap::new()), notifications, inbound_limit: Arc::new(Semaphore::new(4)), shutdown, @@ -380,17 +407,84 @@ impl Particle { Some(warning) } + fn peer_in_backoff(&self, peer_key: &str) -> bool { + let now = Instant::now(); + let mut backoff = self.peer_backoff.lock().expect("peer backoff lock"); + backoff.retain(|_, until| *until > now); + backoff.contains_key(peer_key) + } + + fn first_repeater_group(&self) -> Option { + let groups = self.groups.read().expect("group store lock").clone(); + let key_ring = self.keys.read().expect("key ring lock").clone(); + groups + .into_iter() + .find(|group| { + !group.outdated + && groups::id_bytes(&group.id) + .is_some_and(|id| key_ring.groups.contains_key(&id)) + }) + .map(|group| group.id) + } + + fn backoff_peer(&self, peer_key: &str) { + self.peer_backoff.lock().expect("peer backoff lock").insert( + peer_key.to_owned(), + Instant::now() + Duration::from_secs(60), + ); + } + + fn log_epoch_error(&self, error: &LinkError, group: &Group, peer_name: &str, peer_key: &str) { + match error { + LinkError::Transport(entangle_transport::TransportError::TooFarBehind { + local, + current, + }) => { + self.backoff_peer(peer_key); + if self + .too_far_warned + .lock() + .expect("epoch warning lock") + .insert(groups::id_bytes(&group.id).unwrap_or([0; 4])) + { + warn!( + "group {} is too far out of date (this device is at epoch {}, a member is at {}); re-invite this device", + group.meta.name, local, current + ); + } + } + LinkError::Transport(entangle_transport::TransportError::PeerTooFarBehind { + peer_epoch: _, + }) => { + self.backoff_peer(peer_key); + if self + .peer_too_far_logged + .lock() + .expect("peer epoch log lock") + .insert(peer_key.to_owned()) + { + info!( + "{} is too far behind on group {}; re-invite it", + peer_name, group.meta.name + ); + } + } + _ => {} + } + } + fn group_member(&self, group_id: [u8; 4], particle_id: &str, static_key: &[u8; 32]) -> bool { self.groups .read() .expect("group store lock") .iter() - .find(|group| groups::group_id_bytes(&group.key) == group_id) + .find(|group| !group.outdated && groups::id_bytes(&group.id) == Some(group_id)) .is_some_and(|group| { - group.members.iter().any(|member| { - member.particle_id == particle_id - && member.static_key == hex::encode(static_key) - }) + !group.meta.leaving + && group.members.iter().any(|member| { + member.particle_id == particle_id + && member.static_key == hex::encode(static_key) + }) }) } @@ -400,10 +494,12 @@ impl Particle { .expect("group store lock") .iter() .filter(|group| { - group.members.iter().any(|member| { - member.particle_id == particle_id - && member.static_key == hex::encode(static_key) - }) + !group.outdated + && !group.meta.leaving + && group.members.iter().any(|member| { + member.particle_id == particle_id + && member.static_key == hex::encode(static_key) + }) }) .map(|group| group.meta.name.clone()) .collect() @@ -420,6 +516,7 @@ impl Particle { .read() .expect("group store lock") .iter() + .filter(|group| !group.outdated && !group.meta.leaving) .map(|group| group.id.clone()) .collect(), version: self.manifest.version.clone(), @@ -514,7 +611,9 @@ impl Particle { *self.groups.write().expect("group store lock") = current.clone(); *self.keys.write().expect("key ring lock") = Arc::new(build_key_ring(self._identity.static_private, ¤t)); - self.field.write().expect("field lock").clear(); + if !current.iter().any(|group| group.meta.leaving) { + self.field.write().expect("field lock").clear(); + } self.groups_reload_warned.store(false, Ordering::Relaxed); if let Err(error) = self.restart_resonator().await { self.log_discovery_failure(&error); @@ -545,6 +644,125 @@ impl Particle { self.log_discovery_failure(&error); } } + self.drive_leaving_groups().await; + } + } + + async fn drive_leaving_groups(&self) { + let groups = self.groups.read().expect("group store lock").clone(); + let self_id = self.manifest.particle_id.to_string(); + for group in groups + .into_iter() + .filter(|group| group.meta.leaving && !group.outdated) + { + let started = { + let mut started = self.leave_started.lock().expect("leave timer lock"); + *started.entry(group.id.clone()).or_insert_with(Instant::now) + }; + let group_id = group.id.clone(); + let mut status = groups::read_status(&self.data_root, &group_id) + .ok() + .flatten() + .filter(|status| status.op == "leave") + .unwrap_or(groups::GroupStatus { + epoch: group.meta.epoch, + op: "leave".into(), + confirmed: Vec::new(), + }); + let total = group + .members + .iter() + .filter(|member| member.particle_id != self_id) + .count(); + let pending = group + .members + .iter() + .filter(|member| member.particle_id != self_id) + .filter(|member| !status.confirmed.contains(&member.particle_id)) + .cloned() + .collect::>(); + for member in pending { + if self.peer_in_backoff(&member.static_key) { + continue; + } + let Ok(static_key_bytes) = hex::decode(&member.static_key) else { + continue; + }; + let Ok(static_key) = <[u8; 32]>::try_from(static_key_bytes) else { + continue; + }; + let Some(id) = ParticleId::parse(member.particle_id.clone()).ok() else { + continue; + }; + let peer = self + .field + .read() + .expect("field lock") + .by_id(&id) + .filter(|peer| peer.static_key.is_none_or(|key| key == static_key)); + let Some(peer) = peer else { + continue; + }; + let Some(group_bytes) = groups::id_bytes(&group.id) else { + continue; + }; + let mut delivered = false; + for address in &peer.resonance.addrs { + if peer.resonance.port == 0 { + continue; + } + let route = Route::Direct(SocketAddr::new(*address, peer.resonance.port)); + if let Ok(stream) = self.sync_handshake(&route, static_key, group_bytes).await { + if entangle_link::leave_on(stream, Duration::from_secs(5)) + .await + .is_ok() + { + delivered = true; + break; + } + } + } + if !delivered + && self + .relay_directory + .lock() + .expect("relay directory lock") + .contains(&static_key) + { + let dialer = self + .repeater + .read() + .expect("repeater dialer lock") + .as_ref() + .filter(|dialer| !dialer.is_closed()) + .cloned(); + if let Some(dialer) = dialer { + let route = Route::Tunnel(Arc::new(RepeaterOpener { + dialer, + target: static_key, + })); + if let Ok(stream) = + self.sync_handshake(&route, static_key, group_bytes).await + { + delivered = entangle_link::leave_on(stream, Duration::from_secs(5)) + .await + .is_ok(); + } + } + } + if delivered { + status.confirmed.push(member.particle_id); + status.confirmed.sort(); + let _ = groups::write_status(&self.data_root, &group_id, &status); + } + } + if status.confirmed.len() >= total || started.elapsed() >= Duration::from_secs(60) { + let _ = groups::leave(&self.data_root, &group_id); + self.leave_started + .lock() + .expect("leave timer lock") + .remove(&group_id); + } } } @@ -593,8 +811,8 @@ impl Particle { } continue; }; - let has_groups = !self.keys.read().expect("key ring lock").groups.is_empty(); - if !self._identity.persistent || !has_groups { + if !self._identity.persistent { + let has_groups = self.first_repeater_group().is_some(); debug!( persistent = self._identity.persistent, has_groups, "repeater connection gated by identity or group membership" @@ -608,13 +826,12 @@ impl Particle { } continue; } - let Some(group) = self - .groups - .read() - .expect("group store lock") - .first() - .map(|group| group.id.clone()) - else { + let Some(group) = self.first_repeater_group() else { + debug!( + persistent = self._identity.persistent, + has_groups = false, + "repeater connection gated by identity or group membership" + ); self.clear_repeater(); if !self .wait_or_shutdown(&mut shutdown, Duration::from_secs(1)) @@ -768,13 +985,8 @@ impl Particle { } } _ = health_poll.tick() => { - let has_groups = !self.keys.read().expect("key ring lock").groups.is_empty(); - let current_group = self - .groups - .read() - .expect("group store lock") - .first() - .map(|group| group.id.clone()); + let current_group = self.first_repeater_group(); + let has_groups = current_group.is_some(); let closed = dialer.is_closed(); let group_changed = current_group.as_deref() != Some(group); if closed || !has_groups || group_changed { @@ -1260,7 +1472,10 @@ impl Particle { if !member { return Err(format!("not a member of {}", group.meta.name)); } - (groups::group_id_bytes(&group.key), group.meta.name.clone()) + ( + groups::id_bytes(&group.id).ok_or_else(unreachable)?, + group.meta.name.clone(), + ) } None => { let group_id = self @@ -1748,14 +1963,36 @@ impl Particle { expected_static: [u8; 32], group_id: [u8; 4], ) -> Result { - let stream = entangle_link::connect_secure( + let result = entangle_link::connect_secure_with_handler( route, self.keys.clone(), group_id, + self, Duration::from_secs(5), ) - .await - .map_err(|error| error.to_string())?; + .await; + let stream = match result { + Ok(stream) => stream, + Err(error) => { + let group = self + .groups + .read() + .expect("group store lock") + .iter() + .find(|group| groups::id_bytes(&group.id) == Some(group_id)) + .cloned(); + if let Some(group) = group { + let peer_name = group + .members + .iter() + .find(|member| member.static_key == hex::encode(expected_static)) + .map_or("peer", |member| member.name.as_str()) + .to_owned(); + self.log_epoch_error(&error, &group, &peer_name, &hex::encode(expected_static)); + } + return Err(error.to_string()); + } + }; if stream.remote_static() != expected_static { return Err("remote Noise static key did not match the bound peer key".into()); } @@ -1765,10 +2002,12 @@ impl Particle { fn select_peer_group(&self, peer: &PeerRecord, static_key: &[u8; 32]) -> Option<[u8; 4]> { let groups = self.groups.read().expect("group store lock"); let member_matches = |group: &&Group| { - group.members.iter().any(|member| { - member.particle_id == peer.resonance.particle_id.as_str() - && member.static_key == hex::encode(static_key) - }) + !group.outdated + && !group.meta.leaving + && group.members.iter().any(|member| { + member.particle_id == peer.resonance.particle_id.as_str() + && member.static_key == hex::encode(static_key) + }) }; let advertised = !peer.resonance.groups.is_empty(); groups @@ -1778,11 +2017,10 @@ impl Particle { if advertised { peer.resonance.groups.contains(&group.id) } else { - peer.group_id == Some(groups::group_id_bytes(&group.key)) - || peer.group_id.is_none() + peer.group_id == groups::id_bytes(&group.id) || peer.group_id.is_none() } }) - .map(|group| groups::group_id_bytes(&group.key)) + .and_then(|group| groups::id_bytes(&group.id)) } async fn on_detected(&self, peer: Resonance) { @@ -1817,8 +2055,24 @@ impl Particle { } async fn dial(&self, route: Route, discovery: Option) -> bool { + let peer_key = discovery + .as_ref() + .map(|peer| peer.particle_id.to_string()) + .unwrap_or_else(|| match &route { + Route::Direct(address) => address.to_string(), + Route::Tunnel(opener) => opener.describe(), + }); + if self.peer_in_backoff(&peer_key) { + return true; + } + let peer_name = discovery + .as_ref() + .map_or_else(|| peer_key.clone(), |peer| peer.name.clone()); let groups = self.groups.read().expect("group store lock").clone(); let candidates = groups.into_iter().filter(|group| { + if group.outdated || group.meta.leaving { + return false; + } if discovery .as_ref() .is_some_and(|peer| !peer.groups.contains(&group.id)) @@ -1832,13 +2086,16 @@ impl Particle { .any(|member| member.particle_id == peer.particle_id.as_str()) }) }); - for group in candidates { - let group_id = groups::group_id_bytes(&group.key); - match entangle_link::handshake_stream( + 'groups: for group in candidates { + let Some(group_id) = groups::id_bytes(&group.id) else { + continue; + }; + match entangle_link::handshake_stream_with_handler( &route, self.keys.clone(), group_id, &self.manifest, + self, Duration::from_secs(5), ) .await @@ -1879,6 +2136,7 @@ impl Particle { }, remote_static, group: group_id, + epoch: stream.epoch(), relayed: matches!(&route, Route::Tunnel(_)), }; if let Err(error) = entangle_link::introduce_on( @@ -1952,6 +2210,19 @@ impl Particle { debug!("peer protocol mismatch: {}", manifest.protocol); } Err(error) => { + self.log_epoch_error(&error, &group, &peer_name, &peer_key); + if matches!(&error, LinkError::Stale) { + return true; + } + if matches!( + &error, + LinkError::Transport( + entangle_transport::TransportError::TooFarBehind { .. } + | entangle_transport::TransportError::PeerTooFarBehind { .. } + ) + ) { + continue 'groups; + } let route_description = match &route { Route::Direct(address) => address.to_string(), Route::Tunnel(opener) => opener.describe(), @@ -2150,6 +2421,121 @@ impl Particle { #[async_trait::async_trait] impl LinkHandler for Particle { + fn supports_group_updates(&self) -> bool { + true + } + + fn group_epoch(&self, peer: &PeerInfo) -> u32 { + self.groups + .read() + .expect("group store lock") + .iter() + .find(|group| { + groups::id_bytes(&group.id) == Some(peer.group) + && !group.outdated + && group + .members + .iter() + .any(|member| member.static_key == hex::encode(peer.remote_static)) + }) + .map_or(peer.epoch, |group| group.meta.epoch) + } + + fn peer_particle_id(&self, peer: &PeerInfo) -> Option { + let groups = self.groups.read().ok()?; + let group = groups.iter().find(|group| { + groups::id_bytes(&group.id) == Some(peer.group) + && !group.outdated + && group + .members + .iter() + .any(|member| member.static_key == hex::encode(peer.remote_static)) + })?; + group + .members + .iter() + .find(|member| member.static_key == hex::encode(peer.remote_static)) + .and_then(|member| ParticleId::parse(member.particle_id.clone()).ok()) + } + + fn updates_since(&self, peer: &PeerInfo, epoch: u32) -> Vec { + let groups = self.groups.read().expect("group store lock"); + let Some(group) = groups.iter().find(|group| { + groups::id_bytes(&group.id) == Some(peer.group) + && !group.outdated + && group + .members + .iter() + .any(|member| member.static_key == hex::encode(peer.remote_static)) + }) else { + return Vec::new(); + }; + groups::updates_since(group, epoch).unwrap_or_default() + } + + fn on_updates( + &self, + peer: &PeerInfo, + sender: &ParticleId, + updates: Vec, + ) -> Result { + let member = { + let groups = self + .groups + .read() + .map_err(|_| LinkError::Handler("group store lock poisoned".into()))?; + let group = groups + .iter() + .find(|group| groups::id_bytes(&group.id) == Some(peer.group) && !group.outdated) + .ok_or_else(|| LinkError::Handler("group is not available".into()))?; + group + .members + .iter() + .find(|member| { + member.particle_id == sender.as_str() + && member.static_key == hex::encode(peer.remote_static) + }) + .cloned() + .ok_or_else(|| LinkError::Handler("update sender is not a current member".into()))? + }; + let group_id = hex::encode(peer.group); + let result = groups::apply_updates(&self.data_root, &group_id, &updates, Some(&member)) + .map_err(|error| LinkError::Handler(error.to_string()))?; + Ok(result.epoch) + } + + fn on_leave(&self, peer: &PeerInfo, sender: &ParticleId) -> Result<(), LinkError> { + if !self.group_member(peer.group, sender.as_str(), &peer.remote_static) { + return Err(LinkError::Handler( + "leave sender is not a current member".into(), + )); + } + groups::remove_member(&self.data_root, &hex::encode(peer.group), sender.as_str()) + .map_err(|error| LinkError::Handler(error.to_string())) + } + + fn on_updates_ack(&self, peer: &PeerInfo, epoch: u32) { + let Some(sender) = self.peer_particle_id(peer) else { + return; + }; + let group_id = hex::encode(peer.group); + let Ok(Some(mut status)) = groups::read_status(&self.data_root, &group_id) else { + return; + }; + if status.op != "update" || epoch < status.epoch { + return; + } + if !status.confirmed.iter().any(|id| id == sender.as_str()) { + status.confirmed.push(sender.to_string()); + status.confirmed.sort(); + let _ = groups::write_status(&self.data_root, &group_id, &status); + } + } + + fn on_link_epoch(&self, peer: &PeerInfo, epoch: u32) { + self.on_updates_ack(peer, epoch); + } + fn members_to_introduce( &self, peer: &PeerInfo, @@ -2259,6 +2645,9 @@ impl LinkHandler for Particle { groups: vec![hex::encode(peer.group)], proto: manifest.protocol, }); + if !peer.relayed && !resonance.addrs.contains(&peer.addr.ip()) { + resonance.addrs.push(peer.addr.ip()); + } resonance.particle_id = manifest.particle_id.clone(); resonance.name = manifest.name.clone(); resonance.port = manifest.link_port; @@ -2334,6 +2723,27 @@ impl LinkHandler for Particle { ); continue; } + let revoked = self + .groups + .read() + .expect("group store lock") + .iter() + .find(|group| groups::id_bytes(&group.id) == Some(peer.group)) + .is_some_and(|group| { + group + .meta + .revoked + .iter() + .any(|revocation| revocation.particle_id == member.particle_id) + }); + if revoked { + warn!( + particle_id = %member.particle_id, + group = %hex::encode(peer.group), + "rejecting an introduction for a revoked particle" + ); + continue; + } match groups::add_member(&self.data_root, &group_id, member.clone()) { Ok(true) => added += 1, Ok(false) => {} @@ -2535,6 +2945,7 @@ mod tests { particle_id: id.to_string(), name: name.into(), static_key: hex::encode(entangle_transport::public_key(private)), + sign_key: groups::signing_public_key(private), added_by: added_by.into(), added_at: chrono::Utc::now().to_rfc3339(), } @@ -2545,16 +2956,21 @@ mod tests { name: &str, key: [u8; 32], members: Vec, - role: groups::Role, + _role: groups::Role, ) { + let admin = members.first().expect("test group member").clone(); groups::insert( root, key, groups::GroupMeta { - version: 1, + version: 2, name: name.into(), - role, created_at: chrono::Utc::now().to_rfc3339(), + epoch: 0, + admin: admin.particle_id, + admin_sign_key: admin.sign_key, + revoked: vec![], + leaving: false, }, members, ) @@ -2567,6 +2983,7 @@ mod tests { particle_id: particle_id.into(), name: particle_id.into(), static_key: String::new(), + sign_key: String::new(), added_by: String::new(), added_at: String::new(), }; @@ -2574,12 +2991,19 @@ mod tests { id: groups::group_id(&key), key, meta: groups::GroupMeta { - version: 1, + version: 2, name: name.into(), - role: groups::Role::Admin, created_at: chrono::Utc::now().to_rfc3339(), + epoch: 0, + admin: particle_id.into(), + admin_sign_key: String::new(), + revoked: vec![], + leaving: false, }, members: vec![member(particle_id)], + retained_keys: Default::default(), + updates: vec![], + outdated: false, }; let work = group("work", [0x73; 32], "deadbeef000000000000000000000001"); let home = group("home", [0x74; 32], "deadbeef000000000000000000000002"); @@ -2617,6 +3041,17 @@ mod tests { ); } + fn write_outdated_test_group(root: &Path, name: &str, key: [u8; 32]) { + let path = root.join("groups").join(groups::group_id(&key)); + std::fs::create_dir_all(&path).unwrap(); + std::fs::write(path.join("key"), format!("{}\n", hex::encode(key))).unwrap(); + std::fs::write( + path.join("meta.json"), + serde_json::to_vec(&json!({"version": 1, "name": name})).unwrap(), + ) + .unwrap(); + } + async fn start_test_repeater(root: PathBuf, port: u16) -> (JoinHandle<()>, String, [u8; 32]) { let store = entangle_repeater::store::Store::new(&root); let private = store.key().unwrap(); @@ -2861,66 +3296,482 @@ mod tests { } #[tokio::test] - async fn group_and_subset_sends_filter_by_authenticated_group() { - let schema: Value = - serde_json::from_str(include_str!("../../../schemas/mcp-tools.json")).unwrap(); - let observe_required = schema["tools"] - .as_array() - .unwrap() - .iter() - .find(|tool| tool["name"] == "observe_entangled_states") - .unwrap()["outputSchema"]["properties"]["states"]["items"]["required"] - .as_array() - .unwrap() - .iter() - .map(|value| value.as_str().unwrap().to_owned()) - .collect::>(); + async fn too_far_behind_group_does_not_block_another_shared_group() { let temp = tempfile::tempdir().unwrap(); let root_a = temp.path().join("a"); let root_b = temp.path().join("b"); - let root_c = temp.path().join("c"); let (id_a, private_a) = saved_identity(&root_a); let (id_b, private_b) = saved_identity(&root_b); - let (id_c, private_c) = saved_identity(&root_c); - let member_a_work = test_member(&id_a, "particle-a", &private_a, id_a.as_str()); - let member_b_work = test_member(&id_b, "particle-b", &private_b, id_a.as_str()); - let member_c_work = test_member(&id_c, "particle-c", &private_c, id_a.as_str()); - let member_a_home = member_a_work.clone(); - let member_b_home = member_b_work.clone(); - let work_members = vec![ - member_a_work.clone(), - member_b_work.clone(), - member_c_work.clone(), + let members = vec![ + test_member(&id_a, "particle-a", &private_a, id_a.as_str()), + test_member(&id_b, "particle-b", &private_b, id_a.as_str()), ]; - let home_members = vec![member_a_home, member_b_home]; for (root, role) in [ (&root_a, groups::Role::Admin), (&root_b, groups::Role::Member), - (&root_c, groups::Role::Member), ] { - insert_test_group(root, "work", [0x71; 32], work_members.clone(), role); + insert_test_group(root, "a-stale", [0x61; 32], members.clone(), role.clone()); + insert_test_group(root, "b-current", [0x62; 32], members.clone(), role); } - insert_test_group( - &root_a, - "home", - [0x72; 32], - home_members.clone(), - groups::Role::Admin, + let stale_id = groups::group_id(&[0x61; 32]); + let current_id = groups::group_id(&[0x62; 32]); + for _ in 0..17 { + groups::issue_update( + &root_a, + &stale_id, + entangle_core::UpdateKind::Rekey, + &private_a, + ) + .unwrap(); + } + + let (b, _) = Particle::start(options("particle-b", 0, root_b, None), None) + .await + .unwrap(); + let (a, _) = Particle::start(options("particle-a", 0, root_a, None), None) + .await + .unwrap(); + assert_eq!( + a.groups + .read() + .unwrap() + .iter() + .map(|group| group.meta.name.as_str()) + .collect::>(), + vec!["a-stale", "b-current"] ); - insert_test_group( - &root_b, - "home", - [0x72; 32], - home_members, - groups::Role::Member, + assert!( + a.dial( + Route::Direct(SocketAddr::from(([127, 0, 0, 1], b.manifest.link_port))), + None, + ) + .await ); - let a_port = free_port(); - let b_port = port_other_than(a_port); - let c_port = loop { - let port = free_port(); - if port != a_port && port != b_port { - break port; + let record = a + .field + .read() + .unwrap() + .by_id(&b.manifest.particle_id) + .expect("the peers should link over the current group"); + assert_eq!(record.state, PeerState::Entangled); + assert_eq!( + record.group_id, + Some(groups::id_bytes(¤t_id).unwrap()) + ); + let synced = a + .sync_state(json!({ + "particle_id": b.manifest.particle_id.as_str(), + "state": { "group": "b-current" } + })) + .await + .unwrap(); + assert_eq!(synced["delivered"], true); + assert_eq!(synced["group"], "b-current"); + + a.shutdown().await; + b.shutdown().await; + } + + #[tokio::test] + async fn removal_updates_members_and_rejects_the_revoked_member() { + let temp = tempfile::tempdir().unwrap(); + let root_a = temp.path().join("a"); + let root_b = temp.path().join("b"); + let root_c = temp.path().join("c"); + let (id_a, private_a) = saved_identity(&root_a); + let (id_b, private_b) = saved_identity(&root_b); + let (id_c, private_c) = saved_identity(&root_c); + let members = vec![ + test_member(&id_a, "pc-a", &private_a, id_a.as_str()), + test_member(&id_b, "pc-b", &private_b, id_a.as_str()), + test_member(&id_c, "pc-c", &private_c, id_a.as_str()), + ]; + for (root, role) in [ + (&root_a, groups::Role::Admin), + (&root_b, groups::Role::Member), + (&root_c, groups::Role::Member), + ] { + insert_test_group(root, "work", [0x7a; 32], members.clone(), role); + } + let a_port = free_port(); + let b_port = port_other_than(a_port); + let c_port = loop { + let port = free_port(); + if port != a_port && port != b_port { + break port; + } + }; + let mut options_b = options( + "pc-b", + b_port, + root_b.clone(), + Some(format!("127.0.0.1:{a_port}")), + ); + options_b.no_mdns = true; + let mut options_c = options( + "pc-c", + c_port, + root_c.clone(), + Some(format!("127.0.0.1:{a_port}")), + ); + options_c.no_mdns = true; + let (b, _) = Particle::start(options_b, None).await.unwrap(); + let (c, _) = Particle::start(options_c, None).await.unwrap(); + let mut options_a = options("pc-a", a_port, root_a.clone(), None); + options_a.peer = vec![format!("127.0.0.1:{b_port}"), format!("127.0.0.1:{c_port}")]; + let (a, _) = Particle::start(options_a, None).await.unwrap(); + wait_for_group_pair(&a, &b).await; + wait_for_group_pair(&a, &c).await; + + let group = groups::find(&root_a, "work").unwrap().unwrap(); + groups::issue_update( + &root_a, + &group.id, + entangle_core::UpdateKind::Remove { + particle_id: id_c.to_string(), + }, + &private_a, + ) + .unwrap(); + tokio::time::timeout(Duration::from_secs(12), async { + loop { + let a_reloaded = a + .groups + .read() + .unwrap() + .iter() + .any(|group| group.meta.name == "work" && group.meta.epoch == 1); + let b_reloaded = b + .groups + .read() + .unwrap() + .iter() + .any(|group| group.meta.name == "work" && group.meta.epoch == 1); + let linked = a + .field + .read() + .unwrap() + .by_id(&b.manifest.particle_id) + .is_some_and(|peer| peer.state == PeerState::Entangled); + if a_reloaded && b_reloaded && linked { + break; + } + sleep(Duration::from_millis(25)).await; + } + }) + .await + .expect("members A and B should apply the update and relink"); + let delivered = a + .sync_state(json!({ + "particle_id": id_b.as_str(), + "state": { "message": "still a member" } + })) + .await + .unwrap(); + assert_eq!(delivered["delivered"], true); + assert!(c + .sync_state(json!({ + "particle_id": id_a.as_str(), + "state": { "message": "removed" } + })) + .await + .is_err()); + assert!(c + .sync_state(json!({ + "particle_id": id_b.as_str(), + "state": { "message": "removed" } + })) + .await + .is_err()); + + a.shutdown().await; + b.shutdown().await; + c.shutdown().await; + } + + #[tokio::test] + async fn offline_member_catches_up_from_a_relaying_peer() { + let temp = tempfile::tempdir().unwrap(); + let root_a = temp.path().join("a"); + let root_b = temp.path().join("b"); + let root_c = temp.path().join("c"); + let (id_a, private_a) = saved_identity(&root_a); + let (id_b, private_b) = saved_identity(&root_b); + let (id_c, private_c) = saved_identity(&root_c); + let members = vec![ + test_member(&id_a, "pc-a", &private_a, id_a.as_str()), + test_member(&id_b, "pc-b", &private_b, id_a.as_str()), + test_member(&id_c, "pc-c", &private_c, id_a.as_str()), + ]; + for root in [&root_a, &root_b, &root_c] { + insert_test_group( + root, + "work", + [0x7b; 32], + members.clone(), + groups::Role::Member, + ); + } + let a_port = free_port(); + let b_port = port_other_than(a_port); + let c_port = loop { + let port = free_port(); + if port != a_port && port != b_port { + break port; + } + }; + let mut options_c = options("pc-c", c_port, root_c.clone(), None); + options_c.no_mdns = true; + let (c, _) = Particle::start(options_c, None).await.unwrap(); + let mut options_a = options( + "pc-a", + a_port, + root_a.clone(), + Some(format!("127.0.0.1:{c_port}")), + ); + options_a.no_mdns = true; + let (a, _) = Particle::start(options_a, None).await.unwrap(); + wait_for_group_pair(&a, &c).await; + + let group = groups::find(&root_a, "work").unwrap().unwrap(); + groups::issue_update( + &root_a, + &group.id, + entangle_core::UpdateKind::Rekey, + &private_a, + ) + .unwrap(); + tokio::time::timeout(Duration::from_secs(12), async { + loop { + let caught_up = c + .groups + .read() + .unwrap() + .iter() + .any(|group| group.meta.name == "work" && group.meta.epoch == 1); + if caught_up { + break; + } + sleep(Duration::from_millis(25)).await; + } + }) + .await + .expect("online member should apply the admin's rekey"); + a.shutdown().await; + + let mut options_b = options( + "pc-b", + b_port, + root_b.clone(), + Some(format!("127.0.0.1:{c_port}")), + ); + options_b.no_mdns = true; + let (b, _) = Particle::start(options_b, None).await.unwrap(); + tokio::time::timeout(Duration::from_secs(15), async { + loop { + let caught_up = b + .groups + .read() + .unwrap() + .iter() + .any(|group| group.meta.name == "work" && group.meta.epoch == 1); + let linked = b + .field + .read() + .unwrap() + .by_id(&c.manifest.particle_id) + .is_some_and(|peer| peer.state == PeerState::Entangled); + if caught_up && linked { + break; + } + sleep(Duration::from_millis(25)).await; + } + }) + .await + .expect("offline member should apply relayed updates and relink"); + let delivered = b + .sync_state(json!({ + "particle_id": id_c.as_str(), + "state": { "message": "caught up through a member" } + })) + .await + .unwrap(); + assert_eq!(delivered["delivered"], true); + b.shutdown().await; + c.shutdown().await; + } + + #[tokio::test] + async fn running_member_leave_notifies_peers_and_records_acknowledgements() { + let temp = tempfile::tempdir().unwrap(); + let root_a = temp.path().join("a"); + let root_b = temp.path().join("b"); + let root_c = temp.path().join("c"); + let (id_a, private_a) = saved_identity(&root_a); + let (id_b, private_b) = saved_identity(&root_b); + let (id_c, private_c) = saved_identity(&root_c); + let members = vec![ + test_member(&id_a, "pc-a", &private_a, id_a.as_str()), + test_member(&id_b, "pc-b", &private_b, id_a.as_str()), + test_member(&id_c, "pc-c", &private_c, id_a.as_str()), + ]; + for root in [&root_a, &root_b, &root_c] { + insert_test_group( + root, + "work", + [0x7c; 32], + members.clone(), + groups::Role::Member, + ); + } + let a_port = free_port(); + let b_port = port_other_than(a_port); + let c_port = loop { + let port = free_port(); + if port != a_port && port != b_port { + break port; + } + }; + let mut options_a = options("pc-a", a_port, root_a.clone(), None); + options_a.no_mdns = true; + options_a.peer = vec![format!("127.0.0.1:{b_port}"), format!("127.0.0.1:{c_port}")]; + let mut options_b = options( + "pc-b", + b_port, + root_b.clone(), + Some(format!("127.0.0.1:{a_port}")), + ); + options_b.no_mdns = true; + options_b.peer.push(format!("127.0.0.1:{c_port}")); + let mut options_c = options( + "pc-c", + c_port, + root_c.clone(), + Some(format!("127.0.0.1:{a_port}")), + ); + options_c.no_mdns = true; + options_c.peer.push(format!("127.0.0.1:{b_port}")); + let (a, _) = Particle::start(options_a, None).await.unwrap(); + let (b, _) = Particle::start(options_b, None).await.unwrap(); + let (c, _) = Particle::start(options_c, None).await.unwrap(); + wait_for_group_pair(&a, &b).await; + wait_for_group_pair(&a, &c).await; + wait_for_group_pair(&b, &c).await; + + let group = groups::find(&root_b, "work").unwrap().unwrap(); + groups::set_leaving(&root_b, &group.id, true).unwrap(); + groups::write_status( + &root_b, + &group.id, + &groups::GroupStatus { + epoch: group.meta.epoch, + op: "leave".into(), + confirmed: Vec::new(), + }, + ) + .unwrap(); + tokio::time::timeout(Duration::from_secs(15), async { + loop { + let a_removed = groups::find(&root_a, &group.id) + .unwrap() + .is_some_and(|group| { + group + .members + .iter() + .all(|member| member.particle_id != id_b.as_str()) + }); + let c_removed = groups::find(&root_c, &group.id) + .unwrap() + .is_some_and(|group| { + group + .members + .iter() + .all(|member| member.particle_id != id_b.as_str()) + }); + let status = groups::read_status(&root_b, &group.id).unwrap(); + if a_removed + && c_removed + && groups::find(&root_b, &group.id).unwrap().is_none() + && status.as_ref().is_some_and(|status| { + status.confirmed.len() == 2 + && status.confirmed.contains(&id_a.to_string()) + && status.confirmed.contains(&id_c.to_string()) + }) + { + break; + } + sleep(Duration::from_millis(25)).await; + } + }) + .await + .expect("both members should acknowledge the leave notice"); + a.shutdown().await; + b.shutdown().await; + c.shutdown().await; + } + + #[tokio::test] + async fn group_and_subset_sends_filter_by_authenticated_group() { + let schema: Value = + serde_json::from_str(include_str!("../../../schemas/mcp-tools.json")).unwrap(); + let observe_required = schema["tools"] + .as_array() + .unwrap() + .iter() + .find(|tool| tool["name"] == "observe_entangled_states") + .unwrap()["outputSchema"]["properties"]["states"]["items"]["required"] + .as_array() + .unwrap() + .iter() + .map(|value| value.as_str().unwrap().to_owned()) + .collect::>(); + let temp = tempfile::tempdir().unwrap(); + let root_a = temp.path().join("a"); + let root_b = temp.path().join("b"); + let root_c = temp.path().join("c"); + let (id_a, private_a) = saved_identity(&root_a); + let (id_b, private_b) = saved_identity(&root_b); + let (id_c, private_c) = saved_identity(&root_c); + let member_a_work = test_member(&id_a, "particle-a", &private_a, id_a.as_str()); + let member_b_work = test_member(&id_b, "particle-b", &private_b, id_a.as_str()); + let member_c_work = test_member(&id_c, "particle-c", &private_c, id_a.as_str()); + let member_a_home = member_a_work.clone(); + let member_b_home = member_b_work.clone(); + let work_members = vec![ + member_a_work.clone(), + member_b_work.clone(), + member_c_work.clone(), + ]; + let home_members = vec![member_a_home, member_b_home]; + for (root, role) in [ + (&root_a, groups::Role::Admin), + (&root_b, groups::Role::Member), + (&root_c, groups::Role::Member), + ] { + insert_test_group(root, "work", [0x71; 32], work_members.clone(), role); + } + insert_test_group( + &root_a, + "home", + [0x72; 32], + home_members.clone(), + groups::Role::Admin, + ); + insert_test_group( + &root_b, + "home", + [0x72; 32], + home_members, + groups::Role::Member, + ); + + let a_port = free_port(); + let b_port = port_other_than(a_port); + let c_port = loop { + let port = free_port(); + if port != a_port && port != b_port { + break port; } }; let mut options_b = options( @@ -3324,77 +4175,377 @@ mod tests { assert!(a .sync_state(json!({ "particle_id": b.manifest.particle_id.as_str(), - "state": { "message": "after leave" } + "state": { "message": "after leave" } + })) + .await + .unwrap_err() + .contains("isn't a member of any shared group")); + a.shutdown().await; + b.shutdown().await; + } + + #[tokio::test] + async fn invited_member_is_introduced_across_the_mesh_and_syncs_both_ways() { + let temp = tempfile::tempdir().unwrap(); + let root_a = temp.path().join("a"); + let root_b = temp.path().join("b"); + let root_c = temp.path().join("c"); + let key = [0x63; 32]; + let (id_a, private_a) = saved_identity(&root_a); + let (id_b, private_b) = saved_identity(&root_b); + let (id_c, private_c) = saved_identity(&root_c); + let member_a = test_member(&id_a, "particle-a", &private_a, id_a.as_str()); + let member_c = test_member(&id_c, "particle-c", &private_c, id_a.as_str()); + insert_test_group( + &root_a, + "work", + key, + vec![member_a.clone(), member_c.clone()], + groups::Role::Admin, + ); + insert_test_group( + &root_c, + "work", + key, + vec![member_a, member_c], + groups::Role::Member, + ); + + let a_port = free_port(); + let c_port = port_other_than(a_port); + let b_port = loop { + let port = free_port(); + if port != a_port && port != c_port { + break port; + } + }; + let (c, _) = Particle::start( + options( + "particle-c", + c_port, + root_c.clone(), + Some(format!("127.0.0.1:{b_port}")), + ), + None, + ) + .await + .unwrap(); + let (a, _) = Particle::start( + options( + "particle-a", + a_port, + root_a.clone(), + Some(format!("127.0.0.1:{c_port}")), + ), + None, + ) + .await + .unwrap(); + wait_for_group_pair(&a, &c).await; + + let invite = crate::invite::protocol::InviteServer::new( + &root_a, + groups::group_id(&key), + "particle-a".into(), + crate::invite::code::InviteCode::parse("7-amber-photon-quasar").unwrap(), + 1, + Duration::from_secs(30), + ) + .unwrap(); + let (joiner, inviter) = tokio::io::duplex(1024 * 1024); + let invite_task = tokio::spawn(async move { invite.handle(inviter).await }); + crate::invite::protocol::join( + joiner, + &root_b, + crate::invite::code::InviteCode::parse("7-amber-photon-quasar").unwrap(), + None, + "particle-b", + ) + .await + .unwrap(); + let _ = invite_task.await.unwrap().unwrap(); + + let (b, _) = Particle::start( + options( + "particle-b", + b_port, + root_b.clone(), + Some(format!("127.0.0.1:{a_port}")), + ), + None, + ) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(12), async { + loop { + let c_knows_b = groups::find(&root_c, "work") + .unwrap() + .unwrap() + .members + .iter() + .any(|member| member.particle_id == id_b.as_str()); + let c_linked_b = c + .field + .read() + .unwrap() + .by_id(&b.manifest.particle_id) + .is_some_and(|peer| peer.state == PeerState::Entangled); + if c_knows_b && c_linked_b { + break; + } + sleep(Duration::from_millis(25)).await; + } + }) + .await + .expect("the group watcher should introduce B to C and establish their link"); + for (sender, receiver) in [(&b, &c), (&c, &b)] { + let result = sender + .sync_state(json!({ + "particle_id": receiver.manifest.particle_id.as_str(), + "state": { "mesh": true }, + "timeout_secs": 60 + })) + .await + .unwrap(); + assert_eq!(result["delivered"], true); + } + + let sender_info = PeerInfo { + addr: SocketAddr::from(([127, 0, 0, 1], a_port)), + remote_static: entangle_transport::public_key(&private_a), + group: groups::group_id_bytes(&key), + epoch: 0, + relayed: false, + }; + let untrusted_id = ParticleId::generate(); + let untrusted_private = [0x74; 32]; + let misvouched = test_member( + &untrusted_id, + "untrusted", + &untrusted_private, + id_c.as_str(), + ); + assert_eq!( + c.on_introduce(&sender_info, &id_a, vec![misvouched]) + .await + .unwrap(), + 0 + ); + assert!(groups::find(&root_c, "work") + .unwrap() + .unwrap() + .members + .iter() + .all(|member| member.particle_id != untrusted_id.as_str())); + + let stranger_id = ParticleId::generate(); + let stranger_key = [0x75; 32]; + let stranger_info = PeerInfo { + remote_static: entangle_transport::public_key(&stranger_key), + ..sender_info + }; + assert!(c + .on_introduce(&stranger_info, &stranger_id, vec![]) + .await + .is_err()); + let group = groups::find(&root_a, "work").unwrap().unwrap(); + groups::issue_update( + &root_a, + &group.id, + entangle_core::UpdateKind::Remove { + particle_id: id_b.to_string(), + }, + &private_a, + ) + .unwrap(); + tokio::time::timeout(Duration::from_secs(12), async { + loop { + let caught_up = c + .groups + .read() + .unwrap() + .iter() + .any(|group| group.meta.name == "work" && group.meta.epoch == 1); + if caught_up { + break; + } + sleep(Duration::from_millis(25)).await; + } + }) + .await + .expect("the revoked member update should reach C"); + let revoked_member = test_member(&id_b, "particle-b", &private_b, id_a.as_str()); + assert_eq!( + c.on_introduce(&sender_info, &id_a, vec![revoked_member]) + .await + .unwrap(), + 0 + ); + assert!(groups::find(&root_c, "work") + .unwrap() + .unwrap() + .members + .iter() + .all(|member| member.particle_id != id_b.as_str())); + a.shutdown().await; + b.shutdown().await; + c.shutdown().await; + } + + #[tokio::test] + async fn reinviting_stale_member_refreshes_group_and_restores_links() { + let temp = tempfile::tempdir().unwrap(); + let root_a = temp.path().join("a"); + let root_b = temp.path().join("b"); + let key = [0x66; 32]; + let (id_a, private_a) = saved_identity(&root_a); + let (id_b, private_b) = saved_identity(&root_b); + let members = vec![ + test_member(&id_a, "particle-a", &private_a, id_a.as_str()), + test_member(&id_b, "particle-b", &private_b, id_a.as_str()), + ]; + insert_test_group(&root_a, "work", key, members.clone(), groups::Role::Admin); + insert_test_group(&root_b, "local-work", key, members, groups::Role::Member); + let group_id = groups::group_id(&key); + for _ in 0..18 { + groups::issue_update( + &root_a, + &group_id, + entangle_core::UpdateKind::Rekey, + &private_a, + ) + .unwrap(); + } + + let invite = crate::invite::protocol::InviteServer::new( + &root_a, + group_id.clone(), + "particle-a".into(), + crate::invite::code::InviteCode::parse("7-amber-photon-quasar").unwrap(), + 1, + Duration::from_secs(30), + ) + .unwrap(); + let (joiner, inviter) = tokio::io::duplex(1024 * 1024); + let invite_task = tokio::spawn(async move { invite.handle(inviter).await }); + let joined = crate::invite::protocol::join( + joiner, + &root_b, + crate::invite::code::InviteCode::parse("7-amber-photon-quasar").unwrap(), + None, + "particle-b", + ) + .await + .unwrap(); + invite_task.await.unwrap().unwrap(); + assert!(joined.refreshed); + assert_eq!(joined.group.meta.epoch, 18); + assert_eq!(joined.group.meta.name, "local-work"); + assert_eq!(joined.group.retained_keys.len(), 16); + assert_eq!(joined.group.retained_keys.keys().next(), Some(&2)); + assert_eq!(joined.group.updates.len(), 17); + assert_eq!(joined.group.updates.first().unwrap().epoch, 2); + assert_eq!(joined.group.updates.last().unwrap().epoch, 18); + let mut output = Vec::new(); + crate::write_join_success(&mut output, joined).unwrap(); + assert_eq!( + String::from_utf8(output).unwrap(), + "Refreshed local-work: this device was out of date and now has the current key.\n" + ); + + let (a, _) = Particle::start(options("particle-a", 0, root_a, None), None) + .await + .unwrap(); + let (b, _) = Particle::start(options("particle-b", 0, root_b, None), None) + .await + .unwrap(); + assert!( + a.dial( + Route::Direct(SocketAddr::from(([127, 0, 0, 1], b.manifest.link_port))), + None, + ) + .await + ); + let record = a + .field + .read() + .unwrap() + .by_id(&b.manifest.particle_id) + .expect("refreshed member should link at the current epoch"); + assert_eq!(record.state, PeerState::Entangled); + assert_eq!(record.group_id, Some(groups::id_bytes(&group_id).unwrap())); + let synced = a + .sync_state(json!({ + "particle_id": b.manifest.particle_id.as_str(), + "state": { "message": "refreshed" } })) .await - .unwrap_err() - .contains("isn't a member of any shared group")); + .unwrap(); + assert_eq!(synced["delivered"], true); + a.shutdown().await; b.shutdown().await; } #[tokio::test] - async fn invited_member_is_introduced_across_the_mesh_and_syncs_both_ways() { + async fn refreshed_member_relays_history_to_an_older_member() { let temp = tempfile::tempdir().unwrap(); let root_a = temp.path().join("a"); let root_b = temp.path().join("b"); let root_c = temp.path().join("c"); - let key = [0x63; 32]; + let key = [0x67; 32]; let (id_a, private_a) = saved_identity(&root_a); - let (id_b, _private_b) = saved_identity(&root_b); + let (id_b, private_b) = saved_identity(&root_b); let (id_c, private_c) = saved_identity(&root_c); - let member_a = test_member(&id_a, "particle-a", &private_a, id_a.as_str()); - let member_c = test_member(&id_c, "particle-c", &private_c, id_a.as_str()); - insert_test_group( + let members = vec![ + test_member(&id_a, "particle-a", &private_a, id_a.as_str()), + test_member(&id_b, "particle-b", &private_b, id_a.as_str()), + test_member(&id_c, "particle-c", &private_c, id_a.as_str()), + ]; + for (root, role) in [ + (&root_a, groups::Role::Admin), + (&root_b, groups::Role::Member), + (&root_c, groups::Role::Member), + ] { + insert_test_group(root, "work", key, members.clone(), role); + } + let group_id = groups::group_id(&key); + groups::issue_update( &root_a, - "work", - key, - vec![member_a.clone(), member_c.clone()], - groups::Role::Admin, - ); - insert_test_group( - &root_c, - "work", - key, - vec![member_a, member_c], - groups::Role::Member, - ); - - let a_port = free_port(); - let c_port = port_other_than(a_port); - let b_port = loop { - let port = free_port(); - if port != a_port && port != c_port { - break port; - } - }; - let (c, _) = Particle::start( - options( - "particle-c", - c_port, - root_c.clone(), - Some(format!("127.0.0.1:{b_port}")), - ), - None, + &group_id, + entangle_core::UpdateKind::Rekey, + &private_a, ) - .await .unwrap(); - let (a, _) = Particle::start( - options( - "particle-a", - a_port, - root_a.clone(), - Some(format!("127.0.0.1:{c_port}")), - ), - None, + let second = groups::issue_update( + &root_a, + &group_id, + entangle_core::UpdateKind::Rekey, + &private_a, ) - .await .unwrap(); - wait_for_group_pair(&a, &c).await; + groups::apply_updates(&root_c, &group_id, &second.updates, None).unwrap(); + for _ in 0..16 { + groups::issue_update( + &root_a, + &group_id, + entangle_core::UpdateKind::Rekey, + &private_a, + ) + .unwrap(); + } + assert_eq!( + groups::find(&root_c, &group_id) + .unwrap() + .unwrap() + .meta + .epoch, + 2 + ); let invite = crate::invite::protocol::InviteServer::new( &root_a, - groups::group_id(&key), + group_id.clone(), "particle-a".into(), crate::invite::code::InviteCode::parse("7-amber-photon-quasar").unwrap(), 1, @@ -3403,7 +4554,7 @@ mod tests { .unwrap(); let (joiner, inviter) = tokio::io::duplex(1024 * 1024); let invite_task = tokio::spawn(async move { invite.handle(inviter).await }); - crate::invite::protocol::join( + let joined = crate::invite::protocol::join( joiner, &root_b, crate::invite::code::InviteCode::parse("7-amber-photon-quasar").unwrap(), @@ -3412,91 +4563,69 @@ mod tests { ) .await .unwrap(); - let _ = invite_task.await.unwrap().unwrap(); - - let (b, _) = Particle::start( - options( - "particle-b", - b_port, - root_b.clone(), - Some(format!("127.0.0.1:{a_port}")), - ), - None, - ) - .await - .unwrap(); - tokio::time::timeout(Duration::from_secs(12), async { + invite_task.await.unwrap().unwrap(); + assert_eq!(joined.group.meta.epoch, 18); + assert_eq!(joined.group.retained_keys.get(&2), Some(&second.key)); + assert_eq!(joined.group.updates.first().unwrap().epoch, 2); + assert_eq!(joined.group.updates.last().unwrap().epoch, 18); + + let b_port = free_port(); + let c_port = port_other_than(b_port); + let mut options_b = options("particle-b", b_port, root_b.clone(), None); + options_b.no_mdns = true; + let mut options_c = options( + "particle-c", + c_port, + root_c.clone(), + Some(format!("127.0.0.1:{b_port}")), + ); + options_c.no_mdns = true; + let (b, _) = Particle::start(options_b, None).await.unwrap(); + let (c, _) = Particle::start(options_c, None).await.unwrap(); + assert!( + c.dial( + Route::Direct(SocketAddr::from(([127, 0, 0, 1], b.manifest.link_port))), + None, + ) + .await + ); + tokio::time::timeout(Duration::from_secs(15), async { loop { - let c_knows_b = groups::find(&root_c, "work") - .unwrap() + let caught_up = c + .groups + .read() .unwrap() - .members .iter() - .any(|member| member.particle_id == id_b.as_str()); - let c_linked_b = c + .any(|group| group.id == group_id && group.meta.epoch == 18); + let b_linked = b + .field + .read() + .unwrap() + .by_id(&c.manifest.particle_id) + .is_some_and(|peer| peer.state == PeerState::Entangled); + let c_linked = c .field .read() .unwrap() .by_id(&b.manifest.particle_id) .is_some_and(|peer| peer.state == PeerState::Entangled); - if c_knows_b && c_linked_b { + if caught_up && b_linked && c_linked { break; } sleep(Duration::from_millis(25)).await; } }) .await - .expect("the group watcher should introduce B to C and establish their link"); - for (sender, receiver) in [(&b, &c), (&c, &b)] { - let result = sender - .sync_state(json!({ - "particle_id": receiver.manifest.particle_id.as_str(), - "state": { "mesh": true }, - "timeout_secs": 60 - })) - .await - .unwrap(); - assert_eq!(result["delivered"], true); - } - - let sender_info = PeerInfo { - addr: SocketAddr::from(([127, 0, 0, 1], a_port)), - remote_static: entangle_transport::public_key(&private_a), - group: groups::group_id_bytes(&key), - relayed: false, - }; - let untrusted_id = ParticleId::generate(); - let untrusted_private = [0x74; 32]; - let misvouched = test_member( - &untrusted_id, - "untrusted", - &untrusted_private, - id_c.as_str(), - ); - assert_eq!( - c.on_introduce(&sender_info, &id_a, vec![misvouched]) - .await - .unwrap(), - 0 - ); - assert!(groups::find(&root_c, "work") - .unwrap() - .unwrap() - .members - .iter() - .all(|member| member.particle_id != untrusted_id.as_str())); - - let stranger_id = ParticleId::generate(); - let stranger_key = [0x75; 32]; - let stranger_info = PeerInfo { - remote_static: entangle_transport::public_key(&stranger_key), - ..sender_info - }; - assert!(c - .on_introduce(&stranger_info, &stranger_id, vec![]) + .expect("the refreshed member should relay its update history to C"); + let delivered = c + .sync_state(json!({ + "particle_id": id_b.as_str(), + "state": { "message": "caught up through refreshed member" } + })) .await - .is_err()); - a.shutdown().await; + .unwrap(); + assert_eq!(delivered["delivered"], true); + b.shutdown().await; c.shutdown().await; } @@ -3648,6 +4777,18 @@ mod tests { ) .await .unwrap(); + stream + .send( + &serde_json::to_vec(&entangle_link::Frame::GroupUpdates { updates: vec![] }) + .unwrap(), + ) + .await + .unwrap(); + let updates_ack = stream.recv(Duration::from_secs(2)).await.unwrap(); + assert!(matches!( + serde_json::from_slice::(&updates_ack).unwrap(), + entangle_link::Frame::GroupUpdates { updates } if updates.is_empty() + )); let frame = entangle_link::Frame::SyncOffer { from: a.manifest.particle_id.clone(), sync_id: sync_id.into(), @@ -3751,6 +4892,7 @@ mod tests { addr: SocketAddr::from(([127, 0, 0, 1], 7337)), remote_static: entangle_transport::public_key(&remote_private), group: groups::group_id_bytes(&[0x41; 32]), + epoch: 0, relayed: false, }; let decision = particle @@ -3772,6 +4914,7 @@ mod tests { addr: SocketAddr::from(([127, 0, 0, 1], 7337)), remote_static: [0x22; 32], group: groups::group_id_bytes(&[0x41; 32]), + epoch: 0, relayed: false, }, manifest, @@ -3797,6 +4940,7 @@ mod tests { addr: SocketAddr::from(([127, 0, 0, 1], 7337)), remote_static: [0x22; 32], group: groups::group_id_bytes(&[0x41; 32]), + epoch: 0, relayed: false, }, &offer, @@ -3834,17 +4978,35 @@ mod tests { let address = listener.local_addr().unwrap(); let responder_keys = KeyRing { static_private: unbound_private, - groups: HashMap::from([(group_id, groups::derive_psk(&group_key))]), + groups: HashMap::from([( + group_id, + entangle_transport::GroupKeys { + current: 0, + keys: std::collections::BTreeMap::from([(0, groups::derive_psk(&group_key))]), + }, + )]), }; let accept_task = tokio::spawn(async move { let (stream, _) = listener.accept().await.unwrap(); - entangle_transport::accept_io( + let mut stream = entangle_transport::accept_io( Box::new(stream), &responder_keys, Duration::from_secs(5), ) .await .unwrap(); + let updates = stream.recv(Duration::from_secs(5)).await.unwrap(); + assert!(matches!( + serde_json::from_slice::(&updates).unwrap(), + entangle_link::Frame::GroupUpdates { updates } if updates.is_empty() + )); + stream + .send( + &serde_json::to_vec(&entangle_link::Frame::GroupUpdates { updates: vec![] }) + .unwrap(), + ) + .await + .unwrap(); }); let error = match particle .sync_handshake( @@ -3857,11 +5019,70 @@ mod tests { Ok(_) => panic!("unbound static key must not complete the sync handshake"), Err(error) => error, }; - assert!(error.contains("remote Noise static key did not match")); + assert!( + error.contains("remote Noise static key did not match"), + "unexpected sync handshake error: {error}" + ); accept_task.await.unwrap(); particle.shutdown().await; } + #[tokio::test] + async fn direct_hello_adds_tcp_address_after_repeater_discovery() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("node"); + let (local_id, local_private) = saved_identity(&root); + let id = ParticleId::generate(); + let remote_private = [0x33; 32]; + insert_test_group( + &root, + "test", + [0x41; 32], + vec![ + test_member(&local_id, "receiver", &local_private, local_id.as_str()), + test_member(&id, "peer", &remote_private, local_id.as_str()), + ], + groups::Role::Admin, + ); + let (particle, _) = Particle::start(options("receiver", free_port(), root, None), None) + .await + .unwrap(); + let mut manifest = peer_manifest(id.clone()); + manifest.link_port = 7338; + particle.field.write().unwrap().resonate(Resonance { + particle_id: id.clone(), + name: manifest.name.clone(), + hostname: "via-repeater".into(), + addrs: vec![], + port: 7338, + context: None, + groups: vec![groups::group_id(&[0x41; 32])], + proto: manifest.protocol, + }); + + particle + .on_hello( + &PeerInfo { + addr: SocketAddr::from(([127, 0, 0, 1], 7338)), + remote_static: entangle_transport::public_key(&remote_private), + group: groups::group_id_bytes(&[0x41; 32]), + epoch: 0, + relayed: false, + }, + manifest, + ) + .await + .unwrap(); + + let record = particle.field.read().unwrap().by_id(&id).unwrap(); + assert_eq!( + record.resonance.addrs, + vec![std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)] + ); + assert_eq!(record.path, Some(LinkPath::Direct)); + particle.shutdown().await; + } + #[tokio::test] async fn relayed_hello_preserves_discovery_addresses_and_never_learns_placeholder() { let temp = tempfile::tempdir().unwrap(); @@ -3902,6 +5123,7 @@ mod tests { addr: RELAY_PLACEHOLDER, remote_static: entangle_transport::public_key(&remote_private), group: groups::group_id_bytes(&[0x41; 32]), + epoch: 0, relayed: true, }, manifest, @@ -4025,6 +5247,43 @@ mod tests { let _ = server.await; } + #[tokio::test] + async fn repeater_uses_active_group_when_outdated_group_sorts_first() { + let temp = tempfile::tempdir().unwrap(); + let temp_root = canonical_temp_root(&temp); + let root_a = temp_root.join("a"); + let root_b = temp_root.join("b"); + let repeater_root = temp_root.join("repeater"); + let active_key = [0x41; 32]; + let old_key = [0x11; 32]; + assert_ne!(groups::group_id(&active_key), groups::group_id(&old_key)); + let (server, _, _) = + prepare_test_repeater_pair(&root_a, &root_b, &repeater_root, free_port()).await; + write_outdated_test_group(&root_a, "aaa-legacy", old_key); + + let mut options_a = options("particle-a", 0, root_a.clone(), None); + let mut options_b = options("particle-b", 0, root_b.clone(), None); + options_a.relay_only = true; + options_b.relay_only = true; + let (a, _) = Particle::start(options_a, None).await.unwrap(); + let (b, _) = Particle::start(options_b, None).await.unwrap(); + assert!(a.groups.read().unwrap()[0].outdated); + + wait_for_repeater(&a, &b).await; + sleep(Duration::from_millis(1100)).await; + for particle in [&a, &b] { + assert_eq!( + particle.find_particles(&json!({}))["self"]["repeater"]["connected"], + true + ); + } + + a.shutdown().await; + b.shutdown().await; + server.abort(); + let _ = server.await; + } + #[tokio::test] async fn normal_particle_uses_repeater_for_relay_only_peer() { let temp = tempfile::tempdir().unwrap(); diff --git a/docs/AGENT_SETUP.md b/docs/AGENT_SETUP.md index 4d38bcf..ca5ab7c 100644 --- a/docs/AGENT_SETUP.md +++ b/docs/AGENT_SETUP.md @@ -110,8 +110,8 @@ proceeding. ## Groups -Human-only commands: `group create`, `group invite`, `group join`, and -`group leave`. Agents may run +Human-only commands: `group create`, `group invite`, `group join`, `group +leave`, `group remove`, `group rekey`, and `group admin`. Agents may run `entangle group list`, `entangle group members `, and `entangle peer list`. One device invites with `entangle group invite [--count N]`; the human enters the one-time code on each other device with @@ -121,7 +121,9 @@ group label, not the particle name. `--count N` sets the successful-join limit connection fallback when multicast discovery is blocked. Invites expire after ten minutes, and a wrong code or failed confirmation closes the invite. Ask the human to perform all invite/join commands in their terminal. Agents never -request, display, or relay invite codes. +request, display, or relay invite codes. Agents never remove members, rotate +group keys, transfer administration, or leave groups; these commands require +human judgment and confirmation. Trust is based on shared group membership, and group keys must never be requested or relayed. If `find_entangled_particles.self.warning` is present, report its text verbatim diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 5fa6d55..6874adf 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -22,21 +22,33 @@ eight eight-hex group IDs. Group keys are never advertised. ## Groups and link Each machine can belong to up to eight groups stored at -`/groups//{key,meta.json,members.json}`. The `key` file contains -the 32-byte group key as lowercase hex; `meta.json` stores the group's name and -local role, and `members.json` maps particle IDs to names and Noise static -public keys. Files and directories use private permissions. The group ID is -the first four bytes of +`/groups//{key,meta.json,members.json,keys.json,updates.json}`. +The `key` file contains the current 32-byte group key as lowercase hex; +`meta.json` stores the group name, epoch, admin identity and signing key, and +the current revocation set. `members.json` maps particle IDs to names, Noise +static public keys, and signing public keys. `keys.json` retains up to 16 prior +epoch keys and `updates.json` retains the corresponding signed update chain. +Files and directories use private permissions. The group ID is the first four +bytes of `SHA-256("entangle-group-id-v1" || key)`, encoded as eight lowercase hex -characters. Its Noise PSK is +characters at creation; it remains stable when the key rotates. Its Noise PSK is `SHA-256("entangle-group-psk-v1" || key)`. -Protocol version 4 binds each link to one group. Before the Noise handshake, -the initiator writes a 9-byte preface: `ENTG`, preface version `1`, and the -four group-ID bytes. Both sides use the unchanged +Group admins sign updates with an Ed25519 key derived from the particle's +static private key. Updates form a contiguous epoch chain and carry the new +group key, admin, and revocation set. A member verifies each signature against +the current admin key before applying the next epoch. Updates can be relayed by +any current member; a removed member cannot pass current membership checks. + +Protocol version 5 binds each link to a group and epoch. Before the Noise +handshake, the initiator writes a 13-byte preface: `ENTG`, preface version `2`, +the four group-ID bytes, and the requested epoch as a big-endian `u32`. +Responders negotiate retained epochs with plaintext acknowledgements; a peer +outside the retained-key window is rejected before Noise. Both sides use the +unchanged `Noise_XXpsk3_25519_ChaChaPoly_BLAKE2s` pattern, the selected group's PSK at position 3, and the prologue -`b"entangle-link-v4" || group_id_bytes`. A responder rejects malformed +`b"entangle-link-v5" || group_id_bytes || epoch_be`. A responder rejects malformed prefaces and IDs for groups it does not have. Persistent particles store their 32-byte Noise static private key as 64 lowercase hex characters at `/noise_static.key`; ephemeral identities keep their static key in @@ -90,8 +102,8 @@ sequenceDiagram B-->>A: MCP notification ``` -Protocol version 4 is not wire-compatible with earlier versions; all devices -must use version 4 to communicate. MCP continues to use JSON-RPC 2.0 with +Protocol version 5 is not wire-compatible with earlier versions; all devices +must use version 5 to communicate. MCP continues to use JSON-RPC 2.0 with newline-delimited messages on stdio. Logs go to stderr; stdout belongs exclusively to MCP. diff --git a/docs/SETUP.md b/docs/SETUP.md index ffd3271..e70240f 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -114,6 +114,13 @@ The inviter should stay online briefly after onboarding: existing group members learn about newcomers when the inviter next links to them, using one-hop member introductions. +Group membership administration is human-only. The group admin can use +`entangle group remove ` to remove a member and rotate the key, +`entangle group rekey ` to rotate the key without changing membership, +and `entangle group admin ` to hand administration to a +current member. An admin must hand off administration before leaving a group +with other members. See [playbook 10](playbooks/10-leave-remove-rekey.md). + ## Register as an MCP server Each device runs its own Entangle process. The default name is diff --git a/docs/playbooks/10-leave-remove-rekey.md b/docs/playbooks/10-leave-remove-rekey.md new file mode 100644 index 0000000..9e258d1 --- /dev/null +++ b/docs/playbooks/10-leave-remove-rekey.md @@ -0,0 +1,80 @@ +# 10. Leave, remove, rekey, and hand off + +Use this when a member leaves, an admin removes a device, rotates a group key, +or transfers administration to another member. + +## You'll need + +- Entangle 0.6.0 or later on each device. +- The group already joined on each device. +- A human operator for all membership and key-administration commands. +- The admin's node online to propagate a removal, rekey, or handoff promptly. + +## Steps + +1. **On the current admin's device (your terminal):** Check the group roster and + identify the exact member to remove or promote. + + ```sh + entangle group members work + ``` + +2. **On the admin's device:** Remove a departed or untrusted member. Confirm the + prompt with `y`; this rotates the key and distributes the signed update to + reachable members. + + ```sh + entangle group remove work pc-c + ``` + +3. **On the admin's device:** Rotate a key without changing membership when a + group credential may have been exposed. + + ```sh + entangle group rekey work + ``` + +4. **On the admin's device:** Before the admin leaves a group that still has + other members, transfer administration to a current member. + + ```sh + entangle group admin work pc-b + ``` + +5. **On a member's device:** Leave the group. When its node is running, it + notifies reachable members before removing its local group; when stopped, + the CLI removes the local group immediately. + + ```sh + entangle group leave work + ``` + +Membership changes are human-only. Agents may inspect the roster, but must +never run `group remove`, `group rekey`, `group admin`, or `group leave`. + +## It worked if + +These lines were captured during a local run with separate data directories +and nodes on this machine: + +```text +Removed pc-c from work. Delivered to 1 of 2 members so far (pc-a); pc-d gets it the next time it connects to any member. +pc-a is now the admin of work. Delivered to 1 of 2 members so far (pc-a); pc-d gets it the next time it connects to any member. +Left work. Told 2 of 2 members. +``` + +## If something goes wrong + +- If an update is delivered to only some members, those members can relay its + signed update chain when they next link to another current member. +- A device that is too far out of date must be invited again by the admin. On + joining, it refreshes the existing local group to the current key while + keeping its local group name. The join command prints: + `Refreshed : this device was out of date and now has the current key.` +- A removed device must ask the admin to invite it again; a non-admin cannot + restore its membership. + +## Next + +- [Troubleshooting](11-troubleshooting.md) +- [First group](01-first-group.md) diff --git a/docs/playbooks/11-troubleshooting.md b/docs/playbooks/11-troubleshooting.md index e702c8c..81c6e86 100644 --- a/docs/playbooks/11-troubleshooting.md +++ b/docs/playbooks/11-troubleshooting.md @@ -83,6 +83,9 @@ The cross-network paths were not tested on a real LAN. | `That code was wrong; the invite is now closed. Ask for a new one.` | The code did not authenticate; the inviter closes that invite on a wrong code. | Ask for a new invite code. | | `configured: no` | This device has not saved repeater enrollment in the data directory used by the current process. | Have a person complete repeater enrollment in that same data directory. | | Members are not linked even though both devices list the group. | Membership does not make a blocked route reachable. | Check the link TCP port, static `--peer` address, and repeater enrollment. | +| `group is too far out of date ...; re-invite this device` | The device no longer has a retained key epoch shared with a current member. | Ask the admin to invite it again, then join with the new code. Joining refreshes the stale local group to the current key, keeps its local name, and prints `Refreshed : this device was out of date and now has the current key.` | +| `This device was removed from ; ask its admin to invite it.` | The device is revoked in the group's signed membership state. | Ask the group admin to invite this device again. | +| `group was created by Entangle 0.6; run ... group leave ... and create it again` | This is a pre-v2 group and is not migrated. | Leave the outdated group and create or join a current group again. | ## Next diff --git a/docs/playbooks/README.md b/docs/playbooks/README.md index d8bf6a5..7fc1752 100644 --- a/docs/playbooks/README.md +++ b/docs/playbooks/README.md @@ -13,5 +13,5 @@ Use these walkthroughs for common group, network, repeater, and client setups. | 07 | [Laptop away from home](07-laptop-away-from-home.md) | A configured laptop must connect while off its home LAN | Laptop and reachable repeater | | 08 | [Agents coordinating a test across machines](08-agents-coordinating-a-test-across-machines.md) | Agents need to coordinate a multi-machine test | Agents on two or more devices | | 09 | [Windows with omp](09-windows-omp.md) | You are installing and configuring Entangle on Windows with omp | Windows device and its group peers | -| 10 | Leaving a group, removing a device, rekeying — coming with member removal | Membership administration is available | Group members | +| 10 | [Leave, remove, rekey, and hand off](10-leave-remove-rekey.md) | A member leaves or an admin changes group membership or keys | Group members | | 11 | [Troubleshooting](11-troubleshooting.md) | Discovery, linking, invitations, or repeater enrollment fails | The affected device and relevant peers |