diff --git a/core/binary_protocol/src/consensus/command.rs b/core/binary_protocol/src/consensus/command.rs index ea2f9451de..cf275f8e8a 100644 --- a/core/binary_protocol/src/consensus/command.rs +++ b/core/binary_protocol/src/consensus/command.rs @@ -96,6 +96,8 @@ pub enum Command { // Fresh metadata and partition readiness for primary rebalancing. PrimaryRebalance = 33, SegmentDelete = 34, + CommitReply = 35, + PrepareProgress = 36, } // SAFETY: Command is #[repr(u8)] with no padding bytes. @@ -106,7 +108,7 @@ unsafe impl CheckedBitPattern for Command { type Bits = u8; fn is_valid_bit_pattern(bits: &u8) -> bool { - *bits <= Self::SegmentDelete as u8 + *bits <= Self::PrepareProgress as u8 } } @@ -127,7 +129,7 @@ mod tests { #[test] fn replica_auth_commands_are_valid_bit_patterns() { - for command in 14u8..=34 { + for command in 14u8..=36 { let mut buf: AVec> = AVec::new(16); buf.resize(256, 0); buf[60] = command; @@ -135,7 +137,7 @@ mod tests { } let mut buf: AVec> = AVec::new(16); buf.resize(256, 0); - buf[60] = 35; + buf[60] = 37; assert!(bytemuck::checked::try_from_bytes::(&buf).is_err()); } } diff --git a/core/binary_protocol/src/consensus/header.rs b/core/binary_protocol/src/consensus/header.rs index 222c2b33e3..81dc625db3 100644 --- a/core/binary_protocol/src/consensus/header.rs +++ b/core/binary_protocol/src/consensus/header.rs @@ -1499,6 +1499,166 @@ impl ConsensusHeader for CommitHeader { } } +/// Backup -> primary: echo a heartbeat counter for quorum liveness. +/// +/// This acknowledges connectivity, not journal or state-machine durability. +/// Nodes using this message require a coordinated cluster upgrade. +#[repr(C)] +#[derive(Debug, Clone, Copy, CheckedBitPattern, NoUninit)] +pub struct CommitReplyHeader { + pub checksum: u128, + pub checksum_body: u128, + pub cluster: u128, + pub size: u32, + pub view: u32, + pub release: u32, + pub command: Command, + pub replica: u8, + pub reserved_frame: [u8; 66], + pub timestamp_monotonic: u64, + pub group: u64, + pub partition_incarnation: u64, + pub reserved: [u8; 104], +} + +const _: () = { + assert!(size_of::() == HEADER_SIZE); + assert!(offset_of!(CommitReplyHeader, timestamp_monotonic) == 128); + assert!(offset_of!(CommitReplyHeader, partition_incarnation) == 144); + assert!(offset_of!(CommitReplyHeader, reserved) + size_of::<[u8; 104]>() == HEADER_SIZE); +}; + +impl ConsensusHeader for CommitReplyHeader { + const FRAME_SEALED: bool = true; + const COMMAND: Command = Command::CommitReply; + + fn checksum(&self) -> u128 { + self.checksum + } + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } + fn operation(&self) -> Operation { + Operation::Reserved + } + fn command(&self) -> Command { + self.command + } + fn size(&self) -> u32 { + self.size + } + + fn validate(&self) -> Result<(), ConsensusError> { + if self.command != Self::COMMAND { + return Err(ConsensusError::InvalidCommand { + expected: Self::COMMAND, + found: self.command, + }); + } + if usize::try_from(self.size) != Ok(HEADER_SIZE) { + return Err(ConsensusError::InvalidField( + "commit_reply: invalid size".to_string(), + )); + } + if self.release != 0 { + return Err(ConsensusError::InvalidField("release != 0".to_string())); + } + Ok(()) + } +} + +/// Correlates a replication checkpoint with the backup's contiguous prefixes. +/// +/// Probes must stay ordered with prepares on the sender's queue, so a reply +/// proves that the backup consumed every frame sent before its probe. That FIFO +/// position, not an op, releases transport credit. Progress never counts toward +/// a commit quorum. +/// Replies echo the nonce and observed op. `held_op` is the end of the prefix +/// that the backup holds with proof in the current view, so the primary finds +/// replication gaps from it. `received_op <= held_op` is the part of that +/// prefix that meets the prepare acknowledgment rules, and `ready` certifies it. Probes advertise the primary's applied frontier for retention +/// fallback. +/// Nodes using this message require a coordinated cluster upgrade. +#[repr(C)] +#[derive(Debug, Clone, Copy, CheckedBitPattern, NoUninit)] +pub struct PrepareProgressHeader { + pub checksum: u128, + pub checksum_body: u128, + pub cluster: u128, + pub size: u32, + pub view: u32, + pub release: u32, + pub command: Command, + pub replica: u8, + pub reserved_frame: [u8; 66], + pub nonce: u64, + pub group: u64, + pub partition_incarnation: u64, + pub observed_op: u64, + pub received_op: u64, + pub held_op: u64, + pub commit_op: u64, + pub kind: u8, + pub ready: u8, + pub reserved: [u8; 70], +} + +const _: () = { + assert!(size_of::() == HEADER_SIZE); + assert!(offset_of!(PrepareProgressHeader, nonce) == 128); + assert!(offset_of!(PrepareProgressHeader, partition_incarnation) == 144); + assert!(offset_of!(PrepareProgressHeader, reserved) + size_of::<[u8; 70]>() == HEADER_SIZE); +}; + +impl ConsensusHeader for PrepareProgressHeader { + const FRAME_SEALED: bool = true; + const COMMAND: Command = Command::PrepareProgress; + + fn checksum(&self) -> u128 { + self.checksum + } + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } + fn operation(&self) -> Operation { + Operation::Reserved + } + fn command(&self) -> Command { + self.command + } + fn size(&self) -> u32 { + self.size + } + + fn validate(&self) -> Result<(), ConsensusError> { + if self.command != Self::COMMAND { + return Err(ConsensusError::InvalidCommand { + expected: Self::COMMAND, + found: self.command, + }); + } + if usize::try_from(self.size) != Ok(HEADER_SIZE) { + return Err(ConsensusError::InvalidField( + "prepare_progress: invalid size".to_string(), + )); + } + if self.release != 0 { + return Err(ConsensusError::InvalidField("release != 0".to_string())); + } + if self.kind > Self::REPLY || self.ready > 1 { + return Err(ConsensusError::InvalidField( + "prepare_progress: invalid fields".to_string(), + )); + } + Ok(()) + } +} + +impl PrepareProgressHeader { + pub const PROBE: u8 = 0; + pub const REPLY: u8 = 1; +} + // StartViewChangeHeader - failure detection (header-only) /// Replica suspects primary failure. Header-only. @@ -3156,13 +3316,14 @@ mod tests { } use super::prepare_identity_checksum_bytes; use super::{ - Command, CommitHeader, ConsensusError, ConsensusHeader, DoViewChangeHeader, EvictionHeader, - EvictionReason, ForwardCompletionHeader, ForwardLogoutHeader, ForwardLogoutOutcome, - ForwardLogoutResultHeader, ForwardRegisterHeader, ForwardRegisterOutcome, - ForwardRegisterResultHeader, GenericHeader, HEADER_SIZE, LIFECYCLE_BATCH_HEADER_SIZE, - LIFECYCLE_COMPLETION_SIZE, LIFECYCLE_COMPLETIONS_MAX, Operation, PrepareHeader, - PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, - RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, + Command, CommitHeader, CommitReplyHeader, ConsensusError, ConsensusHeader, + DoViewChangeHeader, EvictionHeader, EvictionReason, ForwardCompletionHeader, + ForwardLogoutHeader, ForwardLogoutOutcome, ForwardLogoutResultHeader, + ForwardRegisterHeader, ForwardRegisterOutcome, ForwardRegisterResultHeader, GenericHeader, + HEADER_SIZE, LIFECYCLE_BATCH_HEADER_SIZE, LIFECYCLE_COMPLETION_SIZE, + LIFECYCLE_COMPLETIONS_MAX, Operation, PrepareHeader, PrepareOkHeader, + PrepareProgressHeader, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, + RequestHeader, RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, RoutedRequestHeader, StartViewChangeHeader, StartViewHeader, StateChunkHeader, StateTransferTargetHeader, }; @@ -3226,6 +3387,8 @@ mod tests { seals!( PrepareOkHeader, CommitHeader, + CommitReplyHeader, + PrepareProgressHeader, StartViewChangeHeader, DoViewChangeHeader, StartViewHeader, @@ -3292,6 +3455,8 @@ mod tests { } check_incarnation!( CommitHeader, + CommitReplyHeader, + PrepareProgressHeader, StartViewChangeHeader, DoViewChangeHeader, StartViewHeader, diff --git a/core/binary_protocol/src/consensus/mod.rs b/core/binary_protocol/src/consensus/mod.rs index 0953de3d22..491d1bdc2e 100644 --- a/core/binary_protocol/src/consensus/mod.rs +++ b/core/binary_protocol/src/consensus/mod.rs @@ -52,15 +52,16 @@ pub use consumer_session::{ }; pub use error::ConsensusError; pub use header::{ - CHECKSUM_UNSEALED, CommitHeader, ConsensusHeader, DVC_HEADERS_MAX, DoViewChangeHeader, - EvictionHeader, EvictionReason, ForwardCompletionHeader, ForwardLogoutHeader, - ForwardLogoutOutcome, ForwardLogoutResultHeader, ForwardRegisterHeader, ForwardRegisterOutcome, - ForwardRegisterResultHeader, GenericHeader, HEADER_SIZE, LIFECYCLE_BATCH_HEADER_SIZE, - LIFECYCLE_COMPLETION_SIZE, LIFECYCLE_COMPLETIONS_MAX, PrepareHeader, PrepareOkHeader, - RESERVED_COMMAND_LEN, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, - RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, - RequestStateTransferHeader, RoutedRequestHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader, - StartViewHeader, StateChunkHeader, StateTransferTargetHeader, frame_body, frame_checksum_bytes, + CHECKSUM_UNSEALED, CommitHeader, CommitReplyHeader, ConsensusHeader, DVC_HEADERS_MAX, + DoViewChangeHeader, EvictionHeader, EvictionReason, ForwardCompletionHeader, + ForwardLogoutHeader, ForwardLogoutOutcome, ForwardLogoutResultHeader, ForwardRegisterHeader, + ForwardRegisterOutcome, ForwardRegisterResultHeader, GenericHeader, HEADER_SIZE, + LIFECYCLE_BATCH_HEADER_SIZE, LIFECYCLE_COMPLETION_SIZE, LIFECYCLE_COMPLETIONS_MAX, + PrepareHeader, PrepareOkHeader, PrepareProgressHeader, RESERVED_COMMAND_LEN, + RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, RequestPreparesHeader, + RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, + RoutedRequestHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader, StartViewHeader, + StateChunkHeader, StateTransferTargetHeader, frame_body, frame_checksum_bytes, prepare_identity_checksum_bytes, read_size_field, }; pub use operation::Operation; diff --git a/core/binary_protocol/src/lib.rs b/core/binary_protocol/src/lib.rs index e8f31d8051..4705ddcd5f 100644 --- a/core/binary_protocol/src/lib.rs +++ b/core/binary_protocol/src/lib.rs @@ -70,20 +70,20 @@ pub mod version; pub use codec::{WireDecode, WireEncode}; pub use consensus::{ - CHECKSUM_UNSEALED, Command, CommitHeader, ConsensusError, ConsensusHeader, ConsumerSession, - ConsumerSessionHeartbeatHeader, DVC_HEADERS_MAX, DoViewChangeHeader, EvictionHeader, - EvictionReason, ForwardCompletionHeader, ForwardLogoutHeader, ForwardLogoutOutcome, - ForwardLogoutResultHeader, ForwardRegisterHeader, ForwardRegisterOutcome, + CHECKSUM_UNSEALED, Command, CommitHeader, CommitReplyHeader, ConsensusError, ConsensusHeader, + ConsumerSession, ConsumerSessionHeartbeatHeader, DVC_HEADERS_MAX, DoViewChangeHeader, + EvictionHeader, EvictionReason, ForwardCompletionHeader, ForwardLogoutHeader, + ForwardLogoutOutcome, ForwardLogoutResultHeader, ForwardRegisterHeader, ForwardRegisterOutcome, ForwardRegisterResultHeader, GenericHeader, HEADER_SIZE, LIFECYCLE_BATCH_HEADER_SIZE, LIFECYCLE_COMPLETION_SIZE, LIFECYCLE_COMPLETIONS_MAX, MAX_CONSUMER_SESSIONS_PER_HEARTBEAT, - Operation, PrepareHeader, PrepareOkHeader, PrimaryRebalanceHeader, PrimaryRebalanceKind, - REJECTION_SECTION_LEN, RESERVED_COMMAND_LEN, RepairPrepareHeader, RepairRangeReplyHeader, - ReplyHeader, RequestHeader, RequestPreparesHeader, RequestStartViewHeader, - RequestStateChunkHeader, RequestStateTransferHeader, RoutedRequestHeader, SIZE_FIELD_OFFSET, - SegmentDeleteHeader, SegmentDeleteKind, StartViewChangeHeader, StartViewHeader, - StateChunkHeader, StateTransferTargetHeader, frame_body, frame_checksum_bytes, - prepare_identity_checksum_bytes, read_size_field, result_code, result_section_len, - write_rejection_section, + Operation, PrepareHeader, PrepareOkHeader, PrepareProgressHeader, PrimaryRebalanceHeader, + PrimaryRebalanceKind, REJECTION_SECTION_LEN, RESERVED_COMMAND_LEN, RepairPrepareHeader, + RepairRangeReplyHeader, ReplyHeader, RequestHeader, RequestPreparesHeader, + RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, + RoutedRequestHeader, SIZE_FIELD_OFFSET, SegmentDeleteHeader, SegmentDeleteKind, + StartViewChangeHeader, StartViewHeader, StateChunkHeader, StateTransferTargetHeader, + frame_body, frame_checksum_bytes, prepare_identity_checksum_bytes, read_size_field, + result_code, result_section_len, write_rejection_section, }; pub use dispatch::{ COMMAND_TABLE, CommandMeta, command_index, lookup_by_operation, lookup_command, diff --git a/core/configs/src/server_config/cluster.rs b/core/configs/src/server_config/cluster.rs index e9ea5bd95e..63020d904b 100644 --- a/core/configs/src/server_config/cluster.rs +++ b/core/configs/src/server_config/cluster.rs @@ -200,9 +200,9 @@ fn default_superblock_wedged_fatal_timeout() -> IggyDuration { pub struct ClusterConfig { pub enabled: bool, pub name: String, - /// Backup-side liveness window for a plane's primary. A replica that - /// sees no primary traffic for this long starts a view change - /// (`normal_heartbeat_timeout`). Raise it on oversubscribed hosts where + /// Liveness window for the primary and its replication quorum. Backups + /// vote after primary silence; a primary without heartbeat replies from + /// a quorum relinquishes leadership. Raise it on oversubscribed hosts where /// scheduling stalls fake primary death; sub-`MIN_CLUSTER_HEARTBEAT_TIMEOUT` /// values (including the `0` / `disabled` / `unlimited` sentinels, which /// all parse to zero) are rejected at boot. On Linux it is also the diff --git a/core/configs/src/server_config/defaults.rs b/core/configs/src/server_config/defaults.rs index 13e5a6f119..e4fd326df6 100644 --- a/core/configs/src/server_config/defaults.rs +++ b/core/configs/src/server_config/defaults.rs @@ -193,6 +193,10 @@ impl Default for PartitionConfig { .wal_bytes_max .parse() .expect("embedded WAL capacity is valid"), + repair_memory_bytes_max: partition + .repair_memory_bytes_max + .parse() + .expect("embedded repair memory capacity is valid"), wal_group_commit_delay_micros: u64::try_from(partition.wal_group_commit_delay_micros) .expect("embedded WAL group commit delay is valid"), validate_checksum: SERVER_CONFIG.partition.validate_checksum, diff --git a/core/configs/src/server_config/partition.rs b/core/configs/src/server_config/partition.rs index b57d3be418..787dec53bf 100644 --- a/core/configs/src/server_config/partition.rs +++ b/core/configs/src/server_config/partition.rs @@ -118,9 +118,9 @@ fn default_offset_reservation_lease() -> NonZeroU32 { pub const DEFAULT_EVICTED_RING_CAPACITY: usize = 65536; /// Upper bound on `evicted_ring_capacity`. The ring exists per multi-replica -/// partition and each retained entry pins a full committed batch, so -/// worst-case pinned memory scales with the partition count; a typo guard, -/// not a sizing endorsement. +/// partition, and each retained entry pins a full committed batch until the +/// byte cap releases it to its segment, so worst-case pinned memory scales +/// with the partition count. A typo guard, not a sizing endorsement. pub const MAX_EVICTED_RING_CAPACITY: usize = 65536; /// Mirrors `partitions::EVICTED_RING_BYTES_MAX`. @@ -164,6 +164,11 @@ fn default_wal_bytes_max() -> IggyByteSize { IggyByteSize::from(DEFAULT_PARTITION_WAL_BYTES_MAX) } +fn default_repair_memory_bytes_max() -> IggyByteSize { + const DEFAULT_REPAIR_MEMORY_BYTES_MAX: u64 = 1024 * 1024 * 1024; + IggyByteSize::from(DEFAULT_REPAIR_MEMORY_BYTES_MAX) +} + /// Ceiling on the group-commit delay. A longer wait costs more than the /// barrier it is meant to amortize. pub const MAX_PARTITION_WAL_GROUP_COMMIT_DELAY_MICROS: u64 = 10_000; @@ -177,9 +182,19 @@ fn default_wal_group_commit_delay_micros() -> u64 { pub struct PartitionConfig { /// Active WAL and pending prepare budget per persisted partition. /// Temporary generation rewrites may require additional disk space. + /// + /// It also bounds the memory of a backup that repairs an unproven log from + /// the head down: that repair holds up to this many bytes of batches per + /// partition, subject to the shared `repair_memory_bytes_max` cap. #[serde(default = "default_wal_bytes_max")] #[config_env(leaf)] pub wal_bytes_max: IggyByteSize, + /// Node-wide allocation cap for unproven repair walks and snapshot repair tails. + /// Exhausted walks fall back to state transfer; exhausted tails require a fresh snapshot. + /// Normal prepare queues and state-transfer buffers have separate limits. + #[serde(default = "default_repair_memory_bytes_max")] + #[config_env(leaf)] + pub repair_memory_bytes_max: IggyByteSize, /// Bounded wait, in microseconds, for more prepares before a persisted /// partition's WAL writer starts its durability barrier. Zero disables it. /// @@ -248,13 +263,15 @@ pub struct PartitionConfig { /// restarting peer can be served from the ring before falling back to /// bulk sync, at the cost of pinned memory per partition. Must be > 0 and /// <= [`MAX_EVICTED_RING_CAPACITY`]. Single-replica partitions retain - /// nothing regardless. + /// nothing regardless. The entries whose batches went back to their + /// segments have their own cap of 16384. pub evicted_ring_capacity: usize, /// Byte ceiling for the evicted ring per partition; whichever ring cap /// (this or [`Self::evicted_ring_capacity`]) trips first evicts. Bounds /// the ring memory a burst of large batches can pin. Must be > 0 and <= - /// [`MAX_EVICTED_RING_BYTES`]. + /// [`MAX_EVICTED_RING_BYTES`]. The headers of the batches that went back to + /// their segments do not count here. #[config_env(leaf)] pub evicted_ring_bytes_max: IggyByteSize, @@ -282,6 +299,11 @@ pub struct PartitionConfig { impl Validatable for PartitionConfig { fn validate(&self) -> Result<(), ConfigurationError> { + let repair_bytes = self.repair_memory_bytes_max.as_bytes_u64(); + if repair_bytes == 0 || usize::try_from(repair_bytes).is_err() { + eprintln!("{COMPONENT} partition.repair_memory_bytes_max must fit in usize and be > 0"); + return Err(ConfigurationError::InvalidConfigurationValue); + } let wal_bytes = self.wal_bytes_max.as_bytes_u64(); if !(MIN_PARTITION_WAL_BYTES_MAX..=MAX_PARTITION_WAL_BYTES_MAX).contains(&wal_bytes) || !wal_bytes.is_multiple_of(4096) @@ -426,6 +448,33 @@ mod tests { ); } + #[test] + fn given_a_repair_memory_limit_when_loading_config_should_preserve_overrides_and_reject_zero() { + const CUSTOM_BYTES: u64 = 8 * 1024 * 1024; + let omitted: PartitionConfig = serde_json::from_str(&partial_table(None)).unwrap(); + assert_eq!( + omitted.repair_memory_bytes_max, + PartitionConfig::default().repair_memory_bytes_max, + "older configuration files use the shipped node-wide limit" + ); + let mut config = PartitionConfig { + repair_memory_bytes_max: IggyByteSize::from(CUSTOM_BYTES), + ..PartitionConfig::default() + }; + assert!(config.validate().is_ok()); + let restored: PartitionConfig = + serde_json::from_str(&serde_json::to_string(&config).unwrap()).unwrap(); + assert_eq!( + restored.repair_memory_bytes_max.as_bytes_u64(), + CUSTOM_BYTES + ); + config.repair_memory_bytes_max = IggyByteSize::from(0_u64); + assert!( + config.validate().is_err(), + "a zero repair budget must fail validation" + ); + } + #[test] fn default_impl_validates() { // `Default` reads the shipped config.toml; the pristine deployment diff --git a/core/consensus/src/impls.rs b/core/consensus/src/impls.rs index cea43028fb..433943ebed 100644 --- a/core/consensus/src/impls.rs +++ b/core/consensus/src/impls.rs @@ -1049,8 +1049,9 @@ impl StateTransferStage { /// drain the journal or correct a stale peer. #[derive(Debug, Clone, Copy)] pub enum CommitOutcome { - /// Nothing to do: the heartbeat was absorbed (or ignored as stale / - /// foreign / wrong-status) without moving `commit_max`. + /// Stale, foreign, or wrong-status heartbeat. Send no reply. + Ignored, + /// Valid heartbeat without a new commit point. Reply for quorum liveness. Accepted, /// `commit_max` advanced; run `commit_journal`. Advanced, @@ -1060,6 +1061,22 @@ pub enum CommitOutcome { RespondStartView, } +impl CommitOutcome { + /// Reply only to a heartbeat accepted from the current primary. + #[must_use] + pub const fn reply(self, header: &iggy_binary_protocol::CommitHeader) -> Option { + match self { + Self::Accepted | Self::Advanced => Some(VsrAction::SendCommitReply { + view: header.view, + group: header.group, + timestamp_monotonic: header.timestamp_monotonic, + target: header.replica, + }), + Self::Ignored | Self::RespondStartView => None, + } + } +} + /// Actions to be taken by the caller after processing a VSR event. #[derive(Debug, Clone)] pub enum VsrAction { @@ -1156,6 +1173,13 @@ pub enum VsrAction { timestamp_monotonic: u64, target: Option, }, + /// Echo a valid primary heartbeat without promising log durability. + SendCommitReply { + view: u32, + group: u64, + timestamp_monotonic: u64, + target: u8, + }, } impl VsrAction { @@ -1178,6 +1202,16 @@ impl VsrAction { } } +/// Whether a received `DoViewChange` is the empty vote of +/// [`VsrAction::is_empty_vote`]. +fn is_empty_vote(header: &DoViewChangeHeader, suffix: &DvcSuffix) -> bool { + header.untrusted_log() + && header.log_view == 0 + && header.op == 0 + && header.commit == 0 + && suffix.is_empty() +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrepareOkOutcome { Accepted { @@ -1199,13 +1233,8 @@ impl PrepareOkOutcome { } } -/// What the primary of the current view knows of each replica's log and has -/// told it of the commit point, so an eager `Commit` reaches a backup only once -/// that backup holds the op it announces. -/// -/// Chain replication forwards a prepare to the last backups after the quorum -/// commits it. A `Commit` sent to every backup at once overtakes the prepare -/// there, and the backup repairs an op that is still on its way to it. +/// Eager commits wait for each backup's confirmed durable progress. A peer +/// paused for progress must not repair fresh operations still waiting to be sent. #[derive(Debug)] struct CommitAnnouncements { /// Commit point the planes asked the backups to learn without waiting for @@ -1385,6 +1414,10 @@ where loopback_notifier: RefCell>, /// Tracks start view change messages received from all replicas (including self) start_view_change_from_all_replicas: RefCell>, + /// The view the votes in `start_view_change_from_all_replicas` are for. Never + /// below `view`, and above it while this replica votes to leave its view + /// without having left it, see [`Self::handle_start_view_change`]. + start_view_change_view: Cell, /// Consecutive unanswered `RequestStartView` probes while Recovering; /// at the `probe_attempts_max` ceiling the replica falls back to an /// election. @@ -1426,8 +1459,8 @@ where /// `StartView` or by starting a view as primary, whose merged log either /// rule makes safe. untrusted_log: Cell, - /// View-change escalations since [`Self::untrusted_log`] was last set, for - /// rate-limiting the halt warning. + /// View-change status timeouts since [`Self::untrusted_log`] was last set, + /// for rate-limiting the halt warning. untrusted_view_change_timeouts: Cell, /// This replica's own uncommitted suffix, with the head, commit point and @@ -1519,10 +1552,14 @@ where /// View and counter of the last emitted or accepted heartbeat. A new /// primary's counter can be lower, so freshness is scoped to its view. heartbeat_watermark: Cell<(u32, u64)>, + /// Only replies to counters emitted after this window began can count. + quorum_heartbeat_floor: Cell, + heartbeat_replies: RefCell>, /// The next tick looks for backups owed an eager `Commit`; see /// [`Self::request_eager_commit`]. eager_commit: Cell, commit_announcements: RefCell, + pub(crate) replication: RefCell, /// Views whose logs were refused for naming a different prepare at an op /// this replica applied; see [`Self::contradicts_applied_log`]. applied_log_contradictions: Cell, @@ -1892,6 +1929,7 @@ impl> VsrConsensus { loopback_queue: RefCell::new(VecDeque::with_capacity(prepare_queue_max)), loopback_notifier: RefCell::new(None), start_view_change_from_all_replicas: RefCell::new(BitSet::with_capacity(REPLICAS_MAX)), + start_view_change_view: Cell::new(0), probe_attempts: Cell::new(0), probe_attempts_max: Cell::new(PROBE_ATTEMPTS_MAX), recovery_election_allowed: Cell::new(true), @@ -1918,8 +1956,11 @@ impl> VsrConsensus { sent_own_do_view_change: Cell::new(false), timeouts: RefCell::new(TimeoutManager::new(timeout_seed)), heartbeat_watermark: Cell::new((0, 0)), + quorum_heartbeat_floor: Cell::new(0), + heartbeat_replies: RefCell::new(BitSet::with_capacity(REPLICAS_MAX)), eager_commit: Cell::new(false), commit_announcements: RefCell::new(CommitAnnouncements::new(replica_count)), + replication: RefCell::new(crate::replication::ReplicationFlow::new(replica_count)), applied_log_contradictions: Cell::new(0), applied_log_contradictions_taken: Cell::new(0), contradiction_reported_view: Cell::new(None), @@ -2028,6 +2069,7 @@ impl> VsrConsensus { pub fn init(&self) { self.status.set(Status::Normal); self.rejoined_on_recovered_log.set(false); + self.reset_quorum_window(); let mut timeouts = self.timeouts.borrow_mut(); if self.is_primary() { // Prepare is deliberately NOT armed here: a fresh primary has an @@ -2037,6 +2079,7 @@ impl> VsrConsensus { // `push_prepare_entry` arms it, timed from that push rather than from // boot. timeouts.start(TimeoutKind::CommitMessage); + timeouts.start(TimeoutKind::CheckQuorum); } else { timeouts.start(TimeoutKind::NormalHeartbeat); } @@ -2105,12 +2148,14 @@ impl> VsrConsensus { /// The primary of a Normal view, unless it ceded at boot /// (`init_as_backup`): then it is a backup by role however the view math /// reads, so it neither retransmits prepares nor announces commits. - fn is_acting_primary(&self) -> bool { + pub(crate) fn is_acting_primary(&self) -> bool { self.is_primary() && self.status.get() == Status::Normal && !self.ceded_primaryship.get() } /// Ask the partition quorum to elect `target` in the next view assigned to it. - /// A ready backup may initiate this too. Refusal leaves consensus unchanged. + /// Only the primary can: its `StartViewChange` leaves its view, so its backups + /// follow it, while that of a backup is only a vote. Refusal leaves consensus + /// unchanged. /// The caller fences the incarnation, drains history work, refreshes the DVC /// suffix and persists the new view before dispatching the returned actions. pub fn request_primary_transfer(&self, expected_view: u32, target: u8) -> Vec { @@ -2118,8 +2163,7 @@ impl> VsrConsensus { if self.group == METADATA_GROUP || target >= self.replica_count || view != expected_view - || !self.is_normal() - || self.has_ceded_primaryship() + || !self.is_acting_primary() || self.is_transferring() || self.view_log_unproven() || self.observed_newer_view.get() > view @@ -2362,6 +2406,7 @@ impl> VsrConsensus { pub fn set_view(&mut self, view: u32) { self.view.set(view); + self.start_view_change_view.set(view); } #[must_use] @@ -3128,9 +3173,8 @@ impl> VsrConsensus { /// `drain_request_queue_into_prepares` could project stale entries /// via `pipeline_prepare_common`, which panics on non-normal status. pub(crate) fn reset_view_change_state(&self) { - self.reset_svc_quorum(); + self.timeouts.borrow_mut().stop(TimeoutKind::CheckQuorum); self.reset_dvc_quorum(); - self.sent_own_start_view_change.set(false); self.sent_own_do_view_change.set(false); // A merge parked for the superseded view may describe a different log, so // drop it and let the new attempt re-derive from the DVCs it collects. @@ -3151,6 +3195,7 @@ impl> VsrConsensus { } /// Timers and retransmissions continue while storage leases defer history changes. + #[allow(clippy::too_many_lines)] pub fn tick_with_history_fence( &self, plane: PlaneKind, @@ -3163,6 +3208,11 @@ impl> VsrConsensus { timeouts.tick(); // Phase 2: Handle fired timeouts + if timeouts.fired(TimeoutKind::CheckQuorum) { + drop(timeouts); + self.handle_check_quorum_timeout(); + timeouts = self.timeouts.borrow_mut(); + } if !history_fenced && timeouts.fired(TimeoutKind::NormalHeartbeat) { drop(timeouts); actions.extend(self.handle_normal_heartbeat_timeout(plane)); @@ -3206,10 +3256,10 @@ impl> VsrConsensus { // primary answers or an election's StartView adopts us. // - ViewChange backup: the election may have concluded with our // copy of the StartView lost; re-requesting it is a - // two-message fix, while the ViewChangeStatus escalation - // backstop burns a fresh cluster-wide election. The would-be - // primary of the view skips the probe (it concludes the view - // itself or escalates). + // two-message fix, while the ViewChangeStatus backstop votes + // for a fresh cluster-wide election. The primary candidate of + // the view skips the probe (it concludes the view itself, or + // probes from its status timeout once it saw a newer view). match self.status.get() { Status::Recovering => { // A probe answered by nobody, repeatedly, means nobody is @@ -3301,8 +3351,8 @@ impl> VsrConsensus { actions } - /// Called when `normal_heartbeat` timeout fires. - /// Backup hasn't heard from primary - start view change. + /// Called when `normal_heartbeat` timeout fires: the backup did not hear its + /// primary, so it votes to replace it. fn handle_normal_heartbeat_timeout(&self, plane: PlaneKind) -> Vec { // A recovering replica makes progress through RequestStartView // retries, not elections; it is quorum-invisible. @@ -3350,7 +3400,12 @@ impl> VsrConsensus { return Vec::new(); } - self.start_election(plane, ViewChangeReason::NormalHeartbeatTimeout) + // The timer keeps running: each firing starts a fresh round of votes, and + // the heartbeat that resets it withdraws the vote. + self.timeouts + .borrow_mut() + .reset(TimeoutKind::NormalHeartbeat); + self.vote_for_next_view(plane, ViewChangeReason::NormalHeartbeatTimeout) } /// Advance to `view + 1` and start a view change (own SVC counted). @@ -3358,6 +3413,110 @@ impl> VsrConsensus { self.enter_view_change(plane, self.view.get() + 1, reason) } + /// Start a fresh round of votes to leave this view, for the next view or for + /// the higher one this replica already votes for. A round drops the votes of + /// the last one, so a peer that heard the primary again stops counting once it + /// stops sending its vote. + fn vote_for_next_view(&self, plane: PlaneKind, reason: ViewChangeReason) -> Vec { + let view = self.start_view_change_view.get().max(self.view.get() + 1); + tracing::info!( + replica = self.replica, + namespace_raw = self.group, + view = self.view.get(), + next_view = view, + reason = reason.as_str(), + "voting to leave the view" + ); + self.start_view_change_view.set(view); + self.reset_svc_quorum(); + self.count_view_change_vote(plane, view, None, reason) + } + + /// Count the vote of `voter` to leave this view for `view`, and the vote of + /// this replica. A view-change quorum of votes enters `view`. The votes carry + /// into it, so the `DoViewChange` goes out at once. + fn count_view_change_vote( + &self, + plane: PlaneKind, + view: u32, + voter: Option, + reason: ViewChangeReason, + ) -> Vec { + if view > self.start_view_change_view.get() { + self.start_view_change_view.set(view); + self.reset_svc_quorum(); + } + let (vote_count, own_vote_is_new) = { + let mut voters = self.start_view_change_from_all_replicas.borrow_mut(); + if let Some(voter) = voter { + voters.insert(usize::from(voter)); + } + let own_vote_is_new = voters.insert(usize::from(self.replica)); + (voters.count(), own_vote_is_new) + }; + if vote_count >= self.quorum_view_change() { + let mut actions = self.enter_view_change(plane, view, reason); + actions.extend(self.send_do_view_change_on_votes(plane)); + return actions; + } + if !own_vote_is_new { + return Vec::new(); + } + self.sent_own_start_view_change.set(true); + self.timeouts + .borrow_mut() + .start(TimeoutKind::StartViewChangeMessage); + let action = VsrAction::SendStartViewChange { + view, + group: self.group, + }; + emit_sim_event( + SimEventKind::ControlMessageScheduled, + &ControlActionLogEvent::from_vsr_action( + ReplicaLogContext::from_consensus(self, plane), + &action, + ), + ); + vec![action] + } + + /// Whether this replica votes to leave its view for `view`, a higher one, + /// with `voter`. + /// + /// A replica in normal operation that heard its primary since its last + /// heartbeat timeout refuses, and so does the primary itself, so a replica + /// that stops hearing a primary its peers still hear cannot depose it. A + /// replica that saw a newer view catches up through its probe instead, so its + /// vote never counts against a primary it did not hear. + fn joins_view_change_vote(&self, voter: u8, view: u32) -> bool { + let current = self.view.get(); + view >= self.start_view_change_view.get() + && !self.has_observed_newer_view() + && (self.status.get() == Status::ViewChange + || self.votes_to_leave_view() + // The primary left its view. + || voter == self.primary_index(current) + // The voter left this view already. Only a replica that entered a + // view alone gets there while its peers stay behind. + || view - current > 1) + } + + /// Whether this replica votes to leave a view it has not left yet. + const fn votes_to_leave_view(&self) -> bool { + self.start_view_change_view.get() > self.view.get() + } + + /// Drop the vote of this replica to leave its view, and the votes it counted + /// with it: it follows a primary of its view again. + fn withdraw_view_change_vote(&self) { + self.start_view_change_view.set(self.view.get()); + self.reset_svc_quorum(); + self.sent_own_start_view_change.set(false); + self.timeouts + .borrow_mut() + .stop(TimeoutKind::StartViewChangeMessage); + } + /// Enter `Status::ViewChange` at `new_view`: count this replica's own SVC, /// arm the view-change timers, and schedule the SVC broadcast. /// @@ -3380,6 +3539,11 @@ impl> VsrConsensus { ) -> Vec { let old_view = self.view.get(); + // Votes counted for `new_view` carry into it. + if self.start_view_change_view.get() != new_view { + self.start_view_change_view.set(new_view); + self.reset_svc_quorum(); + } self.view.set(new_view); self.status.set(Status::ViewChange); self.reset_view_change_state(); @@ -3421,7 +3585,7 @@ impl> VsrConsensus { vec![action] } - /// Resend SVC message if we've started view change. + /// Resend the vote of this replica, while it has one. fn handle_start_view_change_message_timeout(&self, plane: PlaneKind) -> Vec { if !self.sent_own_start_view_change.get() { return Vec::new(); @@ -3432,7 +3596,7 @@ impl> VsrConsensus { .reset(TimeoutKind::StartViewChangeMessage); let action = VsrAction::SendStartViewChange { - view: self.view.get(), + view: self.start_view_change_view.get(), group: self.group, }; emit_sim_event( @@ -3481,7 +3645,11 @@ impl> VsrConsensus { vec![action] } - /// Escalate to next view if stuck in view change. + /// Vote for the next view if this view change is stuck. The replica stays in + /// its view change, so a late `StartView` still lands, and leaves the view + /// only once a view-change quorum votes with it. The primary candidate of the + /// view does not probe, so once it saw a newer view it probes from here: the + /// cluster elected past this view without it. fn handle_view_change_status_timeout(&self, plane: PlaneKind) -> Vec { if self.status.get() != Status::ViewChange { return Vec::new(); @@ -3489,18 +3657,20 @@ impl> VsrConsensus { if self.untrusted_log.get() { self.warn_untrusted_halt(); } - - self.enter_view_change( - plane, - self.view.get() + 1, - ViewChangeReason::ViewChangeStatusTimeout, - ) + if self.has_observed_newer_view() { + self.begin_view_probe(); + return Vec::new(); + } + self.timeouts + .borrow_mut() + .reset(TimeoutKind::ViewChangeStatus); + self.vote_for_next_view(plane, ViewChangeReason::ViewChangeStatusTimeout) } /// No view starts without a quorum of trusted logs or every replica /// reporting, which looks like an ordinary stuck view change unless it says - /// so. Logged on the 1st, 2nd, 4th ... 64th escalation and every 64th after, - /// so a halt that lasts hours stays visible without a line per timeout. + /// so. Logged on the 1st, 2nd, 4th ... 64th status timeout and every 64th + /// after, so a halt that lasts hours stays visible without a line per timeout. fn warn_untrusted_halt(&self) { const STEADY_INTERVAL: u32 = 64; let timeouts = self.untrusted_view_change_timeouts.get().saturating_add(1); @@ -3734,9 +3904,14 @@ impl> VsrConsensus { /// Handle a received `StartViewChange` message. /// - /// "When replica i receives STARTVIEWCHANGE messages for its view-number - /// from f OTHER replicas, it sends a DOVIEWCHANGE message to the node - /// that will be the primary in the new view." + /// For a higher view it is a vote to leave this view for that one. This + /// replica joins the vote only as `Self::joins_view_change_vote` allows, + /// and enters the view once a view-change quorum voted for it, its own vote + /// included. A replica that votes stays in its view meanwhile. + /// + /// For the view this replica changes to, it sends its `DoViewChange` once a + /// view-change quorum voted for that view. With `2f + 1` replicas that is the + /// vote of `f` other replicas, as in the paper. /// /// # Panics /// If `header.group` does not match this replica's namespace. @@ -3760,31 +3935,40 @@ impl> VsrConsensus { let from_replica = header.replica; let msg_view = header.view; - // Ignore SVCs for old views - if msg_view < self.view.get() { - return Vec::new(); - } - - let mut actions = Vec::new(); - - // If SVC is for a higher view, advance to that view if msg_view > self.view.get() { - actions.extend(self.enter_view_change( + if !self.joins_view_change_vote(from_replica, msg_view) { + return Vec::new(); + } + return self.count_view_change_vote( plane, msg_view, + Some(from_replica), ViewChangeReason::ReceivedStartViewChange, - )); + ); } - // Record the SVC from sender + // Normal operation has no use for a vote for its own view, and a replica + // that votes for the next view dropped the votes for this one. + if msg_view < self.view.get() + || self.status.get() != Status::ViewChange + || self.votes_to_leave_view() + { + return Vec::new(); + } self.start_view_change_from_all_replicas .borrow_mut() .insert(from_replica as usize); + self.send_do_view_change_on_votes(plane) + } - // Check if we have f SVCs from OTHER replicas - // We need f SVCs from others to send DVC + /// Send the `DoViewChange` of this replica once a view-change quorum, this + /// replica included, voted for the view it changes to. A `DoViewChange` pulls + /// every replica into its view, so fewer votes than that can depose a primary + /// that the rest still hear. + fn send_do_view_change_on_votes(&self, plane: PlaneKind) -> Vec { + let mut actions = Vec::new(); if !self.sent_own_do_view_change.get() - && self.svc_count_excluding_self() >= self.max_faulty() + && self.svc_count_excluding_self() + 1 >= self.quorum_view_change() { self.sent_own_do_view_change.set(true); @@ -3977,13 +4161,31 @@ impl> VsrConsensus { let mut actions = Vec::new(); - // If DVC is for a higher view, advance to that view + // The sender of a real DVC for a higher view counted a view-change quorum + // of votes for that view. A barred replica sends its empty vote to answer + // any `StartViewChange`, a vote nobody joined included, so that one counts + // only as the vote of its sender. if msg_view > self.view.get() { - actions.extend(self.enter_view_change( - plane, - msg_view, - ViewChangeReason::ReceivedDoViewChange, - )); + if is_empty_vote(header, &msg_suffix) { + if !self.joins_view_change_vote(from_replica, msg_view) { + return actions; + } + actions.extend(self.count_view_change_vote( + plane, + msg_view, + Some(from_replica), + ViewChangeReason::ReceivedDoViewChange, + )); + if self.view.get() != msg_view { + return actions; + } + } else { + actions.extend(self.enter_view_change( + plane, + msg_view, + ViewChangeReason::ReceivedDoViewChange, + )); + } } // Only the primary candidate processes DVCs for quorum @@ -4063,10 +4265,14 @@ impl> VsrConsensus { ); self.status.set(Status::Recovering); self.probe_attempts.set(0); + self.withdraw_view_change_vote(); let mut timeouts = self.timeouts.borrow_mut(); timeouts.stop(TimeoutKind::Prepare); timeouts.stop(TimeoutKind::CommitMessage); + timeouts.stop(TimeoutKind::CheckQuorum); timeouts.stop(TimeoutKind::NormalHeartbeat); + timeouts.stop(TimeoutKind::DoViewChangeMessage); + timeouts.stop(TimeoutKind::ViewChangeStatus); timeouts.start(TimeoutKind::RequestStartViewMessage); } @@ -4169,9 +4375,9 @@ impl> VsrConsensus { /// backups stay silent and the prober retries. Special case: a probe /// FROM the replica that is the current view's primary-by-index proves /// that primary cannot lead (a probing replica has either lost its - /// state or abandoned the view), so a peer receiving it elects - /// immediately instead of waiting out the heartbeat timeout on a slot - /// known to be dead. + /// state or abandoned the view), so a peer receiving it votes for the + /// next view immediately instead of waiting out the heartbeat timeout on + /// a slot known to be dead. /// /// # Panics /// Panics when the probe is routed to the wrong group. @@ -4199,7 +4405,7 @@ impl> VsrConsensus { // through (a backup answers nothing, and the true primary of a // newer view answers with its StartView). if header.view == self.view.get() { - return self.start_election(plane, ViewChangeReason::PrimaryProbedView); + return self.vote_for_next_view(plane, ViewChangeReason::PrimaryProbedView); } return Vec::new(); } @@ -4263,6 +4469,31 @@ impl> VsrConsensus { } } + /// Adopt a state-transfer checkpoint at `commit_op`, whose prepare has + /// `checksum`, and clear the pipeline. The last prepare checksum becomes + /// `checksum` only when the head moves to `commit_op`: a head above it keeps + /// its own, and without a checksum it stays as it is. + /// + /// Raises the commit floor and never lowers the head. A head above the + /// checkpoint came from an adopted `StartView` or from prepares journaled + /// before the install wiped the journal, and the `DoViewChange` suffix still + /// reaches up to it, so a rewound sequencer would stamp a vote whose header op + /// disagrees with its own suffix. Journal repair fetches the ops between the + /// checkpoint and the head, as after a restart. + pub fn install_checkpoint(&self, commit_op: u64, checksum: Option) { + if commit_op > self.commit_min.get() { + self.set_commit_floor(commit_op); + } + if commit_op >= self.sequencer.current_sequence() { + self.sequencer.set_sequence(commit_op); + if let Some(checksum) = checksum { + self.set_last_prepare_checksum(checksum); + } + } + self.clear_pipeline(); + self.advance_commit_max(commit_op); + } + fn finish_view_probe(&self) { self.probe_attempts.set(0); self.timeouts @@ -4608,6 +4839,7 @@ impl> VsrConsensus { } self.advance_commit_max(msg_commit); self.reset_view_change_state(); + self.withdraw_view_change_vote(); // Stale pipeline entries from the old view must be discarded self.view_log_unproven.set(needs_reconciliation); @@ -4673,11 +4905,13 @@ impl> VsrConsensus { /// Handle a `Commit` (heartbeat) message from the primary. /// /// Advances `commit_max` and resets the backup's `NormalHeartbeat` timeout - /// so it doesn't start a spurious view change. Returns `true` if - /// `commit_max` advanced, signalling the caller to run `commit_journal`. + /// so it doesn't start a spurious view change. Returns + /// [`CommitOutcome::Advanced`] if `commit_max` advanced, signalling the + /// caller to run `commit_journal`. /// - /// Only refreshes liveness for a newer heartbeat within its view, so - /// old/replayed messages cannot suppress view changes. + /// Only refreshes liveness for a newer heartbeat within its view, so old or + /// replayed messages cannot suppress the `NormalHeartbeat` vote. Such a + /// heartbeat also withdraws the vote of this replica to replace the primary. /// /// # Panics /// If `header.group` does not match this replica's namespace. @@ -4707,11 +4941,11 @@ impl> VsrConsensus { if header.view > self.view.get() { self.observe_newer_view(header.view); } - return CommitOutcome::Accepted; + return CommitOutcome::Ignored; } if self.status.get() != Status::Normal { - return CommitOutcome::Accepted; + return CommitOutcome::Ignored; } if header.view != self.view.get() { @@ -4722,13 +4956,13 @@ impl> VsrConsensus { if header.view > self.view.get() { self.observe_newer_view(header.view); } - return CommitOutcome::Accepted; + return CommitOutcome::Ignored; } // Tolerant skip, not an assert: the replica handshake proves cluster // membership only, so nothing binds `header.replica` to the sender. if header.replica != self.primary_index(header.view) { - return CommitOutcome::Accepted; + return CommitOutcome::Ignored; } let heartbeat = (header.view, header.timestamp_monotonic); @@ -4737,6 +4971,9 @@ impl> VsrConsensus { self.timeouts .borrow_mut() .reset(TimeoutKind::NormalHeartbeat); + if self.votes_to_leave_view() { + self.withdraw_view_change_vote(); + } } let old_commit_max = self.commit_max.get(); @@ -4748,6 +4985,54 @@ impl> VsrConsensus { } } + /// Count a replica once for a heartbeat emitted during this quorum window. + pub fn handle_commit_reply(&self, header: &iggy_binary_protocol::CommitReplyHeader) { + if !self.is_acting_primary() + || header.cluster != self.cluster + || header.group != self.group + || header.partition_incarnation != self.partition_incarnation.get() + || header.view != self.view.get() + || header.replica >= self.replica_count + || header.replica == self.replica + || header.timestamp_monotonic <= self.quorum_heartbeat_floor.get() + || header.timestamp_monotonic > self.heartbeat_watermark.get().1 + { + return; + } + self.heartbeat_replies + .borrow_mut() + .insert(usize::from(header.replica)); + } + + fn reset_quorum_window(&self) { + self.quorum_heartbeat_floor + .set(self.heartbeat_watermark.get().1); + self.heartbeat_replies.borrow_mut().make_empty(); + } + + fn handle_check_quorum_timeout(&self) { + if !self.is_acting_primary() { + return; + } + let responding = self.heartbeat_replies.borrow().count() + 1; + if responding < self.quorum_replication() { + tracing::warn!( + replica = self.replica, + namespace_raw = self.group, + view = self.view.get(), + responding, + quorum = self.quorum_replication(), + "primary lost heartbeat quorum; relinquishing leadership and probing the view" + ); + // Probing stops admission and heartbeats without replacing history + // that an outstanding file lease may still own. + self.begin_view_probe(); + return; + } + self.reset_quorum_window(); + self.timeouts.borrow_mut().reset(TimeoutKind::CheckQuorum); + } + /// Complete view change as the new primary after collecting DVC quorum. /// /// # Client-table maintenance @@ -4806,8 +5091,8 @@ impl> VsrConsensus { // than its peers can be handed a range it cannot rebuild. // // Refuse rather than panic: a further DoViewChange can raise `commit_max` - // and shrink the range, and otherwise the status timeout escalates. A panic - // would restart into the same merge. + // and shrink the range, and otherwise the status timeout votes to leave the + // view. A panic would restart into the same merge. if merged.op_head.saturating_sub(merged.commit_max) > self.prepare_queue_max as u64 { tracing::error!( replica = self.replica, @@ -4951,6 +5236,11 @@ impl> VsrConsensus { } /// Re-establish the announced head after recovering the physical body log. + /// + /// A certificate that names the adopted view does not prove its head. A + /// state-transfer install and a fenced view window both certify only the + /// prefix the WAL holds, so a recovered head below the adopted one resumes + /// the view too. pub fn resume_adopted_view(&self) { let adopted = self.adopted_view.borrow(); let Some(adopted) = adopted.as_ref() else { @@ -4963,7 +5253,8 @@ impl> VsrConsensus { } else { self.log_view_proven.get() }; - if adopted.view > proven { + let below_adopted_head = self.sequencer.current_sequence() < adopted.op_head; + if adopted.view > proven || (adopted.view == proven && below_adopted_head) { self.log_view.set(adopted.view); self.view_log_unproven.set(true); self.view_log_unproven_through.set(u64::MAX); @@ -5292,14 +5583,19 @@ impl> VsrConsensus { self.log_view_proven.set(self.view.get()); } + // A vote for the next view, cast while the merge waited on repair, ends + // with the view this replica now leads. + self.withdraw_view_change_vote(); + // Update timeouts for normal primary operation + self.reset_quorum_window(); { let mut timeouts = self.timeouts.borrow_mut(); timeouts.stop(TimeoutKind::ViewChangeStatus); timeouts.stop(TimeoutKind::DoViewChangeMessage); - timeouts.stop(TimeoutKind::StartViewChangeMessage); timeouts.stop(TimeoutKind::RequestStartViewMessage); timeouts.start(TimeoutKind::CommitMessage); + timeouts.start(TimeoutKind::CheckQuorum); // If there are uncommitted ops in the rebuilt pipeline, start the // Prepare timeout so that lost PrepareOks trigger retransmission. if rebuild_after < new_op { @@ -5472,7 +5768,7 @@ impl> VsrConsensus { /// Record that `replica` holds `op`, and wake the next tick if that makes /// it owed a `Commit`. - fn note_backup_ack(&self, replica: u8, op: u64) { + pub(crate) fn note_backup_ack(&self, replica: u8, op: u64) { if replica == self.replica { return; } @@ -6226,7 +6522,7 @@ mod pipeline_entry_tests { /// Fixtures every consensus test module needs. #[cfg(test)] pub mod test_bus { - use super::{Command, METADATA_GROUP, Message, StartViewHeader}; + use super::{Command, METADATA_GROUP, Message, StartViewChangeHeader, StartViewHeader}; use iggy_binary_protocol::{CommitHeader, ConsensusHeader}; use message_bus::{BusMessage, MessageBus}; use server_common::MESSAGE_ALIGN; @@ -6284,6 +6580,43 @@ pub mod test_bus { ) } + #[must_use] + pub fn make_commit_reply( + view: u32, + replica: u8, + counter: u64, + ) -> Message { + make_commit(view, replica, counter).transmute_header( + |old, header: &mut iggy_binary_protocol::CommitReplyHeader| { + header.command = Command::CommitReply; + header.cluster = old.cluster; + header.size = old.size; + header.view = old.view; + header.replica = old.replica; + header.group = old.group; + header.timestamp_monotonic = old.timestamp_monotonic; + }, + ) + } + + /// A `StartViewChange` from `replica` that votes for `view`. + /// + /// # Panics + /// Never: the header size fits in `u32`. + #[must_use] + pub fn make_start_view_change(view: u32, replica: u8) -> Message { + Message::::new(std::mem::size_of::()) + .transmute_header(|_, header: &mut StartViewChangeHeader| { + header.command = Command::StartViewChange; + header.cluster = 1; + header.replica = replica; + header.view = view; + header.group = METADATA_GROUP; + header.size = u32::try_from(std::mem::size_of::()).unwrap(); + header.seal(); + }) + } + /// A [`MessageBus`] that accepts everything and remembers nothing. pub struct NoopBus; @@ -6319,7 +6652,7 @@ mod timestamp_clamp_tests { use super::*; use crate::LocalPipeline; - use crate::test_bus::{NoopBus, make_commit, make_start_view}; + use crate::test_bus::{NoopBus, make_commit, make_start_view, make_start_view_change}; use aligned_vec::{AVec, ConstAlign}; use iggy_binary_protocol::CommitHeader; @@ -6827,6 +7160,8 @@ mod timestamp_clamp_tests { const VIEW: u32 = 4; const FIRST_HEAD: u64 = 20; const NEXT_HEAD: u64 = FIRST_HEAD + 1; + /// The backup other than this replica, which votes with it. + const PEER: u8 = 2; let consensus = VsrConsensus::with_clock( 1, 0, @@ -6854,6 +7189,8 @@ mod timestamp_clamp_tests { .handle_normal_heartbeat_timeout(PlaneKind::Metadata) .is_empty() ); + let vote = make_start_view_change(VIEW + 1, PEER); + let _ = consensus.handle_start_view_change(PlaneKind::Metadata, vote.header()); assert_eq!(consensus.status(), Status::ViewChange); assert_eq!(consensus.view(), VIEW + 1); assert!(consensus.view_log_unproven()); @@ -7030,12 +7367,12 @@ mod vsr_consensus_tests { const REPLICAS: u8 = 5; const INCARNATION: u64 = 17; for (replica, view, target, next_view) in [ - (1, 0, 3, 3), - (3, 6, 3, 8), + (0, 0, 3, 3), + (1, 6, 3, 8), (1, 6, 4, 9), - (2, 9, 1, 11), - (2, 4, 0, 5), - (2, u32::MAX - 1, 0, u32::MAX), + (4, 9, 1, 11), + (4, 4, 0, 5), + (4, u32::MAX - 1, 0, u32::MAX), ] { let mut consensus = VsrConsensus::new(1, replica, REPLICAS, 0, StageNoopBus, LocalPipeline::new()); @@ -7063,45 +7400,12 @@ mod vsr_consensus_tests { } #[test] - fn given_backup_transfer_when_primary_receives_svc_should_join_the_target_election() { - const TARGET: u8 = 2; + fn given_a_backup_when_requesting_primary_transfer_should_refuse() { let backup = VsrConsensus::new(1, 1, 3, 0, StageNoopBus, LocalPipeline::new()); - let primary = VsrConsensus::new(1, 0, 3, 0, StageNoopBus, LocalPipeline::new()); backup.init(); - primary.init(); - assert!(!backup.request_primary_transfer(0, TARGET).is_empty()); - assert_eq!(primary.status(), Status::Normal); - - let message = Message::::new(size_of::()) - .transmute_header(|_, header: &mut StartViewChangeHeader| { - header.command = Command::StartViewChange; - header.cluster = 1; - header.replica = backup.replica(); - header.view = backup.view(); - header.group = backup.group(); - header.size = u32::try_from(size_of::()).unwrap(); - header.seal(); - }); - let actions = primary.handle_start_view_change(PlaneKind::Partitions, message.header()); - assert_eq!(primary.view(), u32::from(TARGET)); - assert_eq!(primary.status(), Status::ViewChange); - assert_eq!(primary.log_view(), 0); - assert!( - actions.iter().any(|action| matches!( - action, - VsrAction::SendStartViewChange { view: 2, group: 0 } - )) - ); - assert!(actions.iter().any(|action| matches!( - action, - VsrAction::SendDoViewChange { - target: TARGET, - view: 2, - .. - } - ))); - assert!(primary.pending_view_log().is_none()); + assert!(backup.request_primary_transfer(0, 2).is_empty()); + assert_eq!((backup.view(), backup.status()), (0, Status::Normal)); } #[test] @@ -7166,7 +7470,8 @@ mod vsr_consensus_tests { "recovery barrier", "view start head", ] { - let mut consensus = VsrConsensus::new(1, 2, 5, 0, StageNoopBus, LocalPipeline::new()); + // Replica 1 leads `VIEW`. + let mut consensus = VsrConsensus::new(1, 1, 5, 0, StageNoopBus, LocalPipeline::new()); consensus.set_view(VIEW); consensus.set_log_view(VIEW); consensus.mark_superblock_durable(VIEW, VIEW); @@ -7290,29 +7595,6 @@ mod vsr_consensus_tests { ); } - // No newer view seen means the primary is presumed dead, so the timeout - // must still elect -- the split must not swallow the ordinary case. - #[test] - fn heartbeat_timeout_with_no_newer_view_still_elects() { - let consensus = VsrConsensus::new(1, 1, 3, 0, StageNoopBus, LocalPipeline::new()); - consensus.init(); - - let actions = consensus.handle_normal_heartbeat_timeout(PlaneKind::Metadata); - - assert_eq!( - consensus.status(), - Status::ViewChange, - "a silent primary must still trigger an election" - ); - assert_eq!(consensus.view(), 1, "election advances the view"); - assert!( - actions - .iter() - .any(|action| matches!(action, VsrAction::SendStartViewChange { .. })), - "election must broadcast its SVC" - ); - } - // A recorded view at or below the current one is stale (already caught up) // and must not divert a genuine election, since the guard is a strict `>`. #[test] @@ -7321,12 +7603,13 @@ mod vsr_consensus_tests { consensus.init(); consensus.observe_newer_view(0); - let _ = consensus.handle_normal_heartbeat_timeout(PlaneKind::Metadata); + let actions = consensus.handle_normal_heartbeat_timeout(PlaneKind::Metadata); - assert_eq!( - consensus.status(), - Status::ViewChange, - "a stale observed view must not suppress the election" + assert!( + actions + .iter() + .any(|action| matches!(action, VsrAction::SendStartViewChange { view: 1, .. })), + "a stale observed view must not suppress the vote: {actions:?}" ); } @@ -7362,9 +7645,11 @@ mod vsr_consensus_tests { } #[test] - fn given_a_stalled_candidate_when_it_escalates_should_recover_a_lost_start_view() { + fn given_a_stalled_candidate_when_a_newer_view_settles_should_recover_a_lost_start_view() { const REJOINING_REPLICA: u8 = 1; const NEXT_PRIMARY: u8 = 2; + /// The view `NEXT_PRIMARY` settled while the candidate stalled. + const SETTLED_VIEW: u32 = 2; const REPLICA_COUNT: u8 = 3; const LOCAL_HEAD: u64 = 20; const PRIMARY_HEAD: u64 = LOCAL_HEAD + 1; @@ -7389,13 +7674,6 @@ mod vsr_consensus_tests { assert_eq!(backup.status(), Status::ViewChange); assert!(backup.is_primary_for_view(backup.view())); - for _ in 0..TimeoutManager::VIEW_CHANGE_STATUS_TICKS { - backup.tick(plane); - } - assert_eq!(backup.view(), 2); - assert_eq!(backup.status(), Status::ViewChange); - assert!(!backup.is_primary_for_view(backup.view())); - // Model a settled primary whose initial StartView was withheld by persistence. let mut primary = VsrConsensus::new( 1, @@ -7405,11 +7683,18 @@ mod vsr_consensus_tests { StageNoopBus, LocalPipeline::new(), ); - primary.set_view(backup.view()); - primary.set_log_view(backup.view()); + primary.set_view(SETTLED_VIEW); + primary.set_log_view(SETTLED_VIEW); primary.sequencer().set_sequence(PRIMARY_HEAD); primary.restore_commit_state(LOCAL_HEAD, LOCAL_HEAD); primary.init(); + // Its heartbeat names the settled view. + backup.observe_newer_view(SETTLED_VIEW); + + for _ in 0..TimeoutManager::VIEW_CHANGE_STATUS_TICKS { + backup.tick(plane); + } + assert_eq!((backup.view(), backup.status()), (1, Status::Recovering)); let probe = (0..TimeoutManager::REQUEST_START_VIEW_MESSAGE_TICKS) .flat_map(|_| backup.tick(plane)) @@ -7429,7 +7714,7 @@ mod vsr_consensus_tests { ), _ => None, }) - .expect("an escalated candidate must request the missing StartView as a backup"); + .expect("a candidate behind a settled view must request the missing StartView"); let replies = primary.handle_request_start_view(plane, probe.header()); let [ VsrAction::SendStartView { @@ -7670,7 +7955,17 @@ mod vsr_consensus_tests { VsrConsensus::new(1, 0, 3, METADATA_GROUP, StageNoopBus, LocalPipeline::new()); consensus.init(); for _ in 0..TimeoutManager::NORMAL_HEARTBEAT_TICKS { - let _ = consensus.tick(PlaneKind::Metadata); + for action in consensus.tick(PlaneKind::Metadata) { + if let VsrAction::SendCommit { + view, + timestamp_monotonic, + .. + } = action + { + let reply = test_bus::make_commit_reply(view, 1, timestamp_monotonic); + consensus.handle_commit_reply(reply.header()); + } + } } assert_eq!( consensus.heartbeat_watermark.get(), @@ -7698,8 +7993,8 @@ mod vsr_consensus_tests { } let _ = consensus.tick(PlaneKind::Metadata); assert_eq!( - consensus.view(), - 1, + (consensus.view(), consensus.votes_to_leave_view()), + (1, false), "fresh heartbeats must keep the adopted view alive at tick {tick}, \ enters_election={enters_election}" ); @@ -7709,9 +8004,9 @@ mod vsr_consensus_tests { let _ = consensus.tick(PlaneKind::Metadata); } assert_eq!( - consensus.view(), - 2, - "losing the new primary must still trigger an election" + (consensus.view(), consensus.votes_to_leave_view()), + (1, true), + "losing the new primary must still start a vote to replace it" ); } } @@ -7744,9 +8039,9 @@ mod vsr_consensus_tests { let _ = consensus.tick(PlaneKind::Metadata); } assert_eq!( - consensus.view(), - 2, - "replayed, older-view, and non-primary heartbeats must not suppress an election" + (consensus.view(), consensus.votes_to_leave_view()), + (1, true), + "replayed, older-view, and non-primary heartbeats must not suppress the vote" ); } @@ -8788,7 +9083,7 @@ mod recovery_election_tests { use crate::dvc_merge::suffix_all_present; use crate::test_bus::NoopBus; use crate::view_change_quorum::encode_prepare_headers; - use iggy_binary_protocol::{CommitHeader, HEADER_SIZE, Operation}; + use iggy_binary_protocol::{CommitHeader, CommitReplyHeader, HEADER_SIZE, Operation}; const REPLICA_COUNT: u8 = 3; /// Primary of `OLD_VIEW`, crashed holding only part of what it committed. @@ -8874,12 +9169,12 @@ mod recovery_election_tests { consensus } - /// A backup of `OLD_VIEW` holding the whole committed log, left in the view - /// change its primary's crash started and escalated while nobody else was up. - fn survivor_in_view_change(plane: PlaneKind) -> VsrConsensus { + /// A backup of `OLD_VIEW` holding the whole committed log, in normal + /// operation. + fn backup_in_old_view(replica: u8) -> VsrConsensus { let mut consensus = VsrConsensus::new( 1, - SURVIVOR, + replica, REPLICA_COUNT, group(), NoopBus, @@ -8893,10 +9188,20 @@ mod recovery_election_tests { (COMMITTED - 1..=COMMITTED).rev().map(prepare).collect(), )); consensus.init(); - while consensus.view() <= OLD_VIEW + 1 { + consensus + } + + /// A backup of `OLD_VIEW` holding the whole committed log, voting to replace + /// its crashed primary while nobody else is up to join the vote. + fn survivor_voting_alone(plane: PlaneKind) -> VsrConsensus { + let consensus = backup_in_old_view(SURVIVOR); + while !consensus.votes_to_leave_view() { consensus.tick(plane); } - assert_eq!(consensus.status(), Status::ViewChange); + assert_eq!( + (consensus.status(), consensus.view()), + (Status::Normal, OLD_VIEW) + ); consensus } @@ -9090,7 +9395,28 @@ mod recovery_election_tests { header.group = *group; header.timestamp_monotonic = *timestamp_monotonic; }); - let _ = receiver.handle_commit(heartbeat.header()); + receiver + .handle_commit(heartbeat.header()) + .reply(heartbeat.header()) + .into_iter() + .collect() + } + VsrAction::SendCommitReply { + view, + group, + timestamp_monotonic, + target, + } if *target == to => { + let reply = message(0, |header: &mut CommitReplyHeader| { + header.command = Command::CommitReply; + header.cluster = 1; + header.size = u32::try_from(HEADER_SIZE).unwrap(); + header.replica = from; + header.view = *view; + header.group = *group; + header.timestamp_monotonic = *timestamp_monotonic; + }); + receiver.handle_commit_reply(reply.header()); Vec::new() } _ => Vec::new(), @@ -9166,19 +9492,19 @@ mod recovery_election_tests { } /// Two replicas restart after a double crash, the primary having lost the - /// tail of what it committed. The survivor holding that tail is mid view - /// change, so it answers no probe. Its view-change traffic reaches the - /// restarted pair while they probe (and is ignored) and is then delayed past - /// their fallback election, which an asynchronous network permits. Whatever - /// the restarted pair decide without it must still contain the committed ops. + /// tail of what it committed. The survivor holding that tail is a backup, so + /// it answers no probe. Its votes reach the restarted pair while they probe + /// (and are ignored), and its traffic is then delayed past their fallback + /// election, which an asynchronous network permits. Whatever the restarted + /// pair decide without it must still contain the committed ops. #[test] - fn given_survivor_in_view_change_when_restarted_pair_elects_should_keep_committed_ops() { + fn given_survivor_voting_alone_when_restarted_pair_elects_should_keep_committed_ops() { let plane = PlaneKind::Partitions; let mut cluster = Cluster { replicas: vec![ restarted(RESTARTED_PRIMARY, RECOVERED_HEAD, false), restarted(LAGGING_BACKUP, LAGGING_HEAD, false), - survivor_in_view_change(plane), + survivor_voting_alone(plane), ], plane, do_view_changes: Vec::new(), @@ -9210,13 +9536,13 @@ mod recovery_election_tests { /// restarted pair may not settle a view without it, and once it reports /// the cluster settles holding every committed op. #[test] - fn given_survivor_in_view_change_when_it_stays_reachable_should_settle_with_committed_ops() { + fn given_survivor_voting_alone_when_it_stays_reachable_should_settle_with_committed_ops() { let plane = PlaneKind::Partitions; let mut cluster = Cluster { replicas: vec![ restarted(RESTARTED_PRIMARY, RECOVERED_HEAD, false), restarted(LAGGING_BACKUP, LAGGING_HEAD, false), - survivor_in_view_change(plane), + survivor_voting_alone(plane), ], plane, do_view_changes: Vec::new(), @@ -9229,6 +9555,64 @@ mod recovery_election_tests { cluster.assert_settled(COMMITTED); } + /// A primary that restarts with its whole log probes for its own view. Each + /// backup that gets the probe votes at once, so the group elects before a + /// heartbeat timeout. The backups never tick here, so neither can time out. + #[test] + fn given_a_restarted_primary_when_both_backups_get_its_probe_should_elect_without_a_heartbeat_timeout() + { + const NEXT_PRIMARY: u8 = 1; + let plane = PlaneKind::Partitions; + let mut cluster = Cluster { + replicas: vec![ + restarted(RESTARTED_PRIMARY, COMMITTED, true), + backup_in_old_view(NEXT_PRIMARY), + backup_in_old_view(SURVIVOR), + ], + plane, + do_view_changes: Vec::new(), + broadcast_do_view_changes: false, + delivered: 0, + }; + + let probe = (0..TimeoutManager::NORMAL_HEARTBEAT_TICKS) + .find_map(|_| { + cluster + .replica(RESTARTED_PRIMARY) + .tick(plane) + .into_iter() + .find(|action| matches!(action, VsrAction::SendRequestStartView { .. })) + }) + .expect("a restarted replica must probe before a heartbeat timeout"); + cluster.deliver(VecDeque::from([(RESTARTED_PRIMARY, probe)]), |_, _| true); + + cluster.assert_settled(COMMITTED); + assert!(cluster.replica(NEXT_PRIMARY).is_primary()); + } + + #[test] + fn given_a_one_way_partition_when_heartbeat_replies_cannot_return_should_elect_a_reachable_primary() + { + for plane in [PlaneKind::Metadata, PlaneKind::Partitions] { + let mut cluster = Cluster { + replicas: (0..REPLICA_COUNT).map(backup_in_old_view).collect(), + plane, + do_view_changes: Vec::new(), + broadcast_do_view_changes: true, + delivered: 0, + }; + cluster.run(TICKS, |_, to| to != RESTARTED_PRIMARY); + assert!(!cluster.replica(RESTARTED_PRIMARY).is_acting_primary()); + for replica in [LAGGING_BACKUP, SURVIVOR] { + assert_eq!(cluster.replica(replica).status(), Status::Normal); + assert_eq!(cluster.replica(replica).view(), OLD_VIEW + 1); + } + assert!(cluster.replica(LAGGING_BACKUP).is_primary()); + cluster.run(TICKS, |_, _| true); + cluster.assert_settled(COMMITTED); + } + } + /// The survivor is cut off for good. The restarted pair, a quorum by /// count, must stay unavailable rather than settle on their short logs, and /// recover as soon as the survivor is back. @@ -9239,7 +9623,7 @@ mod recovery_election_tests { replicas: vec![ restarted(RESTARTED_PRIMARY, RECOVERED_HEAD, false), restarted(LAGGING_BACKUP, LAGGING_HEAD, false), - survivor_in_view_change(plane), + survivor_voting_alone(plane), ], plane, do_view_changes: Vec::new(), @@ -9478,7 +9862,7 @@ mod recovery_election_tests { replicas: vec![ restarted(RESTARTED_PRIMARY, COMMITTED, true), restarted(LAGGING_BACKUP, LAGGING_HEAD, true), - survivor_in_view_change(plane), + survivor_voting_alone(plane), ], plane, do_view_changes: Vec::new(), @@ -9521,7 +9905,7 @@ mod recovery_election_tests { replicas: vec![ restarted(RESTARTED_PRIMARY, RECOVERED_HEAD, false), candidate, - survivor_in_view_change(plane), + survivor_voting_alone(plane), ], plane, do_view_changes: Vec::new(), @@ -9603,7 +9987,7 @@ mod recovery_election_tests { replicas: vec![ restarted(RESTARTED_PRIMARY, RECOVERED_HEAD, false), lagging, - survivor_in_view_change(plane), + survivor_voting_alone(plane), ], plane, do_view_changes: Vec::new(), @@ -9660,6 +10044,8 @@ mod recovery_election_tests { )); consensus.set_recovered_log_complete(false); consensus.init(); + // Its primary crashed, so it votes to replace it. + let _ = consensus.handle_normal_heartbeat_timeout(plane); consensus }; let view = OLD_VIEW + 1; @@ -9668,7 +10054,7 @@ mod recovery_election_tests { replicas: vec![ restarted(RESTARTED_PRIMARY, COMMITTED, false), candidate, - survivor_in_view_change(plane), + survivor_voting_alone(plane), ], plane, do_view_changes: Vec::new(), @@ -9802,7 +10188,7 @@ mod recovery_election_tests { replicas: vec![ probing(RESTARTED_PRIMARY), probing(LAGGING_BACKUP), - survivor_in_view_change(plane), + survivor_voting_alone(plane), ], plane, do_view_changes: Vec::new(), @@ -9961,7 +10347,7 @@ mod recovery_election_tests { replicas: vec![ restarted(RESTARTED_PRIMARY, RECOVERED_HEAD, false), candidate, - survivor_in_view_change(plane), + survivor_voting_alone(plane), ], plane, do_view_changes: Vec::new(), @@ -10047,6 +10433,379 @@ mod recovery_election_tests { } } +#[cfg(test)] +mod check_quorum_tests { + use super::*; + use crate::test_bus::NoopBus; + use iggy_binary_protocol::CommitReplyHeader; + + const WINDOW_TICKS: u64 = 8; + + fn primary(replicas: u8) -> VsrConsensus { + let consensus = VsrConsensus::new( + 1, + 0, + replicas, + METADATA_GROUP, + NoopBus, + LocalPipeline::new(), + ); + consensus + .timeouts + .borrow_mut() + .set_normal_heartbeat_ticks(WINDOW_TICKS); + consensus.timeouts.borrow_mut().set_commit_message_ticks(1); + consensus.init(); + consensus + } + + fn reply(replica: u8, counter: u64) -> CommitReplyHeader { + *test_bus::make_commit_reply(0, replica, counter).header() + } + + #[test] + fn given_idle_quorums_when_fresh_heartbeats_are_answered_should_retain_the_primary() { + for replicas in 1..=5 { + let consensus = primary(replicas); + for _ in 0..3 * WINDOW_TICKS { + for action in consensus.tick(PlaneKind::Metadata) { + if let VsrAction::SendCommit { + timestamp_monotonic, + .. + } = action + { + for replica in 1..u8::try_from(consensus.quorum_replication()).unwrap() { + consensus.handle_commit_reply(&reply(replica, timestamp_monotonic)); + } + } + } + assert!(consensus.is_acting_primary(), "{replicas} replicas"); + } + } + } + + #[test] + fn given_duplicate_replies_below_quorum_when_history_is_leased_should_stop_leading() { + let consensus = primary(5); + for _ in 0..WINDOW_TICKS { + for action in consensus.tick_with_history_fence(PlaneKind::Metadata, true) { + if let VsrAction::SendCommit { + timestamp_monotonic, + .. + } = action + { + for _ in 0..5 { + consensus.handle_commit_reply(&reply(1, timestamp_monotonic)); + } + } + } + } + assert_eq!(consensus.status(), Status::Recovering); + assert_eq!( + consensus.view(), + 0, + "step-down must not replace leased history" + ); + assert!( + !consensus + .timeouts + .borrow() + .is_ticking(TimeoutKind::CommitMessage) + ); + assert!( + !consensus + .timeouts + .borrow() + .is_ticking(TimeoutKind::CheckQuorum) + ); + } + + #[test] + fn given_a_delayed_reply_when_replayed_in_the_next_window_should_not_extend_leadership() { + let consensus = primary(3); + let _ = consensus.commit_heartbeat(); + let delayed = reply(1, 1); + let _ = consensus.commit_heartbeat(); + consensus.handle_commit_reply(&delayed); + for _ in 0..WINDOW_TICKS { + consensus.tick(PlaneKind::Metadata); + } + assert!( + consensus.is_acting_primary(), + "a reply need not echo the latest broadcast" + ); + for _ in 0..WINDOW_TICKS { + consensus.handle_commit_reply(&delayed); + consensus.tick(PlaneKind::Metadata); + } + assert_eq!( + consensus.status(), + Status::Recovering, + "a previous-window reply cannot count again" + ); + } + + #[test] + fn given_foreign_or_unsent_replies_when_a_window_expires_should_not_count_them() { + let valid = reply(1, 1); + let invalid = [ + CommitReplyHeader { + cluster: 2, + ..valid + }, + CommitReplyHeader { + group: METADATA_GROUP + 1, + ..valid + }, + CommitReplyHeader { + partition_incarnation: 1, + ..valid + }, + CommitReplyHeader { view: 1, ..valid }, + CommitReplyHeader { + replica: 0, + ..valid + }, + CommitReplyHeader { + replica: 3, + ..valid + }, + CommitReplyHeader { + timestamp_monotonic: 0, + ..valid + }, + CommitReplyHeader { + timestamp_monotonic: 2, + ..valid + }, + ]; + for header in invalid { + let consensus = primary(3); + let _ = consensus.commit_heartbeat(); + consensus.handle_commit_reply(&header); + for _ in 0..WINDOW_TICKS { + consensus.tick(PlaneKind::Metadata); + } + assert_eq!(consensus.status(), Status::Recovering, "{header:?}"); + } + } +} + +#[cfg(test)] +mod view_change_vote_tests { + //! A replica that stops hearing its primary votes to replace it, and leaves + //! its view only once a view-change quorum votes with it. + + use super::*; + use crate::LocalPipeline; + use crate::test_bus::{NoopBus, make_commit, make_start_view_change}; + + const REPLICA_COUNT: u8 = 3; + const VIEW: u32 = 0; + const NEXT_VIEW: u32 = VIEW + 1; + /// Leads `VIEW`. + const PRIMARY: u8 = 0; + /// Leads `NEXT_VIEW`. + const CANDIDATE: u8 = 1; + const BACKUP: u8 = 2; + const PLANE: PlaneKind = PlaneKind::Metadata; + + fn replica(replica: u8) -> VsrConsensus { + replica_of(REPLICA_COUNT, replica) + } + + fn replica_of(replica_count: u8, replica: u8) -> VsrConsensus { + let consensus = VsrConsensus::new( + 1, + replica, + replica_count, + METADATA_GROUP, + NoopBus, + LocalPipeline::new(), + ); + consensus.init(); + consensus + } + + /// A replica that entered `NEXT_VIEW` alone, because nobody answered its view + /// probe. + fn alone_in_next_view(replica_count: u8, replica: u8) -> VsrConsensus { + let consensus = replica_of(replica_count, replica); + consensus.begin_view_probe(); + let probing = + TimeoutManager::REQUEST_START_VIEW_MESSAGE_TICKS * u64::from(PROBE_ATTEMPTS_MAX); + let actions = (0..probing) + .flat_map(|_| consensus.tick(PLANE)) + .collect::>(); + assert!(!sends_do_view_change(&actions), "{actions:?}"); + assert_eq!( + (consensus.status(), consensus.view()), + (Status::ViewChange, NEXT_VIEW) + ); + consensus + } + + fn start_view_change( + consensus: &VsrConsensus, + from: u8, + view: u32, + ) -> Vec { + consensus.handle_start_view_change(PLANE, make_start_view_change(view, from).header()) + } + + fn votes_for(actions: &[VsrAction], view: u32) -> bool { + actions.iter().any(|action| { + matches!(action, VsrAction::SendStartViewChange { view: voted, .. } if *voted == view) + }) + } + + fn sends_do_view_change(actions: &[VsrAction]) -> bool { + actions + .iter() + .any(|action| matches!(action, VsrAction::SendDoViewChange { .. })) + } + + #[test] + fn given_a_heartbeat_timeout_should_vote_for_the_next_view_without_leaving_its_own() { + let consensus = replica(BACKUP); + + let actions = consensus.handle_normal_heartbeat_timeout(PLANE); + + assert!(votes_for(&actions, NEXT_VIEW), "{actions:?}"); + assert_eq!(consensus.status(), Status::Normal); + assert_eq!(consensus.view(), VIEW); + } + + #[test] + fn given_a_backup_that_hears_its_primary_when_a_peer_votes_should_not_join() { + let consensus = replica(BACKUP); + + let actions = start_view_change(&consensus, CANDIDATE, NEXT_VIEW); + + assert!(actions.is_empty(), "{actions:?}"); + assert_eq!(consensus.status(), Status::Normal); + assert_eq!(consensus.view(), VIEW); + } + + #[test] + fn given_its_vote_when_a_peer_votes_too_should_enter_the_view_change_and_send_its_do_view_change() + { + let consensus = replica(BACKUP); + let _ = consensus.handle_normal_heartbeat_timeout(PLANE); + + let actions = start_view_change(&consensus, CANDIDATE, NEXT_VIEW); + + assert!(sends_do_view_change(&actions), "{actions:?}"); + assert_eq!(consensus.status(), Status::ViewChange); + assert_eq!(consensus.view(), NEXT_VIEW); + } + + #[test] + fn given_its_vote_when_the_primary_heartbeats_should_withdraw_the_vote() { + let consensus = replica(BACKUP); + let _ = consensus.handle_normal_heartbeat_timeout(PLANE); + + let _ = consensus.handle_commit(make_commit(VIEW, PRIMARY, 1).header()); + + let resent = (0..TimeoutManager::START_VIEW_CHANGE_MESSAGE_TICKS) + .flat_map(|_| consensus.tick(PLANE)) + .collect::>(); + assert!(!votes_for(&resent, NEXT_VIEW), "{resent:?}"); + let actions = start_view_change(&consensus, CANDIDATE, NEXT_VIEW); + assert!(actions.is_empty(), "{actions:?}"); + assert_eq!(consensus.status(), Status::Normal); + assert_eq!(consensus.view(), VIEW); + } + + #[test] + fn given_the_primary_leaves_its_view_should_follow_it() { + let consensus = replica(BACKUP); + + let actions = start_view_change(&consensus, PRIMARY, NEXT_VIEW); + + assert!(sends_do_view_change(&actions), "{actions:?}"); + assert_eq!(consensus.view(), NEXT_VIEW); + } + + #[test] + fn given_a_vote_two_views_ahead_should_join_it() { + let consensus = replica(BACKUP); + + let actions = start_view_change(&consensus, CANDIDATE, NEXT_VIEW + 1); + + assert!(sends_do_view_change(&actions), "{actions:?}"); + assert_eq!(consensus.view(), NEXT_VIEW + 1); + } + + #[test] + fn given_a_view_change_status_timeout_should_vote_for_the_next_view_without_entering_it() { + let consensus = replica(BACKUP); + let _ = start_view_change(&consensus, PRIMARY, NEXT_VIEW); + + let actions = consensus.handle_view_change_status_timeout(PLANE); + + assert!(votes_for(&actions, NEXT_VIEW + 1), "{actions:?}"); + assert_eq!(consensus.status(), Status::ViewChange); + assert_eq!(consensus.view(), NEXT_VIEW); + } + + /// A barred replica answers every `StartViewChange` with an empty vote, so + /// the vote alone must not depose a primary. + #[test] + fn given_an_empty_vote_for_the_next_view_should_keep_the_primary_in_its_view() { + let consensus = replica(PRIMARY); + let message = Message::::new(size_of::()) + .transmute_header(|_, header: &mut DoViewChangeHeader| { + header.command = Command::DoViewChange; + header.cluster = 1; + header.replica = BACKUP; + header.view = NEXT_VIEW; + header.group = METADATA_GROUP; + header.flags = DoViewChangeHeader::FLAG_UNTRUSTED_LOG; + header.size = u32::try_from(size_of::()).unwrap(); + }); + + let actions = consensus.handle_do_view_change(PLANE, message.header(), &[]); + + assert!(actions.is_empty(), "{actions:?}"); + assert!(consensus.is_primary()); + assert_eq!(consensus.status(), Status::Normal); + assert_eq!(consensus.view(), VIEW); + } + + /// With 4 replicas `max_faulty` is 1, but a view-change quorum is 3. A + /// `DoViewChange` pulls every replica into its view, so one vote beside a + /// replica that entered the view alone must not send one. + #[test] + fn given_four_replicas_when_a_lone_view_change_counts_votes_should_send_its_do_view_change_at_a_quorum() + { + const FOURTH: u8 = 3; + let consensus = alone_in_next_view(4, BACKUP); + + let one_vote = start_view_change(&consensus, FOURTH, NEXT_VIEW); + let quorum = start_view_change(&consensus, PRIMARY, NEXT_VIEW); + + assert!(!sends_do_view_change(&one_vote), "{one_vote:?}"); + assert!(sends_do_view_change(&quorum), "{quorum:?}"); + } + + /// With 2 replicas a view change needs the `DoViewChange` of both, so a + /// replica alone in the view sends none until its peer votes. + #[test] + fn given_two_replicas_when_a_replica_is_alone_in_a_view_change_should_wait_for_the_vote_of_its_peer() + { + let consensus = alone_in_next_view(2, PRIMARY); + + let alone = (0..TimeoutManager::DO_VIEW_CHANGE_MESSAGE_TICKS) + .flat_map(|_| consensus.tick(PLANE)) + .collect::>(); + let with_peer = start_view_change(&consensus, CANDIDATE, NEXT_VIEW); + + assert!(!sends_do_view_change(&alone), "{alone:?}"); + assert!(sends_do_view_change(&with_peer), "{with_peer:?}"); + } +} + #[cfg(test)] mod history_certificate_tests { use super::*; @@ -10364,6 +11123,37 @@ mod adopted_view_claim_tests { ); } + /// A partition WAL can name the adopted view below its head: an install to a + /// checkpoint and a fenced view window both certify only the prefix they hold. + #[test] + fn given_a_wal_certified_below_the_adopted_head_when_the_backup_restarts_should_keep_the_committed_head() + { + let history = history_in(PARTITION_GROUP); + let adopted = adopt_unrepaired(&history); + let durable = adopted.vsr_state(0, 0); + + let restarted = backup_in( + PARTITION_GROUP, + ADOPTED_VIEW, + durable.view, + HEAD - 1, + HEAD - 2, + ); + restarted.restore_adopted_view(durable.adopted_view.as_ref()); + restarted.resume_adopted_view(); + restarted.fence_view_log(HEAD, 0); + let adopted_headers = restarted.adopted_view_log().unwrap().headers; + restarted.set_local_dvc_suffix(DvcSuffix::new(adopted_headers, 0, 0b10)); + assert_eq!(restarted.sequencer().current_sequence(), HEAD); + assert!(restarted.view_log_unproven()); + + assert_eq!( + merged_head(&restarted, [dvc_of(&restarted), old_primary(&history)]), + HEAD, + "the merge must keep op {HEAD}, which view {OLD_VIEW} committed" + ); + } + #[test] fn given_repaired_adopted_bodies_when_restoring_should_keep_the_older_certificate_binding() { const CERTIFIED_HEAD: u64 = HEAD - 3; diff --git a/core/consensus/src/lib.rs b/core/consensus/src/lib.rs index 9aed5ddad1..6df0a8d203 100644 --- a/core/consensus/src/lib.rs +++ b/core/consensus/src/lib.rs @@ -198,6 +198,11 @@ mod impls; pub use impls::*; mod plane_mux; pub use plane_mux::*; +mod replication; +pub use replication::{ + BackupPrefix, REPLICATION_BATCH_BYTES_MAX, ReplicationProgress, ReplicationSend, + ReplicationStats, ReplicationWork, +}; mod plane_helpers; pub use plane_helpers::*; mod metadata_helpers; diff --git a/core/consensus/src/observability.rs b/core/consensus/src/observability.rs index 92552b89c9..6650c2da86 100644 --- a/core/consensus/src/observability.rs +++ b/core/consensus/src/observability.rs @@ -134,6 +134,7 @@ pub enum ControlActionKind { RebuildPipeline, CommitJournal, SendCommit, + SendCommitReply, } impl ControlActionKind { @@ -150,6 +151,7 @@ impl ControlActionKind { Self::RebuildPipeline => "rebuild_pipeline", Self::CommitJournal => "commit_journal", Self::SendCommit => "send_commit", + Self::SendCommitReply => "send_commit_reply", } } } @@ -422,6 +424,14 @@ impl ControlActionLogEvent { commit: Some(commit), untrusted_log: None, }, + VsrAction::SendCommitReply { target, .. } => Self { + replica, + action: ControlActionKind::SendCommitReply, + target_replica: Some(target), + op: None, + commit: None, + untrusted_log: None, + }, } } } diff --git a/core/consensus/src/plane_helpers.rs b/core/consensus/src/plane_helpers.rs index d08bda3fe9..0fe8620c3d 100644 --- a/core/consensus/src/plane_helpers.rs +++ b/core/consensus/src/plane_helpers.rs @@ -17,7 +17,7 @@ use crate::{ Consensus, DvcHeaderKind, IgnoreReason, MergedLog, Pipeline, PipelineEntry, PlaneKind, - PrepareOkOutcome, Sequencer, Status, VsrConsensus, dvc_header_kind, + PrepareOkOutcome, ReplicationSend, Sequencer, Status, VsrConsensus, dvc_header_kind, }; use iggy_binary_protocol::consensus::{REJECTION_SECTION_LEN, write_rejection_section}; use iggy_binary_protocol::{ @@ -31,25 +31,24 @@ use server_common::{ }; use std::{error::Error, fmt, mem::size_of, ops::AsyncFnOnce}; -/// Failure to route or forward a prepare through the replication chain. +/// Failure to enqueue a prepare for replication. #[derive(Debug)] #[non_exhaustive] -pub enum ChainReplicationError { +pub enum PrepareReplicationError { MalformedPrepare, UnexpectedCommand { command: Command }, CommittedPrepare { op: u64, commit_min: u64 }, - SelfRoute { replica: u8 }, Transport(SendError), } -impl ChainReplicationError { +impl PrepareReplicationError { #[must_use] pub const fn is_transport(&self) -> bool { matches!(self, Self::Transport(_)) } } -impl fmt::Display for ChainReplicationError { +impl fmt::Display for PrepareReplicationError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::MalformedPrepare => formatter.write_str("malformed prepare frame"), @@ -60,18 +59,12 @@ impl fmt::Display for ChainReplicationError { formatter, "prepare op {op} is not above committed op {commit_min}" ), - Self::SelfRoute { replica } => { - write!( - formatter, - "replication chain routes replica {replica} to itself" - ) - } Self::Transport(error) => error.fmt(formatter), } } } -impl Error for ChainReplicationError { +impl Error for PrepareReplicationError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { Self::Transport(error) => Some(error), @@ -80,7 +73,7 @@ impl Error for ChainReplicationError { } } -impl From for ChainReplicationError { +impl From for PrepareReplicationError { fn from(error: SendError) -> Self { Self::Transport(error) } @@ -130,151 +123,122 @@ where header.op <= consensus.commit_min() } -/// Shared chain-replication forwarding, skipping disconnected replicas. -/// -/// Borrows the message, makes a deep copy for the wire, and lets the caller -/// retain ownership for journal append. +/// Enqueue one shared wire copy for every backup. Backups never forward it. /// /// # Errors /// -/// Returns an error if the prepare cannot be routed or the bus cannot deliver -/// it to a connected replica before the end of the chain. -/// Other transport errors remain covered by VSR prepare retransmission. +/// Returns the first enqueue error after attempting every backup. The bus +/// bounds each peer's queue; a full queue must not skip another peer. #[allow(clippy::future_not_send)] -pub async fn replicate_to_next_in_chain( +pub async fn replicate_prepare( consensus: &VsrConsensus, message: &Message, -) -> Result<(), ChainReplicationError> +) -> Result<(), PrepareReplicationError> where B: MessageBus, P: Pipeline, { - let Some(next) = replication_target(consensus, message.header())? else { + if !primary_sends_prepare(consensus, message.header())? { return Ok(()); - }; + } let frozen = message.deep_copy().into_generic().into_frozen(); - forward_to_connected_replica( - consensus, - frozen, - next, - consensus.primary_index(message.header().view), - ) - .await + enqueue_prepare_for_backups(consensus, frozen).await } -/// Forward an already validated frozen prepare to the next replica without -/// copying its payload, skipping disconnected replicas. +/// Enqueue a frozen prepare for every backup without copying its payload. /// /// # Errors /// -/// Returns an error if the frame is malformed, cannot be routed, or the bus -/// cannot deliver it to a connected replica before the end of the chain. +/// Returns an error for a malformed frame, or the first enqueue error after +/// attempting every backup. #[allow(clippy::future_not_send)] -pub async fn replicate_frozen_to_next_in_chain( +pub async fn replicate_frozen_prepare( consensus: &VsrConsensus, message: Frozen, -) -> Result<(), ChainReplicationError> +) -> Result<(), PrepareReplicationError> where B: MessageBus, P: Pipeline, { let header = frozen_prepare_header(&message)?; - let Some(next) = replication_target(consensus, &header)? else { + if !primary_sends_prepare(consensus, &header)? { return Ok(()); - }; - forward_to_connected_replica( - consensus, - message, - next, - consensus.primary_index(header.view), - ) - .await + } + enqueue_prepare_for_backups(consensus, message).await } #[allow(clippy::future_not_send)] -async fn forward_to_connected_replica( +async fn enqueue_prepare_for_backups( consensus: &VsrConsensus, message: Frozen, - mut next: u8, - primary: u8, -) -> Result<(), ChainReplicationError> +) -> Result<(), PrepareReplicationError> where B: MessageBus, P: Pipeline, { - loop { - match consensus - .message_bus() - .send_to_replica(next, message.clone()) + let op = frozen_prepare_header(&message)?.op; + let mut first_error = None; + for replica in 0..consensus.replica_count() { + if replica == consensus.replica() { + continue; + } + if let Err(error) = consensus + .send_replicated_prepare(replica, op, message.clone(), ReplicationSend::Live) .await + && first_error.is_none() { - Ok(()) => return Ok(()), - Err(error @ (SendError::ReplicaNotConnected(_) | SendError::ConnectionClosed)) => { - next = (next + 1) % consensus.replica_count(); - if next == primary { - return Err(error.into()); - } - } - Err(error) => return Err(error.into()), + first_error = Some(error); } } + first_error.map_or(Ok(()), |error| Err(error.into())) } fn frozen_prepare_header( message: &Frozen, -) -> Result { +) -> Result { let header_bytes = message .as_slice() .get(..size_of::()) - .ok_or(ChainReplicationError::MalformedPrepare)?; + .ok_or(PrepareReplicationError::MalformedPrepare)?; let header = bytemuck::checked::try_from_bytes::(header_bytes) .copied() - .map_err(|_| ChainReplicationError::MalformedPrepare)?; + .map_err(|_| PrepareReplicationError::MalformedPrepare)?; header .validate() - .map_err(|_| ChainReplicationError::MalformedPrepare)?; + .map_err(|_| PrepareReplicationError::MalformedPrepare)?; let frame_size = - usize::try_from(header.size).map_err(|_| ChainReplicationError::MalformedPrepare)?; + usize::try_from(header.size).map_err(|_| PrepareReplicationError::MalformedPrepare)?; if !(size_of::()..=message.len()).contains(&frame_size) { - return Err(ChainReplicationError::MalformedPrepare); + return Err(PrepareReplicationError::MalformedPrepare); } Ok(header) } -fn replication_target( +fn primary_sends_prepare( consensus: &VsrConsensus, header: &PrepareHeader, -) -> Result, ChainReplicationError> +) -> Result where B: MessageBus, P: Pipeline, { if header.command != Command::Prepare { - return Err(ChainReplicationError::UnexpectedCommand { + return Err(PrepareReplicationError::UnexpectedCommand { command: header.command, }); } let commit_min = consensus.commit_min(); if header.op <= commit_min { - return Err(ChainReplicationError::CommittedPrepare { + return Err(PrepareReplicationError::CommittedPrepare { op: header.op, commit_min, }); } - let next = (consensus.replica() + 1) % consensus.replica_count(); - let primary = consensus.primary_index(header.view); - - if next == primary { - return Ok(None); - } - - if next == consensus.replica() { - return Err(ChainReplicationError::SelfRoute { - replica: consensus.replica(), - }); - } - Ok(Some(next)) + Ok(consensus.is_acting_primary() + && !consensus.has_observed_newer_view() + && !consensus.is_transferring() + && header.view == consensus.view()) } /// Re-stamp a stored prepare with the current view before retransmission. @@ -370,12 +334,8 @@ where return Err(IgnoreReason::NewerView); } - // Deposed-primary prepares can still be in flight after a view change: - // replicate_to_next_in_chain stops the ring at primary_index(header.view), - // not the current view's primary, so a stale prepare reaches the new - // primary and would hit its sequencer invariants. Uncommitted old-view - // ops are decided by the view change, never by late delivery. Message - // repair, once it lands, fetches prepares via its own path, not here. + // Late fanout from a deposed primary must not decide uncommitted old-view + // ops. View change and message repair recover them through their own paths. if header.view < consensus.view() { return Err(IgnoreReason::OlderView); } @@ -1449,103 +1409,125 @@ mod tests { assert!(matches!( frozen_prepare_header(&malformed), - Err(ChainReplicationError::MalformedPrepare) + Err(PrepareReplicationError::MalformedPrepare) )); } #[test] - fn given_disconnected_chain_peers_when_forwarding_should_reach_the_next_peer_immediately() { - for frozen in [false, true] { - for (replica, count, view, disconnected, expected) in [ - (0, 3, 0, vec![1], 2), - (1, 3, 1, vec![2], 0), - (2, 5, 1, vec![3, 4], 0), - ] { - let consensus = - VsrConsensus::new(1, replica, count, 0, SpyBus::new(), LocalPipeline::new()); - consensus.init(); - let bus = consensus.message_bus(); - for peer in &disconnected { - bus.failures - .borrow_mut() - .insert(*peer, SendError::ReplicaNotConnected(*peer)); - } - let message = prepare_message(1, 0, 42).transmute_header( - |old, header: &mut PrepareHeader| { - *header = old; - header.view = view; - }, - ); - let result = if frozen { - let prepare = message.deep_copy().into_frozen(); - let pointer = prepare.as_slice().as_ptr(); - let result = futures::executor::block_on(replicate_frozen_to_next_in_chain( - &consensus, prepare, - )); - if let Some((_, sent)) = bus.sent.borrow().first() { - assert_eq!( - sent.as_slice().as_ptr(), - pointer, - "forwarding must not copy the payload" + fn given_a_primary_when_replicating_should_enqueue_every_backup_independently() { + for count in [3, 5] { + for primary in 0..count { + for frozen in [false, true] { + let mut consensus = VsrConsensus::new( + 1, + primary, + count, + 0, + SpyBus::new(), + LocalPipeline::new(), + ); + consensus.set_view(u32::from(primary)); + consensus.set_log_view(u32::from(primary)); + consensus.init(); + let message = prepare_message(1, 0, 42).transmute_header( + |old, header: &mut PrepareHeader| { + *header = old; + header.view = u32::from(primary); + }, + ); + let result = if frozen { + futures::executor::block_on(replicate_frozen_prepare( + &consensus, + message.deep_copy().into_frozen(), + )) + } else { + futures::executor::block_on(replicate_prepare(&consensus, &message)) + }; + result.unwrap(); + let sent = consensus.message_bus().sent.borrow(); + let expected: Vec<_> = (0..count).filter(|peer| *peer != primary).collect(); + assert_eq!( + sent.iter().map(|(peer, _)| *peer).collect::>(), + expected + ); + for (_, copy) in sent.iter() { + assert_eq!(copy.as_slice(), message.as_slice()); + assert!( + copy.shares_allocation(&sent[0].1), + "fanout shares one wire allocation" ); } - result - } else { - futures::executor::block_on(replicate_to_next_in_chain(&consensus, &message)) - }; - result.expect("a disconnected peer must not delay forwarding to the live suffix"); - assert_eq!( - bus.attempts.borrow().as_slice(), - [disconnected, vec![expected]].concat() - ); - let sent = bus.sent.borrow(); - assert_eq!(sent.len(), 1); - assert_eq!(sent[0].0, expected); - assert_eq!(sent[0].1.as_slice(), message.as_slice()); - assert_eq!( - consensus.commit_max(), - 0, - "forwarding is not an acknowledgement" - ); + assert_eq!(consensus.commit_max(), 0, "enqueueing is not an ack"); + } } } } #[test] - fn given_a_chain_boundary_when_forwarding_should_never_wrap_to_the_primary() { - for (replica, count, connected, expected_attempts) in [ - (0, 3, true, vec![1]), - (0, 3, false, vec![1, 2]), - (1, 3, false, vec![2]), - (2, 3, false, vec![]), - (0, 1, false, vec![]), - ] { - let consensus = - VsrConsensus::new(1, replica, count, 0, SpyBus::new(), LocalPipeline::new()); - consensus.init(); - consensus.message_bus().reject_sends.set(!connected); - let result = futures::executor::block_on(replicate_frozen_to_next_in_chain( - &consensus, - prepare_message(1, 0, 42).into_frozen(), - )); - assert_eq!(result.is_ok(), connected || expected_attempts.is_empty()); - assert_eq!( - *consensus.message_bus().attempts.borrow(), - expected_attempts - ); + fn given_one_full_replica_queue_when_replicating_should_still_enqueue_the_healthy_backup() { + let consensus = VsrConsensus::new(1, 0, 3, 0, SpyBus::new(), LocalPipeline::new()); + consensus.init(); + consensus + .message_bus() + .failures + .borrow_mut() + .insert(1, SendError::Backpressure); + let result = futures::executor::block_on(replicate_frozen_prepare( + &consensus, + prepare_message(1, 0, 42).into_frozen(), + )); + assert!( + result.is_err(), + "the failed enqueue remains visible for retransmission" + ); + assert_eq!(*consensus.message_bus().attempts.borrow(), [1, 2]); + assert_eq!(consensus.message_bus().sent.borrow()[0].0, 2); + } + + #[test] + fn given_a_backup_or_singleton_when_replicating_should_enqueue_nothing() { + for count in [1, 3, 5] { + for view in 0..count { + for replica in 0..count { + if count > 1 && replica == view { + continue; + } + let consensus = VsrConsensus::new( + 1, + replica, + count, + 0, + SpyBus::new(), + LocalPipeline::new(), + ); + consensus.init(); + let prepare = prepare_message(1, 0, 42).transmute_header( + |old, header: &mut PrepareHeader| { + *header = old; + header.view = u32::from(view); + }, + ); + futures::executor::block_on(replicate_frozen_prepare( + &consensus, + prepare.into_frozen(), + )) + .unwrap(); + assert!(consensus.message_bus().attempts.borrow().is_empty()); + } + } } } #[test] - fn given_a_transport_error_when_forwarding_should_skip_only_disconnected_peers() { + fn given_an_enqueue_error_when_replicating_should_attempt_every_other_backup() { for error in [ + SendError::ReplicaNotConnected(1), SendError::ConnectionClosed, SendError::Backpressure, SendError::ReplicaRouteMissing(1), SendError::ReplicaForwardFailed(1), SendError::BusShuttingDown, ] { - let skip = matches!(error, SendError::ConnectionClosed); let consensus = VsrConsensus::new(1, 0, 3, 0, SpyBus::new(), LocalPipeline::new()); consensus.init(); consensus @@ -1553,15 +1535,13 @@ mod tests { .failures .borrow_mut() .insert(1, error); - let result = futures::executor::block_on(replicate_frozen_to_next_in_chain( + let result = futures::executor::block_on(replicate_frozen_prepare( &consensus, prepare_message(1, 0, 42).into_frozen(), )); - assert_eq!(result.is_ok(), skip); - assert_eq!( - consensus.message_bus().attempts.borrow().as_slice(), - if skip { &[1, 2][..] } else { &[1][..] }, - ); + assert!(result.is_err()); + assert_eq!(*consensus.message_bus().attempts.borrow(), [1, 2]); + assert_eq!(consensus.message_bus().sent.borrow()[0].0, 2); } } @@ -1572,8 +1552,8 @@ mod tests { let prepare = prepare_message(0, 0, 0); assert!(matches!( - replication_target(&consensus, prepare.header()), - Err(ChainReplicationError::CommittedPrepare { + primary_sends_prepare(&consensus, prepare.header()), + Err(PrepareReplicationError::CommittedPrepare { op: 0, commit_min: 0 }) @@ -1992,13 +1972,18 @@ mod tests { let local = suffix_headers(2, 4, 0); consensus.set_local_dvc_suffix(crate::view_change_quorum::DvcSuffix::new(local, 0, 0b110)); - let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, 5)); - // Two peers abstain, reaching the quorum of 3 and leaving op 4 one nack short. + // The first brings this replica into view 5, which it announces, before f + // StartViewChanges arrive, so it sends no DoViewChange of its own. for replica in [1u8, 2] { let (dvc, body) = dvc_numbers_only(replica, 5, 0, 4, 2); let actions = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); - assert!(actions.is_empty()); + assert!( + actions + .iter() + .all(|action| matches!(action, VsrAction::SendStartViewChange { view: 5, .. })), + "{actions:?}" + ); } assert!( consensus.pending_view_log().is_none(), @@ -2345,9 +2330,10 @@ mod tests { /// /// A parked merge leaves the replica in `ViewChange` announcing nothing if the /// bodies never arrive, which is the intended trade against losing data but has - /// to stay recoverable: the status timeout fires, escalates, and drops the parked - /// log. Reusing a log merged for a superseded view would leak a truncation - /// decided there into a view that never voted for it. + /// to stay recoverable: the status timeout votes for the next view, and the + /// quorum that joins the vote drops the parked log. Reusing a log merged for a + /// superseded view would leak a truncation decided there into a view that never + /// voted for it. /// A `StartView` from the view's primary, optionally carrying the view's /// canonical headers. fn start_view_with_suffix( @@ -2504,7 +2490,8 @@ mod tests { } #[test] - fn given_parked_view_change_when_status_timeout_fires_should_escalate_and_drop_merged_log() { + fn given_parked_view_change_when_a_quorum_votes_after_its_status_timeout_should_drop_merged_log() + { let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); consensus.init(); consensus.restore_commit_state(2, 2); @@ -2520,26 +2507,24 @@ mod tests { assert_eq!(consensus.status(), Status::ViewChange); let parked_view = consensus.view(); - // `VIEW_CHANGE_STATUS_TICKS` is 500; tick past it. Escalation shows as the - // view advancing, since the 50-tick SVC retransmit also emits a send. - let mut escalated = false; - for _ in 0..600 { - let _ = consensus.tick(PlaneKind::Metadata); - if consensus.view() > parked_view { - escalated = true; - break; - } - } - - assert!( - escalated, - "a parked view change must still escalate on the status timeout" - ); + // The status timeout votes for the next view and keeps the parked merge, + // which can still start the view once repair covers it. + let next_view = parked_view + 1; + let voted = (0..crate::TimeoutManager::VIEW_CHANGE_STATUS_TICKS) + .flat_map(|_| consensus.tick(PlaneKind::Metadata)) + .any(|action| { + matches!(action, VsrAction::SendStartViewChange { view, .. } if view == next_view) + }); assert!( - consensus.view() > parked_view, - "escalation must advance the view past {parked_view}, got {}", - consensus.view() + voted, + "a parked view change must still vote on the status timeout" ); + assert_eq!(consensus.view(), parked_view); + assert!(consensus.pending_view_log().is_some()); + + let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, next_view)); + + assert_eq!(consensus.view(), next_view); assert!( consensus.pending_view_log().is_none(), "the superseded merged log must be dropped, not carried into the next view" diff --git a/core/consensus/src/replication.rs b/core/consensus/src/replication.rs new file mode 100644 index 0000000000..c6576a672a --- /dev/null +++ b/core/consensus/src/replication.rs @@ -0,0 +1,1207 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::{Consensus, Pipeline, PipelineEntry, Sequencer, VsrConsensus}; +use iggy_binary_protocol::{Command, ConsensusHeader, HEADER_SIZE, PrepareProgressHeader}; +use message_bus::{MessageBus, SendError}; +use server_common::{MESSAGE_ALIGN, Message, iobuf::Frozen}; +use std::mem::take; + +pub const REPLICATION_BATCH_BYTES_MAX: u64 = 32 * 1024 * 1024; +/// A checkpoint at half the window returns its credit before the window fills, +/// as long as the backup consumes half a window within one round trip. +const CHECKPOINT_BYTES: u64 = REPLICATION_BATCH_BYTES_MAX / 2; +const PROGRESS_INTERVAL_MICROS: u64 = 1_000_000; +/// Probes and their replies are header-only frames. +#[allow(clippy::cast_possible_truncation)] +const PROGRESS_FRAME_BYTES: u32 = HEADER_SIZE as u32; + +/// Bounds the prepare bytes in flight to each backup by the credit that +/// checkpoints return. +/// +/// A checkpoint is a progress probe on the same FIFO link as the prepares, so +/// its reply proves that the backup consumed every frame sent before it. The +/// reply credits only that prefix, and no send waits for it. Payloads remain in +/// the journal, never in this per-peer state. +#[derive(Debug)] +pub struct ReplicationFlow { + epoch: Option<(u32, u64)>, + nonce: u64, + peers: Vec, + stats: ReplicationStats, +} + +#[derive(Debug, Clone, Copy, Default)] +struct PeerReplication { + /// Every copy sent in this epoch, retransmits included. + sent_bytes: u64, + /// The prefix of `sent_bytes` that a checkpoint reply proved consumed. + credited_bytes: u64, + sent_through: u64, + /// The next op of the ordered resend. Live sends wait until it reaches them. + replay_from: Option, + checkpoint: Option, + /// The backup drops or cannot acknowledge prepares, so nothing is sent. + busy: bool, + /// When the tick sends a new checkpoint or resends the outstanding one. + due: u64, + /// Replay waits until then after history was unavailable. + retry_at: u64, +} + +impl PeerReplication { + const fn outstanding(&self) -> u64 { + self.sent_bytes.saturating_sub(self.credited_bytes) + } + + /// Whether the window takes a frame of `size` once `queued` more bytes are + /// sent. An empty window takes any frame, so one legal frame larger than + /// the window still makes progress. + const fn window_takes(&self, queued: u64, size: u64) -> bool { + let outstanding = self.outstanding().saturating_add(queued); + outstanding == 0 || outstanding.saturating_add(size) <= REPLICATION_BATCH_BYTES_MAX + } +} + +#[derive(Debug, Clone, Copy)] +struct Checkpoint { + nonce: u64, + through: u64, + bytes: u64, + /// A gap at or above the cursor of issue time was still waiting for replay, + /// so the reply cannot move the cursor back to it. + replay_from: Option, +} + +#[derive(Debug, Clone, Copy)] +pub enum ReplicationWork { + Probe { + replica: u8, + nonce: u64, + through: u64, + }, + Replay { + replica: u8, + from_op: u64, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReplicationProgress { + Busy, + Ready { received_op: u64 }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReplicationSend { + /// A new op from the primary's prepare path. + Live, + /// An ordered resend from the replay cursor. + Replay, + /// A prepare-timeout resend of one op. It never moves the cursor, so a + /// healthy backup gets no replay. + Retransmit, +} + +/// Replication events since the previous [`VsrConsensus::take_replication_stats`]. +/// Both stay at zero while every backup keeps up. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct ReplicationStats { + /// Live prepares that the window, the replay cursor or a busy backup refused. + pub live_refusals: u64, + /// Prepares sent by replay. + pub replayed: u64, +} + +/// A backup's answer to a checkpoint. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct BackupPrefix { + /// Every op through it is applied, or is in the journal with proof in the + /// current view. An op above the commit need not be durable yet. + pub held_op: u64, + /// The part of the held prefix that also meets the `PrepareOk` rules. + pub received_op: u64, +} + +impl ReplicationFlow { + pub fn new(replica_count: u8) -> Self { + Self { + epoch: None, + nonce: 0, + peers: vec![PeerReplication::default(); usize::from(replica_count)], + stats: ReplicationStats::default(), + } + } + + fn in_epoch(&mut self, view: u32, incarnation: u64) { + if self.epoch != Some((view, incarnation)) { + self.epoch = Some((view, incarnation)); + self.peers.fill(PeerReplication::default()); + } + } + + fn admit(&mut self, replica: u8, op: u64, size: u64, send: ReplicationSend) -> bool { + let peer = &mut self.peers[usize::from(replica)]; + let window_full = !peer.window_takes(0, size); + let out_of_order = match send { + ReplicationSend::Live => peer.replay_from.is_some_and(|from| from != op), + ReplicationSend::Replay => peer.replay_from != Some(op), + ReplicationSend::Retransmit => false, + }; + if !peer.busy && !window_full && !out_of_order { + return true; + } + if window_full && peer.checkpoint.is_none() { + peer.due = 0; + } + if send == ReplicationSend::Live { + peer.replay_from.get_or_insert(op); + self.stats.live_refusals += 1; + } + false + } + + /// Returns the checkpoint that must follow this frame on its link. + fn sent( + &mut self, + replica: u8, + op: u64, + size: u64, + send: ReplicationSend, + now: u64, + ) -> Option { + let peer = &mut self.peers[usize::from(replica)]; + if peer.outstanding() == 0 && peer.checkpoint.is_none() { + peer.due = now.saturating_add(PROGRESS_INTERVAL_MICROS); + } + peer.sent_bytes = peer.sent_bytes.saturating_add(size); + peer.sent_through = peer.sent_through.max(op); + if send != ReplicationSend::Retransmit && peer.replay_from == Some(op) { + peer.replay_from = Some(op.saturating_add(1)); + } + if send == ReplicationSend::Replay { + self.stats.replayed += 1; + } + let due = peer.checkpoint.is_none() && peer.outstanding() >= CHECKPOINT_BYTES; + due.then(|| self.checkpoint(replica, now)) + } + + fn failed(&mut self, replica: u8, op: u64) { + let peer = &mut self.peers[usize::from(replica)]; + peer.replay_from = Some(peer.replay_from.map_or(op, |from| from.min(op))); + peer.due = 0; + } + + fn defer(&mut self, replica: u8, now: u64) { + let peer = &mut self.peers[usize::from(replica)]; + peer.due = 0; + peer.retry_at = now.saturating_add(PROGRESS_INTERVAL_MICROS); + } + + /// A checkpoint counts only once its frame is queued. Dropping one that the + /// peer queue refused lets the next send issue a new one, instead of the + /// window waiting for the interval to end. + fn probe_failed(&mut self, replica: u8, nonce: u64) { + let peer = &mut self.peers[usize::from(replica)]; + if peer + .checkpoint + .is_some_and(|checkpoint| checkpoint.nonce == nonce) + { + peer.checkpoint = None; + } + } + + fn checkpoint(&mut self, replica: u8, now: u64) -> ReplicationWork { + self.nonce = self.nonce.wrapping_add(1).max(1); + let peer = &mut self.peers[usize::from(replica)]; + peer.checkpoint = Some(Checkpoint { + nonce: self.nonce, + through: peer.sent_through, + bytes: peer.sent_bytes, + replay_from: peer.replay_from, + }); + peer.due = now.saturating_add(PROGRESS_INTERVAL_MICROS); + ReplicationWork::Probe { + replica, + nonce: self.nonce, + through: peer.sent_through, + } + } + + fn work(&mut self, replica: u8, head: u64, now: u64) -> Option { + let peer = &mut self.peers[usize::from(replica)]; + let backoff = now < peer.retry_at; + if backoff && peer.outstanding() == 0 { + return None; + } + if peer.replay_from.is_some_and(|from| from > head) { + peer.replay_from = None; + } + if now >= peer.due { + if let Some(checkpoint) = peer.checkpoint { + // A lost checkpoint goes out again with its nonce and prefix. + peer.due = now.saturating_add(PROGRESS_INTERVAL_MICROS); + return Some(ReplicationWork::Probe { + replica, + nonce: checkpoint.nonce, + through: checkpoint.through, + }); + } + if peer.outstanding() > 0 || peer.replay_from.is_some() || peer.busy { + return Some(self.checkpoint(replica, now)); + } + } + if backoff || peer.busy { + return None; + } + peer.replay_from + .map(|from_op| ReplicationWork::Replay { replica, from_op }) + } + + fn reply( + &mut self, + header: &PrepareProgressHeader, + head: u64, + now: u64, + ) -> Option { + let peer = &mut self.peers[usize::from(header.replica)]; + let checkpoint = peer.checkpoint?; + let ready = header.ready != 0; + if (checkpoint.nonce, checkpoint.through) != (header.nonce, header.observed_op) + || (ready && header.received_op > header.held_op) + { + return None; + } + peer.checkpoint = None; + // The reply proves FIFO consumption, not readiness, so Busy credits too. + peer.credited_bytes = peer.credited_bytes.max(checkpoint.bytes); + if !ready { + peer.busy = true; + let next = peer.sent_through.saturating_add(1); + peer.replay_from.get_or_insert(next); + // Readiness can change before the interval of a lost probe ends. + peer.due = 0; + return Some(ReplicationProgress::Busy); + } + peer.busy = false; + peer.retry_at = 0; + peer.due = now.saturating_add(PROGRESS_INTERVAL_MICROS); + let held_op = header.held_op.min(head); + // A newer head alone never starts a replay. + let gap = held_op < checkpoint.through + && checkpoint.replay_from.is_none_or(|from| held_op + 1 < from); + // Repair or a state transfer can carry the backup past the cursor. The + // primary can drop the op at the cursor by then, and a replay from it + // would block live sends for good. + if gap || peer.replay_from.is_some_and(|from| from <= held_op) { + peer.replay_from = Some(held_op + 1); + } + Some(ReplicationProgress::Ready { + received_op: header.received_op.min(head), + }) + } +} + +impl VsrConsensus +where + B: MessageBus, + P: Pipeline, +{ + /// Send one prepare copy to a backup under its replication window. Once + /// half the window is outstanding, a checkpoint follows the copy. + /// + /// Returns `Ok(false)` when the flow refuses the copy. + /// + /// # Errors + /// + /// Returns the bus error when the copy cannot be enqueued. + #[allow(clippy::future_not_send)] + pub async fn send_replicated_prepare( + &self, + replica: u8, + op: u64, + prepare: Frozen, + send: ReplicationSend, + ) -> Result { + let size = prepare.len() as u64; + if !self.admit_replication(replica, op, size, send) { + return Ok(false); + } + if let Err(error) = self.message_bus().send_to_replica(replica, prepare).await { + self.record_replication_failure(replica, op); + return Err(error); + } + if let Some(ReplicationWork::Probe { nonce, through, .. }) = + self.record_replication_send(replica, op, size, send) + { + self.send_replication_probe(replica, nonce, through).await; + } + Ok(true) + } + + pub fn admit_replication( + &self, + replica: u8, + op: u64, + size: u64, + send: ReplicationSend, + ) -> bool { + if !self.is_acting_primary() || self.has_observed_newer_view() || self.is_transferring() { + return false; + } + let mut flow = self.replication.borrow_mut(); + flow.in_epoch(self.view(), self.partition_incarnation()); + flow.admit(replica, op, size, send) + } + + /// Whether the window of `replica` takes a frame of `size` behind `queued` + /// bytes that are not sent yet. A replay asks before it reads a body, so + /// it reads no body that the window of [`Self::send_replicated_prepare`] + /// then refuses. + pub fn replication_window_takes(&self, replica: u8, queued: u64, size: u64) -> bool { + self.replication.borrow().peers[usize::from(replica)].window_takes(queued, size) + } + + /// Returns the checkpoint that must follow the sent copy. + #[must_use] + pub fn record_replication_send( + &self, + replica: u8, + op: u64, + size: u64, + send: ReplicationSend, + ) -> Option { + self.replication.borrow_mut().sent( + replica, + op, + size, + send, + self.message_bus().monotonic_micros(), + ) + } + + pub fn record_replication_failure(&self, replica: u8, op: u64) { + self.replication.borrow_mut().failed(replica, op); + } + + pub fn defer_replication(&self, replica: u8) { + let now = self.message_bus().monotonic_micros(); + self.replication.borrow_mut().defer(replica, now); + } + + pub fn replication_work(&self, replica: u8) -> Option { + if !self.is_acting_primary() + || self.has_observed_newer_view() + || self.is_transferring() + || self.has_untrusted_log() + { + return None; + } + let mut flow = self.replication.borrow_mut(); + flow.in_epoch(self.view(), self.partition_incarnation()); + flow.work( + replica, + self.sequencer().current_sequence(), + self.message_bus().monotonic_micros(), + ) + } + + /// Send the probe of a [`ReplicationWork::Probe`] behind every frame + /// already queued for `replica`. + #[allow(clippy::future_not_send)] + pub async fn send_replication_probe(&self, replica: u8, nonce: u64, through: u64) { + let probe = Message::::new(HEADER_SIZE).transmute_header( + |_, header: &mut PrepareProgressHeader| { + header.command = Command::PrepareProgress; + header.size = PROGRESS_FRAME_BYTES; + header.cluster = self.cluster(); + header.view = self.view(); + header.replica = self.replica(); + header.group = self.group(); + header.partition_incarnation = self.partition_incarnation(); + header.nonce = nonce; + header.observed_op = through; + header.commit_op = self.commit_min(); + header.kind = PrepareProgressHeader::PROBE; + header.seal(); + }, + ); + if self + .message_bus() + .send_to_replica(replica, probe.into_generic().into_frozen()) + .await + .is_err() + { + self.replication.borrow_mut().probe_failed(replica, nonce); + } + } + + pub fn handle_prepare_progress( + &self, + header: &PrepareProgressHeader, + ) -> Option { + if !self.is_acting_primary() + || self.has_observed_newer_view() + || self.is_transferring() + || header.kind != PrepareProgressHeader::REPLY + || header.cluster != self.cluster() + || header.group != self.group() + || header.view != self.view() + || header.partition_incarnation != self.partition_incarnation() + || header.replica >= self.replica_count() + || header.replica == self.replica() + { + return None; + } + let mut flow = self.replication.borrow_mut(); + flow.in_epoch(self.view(), self.partition_incarnation()); + let progress = flow.reply( + header, + self.sequencer().current_sequence(), + self.message_bus().monotonic_micros(), + )?; + if let ReplicationProgress::Ready { received_op } = progress { + self.note_backup_ack(header.replica, received_op); + } + Some(progress) + } + + pub fn take_replication_stats(&self) -> ReplicationStats { + take(&mut self.replication.borrow_mut().stats) + } +} + +#[cfg(test)] +mod tests { + use super::{ + CHECKPOINT_BYTES, PROGRESS_INTERVAL_MICROS, REPLICATION_BATCH_BYTES_MAX, ReplicationFlow, + ReplicationProgress, ReplicationSend, ReplicationStats, ReplicationWork, + }; + use crate::{LocalPipeline, Sequencer, VsrConsensus, impls::test_bus::NoopBus}; + use iggy_binary_protocol::{Command, ConsensusHeader, HEADER_SIZE, PrepareProgressHeader}; + use server_common::Message; + use std::collections::VecDeque; + + const MIB: u64 = 1024 * 1024; + + fn ready( + replica: u8, + nonce: u64, + observed_op: u64, + held_op: u64, + received_op: u64, + ) -> Message { + Message::::new(HEADER_SIZE).transmute_header( + |_, header: &mut PrepareProgressHeader| { + header.command = Command::PrepareProgress; + header.cluster = 1; + header.size = u32::try_from(HEADER_SIZE).unwrap(); + header.replica = replica; + header.kind = PrepareProgressHeader::REPLY; + header.ready = 1; + header.nonce = nonce; + header.observed_op = observed_op; + header.held_op = held_op; + header.received_op = received_op; + header.seal(); + }, + ) + } + + fn busy(replica: u8, nonce: u64, observed_op: u64) -> Message { + ready(replica, nonce, observed_op, 0, 0).transmute_header( + |old, header: &mut PrepareProgressHeader| { + *header = old; + header.ready = 0; + }, + ) + } + + /// Sends a copy the flow must admit, and returns the checkpoint behind it. + fn send( + flow: &mut ReplicationFlow, + replica: u8, + op: u64, + size: u64, + send: ReplicationSend, + now: u64, + ) -> Option { + assert!( + flow.admit(replica, op, size, send), + "{send:?} op {op} to replica {replica} was refused" + ); + flow.sent(replica, op, size, send, now) + } + + /// Replica 2 answers like a persisted topic whose WAL write of the last op + /// is not durable yet. Replies arrive two ops after their checkpoint, so + /// each checkpoint is out while the stream continues. + #[test] + fn given_healthy_backups_when_the_stream_passes_many_windows_should_never_refuse_or_replay() { + const SIZE: u64 = MIB; + const OPS: u64 = 10 * REPLICATION_BATCH_BYTES_MAX / SIZE; + const REPLY_DELAY_OPS: u64 = 2; + const OP_INTERVAL_MICROS: u64 = 10_000; + let mut flow = ReplicationFlow::new(3); + flow.in_epoch(0, 1); + let mut replies = VecDeque::new(); + for op in 1..=OPS { + let now = op * OP_INTERVAL_MICROS; + for replica in 1..3 { + let checkpoint = send(&mut flow, replica, op, SIZE, ReplicationSend::Live, now); + for work in [checkpoint, flow.work(replica, op, now)] + .into_iter() + .flatten() + { + let ReplicationWork::Probe { nonce, through, .. } = work else { + panic!("replica {replica} replays at op {op}: {work:?}"); + }; + replies.push_back((op + REPLY_DELAY_OPS, replica, nonce, through)); + } + } + while let Some(&(at, replica, nonce, through)) = replies.front() + && at <= op + { + replies.pop_front(); + let received = if replica == 2 { through - 1 } else { through }; + let reply = ready(replica, nonce, through, through, received); + assert!(matches!( + flow.reply(reply.header(), op, now), + Some(ReplicationProgress::Ready { .. }) + )); + } + } + assert_eq!(flow.stats, ReplicationStats::default()); + } + + #[test] + fn given_bytes_sent_after_a_checkpoint_when_its_reply_arrives_should_credit_only_that_prefix() { + const SIZE: u64 = CHECKPOINT_BYTES / 2; + let mut flow = ReplicationFlow::new(3); + flow.in_epoch(0, 1); + assert!(send(&mut flow, 1, 1, SIZE, ReplicationSend::Live, 0).is_none()); + let Some(ReplicationWork::Probe { nonce, through, .. }) = + send(&mut flow, 1, 2, SIZE, ReplicationSend::Live, 0) + else { + panic!("half a window outstanding must send a checkpoint"); + }; + assert!(send(&mut flow, 1, 3, SIZE, ReplicationSend::Live, 0).is_none()); + assert!(send(&mut flow, 1, 4, SIZE, ReplicationSend::Live, 0).is_none()); + assert!(!flow.admit(1, 4, 1, ReplicationSend::Retransmit)); + assert!( + matches!( + flow.work(1, 4, PROGRESS_INTERVAL_MICROS), + Some(ReplicationWork::Probe { nonce: resent, through: 2, .. }) if resent == nonce + ), + "a lost checkpoint goes out again with its nonce and prefix" + ); + for stale in [ + ready(1, nonce + 1, through, through, through), + ready(1, nonce, through + 1, through, through), + ready(1, nonce, through, through - 1, through), + ] { + assert_eq!(flow.reply(stale.header(), 4, 0), None); + } + let reply = ready(1, nonce, through, through, through); + assert!(flow.reply(reply.header(), 4, 0).is_some()); + assert_eq!( + flow.reply(reply.header(), 4, 0), + None, + "a duplicate reply credits nothing" + ); + assert!(flow.admit(1, 4, CHECKPOINT_BYTES, ReplicationSend::Retransmit)); + assert!( + !flow.admit(1, 4, CHECKPOINT_BYTES + 1, ReplicationSend::Retransmit), + "bytes sent after the checkpoint stay charged" + ); + } + + #[test] + fn given_a_checkpoint_the_peer_queue_refused_when_a_copy_follows_should_send_a_new_one() { + let mut flow = ReplicationFlow::new(3); + flow.in_epoch(0, 1); + let Some(ReplicationWork::Probe { nonce: refused, .. }) = + send(&mut flow, 1, 1, CHECKPOINT_BYTES, ReplicationSend::Live, 0) + else { + panic!("half a window outstanding must send a checkpoint"); + }; + flow.probe_failed(1, refused); + let Some(ReplicationWork::Probe { + nonce, through: 2, .. + }) = send(&mut flow, 1, 2, 1, ReplicationSend::Live, 0) + else { + panic!("the next copy must carry a new checkpoint, not wait for the interval"); + }; + flow.probe_failed(1, refused); + assert_eq!(flow.reply(ready(1, refused, 1, 1, 1).header(), 2, 0), None); + assert!( + flow.reply(ready(1, nonce, 2, 2, 2).header(), 2, 0) + .is_some(), + "a failure of an older checkpoint must keep the new one" + ); + assert_eq!(flow.peers[1].outstanding(), 0); + } + + #[test] + fn given_a_lost_prepare_when_a_checkpoint_finds_the_gap_should_replay_it_before_live_sends() { + const SIZE: u64 = 100; + let mut flow = ReplicationFlow::new(3); + flow.in_epoch(0, 1); + for op in 1..=3 { + assert!(send(&mut flow, 1, op, SIZE, ReplicationSend::Live, 0).is_none()); + } + let mut now = PROGRESS_INTERVAL_MICROS; + let Some(ReplicationWork::Probe { + nonce, through: 3, .. + }) = flow.work(1, 3, now) + else { + panic!("the timer must checkpoint outstanding bytes"); + }; + let lost_second = ready(1, nonce, 3, 1, 1); + assert!(flow.reply(lost_second.header(), 3, now).is_some()); + assert!(!flow.admit(1, 4, SIZE, ReplicationSend::Live)); + now += PROGRESS_INTERVAL_MICROS; + let Some(ReplicationWork::Probe { + nonce: early, + through: 3, + .. + }) = flow.work(1, 4, now) + else { + panic!("the timer must checkpoint a waiting replay"); + }; + assert!( + matches!( + flow.work(1, 4, now), + Some(ReplicationWork::Replay { from_op: 2, .. }) + ), + "replay must not wait for the checkpoint" + ); + for op in 2..=4 { + assert!(send(&mut flow, 1, op, SIZE, ReplicationSend::Replay, now).is_none()); + } + let before_replay = ready(1, early, 3, 1, 1); + assert!(flow.reply(before_replay.header(), 4, now).is_some()); + assert!( + flow.admit(1, 5, SIZE, ReplicationSend::Live), + "a checkpoint sent before the replay must not move the cursor back" + ); + assert_eq!( + flow.stats, + ReplicationStats { + live_refusals: 1, + replayed: 3 + } + ); + } + + #[test] + fn given_a_healthy_backup_when_a_prepare_retransmits_should_charge_it_without_a_replay() { + const SIZE: u64 = 100; + let mut flow = ReplicationFlow::new(3); + flow.in_epoch(0, 1); + for op in 1..=2 { + assert!(send(&mut flow, 1, op, SIZE, ReplicationSend::Live, 0).is_none()); + } + assert!(send(&mut flow, 1, 1, SIZE, ReplicationSend::Retransmit, 0).is_none()); + assert_eq!(flow.peers[1].outstanding(), 3 * SIZE); + assert!(flow.admit(1, 3, SIZE, ReplicationSend::Live)); + assert!(flow.work(1, 3, 0).is_none()); + assert_eq!(flow.stats, ReplicationStats::default()); + } + + #[test] + fn given_a_busy_backup_when_prepares_arrive_should_send_nothing_until_a_ready_reply() { + const SIZE: u64 = 100; + let mut flow = ReplicationFlow::new(3); + flow.in_epoch(0, 1); + for op in 1..=2 { + assert!(send(&mut flow, 1, op, SIZE, ReplicationSend::Live, 0).is_none()); + } + let now = PROGRESS_INTERVAL_MICROS; + let Some(ReplicationWork::Probe { nonce, through, .. }) = flow.work(1, 2, now) else { + panic!("the timer must checkpoint outstanding bytes"); + }; + assert_eq!( + flow.reply(busy(1, nonce, through).header(), 2, now), + Some(ReplicationProgress::Busy) + ); + assert_eq!( + flow.peers[1].outstanding(), + 0, + "a busy reply still proves FIFO consumption" + ); + assert!( + !flow.admit(1, 3, SIZE, ReplicationSend::Live), + "a live op equal to the cursor waits too" + ); + assert!(!flow.admit(1, 2, SIZE, ReplicationSend::Retransmit)); + let Some(ReplicationWork::Probe { + nonce: recheck, + through: 2, + .. + }) = flow.work(1, 3, now) + else { + panic!("the next tick must ask a busy backup again"); + }; + assert!( + flow.work(1, 3, now).is_none(), + "a busy backup gets no replay" + ); + assert!( + flow.reply(ready(1, recheck, 2, 2, 2).header(), 3, now) + .is_some() + ); + assert!( + flow.admit(1, 3, SIZE, ReplicationSend::Live), + "a ready reply resumes after the held prefix" + ); + } + + /// Busy replies during a state transfer park the cursor on an op that the + /// primary later drops from memory. A replay from there never succeeds, so + /// the cursor would refuse every live send. + #[test] + fn given_a_backup_that_repair_moved_past_the_cursor_when_it_answers_ready_should_replay_after_its_held_prefix() + { + const SIZE: u64 = 100; + let mut flow = ReplicationFlow::new(3); + flow.in_epoch(0, 1); + for op in 1..=3 { + assert!(send(&mut flow, 1, op, SIZE, ReplicationSend::Live, 0).is_none()); + } + let mut now = PROGRESS_INTERVAL_MICROS; + let Some(ReplicationWork::Probe { nonce, through, .. }) = flow.work(1, 3, now) else { + panic!("the timer must checkpoint outstanding bytes"); + }; + assert_eq!( + flow.reply(busy(1, nonce, through).header(), 3, now), + Some(ReplicationProgress::Busy) + ); + assert!(!flow.admit(1, 4, SIZE, ReplicationSend::Live)); + let Some(ReplicationWork::Probe { + nonce: repaired, + through: 3, + .. + }) = flow.work(1, 10, now) + else { + panic!("the next tick must ask a busy backup again"); + }; + assert_eq!( + flow.reply(ready(1, repaired, 3, 8, 8).header(), 10, now), + Some(ReplicationProgress::Ready { received_op: 8 }) + ); + assert!( + matches!( + flow.work(1, 10, now), + Some(ReplicationWork::Replay { from_op: 9, .. }) + ), + "replay must resume after the held prefix, not at the parked cursor" + ); + for op in 9..=10 { + assert!(send(&mut flow, 1, op, SIZE, ReplicationSend::Replay, now).is_none()); + } + assert!(send(&mut flow, 1, 11, SIZE, ReplicationSend::Live, now).is_none()); + + now += PROGRESS_INTERVAL_MICROS; + let Some(ReplicationWork::Probe { + nonce: installed, + through: 11, + .. + }) = flow.work(1, 11, now) + else { + panic!("the timer must checkpoint outstanding bytes"); + }; + assert!( + flow.reply(ready(1, installed, 11, 5, 5).header(), 11, now) + .is_some() + ); + assert!( + matches!( + flow.work(1, 11, now), + Some(ReplicationWork::Replay { from_op: 6, .. }) + ), + "an install that drops held ops must restart replay at the first lost op" + ); + + now += PROGRESS_INTERVAL_MICROS; + let Some(ReplicationWork::Probe { + nonce: caught_up, + through: 11, + .. + }) = flow.work(1, 11, now) + else { + panic!("the timer must checkpoint a waiting replay"); + }; + assert!( + flow.reply(ready(1, caught_up, 11, 6, 6).header(), 11, now) + .is_some() + ); + assert!( + matches!( + flow.work(1, 11, now), + Some(ReplicationWork::Replay { from_op: 7, .. }) + ), + "a backup that holds the op at the cursor must not get it again" + ); + } + + #[test] + fn given_a_silent_backup_when_sends_continue_should_bound_its_bytes_and_keep_other_peers_sending() + { + const SIZE: u64 = MIB; + let mut flow = ReplicationFlow::new(3); + flow.in_epoch(0, 1); + let mut op = 1; + while flow.admit(1, op, SIZE, ReplicationSend::Live) { + let _ = flow.sent(1, op, SIZE, ReplicationSend::Live, 0); + op += 1; + } + assert_eq!(flow.peers[1].outstanding(), REPLICATION_BATCH_BYTES_MAX); + assert!( + send( + &mut flow, + 2, + 1, + REPLICATION_BATCH_BYTES_MAX + 1, + ReplicationSend::Live, + 0 + ) + .is_some(), + "a silent peer must not stop another peer, and one frame larger than the window passes" + ); + assert!(!flow.admit(2, 2, 1, ReplicationSend::Live)); + } + + #[test] + fn given_a_full_replay_window_when_deferred_should_drain_credit_before_history_backoff() { + let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); + consensus.init(); + consensus.sequencer().set_sequence(2); + assert!(consensus.admit_replication(1, 1, 1, ReplicationSend::Live)); + consensus.record_replication_failure(1, 1); + assert!(consensus.admit_replication( + 1, + 1, + REPLICATION_BATCH_BYTES_MAX, + ReplicationSend::Replay + )); + let Some(ReplicationWork::Probe { nonce, through, .. }) = consensus + .record_replication_send(1, 1, REPLICATION_BATCH_BYTES_MAX, ReplicationSend::Replay) + else { + panic!("a full window must send a checkpoint behind its last frame"); + }; + assert!(!consensus.admit_replication(1, 2, 1, ReplicationSend::Replay)); + consensus.defer_replication(1); + assert!( + matches!( + consensus.replication_work(1), + Some(ReplicationWork::Probe { nonce: resent, .. }) if resent == nonce + ), + "a full window must drain without the unavailable-history delay" + ); + assert!( + consensus + .handle_prepare_progress(ready(1, nonce, through, 1, 1).header()) + .is_some() + ); + assert!( + matches!( + consensus.replication_work(1), + Some(ReplicationWork::Replay { from_op: 2, .. }) + ), + "a drained window must resume replay without the stale history delay" + ); + + assert!(consensus.admit_replication(2, 1, 1, ReplicationSend::Live)); + consensus.record_replication_failure(2, 1); + consensus.defer_replication(2); + assert!( + consensus.replication_work(2).is_none(), + "unavailable history without outstanding bytes must still back off" + ); + } + + #[test] + fn given_progress_replies_when_the_view_incarnation_or_nonce_is_stale_should_neither_release_credit_nor_commit() + { + let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); + consensus.init(); + consensus.sequencer().set_sequence(2); + assert!(consensus.admit_replication( + 1, + 1, + REPLICATION_BATCH_BYTES_MAX, + ReplicationSend::Live + )); + let Some(ReplicationWork::Probe { nonce, through, .. }) = consensus + .record_replication_send(1, 1, REPLICATION_BATCH_BYTES_MAX, ReplicationSend::Live) + else { + panic!("a full window must send a checkpoint behind its last frame"); + }; + for stale in ["view", "incarnation", "nonce", "peer", "cluster", "group"] { + let message = ready(1, nonce, through, 1, 1).transmute_header( + |old, header: &mut PrepareProgressHeader| { + *header = old; + match stale { + "view" => header.view += 1, + "incarnation" => header.partition_incarnation += 1, + "nonce" => header.nonce += 1, + "peer" => header.replica = 2, + "cluster" => header.cluster += 1, + "group" => header.group += 1, + _ => unreachable!(), + } + }, + ); + assert_eq!(consensus.handle_prepare_progress(message.header()), None); + assert!( + !consensus.admit_replication(1, 2, 1, ReplicationSend::Retransmit), + "accepted stale {stale}" + ); + } + let _ = consensus.handle_prepare_progress(ready(1, nonce, through, 1, 1).header()); + assert!(consensus.admit_replication(1, 2, 1, ReplicationSend::Live)); + assert_eq!(consensus.commit_min(), 0); + assert_eq!(consensus.commit_max(), 0); + } + + const MODEL_BACKUP: u8 = 1; + + enum Frame { + Prepare { op: u64, size: u64 }, + Probe { nonce: u64, through: u64 }, + } + + /// A primary that sends to one backup over a FIFO link, and replays only + /// its newest ops. While faults are on, the backup drops frames, sends + /// fail, replies get lost, the backup answers Busy, and repair from another + /// replica moves the backup forward. + struct Model { + flow: ReplicationFlow, + random: u64, + faults: bool, + now: u64, + head: u64, + sizes: Vec, + link: VecDeque, + replies: VecDeque>, + pushed_bytes: u64, + consumed_bytes: u64, + held: u64, + } + + impl Model { + const TICK_MICROS: u64 = 10_000; + const SIZE_MAX: u64 = 4 * MIB; + const REPLAY_OPS_MAX: u64 = 8; + const RETAINED_OPS: u64 = 16; + const FAULT_ONE_IN: u64 = 10; + + fn new(seed: u64) -> Self { + let mut flow = ReplicationFlow::new(2); + flow.in_epoch(0, 1); + Self { + flow, + random: seed, + faults: true, + now: 0, + head: 0, + sizes: Vec::new(), + link: VecDeque::new(), + replies: VecDeque::new(), + pushed_bytes: 0, + consumed_bytes: 0, + held: 0, + } + } + + /// Xorshift, so the run needs no new dependency. + fn below(&mut self, bound: u64) -> u64 { + self.random ^= self.random << 13; + self.random ^= self.random >> 7; + self.random ^= self.random << 17; + self.random % bound + } + + fn fault(&mut self) -> bool { + self.faults && self.below(Self::FAULT_ONE_IN) == 0 + } + + fn step(&mut self) { + match self.below(6) { + 0 => { + self.head += 1; + let size = 1 + self.below(Self::SIZE_MAX); + self.sizes.push(size); + self.send(self.head, ReplicationSend::Live); + } + 1 if self.head > 0 => { + let op = self.head - self.below(self.head.min(Self::RETAINED_OPS)); + self.send(op, ReplicationSend::Retransmit); + } + 2 => self.tick(), + 3 => self.deliver_frame(), + 4 if self.fault() => { + let behind = self.below(Self::RETAINED_OPS); + self.repair(behind); + } + _ => self.deliver_reply(), + } + self.check(); + } + + /// The backup holds every op up to `behind` ops before the head. + fn repair(&mut self, behind: u64) { + self.held = self.held.max(self.head.saturating_sub(behind)); + } + + fn tick(&mut self) { + self.now += Self::TICK_MICROS; + if let Some(work) = self.flow.work(MODEL_BACKUP, self.head, self.now) { + self.run(work); + } + } + + fn run(&mut self, work: ReplicationWork) { + match work { + ReplicationWork::Probe { nonce, through, .. } => { + if self.fault() { + self.flow.probe_failed(MODEL_BACKUP, nonce); + } else { + self.link.push_back(Frame::Probe { nonce, through }); + } + } + ReplicationWork::Replay { from_op, .. } + if from_op + Self::RETAINED_OPS <= self.head => + { + self.flow.defer(MODEL_BACKUP, self.now); + } + ReplicationWork::Replay { from_op, .. } => { + let to_op = self.head.min(from_op + Self::REPLAY_OPS_MAX - 1); + for op in from_op..=to_op { + if !self.send(op, ReplicationSend::Replay) { + break; + } + } + } + } + } + + fn send(&mut self, op: u64, send: ReplicationSend) -> bool { + let size = self.sizes[usize::try_from(op - 1).unwrap()]; + if !self.flow.admit(MODEL_BACKUP, op, size, send) { + return false; + } + if self.fault() { + self.flow.failed(MODEL_BACKUP, op); + return false; + } + self.link.push_back(Frame::Prepare { op, size }); + self.pushed_bytes += size; + if let Some(work) = self.flow.sent(MODEL_BACKUP, op, size, send, self.now) { + self.run(work); + } + true + } + + fn deliver_frame(&mut self) { + let Some(frame) = self.link.pop_front() else { + return; + }; + let dropped = self.fault(); + match frame { + Frame::Prepare { op, size } => { + self.consumed_bytes += size; + if !dropped && op == self.held + 1 { + self.held = op; + } + } + Frame::Probe { nonce, through } if !dropped => { + let reply = if self.fault() { + busy(MODEL_BACKUP, nonce, through) + } else { + ready(MODEL_BACKUP, nonce, through, self.held, self.held) + }; + self.replies.push_back(reply); + } + Frame::Probe { .. } => {} + } + } + + fn deliver_reply(&mut self) { + let Some(reply) = self.replies.pop_front() else { + return; + }; + if !self.fault() { + let _ = self.flow.reply(reply.header(), self.head, self.now); + } + } + + fn check(&self) { + let peer = &self.flow.peers[usize::from(MODEL_BACKUP)]; + assert_eq!(peer.sent_bytes, self.pushed_bytes); + assert!(peer.credited_bytes <= peer.sent_bytes); + assert!( + peer.credited_bytes <= self.consumed_bytes, + "credit {} passed the {} bytes the backup consumed", + peer.credited_bytes, + self.consumed_bytes + ); + assert!(peer.outstanding() <= REPLICATION_BATCH_BYTES_MAX); + } + } + + #[test] + fn given_random_losses_failures_and_busy_answers_when_the_flow_runs_should_keep_its_invariants_and_deliver_every_op() + { + const SEEDS: u64 = 16; + const STEPS: u32 = 20_000; + const SETTLE_TICKS: u32 = 2_000; + for seed in 1..=SEEDS { + let mut model = Model::new(seed); + for _ in 0..STEPS { + model.step(); + } + let stats = model.flow.stats; + assert!( + stats.live_refusals > 0 && stats.replayed > 0, + "seed {seed}: the faults must force refusals and replays, or the run proves nothing: \ + {stats:?}" + ); + model.faults = false; + // Only repair delivers the ops that the primary no longer holds. + model.repair(Model::RETAINED_OPS); + for _ in 0..SETTLE_TICKS { + if model.held == model.head { + break; + } + model.tick(); + while !model.link.is_empty() { + model.deliver_frame(); + } + while !model.replies.is_empty() { + model.deliver_reply(); + } + model.check(); + } + assert_eq!( + model.held, model.head, + "seed {seed}: the backup must hold every op once the faults stop" + ); + } + } +} diff --git a/core/consensus/src/vsr_timeout.rs b/core/consensus/src/vsr_timeout.rs index 9781520e8a..e9382aafb6 100644 --- a/core/consensus/src/vsr_timeout.rs +++ b/core/consensus/src/vsr_timeout.rs @@ -121,6 +121,7 @@ impl Timeout { pub enum TimeoutKind { Prepare, CommitMessage, + CheckQuorum, NormalHeartbeat, StartViewChangeMessage, ViewChangeStatus, @@ -136,6 +137,7 @@ pub enum TimeoutKind { pub struct TimeoutManager { prepare: Timeout, commit_message: Timeout, + check_quorum: Timeout, normal_heartbeat: Timeout, start_view_change_message: Timeout, do_view_change_message: Timeout, @@ -179,6 +181,7 @@ impl TimeoutManager { Self { prepare: Timeout::new(replica_id, Self::PREPARE_TICKS), commit_message: Timeout::new(replica_id, Self::COMMIT_MESSAGE_TICKS), + check_quorum: Timeout::new(replica_id, Self::NORMAL_HEARTBEAT_TICKS), normal_heartbeat: Timeout::new(replica_id, Self::NORMAL_HEARTBEAT_TICKS), start_view_change_message: Timeout::new( replica_id, @@ -196,12 +199,13 @@ impl TimeoutManager { } } - /// Override the backup's primary-liveness window (`[cluster] + /// Override the backup's primary-liveness and primary's quorum window (`[cluster] /// heartbeat_timeout`). Replaces the timeout object, so any countdown /// already in flight is discarded: call before the replica starts /// ticking (i.e. before `init` / `init_as_backup`). pub const fn set_normal_heartbeat_ticks(&mut self, ticks: u64) { self.normal_heartbeat = Timeout::new(self.normal_heartbeat.id, ticks); + self.check_quorum = Timeout::new(self.check_quorum.id, ticks); } /// Override the primary's commit-broadcast interval (`[cluster] @@ -256,6 +260,7 @@ impl TimeoutManager { pub const fn tick(&mut self) { self.prepare.tick(); self.commit_message.tick(); + self.check_quorum.tick(); self.normal_heartbeat.tick(); self.start_view_change_message.tick(); self.do_view_change_message.tick(); @@ -273,6 +278,7 @@ impl TimeoutManager { match kind { TimeoutKind::Prepare => &self.prepare, TimeoutKind::CommitMessage => &self.commit_message, + TimeoutKind::CheckQuorum => &self.check_quorum, TimeoutKind::NormalHeartbeat => &self.normal_heartbeat, TimeoutKind::StartViewChangeMessage => &self.start_view_change_message, TimeoutKind::ViewChangeStatus => &self.view_change_status, @@ -285,6 +291,7 @@ impl TimeoutManager { match kind { TimeoutKind::Prepare => &mut self.prepare, TimeoutKind::CommitMessage => &mut self.commit_message, + TimeoutKind::CheckQuorum => &mut self.check_quorum, TimeoutKind::NormalHeartbeat => &mut self.normal_heartbeat, TimeoutKind::StartViewChangeMessage => &mut self.start_view_change_message, TimeoutKind::ViewChangeStatus => &mut self.view_change_status, @@ -309,6 +316,7 @@ impl TimeoutManager { let timeout = match kind { TimeoutKind::Prepare => &mut self.prepare, TimeoutKind::CommitMessage => &mut self.commit_message, + TimeoutKind::CheckQuorum => &mut self.check_quorum, TimeoutKind::NormalHeartbeat => &mut self.normal_heartbeat, TimeoutKind::StartViewChangeMessage => &mut self.start_view_change_message, TimeoutKind::ViewChangeStatus => &mut self.view_change_status, diff --git a/core/integration/tests/cluster/partition_primary_routing.rs b/core/integration/tests/cluster/partition_primary_routing.rs index 9806bade41..367216ce13 100644 --- a/core/integration/tests/cluster/partition_primary_routing.rs +++ b/core/integration/tests/cluster/partition_primary_routing.rs @@ -731,6 +731,7 @@ async fn given_partitions_on_different_primaries_when_sending_round_robin_should } #[iggy_harness(cluster_nodes = 5, server(sharding.cpu_allocation = "2"))] +#[ignore = "slow: about two minutes against a five-node cluster, run with --run-ignored only"] async fn given_balanced_primaries_when_a_replica_fails_and_returns_should_rebalance_without_data_loss( harness: &mut TestHarness, ) { diff --git a/core/integration/tests/cluster/partition_state_transfer.rs b/core/integration/tests/cluster/partition_state_transfer.rs index 8f6270692d..6eb5a7d489 100644 --- a/core/integration/tests/cluster/partition_state_transfer.rs +++ b/core/integration/tests/cluster/partition_state_transfer.rs @@ -27,7 +27,8 @@ //! `RangeEvicted`, the repaired window cannot connect to recovered state, //! and `complete_repair` returns the `FloorRefused` conversion trigger. -use std::collections::HashMap; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::os::unix::fs::{FileExt, MetadataExt}; use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::Arc; @@ -74,8 +75,9 @@ const KILL_GATE_POLL: Duration = Duration::from_millis(10); const CONVERSION_MARKER: &str = "partition repair floor unreachable; converting to state transfer"; const INSTALL_MARKER: &str = "partition state transfer installed"; const INSTALL_FAILED_MARKER: &str = "partition state transfer install failed"; -/// The `StaleTransfer` reason, logged inside an [`INSTALL_FAILED_MARKER`] line. -const STALE_INSTALL_MARKER: &str = "is below the local commit frontier"; +/// The `OfferErasesHeldOps` reason, logged inside an [`INSTALL_FAILED_MARKER`] +/// line. +const HELD_OPS_INSTALL_MARKER: &str = "would erase held ops"; /// Logged once per transfer, at the last byte of the last artifact. The /// descriptor-time "serving" line only proves a request ARRIVED; this proves the /// pull ran. @@ -102,6 +104,32 @@ const ISOLATED_SENDS: u32 = 100; const WRITE_INTERVAL: Duration = Duration::from_secs(1); /// How long node 2 must go without another conversion once it installed. const CAUGHT_UP_WINDOW: Duration = Duration::from_secs(10); +/// Logged by a receiver that links segments it already holds into a +/// transfer. Its `reused_len` field counts the bytes that it did not pull. +const REUSE_MARKER: &str = "reused segments that this node already holds"; +/// Repeated-install specs must exhaust snapshot-tail retention as well as the +/// ring; otherwise the next gap can correctly heal without another snapshot. +const SNAPSHOT_ONLY_REPAIR_MEMORY_BYTES_MAX: &str = "1 B"; +/// Logged by a receiver that adopts the staged copies of an earlier attempt. +/// Its `adopted_len` field counts the bytes that it did not pull. +const ADOPTED_MARKER: &str = "adopted staged segments from an earlier transfer attempt"; +/// Logged by a receiver that fills an artifact with the bytes of its own +/// segment. Its `seeded_len` field counts them. +const SEED_MARKER: &str = "seeded a partition transfer artifact from the local segment"; +/// Logged by a receiver whose seeded artifact failed the manifest checksum. +const PREFIX_MISMATCH_MARKER: &str = "the local segment prefix does not match the offered artifact"; +/// Logged once per session before the install. Its `pulled_len` field counts +/// the chunk bytes that crossed the network. +const PULLED_MARKER: &str = "partition transfer pulled every artifact"; +/// Logged by a requester that schedules another transfer attempt, for +/// example after a refusal. +const REARM_MARKER: &str = "partition transfer did not land; scheduling a re-arm"; +/// Bulky sends per round of the reuse spec: 10 MiB, more than one +/// [`SEGMENT_SIZE_MULTI_ARTIFACT`], so each round seals at least one segment. +const BULKY_SENDS_PER_ROUND: u32 = 40; +/// Transfers in the reuse spec. The second one must reuse a sealed segment +/// that the first one pulled. +const REUSE_ROUNDS: usize = 2; /// Transfer end-to-end: adoption, repair round-trip, conversion, chunk pull, /// install, tail repair. CI runners are slow; bound without hanging the suite. @@ -382,7 +410,6 @@ async fn given_transfer_peer_dies_when_stalled_should_leave_dead_peer_and_recove /// needed: the pull itself is a few KiB. #[tokio::test(flavor = "multi_thread")] #[parallel] -#[ignore = "known bug: under sustained writes the backup refuses every transfer as stale (transfer frontier below the local commit frontier); run with --run-ignored only"] async fn given_sustained_writes_when_a_node_falls_past_the_evicted_ring_should_rejoin_through_one_transfer() { let mut harness = relayed_cluster(); @@ -408,12 +435,10 @@ async fn given_sustained_writes_when_a_node_falls_past_the_evicted_ring_should_r .expect("restore node 2's peer links"); await_marker(&harness, LAGGING_NODE, CONVERSION_MARKER).await; - // Today every install is refused with "transfer frontier X is below the - // local commit frontier X+1". The first request only pins the offer at the - // primary's commit point (the build answers "flush is pending"), and the - // pull runs on the next request, one `repair_retry_interval` later. Node 2 - // journals the op committed in between, and the install refuses an offer - // below an op it holds. + // The first request only pins the offer at the primary's commit point (the + // build answers "flush is pending"), and the pull runs on the next request, + // one `repair_retry_interval` later. Node 2 journals the ops committed in + // between, so it holds ops above the offer it installs. let deadline = Instant::now() + TRANSFER_BUDGET; while harness .node(LAGGING_NODE) @@ -424,15 +449,15 @@ async fn given_sustained_writes_when_a_node_falls_past_the_evicted_ring_should_r Instant::now() < deadline, "node 2 fell past the evicted ring and never installed a state transfer within \ {TRANSFER_BUDGET:?} while one message per second was written: the peers fully \ - served {} pull(s) and node 2 failed {} install(s), {} of them as stale, so a \ - backup cannot rejoin under sustained writes", + served {} pull(s) and node 2 failed {} install(s), {} of them for held ops, so \ + a backup cannot rejoin under sustained writes", fully_served_count(&harness), harness .node(LAGGING_NODE) .stdout_occurrences(INSTALL_FAILED_MARKER), harness .node(LAGGING_NODE) - .stdout_occurrences(STALE_INSTALL_MARKER), + .stdout_occurrences(HELD_OPS_INSTALL_MARKER), ); sleep(MARKER_POLL).await; } @@ -469,6 +494,298 @@ async fn given_sustained_writes_when_a_node_falls_past_the_evicted_ring_should_r ); } +/// A node that falls past the evicted ring must link the sealed segments it +/// already holds into each transfer and pull only the rest. +/// +/// Node 2 holds sealed segments that it wrote itself, then falls behind twice. +/// Each segment that node 2 held sealed at the cut must keep its inode through +/// the install: the install renamed a link to it, and none of its bytes +/// crossed the network. A pulled segment always lands in a new file, because +/// the old one still exists when the pull writes it. The second round must +/// also keep a segment that the first round pulled, which proves that the +/// install stamps the segments it installs. The reuse line must count exactly +/// the bytes of the kept segments. +#[tokio::test(flavor = "multi_thread")] +#[parallel] +async fn given_sealed_segments_on_a_lagging_node_when_it_transfers_twice_should_pull_only_the_rest() +{ + let mut harness = + relayed_cluster_with_repair_budget(Some(SNAPSHOT_ONLY_REPAIR_MEMORY_BYTES_MAX)); + harness.start().await.expect("start the cluster"); + let client = harness + .root_client_for_node(0) + .await + .expect("connect a root client to the node"); + seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT)).await; + produce_bulky(&client, 2 * BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; + let mut expected: Vec = (0..2 * BULKY_SENDS_PER_ROUND) + .map(bulky_payload_prefix) + .collect(); + let data_path = harness.node(LAGGING_NODE).data_path(); + let mut kept_before = BTreeSet::new(); + + for round in 1..=REUSE_ROUNDS { + // Node 2 must hold every committed payload, so the segments that it + // holds sealed are final before the cut. + await_installed_payloads(&harness, LAGGING_NODE, &expected).await; + let held = segment_files(&data_path); + let sealed: Vec = held.keys().rev().skip(1).copied().collect(); + assert!( + !sealed.is_empty(), + "round {round}: node 2 must hold a sealed segment before the cut" + ); + if round > 1 { + assert!( + sealed.iter().any(|start| !kept_before.contains(start)), + "round {round}: node 2 must hold a sealed segment that the last transfer \ + pulled, or this round cannot prove that the install stamps it" + ); + } + let installs = harness + .node(LAGGING_NODE) + .stdout_occurrences(INSTALL_MARKER); + let reuses = harness.node(LAGGING_NODE).stdout_occurrences(REUSE_MARKER); + + harness + .isolate_node(LAGGING_NODE) + .expect("cut node 2 off from its peers"); + produce_bulky(&client, BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; + produce(&client, ISOLATED_SENDS).await; + harness + .heal_node(LAGGING_NODE) + .expect("restore node 2's peer links"); + await_new_marker(&harness, LAGGING_NODE, INSTALL_MARKER, installs).await; + expected.extend((0..BULKY_SENDS_PER_ROUND).map(bulky_payload_prefix)); + expected.extend((0..ISOLATED_SENDS).map(|sequence| format!("message-{sequence}"))); + await_installed_payloads(&harness, LAGGING_NODE, &expected).await; + + let installed = segment_files(&data_path); + let kept: BTreeMap = held + .iter() + .filter(|(start, file)| installed.get(start) == Some(file)) + .map(|(start, (_, len))| (*start, *len)) + .collect(); + for start in &sealed { + assert!( + kept.contains_key(start), + "round {round}: node 2 held segment {start} sealed, but the install replaced \ + its file, so the transfer copied bytes that node 2 already had" + ); + } + assert_eq!( + reused_len_after(&harness, LAGGING_NODE, reuses), + kept.values().sum::(), + "round {round}: the reuse line must count exactly the bytes of the kept segments" + ); + kept_before = kept.into_keys().collect(); + } +} + +/// A node that falls past the evicted ring while the primary keeps writing +/// the segment that the node holds a prefix of must pull only the rest. +/// +/// Each round, node 2 holds the leading bytes of the segment that the +/// primary writes on during the cut. The transfer must seed that artifact +/// with them, so the pulled bytes are the manifest minus the reused and the +/// held bytes. Between the rounds, node 2 fills and seals the segment that the +/// transfer installed as active. The next transfer must keep that segment like +/// any other sealed one, which proves that its checksum memo grew with it. +#[tokio::test(flavor = "multi_thread")] +#[parallel] +async fn given_a_segment_prefix_on_a_lagging_node_when_it_transfers_should_pull_only_the_suffix() { + let mut harness = + relayed_cluster_with_repair_budget(Some(SNAPSHOT_ONLY_REPAIR_MEMORY_BYTES_MAX)); + harness.start().await.expect("start the cluster"); + let client = harness + .root_client_for_node(0) + .await + .expect("connect a root client to the node"); + seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT)).await; + produce_bulky(&client, 2 * BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; + let mut expected: Vec = (0..2 * BULKY_SENDS_PER_ROUND) + .map(bulky_payload_prefix) + .collect(); + let data_path = harness.node(LAGGING_NODE).data_path(); + let mut installed_active: Option = None; + + for round in 1..=REUSE_ROUNDS { + await_installed_payloads(&harness, LAGGING_NODE, &expected).await; + let held = segment_files(&data_path); + let (active_start, active_len) = held_prefix(&held, round); + if let Some(installed) = installed_active { + assert!( + installed < active_start, + "round {round}: the segment that the last transfer installed as active must \ + seal on node 2 before the cut" + ); + } + let installs = harness + .node(LAGGING_NODE) + .stdout_occurrences(INSTALL_MARKER); + + harness + .isolate_node(LAGGING_NODE) + .expect("cut node 2 off from its peers"); + produce_bulky(&client, BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; + produce(&client, ISOLATED_SENDS).await; + harness + .heal_node(LAGGING_NODE) + .expect("restore node 2's peer links"); + await_new_marker(&harness, LAGGING_NODE, INSTALL_MARKER, installs).await; + expected.extend((0..BULKY_SENDS_PER_ROUND).map(bulky_payload_prefix)); + expected.extend((0..ISOLATED_SENDS).map(|sequence| format!("message-{sequence}"))); + await_installed_payloads(&harness, LAGGING_NODE, &expected).await; + + let session = installed_session(&harness, LAGGING_NODE); + // Node 2 can persist bytes of its active segment after the files were + // listed, so the seed holds at least what the listing saw. + let (seeded_len, _) = session.seed.unwrap_or_else(|| { + panic!( + "round {round}: node 2 held {active_len} bytes of segment {active_start}, \ + but it did not seed them" + ) + }); + assert!( + seeded_len >= active_len, + "round {round}: the seed must hold the {active_len} bytes that node 2 held, \ + got {seeded_len}" + ); + assert_eq!( + session.pulled_len, + session.total_len - session.reused_len - seeded_len, + "round {round}: node 2 seeded {seeded_len} bytes of segment {active_start}, so the \ + transfer must pull the manifest minus those and the reused bytes" + ); + let installed = segment_files(&data_path); + if let Some(sealed) = installed_active { + assert_eq!( + installed.get(&sealed).map(|(inode, _)| inode), + held.get(&sealed).map(|(inode, _)| inode), + "round {round}: node 2 sealed segment {sealed} after the last transfer \ + installed it as active, but this install replaced its file, so the transfer \ + pulled bytes that node 2 already had" + ); + } + installed_active = installed.keys().next_back().copied(); + // With node 2 connected: fill and seal the segment that this transfer + // installed as active. The small sends leave bytes in the segment + // after it, which the next round seeds. + produce_bulky(&client, BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; + produce(&client, ISOLATED_SENDS).await; + expected.extend((0..BULKY_SENDS_PER_ROUND).map(bulky_payload_prefix)); + expected.extend((0..ISOLATED_SENDS).map(|sequence| format!("message-{sequence}"))); + } +} + +/// A held prefix with other bytes than the offered segment must not reach the +/// install. The checksum of the seeded artifact fails, and the transfer pulls +/// that artifact whole in the same session. +#[tokio::test(flavor = "multi_thread")] +#[parallel] +async fn given_a_damaged_segment_prefix_when_it_transfers_should_pull_the_whole_artifact() { + let mut harness = relayed_cluster(); + harness.start().await.expect("start the cluster"); + let client = harness + .root_client_for_node(0) + .await + .expect("connect a root client to the node"); + seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT)).await; + produce_bulky(&client, 2 * BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; + let mut expected: Vec = (0..2 * BULKY_SENDS_PER_ROUND) + .map(bulky_payload_prefix) + .collect(); + let data_path = harness.node(LAGGING_NODE).data_path(); + await_installed_payloads(&harness, LAGGING_NODE, &expected).await; + let held = segment_files(&data_path); + let (active_start, active_len) = held_prefix(&held, 1); + let installs = harness + .node(LAGGING_NODE) + .stdout_occurrences(INSTALL_MARKER); + let mismatches = harness + .node(LAGGING_NODE) + .stdout_occurrences(PREFIX_MISMATCH_MARKER); + + harness + .isolate_node(LAGGING_NODE) + .expect("cut node 2 off from its peers"); + flip_segment_byte(&data_path, active_start, active_len / 2); + produce_bulky(&client, BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; + produce(&client, ISOLATED_SENDS).await; + harness + .heal_node(LAGGING_NODE) + .expect("restore node 2's peer links"); + await_new_marker(&harness, LAGGING_NODE, INSTALL_MARKER, installs).await; + expected.extend((0..BULKY_SENDS_PER_ROUND).map(bulky_payload_prefix)); + expected.extend((0..ISOLATED_SENDS).map(|sequence| format!("message-{sequence}"))); + await_installed_payloads(&harness, LAGGING_NODE, &expected).await; + + assert_eq!( + harness + .node(LAGGING_NODE) + .stdout_occurrences(PREFIX_MISMATCH_MARKER), + mismatches + 1, + "the seeded artifact must fail its checksum once, and the whole pull must pass" + ); + let session = installed_session(&harness, LAGGING_NODE); + let (seeded_len, artifact_len) = session + .seed + .expect("node 2 must seed the artifact before its checksum fails"); + assert!( + seeded_len >= active_len, + "the seed must hold the {active_len} bytes that node 2 held, got {seeded_len}" + ); + assert_eq!( + session.pulled_len, + session.total_len - session.reused_len + artifact_len - seeded_len, + "the transfer must pull the suffix, then the whole artifact once the checksum failed" + ); +} + +/// The first request of a transfer finds the plan of the primary unpinned, +/// because the pin waits for the next I/O turn of the partition. The primary +/// must keep that request and answer it once the plan is pinned. A refusal +/// makes the requester wait out a whole re-arm interval for nothing. +#[tokio::test(flavor = "multi_thread")] +#[parallel] +async fn given_an_unpinned_plan_when_a_transfer_is_requested_should_answer_without_a_re_arm() { + let mut harness = relayed_cluster(); + harness.start().await.expect("start the cluster"); + let client = harness + .root_client_for_node(0) + .await + .expect("connect a root client to the node"); + seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT)).await; + produce_bulky(&client, BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; + let mut expected: Vec = (0..BULKY_SENDS_PER_ROUND) + .map(bulky_payload_prefix) + .collect(); + await_installed_payloads(&harness, LAGGING_NODE, &expected).await; + let installs = harness + .node(LAGGING_NODE) + .stdout_occurrences(INSTALL_MARKER); + let rearms = harness.node(LAGGING_NODE).stdout_occurrences(REARM_MARKER); + + harness + .isolate_node(LAGGING_NODE) + .expect("cut node 2 off from its peers"); + produce_bulky(&client, BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; + produce(&client, ISOLATED_SENDS).await; + harness + .heal_node(LAGGING_NODE) + .expect("restore node 2's peer links"); + await_new_marker(&harness, LAGGING_NODE, INSTALL_MARKER, installs).await; + expected.extend((0..BULKY_SENDS_PER_ROUND).map(bulky_payload_prefix)); + expected.extend((0..ISOLATED_SENDS).map(|sequence| format!("message-{sequence}"))); + await_installed_payloads(&harness, LAGGING_NODE, &expected).await; + + assert_eq!( + harness.node(LAGGING_NODE).stdout_occurrences(REARM_MARKER), + rearms, + "the primary must answer the first transfer request of node 2 with an offer, \ + so node 2 must not schedule a re-arm before the install" + ); +} + async fn restart_lagging_node_and_await_install(harness: &mut TestHarness) { // Node 2 holds a durable prefix, then misses enough traffic that the // survivors' ring moves past its durable end: its repaired window cannot @@ -515,13 +832,23 @@ async fn restart_lagging_node_and_await_install(harness: &mut TestHarness) { /// links, so the one that cuts a node off builds its cluster here with the same /// server settings. fn relayed_cluster() -> TestHarness { - let overrides = HashMap::from([ + relayed_cluster_with_repair_budget(None) +} + +fn relayed_cluster_with_repair_budget(repair_memory_bytes_max: Option<&str>) -> TestHarness { + let mut overrides = HashMap::from([ ("sharding.cpu_allocation".to_owned(), "0..1".to_owned()), ( "partition.evicted_ring_capacity".to_owned(), "64".to_owned(), ), ]); + if let Some(bytes_max) = repair_memory_bytes_max { + overrides.insert( + "partition.repair_memory_bytes_max".to_owned(), + bytes_max.to_owned(), + ); + } let extra_envs = resolve_config_paths(&overrides).expect("valid server config paths"); TestHarness::builder() .server(TestServerConfig::builder().extra_envs(extra_envs).build()) @@ -594,6 +921,123 @@ fn fully_served_count(harness: &TestHarness) -> usize { .sum() } +/// `reused_len` of the first reuse line that `node` logged after `baseline` +/// earlier ones. +fn reused_len_after(harness: &TestHarness, node: usize, baseline: usize) -> u64 { + let log = harness.node(node).stdout_plain(); + let line = log + .lines() + .filter(|line| line.contains(REUSE_MARKER)) + .nth(baseline) + .unwrap_or_else(|| panic!("node {node} logged no {REUSE_MARKER:?} line for the transfer")); + logged_number(line, "reused_len") +} + +/// What the last session that `node` pulled to the end logged. +struct PulledSession { + total_len: u64, + /// The bytes that it staged without a pull: `reused_len` of its reuse + /// line plus `adopted_len` of the adopted line before it. + reused_len: u64, + /// `seeded_len` and `artifact_len` of its seed line. + seed: Option<(u64, u64)>, + pulled_len: u64, +} + +/// The lines of one session come in order: adopted and reuse when its reuse +/// scan lands, the seed when the pull reaches that artifact, the pulled line +/// before the install. +fn installed_session(harness: &TestHarness, node: usize) -> PulledSession { + let log = harness.node(node).stdout_plain(); + let mut adopted = None; + let mut reuse = None; + let mut seed = None; + let mut session = None; + for line in log.lines() { + if line.contains(ADOPTED_MARKER) { + adopted = Some(line); + } else if line.contains(REUSE_MARKER) { + reuse = Some((line, adopted.take())); + seed = None; + } else if line.contains(SEED_MARKER) { + seed = Some(line); + } else if line.contains(PULLED_MARKER) { + session = Some((reuse, seed, line)); + } + } + let (reuse, seed, pulled) = + session.unwrap_or_else(|| panic!("node {node} logged no {PULLED_MARKER:?} line")); + let (reuse, adopted) = + reuse.unwrap_or_else(|| panic!("node {node} logged no {REUSE_MARKER:?} line")); + PulledSession { + total_len: logged_number(reuse, "total_len"), + reused_len: logged_number(reuse, "reused_len") + + adopted.map_or(0, |line| logged_number(line, "adopted_len")), + seed: seed.map(|line| { + ( + logged_number(line, "seeded_len"), + logged_number(line, "artifact_len"), + ) + }), + pulled_len: logged_number(pulled, "pulled_len"), + } +} + +fn logged_number(line: &str, field: &str) -> u64 { + let label = format!(" {field}="); + let start = line + .find(&label) + .unwrap_or_else(|| panic!("the line has no {label:?} field: {line}")) + + label.len(); + line[start..] + .split(|character: char| !character.is_ascii_digit()) + .next() + .and_then(|digits| digits.parse().ok()) + .unwrap_or_else(|| panic!("the line has no number in {label:?}: {line}")) +} + +/// The active segment of `held` and its length. Sealed segments must exist, +/// so the transfer logs a reuse line, and the active one must hold bytes, so +/// it can seed an artifact. +fn held_prefix(held: &BTreeMap, round: usize) -> (u64, u64) { + assert!( + held.len() > 1, + "round {round}: node 2 must hold a sealed segment before the cut" + ); + let (&start, &(_, len)) = held + .last_key_value() + .expect("node 2 holds a segment before the cut"); + assert!( + len > 0, + "round {round}: node 2's active segment must hold bytes before the cut" + ); + (start, len) +} + +/// Invert one byte of the segment log at `start_offset`. +fn flip_segment_byte(root: &Path, start_offset: u64, position: u64) { + let path = walk(root, &mut |path| { + is_segment_log(path) + && path + .file_stem() + .and_then(|stem| stem.to_str()) + .and_then(|stem| stem.parse::().ok()) + == Some(start_offset) + }) + .unwrap_or_else(|| panic!("no segment log starts at offset {start_offset}")); + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(&path) + .expect("open the segment log"); + let mut byte = [0_u8; 1]; + file.read_exact_at(&mut byte, position) + .expect("read the segment byte"); + byte[0] = !byte[0]; + file.write_all_at(&byte, position) + .expect("write the segment byte"); +} + async fn connect_any(harness: &TestHarness, nodes: &[usize]) -> Option { for &node in nodes { if let Ok(builder) = harness.node(node).tcp_client() @@ -655,7 +1099,11 @@ async fn produce(client: &IggyClient, count: u32) { /// `payload_len` filler bytes so the on-disk segment grows fast. async fn produce_bulky(client: &IggyClient, count: u32, payload_len: usize) { for sequence in 0..count { - let payload = format!("bulky-{sequence}-{}", "x".repeat(payload_len)); + let payload = format!( + "{}{}", + bulky_payload_prefix(sequence), + "x".repeat(payload_len) + ); let mut messages = vec![IggyMessage::from_str(&payload).expect("message")]; client .send_messages( @@ -669,6 +1117,12 @@ async fn produce_bulky(client: &IggyClient, count: u32, payload_len: usize) { } } +/// The leading bytes of the payload that [`produce_bulky`] sends as its +/// `sequence`-th message. +fn bulky_payload_prefix(sequence: u32) -> String { + format!("bulky-{sequence}-") +} + async fn seed_partition(client: &IggyClient) { seed_topic(client, None).await; produce(client, MESSAGES_COUNT).await; @@ -856,6 +1310,25 @@ fn segment_log_count(root: &Path) -> usize { count } +/// Inode and length of each segment log under `root`, by base offset. A +/// segment that keeps its inode through an install was linked, not copied. +fn segment_files(root: &Path) -> BTreeMap { + let mut files = BTreeMap::new(); + let _ = walk(root, &mut |path| { + if is_segment_log(path) + && let Some(start_offset) = path + .file_stem() + .and_then(|stem| stem.to_str()) + .and_then(|stem| stem.parse().ok()) + && let Ok(metadata) = std::fs::metadata(path) + { + files.insert(start_offset, (metadata.ino(), metadata.len())); + } + false + }); + files +} + /// A segment `.log`, named for its 20-digit zero-padded base offset (see /// `partitions::state_transfer`'s path builders). fn is_segment_log(path: &Path) -> bool { diff --git a/core/integration/tests/server/external_auth.rs b/core/integration/tests/server/external_auth.rs index 1a788b0007..a6688b88be 100644 --- a/core/integration/tests/server/external_auth.rs +++ b/core/integration/tests/server/external_auth.rs @@ -25,6 +25,7 @@ use std::collections::HashMap; use std::time::Duration; use iggy::prelude::*; +use iggy_common::BINARY_OPERATION_TIMEOUT; use integration::harness::ext_auth_server::{ EXT_AUTH_IGGY_USERNAME, EXT_AUTH_PASSWORD, EXT_AUTH_USERNAME, EXT_DENIED_PASSWORD, EXT_DENIED_USERNAME, ExtAuthServer, @@ -359,7 +360,9 @@ mod serialized { #[tokio::test] async fn given_ext_auth_when_callout_cap_saturated_should_return_retryable_error() { - let ext_server = ExtAuthServer::start_slow(Duration::from_secs(30)).await; + // The overflow login retries until its operation timeout, so a callout + // that ends before it frees a slot and lets a retry reach the service. + let ext_server = ExtAuthServer::start_slow(BINARY_OPERATION_TIMEOUT * 2).await; let mut envs = ext_auth_envs(&ext_server.url()); envs.insert("IGGY_EXTERNAL_AUTH_TIMEOUT".to_owned(), "60 s".to_owned()); envs.insert( diff --git a/core/metadata/src/applied_frontier.rs b/core/metadata/src/applied_frontier.rs index 812da20236..8034160ddf 100644 --- a/core/metadata/src/applied_frontier.rs +++ b/core/metadata/src/applied_frontier.rs @@ -17,11 +17,12 @@ //! The node-wide applied metadata frontier and the wait a read parks on. +use std::cell::Cell; use std::future::Future; use std::pin::Pin; -use std::sync::Mutex; use std::sync::PoisonError; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; use std::task::{Context, Poll, Waker}; use std::time::{Duration, Instant}; @@ -56,6 +57,37 @@ pub struct AppliedFrontier { waiters: Mutex, read_budget: Duration, partition_freshness: Option, + recovering_shards: AtomicUsize, +} + +/// A shard's recovery state, shared with the node's primary placement probes. +/// Dropping a shard releases its registration even during interrupted shutdown. +pub struct PartitionRecovery { + frontier: Arc, + active: Cell, +} + +impl PartitionRecovery { + pub fn set_active(&self, active: bool) { + if self.active.replace(active) == active { + return; + } + if active { + self.frontier + .recovering_shards + .fetch_add(1, Ordering::Release); + } else { + self.frontier + .recovering_shards + .fetch_sub(1, Ordering::Release); + } + } +} + +impl Drop for PartitionRecovery { + fn drop(&mut self) { + self.set_active(false); + } } /// Volatile quorum evidence. Recovery restores metadata, never freshness. @@ -113,9 +145,23 @@ impl AppliedFrontier { }), read_budget, partition_freshness: None, + recovering_shards: AtomicUsize::new(0), + } + } + + #[must_use] + pub fn track_partition_recovery(self: &Arc) -> PartitionRecovery { + PartitionRecovery { + frontier: Arc::clone(self), + active: Cell::new(false), } } + #[must_use] + pub fn has_partition_recovery(&self) -> bool { + self.recovering_shards.load(Ordering::Acquire) != 0 + } + /// Arm admission for a multi-node process. `epoch` must use the bus clock; /// a newly constructed frontier has no credit, including after recovery. #[must_use] @@ -346,6 +392,24 @@ mod tests { use std::task::{Context, Poll}; use std::time::{Duration, Instant}; + #[test] + fn given_recovering_shards_when_one_finishes_should_wait_for_the_last_registration() { + let frontier = Arc::new(AppliedFrontier::default()); + let first = frontier.track_partition_recovery(); + let second = frontier.track_partition_recovery(); + assert!(!frontier.has_partition_recovery()); + first.set_active(true); + first.set_active(true); + second.set_active(true); + first.set_active(false); + assert!(frontier.has_partition_recovery()); + drop(second); + assert!(!frontier.has_partition_recovery()); + first.set_active(true); + drop(first); + assert!(!frontier.has_partition_recovery()); + } + #[test] fn given_quorum_credit_when_the_deadline_arrives_should_expire_without_an_owner_tick() { let epoch = Instant::now(); diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs index edb7f38d73..f6b7931d0a 100644 --- a/core/metadata/src/impls/metadata.rs +++ b/core/metadata/src/impls/metadata.rs @@ -44,7 +44,7 @@ use consensus::{ build_result_rejection_reply, emit_sim_event, fatal, fence_old_prepare_by_commit, is_caught_up_primary, is_caught_up_primary_except_barrier, panic_if_hash_chain_would_break_in_same_view, peek_committable_head, pipeline_prepare_common, - register_preflight, replicate_preflight, replicate_to_next_in_chain, request_preflight, + register_preflight, replicate_preflight, replicate_prepare, request_preflight, send_eviction_to_client, send_prepare_ok as send_prepare_ok_common, verify_prepare_integrity, }; use futures::channel::oneshot; @@ -1932,15 +1932,13 @@ where // Durable here but not yet committed, and the primary is retransmitting // it: our original PrepareOk was lost (e.g. the primary's inbox - // overflowed under a client burst). Re-forward the tail down the chain - // so a downstream replica that missed it recovers, then re-ack ONLY - // the retransmitted op. The primary's retransmit cycle walks every + // overflowed under a client burst). Re-ack only the retransmitted op. + // The primary's retransmit cycle walks every // un-acked op in the window (`prepare_timeout_targets`), so a lost ack // for a lower op gets its own retransmit and its own re-ack; re-acking the // whole suffix here is O(window^2) PrepareOks per cycle across the // backups, which can overflow the primary's inbox -- the very failure - // this path recovers from. Both downstream and primary are idempotent - // on a duplicate (replica, op). + // this path recovers from. The primary is idempotent on duplicate acks. #[allow(clippy::cast_possible_truncation)] if journal.handle().header(header.op as usize).is_some() { warn!( @@ -1951,7 +1949,7 @@ where op = header.op, commit = consensus.commit_max(), operation = ?header.operation, - "journal already holds prepare, re-forwarding + re-acking it" + "journal already holds prepare, re-acking it" ); self.replicate(&message).await; self.send_prepare_ok(&header).await; @@ -5094,8 +5092,8 @@ where journal.handle().header(op).is_some(), "replicate: prepare must be durable in local journal before chain-forward" ); - if let Err(e) = replicate_to_next_in_chain(consensus, message).await { - tracing::warn!(op = header.op, error = ?e, "chain replication failed"); + if let Err(e) = replicate_prepare(consensus, message).await { + tracing::warn!(op = header.op, error = ?e, "prepare replication enqueue failed"); } } diff --git a/core/metadata/src/lib.rs b/core/metadata/src/lib.rs index af44a48aa2..121f0931ef 100644 --- a/core/metadata/src/lib.rs +++ b/core/metadata/src/lib.rs @@ -25,7 +25,7 @@ pub mod stm; // The node-wide read frontier the read gates park on; minted by the bootstrap // before any shard exists, so it is named outside `impls::`. -pub use applied_frontier::{AppliedFrontier, Reached}; +pub use applied_frontier::{AppliedFrontier, PartitionRecovery, Reached}; // Re-export IggyMetadata for use in other modules pub use impls::metadata::{ diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs index 8bd6c2e09f..35b02c6c91 100644 --- a/core/partitions/src/iggy_partition.rs +++ b/core/partitions/src/iggy_partition.rs @@ -20,7 +20,7 @@ use crate::consumer_offset_capacity::{ DurableOffsetState, ExplicitOffsetStores, }; use crate::journal::{ - MessageLookup, PartitionJournal, PartitionJournalMemStorage, RetainedBatchMeta, + BodyPosition, MessageLookup, PartitionJournal, PartitionJournalMemStorage, RetainedBatchMeta, }; use crate::log::JournalInfo; use crate::log::SegmentedLog; @@ -52,7 +52,7 @@ use consensus::{ SimEventKind, VsrConsensus, ack_preflight, ack_quorum_reached, build_deny_reply_from_request, build_reply_from_request, build_reply_message, dvc_header_kind, emit_namespace_progress_event, emit_partition_diag, emit_sim_event, fence_old_prepare_by_commit, repair_session_live, - repaired_frontier_update, repaired_prepare_evidence, replicate_frozen_to_next_in_chain, + repaired_frontier_update, repaired_prepare_evidence, replicate_frozen_prepare, replicate_preflight, restamp_prepare_view, send_prepare_ok as send_prepare_ok_common, verify_prepare_integrity, }; @@ -107,6 +107,8 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use tokio::sync::Mutex as TokioMutex; use tracing::{debug, error, warn}; +const PREPARE_GAP_LOG_INTERVAL_MICROS: u64 = 1_000_000; + /// Which of a partition's offset counters are live. /// /// Two bits, not one: a reservation-seeded boot makes the append counter live @@ -200,6 +202,8 @@ where pub(crate) materialization_missing: bool, recovered_log_view: Option, pending_persisted_acks: RefCell>, + /// Without a WAL, an install must still preserve ops admitted to an ack. + pub(crate) acknowledged_op: Cell, /// In-flight journal repair: /// set when the recovery handshake finds this replica behind the group's /// commit frontier, cleared when `RepairDone` completes the walk. @@ -228,6 +232,7 @@ where /// the rest of the tick reads the partition through, so a `&mut` here would /// buy a second lookup of the same group per tick. prepare_gap_drops: Cell, + prepare_gap_log_after: Cell, /// Fault injection for the harness, armed by /// [`Self::inject_commit_failure`]. Behind a feature only the simulator's /// DEV-dependencies turn on, so no build that ships this crate compiles the @@ -262,7 +267,7 @@ where io_notifier: RefCell>, io_bytes_max: Cell, io_sequence: Cell, - io_active: Rc, + pub(crate) io_active: Rc, pub(crate) transition: Option, deferred_views: Vec, pub(crate) pending_consumer_offset_commits: HashMap, @@ -278,6 +283,15 @@ where /// First op of a broken fenced window the I/O lane is truncating. The fence's /// follow-up runs when that truncation lands. unproven_drop_from: Option, + /// `(commit_min, bytes)`: the bytes that repair journaled above that + /// `commit_min` while the view log was unproven. See + /// [`Self::proof_walk_over_budget`]. + proof_walk: (u64, u64), + pub(crate) proof_walk_memory: Option, + pub(crate) served_repair_tails: RefCell>, + /// The hole of the proof walk and the highest op held below it when the + /// last repair request went out. See [`Self::next_proof_walk_pull`]. + proof_walk_requested: Cell<(u64, Option)>, /// Identity shared with pending polls and replaced when their history retires. poll_history: PollHistoryId, incarnation: crate::PartitionIncarnation, @@ -408,15 +422,20 @@ where /// segment extends its own hasher over the bytes it gained. Without this, /// EVERY offer build re-reads and re-hashes all retained bytes on the pump /// -- and a committing primary advances `commit_op` each round, so the offer - /// cache alone never saves the pass. Swept against the live chain at build - /// time; cleared wherever segment files are unlinked-and-recreated - /// (install, converge). + /// cache alone never saves the pass. Retention drops the entry of each + /// segment it retires. An install replaces the map with the stamps of the + /// segments that it installs, and a converge clears the map. pub(crate) segment_checksum_cache: RefCell>, - /// Receiving-side memo of the last staged-segment reuse scan, so a peer - /// rotation against the same segment set does not re-read and re-walk every - /// staged file. See [`crate::state_transfer::ReuseScanMemo`]. + /// Verified staging files retained across partial scans and advancing offers. + /// See [`crate::state_transfer::ReuseScanMemo`]. pub(crate) reuse_scan_memo: RefCell>, + /// Resolves when the last staging writer ends. Reuse scans and spills wait + /// for it so abandoned tasks cannot race a new session on the same paths. + pub(crate) staging_work_ended: Option>, + /// The checksum state of the last segment that a transfer spilled. The + /// install keeps it as the memo of the segment it opens as active. + pub(crate) spilled_hasher: RefCell>, /// Serving-side offer cache, keyed by the `commit_op` it was built at, so /// simultaneous rejoiners share one manifest instead of re-reading every /// segment per requester. Invalidated by segment removal (same commit @@ -701,6 +720,7 @@ struct PendingMaterialization { durable_offset: Option, phase: MaterializationPhase, save: Option, + checksum: Option>, } #[derive(Clone, Copy)] @@ -910,9 +930,11 @@ where materialization_missing: false, recovered_log_view: None, pending_persisted_acks: RefCell::new(BTreeMap::new()), + acknowledged_op: Cell::new(0), repair: None, gap_ticks: Cell::new(0), prepare_gap_drops: Cell::new(0), + prepare_gap_log_after: Cell::new(0), #[cfg(any(test, feature = "fault-injection"))] injected_commit_failure: false, recovered_durable_offset: None, @@ -923,6 +945,10 @@ where recovered_unproven: false, recovered_commit: 0, unproven_drop_from: None, + proof_walk: (0, 0), + proof_walk_memory: None, + served_repair_tails: RefCell::new(HashMap::new()), + proof_walk_requested: Cell::new((0, None)), pending_commit: None, local_request_order: Cell::new(0), no_ack: VecDeque::new(), @@ -977,6 +1003,8 @@ where transfer_rearm: None, segment_checksum_cache: RefCell::new(std::collections::HashMap::new()), reuse_scan_memo: RefCell::new(None), + staging_work_ended: None, + spilled_hasher: RefCell::new(None), transfer_offer_cache: RefCell::new(None), transfer_plan: RefCell::new(None), }; @@ -1075,7 +1103,7 @@ where .map(|installed| installed.partition_op) } - /// Pending here or already applied, so another copy would only add a prepare. + /// Pending, applied, or superseded in this incarnation, so no prepare is needed. #[must_use] pub fn consumer_group_owner_admitted( &self, @@ -1085,7 +1113,12 @@ where || self .consumer_group_owners .get(&installation.group_id) - .is_some_and(|installed| installed.installation == *installation) + .is_some_and(|installed| { + let installed = installed.installation; + installed.incarnation == installation.incarnation + && (installed == *installation + || installed.owner.generation > installation.owner.generation) + }) } #[must_use] @@ -1803,6 +1836,10 @@ where && (self.consensus.state_transfer_stage() == consensus::StateTransferStage::Idle || self.consensus.status() == consensus::Status::ViewChange) { + // Without a certified log this replica has nothing to elect on, and + // the shard drops its votes, so an election after unanswered probes + // only lifts its view above the primary that must answer them. + self.consensus.set_recovery_election_allowed(false); self.consensus.begin_view_probe(); if self.consensus.state_transfer_stage() == consensus::StateTransferStage::Idle { self.consensus.begin_state_transfer_await(); @@ -2241,6 +2278,7 @@ where } pub async fn drive_persistence(&mut self) { + self.refresh_proof_walk_memory(); if self.fatal.is_some() { return; } @@ -2400,24 +2438,34 @@ where } pub fn register_rebuilt_ack(&self, header: &PrepareHeader) -> bool { + let ready = self.prepare_ack_ready(header); + if ready { + self.record_acknowledged_op(header.op); + } else { + self.pending_persisted_acks + .borrow_mut() + .insert(header.op, *header); + } + ready + } + + fn prepare_ack_ready(&self, header: &PrepareHeader) -> bool { // State-transfer admission uses the accepted WAL head to protect every // acknowledged op, including replicated prepares adopted in a new view. let accepted = self .persistence .as_ref() .is_none_or(|persistence| header.op <= persistence.head()); - let durable = accepted + accepted && (!self.requires_persistence(header.operation) || self.persistence.as_ref().is_some_and(|persistence| { persistence.is_durable(header) && persistence.certified_log_view() == Some(self.consensus.log_view()) - })); - if !durable { - self.pending_persisted_acks - .borrow_mut() - .insert(header.op, *header); - } - durable + })) + } + + fn record_acknowledged_op(&self, op: u64) { + self.acknowledged_op.set(self.acknowledged_op.get().max(op)); } fn submit_prepare_persistence(&self, prepare: Frozen<4096>, operation: Operation) -> bool { @@ -5795,7 +5843,7 @@ where self.append_stamped_messages(message, batch).await } #[must_use] - fn namespace(&self) -> IggyNamespace { + pub(crate) fn namespace(&self) -> IggyNamespace { IggyNamespace::from_raw(self.consensus.group()) } @@ -5818,6 +5866,17 @@ where } } + /// [`Self::notify_io`] for a detached task that outlives this borrow. + pub(crate) fn io_wake(&self) -> impl Fn() + 'static { + let notifier = self.io_notifier.borrow().clone(); + let (namespace, incarnation) = (self.namespace(), self.incarnation); + move || { + if let Some(notifier) = ¬ifier { + notifier(namespace, incarnation); + } + } + } + #[must_use] pub fn needs_io_retry(&self) -> bool { if self.io_active.is_interrupted() { @@ -6163,8 +6222,8 @@ where .journal() .inner .with_entries(|entries| entries.len().saturating_sub(prefix)); - // Eviction rebuilds the retained tail. Amortize partial flushes so - // a slow writer cannot make us rebuild a large tail after each write. + // A partial flush waits until it frees at least as many entries as + // it keeps, so a slow writer gets fewer and larger flushes. if (ready_through == through || prefix >= retained) && self.begin_materialization( ready_through, @@ -6291,10 +6350,18 @@ where if !self.is_shutting_down() && self.transfer.as_ref().is_some_and(|session| { session.target_accepted - && session - .artifacts - .iter() - .all(consensus::ChunkProgress::complete) + && (session + .scan + .as_ref() + .is_some_and(crate::state_transfer::LocalWork::landed) + || session + .seed + .as_ref() + .is_some_and(crate::state_transfer::PrefixSeed::landed) + || session + .artifacts + .iter() + .all(consensus::ChunkProgress::complete)) }) && self.consensus.state_transfer_stage() == consensus::StateTransferStage::Fetching { @@ -6731,6 +6798,16 @@ where } } + /// Whether the next offer round hashes against a pinned plan, instead of + /// returning at once to wait for the pin. + #[must_use] + pub fn transfer_plan_pinned(&self) -> bool { + matches!( + *self.transfer_plan.borrow(), + Some(TransferPlanSlot::Pinned(_)) + ) + } + /// The plan an offer round hashes against, or `None` while the flush that /// has to come before it runs. A mounted owner pins the plan in the I/O /// turn that finds every applied op in its segment, so commits landing @@ -7486,6 +7563,17 @@ where .await; return; } + if self.consumer_group_owner_admitted(&installation) { + Self::send_partition_deny_or_log( + consensus, + message.header(), + IggyError::TransientNotAccepted.as_code(), + "ownership installation is already applied or superseded", + reply.take(), + ) + .await; + return; + } Some(installation) } else { None @@ -8437,15 +8525,12 @@ where let journal_holds_op = self.log.journal().inner.holds_op(header.op); if journal_holds_op { - // Retransmit after downstream flap: durable here but commit - // hasn't caught up. Re-forward + re-ACK so primary's view of - // us is consistent. Both downstream and primary are idempotent - // on duplicate (replica, op). + // Re-ack retransmissions so a lost acknowledgement cannot stall commit. emit_partition_diag( tracing::Level::DEBUG, &PartitionDiagEvent::new( self.diag_ctx(), - "journal already holds prepare, re-forwarding + re-acking", + "journal already holds prepare, re-acking", ) .with_operation(header.operation) .with_op(header.op), @@ -8492,13 +8577,12 @@ where .map(|_| frozen_for_forward.clone()); let consensus = self.consensus(); if consensus.is_follower() && header.op > consensus.sequencer().current_sequence() { - consensus.sequencer().set_sequence(header.op); - consensus.set_last_prepare_checksum(header.checksum); + // Repair can hold this op above a hole, so the head moves only + // along the contiguous run. + self.advance_head_over_journaled_run(); consensus.observe_prepare_timestamp(header.timestamp); } - if let Err(error) = - replicate_frozen_to_next_in_chain(consensus, frozen_for_forward).await - { + if let Err(error) = replicate_frozen_prepare(consensus, frozen_for_forward).await { let is_transport_error = error.is_transport(); emit_partition_diag( if is_transport_error { @@ -8508,7 +8592,7 @@ where }, &PartitionDiagEvent::new( self.diag_ctx(), - "failed to re-forward retransmitted prepare to next in chain", + "failed to enqueue retransmitted prepare for replication", ) .with_operation(header.operation) .with_op(header.op) @@ -8540,32 +8624,28 @@ where .inner .repaired_window_shape(self.consensus().commit_min(), header.op - 1) .complete - && if header.op == 1 { - header.parent == 0 - } else { - self.log - .journal() - .inner - .repair_header(header.op - 1) - .is_some_and(|previous| previous.checksum == header.parent) - }; + && self.chains_to_journaled_predecessor(&header); if header.op != current_op + 1 && !fills_announced_gap { // `sequence` is what separates the two shapes this line covers: // a forward gap (op above the sequencer, the hole the repair // driver closes) and a retransmit of an op this replica already // sequenced. Without it they read identically. - emit_partition_diag( - tracing::Level::WARN, - &PartitionDiagEvent::new( - self.diag_ctx(), - "dropping out-of-order prepare (gap)", - ) - .with_operation(header.operation) - .with_op(header.op) - .with_sequence(current_op), + self.record_prepare_gap(&header, current_op, "dropping out-of-order prepare (gap)"); + // An op at or below the head is a retransmit, which the primary + // sends straight to every backup that has not acknowledged it. + return; + } + // While a transfer re-arm waits, an op above a hole only grows a + // journal that the install wipes. A `StartView` adopted in the wait + // lifts the head above the hole, so the next live op passes the gap + // check. The lowest missing op still lands, so the live stream can + // fill the hole. + if self.transfer_rearm.is_some() && header.op > self.held_through() + 1 { + self.record_prepare_gap( + &header, + current_op, + "dropping prepare above a hole while a transfer re-arm waits", ); - self.prepare_gap_drops - .set(self.prepare_gap_drops.get().saturating_add(1)); return; } } else { @@ -8588,7 +8668,7 @@ where // - above the sequencer: journaling an op the sequencer has not // assigned yet means the next local assignment would collide // with it. Unreachable today (view fences run first, one - // primary per view, the chain stops before the primary), so + // primary per view, fanout targets only backups), so // trip the invariant in debug; in release log loudly and drop // rather than crash a library or corrupt op assignment. if header.op > current_op { @@ -8622,9 +8702,7 @@ where ); } } - // Forward only after apply_replicated_operation journals the prepare. - // The journal and network share the frozen allocation, so the bytes - // retained for repair are exactly the bytes sent downstream. + // The journal and primary fanout share the frozen allocation. let replicated_result = if is_backup && header.operation == Operation::SendMessages { self.append_received_send_messages_to_journal(message).await } else { @@ -8705,10 +8783,18 @@ where if header.op >= current_op { consensus.sequencer().set_sequence(header.op); consensus.set_last_prepare_checksum(header.checksum); + // A filled hole joins the run that repair journaled above it. + if header + .op + .checked_add(1) + .is_some_and(|next| self.log.journal().inner.holds_op(next)) + { + self.advance_head_over_journaled_run(); + } } consensus.observe_prepare_timestamp(header.timestamp); } - if let Err(error) = replicate_frozen_to_next_in_chain(consensus, frozen_for_forward).await { + if let Err(error) = replicate_frozen_prepare(consensus, frozen_for_forward).await { let is_transport_error = error.is_transport(); emit_partition_diag( if is_transport_error { @@ -8718,7 +8804,7 @@ where }, &PartitionDiagEvent::new( self.diag_ctx(), - "failed to replicate prepare to next in chain", + "failed to enqueue prepare for replication", ) .with_operation(header.operation) .with_op(header.op) @@ -8747,6 +8833,65 @@ where self.send_prepare_ok(&header).await; } + /// Whether `header` chains onto the entry this replica holds one op below + /// it, resident or retained for repair. + fn chains_to_journaled_predecessor(&self, header: &PrepareHeader) -> bool { + if header.op == 1 { + header.parent == 0 + } else { + self.log + .journal() + .inner + .repair_header(header.op - 1) + .is_some_and(|previous| previous.checksum == header.parent) + } + } + + fn record_prepare_gap(&self, header: &PrepareHeader, sequence: u64, message: &'static str) { + self.prepare_gap_drops + .set(self.prepare_gap_drops.get().saturating_add(1)); + let now = self.consensus().message_bus().monotonic_micros(); + if now < self.prepare_gap_log_after.get() { + return; + } + self.prepare_gap_log_after + .set(now.saturating_add(PREPARE_GAP_LOG_INTERVAL_MICROS)); + emit_partition_diag( + tracing::Level::WARN, + &PartitionDiagEvent::new(self.diag_ctx(), message) + .with_operation(header.operation) + .with_op(header.op) + .with_sequence(sequence), + ); + } + + /// Move the head along the contiguous run the journal holds above it. + /// + /// DVC advertises the head and elections pick the highest, so a head above + /// a hole can win a view it cannot walk. A hole stops the head, and the op + /// that fills it, from repair or from the live stream, moves the head over + /// every op journaled above it meanwhile. The checksum comes from the + /// journaled header at the new head, so a lower backfill never rewinds the + /// parent the next prepare chains onto. No append observes the timestamps + /// of the passed ops: repair does not, and their live copies arrive below + /// the head. The head's timestamp therefore joins the floor here. + fn advance_head_over_journaled_run(&self) { + let consensus = self.consensus(); + let journal = &self.log.journal().inner; + let Some((head, checksum)) = + repaired_frontier_update(consensus.sequencer().current_sequence(), |op| { + journal.header_by_op(op) + }) + else { + return; + }; + consensus.sequencer().set_sequence(head); + consensus.set_last_prepare_checksum(checksum); + if let Some(header) = journal.header_by_op(head) { + consensus.observe_prepare_timestamp(header.timestamp); + } + } + #[allow(clippy::future_not_send)] pub async fn on_ack(&mut self, message: Message, config: &PartitionsConfig) { if self.fatal.is_some() { @@ -8834,6 +8979,7 @@ where /// replica ask its peers for it. #[allow(clippy::future_not_send)] pub async fn commit_journal(&mut self, config: &PartitionsConfig) { + self.refresh_proof_walk_memory(); if self.transition.is_some() || self.io_active.is_retiring() { self.notify_io(); return; @@ -9183,13 +9329,32 @@ where } else { 0 }; - if (validated.base_offset, validated.base_timestamp) != (expected_offset, header.timestamp) - { + if validated.base_timestamp != header.timestamp { return Err(IggyError::CannotAppendMessage); } - self.append_stamped_messages(message, validated) + if validated.base_offset == expected_offset { + return self + .append_stamped_messages(message, validated) + .await + .map(|journaled| journaled.prepare); + } + // Repair can journal ops above a hole and lift the dirty offset past + // it, so the live prepare that fills the hole starts below that offset. + // Chained to the entry below it, it carries the offsets the primary + // stamped, which repair would journal for this op as well. + let fills_hole = validated.base_offset < expected_offset + && self + .log + .journal() + .inner + .last_op() + .is_some_and(|last| last > header.op) + && self.chains_to_journaled_predecessor(&header); + if !fills_hole { + return Err(IggyError::CannotAppendMessage); + } + self.append_canonical_send_messages(message, validated) .await - .map(|journaled| journaled.prepare) } async fn append_stamped_messages( @@ -9307,6 +9472,40 @@ where .held_through(self.consensus.commit_min()) } + /// The held prefix finds replication gaps. The received prefix inside it + /// follows the same per-operation durability contract as prepare acks. + #[must_use] + pub fn replication_progress(&self) -> Option { + if !self.consensus.is_normal() + || self.consensus.is_transferring() + || self.consensus.has_untrusted_log() + || self.view_fenced() + || self.read_history_is_changing() + { + return None; + } + let held_through = self.held_through(); + let mut held = self.consensus.commit_min(); + let mut received = None; + while held < held_through { + let next = held + 1; + let Some(header) = self.log.journal().inner.repair_header(next) else { + break; + }; + if !self.consensus.view_log_entry_proven(next) { + break; + } + if received.is_none() && !self.prepare_ack_ready(&header) { + received = Some(held); + } + held = next; + } + Some(consensus::BackupPrefix { + held_op: held, + received_op: received.unwrap_or(held), + }) + } + /// Whether an anchored fence holds every entry of its window, so a proof /// now concludes instead of waiting on repair. #[must_use] @@ -9750,6 +9949,10 @@ where } self.offset_reservations_need_resync.set(true); self.truncated_since_scan.set(true); + drop(entries); + if self.log.journal().inner.is_empty() { + self.reset_proof_walk_memory(); + } Ok(removed) } @@ -9927,6 +10130,7 @@ where durable_offset: None, phase: MaterializationPhase::Inspect, save: None, + checksum: None, }); Ok(true) } @@ -10122,8 +10326,8 @@ where .map(|pending| pending.index_span_limit) }); self.log.journal().inner.with_entries(|entries| { - let entries = entries - .get(first_entry..prefix_len) + let entries = (first_entry <= prefix_len && prefix_len <= entries.len()) + .then(|| entries.range(first_entry..prefix_len)) .ok_or(IggyError::CannotWriteToFile)?; let mut backing = 0_usize; let mut file_position = position; @@ -10205,9 +10409,10 @@ where .ok_or(IggyError::CannotWriteToFile)?; let completes = chunk.end_entry == save.chunk.end_entry; let batches = self.log.journal().inner.with_entries(|entries| { - let entries = entries - .get(chunk.first_entry..chunk.end_entry) - .ok_or(IggyError::CannotWriteToFile)?; + let entries = (chunk.first_entry <= chunk.end_entry + && chunk.end_entry <= entries.len()) + .then(|| entries.range(chunk.first_entry..chunk.end_entry)) + .ok_or(IggyError::CannotWriteToFile)?; let mut batches = Vec::with_capacity(chunk.buffers); for entry in entries { if materialization_batch_meta(entry, chunk.persisted_end)?.is_some() { @@ -10368,12 +10573,10 @@ where } /// Evict the committed prefix (the `count` front entries read by - /// `committed_prefix_len`) and reset `journal.info` to reflect only the - /// uncommitted tail left resident, so the next persist threshold counts that - /// tail alone. Call once the prefix is durable, or when there is nothing to - /// persist. The retained tail's accounting is folded from the meta - /// `evict_prefix` surfaced during its re-append, so the tail is not decoded - /// a second time. + /// `committed_prefix_len`) and set the message count and size in + /// `journal.info` to the batches left resident, so the next persist + /// threshold counts that tail alone. Call once the prefix is durable, or + /// when there is nothing to persist. fn evict_committed_prefix(&mut self, count: usize) { if count == 0 { return; @@ -10382,24 +10585,21 @@ where pending.evictable = pending.evictable.max(count); return; } - let retained = self.log.journal().inner.evict_prefix(count); - // Eviction drains the ring and re-appends the retained tail, so the slots - // a suffix snapshot describes move under it without the head or the - // commit point changing. + let persistence = self.persistence.as_deref(); + let journal = &self.log.journal().inner; + journal.evict_prefix_located(count, |op| { + persistence?.segment_reference(op).map(BodyPosition::from) + }); + // Every retained entry moves `count` slots towards the front, so the + // slots a suffix snapshot describes move under it without the head or + // the commit point changing. self.consensus.note_journal_mutation(); - let mut retained_info = JournalInfo::default(); - for (_, meta) in &retained { - if let Some(meta) = meta { - accumulate_committed_info( - &mut retained_info, - meta.base_offset, - meta.base_timestamp, - meta.total_size, - meta.message_count, - ); - } - } - self.log.journal_mut().info = retained_info; + // Appends reject a batch that overflows the count, so it fits. + let messages_count = u32::try_from(journal.resident_messages()).unwrap_or(u32::MAX); + let size = IggyByteSize::from(journal.resident_message_size()); + let info = &mut self.log.journal_mut().info; + info.messages_count = messages_count; + info.size = size; } async fn handle_committed_entries( @@ -11757,7 +11957,14 @@ where index_position: index_writer.position(), index_writer, }; - let frozen_batches = if bodies_written.is_some() { + let checksum = if written == 0 { + self.take_segment_hasher(target.segment_size) + } else { + self.materialization + .as_mut() + .and_then(|pending| pending.checksum.take()) + }; + let frozen_batches = if bodies_written.is_some() && checksum.is_none() { Vec::new() } else { frozen_batches @@ -11775,6 +11982,7 @@ where bodies_written, written, completes, + checksum, }; Ok(Some(job)) } @@ -11839,6 +12047,19 @@ where } target.index_writer.advance(saved_indexes); self.log.segments_mut()[segment_index].size = IggyByteSize::from(size); + if saved == 0 { + if let Some(pending) = self.materialization.as_mut() { + pending.checksum = result.checksum; + } + } else if let Some(hasher) = result.checksum { + self.segment_checksum_cache.borrow_mut().insert( + target.segment_start, + crate::state_transfer::SegmentChecksumMemo::Hashing { + hashed_len: size, + hasher, + }, + ); + } Ok(()) } @@ -11959,6 +12180,7 @@ where self.log.index_writers_mut()[sealed_index] = None; self.log.indexes_mut()[sealed_index] = None; self.log.reset_read_state(sealed_index); + self.stamp_sealed_segment(target.segment_start); debug!( namespace_raw = self.namespace().inner(), sealed_end = target.segment_end, @@ -12091,8 +12313,15 @@ where fn capture_segment_removal(&mut self) -> Option> { let (segment, storage) = self.log.retire_front()?; + self.log + .journal() + .inner + .forget_segment(segment.start_offset); let (messages, indexes) = storage.segment_and_index_paths(); let anchor = self.anchor_cleanup_path(segment.start_offset); + self.segment_checksum_cache + .borrow_mut() + .remove(&segment.start_offset); self.maintenance.removing = Some(segment); self.clear_state_transfer_offer_cache(); Some(crate::PartitionIoJob::RemoveSegment { @@ -12509,14 +12738,63 @@ where } } + /// Bytes that repair journaled above `commit_min` while the view log was + /// unproven. + const fn proof_walk_bytes(&self) -> u64 { + let (commit_min, bytes) = self.proof_walk; + if commit_min == self.consensus().commit_min() { + bytes + } else { + 0 + } + } + + fn refresh_proof_walk_memory(&mut self) { + if !self.consensus.view_log_unproven() || self.proof_walk.0 != self.consensus.commit_min() { + self.reset_proof_walk_memory(); + } + } + + pub(crate) fn reset_proof_walk_memory(&mut self) { + if let Some(memory) = &mut self.proof_walk_memory { + memory.clear(); + } + self.proof_walk = (self.consensus.commit_min(), 0); + } + + /// Whether an unproven repair exceeded its WAL budget or exhausted the + /// shared allocation budget before accepting another frame. + /// + /// The walk fetches ops from the adopted head down, and nothing above + /// `commit_min` commits before it reaches `commit_min`, so every body it + /// fetches stays in memory until then. The WAL budget already caps the + /// prepares that a partition holds above its WAL checkpoint, so the walk + /// can hold as much, plus the one body that crosses it. Past that, state + /// transfer takes the window. Bodies that a later truncation drops still + /// count, so the bound errs toward transfer. The node's allocation budget + /// also applies without a WAL and never admits the body that exceeds it. + fn proof_walk_over_budget(&self) -> bool { + self.consensus().view_log_unproven() + && (self + .proof_walk_memory + .as_ref() + .is_some_and(crate::repair_memory::RepairMemoryReservation::exhausted) + || self + .persistence + .as_ref() + .is_some_and(|persistence| self.proof_walk_bytes() > persistence.capacity())) + } + /// Ingest one repaired prepare: journal + stage it exactly like a live /// replicated op, minus the view-number check, the gap check, and the ack (the /// op is already committed cluster-wide; there is nobody to ack to). The /// commit walk runs at `RepairDone`, after the floor is known. + #[allow(clippy::too_many_lines)] pub async fn apply_repaired_prepare( &mut self, message: Message, ) -> RepairIngest { + self.refresh_proof_walk_memory(); let header = *message.header(); if self.view_truncation_pending() || !self.prepare_history_available(&header) { return RepairIngest::Handled; @@ -12599,12 +12877,30 @@ where self.restart_repair_budget(); return RepairIngest::Handled; } + // A walk past its budget journals nothing more, and `complete_repair` + // hands the window to state transfer. + if self.proof_walk_over_budget() { + return RepairIngest::Handled; + } + let charge = if self.consensus.view_log_unproven() { + message.retained_allocation_size().unwrap_or(usize::MAX) + } else { + 0 + }; + if let Some(memory) = &mut self.proof_walk_memory + && !memory.try_add(charge) + { + return RepairIngest::Handled; + } let applied = if header.operation == Operation::SendMessages { self.append_repaired_send_messages(message).await } else { self.apply_replicated_operation(message).await.map(|_| ()) }; if let Err(error) = applied { + if let Some(memory) = &mut self.proof_walk_memory { + memory.release(charge); + } warn!( target: "iggy.partitions.diag", plane = "partitions", @@ -12619,27 +12915,16 @@ where // its peer serves it again every round, and a refund per serve would // pin the session to that peer. self.restart_repair_budget(); + if self.consensus().view_log_unproven() { + self.proof_walk = ( + self.consensus().commit_min(), + self.proof_walk_bytes() + u64::from(header.size), + ); + } self.consensus.note_repaired_view_header(header, evidence); + self.refresh_proof_walk_memory(); self.persist_repaired_prefix(); - // Advance the sequencer only along the CONTIGUOUS journaled - // frontier. DVC advertises `op = sequencer.current_sequence()` and - // elections pick the max, so bumping straight to a repaired op that - // sits above an unfilled hole would let this replica win a view it - // cannot walk. A dropped frame stalls the frontier here; the stall - // retry refills the hole and the next apply resumes the advance - // (walking over ops that were journaled out of order meanwhile). - // The checksum moves only with the frontier and is read from the - // journal header at the new head: a lower backfill must not rewind - // the parent the next prepare chains onto. - let previous_frontier = self.consensus().sequencer().current_sequence(); - let update = repaired_frontier_update(previous_frontier, |op| { - self.log.journal().inner.header_by_op(op) - }); - if let Some((frontier, frontier_checksum)) = update { - let consensus = self.consensus(); - consensus.sequencer().set_sequence(frontier); - consensus.set_last_prepare_checksum(frontier_checksum); - } + self.advance_head_over_journaled_run(); self.offset_reservations_need_resync.set(true); RepairIngest::Handled } @@ -12656,10 +12941,31 @@ where let Some(mut session) = self.repair else { return RepairConclusion::Done; }; - if !self.consensus().is_normal() || self.consensus().view() != session.view { + if !consensus::repair_session_live(self.consensus()) + || self.consensus().view() != session.view + { self.repair = None; return RepairConclusion::Done; } + if self.proof_walk_over_budget() { + let node_memory_exhausted = self + .proof_walk_memory + .as_ref() + .is_some_and(crate::repair_memory::RepairMemoryReservation::exhausted); + tracing::info!( + target: "iggy.partitions.diag", + plane = "partitions", + namespace_raw = self.namespace().inner(), + walk_bytes = self.proof_walk_bytes(), + node_memory_exhausted, + "unproven repair walk is past its memory budget, so state transfer takes the window" + ); + self.repair = None; + return RepairConclusion::FloorRefused { + floor: self.consensus().commit_min(), + to_op: session.commit_to_op, + }; + } let journal = &self.log.journal().inner; // Apply a resident prefix normally instead of substituting the peer's // floor for operations recovered after boot. @@ -12754,13 +13060,8 @@ where return conclusion; } } - if self - .persistence - .as_ref() - .is_some_and(|persistence| !persistence.is_durable_through(session.fetch_to_op)) - { - return RepairConclusion::InProgress; - } + // Apply verified prefixes before the whole window is durable so a + // checkpoint can reclaim WAL capacity for the remaining repair. let before = self.consensus().commit_min(); self.commit_journal(config).await; let commit_min = self.consensus().commit_min(); @@ -12890,6 +13191,25 @@ where Some(RepairConclusion::InProgress) } + /// Whether another bounded repair response can fit before checkpointing. + /// Unproven histories and unresolved floors must fetch before they can + /// apply a prefix, so WAL backpressure must not stop their proof walks. + #[must_use] + pub fn repair_fetch_ready(&self) -> bool { + self.view_fenced() + || self.consensus.has_untrusted_log() + || self.consensus.view_log_unproven() + || self.consensus.commit_min() >= self.consensus.commit_max() + || self.repair.is_some_and(|session| { + session + .floor + .is_some_and(|floor| floor > self.consensus.commit_min()) + }) + || self.persistence.as_ref().is_none_or(|persistence| { + persistence.has_capacity(journal::partition_journal::PREPARE_BYTES_MAX) + }) + } + /// Where the next pull of the repair window starts: past the resident /// prefix above a floor that still holds the walk, else past the held run /// above `commit_min`. `None` without a session. @@ -12929,6 +13249,33 @@ where Some(from_op) } + /// Highest op held below the hole that the proof walk waits on, for the + /// repair request about to go out, whose window starts above it. A donor + /// serves a window from its bottom up to a disk byte budget, so a window + /// that starts at held ops gets them again on every round and never + /// reaches the hole. Records the hole with it for + /// [`Self::next_proof_walk_pull`]. + pub fn note_proof_walk_request(&self) -> Option { + let hole = self.consensus().view_log_repair_op()?; + let held_below = self.log.journal().inner.held_below(hole); + self.proof_walk_requested.set((hole, held_below)); + held_below + } + + /// The unproven hole to pull again right after a `RepairDone` when the + /// walk gained ops below it since the last request, or `None`. The next + /// window then starts above them. A round that brought nothing leaves the + /// hole to the retry timer. + #[must_use] + pub fn next_proof_walk_pull(&self) -> Option { + let hole = self + .consensus() + .view_log_repair_op() + .filter(|&hole| !self.consensus().view_log_entry_proven(hole))?; + let walked = (hole, self.log.journal().inner.held_below(hole)); + (walked != self.proof_walk_requested.get()).then_some(hole) + } + /// The lowest resident `SendMessages` batch in `(floor, commit_to_op]`: /// where the repair window begins in offset space. /// @@ -13007,16 +13354,26 @@ where if !self.prepare_history_available(message.header()) { return Err(IggyError::HistoryUnavailable); } + let batch = decode_prepare_slice(message.as_slice())?.header; + self.append_canonical_send_messages(message, batch).await?; + Ok(()) + } - let (base_offset, base_timestamp, total_size, message_count) = { - let batch = decode_prepare_slice(message.as_slice())?; - ( - batch.header.base_offset, - batch.header.base_timestamp, - batch.header.total_size() as u64, - batch.message_count(), - ) - }; + /// Journal a `SendMessages` prepare at the offsets its batch carries: a + /// repaired op, or a live op that fills a hole below ops repair journaled. + /// The dirty offset only grows, so a fill below it leaves it in place. The + /// caller holds the write lock. + async fn append_canonical_send_messages( + &mut self, + message: Message, + batch: BatchHeader, + ) -> Result, IggyError> { + let (base_offset, base_timestamp, total_size, message_count) = ( + batch.base_offset, + batch.base_timestamp, + batch.total_size() as u64, + batch.message_count, + ); if message_count == 0 { return Err(IggyError::InvalidCommand); } @@ -13052,14 +13409,14 @@ where journal_info.max_timestamp = journal_info.max_timestamp.max(base_timestamp); let frozen = message.into_frozen(); - self.journal_append(frozen).await?; + self.journal_append(frozen.clone()).await?; self.note_append_live(); self.dirty_offset .store(dirty.max(last_offset), Ordering::Relaxed); self.log.segments_mut()[segment_index].current_position = next_position; self.log.journal_mut().info = journal_info; - Ok(()) + Ok(frozen) } /// Retirement requests name sessions rather than histories; their prepares @@ -13111,6 +13468,7 @@ where // consumer-offset ops (via `apply_replicated_operation`) append // to that journal before `send_prepare_ok` fires, so every op // that reaches here is journal-backed and ACKs as durable. + self.record_acknowledged_op(header.op); send_prepare_ok_common(self.consensus(), header, true).await } @@ -13269,7 +13627,7 @@ fn peek_op(entry: &Frozen<4096>) -> u64 { .op } -fn materialization_batch_meta( +pub fn materialization_batch_meta( entry: &Frozen<4096>, persisted_end: Option, ) -> Result, IggyError> { @@ -13620,8 +13978,8 @@ mod tests { use consensus::LocalPipeline; use iggy_binary_protocol::batch::BATCH_MESSAGE_HEADER_SIZE; use iggy_binary_protocol::{ - Command, CommitHeader, ConsensusHeader, ReplyHeader, StartViewHeader, WireConsumer, - WireEncode, WireName, + Command, CommitHeader, CommitReplyHeader, ConsensusHeader, HEADER_SIZE, ReplyHeader, + StartViewHeader, WireConsumer, WireEncode, WireName, }; use iggy_common::Aes256GcmEncryptor; use iggy_common::Durability::{Persisted, Replicated}; @@ -13846,6 +14204,72 @@ mod tests { assert_eq!(offsets.next_offset, 3); } + /// A tail whose bodies the ring released to segments does not keep a + /// plan pinned. The receiver would repair it from disk, which moves far + /// less per round than repair from memory, so the source re-pins once the + /// tail of the plan leaves memory. + #[compio::test] + async fn given_a_transfer_pin_whose_tail_left_memory_when_hashing_finishes_should_request_a_fresh_plan() + { + let root = tempfile::tempdir().unwrap(); + let (mut partition, _, config, mut parent) = + partition_with_committed_send(root.path(), iggy_common::Durability::Replicated, 3) + .await; + partition.set_io_notifier( + Rc::new(|_, _| {}), + crate::largest_legal_job_charge().unwrap(), + ); + let journal = &partition.log.journal().inner; + journal.set_repair_retention(true); + journal.set_ring_caps(crate::EVICTED_RING_CAPACITY, 1); + assert!(matches!( + partition.state_transfer_offer(&config).await, + Err(crate::state_transfer::PartitionTransferUnavailable::FlushPending) + )); + settle_partition_io(&mut partition, &config).await; + + let persistence = Rc::clone(partition.persistence.as_ref().unwrap()); + for op in 2..=3 { + let prepare = checksummed_segment_prepare(op, parent, op - 1, b"tail"); + let header = *prepare.header(); + parent = header.checksum; + persistence + .append(prepare.clone().into_frozen(), false) + .unwrap(); + assert!(persistence.start()); + Rc::clone(&persistence).run().await; + partition.consensus.with_pipeline_mut(|pipeline| { + pipeline.push(PipelineEntry::new(header)); + }); + partition + .append_repaired_send_messages(prepare) + .await + .unwrap(); + partition.consensus.advance_commit_max(op); + partition.commit_journal(&config).await; + settle_partition_io(&mut partition, &config).await; + } + assert_eq!(partition.consensus().commit_min(), 3); + assert!( + partition.log.journal().inner.evicted_header(2).is_some(), + "the byte cap released the body of op 2 to its segment" + ); + + let offer = partition.state_transfer_offer(&config).await; + assert!( + matches!( + offer, + Err(crate::state_transfer::PartitionTransferUnavailable::FlushPending) + ), + "{:?}", + offer.map(|offer| offer.commit_op) + ); + assert!(matches!( + *partition.transfer_plan.borrow(), + Some(TransferPlanSlot::Wanted) + )); + } + #[compio::test] async fn given_ops_committing_during_a_transfer_when_the_receiver_installs_should_land_on_the_pinned_op() { @@ -13901,7 +14325,7 @@ mod tests { let mut bytes = std::fs::read(&segment.log_path).unwrap(); bytes.truncate(usize::try_from(segment.entry.len).unwrap()); let staged = receiver - .spill_transfer_segment(&segment.entry, bytes) + .spill_transfer_segment(&segment.entry, bytes, None) .await .unwrap(); receiver @@ -13956,6 +14380,873 @@ mod tests { assert!(!partition.maintenance.transfer_flush); } + /// Removes every segment log of `partition`, then builds an offer. The + /// offer can only take each stamp from the checksum memo, and each stamp + /// must equal the manifest checksum of the bytes that the file held. + async fn assert_offer_reads_no_segment_file(partition: &mut IggyPartition) { + let config = repair_config(); + let expected = segment_artifacts_on_disk(partition); + let directory = partition.partition_dir().unwrap(); + for artifact in &expected { + std::fs::remove_file(format!("{directory}/{:020}.log", artifact.frontier)).unwrap(); + } + + assert!(matches!( + partition.state_transfer_offer(&config).await, + Err(PartitionTransferUnavailable::FlushPending) + )); + settle_partition_io(partition, &config).await; + let offer = partition.state_transfer_offer(&config).await.unwrap(); + + let offered: Vec<_> = offer.segments.iter().map(|segment| segment.entry).collect(); + assert_eq!(offered, expected); + } + + /// A save that the I/O byte cap splits into pieces feeds the memo the + /// bytes of every piece, in file order. + #[compio::test] + async fn given_saves_split_into_pieces_when_building_an_offer_should_not_read_the_segments() { + const OPS: u64 = 4; + let config = repair_config(); + let (directory, mut partition) = + Box::pin(disk_poll_partition_with_replicas(&config, 3)).await; + let job_charge_max = crate::largest_legal_job_charge().unwrap(); + partition.set_io_notifier(Rc::new(|_, _| {}), job_charge_max); + for op in 1..=OPS { + journal_send_batch(&mut partition, op).await; + } + let piece_charge = partition + .log + .journal() + .inner + .with_entries(|entries| { + crate::io::materialization_allocation_charge( + std::slice::from_ref(&entries[0]), + 1, + crate::iggy_index::IGGY_INDEX_SIZE, + ) + }) + .unwrap(); + partition.io_bytes_max.set(piece_charge); + partition.consensus().advance_commit_max(OPS); + settle_partition_io(&mut partition, &config).await; + partition.io_bytes_max.set(job_charge_max); + assert!(partition.fatal().is_none()); + assert_eq!( + std::fs::metadata(directory.path().join(format!("{:020}.index", 0))) + .unwrap() + .len(), + crate::iggy_index::IGGY_INDEX_SIZE as u64, + "one save must write every batch, so the cap split it into pieces" + ); + + Box::pin(assert_offer_reads_no_segment_file(&mut partition)).await; + } + + /// A save whose bodies the partition WAL already wrote feeds the memo the + /// same bytes that the segment file holds. The group has peers, so its + /// journal keeps the committed prepare that an offer carries. + #[compio::test] + async fn given_segment_bodies_written_by_the_wal_when_building_an_offer_should_not_read_them() { + const REPLICAS: u8 = 3; + for durability in [ + iggy_common::Durability::Replicated, + iggy_common::Durability::Persisted, + ] { + let root = tempfile::tempdir().unwrap(); + let (mut partition, _replies, _config, _parent) = + partition_with_committed_send(root.path(), durability, REPLICAS).await; + assert!( + partition + .log + .messages_writers() + .last() + .is_some_and(Option::is_none), + "{durability:?}: the segment must have no writer, so the WAL wrote its bodies" + ); + partition.set_io_notifier( + Rc::new(|_, _| {}), + crate::largest_legal_job_charge().unwrap(), + ); + + Box::pin(assert_offer_reads_no_segment_file(&mut partition)).await; + } + } + + /// One-message batches each segment of [`partition_with_sealed_segments`] + /// holds before it rolls. + const BATCHES_PER_SEGMENT: u64 = 2; + + /// `count` one-message records. Each call gives its messages new random + /// ids, so two calls give records of the same lengths but other bytes. + fn segment_records(count: u64) -> Vec> { + (0..count) + .map(|_| build_segment_record(IggyNamespace::new(1, 1, 0), 0)) + .collect() + } + + /// A primary on disk whose segments roll every [`BATCHES_PER_SEGMENT`] + /// batches, with one op per record committed and written to its segments. + async fn partition_with_sealed_segments( + records: &[Vec], + ) -> (tempfile::TempDir, IggyPartition) { + let config = repair_config(); + let (directory, mut partition) = + Box::pin(disk_poll_partition_with_replicas(&config, 3)).await; + let batch_len = build_segment_record(partition.namespace(), 0).len() as u64; + let segment_size = IggyByteSize::from(BATCHES_PER_SEGMENT * batch_len); + partition.runtime_options.segment_size = Some(segment_size); + partition.log.active_segment_mut().max_size = segment_size; + partition.set_io_notifier( + Rc::new(|_, _| {}), + crate::largest_legal_job_charge().unwrap(), + ); + Box::pin(commit_records(&mut partition, 1, records)).await; + assert_eq!(partition.consensus().commit_min(), records.len() as u64); + (directory, partition) + } + + /// Journals `records` as the ops from `first_op` on, and commits and + /// writes each one before the next. + async fn commit_records( + partition: &mut IggyPartition, + first_op: u64, + records: &[Vec], + ) { + let config = repair_config(); + for (op, record) in (first_op..).zip(records) { + journal_send_record(partition, op, record).await; + partition.consensus().advance_commit_max(op); + settle_partition_io(partition, &config).await; + } + } + + /// The offer `partition` serves once the I/O turn pinned its plan. + async fn serve_offer( + partition: &mut IggyPartition, + ) -> Rc { + let config = repair_config(); + assert!(matches!( + partition.state_transfer_offer(&config).await, + Err(PartitionTransferUnavailable::FlushPending) + )); + settle_partition_io(partition, &config).await; + partition.state_transfer_offer(&config).await.unwrap() + } + + /// The manifest entry of each segment of `partition`, read from its file. + fn segment_artifacts_on_disk( + partition: &IggyPartition, + ) -> Vec { + let directory = partition.partition_dir().unwrap(); + partition + .log + .segments() + .iter() + .map(|segment| { + let bytes = + std::fs::read(format!("{directory}/{:020}.log", segment.start_offset)).unwrap(); + assert_eq!(bytes.len() as u64, segment.size.as_bytes_u64()); + consensus::StateArtifact::for_bytes( + consensus::state_manifest::artifact_kind::SEGMENT_LOG, + segment.start_offset, + &bytes, + ) + }) + .collect() + } + + /// The append feeds the checksum memo of each segment it writes, so the + /// offer takes every stamp from memory and reads no segment file. The test + /// removes the files before the build to prove that. The stamps must still + /// equal the manifest checksum of the bytes that the files held. + #[compio::test] + async fn given_segments_written_by_this_process_when_building_an_offer_should_not_read_them() { + const OPS: u64 = 5; + let (directory, mut partition) = + Box::pin(partition_with_sealed_segments(&segment_records(OPS))).await; + let expected = segment_artifacts_on_disk(&partition); + assert_eq!(expected.len(), 3, "two sealed segments and the active one"); + for artifact in &expected { + std::fs::remove_file( + directory + .path() + .join(format!("{:020}.log", artifact.frontier)), + ) + .unwrap(); + } + + let offer = Box::pin(serve_offer(&mut partition)).await; + + assert_eq!(offer.commit_op, OPS); + let offered: Vec<_> = offer.segments.iter().map(|segment| segment.entry).collect(); + assert_eq!(offered, expected); + } + + /// Ops the receivers of the reuse tests commit: two sealed segments and + /// one batch in the active segment. + const RECEIVER_OPS: u64 = 5; + /// Ops the origin of the reuse tests commits: three sealed segments and + /// one batch in the active segment. + const ORIGIN_OPS: u64 = 7; + + /// Runs a reuse scan the way a session does: in a detached task, and then + /// accepted on the pump. Returns what the pump accepted. + async fn reuse_scan( + receiver: &mut IggyPartition, + manifest: &[consensus::StateArtifact], + ) -> Vec<(u32, crate::state_transfer::StagedSegmentMeta)> { + let scan = receiver.begin_reuse_scan(manifest); + let _ = receiver.staging_work_ended.take().unwrap().await; + let scanned = scan.result.borrow_mut().take().unwrap(); + receiver.accept_reuse_scan(scanned) + } + + /// Installs `offer` the way the shard does: the reuse scan first, then a + /// spill of each segment that the scan did not adopt. Returns the manifest + /// indexes that the scan adopted. + async fn install_offer_with_reuse( + receiver: &mut IggyPartition, + offer: &crate::state_transfer::PartitionStateTransferOffer, + ) -> Vec { + let mut staged = vec![None; offer.segments.len()]; + let mut adopted = Vec::new(); + for (index, meta) in reuse_scan(receiver, &offer.manifest()).await { + adopted.push(index); + staged[index as usize] = Some(meta); + } + for (slot, source) in staged.iter_mut().zip(&offer.segments) { + if slot.is_none() { + let mut bytes = std::fs::read(&source.log_path).unwrap(); + bytes.truncate(usize::try_from(source.entry.len).unwrap()); + *slot = Some( + receiver + .spill_transfer_segment(&source.entry, bytes, None) + .await + .unwrap(), + ); + } + } + receiver + .queue_state_transfer_install( + offer.commit_op, + staged.into_iter().map(Option::unwrap).collect(), + &offer.offsets.1, + 0, + ) + .unwrap(); + Box::pin(finish_install_io(receiver)).await.unwrap(); + adopted + } + + /// The offered bytes of each segment of `offer`. + fn offered_segment_bytes( + offer: &crate::state_transfer::PartitionStateTransferOffer, + ) -> Vec> { + offer + .segments + .iter() + .map(|source| { + let mut bytes = std::fs::read(&source.log_path).unwrap(); + bytes.truncate(usize::try_from(source.entry.len).unwrap()); + bytes + }) + .collect() + } + + /// The bytes of each segment file in the partition directory, in offset + /// order. + fn segment_file_bytes(directory: &Path) -> Vec> { + let mut paths: Vec<_> = std::fs::read_dir(directory) + .unwrap() + .map(|entry| entry.unwrap().path()) + .filter(|path| path.extension().is_some_and(|extension| extension == "log")) + .collect(); + paths.sort(); + paths + .iter() + .map(|path| std::fs::read(path).unwrap()) + .collect() + } + + #[cfg(target_os = "linux")] + fn log_inode(directory: &Path, start_offset: u64) -> u64 { + std::fs::metadata(directory.join(format!("{start_offset:020}.log"))) + .unwrap() + .ino() + } + + /// A receiver that already holds sealed segments of an offer links them to + /// the staging names instead of pulling them. The install then removes the + /// old names, sweeps the other staging files and renames the links, so the + /// installed sealed segments are the files that the receiver wrote. The + /// install also stamps every installed segment with its checksum, which + /// lets the next transfer reuse them too. + #[cfg(target_os = "linux")] + #[compio::test] + async fn given_receiver_holding_sealed_segments_when_installing_an_offer_should_reuse_them_without_a_copy() + { + let records = segment_records(ORIGIN_OPS); + let (_origin_directory, mut origin) = + Box::pin(partition_with_sealed_segments(&records)).await; + let held = &records[..usize::try_from(RECEIVER_OPS).unwrap()]; + let (directory, mut receiver) = Box::pin(partition_with_sealed_segments(held)).await; + let offer = Box::pin(serve_offer(&mut origin)).await; + let sealed = [0, BATCHES_PER_SEGMENT]; + let written = sealed.map(|start_offset| log_inode(directory.path(), start_offset)); + + let adopted = Box::pin(install_offer_with_reuse(&mut receiver, &offer)).await; + + assert_eq!( + adopted, + [0, 1], + "the two sealed segments that the receiver holds" + ); + assert_eq!( + sealed.map(|start_offset| log_inode(directory.path(), start_offset)), + written, + "the installed sealed segments are the files that the receiver wrote" + ); + assert_eq!( + segment_file_bytes(directory.path()), + offered_segment_bytes(&offer) + ); + assert!( + std::fs::read_dir(directory.path()) + .unwrap() + .all(|entry| !entry + .unwrap() + .path() + .to_string_lossy() + .ends_with(".staging")), + "the install renamed or swept every staging name" + ); + assert_eq!(receiver.consensus().commit_min(), ORIGIN_OPS); + let memos = receiver.segment_checksum_cache.borrow(); + for entry in offer.segments.iter().map(|source| source.entry) { + assert_eq!( + memos + .get(&entry.frontier) + .and_then(|memo| memo.checksum_of(entry.len)), + Some(entry.checksum), + "segment {} keeps its installed checksum", + entry.frontier + ); + } + } + + #[compio::test] + async fn given_cached_sealed_segment_corruption_when_installing_should_fetch_verified_bytes() { + let records = segment_records(ORIGIN_OPS); + let (_origin_directory, mut origin) = + Box::pin(partition_with_sealed_segments(&records)).await; + let (directory, mut receiver) = Box::pin(partition_with_sealed_segments( + &records[..usize::try_from(RECEIVER_OPS).unwrap()], + )) + .await; + let offer = Box::pin(serve_offer(&mut origin)).await; + let entry = offer.segments[0].entry; + let path = directory.path().join(format!("{:020}.log", entry.frontier)); + let mut corrupted = std::fs::read(&path).unwrap(); + *corrupted.last_mut().unwrap() ^= 1; + std::fs::write(&path, &corrupted).unwrap(); + assert_eq!(corrupted.len() as u64, entry.len); + assert_eq!( + receiver.segment_checksum_cache.borrow()[&entry.frontier].checksum_of(entry.len), + Some(entry.checksum), + "the append checksum still describes the original bytes" + ); + + let adopted = Box::pin(install_offer_with_reuse(&mut receiver, &offer)).await; + + assert_eq!(adopted, [1], "the corrupt sealed segment must be fetched"); + assert_eq!( + segment_file_bytes(directory.path()), + offered_segment_bytes(&offer) + ); + assert_eq!(receiver.consensus().commit_min(), ORIGIN_OPS); + } + + #[compio::test] + async fn given_corrupt_sealed_index_when_reusing_should_rebuild_from_verified_log() { + let records = segment_records(ORIGIN_OPS); + let (_origin_directory, mut origin) = + Box::pin(partition_with_sealed_segments(&records)).await; + let (directory, mut receiver) = Box::pin(partition_with_sealed_segments( + &records[..usize::try_from(RECEIVER_OPS).unwrap()], + )) + .await; + let offer = Box::pin(serve_offer(&mut origin)).await; + let source = &offer.segments[0]; + let expected = origin + .spill_transfer_segment( + &source.entry, + std::fs::read(&source.log_path).unwrap(), + None, + ) + .await + .unwrap(); + let expected_index = std::fs::read(&expected.index_staging).unwrap(); + let index_path = directory + .path() + .join(format!("{:020}.index", source.entry.frontier)); + let mut corrupted = std::fs::read(&index_path).unwrap(); + corrupted.fill(u8::MAX); + std::fs::write(&index_path, &corrupted).unwrap(); + + let adopted = Box::pin(install_offer_with_reuse(&mut receiver, &offer)).await; + + assert_eq!(adopted, [0, 1], "valid logs remain reusable"); + assert_eq!(std::fs::read(index_path).unwrap(), expected_index); + assert_eq!( + segment_file_bytes(directory.path()), + offered_segment_bytes(&offer) + ); + } + + #[compio::test] + async fn given_pending_reuse_scan_when_tearing_down_should_wait_for_its_task() { + let (_directory, mut receiver) = Box::pin(partition_with_sealed_segments( + &segment_records(RECEIVER_OPS), + )) + .await; + let manifest = Box::pin(serve_offer(&mut receiver)).await.manifest(); + let (release, previous) = futures::channel::oneshot::channel::<()>(); + receiver.staging_work_ended = Some(previous); + let scan = receiver.begin_reuse_scan(&manifest); + let mut drain = Box::pin(receiver.capture_teardown().drain()); + + assert!( + futures::poll!(&mut drain).is_pending(), + "teardown must wait even before the detached scan is first polled" + ); + assert!(scan.stop()); + assert!( + futures::poll!(&mut drain).is_pending(), + "cancellation does not prove that the file task has settled" + ); + drop(release); + drain.await.unwrap(); + assert!( + scan.result.borrow().is_some(), + "the scan finished before teardown" + ); + } + + #[compio::test] + async fn given_a_pending_spill_when_tearing_down_should_wait_for_its_task() { + let (directory, mut receiver) = Box::pin(partition_with_sealed_segments(&segment_records( + RECEIVER_OPS, + ))) + .await; + let offer = Box::pin(serve_offer(&mut receiver)).await; + let source = &offer.segments[0]; + let bytes = std::fs::read(&source.log_path).unwrap(); + let (release, previous) = futures::channel::oneshot::channel::<()>(); + receiver.staging_work_ended = Some(previous); + let spill = receiver.begin_transfer_segment_spill(&source.entry, bytes, None); + let mut drain = Box::pin(receiver.capture_teardown().drain()); + + assert!(futures::poll!(&mut drain).is_pending()); + assert!(spill.stop()); + assert!( + futures::poll!(&mut drain).is_pending(), + "cancellation must not release the staging writer before it settles" + ); + drop(release); + drain.await.unwrap(); + + assert!(matches!( + spill.result.borrow_mut().take().unwrap(), + Err(crate::state_transfer::SpillError::Cancelled) + )); + assert!( + !directory + .path() + .join("00000000000000000000.log.staging") + .exists(), + "the cancelled task must not write after teardown" + ); + } + + #[compio::test] + async fn given_an_abandoned_spill_when_a_reuse_scan_starts_should_wait_for_the_writer() { + let (_directory, mut receiver) = Box::pin(partition_with_sealed_segments( + &segment_records(RECEIVER_OPS), + )) + .await; + let offer = Box::pin(serve_offer(&mut receiver)).await; + let source = &offer.segments[0]; + let bytes = std::fs::read(&source.log_path).unwrap(); + let (release, previous) = futures::channel::oneshot::channel::<()>(); + receiver.staging_work_ended = Some(previous); + let spill = receiver.begin_transfer_segment_spill(&source.entry, bytes.clone(), None); + assert!(spill.stop()); + let scan = receiver.begin_reuse_scan(&offer.manifest()); + server_common::yield_to_reactor().await; + assert!( + !scan.landed(), + "a replacement scan must not overtake the abandoned staging writer" + ); + + drop(release); + let _ = receiver.staging_work_ended.take().unwrap().await; + + assert!(matches!( + spill.result.borrow_mut().take().unwrap(), + Err(crate::state_transfer::SpillError::Cancelled) + )); + let reused = receiver.accept_reuse_scan(scan.result.borrow_mut().take().unwrap()); + let staged = &reused.iter().find(|(index, _)| *index == 0).unwrap().1; + assert_eq!(std::fs::read(&staged.log_staging).unwrap(), bytes); + } + + #[compio::test] + async fn given_singleton_materialization_when_writing_should_skip_transfer_checksums() { + let config = repair_config(); + let (directory, mut partition) = Box::pin(disk_poll_partition(&config)).await; + let record = build_segment_record(partition.namespace(), 0); + Box::pin(commit_records( + &mut partition, + 1, + std::slice::from_ref(&record), + )) + .await; + + assert_eq!(partition.consensus().commit_min(), 1); + assert_eq!(segment_file_bytes(directory.path())[0].len(), record.len()); + assert!( + partition.segment_checksum_cache.borrow().is_empty(), + "a singleton has no state-transfer peer to serve" + ); + } + + /// A sealed segment takes no more appends, so its memo keeps only the + /// stamp, whether the appends or an offer read hashed it. The active + /// segment keeps its hasher for the bytes that it gains. + #[compio::test] + async fn given_sealed_segments_when_their_memos_cover_them_should_keep_no_hasher() { + let (_directory, mut partition) = Box::pin(partition_with_sealed_segments( + &segment_records(RECEIVER_OPS), + )) + .await; + let active = partition.log.active_segment().start_offset; + let hashing = |partition: &IggyPartition| -> Vec { + partition + .segment_checksum_cache + .borrow() + .iter() + .filter(|(_, memo)| { + matches!( + memo, + crate::state_transfer::SegmentChecksumMemo::Hashing { .. } + ) + }) + .map(|(start_offset, _)| *start_offset) + .collect() + }; + assert_eq!(hashing(&partition), [active], "after the appends"); + + partition.segment_checksum_cache.borrow_mut().clear(); + Box::pin(serve_offer(&mut partition)).await; + + assert_eq!(hashing(&partition), [active], "after an offer read"); + assert_eq!( + partition.segment_checksum_cache.borrow().len(), + partition.log.segments().len() + ); + } + + /// A receiver links every sealed segment of an offer that it holds, however + /// many it holds, so the transfer pulls none of them. + #[cfg(unix)] + #[compio::test] + async fn given_many_held_sealed_segments_when_installing_an_offer_should_link_every_one() { + const SEALED: u64 = 12; + let records = segment_records(SEALED * BATCHES_PER_SEGMENT + 1); + let (_origin_directory, mut origin) = + Box::pin(partition_with_sealed_segments(&records)).await; + let (_directory, mut receiver) = Box::pin(partition_with_sealed_segments(&records)).await; + let offer = Box::pin(serve_offer(&mut origin)).await; + + let adopted = Box::pin(install_offer_with_reuse(&mut receiver, &offer)).await; + + assert_eq!( + adopted, + (0..u32::try_from(SEALED).unwrap()).collect::>(), + "every held sealed segment" + ); + } + + #[cfg(target_os = "linux")] + #[compio::test] + async fn given_a_partial_reuse_scan_when_an_offer_advances_should_keep_verified_files() { + for spill_other in [false, true] { + let records = segment_records(ORIGIN_OPS + 1); + let (_origin_directory, mut origin) = + Box::pin(partition_with_sealed_segments(&records)).await; + let (directory, mut receiver) = Box::pin(partition_with_sealed_segments( + &records[..usize::try_from(RECEIVER_OPS).unwrap()], + )) + .await; + let previous = Box::pin(serve_offer(&mut receiver)).await.manifest(); + let scan = receiver.begin_reuse_scan(&previous); + let _ = receiver.staging_work_ended.take().unwrap().await; + let mut scanned = scan.result.borrow_mut().take().unwrap(); + assert!(scanned.reused.len() > 1); + // Model a deadline between artifacts using files verified by the real scan. + for (_, unused) in scanned.reused.split_off(1) { + std::fs::remove_file(unused.log_staging).unwrap(); + std::fs::remove_file(unused.index_staging).unwrap(); + } + let first = scanned.reused[0].1.clone(); + receiver.accept_reuse_scan(scanned); + let index = std::fs::File::open(&first.index_staging).unwrap(); + let verified_inode = index.metadata().unwrap().ino(); + let offer = Box::pin(serve_offer(&mut origin)).await; + let manifest = offer.manifest(); + assert_ne!(manifest, previous, "the source kept accepting writes"); + let stopped = receiver.begin_reuse_scan(&manifest); + assert!(stopped.stop()); + let _ = receiver.staging_work_ended.take().unwrap().await; + receiver.accept_reuse_scan(stopped.result.borrow_mut().take().unwrap()); + if spill_other { + let source = offer.segments.last().unwrap(); + let mut bytes = std::fs::read(&source.log_path).unwrap(); + bytes.truncate(usize::try_from(source.entry.len).unwrap()); + receiver + .spill_transfer_segment(&source.entry, bytes, None) + .await + .unwrap(); + } + + let reused = reuse_scan(&mut receiver, &manifest).await; + + assert!(reused.iter().any(|(position, _)| *position == 0)); + assert_eq!( + std::fs::metadata(&first.index_staging).unwrap().ino(), + verified_inode, + "spill_other={spill_other}: verified staging must not be rebuilt on every retry" + ); + drop(index); + Box::pin(install_offer_with_reuse(&mut receiver, &offer)).await; + assert_eq!( + segment_file_bytes(directory.path()), + offered_segment_bytes(&offer) + ); + } + } + + #[compio::test] + async fn given_verified_staging_when_a_spill_replaces_it_should_revalidate_for_an_older_offer() + { + let (_directory, mut receiver) = Box::pin(partition_with_sealed_segments( + &segment_records(RECEIVER_OPS), + )) + .await; + let own = Box::pin(serve_offer(&mut receiver)).await; + reuse_scan(&mut receiver, &own.manifest()).await; + let (_origin_directory, mut origin) = + Box::pin(partition_with_sealed_segments(&segment_records(ORIGIN_OPS))).await; + let offer = Box::pin(serve_offer(&mut origin)).await; + let source = &offer.segments[0]; + assert_ne!(source.entry.checksum, own.segments[0].entry.checksum); + receiver + .spill_transfer_segment( + &source.entry, + std::fs::read(&source.log_path).unwrap(), + None, + ) + .await + .unwrap(); + + let reused = reuse_scan(&mut receiver, &own.manifest()).await; + + let first = &reused.iter().find(|(index, _)| *index == 0).unwrap().1; + assert_eq!( + std::fs::read(&first.log_staging).unwrap(), + std::fs::read(&own.segments[0].log_path).unwrap(), + "an older offer must recover its verified bytes after a staging replacement" + ); + } + + /// A scan that is asked to stop before its first entry stages nothing and + /// leaves no memo, so the next scan of the same segment set runs whole. + #[cfg(unix)] + #[compio::test] + async fn given_a_stopped_reuse_scan_when_it_lands_should_reuse_nothing_and_leave_no_memo() { + let (_directory, mut receiver) = Box::pin(partition_with_sealed_segments( + &segment_records(RECEIVER_OPS), + )) + .await; + let manifest = Box::pin(serve_offer(&mut receiver)).await.manifest(); + let scan = receiver.begin_reuse_scan(&manifest); + assert!(scan.stop()); + + let _ = receiver.staging_work_ended.take().unwrap().await; + + let scanned = scan.result.borrow_mut().take().unwrap(); + assert_eq!(scanned.reused, Vec::new()); + assert!(scanned.memoized.is_empty(), "a stopped scan saw no entry"); + assert_eq!(receiver.accept_reuse_scan(scanned), Vec::new()); + assert!(receiver.reuse_scan_memo.borrow().is_none()); + assert_eq!( + reuse_scan(&mut receiver, &manifest).await.len(), + 2, + "the next scan links both held sealed segments" + ); + } + + /// A link that fails leaves the staging names as they were, so the staged + /// copy of an earlier attempt still serves the transfer. + #[cfg(target_os = "linux")] + #[compio::test] + async fn given_a_failing_link_when_a_staged_copy_exists_should_adopt_the_copy() { + let (directory, mut receiver) = Box::pin(partition_with_sealed_segments(&segment_records( + RECEIVER_OPS, + ))) + .await; + let offer = Box::pin(serve_offer(&mut receiver)).await; + let held = &offer.segments[0]; + receiver + .spill_transfer_segment(&held.entry, std::fs::read(&held.log_path).unwrap(), None) + .await + .unwrap(); + let staged = directory + .path() + .join(format!("{:020}.log.staging", held.entry.frontier)); + let copy = std::fs::metadata(&staged).unwrap().ino(); + // A directory at the scratch name fails the link. + std::fs::create_dir( + directory + .path() + .join(format!("{:020}.log.link.staging", held.entry.frontier)), + ) + .unwrap(); + + let adopted = reuse_scan(&mut receiver, &offer.manifest()).await; + + assert!( + adopted + .iter() + .any(|(_, meta)| meta.start_offset == held.entry.frontier && !meta.linked) + ); + assert_eq!( + std::fs::metadata(&staged).unwrap().ino(), + copy, + "the staging name still names the staged copy" + ); + } + + /// A spill gives a staging name that links a live segment a new file. + /// Writing through the link would change the live segment. + #[cfg(target_os = "linux")] + #[compio::test] + async fn given_a_linked_staging_name_when_spilling_other_bytes_should_keep_the_live_segment() { + let (directory, mut receiver) = Box::pin(partition_with_sealed_segments(&segment_records( + RECEIVER_OPS, + ))) + .await; + let own = Box::pin(serve_offer(&mut receiver)).await; + reuse_scan(&mut receiver, &own.manifest()).await; + let paths = ["log", "index"].map(|kind| directory.path().join(format!("{:020}.{kind}", 0))); + let held = paths.clone().map(|path| std::fs::read(path).unwrap()); + let staged_log = directory.path().join(format!("{:020}.log.staging", 0)); + assert_eq!( + std::fs::metadata(&staged_log).unwrap().ino(), + log_inode(directory.path(), 0), + "the scan linked the first segment" + ); + let (_origin_directory, mut origin) = + Box::pin(partition_with_sealed_segments(&segment_records(ORIGIN_OPS))).await; + let offer = Box::pin(serve_offer(&mut origin)).await; + let source = &offer.segments[0]; + + receiver + .spill_transfer_segment( + &source.entry, + std::fs::read(&source.log_path).unwrap(), + None, + ) + .await + .unwrap(); + + assert!( + paths.map(|path| std::fs::read(path).unwrap()) == held, + "the live segment keeps its bytes" + ); + assert_ne!( + std::fs::metadata(&staged_log).unwrap().ino(), + log_inode(directory.path(), 0) + ); + } + + /// A sealed segment with the base offset and the length of an offered one + /// but other bytes fails the checksum match. The receiver then pulls the + /// offered segment, and the install replaces the file. + #[compio::test] + async fn given_receiver_segment_with_other_bytes_when_installing_an_offer_should_copy_it() { + let (_origin_directory, mut origin) = + Box::pin(partition_with_sealed_segments(&segment_records(ORIGIN_OPS))).await; + let (directory, mut receiver) = Box::pin(partition_with_sealed_segments(&segment_records( + RECEIVER_OPS, + ))) + .await; + let offer = Box::pin(serve_offer(&mut origin)).await; + let held = segment_file_bytes(directory.path()); + let offered = offered_segment_bytes(&offer); + assert_eq!(held[0].len(), offered[0].len()); + assert_ne!(held[0], offered[0]); + + let adopted = Box::pin(install_offer_with_reuse(&mut receiver, &offer)).await; + + assert!( + adopted.is_empty(), + "no held segment matches an offered checksum" + ); + assert_eq!(segment_file_bytes(directory.path()), offered); + assert_eq!(receiver.consensus().commit_min(), ORIGIN_OPS); + } + + /// The segment that was active at an install keeps a hasher, so the + /// appends that seal it extend its checksum. The next transfer then links + /// it like any other sealed segment instead of pulling it again. + #[compio::test] + async fn given_installed_active_segment_when_appends_seal_it_should_reuse_it_in_the_next_transfer() + { + let records = segment_records(ORIGIN_OPS + BATCHES_PER_SEGMENT); + let (origin_held, appended) = records.split_at(usize::try_from(ORIGIN_OPS).unwrap()); + let (_origin_directory, mut origin) = + Box::pin(partition_with_sealed_segments(origin_held)).await; + let receiver_held = &records[..usize::try_from(RECEIVER_OPS).unwrap()]; + let (_directory, mut receiver) = + Box::pin(partition_with_sealed_segments(receiver_held)).await; + let first = Box::pin(serve_offer(&mut origin)).await; + Box::pin(install_offer_with_reuse(&mut receiver, &first)).await; + for partition in [&mut origin, &mut receiver] { + Box::pin(commit_records(partition, ORIGIN_OPS + 1, appended)).await; + } + let second = Box::pin(serve_offer(&mut origin)).await; + let active_at_install = first.segments.last().unwrap().entry.frontier; + + let adopted: Vec = reuse_scan(&mut receiver, &second.manifest()) + .await + .into_iter() + .map(|(_, meta)| meta.start_offset) + .collect(); + + assert_eq!( + adopted, + [ + 0, + BATCHES_PER_SEGMENT, + 2 * BATCHES_PER_SEGMENT, + active_at_install + ], + "every sealed segment, the one active at the install included" + ); + } + #[compio::test] async fn given_wal_lane_published_persisted_runs_when_idle_should_leave_the_tick_retry_set() { const RUNS: u64 = 4; @@ -14059,13 +15350,127 @@ mod tests { matches!( refused, Err( - crate::state_transfer::PartitionInstallError::StaleTransfer { + crate::state_transfer::PartitionInstallError::OfferErasesHeldOps { commit_op: 1, - commit_min: 3, + held_max: 3, } ) ), - "expected a stale refusal, got {refused:?}" + "expected a held-op refusal, got {refused:?}" + ); + } + + #[compio::test] + async fn given_a_no_wal_ack_above_commit_max_when_installing_should_preserve_the_held_op() { + const HELD_OP: u64 = 2; + for rebuilt in [false, true] { + let (_directory, mut receiver) = transfer_receiver(); + receiver.runtime_options.durability = iggy_common::Durability::Replicated; + let held = checksummed_segment_prepare(HELD_OP, 0, HELD_OP - 1, b"acknowledged"); + let header = *held.header(); + receiver + .log + .journal() + .inner + .append(held.into_frozen()) + .await + .unwrap(); + receiver.consensus.sequencer().set_sequence(HELD_OP); + if rebuilt { + assert!(receiver.register_rebuilt_ack(&header)); + } else { + assert!(receiver.send_prepare_ok(&header).await); + } + assert_eq!(receiver.consensus.commit_max(), 0); + let offsets = crate::state_transfer::ConsumerOffsetsWire { + dedup_capacity: consensus::CLIENTS_TABLE_MAX, + ..Default::default() + }; + let result = receiver + .install_state_transfer(&repair_config(), 1, Vec::new(), &offsets.encode()) + .await; + assert!( + matches!( + result, + Err( + crate::state_transfer::PartitionInstallError::OfferErasesHeldOps { + commit_op: 1, + held_max: HELD_OP + } + ) + ), + "acknowledged operation was not protected: {result:?}" + ); + assert_eq!( + receiver + .log + .journal() + .inner + .header_by_op(HELD_OP) + .unwrap() + .checksum, + header.checksum + ); + } + } + + #[compio::test] + async fn given_an_unacknowledged_no_wal_tail_when_installing_should_allow_repair_after_transfer() + { + let (_directory, mut receiver) = transfer_receiver(); + let tail = checksummed_segment_prepare(2, 0, 1, b"unacknowledged"); + receiver + .log + .journal() + .inner + .append(tail.into_frozen()) + .await + .unwrap(); + receiver.consensus.sequencer().set_sequence(2); + let offsets = crate::state_transfer::ConsumerOffsetsWire { + dedup_capacity: consensus::CLIENTS_TABLE_MAX, + ..Default::default() + }; + receiver + .install_state_transfer(&repair_config(), 1, Vec::new(), &offsets.encode()) + .await + .unwrap(); + assert!(receiver.log.journal().inner.is_empty()); + assert_eq!(receiver.consensus.commit_min(), 1); + } + + /// An install below the head keeps the head, and an offer at or below the + /// known commit keeps the commit point, so only the journal wipe can tell a + /// suffix snapshot taken before the install that the bodies it offers are + /// gone. + #[compio::test] + async fn given_a_suffix_snapshot_when_an_install_keeps_the_head_should_invalidate_it() { + const HEAD: u64 = 3; + let (_directory, mut receiver) = transfer_receiver(); + receiver.consensus().sequencer().set_sequence(HEAD); + receiver.consensus().advance_commit_max(HEAD); + receiver + .consensus() + .set_local_dvc_suffix(consensus::DvcSuffix::empty()); + assert!(!receiver.consensus().local_dvc_suffix_stale()); + let offsets = crate::state_transfer::ConsumerOffsetsWire { + dedup_capacity: consensus::CLIENTS_TABLE_MAX, + ..Default::default() + }; + + let installed = receiver + .install_state_transfer(&repair_config(), 1, Vec::new(), &offsets.encode()) + .await; + + assert!( + installed.is_ok(), + "the offer erases no held op, got {installed:?}" + ); + assert_eq!(receiver.consensus().sequencer().current_sequence(), HEAD); + assert!( + receiver.consensus().local_dvc_suffix_stale(), + "a suffix snapshot taken before the install still matches the log, so the \ + next DoViewChange offers bodies the install wiped" ); } @@ -14187,6 +15592,217 @@ mod tests { ); } + /// Two batches: the first is what a lagging replica holds, the second what + /// the primary appended after it. + fn held_and_pulled_batches(held: &[u8]) -> (Vec, usize) { + let first = checksummed_segment_prepare(1, 0, 0, held); + let second = checksummed_segment_prepare(2, first.header().checksum, 1, b"pulled"); + let mut body = first.as_slice()[size_of::()..].to_vec(); + let prefix_len = body.len(); + body.extend_from_slice(&second.as_slice()[size_of::()..]); + (body, prefix_len) + } + + #[compio::test] + async fn given_a_held_prefix_when_spilling_the_artifact_should_stage_what_a_whole_pull_stages() + { + let directory = tempfile::tempdir().unwrap(); + let (mut partition, _) = recording_partition_at(1, 3); + partition.set_partition_dir(directory.path().to_string_lossy().into_owned()); + let (body, prefix_len) = held_and_pulled_batches(b"held"); + let artifact = consensus::StateArtifact::for_bytes( + consensus::state_manifest::artifact_kind::SEGMENT_LOG, + 0, + &body, + ); + let held = directory.path().join("held.log"); + std::fs::write(&held, &body[..prefix_len]).unwrap(); + let (mut bytes, prefix) = crate::state_transfer::read_segment_prefix( + held.to_str().unwrap(), + artifact, + prefix_len as u64, + &Cell::new(false), + ) + .await + .unwrap(); + bytes.extend_from_slice(&body[prefix_len..]); + + let seeded = partition + .spill_transfer_segment(&artifact, bytes, Some(prefix)) + .await + .unwrap(); + let seeded_index = std::fs::read(&seeded.index_staging).unwrap(); + let pulled = partition + .spill_transfer_segment(&artifact, body, None) + .await + .unwrap(); + assert_eq!( + seeded, pulled, + "the seeded spill must stage the same segment" + ); + assert_eq!( + seeded_index, + std::fs::read(&pulled.index_staging).unwrap(), + "the walk resumed after the prefix must rebuild the same index" + ); + } + + #[compio::test] + async fn given_a_held_prefix_with_other_bytes_when_spilling_should_fail_the_checksum() { + let directory = tempfile::tempdir().unwrap(); + let (mut partition, _) = recording_partition_at(1, 3); + partition.set_partition_dir(directory.path().to_string_lossy().into_owned()); + let (body, prefix_len) = held_and_pulled_batches(b"held"); + let (other, other_len) = held_and_pulled_batches(b"HELD"); + assert_eq!(prefix_len, other_len); + let artifact = consensus::StateArtifact::for_bytes( + consensus::state_manifest::artifact_kind::SEGMENT_LOG, + 0, + &body, + ); + let held = directory.path().join("held.log"); + std::fs::write(&held, &other[..prefix_len]).unwrap(); + let (mut bytes, prefix) = crate::state_transfer::read_segment_prefix( + held.to_str().unwrap(), + artifact, + prefix_len as u64, + &Cell::new(false), + ) + .await + .unwrap(); + bytes.extend_from_slice(&body[prefix_len..]); + + let spilled = partition + .spill_transfer_segment(&artifact, bytes, Some(prefix)) + .await; + assert!( + matches!( + spilled, + Err(crate::state_transfer::SpillError::ManifestChecksum { frontier: 0 }) + ), + "a valid prefix with other bytes must fail the manifest checksum, got {spilled:?}" + ); + } + + #[compio::test] + async fn given_a_dropped_session_when_its_prefix_read_runs_should_stop_without_filling_a_buffer() + { + const REACTOR_TURNS_MAX: usize = 1024; + let directory = tempfile::tempdir().unwrap(); + let (mut partition, _) = recording_partition_at(1, 3); + let (body, prefix_len) = held_and_pulled_batches(b"held"); + let artifact = consensus::StateArtifact::for_bytes( + consensus::state_manifest::artifact_kind::SEGMENT_LOG, + 0, + &body, + ); + let held = directory.path().join("held.log"); + std::fs::write(&held, &body[..prefix_len]).unwrap(); + partition.transfer = Some(crate::state_transfer::PartitionTransferSession { + nonce: 1, + peer: 0, + commit_op: 0, + artifacts: vec![crate::state_transfer::TransferArtifact::Pending( + consensus::ArtifactProgress { + entry: artifact, + buf: Vec::new(), + }, + )], + target_accepted: true, + idle_ticks: 0, + spill: None, + scan: None, + seed: Some(crate::state_transfer::PrefixSeed::Planned { + artifact: 0, + log_path: held.to_str().unwrap().to_owned(), + len: prefix_len as u64, + }), + pulled_len: 0, + }); + assert!( + partition.drive_prefix_seed(), + "the pull must wait for the read" + ); + let Some(crate::state_transfer::PrefixSeed::Reading { read, .. }) = partition + .transfer + .as_ref() + .and_then(|session| session.seed.as_ref()) + else { + panic!("the read must start"); + }; + let landed = Rc::clone(&read.result); + + partition.transfer = None; + for _ in 0..REACTOR_TURNS_MAX { + if landed.borrow().is_some() { + break; + } + server_common::yield_to_reactor().await; + } + + let read = landed.borrow(); + assert!( + matches!( + read.as_ref(), + Some(Err(error)) if error.kind() == std::io::ErrorKind::Interrupted + ), + "a read whose session dropped must stop before it fills a buffer, got {read:?}" + ); + } + + /// A backup that fetches one segment artifact, whose prefix read landed. + fn fetching_with_a_landed_prefix_read() -> IggyPartition { + let (mut partition, _) = recording_partition_at(1, 3); + partition + .consensus() + .set_state_transfer_stage(consensus::StateTransferStage::AwaitingTarget); + partition + .consensus() + .set_state_transfer_stage(consensus::StateTransferStage::Fetching); + partition.transfer = Some(crate::state_transfer::PartitionTransferSession { + nonce: 1, + peer: 0, + commit_op: 0, + artifacts: vec![crate::state_transfer::TransferArtifact::Pending( + consensus::ArtifactProgress { + entry: consensus::StateArtifact::for_bytes( + consensus::state_manifest::artifact_kind::SEGMENT_LOG, + 0, + b"segment", + ), + buf: Vec::new(), + }, + )], + target_accepted: true, + idle_ticks: 0, + spill: None, + scan: None, + seed: Some(crate::state_transfer::PrefixSeed::Reading { + artifact: 0, + read: crate::state_transfer::LocalWork { + result: Rc::new(RefCell::new(Some(Err(std::io::Error::other( + "the read failed", + ))))), + ..crate::state_transfer::LocalWork::default() + }, + }), + pulled_len: 0, + }); + partition + } + + #[compio::test] + async fn given_a_landed_prefix_read_when_resuming_io_should_hand_the_pull_its_turn() { + let mut partition = fetching_with_a_landed_prefix_read(); + + let step = partition.resume_io(&repair_config()).await; + + assert!( + matches!(step, crate::PartitionIoStep::TransferReady), + "a landed read must resume the pull before its stall sweep" + ); + } + #[compio::test] async fn transfer_establishes_wal_body_ownership_without_losing_the_active_index_writer() { for durability in [ @@ -14225,7 +15841,7 @@ mod tests { ); staged.push( partition - .spill_transfer_segment(&artifact, body) + .spill_transfer_segment(&artifact, body, None) .await .unwrap(), ); @@ -14295,7 +15911,7 @@ mod tests { ); staged.push( partition - .spill_transfer_segment(&artifact, body) + .spill_transfer_segment(&artifact, body, None) .await .unwrap(), ); @@ -15801,6 +17417,110 @@ mod tests { writer.await; } + /// Past the byte cap the evicted ring releases the bodies that the WAL + /// wrote to segments and keeps their headers. Each body reads back from + /// its segment as the replicated prepare, after the WAL checkpoint too: + /// ops 1 and 2 in one read from the first segment, op 3 from the second. + /// Retention of a segment forgets the ops in it. + #[compio::test] + async fn given_bodies_past_the_ring_byte_cap_when_read_back_should_return_the_replicated_prepares() + { + const RECORDS_PER_SEGMENT: u64 = 2; + const OPS: u64 = 4; + const PAYLOAD: [u8; 4096] = [7; 4096]; + + let root = tempfile::tempdir().unwrap(); + let mut config = repair_config(); + config.path_layout.streams_root = root.path().to_string_lossy().into_owned(); + let directory = std::path::PathBuf::from(config.get_partition_path(1, 1, 0)); + std::fs::create_dir_all(&directory).unwrap(); + let (mut partition, _) = recording_partition_at(1, 3); + partition.set_partition_dir(directory.to_string_lossy().into_owned()); + partition.runtime_options.durability = iggy_common::Durability::Replicated; + partition.runtime_options.consumer_offset_durability = iggy_common::Durability::Replicated; + let prepare_len = checksummed_segment_prepare(1, 0, 0, &PAYLOAD) + .as_slice() + .len(); + let segment_size = IggyByteSize::from( + RECORDS_PER_SEGMENT * (prepare_len - size_of::()) as u64, + ); + partition.runtime_options.segment_size = Some(segment_size); + partition.log.active_segment_mut().max_size = segment_size; + partition.open_persistence().await.unwrap(); + partition.log.retire_front().unwrap(); + partition.install_empty_segment(&config, 0).await.unwrap(); + partition.set_io_notifier( + Rc::new(|_, _| {}), + crate::largest_legal_job_charge().unwrap(), + ); + // Room for one body, but not for two. + partition + .log + .journal() + .inner + .set_ring_caps(crate::EVICTED_RING_CAPACITY, (prepare_len * 3 / 2) as u64); + let persistence = Rc::clone(partition.persistence.as_ref().unwrap()); + + let mut prepares = Vec::new(); + let mut parent = 0; + let mut writer = hold_wal_writer(&persistence); + for offset in 0..OPS { + let op = offset + 1; + let prepare = checksummed_segment_prepare(op, parent, offset, &PAYLOAD); + parent = prepare.header().checksum; + prepares.push(prepare.clone()); + partition.on_replicate(prepare).await; + writer.await; + writer = hold_wal_writer(&persistence); + commit_on_heartbeat(&mut partition, &config, op).await; + settle_partition_io(&mut partition, &config).await; + } + assert_eq!( + partition.log.journal().inner.repair_retained_from(), + Some(1), + "repair serves every committed op" + ); + + persistence.request_checkpoint(); + partition.checkpoint_persistence(&config).await; + settle_partition_io(&mut partition, &config).await; + writer.await; + settle_partition_io(&mut partition, &config).await; + let released: Vec<_> = prepares[..prepares.len() - 1] + .iter() + .map(|prepare| { + partition + .log + .journal() + .inner + .evicted_header(prepare.header().op) + .expect("the byte cap released this body") + }) + .collect(); + let mut bodies = crate::ReleasedRead::new(&partition.log, released.clone()) + .read() + .await; + for (evicted, prepare) in released.iter().zip(&prepares) { + let body = bodies + .take(evicted) + .expect("the read brings every body back"); + assert_eq!(body.as_slice(), prepare.as_slice()); + } + + partition + .remove_sealed_segments_up_to(RECORDS_PER_SEGMENT - 1) + .await; + let crate::PartitionIoStep::Ready(plan) = partition.resume_io(&config).await else { + panic!("retention of the checkpointed segment is due"); + }; + execute_partition_io_plan(&mut partition, &config, plan).await; + assert_eq!( + partition.log.journal().inner.repair_retained_from(), + Some(RECORDS_PER_SEGMENT + 1), + "repair answers RangeEvicted for the ops of the removed segment" + ); + } + #[compio::test] async fn given_transfer_flush_when_tick_promotes_a_queued_send_should_ack_it_on_the_primary() { let root = tempfile::tempdir().unwrap(); @@ -16194,11 +17914,67 @@ mod tests { } pub(super) fn test_partition() -> IggyPartition { + test_partition_with_replicas(1) + } + + #[compio::test] + async fn given_snapshot_transfer_when_writes_evict_its_tail_should_release_only_confirmed_progress_and_expire_idle_peers() + { + const PEER: u8 = 1; + const NONCE: u128 = 1; + const BYTES: usize = 64 * 1024; + let mut partition = test_partition_with_replicas(3); + let budget = Arc::new(crate::RepairMemoryBudget::new(BYTES)); + partition.proof_walk_memory = Some(budget.reservation()); + let journal = &partition.log.journal().inner; + journal.set_ring_caps(1, u64::MAX); + assert!(partition.retain_transfer_tail(PEER, NONCE, 0)); + assert!(partition.transfer_tail_available(PEER, NONCE, 1)); + for op in 1..=8 { + let prepare = checksummed_segment_prepare(op, 0, op - 1, b"tail"); + journal.append(prepare.into_frozen()).await.unwrap(); + journal.evict_prefix(1); + } + assert_eq!(journal.repair_retained_from(), Some(2)); + partition.advance_transfer_tail(PEER, consensus::ReplicationProgress::Busy, 9); + assert_eq!(journal.repair_retained_from(), Some(2)); + partition.advance_transfer_tail( + PEER, + consensus::ReplicationProgress::Ready { received_op: 4 }, + 10, + ); + assert_eq!(journal.repair_retained_from(), Some(5)); + partition.expire_transfer_tails(19, 10); + assert!(partition.touch_transfer_tail(PEER, NONCE, 19)); + partition.renew_transfer_tail_for_repair(PEER, 28); + partition.expire_transfer_tails(29, 10); + assert_eq!( + journal.repair_retained_from(), + Some(5), + "a pull keeps the suffix without proving its prefix" + ); + partition.expire_transfer_tails(38, 10); + assert_eq!(journal.repair_retained_from(), Some(8)); + assert!(!partition.touch_transfer_tail(PEER, NONCE, 29)); + assert!( + !partition.retain_transfer_tail(PEER, NONCE, 29), + "an expired nonce must not rebuild a different manifest" + ); + assert!(partition.retain_transfer_tail(PEER, NONCE + 1, 29)); + assert!( + !partition.transfer_tail_available(PEER, NONCE + 1, 1), + "an old snapshot cannot promise already-missing operations" + ); + assert!(partition.transfer_tail_available(PEER, NONCE + 1, 8)); + assert!(budget.reservation().try_add(BYTES)); + } + + fn test_partition_with_replicas(replica_count: u8) -> IggyPartition { let namespace = IggyNamespace::new(1, 1, 0); let consensus = VsrConsensus::new( TEST_CLUSTER, 0, - 1, + replica_count, namespace.inner(), IggyMessageBus::new(0), LocalPipeline::new(), @@ -16231,9 +18007,16 @@ mod tests { /// Keep the returned directory alive until every read using those files ends. async fn disk_poll_partition( config: &PartitionsConfig, + ) -> (tempfile::TempDir, IggyPartition) { + Box::pin(disk_poll_partition_with_replicas(config, 1)).await + } + + async fn disk_poll_partition_with_replicas( + config: &PartitionsConfig, + replica_count: u8, ) -> (tempfile::TempDir, IggyPartition) { let directory = tempfile::tempdir().expect("create partition directory"); - let mut partition = test_partition(); + let mut partition = test_partition_with_replicas(replica_count); partition.set_partition_dir(directory.path().to_string_lossy().into_owned()); partition.log.retire_front().expect("retire empty segment"); partition @@ -17552,7 +19335,7 @@ mod tests { &body, ); let staged = backup - .spill_transfer_segment(&artifact, body) + .spill_transfer_segment(&artifact, body, None) .await .unwrap(); let offsets = crate::state_transfer::ConsumerOffsetsWire { @@ -17578,7 +19361,7 @@ mod tests { matches!( installed, Err( - crate::state_transfer::PartitionInstallError::StaleTransfer { + crate::state_transfer::PartitionInstallError::OfferErasesHeldOps { commit_op: 1, .. } @@ -17734,6 +19517,54 @@ mod tests { assert!(backup.pending_persisted_acks.borrow().is_empty()); } + #[compio::test] + async fn given_unflushed_wal_when_reporting_replication_progress_should_honor_topic_durability() + { + for durability in [ + iggy_common::Durability::Replicated, + iggy_common::Durability::Persisted, + ] { + let directory = tempfile::tempdir().unwrap(); + let (mut backup, _) = recording_partition_at(1, 3); + backup.set_partition_dir(directory.path().to_string_lossy().into_owned()); + backup.runtime_options.durability = durability; + backup.open_persistence().await.unwrap(); + let prepare = checksummed_segment_prepare(1, 0, 0, b"written"); + let header = *prepare.header(); + backup.consensus().sequencer().set_sequence(header.op); + backup + .log + .journal() + .inner + .append(prepare.clone().into_frozen()) + .await + .unwrap(); + let persistence = Rc::clone(backup.persistence.as_ref().unwrap()); + persistence.append(prepare.into_frozen(), false).unwrap(); + assert!(persistence.start()); + Rc::clone(&persistence).run().await; + assert!(persistence.is_written(&header)); + assert!(!persistence.is_durable(&header)); + + let expected = Some(consensus::BackupPrefix { + held_op: header.op, + received_op: u64::from(!durability.is_persisted()), + }); + assert_eq!( + backup.replication_progress(), + expected, + "{durability:?}: the held prefix finds gaps, and the received prefix follows \ + the prepare acknowledgment contract" + ); + backup.maintenance.flush_through = Some(header.op); + assert_eq!( + backup.replication_progress(), + expected, + "{durability:?}: ordinary materialization does not replace the held history" + ); + } + } + #[compio::test] async fn given_unacked_repair_above_a_wal_hole_when_installing_should_allow_recovery() { let directory = tempfile::tempdir().unwrap(); @@ -17752,7 +19583,9 @@ mod tests { .await .unwrap(); backup.drive_persistence().await; - backup.consensus().advance_commit_max(1); + // Op 3 is committed but unacked here: the WAL hole at op 2 holds back + // its ack, so it must not block the install. + backup.consensus().advance_commit_max(3); let third = checksummed_segment_prepare(3, 0, 2, b"unacked"); let header = *third.header(); backup @@ -17772,7 +19605,7 @@ mod tests { &body, ); let staged = backup - .spill_transfer_segment(&artifact, body) + .spill_transfer_segment(&artifact, body, None) .await .unwrap(); let offsets = crate::state_transfer::ConsumerOffsetsWire { @@ -18000,6 +19833,84 @@ mod tests { } } + #[compio::test] + async fn given_a_backup_that_cannot_journal_a_prepare_when_it_arrives_should_neither_forward_nor_ack() + { + const MIDDLE: u8 = 1; + const TAIL: u8 = 2; + /// Led by replica 0 in a group of three, like view 0, so only the view + /// number distinguishes a prepare from another view. + const LATER_VIEW: u32 = 3; + let first = checksummed_segment_prepare(1, 0, 0, b"first"); + let above_gap = checksummed_segment_prepare(2, first.header().checksum, 1, b"second"); + let later_view = first + .clone() + .transmute_header(|original, header: &mut PrepareHeader| { + *header = original; + header.view = LATER_VIEW; + header.checksum = header.identity_checksum(); + }); + let cases = [ + (MIDDLE, true, &first), + (MIDDLE, false, &above_gap), + (MIDDLE, true, &later_view), + (TAIL, true, &first), + (TAIL, false, &above_gap), + ]; + for (replica, awaiting_target, prepare) in cases { + let (mut backup, _) = recording_partition_at(replica, 3); + let sent = backup.consensus().message_bus().sent_to_replicas.clone(); + if awaiting_target { + backup.consensus().begin_state_transfer_await(); + } + let header = *prepare.header(); + let prepare = prepare.clone(); + backup.on_replicate(prepare).await; + + let case = format!( + "replica {replica}, awaiting target {awaiting_target}, op {}, view {}", + header.op, header.view + ); + assert!(!backup.log.journal().inner.holds_op(header.op), "{case}"); + assert!( + sent.borrow().is_empty(), + "a backup never forwards or acknowledges refused data: {case}" + ); + } + } + + /// A backup that awaits a transfer target relays nothing outside `Normal` + /// status, where its view is not settled, and nothing at or below its + /// commit point, where the prepare is a late copy of a committed op. + #[compio::test] + async fn given_a_backup_outside_normal_status_or_at_its_commit_point_when_a_prepare_arrives_should_not_relay_it() + { + const MIDDLE: u8 = 1; + const OP: u64 = 1; + type Setup = fn(&IggyPartition); + let prepare = checksummed_segment_prepare(OP, 0, 0, b"first"); + let cases: [(&str, Setup); 2] = [ + ("outside normal status", |backup| { + backup.consensus().begin_view_probe(); + }), + ("at its commit point", |backup| { + backup.consensus().restore_commit_state(OP, OP); + }), + ]; + let mut relayed = Vec::new(); + for (case, setup) in cases { + let (mut backup, _) = recording_partition_at(MIDDLE, 3); + let sent = backup.consensus().message_bus().sent_to_replicas.clone(); + setup(&backup); + backup.consensus().begin_state_transfer_await(); + backup.on_replicate(prepare.clone()).await; + if !sent.borrow().is_empty() { + relayed.push(case); + } + } + assert!(relayed.is_empty(), "relayed a prepare {relayed:?}"); + } + async fn partition_with_pending_durable_ack() -> ( tempfile::TempDir, IggyPartition, @@ -18336,7 +20247,7 @@ mod tests { &segment_body, ); let staged = partition - .spill_transfer_segment(&artifact, segment_body) + .spill_transfer_segment(&artifact, segment_body, None) .await .unwrap(); let offsets = crate::state_transfer::ConsumerOffsetsWire { @@ -19198,7 +21109,12 @@ mod tests { None, None, ); - partition.on_replicate(prepare).await; + partition.apply_replicated_operation(prepare).await.unwrap(); + assert_eq!( + partition.acknowledged_op.get(), + installed_op, + "the successor may only be discarded if it was never acknowledged" + ); partition.consensus.set_view(1); partition.resynchronize_consumer_offset_reservations(); assert_eq!(partition.owner_install_barrier, installed_op + 1); @@ -19754,7 +21670,8 @@ mod tests { (installed, installed.owner), (higher, higher.owner), ] { - let op = partition.consensus.sequencer().current_sequence() + 1; + let op = partition.consensus.sequencer().current_sequence() + + u64::from(admitted.owner.generation > installed.owner.generation); partition .on_request(owner_install_request(&partition, &admitted), None) .await; @@ -19774,6 +21691,51 @@ mod tests { } } + #[compio::test] + async fn given_superseded_or_applied_owner_installs_when_resubmitted_should_not_consume_ops() { + const GROUP_ID: u64 = 7; + const FIRST_CLIENT: u128 = 42; + let mut partition = test_partition(); + install_test_owner(&mut partition, GROUP_ID, FIRST_CLIENT).await; + let stale = partition.consumer_group_owners[&GROUP_ID].installation; + let installed_op = install_test_owner(&mut partition, GROUP_ID, FIRST_CLIENT + 1).await; + let installed = partition.consumer_group_owners[&GROUP_ID].installation; + for installation in [stale, installed] { + let (sender, receiver) = consensus::oneshot_channel(); + partition + .on_request( + owner_install_request(&partition, &installation), + Some(sender), + ) + .await; + assert_eq!( + partition.consensus.sequencer().current_sequence(), + installed_op, + "an applied or superseded installation must not consume a log entry" + ); + assert_eq!( + receiver.await.unwrap().header().status, + IggyError::TransientNotAccepted.as_code() + ); + assert!(partition.consumer_group_owner_admitted(&installation)); + assert!(partition.pending_owner_install.get().is_none()); + assert_eq!( + partition.consumer_group_owner(GROUP_ID), + Some(installed.owner) + ); + } + let foreign = InstallConsumerGroupOwnerRequest { + incarnation: installed.incarnation + 1, + ..stale + }; + assert!(!partition.consumer_group_owner_admitted(&foreign)); + install_test_owner(&mut partition, GROUP_ID, FIRST_CLIENT + 2).await; + assert_eq!( + partition.consensus.sequencer().current_sequence(), + installed_op + 1 + ); + } + #[compio::test] async fn given_confirmed_group_deletion_when_retired_and_restarted_should_reject_late_owner() { const LIVE_GROUP_ID: u64 = 1; @@ -23368,7 +25330,7 @@ mod tests { } #[compio::test] - async fn wal_backpressure_on_an_intermediate_replica_does_not_stop_forwarding() { + async fn wal_backpressure_on_one_backup_does_not_stop_the_other_backup() { let (mut primary, _) = recording_partition_at(0, 3); let namespace = primary.namespace(); let request = checksumless_send_request(namespace, 1); @@ -23402,20 +25364,31 @@ mod tests { .exhaust_capacity_for_test(); let stalled_header = *message.header(); backup.on_replicate(message).await; - { - let sent = backup.consensus().message_bus().sent_to_replicas.borrow(); - let (_, forwarded) = sent + assert_eq!(backup.consensus().sequencer().current_sequence(), 1); + assert_eq!(backup.persistence.as_ref().unwrap().head(), 0); + assert!( + backup + .consensus() + .message_bus() + .sent_to_replicas + .borrow() + .is_empty() + ); + let message = + Message::::try_from(Owned::copy_from_slice(forwarded.as_slice())) + .unwrap(); + primary.on_replicate(message).await; + let healthy_copy = { + let sent = primary.consensus().message_bus().sent_to_replicas.borrow(); + let (_, frame) = sent .iter() .find(|(target, _)| *target == 2) - .expect("the downstream replica receives the prepare"); - let header = bytemuck::checked::from_bytes::( - &forwarded.as_slice()[..size_of::()], - ); - assert_eq!(header.command, Command::Prepare); - assert_eq!(backup.consensus().sequencer().current_sequence(), 1); - assert_eq!(backup.persistence.as_ref().unwrap().head(), 0); - assert!(!sent.iter().any(|(target, _)| *target == 0)); - } + .expect("the primary sends directly to the other backup"); + Message::::try_from(Owned::copy_from_slice(frame.as_slice())).unwrap() + }; + let (mut healthy, _) = recording_partition_at(2, 3); + healthy.on_replicate(healthy_copy).await; + assert!(healthy.log.journal().inner.holds_op(1)); assert!(!backup.register_rebuilt_ack(&stalled_header)); assert!(backup.pending_persisted_acks.borrow().contains_key(&1)); @@ -23877,8 +25850,8 @@ mod tests { } /// Runs the tick's superblock step between install steps, as the shard does. - async fn finish_install_io( - partition: &mut IggyPartition, + async fn finish_install_io( + partition: &mut IggyPartition, ) -> Result< crate::state_transfer::PartitionInstallOutcome, crate::state_transfer::PartitionInstallError, @@ -26145,6 +28118,132 @@ mod tests { assert!(!reopened.register_rebuilt_ack(&headers[0])); } + /// An install below the head of an adopted `StartView` leaves the WAL without + /// that view's log. A restart before repair must restore the adopted head, or + /// the replica votes for the view with a head at the install op. + #[compio::test] + async fn given_an_install_below_the_adopted_head_when_reopening_should_resume_the_adopted_view() + { + const BACKUP: u8 = 1; + const ADOPTED: u32 = 2; + const HELD: u64 = 1; + const HEAD: u64 = 3; + let directory = tempfile::tempdir().unwrap(); + let (config, mut partition) = replica_with_prepare_history(directory.path(), BACKUP).await; + let partition_dir = config.get_partition_path(1, 1, 0); + let store = Rc::new(PingPongSuperblock::open(&partition_dir).await.unwrap()); + partition.set_superblock(store.clone(), None); + let held = checksummed_segment_prepare(HELD, 0, 0, b"retained"); + partition.on_replicate(held.clone()).await; + let persistence = partition.persistence.as_ref().unwrap().clone(); + persistence + .drain_with_timeout(compio::runtime::time::sleep) + .await + .unwrap(); + let mut parent = *held.header(); + let mut headers = Vec::new(); + for op in HELD + 1..=HEAD { + let mut header = PrepareHeader { + op, + request: op, + timestamp: op, + view: ADOPTED, + parent: parent.checksum, + ..parent + }; + header.checksum = header.identity_checksum(); + parent = header; + headers.push(header); + } + headers.reverse(); + let start_view = Message::::new(size_of::< + iggy_binary_protocol::StartViewHeader, + >()) + .transmute_header(|_, header: &mut iggy_binary_protocol::StartViewHeader| { + header.command = Command::StartView; + header.cluster = TEST_CLUSTER; + header.group = partition.namespace().inner(); + header.view = ADOPTED; + header.replica = 2; + header.op = HEAD; + header.commit = HEAD - 1; + header.size = + u32::try_from(size_of::()).unwrap(); + }); + partition.consensus().handle_start_view( + consensus::PlaneKind::Partitions, + start_view.header(), + bytemuck::cast_slice(&headers), + ); + assert!(partition.persist_superblock_if_needed().await); + + let body = held.as_slice()[size_of::()..].to_vec(); + let artifact = consensus::StateArtifact::for_bytes( + consensus::state_manifest::artifact_kind::SEGMENT_LOG, + 0, + &body, + ); + let staged = partition + .spill_transfer_segment(&artifact, body, None) + .await + .unwrap(); + let offsets = crate::state_transfer::ConsumerOffsetsWire { + dedup_capacity: consensus::CLIENTS_TABLE_MAX, + prepare_checksum: Some(held.header().checksum), + checkpoint_prepare: held.as_slice().to_vec(), + next_offset: 1, + ..Default::default() + }; + partition.consensus().begin_state_transfer_await(); + partition + .install_state_transfer(&repair_config(), HELD, vec![staged], &offsets.encode()) + .await + .unwrap(); + assert_eq!(partition.consensus().sequencer().current_sequence(), HEAD); + assert!(partition.consensus().view_log_unproven()); + let certified_after_install = persistence.certified_log_view(); + let segment_size = partition.log.active_segment().max_size; + drop(persistence); + drop(partition); + drop(store); + + let (store, latest) = PingPongSuperblock::open_with_latest(&partition_dir) + .await + .unwrap(); + let journal::superblock::SuperblockContents::Present(bytes) = latest else { + panic!("adopted record must be durable"); + }; + let state = consensus::VsrState::try_from(bytes.as_slice()).unwrap(); + let (mut reopened, _) = recording_partition_at(BACKUP, 3); + reopened.consensus.set_view(state.view); + reopened.consensus.set_log_view(state.log_view); + reopened + .consensus + .mark_superblock_durable(state.view, state.log_view); + reopened.set_superblock(Rc::new(store), Some(&state)); + reopened.set_partition_dir(partition_dir); + reopened.log.active_segment_mut().max_size = segment_size; + reopened.open_persistence().await.unwrap(); + assert!( + !reopened.requires_state_transfer(), + "the certificate after the install is below the persisted log view" + ); + assert_eq!( + reopened.consensus().adopted_view_head(), + Some((ADOPTED, HEAD)) + ); + assert_eq!( + ( + reopened.consensus().sequencer().current_sequence(), + reopened.consensus().view_log_unproven(), + ), + (HEAD, true), + "the restart dropped the adopted head: certified view after the install \ + {certified_after_install:?}, state log_view {}", + state.log_view + ); + } + /// The WAL drops a divergent run before the journal does. The canonical /// tail cannot extend that WAL branch and exists only in the journal. /// A crash before refill leaves the committed prefix uncertified, @@ -28240,6 +30339,8 @@ mod tests { max_timestamp: 0, log_staging: directory.path().join("00000000000000000000.log.staging"), index_staging: directory.path().join("00000000000000000000.index.staging"), + checksum: 0, + linked: false, }; let offered_state = crate::state_transfer::ConsumerOffsetsWire { dedup_capacity: consensus::CLIENTS_TABLE_MAX, @@ -28535,6 +30636,8 @@ mod tests { max_timestamp: 0, log_staging: directory.path().join("00000000000000000000.log.staging"), index_staging: directory.path().join("00000000000000000000.index.staging"), + checksum: 0, + linked: false, }; let offered_state = crate::state_transfer::ConsumerOffsetsWire { owner_retirement: None, @@ -29105,6 +31208,34 @@ mod tests { (partition, replies, header) } + fn answer_partition_heartbeat( + partition: &IggyPartition, + action: &consensus::VsrAction, + ) { + if let consensus::VsrAction::SendCommit { + view, + group, + timestamp_monotonic, + .. + } = action + { + let reply = Message::::new(HEADER_SIZE).transmute_header( + |_, header: &mut CommitReplyHeader| { + header.command = Command::CommitReply; + header.size = u32::try_from(HEADER_SIZE).unwrap(); + header.cluster = partition.consensus().cluster(); + header.view = *view; + header.replica = 1; + header.group = *group; + header.partition_incarnation = partition.consensus().partition_incarnation(); + header.timestamp_monotonic = *timestamp_monotonic; + header.seal(); + }, + ); + partition.consensus().handle_commit_reply(reply.header()); + } + } + #[compio::test] async fn given_an_armed_self_ack_when_the_local_write_is_retried_should_not_send_a_second_prepare_ok() { @@ -29154,6 +31285,7 @@ mod tests { persistence.release_capacity_for_test(); } for action in partition.consensus().tick(PlaneKind::Partitions) { + answer_partition_heartbeat(&partition, &action); if let consensus::VsrAction::RetryLocalWrite { op } = action { partition.retry_local_write(op).await; retries += 1; @@ -29199,6 +31331,7 @@ mod tests { for _ in 0..consensus::TimeoutManager::PREPARE_TICKS * 40 { for action in partition.consensus().tick(PlaneKind::Partitions) { + answer_partition_heartbeat(&partition, &action); if let consensus::VsrAction::RetryLocalWrite { op } = action { partition.retry_local_write(op).await; } @@ -32155,10 +34288,115 @@ mod tests { ); } - /// While the WAL is not durable through `fetch_to_op`, `complete_repair` - /// waits and only the shard's walk backstop moves `commit_min`, so stall - /// rounds re-serve frames this journal holds. Modeled without a WAL, one - /// walk slice per round: the walk is progress, so the peer is kept. + #[compio::test] + async fn given_a_repair_larger_than_the_wal_when_a_prefix_is_durable_should_apply_it_before_fetching_the_rest() + { + const OPS: u64 = 3; + const PAYLOAD_BYTES: usize = 48 * 1024 * 1024; + let root = tempfile::tempdir().unwrap(); + let mut config = repair_config(); + config.segment_size = IggyByteSize::from(4 * PAYLOAD_BYTES as u64); + config.path_layout.streams_root = root.path().to_string_lossy().into_owned(); + let directory = std::path::PathBuf::from(config.get_partition_path(1, 1, 0)); + std::fs::create_dir_all(&directory).unwrap(); + let (mut backup, _) = recording_partition_at(1, 3); + backup.set_partition_dir(directory.to_string_lossy().into_owned()); + set_offset_dirs_under(&mut backup, &directory); + backup.runtime_options.durability = iggy_common::Durability::Persisted; + backup.runtime_options.segment_size = Some(config.segment_size); + let payload = vec![1; PAYLOAD_BYTES]; + let parents = segment_chain_parents(OPS, &payload); + let prepare = |op: u64| { + checksummed_segment_prepare( + op, + parents[usize::try_from(op - 1).unwrap()], + op - 1, + &payload, + ) + }; + let frame_bytes = prepare(1).total_len(); + backup.log.retire_front().unwrap(); + backup.install_empty_segment(&config, 0).await.unwrap(); + backup + .open_persistence_with_capacity( + journal::partition_journal::PARTITION_WAL_CAPACITY_MIN, + std::time::Duration::ZERO, + ) + .await + .unwrap(); + backup.consensus().advance_commit_max(OPS); + backup.repair = Some(RepairSession { + floor: None, + ..armed_session(OPS, 0) + }); + let persistence = Rc::clone(backup.persistence.as_ref().unwrap()); + assert!(persistence.segment_checkpoint().is_some()); + + persistence.append(prepare(1).into_frozen(), true).unwrap(); + assert!(persistence.start()); + backup.apply_repaired_prepare(prepare(1)).await; + Rc::clone(&persistence).run().await; + assert!(persistence.is_durable_through(1)); + persistence.append(prepare(2).into_frozen(), true).unwrap(); + assert!(persistence.start()); + backup.apply_repaired_prepare(prepare(2)).await; + assert!(!persistence.has_capacity(frame_bytes)); + assert!(!persistence.is_durable_through(2)); + assert!(!backup.repair_fetch_ready()); + + assert_eq!( + backup.complete_repair(&config).await, + RepairConclusion::InProgress + ); + assert_eq!( + backup.consensus().commit_min(), + 1, + "the verified durable prefix must apply before the rest of the window can fit" + ); + assert!(backup.repair.is_some()); + Rc::clone(&persistence).run().await; + settle_partition_io(&mut backup, &config).await; + assert_eq!( + backup.complete_repair(&config).await, + RepairConclusion::InProgress + ); + assert_eq!(backup.consensus().commit_min(), 2); + backup.checkpoint_persistence(&config).await; + settle_partition_io(&mut backup, &config).await; + persistence + .drain_with_timeout(|timeout| backup.consensus.message_bus().sleep(timeout)) + .await + .unwrap(); + settle_partition_io(&mut backup, &config).await; + assert!( + persistence.has_capacity(frame_bytes), + "applying and checkpointing the prefix must reclaim WAL capacity" + ); + assert!(backup.repair_fetch_ready()); + + backup.apply_repaired_prepare(prepare(3)).await; + backup.persist_repaired_prefix(); + persistence + .drain_with_timeout(|timeout| backup.consensus.message_bus().sleep(timeout)) + .await + .unwrap(); + settle_partition_io(&mut backup, &config).await; + assert_eq!( + backup.complete_repair(&config).await, + RepairConclusion::Done, + "commit={}, head={}, durable={}, sequence={}, fatal={:?}", + backup.consensus().commit_min(), + persistence.head(), + persistence.durable_op(), + backup.consensus().sequencer().current_sequence(), + backup.fatal(), + ); + assert_eq!(backup.consensus().commit_min(), OPS); + assert!(backup.fatal().is_none()); + } + + /// A bounded commit walk can leave already fetched frames for the next + /// round. Advancing the walk keeps the peer's repair budget intact. #[compio::test] async fn given_advancing_repair_walk_when_peer_re_serves_held_frames_should_keep_peer() { const ROUNDS: u32 = crate::types::REPAIR_MAX_STALL_RETRIES + 1; @@ -32355,6 +34593,10 @@ mod tests { artifacts: Vec::new(), target_accepted: false, idle_ticks: 0, + spill: None, + scan: None, + seed: None, + pulled_len: 0, }); } assert!(partition.burn_transfer_attempt(), "budget exhausts"); @@ -32524,6 +34766,185 @@ mod tests { assert!(backup.repair.is_none()); } + /// The repair walk of an unproven log runs from the adopted head down and + /// commits nothing before it reaches `commit_min`. A walk within the WAL + /// byte budget keeps repairing. Once it holds more, it journals nothing + /// more and hands the window to state transfer. + #[compio::test] + async fn given_an_unproven_walk_past_the_wal_byte_budget_when_completing_should_require_state_transfer() + { + const PAYLOAD_BYTES: usize = 4 * 1024 * 1024; + let budget = journal::partition_journal::PARTITION_WAL_CAPACITY_MIN; + let payload = vec![7_u8; PAYLOAD_BYTES]; + let prepare_bytes = u64::try_from( + checksummed_segment_prepare(1, 0, 0, &payload) + .as_slice() + .len(), + ) + .unwrap(); + let journaled = budget / prepare_bytes + 1; + let head = journaled + 2; + let parents = segment_chain_parents(head, &payload); + let prepare = |op: u64| { + let parent = parents[usize::try_from(op - 1).unwrap()]; + checksummed_segment_prepare(op, parent, op - 1, &payload) + }; + let root = tempfile::tempdir().unwrap(); + let (mut backup, _) = recording_partition_at(1, 3); + backup.set_partition_dir(root.path().to_string_lossy().into_owned()); + backup + .open_persistence_with_capacity(budget, std::time::Duration::ZERO) + .await + .unwrap(); + backup.consensus().advance_commit_max(head); + backup.consensus().sequencer().set_sequence(head); + backup.consensus().fence_view_log(head, head); + backup.repair = Some(floorless_session(head, head)); + + backup.apply_repaired_prepare(prepare(head)).await; + assert_eq!( + backup.complete_repair(&repair_config()).await, + RepairConclusion::InProgress, + "a walk within the budget keeps repairing" + ); + for op in (1..head).rev() { + backup.apply_repaired_prepare(prepare(op)).await; + } + + let journal = &backup.log.journal().inner; + let lowest_journaled = head - journaled + 1; + assert!( + journal.header_by_op(lowest_journaled).is_some(), + "the body that crosses the budget is journaled" + ); + assert!( + journal.header_by_op(lowest_journaled - 1).is_none(), + "the walk journals nothing past the budget" + ); + assert_eq!( + backup.complete_repair(&repair_config()).await, + RepairConclusion::FloorRefused { + floor: 0, + to_op: head + } + ); + assert_eq!(backup.consensus().commit_min(), 0); + assert!(backup.repair.is_none()); + } + + #[compio::test] + async fn given_walks_sharing_a_node_budget_when_it_is_full_should_transfer_and_release_discarded_bodies() + { + const HEAD: u64 = 3; + const PAYLOAD: &[u8] = b"repair body"; + let parents = segment_chain_parents(HEAD, PAYLOAD); + let prepare = |op: u64| { + checksummed_segment_prepare( + op, + parents[usize::try_from(op - 1).unwrap()], + op - 1, + PAYLOAD, + ) + }; + let capacity = prepare(HEAD).retained_allocation_size().unwrap(); + let budget = Arc::new(crate::RepairMemoryBudget::new(capacity)); + let backup = || { + let (mut partition, _) = recording_partition_at(1, 3); + partition.proof_walk_memory = Some(budget.reservation()); + partition.consensus().advance_commit_max(HEAD); + partition.consensus().sequencer().set_sequence(HEAD); + partition.consensus().fence_view_log(HEAD, HEAD); + partition.repair = Some(floorless_session(HEAD, HEAD)); + partition + }; + let mut first = backup(); + let mut second = backup(); + first.apply_repaired_prepare(prepare(HEAD)).await; + assert!(first.log.journal().inner.header_by_op(HEAD).is_some()); + assert!( + !budget.reservation().try_add(1), + "the first walk owns the available allocation" + ); + + second.apply_repaired_prepare(prepare(HEAD)).await; + assert!( + second.log.journal().inner.is_empty(), + "an over-budget body must not be journaled" + ); + assert_eq!( + second.complete_repair(&repair_config()).await, + RepairConclusion::FloorRefused { + floor: 0, + to_op: HEAD + }, + "memory pressure must hand this walk to state transfer" + ); + + first.truncate_uncommitted_from(1).await.unwrap(); + assert!( + budget.reservation().try_add(capacity), + "discarding the window releases its reservation" + ); + + let mut third = backup(); + third.apply_repaired_prepare(prepare(HEAD)).await; + assert!(third.log.journal().inner.header_by_op(HEAD).is_some()); + drop(third); + assert!( + budget.reservation().try_add(capacity), + "dropping a partition releases its reservation" + ); + } + + #[compio::test] + async fn given_a_primary_elect_when_repair_memory_is_exhausted_should_request_transfer() { + const VIEW: u32 = 4; + const HEAD: u64 = 3; + let (mut partition, _) = recording_partition_at(1, 3); + let prepare = checksummed_segment_prepare(HEAD, 0, HEAD - 1, b"repair"); + let body = &prepare.as_slice()[..size_of::()]; + let size = size_of::() + body.len(); + let mut dvc = Message::::new(size); + dvc.as_mut_slice()[size_of::()..] + .copy_from_slice(body); + let dvc = + dvc.transmute_header(|_, header: &mut iggy_binary_protocol::DoViewChangeHeader| { + header.command = Command::DoViewChange; + header.size = u32::try_from(size).unwrap(); + header.group = partition.consensus.group(); + header.cluster = TEST_CLUSTER; + header.view = VIEW; + header.log_view = 1; + header.replica = 0; + header.op = HEAD; + header.commit = HEAD; + header.present_bitset = 1; + }); + partition.consensus.handle_do_view_change( + consensus::PlaneKind::Partitions, + dvc.header(), + body, + ); + assert!(partition.consensus.is_primary_elect()); + partition.consensus.fence_view_log(HEAD, HEAD); + partition.repair = Some(RepairSession { + view: VIEW, + ..floorless_session(HEAD, HEAD) + }); + let budget = Arc::new(crate::RepairMemoryBudget::new(1)); + partition.proof_walk_memory = Some(budget.reservation()); + partition.apply_repaired_prepare(prepare).await; + assert!(partition.log.journal().inner.is_empty()); + assert_eq!( + partition.complete_repair(&repair_config()).await, + RepairConclusion::FloorRefused { + floor: 0, + to_op: HEAD + }, + "primary election must not discard the memory-pressure signal" + ); + } + /// The serving peer evicted every op below `RETAINED_FROM` and retains more /// than one repair chunk above it. Every request that starts below /// `RETAINED_FROM` gets `RangeEvicted` and the same first chunk, so the @@ -32828,12 +35249,23 @@ mod tests { /// replicated-apply path (which stamps offsets and re-checksums), with the /// sequencer advanced the way `on_replicate` does after a real append. async fn journal_send_batch(partition: &mut IggyPartition, op: u64) { + let record = build_segment_record(IggyNamespace::new(1, 1, 0), 0); + journal_send_record(partition, op, &record).await; + } + + /// [`journal_send_batch`] with the given `record` as the prepare body. The + /// record builder gives each message a random id, so partitions that must + /// write the same bytes share their records. + async fn journal_send_record( + partition: &mut IggyPartition, + op: u64, + record: &[u8], + ) { let namespace = IggyNamespace::new(1, 1, 0); - let record = build_segment_record(namespace, 0); let header_size = std::mem::size_of::(); let total = header_size + record.len(); let mut message = Message::::new(total); - message.as_mut_slice()[header_size..].copy_from_slice(&record); + message.as_mut_slice()[header_size..].copy_from_slice(record); let message = message.transmute_header(|_, header: &mut PrepareHeader| { header.command = Command::Prepare; header.partition_incarnation = partition.created_revision(); @@ -32911,7 +35343,7 @@ mod tests { partition.consensus().advance_commit_max(3); partition.repair = Some(armed_fetch_session(2, 3, 0)); partition.commit_journal(&repair_config()).await; - let _ = partition.log.journal().inner.evict_prefix(3); + partition.log.journal().inner.evict_prefix(3); let conclusion = partition.complete_repair(&repair_config()).await; @@ -34184,6 +36616,10 @@ mod tests { impl PersistFixture { async fn new(log_path: &str, index_path: &str) -> Self { + Self::with_replicas(log_path, index_path, 1).await + } + + async fn with_replicas(log_path: &str, index_path: &str, replica_count: u8) -> Self { let log_cursor = Rc::new(AtomicU64::new(0)); let index_cursor = Rc::new(AtomicU64::new(0)); let messages_writer = @@ -34194,7 +36630,7 @@ mod tests { .await .expect("open segment index writer"); - let mut partition = test_partition(); + let mut partition = test_partition_with_replicas(replica_count); partition.log.retire_back(); partition.log.add_persisted_segment( Segment::new(0, IggyByteSize::from(1024 * 1024_u64)), @@ -34256,6 +36692,47 @@ mod tests { std::fs::metadata(path).expect("stat file").len() } + #[compio::test] + async fn given_large_saved_appends_when_accepted_should_keep_the_checksum_of_the_written_bytes() + { + const PAYLOAD_BYTES: usize = 12 * 1024 * 1024; + let directory = tempfile::tempdir().unwrap(); + let log_path = directory.path().join("segment.log"); + let index_path = directory.path().join("segment.index"); + let mut fixture = PersistFixture::with_replicas( + log_path.to_str().unwrap(), + index_path.to_str().unwrap(), + 3, + ) + .await; + fixture + .partition + .set_partition_dir(directory.path().to_str().unwrap().to_owned()); + let payload = vec![0x5a; PAYLOAD_BYTES]; + for offset in 0..2 { + let job = fixture.capture(&payload, offset).unwrap(); + assert!( + fixture.partition.segment_checksum_cache.borrow().is_empty(), + "the in-flight append owns its hasher" + ); + let result = job.execute().await; + assert!( + fixture.partition.segment_checksum_cache.borrow().is_empty(), + "file execution cannot publish a stamp before acceptance" + ); + fixture.partition.accept_materialization(result).unwrap(); + let written = std::fs::read(&log_path).unwrap(); + let memos = fixture.partition.segment_checksum_cache.borrow(); + assert_eq!( + memos + .get(&0) + .and_then(|memo| memo.checksum_of(written.len() as u64)), + Some(consensus::state_artifact_checksum(&written)), + "append {offset} must hash exactly the bytes accepted on disk" + ); + } + } + #[compio::test] async fn reference_wal_backlog_keeps_disk_poll_seek_bounded() { const RECORDS: u64 = 8; @@ -34480,7 +36957,7 @@ mod tests { .log .journal() .inner - .with_entries(<[_]>::len), + .with_entries(VecDeque::len), usize::try_from(RECORDS).unwrap() ); } @@ -34917,7 +37394,7 @@ mod tests { ); staged.push( partition - .spill_transfer_segment(&artifact, body) + .spill_transfer_segment(&artifact, body, None) .await .unwrap(), ); diff --git a/core/partitions/src/iggy_partitions.rs b/core/partitions/src/iggy_partitions.rs index 9b0d176669..2f976b1bda 100644 --- a/core/partitions/src/iggy_partitions.rs +++ b/core/partitions/src/iggy_partitions.rs @@ -121,6 +121,7 @@ where fence_published: Rc>, persistence_notifier: RefCell>, io_notifier: RefCell>, + repair_memory: Option>, loopback_ready: Rc, /// Debug-only tripwire: counts live [`Self::with_partition`] borrows so /// `insert` / `remove` can assert the partitions vec is never mutated @@ -167,6 +168,7 @@ where fence_published: Rc::default(), persistence_notifier: RefCell::new(None), io_notifier: RefCell::new(None), + repair_memory: None, loopback_ready: Rc::default(), #[cfg(debug_assertions)] borrow_active: Cell::new(0), @@ -189,12 +191,26 @@ where fence_published: Rc::default(), persistence_notifier: RefCell::new(None), io_notifier: RefCell::new(None), + repair_memory: None, loopback_ready: Rc::default(), #[cfg(debug_assertions)] borrow_active: Cell::new(0), } } + /// Build an empty shard collection using the node's shared repair budget. + #[must_use] + pub fn with_repair_memory( + shard_id: ShardId, + config: PartitionsConfig, + capacity: usize, + budget: Arc, + ) -> Self { + let mut partitions = Self::with_capacity(shard_id, config, capacity); + partitions.repair_memory = Some(budget); + partitions + } + /// This node's key-encryption keys, for wrapping new stream keys. #[must_use] pub fn key_context(&self) -> Arc { @@ -394,6 +410,9 @@ where if let Some((notifier, bytes_max)) = self.io_notifier.borrow().as_ref() { partition.set_io_notifier(Rc::clone(notifier), *bytes_max); } + if let Some(budget) = &self.repair_memory { + partition.proof_walk_memory = Some(budget.reservation()); + } self.install_loopback_notifier(namespace, &partition); partition.publish_current_offset(); partition.set_consumer_group_offsets_reconcile_epoch(Rc::clone( diff --git a/core/partitions/src/install_backup.rs b/core/partitions/src/install_backup.rs index 4c669b8ce7..7037814c07 100644 --- a/core/partitions/src/install_backup.rs +++ b/core/partitions/src/install_backup.rs @@ -235,12 +235,16 @@ mod tests { ] { std::fs::write(root.join(name), name.as_bytes()).unwrap(); } + // A segment that the transfer reuses reaches its name as a link. + std::fs::hard_link(root.join("0.index"), root.join("0.index.staging")).unwrap(); begin(root, &BTreeSet::new()).await.unwrap(); // Model the install's unlink and atomic replacement operations. for name in ["0.log", "offsets/1", "prepares-1/frontier"] { std::fs::remove_file(root.join(name)).unwrap(); std::fs::write(root.join(name), b"replacement").unwrap(); } + std::fs::remove_file(root.join("0.index")).unwrap(); + std::fs::rename(root.join("0.index.staging"), root.join("0.index")).unwrap(); std::fs::write(root.join("99.log"), b"new segment").unwrap(); recover(root).await.unwrap(); for name in [ @@ -254,6 +258,7 @@ mod tests { assert_eq!(std::fs::read(root.join(name)).unwrap(), name.as_bytes()); } assert!(!root.join("99.log").exists()); + assert!(!root.join("0.index.staging").exists()); recover(root).await.unwrap(); } diff --git a/core/partitions/src/io.rs b/core/partitions/src/io.rs index 412c2f9c9b..82ed72f1cd 100644 --- a/core/partitions/src/io.rs +++ b/core/partitions/src/io.rs @@ -24,7 +24,7 @@ use std::task::{Poll, Waker}; use std::time::{Duration, Instant}; use compio::fs::File; -use consensus::VsrState; +use consensus::{StateArtifactHasher, VsrState}; use futures::{StreamExt, TryStreamExt}; use iggy_binary_protocol::PrepareHeader; use iggy_common::MAX_MESSAGE_SIZE_UPPER_BYTES; @@ -72,11 +72,13 @@ pub struct MaterializationIoJob { pub(crate) bodies_written: Option, pub(crate) written: u64, pub(crate) completes: bool, + pub(crate) checksum: Option>, } pub struct MaterializationIoResult { pub(crate) target: MaterializationTarget, pub(crate) outcome: Result<(u64, u64), IggyError>, + pub(crate) checksum: Option>, } pub enum PartitionIoJob { @@ -180,6 +182,7 @@ pub struct CapturedPartitionIo { #[derive(Default)] pub struct PartitionIoQuiescence { active: Cell>, + local_work: Cell, retiring: Cell, deleting: Cell, interrupted: Cell, @@ -208,10 +211,23 @@ impl PartitionIoQuiescence { pub(crate) fn set(&self, active: Option) { self.active.set(active); - if active.is_none() - && let Some(waiter) = self.waiter.borrow_mut().take() - { - waiter.wake(); + if active.is_none() { + self.wake_waiter(); + } + } + + pub(crate) fn begin_local_work(&self) -> bool { + if self.is_retiring() || self.is_interrupted() { + return false; + } + self.local_work.set(self.local_work.get() + 1); + true + } + + pub(crate) fn settle_local_work(&self) { + self.local_work.set(self.local_work.get() - 1); + if self.local_work.get() == 0 { + self.wake_waiter(); } } @@ -248,7 +264,7 @@ impl PartitionIoQuiescence { "partition writer was interrupted", ))); } - if self.active.get().is_none() { + if self.active.get().is_none() && self.local_work.get() == 0 { return Poll::Ready(Ok(())); } *self.waiter.borrow_mut() = Some(context.waker().clone()); @@ -256,6 +272,13 @@ impl PartitionIoQuiescence { }) .await } + + fn wake_waiter(&self) { + let waiter = self.waiter.borrow_mut().take(); + if let Some(waiter) = waiter { + waiter.wake(); + } + } } impl PartitionTeardown { @@ -541,8 +564,23 @@ impl MaterializationIoJob { bodies_written, written, completes, + mut checksum, .. } = self; + for batch in &mut batches { + if batch.len() < size_of::() { + return MaterializationIoResult { + target, + outcome: Err(IggyError::CannotWriteToFile), + checksum: None, + }; + } + *batch = batch.slice(size_of::()..); + } + if let Some(hasher) = checksum.as_mut() { + crate::state_transfer::hash_yielding(hasher, batches.iter().map(Frozen::as_slice)) + .await; + } let outcome = if let Some(saved) = bodies_written { target .index_writer @@ -556,11 +594,9 @@ impl MaterializationIoJob { return MaterializationIoResult { target, outcome: Err(IggyError::CannotWriteToFile), + checksum: None, }; }; - for batch in &mut batches { - *batch = batch.slice(size_of::()..); - } // Both halves must settle, including when either one fails. let (messages, indexes) = futures::future::join( writer.save_frozen_batches_at(&batches, position, completes), @@ -593,7 +629,11 @@ impl MaterializationIoJob { Ok((0, 0)) } }); - MaterializationIoResult { target, outcome } + MaterializationIoResult { + target, + outcome, + checksum, + } } } diff --git a/core/partitions/src/journal.rs b/core/partitions/src/journal.rs index 513f49f1ed..b94018158d 100644 --- a/core/partitions/src/journal.rs +++ b/core/partitions/src/journal.rs @@ -16,31 +16,40 @@ // under the License. use ahash::AHashMap; +use compio::buf::{IntoInner, IoBuf}; +use compio::io::AsyncReadAtExt; use iggy_binary_protocol::{Operation, PrepareHeader}; +use journal::partition_journal::SegmentReference; use journal::{Journal, Storage}; use server_common::{ - iobuf::{Frozen, Owned}, - send_messages::{self, BatchRef, COMMAND_HEADER_SIZE, decode_prepare_slice_trusted}, + iobuf::{Frozen, IOV_MAX, Owned}, + send_messages::{ + self, BatchRef, COMMAND_HEADER_SIZE, decode_prepare_slice, decode_prepare_slice_trusted, + }, }; use std::io; use std::{ cell::{Cell, RefCell, UnsafeCell}, - collections::{BTreeMap, HashMap}, + collections::{BTreeMap, VecDeque}, + fmt::Debug, ops::RangeInclusive, + rc::{Rc, Weak}, }; use tracing::warn; +use crate::log::SegmentedLog; +use crate::repair_memory::RepairMemoryReservation; use crate::{Fragment, PollFragments, PollQueryResult}; const ZERO_LEN: usize = 0; const PREPARE_HEADER_SIZE: usize = std::mem::size_of::(); type JournalBuffer = Frozen<4096>; -/// Decoded `SendMessages` header fields surfaced from a journal (re-)append so a -/// caller can fold segment accounting without a second decode of the same bytes. -/// Raw header values only: the journal stays agnostic of partition-layer -/// accounting types (`JournalInfo` lives in the log layer). `None` is surfaced -/// for non-`SendMessages` ops, which carry no segment bytes. +/// Decoded `SendMessages` header fields: what a resident batch adds to the +/// journal's offset and timestamp indexes and message totals, and what a flush +/// folds into segment accounting. Raw header values only: the journal stays +/// agnostic of partition-layer accounting types (`JournalInfo` lives in the +/// log layer). #[derive(Clone, Copy)] pub struct RetainedBatchMeta { pub base_offset: u64, @@ -49,6 +58,29 @@ pub struct RetainedBatchMeta { pub message_count: u32, } +impl RetainedBatchMeta { + /// `None` for an op that is not `SendMessages` or that carries no + /// messages: such an entry has no offset to index and no segment bytes. + /// + /// Trusted (no batch-hash): every entry here was just stamped by + /// `stamp_prepare_for_persistence` (its checksum recomputed over this exact + /// blob) or was validated by such an append before, so re-hashing the + /// ~1 MiB blob only to read the header is waste. + fn of(header: &PrepareHeader, entry: &JournalBuffer) -> Option { + if header.operation != Operation::SendMessages { + return None; + } + let batch = decode_prepare_slice_trusted(entry.as_slice()).ok()?; + let message_count = batch.message_count(); + (message_count != 0).then(|| Self { + base_offset: batch.header.base_offset, + base_timestamp: batch.header.base_timestamp, + total_size: batch.header.total_size() as u64, + message_count, + }) + } +} + /// What one pass over the journal headers found for a repair window. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct RepairedWindowShape { @@ -57,6 +89,424 @@ pub struct RepairedWindowShape { pub complete: bool, } +/// Where a committed prepare's body sits in the log file of its segment. A +/// segment stores the body without the prepare header. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BodyPosition { + /// Start offset of the segment, which names its log file. + pub segment_start: u64, + pub position: u64, +} + +impl From for BodyPosition { + fn from(reference: SegmentReference) -> Self { + Self { + segment_start: reference.start_offset, + position: reference.position, + } + } +} + +/// A committed prepare the repair ring keeps without its body. The byte cap +/// released the body, which still sits in its segment, so repair reads it +/// back instead of answering `RangeEvicted`. +/// +/// Built only from a whole `SendMessages` prepare, so `header.size` covers the +/// prepare header and a batch. +#[derive(Debug, Clone, Copy)] +pub struct EvictedHeader { + header: PrepareHeader, + body: BodyPosition, + /// Checksum of the released batch. Bytes read back must carry it, so + /// another batch at the same position cannot pass for this op. + batch_checksum: u64, +} + +/// What one released entry costs the evicted ring: 288 bytes, so +/// [`EVICTED_HEADERS_MAX`] of them take 4.5 MiB per partition. +const EVICTED_HEADER_BYTES: u64 = size_of::() as u64; +const _: () = assert!(EVICTED_HEADER_BYTES == 288); + +impl EvictedHeader { + #[must_use] + pub const fn header(&self) -> &PrepareHeader { + &self.header + } + + /// Whether the body of `next` lies right after this one in the same + /// segment file. + const fn followed_by(&self, next: &Self) -> bool { + let body_len = (self.header.size as usize).saturating_sub(PREPARE_HEADER_SIZE); + self.body.segment_start == next.body.segment_start + && self.body.position + body_len as u64 == next.body.position + } + + /// `prepare`, read back for this entry, if its body is the released one. + fn verified(&self, prepare: Owned<4096>) -> io::Result { + let batch = decode_prepare_slice(prepare.as_slice()).map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("the body read back fails its checks: {error}"), + ) + })?; + if batch.header.batch_checksum != self.batch_checksum { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "another batch sits at the evicted body position", + )); + } + Ok(prepare.into()) + } + + /// Start offset of the segment whose log file holds the body. + #[must_use] + pub const fn segment_start(&self) -> u64 { + self.body.segment_start + } + + const fn same_prepare(&self, other: &Self) -> bool { + self.header.op == other.header.op && self.header.checksum == other.header.checksum + } +} + +/// The released bodies that one serve reads back, with the log file of each +/// segment that holds one. It owns what it reads, so a task that runs beside +/// the pump can read the bodies while the log changes. +#[derive(Debug)] +pub struct ReleasedRead { + released: Vec, + /// The log file of each segment by its start offset, `None` when the + /// segment left the log. + files: Vec<(u64, Option)>, +} + +impl ReleasedRead { + #[must_use] + pub fn new(log: &SegmentedLog, released: Vec) -> Self { + let mut files: Vec<(u64, Option)> = Vec::new(); + for evicted in &released { + let segment = evicted.body.segment_start; + if files.iter().all(|(start, _)| *start != segment) { + files.push((segment, log.messages_path(segment))); + } + } + Self { released, files } + } + + /// Reads the bodies back in op order. Bodies that lie back to back in one + /// file come back in one read. + /// + /// The read stops at the first body that it cannot serve: the segment + /// left the log, the read failed, or the bytes there are not the released + /// body, such as a torn or corrupt batch or another batch at the same + /// position. Such bytes are never returned. + pub async fn read(self) -> ReleasedBodies { + let mut bodies = ReleasedBodies { + read: VecDeque::with_capacity(self.released.len()), + failed: None, + }; + if let Err(error) = self.read_runs(&mut bodies.read).await { + bodies.failed = self + .released + .get(bodies.read.len()) + .map(|evicted| (*evicted, error)); + } + bodies + } + + async fn read_runs( + &self, + read: &mut VecDeque<(EvictedHeader, JournalBuffer)>, + ) -> io::Result<()> { + let mut file: Option<(u64, compio::fs::File)> = None; + let runs = self + .released + .chunk_by(EvictedHeader::followed_by) + .flat_map(|run| run.chunks(IOV_MAX)); + for run in runs { + let Some(first) = run.first() else { + continue; + }; + let segment = first.body.segment_start; + let file = match &mut file { + Some((start, file)) if *start == segment => file, + slot => { + let path = self + .files + .iter() + .find(|(start, _)| *start == segment) + .and_then(|(_, path)| path.as_deref()) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "the segment left the log") + })?; + &slot + .insert((segment, compio::fs::File::open(path).await?)) + .1 + } + }; + // A vectored file read in compio fills only the initialized bytes of + // each buffer, so the bodies start zeroed. + let buffers: Vec<_> = run + .iter() + .map(|evicted| { + let size = evicted.header.size as usize; + let mut prepare = Owned::<4096>::zeroed(size); + prepare.as_mut_slice()[..PREPARE_HEADER_SIZE] + .copy_from_slice(bytemuck::bytes_of(&evicted.header)); + prepare.slice(PREPARE_HEADER_SIZE..size) + }) + .collect(); + let compio::BufResult(result, buffers) = file + .read_vectored_exact_at(buffers, first.body.position) + .await; + for (evicted, body) in run.iter().zip(buffers) { + match evicted.verified(body.into_inner()) { + Ok(prepare) => read.push_back((*evicted, prepare)), + // Only the released body itself passes the checks, and a + // failed read leaves zeros where it stopped, so after a failed + // read the first body that fails is where the read stopped. + Err(error) => return Err(result.err().unwrap_or(error)), + } + } + } + Ok(()) + } +} + +/// The bodies that a [`ReleasedRead`] brought back, in op order. +#[derive(Debug)] +pub struct ReleasedBodies { + read: VecDeque<(EvictedHeader, JournalBuffer)>, + /// The body that the read could not serve, and why. The read stopped + /// there. + failed: Option<(EvictedHeader, io::Error)>, +} + +impl ReleasedBodies { + /// The body read back for `evicted`. Callers take ops in ascending order, + /// so the bodies of lower ops are dropped. + pub fn take(&mut self, evicted: &EvictedHeader) -> Option { + while self + .read + .front() + .is_some_and(|(front, _)| front.header.op < evicted.header.op) + { + self.read.pop_front(); + } + if !self + .read + .front() + .is_some_and(|(front, _)| front.same_prepare(evicted)) + { + return None; + } + self.read.pop_front().map(|(_, body)| body) + } + + /// Why the read could not serve `evicted`, if it stopped there. + pub fn take_failure(&mut self, evicted: &EvictedHeader) -> Option { + self.failed + .take_if(|(failed, _)| failed.same_prepare(evicted)) + .map(|(_, error)| error) + } +} + +/// A committed prepare the repair ring holds in memory, with the place of its +/// body in a segment when the WAL wrote it there. +struct RingEntry { + prepare: JournalBuffer, + body: Option, +} + +/// A snapshot's repair suffix. Advancing or dropping the last lease releases +/// its extra retention immediately. The ordinary ring keeps its own caps. +pub struct RepairTailLease { + state: Rc, + retained: Weak>, +} + +struct RepairTailState { + from_op: Cell, + valid: Cell, +} + +impl RepairTailLease { + pub(crate) fn is_valid(&self) -> bool { + self.state.valid.get() + } + + pub(crate) fn advance(&self, from_op: u64) { + self.state + .from_op + .set(self.state.from_op.get().max(from_op)); + if let Some(retained) = self.retained.upgrade() { + retained.borrow_mut().prune(); + } + } +} + +impl Drop for RepairTailLease { + fn drop(&mut self) { + self.state.valid.set(false); + if let Some(retained) = self.retained.upgrade() { + retained.borrow_mut().prune(); + } + } +} + +#[derive(Default)] +struct RepairTail { + leases: Vec>, + bodies: BTreeMap, + headers: BTreeMap, + memory: Option, +} + +impl RepairTail { + fn prune(&mut self) -> Option { + let mut from_op = None; + self.leases.retain(|lease| { + let Some(lease) = lease.upgrade().filter(|lease| lease.valid.get()) else { + return false; + }; + let floor = lease.from_op.get(); + from_op = Some(from_op.map_or(floor, |old: u64| old.min(floor))); + true + }); + let Some(from_op) = from_op else { + self.bodies.clear(); + self.headers.clear(); + self.memory = None; + return None; + }; + while let Some(entry) = self + .bodies + .first_entry() + .filter(|entry| *entry.key() < from_op) + { + let bytes = Self::body_memory_bytes(&entry.remove()); + if let Some(memory) = &mut self.memory { + memory.release(bytes); + } + } + while let Some(entry) = self + .headers + .first_entry() + .filter(|entry| *entry.key() < from_op) + { + entry.remove(); + if let Some(memory) = &mut self.memory { + memory.release(std::mem::size_of::<(u64, EvictedHeader)>()); + } + } + Some(from_op) + } + + fn invalidate(&mut self) { + for lease in &self.leases { + if let Some(lease) = lease.upgrade() { + lease.valid.set(false); + } + } + self.leases.clear(); + self.bodies.clear(); + self.headers.clear(); + self.memory = None; + } + + fn reserve(&mut self, bytes: usize) -> bool { + if self + .memory + .as_mut() + .is_some_and(|memory| memory.try_add(bytes)) + { + return true; + } + self.invalidate(); + false + } + + fn keep_header(&mut self, entry: &EvictedHeader, from_op: Option) { + let op = entry.header.op; + if from_op.is_none_or(|from_op| op < from_op) || self.headers.contains_key(&op) { + return; + } + if let Some(body) = self.bodies.remove(&op) + && let Some(memory) = &mut self.memory + { + memory.release(Self::body_memory_bytes(&body)); + } + if self.reserve(std::mem::size_of::<(u64, EvictedHeader)>()) { + self.headers.insert(op, *entry); + } + } + + fn keep_body(&mut self, op: u64, entry: RingEntry, from_op: Option) { + if from_op.is_none_or(|from_op| op < from_op) + || self.bodies.contains_key(&op) + || self.headers.contains_key(&op) + { + return; + } + if let Some(header) = entry.evicted_header() { + self.keep_header(&header, from_op); + } else if self.reserve(Self::body_memory_bytes(&entry.prepare)) { + self.bodies.insert(op, entry.prepare); + } + } + + fn body_memory_bytes(body: &JournalBuffer) -> usize { + JournalBuffer::allocation_size(body.backing_capacity()) + .and_then(|bytes| bytes.checked_add(std::mem::size_of::<(u64, JournalBuffer)>())) + .unwrap_or(usize::MAX) + } +} + +impl RingEntry { + /// What the ring keeps of this entry once its body leaves memory. `None` + /// when no segment holds the body. + fn evicted_header(&self) -> Option { + let body = self.body?; + let header = *bytemuck::checked::try_from_bytes::( + self.prepare.as_slice().get(..PREPARE_HEADER_SIZE)?, + ) + .ok()?; + if header.operation != Operation::SendMessages { + return None; + } + let batch = decode_prepare_slice_trusted(self.prepare.as_slice()).ok()?; + Some(EvictedHeader { + header, + body, + batch_checksum: batch.header.batch_checksum, + }) + } +} + +/// Drops the oldest entry of either tier of the evicted ring, so the ops it +/// keeps stay one run. Returns the body bytes it frees. +fn drop_oldest_evicted( + ring: &mut BTreeMap, + headers: &mut VecDeque, + retained: &mut RepairTail, + from_op: Option, +) -> u64 { + let oldest_held = ring.first_key_value().map(|(op, _)| *op); + match headers.front() { + Some(evicted) if oldest_held.is_none_or(|held| evicted.header.op < held) => { + if let Some(entry) = headers.pop_front() { + retained.keep_header(&entry, from_op); + } + 0 + } + _ => ring.pop_first().map_or(0, |(op, dropped)| { + let bytes = dropped.prepare.len() as u64; + retained.keep_body(op, dropped, from_op); + bytes + }), + } +} + /// Lookup key for querying messages from the journal. /// /// `ceiling` is the inclusive commit-frontier bound: the resident journal holds @@ -105,7 +555,7 @@ pub struct SelectedBatchSlice { /// /// # Warning — development storage only /// -/// This storage backs the `Journal` trait with a plain `Vec` +/// This storage backs the `Journal` trait with a plain `VecDeque` /// inside an `UnsafeCell`. Writes never hit disk, nothing is `fsync`ed, and /// every entry is lost on process exit. /// @@ -126,11 +576,10 @@ pub struct SelectedBatchSlice { /// production cluster traffic. #[derive(Debug, Default)] pub struct PartitionJournalMemStorage { - entries: UnsafeCell>, - /// Maps byte offset (as if disk-backed) to index in entries Vec - offset_to_index: UnsafeCell>, - /// Current write position (cumulative byte offset) - current_offset: UnsafeCell, + entries: UnsafeCell>, + /// Offset of `entries[0]`. An offset counts the entries written since the + /// last drain, so popping the front leaves every other offset valid. + front_offset: Cell, } impl Storage for PartitionJournalMemStorage { @@ -141,15 +590,8 @@ impl Storage for PartitionJournalMemStorage { } async fn read_at(&self, offset: usize, _buffer: Self::Buffer) -> io::Result { - let offset_to_index = unsafe { &*self.offset_to_index.get() }; - let Some(&index) = offset_to_index.get(&offset) else { - return Ok(Owned::<4096>::zeroed(0).into()); - }; - - let entries = unsafe { &*self.entries.get() }; - Ok(entries - .get(index) - .cloned() + Ok(self + .read_at_sync(offset) .unwrap_or_else(|| Owned::<4096>::zeroed(0).into())) } } @@ -167,24 +609,40 @@ where /// Keeping `op` in the key preserves duplicate timestamps while still /// letting us seek to the closest batch for timestamp-based polling. timestamp_to_op: UnsafeCell>, - headers: UnsafeCell>, + headers: UnsafeCell>, + /// Position of `headers[0]`, counted from the last clear, so the positions + /// in `first_header_by_op` stay valid when the front leaves. + front_position: Cell, /// Commit validation uses the first resident occurrence, including duplicates. first_header_by_op: RefCell>, + /// Messages in the resident `SendMessages` batches. + resident_messages: Cell, + /// Segment bytes of the resident `SendMessages` batches. + resident_message_size: Cell, inner: UnsafeCell>, /// Ring of recently evicted committed entries, keyed by op, retained so /// this replica can serve journal repair for rejoin windows after the - /// entries left the resident journal at flush. Bounded by + /// entries left the resident journal at flush. Bodies past the byte cap + /// move to `evicted_headers` when a segment holds them. Bounded by /// [`EVICTED_RING_CAPACITY`]; requests older than the ring answer /// `RangeEvicted` honestly. - evicted_ring: UnsafeCell>, - /// Running byte total of the buffers held by `evicted_ring`. + evicted_ring: UnsafeCell>, + /// Running byte total of the prepares held by `evicted_ring`. evicted_ring_bytes: Cell, + /// The ring's entries whose bodies the byte cap released to their + /// segments, in op order. At most [`EVICTED_HEADERS_MAX`]. + evicted_headers: UnsafeCell>, + repair_tail: Rc>, + /// Highest op the byte cap released or passed over, so it looks at each + /// entry once. An entry that lands below it later keeps its body until + /// the oldest entries go. + released_through: Cell, /// Entry-count ceiling for `evicted_ring`. Defaults to - /// [`EVICTED_RING_CAPACITY`]; the server overrides it from config at + /// [`EVICTED_RING_CAPACITY`], and the server overrides it from config at /// partition build. evicted_ring_capacity: Cell, /// Byte ceiling for `evicted_ring`. Defaults to - /// [`EVICTED_RING_BYTES_MAX`]; the server overrides it from config at + /// [`EVICTED_RING_BYTES_MAX`], and the server overrides it from config at /// partition build. evicted_ring_bytes_max: Cell, /// Single-replica groups have nobody to repair; retaining evicted @@ -209,40 +667,38 @@ where held_run: Cell<(u64, u64)>, } -/// How many evicted entries each partition retains for repair. Sized to -/// cover a few seconds of traffic around a node restart; anything older is -/// bulk-sync (phase 3) territory. +/// How many evicted entries each partition retains for repair with their +/// bodies in memory. The entries whose bodies went back to their segments +/// have their own cap, `EVICTED_HEADERS_MAX`. pub const EVICTED_RING_CAPACITY: usize = 65536; -/// Byte ceiling for the evicted ring. Every retained entry pins a full -/// batch, so an entry cap alone can consume too much memory across many -/// partitions. Whichever cap trips first evicts. +/// Byte ceiling for the bodies the evicted ring holds in memory. +/// +/// An entry that holds its body pins a full batch, so an entry cap alone can +/// consume too much memory across many partitions. Past this cap the oldest +/// bodies that a segment still holds leave memory, and the oldest entries go +/// when none can leave. The headers of released bodies do not count here. pub const EVICTED_RING_BYTES_MAX: u64 = 64 * 1024 * 1024; +/// How many entries whose bodies went back to their segments each partition +/// keeps for repair. Each one takes 288 bytes, so the cap holds 4.5 MiB per +/// partition, under 8% of the body window that [`EVICTED_RING_BYTES_MAX`] sets. +/// +/// The cap sets how far a backup can fall behind and still catch up by +/// repair: 128 repair rounds at the default chunk of 128 ops. Past it, the +/// backup gets `RangeEvicted` for the older ops and catches up by state +/// transfer, which reuses the sealed segments that the backup holds. +/// A power of two, so the deque that holds the headers, which makes room +/// before each push, never allocates past it. +pub const EVICTED_HEADERS_MAX: usize = 16384; +const _: () = assert!(EVICTED_HEADERS_MAX.is_power_of_two()); + impl Default for PartitionJournal where S: Storage + Default, { fn default() -> Self { - Self { - op_to_storage_offset: UnsafeCell::new(BTreeMap::new()), - offset_to_op: UnsafeCell::new(BTreeMap::new()), - timestamp_to_op: UnsafeCell::new(BTreeMap::new()), - headers: UnsafeCell::new(Vec::new()), - first_header_by_op: RefCell::new(AHashMap::new()), - inner: UnsafeCell::new(JournalInner { - storage: S::default(), - }), - evicted_ring: UnsafeCell::new(BTreeMap::new()), - evicted_ring_bytes: Cell::new(0), - evicted_ring_capacity: Cell::new(EVICTED_RING_CAPACITY), - evicted_ring_bytes_max: Cell::new(EVICTED_RING_BYTES_MAX), - repair_retention: Cell::new(true), - resident_control_ops: Cell::new(0), - evicted_through: Cell::new(0), - out_of_order: Cell::new(false), - held_run: Cell::new((0, 0)), - } + Self::with_storage(S::default()) } } @@ -264,30 +720,33 @@ where impl PartitionJournalMemStorage { /// Synchronous mirror of [`Storage::read_at`] for the poll path. Mem - /// storage never hits the reactor (it copies from an in-memory `Vec`), so - /// the read can run under a partition borrow without crossing an `.await` - /// - the property that keeps poll-read sound. + /// storage never hits the reactor (it copies from an in-memory `VecDeque`), + /// so the read can run under a partition borrow without crossing an + /// `.await` - the property that keeps poll-read sound. fn read_at_sync(&self, offset: usize) -> Option { - let offset_to_index = unsafe { &*self.offset_to_index.get() }; - let index = *offset_to_index.get(&offset)?; let entries = unsafe { &*self.entries.get() }; - entries.get(index).cloned() + entries + .get(offset.checked_sub(self.front_offset.get())?) + .cloned() } fn entries(&self) -> Vec { let entries = unsafe { &*self.entries.get() }; - entries.clone() + entries.iter().cloned().collect() } fn drain(&self) -> Vec { let entries = unsafe { &mut *self.entries.get() }; - let offset_to_index = unsafe { &mut *self.offset_to_index.get() }; - let current_offset = unsafe { &mut *self.current_offset.get() }; - - offset_to_index.clear(); - *current_offset = 0; + self.front_offset.set(0); + std::mem::take(entries).into() + } - std::mem::take(entries) + /// Removes the front entry and keeps the offsets of the others. + fn pop_front(&self) -> Option { + let entries = unsafe { &mut *self.entries.get() }; + let entry = entries.pop_front()?; + self.front_offset.set(self.front_offset.get() + 1); + Some(entry) } fn is_empty(&self) -> bool { @@ -296,21 +755,14 @@ impl PartitionJournalMemStorage { } fn current_offset(&self) -> usize { - let current_offset = unsafe { &*self.current_offset.get() }; - *current_offset + let entries = unsafe { &*self.entries.get() }; + self.front_offset.get() + entries.len() } fn write_at_sync(&self, buf: JournalBuffer) -> usize { let len = buf.len(); let entries = unsafe { &mut *self.entries.get() }; - let offset_to_index = unsafe { &mut *self.offset_to_index.get() }; - let current_offset = unsafe { &mut *self.current_offset.get() }; - - let index = entries.len(); - offset_to_index.insert(*current_offset, index); - entries.push(buf); - *current_offset += len; - + entries.push_back(buf); len } } @@ -333,8 +785,7 @@ impl PartitionJournal { let _ = inner.storage.drain(); } self.clear_resident_indexes(); - unsafe { &mut *self.evicted_ring.get() }.clear(); - self.evicted_ring_bytes.set(0); + self.clear_evicted(); self.evicted_through.set(0); } @@ -342,9 +793,36 @@ impl PartitionJournal { pub fn set_repair_retention(&self, enabled: bool) { self.repair_retention.set(enabled); if !enabled { - let ring = unsafe { &mut *self.evicted_ring.get() }; - ring.clear(); - self.evicted_ring_bytes.set(0); + self.clear_evicted(); + } + } + + fn clear_evicted(&self) { + self.repair_tail.borrow_mut().invalidate(); + unsafe { &mut *self.evicted_ring.get() }.clear(); + unsafe { &mut *self.evicted_headers.get() }.clear(); + self.evicted_ring_bytes.set(0); + self.released_through.set(0); + } + + pub(crate) fn retain_repair_from( + &self, + from_op: u64, + memory: RepairMemoryReservation, + ) -> RepairTailLease { + let state = Rc::new(RepairTailState { + from_op: Cell::new(from_op), + valid: Cell::new(true), + }); + let mut retained = self.repair_tail.borrow_mut(); + retained.prune(); + if retained.memory.is_none() { + retained.memory = Some(memory); + } + retained.leases.push(Rc::downgrade(&state)); + RepairTailLease { + state, + retained: Rc::downgrade(&self.repair_tail), } } @@ -357,13 +835,25 @@ impl PartitionJournal { self.evicted_ring_bytes_max.set(bytes_max); } - /// Entries and payload bytes the repair ring retains, for the shard sweep. + /// Entries the repair ring retains and the bytes they pin, for the shard + /// sweep: the prepare of each entry held in memory, and an + /// [`EvictedHeader`] for each entry whose body went back to its segment. /// /// Excludes unused allocation capacity, alignment and index overhead. /// The newest entry is retained even when it exceeds the byte budget. pub fn evicted_ring_occupancy(&self) -> (usize, u64) { let ring = unsafe { &*self.evicted_ring.get() }; - (ring.len(), self.evicted_ring_bytes.get()) + let headers = unsafe { &*self.evicted_headers.get() }; + let retained = self.repair_tail.borrow(); + ( + ring.len() + headers.len() + retained.bodies.len() + retained.headers.len(), + self.evicted_ring_bytes.get() + + headers.len() as u64 * EVICTED_HEADER_BYTES + + retained + .memory + .as_ref() + .map_or(0, |memory| memory.bytes() as u64), + ) } /// Committed entries already flushed out of the resident journal cannot be @@ -385,12 +875,30 @@ impl PartitionJournal { self.resident_control_ops.get() } + /// Messages in the resident `SendMessages` batches, which the partition's + /// message-count flush threshold reads. + pub const fn resident_messages(&self) -> u64 { + self.resident_messages.get() + } + + /// Segment bytes of the resident `SendMessages` batches, which the + /// partition's size flush threshold reads. + pub const fn resident_message_size(&self) -> u64 { + self.resident_message_size.get() + } + /// Restore the materialized commit point for elections and body repair. pub fn restore_checkpoint_prepare(&self, op: u64, prepare: JournalBuffer) { let ring = unsafe { &mut *self.evicted_ring.get() }; - debug_assert!(ring.is_empty()); + debug_assert!(ring.is_empty() && unsafe { &*self.evicted_headers.get() }.is_empty()); self.evicted_ring_bytes.set(prepare.len() as u64); - ring.insert(op, prepare); + ring.insert( + op, + RingEntry { + prepare, + body: None, + }, + ); } /// Entry bytes for `op`, from the resident journal or the evicted ring. @@ -405,7 +913,9 @@ impl PartitionJournal { } } let ring = unsafe { &*self.evicted_ring.get() }; - ring.get(&op).cloned() + ring.get(&op) + .map(|entry| entry.prepare.clone()) + .or_else(|| self.repair_tail.borrow().bodies.get(&op).cloned()) } /// The lowest resident `SendMessages` op in `ops` with its entry bytes, in @@ -460,8 +970,12 @@ impl PartitionJournal { return Some(header); } let ring = unsafe { &*self.evicted_ring.get() }; - let entry = ring.get(&op)?; - let header_bytes = entry.as_slice().get(..PREPARE_HEADER_SIZE)?; + let retained = self.repair_tail.borrow(); + let prepare = ring + .get(&op) + .map(|entry| &entry.prepare) + .or_else(|| retained.bodies.get(&op))?; + let header_bytes = prepare.as_slice().get(..PREPARE_HEADER_SIZE)?; bytemuck::checked::try_from_bytes::(header_bytes) .ok() .copied() @@ -486,32 +1000,157 @@ impl PartitionJournal { } } let ring = unsafe { &*self.evicted_ring.get() }; - for (op, entry) in ring.range(ops) { + for (op, entry) in ring.range(ops.clone()) { if found.contains_key(op) { continue; } - let Some(header_bytes) = entry.as_slice().get(..PREPARE_HEADER_SIZE) else { + let Some(header_bytes) = entry.prepare.as_slice().get(..PREPARE_HEADER_SIZE) else { continue; }; if let Ok(header) = bytemuck::checked::try_from_bytes::(header_bytes) { found.insert(*op, *header); } } + for (op, entry) in self.repair_tail.borrow().bodies.range(ops) { + if found.contains_key(op) { + continue; + } + let Some(bytes) = entry.as_slice().get(..PREPARE_HEADER_SIZE) else { + continue; + }; + if let Ok(header) = bytemuck::checked::try_from_bytes::(bytes) { + found.insert(*op, *header); + } + } found } /// Oldest op this journal can still serve for repair, including resident - /// backfills older than the retained repair range. + /// backfills older than the retained repair range and bodies read back + /// from segments. pub fn repair_retained_from(&self) -> Option { + let headers = unsafe { &*self.evicted_headers.get() }; + self.oldest_in_memory() + .into_iter() + .chain(headers.front().map(|evicted| evicted.header.op)) + .chain( + self.repair_tail + .borrow() + .headers + .first_key_value() + .map(|(op, _)| *op), + ) + .min() + } + + /// Oldest op from which this journal holds every body up to its head in + /// memory, so repair serves that tail without a disk read. + pub fn memory_retained_from(&self) -> Option { + let headers = unsafe { &*self.evicted_headers.get() }; + let oldest = self.oldest_in_memory()?; + let newest_released = headers + .back() + .map(|entry| entry.header.op) + .into_iter() + .chain( + self.repair_tail + .borrow() + .headers + .last_key_value() + .map(|(op, _)| *op), + ) + .max(); + Some(newest_released.map_or(oldest, |op| oldest.max(op.saturating_add(1)))) + } + + fn oldest_in_memory(&self) -> Option { let ring = unsafe { &*self.evicted_ring.get() }; let resident = unsafe { &*self.op_to_storage_offset.get() }; ring.first_key_value() .map(|(op, _)| *op) .into_iter() .chain(resident.first_key_value().map(|(op, _)| *op)) + .chain( + self.repair_tail + .borrow() + .bodies + .first_key_value() + .map(|(op, _)| *op), + ) .min() } + /// The header the evicted ring keeps for `op` after the byte cap released + /// its body to a segment. [`Self::repair_entry`] and + /// [`Self::repair_header`] do not serve such an op: its body must be read + /// back with a [`ReleasedRead`]. + #[must_use] + pub fn evicted_header(&self, op: u64) -> Option { + let headers = unsafe { &*self.evicted_headers.get() }; + headers + .binary_search_by_key(&op, |evicted| evicted.header.op) + .ok() + .and_then(|index| headers.get(index).copied()) + .or_else(|| self.repair_tail.borrow().headers.get(&op).copied()) + } + + /// Forgets the released entries whose bodies lie in segments up to the + /// one at `start_offset`, once retention deletes it, so repair answers + /// `RangeEvicted` for them. + pub fn forget_segment(&self, start_offset: u64) { + if self + .repair_tail + .borrow() + .headers + .values() + .any(|entry| entry.body.segment_start <= start_offset) + { + self.repair_tail.borrow_mut().invalidate(); + } + let headers = unsafe { &*self.evicted_headers.get() }; + let through = headers + .iter() + .take_while(|evicted| evicted.body.segment_start <= start_offset) + .last() + .map(|evicted| evicted.header.op); + if let Some(through) = through { + self.forget_released_through(through); + } + } + + /// Forgets every evicted entry at or below the released entry at `op`, + /// so repair answers `RangeEvicted` for them. A released entry is never + /// the newest one, so the commit point stays. + pub fn forget_released_through(&self, op: u64) { + let retained_affected = { + let retained = self.repair_tail.borrow(); + retained + .headers + .first_key_value() + .is_some_and(|(first, _)| *first <= op) + || retained + .bodies + .first_key_value() + .is_some_and(|(first, _)| *first <= op) + }; + if retained_affected { + self.repair_tail.borrow_mut().invalidate(); + } + let ring = unsafe { &mut *self.evicted_ring.get() }; + let headers = unsafe { &mut *self.evicted_headers.get() }; + let mut bytes = self.evicted_ring_bytes.get(); + while let Some(entry) = ring.first_entry().filter(|entry| *entry.key() <= op) { + bytes -= entry.remove().prepare.len() as u64; + } + while headers + .front() + .is_some_and(|evicted| evicted.header.op <= op) + { + headers.pop_front(); + } + self.evicted_ring_bytes.set(bytes); + } + /// Synchronous resident-range poll read. Never awaits (mem storage reads /// are pure memory copies), so a partition borrow held across it cannot span /// a scheduler yield. The poll path uses this; the disk tier, which does @@ -589,8 +1228,8 @@ impl PartitionJournal { } /// Inspect resident allocations without pinning extra payload references. - /// The closure cannot retain the borrowed entry vector across owner mutations. - pub(crate) fn with_entries(&self, inspect: impl FnOnce(&[JournalBuffer]) -> R) -> R { + /// The closure cannot retain the borrowed entry deque across owner mutations. + pub(crate) fn with_entries(&self, inspect: impl FnOnce(&VecDeque) -> R) -> R { let inner = unsafe { &*self.inner.get() }; let entries = unsafe { &*inner.storage.entries.get() }; inspect(entries) @@ -647,127 +1286,206 @@ impl PartitionJournal { ordered.sort_by_key(|(op, _)| *op); self.clear_resident_indexes(); for (_, entry) in ordered { - self.append_with_meta_sync(entry); + self.append_sync(entry); } true } - /// Remove the persisted prefix and return the retained entries with their batch metadata. - pub fn evict_prefix(&self, count: usize) -> Vec<(JournalBuffer, Option)> { - let all_entries = { - let inner = unsafe { &*self.inner.get() }; - inner.storage.drain() - }; - // Ops are positional against `headers` until the clear below; capture - // the evicted prefix's ops first so the ring stays op-addressable. - let evicted_ops: Vec = { - let headers = unsafe { &*self.headers.get() }; - headers.iter().take(count).map(|header| header.op).collect() + /// Remove the persisted prefix: the `count` front entries. + pub fn evict_prefix(&self, count: usize) { + self.evict_prefix_located(count, |_| None); + } + + /// [`Self::evict_prefix`] for a journal whose bodies also sit in segments. + /// `locate` names where the body of an evicted op lies, so the evicted + /// ring can release that body from memory and still serve the op. + /// + /// Costs O(`count`), however many entries stay resident: a lagging backup + /// holds hundreds of thousands and evicts after every flush. Two rare + /// shapes rebuild the retained tail instead, because a popped entry would + /// leave a stale index behind: a journal out of op order, and a prefix op + /// that has another copy in the retained tail. + pub fn evict_prefix_located(&self, count: usize, locate: impl Fn(u64) -> Option) { + let evicted = if self.out_of_order.get() || self.retains_copy_of_prefix(count) { + self.rebuild_without_prefix(count) + } else { + self.pop_prefix(count) }; - if let Some(&highest) = evicted_ops.iter().max() { + if let Some(highest) = evicted.iter().map(|(op, _)| *op).max() { self.evicted_through .set(self.evicted_through.get().max(highest)); } - self.clear_resident_indexes(); + if !self.repair_retention.get() { + return; + } + let ring = unsafe { &mut *self.evicted_ring.get() }; + let mut ring_bytes = self.evicted_ring_bytes.get(); + for (op, prepare) in evicted { + ring_bytes += prepare.len() as u64; + let entry = RingEntry { + prepare, + body: locate(op), + }; + if let Some(previous) = ring.insert(op, entry) { + ring_bytes -= previous.prepare.len() as u64; + } + } + self.evicted_ring_bytes.set(ring_bytes); + self.trim_evicted_ring(); + } - let mut all_entries = all_entries.into_iter(); - if self.repair_retention.get() { - let ring = unsafe { &mut *self.evicted_ring.get() }; - let mut ring_bytes = self.evicted_ring_bytes.get(); - for op in evicted_ops { - let Some(entry) = all_entries.next() else { - break; - }; - ring_bytes += entry.len() as u64; - if let Some(previous) = ring.insert(op, entry) { - ring_bytes -= previous.len() as u64; - } - while ring.len() > self.evicted_ring_capacity.get() - || (ring_bytes > self.evicted_ring_bytes_max.get() && ring.len() > 1) - { - if let Some((_, dropped)) = ring.pop_first() { - ring_bytes -= dropped.len() as u64; - } + /// Whether an op in the `count` front entries has a copy behind them. The + /// op index names the last copy, so one lookup per op answers it. + fn retains_copy_of_prefix(&self, count: usize) -> bool { + let headers = unsafe { &*self.headers.get() }; + let op_to_storage_offset = unsafe { &*self.op_to_storage_offset.get() }; + let inner = unsafe { &*self.inner.get() }; + let retained_from = inner.storage.current_offset() - headers.len().saturating_sub(count); + headers.iter().take(count).any(|header| { + op_to_storage_offset + .get(&header.op) + .is_some_and(|&storage_offset| storage_offset >= retained_from) + }) + } + + /// Pops the `count` front entries and removes them from every index. The + /// caller makes sure that no evicted op has a retained copy, so each index + /// ends as a rebuild of the retained tail would leave it. + fn pop_prefix(&self, count: usize) -> Vec<(u64, JournalBuffer)> { + let headers = unsafe { &mut *self.headers.get() }; + let inner = unsafe { &*self.inner.get() }; + let op_to_storage_offset = unsafe { &mut *self.op_to_storage_offset.get() }; + let offset_to_op = unsafe { &mut *self.offset_to_op.get() }; + let timestamp_to_op = unsafe { &mut *self.timestamp_to_op.get() }; + let mut first_header_by_op = self.first_header_by_op.borrow_mut(); + let mut evicted = Vec::with_capacity(count.min(headers.len())); + while evicted.len() < count { + // The length-lock pairs every header with one stored entry. + let (Some(header), Some(entry)) = (headers.pop_front(), inner.storage.pop_front()) + else { + break; + }; + self.front_position.set(self.front_position.get() + 1); + let op = header.op; + op_to_storage_offset.remove(&op); + first_header_by_op.remove(&op); + if header.operation != Operation::SendMessages { + self.resident_control_ops + .set(self.resident_control_ops.get() - 1); + } + if let Some(meta) = RetainedBatchMeta::of(&header, &entry) { + // A later batch at the same offset keeps the key. + if offset_to_op.get(&meta.base_offset) == Some(&op) { + offset_to_op.remove(&meta.base_offset); } + timestamp_to_op.remove(&(meta.base_timestamp, op)); + self.resident_messages + .set(self.resident_messages.get() - u64::from(meta.message_count)); + self.resident_message_size + .set(self.resident_message_size.get() - meta.total_size); } - self.evicted_ring_bytes.set(ring_bytes); - } else { - // Consume without retaining: the iterator itself must still - // advance past the evicted prefix so the retained tail below is - // aligned. - for _ in &evicted_ops { - if all_entries.next().is_none() { - break; + evicted.push((op, entry)); + } + // Every op above the highest evicted one is still resident, so the + // cached run keeps that part instead of a full walk on the next probe. + if let Some(highest) = evicted.iter().map(|(op, _)| *op).max() { + let (run_floor, run_through) = self.held_run.get(); + let floor = run_floor.max(highest); + self.held_run.set(if floor < run_through { + (floor, run_through) + } else { + (0, 0) + }); + } + evicted + } + + /// Evicts the `count` front entries by draining the journal and appending + /// the retained tail again. + fn rebuild_without_prefix(&self, count: usize) -> Vec<(u64, JournalBuffer)> { + let entries = { + let inner = unsafe { &*self.inner.get() }; + inner.storage.drain() + }; + // Positional against `headers` until the clear below. + let ops: Vec = { + let headers = unsafe { &*self.headers.get() }; + headers.iter().map(|header| header.op).collect() + }; + self.clear_resident_indexes(); + let mut entries = ops.into_iter().zip(entries); + let evicted = entries.by_ref().take(count).collect(); + for (_, entry) in entries { + self.append_sync(entry); + } + evicted + } + + /// Keeps the evicted ring within its caps. Past the byte cap the oldest + /// bodies that a segment holds leave memory first, and the oldest entries + /// go when none can leave. The byte cap never takes the newest entry, so + /// the commit point stays servable for a view change. A released body + /// keeps its header, and the oldest entries make room for it first once + /// [`EVICTED_HEADERS_MAX`] headers are kept. + fn trim_evicted_ring(&self) { + let mut retained = self.repair_tail.borrow_mut(); + let retained_from = retained.prune(); + let ring = unsafe { &mut *self.evicted_ring.get() }; + let headers = unsafe { &mut *self.evicted_headers.get() }; + let mut bytes = self.evicted_ring_bytes.get(); + let bytes_max = self.evicted_ring_bytes_max.get(); + let newest = ring.last_key_value().map(|(op, _)| *op); + while bytes > bytes_max { + let Some((&op, entry)) = ring + .range(self.released_through.get().saturating_add(1)..) + .next() + .filter(|(op, _)| Some(**op) != newest) + else { + break; + }; + self.released_through.set(op); + if let Some(evicted) = entry.evicted_header() { + bytes -= entry.prepare.len() as u64; + ring.remove(&op); + while headers.len() >= EVICTED_HEADERS_MAX { + bytes -= drop_oldest_evicted(ring, headers, &mut retained, retained_from); } + headers.push_back(evicted); } } - let retained: Vec = all_entries.collect(); - let mut result = Vec::with_capacity(retained.len()); - for entry in retained { - let meta = self.append_with_meta_sync(entry.clone()); - result.push((entry, meta)); + while ring.len() > self.evicted_ring_capacity.get() || (bytes > bytes_max && ring.len() > 1) + { + bytes -= drop_oldest_evicted(ring, headers, &mut retained, retained_from); } - - result + self.evicted_ring_bytes.set(bytes); } - /// `append`, additionally returning the decoded `RetainedBatchMeta` for a - /// `SendMessages` entry so the eviction path folds its accounting without a - /// second decode of the same bytes. - /// /// INVARIANT (length-lock): the header is pushed before `storage.write_at`, /// so `headers[i]` and the entry at storage index `i` stay positionally /// paired - materialization reads entries by that position. `MemStorage::write_at` /// is infallible, so the push never runs ahead of a failed write. A future /// fallible `Storage` MUST roll the header push back on a write error (or /// write before pushing the header) or the zip desyncs. - fn append_with_meta_sync(&self, entry: JournalBuffer) -> Option { + fn append_sync(&self, entry: JournalBuffer) { let header_bytes = &entry[..PREPARE_HEADER_SIZE]; let header = *bytemuck::checked::try_from_bytes::(header_bytes) .expect("partition journal append expects a valid prepare header"); let op = header.op; - // One decode feeds both the offset/timestamp index and the surfaced - // accounting meta. Both are keyed on `base_timestamp`, the broker - // append time stamped into replies: the seek hint must live on the - // same clock as `select_batch_slice`'s filter or timestamp polls seek - // to the wrong resident entry. - // Trusted (no batch-hash): every entry reaching append was just stamped - // by `stamp_prepare_for_persistence` (its checksum recomputed over this - // exact blob) or re-appended from an already-validated resident entry, - // so re-hashing the ~1 MiB blob here only to read the header is waste. - let (index_offset_timestamp, meta) = if header.operation == Operation::SendMessages { - match decode_prepare_slice_trusted(entry.as_slice()) { - Ok(batch) if batch.message_count() != 0 => { - let message_count = batch.message_count(); - let meta = RetainedBatchMeta { - base_offset: batch.header.base_offset, - base_timestamp: batch.header.base_timestamp, - total_size: batch.header.total_size() as u64, - message_count, - }; - ( - Some((batch.header.base_offset, batch.header.base_timestamp)), - Some(meta), - ) - } - _ => (None, None), - } - } else { - (None, None) - }; + let meta = RetainedBatchMeta::of(&header, &entry); { let headers = unsafe { &mut *self.headers.get() }; - if headers.last().is_some_and(|last| last.op > op) { + if headers.back().is_some_and(|last| last.op > op) { self.out_of_order.set(true); } if !headers.is_empty() { + let position = self.front_position.get() + headers.len(); self.first_header_by_op .borrow_mut() .entry(op) - .or_insert(headers.len()); + .or_insert(position); } - headers.push(header); + headers.push_back(header); }; let storage_offset = { @@ -786,15 +1504,22 @@ impl PartitionJournal { .set(self.resident_control_ops.get() + 1); } - if let Some((offset, timestamp)) = index_offset_timestamp { + if let Some(meta) = meta { let offset_to_op = unsafe { &mut *self.offset_to_op.get() }; - offset_to_op.insert(offset, op); + offset_to_op.insert(meta.base_offset, op); + // Keyed on `base_timestamp`, the broker append time stamped into + // replies: the seek hint must live on the same clock as + // `select_batch_slice`'s filter or timestamp polls seek to the + // wrong resident entry. let timestamp_to_op = unsafe { &mut *self.timestamp_to_op.get() }; - timestamp_to_op.insert((timestamp, op), op); - } + timestamp_to_op.insert((meta.base_timestamp, op), op); - meta + self.resident_messages + .set(self.resident_messages.get() + u64::from(meta.message_count)); + self.resident_message_size + .set(self.resident_message_size.get() + meta.total_size); + } } pub fn is_empty(&self) -> bool { @@ -847,11 +1572,17 @@ where op_to_storage_offset: UnsafeCell::new(BTreeMap::new()), offset_to_op: UnsafeCell::new(BTreeMap::new()), timestamp_to_op: UnsafeCell::new(BTreeMap::new()), - headers: UnsafeCell::new(Vec::new()), + headers: UnsafeCell::new(VecDeque::new()), + front_position: Cell::new(0), inner: UnsafeCell::new(JournalInner { storage }), first_header_by_op: RefCell::new(AHashMap::new()), + resident_messages: Cell::new(0), + resident_message_size: Cell::new(0), evicted_ring: UnsafeCell::new(BTreeMap::new()), evicted_ring_bytes: Cell::new(0), + evicted_headers: UnsafeCell::new(VecDeque::new()), + repair_tail: Rc::default(), + released_through: Cell::new(0), evicted_ring_capacity: Cell::new(EVICTED_RING_CAPACITY), evicted_ring_bytes_max: Cell::new(EVICTED_RING_BYTES_MAX), repair_retention: Cell::new(true), @@ -865,22 +1596,27 @@ where /// Forget every resident header and index. The caller drained the entries. fn clear_resident_indexes(&self) { unsafe { &mut *self.headers.get() }.clear(); + self.front_position.set(0); self.first_header_by_op.borrow_mut().clear(); unsafe { &mut *self.op_to_storage_offset.get() }.clear(); unsafe { &mut *self.offset_to_op.get() }.clear(); self.held_run.set((0, 0)); unsafe { &mut *self.timestamp_to_op.get() }.clear(); self.resident_control_ops.set(0); + self.resident_messages.set(0); + self.resident_message_size.set(0); self.out_of_order.set(false); } pub fn header_by_op(&self, op: u64) -> Option { let headers = unsafe { &*self.headers.get() }; - if headers.first().is_some_and(|header| header.op == op) { - return headers.first().copied(); + if headers.front().is_some_and(|header| header.op == op) { + return headers.front().copied(); } - let index = *self.first_header_by_op.borrow().get(&op)?; - let header = headers.get(index); + let position = *self.first_header_by_op.borrow().get(&op)?; + let header = position + .checked_sub(self.front_position.get()) + .and_then(|index| headers.get(index)); debug_assert!(header.is_some_and(|header| header.op == op)); header.copied() } @@ -923,6 +1659,15 @@ where op_to_storage_offset.contains_key(&op) } + /// Highest resident op below `op`. + pub fn held_below(&self, op: u64) -> Option { + let op_to_storage_offset = unsafe { &*self.op_to_storage_offset.get() }; + op_to_storage_offset + .range(..op) + .next_back() + .map(|(held, _)| *held) + } + /// Highest op `h >= floor` such that every op in `floor + 1..=h` is resident. pub fn held_through(&self, floor: u64) -> u64 { self.held_through_until(floor, u64::MAX) @@ -1033,6 +1778,10 @@ where /// `limit` bounds what one caller commits in a single pass. The run is /// re-derived from the caller's own `commit_min` every time, so a truncated /// answer is resumed, not lost. + /// + /// Costs one index lookup per returned header, however many headers are + /// resident: a lagging backup holds hundreds of thousands, and the commit + /// walk calls this on every tick. pub fn committed_headers_from( &self, from_op: u64, @@ -1047,37 +1796,20 @@ where if from_op > commit_max { return Vec::new(); } - let headers = unsafe { &*self.headers.get() }; - // Sized off the RUN the caller asked for, capped by the headers that - // could possibly cover it. Sizing off `headers.len()` alone would make - // a one-op run allocate a slot per resident header, and `commit_max` is - // a cluster frontier this replica may be arbitrarily far below, so - // neither bound can be dropped. `saturating_add` because `commit_max` - // is `u64::MAX` on a saturated frontier, where `+ 1` would panic in - // debug and wrap to an empty span (a walk that never resumes) in - // release. - let span = (commit_max - from_op) - .saturating_add(1) - .min(limit as u64) - .min(headers.len() as u64); - // `span` is a `min` of two `usize`-derived values on a 64-bit target, - // so the narrowing is lossless; `slot` is then compared against it as - // a `u64` BEFORE any cast, so nothing depends on the cast to bound it. - #[allow(clippy::cast_possible_truncation)] - let span = span as usize; - let mut slots: Vec> = vec![None; span]; - for header in headers { - if header.op < from_op || header.op - from_op >= span as u64 { - continue; - } - #[allow(clippy::cast_possible_truncation)] - let slot = (header.op - from_op) as usize; - // Keep the first resident occurrence, matching `header_by_op`. - slots[slot].get_or_insert(*header); - } + // `saturating_add` because `commit_max` is `u64::MAX` on a saturated + // frontier, where `+ 1` would panic in debug and wrap to an empty span + // (a walk that never resumes) in release. + let span = (commit_max - from_op).saturating_add(1).min(limit as u64); // Stops at the first hole: a replication gap must not be skipped, or - // `advance_commit_min`'s sequential contract breaks. - slots.into_iter().map_while(|header| header).collect() + // `advance_commit_min`'s sequential contract breaks. `header_by_op` + // returns the first resident occurrence of an op. + (0..span) + .map_while(|index| { + from_op + .checked_add(index) + .and_then(|op| self.header_by_op(op)) + }) + .collect() } /// Oldest message offset still resident in the in-memory journal, if @@ -1174,7 +1906,7 @@ impl Journal for PartitionJournal { } async fn append(&self, entry: Self::Entry) -> io::Result<()> { - self.append_with_meta_sync(entry); + self.append_sync(entry); Ok(()) } @@ -1202,7 +1934,7 @@ impl Journal for PartitionJournal { inner.storage.drain() }; // Positional against `headers` until the clear below (see the length-lock - // invariant on `append_with_meta_sync`), so the ops are captured first. + // invariant on `append_sync`), so the ops are captured first. let entry_ops: Vec = { let headers = unsafe { &*self.headers.get() }; headers.iter().map(|header| header.op).collect() @@ -1216,7 +1948,7 @@ impl Journal for PartitionJournal { continue; } // Replays bytes this journal already accepted once, so it cannot fail. - self.append_with_meta_sync(entry); + self.append_sync(entry); } Ok(removed) } @@ -1502,6 +2234,7 @@ pub fn select_resident( #[cfg(test)] mod tests { use super::*; + use crate::RepairMemoryBudget; use bytes::Bytes; use iggy_binary_protocol::{Command, HEADER_SIZE}; use journal::Journal; @@ -1511,6 +2244,151 @@ mod tests { decode_batch_slice, }; use server_common::sharding::IggyNamespace; + use std::sync::Arc; + + #[compio::test] + async fn given_a_small_snapshot_tail_with_a_large_backing_buffer_when_retained_should_charge_and_release_the_allocation() + { + const BACKING_CAPACITY: usize = 64 * 1024; + let mut backing = Owned::<4096>::with_capacity(BACKING_CAPACITY); + backing.extend_from_slice(build_prepare(1, HEADER_SIZE).as_slice()); + let prepare = Frozen::from(backing); + let charge = Frozen::<4096>::allocation_size(prepare.backing_capacity()).unwrap() + + size_of::<(u64, JournalBuffer)>(); + let budget = Arc::new(RepairMemoryBudget::new(charge)); + let journal = PartitionJournal::::default(); + journal.set_ring_caps(1, u64::MAX); + let lease = journal.retain_repair_from(1, budget.reservation()); + journal.append(prepare).await.unwrap(); + journal.evict_prefix(1); + journal + .append(build_prepare(2, HEADER_SIZE).into_frozen()) + .await + .unwrap(); + journal.evict_prefix(1); + + assert!(lease.is_valid()); + assert!(journal.repair_entry(1).is_some()); + assert!( + !budget.reservation().try_add(1), + "the retained frame must charge its complete backing allocation" + ); + lease.advance(2); + assert!(journal.repair_entry(1).is_none()); + assert!( + budget.reservation().try_add(charge), + "advancing the lease must refund the complete allocation charge" + ); + } + + #[compio::test] + async fn given_snapshot_leases_when_the_ring_evicts_should_retain_until_every_receiver_advances() + { + const OPS: u64 = 8; + let journal = PartitionJournal::::default(); + journal.set_ring_caps(1, u64::MAX); + let budget = Arc::new(RepairMemoryBudget::new(64 * 1024)); + let first = journal.retain_repair_from(2, budget.reservation()); + let second = journal.retain_repair_from(4, budget.reservation()); + for op in 1..=OPS { + journal + .append(build_prepare(op, HEADER_SIZE).into_frozen()) + .await + .unwrap(); + journal.evict_prefix(1); + } + assert!(journal.repair_entry(1).is_none()); + for op in 2..=OPS { + assert_eq!( + journal.repair_entry(op).unwrap().as_slice(), + build_prepare(op, HEADER_SIZE).as_slice() + ); + } + assert_eq!(journal.repair_retained_from(), Some(2)); + assert_eq!(journal.repair_headers_in(2..=OPS).len(), 7); + first.advance(6); + assert_eq!(journal.repair_retained_from(), Some(4)); + drop(second); + assert_eq!(journal.repair_retained_from(), Some(6)); + drop(first); + assert_eq!(journal.repair_retained_from(), Some(OPS)); + assert!( + budget.reservation().try_add(64 * 1024), + "all extra retention must release its budget" + ); + } + + #[compio::test] + async fn given_disk_backed_snapshot_tail_when_headers_leave_the_ring_should_keep_locations_without_bodies() + { + let journal = PartitionJournal::::default(); + journal.set_ring_caps(1, 1); + let budget = Arc::new(RepairMemoryBudget::new(64 * 1024)); + let lease = journal.retain_repair_from(1, budget.reservation()); + let ops = u64::try_from(EVICTED_HEADERS_MAX + 2).unwrap(); + for op in 1..=ops { + journal + .append(build_message_prepare(op, op - 1, 1, 1)) + .await + .unwrap(); + } + journal.evict_prefix_located(usize::try_from(ops).unwrap(), |op| { + Some(BodyPosition { + segment_start: 0, + position: op, + }) + }); + assert!(lease.is_valid()); + assert!(journal.repair_entry(1).is_none()); + assert_eq!(journal.evicted_header(1).unwrap().body.position, 1); + assert_eq!(journal.repair_retained_from(), Some(1)); + assert_eq!(journal.memory_retained_from(), Some(ops)); + journal.forget_segment(0); + assert!( + !lease.is_valid(), + "GC must invalidate the promise before its disk bodies disappear" + ); + assert!(journal.evicted_header(1).is_none()); + assert!(budget.reservation().try_add(64 * 1024)); + } + + #[compio::test] + async fn given_snapshot_tail_and_unproven_walk_when_the_shared_cap_is_full_should_invalidate_without_blocking_writes() + { + let charge = build_prepare(1, HEADER_SIZE) + .retained_allocation_size() + .unwrap() + + size_of::<(u64, JournalBuffer)>(); + let budget = Arc::new(RepairMemoryBudget::new(2 * charge)); + let mut walk = budget.reservation(); + assert!(walk.try_add(charge)); + let journal = PartitionJournal::::default(); + journal.set_ring_caps(1, u64::MAX); + let lease = journal.retain_repair_from(1, budget.reservation()); + for op in 1..=2 { + journal + .append(build_prepare(op, HEADER_SIZE).into_frozen()) + .await + .unwrap(); + journal.evict_prefix(1); + } + assert!(lease.is_valid()); + journal + .append(build_prepare(3, HEADER_SIZE).into_frozen()) + .await + .unwrap(); + journal.evict_prefix(1); + assert!(!lease.is_valid()); + assert_eq!(journal.repair_retained_from(), Some(3)); + assert_eq!(journal.evicted_ring_occupancy(), (1, HEADER_SIZE as u64)); + assert!(budget.reservation().try_add(charge)); + drop(walk); + let replacement = journal.retain_repair_from(3, budget.reservation()); + journal.clear_all(); + assert!(!replacement.is_valid()); + assert_eq!(journal.evicted_ring_occupancy(), (0, 0)); + assert!(budget.reservation().try_add(2 * charge)); + } fn build_prepare(op: u64, size: usize) -> Message { Message::::new(size).transmute_header(|_, h: &mut PrepareHeader| { @@ -1730,6 +2608,123 @@ mod tests { assert!(journal.repair_entry(8).is_none()); } + /// A body that no segment holds stays in memory while a newer body leaves + /// it, so the window that memory holds up to the head starts above the + /// newest released op. + #[compio::test] + async fn given_an_unlocated_body_below_a_released_one_when_asked_for_the_memory_window_should_start_above_the_release() + { + const PAYLOAD_LEN: usize = 64; + let journal = PartitionJournal::::default(); + let body_len = build_message_prepare(1, 0, 1, PAYLOAD_LEN).len() as u64; + journal.set_ring_caps(EVICTED_RING_CAPACITY, 2 * body_len); + for op in 1..=3 { + journal + .append(build_message_prepare(op, op - 1, 1, PAYLOAD_LEN)) + .await + .unwrap(); + } + journal.evict_prefix_located(3, |op| { + (op == 2).then_some(BodyPosition { + segment_start: 0, + position: 0, + }) + }); + assert!(journal.repair_entry(1).is_some()); + assert!(journal.evicted_header(2).is_some()); + assert_eq!(journal.repair_retained_from(), Some(1)); + assert_eq!(journal.memory_retained_from(), Some(3)); + } + + /// Released headers take no room from the bodies the byte cap keeps in + /// memory, and a newest body past the cap takes no header away. + #[compio::test] + async fn given_released_headers_when_bodies_pass_the_byte_cap_should_keep_both_windows() { + const OPS: u64 = 20; + const BODIES_KEPT: u64 = 3; + const PAYLOAD_LEN: usize = 64; + let located = |op| { + Some(BodyPosition { + segment_start: 0, + position: op, + }) + }; + let journal = PartitionJournal::::default(); + let body_len = build_message_prepare(1, 0, 1, PAYLOAD_LEN).len() as u64; + journal.set_ring_caps(EVICTED_RING_CAPACITY, BODIES_KEPT * body_len); + for op in 1..=OPS { + journal + .append(build_message_prepare(op, op - 1, 1, PAYLOAD_LEN)) + .await + .unwrap(); + } + journal.evict_prefix_located(usize::try_from(OPS).unwrap(), located); + for op in 1..=OPS - BODIES_KEPT { + assert!( + journal.evicted_header(op).is_some(), + "op {op} lost its header" + ); + } + for op in OPS - BODIES_KEPT + 1..=OPS { + assert!(journal.repair_entry(op).is_some(), "op {op} lost its body"); + } + + let large = build_message_prepare(OPS + 1, OPS, 1, PAYLOAD_LEN * 64); + assert!(large.len() as u64 > BODIES_KEPT * body_len); + journal.append(large).await.unwrap(); + journal.evict_prefix_located(1, located); + assert_eq!(journal.repair_retained_from(), Some(1)); + for op in 1..=OPS { + assert!( + journal.evicted_header(op).is_some(), + "op {op} lost its header" + ); + } + assert!(journal.repair_entry(OPS + 1).is_some()); + } + + /// Each release makes room in the header window first, so a full window + /// never grows its allocation past its cap. + #[compio::test] + async fn given_more_releases_than_the_header_cap_when_trimming_should_keep_the_deque_within_it() + { + const BATCH: u64 = 1024; + let headers_max = EVICTED_HEADERS_MAX; + let ops = u64::try_from(headers_max).unwrap() + 2; + let journal = PartitionJournal::::default(); + journal.set_ring_caps(EVICTED_RING_CAPACITY, 1); + let template = build_message_prepare(1, 0, 1, 1); + for first in (1..=ops).step_by(usize::try_from(BATCH).unwrap()) { + let last = (first + BATCH - 1).min(ops); + for op in first..=last { + let mut prepare = Message::::new(template.len()); + prepare.as_mut_slice().copy_from_slice(template.as_slice()); + let prepare = prepare.transmute_header(|header, stamped: &mut PrepareHeader| { + *stamped = header; + stamped.op = op; + }); + journal.append(prepare.into_frozen()).await.unwrap(); + } + journal.evict_prefix_located(usize::try_from(last - first + 1).unwrap(), |op| { + Some(BodyPosition { + segment_start: 0, + position: op, + }) + }); + } + let headers = unsafe { &*journal.evicted_headers.get() }; + assert!( + headers.capacity() <= headers_max, + "the header deque grew to {} slots", + headers.capacity() + ); + assert_eq!(headers.len(), headers_max); + assert_eq!( + journal.repair_retained_from(), + Some(ops - u64::try_from(headers_max).unwrap()) + ); + } + #[compio::test] async fn repair_headers_in_serves_the_commit_point_from_the_evicted_ring() { // Blank AT the commit point is the one slot a merge can neither adopt nor @@ -1839,9 +2834,9 @@ mod tests { "read must not evict op 1" ); - let retained = journal.evict_prefix(committed.len()); + journal.evict_prefix(committed.len()); assert_eq!( - retained.len(), + journal.resident_entries().len(), 2, "ops 3 and 4 stay resident after eviction" ); @@ -1852,7 +2847,7 @@ mod tests { let header3 = journal.header_by_op(3).expect("op 3 must be retained"); let header4 = journal.header_by_op(4).expect("op 4 must be retained"); - // Retained entries are still byte-readable after the storage rebuild. + // Retained entries are still byte-readable after the eviction. for header in [header3, header4] { let entry = journal .entry(&header) @@ -1867,8 +2862,7 @@ mod tests { // Advancing the frontier flushes the rest with no gap. let committed = journal.committed_prefix(4); - let rest = journal.evict_prefix(committed.len()); - assert!(rest.is_empty(), "ops 3 and 4 flush on the next evict"); + journal.evict_prefix(committed.len()); assert!(journal.is_empty(), "journal is empty once all ops flushed"); } @@ -1899,8 +2893,12 @@ mod tests { .collect(); assert_eq!(ops, vec![1, 2], "prefix stops before the op 3 gap"); - let retained = journal.evict_prefix(committed.len()); - assert_eq!(retained.len(), 1, "op 4 stays retained past the gap"); + journal.evict_prefix(committed.len()); + assert_eq!( + journal.resident_entries().len(), + 1, + "op 4 stays retained past the gap" + ); assert!(journal.header_by_op(4).is_some(), "op 4 still resident"); } @@ -1998,6 +2996,27 @@ mod tests { ); } + #[compio::test] + async fn given_reordered_duplicates_when_walking_should_follow_op_order_and_first_copy() { + let journal = PartitionJournal::::default(); + for (op, checksum) in [(3, 30), (1, 10), (4, 40), (2, 20), (1, 11), (2, 21)] { + let entry = build_prepare(op, HEADER_SIZE).transmute_header( + |mut old, header: &mut PrepareHeader| { + old.checksum = checksum; + *header = old; + }, + ); + journal.append(entry.into_frozen()).await.unwrap(); + } + + let run: Vec<_> = journal + .committed_headers_from(1, u64::MAX, usize::MAX) + .iter() + .map(|header| (header.op, header.checksum)) + .collect(); + assert_eq!(run, vec![(1, 10), (2, 20), (3, 30), (4, 40)]); + } + /// Three-message batch with the broker append time (`base_timestamp`) /// deliberately AFTER every producer stamp (`origin_timestamp` + deltas), /// the layout every real batch has (the broker stamps later than the @@ -2116,7 +3135,7 @@ mod tests { ); assert_eq!(journal.resident_control_ops(), 3); - // Eviction rebuilds the poll view from the retained tail. + // Eviction drops the evicted batches from the poll view. journal.evict_prefix(2); assert_eq!(entry_ops(&journal.resident_message_entries()), vec![3, 5]); assert_eq!(journal.resident_control_ops(), 2); @@ -2154,6 +3173,196 @@ mod tests { assert_eq!(journal.resident_control_ops(), 0); } + /// What readers see of the resident journal. The storage offsets of an + /// eviction and of a rebuild differ, so entries are read back through them + /// instead of compared. Probing `held_through` also seeds its cached run. + #[derive(Debug, PartialEq)] + struct ResidentView { + entry_ops: Vec, + headers: Vec>, + reads: Vec<(u64, Option)>, + offsets: BTreeMap, + timestamps: BTreeMap<(u64, u64), u64>, + held_through: Vec, + control_ops: usize, + totals: (u64, u64), + out_of_order: bool, + } + + fn resident_view( + journal: &PartitionJournal, + top_op: u64, + ) -> ResidentView { + let inner = unsafe { &*journal.inner.get() }; + let entries = journal.resident_entries(); + let folded = entries.iter().fold((0, 0), |(messages, size), entry| { + let header = + bytemuck::checked::try_from_bytes::(&entry[..PREPARE_HEADER_SIZE]) + .expect("entry holds a valid prepare header"); + RetainedBatchMeta::of(header, entry).map_or((messages, size), |meta| { + ( + messages + u64::from(meta.message_count), + size + meta.total_size, + ) + }) + }); + let totals = (journal.resident_messages(), journal.resident_message_size()); + assert_eq!(totals, folded, "the totals must count the resident batches"); + ResidentView { + entry_ops: entry_ops(&entries), + headers: (0..=top_op).map(|op| journal.header_by_op(op)).collect(), + reads: unsafe { &*journal.op_to_storage_offset.get() } + .iter() + .map(|(&op, &storage_offset)| { + let entry = inner.storage.read_at_sync(storage_offset); + (op, entry.map(|entry| entry_ops(&[entry])[0])) + }) + .collect(), + offsets: unsafe { &*journal.offset_to_op.get() }.clone(), + timestamps: unsafe { &*journal.timestamp_to_op.get() }.clone(), + held_through: (0..=top_op) + .map(|floor| journal.held_through(floor)) + .collect(), + control_ops: journal.resident_control_ops(), + totals, + out_of_order: journal.out_of_order.get(), + } + } + + async fn journal_with( + entries: &[Frozen<4096>], + ) -> PartitionJournal { + let journal = PartitionJournal::::default(); + for entry in entries { + journal.append(entry.clone()).await.expect("append"); + } + journal + } + + #[compio::test] + async fn given_journal_shapes_when_evicting_a_prefix_should_leave_what_a_rebuild_leaves() { + // `(op, base offset)` per entry, `None` for an offset op. The shapes: + // batches between offset ops, two batches at one base offset, and a + // duplicated op that a prefix either takes whole or splits. + let shapes: [&[(u64, Option)]; 3] = [ + &[ + (1, Some(0)), + (2, None), + (3, Some(3)), + (4, None), + (5, Some(6)), + (6, Some(9)), + (7, None), + ], + &[(1, Some(0)), (2, Some(3)), (3, Some(3)), (4, Some(6))], + &[(1, Some(0)), (2, None), (2, None), (3, Some(3)), (4, None)], + ]; + for shape in shapes { + let entries: Vec<_> = shape + .iter() + .zip(0u128..) + .map(|(&(op, base_offset), checksum)| { + base_offset.map_or_else( + || { + build_prepare(op, HEADER_SIZE + 16) + .transmute_header(|mut old, header: &mut PrepareHeader| { + old.checksum = checksum; + *header = old; + }) + .into_frozen() + }, + |base_offset| build_message_prepare(op, base_offset, 3, 8), + ) + }) + .collect(); + let next_op = shape.iter().map(|(op, _)| op).max().unwrap() + 1; + let next = build_message_prepare(next_op, 100, 3, 8); + for count in 0..=entries.len() + 1 { + let popped = journal_with(&entries).await; + let rebuilt = journal_with(&entries).await; + resident_view(&popped, next_op); + resident_view(&rebuilt, next_op); + + popped.evict_prefix(count); + rebuilt.rebuild_without_prefix(count); + assert_eq!( + resident_view(&popped, next_op), + resident_view(&rebuilt, next_op), + "shape {shape:?}, evicting {count}" + ); + + // Positions taken after the eviction must stay right too. + for journal in [&popped, &rebuilt] { + journal.append(next.clone()).await.expect("append"); + } + popped.evict_prefix(1); + rebuilt.rebuild_without_prefix(1); + assert_eq!( + resident_view(&popped, next_op), + resident_view(&rebuilt, next_op), + "shape {shape:?}, evicting {count} and then 1" + ); + } + } + } + + #[compio::test] + async fn given_a_backfill_inside_the_prefix_when_evicting_should_flush_the_tail_in_order() { + let journal = PartitionJournal::::default(); + for op in [1u64, 2, 5, 3, 4] { + journal + .append(build_control_prepare(op)) + .await + .expect("append"); + } + assert_eq!(journal.committed_prefix_len(0, u64::MAX), 0); + + journal.evict_prefix(3); + + assert_eq!(entry_ops(&journal.resident_entries()), vec![3, 4]); + assert_eq!( + journal.committed_prefix_len(0, u64::MAX), + 2, + "the out-of-order pair left with the prefix" + ); + } + + #[compio::test] + async fn given_every_rebuild_when_counting_resident_messages_should_count_the_batches() { + let journal = PartitionJournal::::default(); + for (op, base_offset) in [(1u64, 0u64), (3, 6), (4, 9), (2, 3)] { + journal + .append(build_message_prepare(op, base_offset, 3, 8)) + .await + .expect("append"); + } + let batch_size = journal.resident_message_size() / 4; + assert_eq!(journal.resident_messages(), 12); + + assert!(journal.sort_by_op()); + assert_eq!(journal.resident_messages(), 12, "sorting keeps every batch"); + + journal.evict_prefix(1); + assert_eq!(journal.resident_messages(), 9); + assert_eq!(journal.resident_message_size(), 3 * batch_size); + + journal.truncate_from(4).await.expect("truncate"); + assert_eq!(journal.resident_messages(), 6, "ops 2 and 3 stay"); + + journal.commit(); + assert_eq!(journal.resident_messages(), 0); + + journal + .append(build_message_prepare(5, 12, 3, 8)) + .await + .expect("append"); + journal.clear_all(); + assert_eq!( + (journal.resident_messages(), journal.resident_message_size()), + (0, 0) + ); + } + fn offset_lookup(offset: u64, count: u32) -> MessageLookup { MessageLookup::Offset { offset, diff --git a/core/partitions/src/lib.rs b/core/partitions/src/lib.rs index 1ead193566..085d06f0c2 100644 --- a/core/partitions/src/lib.rs +++ b/core/partitions/src/lib.rs @@ -33,11 +33,13 @@ pub mod offset_storage; mod partition_storage; mod persistence; mod poll_plan; +mod repair_memory; pub use persistence::FileSyncBarrier; pub(crate) use persistence::PersistenceDrain; pub use persistence::{ PartitionPersistence, PersistenceCompletion, PersistenceMetrics, PersistenceNotifier, }; +pub use repair_memory::RepairMemoryBudget; mod segment; pub mod segment_anchor; mod segment_recovery; @@ -70,7 +72,10 @@ pub use io::{ PartitionIoResult, PartitionIoStep, PartitionTeardown, PreparedPrimaryTransfer, largest_legal_job_charge, }; -pub use journal::{EVICTED_RING_BYTES_MAX, EVICTED_RING_CAPACITY}; +pub use journal::{ + BodyPosition, EVICTED_RING_BYTES_MAX, EVICTED_RING_CAPACITY, EvictedHeader, ReleasedBodies, + ReleasedRead, +}; pub use view_fence::ViewWindowVerdict; /// Offsets a partition claims in its superblock ahead of the mint counter diff --git a/core/partitions/src/log.rs b/core/partitions/src/log.rs index da36addce0..ed8bb58755 100644 --- a/core/partitions/src/log.rs +++ b/core/partitions/src/log.rs @@ -351,6 +351,16 @@ where &self.storage } + /// Path of the log file of the segment that starts at `start_offset`. + #[must_use] + pub fn messages_path(&self, start_offset: u64) -> Option { + let slot = self + .segments + .binary_search_by_key(&start_offset, |segment| segment.start_offset) + .ok()?; + self.storage.get(slot)?.segment_and_index_paths().0 + } + pub const fn messages_writers(&self) -> &Vec>> { &self.messages_writers } diff --git a/core/partitions/src/persistence.rs b/core/partitions/src/persistence.rs index 064abbf86a..2d4f093b1d 100644 --- a/core/partitions/src/persistence.rs +++ b/core/partitions/src/persistence.rs @@ -686,6 +686,12 @@ impl PartitionPersistence { !self.retired.get() && self.failure.borrow().is_none() && op <= self.written_head.get() } + /// Where the WAL wrote the body of `op` in a segment, while the WAL still + /// holds the op. + pub fn segment_reference(&self, op: u64) -> Option { + self.segment_references.borrow().get(&op).copied() + } + pub const fn segment_checkpoint(&self) -> Option { self.segment_checkpoint.get() } @@ -1133,6 +1139,11 @@ impl PartitionPersistence { self.accepted_head.get() } + /// The WAL byte budget, `wal_bytes_max` in the server configuration. + pub const fn capacity(&self) -> u64 { + self.capacity + } + pub fn retire(&self) { self.retired.set(true); if let Some(lease) = &self.lease { diff --git a/core/partitions/src/repair_memory.rs b/core/partitions/src/repair_memory.rs new file mode 100644 index 0000000000..dd4dedb29e --- /dev/null +++ b/core/partitions/src/repair_memory.rs @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// Allocation budget shared by unproven repairs and snapshot repair tails. +/// +/// Reservations precede journaling and last until the proof finishes or the +/// retained window is discarded. Normal prepare queues have separate limits. +#[derive(Debug)] +pub struct RepairMemoryBudget { + bytes_max: usize, + used: AtomicUsize, +} + +impl RepairMemoryBudget { + #[must_use] + pub const fn new(bytes_max: usize) -> Self { + Self { + bytes_max, + used: AtomicUsize::new(0), + } + } + + pub(crate) fn reservation(self: &Arc) -> RepairMemoryReservation { + RepairMemoryReservation { + budget: Arc::clone(self), + bytes: 0, + exhausted: false, + } + } +} + +pub struct RepairMemoryReservation { + budget: Arc, + bytes: usize, + exhausted: bool, +} + +impl RepairMemoryReservation { + pub(crate) fn sibling(&self) -> Self { + self.budget.reservation() + } + + pub(crate) const fn bytes(&self) -> usize { + self.bytes + } + + pub(crate) fn try_add(&mut self, bytes: usize) -> bool { + if self.exhausted { + return false; + } + if bytes == 0 { + return true; + } + if self + .budget + .used + .try_update(Ordering::Relaxed, Ordering::Relaxed, |used| { + used.checked_add(bytes) + .filter(|next| *next <= self.budget.bytes_max) + }) + .is_err() + { + self.exhausted = true; + return false; + } + self.bytes += bytes; + true + } + + pub(crate) fn release(&mut self, bytes: usize) { + if bytes == 0 { + return; + } + self.bytes -= bytes; + self.budget.used.fetch_sub(bytes, Ordering::Relaxed); + } + + pub(crate) const fn exhausted(&self) -> bool { + self.exhausted + } + + pub(crate) fn clear(&mut self) { + self.release(self.bytes); + self.exhausted = false; + } +} + +impl Drop for RepairMemoryReservation { + fn drop(&mut self) { + self.clear(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Barrier; + use std::thread; + + #[test] + fn given_concurrent_shards_when_reserving_should_share_one_limit_and_release_on_drop() { + const SHARDS: usize = 8; + const CHARGE: usize = 1024; + const CAPACITY: usize = 3 * CHARGE; + let budget = Arc::new(RepairMemoryBudget::new(CAPACITY)); + let barrier = Arc::new(Barrier::new(SHARDS)); + let accepted = AtomicUsize::new(0); + thread::scope(|scope| { + for _ in 0..SHARDS { + scope.spawn(|| { + let mut reservation = budget.reservation(); + barrier.wait(); + if reservation.try_add(CHARGE) { + accepted.fetch_add(1, Ordering::Relaxed); + } + barrier.wait(); + }); + } + }); + assert_eq!(accepted.load(Ordering::Relaxed), CAPACITY / CHARGE); + assert_eq!(budget.used.load(Ordering::Relaxed), 0); + assert!(budget.reservation().try_add(CAPACITY)); + } +} diff --git a/core/partitions/src/state_transfer.rs b/core/partitions/src/state_transfer.rs index e19bc128c4..bed2c2ad14 100644 --- a/core/partitions/src/state_transfer.rs +++ b/core/partitions/src/state_transfer.rs @@ -29,6 +29,8 @@ use crate::IggyPartition; use crate::iggy_partition::{InstalledConsumerGroupOwner, InstalledPartitionHistory}; +use crate::io::PartitionIoQuiescence; +use crate::journal::RepairTailLease; use crate::offset_storage::{discard_offset_replacement, stage_offset_replacement}; use crate::partition_storage::{read_revision_record, write_revision_record}; use crate::segment_anchor::ANCHOR_SUFFIX; @@ -58,13 +60,18 @@ use server_common::iobuf::Owned; use server_common::send_messages::{decode_batch_slice, decode_prepare_slice}; use server_common::sharding::IggyNamespace; use server_common::yield_to_reactor; -use std::collections::{BTreeSet, HashSet}; +use std::cell::{Cell, RefCell}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::fmt; use std::future::Future; use std::mem::size_of; use std::path::{Path, PathBuf}; use std::rc::Rc; use std::sync::atomic::Ordering; +use std::task::Poll; + +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; /// Current state-transfer offsets format, including the prepare-chain anchor. pub(crate) const CONSUMER_OFFSETS_MAGIC: [u8; 4] = *b"ICO1"; @@ -135,9 +142,211 @@ pub struct PartitionTransferSession { /// Whether a descriptor has been accepted (an accepted EMPTY manifest is /// distinguishable from "still waiting"). pub target_accepted: bool, - /// Ticks with no frame progress; at the configured repair-retry - /// threshold the missing piece is re-requested. + /// Ticks since the last frame progress or chunk request. At the + /// configured repair-retry threshold, the missing piece is requested + /// again. pub idle_ticks: u32, + /// The reuse scan of the accepted manifest. The pull waits for it, + /// because the scan writes the staging names that the pull spills to. + pub scan: Option>, + /// The completed segment being validated and synced outside the pump. + pub spill: Option, + /// The leading bytes of one segment artifact that this replica already + /// holds, so the pull asks only for the rest. + pub seed: Option, + /// Chunk payload bytes that crossed the network in this session. + pub pulled_len: u64, +} + +/// The local copy of the leading bytes of one offered segment artifact. +/// +/// A lagging replica holds a prefix of the segment the primary is still +/// writing: the same base offset and fewer bytes. The pull then fetches only +/// the suffix. The spill checks the whole artifact against the manifest +/// checksum, so a prefix with other bytes costs one more pull of that +/// artifact and nothing else. +#[derive(Debug)] +pub enum PrefixSeed { + /// Chosen when the reuse scan lands. The read starts when the pull + /// reaches `artifact`, so the receiver holds one artifact buffer at a + /// time. + Planned { + artifact: u32, + log_path: String, + len: u64, + }, + /// A detached task reads, hashes and walks the prefix. + Reading { + artifact: u32, + read: LocalWork, + }, + /// The prefix bytes sit in the artifact buffer. + Seeded { artifact: u32, prefix: SeededPrefix }, +} + +/// Work that a detached task runs for a transfer session. +/// +/// A prefix read and a reuse scan are such work. The pump does not wait for +/// it: the task fills `result` and wakes the partition. Dropping this state +/// asks the task to stop. +#[derive(Debug)] +pub struct LocalWork { + pub result: Rc>>, + pub cancel: LocalWorkCancel, +} + +impl Default for LocalWork { + fn default() -> Self { + Self { + result: Rc::default(), + cancel: LocalWorkCancel::default(), + } + } +} + +impl LocalWork { + /// Whether the task finished, so the session can go on. + #[must_use] + pub fn landed(&self) -> bool { + self.result.borrow().is_some() + } + + /// Asks the task to stop at its next check. Returns `false` when it was + /// asked before. + #[must_use] + pub fn stop(&self) -> bool { + !self.cancel.0.replace(true) + } +} + +/// Asks local work to stop when the state that waits for it drops. +/// +/// That state drops with its session. A wait that passes its bound asks with +/// [`LocalWork::stop`] instead, and keeps the state until the work lands. The +/// task checks the flag between its steps. A prefix read then frees its +/// buffer of up to one segment early. +#[derive(Debug, Default)] +pub struct LocalWorkCancel(Rc>); + +impl Drop for LocalWorkCancel { + fn drop(&mut self) { + self.0.set(true); + } +} + +/// What a reuse scan staged without a pull. +#[derive(Debug, Default)] +pub struct ReuseScanned { + /// The manifest index and the staged meta of each segment that the scan + /// linked or adopted. + pub reused: Vec<(u32, StagedSegmentMeta)>, + /// Verified files that a cancelled scan did not revisit. + pub(crate) memoized: BTreeMap, +} + +#[derive(Debug)] +/// Owns the completed artifact while the pull waits for its staging writer. +pub struct SegmentSpill { + pub artifact: usize, + pub seeded: bool, + pub work: LocalWork>, +} + +/// A durable staging pair and the checksum state needed if it becomes active. +pub struct SpilledSegment { + meta: StagedSegmentMeta, + hasher: SpilledHasher, +} + +impl fmt::Debug for SpilledSegment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SpilledSegment") + .field("meta", &self.meta) + .finish_non_exhaustive() + } +} + +/// What a landed reuse scan staged, for the log of the receiver. +#[derive(Debug, Default)] +pub struct ReusedSegments { + /// Staged copies of an earlier attempt. + pub adopted: usize, + pub adopted_len: u64, + /// Sealed segments that this replica held. + pub linked: usize, + pub linked_len: u64, + pub artifacts: usize, + pub total_len: u64, +} + +impl ReusedSegments { + /// `artifacts` are the slots of the session before the scan staged any. + fn new(artifacts: &[TransferArtifact], reused: &[(u32, StagedSegmentMeta)]) -> Self { + let mut segments = Self { + artifacts: artifacts.len(), + total_len: artifacts + .iter() + .filter_map(TransferArtifact::pending) + .map(|progress| progress.entry.len) + .sum(), + ..Self::default() + }; + for (_, meta) in reused { + if meta.linked { + segments.linked += 1; + segments.linked_len += meta.size; + } else { + segments.adopted += 1; + segments.adopted_len += meta.size; + } + } + segments + } +} + +impl PrefixSeed { + const fn artifact(&self) -> u32 { + match self { + Self::Planned { artifact, .. } + | Self::Reading { artifact, .. } + | Self::Seeded { artifact, .. } => *artifact, + } + } + + /// Whether the prefix read finished and the pull can continue. + #[must_use] + pub fn landed(&self) -> bool { + matches!(self, Self::Reading { read, .. } if read.landed()) + } +} + +/// The prefix bytes of a prefix read, with the hash and walk state after them. +type PrefixRead = std::io::Result<(Vec, SeededPrefix)>; + +/// The hash and walk state after the first `len` bytes of a segment artifact, +/// which the spill continues over the pulled suffix. +pub struct SeededPrefix { + len: u64, + hasher: Box, + /// `None` when the prefix failed its walk. The spill then walks the whole + /// artifact, once the checksum proved that the bytes are the offered ones. + walk: Option, +} + +impl fmt::Debug for SeededPrefix { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SeededPrefix") + .field("len", &self.len) + .finish_non_exhaustive() + } +} + +/// The checksum state of a spilled segment, which can extend as the segment +/// grows. +pub(crate) struct SpilledHasher { + start_offset: u64, + size: u64, + hasher: Box, } /// A scheduled transfer re-arm (see `IggyPartition::transfer_rearm`). @@ -238,14 +447,18 @@ pub struct StagedSegmentMeta { /// `.staging` extension is invisible to boot recovery, which filters on /// `extension == "log"`. pub log_staging: PathBuf, - /// The locally rebuilt sparse index for the staged log, one entry per - /// batch (denser than the origin's per-flush-chunk index; recovery is - /// sparse-tolerant either way). + /// The sparse index rebuilt from the verified log at one entry per + /// `INDEX_STRIDE_BYTES` of payload. pub index_staging: PathBuf, + /// The manifest checksum of the staged log, which the install keeps as + /// the stamp of the installed segment. + pub checksum: u64, + /// The staged log is a hard link to a sealed segment that this replica + /// held, so the transfer neither pulled nor rewrote its payload. + pub linked: bool, } -/// Memoized streaming checksum state for one segment file: the hasher fed -/// exactly `hashed_len` of its bytes, plus the stamp at that length. +/// Memoized checksum state for one segment file. /// /// Keyed per segment rather than per `(base offset, size)` pair so the ACTIVE /// segment extends its own hasher as it grows, instead of missing the memo on @@ -253,35 +466,51 @@ pub struct StagedSegmentMeta { /// new file at an existing base offset (install, converge) clears the whole /// map, which is what makes "same base offset, longer file, same leading bytes" /// hold. -pub(crate) struct SegmentChecksumMemo { - hashed_len: u64, - /// The stamp is NOT cached alongside: `StateArtifactHasher::finish` takes - /// `&self`, so it is a read of this hasher, and a second copy is just a - /// field that can drift. - hasher: StateArtifactHasher, +/// +/// Three paths fill it. The append feeds the bytes of every chunk it +/// materializes, so a segment that this process wrote from its first byte +/// needs no read. The install stamps each segment it installs with the +/// checksum it verified. The offer reads a file only for the bytes that no +/// memo covers, which after a restart is every segment of the first offer. +pub(crate) enum SegmentChecksumMemo { + /// The hasher fed exactly `hashed_len` bytes of the file. A read or an + /// append extends it. + Hashing { + hashed_len: u64, + /// The stamp is NOT cached alongside: `StateArtifactHasher::finish` + /// takes `&self`, so it is a read of this hasher, and a second copy + /// is just a field that can drift. Boxed, because the hasher state is + /// hundreds of bytes and every stamp in the map would pay that size. + hasher: Box, + }, + /// The checksum of the first `len` bytes, as an install verified them or + /// as the hasher of a sealed segment ended. Nothing can extend it, because + /// it holds no hasher. An append drops it, and a read starts again from + /// byte zero. + Stamped { len: u64, checksum: u64 }, } impl SegmentChecksumMemo { - fn new() -> Self { - Self { - hashed_len: 0, - hasher: StateArtifactHasher::new(), + /// The artifact stamp over the first `len` bytes, if the memo ends there. + pub(crate) fn checksum_of(&self, len: u64) -> Option { + match self { + Self::Hashing { hashed_len, hasher } => (*hashed_len == len).then(|| hasher.finish()), + Self::Stamped { + len: stamped_len, + checksum, + } => (*stamped_len == len).then_some(*checksum), } } } -/// The result of the last staged-segment reuse scan. +/// Verified staging files, keyed by segment start offset. /// -/// A scan reads every length-matching staged file whole, verifies it, and -/// re-walks every batch -- sequentially, on the pump. Peer rotation and stall -/// re-arms mint a fresh session against the same segment set, so without this -/// the full cost is re-paid per arm. `digest` covers every `SEGMENT_LOG` -/// manifest entry, so a hit means the new offer expects byte-identical staged -/// files; any write to a staging file, or any unlink of one, drops the memo, so -/// a hit can never describe bytes that were replaced meanwhile. +/// Partial scans retain their completed files across retries and advancing +/// offers. Each hit must match the offered length and checksum. Replacing a +/// staging file invalidates its entry; installs and convergence clear the map. +/// Live segments are verified and their indexes rebuilt before entering it. pub(crate) struct ReuseScanMemo { - digest: u64, - adopted: Vec<(u32, StagedSegmentMeta)>, + adopted: BTreeMap, } impl StagedSegmentMeta { @@ -305,6 +534,8 @@ impl StagedSegmentMeta { max_timestamp: stats.max_timestamp, log_staging, index_staging, + checksum: entry.checksum, + linked: false, } } } @@ -909,6 +1140,35 @@ mod tests { use super::*; use iggy_binary_protocol::primitives::partition_history::ConsumerGroupOwner; + #[test] + fn given_a_prefix_read_that_stops_without_a_result_should_land_an_error_and_wake() { + let result = Rc::new(RefCell::new(None)); + let woken = Rc::new(Cell::new(false)); + let io = Rc::new(PartitionIoQuiescence::default()); + assert!(io.begin_local_work()); + let landing = Landing { + io: Rc::clone(&io), + result: Rc::clone(&result), + wake: { + let woken = Rc::clone(&woken); + move || woken.set(true) + }, + stopped: prefix_read_stopped, + }; + + drop(landing); + + assert!( + matches!(result.borrow().as_ref(), Some(Err(_))), + "the pull must see a failed read and fetch the whole artifact" + ); + assert!(woken.get(), "the partition must resume the pull"); + assert!( + io.is_interrupted(), + "teardown must keep interrupted I/O fenced" + ); + } + #[test] fn given_segment_read_error_when_classifying_should_tell_local_faults_from_stale_offers() { const EIO: i32 = 5; @@ -1712,114 +1972,150 @@ pub(crate) async fn walk_segment_payload( base_offset: u64, bytes: &[u8], ) -> Result<(SegmentWalkStats, Vec), SegmentWalkError> { - let mut position = 0usize; - let mut next_offset = base_offset; - let mut stats: Option = None; - let mut index_bytes = Vec::new(); - let mut indexed_position: Option = None; - let mut since_yield = 0usize; - while position < bytes.len() { - // The walk re-hashes every message (`decode_batch_slice` verifies - // `batch_checksum`), so a multi-GiB artifact is a long CPU pass on the - // pump task. What these yields buy is NOT tick liveness: the consensus - // tick is a sibling `select_biased!` arm of this same task and arms are - // not polled while another arm's body awaits, so every group's tick and - // heartbeat on this shard stay frozen for the duration either way (see - // the tick-starvation TODO in `shard::router`). They buy the reactor: - // detached tasks and io_uring completions make progress instead of - // waiting out the whole pass. Moving the verify + walk off the pump is - // what would fix the tick, and the nonce re-check after the spill is - // already shaped for that. - if since_yield >= OFFER_HASH_CHUNK_LEN { - since_yield = 0; - yield_to_reactor().await; - } - let batch = - decode_batch_slice(&bytes[position..]).map_err(|source| SegmentWalkError::Batch { - position: position as u64, - source, - })?; - let header = batch.header; - if stats.is_none() && header.base_offset != base_offset { - return Err(SegmentWalkError::BaseOffsetMismatch { - expected: base_offset, - actual: header.base_offset, - }); - } - if header.base_offset != next_offset { - return Err(SegmentWalkError::NonContiguous { - expected: next_offset, - actual: header.base_offset, - }); - } - if header.message_count == 0 { - return Err(SegmentWalkError::Batch { - position: position as u64, - source: iggy_common::IggyError::InvalidMessagesCount, - }); + let mut walk = SegmentWalk::new(base_offset); + walk.advance(bytes).await?; + walk.finish() +} + +/// A walk that stops at the end of the bytes it was given and resumes there, +/// so a seeded prefix and the suffix that completes it are each walked once. +pub(crate) struct SegmentWalk { + base_offset: u64, + /// Payload bytes walked so far. + position: usize, + next_offset: u64, + stats: Option, + index_bytes: Vec, + indexed_position: Option, +} + +impl SegmentWalk { + pub(crate) const fn new(base_offset: u64) -> Self { + Self { + base_offset, + position: 0, + next_offset: base_offset, + stats: None, + index_bytes: Vec::new(), + indexed_position: None, } - // Peer-controlled operands under a reject-on-first-invalid contract: - // checked, not saturating -- a clamp would misdirect the diagnostic. - let Some(batch_end) = header - .base_offset - .checked_add(u64::from(header.message_count) - 1) - else { - return Err(SegmentWalkError::OffsetOverflow { - position: position as u64, - }); - }; - // The append-time canonical stamp, exactly what the flush path writes - // into index entries and segment bounds; `origin_timestamp` is - // client-supplied and would give the installed replica a divergent - // timestamp column (polls and retention keyed differently per node). - let timestamp = header.base_timestamp; - // STRIDED, not one entry per batch: the origin writes one entry per - // flush chunk, and a per-batch index is dense enough that a transferred - // segment never fits the sealed-index residency cap - // (`poll_plan::SEALED_INDEX_RESIDENT_MAX_BYTES`), so every sealed poll - // would fall back to binary-searching the file with single-entry preads - // -- a slow path `poll_plan` reserves for a - // `messages_required_to_save = 1` misconfiguration, which a transfer - // would otherwise produce unconditionally. Both consumers do lower-bound - // lookups and recovery walks forward from the last entry by design, so - // sparser is correct; the first batch always gets one. - let stride_reached = indexed_position - .is_none_or(|indexed| position.saturating_sub(indexed) >= INDEX_STRIDE_BYTES); - if stride_reached { - indexed_position = Some(position); - index_bytes.extend_from_slice(&header.base_offset.to_le_bytes()); - index_bytes.extend_from_slice(×tamp.to_le_bytes()); - index_bytes.extend_from_slice(&(position as u64).to_le_bytes()); - } - stats = Some(stats.map_or( - SegmentWalkStats { - end_offset: batch_end, - start_timestamp: timestamp, - end_timestamp: timestamp, - max_timestamp: timestamp, - }, - |previous| SegmentWalkStats { - end_offset: batch_end, - start_timestamp: previous.start_timestamp, - end_timestamp: timestamp, - max_timestamp: previous.max_timestamp.max(timestamp), - }, - )); - next_offset = batch_end - .checked_add(1) - .ok_or(SegmentWalkError::OffsetOverflow { - position: position as u64, + } + + /// Walk `bytes`, the payload from its first byte, from where the last + /// call stopped to its end. + async fn advance(&mut self, bytes: &[u8]) -> Result<(), SegmentWalkError> { + let mut since_yield = 0usize; + while self.position < bytes.len() { + // The walk re-hashes every message (`decode_batch_slice` verifies + // `batch_checksum`), so a multi-GiB artifact is a long CPU pass. On the + // pump task, what these yields buy is NOT tick liveness: the consensus + // tick is a sibling `select_biased!` arm of this same task and arms are + // not polled while another arm's body awaits, so every group's tick and + // heartbeat on this shard stay frozen for the duration either way (see + // the tick-starvation TODO in `shard::router`). They buy the reactor: + // detached tasks and io_uring completions make progress instead of + // waiting out the whole pass. In a detached task, such as the prefix + // seed read, the same yields let the pump run. + if since_yield >= OFFER_HASH_CHUNK_LEN { + since_yield = 0; + yield_to_reactor().await; + } + let batch = decode_batch_slice(&bytes[self.position..]).map_err(|source| { + SegmentWalkError::Batch { + position: self.position as u64, + source, + } })?; - // No trailing-bytes guard: the header decode floors `batch_length` - // at the 256-byte command header (no zero-step loop is possible) - // and `decode_batch_slice` already rejects a body shorter than - // `total_size()`. - position += header.total_size(); - since_yield += header.total_size(); - } - stats.map_or(Err(SegmentWalkError::Empty), |stats| { - Ok((stats, index_bytes)) - }) + let header = batch.header; + if self.stats.is_none() && header.base_offset != self.base_offset { + return Err(SegmentWalkError::BaseOffsetMismatch { + expected: self.base_offset, + actual: header.base_offset, + }); + } + if header.base_offset != self.next_offset { + return Err(SegmentWalkError::NonContiguous { + expected: self.next_offset, + actual: header.base_offset, + }); + } + if header.message_count == 0 { + return Err(SegmentWalkError::Batch { + position: self.position as u64, + source: iggy_common::IggyError::InvalidMessagesCount, + }); + } + // Peer-controlled operands under a reject-on-first-invalid contract: + // checked, not saturating -- a clamp would misdirect the diagnostic. + let Some(batch_end) = header + .base_offset + .checked_add(u64::from(header.message_count) - 1) + else { + return Err(SegmentWalkError::OffsetOverflow { + position: self.position as u64, + }); + }; + // The append-time canonical stamp, exactly what the flush path writes + // into index entries and segment bounds; `origin_timestamp` is + // client-supplied and would give the installed replica a divergent + // timestamp column (polls and retention keyed differently per node). + let timestamp = header.base_timestamp; + // STRIDED, not one entry per batch: the origin writes one entry per + // flush chunk, and a per-batch index is dense enough that a transferred + // segment never fits the sealed-index residency cap + // (`poll_plan::SEALED_INDEX_RESIDENT_MAX_BYTES`), so every sealed poll + // would fall back to binary-searching the file with single-entry preads + // -- a slow path `poll_plan` reserves for a + // `messages_required_to_save = 1` misconfiguration, which a transfer + // would otherwise produce unconditionally. Both consumers do lower-bound + // lookups and recovery walks forward from the last entry by design, so + // sparser is correct; the first batch always gets one. + let stride_reached = self + .indexed_position + .is_none_or(|indexed| self.position.saturating_sub(indexed) >= INDEX_STRIDE_BYTES); + if stride_reached { + self.indexed_position = Some(self.position); + self.index_bytes + .extend_from_slice(&header.base_offset.to_le_bytes()); + self.index_bytes.extend_from_slice(×tamp.to_le_bytes()); + self.index_bytes + .extend_from_slice(&(self.position as u64).to_le_bytes()); + } + self.stats = Some(self.stats.map_or( + SegmentWalkStats { + end_offset: batch_end, + start_timestamp: timestamp, + end_timestamp: timestamp, + max_timestamp: timestamp, + }, + |previous| SegmentWalkStats { + end_offset: batch_end, + start_timestamp: previous.start_timestamp, + end_timestamp: timestamp, + max_timestamp: previous.max_timestamp.max(timestamp), + }, + )); + self.next_offset = + batch_end + .checked_add(1) + .ok_or(SegmentWalkError::OffsetOverflow { + position: self.position as u64, + })?; + // No trailing-bytes guard: the header decode floors `batch_length` + // at the 256-byte command header (no zero-step loop is possible) + // and `decode_batch_slice` already rejects a body shorter than + // `total_size()`. + self.position += header.total_size(); + since_yield += header.total_size(); + } + Ok(()) + } + + fn finish(self) -> Result<(SegmentWalkStats, Vec), SegmentWalkError> { + self.stats.map_or(Err(SegmentWalkError::Empty), |stats| { + Ok((stats, self.index_bytes)) + }) + } } /// One offered segment: its manifest entry plus WHERE its bytes live. @@ -1900,11 +2196,23 @@ impl PartitionStateTransferOffer { /// these lengths, so hash rounds that span them still describe `commit_op`. pub(crate) struct TransferPlan { commit_op: u64, - /// `(start_offset, len, log_path)` of each non-empty segment. - segments: Vec<(u64, u64, String)>, + /// Each non-empty segment, in chain order. + segments: Vec, offsets: ConsumerOffsetsWire, } +/// One segment as a plan captured it. +pub(crate) struct PlannedSegment { + start_offset: u64, + len: u64, + log_path: String, + /// The artifact stamp over `len` bytes once known. The pin takes it from + /// the memo, or the round whose read completes the segment records it. + /// It is kept here because appends after the pin move the memo of the + /// active segment past `len`, so a later round cannot read it there. + checksum: Cell>, +} + /// How far a serving partition got toward the plan its offer rounds share. pub(crate) enum TransferPlanSlot { /// A request waits for the flush that has to come before the capture. @@ -1921,6 +2229,7 @@ pub(crate) enum TransferPlanSlot { #[derive(Debug)] pub enum PartitionTransferUnavailable { NotCaughtUpPrimary, + RepairTailExpired, /// In-memory / simulated partition: nothing on disk to serve. NoPartitionDir, RepairInProgress, @@ -1984,6 +2293,7 @@ impl PartitionTransferUnavailable { | Self::RepairInProgress | Self::NothingCommitted | Self::SegmentSetChanged + | Self::RepairTailExpired | Self::FlushPending | Self::OfferBuildInProgress { .. } => true, Self::NoPartitionDir @@ -2008,6 +2318,7 @@ impl fmt::Display for PartitionTransferUnavailable { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::NotCaughtUpPrimary => write!(f, "not the caught-up primary of this group"), + Self::RepairTailExpired => write!(f, "the snapshot's retained repair tail expired"), Self::MissingPrepareChecksum { op } => { write!(f, "partition has no checksum for committed op {op}") } @@ -2076,12 +2387,18 @@ pub enum PartitionInstallError { kind: ConsumerKind, }, /// `commit_op` fell below this replica's commit frontier; installing - /// would rewind `commit_min` (the anti-rewind assert, as a refusal) or - /// erase a committed op this replica holds. + /// would rewind `commit_min` (the anti-rewind assert, as a refusal). StaleTransfer { commit_op: u64, commit_min: u64, }, + /// The install wipes the journal, which holds an op above `commit_op` that + /// this replica may have acknowledged: one at or below `held_max`, its + /// accepted WAL head, or the known commit without a WAL. + OfferErasesHeldOps { + commit_op: u64, + held_max: u64, + }, /// The incoming frontier could not be made durable before the swap, so the /// install refuses rather than enter a window whose only durable witness /// would be the segments the failure path quarantines away. @@ -2171,6 +2488,14 @@ impl fmt::Display for PartitionInstallError { f, "transfer frontier {commit_op} is below the local commit frontier {commit_min}" ), + Self::OfferErasesHeldOps { + commit_op, + held_max, + } => write!( + f, + "transfer commit op {commit_op} would erase held ops up to {held_max} that \ + this replica may have acknowledged" + ), Self::FrontierNotDurable { frontier } => write!( f, "could not record the incoming offset frontier {frontier} before the swap" @@ -2231,6 +2556,11 @@ impl From for PartitionInstallError { /// `segment_recovery::sweep_scratch_files_and_collect_offsets`. pub const STAGING_SUFFIX: &str = ".staging"; +/// Extension of the scratch name that a link takes before its rename to the +/// staging name. It ends in `STAGING_SUFFIX`, so the staging sweeps and the +/// boot sweep remove a scratch name that a crash or a failed unlink left. +const LINK_SCRATCH_EXTENSION: &str = "link.staging"; + /// Staging-file names inside the partition directory. fn staging_paths(partition_dir: &str, start_offset: u64) -> (PathBuf, PathBuf) { ( @@ -2631,11 +2961,120 @@ pub(crate) async fn fsync_dir(partition_dir: &str) -> std::io::Result<()> { .await } +pub(crate) struct ServedRepairTail { + nonce: u128, + lease: Option, + last_active_tick: u64, +} + impl IggyPartition where B: MessageBus, SB: SuperblockStore, { + /// Pin before building the manifest: flushing and hashing can span several + /// rounds while writes keep evicting the suffix the receiver will need. + pub fn retain_transfer_tail(&self, peer: u8, nonce: u128, tick: u64) -> bool { + let Some(memory) = &self.proof_walk_memory else { + return true; + }; + let mut tails = self.served_repair_tails.borrow_mut(); + if let Some(tail) = tails.get_mut(&peer).filter(|tail| tail.nonce == nonce) { + tail.last_active_tick = tick; + return tail.lease.as_ref().is_some_and(RepairTailLease::is_valid); + } + let journal = &self.log.journal().inner; + let from_op = journal + .repair_retained_from() + .unwrap_or_else(|| self.consensus().commit_min().saturating_add(1)); + // The replacement must pin before the previous lease drops. + let lease = journal.retain_repair_from(from_op, memory.sibling()); + tails.insert( + peer, + ServedRepairTail { + nonce, + lease: Some(lease), + last_active_tick: tick, + }, + ); + true + } + + /// A descriptor is usable only while its complete post-snapshot suffix is + /// still retained. An exhausted or expired lease needs a new nonce. + pub fn transfer_tail_available(&self, peer: u8, nonce: u128, commit_op: u64) -> bool { + if self.proof_walk_memory.is_none() { + return true; + } + let tails = self.served_repair_tails.borrow(); + let Some(lease) = tails + .get(&peer) + .filter(|tail| tail.nonce == nonce) + .and_then(|tail| tail.lease.as_ref()) + .filter(|lease| lease.is_valid()) + else { + return false; + }; + let from_op = commit_op.saturating_add(1); + if self + .log + .journal() + .inner + .repair_retained_from() + .is_some_and(|oldest| oldest > from_op) + { + return false; + } + lease.advance(from_op); + true + } + + pub fn touch_transfer_tail(&self, peer: u8, nonce: u128, tick: u64) -> bool { + if self.proof_walk_memory.is_none() { + return true; + } + let mut tails = self.served_repair_tails.borrow_mut(); + let Some(tail) = tails.get_mut(&peer).filter(|tail| tail.nonce == nonce) else { + return false; + }; + tail.last_active_tick = tick; + tail.lease.as_ref().is_some_and(RepairTailLease::is_valid) + } + + /// Pull repair can precede a view-fenced progress reply. Its requests keep + /// the suffix alive without claiming that any prefix has been proven. + pub fn renew_transfer_tail_for_repair(&self, peer: u8, tick: u64) { + if let Some(tail) = self.served_repair_tails.borrow_mut().get_mut(&peer) { + tail.last_active_tick = tick; + } + } + + /// Only a correlated replication reply can release the acknowledged prefix. + /// Busy replies renew the lease while the receiver validates and installs. + pub fn advance_transfer_tail( + &self, + peer: u8, + progress: consensus::ReplicationProgress, + tick: u64, + ) { + if let Some(tail) = self.served_repair_tails.borrow_mut().get_mut(&peer) { + tail.last_active_tick = tick; + if let Some(lease) = &tail.lease + && let consensus::ReplicationProgress::Ready { received_op } = progress + { + lease.advance(received_op.saturating_add(1)); + } + } + } + + pub fn expire_transfer_tails(&self, tick: u64, idle_ticks_max: u64) { + for tail in self.served_repair_tails.borrow_mut().values_mut() { + if tick.saturating_sub(tail.last_active_tick) >= idle_ticks_max { + tail.lease = None; + } + } + } + fn plan_transfer_offset_writes( &self, offsets_wire: &ConsumerOffsetsWire, @@ -2696,21 +3135,26 @@ where offer } + /// Whether this replica serves its group's state transfer, which only the + /// primary does. An I/O owner pins the plan in a turn where the segments + /// end at `commit_min`, so a primary still applying what it committed can + /// serve. The inline flush runs through `commit_max` and needs the two + /// equal. + #[must_use] + pub fn serves_state_transfer(&self) -> bool { + if self.has_io_dispatcher() { + consensus::is_settled_primary(self.consensus()) + } else { + consensus::is_caught_up_primary(self.consensus()) + } + } + #[allow(clippy::too_many_lines)] async fn build_state_transfer_offer( &mut self, config: &PartitionsConfig, ) -> Result, PartitionTransferUnavailable> { - // An I/O owner pins the plan in a turn where the segments end at - // `commit_min`, so a primary still applying what it committed can - // serve. The inline flush runs through `commit_max` and needs the two - // equal. - let serving = if self.has_io_dispatcher() { - consensus::is_settled_primary(self.consensus()) - } else { - consensus::is_caught_up_primary(self.consensus()) - }; - if !serving { + if !self.serves_state_transfer() { return Err(PartitionTransferUnavailable::NotCaughtUpPrimary); } if self.partition_dir.is_none() { @@ -2787,37 +3231,44 @@ where // until the offer completes. let mut budget = OFFER_HASH_BUDGET_PER_ROUND_BYTES; let mut segments = Vec::with_capacity(planned.len()); - for (start_offset, size, log_path) in planned { - let Some(checksum) = self - .segment_checksum(*start_offset, *size, log_path, &mut budget) - .await? - else { - // CUMULATIVE across rounds, read back off the memo: per-round - // figures are constant by construction (a partial round always - // spends exactly the budget and always stops inside one - // segment), so they render identically on round 1 and round 30 - // and an operator cannot tell a converging pass from a wedged - // one. This is the only window onto a multi-round build. - let hashed = self.hashed_prefix_len(planned); - let total = planned.iter().map(|(_, size, _)| *size).sum::(); - // The completing round's sweep is skipped on this path, so - // prune here too: retention GC can unlink segments across a - // long build, and their memos would otherwise accumulate until - // some round finally runs the loop to the end. - self.retain_segment_checksum_memos(planned); - return Err(PartitionTransferUnavailable::OfferBuildInProgress { - hashed, - remaining: total.saturating_sub(hashed), - }); + for segment in planned { + let checksum = if let Some(checksum) = segment.checksum.get() { + checksum + } else { + let Some(checksum) = self + .segment_checksum( + segment.start_offset, + segment.len, + &segment.log_path, + &mut budget, + ) + .await? + else { + // CUMULATIVE across rounds, read back off the memo: + // per-round figures are constant by construction (a + // partial round always spends exactly the budget and + // always stops inside one segment), so they render + // identically on round 1 and round 30 and an operator + // cannot tell a converging pass from a wedged one. + // This is the only window onto a multi-round build. + let hashed = self.hashed_prefix_len(planned); + let total = planned.iter().map(|segment| segment.len).sum::(); + return Err(PartitionTransferUnavailable::OfferBuildInProgress { + hashed, + remaining: total.saturating_sub(hashed), + }); + }; + segment.checksum.set(Some(checksum)); + checksum }; segments.push(SegmentArtifactSource { entry: consensus::StateArtifact { kind: artifact_kind::SEGMENT_LOG, - frontier: *start_offset, - len: *size, + frontier: segment.start_offset, + len: segment.len, checksum, }, - log_path: log_path.clone(), + log_path: segment.log_path.clone(), }); } @@ -2837,28 +3288,25 @@ where .collect(); // Append-only within a segment instance, so a live size BELOW the // planned one means the file was replaced rather than extended. - let changed = planned.iter().any(|(start_offset, size, _)| { - live.get(start_offset) - .is_none_or(|live_size| live_size < size) + let changed = planned.iter().any(|segment| { + live.get(&segment.start_offset) + .is_none_or(|live_size| *live_size < segment.len) }); if changed { return Err(PartitionTransferUnavailable::SegmentSetChanged); } - // Sweep memo entries whose segment left the chain (GC), so the map - // tracks the live chain rather than growing with history. - self.segment_checksum_cache - .borrow_mut() - .retain(|start_offset, _| live.contains_key(start_offset)); } - // A completed hash can outlive the repair tail its snapshot needs. - // Re-pin while keeping the memos, so only newly appended bytes are hashed. + // A completed hash can outlive the repair tail its snapshot needs. A + // tail that only segments still hold counts as gone, because repair + // from disk is slow. Re-pin while keeping the memos, so only newly + // appended bytes are hashed. if self.consensus().commit_min() > plan.commit_op && self .log .journal() .inner - .repair_retained_from() + .memory_retained_from() .is_none_or(|retained_from| retained_from > plan.commit_op.saturating_add(1)) { *self.transfer_plan.borrow_mut() = Some(TransferPlanSlot::Wanted); @@ -2891,6 +3339,7 @@ where pub(crate) fn capture_transfer_plan( &self, ) -> Result { + let memos = self.segment_checksum_cache.borrow(); let mut segments = Vec::with_capacity(self.log.segments().len()); for (segment, storage) in self.log.segments().iter().zip(self.log.storages()) { let size = segment.size.as_bytes_u64(); @@ -2904,7 +3353,16 @@ where source: std::io::Error::other("segment holds bytes but no backing file"), }); }; - segments.push((segment.start_offset, size, log_path)); + segments.push(PlannedSegment { + start_offset: segment.start_offset, + len: size, + log_path, + checksum: Cell::new( + memos + .get(&segment.start_offset) + .and_then(|memo| memo.checksum_of(size)), + ), + }); } Ok(TransferPlan { commit_op: self.consensus().commit_min(), @@ -2913,37 +3371,51 @@ where }) } - /// Bytes of `planned` the memo already covers, clamped per segment to the - /// planned length so a memo carrying an active segment's later growth - /// cannot report more than this offer will hash. - fn hashed_prefix_len(&self, planned: &[(u64, u64, String)]) -> u64 { + /// Bytes of `planned` already hashed, clamped per segment to the planned + /// length so a memo carrying an active segment's later growth cannot + /// report more than this offer will hash. + fn hashed_prefix_len(&self, planned: &[PlannedSegment]) -> u64 { let memos = self.segment_checksum_cache.borrow(); planned .iter() - .map(|(start_offset, size, _)| { - memos - .get(start_offset) - .map_or(0, |memo| memo.hashed_len.min(*size)) + .map(|segment| { + if segment.checksum.get().is_some() { + return segment.len; + } + match memos.get(&segment.start_offset) { + Some(SegmentChecksumMemo::Hashing { hashed_len, .. }) => { + (*hashed_len).min(segment.len) + } + Some(SegmentChecksumMemo::Stamped { .. }) | None => 0, + } }) .sum() } - /// Drop memo entries whose segment is no longer in `planned`, which is the - /// live chain as of this round. - /// - /// Set-based rather than a scan per entry: this runs on every - /// budget-exhausted round, inside the frame body the budget exists to - /// bound, and `planned` is capped by `STATE_MANIFEST_ENTRIES_MAX` rather - /// than by anything an operator sized, so the quadratic form dominates the - /// hashing it was meant to make room for. - fn retain_segment_checksum_memos(&self, planned: &[(u64, u64, String)]) { - let live: HashSet = planned - .iter() - .map(|(start_offset, _, _)| *start_offset) - .collect(); - self.segment_checksum_cache - .borrow_mut() - .retain(|start_offset, _| live.contains(start_offset)); + /// Move the append's hasher into its file job. A shorter memo belongs to + /// an offer read and stays available for that read to finish. + pub(crate) fn take_segment_hasher(&self, chunk_start: u64) -> Option> { + if self.consensus().replica_count() == 1 { + return None; + } + self.partition_dir.as_ref()?; + let start_offset = self.log.active_segment().start_offset; + let mut memos = self.segment_checksum_cache.borrow_mut(); + match (chunk_start, memos.remove(&start_offset)) { + (0, _) => Some(Box::default()), + (_, Some(SegmentChecksumMemo::Hashing { hashed_len, hasher })) + if hashed_len == chunk_start => + { + Some(hasher) + } + (_, Some(memo @ SegmentChecksumMemo::Hashing { hashed_len, .. })) + if hashed_len < chunk_start => + { + memos.insert(start_offset, memo); + None + } + _ => None, + } } /// The artifact stamp over the first `size` bytes of a segment file, @@ -2975,12 +3447,16 @@ where .segment_checksum_cache .borrow_mut() .remove(&start_offset); - let mut memo = match memo { + let (hashed_len, mut hasher) = match memo { // `<=`, so the already-hashed case falls through to the shared tail: // `hash_segment_range` returns before opening the file when // `from == to`, and the finish + reinsert below is the same work the // separate arm did. - Some(memo) if memo.hashed_len <= size => memo, + Some(SegmentChecksumMemo::Hashing { hashed_len, hasher }) if hashed_len <= size => { + (hashed_len, hasher) + } + // A stamp holds no hasher to resume, so the read starts at byte 0. + Some(SegmentChecksumMemo::Stamped { .. }) => (0, Box::default()), // Segment bytes are append-only within one segment instance (the // failed-index-save path rewinds the writer cursor and returns // BEFORE the size increment), and every path that plants a fresh @@ -2991,34 +3467,60 @@ where shrunk.is_none(), "segment {start_offset} shrank to {size} bytes below its memo" ); - SegmentChecksumMemo::new() + (0, Box::default()) } }; // Clamped to the round's remaining budget, so a single multi-GiB // segment is split across rounds rather than being the granularity // floor. `finish` does not consume the hasher, so a partial pass is // simply a memo nobody stamps yet. - let target = size.min(memo.hashed_len.saturating_add(*budget)); - let hashed = target.saturating_sub(memo.hashed_len); + let target = size.min(hashed_len.saturating_add(*budget)); + let spent = target.saturating_sub(hashed_len); // Dropped on failure, not reinserted: the hasher is fed chunk by chunk // and a mid-range error leaves it holding bytes `hashed_len` does not // account for, so resuming from it would stamp a checksum over a // doubly-fed prefix. Losing the partial pass is the cheap side. - hash_segment_range(log_path, memo.hashed_len, target, &mut memo.hasher, None) + hash_segment_range(log_path, hashed_len, target, &mut hasher, None) .await .map_err(|source| PartitionTransferUnavailable::SegmentUnreadable { start_offset, source, })?; - memo.hashed_len = target; - let checksum = (target == size).then(|| memo.hasher.finish()); - self.segment_checksum_cache - .borrow_mut() - .insert(start_offset, memo); - *budget = budget.saturating_sub(hashed); + let checksum = (target == size).then(|| hasher.finish()); + self.segment_checksum_cache.borrow_mut().insert( + start_offset, + SegmentChecksumMemo::Hashing { + hashed_len: target, + hasher, + }, + ); + self.stamp_sealed_segment(start_offset); + *budget = budget.saturating_sub(spent); Ok(checksum) } + /// Replace the hasher of a sealed segment with its stamp once the memo + /// covers every byte. Nothing appends to a sealed segment, so its hasher + /// can never take another byte, and the stamp is 16 bytes where the + /// hasher is hundreds in two heap blocks. + pub(crate) fn stamp_sealed_segment(&self, start_offset: u64) { + let segments = self.log.segments(); + let Ok(position) = + segments.binary_search_by_key(&start_offset, |segment| segment.start_offset) + else { + return; + }; + let segment = &segments[position]; + let len = segment.size.as_bytes_u64(); + let mut memos = self.segment_checksum_cache.borrow_mut(); + if segment.sealed + && let Some(memo) = memos.get_mut(&start_offset) + && let Some(checksum) = memo.checksum_of(len) + { + *memo = SegmentChecksumMemo::Stamped { len, checksum }; + } + } + /// Release the cached offer once no requester holds one (the shard's /// offer-expiry sweep), and the plan of a build in progress with it: both /// name segments that retention may be about to unlink. @@ -3151,198 +3653,363 @@ where self.offsets_wire_snapshot().map(|wire| wire.consumers) } - /// Validate one completed `SEGMENT_LOG` artifact and spill it to staging - /// files, returning the walk metadata. Frees receiver memory as it goes: - /// after this the session drops the artifact's buffer. - /// - /// # Errors - /// `Err(walk error description)` when the payload fails validation (the - /// caller charges the decode budget), or a staging-write failure - /// description. - pub async fn spill_transfer_segment( + /// Validate and sync a completed segment without holding the shard pump. + /// Staging writers are serialized, including work of abandoned sessions. + pub fn begin_transfer_segment_spill( + &mut self, + entry: &consensus::StateArtifact, + bytes: Vec, + prefix: Option, + ) -> LocalWork> { + let job = self.plan_segment_spill(entry, bytes, prefix); + self.spawn_staging_work(segment_spill_stopped, move |cancelled| async move { + job?.run(&cancelled).await + }) + } + + /// Publish only the result owned by the current transfer session. + pub fn accept_transfer_segment_spill(&self, spilled: SpilledSegment) -> StagedSegmentMeta { + let SpilledSegment { meta, hasher } = spilled; + *self.spilled_hasher.borrow_mut() = Some(hasher); + self.reuse_scan_memo + .borrow_mut() + .get_or_insert_with(|| ReuseScanMemo { + adopted: BTreeMap::new(), + }) + .adopted + .insert(meta.start_offset, meta.clone()); + meta + } + + fn plan_segment_spill( + &self, + entry: &consensus::StateArtifact, + bytes: Vec, + prefix: Option, + ) -> Result { + let partition_dir = self + .partition_dir + .clone() + .ok_or(SpillError::NoPartitionDir)?; + // A replacement must invalidate the proof before any writer starts. + if let Some(memo) = self.reuse_scan_memo.borrow_mut().as_mut() { + memo.adopted.remove(&entry.frontier); + } + Ok(SegmentSpillJob { + partition_dir, + entry: *entry, + bytes, + prefix, + }) + } + + #[cfg(test)] + pub(crate) async fn spill_transfer_segment( &self, entry: &consensus::StateArtifact, bytes: Vec, + prefix: Option, ) -> Result { - let Some(partition_dir) = self.partition_dir.clone() else { - return Err(SpillError::NoPartitionDir); + let spilled = self + .plan_segment_spill(entry, bytes, prefix)? + .run(&Cell::new(false)) + .await?; + Ok(self.accept_transfer_segment_spill(spilled)) + } + + /// Start the reuse scan of `manifest` in a detached task, so that its + /// links, syncs and reads of staged copies do not hold the shard pump. + /// + /// The scan stages every segment of the manifest that this replica can + /// supply without a pull: a sealed live segment that holds the bytes of + /// an entry, through hard links, or else the staged copy of an earlier + /// attempt. It sweeps the staging strays that match no entry. Scans and + /// spills wait for the preceding writer, so an abandoned task cannot race + /// the next session on the same staging names. + pub fn begin_reuse_scan( + &mut self, + manifest: &[consensus::StateArtifact], + ) -> LocalWork { + let job = self.plan_reuse_scan(manifest); + self.spawn_staging_work(ReuseScanned::default, move |cancelled| async move { + match job { + Some(job) => job.run(&cancelled).await, + None => ReuseScanned::default(), + } + }) + } + + /// Hand a landed reuse scan to the session: stage the segments that it + /// reused, and plan the prefix seed among the rest. Returns `Pending` + /// while the scan runs, because the pull waits for it. The call that + /// finds the scan landed returns what it reused. + pub fn drive_reuse_scan(&mut self) -> Poll> { + let Some(scan) = self + .transfer + .as_ref() + .and_then(|session| session.scan.as_ref()) + else { + return Poll::Ready(None); }; - // This write replaces whatever the reuse memo recorded for this path, - // so the memo cannot outlive it: a later scan against an older offer - // must re-read the file rather than trust a walk of the old bytes. - self.reuse_scan_memo.borrow_mut().take(); - // Artifact-level integrity FIRST, exactly as the metadata plane - // verifies every artifact before decoding: the walk's per-batch - // checksums prove batch bodies, not that these are the bytes the - // manifest promised (length alone is implied by completion). - if !verify_state_artifact_yielding(entry, &bytes).await { - return Err(SpillError::ManifestChecksum { - frontier: entry.frontier, - }); + let Some(scanned) = scan.result.borrow_mut().take() else { + return Poll::Pending; + }; + let reused = self.accept_reuse_scan(scanned); + let Some(session) = self.transfer.as_mut() else { + return Poll::Ready(None); + }; + let segments = ReusedSegments::new(&session.artifacts, &reused); + session.scan = None; + for (index, meta) in reused { + session.artifacts[index as usize] = TransferArtifact::Staged(meta); } - let (stats, index_bytes) = walk_segment_payload(entry.frontier, &bytes) - .await - .map_err(SpillError::Walk)?; - let (log_staging, index_staging) = staging_paths(&partition_dir, entry.frontier); - let index_size = index_bytes.len() as u64; - // Two writes, not a loop: each moves its buffer into compio's - // owned-buffer API, so a segment-sized payload is never copied. - write_staging_file(&log_staging, bytes) - .await - .map_err(|source| SpillError::StagingIo { - path: log_staging.clone(), - source, - })?; - write_staging_file(&index_staging, index_bytes) - .await - .map_err(|source| SpillError::StagingIo { - path: index_staging.clone(), - source, - })?; - fsync_dir(&partition_dir) - .await - .map_err(|source| SpillError::StagingIo { - path: PathBuf::from(&partition_dir), - source, - })?; - Ok(StagedSegmentMeta::from_walk( - entry, - stats, - index_size, - log_staging, - index_staging, - )) + let seed = self + .transfer + .as_ref() + .and_then(|session| self.plan_prefix_seed(&session.artifacts)); + if let Some(session) = self.transfer.as_mut() { + session.seed = seed; + } + // The I/O turn that reports a landed scan leaves the ready queue ahead + // of any other I/O work that is due, so that work takes the next turn + // instead of the next notification. + self.notify_io(); + Poll::Ready(Some(segments)) } - /// Adopt an already-verified staged log without rewriting it: walk the - /// payload once (validation + a rebuilt sparse index), write only the - /// index sidecar, and return the walk metadata. The reuse scan calls - /// this after `verify_state_artifact` proved the bytes match the - /// manifest; rewriting the byte-identical log (and re-verifying a third - /// time) is exactly the work reuse exists to skip. The index write - /// stays: the scan never checks `.index.staging`, and a missing sidecar - /// would hand the install a missing rename source. - /// - /// No directory fsync here: every sidecar lands in the same directory, so - /// the caller fsyncs ONCE after its loop instead of once per adoption. - async fn adopt_staged_segment( - &self, - entry: &consensus::StateArtifact, - bytes: &[u8], - ) -> Result { - let Some(partition_dir) = self.partition_dir.clone() else { - return Err(SpillError::NoPartitionDir); + /// The reuse scan of `manifest`, planned on the pump. `None` without a + /// partition directory. + fn plan_reuse_scan(&self, manifest: &[consensus::StateArtifact]) -> Option { + let partition_dir = self.partition_dir.clone()?; + // The scan owns the memo while it can replace staging names. + let memoized = self + .reuse_scan_memo + .borrow_mut() + .take() + .map_or_else(BTreeMap::new, |memo| memo.adopted); + let entries = manifest + .iter() + .zip(0u32..) + .filter(|(entry, _)| entry.kind == artifact_kind::SEGMENT_LOG) + .map(|(entry, index)| (index, *entry, self.held_segment(entry))) + .collect(); + Some(ReuseScanJob { + partition_dir, + namespace_raw: self.consensus().group(), + memoized, + entries, + }) + } + + /// Select a sealed segment by its append checksum. The detached scan + /// verifies its on-disk bytes before reusing it. + fn held_segment(&self, entry: &consensus::StateArtifact) -> Option { + let position = self.sealed_segment_holding(entry.frontier, entry.len, entry.checksum)?; + let (Some(log_path), _) = self.log.storages()[position].segment_and_index_paths() else { + return None; }; - let (stats, index_bytes) = walk_segment_payload(entry.frontier, bytes) - .await - .map_err(SpillError::Walk)?; - let (log_staging, index_staging) = staging_paths(&partition_dir, entry.frontier); - let index_size = index_bytes.len() as u64; - write_staging_file(&index_staging, index_bytes) - .await - .map_err(|source| SpillError::StagingIo { - path: index_staging.clone(), - source, - })?; - Ok(StagedSegmentMeta::from_walk( - entry, - stats, - index_size, - log_staging, - index_staging, - )) + Some(HeldSegment { log_path }) } - /// Scan the partition directory for staging files left by an earlier - /// session and adopt every one that matches a manifest entry byte-for- - /// byte (length + artifact checksum + full re-walk). Sealed segments are - /// immutable, so on a retry or peer re-target typically only the active - /// segment and the offsets artifact re-pull. Staging strays matching no - /// entry are swept. + /// The position of the sealed live segment at `start_offset` whose memo + /// stamps its `len` bytes with `checksum`. + fn sealed_segment_holding(&self, start_offset: u64, len: u64, checksum: u64) -> Option { + let segments = self.log.segments(); + let position = segments + .binary_search_by_key(&start_offset, |segment| segment.start_offset) + .ok()?; + let segment = &segments[position]; + let stamp = self + .segment_checksum_cache + .borrow() + .get(&start_offset) + .and_then(|memo| memo.checksum_of(len)); + (segment.sealed && segment.size.as_bytes_u64() == len && stamp == Some(checksum)) + .then_some(position) + } + + /// Keep verified files, including progress made before a scan stopped. /// - /// A scan against a segment set this partition already scanned short- - /// circuits through `ReuseScanMemo`: rotating to another peer would - /// otherwise re-read and re-walk every staged file, up to 2 GiB each, - /// sequentially, on the pump. - pub async fn reuse_staged_segments( + /// The pump ran while the scan did, so a linked segment must still be + /// sealed and stamped as the manifest expects. Retention can have + /// removed it. An adopted copy needs no check, because only a scan, its + /// session and the install of that session write staging names, and the + /// session neither pulls nor installs before its scan lands. + pub(crate) fn accept_reuse_scan( &self, - manifest: &[consensus::StateArtifact], + mut scanned: ReuseScanned, ) -> Vec<(u32, StagedSegmentMeta)> { - let Some(partition_dir) = self.partition_dir.clone() else { - return Vec::new(); + let valid = |meta: &StagedSegmentMeta| { + !meta.linked + || self + .sealed_segment_holding(meta.start_offset, meta.size, meta.checksum) + .is_some() }; - let digest = segment_manifest_digest(manifest); - // Cloned out of the borrow: the file re-checks below await. - let memoized = self - .reuse_scan_memo - .borrow() - .as_ref() - .filter(|memo| memo.digest == digest) - .map(|memo| memo.adopted.clone()); - if let Some(adopted) = memoized { - // The memo proves the bytes were validated; only their continued - // presence needs re-checking (an install or a converge between the - // two scans unlinks them). - let mut intact = true; - for (_, meta) in &adopted { - let log_matches = matches!( - compio::fs::metadata(&meta.log_staging).await, - Ok(metadata) if metadata.len() == meta.size - ); - if !log_matches || compio::fs::metadata(&meta.index_staging).await.is_err() { - intact = false; - break; - } - } - if intact { - return adopted; - } - self.reuse_scan_memo.borrow_mut().take(); + let reused: Vec<_> = scanned + .reused + .into_iter() + .filter(|(_, meta)| valid(meta)) + .collect(); + scanned.memoized.retain(|_, meta| valid(meta)); + for (_, meta) in &reused { + scanned.memoized.insert(meta.start_offset, meta.clone()); } - let mut adopted = Vec::new(); - let mut matched_paths = Vec::new(); - for (index, entry) in manifest.iter().enumerate() { + *self.reuse_scan_memo.borrow_mut() = + (!scanned.memoized.is_empty()).then_some(ReuseScanMemo { + adopted: scanned.memoized, + }); + reused + } + + /// The segment artifact whose leading bytes this replica holds: a live + /// segment at the base offset of an entry that no reuse claimed, and + /// shorter than that entry. Such a segment is the one the primary kept + /// writing after this replica fell behind. The spill proves the prefix. + fn plan_prefix_seed(&self, artifacts: &[TransferArtifact]) -> Option { + let segments = self.log.segments(); + artifacts.iter().zip(0u32..).find_map(|(slot, artifact)| { + let entry = slot.pending()?.entry; if entry.kind != artifact_kind::SEGMENT_LOG { - continue; + return None; } - let (log_staging, _) = staging_paths(&partition_dir, entry.frontier); - // Length short-circuit BEFORE reading: length is the first - // conjunct of the artifact check anyway, and the common retry - // case is precisely an active-segment length mismatch -- no - // point reading up to 2 GiB just to discard it. - match compio::fs::metadata(&log_staging).await { - Ok(metadata) if metadata.len() == entry.len => {} - _ => continue, + let position = segments + .binary_search_by_key(&entry.frontier, |segment| segment.start_offset) + .ok()?; + let len = segments[position].size.as_bytes_u64(); + if len == 0 || len >= entry.len { + return None; } - let Ok(bytes) = compio::fs::read(&log_staging).await else { - continue; - }; - if !verify_state_artifact_yielding(entry, &bytes).await { - continue; + let (log_path, _) = self.log.storages()[position].segment_and_index_paths(); + Some(PrefixSeed::Planned { + artifact, + log_path: log_path?, + len, + }) + }) + } + + /// Start or finish the prefix read of the artifact that the pull reached. + /// + /// Returns `true` while the read runs. The next chunk request then waits, + /// because that request asks for the bytes that the read supplies. The read runs + /// in a detached task, so a read of up to a whole segment does not hold the + /// pump. A failed read leaves the artifact to a whole pull. + pub fn drive_prefix_seed(&mut self) -> bool { + let namespace_raw = self.namespace().inner(); + let Some(session) = self.transfer.as_mut() else { + return false; + }; + let next = session + .artifacts + .iter() + .position(|artifact| !consensus::ChunkProgress::complete(artifact)); + if session + .seed + .as_ref() + .is_none_or(|seed| Some(seed.artifact() as usize) != next) + { + return false; + } + match session.seed.take() { + Some(PrefixSeed::Planned { + artifact, + log_path, + len, + }) => { + let Some(entry) = session.artifacts[artifact as usize] + .pending() + .map(|progress| progress.entry) + else { + return false; + }; + let read = + self.spawn_local_work(prefix_read_stopped, move |cancelled| async move { + read_segment_prefix(&log_path, entry, len, &cancelled).await + }); + if let Some(session) = self.transfer.as_mut() { + session.seed = Some(PrefixSeed::Reading { artifact, read }); + } + true } - if let Ok(meta) = self.adopt_staged_segment(entry, &bytes).await { - matched_paths.push(meta.log_staging.clone()); - matched_paths.push(meta.index_staging.clone()); - #[allow(clippy::cast_possible_truncation)] - adopted.push((index as u32, meta)); + Some(PrefixSeed::Reading { artifact, read }) => { + let Some(landed) = read.result.borrow_mut().take() else { + session.seed = Some(PrefixSeed::Reading { artifact, read }); + return true; + }; + match (landed, session.artifacts[artifact as usize].pending_mut()) { + (Ok((bytes, prefix)), Some(progress)) if progress.buf.is_empty() => { + tracing::info!( + namespace_raw, + artifact, + seeded_len = prefix.len, + artifact_len = progress.entry.len, + "seeded a partition transfer artifact from the local segment" + ); + progress.buf = bytes; + session.seed = Some(PrefixSeed::Seeded { artifact, prefix }); + } + (Err(error), _) => tracing::info!( + namespace_raw, + artifact, + %error, + "cannot read the local segment prefix, so the pull fetches the \ + whole artifact" + ), + _ => {} + } + // The I/O turn that reports a landed read leaves the ready + // queue ahead of any other I/O work that is due, so that work + // takes the next turn instead of the next notification. + self.notify_io(); + false + } + seed => { + session.seed = seed; + false } } - // Every rebuilt sidecar landed in the same directory, so one fsync - // covers them all. Its failure discards EVERY adoption: the per-adopt - // filter above no longer sees a durability failure, and an undurable - // sidecar handed to the install is a missing rename source after a - // crash. - if !adopted.is_empty() && fsync_dir(&partition_dir).await.is_err() { - adopted.clear(); - matched_paths.clear(); + } + + fn spawn_staging_work + 'static>( + &mut self, + stopped: fn() -> T, + work: impl FnOnce(Rc>) -> F + 'static, + ) -> LocalWork { + let (ended, receiver) = futures::channel::oneshot::channel::<()>(); + let previous = self.staging_work_ended.replace(receiver); + self.spawn_local_work(stopped, move |cancelled| async move { + let _ended = ended; + if let Some(previous) = previous { + let _ = previous.await; + } + work(cancelled).await + }) + } + + /// Run `work` in a detached task, so that the pump does not wait for it. + /// `work` gets the flag that asks it to stop. + fn spawn_local_work + 'static>( + &self, + stopped: fn() -> T, + work: impl FnOnce(Rc>) -> F, + ) -> LocalWork { + let local = LocalWork::default(); + if !self.io_active.begin_local_work() { + *local.result.borrow_mut() = Some(stopped()); + self.notify_io(); + return local; } - // Sweep strays: anything staged that no adopted meta claims. - sweep_staging_except(&partition_dir, matched_paths.into_iter().collect()).await; - *self.reuse_scan_memo.borrow_mut() = Some(ReuseScanMemo { - digest, - adopted: adopted.clone(), + let landing = Landing { + io: Rc::clone(&self.io_active), + result: Rc::clone(&local.result), + wake: self.io_wake(), + stopped, + }; + let work = work(Rc::clone(&local.cancel.0)); + self.consensus().message_bus().spawn(async move { + landing.land(work.await); }); - adopted + local } /// Install a fully transferred partition state: swap the staged segment @@ -4159,7 +4826,9 @@ where fn clear_install_journal(&mut self) { self.log.journal().inner.clear_all(); + self.reset_proof_walk_memory(); self.log.journal_mut().info = crate::log::JournalInfo::default(); + self.consensus().note_journal_mutation(); } /// Clear the offset table for the installed one, which keeps `emptied_at`. @@ -4225,6 +4894,39 @@ where self.set_offset_space_used(install.next_offset > 0); self.recovered_durable_offset = install.staged.last().map(|meta| meta.end_offset); self.installed_frontier = (install.next_offset > 0).then_some(install.next_offset); + // Each staged log matched its manifest checksum, so a later offer or + // reuse scan can trust these stamps without reading the files. + let mut memos: std::collections::HashMap = install + .staged + .iter() + .map(|meta| { + ( + meta.start_offset, + SegmentChecksumMemo::Stamped { + len: meta.size, + checksum: meta.checksum, + }, + ) + }) + .collect(); + // The active segment grows after the install, and its first append + // drops a stamp, so the next transfer after it seals pulls it whole. + // The hasher of its spill extends with each append instead. + if let Some(spilled) = self.spilled_hasher.borrow_mut().take() + && let Some(active) = install.staged.last() + && spilled.start_offset == active.start_offset + && spilled.size == active.size + && spilled.hasher.finish() == active.checksum + { + memos.insert( + active.start_offset, + SegmentChecksumMemo::Hashing { + hashed_len: active.size, + hasher: spilled.hasher, + }, + ); + } + *self.segment_checksum_cache.borrow_mut() = memos; self.stats.zero_out_all(); self.stats.increment_segments_count( u32::try_from(self.log.segments().len()).expect("manifest segment count fits u32"), @@ -4247,17 +4949,10 @@ where install.commit_op, Owned::<4096>::copy_from_slice(&install.offsets_wire.checkpoint_prepare).into(), ); + self.consensus().note_journal_mutation(); } - let consensus = self.consensus(); - if install.commit_op > consensus.commit_min() { - consensus.set_commit_floor(install.commit_op); - } - consensus.sequencer().set_sequence(install.commit_op); - if let Some(checksum) = install.offsets_wire.prepare_checksum { - consensus.set_last_prepare_checksum(checksum); - } - consensus.clear_pipeline(); - consensus.advance_commit_max(install.commit_op); + self.consensus() + .install_checkpoint(install.commit_op, install.offsets_wire.prepare_checksum); // The cleared journal and pipeline held any unapplied install, and the // checkpoint already carries every owner installed up to commit_op. self.pending_owner_install.set(None); @@ -4318,25 +5013,24 @@ where commit_min, }); } - // The install wipes the journal and rewinds the sequencer to - // `commit_op`. The primary may have counted this replica's ack before - // its commit reached us. Acks wait for the contiguous WAL head, so - // protect held ops through that head as well as the known commit. - // Ops known only from heartbeats are not held: a transferring replica - // journals nothing, and repair fetches them after install. Refusing on - // `commit_max` alone would refuse every offer under steady writes: - // heartbeats carry the primary's `commit_min` past whatever op an offer - // pinned. + // The install wipes the journal. The primary may have counted this + // replica's ack before its commit reached us. Acks wait for the + // contiguous WAL head, so protect held ops through that head. Held ops + // above it were never acked, and repair fetches them again after + // install. Protecting them as well refuses every offer under steady + // writes: while the serving peer builds the offer, repair journals the + // committed tail past the op the offer pinned. Without a WAL, protect + // the recorded acknowledgement frontier and the known commit instead. let commit_max = self.consensus().commit_max(); - let held_max = self - .persistence - .as_ref() - .map_or(commit_max, |persistence| persistence.head().max(commit_max)); + let held_max = self.persistence.as_ref().map_or_else( + || commit_max.max(self.acknowledged_op.get()), + |persistence| persistence.head(), + ); let journal = &self.log.journal().inner; if commit_op < held_max && journal.holds_op_in(commit_op + 1..=held_max) { - return Err(PartitionInstallError::StaleTransfer { + return Err(PartitionInstallError::OfferErasesHeldOps { commit_op, - commit_min: held_max, + held_max, }); } let offsets_wire = ConsumerOffsetsWire::decode(offsets_bytes)?; @@ -4549,6 +5243,7 @@ where #[derive(Debug)] pub enum SpillError { NoPartitionDir, + Cancelled, /// The received bytes are not what the manifest promised. ManifestChecksum { frontier: u64, @@ -4566,6 +5261,7 @@ impl fmt::Display for SpillError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::NoPartitionDir => write!(f, "partition has no on-disk directory"), + Self::Cancelled => write!(f, "the transfer abandoned its segment spill"), Self::ManifestChecksum { frontier } => write!( f, "segment artifact at base offset {frontier} fails its manifest checksum" @@ -4614,6 +5310,12 @@ const OFFER_HASH_CHUNK_LEN: usize = 4 << 20; /// every storage class. const OFFER_HASH_BUDGET_PER_ROUND_BYTES: u64 = 256 * 1024 * 1024; +/// Bytes a prefix read reads between two checks of its cancel flag. A dropped +/// seed stops the read within one piece: 64 MiB, under a second on a +/// 125 MB/s cloud volume. Each piece opens the file once and allocates one +/// read chunk, which costs little next to 16 chunk reads. +const PREFIX_READ_PIECE_LEN: u64 = 16 * OFFER_HASH_CHUNK_LEN as u64; + /// Feed bytes `[from, to)` of `path` into `hasher`, read in /// [`OFFER_HASH_CHUNK_LEN`] chunks with one reactor yield per chunk, appending /// each chunk to `sink` when one is given. @@ -4666,11 +5368,8 @@ async fn hash_segment_range( /// against the manifest entry, chunked through `hash_segment_range` with one /// reactor yield per chunk. /// -/// The serving side runs this on the pump to answer a single chunk request, so -/// it reads and hashes in chunks rather than in one pass. The yields keep the -/// REACTOR moving (detached tasks, `io_uring` completions); they do not keep this -/// shard's consensus ticks alive, which are a sibling select arm of the same -/// task and stay frozen for the duration. +/// The serving side caches only artifacts that fit one chunk. Larger files +/// use bounded range reads and are verified by the receiver before install. /// /// The file may legitimately be LONGER than the entry (an active segment that /// kept appending after the offer was built); the artifact is the prefix. @@ -4695,6 +5394,379 @@ pub async fn load_verified_segment_artifact( Ok(bytes) } +/// Read a requested segment range without loading the rest of the file. +/// +/// The caller bounds `len` by the manifest and chunk limit. The receiver +/// verifies the complete artifact before installing it, including its prefix. +/// +/// # Errors +/// Returns the classified open or read error, including a short range. +pub async fn load_segment_chunk( + log_path: &str, + offset: u64, + len: usize, +) -> Result, SegmentLoadError> { + let file = compio::fs::File::open(log_path) + .await + .map_err(SegmentLoadError::classify)?; + let compio::BufResult(result, bytes) = file.read_exact_at(vec![0; len], offset).await; + result.map_err(SegmentLoadError::classify)?; + Ok(bytes) +} + +/// Read, hash and walk the first `len` bytes of a local segment file into a +/// buffer sized for the whole artifact, so the pulled suffix appends in place. +/// Once `cancelled` is set, the read stops before its next piece or the walk +/// with [`std::io::ErrorKind::Interrupted`] and frees the buffer. +pub(crate) async fn read_segment_prefix( + log_path: &str, + entry: consensus::StateArtifact, + len: u64, + cancelled: &Cell, +) -> std::io::Result<(Vec, SeededPrefix)> { + let capacity = usize::try_from(entry.len).map_err(std::io::Error::other)?; + let mut bytes = Vec::with_capacity(capacity); + let mut hasher = Box::::default(); + let mut position = 0; + loop { + if cancelled.get() { + return Err(std::io::Error::new( + std::io::ErrorKind::Interrupted, + "the transfer gave up this prefix read", + )); + } + if position == len { + break; + } + let end = len.min(position.saturating_add(PREFIX_READ_PIECE_LEN)); + hash_segment_range(log_path, position, end, &mut hasher, Some(&mut bytes)).await?; + position = end; + } + let mut walk = SegmentWalk::new(entry.frontier); + let walk = walk.advance(&bytes).await.is_ok().then_some(walk); + Ok((bytes, SeededPrefix { len, hasher, walk })) +} + +/// Lands the result of local work and wakes the partition. A task that stops +/// without a result, such as one that panicked, drops this guard with an +/// empty slot. The guard then lands `stopped()`, so the session does not wait +/// forever. +struct Landing { + io: Rc, + result: Rc>>, + wake: W, + stopped: fn() -> T, +} + +impl Landing { + fn land(self, result: T) { + *self.result.borrow_mut() = Some(result); + } +} + +impl Drop for Landing { + fn drop(&mut self) { + if self.result.borrow().is_none() { + self.io.interrupt(); + } + self.io.settle_local_work(); + self.result.borrow_mut().get_or_insert_with(self.stopped); + (self.wake)(); + } +} + +/// A prefix read that stopped without a result, so the artifact falls back to +/// a whole pull. +fn prefix_read_stopped() -> PrefixRead { + Err(std::io::Error::other( + "the prefix read stopped without a result", + )) +} + +const fn segment_spill_stopped() -> Result { + Err(SpillError::Cancelled) +} + +struct SegmentSpillJob { + partition_dir: String, + entry: consensus::StateArtifact, + bytes: Vec, + prefix: Option, +} + +impl SegmentSpillJob { + async fn run(self, cancelled: &Cell) -> Result { + if cancelled.get() { + return Err(SpillError::Cancelled); + } + let Self { + partition_dir, + entry, + bytes, + prefix, + } = self; + let (mut hasher, walk, hashed_len) = match prefix { + Some(prefix) => (prefix.hasher, prefix.walk, prefix.len), + None => (Box::default(), None, 0), + }; + // Artifact-level integrity FIRST, exactly as the metadata plane + // verifies every artifact before decoding: the walk's per-batch + // checksums prove batch bodies, not that these are the bytes the + // manifest promised (length alone is implied by completion). + let unhashed = usize::try_from(hashed_len) + .ok() + .and_then(|hashed_len| bytes.get(hashed_len..)); + let Some(unhashed) = unhashed.filter(|_| bytes.len() as u64 == entry.len) else { + return Err(SpillError::ManifestChecksum { + frontier: entry.frontier, + }); + }; + hash_yielding(&mut hasher, [unhashed]).await; + if cancelled.get() { + return Err(SpillError::Cancelled); + } + if hasher.finish() != entry.checksum { + return Err(SpillError::ManifestChecksum { + frontier: entry.frontier, + }); + } + let mut walk = walk.unwrap_or_else(|| SegmentWalk::new(entry.frontier)); + walk.advance(&bytes).await.map_err(SpillError::Walk)?; + let (stats, index_bytes) = walk.finish().map_err(SpillError::Walk)?; + if cancelled.get() { + return Err(SpillError::Cancelled); + } + let (log_staging, index_staging) = staging_paths(&partition_dir, entry.frontier); + let index_size = index_bytes.len() as u64; + // Two writes, not a loop: each moves its buffer into compio's + // owned-buffer API, so a segment-sized payload is never copied. + write_staging_file(&log_staging, bytes) + .await + .map_err(|source| SpillError::StagingIo { + path: log_staging.clone(), + source, + })?; + if cancelled.get() { + return Err(SpillError::Cancelled); + } + write_staging_file(&index_staging, index_bytes) + .await + .map_err(|source| SpillError::StagingIo { + path: index_staging.clone(), + source, + })?; + fsync_dir(&partition_dir) + .await + .map_err(|source| SpillError::StagingIo { + path: PathBuf::from(&partition_dir), + source, + })?; + if cancelled.get() { + return Err(SpillError::Cancelled); + } + Ok(SpilledSegment { + meta: StagedSegmentMeta::from_walk( + &entry, + stats, + index_size, + log_staging, + index_staging, + ), + hasher: SpilledHasher { + start_offset: entry.frontier, + size: entry.len, + hasher, + }, + }) + } +} + +/// A reuse scan that a detached task runs. It holds no partition state, so +/// the pump runs while it does. +struct ReuseScanJob { + partition_dir: String, + namespace_raw: u64, + memoized: BTreeMap, + /// The manifest index of each segment entry, the entry, and the live + /// segment that holds its bytes. + entries: Vec<(u32, consensus::StateArtifact, Option)>, +} + +impl ReuseScanJob { + /// Checks cancellation between files, retaining verified progress without + /// sweeping files that it did not reach. + async fn run(mut self, cancelled: &Cell) -> ReuseScanned { + let mut reused = Vec::new(); + let mut matched_paths = Vec::new(); + let mut complete = true; + for (index, entry, held) in &self.entries { + if cancelled.get() { + complete = false; + break; + } + let memoized = self + .memoized + .remove(&entry.frontier) + .filter(|meta| (meta.size, meta.checksum) == (entry.len, entry.checksum)); + let meta = if let Some(meta) = memoized + && matches!( + compio::fs::metadata(&meta.log_staging).await, + Ok(metadata) if metadata.len() == meta.size + ) + && matches!( + compio::fs::metadata(&meta.index_staging).await, + Ok(metadata) if metadata.len() == meta.index_size + ) { + Some(meta) + } else { + self.reuse(entry, held.as_ref()).await + }; + let Some(meta) = meta else { + continue; + }; + matched_paths.push(meta.log_staging.clone()); + matched_paths.push(meta.index_staging.clone()); + reused.push((*index, meta)); + } + // Every rebuilt sidecar and every link landed in the same directory, + // so one fsync covers them all. Its failure discards EVERY adoption: + // the per-adopt filter above no longer sees a durability failure, and + // an undurable sidecar handed to the install is a missing rename + // source after a crash. + if !reused.is_empty() && fsync_dir(&self.partition_dir).await.is_err() { + reused.clear(); + matched_paths.clear(); + } + // A stopped scan cannot tell a stray from a staged copy of an entry + // that it did not reach. + if complete { + // Sweep strays: anything staged that no adopted meta claims. + sweep_staging_except(&self.partition_dir, matched_paths.into_iter().collect()).await; + self.memoized.clear(); + } + ReuseScanned { + reused, + memoized: self.memoized, + } + } + + /// Prefer a verified live match, then try the staged copy of an earlier + /// attempt. Either path rebuilds the index from verified log bytes. + async fn reuse( + &self, + entry: &consensus::StateArtifact, + held: Option<&HeldSegment>, + ) -> Option { + if let Some(held) = held + && let Some(meta) = held + .link(&self.partition_dir, entry, self.namespace_raw) + .await + { + return Some(meta); + } + let (log_staging, _) = staging_paths(&self.partition_dir, entry.frontier); + // Length short-circuit BEFORE reading: length is the first + // conjunct of the artifact check anyway, and the common retry + // case is precisely an active-segment length mismatch -- no + // point reading up to 2 GiB just to discard it. + match compio::fs::metadata(&log_staging).await { + Ok(metadata) if metadata.len() == entry.len => {} + _ => return None, + } + let bytes = compio::fs::read(&log_staging).await.ok()?; + if !verify_state_artifact_yielding(entry, &bytes).await { + return None; + } + adopt_staged_segment(&self.partition_dir, entry, &bytes) + .await + .ok() + } +} + +/// A sealed live log selected by the pump for verification and reuse. +struct HeldSegment { + log_path: String, +} + +impl HeldSegment { + /// Verify and hard-link the log, rebuilding its index from the same bytes. + /// + /// Each link is synced before it takes the staging name, because an + /// installed segment must be as durable as a spilled one, and the live + /// writes of this segment did not have to sync. A staging name that + /// already names the live file is such a link from an earlier scan, so + /// it costs nothing. + async fn link( + &self, + partition_dir: &str, + entry: &consensus::StateArtifact, + namespace_raw: u64, + ) -> Option { + let log_metadata = compio::fs::metadata(&self.log_path).await.ok()?; + // The install opens the log at exactly the manifest length. + if log_metadata.len() != entry.len { + return None; + } + let bytes = compio::fs::read(&self.log_path).await.ok()?; + if !verify_state_artifact_yielding(entry, &bytes).await { + return None; + } + let (log_staging, _) = staging_paths(partition_dir, entry.frontier); + if !names_file_of(&log_metadata, &log_staging).await + && let Err(error) = link_durably(Path::new(&self.log_path), &log_staging).await + { + tracing::warn!( + %error, + namespace_raw, + path = %log_staging.display(), + "cannot link a held segment to its staging name" + ); + return None; + } + let mut meta = adopt_staged_segment(partition_dir, entry, &bytes) + .await + .ok()?; + meta.linked = true; + Some(meta) + } +} + +/// Adopt an already-verified staged log without rewriting it: walk the +/// payload once (validation + a rebuilt sparse index), write only the +/// index sidecar, and return the walk metadata. The reuse scan calls +/// this after `verify_state_artifact` proved the bytes match the +/// manifest; rewriting the byte-identical log (and re-verifying a third +/// time) is exactly the work reuse exists to skip. The index write +/// stays: the scan never checks `.index.staging`, and a missing sidecar +/// would hand the install a missing rename source. +/// +/// No directory fsync here: every sidecar lands in the same directory, so +/// the caller fsyncs ONCE after its loop instead of once per adoption. +async fn adopt_staged_segment( + partition_dir: &str, + entry: &consensus::StateArtifact, + bytes: &[u8], +) -> Result { + let (stats, index_bytes) = walk_segment_payload(entry.frontier, bytes) + .await + .map_err(SpillError::Walk)?; + let (log_staging, index_staging) = staging_paths(partition_dir, entry.frontier); + let index_size = index_bytes.len() as u64; + write_staging_file(&index_staging, index_bytes) + .await + .map_err(|source| SpillError::StagingIo { + path: index_staging.clone(), + source, + })?; + Ok(StagedSegmentMeta::from_walk( + entry, + stats, + index_size, + log_staging, + index_staging, + )) +} + /// Why a served segment could not be handed to a requester. /// /// The split is the whole point: a short read is what a concurrent GC @@ -4768,33 +5840,31 @@ async fn verify_state_artifact_yielding(entry: &consensus::StateArtifact, bytes: return false; } let mut hasher = StateArtifactHasher::new(); - for chunk in bytes.chunks(OFFER_HASH_CHUNK_LEN) { - hasher.update(chunk); - yield_to_reactor().await; - } + hash_yielding(&mut hasher, [bytes]).await; hasher.finish() == entry.checksum } -pub(crate) fn numeric_offset_id(path: &str) -> Option { - Path::new(path).file_name()?.to_str()?.parse().ok() +pub(crate) async fn hash_yielding<'a>( + hasher: &mut StateArtifactHasher, + slices: impl IntoIterator, +) { + let mut remaining = OFFER_HASH_CHUNK_LEN; + for mut bytes in slices { + while !bytes.is_empty() { + let (chunk, rest) = bytes.split_at(bytes.len().min(remaining)); + hasher.update(chunk); + bytes = rest; + remaining -= chunk.len(); + if remaining == 0 { + yield_to_reactor().await; + remaining = OFFER_HASH_CHUNK_LEN; + } + } + } } -/// Stamp over every `SEGMENT_LOG` entry of a manifest, keying -/// [`ReuseScanMemo`]. Equal digests mean the two offers expect byte-identical -/// staged files, so a scan already done for one answers the other; the offsets -/// artifact is excluded because the scan never looks at it (and it re-encodes -/// per build, so including it would defeat the memo on every rotation). -fn segment_manifest_digest(manifest: &[consensus::StateArtifact]) -> u64 { - let mut hasher = StateArtifactHasher::new(); - for entry in manifest - .iter() - .filter(|entry| entry.kind == artifact_kind::SEGMENT_LOG) - { - hasher.update(&entry.frontier.to_le_bytes()); - hasher.update(&entry.len.to_le_bytes()); - hasher.update(&entry.checksum.to_le_bytes()); - } - hasher.finish() +pub(crate) fn numeric_offset_id(path: &str) -> Option { + Path::new(path).file_name()?.to_str()?.parse().ok() } fn checked_retry_table( @@ -5054,8 +6124,15 @@ pub(crate) async fn reclaim_retry_checkpoint_files( DiskStorage.sync_directory(directory).await } +/// Write `payload` to a new file at `path`. A staging name can be a hard link +/// to a live segment that the reuse scan linked. Writing through that name in +/// place changes the live segment, so the old name goes first. async fn write_staging_file(path: &Path, payload: Vec) -> std::io::Result<()> { - let mut file = compio::fs::File::create(path) + remove_if_present(path).await?; + let mut file = compio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) .await .note_descriptor_exhaustion(|| format!("creating {}", path.display()))?; let (result, _) = file.write_all_at(payload, 0).await.into(); @@ -5063,3 +6140,52 @@ async fn write_staging_file(path: &Path, payload: Vec) -> std::io::Result<() file.sync_data().await?; Ok(()) } + +async fn remove_if_present(path: &Path) -> std::io::Result<()> { + match compio::fs::remove_file(path).await { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => Err(error), + _ => Ok(()), + } +} + +/// Hard-link `live` to `staged` through a scratch name, so that the staging +/// name only ever names a link whose data is durable, and a failure keeps +/// the staged copy that `staged` held. A rename does nothing when both names +/// are links to one file, so the caller must not link a file that `staged` +/// already names. +async fn link_durably(live: &Path, staged: &Path) -> std::io::Result<()> { + let scratch = staged.with_extension(LINK_SCRATCH_EXTENSION); + let linked = async { + remove_if_present(&scratch).await?; + compio::fs::hard_link(live, &scratch).await?; + compio::fs::File::open(&scratch) + .await + .note_descriptor_exhaustion(|| format!("opening {}", scratch.display()))? + .sync_data() + .await?; + compio::fs::rename(&scratch, staged).await + } + .await; + if linked.is_err() { + // A later sweep of the staging names removes a name that this cannot. + let _ = compio::fs::remove_file(&scratch).await; + } + linked +} + +/// Whether `staged` names the file that `live` describes. A link takes its +/// staging name only once its data is durable, so a scan that finds one +/// there keeps it without another link or sync. +#[cfg(unix)] +async fn names_file_of(live: &compio::fs::Metadata, staged: &Path) -> bool { + compio::fs::metadata(staged) + .await + .is_ok_and(|staged| staged.dev() == live.dev() && staged.ino() == live.ino()) +} + +/// Without inode numbers no scan can prove an earlier link, so each one +/// links again. +#[cfg(not(unix))] +async fn names_file_of(_live: &compio::fs::Metadata, _staged: &Path) -> bool { + false +} diff --git a/core/partitions/src/types.rs b/core/partitions/src/types.rs index 39e42fd079..68cecf19c3 100644 --- a/core/partitions/src/types.rs +++ b/core/partitions/src/types.rs @@ -356,8 +356,10 @@ pub enum RepairConclusion { /// The walk reached the requested frontier; the session was dropped. Done, /// The floor's continuity check failed: ops below it are neither locally - /// durable nor repaired. The session was dropped here -- state transfer - /// supersedes repair -- and the caller arms the transfer. + /// durable nor repaired. The repair walk of an unproven view log that + /// outgrew its byte budget also ends here, with `floor` at `commit_min`. + /// The session was dropped here -- state transfer supersedes repair -- and + /// the caller arms the transfer. FloorRefused { floor: u64, to_op: u64 }, } diff --git a/core/server/config.toml b/core/server/config.toml index 50b603d639..b9627fd952 100644 --- a/core/server/config.toml +++ b/core/server/config.toml @@ -497,8 +497,10 @@ name = "iggy-cluster" # whole ticks: values are rounded down to a multiple of 10ms, and anything under # 10ms is raised to a single tick rather than firing sooner. -# Backup-side liveness window for a consensus plane's primary (duration). -# A replica that sees no primary traffic for this long starts a view change. +# Liveness window for a consensus primary and its replication quorum (duration). +# Backups vote after primary silence. A primary without heartbeat replies from +# a quorum relinquishes leadership. Heartbeat replies require a coordinated +# node upgrade; mixed versions cannot sustain a primary's quorum checks. # Raise it on oversubscribed hosts where scheduling stalls fake primary # death. Must be at least "2s" and at least 4x commit_broadcast_interval: the # primary signals liveness through its commit broadcast, and the window must @@ -1035,8 +1037,19 @@ change_log_activity_capacity = 16384 # partition_wal_prepares_total over partition_wal_batches_total is the prepares per durable group, and # partition_wal_group_commit_waits_total counts the groups that actually took the optional wait # below, so a delay that never fires is distinguishable from one that fires and buys nothing. +# A backup whose log the adopted view has not proven yet repairs it from the +# head down and holds every batch it fetches in memory until the repair reaches +# its commit point. Once it holds more than this budget, it stops and catches up +# by state transfer instead. This bound is per partition, with the shared +# allocation cap below limiting concurrent walks across all shards. wal_bytes_max = "256 MiB" +# Node-wide allocation cap shared by unproven repair walks and snapshot repair +# tails. Exhausted walks fall back to state transfer; exhausted tails require +# a fresh snapshot. Normal prepare queues and transfer buffers have separate limits. +# Must be > 0. +repair_memory_bytes_max = "1 GiB" + # Bounded wait for more prepares before a persisted partition's WAL writer # starts its durability barrier, in microseconds. 0 disables it. Max 10000. # Spends up to this much acknowledgment latency to cut device writes: one @@ -1113,19 +1126,26 @@ consumer_offsets_max = 4096 # For superblock compatibility, see superblock_wedged_fatal_timeout in [cluster]. offset_reservation_lease = 65536 -# Entries the evicted ring retains per multi-replica partition for journal -# repair after a peer rejoins. Larger widens the window a restarting peer can be -# served from the ring before falling back to bulk sync, at the cost of pinned -# memory per partition. Must be > 0 and <= 65536. Single-replica partitions -# retain nothing regardless. +# Entries the evicted ring retains with their batches in memory per +# multi-replica partition for journal repair after a peer rejoins. Larger +# widens the window a restarting peer can be served from memory, at the cost of +# pinned memory per partition. Must be > 0 and <= 65536. Single-replica +# partitions retain nothing regardless. The ops whose batches the ring released +# to their segments do not count here. The ring keeps up to 16384 of them, 288 +# bytes each, and repair reads their batches back from disk. evicted_ring_capacity = 65536 -# Payload-byte ceiling per partition and replica; the first ring cap reached -# evicts the lowest ops, retaining the newest entry even if it alone exceeds -# this budget. Must be > 0 and <= "256 MiB". Allocation capacity, alignment and -# index overhead are additional memory, especially for small offset commits. -# There is no node-wide budget. partition_repair_ring_bytes reports payload -# bytes summed across a shard's partitions, not their allocated memory or RSS. +# Byte ceiling per partition and replica for the batches the ring holds in +# memory. Past the cap the oldest batches that a segment holds leave memory +# first, and repair reads them back from disk. When none can leave, the lowest +# ops go, but the newest entry stays even if it alone exceeds this budget. Must +# be > 0 and <= "256 MiB". The ops whose batches left memory take up to +# 4.5 MiB more. Allocation capacity, alignment and index overhead are +# additional memory, especially for small offset commits. There is no node-wide +# budget. +# partition_repair_ring_bytes reports these bytes plus 288 bytes for each +# released op, summed across a shard's partitions, not their allocated memory +# or RSS. evicted_ring_bytes_max = "64 MiB" # Byte budget for segment payloads a SERVING shard keeps resident to answer diff --git a/core/server/src/boot/mod.rs b/core/server/src/boot/mod.rs index 9a0f7e11a1..238260c882 100644 --- a/core/server/src/boot/mod.rs +++ b/core/server/src/boot/mod.rs @@ -345,6 +345,11 @@ pub fn bootstrap( &config, total_shards, )?)); + let repair_memory = Arc::new(partitions::RepairMemoryBudget::new( + usize::try_from(config.partition.repair_memory_bytes_max.as_bytes_u64()).map_err(|_| { + ServerError::Config(configs::ConfigurationError::InvalidConfigurationValue) + })?, + )); // Single-shot bundle handoff (see `MetadataHandoff`): shard 0 sends // one cloned `ServerMetadataBundle` per peer; each peer drains @@ -407,6 +412,7 @@ pub fn bootstrap( let config_for_shard = Arc::clone(&config); let shutdown_flag_for_shard = Arc::clone(&shutdown_flag); let owner_table_for_shard = Arc::clone(&owner_table); + let repair_memory_for_shard = Arc::clone(&repair_memory); let metadata_handoff_for_shard = if shard_id == 0 { MetadataHandoff::Owner { bundle_tx: metadata_bundle_tx.clone(), @@ -448,6 +454,7 @@ pub fn bootstrap( metadata_handoff_for_shard, barrier_for_shard, owner_table_for_shard, + repair_memory_for_shard, roster_cells_for_shard, applied_frontier_for_shard, shard_metrics_for_shard, @@ -522,6 +529,7 @@ async fn shard_main( metadata_handoff: MetadataHandoff, barrier: BootstrapBarrier, owner_table: Arc, + repair_memory: Arc, roster_cells: RosterCells, metadata_applied_frontier: Arc, shard_metrics_all: Vec, @@ -752,6 +760,7 @@ async fn shard_main( reply_inbox, shard_metrics, &roster_cells, + repair_memory, )) .await?; diff --git a/core/server/src/boot/recovery.rs b/core/server/src/boot/recovery.rs index dd1633873f..504479407f 100644 --- a/core/server/src/boot/recovery.rs +++ b/core/server/src/boot/recovery.rs @@ -107,6 +107,7 @@ pub(in crate::boot) async fn build_shard_for_thread( reply_inbox: ShardReceiver, metrics: ShardMetrics, roster_cells: &RosterCells, + repair_memory: Arc, ) -> Result { let shard_local_id = ShardId::new(shard_id); let retired_key: iggy_common::HeaderKey = @@ -172,7 +173,7 @@ pub(in crate::boot) async fn build_shard_for_thread( refuse_partitionless_stream_keys(metadata.mux_stm.streams(), &key_context, solo)?; } refuse_unreadable_topics(metadata.mux_stm.streams())?; - let partitions = IggyPartitions::with_capacity( + let partitions = IggyPartitions::with_repair_memory( shard_local_id, PartitionsConfig { messages_required_to_save: iggy_common::DEFAULT_MESSAGES_REQUIRED_TO_SAVE, @@ -188,6 +189,7 @@ pub(in crate::boot) async fn build_shard_for_thread( }, }, owned_partitions_capacity, + repair_memory, ); let shards_table = PapayaShardsTable::with_capacity(total_partitions); diff --git a/core/server/src/boot/threads.rs b/core/server/src/boot/threads.rs index a5e72f96a1..f6baf66fee 100644 --- a/core/server/src/boot/threads.rs +++ b/core/server/src/boot/threads.rs @@ -668,6 +668,7 @@ pub(in crate::boot) fn run_shard_thread( metadata_handoff: MetadataHandoff, barrier: BootstrapBarrier, owner_table: Arc, + repair_memory: Arc, roster_cells: RosterCells, metadata_applied_frontier: Arc, shard_metrics_all: Vec, @@ -718,6 +719,7 @@ pub(in crate::boot) fn run_shard_thread( metadata_handoff, barrier, owner_table, + repair_memory, roster_cells, metadata_applied_frontier, shard_metrics_all, diff --git a/core/server/src/dispatch/submit.rs b/core/server/src/dispatch/submit.rs index 1caff8d492..3fe30b03d5 100644 --- a/core/server/src/dispatch/submit.rs +++ b/core/server/src/dispatch/submit.rs @@ -646,7 +646,8 @@ mod tests { use iggy_binary_protocol::requests::consumer_groups::CreateConsumerGroupRequest; use iggy_binary_protocol::requests::topics::DeleteTopicRequest; use iggy_binary_protocol::{ - ForwardCompletionHeader, GenericHeader, Operation, WireEncode, WireIdentifier, WireName, + Command, ConsensusHeader, ForwardCompletionHeader, GenericHeader, HEADER_SIZE, Operation, + StartViewChangeHeader, WireEncode, WireIdentifier, WireName, }; use iggy_common::{IggyDuration, IggyError}; use journal::prepare_journal::PrepareJournal; @@ -1115,9 +1116,21 @@ mod tests { for _ in 0..TimeoutManager::NORMAL_HEARTBEAT_TICKS { consensus.tick(PlaneKind::Metadata); } + let vote = Message::::new(HEADER_SIZE).transmute_header( + |_, header: &mut StartViewChangeHeader| { + header.command = Command::StartViewChange; + header.cluster = consensus.cluster(); + header.group = consensus.group(); + header.view = 1; + header.replica = PRIMARY; + header.size = u32::try_from(HEADER_SIZE).unwrap(); + header.seal(); + }, + ); + consensus.handle_start_view_change(PlaneKind::Metadata, vote.header()); assert!( !consensus.is_normal(), - "a silent primary starts a view change" + "a quorum of votes starts a view change" ); assert_eq!( metadata diff --git a/core/server_common/src/consensus_message.rs b/core/server_common/src/consensus_message.rs index c981d4e373..cf73e993bb 100644 --- a/core/server_common/src/consensus_message.rs +++ b/core/server_common/src/consensus_message.rs @@ -20,10 +20,11 @@ use crate::sharding::METADATA_GROUP; use aligned_vec::{AVec, ConstAlign}; use futures::channel::oneshot; use iggy_binary_protocol::{ - Command, CommitHeader, ConsensusError, ConsensusHeader, ConsumerSessionHeartbeatHeader, - DoViewChangeHeader, ForwardCompletionHeader, ForwardLogoutHeader, ForwardLogoutResultHeader, - ForwardRegisterHeader, ForwardRegisterResultHeader, GenericHeader, HEADER_SIZE, Operation, - PrepareHeader, PrepareOkHeader, PrimaryRebalanceHeader, RepairPrepareHeader, + Command, CommitHeader, CommitReplyHeader, ConsensusError, ConsensusHeader, + ConsumerSessionHeartbeatHeader, DoViewChangeHeader, ForwardCompletionHeader, + ForwardLogoutHeader, ForwardLogoutResultHeader, ForwardRegisterHeader, + ForwardRegisterResultHeader, GenericHeader, HEADER_SIZE, Operation, PrepareHeader, + PrepareOkHeader, PrepareProgressHeader, PrimaryRebalanceHeader, RepairPrepareHeader, RepairRangeReplyHeader, RequestHeader, RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, RoutedRequestHeader, SegmentDeleteHeader, StartViewChangeHeader, StartViewHeader, StateChunkHeader, StateTransferTargetHeader, @@ -477,6 +478,13 @@ where self.backing.into_frozen() } + /// Allocation charge after freezing this frame into a retained journal entry. + /// Returns `None` if its allocation layout cannot be represented. + #[must_use] + pub fn retained_allocation_size(&self) -> Option { + Frozen::::allocation_size(self.backing.owned.backing_capacity()) + } + /// # Panics /// /// Panics if `H` and `T` have different sizes, or if the rewritten header @@ -657,6 +665,8 @@ pub enum MessageBag { DoViewChange(Message), StartView(Message), Commit(Message), + CommitReply(Message), + PrepareProgress(Message), RequestStartView(Message), RequestPrepares(Message), /// A journaled prepare re-sent verbatim for repair (command byte @@ -721,6 +731,8 @@ impl MessageBag { | Command::PrepareOk | Command::Reply | Command::Commit + | Command::CommitReply + | Command::PrepareProgress | Command::StartViewChange | Command::DoViewChange | Command::StartView @@ -784,6 +796,14 @@ impl MessageBag { let header = message.header(); (header.operation(), header.group) } + Self::CommitReply(message) => { + let header = message.header(); + (header.operation(), header.group) + } + Self::PrepareProgress(message) => { + let header = message.header(); + (header.operation(), header.group) + } Self::RequestStartView(message) => { let header = message.header(); (header.operation(), header.group) @@ -844,6 +864,8 @@ impl MessageBag { Self::DoViewChange(message) => message.into_generic(), Self::StartView(message) => message.into_generic(), Self::Commit(message) => message.into_generic(), + Self::CommitReply(message) => message.into_generic(), + Self::PrepareProgress(message) => message.into_generic(), Self::RequestStartView(message) => message.into_generic(), Self::RequestPrepares(message) => message.into_generic(), Self::RepairPrepare(message) => message.into_generic(), @@ -873,6 +895,8 @@ impl MessageBag { Self::DoViewChange(message) => message.header().command, Self::StartView(message) => message.header().command, Self::Commit(message) => message.header().command, + Self::CommitReply(message) => message.header().command, + Self::PrepareProgress(message) => message.header().command, Self::RequestStartView(message) => message.header().command, Self::RequestPrepares(message) => message.header().command, Self::RepairPrepare(message) => message.header().command(), @@ -902,6 +926,8 @@ impl MessageBag { Self::DoViewChange(message) => message.header().size(), Self::StartView(message) => message.header().size(), Self::Commit(message) => message.header().size(), + Self::CommitReply(message) => message.header().size(), + Self::PrepareProgress(message) => message.header().size(), Self::RequestStartView(message) => message.header().size(), Self::RequestPrepares(message) => message.header().size(), Self::RepairPrepare(message) => message.header().size(), @@ -931,6 +957,8 @@ impl MessageBag { Self::DoViewChange(message) => message.header().operation(), Self::StartView(message) => message.header().operation(), Self::Commit(message) => message.header().operation(), + Self::CommitReply(message) => message.header().operation(), + Self::PrepareProgress(message) => message.header().operation(), Self::RequestStartView(message) => message.header().operation(), Self::RequestPrepares(message) => message.header().operation(), Self::RepairPrepare(message) => message.header().operation(), @@ -1042,6 +1070,12 @@ where )), Command::StartView => Ok(Self::StartView(value.try_into_typed::()?)), Command::Commit => Ok(Self::Commit(value.try_into_typed::()?)), + Command::CommitReply => Ok(Self::CommitReply( + value.try_into_typed::()?, + )), + Command::PrepareProgress => Ok(Self::PrepareProgress( + value.try_into_typed::()?, + )), Command::RequestStartView => Ok(Self::RequestStartView( value.try_into_typed::()?, )), @@ -1108,6 +1142,7 @@ where #[cfg(test)] mod tests { use super::*; + use crate::sharding::IggyNamespace; use iggy_binary_protocol::{ ForwardLogoutHeader, ForwardLogoutOutcome, ForwardLogoutResultHeader, HEADER_SIZE, Operation, PrimaryRebalanceKind, ReplyHeader, RequestHeader, frame_checksum_bytes, @@ -1435,6 +1470,24 @@ mod tests { assert_ne!(bag.routing().1, METADATA_GROUP); } + #[test] + fn given_heartbeat_replies_when_classifying_links_should_preserve_their_group() { + for group in [METADATA_GROUP, IggyNamespace::new(1, 1, 0).inner()] { + let reply = Message::::new(HEADER_SIZE).transmute_header( + |_, header: &mut CommitReplyHeader| { + header.command = Command::CommitReply; + header.size = u32::try_from(HEADER_SIZE).unwrap(); + header.group = group; + header.timestamp_monotonic = 1; + header.seal(); + }, + ); + let bag = MessageBag::try_from(reply.into_generic()).unwrap(); + assert!(!MessageBag::routes_to_metadata(bag.command())); + assert_eq!(bag.routing().1, group); + } + } + fn primary_rebalance_frame(kind: PrimaryRebalanceKind) -> Message { Message::::new(HEADER_SIZE).transmute_header( |_, header: &mut PrimaryRebalanceHeader| { @@ -1721,6 +1774,7 @@ mod tests { Command::DoViewChange, Command::StartView, Command::Commit, + Command::CommitReply, ] { let bag = dispatch(cmd, 256).expect("dispatch"); assert_eq!(bag.command(), cmd, "round-trip for {cmd:?}"); diff --git a/core/server_common/src/iobuf.rs b/core/server_common/src/iobuf.rs index 0386938e2a..3d41ac45bb 100644 --- a/core/server_common/src/iobuf.rs +++ b/core/server_common/src/iobuf.rs @@ -118,6 +118,12 @@ impl Owned { &self.inner } + /// Capacity of the allocation retained when this buffer is frozen. + #[must_use] + pub fn backing_capacity(&self) -> usize { + self.inner.capacity() + } + pub fn as_mut_slice(&mut self) -> &mut [u8] { &mut self.inner } diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index 586ca01e03..f2717de183 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs @@ -25,6 +25,7 @@ pub use partition_io::{PartitionIoLimits, PartitionIoLimitsError}; mod poll; mod primary_rebalance; mod router; +mod served_reads; pub mod shards_table; #[cfg(test)] mod test_support; @@ -42,10 +43,11 @@ pub use router::CONSENSUS_TICK_INTERVAL; #[cfg(feature = "simulator")] use consensus::LocalPipeline; use consensus::{ - AdoptedView, ChunkProgress, CommitOutcome, Consensus, ConsensusClock, DVC_HEADERS_MAX, - DvcHeaderKind, DvcSuffix, FatalReason, MergedLog, MetadataHandle, MuxPlane, PartitionsHandle, - Pipeline, Plane, PlaneKind, RepairEvidence, STATE_TRANSFER_MAX_DECODE_RETRIES, - STATE_TRANSFER_MAX_STALL_RETRIES, Sequencer, StartViewRefusal, Status, VsrAction, VsrConsensus, + AdoptedView, BackupPrefix, ChunkProgress, CommitOutcome, Consensus, ConsensusClock, + DVC_HEADERS_MAX, DvcHeaderKind, DvcSuffix, FatalReason, MergedLog, MetadataHandle, MuxPlane, + PartitionsHandle, Pipeline, Plane, PlaneKind, RepairEvidence, ReplicationSend, + ReplicationStats, STATE_TRANSFER_MAX_DECODE_RETRIES, STATE_TRANSFER_MAX_STALL_RETRIES, + Sequencer, StartViewRefusal, Status, VsrAction, VsrConsensus, build_deny_reply_from_request_header, dvc_blank, dvc_header_kind, encode_prepare_headers, fatal, repaired_frontier_update, repaired_prepare_evidence, restamp_prepare_view, verify_prepare_integrity, @@ -55,14 +57,14 @@ use crossfire::AsyncRxTrait; use futures::FutureExt; use iggy_binary_protocol::responses::system::GetMetadataChangesResponse; use iggy_binary_protocol::{ - CHECKSUM_UNSEALED, Command, CommitHeader, ConsensusHeader, ConsumerSession, + CHECKSUM_UNSEALED, Command, CommitHeader, CommitReplyHeader, ConsensusHeader, ConsumerSession, ConsumerSessionHeartbeatHeader, DoViewChangeHeader, ForwardCompletionHeader, ForwardLogoutHeader, ForwardLogoutResultHeader, ForwardRegisterHeader, ForwardRegisterResultHeader, GenericHeader, Operation, PrepareHeader, PrepareOkHeader, - RepairPrepareHeader, RepairRangeReplyHeader, RequestPreparesHeader, RequestStartViewHeader, - RequestStateChunkHeader, RequestStateTransferHeader, RoutedRequestHeader, SegmentDeleteHeader, - SegmentDeleteKind, StartViewChangeHeader, StartViewHeader, StateChunkHeader, - StateTransferTargetHeader, WireDecode, + PrepareProgressHeader, RepairPrepareHeader, RepairRangeReplyHeader, RequestPreparesHeader, + RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, + RoutedRequestHeader, SegmentDeleteHeader, SegmentDeleteKind, StartViewChangeHeader, + StartViewHeader, StateChunkHeader, StateTransferTargetHeader, WireDecode, }; #[cfg(feature = "simulator")] use iggy_common::PartitionStats; @@ -82,11 +84,12 @@ use metadata::IggyMetadata; use metadata::impls::metadata::{PartitionPlacementView, StreamsFrontend, metadata_wal_head}; use metadata::stm::StateMachine; use metadata::{BoundSession, CommitWalkStop, MetadataSubmitError}; -use partitions::state_transfer::TransferArtifact; +use partitions::state_transfer::{PrefixSeed, TransferArtifact}; use partitions::{ FatalCommit, IggyPartition, IggyPartitions, PollFragments, PollingArgs, PollingConsumer, RepairIngest, }; +use served_reads::{LandedRead, ServedReadKey}; use server_common::sharding::{ IggyNamespace, LIST_CLIENTS_GATHER_TIMEOUT, PartitionLocation, ShardId, }; @@ -95,8 +98,10 @@ use shards_table::ShardsTable; use std::cell::{Cell, RefCell}; use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; use std::future::Future; +use std::io; use std::rc::Rc; use std::sync::Arc; +use std::task::Poll; use std::time::{Duration, Instant}; pub type ShardPlane = @@ -1220,6 +1225,16 @@ impl ShardFrame { /// `[cluster] repair_chunk_max` at bootstrap. pub const REPAIR_CHUNK_MAX: u64 = 128; +/// Prepare bytes one partition `RequestPrepares` round serves. +/// +/// Bodies that the ring released come back in one detached read per round, +/// and a shard runs one such read at a time, so this bounds the bytes that +/// read holds. Bodies that lie back to back in a segment come back in one +/// read. Resident responses share the bound so receivers can reserve WAL +/// space. The first body always goes, so a batch above the budget still +/// makes progress. +const REPAIR_BYTES_MAX: u64 = 8 * 1024 * 1024; + /// One in-flight metadata journal-repair stream (shard 0 only). #[derive(Debug, Clone, Copy)] struct MetadataRepairSession { @@ -1305,6 +1320,13 @@ const DEFAULT_BUS_MAX_MESSAGE_SIZE: usize = 64 * 1024 * 1024; /// protocol has no completion frame) or gave up. const STATE_TRANSFER_OFFER_EXPIRY_MULTIPLE: u32 = 10; +/// A parked partition transfer request waits for its plan pin for at most the +/// repair-retry interval divided by this. A requester re-sends a request that +/// one whole interval left unanswered, and that re-send burns a stall retry. +/// The refusal goes out with half the interval to spare for the network and +/// for the tick skew between the nodes. +const PARKED_TRANSFER_REQUEST_WAIT_DIVISOR: u32 = 2; + /// Lifetime of a FULLY SERVED offer, as a multiple of the repair-retry /// interval. It only has to outlive the receiver re-requesting a lost final /// chunk, but the receiver's stall re-request fires at exactly one such @@ -1313,6 +1335,16 @@ const STATE_TRANSFER_OFFER_EXPIRY_MULTIPLE: u32 = 10; /// drops the session with every byte already downloaded). const STATE_TRANSFER_SERVED_EXPIRY_MULTIPLE: u32 = 3; +/// A receiver waits for local work, a reuse scan or a prefix read, for at +/// most the serving offer's idle lifetime divided by this. The wait counts +/// from the last chunk request. A scan and a read can follow each other with +/// no request between them, so the bound covers both. The pull sends nothing +/// while it waits, so the offer ages the whole time. The other half of the +/// lifetime lets the first request after the wait reach the offer before it +/// expires. A scan that is asked to stop still finishes the file operation +/// that it runs. +const LOCAL_WORK_WAIT_DIVISOR: u32 = 2; + /// One in-flight metadata state transfer (shard 0 only): a cluster-restart /// rejoin replacing its snapshot-shaped state (metadata snapshot + client /// table) from the live primary before tail repair. @@ -1619,6 +1651,16 @@ impl<'a> TransferDescriptor<'a> { } } +/// A partition `RequestStateTransfer` that waits on the serving side for its +/// plan pin. The pin lands in an I/O turn after the request arrives. A refusal +/// costs the requester a whole re-arm interval, because a transient refusal +/// does not tell it that only the pin is missing. +struct ParkedTransferRequest { + header: RequestStateTransferHeader, + /// Ticks since the first arrival of this nonce. + waited_ticks: u32, +} + /// What `on_request_state_chunk` decided inside its offers borrow; the wire /// sends run after the borrow drops. enum ChunkReply { @@ -1715,6 +1757,7 @@ where /// `(namespace, requester replica id)`. Bounded by the replica count times /// the groups this shard serves; replaced per fresh nonce. state_transfer_offers: RefCell>, + partition_recovery: metadata::PartitionRecovery, /// Partition groups with an offer build under way but no offer yet, keyed /// by namespace and carrying ticks since the last request that advanced it. /// @@ -1724,6 +1767,16 @@ where /// clock as an idle offer, since a requester that walked away leaves /// nothing else to release the slot. partition_offer_builds: RefCell>, + /// Partition groups refused a served transfer for want of a slot, in the + /// order of their first refusal, each with the ticks since its last + /// refused request. Free slots go to the front of this queue, and a group + /// that stops asking ages out on the clock of an abandoned build. + partition_transfer_waiters: RefCell>, + /// Partition transfer requests that wait for their plan pin, keyed like + /// `state_transfer_offers`. The tick answers each one. [`BTreeMap`], not + /// `HashMap`: the tick answers in key order, so a replay stays + /// deterministic. + parked_partition_transfer_requests: RefCell>, /// See [`ServedSegmentCache`]. served_segment_cache: RefCell, @@ -1779,6 +1832,8 @@ where /// displace their results. Only the owner pump validates completions. poll_completions: poll::completion::PollCompletionLane, partition_io: partition_io::PartitionIoLane, + /// Disk reads that answer peers, which the pump serves once they land. + served_reads: served_reads::ServedReads, cooperative_events: Cell, deferred_polls: poll::deferred::DeferredPolls, @@ -1889,6 +1944,7 @@ where /// cursor the leading groups would spend the whole budget on every pass and /// the tail would never be reached. See [`rotate_sweep_to_cursor`]. partition_walk_cursor: Cell>, + replication_replay_cursor: Cell>, partition_placement_cursor: Cell>, /// Namespaces still owed a placement report in the current interval. partition_placement_owed: Cell, @@ -2092,6 +2148,7 @@ where let nonce_seed = boot_nonce(metadata.consensus.as_ref()); // Metadata admits a stream's wrap set with the keys its partitions open. let metadata = metadata.with_stream_key_context(partitions.key_context()); + let partition_recovery = metadata.applied_frontier().track_partition_recovery(); let plane = MuxPlane::new(variadic!(metadata, partitions)); let ShardIdentity { id, name } = identity; let poll_completions = @@ -2117,6 +2174,7 @@ where reply_inbox, poll_completions, partition_io, + served_reads: served_reads::ServedReads::default(), cooperative_events: Cell::new(0), deferred_polls: poll::deferred::DeferredPolls::new(metrics.deferred_polls.clone()), shards_table, @@ -2136,7 +2194,10 @@ where metadata_gap_ticks: Cell::new(0), metadata_walk_stuck_op: Cell::new(0), state_transfer_offers: RefCell::new(HashMap::new()), + partition_recovery, partition_offer_builds: RefCell::new(HashMap::new()), + partition_transfer_waiters: RefCell::new(Vec::new()), + parked_partition_transfer_requests: RefCell::new(BTreeMap::new()), served_segment_cache: RefCell::new(ServedSegmentCache::default()), register_forwards: RefCell::new(HashMap::new()), logout_forwards: RefCell::new(HashMap::new()), @@ -2149,6 +2210,7 @@ where repair_gap_debounce_ticks: Cell::new(partitions::REPAIR_RETRY_TICKS), partition_repairs_inflight: Cell::new(0), partition_walk_cursor: Cell::new(None), + replication_replay_cursor: Cell::new(None), partition_placement_cursor: Cell::new(None), partition_placement_owed: Cell::new(0), primary_rebalance: RefCell::default(), @@ -2977,6 +3039,7 @@ where let nonce_seed = boot_nonce(metadata.consensus.as_ref()); // Metadata admits a stream's wrap set with the keys its partitions open. let metadata = metadata.with_stream_key_context(partitions.key_context()); + let partition_recovery = metadata.applied_frontier().track_partition_recovery(); let plane = MuxPlane::new(variadic!(metadata, partitions)); let ShardIdentity { id, name } = identity; let host: Rc = Rc::new(NoopHost); @@ -3009,6 +3072,7 @@ where .expect("simulator supports the production partition I/O limits"), &metrics, ), + served_reads: served_reads::ServedReads::default(), cooperative_events: Cell::new(0), shards_table, partition_consensus, @@ -3026,7 +3090,10 @@ where metadata_gap_ticks: Cell::new(0), metadata_walk_stuck_op: Cell::new(0), state_transfer_offers: RefCell::new(HashMap::new()), + partition_recovery, partition_offer_builds: RefCell::new(HashMap::new()), + partition_transfer_waiters: RefCell::new(Vec::new()), + parked_partition_transfer_requests: RefCell::new(BTreeMap::new()), served_segment_cache: RefCell::new(ServedSegmentCache::default()), register_forwards: RefCell::new(HashMap::new()), logout_forwards: RefCell::new(HashMap::new()), @@ -3039,6 +3106,7 @@ where repair_gap_debounce_ticks: Cell::new(partitions::REPAIR_RETRY_TICKS), partition_repairs_inflight: Cell::new(0), partition_walk_cursor: Cell::new(None), + replication_replay_cursor: Cell::new(None), partition_placement_cursor: Cell::new(None), partition_placement_owed: Cell::new(0), primary_rebalance: RefCell::default(), @@ -4048,6 +4116,8 @@ where MessageBag::DoViewChange(msg) => self.on_do_view_change(msg).await, MessageBag::StartView(msg) => self.on_start_view(msg).await, MessageBag::Commit(ref msg) => self.on_commit(msg).await, + MessageBag::CommitReply(ref msg) => self.on_commit_reply(msg), + MessageBag::PrepareProgress(ref msg) => self.on_prepare_progress(msg).await, MessageBag::RequestStartView(ref msg) => self.on_request_start_view(msg).await, MessageBag::RequestPrepares(ref msg) => self.on_request_prepares(msg).await, MessageBag::RepairPrepare(msg) => self.on_repair_prepare(msg).await, @@ -5813,7 +5883,11 @@ where // Ahead of `handle_commit`, which drops a lower-view heartbeat from // the real primary once this replica's view climbed alone. consensus.observe_primary(header.view, header.replica); - match consensus.handle_commit(&header) { + let outcome = consensus.handle_commit(&header); + if let Some(action) = outcome.reply(&header) { + dispatch_vsr_actions::(consensus, None, &[action]).await; + } + match outcome { CommitOutcome::Advanced => { // Mid-transfer the pre-install STM must not walk: the // snapshot being installed already contains those ops. @@ -5851,7 +5925,7 @@ where respond_start_view::(consensus).await; } } - CommitOutcome::Accepted => {} + CommitOutcome::Accepted | CommitOutcome::Ignored => {} } return; } @@ -5872,7 +5946,11 @@ where // elections nobody joined drops the real primary's lower-view // heartbeats there, and they are how it learns whom to repair from. consensus.observe_primary(header.view, header.replica); - match consensus.handle_commit(&header) { + let outcome = consensus.handle_commit(&header); + if let Some(action) = outcome.reply(&header) { + dispatch_vsr_actions::(consensus, None, &[action]).await; + } + match outcome { CommitOutcome::Advanced => { if !partition.consensus().is_transferring() { partition.commit_journal(config).await; @@ -5899,7 +5977,21 @@ where respond_start_view::(consensus).await; } } - CommitOutcome::Accepted => {} + CommitOutcome::Accepted | CommitOutcome::Ignored => {} + } + } + + fn on_commit_reply(&self, msg: &Message) { + let header = msg.header(); + let planes = self.plane.inner(); + if let Some(consensus) = &planes.0.consensus + && consensus.group() == header.group + { + consensus.handle_commit_reply(header); + return; + } + if let Some(partition) = planes.1.0.get_by_ns(&IggyNamespace::from_raw(header.group)) { + partition.consensus().handle_commit_reply(header); } } @@ -5967,23 +6059,308 @@ where dispatch_partition_wire_actions::(consensus, partition, wire_actions).await; } + #[allow(clippy::future_not_send)] + async fn on_prepare_progress(&self, msg: &Message) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: + Journal, Header = PrepareHeader>, + { + let header = *msg.header(); + let planes = self.plane.inner(); + let (consensus, progress) = if let Some(consensus) = &planes.0.consensus + && consensus.group() == header.group + { + if header.kind == PrepareProgressHeader::REPLY { + let _ = consensus.handle_prepare_progress(&header); + return; + } + let ready = consensus.is_normal() + && !consensus.is_transferring() + && !consensus.has_untrusted_log() + && consensus.history_certified(); + let mut received = consensus.commit_min(); + if ready && let Some(journal) = &planes.0.journal { + let journal = journal.handle(); + while received < consensus.sequencer().current_sequence() { + let next = received + 1; + let Ok(index) = usize::try_from(next) else { + break; + }; + if !consensus.view_log_entry_proven(next) || journal.header(index).is_none() { + break; + } + received = next; + } + } + // The metadata walk has no durability test, so one value is both prefixes. + let prefix = BackupPrefix { + held_op: received, + received_op: received, + }; + (consensus, ready.then_some(prefix)) + } else { + let Some(partition) = planes.1.0.get_by_ns(&IggyNamespace::from_raw(header.group)) + else { + return; + }; + let consensus = partition.consensus(); + if header.kind == PrepareProgressHeader::REPLY { + if let Some(progress) = consensus.handle_prepare_progress(&header) { + partition.advance_transfer_tail(header.replica, progress, self.ticks.get()); + } + return; + } + (consensus, partition.replication_progress()) + }; + if header.kind != PrepareProgressHeader::PROBE + || header.nonce == 0 + || header.cluster != consensus.cluster() + || header.view != consensus.view() + || header.partition_incarnation != consensus.partition_incarnation() + || header.replica != consensus.primary_index(consensus.view()) + || header.replica == consensus.replica() + { + return; + } + if consensus.is_normal() && !consensus.is_transferring() { + consensus.advance_commit_max(header.commit_op); + } + let reply = Message::::new(size_of::()) + .transmute_header(|_, reply: &mut PrepareProgressHeader| { + *reply = header; + reply.replica = consensus.replica(); + reply.kind = PrepareProgressHeader::REPLY; + reply.ready = u8::from(progress.is_some()); + let prefix = progress.unwrap_or_default(); + reply.held_op = prefix.held_op; + reply.received_op = prefix.received_op; + reply.seal(); + }); + let _ = self + .bus + .send_to_replica(header.replica, reply.into_generic().into_frozen()) + .await; + } + + #[allow(clippy::future_not_send)] + async fn drive_replication(&self, namespaces: &[IggyNamespace]) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: + Journal, Header = PrepareHeader>, + { + let metadata_group = self + .plane + .metadata() + .consensus + .as_ref() + .map(VsrConsensus::group); + let groups = metadata_group + .into_iter() + .chain(namespaces.iter().map(IggyNamespace::inner)); + let cursor = self.replication_replay_cursor.get(); + // Prefer keys after the cursor even when the commit-walk order rotates. + let rank = |key| (cursor.is_some_and(|last| key <= last), key); + let mut replay: Option<(u64, u8, u64)> = None; + let mut stats = ReplicationStats::default(); + for group in groups { + let Some(consensus) = self.replication_consensus(group) else { + continue; + }; + let taken = consensus.take_replication_stats(); + stats.live_refusals += taken.live_refusals; + stats.replayed += taken.replayed; + for replica in 0..consensus.replica_count() { + if replica == consensus.replica() { + continue; + } + match consensus.replication_work(replica) { + Some(consensus::ReplicationWork::Probe { nonce, through, .. }) => { + consensus + .send_replication_probe(replica, nonce, through) + .await; + } + Some(consensus::ReplicationWork::Replay { from_op, .. }) + if !self.served_reads.reading(ServedReadKey::Prepares { + group, + target: replica, + live: true, + }) && replay.is_none_or(|(selected_group, selected_peer, _)| { + rank((group, replica)) < rank((selected_group, selected_peer)) + }) => + { + replay = Some((group, replica, from_op)); + } + Some(consensus::ReplicationWork::Replay { .. }) | None => {} + } + } + } + self.metrics.record_replication(stats); + // One bounded range per tick keeps replay I/O independent of group count. + let Some((group, replica, from_op)) = replay else { + return; + }; + self.replication_replay_cursor.set(Some((group, replica))); + let request = { + let Some(consensus) = self.replication_consensus(group) else { + return; + }; + Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestPreparesHeader| { + header.command = Command::RequestPrepares; + header.size = u32::try_from(size_of::()) + .expect("header size fits u32"); + header.replica = replica; + header.group = group; + header.partition_incarnation = consensus.partition_incarnation(); + header.from_op = from_op; + header.to_op = consensus.sequencer().current_sequence(); + }) + }; + let served = self.serve_prepares(&request, true, None).await; + self.defer_unserved_replay(group, replica, from_op, served); + } + + /// A replay that served nothing waits before the next one, so a body or a + /// peer that is not there is not asked for on every tick. + fn defer_unserved_replay(&self, group: u64, replica: u8, from_op: u64, served: Option) + where + B: MessageBus, + { + if served.is_some_and(|served| served < from_op) + && let Some(consensus) = self.replication_consensus(group) + { + consensus.defer_replication(replica); + } + } + + /// Serve the oldest request whose disk read landed, against the state of + /// now, as the pump serves any request. + #[allow(clippy::future_not_send)] + pub(crate) async fn serve_landed_read(&self) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: + Journal, Header = PrepareHeader>, + { + match self.served_reads.take_landed() { + Some(LandedRead::Prepares { + request, + live, + mut bodies, + }) => { + let served = self.serve_prepares(&request, live, Some(&mut bodies)).await; + if live { + let header = request.header(); + self.defer_unserved_replay( + header.group, + header.replica, + header.from_op, + served, + ); + } + } + Some(LandedRead::Chunk { + header, + artifact, + load, + }) => { + self.serve_partition_chunk(header, Some((artifact, load))) + .await; + } + None => {} + } + } + + /// Serve landed reads, as the pump does, until no read runs or waits. + #[cfg(test)] + #[allow(clippy::future_not_send)] + pub(crate) async fn settle_served_reads(&self) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: + Journal, Header = PrepareHeader>, + { + while !self.served_reads.is_empty() { + self.served_reads.landed().await; + self.serve_landed_read().await; + } + } + + fn replication_consensus(&self, group: u64) -> Option<&VsrConsensus> + where + B: MessageBus, + { + let planes = self.plane.inner(); + if let Some(consensus) = &planes.0.consensus + && consensus.group() == group + { + return Some(consensus); + } + let partition = planes.1.0.get_by_ns(&IggyNamespace::from_raw(group))?; + (!partition.read_history_is_changing()).then(|| partition.consensus()) + } + + #[allow(clippy::future_not_send)] + async fn send_range_prepare( + &self, + consensus: &VsrConsensus, + target: u8, + op: u64, + entry: Frozen, + live: bool, + ) -> bool + where + B: MessageBus, + { + if !live { + return self.send_repair_prepare(target, entry).await; + } + send_replayed_prepare(consensus, target, op, entry, ReplicationSend::Replay).await + } + + #[allow(clippy::future_not_send)] + async fn on_request_prepares(&self, msg: &Message) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: + Journal, Header = PrepareHeader>, + { + self.serve_prepares(msg, false, None).await; + } + /// Serve a repair range from this replica's journal: stream /// `RepairPrepare` frames (stored prepares verbatim, command byte /// rewritten) in op order, prefixed by `RangeEvicted` when the front of /// the range is no longer retained, terminated by `RepairDone`. + /// + /// Returns the last op served. A partition range that needs bodies the + /// ring released sends nothing and returns `None`: a detached read brings + /// the bodies back, and the pump serves the request again with them in + /// `landed`. #[allow(clippy::future_not_send, clippy::too_many_lines)] - async fn on_request_prepares(&self, msg: &Message) + async fn serve_prepares( + &self, + msg: &Message, + live: bool, + mut landed: Option<&mut partitions::ReleasedBodies>, + ) -> Option where B: MessageBus, MJ: JournalHandle, ::Target: Journal, Header = PrepareHeader>, - M: StreamsFrontend, { let header = *msg.header(); let target = header.replica; - // Snapshot the config-overridable chunk ceiling once; both plane - // branches below serve the same per-round window. + // Pull repair keeps its configured operation limit. Live partition + // replay uses the byte bound so small prepares can fill each turn. let repair_chunk_max = self.repair_chunk_max.get(); let planes = self.plane.inner(); if let Some(ref consensus) = planes.0.consensus @@ -5993,18 +6370,22 @@ where // Other requesters may receive only those peers' committed prefix. if consensus.is_transferring() { // Transfer can replace these entries. - return; + return Some(header.from_op.saturating_sub(1)); } if !matches!(consensus.status(), Status::Normal | Status::ViewChange) { - return; + return Some(header.from_op.saturating_sub(1)); } let Some(journal) = planes.0.journal.as_ref() else { - return; + return Some(header.from_op.saturating_sub(1)); }; let journal = journal.handle(); let cluster = consensus.cluster(); let self_id = consensus.replica(); - let to_op = repair_serve_ceiling(&header, consensus); + let to_op = if live { + header.to_op.min(consensus.sequencer().current_sequence()) + } else { + repair_serve_ceiling(&header, consensus) + }; // Skip the compacted prefix (below the snapshot floor) in one // RangeEvicted notice, then serve contiguously until the range // ends or the WAL runs out. @@ -6016,13 +6397,16 @@ where // servable anyway. The partition arm jumps to `retained_from` likewise. let mut from_op = header.from_op.max(journal.snapshot_op() + 1); if !consensus.view_log_entry_proven(from_op) { - return; + return Some(header.from_op.saturating_sub(1)); } #[allow(clippy::cast_possible_truncation)] while from_op <= to_op && journal.header(from_op as usize).is_none() { from_op += 1; } if from_op > to_op { + if live { + return Some(header.from_op.saturating_sub(1)); + } // Nothing in the requested range is retained. Answer the // eviction honestly: a bare `RepairDone(to_op)` here would // claim full coverage while serving zero prepares, and the @@ -6049,9 +6433,12 @@ where consensus.partition_incarnation(), ) .await; - return; + return Some(header.from_op.saturating_sub(1)); } if from_op > header.from_op { + if live { + return Some(header.from_op.saturating_sub(1)); + } self.send_repair_range_reply( cluster, self_id, @@ -6074,47 +6461,72 @@ where let Some(entry_header) = journal.header(op as usize).map(|h| *h) else { break; }; + // Each earlier frame of this loop is already sent. + if live + && !consensus.replication_window_takes(target, 0, u64::from(entry_header.size)) + { + break; + } let Some(entry) = journal.entry(&entry_header).await else { break; }; if !self - .send_repair_prepare(target, entry.into_generic().into_frozen()) + .send_range_prepare( + consensus, + target, + op, + entry.into_generic().into_frozen(), + live, + ) .await { break; } served_through = op; } - self.send_repair_range_reply( - cluster, - self_id, - target, - Command::RepairDone, - header.nonce, - served_through, - header.group, - consensus.partition_incarnation(), - ) - .await; - return; + if !live { + self.send_repair_range_reply( + cluster, + self_id, + target, + Command::RepairDone, + header.nonce, + served_through, + header.group, + consensus.partition_incarnation(), + ) + .await; + } + return Some(served_through); } let namespace = IggyNamespace::from_raw(header.group); let Some(partition) = planes.1.0.get_mut_by_ns(&namespace) else { - return; + return Some(header.from_op.saturating_sub(1)); }; if !self.partition_incarnation_matches( partition, header.partition_incarnation, Command::RequestPrepares, ) { - return; + return Some(header.from_op.saturating_sub(1)); } if !matches!( partition.consensus().status(), Status::Normal | Status::ViewChange ) || partition.view_truncation_pending() { - return; + return Some(header.from_op.saturating_sub(1)); + } + let read_key = ServedReadKey::Prepares { + group: header.group, + target, + live, + }; + if landed.is_none() && self.served_reads.reading(read_key) { + return None; + } + if !live { + partition.renew_transfer_tail_for_repair(target, self.ticks.get()); } let cluster = partition.consensus().cluster(); let self_id = partition.consensus().replica(); @@ -6124,7 +6536,13 @@ where // until the body is journaled, and the primary's retransmit is dropped // by the backup gap check (adoption already advanced its sequencer to // the head), so repair is the only channel that can deliver them. - let to_op = repair_serve_ceiling(&header, partition.consensus()); + let to_op = if live { + header + .to_op + .min(partition.consensus().sequencer().current_sequence()) + } else { + repair_serve_ceiling(&header, partition.consensus()) + }; let fence_floor = partition.view_fenced().then(|| partition.fence_floor()); // `None` means the journal holds NOTHING, not "nothing was evicted": // the partition journal is memory-only and `clear_all` wipes the @@ -6145,9 +6563,86 @@ where .unwrap_or_else(|| partition.consensus().commit_min().saturating_add(1)); let mut from_op = header.from_op; if !partition.consensus().view_log_entry_proven(from_op) { - return; + return Some(header.from_op.saturating_sub(1)); + } + let range_evicted = retained_from > from_op; + if range_evicted { + if live { + return Some(header.from_op.saturating_sub(1)); + } + from_op = retained_from; + } + let chunk_ops_max = if live { + REPAIR_BYTES_MAX / size_of::() as u64 + } else { + repair_chunk_max + }; + let chunk_end = to_op.min(from_op.saturating_add(chunk_ops_max - 1)); + let journal = &partition.log.journal().inner; + // Every op is checked and taken before the first await. A body that the + // ring released comes from `landed`, and `None` stands for one that no + // read brought back yet. + let mut entries = Vec::new(); + let mut released = Vec::new(); + let mut response_bytes = 0; + for op in from_op..=chunk_end { + if !partition.consensus().view_log_entry_proven(op) { + break; + } + let (held, evicted) = match journal.repair_header(op) { + Some(held) => (held, None), + None => match journal.evicted_header(op) { + Some(evicted) => (*evicted.header(), Some(evicted)), + None => break, + }, + }; + if !fenced_serve_admits(fence_floor, op, || { + partition.consensus().adopted_view_vouches_for(&held) + }) { + break; + } + let size = u64::from(held.size); + if response_bytes > 0 && response_bytes + size > REPAIR_BYTES_MAX { + break; + } + if live + && !partition + .consensus() + .replication_window_takes(target, response_bytes, size) + { + break; + } + response_bytes += size; + if let Some(evicted) = evicted { + let body = landed + .as_deref_mut() + .and_then(|landed| landed.take(&evicted)); + if body.is_none() { + released.push(evicted); + } + entries.push(body); + } else { + let Some(entry) = journal.repair_entry(op) else { + break; + }; + entries.push(Some(entry)); + } + } + if landed.is_none() && !released.is_empty() { + // Bodies that lie back to back in a segment take one read. Nothing + // goes out before they land, so the frames keep op order. + let read = partitions::ReleasedRead::new(&partition.log, released); + let request = msg.clone(); + self.served_reads.start(&self.bus, read_key, async move { + LandedRead::Prepares { + request, + live, + bodies: read.read().await, + } + }); + return None; } - if retained_from > from_op { + if range_evicted { self.send_repair_range_reply( cluster, self_id, @@ -6159,50 +6654,73 @@ where partition.created_revision(), ) .await; - from_op = retained_from; } - let chunk_end = to_op.min(from_op.saturating_add(repair_chunk_max - 1)); + // The serve ends at the first body that no read brought back. + let failed = released.first().and_then(|unread| { + let error = landed.as_mut()?.take_failure(unread)?; + Some((unread.header().op, error)) + }); let mut served_through = from_op.saturating_sub(1); - for op in from_op..=chunk_end { - if !partition.consensus().view_log_entry_proven(op) { - break; - } - let Some(held) = partition.log.journal().inner.repair_header(op) else { - break; - }; - if !fenced_serve_admits(fence_floor, op, || { - partition.consensus().adopted_view_vouches_for(&held) - }) { - break; - } - let Some(entry) = partition.log.journal().inner.repair_entry(op) else { + for (op, entry) in (from_op..).zip(entries) { + let Some(entry) = entry else { break; }; - if !self.send_repair_prepare(target, entry).await { + if !self + .send_range_prepare(partition.consensus(), target, op, entry, live) + .await + { break; } served_through = op; } - self.send_repair_range_reply( - cluster, - self_id, - target, - Command::RepairDone, - header.nonce, - served_through, - header.group, - partition.created_revision(), - ) - .await; - tracing::info!( - shard = self.id, - namespace_raw = header.group, - target, - from_op = header.from_op, - to_op, - served_through, - "served partition repair range" - ); + if let Some((op, error)) = failed { + // Bytes that are not the body and a body that left its file stay + // that way. Any other error can pass, so a later round reads again. + if matches!( + error.kind(), + io::ErrorKind::InvalidData | io::ErrorKind::UnexpectedEof | io::ErrorKind::NotFound + ) { + tracing::warn!( + shard = self.id, + namespace_raw = header.group, + op, + %error, + "cannot serve a repair body from its segment, so ops up to it answer RangeEvicted" + ); + journal.forget_released_through(op); + } else { + tracing::warn!( + shard = self.id, + namespace_raw = header.group, + op, + %error, + "cannot read a repair body from its segment, so a later round reads it again" + ); + } + } + if !live { + self.send_repair_range_reply( + cluster, + self_id, + target, + Command::RepairDone, + header.nonce, + served_through, + header.group, + partition.created_revision(), + ) + .await; + tracing::info!( + shard = self.id, + namespace_raw = header.group, + target, + from_op = header.from_op, + to_op, + served_through, + "served partition repair range" + ); + } + Some(served_through) } /// Ingest one repaired prepare. Metadata journals it into the WAL (the @@ -6565,6 +7083,7 @@ where session.to_op, header.group, consensus.partition_incarnation(), + None, ) .await; } @@ -6759,9 +7278,10 @@ where // delivery on a lossy bus). Proof moves backward through gaps, // then may need a higher canonical header's body. Either change // pulls the next chunk without waiting for the retry timer, and so - // do a held run that reached the chunk just requested and a - // floor-held resident prefix that grew. A shorter run is left to - // the retry timer. + // do a held run that reached the chunk just requested, a + // floor-held resident prefix that grew and an unproven walk that + // gained ops below its hole. A shorter run is left to the retry + // timer. if let partitions::RepairConclusion::FloorRefused { floor, to_op } = partition.complete_repair(&config).await { @@ -6835,8 +7355,13 @@ where ) }) }) + .or_else(|| partition.next_proof_walk_pull()) .or_else(|| partition.next_floor_pull()); let next = from_op.and_then(|from_op| { + if !partition.repair_fetch_ready() { + partition.repair.as_mut()?.requested_from = None; + return None; + } let live = partition.repair.as_mut()?; live.from_op = from_op; live.requested_from = Some(from_op); @@ -6851,6 +7376,7 @@ where to_op, header.group, partition.created_revision(), + partition.note_proof_walk_request(), ) .await; } @@ -6863,7 +7389,8 @@ where /// Ask `target` to stream its journaled prepares in `[from_op, to_op]` /// for `namespace`; answered by `RepairPrepare` frames terminated with /// `RepairDone` (prefixed by `RangeEvicted` when the front of the range - /// is no longer retained). + /// is no longer retained). `held_below_hole` is the highest op that a + /// partition holds below the hole of its unproven proof walk. #[allow(clippy::too_many_arguments)] #[allow(clippy::future_not_send, clippy::cast_possible_truncation)] async fn send_request_prepares

( @@ -6875,12 +7402,14 @@ where to_op: u64, namespace: u64, partition_incarnation: u64, + held_below_hole: Option, ) where B: MessageBus, P: Pipeline, { // Proof walks backward. End the chunk at its hole instead of re-serving - // the already held suffix for every missing predecessor. + // the already held suffix for every missing predecessor, and start it + // above the ops held below the hole, which the donor would serve first. let (from_op, to_op) = consensus .view_log_repair_op() .map_or((from_op, to_op), |missing| { @@ -6890,7 +7419,8 @@ where ( missing .saturating_sub(self.repair_chunk_max.get() - 1) - .max(consensus.commit_min() + 1), + .max(consensus.commit_min() + 1) + .max(held_below_hole.map_or(0, |held| held + 1)), missing, ) }); @@ -7235,6 +7765,7 @@ where to_op, consensus.group(), consensus.partition_incarnation(), + None, ) .await; } @@ -7379,6 +7910,7 @@ where to_op, consensus.group(), consensus.partition_incarnation(), + None, ) .await; } @@ -7579,6 +8111,7 @@ where to_op, consensus.group(), consensus.partition_incarnation(), + None, ) .await; } @@ -8749,6 +9282,11 @@ where self.ticks.set(self.ticks.get().wrapping_add(1)); let partitions = self.plane.partitions(); let repair_retry_ticks = self.repair_retry_ticks.get(); + // The serving side ages an idle offer on this clock, `max(1)` included. + let local_work_ticks_max = repair_retry_ticks + .max(1) + .saturating_mul(STATE_TRANSFER_OFFER_EXPIRY_MULTIPLE) + / LOCAL_WORK_WAIT_DIVISOR; self.partition_io.tick(); for identity in self.partition_io.interrupted() { if let Some(partition) = partitions.get_io_owner(&identity.namespace) { @@ -8787,6 +9325,11 @@ where let mut untrusted_logs = 0usize; for namespace in namespace_scratch.iter() { if let Some(partition) = partitions.get_mut_by_ns(namespace) { + partition.expire_transfer_tails( + self.ticks.get(), + u64::from(repair_retry_ticks.max(1)) + * u64::from(STATE_TRANSFER_OFFER_EXPIRY_MULTIPLE), + ); partition.drive_persistence().await; untrusted_logs += usize::from(partition.consensus().has_untrusted_log()); let (entries, bytes) = partition.repair_ring_occupancy(); @@ -8806,6 +9349,7 @@ where } } } + self.drive_replication(namespace_scratch).await; self.metrics.record_persistence(&persistence_metrics); self.partition_io.record_metrics(&self.metrics); self.metrics @@ -9015,8 +9559,9 @@ where continue; } let proof_op = partition.consensus().view_log_repair_op(); + let fetch_ready = partition.repair_fetch_ready(); let retry_due = partition.repair.as_mut().is_some_and(|session| { - if !session_live { + if !session_live || !fetch_ready { return false; } session.idle_ticks += 1; @@ -9180,6 +9725,7 @@ where to_op, namespace.inner(), partition.created_revision(), + partition.note_proof_walk_request(), ) .await; } @@ -9333,12 +9879,53 @@ where // the rejoin behind it) forever. Budget-bounded: a peer that died // mid-transfer is abandoned back to journal repair, which // re-targets the current primary. + let mut prefix_read_expired = false; + let mut reuse_scan_expired = false; + let mut local_work_landed = false; + let mut local_work_keepalive = None; let transfer_stalled = { let Some(partition) = partitions.get_mut_by_ns(&namespace) else { continue; }; partition.transfer.as_mut().and_then(|session| { session.idle_ticks += 1; + // A nonce-preserving retry renews the same pinned offer + // while this node validates or syncs its local files. + if session.target_accepted + && (session.scan.is_some() + || session.spill.is_some() + || matches!(session.seed, Some(PrefixSeed::Reading { .. }))) + && session.idle_ticks.is_multiple_of(repair_retry_ticks.max(1)) + { + local_work_keepalive = Some((session.peer, session.nonce)); + } + // Local work, not the peer, holds the pull, so a slow disk + // burns no retry of the peer. Work that outlives its bound + // is asked to stop, and the pull goes on once it lands. A + // stopped prefix read thus frees its artifact buffer before + // the pull fetches the whole artifact into a new one, and a + // stopped reuse scan leaves the rest to the pull. Work that + // landed resumes the pull here when no I/O turn did. + if let Some(scan) = &session.scan { + if scan.landed() { + local_work_landed = true; + } else if session.idle_ticks >= local_work_ticks_max && scan.stop() { + reuse_scan_expired = true; + } + return None; + } + if let Some(spill) = &session.spill { + local_work_landed = spill.work.landed(); + return None; + } + if let Some(PrefixSeed::Reading { read, .. }) = &session.seed { + if read.landed() { + local_work_landed = true; + } else if session.idle_ticks >= local_work_ticks_max && read.stop() { + prefix_read_expired = true; + } + return None; + } if session.idle_ticks < repair_retry_ticks { return None; } @@ -9346,6 +9933,31 @@ where Some((session.peer, session.nonce, session.target_accepted)) }) }; + if reuse_scan_expired { + tracing::info!( + shard = self.id, + namespace_raw = namespace.inner(), + "the reuse scan outlived its bound, so the pull fetches the segments \ + that it did not reuse" + ); + } + if prefix_read_expired { + tracing::info!( + shard = self.id, + namespace_raw = namespace.inner(), + "the local segment prefix read outlived its bound, so it is asked to \ + stop before the pull goes on" + ); + } + if let Some((peer, nonce)) = local_work_keepalive + && let Some(partition) = partitions.get_by_ns(&namespace) + { + self.send_request_state_transfer(partition.consensus(), peer, nonce) + .await; + } + if local_work_landed { + self.on_partition_transfer_progress(namespace.inner()).await; + } if let Some((peer, nonce, target_accepted)) = transfer_stalled { let Some(partition) = partitions.get_mut_by_ns(&namespace) else { continue; @@ -9359,8 +9971,8 @@ where ); // Staging files are KEPT: a later attempt adopts every // segment whose manifest entry still matches. The shared - // path charges the failure, rotates the peer, schedules - // the re-arm, and re-arms journal repair meanwhile. + // path charges the failure, rotates the peer and schedules + // the re-arm. self.abandon_or_rearm_partition_transfer(partition, peer) .await; } else if target_accepted { @@ -9375,13 +9987,21 @@ where // Scheduled transfer re-arm: count the backoff down and fire // once nothing else recovered the partition in the meantime (a // live session or a non-Idle stage owns the slot; the pending - // entry is then dropped as superseded). + // entry is then dropped as superseded). A journal that holds every + // op through `commit_max` is recovered as well, by the live stream + // or by repair, and the commit walk owns the rest. That re-arm + // stands down at once, not at the end of its backoff: while it + // waits, every repair route stays closed, so a gap that opens + // meanwhile heals only through a transfer. let rearm_peer = { let Some(partition) = partitions.get_mut_by_ns(&namespace) else { continue; }; + let journal_recovered = partition.transfer_rearm.is_some() + && !partition.requires_state_transfer() + && partition.held_through() >= partition.consensus().commit_max(); match partition.transfer_rearm.as_mut() { - Some(pending) if pending.after_ticks > 0 => { + Some(pending) if pending.after_ticks > 0 && !journal_recovered => { pending.after_ticks -= 1; None } @@ -9391,9 +10011,7 @@ where if partition.transfer.is_none() && partition.consensus().state_transfer_stage() == consensus::StateTransferStage::Idle - && (partition.requires_state_transfer() - || partition.consensus().commit_min() - < partition.consensus().commit_max()) + && !journal_recovered { Some(peer) } else { @@ -9427,18 +10045,25 @@ where // Count after every rotation, transfer abandonment and scheduled arm, // including paths that skip the rest of their partition's sweep. let mut repairs_live = 0usize; + let mut recovering = false; // A stranded key stays until its own store or delete succeeds, so a // gauge that never falls is the operator's only signal. let mut stranded = [0usize; ConsumerKind::COUNT]; for namespace in partitions.namespaces() { if let Some(partition) = partitions.get_by_ns(namespace) { repairs_live += usize::from(partition.repair.is_some()); + recovering |= partition.repair.is_some() + || partition.transfer.is_some() + || partition.transfer_rearm.is_some() + || partition.consensus().is_transferring() + || partition.consensus().has_untrusted_log(); for kind in ConsumerKind::ALL { stranded[kind.index()] += partition.stranded_consumer_offset_count(kind); } } } self.partition_repairs_inflight.set(repairs_live); + self.partition_recovery.set_active(recovering); for kind in ConsumerKind::ALL { self.metrics .set_consumer_offsets_stranded(kind, stranded[kind.index()]); @@ -9570,39 +10195,115 @@ where usize::try_from(slots).unwrap_or(usize::MAX).max(1) } - fn may_serve_another_partition_transfer(&self, namespace: u64) -> bool { + /// Admit a served transfer of `namespace`, or queue the group for a free + /// slot. A group with a build under way or an offer still being pulled + /// holds a slot, and its requesters share it. A fully served offer holds + /// none: nothing pulls its payload any more, so the cache evicts it ahead + /// of the payloads that live pulls keep using. Free slots go to refused + /// groups in the order of their first refusal, so a group whose transfer + /// just finished queues behind the groups that waited while it was served. + fn admit_partition_transfer(&self, namespace: u64) -> bool { let builds = self.partition_offer_builds.borrow(); - if builds.contains_key(&namespace) { - return true; - } let offers = self.state_transfer_offers.borrow(); - let mut served: Vec = offers - .iter() - .filter(|(_, served)| matches!(served.offer, ServedOffer::Partition(_))) - .map(|((offer_namespace, _), _)| *offer_namespace) - .collect(); - if served.contains(&namespace) { - return true; - } // Builds count too. A multi-round checksum pass holds no offer yet, so // counting only completed offers admitted every requester's whole // in-flight set at once and let each run its own pass: the frame bodies // stay bounded, but the pump carries N budgets per round-cycle and // every other frame, produce included, queues behind them. - served.extend(builds.keys().copied()); - served.sort_unstable(); - served.dedup(); - served.len() < self.partition_transfer_admission_cap() + let mut holders: Vec = offers + .iter() + .filter(|(_, served)| { + !served.fully_served && matches!(served.offer, ServedOffer::Partition(_)) + }) + .map(|((offer_namespace, _), _)| *offer_namespace) + .chain(builds.keys().copied()) + .collect(); + if holders.contains(&namespace) { + return true; + } + holders.sort_unstable(); + holders.dedup(); + let free = self + .partition_transfer_admission_cap() + .saturating_sub(holders.len()); + let mut waiters = self.partition_transfer_waiters.borrow_mut(); + let queued = waiters.iter().position(|(waiter, _)| *waiter == namespace); + if queued.unwrap_or(waiters.len()) < free { + if let Some(position) = queued { + waiters.remove(position); + } + return true; + } + match queued { + Some(position) => waiters[position].1 = 0, + None => waiters.push((namespace, 0)), + } + false } /// Serve one partition `RequestStateTransfer`: build (or re-serve) this /// group's offer and answer with the descriptor. - #[allow(clippy::future_not_send, clippy::too_many_lines)] + #[allow(clippy::future_not_send)] async fn on_partition_request_state_transfer(&self, msg: &Message) where B: MessageBus, { - let header = *msg.header(); + self.serve_partition_transfer_request(*msg.header(), None) + .await; + } + + /// Answer the parked partition transfer requests: with the offer once the + /// plan is pinned, or with a refusal once a request waited its bound. + /// + /// A tick starts at most one offer build, because a build reads and hashes + /// up to a whole round budget, and the ticks and heartbeats of every group + /// on this shard wait for the tick to end. The other pinned requests stay + /// parked, and their wait keeps counting. + /// + /// The bound refuses only a request that still waits for its pin, so a + /// pinned request builds on a later tick even past its bound. It is late + /// by at most one tick per other pinned request. Admission holds the + /// parked requests to its cap times the requesters of a group, 2 x 2 for 3 + /// replicas at the defaults, so the answer still arrives inside the half + /// retry interval that the bound leaves before the requester re-sends. + /// With a larger cap, a request can miss that window, and its re-send then + /// builds the offer in its own frame. + #[allow(clippy::future_not_send)] + async fn serve_parked_partition_transfer_requests(&self) + where + B: MessageBus, + { + let parked = std::mem::take(&mut *self.parked_partition_transfer_requests.borrow_mut()); + let mut build_started = false; + for (key, mut request) in parked { + request.waited_ticks += 1; + let builds = self + .plane + .partitions() + .get_by_ns(&IggyNamespace::from_raw(key.0)) + .is_some_and(IggyPartition::transfer_plan_pinned); + if builds && build_started { + self.parked_partition_transfer_requests + .borrow_mut() + .insert(key, request); + continue; + } + build_started |= builds; + self.serve_partition_transfer_request(request.header, Some(request.waited_ticks)) + .await; + } + } + + /// `parked_wait` holds the ticks that a parked request waited when the + /// tick answers it, and `None` for a request from the wire. + #[allow(clippy::future_not_send, clippy::too_many_lines)] + async fn serve_partition_transfer_request( + &self, + header: RequestStateTransferHeader, + parked_wait: Option, + ) where + B: MessageBus, + { // Before anything keyed by the requester: the offer map's documented // bound is the replica count, and an unvalidated id makes it 256 entries // per served group, each pinning an offer for its full expiry. @@ -9625,6 +10326,16 @@ where ) { return; } + // Only a request that passed the gates replaces a parked one. A re-send + // keeps the wait that its nonce already spent parked, so re-sends + // cannot hold the build slot past the bound. + let waited_ticks = parked_wait.unwrap_or_else(|| { + self.parked_partition_transfer_requests + .borrow_mut() + .remove(&(header.group, header.replica)) + .filter(|parked| parked.header.nonce == header.nonce) + .map_or(0, |parked| parked.waited_ticks) + }); let cluster = partition.consensus().cluster(); let self_id = partition.consensus().replica(); // First-wins per (requester, nonce), exactly as the metadata arm: a @@ -9644,39 +10355,27 @@ where served.idle_ticks = 0; Some(Rc::clone(offer)) }); + let serves_transfer = partition.serves_state_transfer(); + let admitted = + cached.is_some() || !serves_transfer || self.admit_partition_transfer(header.group); + let tail_available = !serves_transfer + || (admitted + && partition.retain_transfer_tail(header.replica, header.nonce, self.ticks.get())); // One resolve, one send: the three outcomes differ only in the // descriptor they produce, and duplicating the send made it possible for // them to drift on the progress they advertise. let offer = match cached { - Some(offer) => { - tracing::debug!( - shard = self.id, - namespace_raw = header.group, - requester = header.replica, - "re-answering a partition state transfer request from the offer \ - already served" - ); - Some(offer) - } - None if !self.may_serve_another_partition_transfer(header.group) => { - // Admission control, because the served-payload budget is a - // BYTE budget and the pulls that overrun it do not degrade - // gracefully. Each concurrent pull holds a different segment - // resident, so admitting more distinct groups than the budget - // has max-size slots makes them evict each other on every - // chunk: every request then re-reads and re-hashes a whole - // segment to serve one 256 KiB range, and a per-chunk serve - // that outruns the requester's stall interval exhausts its - // retry budget, so the pull rotates peers and never converges. - // Refusing the surplus is what makes the admitted ones finish. - tracing::info!( - shard = self.id, - namespace_raw = header.group, - requester = header.replica, - "already serving as many partition transfers as the served-payload \ - budget holds; refusing until one completes" - ); + _ if !tail_available => { + if !admitted { + tracing::info!( + shard = self.id, + namespace_raw = header.group, + requester = header.replica, + "already serving as many partition transfers as the served-payload \ + budget holds; refusing until a free slot reaches this group" + ); + } let (view, commit_max) = serving_progress(partition); self.send_state_transfer_target( cluster, @@ -9690,6 +10389,16 @@ where .await; return; } + Some(offer) => { + tracing::debug!( + shard = self.id, + namespace_raw = header.group, + requester = header.replica, + "re-answering a partition state transfer request from the offer \ + already served" + ); + Some(offer) + } None => { // Claim the admission slot for the whole build, not just for a // completed offer: the checksum pass runs over several rounds @@ -9697,7 +10406,15 @@ where self.partition_offer_builds .borrow_mut() .insert(header.group, 0); - match partition.state_transfer_offer(&config).await { + let built = partition.state_transfer_offer(&config).await.and_then(|offer| { + if partition.transfer_tail_available(header.replica, header.nonce, offer.commit_op) { + Ok(offer) + } else { + partition.clear_state_transfer_offer_cache(); + Err(partitions::state_transfer::PartitionTransferUnavailable::RepairTailExpired) + } + }); + match built { Ok(offer) => { self.partition_offer_builds .borrow_mut() @@ -9722,6 +10439,23 @@ where ); Some(offer) } + // Only the plan pin is missing, and an I/O turn pins it + // soon. The build slot stays claimed, as for any resumable + // refusal, and the tick answers the request. + Err(partitions::state_transfer::PartitionTransferUnavailable::FlushPending) + if waited_ticks + < self.repair_retry_ticks.get() + / PARKED_TRANSFER_REQUEST_WAIT_DIVISOR => + { + self.parked_partition_transfer_requests.borrow_mut().insert( + (header.group, header.replica), + ParkedTransferRequest { + header, + waited_ticks, + }, + ); + return; + } Err(reason) => { // The ACTUAL reason: "not the caught-up primary" is routine // (the requester re-targets), an unreadable segment is an @@ -9786,27 +10520,44 @@ where .await; } + #[allow(clippy::future_not_send)] + async fn on_partition_request_state_chunk(&self, msg: &Message) + where + B: MessageBus, + { + self.serve_partition_chunk(*msg.header(), None).await; + } + /// Serve one partition chunk. Segment payloads are loaded from disk on /// demand into the shard-wide [`ServedSegmentCache`] (content-addressed, /// so simultaneous rejoiners share one resident copy); a load or /// re-verification failure (GC unlinked the file, bytes changed) evicts /// the offer and tells the requester to restart with a fresh one -- /// which then reflects the current segment set, so the retry converges. + /// + /// A detached read loads the segment bytes, and the pump serves the + /// request again with them in `loaded`, beside the artifact they came from. #[allow( clippy::future_not_send, clippy::cast_possible_truncation, clippy::too_many_lines )] - async fn on_partition_request_state_chunk(&self, msg: &Message) - where + async fn serve_partition_chunk( + &self, + header: RequestStateChunkHeader, + mut loaded: Option<( + consensus::StateArtifact, + Result, partitions::state_transfer::SegmentLoadError>, + )>, + ) where B: MessageBus, { - // TODO: Move partition offer hashing, segment loads and receiver spills - // into bounded file-lane jobs so they cannot stall consensus ticks. + // TODO: Move partition offer hashing off the pump, so it cannot stall + // consensus ticks. // Pass 1 inside the borrow decides; a segment artifact that is not - // resident exits with its path and is loaded OUTSIDE the borrow (a - // RefCell borrow must not be held across an await), then pass 2 - // stores and serves it. + // resident exits with its path and a detached read loads it (a + // RefCell borrow must not be held across an await), then pass 2 of + // the serve that gets the bytes stores and serves them. enum ChunkAttempt { Reply(Option), Load { @@ -9815,11 +10566,17 @@ where }, } - let header = *msg.header(); // The requester id keys the offer map, whose bound is the replica count. if !self.peer_is_known(header.replica, "RequestStateChunk") { return; } + let read_key = ServedReadKey::Chunk { + group: header.group, + requester: header.replica, + }; + if loaded.is_none() && self.served_reads.reading(read_key) { + return; + } let planes = self.plane.inner(); let Some(partition) = planes.1.0.get_by_ns(&IggyNamespace::from_raw(header.group)) else { return; @@ -9836,6 +10593,7 @@ where let serving_view = partition.consensus().view(); let serving_commit_max = partition.consensus().commit_max(); let chunk_len_max = self.state_chunk_len_max(); + let mut loaded_chunk = None; let reply = loop { let attempt = 'attempt: { let mut offers = self.state_transfer_offers.borrow_mut(); @@ -9847,6 +10605,11 @@ where transient: true, })); }; + if !partition.touch_transfer_tail(header.replica, header.nonce, self.ticks.get()) { + break 'attempt ChunkAttempt::Reply(Some(ChunkReply::Unavailable { + transient: true, + })); + } let ServedOffer::Partition(offer) = &served.offer else { break 'attempt ChunkAttempt::Reply(Some(ChunkReply::Unavailable { transient: true, @@ -9856,11 +10619,14 @@ where let artifact = header.artifact as usize; // Keeps a segment payload alive past the cache borrow below. let segment_payload: Rc>; - let artifact_bytes: &[u8] = match offer.artifact_at(artifact) { + let (artifact_bytes, start): (&[u8], usize) = match offer.artifact_at(artifact) { Some(partitions::state_transfer::PartitionArtifactSource::Offsets(bytes)) => { - bytes + (bytes, header.offset as usize) } Some(partitions::state_transfer::PartitionArtifactSource::Segment(source)) => { + if header.offset >= source.entry.len || header.len == 0 { + break 'attempt ChunkAttempt::Reply(None); + } match self .served_segment_cache .borrow_mut() @@ -9868,20 +10634,23 @@ where { Some(payload) => { segment_payload = payload; - &segment_payload + (&segment_payload, header.offset as usize) } None => { - break 'attempt ChunkAttempt::Load { - log_path: source.log_path.clone(), - entry: source.entry, - }; + if let Some(bytes) = loaded_chunk.as_deref() { + (bytes, 0) + } else { + break 'attempt ChunkAttempt::Load { + log_path: source.log_path.clone(), + entry: source.entry, + }; + } } } } // Index past the manifest: requester bug or stale frame. None => break 'attempt ChunkAttempt::Reply(None), }; - let start = header.offset as usize; // `start >= len` is the empty-chunk livelock refusal; see the // metadata arm for the full story. if start >= artifact_bytes.len() { @@ -9931,24 +10700,54 @@ where match attempt { ChunkAttempt::Reply(reply) => break reply, ChunkAttempt::Load { log_path, entry } => { - // Chunked read + incremental hash: this runs on the pump to - // answer ONE 256 KiB chunk request, so a whole-file read - // plus a single hash pass over up to a segment would be one - // long uninterruptible CPU+IO block. The chunking keeps the - // REACTOR moving; this shard's consensus ticks are a sibling - // select arm of the same task and stay frozen either way. - let loaded = partitions::state_transfer::load_verified_segment_artifact( - &log_path, &entry, - ) - .await; - let reason = match loaded { + // A whole-segment read can outlive the requester's retry + // budget. Cache only artifacts that fit one chunk; the + // receiver verifies larger artifacts before install. + let cache_whole = header.offset == 0 && entry.len <= chunk_len_max as u64; + // The offer can change while a read runs. Bytes of another + // artifact must never serve, or fill the cache for, this one. + let Some(load) = loaded + .take() + .and_then(|(artifact, load)| (artifact == entry).then_some(load)) + else { + let len = u64::from(header.len) + .min(chunk_len_max as u64) + .min(entry.len - header.offset) + as usize; + self.served_reads.start(&self.bus, read_key, async move { + let load = if cache_whole { + partitions::state_transfer::load_verified_segment_artifact( + &log_path, &entry, + ) + .await + } else { + partitions::state_transfer::load_segment_chunk( + &log_path, + header.offset, + len, + ) + .await + }; + LandedRead::Chunk { + header, + artifact: entry, + load, + } + }); + return; + }; + let reason = match load { Ok(bytes) => { - self.served_segment_cache.borrow_mut().insert( - header.group, - entry.checksum, - Rc::new(bytes), - self.served_segment_cache_bytes_max.get(), - ); + if cache_whole { + self.served_segment_cache.borrow_mut().insert( + header.group, + entry.checksum, + Rc::new(bytes), + self.served_segment_cache_bytes_max.get(), + ); + } else { + loaded_chunk = Some(bytes); + } continue; } Err(reason) => reason, @@ -10232,14 +11031,13 @@ where /// Arm a fresh partition transfer session against `peer` and request its /// descriptor. Every partition arming site goes through here. Drops any - /// repair session (transfer supersedes repair; a transfer-unavailable - /// fallback re-arms repair fresh) and leaves the stage to its callers (they - /// own the `AwaitingTarget` transition). + /// repair session (transfer supersedes repair) and leaves the stage to its + /// callers (they own the `AwaitingTarget` transition). /// /// Refuses past [`Self::PARTITION_TRANSFERS_INFLIGHT_MAX`]: the arm /// converts to a scheduled re-arm (no failure charged -- the local slot - /// shortage is not the peer's fault) and the stage returns to Idle so - /// journal repair keeps the gap visible meanwhile. + /// shortage is not the peer's fault) and the stage returns to Idle, which + /// is the stage the re-arm fires from. /// /// `transfers_inflight` is computed by the caller BEFORE it formed its /// `&mut partition`: the counting scan takes shared borrows of every @@ -10292,6 +11090,10 @@ where artifacts: Vec::new(), target_accepted: false, idle_ticks: 0, + spill: None, + scan: None, + seed: None, + pulled_len: 0, }); self.send_request_state_transfer(partition.consensus(), peer, nonce) .await; @@ -10321,8 +11123,13 @@ where B: MessageBus + 'static, T: ShardsTable, { + // A scheduled transfer re-arm owns recovery, as at the gap detector and + // the floor-refused arm: it exists because repair could not fill the + // gap, and frames that a session journals above the gap meanwhile only + // wait for the install to wipe them. if !partition.consensus().is_normal() || partition.consensus().is_transferring() + || partition.transfer_rearm.is_some() || partition.repair.is_some() || partition.view_truncation_pending() { @@ -10455,6 +11262,7 @@ where fetch_to_op, partition.consensus().group(), partition.created_revision(), + partition.note_proof_walk_request(), ) .await; true @@ -10494,6 +11302,7 @@ where to_op, partition.consensus().group(), partition.created_revision(), + partition.note_proof_walk_request(), ) .await; } @@ -10545,6 +11354,7 @@ where to_op, partition.consensus().group(), partition.created_revision(), + partition.note_proof_walk_request(), ) .await; } @@ -10629,6 +11439,7 @@ where to_op, namespace, partition.created_revision(), + partition.note_proof_walk_request(), ) .await; } @@ -10824,36 +11635,20 @@ where // and re-run the staging scan for nothing. return; } - let reused = partition.reuse_staged_segments(&entries).await; - if !reused.is_empty() { - tracing::info!( - shard = self.id, - namespace_raw = header.group, - peer = header.replica, - adopted = reused.len(), - artifacts = entries.len(), - "adopted staged segments from an earlier transfer attempt" - ); - } - // Re-check the nonce AFTER the await: the staging scan yields, and a - // session re-minted underneath it must not get stamped with this - // (now stale) offer's commit_op and manifest. - let Some(session) = partition - .transfer - .as_mut() - .filter(|session| session.nonce == header.nonce) - else { + let scan = partition.begin_reuse_scan(&entries); + let Some(session) = partition.transfer.as_mut() else { return; }; session.target_accepted = true; session.commit_op = header.commit_op; + session.scan = Some(scan); // No reservation here: the manifest's total is bounded only by // `PARTITION_TRANSFER_TOTAL_LEN_MAX` (1 TiB), so reserving every // artifact up front is an eager address-space commit of the whole // manifest -- times the in-flight cap -- which turns fatal under strict // overcommit, `RLIMIT_AS`, or cgroup accounting, and contradicts the // session's own promise to bound receiver memory to ONE in-flight - // artifact. Artifacts adopted by the reuse scan below would also be + // artifact. Artifacts that the reuse scan adopts would also be // reserved and then overwritten with `Staged`, making the retry path's // reservation pure waste. `append_chunk` reserves the declared length on // an artifact's FIRST chunk instead, and only ever for the artifact the @@ -10868,9 +11663,6 @@ where }) .collect(); session.idle_ticks = 0; - for (index, meta) in reused { - session.artifacts[index as usize] = TransferArtifact::Staged(meta); - } let consensus = partition.consensus(); if consensus.state_transfer_stage() == consensus::StateTransferStage::AwaitingTarget { consensus.set_state_transfer_stage(consensus::StateTransferStage::Fetching); @@ -10921,6 +11713,9 @@ where if session.peer != header.replica { return; } + if session.spill.is_some() { + return; + } let payload = &msg.as_slice()[size_of::()..header.size as usize]; if !consensus::append_chunk( &mut session.artifacts, @@ -10931,6 +11726,7 @@ where return; } session.idle_ticks = 0; + session.pulled_len += payload.len() as u64; } partition.note_transfer_progress(); self.on_partition_transfer_progress(header.group).await; @@ -10944,7 +11740,6 @@ where where B: MessageBus + 'static, T: ShardsTable, - M: StreamsFrontend, { let planes = self.plane.inner(); let config = planes.1.0.config().clone(); @@ -10977,6 +11772,62 @@ where if !session.target_accepted { return; } + // The reuse scan writes the staging names that a spill writes, so + // nothing spills or installs before it lands. + if self.drive_reuse_scan(partition, namespace) { + return; + } + let landed_spill = partition.transfer.as_mut().and_then(|session| { + let spill = session.spill.as_ref()?; + let result = spill.work.result.borrow_mut().take()?; + let artifact = spill.artifact; + let seeded = spill.seeded; + session.spill = None; + Some((artifact, seeded, session.peer, result)) + }); + if let Some((index, seeded, peer, result)) = landed_spill { + match result { + Ok(spilled) => { + let meta = partition.accept_transfer_segment_spill(spilled); + if let Some(session) = partition.transfer.as_mut() { + session.artifacts[index] = TransferArtifact::Staged(meta); + } + } + // The local prefix is not a prefix of the offered segment. The + // spill took the buffer, so the artifact pulls again from its + // first byte, and nothing about the peer is charged. + Err(partitions::state_transfer::SpillError::ManifestChecksum { .. }) if seeded => { + tracing::warn!( + shard = self.id, + namespace_raw = namespace, + artifact = index, + "the local segment prefix does not match the offered artifact, \ + so the pull fetches the whole artifact" + ); + return Box::pin(self.on_partition_transfer_progress(namespace)).await; + } + Err(reason) => { + tracing::warn!( + shard = self.id, + namespace_raw = namespace, + artifact = index, + %reason, + "partition transfer segment failed validation at spill" + ); + self.abandon_or_rearm_partition_transfer(partition, peer) + .await; + return; + } + } + return Box::pin(self.on_partition_transfer_progress(namespace)).await; + } + let Some(session) = partition.transfer.as_ref() else { + return; + }; + + if session.spill.is_some() { + return; + } // Spill any segment artifact that just completed, freeing its buffer. let spill_candidate = session @@ -10993,45 +11844,38 @@ where .then_some(index) }); if let Some(index) = spill_candidate { - let (entry, bytes, peer, nonce) = { + let (entry, bytes, prefix) = { let Some(session) = partition.transfer.as_mut() else { return; }; + let prefix = match session.seed.take() { + Some(partitions::state_transfer::PrefixSeed::Seeded { artifact, prefix }) + if artifact as usize == index => + { + Some(prefix) + } + seed => { + session.seed = seed; + None + } + }; let Some(progress) = session.artifacts[index].pending_mut() else { return; }; let bytes = std::mem::take(&mut progress.buf); - (progress.entry, bytes, session.peer, session.nonce) + (progress.entry, bytes, prefix) }; - match partition.spill_transfer_segment(&entry, bytes).await { - Ok(meta) => { - // Nonce re-check across the spill await: a session - // re-minted underneath it has a fresh (possibly empty) - // artifact vec, and the stale index would panic. Pump- - // serial today, but nothing enforces that. - if let Some(session) = partition - .transfer - .as_mut() - .filter(|session| session.nonce == nonce) - { - session.artifacts[index] = TransferArtifact::Staged(meta); - } - } - Err(reason) => { - tracing::warn!( - shard = self.id, - namespace_raw = namespace, - artifact = index, - %reason, - "partition transfer segment failed validation at spill" - ); - self.abandon_or_rearm_partition_transfer(partition, peer) - .await; - return; - } + let seeded = prefix.is_some(); + let work = partition.begin_transfer_segment_spill(&entry, bytes, prefix); + if let Some(session) = partition.transfer.as_mut() { + session.idle_ticks = 0; + session.spill = Some(partitions::state_transfer::SegmentSpill { + artifact: index, + seeded, + work, + }); } - // Tail-call for the next candidate / chunk request. - return Box::pin(self.on_partition_transfer_progress(namespace)).await; + return; } let Some(session) = partition.transfer.as_ref() else { @@ -11052,6 +11896,13 @@ where let Some(session) = partition.transfer.take() else { return; }; + tracing::info!( + shard = self.id, + namespace_raw = namespace, + artifacts = session.artifacts.len(), + pulled_len = session.pulled_len, + "partition transfer pulled every artifact" + ); let commit_op = session.commit_op; let peer = session.peer; let mut offsets_bytes: Option> = None; @@ -11200,7 +12051,7 @@ where shard = self.id, namespace_raw = namespace, %error, - "partition state transfer install failed; falling back to journal repair" + "partition state transfer install failed; scheduling a re-arm" ); self.abandon_or_rearm_partition_transfer(partition, peer) .await; @@ -11214,8 +12065,8 @@ where /// undecodable artifact) re-ran the full pull -- including the serving /// primary's whole-segment reads -- at network round-trip rate, and the /// generation-keyed budget never exhausted on a committing cluster. - /// Journal repair is re-armed in the meantime so the gap stays visible - /// and anything repairable heals without waiting out the backoff. + /// Journal repair does not run meanwhile: the re-arm owns recovery (see + /// [`Self::maybe_request_partition_repair`]). #[allow(clippy::future_not_send)] async fn abandon_or_rearm_partition_transfer( &self, @@ -11309,7 +12160,6 @@ where rotate: bool, ) where B: MessageBus + 'static, - T: ShardsTable, { partition.transfer = None; let consensus = partition.consensus(); @@ -11346,7 +12196,6 @@ where }); let config = self.plane.partitions().config().clone(); partition.commit_journal(&config).await; - self.maybe_request_partition_repair(partition, peer).await; } /// Ask for the next missing partition chunk (first unspilled, incomplete @@ -11375,12 +12224,25 @@ where else { return; }; - let request = partition.transfer.as_ref().and_then(|session| { + if partition + .transfer + .as_ref() + .is_some_and(|session| session.spill.is_some()) + { + return; + } + if self.drive_reuse_scan(partition, namespace) || partition.drive_prefix_seed() { + return; + } + let request = partition.transfer.as_mut().and_then(|session| { if !session.target_accepted { return None; } let (index, offset, len) = consensus::next_pending_chunk(&session.artifacts, chunk_len_max)?; + // The serving offer ages from the last chunk that it served, and + // the bound on local work counts the same span. + session.idle_ticks = 0; Some((session.nonce, session.peer, index, offset, len)) }); let consensus_ids = { @@ -11403,6 +12265,47 @@ where } } + /// Hand a landed reuse scan to its session and log what it staged. + /// Returns `true` while the scan runs, because the pull waits for it. + fn drive_reuse_scan(&self, partition: &mut IggyPartition, namespace: u64) -> bool + where + B: MessageBus, + { + let Some(peer) = partition.transfer.as_ref().map(|session| session.peer) else { + return false; + }; + let Poll::Ready(reused) = partition.drive_reuse_scan() else { + return true; + }; + let Some(reused) = reused else { + return false; + }; + if reused.adopted > 0 { + tracing::info!( + shard = self.id, + namespace_raw = namespace, + peer, + adopted = reused.adopted, + adopted_len = reused.adopted_len, + artifacts = reused.artifacts, + "adopted staged segments from an earlier transfer attempt" + ); + } + if reused.linked > 0 { + tracing::info!( + shard = self.id, + namespace_raw = namespace, + peer, + reused = reused.linked, + reused_len = reused.linked_len, + artifacts = reused.artifacts, + total_len = reused.total_len, + "reused segments that this node already holds" + ); + } + false + } + /// Drop serving-side state-transfer offers that stopped being pulled. /// /// An offer pins its plane's payload for as long as it lives -- the metadata @@ -11446,6 +12349,15 @@ where } live }); + // A waiter that stopped asking (it installed from another peer, + // re-targeted or crashed) would otherwise keep the next free slot + // reserved for good. + self.partition_transfer_waiters + .borrow_mut() + .retain_mut(|(_, idle_ticks)| { + *idle_ticks += 1; + *idle_ticks < idle_expiry_ticks + }); let mut offers = self.state_transfer_offers.borrow_mut(); let namespaces_before: Vec = offers.keys().map(|(namespace, _)| *namespace).collect(); offers.retain(|(namespace, requester), served| { @@ -13760,7 +14672,14 @@ async fn dispatch_vsr_actions( continue; }; for replica in replicas { - send(*replica, frozen.clone()).await; + let _ = send_replayed_prepare( + consensus, + *replica, + header.op, + frozen.clone(), + ReplicationSend::Retransmit, + ) + .await; } } } @@ -13785,6 +14704,26 @@ async fn dispatch_vsr_actions( // Handled by the caller, which owns the plane's journal. Metadata // ignores `RetryLocalWrite` and re-acks via `repair_primary_self_acks`. VsrAction::CommitJournal | VsrAction::RetryLocalWrite { .. } => {} + VsrAction::SendCommitReply { + view, + group, + timestamp_monotonic, + target, + } => { + let message = Message::::new(size_of::()) + .transmute_header(|_, header: &mut CommitReplyHeader| { + header.command = Command::CommitReply; + header.partition_incarnation = consensus.partition_incarnation(); + header.cluster = cluster; + header.replica = self_id; + header.view = *view; + header.group = *group; + header.timestamp_monotonic = *timestamp_monotonic; + header.size = size_of::() as u32; + header.seal(); + }); + send(*target, message.into_generic().into_frozen()).await; + } VsrAction::SendCommit { view, commit, @@ -13860,16 +14799,9 @@ async fn dispatch_partition_journal_actions( P: Pipeline, SB: SuperblockStore, { - let bus = consensus.message_bus(); let self_id = consensus.replica(); let journal = &partition.log.journal().inner; - let send = |target: u8, msg: Frozen| async move { - if let Err(e) = bus.send_to_replica(target, msg).await { - tracing::debug!(replica = self_id, target, "bus send failed: {e}"); - } - }; - // Same durable-before-send tripwire as `dispatch_vsr_actions`: this // dispatcher emits view-scoped `SendPrepareOk` too, and all callers are // persist-gated today -- assert it so a future bypass cannot slip @@ -13918,7 +14850,14 @@ async fn dispatch_partition_journal_actions( continue; }; for replica in replicas { - send(*replica, prepare.clone()).await; + let _ = send_replayed_prepare( + consensus, + *replica, + header.op, + prepare.clone(), + ReplicationSend::Retransmit, + ) + .await; } } } @@ -13940,6 +14879,32 @@ async fn dispatch_partition_journal_actions( } } +#[allow(clippy::future_not_send)] +async fn send_replayed_prepare( + consensus: &VsrConsensus, + target: u8, + op: u64, + entry: Frozen, + send: ReplicationSend, +) -> bool +where + B: MessageBus, + P: Pipeline, +{ + if !consensus.is_primary() || !consensus.is_normal() || consensus.is_transferring() { + return false; + } + let Some(entry) = restamp_prepare_view(entry, consensus.view()) else { + return false; + }; + matches!( + consensus + .send_replicated_prepare(target, op, entry, send) + .await, + Ok(true) + ) +} + #[cfg(test)] mod warn_pace_tests { use super::*; @@ -14917,15 +15882,14 @@ mod dvc_suffix_window_tests { #[cfg(test)] mod installed_transfer_dvc_tests { //! A lagging replica adopts a `StartView` and later installs a state - //! transfer below that view's head. The install rewinds the sequencer to - //! the transfer's op and keeps the adopted view. Each `DoViewChange` the - //! replica sends from then on carries the transfer op in its header but - //! starts its suffix at the adopted head, and its peers drop every one as - //! unreadable. With a third replica down, or back with an untrusted log, - //! no quorum of readable votes forms and the view change never ends. + //! transfer below that view's head. The install keeps that head, so each + //! `DoViewChange` the replica sends afterwards stays readable and + //! describes every uncommitted op of the adopted view. With a third + //! replica down, or back with an untrusted log, the view change has no + //! other quorum of readable votes. use super::*; - use consensus::{LocalPipeline, dvc_suffix_decode}; + use consensus::{LocalPipeline, TimeoutManager, dvc_suffix_decode}; use iggy_common::{IggyByteSize, PartitionStats}; use journal::prepare_journal::PrepareJournal; use message_bus::IggyMessageBus; @@ -14946,10 +15910,9 @@ mod installed_transfer_dvc_tests { const COMMIT: u64 = HEAD - 2; #[compio::test] - #[ignore = "known bug: a transfer installed below the adopted view head makes the replica's DoViewChange unreadable; run with --ignored"] async fn given_a_transfer_installed_below_the_adopted_head_when_a_view_change_follows_should_send_a_readable_do_view_change() { - // Inside the 128-entry window, so only the head disagrees. + // Inside the 128-entry window: only the header op can disagree with the suffix. let transfer = HEAD - 100; // Boxed: the setup future carries a whole partition. let (vote, message) = Box::pin(vote_after_install(transfer)).await; @@ -14976,7 +15939,6 @@ mod installed_transfer_dvc_tests { } #[compio::test] - #[ignore = "known bug: a transfer installed far below the adopted view head leaves uncommitted ops outside the DoViewChange window; run with --ignored"] async fn given_a_transfer_installed_far_below_the_adopted_head_when_a_view_change_follows_should_describe_every_uncommitted_op() { // Past the 128-entry window. @@ -14999,8 +15961,9 @@ mod installed_transfer_dvc_tests { } /// The lagging replica's vote for `NEXT_VIEW`, and the frame the shard sends - /// for it. Before the vote, the replica adopts the `ADOPTED_VIEW` `StartView` - /// and then installs a peer's state transfer at `transfer`. + /// for it. Before the vote, the replica adopts the `ADOPTED_VIEW` `StartView`, + /// installs a peer's state transfer at `transfer`, and then stops hearing the + /// primary of `ADOPTED_VIEW`, which is down. #[allow(clippy::future_not_send)] async fn vote_after_install(transfer: u64) -> (VsrAction, Message) { let bus = IggyMessageBus::new(0); @@ -15045,10 +16008,9 @@ mod installed_transfer_dvc_tests { // Repair finds the gap evicted at every peer, so a transfer follows. The // install cannot run here: its offsets artifact is a type private to - // `partitions`. So these are its journal wipe and then the steps of - // `publish_installed_consensus`, in its order, with a checkpoint prepare - // built here in place of the origin's. They rewind the sequencer to - // `transfer` and keep the adopted view. + // `partitions`. So these are its journal wipe and then the consensus step + // of `publish_installed_consensus`, with a checkpoint prepare built here in + // place of the origin's. let checkpoint = Message::::new(size_of::()) .transmute_header(|_, header: &mut PrepareHeader| { *header = prepare_header(transfer, 0); @@ -15058,12 +16020,19 @@ mod installed_transfer_dvc_tests { transfer, Owned::::copy_from_slice(checkpoint.as_slice()).into(), ); - let consensus = partition.consensus(); - consensus.set_commit_floor(transfer); - consensus.sequencer().set_sequence(transfer); - consensus.set_last_prepare_checksum(checkpoint.header().checksum); - consensus.clear_pipeline(); - consensus.advance_commit_max(transfer); + partition.consensus().note_journal_mutation(); + partition + .consensus() + .install_checkpoint(transfer, Some(checkpoint.header().checksum)); + + // The heartbeat timeout votes to leave the view, so the `StartViewChange` + // of replica 1 completes a quorum of votes. + let voted = (0..TimeoutManager::NORMAL_HEARTBEAT_TICKS) + .flat_map(|_| partition.consensus().tick(PlaneKind::Partitions)) + .any(|action| { + matches!(action, VsrAction::SendStartViewChange { view, .. } if view == NEXT_VIEW) + }); + assert!(voted, "the lagging replica votes to leave the view"); // `on_start_view_change`: snapshot the suffix, then vote. refresh_partition_dvc_suffix(&partition); @@ -15163,8 +16132,8 @@ mod adopted_view_retention_tests { use super::{adopted_view_headers, build_dvc_suffix, refresh_metadata_dvc_suffix}; use consensus::{ CLIENTS_TABLE_MAX, ClientTable, Consensus, JoinMode, LocalPipeline, PlaneKind, Project, - Sequencer, StateTransferStage, ViewLogProof, ViewRestore, VsrAction, VsrConsensus, - encode_prepare_headers, send_prepare_ok, view_log_proof, + Sequencer, StateTransferStage, TimeoutManager, ViewLogProof, ViewRestore, VsrAction, + VsrConsensus, encode_prepare_headers, send_prepare_ok, view_log_proof, }; use iggy_binary_protocol::{ Command, DoViewChangeHeader, Operation, PrepareHeader, PrepareOkHeader, @@ -15377,6 +16346,8 @@ mod adopted_view_retention_tests { }; lagging.fence_view_log(op, unproven_through); + vote_on_heartbeat_timeout(&restarted, NEXT_VIEW); + vote_on_heartbeat_timeout(&lagging, NEXT_VIEW); refresh_suffix(&restarted, journal); refresh_suffix(&lagging, lagging_journal); let vote = restarted @@ -15555,6 +16526,8 @@ mod adopted_view_retention_tests { }); reconcile(&lagging, lagging_journal); + vote_on_heartbeat_timeout(restarted, NEXT_VIEW); + vote_on_heartbeat_timeout(&lagging, NEXT_VIEW); refresh_metadata_dvc_suffix(restarted, metadata.journal.as_ref()); refresh_suffix(&lagging, lagging_journal); let vote = restarted @@ -15669,14 +16642,16 @@ mod adopted_view_retention_tests { ); next_primary.handle_commit(heartbeat.header()); next_primary.advance_commit_min(COMMITTED); - let mut replacement = original.clone(); + let replacement = original.clone(); let abandoned = projected_stream(&old_primary, "abandoned"); old_primary.pipeline_message(PlaneKind::Metadata, &abandoned); original.push(*abandoned.header()); old_primary.extend_history_certificate(HEAD, abandoned.header().checksum); // Replica 1 is isolated. The two remaining replicas decide view 11 - // through timer-produced SVCs and the DVCs their handlers emit. + // through the votes of their heartbeat timeouts and the DVCs their + // handlers emit. + vote_on_heartbeat_timeout(&lagging, ADOPTED_VIEW); elect_from_handlers( &next_primary, &lagging, @@ -15689,32 +16664,9 @@ mod adopted_view_retention_tests { assert_eq!(lagging.log_view_claim(), ADOPTED_VIEW); assert!(lagging.view_log_unproven()); let uncommitted = projected_stream(&next_primary, "replacement"); - next_primary.pipeline_message(PlaneKind::Metadata, &uncommitted); - replacement.push(*uncommitted.header()); - next_primary.extend_history_certificate(HEAD, uncommitted.header().checksum); assert_ne!(abandoned.header().checksum, uncommitted.header().checksum); - // Drop all repair traffic to replica 0 and the intermediate elections. - // Its timer eventually nominates replica 2 again. One body plus one nack - // retains the uncommitted replacement, which that primary announces. - elect_from_handlers( - &next_primary, - &lagging, - &replacement, - &lagging_journal, - LATER_VIEW, - ); - assert_eq!(next_primary.pending_view_log().unwrap().op_head, HEAD); - announce_from_handler(&next_primary, &lagging, &replacement, &lagging_journal); - assert_eq!(lagging.log_view_claim(), LATER_VIEW); - assert_eq!(lagging.commit_min(), COMMON); - assert_eq!( - lagging.pending_view_log().unwrap().headers[0].checksum, - uncommitted.header().checksum - ); - - let lagging = if restart { - let record = lagging.vsr_state(COMMON, 0); + let restore = |record: &consensus::VsrState| { let record = consensus::VsrState::try_from(record.to_bytes().as_slice()).unwrap(); let mut recovered = replica(BACKUP); @@ -15736,6 +16688,37 @@ mod adopted_view_retention_tests { recovered.resume_adopted_view(); recovered.init_as_backup(); recovered.begin_view_probe(); + recovered + }; + + // A restarted replica whose probes go unanswered enters the next view + // alone, so a replica that restarts while cut off can reach a view that + // no quorum voted for, and the merge of its stitched unproven headers + // must still hold. Replica 0 adopted the `LATER_VIEW` `StartView` of + // replica 2, which kept the uncommitted replacement above `COMMITTED`, + // and later restarted in the view before `target_view`. + let lagging = restore(&consensus::VsrState { + view: target_view - 1, + log_view: LATER_VIEW, + adopted_view: Some(consensus::AdoptedView { + view: LATER_VIEW, + group: METADATA_GROUP, + partition_incarnation: 0, + op_head: HEAD, + commit_max: COMMITTED, + headers: vec![*uncommitted.header(), *committed.header()], + }), + ..lagging.vsr_state(COMMON, 0) + }); + assert_eq!(lagging.log_view_claim(), LATER_VIEW); + assert_eq!(lagging.commit_min(), COMMON); + assert_eq!( + lagging.pending_view_log().unwrap().headers[0].checksum, + uncommitted.header().checksum + ); + + let lagging = if restart { + let recovered = restore(&lagging.vsr_state(COMMON, 0)); assert_eq!(recovered.adopted_view_head(), Some((LATER_VIEW, HEAD))); assert!(!recovered.history_certified()); recovered @@ -15978,6 +16961,22 @@ mod adopted_view_retention_tests { )); } + /// Tick `consensus` until its heartbeat timeout fires. The replica no longer + /// hears its primary, so it votes for `view`, and the `StartViewChange` of a + /// survivor that votes the same completes a quorum of votes. + fn vote_on_heartbeat_timeout(consensus: &VsrConsensus, view: u32) { + let voted = (0..TimeoutManager::NORMAL_HEARTBEAT_TICKS) + .flat_map(|_| consensus.tick(PlaneKind::Metadata)) + .any(|action| { + matches!(action, VsrAction::SendStartViewChange { view: next, .. } if next == view) + }); + assert!( + voted, + "replica {} votes for view {view}", + consensus.replica() + ); + } + /// The proof `reconcile_metadata_view_divergence` takes of an adopted view /// once the replica is not transferring, over a journal answering `held`. fn reconcile(consensus: &VsrConsensus, held: impl Fn(u64) -> Option) { @@ -17020,7 +18019,7 @@ mod partition_placement_tests { use std::mem::size_of; use std::sync::Arc; - type PlacementShard = IggyShard< + pub type PlacementShard = IggyShard< Rc, PrepareJournal, IggySnapshot, @@ -17029,20 +18028,30 @@ mod partition_placement_tests { >; pub fn owner(shard_id: u16) -> (PlacementShard, Receiver) { + owner_with_repair_memory(shard_id, None) + } + + pub fn owner_with_repair_memory( + shard_id: u16, + repair_memory: Option>, + ) -> (PlacementShard, Receiver) { let bus = Rc::new(IggyMessageBus::new(shard_id)); let segment_size = IggyByteSize::from(1_048_576_u64); - let partitions = IggyPartitions::new( - ShardId::new(shard_id), - PartitionsConfig { - messages_required_to_save: 100, - size_of_messages_required_to_save: segment_size, - validate_checksum: true, - segment_size, - preallocate_segments: false, - key_context: Arc::default(), - path_layout: PartitionPathLayout::default(), - }, - ); + let config = PartitionsConfig { + messages_required_to_save: 100, + size_of_messages_required_to_save: segment_size, + validate_checksum: true, + segment_size, + preallocate_segments: false, + key_context: Arc::default(), + path_layout: PartitionPathLayout::default(), + }; + let partitions = match repair_memory { + Some(budget) => { + IggyPartitions::with_repair_memory(ShardId::new(shard_id), config, 0, budget) + } + None => IggyPartitions::new(ShardId::new(shard_id), config), + }; let metadata = IggyMetadata::new( None, None, @@ -17065,7 +18074,7 @@ mod partition_placement_tests { (shard, target) } - fn insert(shard: &PlacementShard, id: usize, revision: u64, view: u32) -> IggyNamespace { + pub fn insert(shard: &PlacementShard, id: usize, revision: u64, view: u32) -> IggyNamespace { let namespace = IggyNamespace::new(0, 0, id); let mut consensus = VsrConsensus::new( 1, @@ -17978,3 +18987,362 @@ mod untrusted_do_view_change_tests { )); } } + +#[cfg(test)] +mod replication_flow_tests { + use super::*; + use crate::partition_placement_tests::{PlacementShard, insert, owner_with_repair_memory}; + use consensus::REPLICATION_BATCH_BYTES_MAX; + use journal::prepare_journal::PrepareJournal; + use message_bus::IggyMessageBus; + use server_common::iobuf::Owned; + use server_common::sharding::METADATA_GROUP; + use std::collections::BTreeSet; + use std::sync::Arc; + + type SentFrames = Rc)>>>; + + fn recording_shard() -> (PlacementShard, SentFrames) { + const REPAIR_MEMORY_BYTES_MAX: usize = 64 * 1024; + let (shard, _) = owner_with_repair_memory( + 0, + Some(Arc::new(partitions::RepairMemoryBudget::new( + REPAIR_MEMORY_BYTES_MAX, + ))), + ); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + shard + .bus + .set_replica_forward_fn(Box::new(move |replica, _, frame| { + captured.borrow_mut().push((replica, frame)); + Ok(()) + })); + for replica in 1..3 { + crate::test_support::connect_replica_on(&shard.bus, replica, 1); + } + (shard, sent) + } + + fn prepare(group: u64, op: u64) -> Message { + Message::::new(size_of::()).transmute_header( + |_, header: &mut PrepareHeader| { + header.command = Command::Prepare; + header.operation = Operation::StoreConsumerOffset; + header.cluster = 1; + header.group = group; + header.op = op; + header.timestamp = op; + header.size = u32::try_from(size_of::()).unwrap(); + header.checksum = header.identity_checksum(); + }, + ) + } + + fn ready_reply( + consensus: &VsrConsensus>, + replica: u8, + nonce: u64, + observed_op: u64, + received_op: u64, + ) -> Message { + Message::::new(size_of::()).transmute_header( + |_, header: &mut PrepareProgressHeader| { + header.command = Command::PrepareProgress; + header.cluster = consensus.cluster(); + header.group = consensus.group(); + header.view = consensus.view(); + header.partition_incarnation = consensus.partition_incarnation(); + header.replica = replica; + header.kind = PrepareProgressHeader::REPLY; + header.ready = 1; + header.nonce = nonce; + header.observed_op = observed_op; + header.held_op = received_op; + header.received_op = received_op; + header.size = u32::try_from(size_of::()).unwrap(); + header.seal(); + }, + ) + } + + #[compio::test] + async fn given_a_full_transfer_admission_when_another_group_requests_should_not_pin_its_repair_tail() + { + const NONCE: u128 = 1; + let (shard, _) = recording_shard(); + let namespace = insert(&shard, 0, 0, 0); + let busy = insert(&shard, 1, 0, 0); + shard.served_segment_cache_bytes_max.set(0); + shard + .partition_offer_builds + .borrow_mut() + .insert(busy.inner(), 0); + let partition = shard.plane.partitions().get_by_ns(&namespace).unwrap(); + assert!(partition.serves_state_transfer()); + assert!(!partition.touch_transfer_tail(1, NONCE, 0)); + let request = + Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestStateTransferHeader| { + header.command = Command::RequestStateTransfer; + header.cluster = 1; + header.replica = 1; + header.group = namespace.inner(); + header.partition_incarnation = partition.consensus().partition_incarnation(); + header.nonce = NONCE; + header.size = u32::try_from(size_of::()).unwrap(); + header.seal(); + }); + + shard + .serve_partition_transfer_request(*request.header(), None) + .await; + + assert!( + !partition.touch_transfer_tail(1, NONCE, 0), + "a refused transfer must not retain a live repair-tail lease" + ); + assert_eq!( + shard.partition_transfer_waiters.borrow().as_slice(), + &[(namespace.inner(), 0)] + ); + } + + #[compio::test] + async fn given_a_delayed_progress_probe_when_prepares_retransmit_should_share_credit_and_resume_after_its_reply() + { + let (shard, sent) = recording_shard(); + let namespace = insert(&shard, 0, 0, 0); + let partition = shard.plane.partitions().get_by_ns(&namespace).unwrap(); + let entry = prepare(namespace.inner(), 1); + let header = *entry.header(); + partition + .log + .journal() + .inner + .append(entry.into_frozen()) + .await + .unwrap(); + let consensus = partition.consensus(); + consensus.sequencer().set_sequence(1); + assert!(consensus.admit_replication( + 1, + 1, + REPLICATION_BATCH_BYTES_MAX, + ReplicationSend::Live + )); + let Some(consensus::ReplicationWork::Probe { nonce, through, .. }) = consensus + .record_replication_send(1, 1, REPLICATION_BATCH_BYTES_MAX, ReplicationSend::Live) + else { + panic!("a full window must send a checkpoint behind its last frame"); + }; + let retry = VsrAction::RetransmitPrepares { + targets: vec![(header, vec![1, 2])], + }; + + dispatch_partition_journal_actions(consensus, partition, &[retry]).await; + + assert_eq!( + sent.borrow() + .iter() + .map(|(replica, _)| *replica) + .collect::>(), + vec![2] + ); + assert!( + !consensus.admit_replication( + 2, + 2, + REPLICATION_BATCH_BYTES_MAX, + ReplicationSend::Retransmit + ), + "a successful timeout retry must consume the same byte credit as replay" + ); + let reply = ready_reply(consensus, 1, nonce, through, 1); + assert!(consensus.handle_prepare_progress(reply.header()).is_some()); + assert_eq!( + consensus.commit_max(), + 0, + "progress is not a PrepareOk vote" + ); + sent.borrow_mut().clear(); + dispatch_partition_journal_actions( + consensus, + partition, + &[VsrAction::RetransmitPrepares { + targets: vec![(header, vec![1])], + }], + ) + .await; + assert_eq!( + sent.borrow().len(), + 1, + "the duplicate must still recover a lost PrepareOk" + ); + assert_eq!(sent.borrow()[0].0, 1); + } + + #[compio::test] + async fn given_a_metadata_progress_probe_when_prepares_retransmit_should_share_the_peer_window() + { + const JOURNAL_SLOTS: usize = 128; + let (shard, sent) = recording_shard(); + let consensus = VsrConsensus::new( + 1, + 0, + 3, + METADATA_GROUP, + Rc::clone(&shard.bus), + consensus::LocalPipeline::new(), + ); + consensus.init(); + let directory = tempfile::tempdir().unwrap(); + let journal = PrepareJournal::open_with_slots( + &directory.path().join("journal.wal"), + 0, + JOURNAL_SLOTS, + ) + .await + .unwrap(); + let entry = prepare(METADATA_GROUP, 1); + let header = *entry.header(); + journal.append(entry).await.unwrap(); + consensus.sequencer().set_sequence(1); + assert!(consensus.admit_replication( + 1, + 1, + REPLICATION_BATCH_BYTES_MAX, + ReplicationSend::Live + )); + let Some(consensus::ReplicationWork::Probe { nonce, through, .. }) = consensus + .record_replication_send(1, 1, REPLICATION_BATCH_BYTES_MAX, ReplicationSend::Live) + else { + panic!("a full metadata window must send a checkpoint behind its last frame"); + }; + dispatch_vsr_actions( + &consensus, + Some(&journal), + &[VsrAction::RetransmitPrepares { + targets: vec![(header, vec![1, 2])], + }], + ) + .await; + assert_eq!( + sent.borrow() + .iter() + .map(|(replica, _)| *replica) + .collect::>(), + vec![2] + ); + assert!(!consensus.admit_replication( + 2, + 2, + REPLICATION_BATCH_BYTES_MAX, + ReplicationSend::Retransmit + )); + let reply = ready_reply(&consensus, 1, nonce, through, 1); + assert!(consensus.handle_prepare_progress(reply.header()).is_some()); + sent.borrow_mut().clear(); + dispatch_vsr_actions( + &consensus, + Some(&journal), + &[VsrAction::RetransmitPrepares { + targets: vec![(header, vec![1])], + }], + ) + .await; + assert_eq!( + sent.borrow().len(), + 1, + "metadata retries resume after the correlated reply" + ); + } + + #[compio::test] + async fn given_several_groups_and_peers_in_replay_when_ticks_advance_should_bound_each_turn_and_visit_every_pair() + { + const GROUPS: usize = 3; + const OPS: u64 = 3; + const PULL_ROUND_OPS_MAX: u64 = 1; + let (shard, sent) = recording_shard(); + shard.repair_chunk_max.set(PULL_ROUND_OPS_MAX); + let mut expected = BTreeSet::new(); + for id in 0..GROUPS { + let namespace = insert(&shard, id, 0, 0); + let partition = shard.plane.partitions().get_by_ns(&namespace).unwrap(); + for op in 1..=OPS { + partition + .log + .journal() + .inner + .append(prepare(namespace.inner(), op).into_frozen()) + .await + .unwrap(); + } + let consensus = partition.consensus(); + consensus.sequencer().set_sequence(OPS); + for replica in 1..3 { + assert!(consensus.admit_replication( + replica, + 1, + size_of::() as u64, + ReplicationSend::Live + )); + consensus.record_replication_failure(replica, 1); + let Some(consensus::ReplicationWork::Probe { nonce, through, .. }) = + consensus.replication_work(replica) + else { + panic!("a missing prefix must request progress"); + }; + let reply = ready_reply(consensus, replica, nonce, through, 0); + assert!(consensus.handle_prepare_progress(reply.header()).is_some()); + expected.insert((namespace.inner(), replica)); + } + } + let mut observed = BTreeSet::new(); + for tick in 1..=expected.len() { + let op = OPS + u64::try_from(tick).unwrap(); + for id in 0..GROUPS { + let namespace = IggyNamespace::new(0, 0, id); + let partition = shard.plane.partitions().get_by_ns(&namespace).unwrap(); + partition + .log + .journal() + .inner + .append(prepare(namespace.inner(), op).into_frozen()) + .await + .unwrap(); + partition.consensus().sequencer().set_sequence(op); + } + sent.borrow_mut().clear(); + assert!(shard.tick_partitions(&mut Vec::new()).await.is_none()); + let frames = sent.borrow(); + let prepares: Vec<_> = frames + .iter() + .filter_map(|(replica, frame)| { + let message = Message::::try_from(Owned::copy_from_slice( + frame.as_slice(), + )) + .ok()?; + let header = message.header(); + (header.command == Command::Prepare).then_some((header.group, *replica)) + }) + .collect(); + assert!( + u64::try_from(prepares.len()).unwrap() > PULL_ROUND_OPS_MAX, + "live replay must not inherit the pull repair operation-count limit" + ); + let pairs: BTreeSet<_> = prepares.into_iter().collect(); + assert_eq!( + pairs.len(), + 1, + "one shard tick must drive only one bounded replay range" + ); + assert!( + observed.insert(*pairs.first().unwrap()), + "a busy pair must not starve another pair" + ); + } + assert_eq!(observed, expected); + } +} diff --git a/core/shard/src/metrics.rs b/core/shard/src/metrics.rs index 6a25f91ec7..d536227758 100644 --- a/core/shard/src/metrics.rs +++ b/core/shard/src/metrics.rs @@ -47,6 +47,7 @@ use std::hash::{Hash, Hasher}; use std::sync::{Arc, OnceLock}; use std::time::Duration; +use consensus::ReplicationStats; use iggy_binary_protocol::{COMMAND_TABLE, Command, command_index}; use iggy_common::ConsumerKind; use message_bus::{ReplicaLinkMetrics, ReplicaReadMetrics}; @@ -376,6 +377,8 @@ pub struct ShardMetrics { replica_socket_reads_total: Counter, replica_inbound_frames_total: Counter, replica_outbound_drops_total: Counter, + replication_live_refusals_total: Counter, + replication_replayed_prepares_total: Counter, replica_bundle_teardowns_total: Counter, replica_preamble_refusals_total: Counter, replica_handshake_failures_total: Counter, @@ -466,6 +469,8 @@ impl ShardMetrics { replica_socket_reads_total: Counter::default(), replica_inbound_frames_total: Counter::default(), replica_outbound_drops_total: Counter::default(), + replication_live_refusals_total: Counter::default(), + replication_replayed_prepares_total: Counter::default(), replica_bundle_teardowns_total: Counter::default(), replica_preamble_refusals_total: Counter::default(), replica_handshake_failures_total: Counter::default(), @@ -544,6 +549,31 @@ impl ShardMetrics { .set(i64::try_from(stats.bundles_up).unwrap_or(i64::MAX)); } + /// Fold one sweep's replication flow events in, drained by + /// `VsrConsensus::take_replication_stats`. Both stay at zero while every + /// backup keeps up, so a nonzero value on a run with no fault means that + /// flow control held prepares back. + pub fn record_replication(&self, stats: ReplicationStats) { + self.replication_live_refusals_total + .inc_by(stats.live_refusals); + self.replication_replayed_prepares_total + .inc_by(stats.replayed); + } + + /// Snapshot of `replication_live_refusals_total`. Test/simulator accessor. + #[cfg(any(test, feature = "simulator"))] + #[must_use] + pub fn replication_live_refusals_value(&self) -> u64 { + self.replication_live_refusals_total.get() + } + + /// Snapshot of `replication_replayed_prepares_total`. Test/simulator accessor. + #[cfg(any(test, feature = "simulator"))] + #[must_use] + pub fn replication_replayed_prepares_value(&self) -> u64 { + self.replication_replayed_prepares_total.get() + } + fn register_persistence(&self, registry: &mut Registry) { registry.register( "partition_wal_disk_bytes", @@ -1209,6 +1239,16 @@ impl ShardMetrics { "replica frames dropped because a peer send queue was full", self.replica_outbound_drops_total.clone(), ); + registry.register( + "replication_live_refusals", + "live prepares that a backup's replication window, replay cursor or busy state refused", + self.replication_live_refusals_total.clone(), + ); + registry.register( + "replication_replayed_prepares", + "prepares resent to a backup by ordered replay", + self.replication_replayed_prepares_total.clone(), + ); registry.register( "replica_bundle_teardowns", "replica bundles torn down because a link closed or the bundle stayed incomplete", diff --git a/core/shard/src/partition_io.rs b/core/shard/src/partition_io.rs index 1c6841a652..d920751188 100644 --- a/core/shard/src/partition_io.rs +++ b/core/shard/src/partition_io.rs @@ -797,8 +797,8 @@ mod tests { use consensus::{ ArtifactProgress, Consensus, DvcSuffix, LocalPipeline, PartitionsHandle, Pipeline, Plane, - Sequencer, StateArtifact, StateTransferStage, TimeoutManager, VsrAction, VsrConsensus, - artifact_kind, + REPLICATION_BATCH_BYTES_MAX, ReplicationSend, Sequencer, StateArtifact, StateTransferStage, + TimeoutManager, VsrAction, VsrConsensus, artifact_kind, }; use futures::FutureExt; use futures::channel::oneshot; @@ -828,7 +828,8 @@ mod tests { use metadata::stm::user::{Users, UsersInner}; use metadata::{IggyMetadata, MuxStateMachine}; use partitions::state_transfer::{ - PartitionArtifactSource, PartitionTransferSession, TransferArtifact, + LocalWork, PartitionArtifactSource, PartitionTransferSession, PrefixSeed, ReuseScanned, + StagedSegmentMeta, TransferArtifact, }; use partitions::{ IggyIndexWriter, IggyPartition, IggyPartitions, MessagesWriter, PartitionPathLayout, @@ -845,6 +846,7 @@ mod tests { use crate::host::NoopHost; use crate::metrics::ShardMetrics; use crate::shards_table::{PapayaShardsTable, ShardsTable}; + use crate::test_support::connect_replica_on; use crate::{ IggyShard, LifecycleFrame, PARTITION_REPAIRS_INFLIGHT_MAX, PartitionConsensusConfig, REPAIR_GAP_DEBOUNCE_TICKS_MIN, Receiver, ReconcileOp, ReplicaTopology, ShardFrame, @@ -1019,6 +1021,7 @@ mod tests { 1, group, consensus.partition_incarnation(), + None, ) .await; owner @@ -1124,6 +1127,464 @@ mod tests { assert_eq!(control_frame_incarnation(frame.as_slice()), INCARNATION); } + #[compio::test] + async fn given_a_served_snapshot_when_later_writes_overflow_the_ring_should_keep_its_repair_tail() + { + const NONCE: u128 = 1; + const OPS: u64 = 8; + const BUDGET: usize = 64 * 1024; + let bus = Rc::new(IggyMessageBus::new(0)); + let targets = record_transfer_targets(&bus); + connect_replica_on(&bus, TRANSFER_REQUESTER, 1); + let (mut owner, _sender) = test_owner_with_repair_memory( + &bus, + None, + Some(Arc::new(partitions::RepairMemoryBudget::new(BUDGET))), + ); + owner.partition_consensus.replica_count = TRANSFER_REQUESTER + 1; + let namespace = IggyNamespace::new(0, 0, 0); + let directory = tempfile::tempdir().unwrap(); + let partitions = owner.plane.partitions(); + let partition = partitions.get_mut_by_ns(&namespace).unwrap(); + partition.set_created_revision(SERVED_INCARNATION); + attach_segment_files(partition, directory.path(), false).await; + let journal = &partition.log.journal().inner; + journal.set_repair_retention(true); + journal.set_ring_caps(1, u64::MAX); + for op in 1..=OPS { + let request = send_request(namespace, op).transmute_header( + |old, header: &mut RoutedRequestHeader| { + *header = old; + header.partition_incarnation = SERVED_INCARNATION; + }, + ); + let (reply, replied) = consensus::oneshot_channel(); + partitions.on_request_with_reply(request, Some(reply)).await; + owner + .process_loopback(&mut crate::router::LoopbackRound::default()) + .await; + assert_eq!(replied.await.unwrap().header().status, 0); + if op == 1 { + owner + .on_message(request_state_transfer(namespace, NONCE, SERVED_INCARNATION)) + .await; + assert_eq!( + targets.borrow().last(), + Some(&(namespace.inner(), NONCE, OFFERED)) + ); + } + } + let partition = partitions.get_by_ns(&namespace).unwrap(); + // Flushing also exercises eviction after the offer's pinned cut. + partition + .log + .journal() + .inner + .evict_prefix(usize::try_from(OPS).unwrap()); + let journal = &partition.log.journal().inner; + assert_eq!(journal.repair_retained_from(), Some(2)); + for op in 2..=OPS { + assert!( + journal.repair_entry(op).is_some(), + "snapshot tail lost op {op}" + ); + } + owner.ticks.set(9); + let repair = Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestPreparesHeader| { + header.command = Command::RequestPrepares; + header.size = u32::try_from(size_of::()).unwrap(); + header.cluster = 1; + header.group = namespace.inner(); + header.replica = TRANSFER_REQUESTER; + header.partition_incarnation = SERVED_INCARNATION; + header.nonce = NONCE + 1; + header.from_op = 2; + header.to_op = OPS; + header.seal(); + }); + owner.on_message(MessageBag::RequestPrepares(repair)).await; + partition.expire_transfer_tails(10, 10); + assert_eq!( + journal.repair_retained_from(), + Some(2), + "active pull repair must renew the snapshot suffix" + ); + partition.expire_transfer_tails(19, 10); + assert_eq!(journal.repair_retained_from(), Some(OPS)); + owner + .on_message(request_state_transfer(namespace, NONCE, SERVED_INCARNATION)) + .await; + assert_eq!( + targets.borrow().last(), + Some(&(namespace.inner(), NONCE, REFUSED)) + ); + assert_eq!( + owner.state_transfer_offers.borrow()[&(namespace.inner(), TRANSFER_REQUESTER)].nonce, + NONCE + ); + } + + #[compio::test] + async fn given_a_large_transfer_artifact_when_requesting_a_range_should_read_only_that_range() { + const NONCE: u128 = 1; + const PREFIX: &[u8] = b"held prefix"; + const SUFFIX: &[u8] = b"remaining segment"; + const CHUNK_LEN: u32 = 4; + for offset in [0, PREFIX.len()] { + let bus = Rc::new(IggyMessageBus::new(0)); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + captured.borrow_mut().push(frame); + Ok(()) + })); + connect_replica_on(&bus, TRANSFER_REQUESTER, 1); + let (mut owner, _sender) = test_owner(&bus, None); + owner.partition_consensus.replica_count = TRANSFER_REQUESTER + 1; + let namespace = IggyNamespace::new(0, 0, 0); + owner + .plane + .partitions() + .get_mut_by_ns(&namespace) + .unwrap() + .set_created_revision(SERVED_INCARNATION); + let directory = tempfile::tempdir().unwrap(); + let log_path = directory.path().join("segment.log"); + let mut bytes = [PREFIX, SUFFIX].concat(); + bytes.resize(crate::STATE_CHUNK_LEN as usize + 1, 0); + let entry = StateArtifact::for_bytes(artifact_kind::SEGMENT_LOG, 0, &bytes); + let expected = bytes[offset..offset + CHUNK_LEN as usize].to_vec(); + // Unrequested bytes need not be read to answer this chunk. + bytes[..offset].fill(0); + bytes.truncate(offset + CHUNK_LEN as usize); + std::fs::write(&log_path, &bytes).unwrap(); + let offer = partitions::state_transfer::PartitionStateTransferOffer { + commit_op: 0, + segments: vec![partitions::state_transfer::SegmentArtifactSource { + entry, + log_path: log_path.to_str().unwrap().to_owned(), + }], + offsets: ( + StateArtifact::for_bytes(artifact_kind::CONSUMER_OFFSETS, 0, &[]), + Rc::new(Vec::new()), + ), + }; + owner.state_transfer_offers.borrow_mut().insert( + (namespace.inner(), TRANSFER_REQUESTER), + crate::ServedStateTransfer { + nonce: NONCE, + offer: crate::ServedOffer::Partition(Rc::new(offer)), + idle_ticks: 0, + fully_served: false, + }, + ); + let request = + Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestStateChunkHeader| { + header.command = Command::RequestStateChunk; + header.size = u32::try_from(size_of::()).unwrap(); + header.cluster = 1; + header.group = namespace.inner(); + header.replica = TRANSFER_REQUESTER; + header.partition_incarnation = SERVED_INCARNATION; + header.nonce = NONCE; + header.offset = offset as u64; + header.len = CHUNK_LEN; + header.seal(); + }); + owner + .on_message(MessageBag::RequestStateChunk(request)) + .await; + assert!( + sent.borrow().is_empty(), + "the chunk goes out once its read lands" + ); + owner.settle_served_reads().await; + let frames = sent.borrow(); + assert_eq!(frames.len(), 1, "one reply for the requested chunk"); + assert_eq!( + frames[0].as_slice()[std::mem::offset_of!(GenericHeader, command)], + Command::StateChunk as u8, + "the held prefix and unread tail must not be loaded" + ); + let chunk = + Message::::try_from(Owned::copy_from_slice(frames[0].as_slice())) + .unwrap(); + assert_eq!(chunk.header().offset, offset as u64); + assert_eq!(chunk.body(), expected); + assert!( + !owner.state_transfer_offers.borrow()[&(namespace.inner(), TRANSFER_REQUESTER)] + .fully_served + ); + } + } + + /// The offer can change while a chunk read runs. Bytes read for the + /// artifact of the old offer never answer the request or fill the cache: + /// the serve reads the artifact of the offer of now. + #[compio::test] + async fn given_a_chunk_read_when_the_offer_changes_before_it_lands_should_serve_the_new_artifact() + { + const NONCE: u128 = 1; + const OLD: &[u8] = b"old segment bytes"; + const NEW: &[u8] = b"new segment bytes"; + const CHUNK_LEN: u32 = 8; + let bus = Rc::new(IggyMessageBus::new(0)); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + captured.borrow_mut().push(frame); + Ok(()) + })); + connect_replica_on(&bus, TRANSFER_REQUESTER, 1); + let (mut owner, _sender) = test_owner(&bus, None); + owner.partition_consensus.replica_count = TRANSFER_REQUESTER + 1; + let namespace = IggyNamespace::new(0, 0, 0); + owner + .plane + .partitions() + .get_mut_by_ns(&namespace) + .unwrap() + .set_created_revision(SERVED_INCARNATION); + let directory = tempfile::tempdir().unwrap(); + let offer_of = |name: &str, bytes: &[u8]| { + let log_path = directory.path().join(name); + std::fs::write(&log_path, bytes).unwrap(); + let offer = partitions::state_transfer::PartitionStateTransferOffer { + commit_op: 0, + segments: vec![partitions::state_transfer::SegmentArtifactSource { + entry: StateArtifact::for_bytes(artifact_kind::SEGMENT_LOG, 0, bytes), + log_path: log_path.to_str().unwrap().to_owned(), + }], + offsets: ( + StateArtifact::for_bytes(artifact_kind::CONSUMER_OFFSETS, 0, &[]), + Rc::new(Vec::new()), + ), + }; + crate::ServedStateTransfer { + nonce: NONCE, + offer: crate::ServedOffer::Partition(Rc::new(offer)), + idle_ticks: 0, + fully_served: false, + } + }; + let offer_key = (namespace.inner(), TRANSFER_REQUESTER); + owner + .state_transfer_offers + .borrow_mut() + .insert(offer_key, offer_of("old.log", OLD)); + let request = Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestStateChunkHeader| { + header.command = Command::RequestStateChunk; + header.size = u32::try_from(size_of::()).unwrap(); + header.cluster = 1; + header.group = namespace.inner(); + header.replica = TRANSFER_REQUESTER; + header.partition_incarnation = SERVED_INCARNATION; + header.nonce = NONCE; + header.len = CHUNK_LEN; + header.seal(); + }); + + owner + .on_message(MessageBag::RequestStateChunk(request)) + .await; + owner + .state_transfer_offers + .borrow_mut() + .insert(offer_key, offer_of("new.log", NEW)); + owner.settle_served_reads().await; + + let frames = sent.borrow(); + assert_eq!(frames.len(), 1, "one reply for the requested chunk"); + let chunk = + Message::::try_from(Owned::copy_from_slice(frames[0].as_slice())) + .unwrap(); + assert_eq!(chunk.body(), &NEW[..CHUNK_LEN as usize]); + } + + /// A request frame that fails the gates, here one with a stale partition + /// incarnation, must leave the parked request of its requester in place, + /// so the tick answers that request once its plan pins. + #[compio::test] + async fn given_a_parked_transfer_request_when_a_stale_request_arrives_should_answer_the_parked_one() + { + const PARKED_NONCE: u128 = 1; + const STALE_NONCE: u128 = 2; + const STALE_INCARNATION: u64 = SERVED_INCARNATION - 1; + let bus = Rc::new(IggyMessageBus::new(0)); + let targets = record_transfer_targets(&bus); + let namespace = IggyNamespace::new(0, 0, 0); + let directory = tempfile::tempdir().unwrap(); + let (owner, _sender) = serving_shard(&bus, &[namespace], directory.path()).await; + owner + .on_message(request_state_transfer( + namespace, + PARKED_NONCE, + SERVED_INCARNATION, + )) + .await; + assert_eq!(parked_nonce(&owner, namespace), Some(PARKED_NONCE)); + + owner + .on_message(request_state_transfer( + namespace, + STALE_NONCE, + STALE_INCARNATION, + )) + .await; + + assert_eq!( + parked_nonce(&owner, namespace), + Some(PARKED_NONCE), + "a request that fails the gates must not replace the parked one" + ); + pin_transfer_plans(&owner, &[namespace]).await; + owner.serve_parked_partition_transfer_requests().await; + assert_eq!( + *targets.borrow(), + [(namespace.inner(), PARKED_NONCE, OFFERED)], + "the tick must answer the parked request with its offer" + ); + } + + /// Two parked requests whose plans pin together at their bound must start + /// their offer builds on two ticks. One build reads and hashes up to a + /// whole round budget, and every timer of the shard waits for the tick + /// that runs it. The bound refuses only a request that still waits for its + /// pin, so the request that misses the bound tick builds on the next one. + #[compio::test] + async fn given_two_parked_transfer_requests_when_their_plans_pin_at_the_bound_should_answer_both_on_two_ticks() + { + const NONCE: u128 = 1; + let bus = Rc::new(IggyMessageBus::new(0)); + let targets = record_transfer_targets(&bus); + let namespaces = [IggyNamespace::new(0, 0, 0), IggyNamespace::new(0, 0, 1)]; + let directory = tempfile::tempdir().unwrap(); + let (owner, _sender) = serving_shard(&bus, &namespaces, directory.path()).await; + for namespace in namespaces { + owner + .on_message(request_state_transfer(namespace, NONCE, SERVED_INCARNATION)) + .await; + } + for _ in 1..PARKED_WAIT_TICKS { + owner.serve_parked_partition_transfer_requests().await; + } + assert!(targets.borrow().is_empty()); + pin_transfer_plans(&owner, &namespaces).await; + + owner.serve_parked_partition_transfer_requests().await; + assert_eq!( + *targets.borrow(), + [(namespaces[0].inner(), NONCE, OFFERED)], + "the bound tick must start one offer build" + ); + assert_eq!(parked_nonce(&owner, namespaces[1]), Some(NONCE)); + owner.serve_parked_partition_transfer_requests().await; + + assert_eq!( + *targets.borrow(), + [ + (namespaces[0].inner(), NONCE, OFFERED), + (namespaces[1].inner(), NONCE, OFFERED), + ], + "the request that missed the bound tick must get its offer, not a refusal" + ); + } + + /// A parked request whose plan does not pin must get one transient refusal + /// at its bound, before the requester re-sends and burns a stall retry. + /// The build slot stays claimed, as for any resumable refusal. + #[compio::test] + async fn given_a_parked_transfer_request_when_its_plan_does_not_pin_should_refuse_it_at_the_bound() + { + const NONCE: u128 = 1; + let bus = Rc::new(IggyMessageBus::new(0)); + let targets = record_transfer_targets(&bus); + let namespace = IggyNamespace::new(0, 0, 0); + let directory = tempfile::tempdir().unwrap(); + let (owner, _sender) = serving_shard(&bus, &[namespace], directory.path()).await; + owner + .on_message(request_state_transfer(namespace, NONCE, SERVED_INCARNATION)) + .await; + assert_eq!(parked_nonce(&owner, namespace), Some(NONCE)); + assert!( + owner + .partition_offer_builds + .borrow() + .contains_key(&namespace.inner()) + ); + + for _ in 1..PARKED_WAIT_TICKS { + owner.serve_parked_partition_transfer_requests().await; + } + assert!( + targets.borrow().is_empty(), + "the request must wait its whole bound" + ); + owner.serve_parked_partition_transfer_requests().await; + + assert_eq!(*targets.borrow(), [(namespace.inner(), NONCE, REFUSED)]); + assert_eq!(parked_nonce(&owner, namespace), None); + assert!( + owner + .partition_offer_builds + .borrow() + .contains_key(&namespace.inner()), + "the build slot must stay claimed for the next request" + ); + } + + /// A re-send of a parked request keeps the wait of its nonce, so re-sends + /// cannot hold the build slot past the bound. A request with a new nonce + /// replaces the parked one and waits a whole bound of its own. + #[compio::test] + async fn given_a_parked_transfer_request_when_it_is_re_sent_should_refuse_it_at_the_bound_of_its_nonce() + { + const NONCE: u128 = 1; + const NEW_NONCE: u128 = 2; + const RE_SENT_AT: u32 = PARKED_WAIT_TICKS / 2; + for (re_sent, refused_at) in [ + (NONCE, PARKED_WAIT_TICKS), + (NEW_NONCE, RE_SENT_AT + PARKED_WAIT_TICKS), + ] { + let bus = Rc::new(IggyMessageBus::new(0)); + let targets = record_transfer_targets(&bus); + let namespace = IggyNamespace::new(0, 0, 0); + let directory = tempfile::tempdir().unwrap(); + let (owner, _sender) = serving_shard(&bus, &[namespace], directory.path()).await; + owner + .on_message(request_state_transfer(namespace, NONCE, SERVED_INCARNATION)) + .await; + for _ in 0..RE_SENT_AT { + owner.serve_parked_partition_transfer_requests().await; + } + owner + .on_message(request_state_transfer( + namespace, + re_sent, + SERVED_INCARNATION, + )) + .await; + + for _ in RE_SENT_AT + 1..refused_at { + owner.serve_parked_partition_transfer_requests().await; + } + assert!( + targets.borrow().is_empty(), + "nonce {re_sent} must wait until tick {refused_at}" + ); + owner.serve_parked_partition_transfer_requests().await; + + assert_eq!( + *targets.borrow(), + [(namespace.inner(), re_sent, REFUSED)], + "nonce {re_sent} must get its refusal at tick {refused_at}" + ); + } + } + #[compio::test] async fn given_a_partition_incarnation_when_foreign_repair_or_transfer_requests_arrive_should_not_reply() { @@ -1265,6 +1726,10 @@ mod tests { })], target_accepted: command == Command::StateChunk, idle_ticks: IDLE_TICKS, + spill: None, + scan: None, + seed: None, + pulled_len: 0, }); } partitions.insert(namespace, partition); @@ -2136,6 +2601,8 @@ mod tests { owner.partition_io.notifier(), owner.partition_io.limits.bytes_max(), ); + // A vote two views ahead shows that replica 1 left view 0, so even its + // primary joins the vote and enters the view change. let message = Message::::new(size_of::()) .transmute_header(|_, header: &mut StartViewChangeHeader| { header.command = Command::StartViewChange; @@ -2143,7 +2610,7 @@ mod tests { header.cluster = 1; header.group = namespace.inner(); header.replica = 1; - header.view = 1; + header.view = 2; header.seal(); }); owner.on_start_view_change(message).await; @@ -2383,6 +2850,10 @@ mod tests { })], target_accepted: true, idle_ticks: 0, + spill: None, + scan: None, + seed: None, + pulled_len: 0, }); owner @@ -3543,16 +4014,28 @@ mod tests { } } - /// A spent ring that cannot convert keeps rotating at the pace of the + /// A spent ring that cannot convert never arms a transfer. A primary has no + /// primary to transfer from, so it keeps rotating at the pace of the /// budget, one rotation per spent budget. A backup with a scheduled re-arm - /// waits for it, and a primary has no primary to transfer from. + /// leaves recovery to the re-arm, so its spent session ends without a + /// successor. #[compio::test] - async fn given_ring_that_cannot_convert_when_ring_is_spent_should_rotate_once_per_budget() { + async fn given_ring_that_cannot_convert_when_ring_is_spent_should_never_arm_a_transfer() { const REARM_BACKOFF_TICKS: u32 = 1024; let budget = REPAIR_MAX_STALL_RETRIES as usize; - for (case, replica, peer) in [ - ("a backup waiting to re-arm", REPAIR_BACKUP, REPAIR_PRIMARY), - ("a primary", REPAIR_PRIMARY, REPAIR_BACKUP), + for (case, replica, peer, successor) in [ + ( + "a backup waiting to re-arm", + REPAIR_BACKUP, + REPAIR_PRIMARY, + None, + ), + ( + "a primary", + REPAIR_PRIMARY, + REPAIR_BACKUP, + Some(REPAIR_NEXT_PEER), + ), ] { let bus = Rc::new(IggyMessageBus::new(0)); let (owner, _sender) = test_owner(&bus, None); @@ -3575,53 +4058,506 @@ mod tests { owner.plane.partitions().insert(namespace, partition); let mut expected = vec![(Some(peer), None); budget]; - expected.extend(vec![(Some(REPAIR_NEXT_PEER), None); budget + 1]); - expected.push((Some(peer), None)); + expected.extend(vec![(successor, None); budget + 1]); + expected.push((successor.map(|_| peer), None)); let observed = stall_rounds(&owner, namespace, expected.len()).await; assert_eq!( observed, expected, - "{case} must rotate once per spent budget and never arm a transfer" + "{case} must rotate only without a scheduled re-arm and never arm a transfer" ); } } - /// A spent ring rotates as before, here dropping the session of a group - /// of two, when a transfer cannot take over. A scheduled re-arm keeps its - /// backoff, a busy stage belongs to the transfer in flight, a session can - /// outlive its stage when an election clears the stage, and a transfer - /// carries no uncommitted suffix. + /// A prefix read that has not landed by its bound is asked to stop. The + /// pull waits until the read lands, so the read frees its artifact buffer + /// before the pull fetches the whole artifact into a new one. The pull + /// then asks for the whole artifact while the serving offer still lives + /// and burns no retry of the peer. From then on the stall sweep runs as + /// for any pull. #[compio::test] - async fn given_transfer_that_cannot_take_over_when_ring_is_spent_should_rotate_instead_of_arming() + async fn given_a_prefix_read_that_does_not_land_by_its_bound_when_it_lands_should_pull_the_artifact_whole() { - const REARM_BACKOFF_TICKS: u32 = 1024; - const TRANSFER_NONCE: u128 = 7; - const SUFFIX_OPS: u64 = 2; - type Setup = fn(&mut IggyPartition, HeldSuperblock>); - let cases: [(&str, u64, Setup); 4] = [ - ("an uncommitted suffix", LAGGING_COMMIT_MIN, |partition| { - partition.repair = partition.repair.map(|session| RepairSession { - fetch_to_op: session.commit_to_op + SUFFIX_OPS, - ..session - }); - }), - ("a scheduled re-arm", LAGGING_COMMIT_MAX, |partition| { - partition.transfer_rearm = Some(partitions::state_transfer::PendingTransferRearm { - peer: REPAIR_PRIMARY, - after_ticks: REARM_BACKOFF_TICKS, - }); - }), - ("a transfer stage", LAGGING_COMMIT_MAX, |partition| { - partition.consensus().begin_state_transfer_await(); - }), - ("a transfer session", LAGGING_COMMIT_MAX, |partition| { - partition.transfer = Some(PartitionTransferSession { - nonce: TRANSFER_NONCE, - peer: REPAIR_PRIMARY, - commit_op: 0, - artifacts: Vec::new(), - target_accepted: false, + const INCARNATION: u64 = 7; + const NONCE: u128 = 7; + const RETRY_TICKS: u32 = 2; + const OFFSETS: &[u8] = b"offsets"; + let bus = Rc::new(IggyMessageBus::new(0)); + let requests = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&requests); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + let message = + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(); + if message.header().command == Command::RequestStateChunk { + let request = Message::::try_from(Owned::copy_from_slice( + frame.as_slice(), + )) + .unwrap(); + let header = request.header(); + captured + .borrow_mut() + .push((header.artifact, header.offset, header.len)); + } + Ok(()) + })); + connect_replica_on(&bus, REPAIR_PRIMARY, 1); + let (owner, _sender) = test_owner(&bus, None); + owner.set_repair_retry_ticks(RETRY_TICKS); + let namespace = IggyNamespace::new(0, 0, 0); + let directory = tempfile::tempdir().unwrap(); + let segment = send_request(namespace, 1).body().to_vec(); + let partitions = owner.plane.partitions(); + partitions.remove(&namespace).unwrap(); + let mut partition = + repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); + partition.repair = None; + partition.set_created_revision(INCARNATION); + partition.set_partition_dir(directory.path().to_str().unwrap().to_owned()); + partition + .consensus() + .set_state_transfer_stage(StateTransferStage::AwaitingTarget); + partition + .consensus() + .set_state_transfer_stage(StateTransferStage::Fetching); + partition.transfer = Some(PartitionTransferSession { + nonce: NONCE, + peer: REPAIR_PRIMARY, + commit_op: 0, + artifacts: vec![ + TransferArtifact::Pending(ArtifactProgress { + entry: StateArtifact::for_bytes(artifact_kind::SEGMENT_LOG, 0, &segment), + buf: Vec::new(), + }), + TransferArtifact::Pending(ArtifactProgress { + entry: StateArtifact::for_bytes(artifact_kind::CONSUMER_OFFSETS, 0, OFFSETS), + buf: Vec::new(), + }), + ], + target_accepted: true, + idle_ticks: 0, + spill: None, + scan: None, + seed: Some(PrefixSeed::Reading { + artifact: 0, + read: LocalWork::default(), + }), + pulled_len: 0, + }); + partitions.insert(namespace, partition); + let bound = RETRY_TICKS * crate::STATE_TRANSFER_OFFER_EXPIRY_MULTIPLE + / crate::LOCAL_WORK_WAIT_DIVISOR; + let whole = (0, 0, u32::try_from(segment.len()).unwrap()); + + stall_rounds(&owner, namespace, bound as usize).await; + assert!( + requests.borrow().is_empty(), + "the pull must wait for its read" + ); + let read = || { + let Some(PrefixSeed::Reading { read, .. }) = &partitions + .get_by_ns(&namespace) + .unwrap() + .transfer + .as_ref() + .unwrap() + .seed + else { + panic!("the session must keep the read until it lands"); + }; + read + }; + assert!(!read().stop(), "the bound must ask the read to stop"); + stall_rounds(&owner, namespace, RETRY_TICKS as usize).await; + assert!( + requests.borrow().is_empty(), + "the pull must not fetch the artifact while the stopped read holds its buffer" + ); + *read().result.borrow_mut() = Some(Err(io::Error::new( + io::ErrorKind::Interrupted, + "the transfer gave up this prefix read", + ))); + stall_rounds(&owner, namespace, 1).await; + assert_eq!( + *requests.borrow(), + [whole], + "the pull must ask for the whole artifact once the read lands" + ); + assert_eq!( + partitions + .get_by_ns(&namespace) + .unwrap() + .transfer_attempts(), + 0, + "a slow local read must not burn a retry of the peer" + ); + stall_rounds(&owner, namespace, RETRY_TICKS as usize).await; + assert_eq!( + ( + requests.borrow().clone(), + partitions + .get_by_ns(&namespace) + .unwrap() + .transfer_attempts() + ), + (vec![whole, whole], 1), + "the stall sweep must re-request the whole artifact after the bound" + ); + + let size = size_of::() + segment.len(); + let mut chunk = Message::::new(size); + chunk.as_mut_slice()[size_of::()..].copy_from_slice(&segment); + let chunk = chunk.transmute_header(|_, header: &mut StateChunkHeader| { + header.command = Command::StateChunk; + header.size = u32::try_from(size).unwrap(); + header.cluster = 1; + header.group = namespace.inner(); + header.replica = REPAIR_PRIMARY; + header.partition_incarnation = INCARNATION; + header.nonce = NONCE; + header.seal(); + }); + owner.on_message(MessageBag::StateChunk(chunk)).await; + settle_transfer_spill(&owner, namespace).await; + + let session = partitions + .get_by_ns(&namespace) + .unwrap() + .transfer + .as_ref() + .unwrap(); + assert!( + matches!(session.artifacts[0], TransferArtifact::Staged(_)), + "the whole pull must stage the artifact" + ); + assert_eq!( + requests.borrow().last(), + Some(&(1, 0, u32::try_from(OFFSETS.len()).unwrap())), + "the session must go on to its next artifact" + ); + } + + /// The I/O turn that hands a landed prefix read to the pull leaves the + /// ready queue. The partition takes another turn, so I/O work that the + /// read went ahead of does not wait for the next notification. + #[compio::test] + async fn given_a_landed_prefix_read_when_its_turn_resumes_the_pull_should_keep_the_partition_ready() + { + let bus = Rc::new(IggyMessageBus::new(0)); + bus.set_replica_forward_fn(Box::new(|_, _, _| Ok(()))); + connect_replica_on(&bus, REPAIR_PRIMARY, 1); + let (owner, _sender) = test_owner(&bus, None); + let namespace = IggyNamespace::new(0, 0, 0); + let partitions = owner.plane.partitions(); + partitions.remove(&namespace).unwrap(); + let mut partition = + repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); + partition.repair = None; + partition + .consensus() + .set_state_transfer_stage(StateTransferStage::AwaitingTarget); + partition + .consensus() + .set_state_transfer_stage(StateTransferStage::Fetching); + partition.transfer = Some(PartitionTransferSession { + nonce: 1, + peer: REPAIR_PRIMARY, + commit_op: 0, + artifacts: vec![TransferArtifact::Pending(ArtifactProgress { + entry: StateArtifact::for_bytes(artifact_kind::SEGMENT_LOG, 0, b"segment"), + buf: Vec::new(), + })], + target_accepted: true, + idle_ticks: 0, + spill: None, + scan: None, + seed: Some(PrefixSeed::Reading { + artifact: 0, + read: LocalWork { + result: Rc::new(RefCell::new(Some(Err(io::Error::other("the read failed"))))), + ..LocalWork::default() + }, + }), + pulled_len: 0, + }); + partitions.insert(namespace, partition); + partitions.set_io_notifier( + owner.partition_io.notifier(), + owner.partition_io.limits.bytes_max(), + ); + while let Some((ready, incarnation)) = owner.partition_io.head() { + owner.partition_io.pop_ready(ready, incarnation); + } + let incarnation = partitions.get_io_owner(&namespace).unwrap().incarnation(); + partitions.get_io_owner(&namespace).unwrap().notify_io(); + + owner.service_partition_io().await; + + assert!( + partitions + .get_by_ns(&namespace) + .unwrap() + .transfer + .as_ref() + .unwrap() + .seed + .is_none(), + "the turn must hand the landed read to the pull" + ); + assert_eq!( + owner.partition_io.head(), + Some((namespace, incarnation)), + "the partition must keep a turn for the work that the read went ahead of" + ); + } + + /// A spent ring rotates as before, here dropping the session of a group + /// of two, when a transfer cannot take over. A scheduled re-arm keeps its + /// backoff, a busy stage belongs to the transfer in flight, a session can + /// outlive its stage when an election clears the stage, and a transfer + /// carries no uncommitted suffix. + /// A reuse scan that outlives its bound is asked to stop. The pull still + /// waits until the task lands, because the scan writes the staging names + /// that the pull spills to. It then fetches every segment that the scan did + /// not reuse and finishes. The wait burns no retry of the peer. + #[compio::test] + async fn given_a_reuse_scan_that_does_not_land_by_its_bound_when_it_lands_should_pull_the_rest_and_finish() + { + let receiver = scanning_receiver(); + let bound = SCANNING_RETRY_TICKS * crate::STATE_TRANSFER_OFFER_EXPIRY_MULTIPLE + / crate::LOCAL_WORK_WAIT_DIVISOR; + let partitions = receiver.owner.plane.partitions(); + let session = || { + partitions + .get_by_ns(&receiver.namespace) + .unwrap() + .transfer + .as_ref() + }; + + stall_rounds(&receiver.owner, receiver.namespace, bound as usize).await; + assert!( + receiver.requests.borrow().is_empty(), + "the pull must wait for its scan" + ); + let scan = session().unwrap().scan.as_ref().unwrap(); + assert!(!scan.stop(), "the bound must ask the scan to stop"); + *scan.result.borrow_mut() = Some(ReuseScanned::default()); + stall_rounds(&receiver.owner, receiver.namespace, 1).await; + assert_eq!( + *receiver.requests.borrow(), + [(0, 0, u32::try_from(receiver.segment.len()).unwrap())], + "the pull must fetch the segment that the scan did not reuse" + ); + receiver + .owner + .on_message(MessageBag::StateChunk(receiver.chunk(0, &receiver.segment))) + .await; + settle_transfer_spill(&receiver.owner, receiver.namespace).await; + receiver + .owner + .on_message(MessageBag::StateChunk(receiver.chunk(1, SCANNING_OFFSETS))) + .await; + + assert_eq!( + receiver.requests.borrow().last(), + Some(&(1, 0, u32::try_from(SCANNING_OFFSETS.len()).unwrap())), + ); + assert!( + session().is_none(), + "the session must finish its pull and hand it to the install" + ); + assert_eq!( + partitions + .get_by_ns(&receiver.namespace) + .unwrap() + .transfer_attempts(), + 0, + "a slow scan must not burn a retry of the peer" + ); + } + + #[compio::test] + async fn given_local_transfer_work_when_ticks_pass_should_refresh_the_same_offer() { + let receiver = scanning_receiver(); + stall_rounds( + &receiver.owner, + receiver.namespace, + 3 * SCANNING_RETRY_TICKS as usize, + ) + .await; + + assert_eq!( + *receiver.offer_requests.borrow(), + [( + receiver.namespace.inner(), + SCANNING_INCARNATION, + SCANNING_NONCE + ); 3], + "local work must renew the accepted offer instead of minting a new session" + ); + assert_eq!( + receiver + .owner + .plane + .partitions() + .get_by_ns(&receiver.namespace) + .unwrap() + .transfer_attempts(), + 0, + "waiting for local I/O must not spend the peer's retry budget" + ); + } + + #[compio::test] + async fn given_a_completed_artifact_when_receiving_its_last_chunk_should_queue_staging() { + let receiver = scanning_receiver(); + let partitions = receiver.owner.plane.partitions(); + *partitions + .get_by_ns(&receiver.namespace) + .unwrap() + .transfer + .as_ref() + .unwrap() + .scan + .as_ref() + .unwrap() + .result + .borrow_mut() = Some(ReuseScanned::default()); + receiver + .owner + .on_partition_transfer_progress(receiver.namespace.inner()) + .await; + + receiver + .owner + .on_message(MessageBag::StateChunk(receiver.chunk(0, &receiver.segment))) + .await; + + assert!( + !receiver + .directory + .path() + .join("00000000000000000000.log.staging") + .exists(), + "receiving a chunk must return before staging I/O runs on the shard pump" + ); + settle_transfer_spill(&receiver.owner, receiver.namespace).await; + assert!(matches!( + partitions + .get_by_ns(&receiver.namespace) + .unwrap() + .transfer + .as_ref() + .unwrap() + .artifacts[0], + TransferArtifact::Staged(_) + )); + } + + /// The pull neither requests nor spills while its reuse scan runs, and the + /// wait burns no retry of the peer. Once the scan lands, the pull fetches + /// only what the scan did not reuse. + #[compio::test] + async fn given_a_running_reuse_scan_when_the_pull_progresses_should_request_nothing_until_it_lands() + { + let receiver = scanning_receiver(); + let partitions = receiver.owner.plane.partitions(); + let session = || { + partitions + .get_by_ns(&receiver.namespace) + .unwrap() + .transfer + .as_ref() + .unwrap() + }; + + receiver + .owner + .on_partition_transfer_progress(receiver.namespace.inner()) + .await; + stall_rounds( + &receiver.owner, + receiver.namespace, + 2 * SCANNING_RETRY_TICKS as usize, + ) + .await; + assert!( + receiver.requests.borrow().is_empty(), + "the pull must wait for its scan" + ); + assert_eq!( + partitions + .get_by_ns(&receiver.namespace) + .unwrap() + .transfer_attempts(), + 0, + "a running scan must not burn a retry of the peer" + ); + let adopted = StagedSegmentMeta { + start_offset: 0, + end_offset: 0, + index_size: 0, + size: receiver.segment.len() as u64, + start_timestamp: 0, + end_timestamp: 0, + max_timestamp: 0, + log_staging: PathBuf::new(), + index_staging: PathBuf::new(), + checksum: 0, + linked: false, + }; + let mut scanned = ReuseScanned::default(); + scanned.reused.push((0, adopted)); + *session().scan.as_ref().unwrap().result.borrow_mut() = Some(scanned); + stall_rounds(&receiver.owner, receiver.namespace, 1).await; + + assert_eq!( + *receiver.requests.borrow(), + [(1, 0, u32::try_from(SCANNING_OFFSETS.len()).unwrap())], + "the pull must fetch only what the scan did not reuse" + ); + assert!(matches!( + session().artifacts[0], + TransferArtifact::Staged(_) + )); + } + + #[compio::test] + async fn given_transfer_that_cannot_take_over_when_ring_is_spent_should_rotate_instead_of_arming() + { + const REARM_BACKOFF_TICKS: u32 = 1024; + const TRANSFER_NONCE: u128 = 7; + const SUFFIX_OPS: u64 = 2; + type Setup = fn(&mut IggyPartition, HeldSuperblock>); + let cases: [(&str, u64, Setup); 4] = [ + ("an uncommitted suffix", LAGGING_COMMIT_MIN, |partition| { + partition.repair = partition.repair.map(|session| RepairSession { + fetch_to_op: session.commit_to_op + SUFFIX_OPS, + ..session + }); + }), + ("a scheduled re-arm", LAGGING_COMMIT_MAX, |partition| { + partition.transfer_rearm = Some(partitions::state_transfer::PendingTransferRearm { + peer: REPAIR_PRIMARY, + after_ticks: REARM_BACKOFF_TICKS, + }); + }), + ("a transfer stage", LAGGING_COMMIT_MAX, |partition| { + partition.consensus().begin_state_transfer_await(); + }), + ("a transfer session", LAGGING_COMMIT_MAX, |partition| { + partition.transfer = Some(PartitionTransferSession { + nonce: TRANSFER_NONCE, + peer: REPAIR_PRIMARY, + commit_op: 0, + artifacts: Vec::new(), + target_accepted: false, idle_ticks: 0, + spill: None, + scan: None, + seed: None, + pulled_len: 0, }); }), ]; @@ -3802,20 +4738,218 @@ mod tests { ); } - /// A rotation that the live-session ceiling refuses opens no session, so - /// it does not count toward the ring. The detector re-arms the primary - /// later, and the ring still reaches the other backup before it converts. + /// The order a lagging backup saw under load. Its repair peer had evicted + /// the bottom of the window, so the reply carried `RangeEvicted` and then + /// ops above a hole. The backup armed a transfer, the peer refused it as + /// transient, and the live stream then delivered the hole. The live + /// prepares must fill the hole and carry the head over the run that repair + /// journaled, and the scheduled re-arm must not start a transfer while the + /// commit walk catches up. #[compio::test] - async fn given_rotation_refused_at_session_ceiling_when_ring_resumes_should_ask_next_peer_before_transfer() - { + async fn given_repair_above_a_hole_when_the_live_stream_fills_it_should_need_no_transfer() { + /// Committed and applied before the repair round. + const APPLIED: u64 = 4; + /// The lowest op the repair peer still retains, so the hole + /// `APPLIED + 1..RETAINED_FROM` is 28 ops long. + const RETAINED_FROM: u64 = APPLIED + 29; + /// The end of the one chunk the peer serves from `RETAINED_FROM`. + const SERVED_THROUGH: u64 = RETAINED_FROM + crate::REPAIR_CHUNK_MAX - 1; + /// The group head and commit point: four bounded commit walks above + /// `APPLIED`, so the re-arm comes due while the walk still runs. + const HEAD: u64 = APPLIED + 4 * partitions::COMMIT_WALK_OPS_MAX as u64; + const NONCE: u128 = 1; let bus = Rc::new(IggyMessageBus::new(0)); - let (owner, _sender) = test_owner(&bus, None); + let (mut owner, _sender) = test_owner(&bus, None); + owner.partition_consensus.replica_count = 3; owner.set_repair_retry_ticks(1); - owner.set_repair_gap_debounce_ticks(REPAIR_GAP_DEBOUNCE_TICKS_MIN); let namespace = IggyNamespace::new(0, 0, 0); - owner.plane.partitions().insert( - namespace, - repairing_backup(&bus, namespace, LAGGING_COMMIT_MIN, LAGGING_COMMIT_MAX), + let prepares = primary_prepares(&bus, namespace, HEAD).await; + let applied = usize::try_from(APPLIED).unwrap(); + let mut backup = repairing_backup(&bus, namespace, 0, 0); + backup.repair = None; + for prepare in &prepares[..applied] { + backup.on_replicate(prepare.clone()).await; + } + backup.consensus().advance_commit_max(APPLIED); + backup + .commit_journal(owner.plane.partitions().config()) + .await; + assert_eq!(backup.consensus().commit_min(), APPLIED); + backup.consensus().advance_commit_max(HEAD); + backup.repair = Some(RepairSession { + nonce: NONCE, + view: REPAIR_VIEW, + from_op: APPLIED + 1, + commit_to_op: HEAD, + fetch_to_op: HEAD, + floor: None, + peer: REPAIR_PRIMARY, + idle_ticks: 0, + floor_pulled_from: 0, + requested_from: Some(APPLIED + 1), + }); + owner.plane.partitions().remove(&namespace).unwrap(); + owner.plane.partitions().insert(namespace, backup); + + let evicted = repair_done(namespace, REPAIR_PRIMARY, NONCE, RETAINED_FROM) + .transmute_header(|done, header: &mut RepairRangeReplyHeader| { + *header = done; + header.command = Command::RangeEvicted; + header.seal(); + }); + owner.on_repair_range_reply(&evicted).await; + for op in RETAINED_FROM..=SERVED_THROUGH { + let prepare = prepares[usize::try_from(op - 1).unwrap()].clone(); + owner.on_repair_prepare(as_repair_prepare(prepare)).await; + } + owner + .on_repair_range_reply(&repair_done( + namespace, + REPAIR_PRIMARY, + NONCE, + SERVED_THROUGH, + )) + .await; + let transfer_nonce = { + let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap(); + partition + .transfer + .as_ref() + .expect("the floor the peer evicted past arms a transfer") + .nonce + }; + let refusal = + Message::::new(size_of::()) + .transmute_header(|_, header: &mut StateTransferTargetHeader| { + header.command = Command::StateTransferTarget; + header.size = u32::try_from(size_of::()).unwrap(); + header.cluster = 1; + header.group = namespace.inner(); + header.replica = REPAIR_PRIMARY; + header.nonce = transfer_nonce; + header.unavailable_transient = 1; + header.seal(); + }); + owner + .on_message(MessageBag::StateTransferTarget(refusal)) + .await; + { + let partition = owner.plane.partitions().get_mut_by_ns(&namespace).unwrap(); + assert!(partition.transfer.is_none()); + assert!(partition.transfer_rearm.is_some()); + for prepare in &prepares[applied..] { + partition.on_replicate(prepare.clone()).await; + } + assert_eq!( + partition.held_through(), + HEAD, + "the live stream must fill the hole below the repaired run" + ); + assert_eq!(partition.consensus().sequencer().current_sequence(), HEAD); + } + + let rounds = usize::try_from(HEAD - APPLIED).unwrap() / partitions::COMMIT_WALK_OPS_MAX + 1; + let observed = stall_rounds(&owner, namespace, rounds).await; + + assert!( + observed.iter().all(|(_, transfer)| transfer.is_none()), + "a journal that holds every committed op needs no transfer: {observed:?}" + ); + let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap(); + assert_eq!(partition.consensus().commit_min(), HEAD); + assert!(partition.transfer_rearm.is_none()); + } + + /// A live copy of an op that repair journaled above a hole leaves the head + /// below the hole. A head above a hole can win a view change that this + /// replica cannot walk. + #[compio::test] + async fn given_repaired_op_above_a_hole_when_its_live_copy_arrives_should_keep_the_head_below_the_hole() + { + let bus = Rc::new(IggyMessageBus::new(0)); + let namespace = IggyNamespace::new(0, 0, 0); + let (mut backup, [below, _, above]) = + Box::pin(backup_with_repaired_op_above_a_hole(&bus, namespace)).await; + + backup.on_replicate(above).await; + + assert_eq!( + backup.consensus().sequencer().current_sequence(), + below.header().op, + "the head must stay below the hole" + ); + } + + /// A live prepare that fills a hole below a repaired op carries offsets + /// below the dirty offset, so only its link to the entry below it shows + /// that it belongs to this log. A copy with another parent stays out of the + /// journal, and the chained copy then fills the hole. + #[compio::test] + async fn given_hole_below_a_repaired_op_when_a_live_prepare_does_not_chain_to_it_should_not_journal_it() + { + let bus = Rc::new(IggyMessageBus::new(0)); + let namespace = IggyNamespace::new(0, 0, 0); + let (mut backup, [_, hole, _]) = + Box::pin(backup_with_repaired_op_above_a_hole(&bus, namespace)).await; + let op = hole.header().op; + let unchained = hole + .clone() + .transmute_header(|original, header: &mut PrepareHeader| { + *header = original; + header.parent = !original.parent; + header.checksum = header.identity_checksum(); + }); + + backup.on_replicate(unchained).await; + let unchained_journaled = backup.log.journal().inner.holds_op(op); + backup.on_replicate(hole).await; + + assert_eq!( + (unchained_journaled, backup.log.journal().inner.holds_op(op)), + (false, true), + "(unchained copy journaled, chained copy journaled)" + ); + } + + /// A backup that journaled the first of three ops from the live stream and + /// the third from repair, so the second is a hole below a repaired op. + /// Returned with the three prepares. + #[allow(clippy::future_not_send)] + async fn backup_with_repaired_op_above_a_hole( + bus: &Rc, + namespace: IggyNamespace, + ) -> ( + IggyPartition, HeldSuperblock>, + [Message; 3], + ) { + let mut prepares = primary_prepares(bus, namespace, 3).await; + let above = prepares.pop().unwrap(); + let hole = prepares.pop().unwrap(); + let below = prepares.pop().unwrap(); + let mut backup = repairing_backup(bus, namespace, 0, above.header().op); + backup.on_replicate(below.clone()).await; + backup.apply_repaired_prepare(above.clone()).await; + assert!( + backup.log.journal().inner.holds_op(above.header().op), + "repair journals the op above the hole" + ); + (backup, [below, hole, above]) + } + + /// A rotation that the live-session ceiling refuses opens no session, so + /// it does not count toward the ring. The detector re-arms the primary + /// later, and the ring still reaches the other backup before it converts. + #[compio::test] + async fn given_rotation_refused_at_session_ceiling_when_ring_resumes_should_ask_next_peer_before_transfer() + { + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = test_owner(&bus, None); + owner.set_repair_retry_ticks(1); + owner.set_repair_gap_debounce_ticks(REPAIR_GAP_DEBOUNCE_TICKS_MIN); + let namespace = IggyNamespace::new(0, 0, 0); + owner.plane.partitions().insert( + namespace, + repairing_backup(&bus, namespace, LAGGING_COMMIT_MIN, LAGGING_COMMIT_MAX), ); let budget = REPAIR_MAX_STALL_RETRIES as usize; let mut expected = vec![(Some(REPAIR_PRIMARY), None); budget]; @@ -4050,8 +5184,11 @@ mod tests { } } + /// A transfer abandoned at the session ceiling schedules a re-arm and opens + /// no repair session while it waits, so its slot goes to a waiting group. #[compio::test] - async fn given_transfer_abandoned_at_ceiling_when_repair_rearms_should_count_new_session() { + async fn given_transfer_abandoned_at_ceiling_when_its_re_arm_waits_should_give_the_slot_to_a_waiting_group() + { const OTHER_SESSIONS: usize = PARTITION_REPAIRS_INFLIGHT_MAX - 1; const SWEEPS: usize = 2; for retry_ticks in [1, partitions::REPAIR_RETRY_TICKS] { @@ -4105,21 +5242,29 @@ mod tests { live, "interval {retry_ticks}, sweep {sweep}" ); - assert!( - partitions - .get_by_ns(&transferring) - .unwrap() - .repair - .is_some() + let abandoned = partitions.get_by_ns(&transferring).unwrap(); + assert_eq!( + ( + abandoned.repair.is_some(), + abandoned.transfer_rearm.is_some(), + partitions.get_by_ns(&waiting).unwrap().repair.is_some(), + ), + (false, true, true), + "(abandoned group repairs, its re-arm waits, waiting group repairs) at \ + interval {retry_ticks}, sweep {sweep}" ); - assert!(partitions.get_by_ns(&waiting).unwrap().repair.is_none()); } } } + /// A transfer that supersedes a partly spent repair restarts the repair + /// budget. Once the transfer is abandoned, no repair session opens while + /// its re-arm waits, and the re-arm then arms the transfer again. #[compio::test] - async fn given_partly_spent_repair_when_transfer_supersedes_it_should_restart_fallback_budget() - { + async fn given_superseded_repair_when_its_transfer_is_abandoned_should_open_no_repair_until_the_re_arm() + { + /// Long enough for a repair session to spend a whole budget and rotate + /// while the re-arm waits. const REARM_BACKOFF_TICKS: u32 = 2 * (REPAIR_MAX_STALL_RETRIES + 1); let bus = Rc::new(IggyMessageBus::new(0)); let (owner, _sender) = test_owner(&bus, None); @@ -4143,130 +5288,611 @@ mod tests { owner .abandon_or_rearm_partition_transfer(&mut backup, REPAIR_PRIMARY) .await; - backup.transfer_rearm.as_mut().unwrap().after_ticks = REARM_BACKOFF_TICKS; + assert!(backup.repair.is_none()); + let rearm = backup.transfer_rearm.as_mut().unwrap(); + rearm.after_ticks = REARM_BACKOFF_TICKS; + let rearm_peer = rearm.peer; owner.plane.partitions().insert(namespace, backup); - let budget = REPAIR_MAX_STALL_RETRIES as usize; - let observed = stall_rounds(&owner, namespace, budget + 1).await; - let mut expected = vec![(Some(REPAIR_PRIMARY), None); budget]; - expected.push((Some(REPAIR_NEXT_PEER), None)); + let waiting = REARM_BACKOFF_TICKS as usize; + let observed = stall_rounds(&owner, namespace, waiting + 1).await; + let mut expected = vec![(None, None); waiting]; + expected.push((None, Some(rearm_peer))); assert_eq!( observed, expected, - "fallback repair must get the whole stall budget" + "a repair session must not open while the re-arm waits" ); } - #[allow(clippy::future_not_send)] - async fn capture_first_write( - owner: &IoTestShard, - ) -> (usize, partitions::CapturedPartitionIo) { - let lane = &owner.partition_io; - let partitions = owner.plane.partitions(); - partitions.set_io_notifier(lane.notifier(), lane.limits.bytes_max()); + /// A lagging backup whose peers evicted the ops it misses converts its + /// repair to a state transfer. Each transient refusal schedules a re-arm, + /// and no repair session opens while it waits. The frames that peers + /// serve above their floor and the live prepares above the hole therefore + /// stay out of the journal until the transfer installs. + #[compio::test] + async fn given_peers_serving_above_their_floor_when_the_transfer_is_refused_repeatedly_should_not_grow_the_journal() + { + const FIRST_FLOOR: u64 = 4; + const REFUSALS: u64 = 3; + /// Room for one more served reply per refusal above the first one. + const COMMIT_MAX: u64 = FIRST_FLOOR + (REFUSALS + 1) * FRAMES_ABOVE_FLOOR; + /// A flat re-arm fires on the second tick at a one-tick retry interval, + /// so four ticks leave slack. + const REARM_TICKS_MAX: usize = 4; + let bus = Rc::new(IggyMessageBus::new(0)); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + bus.set_replica_forward_fn(Box::new(move |peer, _, frame| { + captured.borrow_mut().push((peer, frame)); + Ok(()) + })); let namespace = IggyNamespace::new(0, 0, 0); - let (reply, result) = consensus::oneshot_channel(); - partitions - .on_request_with_reply(send_request(namespace, 1), Some(reply)) - .await; - drop(result); - let partition = partitions.get_mut_by_ns(&namespace).unwrap(); - for _ in 0..crate::router::COOPERATIVE_EVENT_BUDGET { - match partition.resume_io(partitions.config()).await { - partitions::PartitionIoStep::Ready(plan) => { - let slot = lane - .try_reserve(namespace, partition.incarnation(), plan.allocation_charge) - .unwrap(); - let captured = partition - .capture_io(plan, partitions.config()) - .unwrap() - .unwrap(); - assert_eq!( - captured.identity.continuation, - partitions::PartitionIoContinuation::Superblock - ); - return (slot, captured); + let prepares = primary_prepares(&bus, namespace, COMMIT_MAX + REFUSALS).await; + let mut floor = FIRST_FLOOR; + let (owner, _sender) = floor_converted_backup( + &bus, + namespace, + floor, + &prepares[..usize::try_from(COMMIT_MAX).unwrap()], + ) + .await; + let partitions = owner.plane.partitions(); + let recovery = || { + let partition = partitions.get_by_ns(&namespace).unwrap(); + let journal = &partition.log.journal().inner; + ( + journal.resident_count(), + journal.last_op(), + partition.transfer.is_some(), + ) + }; + let converted = recovery(); + assert_eq!( + converted, + ( + usize::try_from(FRAMES_ABOVE_FLOOR).unwrap(), + Some(FIRST_FLOOR + FRAMES_ABOVE_FLOOR), + true + ), + "the first conversion journals the frames served above the floor and arms a transfer" + ); + sent.borrow_mut().clear(); + + for refusal in 1..=REFUSALS { + let (peer, nonce) = { + let partition = partitions.get_by_ns(&namespace).unwrap(); + let transfer = partition.transfer.as_ref().unwrap(); + (transfer.peer, transfer.nonce) + }; + owner + .on_message(transient_transfer_refusal(namespace, (peer, nonce))) + .await; + let live = prepares[usize::try_from(COMMIT_MAX + refusal - 1).unwrap()].clone(); + partitions + .get_mut_by_ns(&namespace) + .unwrap() + .on_replicate(live) + .await; + for _ in 0..REARM_TICKS_MAX { + let requests: Vec<_> = sent + .borrow_mut() + .drain(..) + .filter_map(|(peer, frame)| { + let request = Message::::try_from( + Owned::copy_from_slice(frame.as_slice()), + ) + .ok()?; + (request.header().command == Command::RequestPrepares) + .then(|| (peer, request.header().nonce)) + }) + .collect(); + for request in requests { + floor += FRAMES_ABOVE_FLOOR; + serve_above_floor(&owner, namespace, request, floor, &prepares).await; } - partitions::PartitionIoStep::Progress => {} - _ => panic!("first send must reach its reservation write"), + if partitions.get_by_ns(&namespace).unwrap().transfer.is_some() { + break; + } + assert!(owner.tick_partitions(&mut Vec::new()).await.is_none()); } + assert_eq!( + recovery(), + converted, + "(resident entries, last op, transfer re-armed) after refusal {refusal}" + ); } - panic!("reservation did not reach file execution"); } - type IoTestMetadata = MuxStateMachine; - type IoTestShard> = - IggyShard; - + /// A backup adopts a `StartView` at the head it holds while a transfer + /// re-arm waits, so the next live prepare passes the gap check above its + /// hole. The backup journals and acknowledges nothing, because the + /// install wipes the journal. The primary sends to the other backups. #[compio::test] - #[allow(clippy::too_many_lines)] - async fn given_delayed_old_repair_when_a_probe_announces_an_appending_head_should_not_commit_it() + async fn given_start_view_adopted_while_a_transfer_re_arm_waits_when_a_live_prepare_arrives_should_drop_it_without_growing_the_journal() { - const REPLICAS: u8 = 5; - const OLD_PRIMARY: usize = 0; - const PRIMARY: usize = 1; - const BACKUP: usize = 4; - const OLD_CLIENT: u128 = 11; - const CLIENT: u128 = 22; - const ROOT_OP: u64 = 1; - const REGISTER_OP: u64 = ROOT_OP + 1; - const STEP_LIMIT: usize = 10_000; - let root_slot = usize::try_from(ROOT_OP).unwrap(); - let register_slot = usize::try_from(REGISTER_OP).unwrap(); - let directory = tempfile::tempdir().unwrap(); - let frames: Vec = (0..REPLICAS) - .map(|_| Rc::new(RefCell::new(Vec::new()))) - .collect(); - let mut owners = Vec::new(); - for replica in 0..REPLICAS { - let bus = Rc::new(IggyMessageBus::new(0)); - let captured = Rc::clone(&frames[usize::from(replica)]); - bus.set_replica_forward_fn(Box::new(move |target, _, frame| { - let message = - Message::::try_from(Owned::copy_from_slice(frame.as_slice())) - .unwrap(); - captured.borrow_mut().push((target, message)); - Ok(()) - })); - for peer in 0..REPLICAS { - if peer != replica { - crate::test_support::connect_replica_on(&bus, peer, 1); - } - } - let (mut owner, inbox_owner) = test_owner(&bus, None); - let consensus = VsrConsensus::new( - 1, - replica, - REPLICAS, - METADATA_GROUP, - Rc::clone(&bus), - LocalPipeline::new(), - ); - consensus.set_incarnation(u128::from(replica) + 1); - consensus.init(); - consensus.mark_superblock_durable(0, 0); - let journal = - PrepareJournal::open(&directory.path().join(format!("metadata-{replica}.wal")), 0) - .await + const FLOOR: u64 = 4; + /// The backup holds every op above the floor through the group head. + const HEAD: u64 = FLOOR + FRAMES_ABOVE_FLOOR; + let bus = Rc::new(IggyMessageBus::new(0)); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + bus.set_replica_forward_fn(Box::new(move |replica, _, frame| { + let message = + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) .unwrap(); - owner.plane.inner_mut().0 = IggyMetadata::new( - Some(consensus), - Some(journal), - None, - None, - IoTestMetadata::new(( - UsersInner::new().into(), - (StreamsInner::default().into(), ()), - )), - None, - ); - owners.push((owner, inbox_owner)); - } - let (owners, _inbox_owners): (Vec<_>, Vec<_>) = owners.into_iter().unzip(); - let metadata = |replica: usize| &owners[replica].plane.inner().0; - let consensus = |replica| metadata(replica).consensus.as_ref().unwrap(); - let journal = |replica| metadata(replica).journal.as_ref().unwrap(); - let take = |source: usize, target: usize, command: Command| { - let mut pending = frames[source].borrow_mut(); - let index = pending + captured + .borrow_mut() + .push((replica, message.header().command)); + Ok(()) + })); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = primary_prepares(&bus, namespace, HEAD + 1).await; + let head = usize::try_from(HEAD).unwrap(); + let (owner, _sender) = + floor_converted_backup(&bus, namespace, FLOOR, &prepares[..head]).await; + let partitions = owner.plane.partitions(); + let transfer = { + let partition = partitions.get_by_ns(&namespace).unwrap(); + let transfer = partition.transfer.as_ref().unwrap(); + (transfer.peer, transfer.nonce) + }; + owner + .on_message(transient_transfer_refusal(namespace, transfer)) + .await; + owner + .on_start_view(start_view_suffix( + namespace, + REPAIR_VIEW, + HEAD, + HEAD, + &[*prepares[head - 1].header()], + )) + .await; + let journal_shape = || { + let journal = &partitions + .get_by_ns(&namespace) + .unwrap() + .log + .journal() + .inner; + (journal.resident_count(), journal.last_op()) + }; + let adopted = journal_shape(); + { + let partition = partitions.get_by_ns(&namespace).unwrap(); + assert!(partition.transfer_rearm.is_some(), "the re-arm still waits"); + assert_eq!(partition.consensus().sequencer().current_sequence(), HEAD); + } + sent.borrow_mut().clear(); + + partitions + .get_mut_by_ns(&namespace) + .unwrap() + .on_replicate(prepares[head].clone()) + .await; + + assert_eq!(journal_shape(), adopted, "(resident entries, last op)"); + assert!( + sent.borrow().is_empty(), + "refused data is neither forwarded nor acknowledged" + ); + } + + /// The live stream fills the hole of a backup whose transfer re-arm waits + /// out a long backoff. The re-arm stands down at once, so a gap that opens + /// later reaches the gap detector and is repaired, not transferred. + #[compio::test] + async fn given_hole_filled_while_a_long_re_arm_waits_when_a_new_gap_opens_should_repair_it() { + /// The ops that the live stream delivers into the hole. + const HEAD: u64 = 3; + /// Far past the rounds below, so only a stand-down lets repair run. + const REARM_BACKOFF_TICKS: u32 = 1024; + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = test_owner(&bus, None); + owner.set_repair_retry_ticks(1); + owner.set_repair_gap_debounce_ticks(REPAIR_GAP_DEBOUNCE_TICKS_MIN); + let namespace = IggyNamespace::new(0, 0, 0); + let mut backup = repairing_backup(&bus, namespace, 0, HEAD); + backup.repair = None; + backup.transfer_rearm = Some(partitions::state_transfer::PendingTransferRearm { + peer: REPAIR_PRIMARY, + after_ticks: REARM_BACKOFF_TICKS, + }); + for prepare in primary_prepares(&bus, namespace, HEAD).await { + backup.on_replicate(prepare).await; + } + assert_eq!(backup.held_through(), HEAD); + owner.plane.partitions().insert(namespace, backup); + + assert!(owner.tick_partitions(&mut Vec::new()).await.is_none()); + { + let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap(); + assert!( + partition.transfer_rearm.is_none(), + "a journal that holds every op through commit_max must stand the re-arm down" + ); + partition.consensus().advance_commit_max(HEAD + 1); + } + let debounce = REPAIR_GAP_DEBOUNCE_TICKS_MIN as usize; + let observed = stall_rounds(&owner, namespace, debounce).await; + + let mut expected = vec![(None, None); debounce - 1]; + expected.push((Some(REPAIR_PRIMARY), None)); + assert_eq!( + observed, expected, + "the new gap must open a repair session, never a transfer" + ); + } + + /// A backup lagging from op 1 to the last of `prepares` whose first repair + /// met a peer that evicted every op up to `floor`. The backup journaled the + /// prepares served above the floor and armed a transfer. + #[allow(clippy::future_not_send)] + async fn floor_converted_backup( + bus: &Rc, + namespace: IggyNamespace, + floor: u64, + prepares: &[Message], + ) -> (IoTestShard, TaggedSender) { + for peer in [REPAIR_NEXT_PEER, REPAIR_PRIMARY] { + connect_replica_on(bus, peer, 1); + } + let (mut owner, sender) = test_owner(bus, None); + owner.partition_consensus.replica_count = 3; + owner.set_repair_retry_ticks(1); + let commit_max = u64::try_from(prepares.len()).unwrap(); + let partitions = owner.plane.partitions(); + partitions.remove(&namespace).unwrap(); + partitions.insert(namespace, repairing_backup(bus, namespace, 0, commit_max)); + let session = partitions.get_by_ns(&namespace).unwrap().repair.unwrap(); + serve_above_floor( + &owner, + namespace, + (session.peer, session.nonce), + floor, + prepares, + ) + .await; + assert!( + partitions.get_by_ns(&namespace).unwrap().transfer.is_some(), + "the evicted floor arms a transfer" + ); + (owner, sender) + } + + /// The answer of the transfer peer that it cannot serve the session + /// `(peer, nonce)` yet. + fn transient_transfer_refusal( + namespace: IggyNamespace, + (peer, nonce): (u8, u128), + ) -> MessageBag { + MessageBag::StateTransferTarget( + Message::::new(size_of::()) + .transmute_header(|_, header: &mut StateTransferTargetHeader| { + header.command = Command::StateTransferTarget; + header.size = u32::try_from(size_of::()).unwrap(); + header.cluster = 1; + header.group = namespace.inner(); + header.replica = peer; + header.nonce = nonce; + header.unavailable_transient = 1; + header.seal(); + }), + ) + } + + /// Prepares a peer serves above its eviction floor in one repair reply. + const FRAMES_ABOVE_FLOOR: u64 = 4; + + /// Answer the repair request `(peer, nonce)` as a peer that evicted every + /// op up to `floor`: `RangeEvicted`, the next [`FRAMES_ABOVE_FLOOR`] + /// prepares, then `RepairDone`. + #[allow(clippy::future_not_send)] + async fn serve_above_floor( + owner: &IoTestShard, + namespace: IggyNamespace, + (peer, nonce): (u8, u128), + floor: u64, + prepares: &[Message], + ) { + let last = floor + FRAMES_ABOVE_FLOOR; + let evicted = repair_done(namespace, peer, nonce, floor + 1).transmute_header( + |done, evicted: &mut RepairRangeReplyHeader| { + *evicted = done; + evicted.command = Command::RangeEvicted; + evicted.seal(); + }, + ); + owner.on_repair_range_reply(&evicted).await; + let served = usize::try_from(floor).unwrap()..usize::try_from(last).unwrap(); + for prepare in &prepares[served] { + owner + .on_repair_prepare(as_repair_prepare(prepare.clone())) + .await; + } + owner + .on_repair_range_reply(&repair_done(namespace, peer, nonce, last)) + .await; + } + + /// A group whose served transfer finished asks again at once. The slot + /// goes to the group that waited while it was served. + #[compio::test] + async fn given_fully_served_group_when_it_asks_again_should_leave_its_slot_to_the_waiting_group() + { + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = one_transfer_slot_owner(&bus); + let [served, waiting] = + [0, 1].map(|partition_id| IggyNamespace::new(0, 0, partition_id).inner()); + serve_partition_offer(&owner, served, false); + assert!(!owner.admit_partition_transfer(waiting)); + + serve_partition_offer(&owner, served, true); + + assert_eq!( + ( + owner.admit_partition_transfer(served), + owner.admit_partition_transfer(waiting) + ), + (false, true), + "(finished group admitted, waiting group admitted)" + ); + } + + /// Refused groups take a freed slot in the order of their first refusal, + /// whichever of them asks first. + #[compio::test] + async fn given_two_waiting_groups_when_a_slot_frees_should_admit_the_oldest_waiter() { + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = one_transfer_slot_owner(&bus); + let [served, oldest, newest] = + [0, 1, 2].map(|partition_id| IggyNamespace::new(0, 0, partition_id).inner()); + serve_partition_offer(&owner, served, false); + assert!(!owner.admit_partition_transfer(oldest)); + assert!(!owner.admit_partition_transfer(newest)); + + owner.state_transfer_offers.borrow_mut().clear(); + + assert_eq!( + ( + owner.admit_partition_transfer(newest), + owner.admit_partition_transfer(oldest) + ), + (false, true), + "(newest waiter admitted, oldest waiter admitted)" + ); + } + + /// A waiter that stops asking ages out on the clock of an abandoned build, + /// so the free slot it held reaches the waiter behind it. + #[compio::test] + async fn given_waiter_that_stops_asking_when_its_idle_window_passes_should_free_its_turn() { + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = one_transfer_slot_owner(&bus); + owner.set_repair_retry_ticks(1); + let [served, silent, asking] = + [0, 1, 2].map(|partition_id| IggyNamespace::new(0, 0, partition_id).inner()); + serve_partition_offer(&owner, served, false); + assert!(!owner.admit_partition_transfer(silent)); + assert!(!owner.admit_partition_transfer(asking)); + owner.state_transfer_offers.borrow_mut().clear(); + + for sweep in 0..crate::STATE_TRANSFER_OFFER_EXPIRY_MULTIPLE { + assert!( + !owner.admit_partition_transfer(asking), + "the silent waiter keeps its turn through sweep {sweep}" + ); + owner.expire_idle_state_transfer_offers(); + } + + assert!(owner.admit_partition_transfer(asking)); + } + + /// A replica that is not the primary of a group refuses a transfer request + /// for it without queueing the group, so a freed slot goes to the next + /// group that asks. + #[compio::test] + async fn given_backup_of_a_group_when_its_transfer_is_requested_should_refuse_without_queueing_it() + { + const REQUESTER: u8 = 0; + const NONCE: u128 = 1; + let bus = Rc::new(IggyMessageBus::new(0)); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + captured.borrow_mut().push(frame); + Ok(()) + })); + connect_replica_on(&bus, REQUESTER, 1); + let (mut owner, _sender) = one_transfer_slot_owner(&bus); + owner.partition_consensus.replica_count = 3; + let [served, backed_up, next] = + [0, 1, 2].map(|partition_id| IggyNamespace::new(0, 0, partition_id)); + let mut backup = repairing_backup(&bus, backed_up, 0, 0); + backup.repair = None; + owner.plane.partitions().insert(backed_up, backup); + serve_partition_offer(&owner, served.inner(), false); + let request = + Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestStateTransferHeader| { + header.command = Command::RequestStateTransfer; + header.size = u32::try_from(size_of::()).unwrap(); + header.cluster = 1; + header.group = backed_up.inner(); + header.replica = REQUESTER; + header.nonce = NONCE; + header.seal(); + }); + + owner.on_partition_request_state_transfer(&request).await; + owner.state_transfer_offers.borrow_mut().clear(); + + let refusals: Vec<_> = sent + .borrow() + .iter() + .map(|frame| { + let reply = Message::::try_from(Owned::copy_from_slice( + frame.as_slice(), + )) + .unwrap(); + ( + reply.header().available, + reply.header().unavailable_transient, + ) + }) + .collect(); + let queued = owner.partition_transfer_waiters.borrow().len(); + assert_eq!( + ( + refusals, + queued, + owner.admit_partition_transfer(next.inner()) + ), + (vec![(0, 1)], 0, true), + "((available, transient) per reply, queued groups, next group admitted)" + ); + } + + /// A test owner whose served-payload budget holds one transfer slot. + fn one_transfer_slot_owner(bus: &Rc) -> (IoTestShard, TaggedSender) { + let (owner, sender) = test_owner(bus, None); + owner.set_served_segment_cache_bytes_max(crate::PARTITION_ARTIFACT_LEN_DEFAULT); + assert_eq!(owner.partition_transfer_admission_cap(), 1); + (owner, sender) + } + + /// Record an offer of the partition `namespace` served to [`REPAIR_BACKUP`]. + fn serve_partition_offer(owner: &IoTestShard, namespace: u64, fully_served: bool) { + let offer = partitions::state_transfer::PartitionStateTransferOffer { + commit_op: 0, + segments: Vec::new(), + offsets: ( + StateArtifact::for_bytes(artifact_kind::CONSUMER_OFFSETS, 0, &[]), + Rc::new(Vec::new()), + ), + }; + owner.state_transfer_offers.borrow_mut().insert( + (namespace, REPAIR_BACKUP), + crate::ServedStateTransfer { + nonce: 0, + offer: crate::ServedOffer::Partition(Rc::new(offer)), + idle_ticks: 0, + fully_served, + }, + ); + } + + #[allow(clippy::future_not_send)] + async fn capture_first_write( + owner: &IoTestShard, + ) -> (usize, partitions::CapturedPartitionIo) { + let lane = &owner.partition_io; + let partitions = owner.plane.partitions(); + partitions.set_io_notifier(lane.notifier(), lane.limits.bytes_max()); + let namespace = IggyNamespace::new(0, 0, 0); + let (reply, result) = consensus::oneshot_channel(); + partitions + .on_request_with_reply(send_request(namespace, 1), Some(reply)) + .await; + drop(result); + let partition = partitions.get_mut_by_ns(&namespace).unwrap(); + for _ in 0..crate::router::COOPERATIVE_EVENT_BUDGET { + match partition.resume_io(partitions.config()).await { + partitions::PartitionIoStep::Ready(plan) => { + let slot = lane + .try_reserve(namespace, partition.incarnation(), plan.allocation_charge) + .unwrap(); + let captured = partition + .capture_io(plan, partitions.config()) + .unwrap() + .unwrap(); + assert_eq!( + captured.identity.continuation, + partitions::PartitionIoContinuation::Superblock + ); + return (slot, captured); + } + partitions::PartitionIoStep::Progress => {} + _ => panic!("first send must reach its reservation write"), + } + } + panic!("reservation did not reach file execution"); + } + + type IoTestMetadata = MuxStateMachine; + type IoTestShard> = + IggyShard; + + #[compio::test] + #[allow(clippy::too_many_lines)] + async fn given_delayed_old_repair_when_a_probe_announces_an_appending_head_should_not_commit_it() + { + const REPLICAS: u8 = 5; + const OLD_PRIMARY: usize = 0; + const PRIMARY: usize = 1; + const BACKUP: usize = 4; + const OLD_CLIENT: u128 = 11; + const CLIENT: u128 = 22; + const ROOT_OP: u64 = 1; + const REGISTER_OP: u64 = ROOT_OP + 1; + const STEP_LIMIT: usize = 10_000; + let root_slot = usize::try_from(ROOT_OP).unwrap(); + let register_slot = usize::try_from(REGISTER_OP).unwrap(); + let directory = tempfile::tempdir().unwrap(); + let frames: Vec = (0..REPLICAS) + .map(|_| Rc::new(RefCell::new(Vec::new()))) + .collect(); + let mut owners = Vec::new(); + for replica in 0..REPLICAS { + let bus = Rc::new(IggyMessageBus::new(0)); + let captured = Rc::clone(&frames[usize::from(replica)]); + bus.set_replica_forward_fn(Box::new(move |target, _, frame| { + let message = + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(); + captured.borrow_mut().push((target, message)); + Ok(()) + })); + for peer in 0..REPLICAS { + if peer != replica { + crate::test_support::connect_replica_on(&bus, peer, 1); + } + } + let (mut owner, inbox_owner) = test_owner(&bus, None); + let consensus = VsrConsensus::new( + 1, + replica, + REPLICAS, + METADATA_GROUP, + Rc::clone(&bus), + LocalPipeline::new(), + ); + consensus.set_incarnation(u128::from(replica) + 1); + consensus.init(); + consensus.mark_superblock_durable(0, 0); + let journal = + PrepareJournal::open(&directory.path().join(format!("metadata-{replica}.wal")), 0) + .await + .unwrap(); + owner.plane.inner_mut().0 = IggyMetadata::new( + Some(consensus), + Some(journal), + None, + None, + IoTestMetadata::new(( + UsersInner::new().into(), + (StreamsInner::default().into(), ()), + )), + None, + ); + owners.push((owner, inbox_owner)); + } + let (owners, _inbox_owners): (Vec<_>, Vec<_>) = owners.into_iter().unzip(); + let metadata = |replica: usize| &owners[replica].plane.inner().0; + let consensus = |replica| metadata(replica).consensus.as_ref().unwrap(); + let journal = |replica| metadata(replica).journal.as_ref().unwrap(); + let take = |source: usize, target: usize, command: Command| { + let mut pending = frames[source].borrow_mut(); + let index = pending .iter() .position(|(replica, message)| { usize::from(*replica) == target && message.header().command == command @@ -5268,7 +6894,7 @@ mod tests { let expected_from = if partition.consensus().view_log_entry_proven(missing) { missing } else { - missing.saturating_sub(CHUNK - 1).max(APPLIED + 1) + missing.saturating_sub(CHUNK - 1).max(LOCAL_HEAD + 1) }; assert_eq!(request.from_op, expected_from); assert_eq!(request.to_op, missing); @@ -5335,7 +6961,7 @@ mod tests { owner.on_repair_range_reply(&reply).await; } assert_eq!(rounds, (NEW_HEAD - LOCAL_HEAD).div_ceil(CHUNK) + 1); - assert_eq!(bodies, NEW_HEAD - APPLIED); + assert_eq!(bodies, NEW_HEAD - LOCAL_HEAD); assert!(requests.borrow().is_empty()); let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap(); assert!(!partition.consensus().view_log_unproven()); @@ -5462,333 +7088,686 @@ mod tests { } } - /// The candidate starts [`HEADLESS_NEXT_VIEW`] while `backup` holds the proven - /// log of [`HEADLESS_VIEW`] and has run nothing since its proof settled. The - /// backup votes only after the held write of `store` persists the new view, - /// and the candidate wins the view with that vote. - #[allow(clippy::future_not_send)] - async fn elect_beside_proven_backup( + /// Both live replicas vote to leave [`HEADLESS_VIEW`]. The vote of the + /// candidate starts [`HEADLESS_NEXT_VIEW`] at `backup`, which holds the proven + /// log of [`HEADLESS_VIEW`] and has run nothing since its proof settled. The + /// backup sends its view change messages only after the held write of `store` + /// persists the new view, and the candidate wins the view with them. + #[allow(clippy::future_not_send, clippy::too_many_lines)] + async fn elect_beside_proven_backup( + namespace: IggyNamespace, + backup: &HeadlessReplica, + candidate: &HeadlessReplica, + store: &HeldSuperblock, + ) { + let backup_consensus = || backup.partition(namespace).consensus(); + let candidate_consensus = || candidate.partition(namespace).consensus(); + let backup_voted = || { + [Command::StartViewChange, Command::DoViewChange] + .into_iter() + .all(|command| backup.sent_to(HEADLESS_CANDIDATE, command, HEADLESS_NEXT_VIEW)) + }; + let candidate_voted = || { + candidate.sent_to( + HEADLESS_BACKUP, + Command::StartViewChange, + HEADLESS_NEXT_VIEW, + ) + }; + assert!(backup_consensus().is_normal()); + assert!(!backup_consensus().view_log_unproven()); + + let mut scratch = Vec::new(); + for _ in 0..2 * TimeoutManager::NORMAL_HEARTBEAT_TICKS { + if candidate_voted() { + break; + } + assert!( + candidate + .owner + .tick_partitions(&mut scratch) + .await + .is_none() + ); + } + assert!( + candidate_voted(), + "the candidate's heartbeat timeout votes to leave the view" + ); + // Only the consensus of the backup ticks, so no persist sweep runs after its + // proof settled. Its own vote stays unsent, and the vote of the candidate + // completes its quorum. + let backup_heartbeat_voted = (0..TimeoutManager::NORMAL_HEARTBEAT_TICKS) + .flat_map(|_| backup_consensus().tick(consensus::PlaneKind::Partitions)) + .any(|action| { + matches!( + action, + VsrAction::SendStartViewChange { view, .. } if view == HEADLESS_NEXT_VIEW + ) + }); + assert!( + backup_heartbeat_voted, + "the backup's heartbeat timeout votes to leave the view" + ); + let (entered, mut started) = oneshot::channel(); + let (release, held) = oneshot::channel(); + *store.entered.borrow_mut() = Some(entered); + *store.held.borrow_mut() = Some(held); + backup + .owner + .on_message(candidate.take(HEADLESS_BACKUP, Command::StartViewChange)) + .await; + assert_eq!(backup_consensus().view(), HEADLESS_NEXT_VIEW); + assert!(!backup_consensus().is_normal()); + + let persisting = async { + while started.try_recv().unwrap().is_none() { + backup.step(&mut scratch).await; + } + }; + compio::time::timeout(REPLY_DEADLINE, Box::pin(persisting)) + .await + .expect("the backup must persist the view once its WAL certifies the view it proved"); + assert!( + !backup_voted(), + "the votes must wait for the superblock write" + ); + release.send(()).unwrap(); + let votes = async { + while !backup_voted() { + backup.step(&mut scratch).await; + } + }; + compio::time::timeout(REPLY_DEADLINE, Box::pin(votes)) + .await + .expect("the backup must vote once its superblock write completes"); + + candidate + .owner + .on_message(backup.take(HEADLESS_CANDIDATE, Command::StartViewChange)) + .await; + candidate + .owner + .on_message(backup.take(HEADLESS_CANDIDATE, Command::DoViewChange)) + .await; + for _ in 0..TimeoutManager::VIEW_CHANGE_STATUS_TICKS { + if candidate_consensus().is_normal() { + break; + } + assert!( + candidate + .owner + .tick_partitions(&mut scratch) + .await + .is_none() + ); + } + assert!(candidate_consensus().is_normal()); + assert!(candidate_consensus().is_primary()); + assert_eq!(candidate_consensus().view(), HEADLESS_NEXT_VIEW); + backup + .owner + .on_message(candidate.take(HEADLESS_BACKUP, Command::StartView)) + .await; + assert!(backup_consensus().is_normal()); + assert_eq!(backup_consensus().view(), HEADLESS_NEXT_VIEW); + } + + /// A backup proves its adopted log through a repair while its WAL still + /// certifies the view it replicated in. The next view change reaches it before + /// any persist sweep certifies the proven view, and that view's primary is down. + #[compio::test] + async fn given_settled_view_proof_when_start_view_change_precedes_wal_certification_should_elect_primary() + { + const LOCAL_HEAD: u64 = 1; + const HEAD: u64 = 2; + let directory = tempfile::tempdir().unwrap(); + let store = Rc::new(HeldSuperblock { + entered: RefCell::new(None), + held: RefCell::new(None), + }); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = primary_prepares(&Rc::new(IggyMessageBus::new(0)), namespace, HEAD).await; + let local = usize::try_from(LOCAL_HEAD).unwrap(); + let backup = Box::pin(HeadlessReplica::journaled( + HEADLESS_BACKUP, + namespace, + &prepares[..local], + Some((directory.path(), &store)), + )) + .await; + let candidate = Box::pin(HeadlessReplica::journaled( + HEADLESS_CANDIDATE, + namespace, + &prepares, + None, + )) + .await; + let start_view = start_view_suffix( + namespace, + HEADLESS_VIEW, + HEAD, + 0, + &[*prepares.last().unwrap().header()], + ); + for replica in [&backup, &candidate] { + replica.owner.on_start_view(start_view.clone()).await; + } + assert!( + !candidate + .partition(namespace) + .consensus() + .view_log_unproven() + ); + assert!(backup.partition(namespace).consensus().view_log_unproven()); + + backup + .owner + .on_repair_prepare(as_repair_prepare(prepares[local].clone())) + .await; + elect_beside_proven_backup(namespace, &backup, &candidate, &store).await; + } + + /// A backup adopts a view whose log ends below an op it held. Dropping that op + /// proves the adopted log with no repair frame, so no frame moves the chain + /// anchor off the dropped op. The next view change reaches the backup before + /// any prepare of the adopted view. + #[compio::test] + async fn given_adopted_head_below_held_op_when_start_view_change_precedes_any_prepare_should_elect_primary() + { + Box::pin(elect_after_dropping_held_op(false)).await; + } + + /// As above, but the adopted head is the commit point. A segment holds it, so + /// the resident headers no longer do. + #[compio::test] + async fn given_evicted_commit_point_below_held_op_when_start_view_change_precedes_any_prepare_should_elect_primary() + { + Box::pin(elect_after_dropping_held_op(true)).await; + } + + /// If `head_committed`, the adopted head is also the commit point, and the + /// backup holds it only in the repair ring. + #[allow(clippy::future_not_send)] + async fn elect_after_dropping_held_op(head_committed: bool) { + const ADOPTED_HEAD: u64 = 1; + const HELD_HEAD: u64 = ADOPTED_HEAD + 1; + let commit = if head_committed { ADOPTED_HEAD } else { 0 }; + let directory = tempfile::tempdir().unwrap(); + let store = Rc::new(HeldSuperblock { + entered: RefCell::new(None), + held: RefCell::new(None), + }); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = + primary_prepares(&Rc::new(IggyMessageBus::new(0)), namespace, HELD_HEAD).await; + let adopted = usize::try_from(ADOPTED_HEAD).unwrap(); + let backup = Box::pin(HeadlessReplica::journaled( + HEADLESS_BACKUP, + namespace, + &prepares, + Some((directory.path(), &store)), + )) + .await; + assert_eq!( + backup.partition(namespace).log.journal().inner.last_op(), + Some(HELD_HEAD) + ); + let candidate = Box::pin(HeadlessReplica::journaled( + HEADLESS_CANDIDATE, + namespace, + &prepares[..adopted], + None, + )) + .await; + if head_committed { + for replica in [&backup, &candidate] { + replica + .partition(namespace) + .consensus() + .restore_commit_state(commit, commit); + } + let journal = &backup.partition(namespace).log.journal().inner; + journal.evict_prefix(adopted); + assert_eq!(journal.resident_entries().len(), prepares.len() - adopted); + assert!(journal.header_by_op(ADOPTED_HEAD).is_none()); + assert!(journal.repair_header(ADOPTED_HEAD).is_some()); + } + let start_view = start_view_suffix( + namespace, + HEADLESS_VIEW, + ADOPTED_HEAD, + commit, + &[*prepares[adopted - 1].header()], + ); + for replica in [&backup, &candidate] { + replica.owner.on_start_view(start_view.clone()).await; + } + // The adoption leaves the drop of the held op to the IO lane, and the + // backup runs nothing once its proof settles. + let mut scratch = Vec::new(); + let settling = async { + while backup.partition(namespace).consensus().view_log_unproven() { + backup.step(&mut scratch).await; + } + }; + compio::time::timeout(REPLY_DEADLINE, Box::pin(settling)) + .await + .expect("dropping the held op must prove the adopted log"); + let partition = backup.partition(namespace); + assert!(partition.repair.is_none()); + assert_eq!( + partition.log.journal().inner.last_op(), + (!head_committed).then_some(ADOPTED_HEAD) + ); + assert_eq!( + partition.consensus().sequencer().current_sequence(), + ADOPTED_HEAD + ); + elect_beside_proven_backup(namespace, &backup, &candidate, &store).await; + } + + /// The candidate becomes the [`HEADLESS_NEXT_VIEW`] primary-elect beside the + /// backup. It truncates the op that the merged log replaces, which clears the + /// WAL certificate of the view it replicated in, and proves the merge through + /// a repair. Returns the backup and the elect, whose view has not started. + #[allow(clippy::future_not_send, clippy::too_many_lines)] + async fn elect_with_settled_merge( namespace: IggyNamespace, - backup: &HeadlessReplica, - candidate: &HeadlessReplica, - store: &HeldSuperblock, - ) { + directory: &Path, + store: &Rc, + ) -> (HeadlessReplica, HeadlessReplica) { + const HEAD: u64 = 2; + let prepares = primary_prepares(&Rc::new(IggyMessageBus::new(0)), namespace, HEAD).await; + let prefix = usize::try_from(HEAD - 1).unwrap(); + // The primary of both views prepared op 2 in the elect's view, lost it, + // and prepared another op 2 in the adopted view. + let merged = + prepares[prefix] + .clone() + .transmute_header(|previous, header: &mut PrepareHeader| { + *header = previous; + header.view = HEADLESS_VIEW; + header.request += 1; + header.checksum = header.identity_checksum(); + }); + let backup = Box::pin(HeadlessReplica::journaled( + HEADLESS_BACKUP, + namespace, + &prepares[..prefix], + None, + )) + .await; + let elect = Box::pin(HeadlessReplica::journaled( + HEADLESS_CANDIDATE, + namespace, + &prepares, + Some((directory, store)), + )) + .await; + backup + .owner + .on_start_view(start_view_suffix( + namespace, + HEADLESS_VIEW, + HEAD, + 0, + &[*merged.header()], + )) + .await; + backup + .owner + .on_repair_prepare(as_repair_prepare(merged.clone())) + .await; let backup_consensus = || backup.partition(namespace).consensus(); - let candidate_consensus = || candidate.partition(namespace).consensus(); - let backup_voted = || { - [Command::StartViewChange, Command::DoViewChange] - .into_iter() - .all(|command| backup.sent_to(HEADLESS_CANDIDATE, command, HEADLESS_NEXT_VIEW)) - }; + let elect_partition = || elect.partition(namespace); assert!(backup_consensus().is_normal()); assert!(!backup_consensus().view_log_unproven()); + assert_eq!(elect_partition().log.journal().inner.last_op(), Some(HEAD)); - // Only the candidate ticks: its heartbeat timeout starts the next view - // while the backup runs nothing after its proof settled. + // The heartbeat timeout of the backup votes to leave the view. The elect + // never left `REPAIR_VIEW`, so that vote is more than one view ahead of it, + // and the elect joins it. let mut scratch = Vec::new(); for _ in 0..2 * TimeoutManager::NORMAL_HEARTBEAT_TICKS { - if candidate_consensus().view() == HEADLESS_NEXT_VIEW { + if backup.sent_to( + HEADLESS_CANDIDATE, + Command::StartViewChange, + HEADLESS_NEXT_VIEW, + ) { break; } - assert!( - candidate - .owner - .tick_partitions(&mut scratch) - .await - .is_none() - ); + assert!(backup.owner.tick_partitions(&mut scratch).await.is_none()); } - assert_eq!(candidate_consensus().view(), HEADLESS_NEXT_VIEW); - let (entered, mut started) = oneshot::channel(); - let (release, held) = oneshot::channel(); - *store.entered.borrow_mut() = Some(entered); - *store.held.borrow_mut() = Some(held); - backup + elect .owner - .on_message(candidate.take(HEADLESS_BACKUP, Command::StartViewChange)) + .on_message(backup.take(HEADLESS_CANDIDATE, Command::StartViewChange)) .await; - assert_eq!(backup_consensus().view(), HEADLESS_NEXT_VIEW); - assert!(!backup_consensus().is_normal()); - - let persisting = async { - while started.try_recv().unwrap().is_none() { - backup.step(&mut scratch).await; - } - }; - compio::time::timeout(REPLY_DEADLINE, Box::pin(persisting)) - .await - .expect("the backup must persist the view once its WAL certifies the view it proved"); - assert!( - !backup_voted(), - "the votes must wait for the superblock write" - ); - release.send(()).unwrap(); - let votes = async { - while !backup_voted() { - backup.step(&mut scratch).await; + let announced = async { + while !elect.sent_to( + HEADLESS_BACKUP, + Command::StartViewChange, + HEADLESS_NEXT_VIEW, + ) { + elect.step(&mut scratch).await; } }; - compio::time::timeout(REPLY_DEADLINE, Box::pin(votes)) + compio::time::timeout(REPLY_DEADLINE, Box::pin(announced)) .await - .expect("the backup must vote once its superblock write completes"); - - candidate + .expect("the elect must announce the view change"); + backup .owner - .on_message(backup.take(HEADLESS_CANDIDATE, Command::StartViewChange)) + .on_message(elect.take(HEADLESS_BACKUP, Command::StartViewChange)) .await; - candidate + assert_eq!(backup_consensus().view(), HEADLESS_NEXT_VIEW); + elect .owner .on_message(backup.take(HEADLESS_CANDIDATE, Command::DoViewChange)) .await; - for _ in 0..TimeoutManager::VIEW_CHANGE_STATUS_TICKS { - if candidate_consensus().is_normal() { - break; + assert!(elect_partition().consensus().is_primary_elect()); + // The IO lane truncates the replaced op, then the elect fetches the merged one. + let truncating = async { + while elect_partition().repair.is_none() { + elect.step(&mut scratch).await; } - assert!( - candidate - .owner - .tick_partitions(&mut scratch) - .await - .is_none() - ); - } - assert!(candidate_consensus().is_normal()); - assert!(candidate_consensus().is_primary()); - assert_eq!(candidate_consensus().view(), HEADLESS_NEXT_VIEW); - backup + }; + compio::time::timeout(REPLY_DEADLINE, Box::pin(truncating)) + .await + .expect("the elect must truncate the replaced op and fetch the merged one"); + assert_eq!( + elect_partition().log.journal().inner.last_op(), + Some(HEAD - 1) + ); + elect .owner - .on_message(candidate.take(HEADLESS_BACKUP, Command::StartView)) + .on_repair_prepare(as_repair_prepare(merged)) .await; - assert!(backup_consensus().is_normal()); - assert_eq!(backup_consensus().view(), HEADLESS_NEXT_VIEW); + assert!(!elect_partition().consensus().view_log_unproven()); + assert!(elect_partition().consensus().is_primary_elect()); + assert_eq!(elect_partition().consensus().log_view(), REPAIR_VIEW); + (backup, elect) } - /// A backup proves its adopted log through a repair while its WAL still - /// certifies the view it replicated in. The next view change reaches it before - /// any persist sweep certifies the proven view, and that view's primary is down. + /// A primary-elect truncates the op that the merged log replaces and proves the + /// merge through a repair. Its `log_view` names the view it replicated in until + /// the view starts, so its WAL must not certify that view over the merged + /// journal in between. #[compio::test] - async fn given_settled_view_proof_when_start_view_change_precedes_wal_certification_should_elect_primary() + async fn given_a_settled_merge_when_the_elect_persists_before_the_view_starts_should_not_certify_its_old_log_view() { - const LOCAL_HEAD: u64 = 1; - const HEAD: u64 = 2; + // Persist calls between the settled proof and the view start, one tick + // apart. The WAL task publishes a queued certificate within the first few. + const WINDOW_ROUNDS: usize = 20; let directory = tempfile::tempdir().unwrap(); let store = Rc::new(HeldSuperblock { entered: RefCell::new(None), held: RefCell::new(None), }); let namespace = IggyNamespace::new(0, 0, 0); - let prepares = primary_prepares(&Rc::new(IggyMessageBus::new(0)), namespace, HEAD).await; - let local = usize::try_from(LOCAL_HEAD).unwrap(); - let backup = Box::pin(HeadlessReplica::journaled( - HEADLESS_BACKUP, - namespace, - &prepares[..local], - Some((directory.path(), &store)), - )) - .await; - let candidate = Box::pin(HeadlessReplica::journaled( - HEADLESS_CANDIDATE, + let (backup, elect) = Box::pin(elect_with_settled_merge( namespace, - &prepares, - None, + directory.path(), + &store, )) .await; - let start_view = start_view_suffix( - namespace, - HEADLESS_VIEW, - HEAD, - 0, - &[*prepares.last().unwrap().header()], - ); - for replica in [&backup, &candidate] { - replica.owner.on_start_view(start_view.clone()).await; - } - assert!( - !candidate - .partition(namespace) - .consensus() - .view_log_unproven() - ); - assert!(backup.partition(namespace).consensus().view_log_unproven()); + let backup_consensus = || backup.partition(namespace).consensus(); + let elect_partition = || elect.partition(namespace); + let mut scratch = Vec::new(); + // The next tick sweep starts the view. Until then the journal is the + // merged log while `log_view` still names the elect's own view. + for _ in 0..WINDOW_ROUNDS { + assert!( + !elect_partition().persist_superblock_if_needed().await, + "the WAL must not certify view {REPAIR_VIEW} over the merged log" + ); + compio::time::sleep(crate::CONSENSUS_TICK_INTERVAL).await; + } + let started = async { + while !elect.sent_to(HEADLESS_BACKUP, Command::StartView, HEADLESS_NEXT_VIEW) { + elect.step(&mut scratch).await; + } + }; + compio::time::timeout(REPLY_DEADLINE, Box::pin(started)) + .await + .expect("the elect must start the view once its WAL certifies it"); + assert!(elect_partition().consensus().is_primary()); + assert_eq!(elect_partition().consensus().log_view(), HEADLESS_NEXT_VIEW); backup .owner - .on_repair_prepare(as_repair_prepare(prepares[local].clone())) + .on_message(elect.take(HEADLESS_BACKUP, Command::StartView)) .await; - elect_beside_proven_backup(namespace, &backup, &candidate, &store).await; - } - - /// A backup adopts a view whose log ends below an op it held. Dropping that op - /// proves the adopted log with no repair frame, so no frame moves the chain - /// anchor off the dropped op. The next view change reaches the backup before - /// any prepare of the adopted view. - #[compio::test] - async fn given_adopted_head_below_held_op_when_start_view_change_precedes_any_prepare_should_elect_primary() - { - Box::pin(elect_after_dropping_held_op(false)).await; + assert!(backup_consensus().is_normal()); + assert_eq!(backup_consensus().view(), HEADLESS_NEXT_VIEW); } - /// As above, but the adopted head is the commit point. A segment holds it, so - /// the resident headers no longer do. + /// As above, but the backup's view change supersedes the elect before its view + /// starts. The third replica is down, so the new view needs the elect's votes, + /// and those wait for a superblock write. #[compio::test] - async fn given_evicted_commit_point_below_held_op_when_start_view_change_precedes_any_prepare_should_elect_primary() + async fn given_a_settled_merge_when_a_view_change_supersedes_the_elect_should_vote_and_elect_a_primary() { - Box::pin(elect_after_dropping_held_op(true)).await; - } - - /// If `head_committed`, the adopted head is also the commit point, and the - /// backup holds it only in the repair ring. - #[allow(clippy::future_not_send)] - async fn elect_after_dropping_held_op(head_committed: bool) { - const ADOPTED_HEAD: u64 = 1; - const HELD_HEAD: u64 = ADOPTED_HEAD + 1; - let commit = if head_committed { ADOPTED_HEAD } else { 0 }; + const SUPERSEDING_VIEW: u32 = HEADLESS_NEXT_VIEW + 1; + const SUPERSEDING_PRIMARY: u8 = HEADLESS_BACKUP; let directory = tempfile::tempdir().unwrap(); let store = Rc::new(HeldSuperblock { entered: RefCell::new(None), held: RefCell::new(None), }); let namespace = IggyNamespace::new(0, 0, 0); - let prepares = - primary_prepares(&Rc::new(IggyMessageBus::new(0)), namespace, HELD_HEAD).await; - let adopted = usize::try_from(ADOPTED_HEAD).unwrap(); - let backup = Box::pin(HeadlessReplica::journaled( - HEADLESS_BACKUP, - namespace, - &prepares, - Some((directory.path(), &store)), - )) - .await; assert_eq!( - backup.partition(namespace).log.journal().inner.last_op(), - Some(HELD_HEAD) + u32::from(SUPERSEDING_PRIMARY), + SUPERSEDING_VIEW % u32::from(HEADLESS_REPLICAS) ); - let candidate = Box::pin(HeadlessReplica::journaled( - HEADLESS_CANDIDATE, + let (backup, elect) = Box::pin(elect_with_settled_merge( namespace, - &prepares[..adopted], - None, + directory.path(), + &store, )) .await; - if head_committed { - for replica in [&backup, &candidate] { - replica - .partition(namespace) - .consensus() - .restore_commit_state(commit, commit); + let backup_consensus = || backup.partition(namespace).consensus(); + let elect_consensus = || elect.partition(namespace).consensus(); + // What the first refresh after the settled proof snapshots. + crate::refresh_partition_dvc_suffix(elect.partition(namespace)); + let settled = elect_consensus().local_dvc_suffix(); + + // Only the backup ticks: its status timeout votes to supersede the elect, + // which runs nothing after its proof settled. The elect joins that vote + // from its view change. + let mut scratch = Vec::new(); + for _ in 0..2 * TimeoutManager::VIEW_CHANGE_STATUS_TICKS { + if backup.sent_to( + HEADLESS_CANDIDATE, + Command::StartViewChange, + SUPERSEDING_VIEW, + ) { + break; } - let journal = &backup.partition(namespace).log.journal().inner; - assert_eq!( - journal.evict_prefix(adopted).len(), - prepares.len() - adopted - ); - assert!(journal.header_by_op(ADOPTED_HEAD).is_none()); - assert!(journal.repair_header(ADOPTED_HEAD).is_some()); + assert!(backup.owner.tick_partitions(&mut scratch).await.is_none()); } - let start_view = start_view_suffix( - namespace, - HEADLESS_VIEW, - ADOPTED_HEAD, - commit, - &[*prepares[adopted - 1].header()], - ); - for replica in [&backup, &candidate] { - replica.owner.on_start_view(start_view.clone()).await; + // `take` returns the first match, so drop the frames of earlier views, the + // backup's retransmits among them. + for replica in [&backup, &elect] { + replica + .sent + .borrow_mut() + .retain(|(_, frame)| frame.header().view == SUPERSEDING_VIEW); } - // The adoption leaves the drop of the held op to the IO lane, and the - // backup runs nothing once its proof settles. - let mut scratch = Vec::new(); - let settling = async { - while backup.partition(namespace).consensus().view_log_unproven() { - backup.step(&mut scratch).await; + assert!(elect_consensus().view_log_merged()); + elect + .owner + .on_message(backup.take(HEADLESS_CANDIDATE, Command::StartViewChange)) + .await; + assert_eq!(elect_consensus().view(), SUPERSEDING_VIEW); + + let voted = async { + while ![Command::StartViewChange, Command::DoViewChange] + .into_iter() + .all(|command| elect.sent_to(SUPERSEDING_PRIMARY, command, SUPERSEDING_VIEW)) + { + elect.step(&mut scratch).await; } }; - compio::time::timeout(REPLY_DEADLINE, Box::pin(settling)) + compio::time::timeout(REPLY_DEADLINE, Box::pin(voted)) .await - .expect("dropping the held op must prove the adopted log"); - let partition = backup.partition(namespace); - assert!(partition.repair.is_none()); + .expect("the superseded elect must persist the view and vote in it"); + let MessageBag::DoViewChange(vote) = elect.take(SUPERSEDING_PRIMARY, Command::DoViewChange) + else { + unreachable!("take returns the command it was asked for"); + }; + let header = *vote.header(); assert_eq!( - partition.log.journal().inner.last_op(), - (!head_committed).then_some(ADOPTED_HEAD) + header.log_view, REPAIR_VIEW, + "the merge never started, so the elect claims the view it replicated in" ); assert_eq!( - partition.consensus().sequencer().current_sequence(), - ADOPTED_HEAD + (header.commit, header.nack_bitset, header.present_bitset), + (0, 0, 0), + "nothing is committed, and an unproven journal neither offers nor nacks an uncommitted op" ); - elect_beside_proven_backup(namespace, &backup, &candidate, &store).await; + let suffix = consensus::dvc_suffix_decode( + vote.body(), + header.op, + header.nack_bitset, + header.present_bitset, + ) + .unwrap(); + assert!( + !suffix.is_empty(), + "the vote must carry the snapshot taken before the view change" + ); + assert_eq!( + (suffix.headers(), header.nack_bitset, header.present_bitset), + ( + settled.headers(), + settled.nack_bitset(), + settled.present_bitset() + ), + "the superseding handler must vote with the snapshot of the settled proof" + ); + + backup + .owner + .on_message(elect.take(SUPERSEDING_PRIMARY, Command::StartViewChange)) + .await; + assert_eq!(backup_consensus().view(), SUPERSEDING_VIEW); + backup + .owner + .on_message(MessageBag::DoViewChange(vote)) + .await; + for _ in 0..TimeoutManager::VIEW_CHANGE_STATUS_TICKS { + if backup_consensus().is_normal() { + break; + } + assert!(backup.owner.tick_partitions(&mut scratch).await.is_none()); + } + assert!(backup_consensus().is_normal()); + assert!(backup_consensus().is_primary()); + assert_eq!(backup_consensus().view(), SUPERSEDING_VIEW); + elect + .owner + .on_message(backup.take(HEADLESS_CANDIDATE, Command::StartView)) + .await; + assert!(elect_consensus().is_normal()); + assert_eq!(elect_consensus().view(), SUPERSEDING_VIEW); } - /// The candidate becomes the [`HEADLESS_NEXT_VIEW`] primary-elect beside the - /// backup. It truncates the op that the merged log replaces, which clears the - /// WAL certificate of the view it replicated in, and proves the merge through - /// a repair. Returns the backup and the elect, whose view has not started. - #[allow(clippy::future_not_send)] - async fn elect_with_settled_merge( - namespace: IggyNamespace, - directory: &Path, - store: &Rc, - ) -> (HeadlessReplica, HeadlessReplica) { - const HEAD: u64 = 2; + /// As above, but the elect adopted the view of the third replica, which goes + /// silent after it prepares one more op. The third replica still votes for the + /// next view with that op, then goes down. The merge truncates nothing, so the + /// elect's WAL still certifies its `log_view`, and the backup's `log_view` is + /// below it. Only the elect's journal then holds every op, so it must stay + /// proven and offer them. + #[compio::test] + async fn given_a_settled_merge_that_kept_the_wal_certificate_when_a_lagging_backup_supersedes_the_elect_should_elect_the_backup_from_the_merged_log() + { + const HEAD: u64 = 3; + const SUPERSEDING_VIEW: u32 = HEADLESS_NEXT_VIEW + 1; + const SUPERSEDING_PRIMARY: u8 = HEADLESS_BACKUP; + let directory = tempfile::tempdir().unwrap(); + let store = Rc::new(HeldSuperblock { + entered: RefCell::new(None), + held: RefCell::new(None), + }); + let namespace = IggyNamespace::new(0, 0, 0); let prepares = primary_prepares(&Rc::new(IggyMessageBus::new(0)), namespace, HEAD).await; - let prefix = usize::try_from(HEAD - 1).unwrap(); - // The primary of both views prepared op 2 in the elect's view, lost it, - // and prepared another op 2 in the adopted view. - let merged = - prepares[prefix] + let adopted = usize::try_from(HEAD - 1).unwrap(); + // The primary prepared the last op after it started the adopted view. + let lagged = + prepares[adopted] .clone() .transmute_header(|previous, header: &mut PrepareHeader| { *header = previous; header.view = HEADLESS_VIEW; - header.request += 1; header.checksum = header.identity_checksum(); }); let backup = Box::pin(HeadlessReplica::journaled( HEADLESS_BACKUP, namespace, - &prepares[..prefix], + &prepares[..1], None, )) .await; let elect = Box::pin(HeadlessReplica::journaled( HEADLESS_CANDIDATE, namespace, - &prepares, - Some((directory, store)), + &prepares[..adopted], + Some((directory.path(), &store)), )) .await; - backup + let backup_consensus = || backup.partition(namespace).consensus(); + let elect_partition = || elect.partition(namespace); + let elect_consensus = || elect.partition(namespace).consensus(); + elect .owner .on_start_view(start_view_suffix( namespace, HEADLESS_VIEW, - HEAD, + HEAD - 1, 0, - &[*merged.header()], + &[*prepares[adopted - 1].header()], )) .await; + assert!(!elect_consensus().view_log_unproven()); + assert_eq!(elect_consensus().log_view(), HEADLESS_VIEW); + + let mut scratch = Vec::new(); + let certified = async { + while !elect_partition().wal_keeps_log_view() { + elect.step(&mut scratch).await; + } + }; + compio::time::timeout(REPLY_DEADLINE, Box::pin(certified)) + .await + .expect("the elect's WAL must certify the view it adopted"); + // The third replica goes silent, so the heartbeat timeout of the elect votes + // to leave the view. + for _ in 0..TimeoutManager::NORMAL_HEARTBEAT_TICKS { + assert!(elect.owner.tick_partitions(&mut scratch).await.is_none()); + } + let voted = async { + while !elect.sent_to( + HEADLESS_BACKUP, + Command::StartViewChange, + HEADLESS_NEXT_VIEW, + ) { + elect.step(&mut scratch).await; + } + }; + compio::time::timeout(REPLY_DEADLINE, Box::pin(voted)) + .await + .expect("the elect must vote to leave the view"); + // The backup never left `REPAIR_VIEW`, so that vote is more than one view + // ahead of it, and the backup joins it. backup .owner - .on_repair_prepare(as_repair_prepare(merged.clone())) + .on_message(elect.take(HEADLESS_BACKUP, Command::StartViewChange)) .await; - let backup_consensus = || backup.partition(namespace).consensus(); - let elect_partition = || elect.partition(namespace); - assert!(backup_consensus().is_normal()); - assert!(!backup_consensus().view_log_unproven()); - assert_eq!(elect_partition().log.journal().inner.last_op(), Some(HEAD)); - - let mut scratch = Vec::new(); - for _ in 0..2 * TimeoutManager::NORMAL_HEARTBEAT_TICKS { - if backup_consensus().view() == HEADLESS_NEXT_VIEW { - break; - } - assert!(backup.owner.tick_partitions(&mut scratch).await.is_none()); - } - assert_eq!(backup_consensus().view(), HEADLESS_NEXT_VIEW); elect .owner .on_message(backup.take(HEADLESS_CANDIDATE, Command::StartViewChange)) .await; + assert_eq!(elect_consensus().view(), HEADLESS_NEXT_VIEW); let announced = async { while !elect.sent_to( HEADLESS_BACKUP, @@ -5801,510 +7780,713 @@ mod tests { compio::time::timeout(REPLY_DEADLINE, Box::pin(announced)) .await .expect("the elect must announce the view change"); - backup - .owner - .on_message(elect.take(HEADLESS_BACKUP, Command::StartViewChange)) - .await; + // The backup's own DoViewChange never arrives, so the quorum is the elect's + // and the third replica's, which offers a body for every op. + let third_replica = DvcSuffix::new( + [&lagged, &prepares[1], &prepares[0]] + .map(|prepare| *prepare.header()) + .to_vec(), + 0, + (1 << HEAD) - 1, + ); elect .owner - .on_message(backup.take(HEADLESS_CANDIDATE, Command::DoViewChange)) + .on_message(MessageBag::DoViewChange(do_view_change_suffix( + namespace, + HEADLESS_PRIMARY, + HEADLESS_NEXT_VIEW, + HEADLESS_VIEW, + HEAD, + &third_replica, + ))) .await; - assert!(elect_partition().consensus().is_primary_elect()); - // The IO lane truncates the replaced op, then the elect fetches the merged one. - let truncating = async { + assert!(elect_consensus().is_primary_elect()); + let fetching = async { while elect_partition().repair.is_none() { elect.step(&mut scratch).await; } }; - compio::time::timeout(REPLY_DEADLINE, Box::pin(truncating)) + compio::time::timeout(REPLY_DEADLINE, Box::pin(fetching)) .await - .expect("the elect must truncate the replaced op and fetch the merged one"); + .expect("the elect must fetch the op it lacks"); assert_eq!( elect_partition().log.journal().inner.last_op(), - Some(HEAD - 1) + Some(HEAD - 1), + "the elect holds a prefix of the merged log, so nothing truncates" ); elect .owner - .on_repair_prepare(as_repair_prepare(merged)) + .on_repair_prepare(as_repair_prepare(lagged.clone())) .await; - assert!(!elect_partition().consensus().view_log_unproven()); - assert!(elect_partition().consensus().is_primary_elect()); - assert_eq!(elect_partition().consensus().log_view(), REPAIR_VIEW); - (backup, elect) + assert!(!elect_consensus().view_log_unproven()); + assert!(elect_consensus().is_primary_elect()); + assert!(elect_partition().wal_keeps_log_view()); + + // Only the backup ticks: its status timeout votes to supersede the elect, + // which runs nothing after its proof settled. The elect joins that vote + // from its view change. + for _ in 0..2 * TimeoutManager::VIEW_CHANGE_STATUS_TICKS { + if backup.sent_to( + HEADLESS_CANDIDATE, + Command::StartViewChange, + SUPERSEDING_VIEW, + ) { + break; + } + assert!(backup.owner.tick_partitions(&mut scratch).await.is_none()); + } + for replica in [&backup, &elect] { + replica + .sent + .borrow_mut() + .retain(|(_, frame)| frame.header().view == SUPERSEDING_VIEW); + } + elect + .owner + .on_message(backup.take(HEADLESS_CANDIDATE, Command::StartViewChange)) + .await; + let voted = async { + while ![Command::StartViewChange, Command::DoViewChange] + .into_iter() + .all(|command| elect.sent_to(SUPERSEDING_PRIMARY, command, SUPERSEDING_VIEW)) + { + elect.step(&mut scratch).await; + } + }; + compio::time::timeout(REPLY_DEADLINE, Box::pin(voted)) + .await + .expect("the superseded elect must vote"); + let MessageBag::DoViewChange(vote) = elect.take(SUPERSEDING_PRIMARY, Command::DoViewChange) + else { + unreachable!("take returns the command it was asked for"); + }; + assert!( + backup_consensus().log_view() < vote.header().log_view, + "no copy the backup holds can stand in for the elect's" + ); + backup + .owner + .on_message(elect.take(SUPERSEDING_PRIMARY, Command::StartViewChange)) + .await; + assert_eq!(backup_consensus().view(), SUPERSEDING_VIEW); + backup + .owner + .on_message(MessageBag::DoViewChange(vote)) + .await; + + let merging = async { + while backup.partition(namespace).repair.is_none() { + backup.step(&mut scratch).await; + } + }; + compio::time::timeout(REPLY_DEADLINE, Box::pin(merging)) + .await + .expect("the backup must merge the elect's log and fetch the ops it lacks"); + for prepare in [&prepares[1], &lagged] { + backup + .owner + .on_repair_prepare(as_repair_prepare(prepare.clone())) + .await; + } + for _ in 0..TimeoutManager::VIEW_CHANGE_STATUS_TICKS { + if backup_consensus().is_normal() { + break; + } + assert!(backup.owner.tick_partitions(&mut scratch).await.is_none()); + } + assert!(backup_consensus().is_normal()); + assert!(backup_consensus().is_primary()); + assert_eq!(backup_consensus().view(), SUPERSEDING_VIEW); + elect + .owner + .on_message(backup.take(HEADLESS_CANDIDATE, Command::StartView)) + .await; + assert!(elect_consensus().is_normal()); + assert_eq!(elect_consensus().view(), SUPERSEDING_VIEW); } - /// A primary-elect truncates the op that the merged log replaces and proves the - /// merge through a repair. Its `log_view` names the view it replicated in until - /// the view starts, so its WAL must not certify that view over the merged - /// journal in between. + /// The walk refuses an unproven view log, so a resident op after + /// `commit_min` cannot move a group whose proof waits on a missing body. + /// When the session ceiling refuses the arm at `StartView` adoption, the + /// gap detector is the only site left to fetch that body. #[compio::test] - async fn given_a_settled_merge_when_the_elect_persists_before_the_view_starts_should_not_certify_its_old_log_view() + async fn given_a_proof_waiting_above_a_resident_next_op_when_the_edge_arm_is_refused_should_arm_from_the_tick() { - // Persist calls between the settled proof and the view start, one tick - // apart. The WAL task publishes a queued certificate within the first few. - const WINDOW_ROUNDS: usize = 20; - let directory = tempfile::tempdir().unwrap(); - let store = Rc::new(HeldSuperblock { - entered: RefCell::new(None), - held: RefCell::new(None), - }); + const APPLIED: u64 = 2; + const LOCAL_HEAD: u64 = 4; + const NEW_HEAD: u64 = 8; + const NEW_VIEW: u32 = 3; + const NEW_PRIMARY: u8 = 0; + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = test_owner(&bus, None); + owner.set_repair_gap_debounce_ticks(REPAIR_GAP_DEBOUNCE_TICKS_MIN); let namespace = IggyNamespace::new(0, 0, 0); - let (backup, elect) = Box::pin(elect_with_settled_merge( - namespace, - directory.path(), - &store, - )) - .await; - let backup_consensus = || backup.partition(namespace).consensus(); - let elect_partition = || elect.partition(namespace); - let mut scratch = Vec::new(); + let prepares = primary_prepares(&bus, namespace, NEW_HEAD).await; + let mut partition = + repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); + partition.repair = None; + for prepare in prepares.iter().take(usize::try_from(LOCAL_HEAD).unwrap()) { + partition.on_replicate(prepare.clone()).await; + } + partition.consensus().advance_commit_max(APPLIED); + partition + .commit_journal(owner.plane.partitions().config()) + .await; + owner.plane.partitions().remove(&namespace).unwrap(); + owner.plane.partitions().insert(namespace, partition); + // Other groups hold every session slot when the view change lands. + owner + .partition_repairs_inflight + .set(PARTITION_REPAIRS_INFLIGHT_MAX); - // The next tick sweep starts the view. Until then the journal is the - // merged log while `log_view` still names the elect's own view. - for _ in 0..WINDOW_ROUNDS { + owner + .on_start_view(start_view_suffix( + namespace, + NEW_VIEW, + NEW_HEAD, + NEW_HEAD, + &[*prepares.last().unwrap().header()], + )) + .await; + let proof_op = { + let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap(); assert!( - !elect_partition().persist_superblock_if_needed().await, - "the WAL must not certify view {REPAIR_VIEW} over the merged log" + partition.repair.is_none(), + "the ceiling must refuse the edge arm" ); - compio::time::sleep(crate::CONSENSUS_TICK_INTERVAL).await; - } - let started = async { - while !elect.sent_to(HEADLESS_BACKUP, Command::StartView, HEADLESS_NEXT_VIEW) { - elect.step(&mut scratch).await; - } + assert_eq!(partition.consensus().commit_min(), APPLIED); + assert!(partition.log.journal().inner.holds_op(APPLIED + 1)); + let proof_op = partition.consensus().view_log_repair_op().unwrap(); + assert!(!partition.log.journal().inner.holds_op(proof_op)); + proof_op }; - compio::time::timeout(REPLY_DEADLINE, Box::pin(started)) - .await - .expect("the elect must start the view once its WAL certifies it"); - assert!(elect_partition().consensus().is_primary()); - assert_eq!(elect_partition().consensus().log_view(), HEADLESS_NEXT_VIEW); - backup - .owner - .on_message(elect.take(HEADLESS_BACKUP, Command::StartView)) + + let mut expected = vec![(None, None); REPAIR_GAP_DEBOUNCE_TICKS_MIN as usize - 1]; + expected.push((Some(NEW_PRIMARY), None)); + let observed = stall_rounds(&owner, namespace, expected.len()).await; + + assert_eq!( + observed, expected, + "the detector must arm once its debounce runs out" + ); + let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap(); + assert_eq!( + partition.repair.map(|session| session.from_op), + Some(proof_op) + ); + } + + /// Host at `namespace` a backup of [`REPAIR_VIEW`] that applied `prepares`. + #[allow(clippy::future_not_send)] + async fn host_applied_backup( + owner: &IoTestShard, + bus: &Rc, + namespace: IggyNamespace, + prepares: &[Message], + ) { + let mut partition = + repairing_replica(bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); + partition.repair = None; + for prepare in prepares { + partition.on_replicate(prepare.clone()).await; + } + let applied = u64::try_from(prepares.len()).unwrap(); + partition.consensus().advance_commit_max(applied); + partition + .commit_journal(owner.plane.partitions().config()) .await; - assert!(backup_consensus().is_normal()); - assert_eq!(backup_consensus().view(), HEADLESS_NEXT_VIEW); + assert_eq!(partition.consensus().commit_min(), applied); + owner.plane.partitions().remove(&namespace).unwrap(); + owner.plane.partitions().insert(namespace, partition); } - /// As above, but the backup's view change supersedes the elect before its view - /// starts. The third replica is down, so the new view needs the elect's votes, - /// and those wait for a superblock write. #[compio::test] - async fn given_a_settled_merge_when_a_view_change_supersedes_the_elect_should_vote_and_elect_a_primary() - { - const SUPERSEDING_VIEW: u32 = HEADLESS_NEXT_VIEW + 1; - const SUPERSEDING_PRIMARY: u8 = HEADLESS_BACKUP; - let directory = tempfile::tempdir().unwrap(); - let store = Rc::new(HeldSuperblock { - entered: RefCell::new(None), - held: RefCell::new(None), - }); + async fn given_a_start_view_that_contradicts_an_applied_op_when_adopting_should_refuse_it() { + const APPLIED: u64 = 2; + const NEW_VIEW: u32 = 3; + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = test_owner(&bus, None); let namespace = IggyNamespace::new(0, 0, 0); + let prepares = primary_prepares(&bus, namespace, APPLIED).await; + host_applied_backup(&owner, &bus, namespace, &prepares).await; + let mut forged = *prepares.last().unwrap().header(); + forged.timestamp += 1; + forged.checksum = forged.identity_checksum(); + + owner + .on_start_view(start_view_suffix( + namespace, + NEW_VIEW, + APPLIED, + APPLIED, + &[forged], + )) + .await; + + // The refusal also fences the partition for resync, behind the tombstone. + let partition = owner.plane.partitions().get_io_owner(&namespace).unwrap(); assert_eq!( - u32::from(SUPERSEDING_PRIMARY), - SUPERSEDING_VIEW % u32::from(HEADLESS_REPLICAS) + partition.consensus().view(), + REPAIR_VIEW, + "a log naming another prepare at applied op {APPLIED} must be refused, not adopted" ); - let (backup, elect) = Box::pin(elect_with_settled_merge( - namespace, - directory.path(), - &store, - )) - .await; - let backup_consensus = || backup.partition(namespace).consensus(); - let elect_consensus = || elect.partition(namespace).consensus(); - // What the first refresh after the settled proof snapshots. - crate::refresh_partition_dvc_suffix(elect.partition(namespace)); - let settled = elect_consensus().local_dvc_suffix(); + assert_eq!(partition.consensus().applied_log_contradictions(), 1); + } - // Only the backup ticks: its status timeout supersedes the elect, which runs - // nothing after its proof settled. - let mut scratch = Vec::new(); - for _ in 0..2 * TimeoutManager::VIEW_CHANGE_STATUS_TICKS { - if backup_consensus().view() == SUPERSEDING_VIEW { - break; - } - assert!(backup.owner.tick_partitions(&mut scratch).await.is_none()); - } - assert_eq!(backup_consensus().view(), SUPERSEDING_VIEW); - // `take` returns the first match, so drop the frames of earlier views, the - // backup's retransmits among them. - for replica in [&backup, &elect] { - replica - .sent - .borrow_mut() - .retain(|(_, frame)| frame.header().view == SUPERSEDING_VIEW); - } - assert!(elect_consensus().view_log_merged()); - elect - .owner - .on_message(backup.take(HEADLESS_CANDIDATE, Command::StartViewChange)) + #[compio::test] + async fn given_a_refused_contradicting_start_view_when_ticking_should_export_it_once() { + const APPLIED: u64 = 2; + const NEW_VIEW: u32 = 3; + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = test_owner(&bus, None); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = primary_prepares(&bus, namespace, APPLIED).await; + host_applied_backup(&owner, &bus, namespace, &prepares).await; + let mut forged = *prepares.last().unwrap().header(); + forged.timestamp += 1; + forged.checksum = forged.identity_checksum(); + owner + .on_start_view(start_view_suffix( + namespace, + NEW_VIEW, + APPLIED, + APPLIED, + &[forged], + )) .await; - assert_eq!(elect_consensus().view(), SUPERSEDING_VIEW); - let voted = async { - while ![Command::StartViewChange, Command::DoViewChange] - .into_iter() - .all(|command| elect.sent_to(SUPERSEDING_PRIMARY, command, SUPERSEDING_VIEW)) - { - elect.step(&mut scratch).await; - } - }; - compio::time::timeout(REPLY_DEADLINE, Box::pin(voted)) - .await - .expect("the superseded elect must persist the view and vote in it"); - let MessageBag::DoViewChange(vote) = elect.take(SUPERSEDING_PRIMARY, Command::DoViewChange) - else { - unreachable!("take returns the command it was asked for"); - }; - let header = *vote.header(); + for _ in 0..2 { + assert!(owner.tick_partitions(&mut Vec::new()).await.is_none()); + } + assert_eq!( - header.log_view, REPAIR_VIEW, - "the merge never started, so the elect claims the view it replicated in" + owner.metrics.partition_applied_log_contradictions_value(), + 1 ); + let partition = owner.plane.partitions().get_io_owner(&namespace).unwrap(); assert_eq!( - (header.commit, header.nack_bitset, header.present_bitset), - (0, 0, 0), - "nothing is committed, and an unproven journal neither offers nor nacks an uncommitted op" + partition.consensus().applied_log_contradictions(), + 1, + "the export must leave the total for the simulator invariants" ); - let suffix = consensus::dvc_suffix_decode( - vote.body(), - header.op, - header.nack_bitset, - header.present_bitset, - ) - .unwrap(); + } + + #[compio::test] + async fn given_a_start_view_contradicting_an_evicted_applied_op_when_refused_should_fence_the_partition_for_resync() + { + const APPLIED: u64 = 2; + const NEW_VIEW: u32 = 3; + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = test_owner(&bus, None); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = primary_prepares(&bus, namespace, APPLIED).await; + host_applied_backup(&owner, &bus, namespace, &prepares).await; + { + let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap(); + let journal = &partition.log.journal().inner; + journal.evict_prefix(usize::try_from(APPLIED).unwrap()); + assert!(journal.header_by_op(APPLIED).is_none()); + assert!(partition.wal_checksum(APPLIED).is_none()); + assert!(partition.applied_header(APPLIED).is_some()); + } + let mut forged = *prepares.last().unwrap().header(); + forged.timestamp += 1; + forged.checksum = forged.identity_checksum(); + + owner + .on_start_view(start_view_suffix( + namespace, + NEW_VIEW, + APPLIED, + APPLIED, + &[forged], + )) + .await; + + let partitions = owner.plane.partitions(); assert!( - !suffix.is_empty(), - "the vote must carry the snapshot taken before the view change" + partitions.is_tombstoned(&namespace), + "an op only the repair ring still holds must fence the partition it refused on" ); + assert_eq!(owner.metrics.partition_applied_divergences_value(), 1); assert_eq!( - (suffix.headers(), header.nack_bitset, header.present_bitset), - ( - settled.headers(), - settled.nack_bitset(), - settled.present_bitset() - ), - "the superseding handler must vote with the snapshot of the settled proof" + partitions + .get_io_owner(&namespace) + .unwrap() + .consensus() + .view(), + REPAIR_VIEW ); + } - backup - .owner - .on_message(elect.take(SUPERSEDING_PRIMARY, Command::StartViewChange)) - .await; - backup - .owner - .on_message(MessageBag::DoViewChange(vote)) - .await; - for _ in 0..TimeoutManager::VIEW_CHANGE_STATUS_TICKS { - if backup_consensus().is_normal() { - break; - } - assert!(backup.owner.tick_partitions(&mut scratch).await.is_none()); - } - assert!(backup_consensus().is_normal()); - assert!(backup_consensus().is_primary()); - assert_eq!(backup_consensus().view(), SUPERSEDING_VIEW); - elect - .owner - .on_message(backup.take(HEADLESS_CANDIDATE, Command::StartView)) - .await; - assert!(elect_consensus().is_normal()); - assert_eq!(elect_consensus().view(), SUPERSEDING_VIEW); + #[compio::test] + async fn given_a_refused_contradicting_metadata_log_when_ticking_should_export_it_once() { + let bus = Rc::new(IggyMessageBus::new(0)); + let (mut owner, _sender) = test_owner(&bus, None); + let consensus = VsrConsensus::new( + 1, + 0, + 1, + METADATA_GROUP, + Rc::clone(&bus), + LocalPipeline::new(), + ); + consensus.init(); + owner.plane.inner_mut().0 = IggyMetadata::new( + Some(consensus), + None, + None, + None, + IoTestMetadata::new(( + UsersInner::new().into(), + (StreamsInner::default().into(), ()), + )), + None, + ); + let announced = PrepareHeader { + checksum: 1, + ..PrepareHeader::default() + }; + let consensus = owner.plane.inner().0.consensus.as_ref().unwrap(); + assert!(consensus.contradicts_applied_log( + consensus::PlaneKind::Metadata, + consensus::ContradictionSource::StartView, + 1, + [&announced], + |_| Some(PrepareHeader { + checksum: 2, + ..announced + }), + )); + + for _ in 0..2 { + owner.tick_metadata().await; + } + + assert_eq!(owner.metrics.metadata_applied_log_contradictions_value(), 1); + assert_eq!( + owner + .plane + .inner() + .0 + .consensus + .as_ref() + .unwrap() + .applied_log_contradictions(), + 1 + ); } - /// As above, but the elect adopted the view of the third replica, which goes - /// silent after it prepares one more op. The third replica still votes for the - /// next view with that op, then goes down. The merge truncates nothing, so the - /// elect's WAL still certifies its `log_view`, and the backup's `log_view` is - /// below it. Only the elect's journal then holds every op, so it must stay - /// proven and offer them. #[compio::test] - async fn given_a_settled_merge_that_kept_the_wal_certificate_when_a_lagging_backup_supersedes_the_elect_should_elect_the_backup_from_the_merged_log() - { - const HEAD: u64 = 3; - const SUPERSEDING_VIEW: u32 = HEADLESS_NEXT_VIEW + 1; - const SUPERSEDING_PRIMARY: u8 = HEADLESS_BACKUP; - let directory = tempfile::tempdir().unwrap(); - let store = Rc::new(HeldSuperblock { - entered: RefCell::new(None), - held: RefCell::new(None), - }); + async fn given_a_start_view_that_matches_the_applied_ops_when_adopting_should_adopt_it() { + const APPLIED: u64 = 2; + const NEW_VIEW: u32 = 3; + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = test_owner(&bus, None); let namespace = IggyNamespace::new(0, 0, 0); - let prepares = primary_prepares(&Rc::new(IggyMessageBus::new(0)), namespace, HEAD).await; - let adopted = usize::try_from(HEAD - 1).unwrap(); - // The primary prepared the last op after it started the adopted view. - let lagged = - prepares[adopted] - .clone() - .transmute_header(|previous, header: &mut PrepareHeader| { - *header = previous; - header.view = HEADLESS_VIEW; - header.checksum = header.identity_checksum(); - }); - let backup = Box::pin(HeadlessReplica::journaled( - HEADLESS_BACKUP, - namespace, - &prepares[..1], - None, - )) - .await; - let elect = Box::pin(HeadlessReplica::journaled( - HEADLESS_CANDIDATE, - namespace, - &prepares[..adopted], - Some((directory.path(), &store)), - )) - .await; - let backup_consensus = || backup.partition(namespace).consensus(); - let elect_partition = || elect.partition(namespace); - let elect_consensus = || elect.partition(namespace).consensus(); - elect - .owner + let prepares = primary_prepares(&bus, namespace, APPLIED).await; + host_applied_backup(&owner, &bus, namespace, &prepares).await; + + owner .on_start_view(start_view_suffix( namespace, - HEADLESS_VIEW, - HEAD - 1, - 0, - &[*prepares[adopted - 1].header()], + NEW_VIEW, + APPLIED, + APPLIED, + &[*prepares.last().unwrap().header()], )) .await; - assert!(!elect_consensus().view_log_unproven()); - assert_eq!(elect_consensus().log_view(), HEADLESS_VIEW); - let mut scratch = Vec::new(); - let certified = async { - while !elect_partition().wal_keeps_log_view() { - elect.step(&mut scratch).await; - } - }; - compio::time::timeout(REPLY_DEADLINE, Box::pin(certified)) - .await - .expect("the elect's WAL must certify the view it adopted"); - for _ in 0..2 * TimeoutManager::NORMAL_HEARTBEAT_TICKS { - if elect_consensus().view() == HEADLESS_NEXT_VIEW { - break; - } - assert!(elect.owner.tick_partitions(&mut scratch).await.is_none()); + let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap(); + assert_eq!(partition.consensus().view(), NEW_VIEW); + assert!(!partition.consensus().view_log_unproven()); + assert_eq!(partition.consensus().applied_log_contradictions(), 0); + } + + #[compio::test] + async fn given_a_divergent_entry_when_removal_is_deferred_should_keep_the_canonical_tail() { + const APPLIED: u64 = 2; + const DIVERGENT: u64 = 3; + const HEAD: u64 = 4; + const NEW_VIEW: u32 = 3; + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = test_owner(&bus, None); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = primary_prepares(&bus, namespace, HEAD).await; + let mut partition = + repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); + partition.repair = None; + for prepare in &prepares { + partition.on_replicate(prepare.clone()).await; } - assert_eq!(elect_consensus().view(), HEADLESS_NEXT_VIEW); - let announced = async { - while !elect.sent_to( - HEADLESS_BACKUP, - Command::StartViewChange, - HEADLESS_NEXT_VIEW, - ) { - elect.step(&mut scratch).await; - } - }; - compio::time::timeout(REPLY_DEADLINE, Box::pin(announced)) - .await - .expect("the elect must announce the view change"); - backup - .owner - .on_message(elect.take(HEADLESS_BACKUP, Command::StartViewChange)) - .await; - elect - .owner - .on_message(backup.take(HEADLESS_CANDIDATE, Command::StartViewChange)) + partition.consensus().advance_commit_max(APPLIED); + partition + .commit_journal(owner.plane.partitions().config()) .await; - // The backup's own vote never arrives, so the quorum is the elect's and the - // third replica's, which offers a body for every op. - let third_replica = DvcSuffix::new( - [&lagged, &prepares[1], &prepares[0]] - .map(|prepare| *prepare.header()) - .to_vec(), - 0, - (1 << HEAD) - 1, + partition + .log + .journal() + .inner + .remove_range(DIVERGENT..=DIVERGENT) + .await + .unwrap(); + let divergent = prepares[usize::try_from(DIVERGENT - 1).unwrap()] + .clone() + .transmute_header(|previous, header: &mut PrepareHeader| { + *header = previous; + header.client += 1; + header.checksum = header.identity_checksum(); + }); + partition + .log + .journal() + .inner + .append(divergent.into_frozen()) + .await + .unwrap(); + let partitions = owner.plane.partitions(); + partitions.remove(&namespace).unwrap(); + partitions.insert(namespace, partition); + partitions.set_io_notifier( + owner.partition_io.notifier(), + owner.partition_io.limits.bytes_max(), ); - elect - .owner - .on_message(MessageBag::DoViewChange(do_view_change_suffix( + + owner + .on_start_view(start_view_suffix( namespace, - HEADLESS_PRIMARY, - HEADLESS_NEXT_VIEW, - HEADLESS_VIEW, + NEW_VIEW, HEAD, - &third_replica, - ))) + HEAD, + &[*prepares.last().unwrap().header()], + )) .await; - assert!(elect_consensus().is_primary_elect()); - let fetching = async { - while elect_partition().repair.is_none() { - elect.step(&mut scratch).await; - } - }; - compio::time::timeout(REPLY_DEADLINE, Box::pin(fetching)) - .await - .expect("the elect must fetch the op it lacks"); - assert_eq!( - elect_partition().log.journal().inner.last_op(), - Some(HEAD - 1), - "the elect holds a prefix of the merged log, so nothing truncates" + assert!( + partitions + .get_by_ns(&namespace) + .unwrap() + .view_truncation_pending() ); - elect - .owner - .on_repair_prepare(as_repair_prepare(lagged.clone())) - .await; - assert!(!elect_consensus().view_log_unproven()); - assert!(elect_consensus().is_primary_elect()); - assert!(elect_partition().wal_keeps_log_view()); - - // Only the backup ticks: its status timeout supersedes the elect, which runs - // nothing after its proof settled. - for _ in 0..2 * TimeoutManager::VIEW_CHANGE_STATUS_TICKS { - if backup_consensus().view() == SUPERSEDING_VIEW { + for _ in 0..crate::router::COOPERATIVE_EVENT_BUDGET { + owner.service_partition_io().await; + if !partitions + .get_by_ns(&namespace) + .unwrap() + .view_truncation_pending() + { break; } - assert!(backup.owner.tick_partitions(&mut scratch).await.is_none()); } - assert_eq!(backup_consensus().view(), SUPERSEDING_VIEW); - for replica in [&backup, &elect] { - replica - .sent - .borrow_mut() - .retain(|(_, frame)| frame.header().view == SUPERSEDING_VIEW); - } - elect - .owner - .on_message(backup.take(HEADLESS_CANDIDATE, Command::StartViewChange)) - .await; - let voted = async { - while ![Command::StartViewChange, Command::DoViewChange] - .into_iter() - .all(|command| elect.sent_to(SUPERSEDING_PRIMARY, command, SUPERSEDING_VIEW)) - { - elect.step(&mut scratch).await; - } - }; - compio::time::timeout(REPLY_DEADLINE, Box::pin(voted)) - .await - .expect("the superseded elect must vote"); - let MessageBag::DoViewChange(vote) = elect.take(SUPERSEDING_PRIMARY, Command::DoViewChange) - else { - unreachable!("take returns the command it was asked for"); - }; - assert!( - backup_consensus().log_view() < vote.header().log_view, - "no copy the backup holds can stand in for the elect's" + let partition = partitions.get_by_ns(&namespace).unwrap(); + assert!(!partition.view_truncation_pending()); + assert!(partition.consensus().view_log_unproven()); + assert_eq!(partition.consensus().commit_min(), APPLIED); + assert!(!partition.log.journal().inner.holds_op(DIVERGENT)); + assert_eq!( + partition.log.journal().inner.header_by_op(HEAD), + Some(*prepares.last().unwrap().header()) ); - backup - .owner - .on_message(elect.take(SUPERSEDING_PRIMARY, Command::StartViewChange)) - .await; - backup - .owner - .on_message(MessageBag::DoViewChange(vote)) - .await; - let merging = async { - while backup.partition(namespace).repair.is_none() { - backup.step(&mut scratch).await; - } - }; - compio::time::timeout(REPLY_DEADLINE, Box::pin(merging)) - .await - .expect("the backup must merge the elect's log and fetch the ops it lacks"); - for prepare in [&prepares[1], &lagged] { - backup - .owner - .on_repair_prepare(as_repair_prepare(prepare.clone())) + let repaired = prepares[usize::try_from(DIVERGENT - 1).unwrap()] + .clone() + .transmute_header( + |header, repaired: &mut iggy_binary_protocol::RepairPrepareHeader| { + repaired.0 = header; + repaired.0.command = Command::RepairPrepare; + }, + ); + owner.on_repair_prepare(repaired).await; + let partition = partitions.get_mut_by_ns(&namespace).unwrap(); + assert!(!partition.consensus().view_log_unproven()); + partition.commit_journal(partitions.config()).await; + assert_eq!(partition.consensus().commit_min(), HEAD); + } + + #[compio::test] + async fn given_a_divergent_partition_run_when_reconciling_should_remove_it_at_once() { + const APPLIED: u64 = 2; + const FIRST: u64 = APPLIED + 1; + const DIVERGENT: u64 = 32; + const LAST: u64 = FIRST + DIVERGENT - 1; + const HEAD: u64 = LAST + 3; + const NEW_VIEW: u32 = 3; + for deferred in [true, false] { + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = test_owner(&bus, None); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = primary_prepares(&bus, namespace, HEAD).await; + let canonical: Vec<_> = prepares + .iter() + .rev() + .take(usize::try_from(HEAD - APPLIED).unwrap()) + .map(|prepare| *prepare.header()) + .collect(); + let view = || start_view_suffix(namespace, NEW_VIEW, HEAD, HEAD, &canonical); + install_divergent_backup(&owner, &bus, namespace, &prepares, APPLIED, FIRST..=LAST) .await; - } - for _ in 0..TimeoutManager::VIEW_CHANGE_STATUS_TICKS { - if backup_consensus().is_normal() { - break; + let partitions = owner.plane.partitions(); + if deferred { + partitions.set_io_notifier( + owner.partition_io.notifier(), + owner.partition_io.limits.bytes_max(), + ); + } + owner.on_start_view(view()).await; + + assert_eq!( + settle_view_truncation(&owner, namespace, FIRST..=LAST).await, + usize::from(deferred), + "one transition truncates the whole proven run" + ); + let partition = partitions.get_mut_by_ns(&namespace).unwrap(); + assert!(partition.consensus().view_log_unproven()); + assert_eq!( + partition.consensus().view_log_repair_op(), + Some(FIRST), + "repair opens at the lowest removed op, so one window refetches the run" + ); + partition.commit_journal(partitions.config()).await; + assert_eq!(partition.consensus().commit_min(), APPLIED); + let journal = &partition.log.journal().inner; + for prepare in prepares.iter().skip(usize::try_from(APPLIED).unwrap()) { + let op = prepare.header().op; + if op <= LAST { + assert!(!journal.holds_op(op), "divergent op {op} must go"); + } else { + assert_eq!( + journal.header_by_op(op), + Some(*prepare.header()), + "canonical op {op} must stay" + ); + } } - assert!(backup.owner.tick_partitions(&mut scratch).await.is_none()); } - assert!(backup_consensus().is_normal()); - assert!(backup_consensus().is_primary()); - assert_eq!(backup_consensus().view(), SUPERSEDING_VIEW); - elect - .owner - .on_message(backup.take(HEADLESS_CANDIDATE, Command::StartView)) - .await; - assert!(elect_consensus().is_normal()); - assert_eq!(elect_consensus().view(), SUPERSEDING_VIEW); } - /// The walk refuses an unproven view log, so a resident op after - /// `commit_min` cannot move a group whose proof waits on a missing body. - /// When the session ceiling refuses the arm at `StartView` adoption, the - /// gap detector is the only site left to fetch that body. #[compio::test] - async fn given_a_proof_waiting_above_a_resident_next_op_when_the_edge_arm_is_refused_should_arm_from_the_tick() + async fn given_a_bridged_header_off_the_view_chain_when_reconciling_a_partition_should_fetch_its_op() { const APPLIED: u64 = 2; - const LOCAL_HEAD: u64 = 4; - const NEW_HEAD: u64 = 8; + const HEAD: u64 = 8; + const BRIDGED: u64 = 5; const NEW_VIEW: u32 = 3; - const NEW_PRIMARY: u8 = 0; let bus = Rc::new(IggyMessageBus::new(0)); let (owner, _sender) = test_owner(&bus, None); - owner.set_repair_gap_debounce_ticks(REPAIR_GAP_DEBOUNCE_TICKS_MIN); let namespace = IggyNamespace::new(0, 0, 0); - let prepares = primary_prepares(&bus, namespace, NEW_HEAD).await; - let mut partition = - repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); - partition.repair = None; - for prepare in prepares.iter().take(usize::try_from(LOCAL_HEAD).unwrap()) { - partition.on_replicate(prepare.clone()).await; - } - partition.consensus().advance_commit_max(APPLIED); - partition - .commit_journal(owner.plane.partitions().config()) - .await; - owner.plane.partitions().remove(&namespace).unwrap(); - owner.plane.partitions().insert(namespace, partition); - // Other groups hold every session slot when the view change lands. - owner - .partition_repairs_inflight - .set(PARTITION_REPAIRS_INFLIGHT_MAX); + let prepares = primary_prepares(&bus, namespace, HEAD).await; + let canonical: Vec<_> = prepares + .iter() + .rev() + .take(usize::try_from(HEAD - APPLIED).unwrap()) + .map(|prepare| *prepare.header()) + .collect(); + install_divergent_backup( + &owner, + &bus, + namespace, + &prepares, + APPLIED, + BRIDGED..=BRIDGED, + ) + .await; + let partitions = owner.plane.partitions(); + partitions + .get_by_ns(&namespace) + .unwrap() + .log + .journal() + .inner + .remove_range(BRIDGED..=BRIDGED) + .await + .unwrap(); + partitions.set_io_notifier( + owner.partition_io.notifier(), + owner.partition_io.limits.bytes_max(), + ); owner .on_start_view(start_view_suffix( namespace, NEW_VIEW, - NEW_HEAD, - NEW_HEAD, - &[*prepares.last().unwrap().header()], + HEAD, + HEAD, + &off_chain_at(&canonical, BRIDGED), )) .await; - let proof_op = { - let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap(); - assert!( - partition.repair.is_none(), - "the ceiling must refuse the edge arm" - ); - assert_eq!(partition.consensus().commit_min(), APPLIED); - assert!(partition.log.journal().inner.holds_op(APPLIED + 1)); - let proof_op = partition.consensus().view_log_repair_op().unwrap(); - assert!(!partition.log.journal().inner.holds_op(proof_op)); - proof_op - }; - - let mut expected = vec![(None, None); REPAIR_GAP_DEBOUNCE_TICKS_MIN as usize - 1]; - expected.push((Some(NEW_PRIMARY), None)); - let observed = stall_rounds(&owner, namespace, expected.len()).await; - assert_eq!( - observed, expected, - "the detector must arm once its debounce runs out" + let partition = partitions.get_by_ns(&namespace).unwrap(); + assert!( + !partition.view_truncation_pending(), + "op {BRIDGED} is not held, so a truncation there removes nothing and the next \ + proof defers it again" ); - let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap(); + assert!(partition.consensus().view_log_unproven()); assert_eq!( - partition.repair.map(|session| session.from_op), - Some(proof_op) + partition.consensus().view_log_repair_op(), + Some(BRIDGED), + "the op under the off-chain header is a gap to fetch" ); + let journal = &partition.log.journal().inner; + for prepare in prepares.iter().skip(usize::try_from(APPLIED).unwrap()) { + let op = prepare.header().op; + assert_eq!( + journal.header_by_op(op), + (op != BRIDGED).then_some(*prepare.header()), + "op {op}: the canonical tail and the unproven entries below the gap stay" + ); + } } - /// Host at `namespace` a backup of [`REPAIR_VIEW`] that applied `prepares`. + /// Service the I/O lane until the view truncation settles, and count the steps + /// that removed part of `divergent`. + #[allow(clippy::future_not_send)] + async fn settle_view_truncation( + owner: &IoTestShard, + namespace: IggyNamespace, + divergent: RangeInclusive, + ) -> usize { + let partitions = owner.plane.partitions(); + let pending = || { + partitions + .get_by_ns(&namespace) + .unwrap() + .view_truncation_pending() + }; + let held = || { + let partition = partitions.get_by_ns(&namespace).unwrap(); + divergent + .clone() + .filter(|op| partition.log.journal().inner.holds_op(*op)) + .count() + }; + let mut still_held = held(); + let mut truncations = 0; + for _ in 0..crate::router::COOPERATIVE_EVENT_BUDGET { + if !pending() { + break; + } + owner.service_partition_io().await; + let now_held = held(); + if now_held < still_held { + truncations += 1; + still_held = now_held; + } + } + assert!(!pending(), "the view truncation never settled"); + truncations + } + + /// Install a backup that journaled `prepares`, applied through `applied`, and + /// holds a stale copy of every op in `divergent`. It inherits the I/O lane only + /// once the test set one. #[allow(clippy::future_not_send)] - async fn host_applied_backup( + async fn install_divergent_backup( owner: &IoTestShard, bus: &Rc, namespace: IggyNamespace, prepares: &[Message], + applied: u64, + divergent: RangeInclusive, ) { let mut partition = repairing_replica(bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); @@ -6312,890 +8494,1257 @@ mod tests { for prepare in prepares { partition.on_replicate(prepare.clone()).await; } - let applied = u64::try_from(prepares.len()).unwrap(); partition.consensus().advance_commit_max(applied); partition .commit_journal(owner.plane.partitions().config()) .await; - assert_eq!(partition.consensus().commit_min(), applied); - owner.plane.partitions().remove(&namespace).unwrap(); - owner.plane.partitions().insert(namespace, partition); - } - - #[compio::test] - async fn given_a_start_view_that_contradicts_an_applied_op_when_adopting_should_refuse_it() { - const APPLIED: u64 = 2; - const NEW_VIEW: u32 = 3; - let bus = Rc::new(IggyMessageBus::new(0)); - let (owner, _sender) = test_owner(&bus, None); - let namespace = IggyNamespace::new(0, 0, 0); - let prepares = primary_prepares(&bus, namespace, APPLIED).await; - host_applied_backup(&owner, &bus, namespace, &prepares).await; - let mut forged = *prepares.last().unwrap().header(); - forged.timestamp += 1; - forged.checksum = forged.identity_checksum(); - - owner - .on_start_view(start_view_suffix( - namespace, - NEW_VIEW, - APPLIED, - APPLIED, - &[forged], - )) - .await; - - // The refusal also fences the partition for resync, behind the tombstone. - let partition = owner.plane.partitions().get_io_owner(&namespace).unwrap(); - assert_eq!( - partition.consensus().view(), - REPAIR_VIEW, - "a log naming another prepare at applied op {APPLIED} must be refused, not adopted" - ); - assert_eq!(partition.consensus().applied_log_contradictions(), 1); - } - - #[compio::test] - async fn given_a_refused_contradicting_start_view_when_ticking_should_export_it_once() { - const APPLIED: u64 = 2; - const NEW_VIEW: u32 = 3; - let bus = Rc::new(IggyMessageBus::new(0)); - let (owner, _sender) = test_owner(&bus, None); - let namespace = IggyNamespace::new(0, 0, 0); - let prepares = primary_prepares(&bus, namespace, APPLIED).await; - host_applied_backup(&owner, &bus, namespace, &prepares).await; - let mut forged = *prepares.last().unwrap().header(); - forged.timestamp += 1; - forged.checksum = forged.identity_checksum(); - owner - .on_start_view(start_view_suffix( - namespace, - NEW_VIEW, - APPLIED, - APPLIED, - &[forged], - )) - .await; - - for _ in 0..2 { - assert!(owner.tick_partitions(&mut Vec::new()).await.is_none()); - } - - assert_eq!( - owner.metrics.partition_applied_log_contradictions_value(), - 1 - ); - let partition = owner.plane.partitions().get_io_owner(&namespace).unwrap(); - assert_eq!( - partition.consensus().applied_log_contradictions(), - 1, - "the export must leave the total for the simulator invariants" - ); - } - - #[compio::test] - async fn given_a_start_view_contradicting_an_evicted_applied_op_when_refused_should_fence_the_partition_for_resync() - { - const APPLIED: u64 = 2; - const NEW_VIEW: u32 = 3; - let bus = Rc::new(IggyMessageBus::new(0)); - let (owner, _sender) = test_owner(&bus, None); - let namespace = IggyNamespace::new(0, 0, 0); - let prepares = primary_prepares(&bus, namespace, APPLIED).await; - host_applied_backup(&owner, &bus, namespace, &prepares).await; + let journal = &partition.log.journal().inner; + journal.remove_range(divergent.clone()).await.unwrap(); + for prepare in prepares + .iter() + .filter(|prepare| divergent.contains(&prepare.header().op)) { - let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap(); - let journal = &partition.log.journal().inner; - journal.evict_prefix(usize::try_from(APPLIED).unwrap()); - assert!(journal.header_by_op(APPLIED).is_none()); - assert!(partition.wal_checksum(APPLIED).is_none()); - assert!(partition.applied_header(APPLIED).is_some()); + let stale = prepare + .clone() + .transmute_header(|previous, header: &mut PrepareHeader| { + *header = previous; + header.client += 1; + header.checksum = header.identity_checksum(); + }); + journal.append(stale.into_frozen()).await.unwrap(); } - let mut forged = *prepares.last().unwrap().header(); - forged.timestamp += 1; - forged.checksum = forged.identity_checksum(); - - owner - .on_start_view(start_view_suffix( - namespace, - NEW_VIEW, - APPLIED, - APPLIED, - &[forged], - )) - .await; - let partitions = owner.plane.partitions(); - assert!( - partitions.is_tombstoned(&namespace), - "an op only the repair ring still holds must fence the partition it refused on" - ); - assert_eq!(owner.metrics.partition_applied_divergences_value(), 1); - assert_eq!( - partitions - .get_io_owner(&namespace) - .unwrap() - .consensus() - .view(), - REPAIR_VIEW - ); + partitions.remove(&namespace).unwrap(); + partitions.insert(namespace, partition); } - #[compio::test] - async fn given_a_refused_contradicting_metadata_log_when_ticking_should_export_it_once() { - let bus = Rc::new(IggyMessageBus::new(0)); - let (mut owner, _sender) = test_owner(&bus, None); - let consensus = VsrConsensus::new( - 1, - 0, - 1, - METADATA_GROUP, - Rc::clone(&bus), - LocalPipeline::new(), - ); - consensus.init(); - owner.plane.inner_mut().0 = IggyMetadata::new( - Some(consensus), - None, - None, - None, - IoTestMetadata::new(( - UsersInner::new().into(), - (StreamsInner::default().into(), ()), - )), - None, - ); - let announced = PrepareHeader { - checksum: 1, - ..PrepareHeader::default() - }; - let consensus = owner.plane.inner().0.consensus.as_ref().unwrap(); - assert!(consensus.contradicts_applied_log( - consensus::PlaneKind::Metadata, - consensus::ContradictionSource::StartView, - 1, - [&announced], - |_| Some(PrepareHeader { - checksum: 2, - ..announced - }), - )); - - for _ in 0..2 { - owner.tick_metadata().await; - } - - assert_eq!(owner.metrics.metadata_applied_log_contradictions_value(), 1); - assert_eq!( - owner - .plane - .inner() - .0 - .consensus - .as_ref() - .unwrap() - .applied_log_contradictions(), - 1 - ); + /// `suffix`, high to low, with the header at `bridged` moved off the view's + /// chain. The header above it goes blank, so the suffix still decodes: only + /// adjacent headers must chain there. + fn off_chain_at(suffix: &[PrepareHeader], bridged: u64) -> Vec { + suffix + .iter() + .map(|header| { + if header.op == bridged + 1 { + consensus::dvc_blank(header.op) + } else if header.op == bridged { + let mut off_chain = *header; + off_chain.client += 1; + off_chain.checksum = off_chain.identity_checksum(); + off_chain + } else { + *header + } + }) + .collect() } #[compio::test] - async fn given_a_start_view_that_matches_the_applied_ops_when_adopting_should_adopt_it() { + async fn given_deferred_view_truncation_when_resuming_should_settle_proof_only_after_success() { const APPLIED: u64 = 2; + const LOCAL_HEAD: u64 = 6; + const NEW_HEAD: u64 = 4; const NEW_VIEW: u32 = 3; - let bus = Rc::new(IggyMessageBus::new(0)); - let (owner, _sender) = test_owner(&bus, None); - let namespace = IggyNamespace::new(0, 0, 0); - let prepares = primary_prepares(&bus, namespace, APPLIED).await; - host_applied_backup(&owner, &bus, namespace, &prepares).await; + for (truncate_fails, repair_fallback) in + [(false, false), (true, false), (false, true), (true, true)] + { + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = test_owner(&bus, None); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = primary_prepares(&bus, namespace, LOCAL_HEAD).await; + let mut partition = + repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); + partition.repair = None; + for prepare in prepares.iter().take(usize::try_from(LOCAL_HEAD).unwrap()) { + partition.on_replicate(prepare.clone()).await; + } + partition.consensus().advance_commit_max(APPLIED); + partition + .commit_journal(owner.plane.partitions().config()) + .await; + let partitions = owner.plane.partitions(); + partitions.remove(&namespace).unwrap(); + partitions.insert(namespace, partition); + partitions.set_io_notifier( + owner.partition_io.notifier(), + owner.partition_io.limits.bytes_max(), + ); + let view = start_view_suffix( + namespace, + NEW_VIEW, + NEW_HEAD, + NEW_HEAD, + &[*prepares[usize::try_from(NEW_HEAD - 1).unwrap()].header()], + ); + if repair_fallback { + let partition = partitions.get_mut_by_ns(&namespace).unwrap(); + partition.consensus().handle_start_view( + consensus::PlaneKind::Partitions, + view.header(), + view.body(), + ); + assert!( + !owner + .maybe_request_partition_repair(partition, REPAIR_PRIMARY) + .await + ); + } else { + owner.on_start_view(view).await; + } + let partition = partitions.get_mut_by_ns(&namespace).unwrap(); + assert!(partition.view_truncation_pending()); + assert!(partition.consensus().view_log_unproven()); + if truncate_fails { + partition.log.active_segment_mut().size = IggyByteSize::from(u64::MAX); + } - owner - .on_start_view(start_view_suffix( - namespace, - NEW_VIEW, - APPLIED, - APPLIED, - &[*prepares.last().unwrap().header()], - )) - .await; + for _ in 0..crate::router::COOPERATIVE_EVENT_BUDGET { + owner.service_partition_io().await; + if !partitions + .get_by_ns(&namespace) + .unwrap() + .view_truncation_pending() + { + break; + } + } - let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap(); - assert_eq!(partition.consensus().view(), NEW_VIEW); - assert!(!partition.consensus().view_log_unproven()); - assert_eq!(partition.consensus().applied_log_contradictions(), 0); + let partition = partitions.get_mut_by_ns(&namespace).unwrap(); + assert!(!partition.view_truncation_pending()); + assert_eq!(partition.consensus().view_log_unproven(), truncate_fails); + partition.commit_journal(partitions.config()).await; + assert_eq!( + partition.consensus().commit_min(), + if truncate_fails { APPLIED } else { NEW_HEAD } + ); + assert_eq!( + partition.log.journal().inner.holds_op(NEW_HEAD + 1), + truncate_fails + ); + } } #[compio::test] - async fn given_a_divergent_entry_when_removal_is_deferred_should_keep_the_canonical_tail() { - const APPLIED: u64 = 2; - const DIVERGENT: u64 = 3; - const HEAD: u64 = 4; + async fn given_commit_only_start_view_when_entries_are_acked_should_keep_the_running_suffix() { + const FIRST_HEAD: u64 = 4; + const NEW_HEAD: u64 = 5; const NEW_VIEW: u32 = 3; let bus = Rc::new(IggyMessageBus::new(0)); let (owner, _sender) = test_owner(&bus, None); let namespace = IggyNamespace::new(0, 0, 0); - let prepares = primary_prepares(&bus, namespace, HEAD).await; + let prepares = primary_prepares(&bus, namespace, NEW_HEAD).await; let mut partition = repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); partition.repair = None; - for prepare in &prepares { + for prepare in prepares.iter().take(usize::try_from(FIRST_HEAD).unwrap()) { partition.on_replicate(prepare.clone()).await; } - partition.consensus().advance_commit_max(APPLIED); - partition - .commit_journal(owner.plane.partitions().config()) + owner.plane.partitions().remove(&namespace).unwrap(); + owner.plane.partitions().insert(namespace, partition); + let suffix: Vec<_> = prepares[..usize::try_from(FIRST_HEAD).unwrap()] + .iter() + .rev() + .map(|prepare| *prepare.header()) + .collect(); + owner + .on_start_view(start_view_suffix( + namespace, NEW_VIEW, FIRST_HEAD, 2, &suffix, + )) .await; - partition - .log - .journal() - .inner - .remove_range(DIVERGENT..=DIVERGENT) - .await - .unwrap(); - let divergent = prepares[usize::try_from(DIVERGENT - 1).unwrap()] - .clone() - .transmute_header(|previous, header: &mut PrepareHeader| { + let partition = owner.plane.partitions().get_mut_by_ns(&namespace).unwrap(); + let prepare = prepares.last().unwrap().clone().transmute_header( + |previous, header: &mut PrepareHeader| { *header = previous; - header.client += 1; + header.view = NEW_VIEW; + header.replica = 0; + header.commit = 3; header.checksum = header.identity_checksum(); - }); - partition - .log - .journal() - .inner - .append(divergent.into_frozen()) - .await - .unwrap(); - let partitions = owner.plane.partitions(); - partitions.remove(&namespace).unwrap(); - partitions.insert(namespace, partition); - partitions.set_io_notifier( - owner.partition_io.notifier(), - owner.partition_io.limits.bytes_max(), + }, ); - + partition.on_replicate(prepare.clone()).await; + assert!(partition.log.journal().inner.holds_op(NEW_HEAD)); owner .on_start_view(start_view_suffix( namespace, NEW_VIEW, - HEAD, - HEAD, - &[*prepares.last().unwrap().header()], + NEW_HEAD, + FIRST_HEAD, + &[*prepare.header()], )) .await; - assert!( - partitions - .get_by_ns(&namespace) - .unwrap() - .view_truncation_pending() - ); - for _ in 0..crate::router::COOPERATIVE_EVENT_BUDGET { - owner.service_partition_io().await; - if !partitions - .get_by_ns(&namespace) - .unwrap() - .view_truncation_pending() - { - break; - } - } - let partition = partitions.get_by_ns(&namespace).unwrap(); - assert!(!partition.view_truncation_pending()); - assert!(partition.consensus().view_log_unproven()); - assert_eq!(partition.consensus().commit_min(), APPLIED); - assert!(!partition.log.journal().inner.holds_op(DIVERGENT)); - assert_eq!( - partition.log.journal().inner.header_by_op(HEAD), - Some(*prepares.last().unwrap().header()) - ); - - let repaired = prepares[usize::try_from(DIVERGENT - 1).unwrap()] - .clone() - .transmute_header( - |header, repaired: &mut iggy_binary_protocol::RepairPrepareHeader| { - repaired.0 = header; - repaired.0.command = Command::RepairPrepare; - }, - ); - owner.on_repair_prepare(repaired).await; - let partition = partitions.get_mut_by_ns(&namespace).unwrap(); - assert!(!partition.consensus().view_log_unproven()); - partition.commit_journal(partitions.config()).await; - assert_eq!(partition.consensus().commit_min(), HEAD); + let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap(); + assert_eq!(partition.consensus().commit_min(), FIRST_HEAD); + assert!(partition.log.journal().inner.holds_op(NEW_HEAD)); } #[compio::test] - async fn given_a_divergent_partition_run_when_reconciling_should_remove_it_at_once() { - const APPLIED: u64 = 2; - const FIRST: u64 = APPLIED + 1; - const DIVERGENT: u64 = 32; - const LAST: u64 = FIRST + DIVERGENT - 1; - const HEAD: u64 = LAST + 3; - const NEW_VIEW: u32 = 3; - for deferred in [true, false] { - let bus = Rc::new(IggyMessageBus::new(0)); - let (owner, _sender) = test_owner(&bus, None); - let namespace = IggyNamespace::new(0, 0, 0); - let prepares = primary_prepares(&bus, namespace, HEAD).await; - let canonical: Vec<_> = prepares - .iter() - .rev() - .take(usize::try_from(HEAD - APPLIED).unwrap()) - .map(|prepare| *prepare.header()) - .collect(); - let view = || start_view_suffix(namespace, NEW_VIEW, HEAD, HEAD, &canonical); - install_divergent_backup(&owner, &bus, namespace, &prepares, APPLIED, FIRST..=LAST) - .await; - let partitions = owner.plane.partitions(); - if deferred { - partitions.set_io_notifier( - owner.partition_io.notifier(), - owner.partition_io.limits.bytes_max(), - ); - } - owner.on_start_view(view()).await; - - assert_eq!( - settle_view_truncation(&owner, namespace, FIRST..=LAST).await, - usize::from(deferred), - "one transition truncates the whole proven run" - ); - let partition = partitions.get_mut_by_ns(&namespace).unwrap(); - assert!(partition.consensus().view_log_unproven()); - assert_eq!( - partition.consensus().view_log_repair_op(), - Some(FIRST), - "repair opens at the lowest removed op, so one window refetches the run" - ); - partition.commit_journal(partitions.config()).await; - assert_eq!(partition.consensus().commit_min(), APPLIED); - let journal = &partition.log.journal().inner; - for prepare in prepares.iter().skip(usize::try_from(APPLIED).unwrap()) { - let op = prepare.header().op; - if op <= LAST { - assert!(!journal.holds_op(op), "divergent op {op} must go"); - } else { - assert_eq!( - journal.header_by_op(op), - Some(*prepare.header()), - "canonical op {op} must stay" - ); - } - } + async fn given_evicted_commit_backlog_when_adopting_start_view_should_preserve_the_ring_floor() + { + const COMMITTED: u64 = 3; + const STAMPED: u64 = 5; + const NEW_HEAD: u64 = 7; + let bus = Rc::new(IggyMessageBus::new(0)); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = primary_prepares(&bus, namespace, NEW_HEAD).await; + let mut partition = + repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); + for prepare in prepares.iter().take(usize::try_from(STAMPED).unwrap()) { + let prepare = if prepare.header().op == STAMPED { + prepare + .clone() + .transmute_header(|previous, header: &mut PrepareHeader| { + *header = previous; + header.commit = STAMPED - 1; + header.checksum = header.identity_checksum(); + }) + } else { + prepare.clone() + }; + partition.on_replicate(prepare).await; } + partition.consensus().advance_commit_max(COMMITTED); + partition + .log + .journal() + .inner + .evict_prefix(usize::try_from(COMMITTED).unwrap()); + assert_eq!(partition.consensus().commit_min(), 0); + let message = start_view_suffix( + namespace, + 3, + NEW_HEAD, + NEW_HEAD, + &[*prepares.last().unwrap().header()], + ); + partition.consensus().handle_start_view( + consensus::PlaneKind::Partitions, + message.header(), + &message.as_slice()[size_of::()..], + ); + let pending = partition.consensus().pending_view_log(); + assert_eq!( + crate::partition_view_divergence_from(&partition, pending.as_ref()), + None + ); + assert!( + partition + .log + .journal() + .inner + .repair_header(COMMITTED) + .is_some() + ); } #[compio::test] - async fn given_a_bridged_header_off_the_view_chain_when_reconciling_a_partition_should_fetch_its_op() - { - const APPLIED: u64 = 2; - const HEAD: u64 = 8; - const BRIDGED: u64 = 5; - const NEW_VIEW: u32 = 3; + async fn given_pending_view_truncation_when_peer_requests_repair_should_not_serve_old_entries() + { + const HEAD: u64 = 4; let bus = Rc::new(IggyMessageBus::new(0)); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + captured.borrow_mut().push(frame); + Ok(()) + })); + crate::test_support::connect_replica_on(&bus, REPAIR_PRIMARY, 1); let (owner, _sender) = test_owner(&bus, None); let namespace = IggyNamespace::new(0, 0, 0); let prepares = primary_prepares(&bus, namespace, HEAD).await; - let canonical: Vec<_> = prepares - .iter() - .rev() - .take(usize::try_from(HEAD - APPLIED).unwrap()) - .map(|prepare| *prepare.header()) - .collect(); - install_divergent_backup( - &owner, - &bus, - namespace, - &prepares, - APPLIED, - BRIDGED..=BRIDGED, - ) - .await; - let partitions = owner.plane.partitions(); - partitions - .get_by_ns(&namespace) - .unwrap() - .log - .journal() - .inner - .remove_range(BRIDGED..=BRIDGED) - .await - .unwrap(); - partitions.set_io_notifier( - owner.partition_io.notifier(), - owner.partition_io.limits.bytes_max(), + let mut partition = + repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); + for prepare in prepares { + partition.on_replicate(prepare).await; + } + owner.plane.partitions().remove(&namespace).unwrap(); + owner.plane.partitions().insert(namespace, partition); + let request = Message::::new(size_of::< + iggy_binary_protocol::RequestPreparesHeader, + >()) + .transmute_header( + |_, header: &mut iggy_binary_protocol::RequestPreparesHeader| { + header.command = Command::RequestPrepares; + header.cluster = 1; + header.group = namespace.inner(); + header.replica = REPAIR_PRIMARY; + header.from_op = 3; + header.to_op = HEAD; + header.size = u32::try_from(std::mem::size_of_val(header)).unwrap(); + header.seal(); + }, ); - - owner - .on_start_view(start_view_suffix( - namespace, - NEW_VIEW, - HEAD, - HEAD, - &off_chain_at(&canonical, BRIDGED), - )) - .await; - - let partition = partitions.get_by_ns(&namespace).unwrap(); + sent.borrow_mut().clear(); + owner.on_request_prepares(&request).await; assert!( - !partition.view_truncation_pending(), - "op {BRIDGED} is not held, so a truncation there removes nothing and the next \ - proof defers it again" + !sent.borrow().is_empty(), + "the unblocked replica serves this range" ); - assert!(partition.consensus().view_log_unproven()); - assert_eq!( - partition.consensus().view_log_repair_op(), - Some(BRIDGED), - "the op under the off-chain header is a gap to fetch" + sent.borrow_mut().clear(); + owner + .plane + .partitions() + .get_mut_by_ns(&namespace) + .unwrap() + .defer_view_truncation(3..=u64::MAX, Vec::new(), None); + owner.on_request_prepares(&request).await; + assert!( + sent.borrow().is_empty(), + "unproven entries must not reach a peer" ); - let journal = &partition.log.journal().inner; - for prepare in prepares.iter().skip(usize::try_from(APPLIED).unwrap()) { - let op = prepare.header().op; - assert_eq!( - journal.header_by_op(op), - (op != BRIDGED).then_some(*prepare.header()), - "op {op}: the canonical tail and the unproven entries below the gap stay" - ); - } } - /// Service the I/O lane until the view truncation settles, and count the steps - /// that removed part of `divergent`. - #[allow(clippy::future_not_send)] - async fn settle_view_truncation( - owner: &IoTestShard, - namespace: IggyNamespace, - divergent: RangeInclusive, - ) -> usize { - let partitions = owner.plane.partitions(); - let pending = || { - partitions - .get_by_ns(&namespace) - .unwrap() - .view_truncation_pending() - }; - let held = || { - let partition = partitions.get_by_ns(&namespace).unwrap(); - divergent - .clone() - .filter(|op| partition.log.journal().inner.holds_op(*op)) - .count() - }; - let mut still_held = held(); - let mut truncations = 0; - for _ in 0..crate::router::COOPERATIVE_EVENT_BUDGET { - if !pending() { - break; - } - owner.service_partition_io().await; - let now_held = held(); - if now_held < still_held { - truncations += 1; - still_held = now_held; + #[compio::test] + async fn given_abandoned_prepares_when_a_backup_requests_repair_should_serve_only_committed_ops() + { + const REPLICAS: u8 = 5; + const DONOR: u8 = 3; + const BACKUP: u8 = 4; + const NEXT_VIEW: u32 = 2; + const NEXT_PRIMARY: u8 = 2; + const COMMITTED: u64 = 2; + const HEAD: u64 = 3; + let bus = Rc::new(IggyMessageBus::new(0)); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + let message = + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(); + if message.header().command == Command::RepairPrepare { + let prepare = Message::::try_from( + Owned::copy_from_slice(frame.as_slice()), + ) + .unwrap(); + captured.borrow_mut().push(prepare.header().0); } + Ok(()) + })); + for requester in [BACKUP, NEXT_PRIMARY] { + crate::test_support::connect_replica_on(&bus, requester, 1); } - assert!(!pending(), "the view truncation never settled"); - truncations - } - - /// Install a backup that journaled `prepares`, applied through `applied`, and - /// holds a stale copy of every op in `divergent`. It inherits the I/O lane only - /// once the test set one. - #[allow(clippy::future_not_send)] - async fn install_divergent_backup( - owner: &IoTestShard, - bus: &Rc, - namespace: IggyNamespace, - prepares: &[Message], - applied: u64, - divergent: RangeInclusive, - ) { - let mut partition = - repairing_replica(bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); - partition.repair = None; + let (owner, _sender) = test_owner(&bus, None); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = primary_prepares(&bus, namespace, HEAD).await; + let mut donor = repairing_replica(&bus, namespace, (DONOR, REPLICAS), REPAIR_PRIMARY, 0, 0); + donor.repair = None; for prepare in prepares { - partition.on_replicate(prepare.clone()).await; + donor.on_replicate(prepare).await; } - partition.consensus().advance_commit_max(applied); - partition + donor.consensus().advance_commit_max(COMMITTED); + donor .commit_journal(owner.plane.partitions().config()) .await; - let journal = &partition.log.journal().inner; - journal.remove_range(divergent.clone()).await.unwrap(); - for prepare in prepares - .iter() - .filter(|prepare| divergent.contains(&prepare.header().op)) - { - let stale = prepare - .clone() - .transmute_header(|previous, header: &mut PrepareHeader| { - *header = previous; - header.client += 1; - header.checksum = header.identity_checksum(); + for voter in [REPAIR_PRIMARY, BACKUP] { + let change = + view_control_message(namespace, Command::StartViewChange, NEXT_VIEW, voter); + let MessageBag::StartViewChange(change) = change else { + unreachable!() + }; + donor + .consensus() + .handle_start_view_change(consensus::PlaneKind::Partitions, change.header()); + } + assert_eq!(donor.consensus().status(), consensus::Status::ViewChange); + owner.plane.partitions().remove(&namespace).unwrap(); + owner.plane.partitions().insert(namespace, donor); + + for (requester, view, through) in [ + (BACKUP, NEXT_VIEW, COMMITTED), + (NEXT_PRIMARY, NEXT_VIEW, HEAD), + (NEXT_PRIMARY, REPAIR_VIEW, COMMITTED), + ] { + let request = Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestPreparesHeader| { + header.command = Command::RequestPrepares; + header.cluster = 1; + header.group = namespace.inner(); + header.replica = requester; + header.view = view; + header.from_op = 1; + header.to_op = HEAD; + header.size = u32::try_from(size_of::()).unwrap(); + header.seal(); }); - journal.append(stale.into_frozen()).await.unwrap(); + sent.borrow_mut().clear(); + owner.on_request_prepares(&request).await; + let served: Vec<_> = sent.borrow().iter().map(|header| header.op).collect(); + assert_eq!( + served, + (1..=through).collect::>(), + "requester {requester} in view {view}" + ); } - let partitions = owner.plane.partitions(); - partitions.remove(&namespace).unwrap(); - partitions.insert(namespace, partition); } - /// `suffix`, high to low, with the header at `bridged` moved off the view's - /// chain. The header above it goes blank, so the suffix still decodes: only - /// adjacent headers must chain there. - fn off_chain_at(suffix: &[PrepareHeader], bridged: u64) -> Vec { - suffix - .iter() - .map(|header| { - if header.op == bridged + 1 { - consensus::dvc_blank(header.op) - } else if header.op == bridged { - let mut off_chain = *header; - off_chain.client += 1; - off_chain.checksum = off_chain.identity_checksum(); - off_chain - } else { - *header - } - }) - .collect() + /// A donor whose evicted ring released the bodies of ops 1..=3 to their + /// segment serves them from the segment, byte for byte. It never serves a + /// corrupt or a torn body: the round stops there, and from then on the + /// ops up to it answer `RangeEvicted`. + #[compio::test] + async fn given_bodies_released_to_a_segment_when_a_backup_requests_them_should_serve_only_intact_ones() + { + let donor = Box::pin(SegmentDonor::new(4)).await; + let SegmentDonor { + prepares, + log_path, + positions, + .. + } = &donor; + + assert_eq!( + donor.answers(1).await, + [ + (Command::RepairPrepare, 1), + (Command::RepairPrepare, 2), + (Command::RepairPrepare, 3), + (Command::RepairPrepare, 4), + (Command::RepairDone, 4), + ] + ); + + // A flipped byte at the end of the body of op 2, and op 3 cut in half. + let mut segment = std::fs::read(log_path).unwrap(); + let op_3_start = usize::try_from(positions[&3]).unwrap(); + segment[op_3_start - 1] ^= 0xFF; + segment + .truncate(op_3_start + (prepares[2].as_slice().len() - size_of::()) / 2); + std::fs::write(log_path, &segment).unwrap(); + assert_eq!( + donor.answers(1).await, + [(Command::RepairPrepare, 1), (Command::RepairDone, 1)], + "the corrupt body of op 2 is not served" + ); + assert_eq!( + donor.answers(2).await, + [(Command::RangeEvicted, 3), (Command::RepairDone, 2)], + "the torn body of op 3 is not served" + ); + assert_eq!( + donor.answers(3).await, + [ + (Command::RangeEvicted, 4), + (Command::RepairPrepare, 4), + (Command::RepairDone, 4), + ] + ); } + /// A round that needs released bodies sends nothing until a detached read + /// brings them back, so the pump goes on while the disk works: another + /// requester's round of a resident op goes out first. A request that + /// repeats the round waits for its read. Once the bodies land, the round + /// goes out in op order and `RepairDone` ends it. #[compio::test] - async fn given_deferred_view_truncation_when_resuming_should_settle_proof_only_after_success() { - const APPLIED: u64 = 2; - const LOCAL_HEAD: u64 = 6; - const NEW_HEAD: u64 = 4; - const NEW_VIEW: u32 = 3; - for (truncate_fails, repair_fallback) in - [(false, false), (true, false), (false, true), (true, true)] - { - let bus = Rc::new(IggyMessageBus::new(0)); - let (owner, _sender) = test_owner(&bus, None); - let namespace = IggyNamespace::new(0, 0, 0); - let prepares = primary_prepares(&bus, namespace, LOCAL_HEAD).await; - let mut partition = - repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); - partition.repair = None; - for prepare in prepares.iter().take(usize::try_from(LOCAL_HEAD).unwrap()) { - partition.on_replicate(prepare.clone()).await; - } - partition.consensus().advance_commit_max(APPLIED); - partition - .commit_journal(owner.plane.partitions().config()) - .await; - let partitions = owner.plane.partitions(); - partitions.remove(&namespace).unwrap(); - partitions.insert(namespace, partition); - partitions.set_io_notifier( - owner.partition_io.notifier(), - owner.partition_io.limits.bytes_max(), - ); - let view = start_view_suffix( - namespace, - NEW_VIEW, - NEW_HEAD, - NEW_HEAD, - &[*prepares[usize::try_from(NEW_HEAD - 1).unwrap()].header()], - ); - if repair_fallback { - let partition = partitions.get_mut_by_ns(&namespace).unwrap(); - partition.consensus().handle_start_view( - consensus::PlaneKind::Partitions, - view.header(), - view.body(), - ); - assert!( - !owner - .maybe_request_partition_repair(partition, REPAIR_PRIMARY) - .await - ); - } else { - owner.on_start_view(view).await; - } - let partition = partitions.get_mut_by_ns(&namespace).unwrap(); - assert!(partition.view_truncation_pending()); - assert!(partition.consensus().view_log_unproven()); - if truncate_fails { - partition.log.active_segment_mut().size = IggyByteSize::from(u64::MAX); - } + async fn given_a_read_of_released_bodies_when_other_requests_arrive_should_serve_them_meanwhile() + { + const OTHER_REQUESTER: u8 = 0; + let donor = Box::pin(SegmentDonor::new(4)).await; + connect_replica_on(&donor.owner.bus, OTHER_REQUESTER, 1); + donor.sent.borrow_mut().clear(); + let request = repair_request(donor.namespace, 1, donor.prepares.len()); + let resident = repair_request(donor.namespace, 4, donor.prepares.len()).transmute_header( + |old, header: &mut RequestPreparesHeader| { + *header = old; + header.replica = OTHER_REQUESTER; + }, + ); - for _ in 0..crate::router::COOPERATIVE_EVENT_BUDGET { - owner.service_partition_io().await; - if !partitions - .get_by_ns(&namespace) - .unwrap() - .view_truncation_pending() - { - break; + let held = donor.owner.served_reads.hold(); + donor.owner.on_request_prepares(&request).await; + donor.owner.on_request_prepares(&request).await; + donor.owner.on_request_prepares(&resident).await; + assert_eq!( + sent_answers(&donor.sent, &donor.prepares), + [(Command::RepairPrepare, 4), (Command::RepairDone, 4)], + "only the resident round goes out while the read waits" + ); + + drop(held); + donor.owner.settle_served_reads().await; + assert_eq!( + sent_answers(&donor.sent, &donor.prepares), + [ + (Command::RepairPrepare, 1), + (Command::RepairPrepare, 2), + (Command::RepairPrepare, 3), + (Command::RepairPrepare, 4), + (Command::RepairDone, 4), + ], + "the repeated request is answered once" + ); + } + + /// The running pump serves other frames while a round waits for its + /// released bodies, and answers that round once the bodies land. + #[compio::test] + async fn given_a_running_pump_when_a_round_waits_for_released_bodies_should_serve_other_frames() + { + const OTHER_REQUESTER: u8 = 0; + const PUMP_WAITS_MAX: usize = 1000; + let donor = &Box::pin(SegmentDonor::new(4)).await; + connect_replica_on(&donor.owner.bus, OTHER_REQUESTER, 1); + donor.sent.borrow_mut().clear(); + let deliver = |request: Message| { + donor + .sender + .try_send(ShardFrame::consensus( + donor.owner.id, + MessageBag::RequestPrepares(request), + )) + .expect("fixture request fits the inbox"); + }; + let sent_frames = |count: usize| async move { + for _ in 0..PUMP_WAITS_MAX { + if donor.sent.borrow().len() >= count { + return; } + donor.owner.bus.sleep(Duration::from_millis(1)).await; } - - let partition = partitions.get_mut_by_ns(&namespace).unwrap(); - assert!(!partition.view_truncation_pending()); - assert_eq!(partition.consensus().view_log_unproven(), truncate_fails); - partition.commit_journal(partitions.config()).await; + panic!( + "the pump sent {} of {count} frames", + donor.sent.borrow().len() + ); + }; + let (stop, stopped) = channel(1); + let pump = donor + .owner + .run_message_pump(stopped, Arc::new(AtomicBool::new(false))); + let exercise = async { + let held = donor.owner.served_reads.hold(); + deliver(repair_request(donor.namespace, 1, donor.prepares.len())); + deliver( + repair_request(donor.namespace, 4, donor.prepares.len()).transmute_header( + |old, header: &mut RequestPreparesHeader| { + *header = old; + header.replica = OTHER_REQUESTER; + }, + ), + ); + sent_frames(2).await; assert_eq!( - partition.consensus().commit_min(), - if truncate_fails { APPLIED } else { NEW_HEAD } + sent_answers(&donor.sent, &donor.prepares), + [(Command::RepairPrepare, 4), (Command::RepairDone, 4)] ); + + drop(held); + sent_frames(5).await; assert_eq!( - partition.log.journal().inner.holds_op(NEW_HEAD + 1), - truncate_fails + sent_answers(&donor.sent, &donor.prepares), + [ + (Command::RepairPrepare, 1), + (Command::RepairPrepare, 2), + (Command::RepairPrepare, 3), + (Command::RepairPrepare, 4), + (Command::RepairDone, 4), + ] ); - } + stop.try_send(()).unwrap(); + }; + let (fault, ()) = futures::join!(pump, exercise); + assert!(fault.is_none()); } + /// The bodies can land after the partition left the shard. The serve + /// checks the state again, so it sends nothing, and a later request reads + /// again. #[compio::test] - async fn given_commit_only_start_view_when_entries_are_acked_should_keep_the_running_suffix() { - const FIRST_HEAD: u64 = 4; - const NEW_HEAD: u64 = 5; - const NEW_VIEW: u32 = 3; - let bus = Rc::new(IggyMessageBus::new(0)); - let (owner, _sender) = test_owner(&bus, None); - let namespace = IggyNamespace::new(0, 0, 0); - let prepares = primary_prepares(&bus, namespace, NEW_HEAD).await; - let mut partition = - repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); - partition.repair = None; - for prepare in prepares.iter().take(usize::try_from(FIRST_HEAD).unwrap()) { - partition.on_replicate(prepare.clone()).await; - } - owner.plane.partitions().remove(&namespace).unwrap(); - owner.plane.partitions().insert(namespace, partition); - let suffix: Vec<_> = prepares[..usize::try_from(FIRST_HEAD).unwrap()] - .iter() - .rev() - .map(|prepare| *prepare.header()) - .collect(); - owner - .on_start_view(start_view_suffix( - namespace, NEW_VIEW, FIRST_HEAD, 2, &suffix, - )) + async fn given_a_read_of_released_bodies_when_the_partition_leaves_should_send_nothing() { + let donor = Box::pin(SegmentDonor::new(4)).await; + let partitions = donor.owner.plane.partitions(); + donor.sent.borrow_mut().clear(); + donor + .owner + .on_request_prepares(&repair_request(donor.namespace, 1, donor.prepares.len())) .await; - let partition = owner.plane.partitions().get_mut_by_ns(&namespace).unwrap(); - let prepare = prepares.last().unwrap().clone().transmute_header( - |previous, header: &mut PrepareHeader| { - *header = previous; - header.view = NEW_VIEW; - header.replica = 0; - header.commit = 3; - header.checksum = header.identity_checksum(); + let partition = partitions.remove(&donor.namespace).unwrap(); + donor.owner.settle_served_reads().await; + assert!(donor.sent.borrow().is_empty()); + + partitions.insert(donor.namespace, partition); + assert_eq!( + donor.answers(1).await, + [ + (Command::RepairPrepare, 1), + (Command::RepairPrepare, 2), + (Command::RepairPrepare, 3), + (Command::RepairPrepare, 4), + (Command::RepairDone, 4), + ] + ); + } + + /// A read starts only after the pump took the bytes of the read before + /// it, so a busy pump holds the bodies of one round, not of every round + /// that waits. + #[compio::test] + async fn given_a_landed_read_when_the_pump_has_not_taken_it_should_start_no_other_read() { + const OTHER_REQUESTER: u8 = 0; + // Ample time for a second read of the fixture segment to land, if it + // could start before the take. + const SECOND_READ_WINDOW: Duration = Duration::from_millis(10); + let donor = Box::pin(SegmentDonor::new(4)).await; + connect_replica_on(&donor.owner.bus, OTHER_REQUESTER, 1); + donor.sent.borrow_mut().clear(); + let round = [ + (Command::RepairPrepare, 1), + (Command::RepairPrepare, 2), + (Command::RepairPrepare, 3), + (Command::RepairPrepare, 4), + (Command::RepairDone, 4), + ]; + let other = repair_request(donor.namespace, 1, donor.prepares.len()).transmute_header( + |old, header: &mut RequestPreparesHeader| { + *header = old; + header.replica = OTHER_REQUESTER; }, ); - partition.on_replicate(prepare.clone()).await; - assert!(partition.log.journal().inner.holds_op(NEW_HEAD)); - owner - .on_start_view(start_view_suffix( - namespace, - NEW_VIEW, - NEW_HEAD, - FIRST_HEAD, - &[*prepare.header()], - )) + donor + .owner + .on_request_prepares(&repair_request(donor.namespace, 1, donor.prepares.len())) .await; - let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap(); - assert_eq!(partition.consensus().commit_min(), FIRST_HEAD); - assert!(partition.log.journal().inner.holds_op(NEW_HEAD)); + donor.owner.on_request_prepares(&other).await; + + donor.owner.served_reads.landed().await; + donor.owner.bus.sleep(SECOND_READ_WINDOW).await; + assert_eq!( + donor.owner.served_reads.landed_count(), + 1, + "the second read waits for the pump to take the first" + ); + + donor.owner.settle_served_reads().await; + assert_eq!( + sent_answers(&donor.sent, &donor.prepares), + [round, round].concat() + ); + } + + /// A replay to a backup reads no released body that the replication + /// window refuses. A window with room for op 1 reads op 1 only, and a full + /// window reads nothing. The body of op 2 is corrupt, so a read of it + /// would drop ops 1..=2 from the ring. + #[compio::test] + async fn given_released_bodies_when_the_replication_window_refuses_them_should_not_read_them() { + const HEAD: u64 = 3; + let bus = Rc::new(IggyMessageBus::new(0)); + let replayed = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&replayed); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + let message = + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(); + if message.header().command == Command::Prepare { + let prepare: Message = message.try_into_typed().unwrap(); + captured.borrow_mut().push(prepare.header().op); + } + Ok(()) + })); + let (owner, _sender) = test_owner(&bus, None); + let namespace = IggyNamespace::new(0, 0, 0); + let (mut primary, prepares) = journaling_primary(&bus, namespace, HEAD, 1).await; + let directory = tempfile::tempdir().unwrap(); + let (log_path, _) = attach_segment_files(&mut primary, directory.path(), false).await; + let positions = release_bodies_to_segment(&primary, &log_path, &prepares); + let mut segment = std::fs::read(&log_path).unwrap(); + segment[usize::try_from(positions[&2]).unwrap()] ^= 0xFF; + std::fs::write(&log_path, &segment).unwrap(); + let partitions = owner.plane.partitions(); + partitions.remove(&namespace).unwrap(); + partitions.insert(namespace, primary); + connect_replica_on(&bus, REPAIR_BACKUP, 1); + let first = u64::from(prepares[0].header().size); + let incarnation = { + let consensus = partitions.get_by_ns(&namespace).unwrap().consensus(); + assert!(consensus.admit_replication(REPAIR_BACKUP, 1, first, ReplicationSend::Live)); + consensus.record_replication_failure(REPAIR_BACKUP, 1); + let _checkpoint = consensus.record_replication_send( + REPAIR_BACKUP, + 1, + REPLICATION_BATCH_BYTES_MAX - first, + ReplicationSend::Retransmit, + ); + consensus.partition_incarnation() + }; + let replay = |from_op| { + Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestPreparesHeader| { + header.command = Command::RequestPrepares; + header.size = u32::try_from(size_of::()).unwrap(); + header.replica = REPAIR_BACKUP; + header.group = namespace.inner(); + header.partition_incarnation = incarnation; + header.from_op = from_op; + header.to_op = HEAD; + }) + }; + let retained_from = || { + partitions + .get_by_ns(&namespace) + .unwrap() + .log + .journal() + .inner + .repair_retained_from() + }; + + assert_eq!( + owner.serve_prepares(&replay(1), true, None).await, + None, + "the body of op 1 is on disk" + ); + owner.settle_served_reads().await; + assert_eq!(*replayed.borrow(), [1], "the window must take op 1 only"); + assert_eq!( + retained_from(), + Some(1), + "the replay must not read the body of op 2 that the window refuses" + ); + assert_eq!(owner.serve_prepares(&replay(2), true, None).await, Some(1)); + assert_eq!(*replayed.borrow(), [1], "a full window must take nothing"); + assert_eq!( + retained_from(), + Some(1), + "a full window must not read a body" + ); } + /// A log file that cannot be opened for now fails the read with an error + /// that can pass, like a file descriptor limit. A link that points at + /// itself stands in for it, because a lower limit would reach the other + /// tests in the process. The round answers without the released ops and + /// keeps them, so a later round serves them. #[compio::test] - async fn given_evicted_commit_backlog_when_adopting_start_view_should_preserve_the_ring_floor() - { - const COMMITTED: u64 = 3; - const STAMPED: u64 = 5; - const NEW_HEAD: u64 = 7; - let bus = Rc::new(IggyMessageBus::new(0)); - let namespace = IggyNamespace::new(0, 0, 0); - let prepares = primary_prepares(&bus, namespace, NEW_HEAD).await; - let mut partition = - repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); - for prepare in prepares.iter().take(usize::try_from(STAMPED).unwrap()) { - let prepare = if prepare.header().op == STAMPED { - prepare - .clone() - .transmute_header(|previous, header: &mut PrepareHeader| { - *header = previous; - header.commit = STAMPED - 1; - header.checksum = header.identity_checksum(); - }) - } else { - prepare.clone() - }; - partition.on_replicate(prepare).await; - } - partition.consensus().advance_commit_max(COMMITTED); - partition - .log - .journal() - .inner - .evict_prefix(usize::try_from(COMMITTED).unwrap()); - assert_eq!(partition.consensus().commit_min(), 0); - let message = start_view_suffix( - namespace, - 3, - NEW_HEAD, - NEW_HEAD, - &[*prepares.last().unwrap().header()], - ); - partition.consensus().handle_start_view( - consensus::PlaneKind::Partitions, - message.header(), - &message.as_slice()[size_of::()..], + async fn given_a_log_file_that_cannot_be_opened_when_a_backup_requests_released_bodies_should_serve_them_once_it_opens() + { + let donor = Box::pin(SegmentDonor::new(3)).await; + let segment = std::fs::read(&donor.log_path).unwrap(); + std::fs::remove_file(&donor.log_path).unwrap(); + std::os::unix::fs::symlink(&donor.log_path, &donor.log_path).unwrap(); + assert_eq!(donor.answers(1).await, [(Command::RepairDone, 0)]); + + std::fs::remove_file(&donor.log_path).unwrap(); + std::fs::write(&donor.log_path, &segment).unwrap(); + assert_eq!( + donor.answers(1).await, + [ + (Command::RepairPrepare, 1), + (Command::RepairPrepare, 2), + (Command::RepairPrepare, 3), + (Command::RepairDone, 3), + ] ); - let pending = partition.consensus().pending_view_log(); + } + + /// A whole batch that passes its own checks, but sits where the ring + /// expects another body, is never served: here the body of op 3 written + /// over the body of op 2, which has the same length. + #[compio::test] + async fn given_another_batch_at_a_released_body_position_when_a_backup_requests_it_should_not_serve_it() + { + let donor = Box::pin(SegmentDonor::new(4)).await; + let body = |op: usize| &donor.prepares[op - 1].as_slice()[size_of::()..]; + assert_eq!(body(2).len(), body(3).len()); + let mut segment = std::fs::read(&donor.log_path).unwrap(); + let op_2_start = usize::try_from(donor.positions[&2]).unwrap(); + segment[op_2_start..op_2_start + body(3).len()].copy_from_slice(body(3)); + std::fs::write(&donor.log_path, &segment).unwrap(); + assert_eq!( - crate::partition_view_divergence_from(&partition, pending.as_ref()), - None + donor.answers(1).await, + [(Command::RepairPrepare, 1), (Command::RepairDone, 1)], + "the batch of op 3 is not served as op 2" ); - assert!( - partition - .log - .journal() - .inner - .repair_header(COMMITTED) - .is_some() + assert_eq!( + donor.answers(2).await, + [ + (Command::RangeEvicted, 3), + (Command::RepairPrepare, 3), + (Command::RepairPrepare, 4), + (Command::RepairDone, 4), + ] ); } + /// Released bodies that lie back to back in a segment file come back in + /// one read. A run ends at a gap in the file and at the end of its + /// segment, also where the next body starts at the same position in the + /// next file. Every body reaches the backup byte for byte. #[compio::test] - async fn given_pending_view_truncation_when_peer_requests_repair_should_not_serve_old_entries() - { - const HEAD: u64 = 4; + async fn given_released_bodies_in_runs_across_two_segments_when_a_backup_requests_them_should_serve_each_one() + { + const HEAD: u64 = 6; + const GAP: usize = 64; + const SECOND_SEGMENT_START: u64 = 1000; let bus = Rc::new(IggyMessageBus::new(0)); let sent = Rc::new(RefCell::new(Vec::new())); let captured = Rc::clone(&sent); bus.set_replica_forward_fn(Box::new(move |_, _, frame| { - captured.borrow_mut().push(frame); + captured.borrow_mut().push( + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(), + ); Ok(()) })); - crate::test_support::connect_replica_on(&bus, REPAIR_PRIMARY, 1); + connect_replica_on(&bus, REPAIR_PRIMARY, 1); let (owner, _sender) = test_owner(&bus, None); let namespace = IggyNamespace::new(0, 0, 0); let prepares = primary_prepares(&bus, namespace, HEAD).await; - let mut partition = + let mut donor = repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); - for prepare in prepares { - partition.on_replicate(prepare).await; + donor.repair = None; + let directory = tempfile::tempdir().unwrap(); + let (first_path, _) = attach_segment_files(&mut donor, directory.path(), false).await; + for prepare in &prepares { + donor.on_replicate(prepare.clone()).await; } + let second_path = + attach_next_segment_file(&mut donor, directory.path(), SECOND_SEGMENT_START).await; + + // Ops 1 and 2 back to back, a gap, then op 3 in the first file. Ops 4 + // and 5 back to back in the second file, op 4 where op 3 ends. + let body = |op: u64| { + &prepares[usize::try_from(op - 1).unwrap()].as_slice()[size_of::()..] + }; + let mut first = Vec::new(); + let mut second = Vec::new(); + let mut positions = BTreeMap::new(); + for op in 1..=2 { + positions.insert(op, (0, first.len())); + first.extend_from_slice(body(op)); + } + first.resize(first.len() + GAP, u8::MAX); + positions.insert(3, (0, first.len())); + first.extend_from_slice(body(3)); + second.resize(first.len(), 0); + for op in 4..=5 { + positions.insert(op, (SECOND_SEGMENT_START, second.len())); + second.extend_from_slice(body(op)); + } + std::fs::write(&first_path, &first).unwrap(); + std::fs::write(&second_path, &second).unwrap(); + release_bodies(&donor, &prepares, |op| { + positions + .get(&op) + .map(|&(segment_start, position)| partitions::BodyPosition { + segment_start, + position: u64::try_from(position).unwrap(), + }) + }); owner.plane.partitions().remove(&namespace).unwrap(); - owner.plane.partitions().insert(namespace, partition); - let request = Message::::new(size_of::< - iggy_binary_protocol::RequestPreparesHeader, - >()) - .transmute_header( - |_, header: &mut iggy_binary_protocol::RequestPreparesHeader| { + owner.plane.partitions().insert(namespace, donor); + + assert_eq!( + repair_answers(&owner, &sent, namespace, 1, &prepares).await, + (1..=HEAD) + .map(|op| (Command::RepairPrepare, op)) + .chain([(Command::RepairDone, HEAD)]) + .collect::>() + ); + } + + /// Asks `owner` for ops `from_op..` of `prepares` as the primary of + /// [`REPAIR_VIEW`], serves the reads that land, and returns what + /// [`sent_answers`] finds. + #[allow(clippy::future_not_send)] + async fn repair_answers( + owner: &IoTestShard, + sent: &RefCell>>, + namespace: IggyNamespace, + from_op: u64, + prepares: &[Message], + ) -> Vec<(Command, u64)> { + sent.borrow_mut().clear(); + owner + .on_request_prepares(&repair_request(namespace, from_op, prepares.len())) + .await; + owner.settle_served_reads().await; + sent_answers(sent, prepares) + } + + /// A request of [`REPAIR_PRIMARY`] in [`REPAIR_VIEW`] for ops + /// `from_op..=head`. + fn repair_request( + namespace: IggyNamespace, + from_op: u64, + head: usize, + ) -> Message { + Message::::new(size_of::()).transmute_header( + |_, header: &mut RequestPreparesHeader| { header.command = Command::RequestPrepares; header.cluster = 1; header.group = namespace.inner(); header.replica = REPAIR_PRIMARY; - header.from_op = 3; - header.to_op = HEAD; - header.size = u32::try_from(std::mem::size_of_val(header)).unwrap(); + header.view = REPAIR_VIEW; + header.from_op = from_op; + header.to_op = u64::try_from(head).unwrap(); + header.size = u32::try_from(size_of::()).unwrap(); header.seal(); }, - ); - sent.borrow_mut().clear(); - owner.on_request_prepares(&request).await; - assert!( - !sent.borrow().is_empty(), - "the unblocked replica serves this range" - ); - sent.borrow_mut().clear(); - owner - .plane - .partitions() - .get_mut_by_ns(&namespace) - .unwrap() - .defer_view_truncation(3..=u64::MAX, Vec::new(), None); - owner.on_request_prepares(&request).await; - assert!( - sent.borrow().is_empty(), - "unproven entries must not reach a peer" - ); + ) + } + + /// Takes the frames in `sent`, and returns the command and op of each. + /// Each `RepairPrepare` must carry its prepare from `prepares` verbatim. + fn sent_answers( + sent: &RefCell>>, + prepares: &[Message], + ) -> Vec<(Command, u64)> { + const COMMAND_OFFSET: usize = std::mem::offset_of!(GenericHeader, command); + sent.borrow_mut() + .drain(..) + .map(|message| { + let command = message.header().command; + if command != Command::RepairPrepare { + let reply: Message = message.try_into_typed().unwrap(); + return (command, reply.header().op); + } + let prepare: Message = + message.try_into_typed().unwrap(); + let op = prepare.header().0.op; + let mut expected = prepares[usize::try_from(op).unwrap() - 1] + .as_slice() + .to_vec(); + expected[COMMAND_OFFSET] = Command::RepairPrepare as u8; + assert_eq!( + prepare.as_slice(), + expected, + "op {op} reaches the backup as journaled" + ); + (command, op) + }) + .collect() } #[compio::test] - async fn given_abandoned_prepares_when_a_backup_requests_repair_should_serve_only_committed_ops() + async fn given_large_resident_repairs_when_serving_should_bound_bytes_and_allow_one_oversized_frame() { - const REPLICAS: u8 = 5; - const DONOR: u8 = 3; - const BACKUP: u8 = 4; - const NEXT_VIEW: u32 = 2; - const NEXT_PRIMARY: u8 = 2; - const COMMITTED: u64 = 2; const HEAD: u64 = 3; - let bus = Rc::new(IggyMessageBus::new(0)); - let sent = Rc::new(RefCell::new(Vec::new())); - let captured = Rc::clone(&sent); - bus.set_replica_forward_fn(Box::new(move |_, _, frame| { - let message = - Message::::try_from(Owned::copy_from_slice(frame.as_slice())) - .unwrap(); - if message.header().command == Command::RepairPrepare { - let prepare = Message::::try_from( - Owned::copy_from_slice(frame.as_slice()), - ) + for payload_bytes in [4 * 1024 * 1024, 12 * 1024 * 1024] { + let bus = Rc::new(IggyMessageBus::new(0)); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + captured.borrow_mut().push( + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(), + ); + Ok(()) + })); + connect_replica_on(&bus, REPAIR_PRIMARY, 1); + let (owner, _sender) = test_owner(&bus, None); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = sized_primary_prepares(&bus, namespace, HEAD, payload_bytes).await; + let mut donor = + repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); + donor.repair = None; + for prepare in &prepares { + donor.on_replicate(prepare.clone()).await; + } + owner.plane.partitions().remove(&namespace).unwrap(); + owner.plane.partitions().insert(namespace, donor); + + let mut from_op = 1; + while from_op <= HEAD { + let answers = repair_answers(&owner, &sent, namespace, from_op, &prepares).await; + let served: Vec<_> = answers + .iter() + .filter(|(command, _)| *command == Command::RepairPrepare) + .map(|(_, op)| *op) + .collect(); + assert!( + !served.is_empty(), + "a legal oversized frame must make progress" + ); + let bytes: usize = served + .iter() + .map(|op| prepares[usize::try_from(op - 1).unwrap()].total_len()) + .sum(); + assert!( + bytes <= usize::try_from(crate::REPAIR_BYTES_MAX).unwrap() || served.len() == 1, + "resident response exceeded the byte budget: {bytes} bytes, ops {served:?}" + ); + let through = *served.last().unwrap(); + assert_eq!(answers.last(), Some(&(Command::RepairDone, through))); + assert_eq!(served, (from_op..=through).collect::>()); + from_op = through + 1; + } + } + } + + /// A primary whose evicted ring released bodies of 1 MiB to their segment + /// reads at most `REPAIR_BYTES_MAX` of them per round, from the + /// bottom of the window. A backup that adopted the view walks its unproven + /// log down from the hole below the adopted head. Each window starts above + /// the bodies the backup holds below the hole, and a round that brought + /// such bodies pulls the next window at once, so the walk reaches the hole + /// and proves the log without a state transfer. + #[compio::test] + async fn given_released_bodies_past_the_disk_budget_when_a_backup_walks_down_should_reach_the_hole() + { + const HEAD: u64 = 40; + const BATCH_BYTES: usize = 1024 * 1024; + const NEW_VIEW: u32 = REPAIR_VIEW + 3; + const EXCHANGES_MAX: usize = 32; + let primary_bus = Rc::new(IggyMessageBus::new(0)); + let backup_bus = Rc::new(IggyMessageBus::new(0)); + let served = capture_replica_frames(&primary_bus); + let requested = capture_replica_frames(&backup_bus); + connect_replica_on(&primary_bus, REPAIR_BACKUP, 1); + connect_replica_on(&backup_bus, REPAIR_PRIMARY, 1); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = sized_primary_prepares(&primary_bus, namespace, HEAD, BATCH_BYTES).await; + served.borrow_mut().clear(); + + let mut consensus = VsrConsensus::new( + 1, + REPAIR_PRIMARY, + 3, + namespace.inner(), + Rc::clone(&primary_bus), + LocalPipeline::new(), + ); + consensus.set_view(NEW_VIEW); + consensus.set_log_view(NEW_VIEW); + consensus.init(); + consensus.advance_commit_max(HEAD); + let mut primary = IggyPartition::with_in_memory_storage( + Arc::new(PartitionStats::default()), + consensus, + IggyByteSize::from(SEGMENT_BYTES), + ); + let directory = tempfile::tempdir().unwrap(); + let (log_path, _) = attach_segment_files(&mut primary, directory.path(), false).await; + for prepare in &prepares { + primary + .log + .journal() + .inner + .append(prepare.clone().into_frozen()) + .await .unwrap(); - captured.borrow_mut().push(prepare.header().0); + } + release_bodies_to_segment(&primary, &log_path, &prepares); + let (primary_owner, _primary_sender) = test_owner(&primary_bus, None); + primary_owner.plane.partitions().remove(&namespace).unwrap(); + primary_owner.plane.partitions().insert(namespace, primary); + + let (backup_owner, _backup_sender) = test_owner(&backup_bus, None); + backup_owner.set_repair_retry_ticks(1); + let mut backup = repairing_replica( + &backup_bus, + namespace, + (REPAIR_BACKUP, 3), + REPAIR_PRIMARY, + 0, + 0, + ); + backup.repair = None; + let partitions = backup_owner.plane.partitions(); + partitions.remove(&namespace).unwrap(); + partitions.insert(namespace, backup); + backup_owner + .on_start_view(start_view_suffix( + namespace, + NEW_VIEW, + HEAD, + HEAD, + &[*prepares.last().unwrap().header()], + )) + .await; + let backup = partitions.get_by_ns(&namespace).unwrap(); + let hole = backup.consensus().view_log_repair_op().unwrap(); + assert!(!backup.consensus().view_log_entry_proven(hole)); + + let mut stall_ticks = 0; + for _ in 0..EXCHANGES_MAX { + if !backup.consensus().view_log_unproven() { + break; } + let requests: Vec<_> = requested + .borrow_mut() + .drain(..) + .filter(|(_, frame)| frame.header().command == Command::RequestPrepares) + .collect(); + if requests.is_empty() { + stall_ticks += 1; + assert!( + backup_owner + .tick_partitions(&mut Vec::new()) + .await + .is_none() + ); + continue; + } + for (peer, request) in requests { + assert_eq!(peer, REPAIR_PRIMARY, "the walk left the primary"); + primary_owner + .on_request_prepares(&request.try_into_typed().unwrap()) + .await; + primary_owner.settle_served_reads().await; + } + let frames: Vec<_> = served.borrow_mut().drain(..).collect(); + for (_, frame) in frames { + match frame.header().command { + Command::RepairPrepare => { + backup_owner + .on_repair_prepare(frame.try_into_typed().unwrap()) + .await; + } + command => { + assert_ne!(command, Command::RangeEvicted, "the primary holds every op"); + backup_owner + .on_repair_range_reply(&frame.try_into_typed().unwrap()) + .await; + } + } + } + } + assert!( + !backup.consensus().view_log_unproven(), + "the walk never reached its hole at op {hole}" + ); + assert_eq!( + stall_ticks, 0, + "a round that brought ops below the hole pulls the next window at once" + ); + assert!(backup.log.journal().inner.holds_op(hole)); + assert!(backup.transfer.is_none()); + assert_eq!( + backup.consensus().state_transfer_stage(), + consensus::StateTransferStage::Idle + ); + } + + /// Frames a shard sent to claimed replicas, each with its replica. + type ReplicaFrames = Rc)>>>; + + /// Every frame the shard on `bus` sends to a claimed replica, with that + /// replica. + fn capture_replica_frames(bus: &IggyMessageBus) -> ReplicaFrames { + let frames = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&frames); + bus.set_replica_forward_fn(Box::new(move |replica, _, frame| { + captured.borrow_mut().push(( + replica, + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(), + )); Ok(()) })); - for requester in [BACKUP, NEXT_PRIMARY] { - crate::test_support::connect_replica_on(&bus, requester, 1); - } - let (owner, _sender) = test_owner(&bus, None); - let namespace = IggyNamespace::new(0, 0, 0); - let prepares = primary_prepares(&bus, namespace, HEAD).await; - let mut donor = repairing_replica(&bus, namespace, (DONOR, REPLICAS), REPAIR_PRIMARY, 0, 0); - donor.repair = None; - for prepare in prepares { - donor.on_replicate(prepare).await; + frames + } + + /// A primary of [`REPAIR_VIEW`] that journaled ops `1..=head` and released + /// every body but the newest one to the log file of its first segment. + struct SegmentDonor { + owner: IoTestShard, + sent: Rc>>>, + namespace: IggyNamespace, + prepares: Vec>, + log_path: PathBuf, + /// The position of each released body in the log file. + positions: BTreeMap, + sender: TaggedSender, + _directory: tempfile::TempDir, + } + + impl SegmentDonor { + #[allow(clippy::future_not_send)] + async fn new(head: u64) -> Self { + let bus = Rc::new(IggyMessageBus::new(0)); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + captured.borrow_mut().push( + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(), + ); + Ok(()) + })); + connect_replica_on(&bus, REPAIR_PRIMARY, 1); + let (owner, sender) = test_owner(&bus, None); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = primary_prepares(&bus, namespace, head).await; + let mut donor = + repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); + donor.repair = None; + let directory = tempfile::tempdir().unwrap(); + let (log_path, _) = attach_segment_files(&mut donor, directory.path(), false).await; + for prepare in &prepares { + donor.on_replicate(prepare.clone()).await; + } + let positions = release_bodies_to_segment(&donor, &log_path, &prepares); + owner.plane.partitions().remove(&namespace).unwrap(); + owner.plane.partitions().insert(namespace, donor); + Self { + owner, + sent, + namespace, + prepares, + log_path, + positions, + sender, + _directory: directory, + } } - donor.consensus().advance_commit_max(COMMITTED); - donor - .commit_journal(owner.plane.partitions().config()) - .await; - let change = - view_control_message(namespace, Command::StartViewChange, NEXT_VIEW, NEXT_PRIMARY); - let MessageBag::StartViewChange(change) = change else { - unreachable!() - }; - donor - .consensus() - .handle_start_view_change(consensus::PlaneKind::Partitions, change.header()); - assert_eq!(donor.consensus().status(), consensus::Status::ViewChange); - owner.plane.partitions().remove(&namespace).unwrap(); - owner.plane.partitions().insert(namespace, donor); - for (requester, view, through) in [ - (BACKUP, NEXT_VIEW, COMMITTED), - (NEXT_PRIMARY, NEXT_VIEW, HEAD), - (NEXT_PRIMARY, REPAIR_VIEW, COMMITTED), - ] { - let request = Message::::new(size_of::()) - .transmute_header(|_, header: &mut RequestPreparesHeader| { - header.command = Command::RequestPrepares; - header.cluster = 1; - header.group = namespace.inner(); - header.replica = requester; - header.view = view; - header.from_op = 1; - header.to_op = HEAD; - header.size = u32::try_from(size_of::()).unwrap(); - header.seal(); - }); - sent.borrow_mut().clear(); - owner.on_request_prepares(&request).await; - let served: Vec<_> = sent.borrow().iter().map(|header| header.op).collect(); - assert_eq!( - served, - (1..=through).collect::>(), - "requester {requester} in view {view}" - ); + /// See [`repair_answers`]. + #[allow(clippy::future_not_send)] + async fn answers(&self, from_op: u64) -> Vec<(Command, u64)> { + repair_answers( + &self.owner, + &self.sent, + self.namespace, + from_op, + &self.prepares, + ) + .await } } + /// Writes the bodies of `prepares`, which `donor` journaled, back to back + /// into the log file of the first segment at `log_path`, and releases + /// them there with [`release_bodies`]. Returns the position of each body + /// in the file. + fn release_bodies_to_segment( + donor: &IggyPartition, HeldSuperblock>, + log_path: &Path, + prepares: &[Message], + ) -> BTreeMap { + let mut segment = Vec::new(); + let mut positions = BTreeMap::new(); + for prepare in prepares { + positions.insert(prepare.header().op, u64::try_from(segment.len()).unwrap()); + segment.extend_from_slice(&prepare.as_slice()[size_of::()..]); + } + std::fs::write(log_path, &segment).unwrap(); + release_bodies(donor, prepares, |op| { + positions + .get(&op) + .map(|&position| partitions::BodyPosition { + segment_start: 0, + position, + }) + }); + positions + } + + /// Moves `prepares`, which `donor` journaled, into its evicted ring with + /// every body but the newest one released to the place `locate` names. + fn release_bodies( + donor: &IggyPartition, HeldSuperblock>, + prepares: &[Message], + locate: impl Fn(u64) -> Option, + ) { + let journal = &donor.log.journal().inner; + // Room for the body of the newest op only. + journal.set_ring_caps( + partitions::EVICTED_RING_CAPACITY, + u64::try_from(prepares.last().unwrap().as_slice().len()).unwrap(), + ); + journal.evict_prefix_located(prepares.len(), locate); + } + /// Seven replicas: the requester committed ops 1..=2 in view 1, adopted view 2 /// at op 2 and learned commit 3 from view 2's primary. A stale donor serves the /// op 3 view 1 prepared, which view 2 replaced. Nothing the requester holds @@ -7386,6 +9935,7 @@ mod tests { let session = partition.repair.unwrap(); assert_eq!(session.peer, PRIMARY); assert_ne!(session.nonce, OLD_NONCE); + // The branch's op 4 is held below the hole, so the window starts above it. let request = requests.borrow_mut().pop().expect("a re-request"); assert_eq!( ( @@ -7394,7 +9944,7 @@ mod tests { request.1.from_op, request.1.to_op ), - (PRIMARY, session.nonce, FORK, HEAD - 1) + (PRIMARY, session.nonce, HEAD - 1, HEAD - 1) ); owner .on_repair_range_reply(&repair_done(namespace, OLD_PEER, OLD_NONCE, HEAD - 1)) @@ -7961,6 +10511,14 @@ mod tests { fn test_owner( bus: &B, store: Option<&Rc>, + ) -> (IoTestShard, TaggedSender) { + test_owner_with_repair_memory(bus, store, None) + } + + fn test_owner_with_repair_memory( + bus: &B, + store: Option<&Rc>, + repair_memory: Option>, ) -> (IoTestShard, TaggedSender) { let shard_id = ShardId::new(0); let config = PartitionsConfig { @@ -7972,7 +10530,10 @@ mod tests { key_context: Arc::default(), path_layout: PartitionPathLayout::default(), }; - let partitions = IggyPartitions::new(shard_id, config); + let partitions = match repair_memory { + Some(budget) => IggyPartitions::with_repair_memory(shard_id, config, 0, budget), + None => IggyPartitions::new(shard_id, config), + }; let routes = PapayaShardsTable::new(); let mut inner = StreamsInner::default(); let mut stream = Stream::default(); @@ -8103,6 +10664,33 @@ mod tests { namespace: IggyNamespace, ops: u64, ) -> Vec> { + sized_primary_prepares(bus, namespace, ops, 1).await + } + + /// [`primary_prepares`] whose one message per send carries `payload_len` + /// bytes. + #[allow(clippy::future_not_send)] + async fn sized_primary_prepares( + bus: &Rc, + namespace: IggyNamespace, + ops: u64, + payload_len: usize, + ) -> Vec> { + journaling_primary(bus, namespace, ops, payload_len).await.1 + } + + /// The primary of [`REPAIR_VIEW`] after it journaled the first `ops` sends + /// of `payload_len` bytes each, with a copy of each prepare. + #[allow(clippy::future_not_send)] + async fn journaling_primary( + bus: &Rc, + namespace: IggyNamespace, + ops: u64, + payload_len: usize, + ) -> ( + IggyPartition, HeldSuperblock>, + Vec>, + ) { let mut consensus = VsrConsensus::new( 1, REPAIR_PRIMARY, @@ -8121,7 +10709,7 @@ mod tests { ); let mut prepares = Vec::new(); for op in 1..=ops { - origin.on_request(send_request(namespace, op), None).await; + Box::pin(origin.on_request(sized_send_request(namespace, op, payload_len), None)).await; let frozen = origin.log.journal().inner.repair_entry(op).unwrap(); let mut prepare = Message::::new(frozen.len()); prepare.as_mut_slice().copy_from_slice(frozen.as_slice()); @@ -8130,7 +10718,154 @@ mod tests { origin.consensus().clear_pipeline(); origin.consensus().drain_loopback_into(&mut Vec::new()); } - prepares + (origin, prepares) + } + + const SCANNING_INCARNATION: u64 = 7; + const SCANNING_NONCE: u128 = 7; + const SCANNING_RETRY_TICKS: u32 = 2; + const SCANNING_OFFSETS: &[u8] = b"offsets"; + + /// A receiver whose session waits for a reuse scan before it pulls a + /// segment and an offsets artifact from `REPAIR_PRIMARY`. + struct ScanningReceiver { + owner: IoTestShard, + _sender: TaggedSender, + /// `(artifact, offset, len)` of each chunk request that it sends. + requests: Rc>>, + offer_requests: Rc>>, + namespace: IggyNamespace, + segment: Vec, + directory: tempfile::TempDir, + } + + impl ScanningReceiver { + fn chunk(&self, artifact: u32, payload: &[u8]) -> Message { + let size = size_of::() + payload.len(); + let mut chunk = Message::::new(size); + chunk.as_mut_slice()[size_of::()..].copy_from_slice(payload); + chunk.transmute_header(|_, header: &mut StateChunkHeader| { + header.command = Command::StateChunk; + header.size = u32::try_from(size).unwrap(); + header.cluster = 1; + header.group = self.namespace.inner(); + header.replica = REPAIR_PRIMARY; + header.partition_incarnation = SCANNING_INCARNATION; + header.nonce = SCANNING_NONCE; + header.artifact = artifact; + header.seal(); + }) + } + } + + fn scanning_receiver() -> ScanningReceiver { + let bus = Rc::new(IggyMessageBus::new(0)); + let requests = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&requests); + let offer_requests = Rc::new(RefCell::new(Vec::new())); + let captured_offers = Rc::clone(&offer_requests); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + let message = + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(); + if message.header().command == Command::RequestStateChunk { + let request = Message::::try_from(Owned::copy_from_slice( + frame.as_slice(), + )) + .unwrap(); + let header = request.header(); + captured + .borrow_mut() + .push((header.artifact, header.offset, header.len)); + } else if message.header().command == Command::RequestStateTransfer { + let request = Message::::try_from( + Owned::copy_from_slice(frame.as_slice()), + ) + .unwrap(); + let header = request.header(); + captured_offers.borrow_mut().push(( + header.group, + header.partition_incarnation, + header.nonce, + )); + } + Ok(()) + })); + connect_replica_on(&bus, REPAIR_PRIMARY, 1); + let (owner, sender) = test_owner(&bus, None); + owner.set_repair_retry_ticks(SCANNING_RETRY_TICKS); + let namespace = IggyNamespace::new(0, 0, 0); + let directory = tempfile::tempdir().unwrap(); + let segment = send_request(namespace, 1).body().to_vec(); + let partitions = owner.plane.partitions(); + partitions.remove(&namespace).unwrap(); + let mut partition = + repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); + partition.repair = None; + partition.set_created_revision(SCANNING_INCARNATION); + partition.set_partition_dir(directory.path().to_str().unwrap().to_owned()); + partition + .consensus() + .set_state_transfer_stage(StateTransferStage::AwaitingTarget); + partition + .consensus() + .set_state_transfer_stage(StateTransferStage::Fetching); + partition.transfer = Some(PartitionTransferSession { + nonce: SCANNING_NONCE, + peer: REPAIR_PRIMARY, + commit_op: 0, + artifacts: vec![ + TransferArtifact::Pending(ArtifactProgress { + entry: StateArtifact::for_bytes(artifact_kind::SEGMENT_LOG, 0, &segment), + buf: Vec::new(), + }), + TransferArtifact::Pending(ArtifactProgress { + entry: StateArtifact::for_bytes( + artifact_kind::CONSUMER_OFFSETS, + 0, + SCANNING_OFFSETS, + ), + buf: Vec::new(), + }), + ], + target_accepted: true, + idle_ticks: 0, + spill: None, + scan: Some(LocalWork::default()), + seed: None, + pulled_len: 0, + }); + partitions.insert(namespace, partition); + ScanningReceiver { + owner, + _sender: sender, + requests, + offer_requests, + namespace, + segment, + directory, + } + } + + #[allow(clippy::future_not_send)] + async fn settle_transfer_spill(owner: &IoTestShard, namespace: IggyNamespace) { + const TURNS_MAX: usize = 1024; + for _ in 0..TURNS_MAX { + let pending = owner + .plane + .partitions() + .get_by_ns(&namespace) + .and_then(|partition| partition.transfer.as_ref()) + .is_some_and(|session| session.spill.is_some()); + if !pending { + return; + } + compio::runtime::time::sleep(Duration::from_millis(1)).await; + owner + .on_partition_transfer_progress(namespace.inner()) + .await; + } + panic!("the queued spill did not settle"); } /// Run `rounds` stall rounds, one per sweep, and record after each the @@ -8200,6 +10935,35 @@ mod tests { (log_path, index_path) } + /// Adds a segment that starts at `start_offset` behind the ones of + /// `partition`, backed by a fresh log file in `directory`, and returns the + /// path of that file. + #[allow(clippy::future_not_send)] + async fn attach_next_segment_file( + partition: &mut IggyPartition, HeldSuperblock>, + directory: &Path, + start_offset: u64, + ) -> PathBuf { + let log_path = directory.join(format!("{start_offset:020}.log")); + let index_path = directory.join(format!("{start_offset:020}.index")); + let storage = SegmentStorage::new( + log_path.to_str().unwrap(), + index_path.to_str().unwrap(), + 0, + 0, + false, + ) + .await + .unwrap(); + partition.log.add_persisted_segment( + Segment::new(start_offset, IggyByteSize::from(SEGMENT_BYTES)), + storage, + None, + None, + ); + log_path + } + fn submit( sender: &TaggedSender, namespace: IggyNamespace, @@ -8218,14 +10982,23 @@ mod tests { /// Each request comes from its own client, because a client has one request /// in flight per partition. fn send_request(namespace: IggyNamespace, request: u64) -> Message { + sized_send_request(namespace, request, 1) + } + + /// [`send_request`] whose message carries `payload_len` bytes. + fn sized_send_request( + namespace: IggyNamespace, + request: u64, + payload_len: usize, + ) -> Message { let mut messages = IggyMessages::with_capacity(1); messages.push(IggyMessage { header: IggyMessageHeader { id: u128::from(request), - payload_length: 1, + payload_length: u32::try_from(payload_len).unwrap(), ..Default::default() }, - payload: vec![1].into(), + payload: vec![1; payload_len].into(), user_headers: None, }); SendMessagesOwned::from_messages(namespace, &messages) @@ -8242,6 +11015,144 @@ mod tests { .unwrap() } + /// Incarnation of the groups that a [`serving_shard`] hosts. + const SERVED_INCARNATION: u64 = 7; + /// The replica that asks a [`serving_shard`] for every transfer. + const TRANSFER_REQUESTER: u8 = 1; + + /// A shard whose primaries of `namespaces` each hold one committed op in + /// segment files under `directory`. Its I/O owner pins their transfer + /// plans, so a transfer request parks until an I/O turn pins its plan. + #[allow(clippy::future_not_send)] + async fn serving_shard( + bus: &Rc, + namespaces: &[IggyNamespace], + directory: &Path, + ) -> (IoTestShard, TaggedSender) { + connect_replica_on(bus, TRANSFER_REQUESTER, 1); + let (mut owner, sender) = test_owner(bus, None); + owner.partition_consensus.replica_count = TRANSFER_REQUESTER + 1; + let partitions = owner.plane.partitions(); + for namespace in namespaces { + let directory = directory.join(namespace.inner().to_string()); + std::fs::create_dir(&directory).unwrap(); + let partition = partitions.get_mut_by_ns(namespace).unwrap(); + partition.set_created_revision(SERVED_INCARNATION); + attach_segment_files(partition, &directory, false).await; + let request = send_request(*namespace, 1).transmute_header( + |old, header: &mut RoutedRequestHeader| { + *header = old; + header.partition_incarnation = SERVED_INCARNATION; + }, + ); + let (reply, replied) = consensus::oneshot_channel(); + partitions.on_request_with_reply(request, Some(reply)).await; + owner + .process_loopback(&mut crate::router::LoopbackRound::default()) + .await; + assert_eq!(replied.await.unwrap().header().status, 0); + } + partitions.set_io_notifier( + owner.partition_io.notifier(), + owner.partition_io.limits.bytes_max(), + ); + (owner, sender) + } + + /// A `RequestStateTransfer` for `namespace` from [`TRANSFER_REQUESTER`]. + fn request_state_transfer( + namespace: IggyNamespace, + nonce: u128, + partition_incarnation: u64, + ) -> MessageBag { + MessageBag::RequestStateTransfer( + Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestStateTransferHeader| { + header.command = Command::RequestStateTransfer; + header.size = u32::try_from(size_of::()).unwrap(); + header.cluster = 1; + header.group = namespace.inner(); + header.replica = TRANSFER_REQUESTER; + header.partition_incarnation = partition_incarnation; + header.nonce = nonce; + header.seal(); + }), + ) + } + + /// `available` and `unavailable_transient` of a transfer target that + /// carries an offer. + const OFFERED: (u8, u8) = (1, 0); + /// `available` and `unavailable_transient` of a transient refusal. + const REFUSED: (u8, u8) = (0, 1); + /// Ticks that a parked request waits for its plan pin at the default + /// repair-retry interval. + const PARKED_WAIT_TICKS: u32 = + partitions::REPAIR_RETRY_TICKS / crate::PARKED_TRANSFER_REQUEST_WAIT_DIVISOR; + + /// `(group, nonce, (available, unavailable_transient))` of each + /// `StateTransferTarget` that a bus forwarded. + type RecordedTargets = Rc>>; + + /// Records each `StateTransferTarget` that `bus` forwards. + fn record_transfer_targets(bus: &IggyMessageBus) -> RecordedTargets { + let targets = Rc::new(RefCell::new(Vec::new())); + let recorded = Rc::clone(&targets); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + let message = + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(); + if message.header().command == Command::StateTransferTarget { + let target = Message::::try_from( + Owned::copy_from_slice(frame.as_slice()), + ) + .unwrap(); + let header = target.header(); + recorded.borrow_mut().push(( + header.group, + header.nonce, + (header.available, header.unavailable_transient), + )); + } + Ok(()) + })); + targets + } + + fn parked_nonce(owner: &IoTestShard, namespace: IggyNamespace) -> Option { + owner + .parked_partition_transfer_requests + .borrow() + .get(&(namespace.inner(), TRANSFER_REQUESTER)) + .map(|parked| parked.header.nonce) + } + + /// Runs I/O turns until the transfer plans of `namespaces` are pinned. A + /// pin waits for one flush, so it gets the [`REPLY_DEADLINE`] that these + /// tests give one write, also on a loaded host. + #[allow(clippy::future_not_send)] + async fn pin_transfer_plans(owner: &IoTestShard, namespaces: &[IggyNamespace]) { + let pinned = || { + namespaces.iter().all(|namespace| { + owner + .plane + .partitions() + .get_by_ns(namespace) + .is_some_and(IggyPartition::transfer_plan_pinned) + }) + }; + let pin = async { + while !pinned() { + owner.service_partition_io().await; + compio::time::sleep(TASK_POLL_INTERVAL).await; + } + }; + assert!( + compio::time::timeout(REPLY_DEADLINE, pin).await.is_ok(), + "the I/O turns must pin every transfer plan" + ); + } + fn view_control_message( namespace: IggyNamespace, command: Command, diff --git a/core/shard/src/poll/completion_tests.rs b/core/shard/src/poll/completion_tests.rs index 22c371c572..f4e4807d21 100644 --- a/core/shard/src/poll/completion_tests.rs +++ b/core/shard/src/poll/completion_tests.rs @@ -2173,6 +2173,10 @@ where artifacts: Vec::new(), target_accepted: false, idle_ticks: 0, + spill: None, + scan: None, + seed: None, + pulled_len: 0, }); } diff --git a/core/shard/src/primary_rebalance.rs b/core/shard/src/primary_rebalance.rs index f594ffee55..3f503bee2a 100644 --- a/core/shard/src/primary_rebalance.rs +++ b/core/shard/src/primary_rebalance.rs @@ -53,6 +53,14 @@ const STABLE_INTERVAL: Duration = Duration::from_secs(5); const GRANT_TIMEOUT: Duration = Duration::from_secs(1); const TRANSFER_TIMEOUT: Duration = Duration::from_secs(30); const PROBES_PER_TICK: usize = 8; +/// A moved primary that keeps its partition this long counts as held. A backup +/// that cannot follow the target loses the view inside it: one view-change +/// status timeout to give up on the view, one more to escalate past it. Sized +/// for the default 5 s timeouts. Also the first wait after a failed move. +const MOVE_HOLD_INTERVAL: Duration = Duration::from_secs(30); +/// Caps the wait after repeated failed moves, so a target that recovered gets +/// primaries back within minutes. +const TARGET_BACKOFF_MAX: Duration = Duration::from_secs(240); #[cfg(test)] mod tests; @@ -70,6 +78,40 @@ pub struct PrimaryRebalance { /// When this node's partition admission last opened under this authority. /// A probe sent before then carried no fresh quorum evidence. admission_opened: Option, + /// Moves a source carried out, watched until the target holds the + /// partition for `MOVE_HOLD_INTERVAL` or loses it to a view change. + moves: Vec, + /// Per replica. A target ready for a move can still be one that a backup + /// cannot hear in time, and only a lost view shows it. Moving to it again + /// at once would repeat the view changes forever. + backoffs: Vec, +} + +struct MovedPrimary { + plan: PartitionPrimaryTransfer, + since: Instant, +} + +#[derive(Clone, Copy, Default)] +struct TargetBackoff { + until: Option, + wait: Duration, +} + +impl TargetBackoff { + fn blocks(self, now: Instant) -> bool { + self.until.is_some_and(|until| now < until) + } + + /// Moves watched together often fail together, for example when the target + /// restarts, so a failure inside a running wait counts once. + fn record_failure(&mut self, now: Instant) { + if self.blocks(now) { + return; + } + self.wait = (self.wait * 2).clamp(MOVE_HOLD_INTERVAL, TARGET_BACKOFF_MAX); + self.until = Some(now + self.wait); + } } #[derive(Clone, Default)] @@ -230,6 +272,8 @@ impl Default for PrimaryRebalance { pending: None, received_probe: None, admission_opened: None, + moves: Vec::new(), + backoffs: Vec::new(), } } } @@ -299,6 +343,7 @@ where replicas ], eligible: vec![false; replicas], + backoffs: vec![TargetBackoff::default(); replicas], stable_since: now, next_move: now, received_probe: state.received_probe, @@ -309,10 +354,13 @@ where let admission_opened = state.admission_opened; for replica in 0..consensus.replica_count() { let index = usize::from(replica); - let ready = replica == consensus.replica() - || state.peers[index] + let ready = if replica == consensus.replica() { + !metadata.applied_frontier().has_partition_recovery() + } else { + state.peers[index] .ready_at - .is_some_and(|ready| now.duration_since(ready) <= PROBE_TIMEOUT); + .is_some_and(|ready| now.duration_since(ready) <= PROBE_TIMEOUT) + }; if state.eligible[index] != ready { state.eligible[index] = ready; changed = true; @@ -363,16 +411,61 @@ where MovePhase::Transfer => TRANSFER_TIMEOUT, }; if invalid || expired { + // The planned view ended after the grant, so watch whether + // the target keeps the partition. + let moved = + (invalid && current.is_some() && pending.phase == MovePhase::Transfer) + .then_some(pending.plan); state.pending = None; + if let Some(plan) = moved { + state.moves.push(MovedPrimary { plan, since: now }); + } } } + let PrimaryRebalance { + moves, backoffs, .. + } = &mut *state; + moves.retain(|moved| { + let target = moved.plan.target; + let (Some(current), Some(backoff)) = ( + metadata.partition_placement_view( + moved.plan.namespace, + moved.plan.created_revision, + ), + backoffs.get_mut(usize::from(target)), + ) else { + return false; + }; + if consensus.primary_index(current.view) != target { + backoff.record_failure(now); + return false; + } + if now.duration_since(moved.since) >= MOVE_HOLD_INTERVAL { + *backoff = TargetBackoff::default(); + return false; + } + true + }); if state.pending.is_none() && now >= state.next_move && now.duration_since(state.stable_since) >= STABLE_INTERVAL { state.next_move = now + PROBE_INTERVAL; - if let Some(plan) = - metadata.primary_rebalance_candidate(&state.eligible, state.cursor) + // After its wait, a backed-off target gets one move at a time + // until one holds. It also sits out as a source: it lost the + // primaries moved to it, so it rarely has any to give. + let mut candidates = state.eligible.clone(); + for ((replica, eligible), backoff) in + candidates.iter_mut().enumerate().zip(&state.backoffs) + { + let probing = backoff.wait > Duration::ZERO + && state + .moves + .iter() + .any(|moved| usize::from(moved.plan.target) == replica); + *eligible &= !backoff.blocks(now) && !probing; + } + if let Some(plan) = metadata.primary_rebalance_candidate(&candidates, state.cursor) { let mut probe = probe_header(consensus, self.next_forward_nonce(consensus.replica())); @@ -452,7 +545,12 @@ where }; reply.ready = u8::from( header.kind == PrimaryRebalanceKind::MetadataProbe - && consensus_ready(consensus), + && consensus_ready(consensus) + && !self + .plane + .metadata() + .applied_frontier() + .has_partition_recovery(), ); self.send_rebalance(header.replica, reply).await; } @@ -570,6 +668,11 @@ where && partition.consensus().is_primary() } else { partition_ready(partition, &header) + && !self + .plane + .metadata() + .applied_frontier() + .has_partition_recovery() } })); if let Some(partition) = partition { @@ -796,6 +899,7 @@ fn partition_ready( probe: &PrimaryRebalanceHeader, ) -> bool { partition_history_ready(partition, probe) + && partition.repair.is_none() && partition.consensus().commit_min() >= probe.commit && partition.consensus().commit_min() == partition.consensus().commit_max() && !partition.history_is_busy() diff --git a/core/shard/src/primary_rebalance/tests.rs b/core/shard/src/primary_rebalance/tests.rs index 2d78f4e256..16630b7b76 100644 --- a/core/shard/src/primary_rebalance/tests.rs +++ b/core/shard/src/primary_rebalance/tests.rs @@ -189,6 +189,7 @@ fn arm(shard: &TestShard, phase: MovePhase) -> PrimaryR view: Some(METADATA_VIEW), peers: vec![Peer::default(); usize::from(REPLICAS)], eligible: vec![true; usize::from(REPLICAS)], + backoffs: vec![TargetBackoff::default(); usize::from(REPLICAS)], pending: Some(PendingMove { plan: PartitionPrimaryTransfer { namespace, @@ -480,6 +481,87 @@ async fn given_a_sent_grant_when_the_new_view_is_unseen_should_hold_the_move_unt } } +#[test] +fn given_failed_moves_to_a_target_when_backing_off_should_double_each_wait_once_up_to_the_cap() { + let mut now = Instant::now(); + let mut backoff = TargetBackoff::default(); + for wait in [ + MOVE_HOLD_INTERVAL, + MOVE_HOLD_INTERVAL * 2, + MOVE_HOLD_INTERVAL * 4, + TARGET_BACKOFF_MAX, + TARGET_BACKOFF_MAX, + ] { + backoff.record_failure(now); + // Another watched move fails inside the same wait. + backoff.record_failure(now + wait / 2); + assert_eq!(backoff.wait, wait); + assert!(backoff.blocks(now + wait / 2)); + now += wait; + assert!(!backoff.blocks(now)); + } +} + +#[compio::test] +async fn given_a_backed_off_target_when_its_probe_move_is_lost_or_held_should_double_or_reset_the_wait() + { + let target_view = PARTITION_VIEW + 1; + for held in [false, true] { + let (shard, _inbox) = owner(); + arm(&shard, MovePhase::Transfer); + let metadata = shard.plane.metadata(); + assert_eq!( + metadata + .consensus + .as_ref() + .unwrap() + .primary_index(target_view), + TARGET + ); + let namespace = IggyNamespace::new(1, 1, 0); + let streams = metadata.mux_stm.streams(); + streams.seed_namespace(namespace, namespace.inner(), PARTITION_VIEW); + let created_revision = streams.created_revision_for_namespace(namespace).unwrap(); + let observe = |view| { + metadata.observe_partition_views(&[PartitionPlacementView { + namespace, + created_revision, + view, + }]); + }; + observe(PARTITION_VIEW); + let moved_at = shard.bus.monotonic_now(); + { + let mut state = shard.primary_rebalance.borrow_mut(); + let pending = state.pending.as_mut().unwrap(); + pending.plan.created_revision = created_revision; + pending.since = moved_at; + // The wait after one failed move ran out, so this move probes the target. + state.backoffs[usize::from(TARGET)] = TargetBackoff { + until: Some(moved_at), + wait: MOVE_HOLD_INTERVAL, + }; + } + + observe(target_view); + tick_with_ready_peers(&shard, moved_at).await; + if held { + tick_with_ready_peers(&shard, moved_at + MOVE_HOLD_INTERVAL).await; + } else { + observe(target_view + 1); + tick_with_ready_peers(&shard, moved_at + PROBE_INTERVAL).await; + } + + let wait = shard.primary_rebalance.borrow().backoffs[usize::from(TARGET)].wait; + let expected = if held { + Duration::ZERO + } else { + MOVE_HOLD_INTERVAL * 2 + }; + assert_eq!(wait, expected, "held={held}"); + } +} + #[compio::test] async fn given_an_unsent_grant_when_the_coordinator_ticks_should_plan_again_before_the_transfer_timeout() { @@ -762,6 +844,7 @@ fn given_partition_probe_when_local_history_is_not_ready_should_refuse() { "recovering", "unproved", "transfer", + "repair", "ceded", "recovery barrier", "transition", @@ -782,6 +865,20 @@ fn given_partition_probe_when_local_history_is_not_ready_should_refuse() { "transfer" => partition .consensus() .set_state_transfer_stage(StateTransferStage::AwaitingTarget), + "repair" => { + partition.repair = Some(partitions::RepairSession { + nonce: NONCE, + view: PARTITION_VIEW, + from_op: COMMITTED + 1, + commit_to_op: COMMITTED + 1, + fetch_to_op: COMMITTED + 1, + floor: None, + peer: SOURCE, + idle_ticks: 0, + floor_pulled_from: 0, + requested_from: None, + }); + } "ceded" => partition.consensus().init_as_backup(), "recovery barrier" => partition.consensus().set_recovery_barrier(COMMITTED + 1), "transition" => { @@ -812,6 +909,45 @@ fn given_busy_source_history_when_capturing_frontier_should_allow_drain_but_not_ assert!(!partition_ready(&partition, &probe)); } +#[compio::test] +async fn given_another_shard_recovering_when_metadata_is_probed_should_refuse_primary_moves_without_blocking_admission() + { + let (shard, _inbox) = owner_at::(None, 1, 3, 0); + let consensus = shard.plane.metadata().consensus.as_ref().unwrap(); + let frontier = shard.plane.metadata().applied_frontier(); + let other_shard = frontier.track_partition_recovery(); + other_shard.set_active(true); + let mut probe = probe_header(consensus, NONCE); + probe.replica = 0; + probe.quorum_age_ns = 0; + shard.on_primary_rebalance(probe).await; + probe.nonce += 1; + shard.on_primary_rebalance(probe).await; + assert_eq!(shard.bus.sent.borrow().last().unwrap().1.ready, 0); + assert!(frontier.partition_requests_allowed(shard.bus.monotonic_now())); + drop(other_shard); + probe.nonce += 1; + shard.on_primary_rebalance(probe).await; + assert_eq!(shard.bus.sent.borrow().last().unwrap().1.ready, 1); +} + +#[compio::test] +async fn given_local_partition_recovery_when_the_coordinator_probes_should_exclude_itself_until_recovery_finishes() + { + let (shard, _inbox) = owner_at::(None, 0, 3, 0); + let recovery = shard + .plane + .metadata() + .applied_frontier() + .track_partition_recovery(); + recovery.set_active(true); + shard.tick_primary_rebalance().await; + assert!(!shard.primary_rebalance.borrow().eligible[0]); + recovery.set_active(false); + shard.tick_primary_rebalance().await; + assert!(shard.primary_rebalance.borrow().eligible[0]); +} + #[compio::test] async fn given_three_or_five_replicas_when_a_quorum_answers_not_ready_should_credit_probe_send_times() { diff --git a/core/shard/src/router.rs b/core/shard/src/router.rs index 6fb87ff1b6..95d3e47335 100644 --- a/core/shard/src/router.rs +++ b/core/shard/src/router.rs @@ -420,6 +420,7 @@ where // offers live on every shard that hosts a serving group -- // parked behind the shard-0 gate they would never expire. self.expire_idle_state_transfer_offers(); + self.serve_parked_partition_transfer_requests().await; // A drain completes without an event, so a pending // revocation wakes the reconciler each tick. A published // fence wakes it once, and retries of an unpublished one @@ -528,7 +529,11 @@ where () = poll_fn(|context| { self.plane.partitions().register_loopback_waker(context.waker()); self.partition_io.register_waker(context.waker()); - if self.has_pending_loopback(&loopback_buf) || self.partition_io.has_ready() { + self.served_reads.register_waker(context.waker()); + if self.has_pending_loopback(&loopback_buf) + || self.partition_io.has_ready() + || self.served_reads.has_landed() + { Poll::Ready(()) } else { Poll::Pending @@ -539,6 +544,9 @@ where } } self.service_partition_io().await; + // After every event, as partition I/O, so a busy inbox cannot + // starve the serves that wait for a disk read. + self.serve_landed_read().await; } self.poll_completions.close(); diff --git a/core/shard/src/served_reads.rs b/core/shard/src/served_reads.rs new file mode 100644 index 0000000000..c096365829 --- /dev/null +++ b/core/shard/src/served_reads.rs @@ -0,0 +1,218 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Disk reads that answer a peer run beside the pump. +//! +//! A repair or a replay can need prepare bodies that the journal released to +//! their segment, and a transfer chunk can need segment bytes. A read on the +//! pump would hold the frames, ticks and replies of every group on the shard +//! until the disk answers. So the pump decides what to serve, a detached task +//! reads the bytes, and the pump serves the request again once they land. That +//! serve checks the state again, so the bytes never answer a request that the +//! state no longer admits. +//! +//! The reads run one at a time per shard, as they did on the pump, and a read +//! starts only after the pump took the bytes of the reads before it. So a +//! shard holds the bytes of two reads at most: the one that the pump serves +//! and the one that runs. A key has at most one read, and a shard holds at +//! most [`SERVED_READS_MAX`] keys. The pump drops a request past either bound, +//! and its requester asks again. + +use std::cell::RefCell; +use std::collections::{HashSet, VecDeque}; +use std::future::poll_fn; +use std::rc::Rc; +use std::task::{Context, Poll, Waker}; + +use consensus::StateArtifact; +use iggy_binary_protocol::{RequestPreparesHeader, RequestStateChunkHeader}; +use journal::local_gate::LocalGate; +use message_bus::MessageBus; +use partitions::ReleasedBodies; +use partitions::state_transfer::SegmentLoadError; +use server_common::Message; + +/// Keys that one shard reads or waits to read for. A waiting key holds its +/// request and the headers of one round, but no bytes. +pub const SERVED_READS_MAX: usize = 64; + +/// What a read serves. A request whose key reads waits for that read. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ServedReadKey { + /// Released bodies that a repair, or a replay when `live`, sends `target`. + Prepares { group: u64, target: u8, live: bool }, + /// Segment bytes of the transfer chunk that `requester` pulls. + Chunk { group: u64, requester: u8 }, +} + +/// A read that landed, with the request that it serves. +pub enum LandedRead { + Prepares { + request: Message, + live: bool, + bodies: ReleasedBodies, + }, + Chunk { + header: RequestStateChunkHeader, + /// The offered artifact that `load` read. + artifact: StateArtifact, + load: Result, SegmentLoadError>, + }, +} + +/// The pump and the read tasks share one state, so a shard holds one pointer. +#[derive(Default)] +pub struct ServedReads { + shared: Rc, +} + +#[derive(Default)] +struct Shared { + keys: RefCell>, + landed: RefCell)>>, + pump: RefCell>, + /// The read whose turn came while the pump had not taken every result. + taken: RefCell>, + gate: LocalGate, +} + +impl Shared { + /// Ready once the pump took every read that landed. + fn poll_taken(&self, context: &Context<'_>) -> Poll<()> { + if self.landed.borrow().is_empty() { + return Poll::Ready(()); + } + self.taken.replace(Some(context.waker().clone())); + Poll::Pending + } +} + +impl ServedReads { + pub(crate) fn reading(&self, key: ServedReadKey) -> bool { + self.shared.keys.borrow().contains(&key) + } + + /// Runs `read` in a detached task once no other read of this shard runs + /// and the pump took every result, and holds `key` until the pump takes + /// this one. Runs nothing when `key` already reads or the shard holds + /// [`SERVED_READS_MAX`] keys. + pub(crate) fn start( + &self, + bus: &B, + key: ServedReadKey, + read: impl Future + 'static, + ) { + { + let mut keys = self.shared.keys.borrow_mut(); + if keys.len() >= SERVED_READS_MAX || !keys.insert(key) { + return; + } + } + let landing = Landing { + shared: Rc::clone(&self.shared), + key, + read: None, + }; + let shared = Rc::clone(&self.shared); + bus.spawn(async move { + let _turn = shared.gate.acquire().await; + poll_fn(|context| shared.poll_taken(context)).await; + landing.land(read.await); + }); + } + + pub(crate) fn register_waker(&self, waker: &Waker) { + let mut current = self.shared.pump.borrow_mut(); + if current + .as_ref() + .is_none_or(|current| !current.will_wake(waker)) + { + *current = Some(waker.clone()); + } + } + + pub(crate) fn has_landed(&self) -> bool { + !self.shared.landed.borrow().is_empty() + } + + /// Takes the oldest landed read and frees its key. A read that stopped + /// without a result frees its key and returns `None`. + pub(crate) fn take_landed(&self) -> Option { + let (key, read) = self.shared.landed.borrow_mut().pop_front()?; + self.shared.keys.borrow_mut().remove(&key); + if let Some(taken) = self.shared.taken.take() { + taken.wake(); + } + read + } + + #[cfg(test)] + pub(crate) fn is_empty(&self) -> bool { + self.shared.keys.borrow().is_empty() + } + + #[cfg(test)] + pub(crate) fn landed_count(&self) -> usize { + self.shared.landed.borrow().len() + } + + #[cfg(test)] + #[allow(clippy::future_not_send)] + pub(crate) async fn landed(&self) { + poll_fn(|context| { + self.register_waker(context.waker()); + if self.has_landed() { + Poll::Ready(()) + } else { + Poll::Pending + } + }) + .await; + } + + /// Holds the turn of this shard's reads, so none runs until the guard drops. + #[cfg(test)] + pub(crate) fn hold(&self) -> journal::local_gate::LocalGateGuard<'_> { + self.shared.gate.try_acquire().expect("no read runs") + } +} + +/// Hands the result of a read to the pump. A task that stops without one, such +/// as one that panicked, hands over nothing, so its key frees either way. +struct Landing { + shared: Rc, + key: ServedReadKey, + read: Option, +} + +impl Landing { + fn land(mut self, read: LandedRead) { + self.read = Some(read); + } +} + +impl Drop for Landing { + fn drop(&mut self) { + self.shared + .landed + .borrow_mut() + .push_back((self.key, self.read.take())); + if let Some(waker) = self.shared.pump.borrow_mut().take() { + waker.wake(); + } + } +} diff --git a/core/simulator/README.md b/core/simulator/README.md index acd51282b1..a7e34edbf4 100644 --- a/core/simulator/README.md +++ b/core/simulator/README.md @@ -202,7 +202,7 @@ replica 2: live | metadata ... commit=1407..1414 barrier=1359 transferring=false The suite lives at the end of `src/lib.rs`, with unit tests next to the modules they cover. Names follow `given_X_when_Y_should_Z` where there is a meaningful given; older behavioural names remain where there is not. Every test that touches replies initialises `MemoryPool` first, as above. -What the suite pins, by example: `workload_replay_is_deterministic` and `multi_shard_replay_is_deterministic` (the seed contract), `committed_metadata_agrees_across_replicas`, `given_advanced_view_when_metadata_replica_restarts_should_recover_view_from_superblock`, `given_superblock_write_fails_when_primary_crashes_should_withhold_votes_and_not_elect`, `checkpointing_cluster_serves_a_chunked_state_transfer`, `shell_detects_partition_borrow_held_across_await`. +What the suite pins, by example: `workload_replay_is_deterministic` and `multi_shard_replay_is_deterministic` (the seed contract), `committed_metadata_agrees_across_replicas`, `given_advanced_view_when_metadata_replica_restarts_should_recover_view_from_superblock`, `given_superblock_write_fails_when_primary_crashes_should_withhold_the_new_view_from_peers`, `checkpointing_cluster_serves_a_chunked_state_transfer`, `shell_detects_partition_borrow_held_across_await`. Shell simulations run the production heartbeat verifier on every shard and the production consumer-group liveness loop on shard 0, using the virtual monotonic clock. Crash stops those tasks; restart rebuilds them. Silent or disconnected clients lose their lease through the same ordered cleanup as the server. Each simulator owns a temporary system directory for liveness fences, retained across replica restarts and removed when the simulator drops. Checkpoint simulations reuse their configured directory without changing its lifetime. diff --git a/core/simulator/src/lib.rs b/core/simulator/src/lib.rs index 4e0c45aff8..75542be834 100644 --- a/core/simulator/src/lib.rs +++ b/core/simulator/src/lib.rs @@ -40,7 +40,7 @@ use executor::{DetExecutor, RunOutcome, TaskId}; use iggy_binary_protocol::primitives::partition_history::{ConsumerGroupOwner, PartitionContext}; use iggy_binary_protocol::{ Command, CommitHeader, DoViewChangeHeader, GenericHeader, PrepareHeader, ReplyHeader, - RequestHeader, RoutedRequestHeader, StartViewChangeHeader, StartViewHeader, + RequestHeader, RoutedRequestHeader, StartViewHeader, }; use iggy_common::{Durability, IggyError}; use message_bus::MessageBus; @@ -918,14 +918,17 @@ impl Simulator { // left `status` at 0, so it reads as a completed handshake and hands // back a session the cluster never granted, silently, whenever that // commit point is nonzero. - if answer.is_none() - && header.client == client_id + if header.client == client_id && header.operation == expected_operation && header.request == request - && !setup_reply_is_transient(&reply) { - answer = Some(reply); - continue; + if setup_reply_is_transient(&reply) { + continue; + } + if answer.is_none() { + answer = Some(reply); + continue; + } } self.deferred_client_replies.push((from, reply)); } @@ -2153,7 +2156,8 @@ fn materialise_partition( /// The partition group, incarnation and view of a message the persist gate /// holds until the replica's view is durable, or `None` for any other message. -/// `PrepareOk` names no incarnation, so it is left out. +/// `PrepareOk` names no incarnation, so it is left out. A `StartViewChange` is +/// left out too, because a replica votes for a view before it enters it. fn view_scoped_partition_frame( message: &Message, ) -> Option<(IggyNamespace, u64, u32)> { @@ -2166,13 +2170,6 @@ fn view_scoped_partition_frame( let header = message.try_as_typed::().ok()?.header(); (header.group, header.partition_incarnation) } - Command::StartViewChange => { - let header = message - .try_as_typed::() - .ok()? - .header(); - (header.group, header.partition_incarnation) - } Command::DoViewChange => { let header = message.try_as_typed::().ok()?.header(); (header.group, header.partition_incarnation) @@ -2229,6 +2226,9 @@ mod slow_replica_tests; #[cfg(test)] mod stale_prepare_commit_tests; +#[cfg(test)] +mod replication_flow_tests; + #[cfg(test)] mod tests { use super::partition_repair_driver_tests::{ @@ -3017,7 +3017,16 @@ mod tests { fn given_no_commit_heartbeat_when_a_partition_fence_commits_should_reach_every_backup_within_a_few_ticks() { const FENCE_STEPS: usize = 400; - let (mut sim, client) = cluster(0x4645_4E43); + let network = packet::PacketSimulatorOptions { + node_count: 3, + client_count: 1, + seed: 0x4645_4E43, + one_way_delay_min: 1, + one_way_delay_mean: 1, + ..packet::PacketSimulatorOptions::default() + }; + let mut sim = Simulator::new(3, std::iter::once(1), network); + let client = SimClient::new(1); let namespace = IggyNamespace::new(0, 0, 0); sim.init_partition(namespace); sim.register_client_with_primary(&client); @@ -3071,7 +3080,28 @@ mod tests { commit_max(&sim, replica).is_some_and(|commit| commit >= fence_op) }) }), - "every backup must learn the fence commit {fence_op} within {EAGER_COMMIT_TICKS} ticks" + "every backup must learn the fence commit {fence_op} within {EAGER_COMMIT_TICKS} ticks: {:?}", + (0..sim.replica_count) + .map(|replica| { + let partition = sim.replicas[usize::from(replica)] + .partition_shard(namespace) + .plane + .partitions() + .get_by_ns(&namespace) + .unwrap(); + let consensus = partition.consensus(); + ( + consensus.status(), + consensus.view(), + consensus.sequencer().current_sequence(), + consensus.commit_min(), + consensus.commit_max(), + partition.log.journal().inner.holds_op(fence_op), + partition.history_deleted(), + partition.repair, + ) + }) + .collect::>() ); } @@ -3438,7 +3468,13 @@ mod tests { for index in 0..SENDS { let send = client.send_messages(fixture, &[Bytes::from(format!("deleted-{index}"))]); let reply = submit_and_wait_for_reply(&mut sim, client.client_id(), 0, send); - assert_eq!(reply.header().status, 0, "send {index} must commit"); + assert_eq!( + reply.header().status, + 0, + "send {index} must commit: {:?}, body {:?}", + reply.header(), + String::from_utf8_lossy(reply.body()) + ); } complete_metadata_op(&mut sim, &client, client.delete_topic(&stream, &topic)); // A later prepare carries the completion's commit point, so the restarted @@ -3618,7 +3654,13 @@ mod tests { for index in 0..SENDS { let send = client.send_messages(fixture, &[Bytes::from(format!("recreated-{index}"))]); let reply = submit_and_wait_for_reply(&mut sim, client.client_id(), 0, send); - assert_eq!(reply.header().status, 0, "send {index} must commit"); + assert_eq!( + reply.header().status, + 0, + "send {index} must commit: {:?}, body {:?}", + reply.header(), + String::from_utf8_lossy(reply.body()) + ); } let committed = |sim: &Simulator, replica_idx: usize| { sim.offsets(replica_idx, fixture) @@ -3964,7 +4006,13 @@ mod tests { for index in 0..SENDS { let send = client.send_messages(fixture, &[Bytes::from(format!("deleted-{index}"))]); let reply = submit_and_wait_for_reply(&mut sim, client.client_id(), 0, send); - assert_eq!(reply.header().status, 0, "send {index} must commit"); + assert_eq!( + reply.header().status, + 0, + "send {index} must commit: {:?}, body {:?}", + reply.header(), + String::from_utf8_lossy(reply.body()) + ); } assert!( (0..PROGRESS_STEPS).any(|_| { @@ -4190,8 +4238,22 @@ mod tests { let (mut sim, client, _) = consumer_group_disconnect_cluster(3, session_count + 2); for partition in 0..PARTITIONS { let namespace = IggyNamespace::new(0, 0, partition); - sim.init_partition(namespace); - sim.seed_stream_topic_partition(namespace); + if sim.replicas[0] + .partition_shard(namespace) + .plane + .partitions() + .get_by_ns(&namespace) + .is_none() + { + sim.init_partition(namespace); + sim.seed_stream_topic_partition(namespace); + } + } + for replica in &sim.replicas { + replica.shards[0] + .plane + .metadata() + .seed_applied_frontier_from_consensus(); } sim.discover_partition_contexts(&client); for partition in 0..PARTITIONS { @@ -4667,11 +4729,11 @@ mod tests { ); } - /// Without a quorum the pending owner never installs, so settling stops at - /// the stale revocation bound and the quiesce check fails the revocation. + /// Without a quorum the primary steps down, so settlement must fail before + /// the convergence oracle can choose a leader to inspect. #[test] - #[should_panic(expected = "pending revocation")] - fn given_a_pending_owner_without_quorum_when_settled_should_fail_the_quiesce_check() { + #[should_panic(expected = "the views never settled")] + fn given_a_pending_owner_without_quorum_when_settling_should_refuse_convergence() { let mut sim = pending_leave_fixture(); sim.replica_crash(1); sim.replica_crash(2); @@ -4916,8 +4978,20 @@ mod tests { const PROGRESS_STEPS: usize = 4_000; for index in 0..count { let send = client.send_messages(namespace, &[Bytes::from(format!("message-{index}"))]); - let reply = submit_and_wait_for_reply(sim, client.client_id(), 0, send); - assert_eq!(reply.header().status, 0, "send {index} must commit"); + let reply = sim.await_setup_reply( + client, + 0, + &send.into_generic(), + Operation::SendMessages, + "commit_sends_everywhere", + ); + assert_eq!( + reply.header().status, + 0, + "send {index} must commit: {:?}, body {:?}", + reply.header(), + String::from_utf8_lossy(reply.body()) + ); } let last = u64::try_from(count - 1).expect("a send count fits u64"); assert!( @@ -5242,7 +5316,26 @@ mod tests { return reply; } } - panic!("request {request_id} did not receive a reply"); + panic!( + "request {request_id} did not receive a reply; metadata {:?}", + sim.replicas + .iter() + .enumerate() + .map(|(node, replica)| ( + node, + sim.metadata_consensus(node).map(|consensus| ( + consensus.status(), + consensus.view(), + consensus.commit_min(), + consensus.commit_max(), + consensus.sequencer().current_sequence(), + consensus.pipeline_head_header() + )), + replica.metadata_journal.last_op(), + replica.shards[0].metadata_repair_window() + )) + .collect::>() + ); } /// `(stream id, created op)` of `name` on `replica`. @@ -6123,6 +6216,10 @@ mod tests { .unwrap() .vsr_state(0, 0) .with_history_certificate(None); + let view_before_crash = sim + .metadata_consensus(usize::from(PEER)) + .expect("shard 0 hosts the metadata plane") + .view(); sim.replica_crash(0); sim.replica_crash(LEGACY); @@ -6145,10 +6242,14 @@ mod tests { Status::Recovering, "an unanswered probe let an unproven log vote" ); - assert_ne!( - with_consensus(&sim, PEER).0, - Status::Normal, - "one peer started a view without a quorum" + let peer = sim + .metadata_consensus(usize::from(PEER)) + .expect("shard 0 hosts the metadata plane"); + // The peer votes to replace the crashed primary but stays in its view. + assert!( + peer.view() == view_before_crash || peer.status() != Status::Normal, + "one peer started view {} without a quorum", + peer.view() ); sim.replica_restart(0); @@ -7433,15 +7534,16 @@ mod tests { } #[test] - fn given_superblock_write_fails_when_primary_crashes_should_withhold_votes_and_not_elect() { - // The gate under test: a view-scoped send (SVC, DVC, StartView) must not go - // out until the new view is durable. Failing one survivor's superblock writes + fn given_superblock_write_fails_when_primary_crashes_should_withhold_the_new_view_from_peers() { + // The gate under test: a view-scoped send (DVC, StartView) must not go out + // until the new view is durable. Failing one survivor's superblock writes // makes its persist gate return false, advancing its view in-memory while - // withholding every view-scoped send. Quorum 2 of 3, so crashing the primary - // leaves one working survivor whose lone vote cannot reach quorum and NO new - // primary is elected. Sending before persisting would instead elect a primary - // with no durable record of the new view, reintroducing split-brain on its - // restart. Positive control: + // withholding every view-scoped send. A StartViewChange is only a vote and + // changes no view, so both survivors enter view 1. The faulted survivor leads + // view 1 and can win it in memory, but its StartView never goes out, so the + // working survivor never starts the view. Sending before persisting would + // instead start a view whose primary has no durable record of it, + // reintroducing split-brain on its restart. Positive control: // `given_advanced_view_when_metadata_replica_restarts_...`, same crash with // healthy superblocks. server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { @@ -7464,8 +7566,8 @@ mod tests { ); // Replica 1 survives and is primary-by-index for view 1. Failing its - // superblock keeps it from durably taking any new view, so it never emits a - // view-scoped vote. + // superblock keeps it from durably taking any new view, so it never sends a + // DoViewChange or a StartView. sim.replicas[1].superblock.set_fail_writes(); sim.replica_crash(0); @@ -7473,24 +7575,18 @@ mod tests { sim.step(); } - for replica in 1..replica_count { - let elected = sim.replicas[replica as usize].shards[0] - .plane - .metadata() - .consensus - .as_ref() - .is_some_and(|consensus| { - consensus.view() > 0 - && consensus.status() == Status::Normal - && consensus.is_primary() - }); - assert!( - !elected, - "replica {replica} must not be elected primary while a survivor's \ - superblock write fails: the persist gate withholds its view-scoped \ - votes, so quorum cannot form" - ); - } + let working = sim.replicas[2].shards[0] + .plane + .metadata() + .consensus + .as_ref() + .expect("replica 2 owns metadata consensus"); + assert!( + working.view() == 0 || working.status() != Status::Normal, + "replica 2 started view {} while the superblock write of its primary fails: \ + the persist gate must withhold the StartView", + working.view() + ); // The faulted replica never durably recorded a new view, so a restart recovers // its old view, never one it might have voted in. @@ -8567,6 +8663,94 @@ mod tests { full_restart_awaiting_transfer_starts_a_view(&[1, 2]); } + /// A backup restarts with its partition files lost while it cannot reach its + /// peers, so nobody answers its view probe. Once the link heals it must adopt + /// the view of the primary that kept leading: only that primary can name the + /// state transfer it waits for. + #[test] + fn given_a_backup_awaiting_a_state_transfer_when_its_probes_go_unanswered_should_adopt_the_settled_view_after_the_heal() + { + const PROGRESS_STEPS: usize = 4_000; + // Three exhausted probes, each one lone view change for a replica + // allowed to elect on its recovered log. + const ISOLATION_STEPS: u64 = 3 + * consensus::TimeoutManager::REQUEST_START_VIEW_MESSAGE_TICKS + * consensus::PROBE_ATTEMPTS_MAX as u64; + let (mut sim, client) = cluster(packet::PacketSimulatorOptions::default().seed); + sim.register_client_with_primary(&client); + let namespace = IggyNamespace::new(1, 1, 0); + sim.seed_stream_topic_partition(namespace); + for replica in &sim.replicas { + materialise_partition(replica, namespace, sim.consumer_offsets_max); + } + let primary = (0..PROGRESS_STEPS) + .find_map(|_| { + sim.step(); + (0..3u8).find(|replica| { + sim.partition_consensus_state(usize::from(*replica), namespace) + .is_some_and(|state| state.status == Status::Normal && state.is_primary) + }) + }) + .expect("the fresh partition elects a primary"); + let awaiting = (primary + 1) % 3; + let set_links = |sim: &mut Simulator, enabled: bool| { + for peer in (0..3u8).filter(|peer| *peer != awaiting) { + sim.network.set_link_filter( + packet::ProcessId::Replica(awaiting), + packet::ProcessId::Replica(peer), + enabled, + ); + sim.network.set_link_filter( + packet::ProcessId::Replica(peer), + packet::ProcessId::Replica(awaiting), + enabled, + ); + } + }; + + sim.replica_crash(awaiting); + set_links(&mut sim, false); + sim.replica_restart(awaiting); + { + let partitions = sim.replicas[usize::from(awaiting)] + .partition_shard(namespace) + .plane + .partitions(); + let partition = partitions + .get_mut_by_ns(&namespace) + .expect("a restarted replica rebuilds the partition"); + partition.force_requires_state_transfer_for_test(); + assert!(partition.requires_state_transfer()); + } + for _ in 0..ISOLATION_STEPS { + sim.step(); + } + let isolated_view = sim + .partition_consensus_state(usize::from(awaiting), namespace) + .expect("the awaiting replica hosts the partition") + .view; + set_links(&mut sim, true); + + let settled = |sim: &Simulator| { + let leader = sim + .partition_consensus_state(usize::from(primary), namespace) + .expect("the primary hosts the partition"); + sim.partition_consensus_state(usize::from(awaiting), namespace) + .is_some_and(|state| state.view == leader.view) + }; + assert!( + (0..PROGRESS_STEPS).any(|_| { + sim.step(); + settled(&sim) + }), + "replica {awaiting} awaits a state transfer and never adopted the view of primary \ + {primary}: it reached view {isolated_view} while cut off, states={:?}", + (0..3) + .map(|replica| sim.partition_consensus_state(replica, namespace)) + .collect::>() + ); + } + fn successful_send_reply_count(replies: &[Message]) -> usize { replies .iter() @@ -12201,6 +12385,9 @@ mod view_change_data_loss_tests { /// the header distinguishes it from a commit and the code is the only signal. pub(crate) fn setup_reply_is_transient(reply: &Message) -> bool { let header = reply.header(); + if workload::TransientRejection::from_code(header.status).is_some() { + return true; + } let Some(body) = reply .as_slice() .get(size_of::()..header.size as usize) @@ -12465,10 +12652,28 @@ mod partition_repair_driver_tests { /// are still declared in each test, where its fault setup is read. macro_rules! withhold_one_prepare { ($namespace:ident, $withheld_op:ident) => { + static MIN_WITHHELD_OP: AtomicU64 = AtomicU64::new(0); + fn arm_withheld_prepare(sim: &Simulator) { + let namespace = IggyNamespace::from_raw($namespace.load(Ordering::Relaxed)); + let primary = sim.primary_index(namespace).unwrap(); + let partition = sim.replicas[usize::from(primary)] + .partition_shard(namespace) + .plane + .partitions() + .get_by_ns(&namespace) + .unwrap(); + MIN_WITHHELD_OP.store( + partition.consensus().sequencer().current_sequence() + 1, + Ordering::Relaxed, + ); + } fn withhold_one_prepare(packet: &Packet) -> bool { let Some(header) = prepare_for(packet, $namespace.load(Ordering::Relaxed)) else { return false; }; + if header.op < MIN_WITHHELD_OP.load(Ordering::Relaxed) { + return false; + } $withheld_op .compare_exchange(0, header.op, Ordering::Relaxed, Ordering::Relaxed) .is_ok() @@ -12862,6 +13067,9 @@ mod partition_repair_driver_tests { /// alone. Without it the retry timer could heal the gap and the test /// would pass with no repair driver at all. fn starve_commit_edge(packet: &Packet) -> bool { + if withhold_one_prepare(packet) { + return true; + } let group = GAP_NS.load(Ordering::Relaxed); if let Some(header) = prepare_for(packet, group) { return header.op == WITHHELD_OP.load(Ordering::Relaxed); @@ -12877,6 +13085,7 @@ mod partition_repair_driver_tests { WITHHELD_OP.store(0, Ordering::Relaxed); produce(&mut sim, &client, namespace, WARMUP_SENDS, "warmup"); + arm_withheld_prepare(&sim); let (_, _, warm_commit_min, _) = group_state(&sim, LAGGING, namespace); assert!( warm_commit_min > 0, @@ -12884,9 +13093,6 @@ mod partition_repair_driver_tests { below would open at the group's first op" ); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(starve_commit_edge); @@ -12896,7 +13102,7 @@ mod partition_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no partition prepare crossed the chain link, so the fault never armed" + "no partition prepare crossed the primary link, so the fault never armed" ); assert!( gap_drops(&sim, LAGGING, namespace) > 0, @@ -12960,6 +13166,9 @@ mod partition_repair_driver_tests { withhold_one_prepare!(GAP_NS, WITHHELD_OP); fn starve_commit_edge(packet: &Packet) -> bool { + if withhold_one_prepare(packet) { + return true; + } let group = GAP_NS.load(Ordering::Relaxed); if let Some(header) = prepare_for(packet, group) { return header.op == WITHHELD_OP.load(Ordering::Relaxed); @@ -12975,10 +13184,8 @@ mod partition_repair_driver_tests { WITHHELD_OP.store(0, Ordering::Relaxed); produce(&mut sim, &client, namespace, WARMUP_SENDS, "warmup"); + arm_withheld_prepare(&sim); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(starve_commit_edge); @@ -12987,7 +13194,7 @@ mod partition_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no partition prepare crossed the chain link, so the fault never armed" + "no partition prepare crossed the primary link, so the fault never armed" ); let (_, _, commit_min, commit_max) = group_state(&sim, LAGGING, namespace); assert!( @@ -13168,7 +13375,7 @@ mod partition_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert!( WITHHELD_COPIES.load(Ordering::Relaxed) > 0, - "no partition prepare crossed the chain link, so the fault never armed" + "no partition prepare crossed the primary link, so the fault never armed" ); assert!( gap_drops(&sim, LAGGING_OF_FIVE, namespace) > 0, @@ -13336,16 +13543,18 @@ mod partition_repair_driver_tests { withhold_one_prepare!(GAP_NS, WITHHELD_OP); - /// Primary -> 2: withhold every direct prepare (live ones ride the - /// chain, so this starves only retransmit heals), the group's commit - /// heartbeats, and its repair terminators. The repaired ops themselves + /// Withhold one live prepare and its retries, commit heartbeats, and + /// repair terminators. Later prepares carry the committed frontier. The repaired ops themselves /// pass, so the window lands resident while `complete_repair`, the walk /// the terminator would run, never fires. fn starve_walk_edges(packet: &Packet) -> bool { - let group = GAP_NS.load(Ordering::Relaxed); - if prepare_for(packet, group).is_some() { + if withhold_one_prepare(packet) { return true; } + let group = GAP_NS.load(Ordering::Relaxed); + if let Some(header) = prepare_for(packet, group) { + return header.op == WITHHELD_OP.load(Ordering::Relaxed); + } if is_repair_done_for(packet, group) { WITHHELD_DONES.fetch_add(1, Ordering::Relaxed); return true; @@ -13353,7 +13562,7 @@ mod partition_repair_driver_tests { is_commit_for(packet, group) } - let (mut sim, client) = cluster(0x5EED_0235); + let (mut sim, client) = cluster(0x5EED_0237); let namespace = IggyNamespace::new(1, 1, 0); sim.init_partition(namespace); sim.register_client_with_primary(&client); @@ -13362,6 +13571,7 @@ mod partition_repair_driver_tests { WITHHELD_DONES.store(0, Ordering::Relaxed); produce(&mut sim, &client, namespace, WARMUP_SENDS, "warmup"); + arm_withheld_prepare(&sim); let (_, _, warm_commit_min, _) = group_state(&sim, LAGGING, namespace); assert!( warm_commit_min > 0, @@ -13369,9 +13579,6 @@ mod partition_repair_driver_tests { below would open at the group's first op" ); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(starve_walk_edges); @@ -13381,7 +13588,7 @@ mod partition_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no partition prepare crossed the chain link, so the fault never armed" + "no partition prepare crossed the primary link, so the fault never armed" ); for _ in 0..STRAND_QUIET_STEPS { @@ -13438,6 +13645,9 @@ mod partition_repair_driver_tests { /// Primary -> 2: withhold this group's commit heartbeats, so the /// edge-triggered backstop cannot be what arms the repair below. fn starve_commit_edge(packet: &Packet) -> bool { + if withhold_one_prepare(packet) { + return true; + } let group = GAP_NS.load(Ordering::Relaxed); if let Some(header) = prepare_for(packet, group) { return header.op == WITHHELD_OP.load(Ordering::Relaxed); @@ -13453,9 +13663,7 @@ mod partition_repair_driver_tests { WITHHELD_OP.store(0, Ordering::Relaxed); produce(&mut sim, &client, namespace, WARMUP_SENDS, "warmup"); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); + arm_withheld_prepare(&sim); *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(starve_commit_edge); @@ -13466,7 +13674,7 @@ mod partition_repair_driver_tests { assert_ne!( WITHHELD_OP.load(Ordering::Relaxed), 0, - "no partition prepare crossed the chain link, so the fault never armed" + "no partition prepare crossed the primary link, so the fault never armed" ); let (_, _, commit_min, commit_max) = group_state(&sim, LAGGING, namespace); assert!( @@ -13595,8 +13803,10 @@ mod partition_repair_driver_tests { WITHHELD_OP.store(0, Ordering::Relaxed); produce(&mut sim, &client, namespace, WARMUP_SENDS, "warmup"); + arm_withheld_prepare(&sim); + *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = + .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(withhold_one_prepare); // Stepped one at a time and stopped on the step that buffered a drop: @@ -13773,10 +13983,25 @@ mod metadata_repair_driver_tests { /// per-test or the siblings in this binary would share one fault. macro_rules! withhold_one_metadata_prepare { ($withheld_op:ident) => { + static MIN_WITHHELD_OP: AtomicU64 = AtomicU64::new(0); + fn arm_withheld_prepare(sim: &Simulator) { + let primary = sim.metadata_primary_index().unwrap(); + MIN_WITHHELD_OP.store( + sim.metadata_consensus(usize::from(primary)) + .unwrap() + .sequencer() + .current_sequence() + + 1, + Ordering::Relaxed, + ); + } fn withhold_one_prepare(packet: &Packet) -> bool { let Some(header) = prepare_for(packet, METADATA_GROUP) else { return false; }; + if header.op < MIN_WITHHELD_OP.load(Ordering::Relaxed) { + return false; + } $withheld_op .compare_exchange(0, header.op, Ordering::Relaxed, Ordering::Relaxed) .is_ok() @@ -13794,6 +14019,9 @@ mod metadata_repair_driver_tests { macro_rules! starve_commit_edge { ($withheld_op:ident) => { fn starve_commit_edge(packet: &Packet) -> bool { + if withhold_one_prepare(packet) { + return true; + } if let Some(header) = prepare_for(packet, METADATA_GROUP) { return header.op == $withheld_op.load(Ordering::Relaxed); } @@ -13852,11 +14080,22 @@ mod metadata_repair_driver_tests { /// Submit `sends` stream creations to the primary, stepping between each. fn create_streams(sim: &mut Simulator, client: &SimClient, sends: usize, tag: &str) { for index in 0..sends { - let msg = client.create_stream(&format!("{tag}-{index}")); - sim.submit_request(client.client_id(), 0, msg.into_generic()); - for _ in 0..STEPS_PER_SEND { - sim.step(); - } + let request = client + .create_stream(&format!("{tag}-{index}")) + .into_generic(); + let reply = sim.await_setup_reply( + client, + 0, + &request, + iggy_binary_protocol::Operation::CreateStream, + "create_streams", + ); + assert_eq!( + reply.header().status, + 0, + "metadata setup {tag}-{index} must commit" + ); + assert_eq!(iggy_binary_protocol::result_code(reply.body()), Some(0)); } } @@ -14040,6 +14279,7 @@ mod metadata_repair_driver_tests { WITHHELD_OP.store(0, Ordering::Relaxed); create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm"); + arm_withheld_prepare(&sim); let (_, _, warm_commit_min, _) = metadata_state(&sim, LAGGING); assert!( warm_commit_min > 0, @@ -14047,9 +14287,6 @@ mod metadata_repair_driver_tests { below would open at the group's first op" ); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(starve_commit_edge); @@ -14059,7 +14296,7 @@ mod metadata_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no metadata prepare crossed the chain link, so the fault never armed" + "no metadata prepare crossed the primary link, so the fault never armed" ); assert!( gap_drops(&sim, LAGGING) > 0, @@ -14123,10 +14360,8 @@ mod metadata_repair_driver_tests { WITHHELD_OP.store(0, Ordering::Relaxed); create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm"); + arm_withheld_prepare(&sim); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(starve_commit_edge); @@ -14135,7 +14370,7 @@ mod metadata_repair_driver_tests { assert_ne!( WITHHELD_OP.load(Ordering::Relaxed), 0, - "no metadata prepare crossed the chain link, so the fault never armed" + "no metadata prepare crossed the primary link, so the fault never armed" ); // Move the primary's retention floor past the whole gap window before @@ -14171,15 +14406,17 @@ mod metadata_repair_driver_tests { static WITHHELD_DONES: AtomicU64 = AtomicU64::new(0); withhold_one_metadata_prepare!(WITHHELD_OP); - /// Primary -> 2: withhold every direct prepare (live ones ride the - /// chain, so this starves only retransmit heals), the group's commit - /// heartbeats, and its repair terminators. The repaired ops themselves + /// Withhold one live prepare and its retries, commit heartbeats, and + /// repair terminators. Later prepares carry the committed frontier. The repaired ops themselves /// pass, so the window lands resident while the `RepairDone` that would /// run the walk never fires. fn starve_walk_edges(packet: &Packet) -> bool { - if prepare_for(packet, METADATA_GROUP).is_some() { + if withhold_one_prepare(packet) { return true; } + if let Some(header) = prepare_for(packet, METADATA_GROUP) { + return header.op == WITHHELD_OP.load(Ordering::Relaxed); + } if is_repair_done_for(packet, METADATA_GROUP) { WITHHELD_DONES.fetch_add(1, Ordering::Relaxed); return true; @@ -14187,12 +14424,13 @@ mod metadata_repair_driver_tests { is_commit_for(packet, METADATA_GROUP) } - let (mut sim, client) = cluster(0x5EED_0242); + let (mut sim, client) = cluster(0x5EED_0243); sim.register_client_with_primary(&client); WITHHELD_OP.store(0, Ordering::Relaxed); WITHHELD_DONES.store(0, Ordering::Relaxed); create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm"); + arm_withheld_prepare(&sim); let (_, _, warm_commit_min, _) = metadata_state(&sim, LAGGING); assert!( warm_commit_min > 0, @@ -14200,9 +14438,6 @@ mod metadata_repair_driver_tests { below would open at the group's first op" ); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(starve_walk_edges); @@ -14212,7 +14447,7 @@ mod metadata_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no metadata prepare crossed the chain link, so the fault never armed" + "no metadata prepare crossed the primary link, so the fault never armed" ); for _ in 0..STRAND_QUIET_STEPS { @@ -14259,16 +14494,14 @@ mod metadata_repair_driver_tests { withhold_one_metadata_prepare!(WITHHELD_OP); starve_commit_edge!(WITHHELD_OP); - let (mut sim, client) = cluster(0x5EED_0245); + let (mut sim, client) = cluster(0x5EED_0246); sim.register_client_with_primary(&client); WITHHELD_OP.store(0, Ordering::Relaxed); sim.replicas[LAGGING as usize].shards[0].set_repair_retry_ticks(ROTATE_RETRY_TICKS); create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm"); + arm_withheld_prepare(&sim); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(starve_commit_edge); @@ -14277,7 +14510,7 @@ mod metadata_repair_driver_tests { assert_ne!( WITHHELD_OP.load(Ordering::Relaxed), 0, - "no metadata prepare crossed the chain link, so the fault never armed" + "no metadata prepare crossed the primary link, so the fault never armed" ); // Corrupt the lowest op repair has put back above the commit point, @@ -14350,7 +14583,14 @@ mod metadata_repair_driver_tests { /// The peer `gap_repair_peer` names for a gap-stopped backup is the /// primary, so the session can only heal by rotating off it. - fn blackhole(_packet: &Packet) -> bool { + fn blackhole(packet: &Packet) -> bool { + if packet.message.header().command == Command::PrepareProgress { + return false; + } + if let Some(header) = prepare_for(packet, METADATA_GROUP) { + return withhold_one_prepare(packet) + || header.op == WITHHELD_OP.load(Ordering::Relaxed); + } true } @@ -14376,6 +14616,7 @@ mod metadata_repair_driver_tests { sim.replicas[LAGGING as usize].shards[0].set_repair_retry_ticks(ROTATE_RETRY_TICKS); create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm"); + arm_withheld_prepare(&sim); let (_, _, warm_commit_min, _) = metadata_state(&sim, LAGGING); assert!( warm_commit_min > 0, @@ -14383,13 +14624,9 @@ mod metadata_repair_driver_tests { below would open at the group's first op" ); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); // Everything, not just this group's commit edge: the primary must be // unable to answer the repair it is about to be asked for, while the - // chain (0 -> 1 -> 2) keeps carrying the prepares that advance - // `commit_max` past the hole. + // later direct prepares keep advancing `commit_max` past the hole. *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(blackhole); @@ -14404,7 +14641,7 @@ mod metadata_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no metadata prepare crossed the chain link, so the fault never armed" + "no metadata prepare crossed the primary link, so the fault never armed" ); for _ in 0..ROTATE_QUIET_STEPS { @@ -14453,6 +14690,7 @@ mod metadata_repair_driver_tests { /// direct reply, so no stall clock fires and no budget is charged: the /// replica never leaves that peer, even once another one can serve the hole. #[test] + #[allow(clippy::too_many_lines)] fn given_a_repair_peer_without_snapshot_when_it_answers_range_evicted_should_rotate_instead_of_transferring() { /// Where the session rotates off the primary (`next_transfer_peer`), and @@ -14463,7 +14701,14 @@ mod metadata_repair_driver_tests { static TRANSFER_REQUESTS: AtomicU64 = AtomicU64::new(0); withhold_one_metadata_prepare!(WITHHELD_OP); - fn blackhole(_packet: &Packet) -> bool { + fn blackhole(packet: &Packet) -> bool { + if packet.message.header().command == Command::PrepareProgress { + return false; + } + if let Some(header) = prepare_for(packet, METADATA_GROUP) { + return withhold_one_prepare(packet) + || header.op == WITHHELD_OP.load(Ordering::Relaxed); + } true } @@ -14475,13 +14720,14 @@ mod metadata_repair_driver_tests { false } - let (mut sim, client) = cluster(0x5EED_0246); + let (mut sim, client) = cluster(0x5EED_0247); sim.register_client_with_primary(&client); WITHHELD_OP.store(0, Ordering::Relaxed); TRANSFER_REQUESTS.store(0, Ordering::Relaxed); sim.replicas[LAGGING as usize].shards[0].set_repair_retry_ticks(ROTATE_RETRY_TICKS); create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm"); + arm_withheld_prepare(&sim); *sim.network .link_drop_packet_fn(ProcessId::Replica(SUCCESSOR), ProcessId::Replica(LAGGING)) = @@ -14498,7 +14744,7 @@ mod metadata_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no metadata prepare crossed the chain link, so the fault never armed" + "no metadata prepare crossed the primary link, so the fault never armed" ); // The hole a lost repair frame leaves (see `SimJournal::forget_op`). The @@ -14602,10 +14848,8 @@ mod metadata_repair_driver_tests { TRANSFER_REQUESTS.store(0, Ordering::Relaxed); create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm"); + arm_withheld_prepare(&sim); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); // Starved so no heartbeat arms the repair before the primary's hole exists. *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = @@ -14618,7 +14862,7 @@ mod metadata_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no metadata prepare crossed the chain link, so the fault never armed" + "no metadata prepare crossed the primary link, so the fault never armed" ); let (_, _, primary_commit_min, _) = metadata_state(&sim, 0); assert!( @@ -14703,10 +14947,8 @@ mod metadata_repair_driver_tests { TRANSFER_REQUESTS.store(0, Ordering::Relaxed); create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm"); + arm_withheld_prepare(&sim); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(starve_commit_edge); @@ -14720,7 +14962,7 @@ mod metadata_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no metadata prepare crossed the chain link, so the fault never armed" + "no metadata prepare crossed the primary link, so the fault never armed" ); for peer in [0, 1] { let (_, _, peer_commit_min, _) = metadata_state(&sim, peer); @@ -14805,6 +15047,7 @@ mod metadata_repair_driver_tests { .set_repair_retry_ticks(ROTATE_RETRY_TICKS); create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm"); + arm_withheld_prepare(&sim); *sim.network.link_drop_packet_fn( ProcessId::Replica(LAGGING_OF_FIVE - 1), @@ -14818,7 +15061,7 @@ mod metadata_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no metadata prepare crossed the chain link, so the fault never armed" + "no metadata prepare crossed the primary link, so the fault never armed" ); // A peer can learn the last gap commit from the primary's next `Commit`, // after the sends end. Waiting one more send interval keeps the gap phase @@ -15750,8 +15993,8 @@ mod metadata_read_frontier_tests { ); assert_eq!( before.header().status, - 0, - "the old primary may serve until its evidence expires: partition {:?}, fresh {}", + IggyError::TransientNotAccepted.as_code(), + "quorum loss must fence reads before metadata evidence expires: partition {:?}, fresh {}", sim.partition_consensus_state(0, namespace), partition_fresh(&sim, 0) ); @@ -15782,11 +16025,11 @@ mod metadata_read_frontier_tests { .consensus .as_ref() .unwrap(); - assert!( - old.is_primary(), - "the isolated node still believes it is primary" + assert_ne!( + old.status(), + Status::Normal, + "the minority cannot regain leadership" ); - assert_eq!(old.view(), 0); let convergence_start = sim.network.current_tick(); for _ in 0..SETUP_TOTAL_STEPS { if [2, 3, 4] @@ -16419,6 +16662,10 @@ mod applied_divergence_resync_tests { artifacts: Vec::new(), target_accepted: false, idle_ticks: 0, + spill: None, + scan: None, + seed: None, + pulled_len: 0, }); let consensus = partition.consensus(); consensus.set_state_transfer_stage(StateTransferStage::AwaitingTarget); @@ -16528,13 +16775,16 @@ mod metadata_walk_stuck_detector_tests { /// Swarm seed whose network elects replica 1 primary of view 1 with a /// committed pipeline head it has not yet walked. - const SEED: u64 = 1723; + const SEED: u64 = 70; const REPLICAS: u8 = 3; const CLIENTS: [u128; 3] = [1, 2, 3]; const VIEW_0_PRIMARY: u8 = 0; /// The replica the faults leave to win view 1. - const PRIMARY_ELECT: usize = 1; + const PRIMARY_ELECT: u8 = 1; + /// Holds the last registration as committed. Its `DoViewChange` would end + /// the view-1 change with that op committed and the pipeline empty. + const COMMITTED_PEER: u8 = 2; /// Replica faults as `(step, replica, crash)`. The view-0 primary goes /// down, which puts the view-1 change on the primary-elect, and comes back @@ -16543,7 +16793,7 @@ mod metadata_walk_stuck_detector_tests { [(130, VIEW_0_PRIMARY, true), (250, VIEW_0_PRIMARY, false)]; /// Past the inherited-head window, then the drain heals the cluster. - const FAULT_STEPS: usize = 800; + const FAULT_STEPS: usize = 1_000; const DRAIN_STEPS: usize = 4_000; /// Only the three `Register`s go through the metadata plane: no client @@ -16553,6 +16803,9 @@ mod metadata_walk_stuck_detector_tests { #[test] fn given_primary_elect_inheriting_a_committed_head_when_the_tick_probes_the_walk_should_keep_its_prepare() { + fn drop_do_view_change(packet: &packet::Packet) -> bool { + packet.message.header().command == Command::DoViewChange + } server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { enabled: false, size: iggy_common::IggyByteSize::from(0u64), @@ -16567,6 +16820,16 @@ mod metadata_walk_stuck_detector_tests { sim.register_client_with_primary(&SimClient::new(id)); } + // A view change needs a quorum of votes, so the peer that also lost the + // view-0 primary takes part in the view-1 change, and its + // `DoViewChange` would commit the last registration. Without it the + // primary-elect merges the log of the restarted view-0 primary, which + // holds that op above its commit point. + *sim.network.link_drop_packet_fn( + ProcessId::Replica(COMMITTED_PEER), + ProcessId::Replica(PRIMARY_ELECT), + ) = Some(drop_do_view_change); + let mut inherited_head_seen = false; for step in 0..FAULT_STEPS { for &(at, replica, crash) in &FAULTS { @@ -16581,6 +16844,10 @@ mod metadata_walk_stuck_detector_tests { sim.step(); inherited_head_seen |= primary_elect_holds_inherited_head(&sim); } + *sim.network.link_drop_packet_fn( + ProcessId::Replica(COMMITTED_PEER), + ProcessId::Replica(PRIMARY_ELECT), + ) = None; // Without this a drifted seed that never builds the shape passes the // drain assertion below without testing anything. assert!( @@ -16613,7 +16880,7 @@ mod metadata_walk_stuck_detector_tests { /// does not count. fn primary_elect_holds_inherited_head(sim: &Simulator) -> bool { let consensus = sim - .metadata_consensus(PRIMARY_ELECT) + .metadata_consensus(usize::from(PRIMARY_ELECT)) .expect("shard 0 hosts the metadata plane"); consensus.is_primary() && consensus.is_normal() diff --git a/core/simulator/src/packet.rs b/core/simulator/src/packet.rs index 30417e2004..05e909e952 100644 --- a/core/simulator/src/packet.rs +++ b/core/simulator/src/packet.rs @@ -77,6 +77,9 @@ pub struct Packet { pub message: Message, /// Tick at which this packet becomes deliverable. pub ready_at: u64, + /// Send order. With [`PacketSimulatorOptions::ordered`], each link + /// delivers in this order. + pub seq: u64, } impl Clone for Packet { @@ -86,6 +89,7 @@ impl Clone for Packet { to: self.to, message: self.message.deep_copy(), ready_at: self.ready_at, + seq: self.seq, } } } @@ -131,6 +135,10 @@ pub struct PacketSimulatorOptions { pub client_count: u8, /// PRNG seed for deterministic behavior. pub seed: u64, + /// Deliver each link in send order, `one_way_delay_min` ticks plus the + /// link's extra delay after the send, and never evict: one TCP connection + /// on an idle network. `one_way_delay_mean` and `link_capacity` do not apply. + pub ordered: bool, } impl Default for PacketSimulatorOptions { @@ -152,6 +160,7 @@ impl Default for PacketSimulatorOptions { node_count: 1, client_count: 0, seed: 0, + ordered: false, } } } @@ -214,6 +223,20 @@ impl PacketSimulatorOptions { } } +/// The oldest packet once it is ready, so a later packet never passes an +/// earlier one, even when the link delay changes between the two sends. +fn remove_oldest_ready(packets: &mut ReadyQueue, tick: u64) -> Option { + let (index, oldest) = packets + .as_slice() + .iter() + .enumerate() + .min_by_key(|(_, packet)| packet.seq)?; + if oldest.ready_at > tick { + return None; + } + Some(packets.remove_at(index)) +} + /// Uniform draw over an enum's variants. /// /// Generic rather than a `match` on a drawn index: a match would silently keep @@ -337,11 +360,13 @@ pub struct PacketSimulator { /// protocol it exercised, which is the difference between covering a path and /// merely compiling it. command_counts: [u64; COMMAND_COUNT_MAX], + /// [`Packet::seq`] of the next submitted packet. + next_seq: u64, } /// One past the highest [`Command`] discriminant, sizing [`COMMAND_LABELS`] and /// the delivery counters. Raising it is part of adding a command. -pub const COMMAND_COUNT_MAX: usize = 35; +pub const COMMAND_COUNT_MAX: usize = 37; /// Names for each [`Command`] discriminant, so a coverage report reads as /// protocol rather than as integers. Indexed by discriminant; the trailing @@ -382,6 +407,8 @@ pub const COMMAND_LABELS: [&str; COMMAND_COUNT_MAX] = [ "ForwardCompletion", "PrimaryRebalance", "SegmentDelete", + "CommitReply", + "PrepareProgress", ]; const _: () = { @@ -475,6 +502,7 @@ impl PacketSimulator { auto_partition_nodes: (0..node_count).collect(), delivered: Vec::new(), command_counts: [0; COMMAND_COUNT_MAX], + next_seq: 0, } } @@ -531,11 +559,15 @@ impl PacketSimulator { /// /// Always enqueues the packet. Loss and replay checks happen at delivery /// time in `step()`. If the link is at capacity, a random existing packet - /// is evicted to make room. + /// is evicted to make room, unless the network is ordered. pub fn submit(&mut self, from: ProcessId, to: ProcessId, message: Message) { let idx = self.link_index(from, to); - let delay = Self::calculate_delay(&mut self.prng, &self.options) - .saturating_add(self.links[idx].extra_delay); + let delay = if self.options.ordered { + self.options.one_way_delay_min + } else { + Self::calculate_delay(&mut self.prng, &self.options) + } + .saturating_add(self.links[idx].extra_delay); let ready_at = self.current_tick.saturating_add(delay); let packet = Packet { @@ -543,10 +575,14 @@ impl PacketSimulator { to, message, ready_at, + seq: self.next_seq, }; + self.next_seq += 1; // If at capacity, evict a random existing packet - if self.links[idx].packets.len() >= self.options.link_capacity as usize { + if !self.options.ordered + && self.links[idx].packets.len() >= self.options.link_capacity as usize + { self.links[idx].packets.remove_random(&mut self.prng); tracing::trace!(?from, ?to, "evicted random packet (link at capacity)"); } @@ -718,7 +754,12 @@ impl PacketSimulator { } loop { - let Some(packet) = link.packets.remove_ready(prng, *current_tick) else { + let next = if options.ordered { + remove_oldest_ready(&mut link.packets, *current_tick) + } else { + link.packets.remove_ready(prng, *current_tick) + }; + let Some(packet) = next else { break; }; diff --git a/core/simulator/src/replication_flow_tests.rs b/core/simulator/src/replication_flow_tests.rs new file mode 100644 index 0000000000..32e06d4fba --- /dev/null +++ b/core/simulator/src/replication_flow_tests.rs @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Flow control must not hold back the prepares of a backup that keeps up. +//! +//! The primary charges each prepare it sends to a backup against a window of +//! `REPLICATION_BATCH_BYTES_MAX` bytes. A progress probe follows the prepares on +//! the same ordered link, and its reply returns the credit of the bytes sent +//! before it. Probes go out at half a window and about once per second, and no +//! prepare waits for a probe reply. So a backup that keeps up never fills its +//! window, and no prepare goes out late or by replay. +//! +//! The network is ordered, like the TCP link between two replicas, and each hop +//! takes one tick. A produce takes four hops: the request, the prepare, the +//! prepare ok and the reply. A produce that takes longer waited for flow +//! control, and the other backup can hide that wait. So the flow counters must +//! also stay at zero. + +use bytes::Bytes; +use consensus::REPLICATION_BATCH_BYTES_MAX; +use server_common::sharding::IggyNamespace; + +use crate::Simulator; +use crate::client::SimClient; +use crate::packet::PacketSimulatorOptions; + +const SEED: u64 = 0x5245_504C; +const REPLICAS: u8 = 3; +/// Leads view 0. +const PRIMARY: u8 = 0; +const ONE_WAY_DELAY_TICKS: u64 = 1; +/// The request, the prepare, the prepare ok and the reply. +const PRODUCE_HOPS: u64 = 4; +const LATENCY_TICKS_MAX: u64 = PRODUCE_HOPS * ONE_WAY_DELAY_TICKS; +/// A client has one produce in flight, so one produce per tick needs more +/// clients than the slowest produce takes ticks. +const CLIENTS: u8 = 16; + +#[test] +fn given_steady_produce_when_it_passes_several_windows_should_answer_without_flow_control_waits() { + const REQUEST_BYTES: u64 = 512 * 1024; + /// Bytes sent to each backup, counted in windows. + const WINDOWS: u64 = 3; + assert_produce_latency( + REQUEST_BYTES, + WINDOWS * REPLICATION_BATCH_BYTES_MAX / REQUEST_BYTES, + ); +} + +#[test] +fn given_steady_produce_when_the_progress_timer_probes_a_backup_should_answer_without_flow_control_waits() + { + /// Never fills a window, so only the timer sends probes. + const REQUEST_BYTES: u64 = 1024; + /// The timer probes a backup 1 s (`PROGRESS_INTERVAL_MICROS`) after its + /// last probe reply: 100 ticks of 10 ms. + const PROBE_INTERVAL_TICKS: u64 = 100; + /// Four intervals, so the timer probes each backup more than once. + const TICKS: u64 = 4 * PROBE_INTERVAL_TICKS; + assert_produce_latency(REQUEST_BYTES, TICKS); +} + +/// Sends one produce per tick for `ticks` ticks, like a producer at a fixed +/// rate. Asserts that each one is answered within [`LATENCY_TICKS_MAX`], and +/// that flow control refused no live prepare and replayed none. +fn assert_produce_latency(request_bytes: u64, ticks: u64) { + server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { + enabled: false, + size: iggy_common::IggyByteSize::from(0u64), + bucket_capacity: 1, + }); + let network = PacketSimulatorOptions { + node_count: REPLICAS, + client_count: CLIENTS, + seed: SEED, + one_way_delay_min: ONE_WAY_DELAY_TICKS, + ordered: true, + ..PacketSimulatorOptions::default() + }; + let client_ids = 1..=u128::from(CLIENTS); + let mut sim = Simulator::new(usize::from(REPLICAS), client_ids.clone(), network); + let namespace = IggyNamespace::new(1, 1, 0); + sim.init_partition(namespace); + let clients: Vec = client_ids.map(SimClient::new).collect(); + for client in &clients { + sim.register_client_with_primary(client); + } + + let payload = Bytes::from(vec![0; usize::try_from(request_bytes).unwrap()]); + let mut sent_at: Vec> = vec![None; clients.len()]; + let mut late = Vec::new(); + // The produces sent last get the whole bound to finish. + for tick in 0..ticks + LATENCY_TICKS_MAX { + if tick < ticks + && let Some(idle) = sent_at.iter().position(Option::is_none) + { + let request = clients[idle].send_messages(namespace, std::slice::from_ref(&payload)); + sim.submit_request(clients[idle].client_id(), PRIMARY, request.into_generic()); + sent_at[idle] = Some(tick); + } + for (_, reply) in sim.step_with_senders() { + let header = reply.header(); + assert_eq!(header.status, 0, "produce failed at tick {tick}"); + let client = clients + .iter() + .position(|client| client.client_id() == header.client) + .expect("reply to a test client"); + let sent = sent_at[client].take().expect("one reply per produce"); + if tick - sent > LATENCY_TICKS_MAX { + late.push((sent, Some(tick - sent))); + } + } + } + late.extend(sent_at.into_iter().flatten().map(|sent| (sent, None))); + late.sort_unstable(); + assert!( + late.is_empty(), + "every produce must be answered within its {PRODUCE_HOPS} hops, so a slower one waited \ + for flow control. (tick sent, ticks to the reply or None without one): {late:?}" + ); + let shards = sim + .replicas + .iter() + .flat_map(|replica| replica.shards.iter()); + let (refusals, replayed) = shards.fold((0, 0), |(refusals, replayed), shard| { + let metrics = shard.metrics(); + ( + refusals + metrics.replication_live_refusals_value(), + replayed + metrics.replication_replayed_prepares_value(), + ) + }); + assert_eq!( + (refusals, replayed), + (0, 0), + "flow control must not refuse or replay prepares while every backup keeps up: \ + (live refusals, replayed prepares)" + ); +} diff --git a/core/simulator/src/slow_replica_tests.rs b/core/simulator/src/slow_replica_tests.rs index 0c22f42e33..cb5b60d900 100644 --- a/core/simulator/src/slow_replica_tests.rs +++ b/core/simulator/src/slow_replica_tests.rs @@ -19,17 +19,33 @@ //! //! Replica 2 is cut off for a moment, long enough to start a view change of its //! own, and every message it sends from then on takes longer than the -//! view-change status timeout to arrive. Its `StartViewChange` for the next -//! view therefore lands after it already timed out into the view after that. -//! Replicas 0 and 1 are Normal and commit together, yet each such -//! `StartViewChange` takes them out of their working view, and the primary -//! keeps moving between them. Nothing but latency is injected after the cut -//! heals: no packet is dropped. +//! view-change status timeout to arrive. A replica that times out into the +//! next view before its `StartViewChange` lands stays one view ahead of every +//! vote it sends, and each vote that lands takes replicas 0 and 1 out of their +//! working view while they commit together. Nothing but latency is injected +//! after the cut heals: no packet is dropped. +//! +//! Replica 2 can also hear a single peer late while every other link stays +//! fast. On a real network a slow egress of replica 2 delays the TCP +//! acknowledgements of the prepares replica 1 forwards to it down the chain, so +//! they queue on that link, and every frame replica 1 sends after them waits in +//! the same queue. Replica 2 then gives up on each view replica 1 leads before +//! its `StartView` arrives, and the rebalancer must not keep handing primaries +//! back to replica 1. +//! +//! A replica that stops hearing a primary its peers still hear must not depose +//! it, on either plane, while a primary that crashed or was cut off is still +//! replaced as fast as before. +use bytes::Bytes; +use consensus::{PartitionsHandle, Status, TimeoutManager}; +use iggy_binary_protocol::{Command, Operation, PrepareHeader, RequestPreparesHeader}; use server_common::sharding::IggyNamespace; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use crate::Simulator; -use crate::packet::ProcessId; +use crate::client::SimClient; +use crate::packet::{Packet, ProcessId}; use crate::partition_repair_driver_tests::cluster; use crate::workload::{ self, Workload, @@ -39,11 +55,30 @@ use crate::workload::{ const SEED: u64 = 0x5052_494D; const SLOW: u8 = 2; +/// Leads the single group of [`Run::single`] and the metadata plane. +const LEADER: u8 = 0; +/// The peer replica 2 hears late: its predecessor in the chain of every group +/// that replica 0 or 1 leads. +const LAGGING_PEER: u8 = 1; /// Groups the rebalancer spreads across the replicas, so each replica leads /// one before the fault. const GROUPS: usize = 3; /// Ticks for the rebalancer to spread the primaries. const SPREAD_TICKS: u64 = 10_000; +/// Ticks for a single group to settle under the workload before a fault, as +/// long as the spread, so the rebalancer had the same time to act. +const SETTLE_TICKS: u64 = SPREAD_TICKS; +/// Ten view-change status timeouts: a replica cut off this long times out +/// again and again. +const PARTITION_TICKS: u64 = 10 * TimeoutManager::VIEW_CHANGE_STATUS_TICKS; +/// A backup times out a heartbeat timeout after the last heartbeat it heard, +/// and that heartbeat left the primary at most one heartbeat interval before +/// the fault. Every backup heard one in that interval, so waiting for the +/// backup that times out last adds nothing to the bound. The view change +/// itself must finish within one retransmit interval of its votes. +const ELECTION_TICKS_MAX: u64 = TimeoutManager::NORMAL_HEARTBEAT_TICKS + + TimeoutManager::COMMIT_MESSAGE_TICKS + + TimeoutManager::START_VIEW_CHANGE_MESSAGE_TICKS; /// Past `NORMAL_HEARTBEAT_TICKS` (500): the cut lasts long enough that the /// group replica 2 led fails over and replica 2 starts a view change itself. const CUT_TICKS: u64 = 600; @@ -51,89 +86,441 @@ const CUT_TICKS: u64 = 600; /// `VIEW_CHANGE_STATUS_TICKS` (500). Below that bound the healthy pair adopts /// replica 2's view before it times out again, and the views settle. const EGRESS_DELAY_TICKS: u64 = 520; -/// Measured after the heal: 300 s of virtual time at 10 ms per tick. +/// One-way delay from the lagging peer to replica 2: 20 s, inside the 15 to +/// 55 s the queue took on a real cluster. Any delay past +/// `VIEW_CHANGE_STATUS_TICKS` (500) loses the `StartView` of that peer. +const LAGGING_PEER_DELAY_TICKS: u64 = 2_000; +/// Measured after the fault starts: 300 s of virtual time at 10 ms per tick. const MEASURE_TICKS: u64 = 30_000; -/// The heal itself may cost a few view changes while replica 2 rejoins: 450 -/// ticks of delay costs up to 3 here, 520 ticks costs about 60. +/// The heal itself may cost a few view changes while replica 2 rejoins: up to 2 +/// per group here with 450, 520 or 1200 ticks of delay. const VIEW_CHANGES_MAX: u32 = 5; +/// Summed over the groups, because the rebalancer backs off per target: one +/// view change when replica 2 leaves the view replica 1 leads at the start, +/// then two for each primary moved to replica 1 that replica 2 cannot follow. +/// The waits after the failed moves, 30, 60 and 120 s, leave room for four such +/// moves in the window, and a fifth would need 450 s. Without the backoff a +/// move fails every few seconds. +const LAGGING_PEER_VIEW_CHANGES_MAX: u32 = 9; #[test] -#[ignore = "known bug: a replica slower than the view-change status timeout keeps the healthy pair leaving its working view; run with --ignored"] -fn given_a_replica_slower_than_the_view_change_timeout_when_it_rejoins_should_not_keep_the_healthy_pair_changing_views() +#[allow(clippy::too_many_lines)] +fn given_a_missed_committed_prepare_when_pull_repair_is_unavailable_should_rejoin_live_delivery_under_load() { - let (mut sim, client) = cluster(SEED); - let namespaces: Vec<_> = (0..GROUPS) - .map(|partition| IggyNamespace::new(1, 1, partition)) - .collect(); - for &namespace in &namespaces { - sim.init_partition(namespace); + const WRITES: usize = 24; + const TICKS_PER_WRITE: usize = 250; + static GROUP: AtomicU64 = AtomicU64::new(0); + static MISSING: AtomicU64 = AtomicU64::new(0); + static WITHHOLD: AtomicBool = AtomicBool::new(true); + static DROPPED: AtomicU64 = AtomicU64::new(0); + + fn withhold_prepare(packet: &Packet) -> bool { + if !WITHHOLD.load(Ordering::Relaxed) || packet.message.header().command != Command::Prepare + { + return false; + } + let message = packet.message.try_as_typed::().unwrap(); + let header = message.header(); + if header.group == GROUP.load(Ordering::Relaxed) + && header.op == MISSING.load(Ordering::Relaxed) + { + DROPPED.fetch_add(1, Ordering::Relaxed); + return true; + } + false + } + + fn block_pull_repair(packet: &Packet) -> bool { + packet.message.header().command == Command::RequestPrepares + && packet + .message + .try_as_typed::() + .unwrap() + .header() + .group + == GROUP.load(Ordering::Relaxed) } + + let (mut sim, client) = cluster(SEED); + let namespace = IggyNamespace::new(1, 1, 0); + sim.init_partition(namespace); sim.register_client_with_primary(&client); - let mut options = WorkloadOptions::new(SEED, sim.replica_count, namespaces.clone()); - options.weights = ActionWeights::partition_only(); - let mut workload = Workload::new(options); - let clients = [client]; - let mut invariants = Invariants::new(); - let mut tick = |sim: &mut Simulator| { - workload::run(sim, &mut workload, &clients, 1, u64::MAX, &mut invariants) - }; - - for _ in 0..SPREAD_TICKS { - tick(&mut sim); - } - set_peer_links(&mut sim, false); + let warmup = client + .send_messages(namespace, &[Bytes::from_static(b"warmup")]) + .into_generic(); + let reply = sim.await_setup_reply( + &client, + LEADER, + &warmup, + Operation::SendMessages, + "warmup prefix", + ); + assert_eq!(reply.header().status, 0); + assert!((0..500).any(|_| { + sim.step(); + sim.partition_consensus_state(usize::from(SLOW), namespace) + .is_some_and(|state| state.commit_min >= 1) + })); + GROUP.store(namespace.inner(), Ordering::Relaxed); + MISSING.store(2, Ordering::Relaxed); + WITHHOLD.store(true, Ordering::Relaxed); + DROPPED.store(0, Ordering::Relaxed); + *sim.network + .link_drop_packet_fn(ProcessId::Replica(LEADER), ProcessId::Replica(SLOW)) = + Some(withhold_prepare); + for peer in [0, 1] { + *sim.network + .link_drop_packet_fn(ProcessId::Replica(SLOW), ProcessId::Replica(peer)) = + Some(block_pull_repair); + } + let repair_prepares = sim.network.command_counts()[Command::RepairPrepare as usize]; + let request = client + .send_messages(namespace, &[Bytes::from_static(b"missed")]) + .into_generic(); + let reply = sim.await_setup_reply( + &client, + LEADER, + &request, + Operation::SendMessages, + "missed prepare setup", + ); + assert_eq!(reply.header().status, 0); + assert!(DROPPED.load(Ordering::Relaxed) > 0); + assert!( + sim.replicas[usize::from(SLOW)] + .partition_shard(namespace) + .plane + .partitions() + .get_by_ns(&namespace) + .unwrap() + .consensus() + .commit_min() + < 2 + ); + WITHHOLD.store(false, Ordering::Relaxed); + + let mut caught_under_load = false; + for index in 0..WRITES { + let request = client + .send_messages(namespace, &[Bytes::from(format!("live-{index}"))]) + .into_generic(); + let reply = sim.await_setup_reply( + &client, + LEADER, + &request, + Operation::SendMessages, + "continued load", + ); + assert_eq!(reply.header().status, 0); + for _ in 0..TICKS_PER_WRITE { + sim.step(); + } + let state = sim + .partition_consensus_state(usize::from(SLOW), namespace) + .unwrap(); + caught_under_load |= index < WRITES - 1 && state.commit_min >= reply.header().op; + assert_eq!((state.status, state.view), (Status::Normal, 0)); + } + assert!( + caught_under_load, + "the replica must catch up before writes stop: source {:?}, backup {:?}, progress {:?}", + sim.partition_consensus_state(usize::from(LEADER), namespace), + sim.partition_consensus_state(usize::from(SLOW), namespace), + sim.replicas[usize::from(SLOW)] + .partition_shard(namespace) + .plane + .partitions() + .get_by_ns(&namespace) + .map(|partition| ( + partition.held_through(), + partition.replication_progress(), + partition.view_fenced(), + partition.repair + )), + ); + assert!(sim.network.delivered_any(Command::PrepareProgress)); + assert_eq!( + sim.network.command_counts()[Command::RepairPrepare as usize], + repair_prepares, + "the committed hole must heal through ordered live replay while pull repair is blocked" + ); +} + +#[test] +fn given_a_replica_slower_than_the_view_change_timeout_when_it_rejoins_should_not_keep_the_healthy_pair_changing_views() + { + let mut run = Run::spread(); + set_peer_links(&mut run.sim, SLOW, false); for _ in 0..CUT_TICKS { - tick(&mut sim); + run.tick(); } - set_peer_links(&mut sim, true); - for peer in (0..sim.replica_count).filter(|&peer| peer != SLOW) { - *sim.network + set_peer_links(&mut run.sim, SLOW, true); + for peer in (0..run.sim.replica_count).filter(|&peer| peer != SLOW) { + *run.sim + .network .link_extra_delay(ProcessId::Replica(SLOW), ProcessId::Replica(peer)) = EGRESS_DELAY_TICKS; } + let changes = run.measure_view_changes(); + for (namespace, &count) in run.namespaces.iter().zip(&changes) { + assert!( + count <= VIEW_CHANGES_MAX, + "replicas 0 and 1 changed view {count} times for {namespace:?} in {MEASURE_TICKS} \ + ticks, while only replica 2 was slow: one replica whose messages arrive later \ + than the view-change status timeout keeps the healthy pair leaving its working \ + view (changes per group {changes:?})" + ); + } +} - let first_views: Vec = namespaces - .iter() - .map(|&namespace| healthy_view(&sim, namespace)) - .collect(); - let mut views = first_views.clone(); - let mut changes = vec![0_u32; GROUPS]; - let mut replies = 0; - for _ in 0..MEASURE_TICKS { - replies += tick(&mut sim); - for (index, &namespace) in namespaces.iter().enumerate() { - let view = healthy_view(&sim, namespace); - if view != views[index] { - changes[index] += 1; - views[index] = view; - } - } +#[test] +fn given_a_replica_that_hears_one_peer_late_when_primaries_are_rebalanced_should_not_keep_the_healthy_pair_changing_views() + { + let mut run = Run::spread(); + *run.sim + .network + .link_extra_delay(ProcessId::Replica(LAGGING_PEER), ProcessId::Replica(SLOW)) = + LAGGING_PEER_DELAY_TICKS; + let changes = run.measure_view_changes(); + let total: u32 = changes.iter().sum(); + assert!( + total <= LAGGING_PEER_VIEW_CHANGES_MAX, + "replicas 0 and 1 changed view {total} times in {MEASURE_TICKS} ticks, while replica 2 \ + heard replica 1 late: replica 2 leaves every view replica 1 leads before it hears of \ + it, and each primary moved back to replica 1 starts the view changes again (changes \ + per group {changes:?})" + ); +} + +#[test] +fn given_a_replica_that_hears_the_primary_late_should_not_depose_it() { + let mut run = Run::single(); + let metadata_view = healthy_metadata_view(&run.sim); + *run.sim + .network + .link_extra_delay(ProcessId::Replica(LEADER), ProcessId::Replica(SLOW)) = + LAGGING_PEER_DELAY_TICKS; + + let changes = run.measure_view_changes(); + + assert_eq!( + (changes[0], healthy_metadata_view(&run.sim)), + (0, metadata_view), + "replica 2 heard replica 0 {LAGGING_PEER_DELAY_TICKS} ticks late, and replicas 0 and 1, \ + which hear it in time, followed replica 2 out of its view: (partition view changes, \ + metadata view)" + ); +} + +#[test] +fn given_a_backup_cut_off_for_many_view_change_timeouts_when_it_heals_should_not_change_the_view() { + let mut run = Run::single(); + let metadata_view = healthy_metadata_view(&run.sim); + let view = healthy_view(&run.sim, run.namespaces[0]); + set_peer_links(&mut run.sim, SLOW, false); + for _ in 0..PARTITION_TICKS { + run.tick(); } + set_peer_links(&mut run.sim, SLOW, true); + + let changes = run.measure_view_changes(); + + assert_eq!( + ( + healthy_view(&run.sim, run.namespaces[0]), + changes[0], + healthy_metadata_view(&run.sim) + ), + (view, 0, metadata_view), + "replica 2 was cut off for {PARTITION_TICKS} ticks, and after the heal replicas 0 and 1 \ + left the view of a primary they never stopped hearing: (partition view, partition view \ + changes after the heal, metadata view)" + ); +} + +#[test] +fn given_a_crashed_primary_should_elect_a_new_one_within_the_election_bound() { + let mut run = Run::single(); + run.sim.replica_crash(LEADER); + + assert!( + run.others_lead_within_election_bound(LEADER), + "replicas 1 and 2 did not elect a partition and a metadata primary within \ + {ELECTION_TICKS_MAX} ticks of the crash of replica 0" + ); +} + +#[test] +fn given_a_primary_cut_off_should_be_replaced_and_rejoin_as_a_backup() { + let mut run = Run::single(); + let namespace = run.namespaces[0]; + set_peer_links(&mut run.sim, LEADER, false); + assert!( - replies > 0, - "the workload must keep committing after the heal" + run.others_lead_within_election_bound(LEADER), + "replicas 1 and 2 did not elect a partition and a metadata primary within \ + {ELECTION_TICKS_MAX} ticks of cutting off replica 0" + ); + let views = run.views_of(LEADER + 1, namespace); + set_peer_links(&mut run.sim, LEADER, true); + for _ in 0..PARTITION_TICKS { + run.tick(); + } + + assert_eq!( + ( + run.views_of(LEADER + 1, namespace), + run.views_of(LEADER, namespace) + ), + (views, views), + "after the heal replica 0 must adopt the views replicas 1 and 2 elected without \ + another view change: ((partition, metadata) of replica 1, of replica 0)" ); - for (index, &namespace) in namespaces.iter().enumerate() { + let rejoined = run + .sim + .partition_consensus_state(usize::from(LEADER), namespace) + .is_some_and(|state| state.status == Status::Normal && !state.is_primary); + assert!(rejoined, "replica 0 must rejoin the partition as a backup"); +} + +/// The groups after the rebalancer spread them, with the workload that keeps +/// them committing. +struct Run { + sim: Simulator, + workload: Workload, + clients: [SimClient; 1], + invariants: Invariants, + namespaces: Vec, +} + +impl Run { + fn spread() -> Self { + Self::new(GROUPS, SPREAD_TICKS) + } + + /// One group that replica 0 leads, like the metadata plane. + fn single() -> Self { + let run = Self::new(1, SETTLE_TICKS); + assert_eq!( + ( + run.sim.primary_index(run.namespaces[0]), + run.sim.metadata_primary_index() + ), + (Some(LEADER), Some(LEADER)), + "replica 0 must lead the group and the metadata plane before the fault" + ); + run + } + + fn new(groups: usize, settle_ticks: u64) -> Self { + let (mut sim, client) = cluster(SEED); + let namespaces: Vec<_> = (0..groups) + .map(|partition| IggyNamespace::new(1, 1, partition)) + .collect(); + for &namespace in &namespaces { + sim.init_partition(namespace); + } + sim.register_client_with_primary(&client); + let mut options = WorkloadOptions::new(SEED, sim.replica_count, namespaces.clone()); + options.weights = ActionWeights::partition_only(); + let mut run = Self { + sim, + workload: Workload::new(options), + clients: [client], + invariants: Invariants::new(), + namespaces, + }; + for _ in 0..settle_ticks { + run.tick(); + } + run + } + + fn tick(&mut self) -> u64 { + workload::run( + &mut self.sim, + &mut self.workload, + &self.clients, + 1, + u64::MAX, + &mut self.invariants, + ) + } + + /// Runs `MEASURE_TICKS` under the fault already in place and returns how + /// often replicas 0 and 1 changed view in each group. + fn measure_view_changes(&mut self) -> Vec { + let mut views: Vec = self + .namespaces + .iter() + .map(|&namespace| healthy_view(&self.sim, namespace)) + .collect(); + let mut changes = vec![0_u32; self.namespaces.len()]; + let mut replies = 0; + for _ in 0..MEASURE_TICKS { + replies += self.tick(); + for (index, &namespace) in self.namespaces.iter().enumerate() { + let view = healthy_view(&self.sim, namespace); + if view != views[index] { + changes[index] += 1; + views[index] = view; + } + } + } assert!( - changes[index] <= VIEW_CHANGES_MAX, - "replicas 0 and 1 changed view {} times for {namespace:?} in {MEASURE_TICKS} ticks \ - (view {} -> {}), while only replica 2 was slow: one replica whose messages \ - arrive later than the view-change status timeout keeps the healthy pair leaving \ - its working view (changes per group {changes:?})", - changes[index], - first_views[index], - views[index], + replies > 0, + "the workload must keep committing under the fault" ); + changes + } + + /// Whether a replica other than `failed` becomes a `Normal` primary of the + /// group and of the metadata plane within `ELECTION_TICKS_MAX`. + fn others_lead_within_election_bound(&mut self, failed: u8) -> bool { + (0..ELECTION_TICKS_MAX).any(|_| { + self.tick(); + let mut others = (0..self.sim.replica_count).filter(|&replica| replica != failed); + let namespace = self.namespaces[0]; + others.clone().any(|replica| { + self.sim + .partition_consensus_state(usize::from(replica), namespace) + .is_some_and(|state| state.is_primary && state.status == Status::Normal) + }) && others.any(|replica| { + self.sim + .metadata_consensus(usize::from(replica)) + .is_some_and(|consensus| { + consensus.is_primary() && consensus.status() == Status::Normal + }) + }) + }) + } + + /// The partition and metadata views of `replica`. + fn views_of(&self, replica: u8, namespace: IggyNamespace) -> (Option, Option) { + ( + self.sim + .partition_consensus_state(usize::from(replica), namespace) + .map(|state| state.view), + self.sim + .metadata_consensus(usize::from(replica)) + .map(consensus::VsrConsensus::view), + ) } } -/// Cuts or restores every link between the slow replica and its peers. -fn set_peer_links(sim: &mut Simulator, enabled: bool) { - for peer in (0..sim.replica_count).filter(|&peer| peer != SLOW) { - sim.network - .set_link_filter(ProcessId::Replica(SLOW), ProcessId::Replica(peer), enabled); - sim.network - .set_link_filter(ProcessId::Replica(peer), ProcessId::Replica(SLOW), enabled); +/// Cuts or restores every link between `replica` and its peers. +fn set_peer_links(sim: &mut Simulator, replica: u8, enabled: bool) { + for peer in (0..sim.replica_count).filter(|&peer| peer != replica) { + sim.network.set_link_filter( + ProcessId::Replica(replica), + ProcessId::Replica(peer), + enabled, + ); + sim.network.set_link_filter( + ProcessId::Replica(peer), + ProcessId::Replica(replica), + enabled, + ); } } @@ -146,3 +533,13 @@ fn healthy_view(sim: &Simulator, namespace: IggyNamespace) -> u32 { .max() .unwrap_or(0) } + +/// The highest metadata view replicas 0 and 1 hold. +fn healthy_metadata_view(sim: &Simulator) -> u32 { + (0..sim.replica_count) + .filter(|&replica| replica != SLOW) + .filter_map(|replica| sim.metadata_consensus(usize::from(replica))) + .map(consensus::VsrConsensus::view) + .max() + .unwrap_or(0) +} diff --git a/core/simulator/src/stale_prepare_commit_tests.rs b/core/simulator/src/stale_prepare_commit_tests.rs index 3e92fb5213..dd8302d2f5 100644 --- a/core/simulator/src/stale_prepare_commit_tests.rs +++ b/core/simulator/src/stale_prepare_commit_tests.rs @@ -26,6 +26,7 @@ use bytes::Bytes; use consensus::{PartitionsHandle, Status}; use futures::FutureExt; +use iggy_binary_protocol::Command; use iggy_common::PollingStrategy; use partitions::{PollingArgs, PollingConsumer}; use server_common::send_messages::{BatchIntegrity, decode_batch_slice_with}; @@ -33,7 +34,7 @@ use server_common::sharding::IggyNamespace; use crate::Simulator; use crate::client::SimClient; -use crate::packet::ProcessId; +use crate::packet::{Packet, ProcessId}; use crate::partition_repair_driver_tests::{cluster, group_state}; const REPLICAS: u8 = 3; @@ -42,8 +43,6 @@ const STEPS_PER_SEND: usize = 12; /// Well past `NORMAL_HEARTBEAT_TICKS` (500), so the majority elects. const ELECTION_STEPS: usize = 5_000; const SETTLE_STEPS: usize = 5_000; -/// Long enough for a lone survivor's view changes to rotate onto the old primary. -const REJOIN_STEPS: usize = 40_000; fn send(sim: &mut Simulator, client: &SimClient, namespace: IggyNamespace, to: u8, payload: &str) { let request = client.send_messages(namespace, &[Bytes::from(payload.to_owned())]); @@ -152,13 +151,17 @@ fn given_an_old_primary_holding_an_uncommitted_prepare_when_it_rejoins_a_view_th // The new view commits different prepares at op 4 and beyond. for index in 0..4 { - send( - &mut sim, + let request = client + .send_messages(namespace, &[Bytes::from(format!("fresh-{index}"))]) + .into_generic(); + let reply = sim.await_setup_reply( &client, - namespace, new_primary, - &format!("fresh-{index}"), + &request, + iggy_binary_protocol::Operation::SendMessages, + "fresh view setup", ); + assert_eq!(reply.header().status, 0); } let (_, _, majority_commit, _) = group_state(&sim, new_primary, namespace); assert!( @@ -218,13 +221,17 @@ fn given_an_old_primary_holding_an_uncommitted_prepare_when_it_is_elected_over_a .expect("replicas 1 and 2 never elected a new view"); let new_primary = primary_of(&sim, 1, namespace, new_view); for index in 0..4 { - send( - &mut sim, + let request = client + .send_messages(namespace, &[Bytes::from(format!("fresh-{index}"))]) + .into_generic(); + let reply = sim.await_setup_reply( &client, - namespace, new_primary, - &format!("fresh-{index}"), + &request, + iggy_binary_protocol::Operation::SendMessages, + "fresh view setup", ); + assert_eq!(reply.header().status, 0); } let (_, _, majority_commit, _) = group_state(&sim, new_primary, namespace); assert!( @@ -232,19 +239,17 @@ fn given_an_old_primary_holding_an_uncommitted_prepare_when_it_is_elected_over_a "the new view committed only through {majority_commit}" ); - // Cut the new primary off and keep the old one apart until the survivor's view - // changes reach a view the old primary leads; then let those two meet. + // Cut the new primary off and let the old primary meet the survivor. The old + // primary's DoViewChange is lost on the way, so the view the survivor leads + // cannot start, and the next view, which the old primary leads, starts. let survivor = (0..REPLICAS) .find(|replica| *replica != OLD_PRIMARY && *replica != new_primary) .expect("three replicas"); isolate(&mut sim, new_primary, true); - (0..REJOIN_STEPS) - .find(|_| { - sim.step(); - let (_, view, _, _) = group_state(&sim, survivor, namespace); - primary_of(&sim, survivor, namespace, view) == OLD_PRIMARY - }) - .expect("the survivor never reached a view the old primary leads"); + *sim.network.link_drop_packet_fn( + ProcessId::Replica(OLD_PRIMARY), + ProcessId::Replica(survivor), + ) = Some(is_do_view_change); connect(&mut sim, OLD_PRIMARY, survivor); let led_view = (0..SETTLE_STEPS) .find_map(|_| { @@ -256,6 +261,10 @@ fn given_an_old_primary_holding_an_uncommitted_prepare_when_it_is_elected_over_a .then_some(view) }) .expect("the old primary never led a view: the primary-elect path did not run"); + *sim.network.link_drop_packet_fn( + ProcessId::Replica(OLD_PRIMARY), + ProcessId::Replica(survivor), + ) = None; isolate(&mut sim, new_primary, false); for _ in 0..SETTLE_STEPS { @@ -308,12 +317,18 @@ fn given_a_fenced_window_wider_than_a_repair_chunk_when_the_old_primary_rejoins_ .expect("replicas 1 and 2 never elected a new view"); let new_primary = primary_of(&sim, 1, namespace, new_view); for index in 0..12 { - send( - &mut sim, + let request = client.send_messages(namespace, &[Bytes::from(format!("fresh-{index}"))]); + let reply = sim.await_setup_reply( &client, - namespace, new_primary, - &format!("fresh-{index}"), + &request.into_generic(), + iggy_binary_protocol::Operation::SendMessages, + "fresh rejoin setup", + ); + assert_eq!( + reply.header().status, + 0, + "fresh operation {index} must commit before rejoining" ); } let (_, _, majority_commit, _) = group_state(&sim, new_primary, namespace); @@ -395,13 +410,17 @@ fn cut_to_catch_up(seed: u64) -> Option { .expect("replicas 1 and 2 never elected a new view"); let new_primary = primary_of(&sim, 1, namespace, new_view); for index in 0..6 { - send( - &mut sim, + let request = client + .send_messages(namespace, &[Bytes::from(format!("fresh-{index}"))]) + .into_generic(); + let reply = sim.await_setup_reply( &client, - namespace, new_primary, - &format!("fresh-{index}"), + &request, + iggy_binary_protocol::Operation::SendMessages, + "fresh view setup", ); + assert_eq!(reply.header().status, 0); } let (_, _, majority_commit, _) = group_state(&sim, new_primary, namespace); @@ -497,6 +516,10 @@ fn primary_of(sim: &Simulator, replica: u8, namespace: IggyNamespace, view: u32) .primary_index(view) } +fn is_do_view_change(packet: &Packet) -> bool { + packet.message.header().command == Command::DoViewChange +} + fn connect(sim: &mut Simulator, left: u8, right: u8) { sim.network .set_link_filter(ProcessId::Replica(left), ProcessId::Replica(right), true); diff --git a/core/simulator/src/workload/state_checker.rs b/core/simulator/src/workload/state_checker.rs index f134a9a2a2..b141471cbe 100644 --- a/core/simulator/src/workload/state_checker.rs +++ b/core/simulator/src/workload/state_checker.rs @@ -432,11 +432,17 @@ fn check_partition_owner( if header.operation == Operation::InstallConsumerGroupOwner { let install = InstallConsumerGroupOwnerRequest::decode_from(body) .expect("a committed owner installation decodes"); - if !owners + let retired = owners .retirement .as_ref() - .is_some_and(|catalog| catalog.retires(install.group_id)) - { + .is_some_and(|catalog| catalog.retires(install.group_id)); + // The partition keeps the higher generation, so a superseded install + // that commits after its successor changes no owner. + let superseded = owners + .installed + .get(&install.group_id) + .is_some_and(|installed| install.owner.generation <= installed.generation); + if !retired && !superseded { owners.installed.insert(install.group_id, install.owner); } return; @@ -908,6 +914,50 @@ mod tests { ); } + /// The partition keeps the higher generation when a superseded owner install + /// commits after its successor (`apply_consumer_group_owner` returns early), + /// so the successor still owns the group and its offset is fenced correctly. + #[test] + fn given_a_superseded_install_committed_after_its_successor_when_the_successor_commits_should_pass_the_oracle() + { + let superseded = ConsumerGroupOwner { + client_id: 0, + session: 0, + generation: 85, + }; + let successor = ConsumerGroupOwner { + client_id: 2, + session: 2, + generation: 88, + }; + let mut owners = PartitionOwners::default(); + for (op, owner) in [(19, successor), (24, superseded)] { + let install = InstallConsumerGroupOwnerRequest { + incarnation: 1, + group_id: 0, + owner, + metadata_op: owner.generation, + }; + check_partition_owner( + &mut owners, + &PrepareHeader { + operation: Operation::InstallConsumerGroupOwner, + op, + ..Default::default() + }, + &install.to_bytes(), + SEED, + ); + } + check_offset( + &mut owners, + successor.client_id, + successor.session, + successor.generation, + Operation::StoreConsumerOffset, + ); + } + fn installed_owner() -> PartitionOwners { PartitionOwners { installed: BTreeMap::from([( diff --git a/foreign/swift/Tests/IggyE2ETests/DeferredPollE2ETests.swift b/foreign/swift/Tests/IggyE2ETests/DeferredPollE2ETests.swift index 63834ddf79..022fa80443 100644 --- a/foreign/swift/Tests/IggyE2ETests/DeferredPollE2ETests.swift +++ b/foreign/swift/Tests/IggyE2ETests/DeferredPollE2ETests.swift @@ -71,10 +71,12 @@ struct DeferredPollE2ETests { streamID: scratch.stream, topicID: scratch.topic, consumer: .consumer(try Identifier(named: E2EEnvironment.uniqueName("swift-e2e-backlog"))), configuration: ReaderConfiguration(partitions: [0, 1], start: .first, options: PollOptions(count: 1, maxWait: .seconds(1)))) var payloads: [String] = [] - for _ in 0..<4 { + // Each fetch in flight takes a data connection of its own, so the + // backlog can answer a few times while the other fetch opens one. + while !payloads.contains("late"), payloads.count < backlog.count / 2 { payloads += try #require(try await reader.next()).messages.map(\.payloadString) } - #expect(payloads.contains("late"), "\(payloads)") + #expect(payloads.contains("late"), "\(payloads.count) messages came without it") try await reader.close() } }