From 39a2965dcb355f7cde39810d38f6dd5e012a5dce Mon Sep 17 00:00:00 2001 From: Hubert Gruszecki Date: Wed, 28 Oct 2026 12:06:21 +0100 Subject: [PATCH 01/10] fix(cluster)!: let slow replicas catch up under load A slow replica could throttle healthy peers, churn views and repeat snapshots after losing their required repair tails. Repair windows could also fill the WAL before a durable prefix was applied. Fan out prepares independently and replay peers in bounded batches with fenced progress replies. Bound replay work per shard tick and charge timeout retries to the same peer window. Apply durable repair prefixes and wait for WAL headroom before further trusted fetches. Recheck busy peers promptly without releasing their batch credit and drain full replay windows before backing off unavailable history. Reuse verified segments and fetch missing suffixes. Move hashing, index reconstruction and spills outside the shard pump with owned cancellation and installation guards. Pin admitted snapshot tails under the shared configurable cap, charge full retained allocations, and keep recovering nodes out of new primary placements. Require PreVote quorum before changing views and step down when fresh heartbeat replies no longer establish quorum. Preserve acknowledged operations without a WAL and reject superseded owner installs before sequencing. BREAKING CHANGE: CommitReply and PrepareProgress require a coordinated server upgrade. The client protocol is unchanged. --- core/binary_protocol/src/consensus/command.rs | 8 +- core/binary_protocol/src/consensus/header.rs | 173 +- core/binary_protocol/src/consensus/mod.rs | 19 +- core/binary_protocol/src/lib.rs | 24 +- core/configs/src/server_config/cluster.rs | 6 +- core/configs/src/server_config/defaults.rs | 4 + core/configs/src/server_config/partition.rs | 59 +- core/consensus/src/impls.rs | 1172 +++++- core/consensus/src/lib.rs | 2 + core/consensus/src/observability.rs | 10 + core/consensus/src/plane_helpers.rs | 390 +- core/consensus/src/replication.rs | 464 +++ core/consensus/src/vsr_timeout.rs | 10 +- .../tests/cluster/partition_state_transfer.rs | 503 ++- .../scenarios/delete_segments_scenario.rs | 2 + core/metadata/src/applied_frontier.rs | 68 +- core/metadata/src/impls/metadata.rs | 16 +- core/metadata/src/lib.rs | 2 +- core/partitions/src/iggy_partition.rs | 2763 ++++++++++++++- core/partitions/src/iggy_partitions.rs | 19 + core/partitions/src/install_backup.rs | 5 + core/partitions/src/io.rs | 60 +- core/partitions/src/journal.rs | 965 ++++- core/partitions/src/lib.rs | 7 +- core/partitions/src/log.rs | 10 + core/partitions/src/persistence.rs | 11 + core/partitions/src/repair_memory.rs | 141 + core/partitions/src/state_transfer.rs | 2010 ++++++++--- core/partitions/src/types.rs | 6 +- core/server/config.toml | 46 +- core/server/src/boot/mod.rs | 9 + core/server/src/boot/recovery.rs | 4 +- core/server/src/boot/threads.rs | 2 + core/server/src/dispatch/submit.rs | 17 +- core/server_common/src/consensus_message.rs | 62 +- core/server_common/src/iobuf.rs | 6 + core/shard/src/lib.rs | 1768 +++++++-- core/shard/src/partition_io.rs | 3154 +++++++++++++++-- core/shard/src/poll/completion_tests.rs | 4 + core/shard/src/primary_rebalance.rs | 116 +- core/shard/src/primary_rebalance/tests.rs | 136 + core/shard/src/router.rs | 1 + core/simulator/README.md | 2 +- core/simulator/src/lib.rs | 544 ++- core/simulator/src/packet.rs | 4 +- core/simulator/src/slow_replica_tests.rs | 535 ++- .../src/stale_prepare_commit_tests.rs | 79 +- core/simulator/src/workload/state_checker.rs | 56 +- 48 files changed, 13471 insertions(+), 2003 deletions(-) create mode 100644 core/consensus/src/replication.rs create mode 100644 core/partitions/src/repair_memory.rs diff --git a/core/binary_protocol/src/consensus/command.rs b/core/binary_protocol/src/consensus/command.rs index ea2f9451de..cf275f8e8a 100644 --- a/core/binary_protocol/src/consensus/command.rs +++ b/core/binary_protocol/src/consensus/command.rs @@ -96,6 +96,8 @@ pub enum Command { // Fresh metadata and partition readiness for primary rebalancing. PrimaryRebalance = 33, SegmentDelete = 34, + CommitReply = 35, + PrepareProgress = 36, } // SAFETY: Command is #[repr(u8)] with no padding bytes. @@ -106,7 +108,7 @@ unsafe impl CheckedBitPattern for Command { type Bits = u8; fn is_valid_bit_pattern(bits: &u8) -> bool { - *bits <= Self::SegmentDelete as u8 + *bits <= Self::PrepareProgress as u8 } } @@ -127,7 +129,7 @@ mod tests { #[test] fn replica_auth_commands_are_valid_bit_patterns() { - for command in 14u8..=34 { + for command in 14u8..=36 { let mut buf: AVec> = AVec::new(16); buf.resize(256, 0); buf[60] = command; @@ -135,7 +137,7 @@ mod tests { } let mut buf: AVec> = AVec::new(16); buf.resize(256, 0); - buf[60] = 35; + buf[60] = 37; assert!(bytemuck::checked::try_from_bytes::(&buf).is_err()); } } diff --git a/core/binary_protocol/src/consensus/header.rs b/core/binary_protocol/src/consensus/header.rs index 222c2b33e3..7f4fee0ecf 100644 --- a/core/binary_protocol/src/consensus/header.rs +++ b/core/binary_protocol/src/consensus/header.rs @@ -1499,6 +1499,160 @@ impl ConsensusHeader for CommitHeader { } } +/// Backup -> primary: echo a heartbeat counter for quorum liveness. +/// +/// This acknowledges connectivity, not journal or state-machine durability. +/// Nodes using this message require a coordinated cluster upgrade. +#[repr(C)] +#[derive(Debug, Clone, Copy, CheckedBitPattern, NoUninit)] +pub struct CommitReplyHeader { + pub checksum: u128, + pub checksum_body: u128, + pub cluster: u128, + pub size: u32, + pub view: u32, + pub release: u32, + pub command: Command, + pub replica: u8, + pub reserved_frame: [u8; 66], + pub timestamp_monotonic: u64, + pub group: u64, + pub partition_incarnation: u64, + pub reserved: [u8; 104], +} + +const _: () = { + assert!(size_of::() == HEADER_SIZE); + assert!(offset_of!(CommitReplyHeader, timestamp_monotonic) == 128); + assert!(offset_of!(CommitReplyHeader, partition_incarnation) == 144); + assert!(offset_of!(CommitReplyHeader, reserved) + size_of::<[u8; 104]>() == HEADER_SIZE); +}; + +impl ConsensusHeader for CommitReplyHeader { + const FRAME_SEALED: bool = true; + const COMMAND: Command = Command::CommitReply; + + fn checksum(&self) -> u128 { + self.checksum + } + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } + fn operation(&self) -> Operation { + Operation::Reserved + } + fn command(&self) -> Command { + self.command + } + fn size(&self) -> u32 { + self.size + } + + fn validate(&self) -> Result<(), ConsensusError> { + if self.command != Self::COMMAND { + return Err(ConsensusError::InvalidCommand { + expected: Self::COMMAND, + found: self.command, + }); + } + if usize::try_from(self.size) != Ok(HEADER_SIZE) { + return Err(ConsensusError::InvalidField( + "commit_reply: invalid size".to_string(), + )); + } + if self.release != 0 { + return Err(ConsensusError::InvalidField("release != 0".to_string())); + } + Ok(()) + } +} + +/// Correlates a replication batch with the backup's contiguous acknowledged prefix. +/// +/// Progress releases transport credit and never counts toward a commit quorum. +/// Replies echo the nonce and observed op; `ready` certifies the received prefix. +/// Probes advertise the primary's applied frontier for retention fallback. +/// Probes must stay ordered with prepares on the sender's queue. +/// Nodes using this message require a coordinated cluster upgrade. +#[repr(C)] +#[derive(Debug, Clone, Copy, CheckedBitPattern, NoUninit)] +pub struct PrepareProgressHeader { + pub checksum: u128, + pub checksum_body: u128, + pub cluster: u128, + pub size: u32, + pub view: u32, + pub release: u32, + pub command: Command, + pub replica: u8, + pub reserved_frame: [u8; 66], + pub nonce: u64, + pub group: u64, + pub partition_incarnation: u64, + pub observed_op: u64, + pub received_op: u64, + pub commit_op: u64, + pub kind: u8, + pub ready: u8, + pub reserved: [u8; 78], +} + +const _: () = { + assert!(size_of::() == HEADER_SIZE); + assert!(offset_of!(PrepareProgressHeader, nonce) == 128); + assert!(offset_of!(PrepareProgressHeader, partition_incarnation) == 144); + assert!(offset_of!(PrepareProgressHeader, reserved) + size_of::<[u8; 78]>() == HEADER_SIZE); +}; + +impl ConsensusHeader for PrepareProgressHeader { + const FRAME_SEALED: bool = true; + const COMMAND: Command = Command::PrepareProgress; + + fn checksum(&self) -> u128 { + self.checksum + } + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } + fn operation(&self) -> Operation { + Operation::Reserved + } + fn command(&self) -> Command { + self.command + } + fn size(&self) -> u32 { + self.size + } + + fn validate(&self) -> Result<(), ConsensusError> { + if self.command != Self::COMMAND { + return Err(ConsensusError::InvalidCommand { + expected: Self::COMMAND, + found: self.command, + }); + } + if usize::try_from(self.size) != Ok(HEADER_SIZE) { + return Err(ConsensusError::InvalidField( + "prepare_progress: invalid size".to_string(), + )); + } + if self.release != 0 { + return Err(ConsensusError::InvalidField("release != 0".to_string())); + } + if self.kind > Self::REPLY || self.ready > 1 { + return Err(ConsensusError::InvalidField( + "prepare_progress: invalid fields".to_string(), + )); + } + Ok(()) + } +} + +impl PrepareProgressHeader { + pub const PROBE: u8 = 0; + pub const REPLY: u8 = 1; +} + // StartViewChangeHeader - failure detection (header-only) /// Replica suspects primary failure. Header-only. @@ -3156,13 +3310,14 @@ mod tests { } use super::prepare_identity_checksum_bytes; use super::{ - Command, CommitHeader, ConsensusError, ConsensusHeader, DoViewChangeHeader, EvictionHeader, - EvictionReason, ForwardCompletionHeader, ForwardLogoutHeader, ForwardLogoutOutcome, - ForwardLogoutResultHeader, ForwardRegisterHeader, ForwardRegisterOutcome, - ForwardRegisterResultHeader, GenericHeader, HEADER_SIZE, LIFECYCLE_BATCH_HEADER_SIZE, - LIFECYCLE_COMPLETION_SIZE, LIFECYCLE_COMPLETIONS_MAX, Operation, PrepareHeader, - PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, - RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, + Command, CommitHeader, CommitReplyHeader, ConsensusError, ConsensusHeader, + DoViewChangeHeader, EvictionHeader, EvictionReason, ForwardCompletionHeader, + ForwardLogoutHeader, ForwardLogoutOutcome, ForwardLogoutResultHeader, + ForwardRegisterHeader, ForwardRegisterOutcome, ForwardRegisterResultHeader, GenericHeader, + HEADER_SIZE, LIFECYCLE_BATCH_HEADER_SIZE, LIFECYCLE_COMPLETION_SIZE, + LIFECYCLE_COMPLETIONS_MAX, Operation, PrepareHeader, PrepareOkHeader, + PrepareProgressHeader, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, + RequestHeader, RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, RoutedRequestHeader, StartViewChangeHeader, StartViewHeader, StateChunkHeader, StateTransferTargetHeader, }; @@ -3226,6 +3381,8 @@ mod tests { seals!( PrepareOkHeader, CommitHeader, + CommitReplyHeader, + PrepareProgressHeader, StartViewChangeHeader, DoViewChangeHeader, StartViewHeader, @@ -3292,6 +3449,8 @@ mod tests { } check_incarnation!( CommitHeader, + CommitReplyHeader, + PrepareProgressHeader, StartViewChangeHeader, DoViewChangeHeader, StartViewHeader, diff --git a/core/binary_protocol/src/consensus/mod.rs b/core/binary_protocol/src/consensus/mod.rs index 0953de3d22..491d1bdc2e 100644 --- a/core/binary_protocol/src/consensus/mod.rs +++ b/core/binary_protocol/src/consensus/mod.rs @@ -52,15 +52,16 @@ pub use consumer_session::{ }; pub use error::ConsensusError; pub use header::{ - CHECKSUM_UNSEALED, CommitHeader, ConsensusHeader, DVC_HEADERS_MAX, DoViewChangeHeader, - EvictionHeader, EvictionReason, ForwardCompletionHeader, ForwardLogoutHeader, - ForwardLogoutOutcome, ForwardLogoutResultHeader, ForwardRegisterHeader, ForwardRegisterOutcome, - ForwardRegisterResultHeader, GenericHeader, HEADER_SIZE, LIFECYCLE_BATCH_HEADER_SIZE, - LIFECYCLE_COMPLETION_SIZE, LIFECYCLE_COMPLETIONS_MAX, PrepareHeader, PrepareOkHeader, - RESERVED_COMMAND_LEN, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, - RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, - RequestStateTransferHeader, RoutedRequestHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader, - StartViewHeader, StateChunkHeader, StateTransferTargetHeader, frame_body, frame_checksum_bytes, + CHECKSUM_UNSEALED, CommitHeader, CommitReplyHeader, ConsensusHeader, DVC_HEADERS_MAX, + DoViewChangeHeader, EvictionHeader, EvictionReason, ForwardCompletionHeader, + ForwardLogoutHeader, ForwardLogoutOutcome, ForwardLogoutResultHeader, ForwardRegisterHeader, + ForwardRegisterOutcome, ForwardRegisterResultHeader, GenericHeader, HEADER_SIZE, + LIFECYCLE_BATCH_HEADER_SIZE, LIFECYCLE_COMPLETION_SIZE, LIFECYCLE_COMPLETIONS_MAX, + PrepareHeader, PrepareOkHeader, PrepareProgressHeader, RESERVED_COMMAND_LEN, + RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, RequestPreparesHeader, + RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, + RoutedRequestHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader, StartViewHeader, + StateChunkHeader, StateTransferTargetHeader, frame_body, frame_checksum_bytes, prepare_identity_checksum_bytes, read_size_field, }; pub use operation::Operation; diff --git a/core/binary_protocol/src/lib.rs b/core/binary_protocol/src/lib.rs index e8f31d8051..4705ddcd5f 100644 --- a/core/binary_protocol/src/lib.rs +++ b/core/binary_protocol/src/lib.rs @@ -70,20 +70,20 @@ pub mod version; pub use codec::{WireDecode, WireEncode}; pub use consensus::{ - CHECKSUM_UNSEALED, Command, CommitHeader, ConsensusError, ConsensusHeader, ConsumerSession, - ConsumerSessionHeartbeatHeader, DVC_HEADERS_MAX, DoViewChangeHeader, EvictionHeader, - EvictionReason, ForwardCompletionHeader, ForwardLogoutHeader, ForwardLogoutOutcome, - ForwardLogoutResultHeader, ForwardRegisterHeader, ForwardRegisterOutcome, + CHECKSUM_UNSEALED, Command, CommitHeader, CommitReplyHeader, ConsensusError, ConsensusHeader, + ConsumerSession, ConsumerSessionHeartbeatHeader, DVC_HEADERS_MAX, DoViewChangeHeader, + EvictionHeader, EvictionReason, ForwardCompletionHeader, ForwardLogoutHeader, + ForwardLogoutOutcome, ForwardLogoutResultHeader, ForwardRegisterHeader, ForwardRegisterOutcome, ForwardRegisterResultHeader, GenericHeader, HEADER_SIZE, LIFECYCLE_BATCH_HEADER_SIZE, LIFECYCLE_COMPLETION_SIZE, LIFECYCLE_COMPLETIONS_MAX, MAX_CONSUMER_SESSIONS_PER_HEARTBEAT, - Operation, PrepareHeader, PrepareOkHeader, PrimaryRebalanceHeader, PrimaryRebalanceKind, - REJECTION_SECTION_LEN, RESERVED_COMMAND_LEN, RepairPrepareHeader, RepairRangeReplyHeader, - ReplyHeader, RequestHeader, RequestPreparesHeader, RequestStartViewHeader, - RequestStateChunkHeader, RequestStateTransferHeader, RoutedRequestHeader, SIZE_FIELD_OFFSET, - SegmentDeleteHeader, SegmentDeleteKind, StartViewChangeHeader, StartViewHeader, - StateChunkHeader, StateTransferTargetHeader, frame_body, frame_checksum_bytes, - prepare_identity_checksum_bytes, read_size_field, result_code, result_section_len, - write_rejection_section, + Operation, PrepareHeader, PrepareOkHeader, PrepareProgressHeader, PrimaryRebalanceHeader, + PrimaryRebalanceKind, REJECTION_SECTION_LEN, RESERVED_COMMAND_LEN, RepairPrepareHeader, + RepairRangeReplyHeader, ReplyHeader, RequestHeader, RequestPreparesHeader, + RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, + RoutedRequestHeader, SIZE_FIELD_OFFSET, SegmentDeleteHeader, SegmentDeleteKind, + StartViewChangeHeader, StartViewHeader, StateChunkHeader, StateTransferTargetHeader, + frame_body, frame_checksum_bytes, prepare_identity_checksum_bytes, read_size_field, + result_code, result_section_len, write_rejection_section, }; pub use dispatch::{ COMMAND_TABLE, CommandMeta, command_index, lookup_by_operation, lookup_command, diff --git a/core/configs/src/server_config/cluster.rs b/core/configs/src/server_config/cluster.rs index e9ea5bd95e..63020d904b 100644 --- a/core/configs/src/server_config/cluster.rs +++ b/core/configs/src/server_config/cluster.rs @@ -200,9 +200,9 @@ fn default_superblock_wedged_fatal_timeout() -> IggyDuration { pub struct ClusterConfig { pub enabled: bool, pub name: String, - /// Backup-side liveness window for a plane's primary. A replica that - /// sees no primary traffic for this long starts a view change - /// (`normal_heartbeat_timeout`). Raise it on oversubscribed hosts where + /// Liveness window for the primary and its replication quorum. Backups + /// vote after primary silence; a primary without heartbeat replies from + /// a quorum relinquishes leadership. Raise it on oversubscribed hosts where /// scheduling stalls fake primary death; sub-`MIN_CLUSTER_HEARTBEAT_TIMEOUT` /// values (including the `0` / `disabled` / `unlimited` sentinels, which /// all parse to zero) are rejected at boot. On Linux it is also the diff --git a/core/configs/src/server_config/defaults.rs b/core/configs/src/server_config/defaults.rs index 13e5a6f119..e4fd326df6 100644 --- a/core/configs/src/server_config/defaults.rs +++ b/core/configs/src/server_config/defaults.rs @@ -193,6 +193,10 @@ impl Default for PartitionConfig { .wal_bytes_max .parse() .expect("embedded WAL capacity is valid"), + repair_memory_bytes_max: partition + .repair_memory_bytes_max + .parse() + .expect("embedded repair memory capacity is valid"), wal_group_commit_delay_micros: u64::try_from(partition.wal_group_commit_delay_micros) .expect("embedded WAL group commit delay is valid"), validate_checksum: SERVER_CONFIG.partition.validate_checksum, diff --git a/core/configs/src/server_config/partition.rs b/core/configs/src/server_config/partition.rs index b57d3be418..787dec53bf 100644 --- a/core/configs/src/server_config/partition.rs +++ b/core/configs/src/server_config/partition.rs @@ -118,9 +118,9 @@ fn default_offset_reservation_lease() -> NonZeroU32 { pub const DEFAULT_EVICTED_RING_CAPACITY: usize = 65536; /// Upper bound on `evicted_ring_capacity`. The ring exists per multi-replica -/// partition and each retained entry pins a full committed batch, so -/// worst-case pinned memory scales with the partition count; a typo guard, -/// not a sizing endorsement. +/// partition, and each retained entry pins a full committed batch until the +/// byte cap releases it to its segment, so worst-case pinned memory scales +/// with the partition count. A typo guard, not a sizing endorsement. pub const MAX_EVICTED_RING_CAPACITY: usize = 65536; /// Mirrors `partitions::EVICTED_RING_BYTES_MAX`. @@ -164,6 +164,11 @@ fn default_wal_bytes_max() -> IggyByteSize { IggyByteSize::from(DEFAULT_PARTITION_WAL_BYTES_MAX) } +fn default_repair_memory_bytes_max() -> IggyByteSize { + const DEFAULT_REPAIR_MEMORY_BYTES_MAX: u64 = 1024 * 1024 * 1024; + IggyByteSize::from(DEFAULT_REPAIR_MEMORY_BYTES_MAX) +} + /// Ceiling on the group-commit delay. A longer wait costs more than the /// barrier it is meant to amortize. pub const MAX_PARTITION_WAL_GROUP_COMMIT_DELAY_MICROS: u64 = 10_000; @@ -177,9 +182,19 @@ fn default_wal_group_commit_delay_micros() -> u64 { pub struct PartitionConfig { /// Active WAL and pending prepare budget per persisted partition. /// Temporary generation rewrites may require additional disk space. + /// + /// It also bounds the memory of a backup that repairs an unproven log from + /// the head down: that repair holds up to this many bytes of batches per + /// partition, subject to the shared `repair_memory_bytes_max` cap. #[serde(default = "default_wal_bytes_max")] #[config_env(leaf)] pub wal_bytes_max: IggyByteSize, + /// Node-wide allocation cap for unproven repair walks and snapshot repair tails. + /// Exhausted walks fall back to state transfer; exhausted tails require a fresh snapshot. + /// Normal prepare queues and state-transfer buffers have separate limits. + #[serde(default = "default_repair_memory_bytes_max")] + #[config_env(leaf)] + pub repair_memory_bytes_max: IggyByteSize, /// Bounded wait, in microseconds, for more prepares before a persisted /// partition's WAL writer starts its durability barrier. Zero disables it. /// @@ -248,13 +263,15 @@ pub struct PartitionConfig { /// restarting peer can be served from the ring before falling back to /// bulk sync, at the cost of pinned memory per partition. Must be > 0 and /// <= [`MAX_EVICTED_RING_CAPACITY`]. Single-replica partitions retain - /// nothing regardless. + /// nothing regardless. The entries whose batches went back to their + /// segments have their own cap of 16384. pub evicted_ring_capacity: usize, /// Byte ceiling for the evicted ring per partition; whichever ring cap /// (this or [`Self::evicted_ring_capacity`]) trips first evicts. Bounds /// the ring memory a burst of large batches can pin. Must be > 0 and <= - /// [`MAX_EVICTED_RING_BYTES`]. + /// [`MAX_EVICTED_RING_BYTES`]. The headers of the batches that went back to + /// their segments do not count here. #[config_env(leaf)] pub evicted_ring_bytes_max: IggyByteSize, @@ -282,6 +299,11 @@ pub struct PartitionConfig { impl Validatable for PartitionConfig { fn validate(&self) -> Result<(), ConfigurationError> { + let repair_bytes = self.repair_memory_bytes_max.as_bytes_u64(); + if repair_bytes == 0 || usize::try_from(repair_bytes).is_err() { + eprintln!("{COMPONENT} partition.repair_memory_bytes_max must fit in usize and be > 0"); + return Err(ConfigurationError::InvalidConfigurationValue); + } let wal_bytes = self.wal_bytes_max.as_bytes_u64(); if !(MIN_PARTITION_WAL_BYTES_MAX..=MAX_PARTITION_WAL_BYTES_MAX).contains(&wal_bytes) || !wal_bytes.is_multiple_of(4096) @@ -426,6 +448,33 @@ mod tests { ); } + #[test] + fn given_a_repair_memory_limit_when_loading_config_should_preserve_overrides_and_reject_zero() { + const CUSTOM_BYTES: u64 = 8 * 1024 * 1024; + let omitted: PartitionConfig = serde_json::from_str(&partial_table(None)).unwrap(); + assert_eq!( + omitted.repair_memory_bytes_max, + PartitionConfig::default().repair_memory_bytes_max, + "older configuration files use the shipped node-wide limit" + ); + let mut config = PartitionConfig { + repair_memory_bytes_max: IggyByteSize::from(CUSTOM_BYTES), + ..PartitionConfig::default() + }; + assert!(config.validate().is_ok()); + let restored: PartitionConfig = + serde_json::from_str(&serde_json::to_string(&config).unwrap()).unwrap(); + assert_eq!( + restored.repair_memory_bytes_max.as_bytes_u64(), + CUSTOM_BYTES + ); + config.repair_memory_bytes_max = IggyByteSize::from(0_u64); + assert!( + config.validate().is_err(), + "a zero repair budget must fail validation" + ); + } + #[test] fn default_impl_validates() { // `Default` reads the shipped config.toml; the pristine deployment diff --git a/core/consensus/src/impls.rs b/core/consensus/src/impls.rs index cea43028fb..433943ebed 100644 --- a/core/consensus/src/impls.rs +++ b/core/consensus/src/impls.rs @@ -1049,8 +1049,9 @@ impl StateTransferStage { /// drain the journal or correct a stale peer. #[derive(Debug, Clone, Copy)] pub enum CommitOutcome { - /// Nothing to do: the heartbeat was absorbed (or ignored as stale / - /// foreign / wrong-status) without moving `commit_max`. + /// Stale, foreign, or wrong-status heartbeat. Send no reply. + Ignored, + /// Valid heartbeat without a new commit point. Reply for quorum liveness. Accepted, /// `commit_max` advanced; run `commit_journal`. Advanced, @@ -1060,6 +1061,22 @@ pub enum CommitOutcome { RespondStartView, } +impl CommitOutcome { + /// Reply only to a heartbeat accepted from the current primary. + #[must_use] + pub const fn reply(self, header: &iggy_binary_protocol::CommitHeader) -> Option { + match self { + Self::Accepted | Self::Advanced => Some(VsrAction::SendCommitReply { + view: header.view, + group: header.group, + timestamp_monotonic: header.timestamp_monotonic, + target: header.replica, + }), + Self::Ignored | Self::RespondStartView => None, + } + } +} + /// Actions to be taken by the caller after processing a VSR event. #[derive(Debug, Clone)] pub enum VsrAction { @@ -1156,6 +1173,13 @@ pub enum VsrAction { timestamp_monotonic: u64, target: Option, }, + /// Echo a valid primary heartbeat without promising log durability. + SendCommitReply { + view: u32, + group: u64, + timestamp_monotonic: u64, + target: u8, + }, } impl VsrAction { @@ -1178,6 +1202,16 @@ impl VsrAction { } } +/// Whether a received `DoViewChange` is the empty vote of +/// [`VsrAction::is_empty_vote`]. +fn is_empty_vote(header: &DoViewChangeHeader, suffix: &DvcSuffix) -> bool { + header.untrusted_log() + && header.log_view == 0 + && header.op == 0 + && header.commit == 0 + && suffix.is_empty() +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrepareOkOutcome { Accepted { @@ -1199,13 +1233,8 @@ impl PrepareOkOutcome { } } -/// What the primary of the current view knows of each replica's log and has -/// told it of the commit point, so an eager `Commit` reaches a backup only once -/// that backup holds the op it announces. -/// -/// Chain replication forwards a prepare to the last backups after the quorum -/// commits it. A `Commit` sent to every backup at once overtakes the prepare -/// there, and the backup repairs an op that is still on its way to it. +/// Eager commits wait for each backup's confirmed durable progress. A peer +/// paused for progress must not repair fresh operations still waiting to be sent. #[derive(Debug)] struct CommitAnnouncements { /// Commit point the planes asked the backups to learn without waiting for @@ -1385,6 +1414,10 @@ where loopback_notifier: RefCell>, /// Tracks start view change messages received from all replicas (including self) start_view_change_from_all_replicas: RefCell>, + /// The view the votes in `start_view_change_from_all_replicas` are for. Never + /// below `view`, and above it while this replica votes to leave its view + /// without having left it, see [`Self::handle_start_view_change`]. + start_view_change_view: Cell, /// Consecutive unanswered `RequestStartView` probes while Recovering; /// at the `probe_attempts_max` ceiling the replica falls back to an /// election. @@ -1426,8 +1459,8 @@ where /// `StartView` or by starting a view as primary, whose merged log either /// rule makes safe. untrusted_log: Cell, - /// View-change escalations since [`Self::untrusted_log`] was last set, for - /// rate-limiting the halt warning. + /// View-change status timeouts since [`Self::untrusted_log`] was last set, + /// for rate-limiting the halt warning. untrusted_view_change_timeouts: Cell, /// This replica's own uncommitted suffix, with the head, commit point and @@ -1519,10 +1552,14 @@ where /// View and counter of the last emitted or accepted heartbeat. A new /// primary's counter can be lower, so freshness is scoped to its view. heartbeat_watermark: Cell<(u32, u64)>, + /// Only replies to counters emitted after this window began can count. + quorum_heartbeat_floor: Cell, + heartbeat_replies: RefCell>, /// The next tick looks for backups owed an eager `Commit`; see /// [`Self::request_eager_commit`]. eager_commit: Cell, commit_announcements: RefCell, + pub(crate) replication: RefCell, /// Views whose logs were refused for naming a different prepare at an op /// this replica applied; see [`Self::contradicts_applied_log`]. applied_log_contradictions: Cell, @@ -1892,6 +1929,7 @@ impl> VsrConsensus { loopback_queue: RefCell::new(VecDeque::with_capacity(prepare_queue_max)), loopback_notifier: RefCell::new(None), start_view_change_from_all_replicas: RefCell::new(BitSet::with_capacity(REPLICAS_MAX)), + start_view_change_view: Cell::new(0), probe_attempts: Cell::new(0), probe_attempts_max: Cell::new(PROBE_ATTEMPTS_MAX), recovery_election_allowed: Cell::new(true), @@ -1918,8 +1956,11 @@ impl> VsrConsensus { sent_own_do_view_change: Cell::new(false), timeouts: RefCell::new(TimeoutManager::new(timeout_seed)), heartbeat_watermark: Cell::new((0, 0)), + quorum_heartbeat_floor: Cell::new(0), + heartbeat_replies: RefCell::new(BitSet::with_capacity(REPLICAS_MAX)), eager_commit: Cell::new(false), commit_announcements: RefCell::new(CommitAnnouncements::new(replica_count)), + replication: RefCell::new(crate::replication::ReplicationFlow::new(replica_count)), applied_log_contradictions: Cell::new(0), applied_log_contradictions_taken: Cell::new(0), contradiction_reported_view: Cell::new(None), @@ -2028,6 +2069,7 @@ impl> VsrConsensus { pub fn init(&self) { self.status.set(Status::Normal); self.rejoined_on_recovered_log.set(false); + self.reset_quorum_window(); let mut timeouts = self.timeouts.borrow_mut(); if self.is_primary() { // Prepare is deliberately NOT armed here: a fresh primary has an @@ -2037,6 +2079,7 @@ impl> VsrConsensus { // `push_prepare_entry` arms it, timed from that push rather than from // boot. timeouts.start(TimeoutKind::CommitMessage); + timeouts.start(TimeoutKind::CheckQuorum); } else { timeouts.start(TimeoutKind::NormalHeartbeat); } @@ -2105,12 +2148,14 @@ impl> VsrConsensus { /// The primary of a Normal view, unless it ceded at boot /// (`init_as_backup`): then it is a backup by role however the view math /// reads, so it neither retransmits prepares nor announces commits. - fn is_acting_primary(&self) -> bool { + pub(crate) fn is_acting_primary(&self) -> bool { self.is_primary() && self.status.get() == Status::Normal && !self.ceded_primaryship.get() } /// Ask the partition quorum to elect `target` in the next view assigned to it. - /// A ready backup may initiate this too. Refusal leaves consensus unchanged. + /// Only the primary can: its `StartViewChange` leaves its view, so its backups + /// follow it, while that of a backup is only a vote. Refusal leaves consensus + /// unchanged. /// The caller fences the incarnation, drains history work, refreshes the DVC /// suffix and persists the new view before dispatching the returned actions. pub fn request_primary_transfer(&self, expected_view: u32, target: u8) -> Vec { @@ -2118,8 +2163,7 @@ impl> VsrConsensus { if self.group == METADATA_GROUP || target >= self.replica_count || view != expected_view - || !self.is_normal() - || self.has_ceded_primaryship() + || !self.is_acting_primary() || self.is_transferring() || self.view_log_unproven() || self.observed_newer_view.get() > view @@ -2362,6 +2406,7 @@ impl> VsrConsensus { pub fn set_view(&mut self, view: u32) { self.view.set(view); + self.start_view_change_view.set(view); } #[must_use] @@ -3128,9 +3173,8 @@ impl> VsrConsensus { /// `drain_request_queue_into_prepares` could project stale entries /// via `pipeline_prepare_common`, which panics on non-normal status. pub(crate) fn reset_view_change_state(&self) { - self.reset_svc_quorum(); + self.timeouts.borrow_mut().stop(TimeoutKind::CheckQuorum); self.reset_dvc_quorum(); - self.sent_own_start_view_change.set(false); self.sent_own_do_view_change.set(false); // A merge parked for the superseded view may describe a different log, so // drop it and let the new attempt re-derive from the DVCs it collects. @@ -3151,6 +3195,7 @@ impl> VsrConsensus { } /// Timers and retransmissions continue while storage leases defer history changes. + #[allow(clippy::too_many_lines)] pub fn tick_with_history_fence( &self, plane: PlaneKind, @@ -3163,6 +3208,11 @@ impl> VsrConsensus { timeouts.tick(); // Phase 2: Handle fired timeouts + if timeouts.fired(TimeoutKind::CheckQuorum) { + drop(timeouts); + self.handle_check_quorum_timeout(); + timeouts = self.timeouts.borrow_mut(); + } if !history_fenced && timeouts.fired(TimeoutKind::NormalHeartbeat) { drop(timeouts); actions.extend(self.handle_normal_heartbeat_timeout(plane)); @@ -3206,10 +3256,10 @@ impl> VsrConsensus { // primary answers or an election's StartView adopts us. // - ViewChange backup: the election may have concluded with our // copy of the StartView lost; re-requesting it is a - // two-message fix, while the ViewChangeStatus escalation - // backstop burns a fresh cluster-wide election. The would-be - // primary of the view skips the probe (it concludes the view - // itself or escalates). + // two-message fix, while the ViewChangeStatus backstop votes + // for a fresh cluster-wide election. The primary candidate of + // the view skips the probe (it concludes the view itself, or + // probes from its status timeout once it saw a newer view). match self.status.get() { Status::Recovering => { // A probe answered by nobody, repeatedly, means nobody is @@ -3301,8 +3351,8 @@ impl> VsrConsensus { actions } - /// Called when `normal_heartbeat` timeout fires. - /// Backup hasn't heard from primary - start view change. + /// Called when `normal_heartbeat` timeout fires: the backup did not hear its + /// primary, so it votes to replace it. fn handle_normal_heartbeat_timeout(&self, plane: PlaneKind) -> Vec { // A recovering replica makes progress through RequestStartView // retries, not elections; it is quorum-invisible. @@ -3350,7 +3400,12 @@ impl> VsrConsensus { return Vec::new(); } - self.start_election(plane, ViewChangeReason::NormalHeartbeatTimeout) + // The timer keeps running: each firing starts a fresh round of votes, and + // the heartbeat that resets it withdraws the vote. + self.timeouts + .borrow_mut() + .reset(TimeoutKind::NormalHeartbeat); + self.vote_for_next_view(plane, ViewChangeReason::NormalHeartbeatTimeout) } /// Advance to `view + 1` and start a view change (own SVC counted). @@ -3358,6 +3413,110 @@ impl> VsrConsensus { self.enter_view_change(plane, self.view.get() + 1, reason) } + /// Start a fresh round of votes to leave this view, for the next view or for + /// the higher one this replica already votes for. A round drops the votes of + /// the last one, so a peer that heard the primary again stops counting once it + /// stops sending its vote. + fn vote_for_next_view(&self, plane: PlaneKind, reason: ViewChangeReason) -> Vec { + let view = self.start_view_change_view.get().max(self.view.get() + 1); + tracing::info!( + replica = self.replica, + namespace_raw = self.group, + view = self.view.get(), + next_view = view, + reason = reason.as_str(), + "voting to leave the view" + ); + self.start_view_change_view.set(view); + self.reset_svc_quorum(); + self.count_view_change_vote(plane, view, None, reason) + } + + /// Count the vote of `voter` to leave this view for `view`, and the vote of + /// this replica. A view-change quorum of votes enters `view`. The votes carry + /// into it, so the `DoViewChange` goes out at once. + fn count_view_change_vote( + &self, + plane: PlaneKind, + view: u32, + voter: Option, + reason: ViewChangeReason, + ) -> Vec { + if view > self.start_view_change_view.get() { + self.start_view_change_view.set(view); + self.reset_svc_quorum(); + } + let (vote_count, own_vote_is_new) = { + let mut voters = self.start_view_change_from_all_replicas.borrow_mut(); + if let Some(voter) = voter { + voters.insert(usize::from(voter)); + } + let own_vote_is_new = voters.insert(usize::from(self.replica)); + (voters.count(), own_vote_is_new) + }; + if vote_count >= self.quorum_view_change() { + let mut actions = self.enter_view_change(plane, view, reason); + actions.extend(self.send_do_view_change_on_votes(plane)); + return actions; + } + if !own_vote_is_new { + return Vec::new(); + } + self.sent_own_start_view_change.set(true); + self.timeouts + .borrow_mut() + .start(TimeoutKind::StartViewChangeMessage); + let action = VsrAction::SendStartViewChange { + view, + group: self.group, + }; + emit_sim_event( + SimEventKind::ControlMessageScheduled, + &ControlActionLogEvent::from_vsr_action( + ReplicaLogContext::from_consensus(self, plane), + &action, + ), + ); + vec![action] + } + + /// Whether this replica votes to leave its view for `view`, a higher one, + /// with `voter`. + /// + /// A replica in normal operation that heard its primary since its last + /// heartbeat timeout refuses, and so does the primary itself, so a replica + /// that stops hearing a primary its peers still hear cannot depose it. A + /// replica that saw a newer view catches up through its probe instead, so its + /// vote never counts against a primary it did not hear. + fn joins_view_change_vote(&self, voter: u8, view: u32) -> bool { + let current = self.view.get(); + view >= self.start_view_change_view.get() + && !self.has_observed_newer_view() + && (self.status.get() == Status::ViewChange + || self.votes_to_leave_view() + // The primary left its view. + || voter == self.primary_index(current) + // The voter left this view already. Only a replica that entered a + // view alone gets there while its peers stay behind. + || view - current > 1) + } + + /// Whether this replica votes to leave a view it has not left yet. + const fn votes_to_leave_view(&self) -> bool { + self.start_view_change_view.get() > self.view.get() + } + + /// Drop the vote of this replica to leave its view, and the votes it counted + /// with it: it follows a primary of its view again. + fn withdraw_view_change_vote(&self) { + self.start_view_change_view.set(self.view.get()); + self.reset_svc_quorum(); + self.sent_own_start_view_change.set(false); + self.timeouts + .borrow_mut() + .stop(TimeoutKind::StartViewChangeMessage); + } + /// Enter `Status::ViewChange` at `new_view`: count this replica's own SVC, /// arm the view-change timers, and schedule the SVC broadcast. /// @@ -3380,6 +3539,11 @@ impl> VsrConsensus { ) -> Vec { let old_view = self.view.get(); + // Votes counted for `new_view` carry into it. + if self.start_view_change_view.get() != new_view { + self.start_view_change_view.set(new_view); + self.reset_svc_quorum(); + } self.view.set(new_view); self.status.set(Status::ViewChange); self.reset_view_change_state(); @@ -3421,7 +3585,7 @@ impl> VsrConsensus { vec![action] } - /// Resend SVC message if we've started view change. + /// Resend the vote of this replica, while it has one. fn handle_start_view_change_message_timeout(&self, plane: PlaneKind) -> Vec { if !self.sent_own_start_view_change.get() { return Vec::new(); @@ -3432,7 +3596,7 @@ impl> VsrConsensus { .reset(TimeoutKind::StartViewChangeMessage); let action = VsrAction::SendStartViewChange { - view: self.view.get(), + view: self.start_view_change_view.get(), group: self.group, }; emit_sim_event( @@ -3481,7 +3645,11 @@ impl> VsrConsensus { vec![action] } - /// Escalate to next view if stuck in view change. + /// Vote for the next view if this view change is stuck. The replica stays in + /// its view change, so a late `StartView` still lands, and leaves the view + /// only once a view-change quorum votes with it. The primary candidate of the + /// view does not probe, so once it saw a newer view it probes from here: the + /// cluster elected past this view without it. fn handle_view_change_status_timeout(&self, plane: PlaneKind) -> Vec { if self.status.get() != Status::ViewChange { return Vec::new(); @@ -3489,18 +3657,20 @@ impl> VsrConsensus { if self.untrusted_log.get() { self.warn_untrusted_halt(); } - - self.enter_view_change( - plane, - self.view.get() + 1, - ViewChangeReason::ViewChangeStatusTimeout, - ) + if self.has_observed_newer_view() { + self.begin_view_probe(); + return Vec::new(); + } + self.timeouts + .borrow_mut() + .reset(TimeoutKind::ViewChangeStatus); + self.vote_for_next_view(plane, ViewChangeReason::ViewChangeStatusTimeout) } /// No view starts without a quorum of trusted logs or every replica /// reporting, which looks like an ordinary stuck view change unless it says - /// so. Logged on the 1st, 2nd, 4th ... 64th escalation and every 64th after, - /// so a halt that lasts hours stays visible without a line per timeout. + /// so. Logged on the 1st, 2nd, 4th ... 64th status timeout and every 64th + /// after, so a halt that lasts hours stays visible without a line per timeout. fn warn_untrusted_halt(&self) { const STEADY_INTERVAL: u32 = 64; let timeouts = self.untrusted_view_change_timeouts.get().saturating_add(1); @@ -3734,9 +3904,14 @@ impl> VsrConsensus { /// Handle a received `StartViewChange` message. /// - /// "When replica i receives STARTVIEWCHANGE messages for its view-number - /// from f OTHER replicas, it sends a DOVIEWCHANGE message to the node - /// that will be the primary in the new view." + /// For a higher view it is a vote to leave this view for that one. This + /// replica joins the vote only as `Self::joins_view_change_vote` allows, + /// and enters the view once a view-change quorum voted for it, its own vote + /// included. A replica that votes stays in its view meanwhile. + /// + /// For the view this replica changes to, it sends its `DoViewChange` once a + /// view-change quorum voted for that view. With `2f + 1` replicas that is the + /// vote of `f` other replicas, as in the paper. /// /// # Panics /// If `header.group` does not match this replica's namespace. @@ -3760,31 +3935,40 @@ impl> VsrConsensus { let from_replica = header.replica; let msg_view = header.view; - // Ignore SVCs for old views - if msg_view < self.view.get() { - return Vec::new(); - } - - let mut actions = Vec::new(); - - // If SVC is for a higher view, advance to that view if msg_view > self.view.get() { - actions.extend(self.enter_view_change( + if !self.joins_view_change_vote(from_replica, msg_view) { + return Vec::new(); + } + return self.count_view_change_vote( plane, msg_view, + Some(from_replica), ViewChangeReason::ReceivedStartViewChange, - )); + ); } - // Record the SVC from sender + // Normal operation has no use for a vote for its own view, and a replica + // that votes for the next view dropped the votes for this one. + if msg_view < self.view.get() + || self.status.get() != Status::ViewChange + || self.votes_to_leave_view() + { + return Vec::new(); + } self.start_view_change_from_all_replicas .borrow_mut() .insert(from_replica as usize); + self.send_do_view_change_on_votes(plane) + } - // Check if we have f SVCs from OTHER replicas - // We need f SVCs from others to send DVC + /// Send the `DoViewChange` of this replica once a view-change quorum, this + /// replica included, voted for the view it changes to. A `DoViewChange` pulls + /// every replica into its view, so fewer votes than that can depose a primary + /// that the rest still hear. + fn send_do_view_change_on_votes(&self, plane: PlaneKind) -> Vec { + let mut actions = Vec::new(); if !self.sent_own_do_view_change.get() - && self.svc_count_excluding_self() >= self.max_faulty() + && self.svc_count_excluding_self() + 1 >= self.quorum_view_change() { self.sent_own_do_view_change.set(true); @@ -3977,13 +4161,31 @@ impl> VsrConsensus { let mut actions = Vec::new(); - // If DVC is for a higher view, advance to that view + // The sender of a real DVC for a higher view counted a view-change quorum + // of votes for that view. A barred replica sends its empty vote to answer + // any `StartViewChange`, a vote nobody joined included, so that one counts + // only as the vote of its sender. if msg_view > self.view.get() { - actions.extend(self.enter_view_change( - plane, - msg_view, - ViewChangeReason::ReceivedDoViewChange, - )); + if is_empty_vote(header, &msg_suffix) { + if !self.joins_view_change_vote(from_replica, msg_view) { + return actions; + } + actions.extend(self.count_view_change_vote( + plane, + msg_view, + Some(from_replica), + ViewChangeReason::ReceivedDoViewChange, + )); + if self.view.get() != msg_view { + return actions; + } + } else { + actions.extend(self.enter_view_change( + plane, + msg_view, + ViewChangeReason::ReceivedDoViewChange, + )); + } } // Only the primary candidate processes DVCs for quorum @@ -4063,10 +4265,14 @@ impl> VsrConsensus { ); self.status.set(Status::Recovering); self.probe_attempts.set(0); + self.withdraw_view_change_vote(); let mut timeouts = self.timeouts.borrow_mut(); timeouts.stop(TimeoutKind::Prepare); timeouts.stop(TimeoutKind::CommitMessage); + timeouts.stop(TimeoutKind::CheckQuorum); timeouts.stop(TimeoutKind::NormalHeartbeat); + timeouts.stop(TimeoutKind::DoViewChangeMessage); + timeouts.stop(TimeoutKind::ViewChangeStatus); timeouts.start(TimeoutKind::RequestStartViewMessage); } @@ -4169,9 +4375,9 @@ impl> VsrConsensus { /// backups stay silent and the prober retries. Special case: a probe /// FROM the replica that is the current view's primary-by-index proves /// that primary cannot lead (a probing replica has either lost its - /// state or abandoned the view), so a peer receiving it elects - /// immediately instead of waiting out the heartbeat timeout on a slot - /// known to be dead. + /// state or abandoned the view), so a peer receiving it votes for the + /// next view immediately instead of waiting out the heartbeat timeout on + /// a slot known to be dead. /// /// # Panics /// Panics when the probe is routed to the wrong group. @@ -4199,7 +4405,7 @@ impl> VsrConsensus { // through (a backup answers nothing, and the true primary of a // newer view answers with its StartView). if header.view == self.view.get() { - return self.start_election(plane, ViewChangeReason::PrimaryProbedView); + return self.vote_for_next_view(plane, ViewChangeReason::PrimaryProbedView); } return Vec::new(); } @@ -4263,6 +4469,31 @@ impl> VsrConsensus { } } + /// Adopt a state-transfer checkpoint at `commit_op`, whose prepare has + /// `checksum`, and clear the pipeline. The last prepare checksum becomes + /// `checksum` only when the head moves to `commit_op`: a head above it keeps + /// its own, and without a checksum it stays as it is. + /// + /// Raises the commit floor and never lowers the head. A head above the + /// checkpoint came from an adopted `StartView` or from prepares journaled + /// before the install wiped the journal, and the `DoViewChange` suffix still + /// reaches up to it, so a rewound sequencer would stamp a vote whose header op + /// disagrees with its own suffix. Journal repair fetches the ops between the + /// checkpoint and the head, as after a restart. + pub fn install_checkpoint(&self, commit_op: u64, checksum: Option) { + if commit_op > self.commit_min.get() { + self.set_commit_floor(commit_op); + } + if commit_op >= self.sequencer.current_sequence() { + self.sequencer.set_sequence(commit_op); + if let Some(checksum) = checksum { + self.set_last_prepare_checksum(checksum); + } + } + self.clear_pipeline(); + self.advance_commit_max(commit_op); + } + fn finish_view_probe(&self) { self.probe_attempts.set(0); self.timeouts @@ -4608,6 +4839,7 @@ impl> VsrConsensus { } self.advance_commit_max(msg_commit); self.reset_view_change_state(); + self.withdraw_view_change_vote(); // Stale pipeline entries from the old view must be discarded self.view_log_unproven.set(needs_reconciliation); @@ -4673,11 +4905,13 @@ impl> VsrConsensus { /// Handle a `Commit` (heartbeat) message from the primary. /// /// Advances `commit_max` and resets the backup's `NormalHeartbeat` timeout - /// so it doesn't start a spurious view change. Returns `true` if - /// `commit_max` advanced, signalling the caller to run `commit_journal`. + /// so it doesn't start a spurious view change. Returns + /// [`CommitOutcome::Advanced`] if `commit_max` advanced, signalling the + /// caller to run `commit_journal`. /// - /// Only refreshes liveness for a newer heartbeat within its view, so - /// old/replayed messages cannot suppress view changes. + /// Only refreshes liveness for a newer heartbeat within its view, so old or + /// replayed messages cannot suppress the `NormalHeartbeat` vote. Such a + /// heartbeat also withdraws the vote of this replica to replace the primary. /// /// # Panics /// If `header.group` does not match this replica's namespace. @@ -4707,11 +4941,11 @@ impl> VsrConsensus { if header.view > self.view.get() { self.observe_newer_view(header.view); } - return CommitOutcome::Accepted; + return CommitOutcome::Ignored; } if self.status.get() != Status::Normal { - return CommitOutcome::Accepted; + return CommitOutcome::Ignored; } if header.view != self.view.get() { @@ -4722,13 +4956,13 @@ impl> VsrConsensus { if header.view > self.view.get() { self.observe_newer_view(header.view); } - return CommitOutcome::Accepted; + return CommitOutcome::Ignored; } // Tolerant skip, not an assert: the replica handshake proves cluster // membership only, so nothing binds `header.replica` to the sender. if header.replica != self.primary_index(header.view) { - return CommitOutcome::Accepted; + return CommitOutcome::Ignored; } let heartbeat = (header.view, header.timestamp_monotonic); @@ -4737,6 +4971,9 @@ impl> VsrConsensus { self.timeouts .borrow_mut() .reset(TimeoutKind::NormalHeartbeat); + if self.votes_to_leave_view() { + self.withdraw_view_change_vote(); + } } let old_commit_max = self.commit_max.get(); @@ -4748,6 +4985,54 @@ impl> VsrConsensus { } } + /// Count a replica once for a heartbeat emitted during this quorum window. + pub fn handle_commit_reply(&self, header: &iggy_binary_protocol::CommitReplyHeader) { + if !self.is_acting_primary() + || header.cluster != self.cluster + || header.group != self.group + || header.partition_incarnation != self.partition_incarnation.get() + || header.view != self.view.get() + || header.replica >= self.replica_count + || header.replica == self.replica + || header.timestamp_monotonic <= self.quorum_heartbeat_floor.get() + || header.timestamp_monotonic > self.heartbeat_watermark.get().1 + { + return; + } + self.heartbeat_replies + .borrow_mut() + .insert(usize::from(header.replica)); + } + + fn reset_quorum_window(&self) { + self.quorum_heartbeat_floor + .set(self.heartbeat_watermark.get().1); + self.heartbeat_replies.borrow_mut().make_empty(); + } + + fn handle_check_quorum_timeout(&self) { + if !self.is_acting_primary() { + return; + } + let responding = self.heartbeat_replies.borrow().count() + 1; + if responding < self.quorum_replication() { + tracing::warn!( + replica = self.replica, + namespace_raw = self.group, + view = self.view.get(), + responding, + quorum = self.quorum_replication(), + "primary lost heartbeat quorum; relinquishing leadership and probing the view" + ); + // Probing stops admission and heartbeats without replacing history + // that an outstanding file lease may still own. + self.begin_view_probe(); + return; + } + self.reset_quorum_window(); + self.timeouts.borrow_mut().reset(TimeoutKind::CheckQuorum); + } + /// Complete view change as the new primary after collecting DVC quorum. /// /// # Client-table maintenance @@ -4806,8 +5091,8 @@ impl> VsrConsensus { // than its peers can be handed a range it cannot rebuild. // // Refuse rather than panic: a further DoViewChange can raise `commit_max` - // and shrink the range, and otherwise the status timeout escalates. A panic - // would restart into the same merge. + // and shrink the range, and otherwise the status timeout votes to leave the + // view. A panic would restart into the same merge. if merged.op_head.saturating_sub(merged.commit_max) > self.prepare_queue_max as u64 { tracing::error!( replica = self.replica, @@ -4951,6 +5236,11 @@ impl> VsrConsensus { } /// Re-establish the announced head after recovering the physical body log. + /// + /// A certificate that names the adopted view does not prove its head. A + /// state-transfer install and a fenced view window both certify only the + /// prefix the WAL holds, so a recovered head below the adopted one resumes + /// the view too. pub fn resume_adopted_view(&self) { let adopted = self.adopted_view.borrow(); let Some(adopted) = adopted.as_ref() else { @@ -4963,7 +5253,8 @@ impl> VsrConsensus { } else { self.log_view_proven.get() }; - if adopted.view > proven { + let below_adopted_head = self.sequencer.current_sequence() < adopted.op_head; + if adopted.view > proven || (adopted.view == proven && below_adopted_head) { self.log_view.set(adopted.view); self.view_log_unproven.set(true); self.view_log_unproven_through.set(u64::MAX); @@ -5292,14 +5583,19 @@ impl> VsrConsensus { self.log_view_proven.set(self.view.get()); } + // A vote for the next view, cast while the merge waited on repair, ends + // with the view this replica now leads. + self.withdraw_view_change_vote(); + // Update timeouts for normal primary operation + self.reset_quorum_window(); { let mut timeouts = self.timeouts.borrow_mut(); timeouts.stop(TimeoutKind::ViewChangeStatus); timeouts.stop(TimeoutKind::DoViewChangeMessage); - timeouts.stop(TimeoutKind::StartViewChangeMessage); timeouts.stop(TimeoutKind::RequestStartViewMessage); timeouts.start(TimeoutKind::CommitMessage); + timeouts.start(TimeoutKind::CheckQuorum); // If there are uncommitted ops in the rebuilt pipeline, start the // Prepare timeout so that lost PrepareOks trigger retransmission. if rebuild_after < new_op { @@ -5472,7 +5768,7 @@ impl> VsrConsensus { /// Record that `replica` holds `op`, and wake the next tick if that makes /// it owed a `Commit`. - fn note_backup_ack(&self, replica: u8, op: u64) { + pub(crate) fn note_backup_ack(&self, replica: u8, op: u64) { if replica == self.replica { return; } @@ -6226,7 +6522,7 @@ mod pipeline_entry_tests { /// Fixtures every consensus test module needs. #[cfg(test)] pub mod test_bus { - use super::{Command, METADATA_GROUP, Message, StartViewHeader}; + use super::{Command, METADATA_GROUP, Message, StartViewChangeHeader, StartViewHeader}; use iggy_binary_protocol::{CommitHeader, ConsensusHeader}; use message_bus::{BusMessage, MessageBus}; use server_common::MESSAGE_ALIGN; @@ -6284,6 +6580,43 @@ pub mod test_bus { ) } + #[must_use] + pub fn make_commit_reply( + view: u32, + replica: u8, + counter: u64, + ) -> Message { + make_commit(view, replica, counter).transmute_header( + |old, header: &mut iggy_binary_protocol::CommitReplyHeader| { + header.command = Command::CommitReply; + header.cluster = old.cluster; + header.size = old.size; + header.view = old.view; + header.replica = old.replica; + header.group = old.group; + header.timestamp_monotonic = old.timestamp_monotonic; + }, + ) + } + + /// A `StartViewChange` from `replica` that votes for `view`. + /// + /// # Panics + /// Never: the header size fits in `u32`. + #[must_use] + pub fn make_start_view_change(view: u32, replica: u8) -> Message { + Message::::new(std::mem::size_of::()) + .transmute_header(|_, header: &mut StartViewChangeHeader| { + header.command = Command::StartViewChange; + header.cluster = 1; + header.replica = replica; + header.view = view; + header.group = METADATA_GROUP; + header.size = u32::try_from(std::mem::size_of::()).unwrap(); + header.seal(); + }) + } + /// A [`MessageBus`] that accepts everything and remembers nothing. pub struct NoopBus; @@ -6319,7 +6652,7 @@ mod timestamp_clamp_tests { use super::*; use crate::LocalPipeline; - use crate::test_bus::{NoopBus, make_commit, make_start_view}; + use crate::test_bus::{NoopBus, make_commit, make_start_view, make_start_view_change}; use aligned_vec::{AVec, ConstAlign}; use iggy_binary_protocol::CommitHeader; @@ -6827,6 +7160,8 @@ mod timestamp_clamp_tests { const VIEW: u32 = 4; const FIRST_HEAD: u64 = 20; const NEXT_HEAD: u64 = FIRST_HEAD + 1; + /// The backup other than this replica, which votes with it. + const PEER: u8 = 2; let consensus = VsrConsensus::with_clock( 1, 0, @@ -6854,6 +7189,8 @@ mod timestamp_clamp_tests { .handle_normal_heartbeat_timeout(PlaneKind::Metadata) .is_empty() ); + let vote = make_start_view_change(VIEW + 1, PEER); + let _ = consensus.handle_start_view_change(PlaneKind::Metadata, vote.header()); assert_eq!(consensus.status(), Status::ViewChange); assert_eq!(consensus.view(), VIEW + 1); assert!(consensus.view_log_unproven()); @@ -7030,12 +7367,12 @@ mod vsr_consensus_tests { const REPLICAS: u8 = 5; const INCARNATION: u64 = 17; for (replica, view, target, next_view) in [ - (1, 0, 3, 3), - (3, 6, 3, 8), + (0, 0, 3, 3), + (1, 6, 3, 8), (1, 6, 4, 9), - (2, 9, 1, 11), - (2, 4, 0, 5), - (2, u32::MAX - 1, 0, u32::MAX), + (4, 9, 1, 11), + (4, 4, 0, 5), + (4, u32::MAX - 1, 0, u32::MAX), ] { let mut consensus = VsrConsensus::new(1, replica, REPLICAS, 0, StageNoopBus, LocalPipeline::new()); @@ -7063,45 +7400,12 @@ mod vsr_consensus_tests { } #[test] - fn given_backup_transfer_when_primary_receives_svc_should_join_the_target_election() { - const TARGET: u8 = 2; + fn given_a_backup_when_requesting_primary_transfer_should_refuse() { let backup = VsrConsensus::new(1, 1, 3, 0, StageNoopBus, LocalPipeline::new()); - let primary = VsrConsensus::new(1, 0, 3, 0, StageNoopBus, LocalPipeline::new()); backup.init(); - primary.init(); - assert!(!backup.request_primary_transfer(0, TARGET).is_empty()); - assert_eq!(primary.status(), Status::Normal); - - let message = Message::::new(size_of::()) - .transmute_header(|_, header: &mut StartViewChangeHeader| { - header.command = Command::StartViewChange; - header.cluster = 1; - header.replica = backup.replica(); - header.view = backup.view(); - header.group = backup.group(); - header.size = u32::try_from(size_of::()).unwrap(); - header.seal(); - }); - let actions = primary.handle_start_view_change(PlaneKind::Partitions, message.header()); - assert_eq!(primary.view(), u32::from(TARGET)); - assert_eq!(primary.status(), Status::ViewChange); - assert_eq!(primary.log_view(), 0); - assert!( - actions.iter().any(|action| matches!( - action, - VsrAction::SendStartViewChange { view: 2, group: 0 } - )) - ); - assert!(actions.iter().any(|action| matches!( - action, - VsrAction::SendDoViewChange { - target: TARGET, - view: 2, - .. - } - ))); - assert!(primary.pending_view_log().is_none()); + assert!(backup.request_primary_transfer(0, 2).is_empty()); + assert_eq!((backup.view(), backup.status()), (0, Status::Normal)); } #[test] @@ -7166,7 +7470,8 @@ mod vsr_consensus_tests { "recovery barrier", "view start head", ] { - let mut consensus = VsrConsensus::new(1, 2, 5, 0, StageNoopBus, LocalPipeline::new()); + // Replica 1 leads `VIEW`. + let mut consensus = VsrConsensus::new(1, 1, 5, 0, StageNoopBus, LocalPipeline::new()); consensus.set_view(VIEW); consensus.set_log_view(VIEW); consensus.mark_superblock_durable(VIEW, VIEW); @@ -7290,29 +7595,6 @@ mod vsr_consensus_tests { ); } - // No newer view seen means the primary is presumed dead, so the timeout - // must still elect -- the split must not swallow the ordinary case. - #[test] - fn heartbeat_timeout_with_no_newer_view_still_elects() { - let consensus = VsrConsensus::new(1, 1, 3, 0, StageNoopBus, LocalPipeline::new()); - consensus.init(); - - let actions = consensus.handle_normal_heartbeat_timeout(PlaneKind::Metadata); - - assert_eq!( - consensus.status(), - Status::ViewChange, - "a silent primary must still trigger an election" - ); - assert_eq!(consensus.view(), 1, "election advances the view"); - assert!( - actions - .iter() - .any(|action| matches!(action, VsrAction::SendStartViewChange { .. })), - "election must broadcast its SVC" - ); - } - // A recorded view at or below the current one is stale (already caught up) // and must not divert a genuine election, since the guard is a strict `>`. #[test] @@ -7321,12 +7603,13 @@ mod vsr_consensus_tests { consensus.init(); consensus.observe_newer_view(0); - let _ = consensus.handle_normal_heartbeat_timeout(PlaneKind::Metadata); + let actions = consensus.handle_normal_heartbeat_timeout(PlaneKind::Metadata); - assert_eq!( - consensus.status(), - Status::ViewChange, - "a stale observed view must not suppress the election" + assert!( + actions + .iter() + .any(|action| matches!(action, VsrAction::SendStartViewChange { view: 1, .. })), + "a stale observed view must not suppress the vote: {actions:?}" ); } @@ -7362,9 +7645,11 @@ mod vsr_consensus_tests { } #[test] - fn given_a_stalled_candidate_when_it_escalates_should_recover_a_lost_start_view() { + fn given_a_stalled_candidate_when_a_newer_view_settles_should_recover_a_lost_start_view() { const REJOINING_REPLICA: u8 = 1; const NEXT_PRIMARY: u8 = 2; + /// The view `NEXT_PRIMARY` settled while the candidate stalled. + const SETTLED_VIEW: u32 = 2; const REPLICA_COUNT: u8 = 3; const LOCAL_HEAD: u64 = 20; const PRIMARY_HEAD: u64 = LOCAL_HEAD + 1; @@ -7389,13 +7674,6 @@ mod vsr_consensus_tests { assert_eq!(backup.status(), Status::ViewChange); assert!(backup.is_primary_for_view(backup.view())); - for _ in 0..TimeoutManager::VIEW_CHANGE_STATUS_TICKS { - backup.tick(plane); - } - assert_eq!(backup.view(), 2); - assert_eq!(backup.status(), Status::ViewChange); - assert!(!backup.is_primary_for_view(backup.view())); - // Model a settled primary whose initial StartView was withheld by persistence. let mut primary = VsrConsensus::new( 1, @@ -7405,11 +7683,18 @@ mod vsr_consensus_tests { StageNoopBus, LocalPipeline::new(), ); - primary.set_view(backup.view()); - primary.set_log_view(backup.view()); + primary.set_view(SETTLED_VIEW); + primary.set_log_view(SETTLED_VIEW); primary.sequencer().set_sequence(PRIMARY_HEAD); primary.restore_commit_state(LOCAL_HEAD, LOCAL_HEAD); primary.init(); + // Its heartbeat names the settled view. + backup.observe_newer_view(SETTLED_VIEW); + + for _ in 0..TimeoutManager::VIEW_CHANGE_STATUS_TICKS { + backup.tick(plane); + } + assert_eq!((backup.view(), backup.status()), (1, Status::Recovering)); let probe = (0..TimeoutManager::REQUEST_START_VIEW_MESSAGE_TICKS) .flat_map(|_| backup.tick(plane)) @@ -7429,7 +7714,7 @@ mod vsr_consensus_tests { ), _ => None, }) - .expect("an escalated candidate must request the missing StartView as a backup"); + .expect("a candidate behind a settled view must request the missing StartView"); let replies = primary.handle_request_start_view(plane, probe.header()); let [ VsrAction::SendStartView { @@ -7670,7 +7955,17 @@ mod vsr_consensus_tests { VsrConsensus::new(1, 0, 3, METADATA_GROUP, StageNoopBus, LocalPipeline::new()); consensus.init(); for _ in 0..TimeoutManager::NORMAL_HEARTBEAT_TICKS { - let _ = consensus.tick(PlaneKind::Metadata); + for action in consensus.tick(PlaneKind::Metadata) { + if let VsrAction::SendCommit { + view, + timestamp_monotonic, + .. + } = action + { + let reply = test_bus::make_commit_reply(view, 1, timestamp_monotonic); + consensus.handle_commit_reply(reply.header()); + } + } } assert_eq!( consensus.heartbeat_watermark.get(), @@ -7698,8 +7993,8 @@ mod vsr_consensus_tests { } let _ = consensus.tick(PlaneKind::Metadata); assert_eq!( - consensus.view(), - 1, + (consensus.view(), consensus.votes_to_leave_view()), + (1, false), "fresh heartbeats must keep the adopted view alive at tick {tick}, \ enters_election={enters_election}" ); @@ -7709,9 +8004,9 @@ mod vsr_consensus_tests { let _ = consensus.tick(PlaneKind::Metadata); } assert_eq!( - consensus.view(), - 2, - "losing the new primary must still trigger an election" + (consensus.view(), consensus.votes_to_leave_view()), + (1, true), + "losing the new primary must still start a vote to replace it" ); } } @@ -7744,9 +8039,9 @@ mod vsr_consensus_tests { let _ = consensus.tick(PlaneKind::Metadata); } assert_eq!( - consensus.view(), - 2, - "replayed, older-view, and non-primary heartbeats must not suppress an election" + (consensus.view(), consensus.votes_to_leave_view()), + (1, true), + "replayed, older-view, and non-primary heartbeats must not suppress the vote" ); } @@ -8788,7 +9083,7 @@ mod recovery_election_tests { use crate::dvc_merge::suffix_all_present; use crate::test_bus::NoopBus; use crate::view_change_quorum::encode_prepare_headers; - use iggy_binary_protocol::{CommitHeader, HEADER_SIZE, Operation}; + use iggy_binary_protocol::{CommitHeader, CommitReplyHeader, HEADER_SIZE, Operation}; const REPLICA_COUNT: u8 = 3; /// Primary of `OLD_VIEW`, crashed holding only part of what it committed. @@ -8874,12 +9169,12 @@ mod recovery_election_tests { consensus } - /// A backup of `OLD_VIEW` holding the whole committed log, left in the view - /// change its primary's crash started and escalated while nobody else was up. - fn survivor_in_view_change(plane: PlaneKind) -> VsrConsensus { + /// A backup of `OLD_VIEW` holding the whole committed log, in normal + /// operation. + fn backup_in_old_view(replica: u8) -> VsrConsensus { let mut consensus = VsrConsensus::new( 1, - SURVIVOR, + replica, REPLICA_COUNT, group(), NoopBus, @@ -8893,10 +9188,20 @@ mod recovery_election_tests { (COMMITTED - 1..=COMMITTED).rev().map(prepare).collect(), )); consensus.init(); - while consensus.view() <= OLD_VIEW + 1 { + consensus + } + + /// A backup of `OLD_VIEW` holding the whole committed log, voting to replace + /// its crashed primary while nobody else is up to join the vote. + fn survivor_voting_alone(plane: PlaneKind) -> VsrConsensus { + let consensus = backup_in_old_view(SURVIVOR); + while !consensus.votes_to_leave_view() { consensus.tick(plane); } - assert_eq!(consensus.status(), Status::ViewChange); + assert_eq!( + (consensus.status(), consensus.view()), + (Status::Normal, OLD_VIEW) + ); consensus } @@ -9090,7 +9395,28 @@ mod recovery_election_tests { header.group = *group; header.timestamp_monotonic = *timestamp_monotonic; }); - let _ = receiver.handle_commit(heartbeat.header()); + receiver + .handle_commit(heartbeat.header()) + .reply(heartbeat.header()) + .into_iter() + .collect() + } + VsrAction::SendCommitReply { + view, + group, + timestamp_monotonic, + target, + } if *target == to => { + let reply = message(0, |header: &mut CommitReplyHeader| { + header.command = Command::CommitReply; + header.cluster = 1; + header.size = u32::try_from(HEADER_SIZE).unwrap(); + header.replica = from; + header.view = *view; + header.group = *group; + header.timestamp_monotonic = *timestamp_monotonic; + }); + receiver.handle_commit_reply(reply.header()); Vec::new() } _ => Vec::new(), @@ -9166,19 +9492,19 @@ mod recovery_election_tests { } /// Two replicas restart after a double crash, the primary having lost the - /// tail of what it committed. The survivor holding that tail is mid view - /// change, so it answers no probe. Its view-change traffic reaches the - /// restarted pair while they probe (and is ignored) and is then delayed past - /// their fallback election, which an asynchronous network permits. Whatever - /// the restarted pair decide without it must still contain the committed ops. + /// tail of what it committed. The survivor holding that tail is a backup, so + /// it answers no probe. Its votes reach the restarted pair while they probe + /// (and are ignored), and its traffic is then delayed past their fallback + /// election, which an asynchronous network permits. Whatever the restarted + /// pair decide without it must still contain the committed ops. #[test] - fn given_survivor_in_view_change_when_restarted_pair_elects_should_keep_committed_ops() { + fn given_survivor_voting_alone_when_restarted_pair_elects_should_keep_committed_ops() { let plane = PlaneKind::Partitions; let mut cluster = Cluster { replicas: vec![ restarted(RESTARTED_PRIMARY, RECOVERED_HEAD, false), restarted(LAGGING_BACKUP, LAGGING_HEAD, false), - survivor_in_view_change(plane), + survivor_voting_alone(plane), ], plane, do_view_changes: Vec::new(), @@ -9210,13 +9536,13 @@ mod recovery_election_tests { /// restarted pair may not settle a view without it, and once it reports /// the cluster settles holding every committed op. #[test] - fn given_survivor_in_view_change_when_it_stays_reachable_should_settle_with_committed_ops() { + fn given_survivor_voting_alone_when_it_stays_reachable_should_settle_with_committed_ops() { let plane = PlaneKind::Partitions; let mut cluster = Cluster { replicas: vec![ restarted(RESTARTED_PRIMARY, RECOVERED_HEAD, false), restarted(LAGGING_BACKUP, LAGGING_HEAD, false), - survivor_in_view_change(plane), + survivor_voting_alone(plane), ], plane, do_view_changes: Vec::new(), @@ -9229,6 +9555,64 @@ mod recovery_election_tests { cluster.assert_settled(COMMITTED); } + /// A primary that restarts with its whole log probes for its own view. Each + /// backup that gets the probe votes at once, so the group elects before a + /// heartbeat timeout. The backups never tick here, so neither can time out. + #[test] + fn given_a_restarted_primary_when_both_backups_get_its_probe_should_elect_without_a_heartbeat_timeout() + { + const NEXT_PRIMARY: u8 = 1; + let plane = PlaneKind::Partitions; + let mut cluster = Cluster { + replicas: vec![ + restarted(RESTARTED_PRIMARY, COMMITTED, true), + backup_in_old_view(NEXT_PRIMARY), + backup_in_old_view(SURVIVOR), + ], + plane, + do_view_changes: Vec::new(), + broadcast_do_view_changes: false, + delivered: 0, + }; + + let probe = (0..TimeoutManager::NORMAL_HEARTBEAT_TICKS) + .find_map(|_| { + cluster + .replica(RESTARTED_PRIMARY) + .tick(plane) + .into_iter() + .find(|action| matches!(action, VsrAction::SendRequestStartView { .. })) + }) + .expect("a restarted replica must probe before a heartbeat timeout"); + cluster.deliver(VecDeque::from([(RESTARTED_PRIMARY, probe)]), |_, _| true); + + cluster.assert_settled(COMMITTED); + assert!(cluster.replica(NEXT_PRIMARY).is_primary()); + } + + #[test] + fn given_a_one_way_partition_when_heartbeat_replies_cannot_return_should_elect_a_reachable_primary() + { + for plane in [PlaneKind::Metadata, PlaneKind::Partitions] { + let mut cluster = Cluster { + replicas: (0..REPLICA_COUNT).map(backup_in_old_view).collect(), + plane, + do_view_changes: Vec::new(), + broadcast_do_view_changes: true, + delivered: 0, + }; + cluster.run(TICKS, |_, to| to != RESTARTED_PRIMARY); + assert!(!cluster.replica(RESTARTED_PRIMARY).is_acting_primary()); + for replica in [LAGGING_BACKUP, SURVIVOR] { + assert_eq!(cluster.replica(replica).status(), Status::Normal); + assert_eq!(cluster.replica(replica).view(), OLD_VIEW + 1); + } + assert!(cluster.replica(LAGGING_BACKUP).is_primary()); + cluster.run(TICKS, |_, _| true); + cluster.assert_settled(COMMITTED); + } + } + /// The survivor is cut off for good. The restarted pair, a quorum by /// count, must stay unavailable rather than settle on their short logs, and /// recover as soon as the survivor is back. @@ -9239,7 +9623,7 @@ mod recovery_election_tests { replicas: vec![ restarted(RESTARTED_PRIMARY, RECOVERED_HEAD, false), restarted(LAGGING_BACKUP, LAGGING_HEAD, false), - survivor_in_view_change(plane), + survivor_voting_alone(plane), ], plane, do_view_changes: Vec::new(), @@ -9478,7 +9862,7 @@ mod recovery_election_tests { replicas: vec![ restarted(RESTARTED_PRIMARY, COMMITTED, true), restarted(LAGGING_BACKUP, LAGGING_HEAD, true), - survivor_in_view_change(plane), + survivor_voting_alone(plane), ], plane, do_view_changes: Vec::new(), @@ -9521,7 +9905,7 @@ mod recovery_election_tests { replicas: vec![ restarted(RESTARTED_PRIMARY, RECOVERED_HEAD, false), candidate, - survivor_in_view_change(plane), + survivor_voting_alone(plane), ], plane, do_view_changes: Vec::new(), @@ -9603,7 +9987,7 @@ mod recovery_election_tests { replicas: vec![ restarted(RESTARTED_PRIMARY, RECOVERED_HEAD, false), lagging, - survivor_in_view_change(plane), + survivor_voting_alone(plane), ], plane, do_view_changes: Vec::new(), @@ -9660,6 +10044,8 @@ mod recovery_election_tests { )); consensus.set_recovered_log_complete(false); consensus.init(); + // Its primary crashed, so it votes to replace it. + let _ = consensus.handle_normal_heartbeat_timeout(plane); consensus }; let view = OLD_VIEW + 1; @@ -9668,7 +10054,7 @@ mod recovery_election_tests { replicas: vec![ restarted(RESTARTED_PRIMARY, COMMITTED, false), candidate, - survivor_in_view_change(plane), + survivor_voting_alone(plane), ], plane, do_view_changes: Vec::new(), @@ -9802,7 +10188,7 @@ mod recovery_election_tests { replicas: vec![ probing(RESTARTED_PRIMARY), probing(LAGGING_BACKUP), - survivor_in_view_change(plane), + survivor_voting_alone(plane), ], plane, do_view_changes: Vec::new(), @@ -9961,7 +10347,7 @@ mod recovery_election_tests { replicas: vec![ restarted(RESTARTED_PRIMARY, RECOVERED_HEAD, false), candidate, - survivor_in_view_change(plane), + survivor_voting_alone(plane), ], plane, do_view_changes: Vec::new(), @@ -10047,6 +10433,379 @@ mod recovery_election_tests { } } +#[cfg(test)] +mod check_quorum_tests { + use super::*; + use crate::test_bus::NoopBus; + use iggy_binary_protocol::CommitReplyHeader; + + const WINDOW_TICKS: u64 = 8; + + fn primary(replicas: u8) -> VsrConsensus { + let consensus = VsrConsensus::new( + 1, + 0, + replicas, + METADATA_GROUP, + NoopBus, + LocalPipeline::new(), + ); + consensus + .timeouts + .borrow_mut() + .set_normal_heartbeat_ticks(WINDOW_TICKS); + consensus.timeouts.borrow_mut().set_commit_message_ticks(1); + consensus.init(); + consensus + } + + fn reply(replica: u8, counter: u64) -> CommitReplyHeader { + *test_bus::make_commit_reply(0, replica, counter).header() + } + + #[test] + fn given_idle_quorums_when_fresh_heartbeats_are_answered_should_retain_the_primary() { + for replicas in 1..=5 { + let consensus = primary(replicas); + for _ in 0..3 * WINDOW_TICKS { + for action in consensus.tick(PlaneKind::Metadata) { + if let VsrAction::SendCommit { + timestamp_monotonic, + .. + } = action + { + for replica in 1..u8::try_from(consensus.quorum_replication()).unwrap() { + consensus.handle_commit_reply(&reply(replica, timestamp_monotonic)); + } + } + } + assert!(consensus.is_acting_primary(), "{replicas} replicas"); + } + } + } + + #[test] + fn given_duplicate_replies_below_quorum_when_history_is_leased_should_stop_leading() { + let consensus = primary(5); + for _ in 0..WINDOW_TICKS { + for action in consensus.tick_with_history_fence(PlaneKind::Metadata, true) { + if let VsrAction::SendCommit { + timestamp_monotonic, + .. + } = action + { + for _ in 0..5 { + consensus.handle_commit_reply(&reply(1, timestamp_monotonic)); + } + } + } + } + assert_eq!(consensus.status(), Status::Recovering); + assert_eq!( + consensus.view(), + 0, + "step-down must not replace leased history" + ); + assert!( + !consensus + .timeouts + .borrow() + .is_ticking(TimeoutKind::CommitMessage) + ); + assert!( + !consensus + .timeouts + .borrow() + .is_ticking(TimeoutKind::CheckQuorum) + ); + } + + #[test] + fn given_a_delayed_reply_when_replayed_in_the_next_window_should_not_extend_leadership() { + let consensus = primary(3); + let _ = consensus.commit_heartbeat(); + let delayed = reply(1, 1); + let _ = consensus.commit_heartbeat(); + consensus.handle_commit_reply(&delayed); + for _ in 0..WINDOW_TICKS { + consensus.tick(PlaneKind::Metadata); + } + assert!( + consensus.is_acting_primary(), + "a reply need not echo the latest broadcast" + ); + for _ in 0..WINDOW_TICKS { + consensus.handle_commit_reply(&delayed); + consensus.tick(PlaneKind::Metadata); + } + assert_eq!( + consensus.status(), + Status::Recovering, + "a previous-window reply cannot count again" + ); + } + + #[test] + fn given_foreign_or_unsent_replies_when_a_window_expires_should_not_count_them() { + let valid = reply(1, 1); + let invalid = [ + CommitReplyHeader { + cluster: 2, + ..valid + }, + CommitReplyHeader { + group: METADATA_GROUP + 1, + ..valid + }, + CommitReplyHeader { + partition_incarnation: 1, + ..valid + }, + CommitReplyHeader { view: 1, ..valid }, + CommitReplyHeader { + replica: 0, + ..valid + }, + CommitReplyHeader { + replica: 3, + ..valid + }, + CommitReplyHeader { + timestamp_monotonic: 0, + ..valid + }, + CommitReplyHeader { + timestamp_monotonic: 2, + ..valid + }, + ]; + for header in invalid { + let consensus = primary(3); + let _ = consensus.commit_heartbeat(); + consensus.handle_commit_reply(&header); + for _ in 0..WINDOW_TICKS { + consensus.tick(PlaneKind::Metadata); + } + assert_eq!(consensus.status(), Status::Recovering, "{header:?}"); + } + } +} + +#[cfg(test)] +mod view_change_vote_tests { + //! A replica that stops hearing its primary votes to replace it, and leaves + //! its view only once a view-change quorum votes with it. + + use super::*; + use crate::LocalPipeline; + use crate::test_bus::{NoopBus, make_commit, make_start_view_change}; + + const REPLICA_COUNT: u8 = 3; + const VIEW: u32 = 0; + const NEXT_VIEW: u32 = VIEW + 1; + /// Leads `VIEW`. + const PRIMARY: u8 = 0; + /// Leads `NEXT_VIEW`. + const CANDIDATE: u8 = 1; + const BACKUP: u8 = 2; + const PLANE: PlaneKind = PlaneKind::Metadata; + + fn replica(replica: u8) -> VsrConsensus { + replica_of(REPLICA_COUNT, replica) + } + + fn replica_of(replica_count: u8, replica: u8) -> VsrConsensus { + let consensus = VsrConsensus::new( + 1, + replica, + replica_count, + METADATA_GROUP, + NoopBus, + LocalPipeline::new(), + ); + consensus.init(); + consensus + } + + /// A replica that entered `NEXT_VIEW` alone, because nobody answered its view + /// probe. + fn alone_in_next_view(replica_count: u8, replica: u8) -> VsrConsensus { + let consensus = replica_of(replica_count, replica); + consensus.begin_view_probe(); + let probing = + TimeoutManager::REQUEST_START_VIEW_MESSAGE_TICKS * u64::from(PROBE_ATTEMPTS_MAX); + let actions = (0..probing) + .flat_map(|_| consensus.tick(PLANE)) + .collect::>(); + assert!(!sends_do_view_change(&actions), "{actions:?}"); + assert_eq!( + (consensus.status(), consensus.view()), + (Status::ViewChange, NEXT_VIEW) + ); + consensus + } + + fn start_view_change( + consensus: &VsrConsensus, + from: u8, + view: u32, + ) -> Vec { + consensus.handle_start_view_change(PLANE, make_start_view_change(view, from).header()) + } + + fn votes_for(actions: &[VsrAction], view: u32) -> bool { + actions.iter().any(|action| { + matches!(action, VsrAction::SendStartViewChange { view: voted, .. } if *voted == view) + }) + } + + fn sends_do_view_change(actions: &[VsrAction]) -> bool { + actions + .iter() + .any(|action| matches!(action, VsrAction::SendDoViewChange { .. })) + } + + #[test] + fn given_a_heartbeat_timeout_should_vote_for_the_next_view_without_leaving_its_own() { + let consensus = replica(BACKUP); + + let actions = consensus.handle_normal_heartbeat_timeout(PLANE); + + assert!(votes_for(&actions, NEXT_VIEW), "{actions:?}"); + assert_eq!(consensus.status(), Status::Normal); + assert_eq!(consensus.view(), VIEW); + } + + #[test] + fn given_a_backup_that_hears_its_primary_when_a_peer_votes_should_not_join() { + let consensus = replica(BACKUP); + + let actions = start_view_change(&consensus, CANDIDATE, NEXT_VIEW); + + assert!(actions.is_empty(), "{actions:?}"); + assert_eq!(consensus.status(), Status::Normal); + assert_eq!(consensus.view(), VIEW); + } + + #[test] + fn given_its_vote_when_a_peer_votes_too_should_enter_the_view_change_and_send_its_do_view_change() + { + let consensus = replica(BACKUP); + let _ = consensus.handle_normal_heartbeat_timeout(PLANE); + + let actions = start_view_change(&consensus, CANDIDATE, NEXT_VIEW); + + assert!(sends_do_view_change(&actions), "{actions:?}"); + assert_eq!(consensus.status(), Status::ViewChange); + assert_eq!(consensus.view(), NEXT_VIEW); + } + + #[test] + fn given_its_vote_when_the_primary_heartbeats_should_withdraw_the_vote() { + let consensus = replica(BACKUP); + let _ = consensus.handle_normal_heartbeat_timeout(PLANE); + + let _ = consensus.handle_commit(make_commit(VIEW, PRIMARY, 1).header()); + + let resent = (0..TimeoutManager::START_VIEW_CHANGE_MESSAGE_TICKS) + .flat_map(|_| consensus.tick(PLANE)) + .collect::>(); + assert!(!votes_for(&resent, NEXT_VIEW), "{resent:?}"); + let actions = start_view_change(&consensus, CANDIDATE, NEXT_VIEW); + assert!(actions.is_empty(), "{actions:?}"); + assert_eq!(consensus.status(), Status::Normal); + assert_eq!(consensus.view(), VIEW); + } + + #[test] + fn given_the_primary_leaves_its_view_should_follow_it() { + let consensus = replica(BACKUP); + + let actions = start_view_change(&consensus, PRIMARY, NEXT_VIEW); + + assert!(sends_do_view_change(&actions), "{actions:?}"); + assert_eq!(consensus.view(), NEXT_VIEW); + } + + #[test] + fn given_a_vote_two_views_ahead_should_join_it() { + let consensus = replica(BACKUP); + + let actions = start_view_change(&consensus, CANDIDATE, NEXT_VIEW + 1); + + assert!(sends_do_view_change(&actions), "{actions:?}"); + assert_eq!(consensus.view(), NEXT_VIEW + 1); + } + + #[test] + fn given_a_view_change_status_timeout_should_vote_for_the_next_view_without_entering_it() { + let consensus = replica(BACKUP); + let _ = start_view_change(&consensus, PRIMARY, NEXT_VIEW); + + let actions = consensus.handle_view_change_status_timeout(PLANE); + + assert!(votes_for(&actions, NEXT_VIEW + 1), "{actions:?}"); + assert_eq!(consensus.status(), Status::ViewChange); + assert_eq!(consensus.view(), NEXT_VIEW); + } + + /// A barred replica answers every `StartViewChange` with an empty vote, so + /// the vote alone must not depose a primary. + #[test] + fn given_an_empty_vote_for_the_next_view_should_keep_the_primary_in_its_view() { + let consensus = replica(PRIMARY); + let message = Message::::new(size_of::()) + .transmute_header(|_, header: &mut DoViewChangeHeader| { + header.command = Command::DoViewChange; + header.cluster = 1; + header.replica = BACKUP; + header.view = NEXT_VIEW; + header.group = METADATA_GROUP; + header.flags = DoViewChangeHeader::FLAG_UNTRUSTED_LOG; + header.size = u32::try_from(size_of::()).unwrap(); + }); + + let actions = consensus.handle_do_view_change(PLANE, message.header(), &[]); + + assert!(actions.is_empty(), "{actions:?}"); + assert!(consensus.is_primary()); + assert_eq!(consensus.status(), Status::Normal); + assert_eq!(consensus.view(), VIEW); + } + + /// With 4 replicas `max_faulty` is 1, but a view-change quorum is 3. A + /// `DoViewChange` pulls every replica into its view, so one vote beside a + /// replica that entered the view alone must not send one. + #[test] + fn given_four_replicas_when_a_lone_view_change_counts_votes_should_send_its_do_view_change_at_a_quorum() + { + const FOURTH: u8 = 3; + let consensus = alone_in_next_view(4, BACKUP); + + let one_vote = start_view_change(&consensus, FOURTH, NEXT_VIEW); + let quorum = start_view_change(&consensus, PRIMARY, NEXT_VIEW); + + assert!(!sends_do_view_change(&one_vote), "{one_vote:?}"); + assert!(sends_do_view_change(&quorum), "{quorum:?}"); + } + + /// With 2 replicas a view change needs the `DoViewChange` of both, so a + /// replica alone in the view sends none until its peer votes. + #[test] + fn given_two_replicas_when_a_replica_is_alone_in_a_view_change_should_wait_for_the_vote_of_its_peer() + { + let consensus = alone_in_next_view(2, PRIMARY); + + let alone = (0..TimeoutManager::DO_VIEW_CHANGE_MESSAGE_TICKS) + .flat_map(|_| consensus.tick(PLANE)) + .collect::>(); + let with_peer = start_view_change(&consensus, CANDIDATE, NEXT_VIEW); + + assert!(!sends_do_view_change(&alone), "{alone:?}"); + assert!(sends_do_view_change(&with_peer), "{with_peer:?}"); + } +} + #[cfg(test)] mod history_certificate_tests { use super::*; @@ -10364,6 +11123,37 @@ mod adopted_view_claim_tests { ); } + /// A partition WAL can name the adopted view below its head: an install to a + /// checkpoint and a fenced view window both certify only the prefix they hold. + #[test] + fn given_a_wal_certified_below_the_adopted_head_when_the_backup_restarts_should_keep_the_committed_head() + { + let history = history_in(PARTITION_GROUP); + let adopted = adopt_unrepaired(&history); + let durable = adopted.vsr_state(0, 0); + + let restarted = backup_in( + PARTITION_GROUP, + ADOPTED_VIEW, + durable.view, + HEAD - 1, + HEAD - 2, + ); + restarted.restore_adopted_view(durable.adopted_view.as_ref()); + restarted.resume_adopted_view(); + restarted.fence_view_log(HEAD, 0); + let adopted_headers = restarted.adopted_view_log().unwrap().headers; + restarted.set_local_dvc_suffix(DvcSuffix::new(adopted_headers, 0, 0b10)); + assert_eq!(restarted.sequencer().current_sequence(), HEAD); + assert!(restarted.view_log_unproven()); + + assert_eq!( + merged_head(&restarted, [dvc_of(&restarted), old_primary(&history)]), + HEAD, + "the merge must keep op {HEAD}, which view {OLD_VIEW} committed" + ); + } + #[test] fn given_repaired_adopted_bodies_when_restoring_should_keep_the_older_certificate_binding() { const CERTIFIED_HEAD: u64 = HEAD - 3; diff --git a/core/consensus/src/lib.rs b/core/consensus/src/lib.rs index 9aed5ddad1..ecc5080caa 100644 --- a/core/consensus/src/lib.rs +++ b/core/consensus/src/lib.rs @@ -198,6 +198,8 @@ mod impls; pub use impls::*; mod plane_mux; pub use plane_mux::*; +mod replication; +pub use replication::{REPLICATION_BATCH_BYTES_MAX, ReplicationProgress, ReplicationWork}; mod plane_helpers; pub use plane_helpers::*; mod metadata_helpers; diff --git a/core/consensus/src/observability.rs b/core/consensus/src/observability.rs index 92552b89c9..6650c2da86 100644 --- a/core/consensus/src/observability.rs +++ b/core/consensus/src/observability.rs @@ -134,6 +134,7 @@ pub enum ControlActionKind { RebuildPipeline, CommitJournal, SendCommit, + SendCommitReply, } impl ControlActionKind { @@ -150,6 +151,7 @@ impl ControlActionKind { Self::RebuildPipeline => "rebuild_pipeline", Self::CommitJournal => "commit_journal", Self::SendCommit => "send_commit", + Self::SendCommitReply => "send_commit_reply", } } } @@ -422,6 +424,14 @@ impl ControlActionLogEvent { commit: Some(commit), untrusted_log: None, }, + VsrAction::SendCommitReply { target, .. } => Self { + replica, + action: ControlActionKind::SendCommitReply, + target_replica: Some(target), + op: None, + commit: None, + untrusted_log: None, + }, } } } diff --git a/core/consensus/src/plane_helpers.rs b/core/consensus/src/plane_helpers.rs index d08bda3fe9..f5081e1c6b 100644 --- a/core/consensus/src/plane_helpers.rs +++ b/core/consensus/src/plane_helpers.rs @@ -31,25 +31,24 @@ use server_common::{ }; use std::{error::Error, fmt, mem::size_of, ops::AsyncFnOnce}; -/// Failure to route or forward a prepare through the replication chain. +/// Failure to enqueue a prepare for replication. #[derive(Debug)] #[non_exhaustive] -pub enum ChainReplicationError { +pub enum PrepareReplicationError { MalformedPrepare, UnexpectedCommand { command: Command }, CommittedPrepare { op: u64, commit_min: u64 }, - SelfRoute { replica: u8 }, Transport(SendError), } -impl ChainReplicationError { +impl PrepareReplicationError { #[must_use] pub const fn is_transport(&self) -> bool { matches!(self, Self::Transport(_)) } } -impl fmt::Display for ChainReplicationError { +impl fmt::Display for PrepareReplicationError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::MalformedPrepare => formatter.write_str("malformed prepare frame"), @@ -60,18 +59,12 @@ impl fmt::Display for ChainReplicationError { formatter, "prepare op {op} is not above committed op {commit_min}" ), - Self::SelfRoute { replica } => { - write!( - formatter, - "replication chain routes replica {replica} to itself" - ) - } Self::Transport(error) => error.fmt(formatter), } } } -impl Error for ChainReplicationError { +impl Error for PrepareReplicationError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { Self::Transport(error) => Some(error), @@ -80,7 +73,7 @@ impl Error for ChainReplicationError { } } -impl From for ChainReplicationError { +impl From for PrepareReplicationError { fn from(error: SendError) -> Self { Self::Transport(error) } @@ -130,151 +123,137 @@ where header.op <= consensus.commit_min() } -/// Shared chain-replication forwarding, skipping disconnected replicas. -/// -/// Borrows the message, makes a deep copy for the wire, and lets the caller -/// retain ownership for journal append. +/// Enqueue one shared wire copy for every backup. Backups never forward it. /// /// # Errors /// -/// Returns an error if the prepare cannot be routed or the bus cannot deliver -/// it to a connected replica before the end of the chain. -/// Other transport errors remain covered by VSR prepare retransmission. +/// Returns the first enqueue error after attempting every backup. The bus +/// bounds each peer's queue; a full queue must not skip another peer. #[allow(clippy::future_not_send)] -pub async fn replicate_to_next_in_chain( +pub async fn replicate_prepare( consensus: &VsrConsensus, message: &Message, -) -> Result<(), ChainReplicationError> +) -> Result<(), PrepareReplicationError> where B: MessageBus, P: Pipeline, { - let Some(next) = replication_target(consensus, message.header())? else { + if !primary_sends_prepare(consensus, message.header())? { return Ok(()); - }; + } let frozen = message.deep_copy().into_generic().into_frozen(); - forward_to_connected_replica( - consensus, - frozen, - next, - consensus.primary_index(message.header().view), - ) - .await + enqueue_prepare_for_backups(consensus, frozen).await } -/// Forward an already validated frozen prepare to the next replica without -/// copying its payload, skipping disconnected replicas. +/// Enqueue a frozen prepare for every backup without copying its payload. /// /// # Errors /// -/// Returns an error if the frame is malformed, cannot be routed, or the bus -/// cannot deliver it to a connected replica before the end of the chain. +/// Returns an error for a malformed frame, or the first enqueue error after +/// attempting every backup. #[allow(clippy::future_not_send)] -pub async fn replicate_frozen_to_next_in_chain( +pub async fn replicate_frozen_prepare( consensus: &VsrConsensus, message: Frozen, -) -> Result<(), ChainReplicationError> +) -> Result<(), PrepareReplicationError> where B: MessageBus, P: Pipeline, { let header = frozen_prepare_header(&message)?; - let Some(next) = replication_target(consensus, &header)? else { + if !primary_sends_prepare(consensus, &header)? { return Ok(()); - }; - forward_to_connected_replica( - consensus, - message, - next, - consensus.primary_index(header.view), - ) - .await + } + enqueue_prepare_for_backups(consensus, message).await } #[allow(clippy::future_not_send)] -async fn forward_to_connected_replica( +async fn enqueue_prepare_for_backups( consensus: &VsrConsensus, message: Frozen, - mut next: u8, - primary: u8, -) -> Result<(), ChainReplicationError> +) -> Result<(), PrepareReplicationError> where B: MessageBus, P: Pipeline, { - loop { + let (op, size) = { + let header = frozen_prepare_header(&message)?; + (header.op, u64::from(header.size)) + }; + let mut first_error = None; + for replica in 0..consensus.replica_count() { + if replica == consensus.replica() { + continue; + } + if !consensus.admit_replication(replica, op, size, false) { + continue; + } match consensus .message_bus() - .send_to_replica(next, message.clone()) + .send_to_replica(replica, message.clone()) .await { - Ok(()) => return Ok(()), - Err(error @ (SendError::ReplicaNotConnected(_) | SendError::ConnectionClosed)) => { - next = (next + 1) % consensus.replica_count(); - if next == primary { - return Err(error.into()); + Ok(()) => { + consensus.record_replication_send(replica, op, size, false); + } + Err(error) => { + consensus.record_replication_failure(replica, op); + if first_error.is_none() { + first_error = Some(error); } } - Err(error) => return Err(error.into()), } } + first_error.map_or(Ok(()), |error| Err(error.into())) } fn frozen_prepare_header( message: &Frozen, -) -> Result { +) -> Result { let header_bytes = message .as_slice() .get(..size_of::()) - .ok_or(ChainReplicationError::MalformedPrepare)?; + .ok_or(PrepareReplicationError::MalformedPrepare)?; let header = bytemuck::checked::try_from_bytes::(header_bytes) .copied() - .map_err(|_| ChainReplicationError::MalformedPrepare)?; + .map_err(|_| PrepareReplicationError::MalformedPrepare)?; header .validate() - .map_err(|_| ChainReplicationError::MalformedPrepare)?; + .map_err(|_| PrepareReplicationError::MalformedPrepare)?; let frame_size = - usize::try_from(header.size).map_err(|_| ChainReplicationError::MalformedPrepare)?; + usize::try_from(header.size).map_err(|_| PrepareReplicationError::MalformedPrepare)?; if !(size_of::()..=message.len()).contains(&frame_size) { - return Err(ChainReplicationError::MalformedPrepare); + return Err(PrepareReplicationError::MalformedPrepare); } Ok(header) } -fn replication_target( +fn primary_sends_prepare( consensus: &VsrConsensus, header: &PrepareHeader, -) -> Result, ChainReplicationError> +) -> Result where B: MessageBus, P: Pipeline, { if header.command != Command::Prepare { - return Err(ChainReplicationError::UnexpectedCommand { + return Err(PrepareReplicationError::UnexpectedCommand { command: header.command, }); } let commit_min = consensus.commit_min(); if header.op <= commit_min { - return Err(ChainReplicationError::CommittedPrepare { + return Err(PrepareReplicationError::CommittedPrepare { op: header.op, commit_min, }); } - let next = (consensus.replica() + 1) % consensus.replica_count(); - let primary = consensus.primary_index(header.view); - - if next == primary { - return Ok(None); - } - - if next == consensus.replica() { - return Err(ChainReplicationError::SelfRoute { - replica: consensus.replica(), - }); - } - Ok(Some(next)) + Ok(consensus.is_acting_primary() + && !consensus.has_observed_newer_view() + && !consensus.is_transferring() + && header.view == consensus.view() + && consensus.replica() == consensus.primary_index(header.view)) } /// Re-stamp a stored prepare with the current view before retransmission. @@ -370,12 +349,8 @@ where return Err(IgnoreReason::NewerView); } - // Deposed-primary prepares can still be in flight after a view change: - // replicate_to_next_in_chain stops the ring at primary_index(header.view), - // not the current view's primary, so a stale prepare reaches the new - // primary and would hit its sequencer invariants. Uncommitted old-view - // ops are decided by the view change, never by late delivery. Message - // repair, once it lands, fetches prepares via its own path, not here. + // Late fanout from a deposed primary must not decide uncommitted old-view + // ops. View change and message repair recover them through their own paths. if header.view < consensus.view() { return Err(IgnoreReason::OlderView); } @@ -1449,103 +1424,125 @@ mod tests { assert!(matches!( frozen_prepare_header(&malformed), - Err(ChainReplicationError::MalformedPrepare) + Err(PrepareReplicationError::MalformedPrepare) )); } #[test] - fn given_disconnected_chain_peers_when_forwarding_should_reach_the_next_peer_immediately() { - for frozen in [false, true] { - for (replica, count, view, disconnected, expected) in [ - (0, 3, 0, vec![1], 2), - (1, 3, 1, vec![2], 0), - (2, 5, 1, vec![3, 4], 0), - ] { - let consensus = - VsrConsensus::new(1, replica, count, 0, SpyBus::new(), LocalPipeline::new()); - consensus.init(); - let bus = consensus.message_bus(); - for peer in &disconnected { - bus.failures - .borrow_mut() - .insert(*peer, SendError::ReplicaNotConnected(*peer)); - } - let message = prepare_message(1, 0, 42).transmute_header( - |old, header: &mut PrepareHeader| { - *header = old; - header.view = view; - }, - ); - let result = if frozen { - let prepare = message.deep_copy().into_frozen(); - let pointer = prepare.as_slice().as_ptr(); - let result = futures::executor::block_on(replicate_frozen_to_next_in_chain( - &consensus, prepare, - )); - if let Some((_, sent)) = bus.sent.borrow().first() { - assert_eq!( - sent.as_slice().as_ptr(), - pointer, - "forwarding must not copy the payload" + fn given_a_primary_when_replicating_should_enqueue_every_backup_independently() { + for count in [3, 5] { + for primary in 0..count { + for frozen in [false, true] { + let mut consensus = VsrConsensus::new( + 1, + primary, + count, + 0, + SpyBus::new(), + LocalPipeline::new(), + ); + consensus.set_view(u32::from(primary)); + consensus.set_log_view(u32::from(primary)); + consensus.init(); + let message = prepare_message(1, 0, 42).transmute_header( + |old, header: &mut PrepareHeader| { + *header = old; + header.view = u32::from(primary); + }, + ); + let result = if frozen { + futures::executor::block_on(replicate_frozen_prepare( + &consensus, + message.deep_copy().into_frozen(), + )) + } else { + futures::executor::block_on(replicate_prepare(&consensus, &message)) + }; + result.unwrap(); + let sent = consensus.message_bus().sent.borrow(); + let expected: Vec<_> = (0..count).filter(|peer| *peer != primary).collect(); + assert_eq!( + sent.iter().map(|(peer, _)| *peer).collect::>(), + expected + ); + for (_, copy) in sent.iter() { + assert_eq!(copy.as_slice(), message.as_slice()); + assert!( + copy.shares_allocation(&sent[0].1), + "fanout shares one wire allocation" ); } - result - } else { - futures::executor::block_on(replicate_to_next_in_chain(&consensus, &message)) - }; - result.expect("a disconnected peer must not delay forwarding to the live suffix"); - assert_eq!( - bus.attempts.borrow().as_slice(), - [disconnected, vec![expected]].concat() - ); - let sent = bus.sent.borrow(); - assert_eq!(sent.len(), 1); - assert_eq!(sent[0].0, expected); - assert_eq!(sent[0].1.as_slice(), message.as_slice()); - assert_eq!( - consensus.commit_max(), - 0, - "forwarding is not an acknowledgement" - ); + assert_eq!(consensus.commit_max(), 0, "enqueueing is not an ack"); + } } } } #[test] - fn given_a_chain_boundary_when_forwarding_should_never_wrap_to_the_primary() { - for (replica, count, connected, expected_attempts) in [ - (0, 3, true, vec![1]), - (0, 3, false, vec![1, 2]), - (1, 3, false, vec![2]), - (2, 3, false, vec![]), - (0, 1, false, vec![]), - ] { - let consensus = - VsrConsensus::new(1, replica, count, 0, SpyBus::new(), LocalPipeline::new()); - consensus.init(); - consensus.message_bus().reject_sends.set(!connected); - let result = futures::executor::block_on(replicate_frozen_to_next_in_chain( - &consensus, - prepare_message(1, 0, 42).into_frozen(), - )); - assert_eq!(result.is_ok(), connected || expected_attempts.is_empty()); - assert_eq!( - *consensus.message_bus().attempts.borrow(), - expected_attempts - ); + fn given_one_full_replica_queue_when_replicating_should_still_enqueue_the_healthy_backup() { + let consensus = VsrConsensus::new(1, 0, 3, 0, SpyBus::new(), LocalPipeline::new()); + consensus.init(); + consensus + .message_bus() + .failures + .borrow_mut() + .insert(1, SendError::Backpressure); + let result = futures::executor::block_on(replicate_frozen_prepare( + &consensus, + prepare_message(1, 0, 42).into_frozen(), + )); + assert!( + result.is_err(), + "the failed enqueue remains visible for retransmission" + ); + assert_eq!(*consensus.message_bus().attempts.borrow(), [1, 2]); + assert_eq!(consensus.message_bus().sent.borrow()[0].0, 2); + } + + #[test] + fn given_a_backup_or_singleton_when_replicating_should_enqueue_nothing() { + for count in [1, 3, 5] { + for view in 0..count { + for replica in 0..count { + if count > 1 && replica == view { + continue; + } + let consensus = VsrConsensus::new( + 1, + replica, + count, + 0, + SpyBus::new(), + LocalPipeline::new(), + ); + consensus.init(); + let prepare = prepare_message(1, 0, 42).transmute_header( + |old, header: &mut PrepareHeader| { + *header = old; + header.view = u32::from(view); + }, + ); + futures::executor::block_on(replicate_frozen_prepare( + &consensus, + prepare.into_frozen(), + )) + .unwrap(); + assert!(consensus.message_bus().attempts.borrow().is_empty()); + } + } } } #[test] - fn given_a_transport_error_when_forwarding_should_skip_only_disconnected_peers() { + fn given_an_enqueue_error_when_replicating_should_attempt_every_other_backup() { for error in [ + SendError::ReplicaNotConnected(1), SendError::ConnectionClosed, SendError::Backpressure, SendError::ReplicaRouteMissing(1), SendError::ReplicaForwardFailed(1), SendError::BusShuttingDown, ] { - let skip = matches!(error, SendError::ConnectionClosed); let consensus = VsrConsensus::new(1, 0, 3, 0, SpyBus::new(), LocalPipeline::new()); consensus.init(); consensus @@ -1553,15 +1550,13 @@ mod tests { .failures .borrow_mut() .insert(1, error); - let result = futures::executor::block_on(replicate_frozen_to_next_in_chain( + let result = futures::executor::block_on(replicate_frozen_prepare( &consensus, prepare_message(1, 0, 42).into_frozen(), )); - assert_eq!(result.is_ok(), skip); - assert_eq!( - consensus.message_bus().attempts.borrow().as_slice(), - if skip { &[1, 2][..] } else { &[1][..] }, - ); + assert!(result.is_err()); + assert_eq!(*consensus.message_bus().attempts.borrow(), [1, 2]); + assert_eq!(consensus.message_bus().sent.borrow()[0].0, 2); } } @@ -1572,8 +1567,8 @@ mod tests { let prepare = prepare_message(0, 0, 0); assert!(matches!( - replication_target(&consensus, prepare.header()), - Err(ChainReplicationError::CommittedPrepare { + primary_sends_prepare(&consensus, prepare.header()), + Err(PrepareReplicationError::CommittedPrepare { op: 0, commit_min: 0 }) @@ -1992,13 +1987,18 @@ mod tests { let local = suffix_headers(2, 4, 0); consensus.set_local_dvc_suffix(crate::view_change_quorum::DvcSuffix::new(local, 0, 0b110)); - let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, 5)); - // Two peers abstain, reaching the quorum of 3 and leaving op 4 one nack short. + // The first brings this replica into view 5, which it announces, before f + // StartViewChanges arrive, so it sends no DoViewChange of its own. for replica in [1u8, 2] { let (dvc, body) = dvc_numbers_only(replica, 5, 0, 4, 2); let actions = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); - assert!(actions.is_empty()); + assert!( + actions + .iter() + .all(|action| matches!(action, VsrAction::SendStartViewChange { view: 5, .. })), + "{actions:?}" + ); } assert!( consensus.pending_view_log().is_none(), @@ -2345,9 +2345,10 @@ mod tests { /// /// A parked merge leaves the replica in `ViewChange` announcing nothing if the /// bodies never arrive, which is the intended trade against losing data but has - /// to stay recoverable: the status timeout fires, escalates, and drops the parked - /// log. Reusing a log merged for a superseded view would leak a truncation - /// decided there into a view that never voted for it. + /// to stay recoverable: the status timeout votes for the next view, and the + /// quorum that joins the vote drops the parked log. Reusing a log merged for a + /// superseded view would leak a truncation decided there into a view that never + /// voted for it. /// A `StartView` from the view's primary, optionally carrying the view's /// canonical headers. fn start_view_with_suffix( @@ -2504,7 +2505,8 @@ mod tests { } #[test] - fn given_parked_view_change_when_status_timeout_fires_should_escalate_and_drop_merged_log() { + fn given_parked_view_change_when_a_quorum_votes_after_its_status_timeout_should_drop_merged_log() + { let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); consensus.init(); consensus.restore_commit_state(2, 2); @@ -2520,26 +2522,24 @@ mod tests { assert_eq!(consensus.status(), Status::ViewChange); let parked_view = consensus.view(); - // `VIEW_CHANGE_STATUS_TICKS` is 500; tick past it. Escalation shows as the - // view advancing, since the 50-tick SVC retransmit also emits a send. - let mut escalated = false; - for _ in 0..600 { - let _ = consensus.tick(PlaneKind::Metadata); - if consensus.view() > parked_view { - escalated = true; - break; - } - } - - assert!( - escalated, - "a parked view change must still escalate on the status timeout" - ); + // The status timeout votes for the next view and keeps the parked merge, + // which can still start the view once repair covers it. + let next_view = parked_view + 1; + let voted = (0..crate::TimeoutManager::VIEW_CHANGE_STATUS_TICKS) + .flat_map(|_| consensus.tick(PlaneKind::Metadata)) + .any(|action| { + matches!(action, VsrAction::SendStartViewChange { view, .. } if view == next_view) + }); assert!( - consensus.view() > parked_view, - "escalation must advance the view past {parked_view}, got {}", - consensus.view() + voted, + "a parked view change must still vote on the status timeout" ); + assert_eq!(consensus.view(), parked_view); + assert!(consensus.pending_view_log().is_some()); + + let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, next_view)); + + assert_eq!(consensus.view(), next_view); assert!( consensus.pending_view_log().is_none(), "the superseded merged log must be dropped, not carried into the next view" diff --git a/core/consensus/src/replication.rs b/core/consensus/src/replication.rs new file mode 100644 index 0000000000..e499c3fed6 --- /dev/null +++ b/core/consensus/src/replication.rs @@ -0,0 +1,464 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::{Consensus, Pipeline, PipelineEntry, Sequencer, VsrConsensus}; +use iggy_binary_protocol::PrepareProgressHeader; +use message_bus::MessageBus; + +pub const REPLICATION_BATCH_BYTES_MAX: u64 = 32 * 1024 * 1024; +const PROGRESS_INTERVAL_MICROS: u64 = 1_000_000; + +/// A probe follows every prepare in its batch on the same FIFO link. No new +/// prepares are sent until its reply, so a reply also drains the batch's credit. +/// Payloads remain in the journal, never in this per-peer state. +#[derive(Debug)] +pub struct ReplicationFlow { + epoch: Option<(u32, u64)>, + nonce: u64, + peers: Vec, +} + +#[derive(Debug, Clone, Copy, Default)] +struct PeerReplication { + sent_through: u64, + bytes: u64, + replay_from: Option, + probe: Option<(u64, u64)>, + due: u64, + retry_at: u64, +} + +#[derive(Debug, Clone, Copy)] +pub enum ReplicationWork { + Probe { + replica: u8, + nonce: u64, + through: u64, + }, + Replay { + replica: u8, + from_op: u64, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReplicationProgress { + Busy, + Ready { received_op: u64 }, +} + +impl ReplicationFlow { + pub fn new(replica_count: u8) -> Self { + Self { + epoch: None, + nonce: 0, + peers: vec![PeerReplication::default(); usize::from(replica_count)], + } + } + + fn in_epoch(&mut self, view: u32, incarnation: u64) { + if self.epoch != Some((view, incarnation)) { + self.epoch = Some((view, incarnation)); + self.peers.fill(PeerReplication::default()); + } + } + + fn admit(&mut self, replica: u8, op: u64, size: u64, replay: bool) -> bool { + let peer = &mut self.peers[usize::from(replica)]; + if peer.probe.is_some() { + return false; + } + if !replay && peer.replay_from.is_some_and(|from| from != op) { + return false; + } + // One legal frame larger than the window must still make progress. + if peer.bytes > 0 && peer.bytes.saturating_add(size) > REPLICATION_BATCH_BYTES_MAX { + peer.replay_from.get_or_insert(op); + peer.due = 0; + return false; + } + true + } + + fn sent(&mut self, replica: u8, op: u64, size: u64, replay: bool, now: u64) { + let peer = &mut self.peers[usize::from(replica)]; + if peer.bytes == 0 { + peer.due = now.saturating_add(PROGRESS_INTERVAL_MICROS); + } + peer.bytes = peer.bytes.saturating_add(size); + peer.sent_through = peer.sent_through.max(op); + if replay || peer.replay_from.is_some() { + peer.replay_from = Some(op.saturating_add(1)); + } + } + + fn failed(&mut self, replica: u8, op: u64) { + let peer = &mut self.peers[usize::from(replica)]; + peer.replay_from = Some(peer.replay_from.map_or(op, |from| from.min(op))); + peer.due = 0; + } + + fn work(&mut self, replica: u8, head: u64, now: u64) -> Option { + let peer = &mut self.peers[usize::from(replica)]; + if now < peer.retry_at && peer.bytes == 0 { + return None; + } + if let Some((nonce, through)) = peer.probe { + if now < peer.due { + return None; + } + peer.due = now.saturating_add(PROGRESS_INTERVAL_MICROS); + return Some(ReplicationWork::Probe { + replica, + nonce, + through, + }); + } + if peer.replay_from.is_some_and(|from| from > head) { + peer.replay_from = None; + } + if now >= peer.due && (peer.bytes > 0 || peer.replay_from.is_some()) { + self.nonce = self.nonce.wrapping_add(1).max(1); + let through = peer.sent_through; + peer.probe = Some((self.nonce, through)); + peer.due = now.saturating_add(PROGRESS_INTERVAL_MICROS); + return Some(ReplicationWork::Probe { + replica, + nonce: self.nonce, + through, + }); + } + peer.replay_from + .map(|from_op| ReplicationWork::Replay { replica, from_op }) + } + + fn reply(&mut self, header: &PrepareProgressHeader, head: u64) -> Option { + let peer = &mut self.peers[usize::from(header.replica)]; + if peer.probe != Some((header.nonce, header.observed_op)) { + return None; + } + if header.ready == 0 { + // Readiness can change before the retry interval for a lost probe expires. + peer.due = 0; + return Some(ReplicationProgress::Busy); + } + peer.probe = None; + peer.bytes = 0; + peer.retry_at = 0; + peer.sent_through = header.received_op.min(head); + peer.replay_from = (peer.sent_through < head).then(|| peer.sent_through + 1); + peer.due = u64::MAX; + Some(ReplicationProgress::Ready { + received_op: peer.sent_through, + }) + } +} + +impl VsrConsensus +where + B: MessageBus, + P: Pipeline, +{ + pub fn admit_replication(&self, replica: u8, op: u64, size: u64, replay: bool) -> bool { + if !self.is_acting_primary() || self.has_observed_newer_view() || self.is_transferring() { + return false; + } + let mut flow = self.replication.borrow_mut(); + flow.in_epoch(self.view(), self.partition_incarnation()); + flow.admit(replica, op, size, replay) + } + + pub fn record_replication_send(&self, replica: u8, op: u64, size: u64, replay: bool) { + self.replication.borrow_mut().sent( + replica, + op, + size, + replay, + self.message_bus().monotonic_micros(), + ); + } + + pub fn record_replication_failure(&self, replica: u8, op: u64) { + self.replication.borrow_mut().failed(replica, op); + } + + pub fn defer_replication(&self, replica: u8) { + let mut flow = self.replication.borrow_mut(); + let peer = &mut flow.peers[usize::from(replica)]; + peer.due = 0; + peer.retry_at = self + .message_bus() + .monotonic_micros() + .saturating_add(PROGRESS_INTERVAL_MICROS); + } + + pub fn replication_work(&self, replica: u8) -> Option { + if !self.is_acting_primary() + || self.has_observed_newer_view() + || self.is_transferring() + || self.has_untrusted_log() + { + return None; + } + let mut flow = self.replication.borrow_mut(); + flow.in_epoch(self.view(), self.partition_incarnation()); + flow.work( + replica, + self.sequencer().current_sequence(), + self.message_bus().monotonic_micros(), + ) + } + + pub fn handle_prepare_progress( + &self, + header: &PrepareProgressHeader, + ) -> Option { + if !self.is_acting_primary() + || self.has_observed_newer_view() + || self.is_transferring() + || header.kind != PrepareProgressHeader::REPLY + || header.cluster != self.cluster() + || header.group != self.group() + || header.view != self.view() + || header.partition_incarnation != self.partition_incarnation() + || header.replica >= self.replica_count() + || header.replica == self.replica() + { + return None; + } + let mut flow = self.replication.borrow_mut(); + flow.in_epoch(self.view(), self.partition_incarnation()); + let progress = flow.reply(header, self.sequencer().current_sequence())?; + if let ReplicationProgress::Ready { received_op } = progress { + self.note_backup_ack(header.replica, received_op); + } + Some(progress) + } +} + +#[cfg(test)] +mod tests { + use super::{ + PROGRESS_INTERVAL_MICROS, REPLICATION_BATCH_BYTES_MAX, ReplicationFlow, ReplicationWork, + }; + use crate::{LocalPipeline, Sequencer, VsrConsensus, impls::test_bus::NoopBus}; + use iggy_binary_protocol::{Command, ConsensusHeader, HEADER_SIZE, PrepareProgressHeader}; + use server_common::Message; + + fn reply(nonce: u64, observed_op: u64, received_op: u64) -> Message { + Message::::new(HEADER_SIZE).transmute_header( + |_, header: &mut PrepareProgressHeader| { + header.command = Command::PrepareProgress; + header.cluster = 1; + header.size = u32::try_from(HEADER_SIZE).unwrap(); + header.replica = 1; + header.kind = PrepareProgressHeader::REPLY; + header.ready = 1; + header.nonce = nonce; + header.observed_op = observed_op; + header.received_op = received_op; + header.seal(); + }, + ) + } + + #[test] + fn given_a_full_replay_batch_when_deferred_should_drain_credit_before_history_backoff() { + let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); + consensus.init(); + consensus.sequencer().set_sequence(2); + assert!(consensus.admit_replication(1, 1, REPLICATION_BATCH_BYTES_MAX, true)); + consensus.record_replication_send(1, 1, REPLICATION_BATCH_BYTES_MAX, true); + assert!(!consensus.admit_replication(1, 2, 1, true)); + consensus.defer_replication(1); + let Some(ReplicationWork::Probe { nonce, through, .. }) = consensus.replication_work(1) + else { + panic!("a full batch must drain without the unavailable-history delay"); + }; + assert!( + consensus + .handle_prepare_progress(reply(nonce, through, 1).header()) + .is_some() + ); + assert!( + matches!( + consensus.replication_work(1), + Some(ReplicationWork::Replay { from_op: 2, .. }) + ), + "a drained batch must resume replay without the stale history delay" + ); + + assert!(consensus.admit_replication(2, 1, 1, true)); + consensus.record_replication_failure(2, 1); + consensus.defer_replication(2); + assert!( + consensus.replication_work(2).is_none(), + "unavailable history without outstanding bytes must still back off" + ); + } + + #[test] + fn given_a_full_peer_window_when_its_probe_is_delayed_should_bound_bytes_and_keep_other_peers_sending() + { + let mut flow = ReplicationFlow::new(3); + flow.in_epoch(0, 1); + assert!(flow.admit(1, 1, REPLICATION_BATCH_BYTES_MAX, false)); + flow.sent(1, 1, REPLICATION_BATCH_BYTES_MAX, false, 0); + assert!(!flow.admit(1, 2, 1, false)); + let Some(ReplicationWork::Probe { nonce, through, .. }) = flow.work(1, 2, 0) else { + panic!("full window must probe"); + }; + assert!(!flow.admit(1, 2, 1, true)); + assert!( + flow.admit(2, 2, 1, false), + "a delayed peer must not stop another peer" + ); + assert!( + matches!(flow.work(1, 3, PROGRESS_INTERVAL_MICROS), Some(ReplicationWork::Probe { nonce: repeated, through: 1, .. }) if repeated == nonce) + ); + let _ = flow.reply(reply(nonce, through, 0).header(), 3); + assert!(matches!( + flow.work(1, 3, PROGRESS_INTERVAL_MICROS), + Some(ReplicationWork::Replay { from_op: 1, .. }) + )); + assert!( + !flow.admit(1, 4, 1, false), + "live traffic cannot overtake replay" + ); + for op in 1..=4 { + assert!(flow.admit(1, op, 1, true)); + flow.sent(1, op, 1, true, PROGRESS_INTERVAL_MICROS); + } + assert!(flow.work(1, 4, PROGRESS_INTERVAL_MICROS).is_none()); + assert!( + flow.admit(1, 5, 1, false), + "enqueueing through the moving head restores live delivery" + ); + } + + #[test] + fn given_an_inflight_local_prepare_when_a_probe_finishes_should_join_replay_without_waiting_for_a_tick() + { + let mut flow = ReplicationFlow::new(3); + flow.in_epoch(0, 1); + flow.sent(1, 1, 100, false, 0); + let Some(ReplicationWork::Probe { nonce, through, .. }) = + flow.work(1, 2, PROGRESS_INTERVAL_MICROS) + else { + panic!("the sent batch must be probed"); + }; + let _ = flow.reply(reply(nonce, through, 1).header(), 2); + assert!( + !flow.admit(1, 3, 100, false), + "a fresh op cannot skip replay" + ); + assert!( + flow.admit(1, 2, 100, false), + "the next queued local prepare must join the ordered stream immediately" + ); + flow.sent(1, 2, 100, false, PROGRESS_INTERVAL_MICROS); + assert!( + flow.admit(1, 3, 100, false), + "continuous live traffic must advance the replay cursor without a timer" + ); + } + + #[test] + fn given_an_enqueue_failure_when_a_correlated_probe_finds_a_gap_should_replay_the_committed_suffix() + { + let mut flow = ReplicationFlow::new(3); + flow.in_epoch(0, 1); + flow.sent(1, 3, 100, false, 0); + flow.failed(1, 4); + let Some(ReplicationWork::Probe { nonce, through, .. }) = flow.work(1, 6, 0) else { + panic!("failed enqueue must probe"); + }; + let _ = flow.reply(reply(nonce + 1, through, 6).header(), 6); + assert!( + !flow.admit(1, 7, 1, false), + "an uncorrelated reply releases no credit" + ); + let busy = + reply(nonce, through, 6).transmute_header(|old, header: &mut PrepareProgressHeader| { + *header = old; + header.ready = 0; + }); + assert_eq!( + flow.reply(busy.header(), 6), + Some(super::ReplicationProgress::Busy) + ); + assert!( + !flow.admit(1, 7, 1, false), + "a busy replica does not establish progress" + ); + assert!( + matches!(flow.work(1, 6, 1), Some(ReplicationWork::Probe { nonce: repeated, through: observed, .. }) if repeated == nonce && observed == through), + "a busy reply must recheck readiness on the next tick without releasing its batch credit" + ); + let _ = flow.reply(reply(nonce, through, 2).header(), 6); + assert!(matches!( + flow.work(1, 6, 1), + Some(ReplicationWork::Replay { from_op: 3, .. }) + )); + flow.in_epoch(1, 1); + let _ = flow.reply(reply(nonce, through, 2).header(), 6); + assert!( + flow.admit(1, 7, 1, false), + "a new view discards the previous sender window" + ); + } + + #[test] + fn given_progress_replies_when_the_view_incarnation_or_nonce_is_stale_should_neither_release_credit_nor_commit() + { + let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); + consensus.init(); + consensus.sequencer().set_sequence(2); + assert!(consensus.admit_replication(1, 1, 1, false)); + consensus.record_replication_send(1, 1, 1, false); + consensus.record_replication_failure(1, 2); + let Some(ReplicationWork::Probe { nonce, through, .. }) = consensus.replication_work(1) + else { + panic!("failed enqueue must probe"); + }; + for stale in ["view", "incarnation", "nonce", "peer", "cluster", "group"] { + let message = reply(nonce, through, 2).transmute_header( + |old, header: &mut PrepareProgressHeader| { + *header = old; + match stale { + "view" => header.view += 1, + "incarnation" => header.partition_incarnation += 1, + "nonce" => header.nonce += 1, + "peer" => header.replica = 2, + "cluster" => header.cluster += 1, + "group" => header.group += 1, + _ => unreachable!(), + } + }, + ); + assert_eq!(consensus.handle_prepare_progress(message.header()), None); + assert!( + !consensus.admit_replication(1, 3, 1, false), + "accepted stale {stale}" + ); + } + let _ = consensus.handle_prepare_progress(reply(nonce, through, 2).header()); + assert!(consensus.admit_replication(1, 3, 1, false)); + assert_eq!(consensus.commit_min(), 0); + assert_eq!(consensus.commit_max(), 0); + } +} diff --git a/core/consensus/src/vsr_timeout.rs b/core/consensus/src/vsr_timeout.rs index 9781520e8a..e9382aafb6 100644 --- a/core/consensus/src/vsr_timeout.rs +++ b/core/consensus/src/vsr_timeout.rs @@ -121,6 +121,7 @@ impl Timeout { pub enum TimeoutKind { Prepare, CommitMessage, + CheckQuorum, NormalHeartbeat, StartViewChangeMessage, ViewChangeStatus, @@ -136,6 +137,7 @@ pub enum TimeoutKind { pub struct TimeoutManager { prepare: Timeout, commit_message: Timeout, + check_quorum: Timeout, normal_heartbeat: Timeout, start_view_change_message: Timeout, do_view_change_message: Timeout, @@ -179,6 +181,7 @@ impl TimeoutManager { Self { prepare: Timeout::new(replica_id, Self::PREPARE_TICKS), commit_message: Timeout::new(replica_id, Self::COMMIT_MESSAGE_TICKS), + check_quorum: Timeout::new(replica_id, Self::NORMAL_HEARTBEAT_TICKS), normal_heartbeat: Timeout::new(replica_id, Self::NORMAL_HEARTBEAT_TICKS), start_view_change_message: Timeout::new( replica_id, @@ -196,12 +199,13 @@ impl TimeoutManager { } } - /// Override the backup's primary-liveness window (`[cluster] + /// Override the backup's primary-liveness and primary's quorum window (`[cluster] /// heartbeat_timeout`). Replaces the timeout object, so any countdown /// already in flight is discarded: call before the replica starts /// ticking (i.e. before `init` / `init_as_backup`). pub const fn set_normal_heartbeat_ticks(&mut self, ticks: u64) { self.normal_heartbeat = Timeout::new(self.normal_heartbeat.id, ticks); + self.check_quorum = Timeout::new(self.check_quorum.id, ticks); } /// Override the primary's commit-broadcast interval (`[cluster] @@ -256,6 +260,7 @@ impl TimeoutManager { pub const fn tick(&mut self) { self.prepare.tick(); self.commit_message.tick(); + self.check_quorum.tick(); self.normal_heartbeat.tick(); self.start_view_change_message.tick(); self.do_view_change_message.tick(); @@ -273,6 +278,7 @@ impl TimeoutManager { match kind { TimeoutKind::Prepare => &self.prepare, TimeoutKind::CommitMessage => &self.commit_message, + TimeoutKind::CheckQuorum => &self.check_quorum, TimeoutKind::NormalHeartbeat => &self.normal_heartbeat, TimeoutKind::StartViewChangeMessage => &self.start_view_change_message, TimeoutKind::ViewChangeStatus => &self.view_change_status, @@ -285,6 +291,7 @@ impl TimeoutManager { match kind { TimeoutKind::Prepare => &mut self.prepare, TimeoutKind::CommitMessage => &mut self.commit_message, + TimeoutKind::CheckQuorum => &mut self.check_quorum, TimeoutKind::NormalHeartbeat => &mut self.normal_heartbeat, TimeoutKind::StartViewChangeMessage => &mut self.start_view_change_message, TimeoutKind::ViewChangeStatus => &mut self.view_change_status, @@ -309,6 +316,7 @@ impl TimeoutManager { let timeout = match kind { TimeoutKind::Prepare => &mut self.prepare, TimeoutKind::CommitMessage => &mut self.commit_message, + TimeoutKind::CheckQuorum => &mut self.check_quorum, TimeoutKind::NormalHeartbeat => &mut self.normal_heartbeat, TimeoutKind::StartViewChangeMessage => &mut self.start_view_change_message, TimeoutKind::ViewChangeStatus => &mut self.view_change_status, diff --git a/core/integration/tests/cluster/partition_state_transfer.rs b/core/integration/tests/cluster/partition_state_transfer.rs index 8f6270692d..6eb5a7d489 100644 --- a/core/integration/tests/cluster/partition_state_transfer.rs +++ b/core/integration/tests/cluster/partition_state_transfer.rs @@ -27,7 +27,8 @@ //! `RangeEvicted`, the repaired window cannot connect to recovered state, //! and `complete_repair` returns the `FloorRefused` conversion trigger. -use std::collections::HashMap; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::os::unix::fs::{FileExt, MetadataExt}; use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::Arc; @@ -74,8 +75,9 @@ const KILL_GATE_POLL: Duration = Duration::from_millis(10); const CONVERSION_MARKER: &str = "partition repair floor unreachable; converting to state transfer"; const INSTALL_MARKER: &str = "partition state transfer installed"; const INSTALL_FAILED_MARKER: &str = "partition state transfer install failed"; -/// The `StaleTransfer` reason, logged inside an [`INSTALL_FAILED_MARKER`] line. -const STALE_INSTALL_MARKER: &str = "is below the local commit frontier"; +/// The `OfferErasesHeldOps` reason, logged inside an [`INSTALL_FAILED_MARKER`] +/// line. +const HELD_OPS_INSTALL_MARKER: &str = "would erase held ops"; /// Logged once per transfer, at the last byte of the last artifact. The /// descriptor-time "serving" line only proves a request ARRIVED; this proves the /// pull ran. @@ -102,6 +104,32 @@ const ISOLATED_SENDS: u32 = 100; const WRITE_INTERVAL: Duration = Duration::from_secs(1); /// How long node 2 must go without another conversion once it installed. const CAUGHT_UP_WINDOW: Duration = Duration::from_secs(10); +/// Logged by a receiver that links segments it already holds into a +/// transfer. Its `reused_len` field counts the bytes that it did not pull. +const REUSE_MARKER: &str = "reused segments that this node already holds"; +/// Repeated-install specs must exhaust snapshot-tail retention as well as the +/// ring; otherwise the next gap can correctly heal without another snapshot. +const SNAPSHOT_ONLY_REPAIR_MEMORY_BYTES_MAX: &str = "1 B"; +/// Logged by a receiver that adopts the staged copies of an earlier attempt. +/// Its `adopted_len` field counts the bytes that it did not pull. +const ADOPTED_MARKER: &str = "adopted staged segments from an earlier transfer attempt"; +/// Logged by a receiver that fills an artifact with the bytes of its own +/// segment. Its `seeded_len` field counts them. +const SEED_MARKER: &str = "seeded a partition transfer artifact from the local segment"; +/// Logged by a receiver whose seeded artifact failed the manifest checksum. +const PREFIX_MISMATCH_MARKER: &str = "the local segment prefix does not match the offered artifact"; +/// Logged once per session before the install. Its `pulled_len` field counts +/// the chunk bytes that crossed the network. +const PULLED_MARKER: &str = "partition transfer pulled every artifact"; +/// Logged by a requester that schedules another transfer attempt, for +/// example after a refusal. +const REARM_MARKER: &str = "partition transfer did not land; scheduling a re-arm"; +/// Bulky sends per round of the reuse spec: 10 MiB, more than one +/// [`SEGMENT_SIZE_MULTI_ARTIFACT`], so each round seals at least one segment. +const BULKY_SENDS_PER_ROUND: u32 = 40; +/// Transfers in the reuse spec. The second one must reuse a sealed segment +/// that the first one pulled. +const REUSE_ROUNDS: usize = 2; /// Transfer end-to-end: adoption, repair round-trip, conversion, chunk pull, /// install, tail repair. CI runners are slow; bound without hanging the suite. @@ -382,7 +410,6 @@ async fn given_transfer_peer_dies_when_stalled_should_leave_dead_peer_and_recove /// needed: the pull itself is a few KiB. #[tokio::test(flavor = "multi_thread")] #[parallel] -#[ignore = "known bug: under sustained writes the backup refuses every transfer as stale (transfer frontier below the local commit frontier); run with --run-ignored only"] async fn given_sustained_writes_when_a_node_falls_past_the_evicted_ring_should_rejoin_through_one_transfer() { let mut harness = relayed_cluster(); @@ -408,12 +435,10 @@ async fn given_sustained_writes_when_a_node_falls_past_the_evicted_ring_should_r .expect("restore node 2's peer links"); await_marker(&harness, LAGGING_NODE, CONVERSION_MARKER).await; - // Today every install is refused with "transfer frontier X is below the - // local commit frontier X+1". The first request only pins the offer at the - // primary's commit point (the build answers "flush is pending"), and the - // pull runs on the next request, one `repair_retry_interval` later. Node 2 - // journals the op committed in between, and the install refuses an offer - // below an op it holds. + // The first request only pins the offer at the primary's commit point (the + // build answers "flush is pending"), and the pull runs on the next request, + // one `repair_retry_interval` later. Node 2 journals the ops committed in + // between, so it holds ops above the offer it installs. let deadline = Instant::now() + TRANSFER_BUDGET; while harness .node(LAGGING_NODE) @@ -424,15 +449,15 @@ async fn given_sustained_writes_when_a_node_falls_past_the_evicted_ring_should_r Instant::now() < deadline, "node 2 fell past the evicted ring and never installed a state transfer within \ {TRANSFER_BUDGET:?} while one message per second was written: the peers fully \ - served {} pull(s) and node 2 failed {} install(s), {} of them as stale, so a \ - backup cannot rejoin under sustained writes", + served {} pull(s) and node 2 failed {} install(s), {} of them for held ops, so \ + a backup cannot rejoin under sustained writes", fully_served_count(&harness), harness .node(LAGGING_NODE) .stdout_occurrences(INSTALL_FAILED_MARKER), harness .node(LAGGING_NODE) - .stdout_occurrences(STALE_INSTALL_MARKER), + .stdout_occurrences(HELD_OPS_INSTALL_MARKER), ); sleep(MARKER_POLL).await; } @@ -469,6 +494,298 @@ async fn given_sustained_writes_when_a_node_falls_past_the_evicted_ring_should_r ); } +/// A node that falls past the evicted ring must link the sealed segments it +/// already holds into each transfer and pull only the rest. +/// +/// Node 2 holds sealed segments that it wrote itself, then falls behind twice. +/// Each segment that node 2 held sealed at the cut must keep its inode through +/// the install: the install renamed a link to it, and none of its bytes +/// crossed the network. A pulled segment always lands in a new file, because +/// the old one still exists when the pull writes it. The second round must +/// also keep a segment that the first round pulled, which proves that the +/// install stamps the segments it installs. The reuse line must count exactly +/// the bytes of the kept segments. +#[tokio::test(flavor = "multi_thread")] +#[parallel] +async fn given_sealed_segments_on_a_lagging_node_when_it_transfers_twice_should_pull_only_the_rest() +{ + let mut harness = + relayed_cluster_with_repair_budget(Some(SNAPSHOT_ONLY_REPAIR_MEMORY_BYTES_MAX)); + harness.start().await.expect("start the cluster"); + let client = harness + .root_client_for_node(0) + .await + .expect("connect a root client to the node"); + seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT)).await; + produce_bulky(&client, 2 * BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; + let mut expected: Vec = (0..2 * BULKY_SENDS_PER_ROUND) + .map(bulky_payload_prefix) + .collect(); + let data_path = harness.node(LAGGING_NODE).data_path(); + let mut kept_before = BTreeSet::new(); + + for round in 1..=REUSE_ROUNDS { + // Node 2 must hold every committed payload, so the segments that it + // holds sealed are final before the cut. + await_installed_payloads(&harness, LAGGING_NODE, &expected).await; + let held = segment_files(&data_path); + let sealed: Vec = held.keys().rev().skip(1).copied().collect(); + assert!( + !sealed.is_empty(), + "round {round}: node 2 must hold a sealed segment before the cut" + ); + if round > 1 { + assert!( + sealed.iter().any(|start| !kept_before.contains(start)), + "round {round}: node 2 must hold a sealed segment that the last transfer \ + pulled, or this round cannot prove that the install stamps it" + ); + } + let installs = harness + .node(LAGGING_NODE) + .stdout_occurrences(INSTALL_MARKER); + let reuses = harness.node(LAGGING_NODE).stdout_occurrences(REUSE_MARKER); + + harness + .isolate_node(LAGGING_NODE) + .expect("cut node 2 off from its peers"); + produce_bulky(&client, BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; + produce(&client, ISOLATED_SENDS).await; + harness + .heal_node(LAGGING_NODE) + .expect("restore node 2's peer links"); + await_new_marker(&harness, LAGGING_NODE, INSTALL_MARKER, installs).await; + expected.extend((0..BULKY_SENDS_PER_ROUND).map(bulky_payload_prefix)); + expected.extend((0..ISOLATED_SENDS).map(|sequence| format!("message-{sequence}"))); + await_installed_payloads(&harness, LAGGING_NODE, &expected).await; + + let installed = segment_files(&data_path); + let kept: BTreeMap = held + .iter() + .filter(|(start, file)| installed.get(start) == Some(file)) + .map(|(start, (_, len))| (*start, *len)) + .collect(); + for start in &sealed { + assert!( + kept.contains_key(start), + "round {round}: node 2 held segment {start} sealed, but the install replaced \ + its file, so the transfer copied bytes that node 2 already had" + ); + } + assert_eq!( + reused_len_after(&harness, LAGGING_NODE, reuses), + kept.values().sum::(), + "round {round}: the reuse line must count exactly the bytes of the kept segments" + ); + kept_before = kept.into_keys().collect(); + } +} + +/// A node that falls past the evicted ring while the primary keeps writing +/// the segment that the node holds a prefix of must pull only the rest. +/// +/// Each round, node 2 holds the leading bytes of the segment that the +/// primary writes on during the cut. The transfer must seed that artifact +/// with them, so the pulled bytes are the manifest minus the reused and the +/// held bytes. Between the rounds, node 2 fills and seals the segment that the +/// transfer installed as active. The next transfer must keep that segment like +/// any other sealed one, which proves that its checksum memo grew with it. +#[tokio::test(flavor = "multi_thread")] +#[parallel] +async fn given_a_segment_prefix_on_a_lagging_node_when_it_transfers_should_pull_only_the_suffix() { + let mut harness = + relayed_cluster_with_repair_budget(Some(SNAPSHOT_ONLY_REPAIR_MEMORY_BYTES_MAX)); + harness.start().await.expect("start the cluster"); + let client = harness + .root_client_for_node(0) + .await + .expect("connect a root client to the node"); + seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT)).await; + produce_bulky(&client, 2 * BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; + let mut expected: Vec = (0..2 * BULKY_SENDS_PER_ROUND) + .map(bulky_payload_prefix) + .collect(); + let data_path = harness.node(LAGGING_NODE).data_path(); + let mut installed_active: Option = None; + + for round in 1..=REUSE_ROUNDS { + await_installed_payloads(&harness, LAGGING_NODE, &expected).await; + let held = segment_files(&data_path); + let (active_start, active_len) = held_prefix(&held, round); + if let Some(installed) = installed_active { + assert!( + installed < active_start, + "round {round}: the segment that the last transfer installed as active must \ + seal on node 2 before the cut" + ); + } + let installs = harness + .node(LAGGING_NODE) + .stdout_occurrences(INSTALL_MARKER); + + harness + .isolate_node(LAGGING_NODE) + .expect("cut node 2 off from its peers"); + produce_bulky(&client, BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; + produce(&client, ISOLATED_SENDS).await; + harness + .heal_node(LAGGING_NODE) + .expect("restore node 2's peer links"); + await_new_marker(&harness, LAGGING_NODE, INSTALL_MARKER, installs).await; + expected.extend((0..BULKY_SENDS_PER_ROUND).map(bulky_payload_prefix)); + expected.extend((0..ISOLATED_SENDS).map(|sequence| format!("message-{sequence}"))); + await_installed_payloads(&harness, LAGGING_NODE, &expected).await; + + let session = installed_session(&harness, LAGGING_NODE); + // Node 2 can persist bytes of its active segment after the files were + // listed, so the seed holds at least what the listing saw. + let (seeded_len, _) = session.seed.unwrap_or_else(|| { + panic!( + "round {round}: node 2 held {active_len} bytes of segment {active_start}, \ + but it did not seed them" + ) + }); + assert!( + seeded_len >= active_len, + "round {round}: the seed must hold the {active_len} bytes that node 2 held, \ + got {seeded_len}" + ); + assert_eq!( + session.pulled_len, + session.total_len - session.reused_len - seeded_len, + "round {round}: node 2 seeded {seeded_len} bytes of segment {active_start}, so the \ + transfer must pull the manifest minus those and the reused bytes" + ); + let installed = segment_files(&data_path); + if let Some(sealed) = installed_active { + assert_eq!( + installed.get(&sealed).map(|(inode, _)| inode), + held.get(&sealed).map(|(inode, _)| inode), + "round {round}: node 2 sealed segment {sealed} after the last transfer \ + installed it as active, but this install replaced its file, so the transfer \ + pulled bytes that node 2 already had" + ); + } + installed_active = installed.keys().next_back().copied(); + // With node 2 connected: fill and seal the segment that this transfer + // installed as active. The small sends leave bytes in the segment + // after it, which the next round seeds. + produce_bulky(&client, BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; + produce(&client, ISOLATED_SENDS).await; + expected.extend((0..BULKY_SENDS_PER_ROUND).map(bulky_payload_prefix)); + expected.extend((0..ISOLATED_SENDS).map(|sequence| format!("message-{sequence}"))); + } +} + +/// A held prefix with other bytes than the offered segment must not reach the +/// install. The checksum of the seeded artifact fails, and the transfer pulls +/// that artifact whole in the same session. +#[tokio::test(flavor = "multi_thread")] +#[parallel] +async fn given_a_damaged_segment_prefix_when_it_transfers_should_pull_the_whole_artifact() { + let mut harness = relayed_cluster(); + harness.start().await.expect("start the cluster"); + let client = harness + .root_client_for_node(0) + .await + .expect("connect a root client to the node"); + seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT)).await; + produce_bulky(&client, 2 * BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; + let mut expected: Vec = (0..2 * BULKY_SENDS_PER_ROUND) + .map(bulky_payload_prefix) + .collect(); + let data_path = harness.node(LAGGING_NODE).data_path(); + await_installed_payloads(&harness, LAGGING_NODE, &expected).await; + let held = segment_files(&data_path); + let (active_start, active_len) = held_prefix(&held, 1); + let installs = harness + .node(LAGGING_NODE) + .stdout_occurrences(INSTALL_MARKER); + let mismatches = harness + .node(LAGGING_NODE) + .stdout_occurrences(PREFIX_MISMATCH_MARKER); + + harness + .isolate_node(LAGGING_NODE) + .expect("cut node 2 off from its peers"); + flip_segment_byte(&data_path, active_start, active_len / 2); + produce_bulky(&client, BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; + produce(&client, ISOLATED_SENDS).await; + harness + .heal_node(LAGGING_NODE) + .expect("restore node 2's peer links"); + await_new_marker(&harness, LAGGING_NODE, INSTALL_MARKER, installs).await; + expected.extend((0..BULKY_SENDS_PER_ROUND).map(bulky_payload_prefix)); + expected.extend((0..ISOLATED_SENDS).map(|sequence| format!("message-{sequence}"))); + await_installed_payloads(&harness, LAGGING_NODE, &expected).await; + + assert_eq!( + harness + .node(LAGGING_NODE) + .stdout_occurrences(PREFIX_MISMATCH_MARKER), + mismatches + 1, + "the seeded artifact must fail its checksum once, and the whole pull must pass" + ); + let session = installed_session(&harness, LAGGING_NODE); + let (seeded_len, artifact_len) = session + .seed + .expect("node 2 must seed the artifact before its checksum fails"); + assert!( + seeded_len >= active_len, + "the seed must hold the {active_len} bytes that node 2 held, got {seeded_len}" + ); + assert_eq!( + session.pulled_len, + session.total_len - session.reused_len + artifact_len - seeded_len, + "the transfer must pull the suffix, then the whole artifact once the checksum failed" + ); +} + +/// The first request of a transfer finds the plan of the primary unpinned, +/// because the pin waits for the next I/O turn of the partition. The primary +/// must keep that request and answer it once the plan is pinned. A refusal +/// makes the requester wait out a whole re-arm interval for nothing. +#[tokio::test(flavor = "multi_thread")] +#[parallel] +async fn given_an_unpinned_plan_when_a_transfer_is_requested_should_answer_without_a_re_arm() { + let mut harness = relayed_cluster(); + harness.start().await.expect("start the cluster"); + let client = harness + .root_client_for_node(0) + .await + .expect("connect a root client to the node"); + seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT)).await; + produce_bulky(&client, BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; + let mut expected: Vec = (0..BULKY_SENDS_PER_ROUND) + .map(bulky_payload_prefix) + .collect(); + await_installed_payloads(&harness, LAGGING_NODE, &expected).await; + let installs = harness + .node(LAGGING_NODE) + .stdout_occurrences(INSTALL_MARKER); + let rearms = harness.node(LAGGING_NODE).stdout_occurrences(REARM_MARKER); + + harness + .isolate_node(LAGGING_NODE) + .expect("cut node 2 off from its peers"); + produce_bulky(&client, BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; + produce(&client, ISOLATED_SENDS).await; + harness + .heal_node(LAGGING_NODE) + .expect("restore node 2's peer links"); + await_new_marker(&harness, LAGGING_NODE, INSTALL_MARKER, installs).await; + expected.extend((0..BULKY_SENDS_PER_ROUND).map(bulky_payload_prefix)); + expected.extend((0..ISOLATED_SENDS).map(|sequence| format!("message-{sequence}"))); + await_installed_payloads(&harness, LAGGING_NODE, &expected).await; + + assert_eq!( + harness.node(LAGGING_NODE).stdout_occurrences(REARM_MARKER), + rearms, + "the primary must answer the first transfer request of node 2 with an offer, \ + so node 2 must not schedule a re-arm before the install" + ); +} + async fn restart_lagging_node_and_await_install(harness: &mut TestHarness) { // Node 2 holds a durable prefix, then misses enough traffic that the // survivors' ring moves past its durable end: its repaired window cannot @@ -515,13 +832,23 @@ async fn restart_lagging_node_and_await_install(harness: &mut TestHarness) { /// links, so the one that cuts a node off builds its cluster here with the same /// server settings. fn relayed_cluster() -> TestHarness { - let overrides = HashMap::from([ + relayed_cluster_with_repair_budget(None) +} + +fn relayed_cluster_with_repair_budget(repair_memory_bytes_max: Option<&str>) -> TestHarness { + let mut overrides = HashMap::from([ ("sharding.cpu_allocation".to_owned(), "0..1".to_owned()), ( "partition.evicted_ring_capacity".to_owned(), "64".to_owned(), ), ]); + if let Some(bytes_max) = repair_memory_bytes_max { + overrides.insert( + "partition.repair_memory_bytes_max".to_owned(), + bytes_max.to_owned(), + ); + } let extra_envs = resolve_config_paths(&overrides).expect("valid server config paths"); TestHarness::builder() .server(TestServerConfig::builder().extra_envs(extra_envs).build()) @@ -594,6 +921,123 @@ fn fully_served_count(harness: &TestHarness) -> usize { .sum() } +/// `reused_len` of the first reuse line that `node` logged after `baseline` +/// earlier ones. +fn reused_len_after(harness: &TestHarness, node: usize, baseline: usize) -> u64 { + let log = harness.node(node).stdout_plain(); + let line = log + .lines() + .filter(|line| line.contains(REUSE_MARKER)) + .nth(baseline) + .unwrap_or_else(|| panic!("node {node} logged no {REUSE_MARKER:?} line for the transfer")); + logged_number(line, "reused_len") +} + +/// What the last session that `node` pulled to the end logged. +struct PulledSession { + total_len: u64, + /// The bytes that it staged without a pull: `reused_len` of its reuse + /// line plus `adopted_len` of the adopted line before it. + reused_len: u64, + /// `seeded_len` and `artifact_len` of its seed line. + seed: Option<(u64, u64)>, + pulled_len: u64, +} + +/// The lines of one session come in order: adopted and reuse when its reuse +/// scan lands, the seed when the pull reaches that artifact, the pulled line +/// before the install. +fn installed_session(harness: &TestHarness, node: usize) -> PulledSession { + let log = harness.node(node).stdout_plain(); + let mut adopted = None; + let mut reuse = None; + let mut seed = None; + let mut session = None; + for line in log.lines() { + if line.contains(ADOPTED_MARKER) { + adopted = Some(line); + } else if line.contains(REUSE_MARKER) { + reuse = Some((line, adopted.take())); + seed = None; + } else if line.contains(SEED_MARKER) { + seed = Some(line); + } else if line.contains(PULLED_MARKER) { + session = Some((reuse, seed, line)); + } + } + let (reuse, seed, pulled) = + session.unwrap_or_else(|| panic!("node {node} logged no {PULLED_MARKER:?} line")); + let (reuse, adopted) = + reuse.unwrap_or_else(|| panic!("node {node} logged no {REUSE_MARKER:?} line")); + PulledSession { + total_len: logged_number(reuse, "total_len"), + reused_len: logged_number(reuse, "reused_len") + + adopted.map_or(0, |line| logged_number(line, "adopted_len")), + seed: seed.map(|line| { + ( + logged_number(line, "seeded_len"), + logged_number(line, "artifact_len"), + ) + }), + pulled_len: logged_number(pulled, "pulled_len"), + } +} + +fn logged_number(line: &str, field: &str) -> u64 { + let label = format!(" {field}="); + let start = line + .find(&label) + .unwrap_or_else(|| panic!("the line has no {label:?} field: {line}")) + + label.len(); + line[start..] + .split(|character: char| !character.is_ascii_digit()) + .next() + .and_then(|digits| digits.parse().ok()) + .unwrap_or_else(|| panic!("the line has no number in {label:?}: {line}")) +} + +/// The active segment of `held` and its length. Sealed segments must exist, +/// so the transfer logs a reuse line, and the active one must hold bytes, so +/// it can seed an artifact. +fn held_prefix(held: &BTreeMap, round: usize) -> (u64, u64) { + assert!( + held.len() > 1, + "round {round}: node 2 must hold a sealed segment before the cut" + ); + let (&start, &(_, len)) = held + .last_key_value() + .expect("node 2 holds a segment before the cut"); + assert!( + len > 0, + "round {round}: node 2's active segment must hold bytes before the cut" + ); + (start, len) +} + +/// Invert one byte of the segment log at `start_offset`. +fn flip_segment_byte(root: &Path, start_offset: u64, position: u64) { + let path = walk(root, &mut |path| { + is_segment_log(path) + && path + .file_stem() + .and_then(|stem| stem.to_str()) + .and_then(|stem| stem.parse::().ok()) + == Some(start_offset) + }) + .unwrap_or_else(|| panic!("no segment log starts at offset {start_offset}")); + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(&path) + .expect("open the segment log"); + let mut byte = [0_u8; 1]; + file.read_exact_at(&mut byte, position) + .expect("read the segment byte"); + byte[0] = !byte[0]; + file.write_all_at(&byte, position) + .expect("write the segment byte"); +} + async fn connect_any(harness: &TestHarness, nodes: &[usize]) -> Option { for &node in nodes { if let Ok(builder) = harness.node(node).tcp_client() @@ -655,7 +1099,11 @@ async fn produce(client: &IggyClient, count: u32) { /// `payload_len` filler bytes so the on-disk segment grows fast. async fn produce_bulky(client: &IggyClient, count: u32, payload_len: usize) { for sequence in 0..count { - let payload = format!("bulky-{sequence}-{}", "x".repeat(payload_len)); + let payload = format!( + "{}{}", + bulky_payload_prefix(sequence), + "x".repeat(payload_len) + ); let mut messages = vec![IggyMessage::from_str(&payload).expect("message")]; client .send_messages( @@ -669,6 +1117,12 @@ async fn produce_bulky(client: &IggyClient, count: u32, payload_len: usize) { } } +/// The leading bytes of the payload that [`produce_bulky`] sends as its +/// `sequence`-th message. +fn bulky_payload_prefix(sequence: u32) -> String { + format!("bulky-{sequence}-") +} + async fn seed_partition(client: &IggyClient) { seed_topic(client, None).await; produce(client, MESSAGES_COUNT).await; @@ -856,6 +1310,25 @@ fn segment_log_count(root: &Path) -> usize { count } +/// Inode and length of each segment log under `root`, by base offset. A +/// segment that keeps its inode through an install was linked, not copied. +fn segment_files(root: &Path) -> BTreeMap { + let mut files = BTreeMap::new(); + let _ = walk(root, &mut |path| { + if is_segment_log(path) + && let Some(start_offset) = path + .file_stem() + .and_then(|stem| stem.to_str()) + .and_then(|stem| stem.parse().ok()) + && let Ok(metadata) = std::fs::metadata(path) + { + files.insert(start_offset, (metadata.ino(), metadata.len())); + } + false + }); + files +} + /// A segment `.log`, named for its 20-digit zero-padded base offset (see /// `partitions::state_transfer`'s path builders). fn is_segment_log(path: &Path) -> bool { diff --git a/core/integration/tests/server/scenarios/delete_segments_scenario.rs b/core/integration/tests/server/scenarios/delete_segments_scenario.rs index 09959c71fa..3f0cabb800 100644 --- a/core/integration/tests/server/scenarios/delete_segments_scenario.rs +++ b/core/integration/tests/server/scenarios/delete_segments_scenario.rs @@ -775,6 +775,8 @@ pub async fn run_multi_consumer_barrier(harness: &mut TestHarness, restart_serve (TOTAL_MESSAGES - 1) as u64, ) .await; + // Stop background polling before the deliberate server restarts. + fast_consumer.shutdown().await.unwrap(); // --- Set up slow standalone consumer: store offset at 0 --- let slow_consumer = Consumer { diff --git a/core/metadata/src/applied_frontier.rs b/core/metadata/src/applied_frontier.rs index 812da20236..8034160ddf 100644 --- a/core/metadata/src/applied_frontier.rs +++ b/core/metadata/src/applied_frontier.rs @@ -17,11 +17,12 @@ //! The node-wide applied metadata frontier and the wait a read parks on. +use std::cell::Cell; use std::future::Future; use std::pin::Pin; -use std::sync::Mutex; use std::sync::PoisonError; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; use std::task::{Context, Poll, Waker}; use std::time::{Duration, Instant}; @@ -56,6 +57,37 @@ pub struct AppliedFrontier { waiters: Mutex, read_budget: Duration, partition_freshness: Option, + recovering_shards: AtomicUsize, +} + +/// A shard's recovery state, shared with the node's primary placement probes. +/// Dropping a shard releases its registration even during interrupted shutdown. +pub struct PartitionRecovery { + frontier: Arc, + active: Cell, +} + +impl PartitionRecovery { + pub fn set_active(&self, active: bool) { + if self.active.replace(active) == active { + return; + } + if active { + self.frontier + .recovering_shards + .fetch_add(1, Ordering::Release); + } else { + self.frontier + .recovering_shards + .fetch_sub(1, Ordering::Release); + } + } +} + +impl Drop for PartitionRecovery { + fn drop(&mut self) { + self.set_active(false); + } } /// Volatile quorum evidence. Recovery restores metadata, never freshness. @@ -113,9 +145,23 @@ impl AppliedFrontier { }), read_budget, partition_freshness: None, + recovering_shards: AtomicUsize::new(0), + } + } + + #[must_use] + pub fn track_partition_recovery(self: &Arc) -> PartitionRecovery { + PartitionRecovery { + frontier: Arc::clone(self), + active: Cell::new(false), } } + #[must_use] + pub fn has_partition_recovery(&self) -> bool { + self.recovering_shards.load(Ordering::Acquire) != 0 + } + /// Arm admission for a multi-node process. `epoch` must use the bus clock; /// a newly constructed frontier has no credit, including after recovery. #[must_use] @@ -346,6 +392,24 @@ mod tests { use std::task::{Context, Poll}; use std::time::{Duration, Instant}; + #[test] + fn given_recovering_shards_when_one_finishes_should_wait_for_the_last_registration() { + let frontier = Arc::new(AppliedFrontier::default()); + let first = frontier.track_partition_recovery(); + let second = frontier.track_partition_recovery(); + assert!(!frontier.has_partition_recovery()); + first.set_active(true); + first.set_active(true); + second.set_active(true); + first.set_active(false); + assert!(frontier.has_partition_recovery()); + drop(second); + assert!(!frontier.has_partition_recovery()); + first.set_active(true); + drop(first); + assert!(!frontier.has_partition_recovery()); + } + #[test] fn given_quorum_credit_when_the_deadline_arrives_should_expire_without_an_owner_tick() { let epoch = Instant::now(); diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs index edb7f38d73..f6b7931d0a 100644 --- a/core/metadata/src/impls/metadata.rs +++ b/core/metadata/src/impls/metadata.rs @@ -44,7 +44,7 @@ use consensus::{ build_result_rejection_reply, emit_sim_event, fatal, fence_old_prepare_by_commit, is_caught_up_primary, is_caught_up_primary_except_barrier, panic_if_hash_chain_would_break_in_same_view, peek_committable_head, pipeline_prepare_common, - register_preflight, replicate_preflight, replicate_to_next_in_chain, request_preflight, + register_preflight, replicate_preflight, replicate_prepare, request_preflight, send_eviction_to_client, send_prepare_ok as send_prepare_ok_common, verify_prepare_integrity, }; use futures::channel::oneshot; @@ -1932,15 +1932,13 @@ where // Durable here but not yet committed, and the primary is retransmitting // it: our original PrepareOk was lost (e.g. the primary's inbox - // overflowed under a client burst). Re-forward the tail down the chain - // so a downstream replica that missed it recovers, then re-ack ONLY - // the retransmitted op. The primary's retransmit cycle walks every + // overflowed under a client burst). Re-ack only the retransmitted op. + // The primary's retransmit cycle walks every // un-acked op in the window (`prepare_timeout_targets`), so a lost ack // for a lower op gets its own retransmit and its own re-ack; re-acking the // whole suffix here is O(window^2) PrepareOks per cycle across the // backups, which can overflow the primary's inbox -- the very failure - // this path recovers from. Both downstream and primary are idempotent - // on a duplicate (replica, op). + // this path recovers from. The primary is idempotent on duplicate acks. #[allow(clippy::cast_possible_truncation)] if journal.handle().header(header.op as usize).is_some() { warn!( @@ -1951,7 +1949,7 @@ where op = header.op, commit = consensus.commit_max(), operation = ?header.operation, - "journal already holds prepare, re-forwarding + re-acking it" + "journal already holds prepare, re-acking it" ); self.replicate(&message).await; self.send_prepare_ok(&header).await; @@ -5094,8 +5092,8 @@ where journal.handle().header(op).is_some(), "replicate: prepare must be durable in local journal before chain-forward" ); - if let Err(e) = replicate_to_next_in_chain(consensus, message).await { - tracing::warn!(op = header.op, error = ?e, "chain replication failed"); + if let Err(e) = replicate_prepare(consensus, message).await { + tracing::warn!(op = header.op, error = ?e, "prepare replication enqueue failed"); } } diff --git a/core/metadata/src/lib.rs b/core/metadata/src/lib.rs index af44a48aa2..121f0931ef 100644 --- a/core/metadata/src/lib.rs +++ b/core/metadata/src/lib.rs @@ -25,7 +25,7 @@ pub mod stm; // The node-wide read frontier the read gates park on; minted by the bootstrap // before any shard exists, so it is named outside `impls::`. -pub use applied_frontier::{AppliedFrontier, Reached}; +pub use applied_frontier::{AppliedFrontier, PartitionRecovery, Reached}; // Re-export IggyMetadata for use in other modules pub use impls::metadata::{ diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs index 8bd6c2e09f..cc79fd3e53 100644 --- a/core/partitions/src/iggy_partition.rs +++ b/core/partitions/src/iggy_partition.rs @@ -20,7 +20,7 @@ use crate::consumer_offset_capacity::{ DurableOffsetState, ExplicitOffsetStores, }; use crate::journal::{ - MessageLookup, PartitionJournal, PartitionJournalMemStorage, RetainedBatchMeta, + BodyPosition, MessageLookup, PartitionJournal, PartitionJournalMemStorage, RetainedBatchMeta, }; use crate::log::JournalInfo; use crate::log::SegmentedLog; @@ -52,7 +52,7 @@ use consensus::{ SimEventKind, VsrConsensus, ack_preflight, ack_quorum_reached, build_deny_reply_from_request, build_reply_from_request, build_reply_message, dvc_header_kind, emit_namespace_progress_event, emit_partition_diag, emit_sim_event, fence_old_prepare_by_commit, repair_session_live, - repaired_frontier_update, repaired_prepare_evidence, replicate_frozen_to_next_in_chain, + repaired_frontier_update, repaired_prepare_evidence, replicate_frozen_prepare, replicate_preflight, restamp_prepare_view, send_prepare_ok as send_prepare_ok_common, verify_prepare_integrity, }; @@ -107,6 +107,8 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use tokio::sync::Mutex as TokioMutex; use tracing::{debug, error, warn}; +const PREPARE_GAP_LOG_INTERVAL_MICROS: u64 = 1_000_000; + /// Which of a partition's offset counters are live. /// /// Two bits, not one: a reservation-seeded boot makes the append counter live @@ -200,6 +202,8 @@ where pub(crate) materialization_missing: bool, recovered_log_view: Option, pending_persisted_acks: RefCell>, + /// Without a WAL, an install must still preserve ops admitted to an ack. + pub(crate) acknowledged_op: Cell, /// In-flight journal repair: /// set when the recovery handshake finds this replica behind the group's /// commit frontier, cleared when `RepairDone` completes the walk. @@ -228,6 +232,7 @@ where /// the rest of the tick reads the partition through, so a `&mut` here would /// buy a second lookup of the same group per tick. prepare_gap_drops: Cell, + prepare_gap_log_after: Cell, /// Fault injection for the harness, armed by /// [`Self::inject_commit_failure`]. Behind a feature only the simulator's /// DEV-dependencies turn on, so no build that ships this crate compiles the @@ -262,7 +267,7 @@ where io_notifier: RefCell>, io_bytes_max: Cell, io_sequence: Cell, - io_active: Rc, + pub(crate) io_active: Rc, pub(crate) transition: Option, deferred_views: Vec, pub(crate) pending_consumer_offset_commits: HashMap, @@ -278,6 +283,15 @@ where /// First op of a broken fenced window the I/O lane is truncating. The fence's /// follow-up runs when that truncation lands. unproven_drop_from: Option, + /// `(commit_min, bytes)`: the bytes that repair journaled above that + /// `commit_min` while the view log was unproven. See + /// [`Self::proof_walk_over_budget`]. + proof_walk: (u64, u64), + pub(crate) proof_walk_memory: Option, + pub(crate) served_repair_tails: RefCell>, + /// The hole of the proof walk and the highest op held below it when the + /// last repair request went out. See [`Self::next_proof_walk_pull`]. + proof_walk_requested: Cell<(u64, Option)>, /// Identity shared with pending polls and replaced when their history retires. poll_history: PollHistoryId, incarnation: crate::PartitionIncarnation, @@ -408,15 +422,20 @@ where /// segment extends its own hasher over the bytes it gained. Without this, /// EVERY offer build re-reads and re-hashes all retained bytes on the pump /// -- and a committing primary advances `commit_op` each round, so the offer - /// cache alone never saves the pass. Swept against the live chain at build - /// time; cleared wherever segment files are unlinked-and-recreated - /// (install, converge). + /// cache alone never saves the pass. Retention drops the entry of each + /// segment it retires. An install replaces the map with the stamps of the + /// segments that it installs, and a converge clears the map. pub(crate) segment_checksum_cache: RefCell>, - /// Receiving-side memo of the last staged-segment reuse scan, so a peer - /// rotation against the same segment set does not re-read and re-walk every - /// staged file. See [`crate::state_transfer::ReuseScanMemo`]. + /// Verified staging files retained across partial scans and advancing offers. + /// See [`crate::state_transfer::ReuseScanMemo`]. pub(crate) reuse_scan_memo: RefCell>, + /// Resolves when the last staging writer ends. Reuse scans and spills wait + /// for it so abandoned tasks cannot race a new session on the same paths. + pub(crate) staging_work_ended: Option>, + /// The checksum state of the last segment that a transfer spilled. The + /// install keeps it as the memo of the segment it opens as active. + pub(crate) spilled_hasher: RefCell>, /// Serving-side offer cache, keyed by the `commit_op` it was built at, so /// simultaneous rejoiners share one manifest instead of re-reading every /// segment per requester. Invalidated by segment removal (same commit @@ -701,6 +720,7 @@ struct PendingMaterialization { durable_offset: Option, phase: MaterializationPhase, save: Option, + checksum: Option>, } #[derive(Clone, Copy)] @@ -910,9 +930,11 @@ where materialization_missing: false, recovered_log_view: None, pending_persisted_acks: RefCell::new(BTreeMap::new()), + acknowledged_op: Cell::new(0), repair: None, gap_ticks: Cell::new(0), prepare_gap_drops: Cell::new(0), + prepare_gap_log_after: Cell::new(0), #[cfg(any(test, feature = "fault-injection"))] injected_commit_failure: false, recovered_durable_offset: None, @@ -923,6 +945,10 @@ where recovered_unproven: false, recovered_commit: 0, unproven_drop_from: None, + proof_walk: (0, 0), + proof_walk_memory: None, + served_repair_tails: RefCell::new(HashMap::new()), + proof_walk_requested: Cell::new((0, None)), pending_commit: None, local_request_order: Cell::new(0), no_ack: VecDeque::new(), @@ -977,6 +1003,8 @@ where transfer_rearm: None, segment_checksum_cache: RefCell::new(std::collections::HashMap::new()), reuse_scan_memo: RefCell::new(None), + staging_work_ended: None, + spilled_hasher: RefCell::new(None), transfer_offer_cache: RefCell::new(None), transfer_plan: RefCell::new(None), }; @@ -1075,7 +1103,7 @@ where .map(|installed| installed.partition_op) } - /// Pending here or already applied, so another copy would only add a prepare. + /// Pending, applied, or superseded in this incarnation, so no prepare is needed. #[must_use] pub fn consumer_group_owner_admitted( &self, @@ -1085,7 +1113,12 @@ where || self .consumer_group_owners .get(&installation.group_id) - .is_some_and(|installed| installed.installation == *installation) + .is_some_and(|installed| { + let installed = installed.installation; + installed.incarnation == installation.incarnation + && (installed == *installation + || installed.owner.generation > installation.owner.generation) + }) } #[must_use] @@ -1803,6 +1836,10 @@ where && (self.consensus.state_transfer_stage() == consensus::StateTransferStage::Idle || self.consensus.status() == consensus::Status::ViewChange) { + // Without a certified log this replica has nothing to elect on, and + // the shard drops its votes, so an election after unanswered probes + // only lifts its view above the primary that must answer them. + self.consensus.set_recovery_election_allowed(false); self.consensus.begin_view_probe(); if self.consensus.state_transfer_stage() == consensus::StateTransferStage::Idle { self.consensus.begin_state_transfer_await(); @@ -2241,6 +2278,7 @@ where } pub async fn drive_persistence(&mut self) { + self.refresh_proof_walk_memory(); if self.fatal.is_some() { return; } @@ -2400,24 +2438,34 @@ where } pub fn register_rebuilt_ack(&self, header: &PrepareHeader) -> bool { + let ready = self.prepare_ack_ready(header); + if ready { + self.record_acknowledged_op(header.op); + } else { + self.pending_persisted_acks + .borrow_mut() + .insert(header.op, *header); + } + ready + } + + fn prepare_ack_ready(&self, header: &PrepareHeader) -> bool { // State-transfer admission uses the accepted WAL head to protect every // acknowledged op, including replicated prepares adopted in a new view. let accepted = self .persistence .as_ref() .is_none_or(|persistence| header.op <= persistence.head()); - let durable = accepted + accepted && (!self.requires_persistence(header.operation) || self.persistence.as_ref().is_some_and(|persistence| { persistence.is_durable(header) && persistence.certified_log_view() == Some(self.consensus.log_view()) - })); - if !durable { - self.pending_persisted_acks - .borrow_mut() - .insert(header.op, *header); - } - durable + })) + } + + fn record_acknowledged_op(&self, op: u64) { + self.acknowledged_op.set(self.acknowledged_op.get().max(op)); } fn submit_prepare_persistence(&self, prepare: Frozen<4096>, operation: Operation) -> bool { @@ -5795,7 +5843,7 @@ where self.append_stamped_messages(message, batch).await } #[must_use] - fn namespace(&self) -> IggyNamespace { + pub(crate) fn namespace(&self) -> IggyNamespace { IggyNamespace::from_raw(self.consensus.group()) } @@ -5818,6 +5866,17 @@ where } } + /// [`Self::notify_io`] for a detached task that outlives this borrow. + pub(crate) fn io_wake(&self) -> impl Fn() + 'static { + let notifier = self.io_notifier.borrow().clone(); + let (namespace, incarnation) = (self.namespace(), self.incarnation); + move || { + if let Some(notifier) = ¬ifier { + notifier(namespace, incarnation); + } + } + } + #[must_use] pub fn needs_io_retry(&self) -> bool { if self.io_active.is_interrupted() { @@ -6291,10 +6350,18 @@ where if !self.is_shutting_down() && self.transfer.as_ref().is_some_and(|session| { session.target_accepted - && session - .artifacts - .iter() - .all(consensus::ChunkProgress::complete) + && (session + .scan + .as_ref() + .is_some_and(crate::state_transfer::LocalWork::landed) + || session + .seed + .as_ref() + .is_some_and(crate::state_transfer::PrefixSeed::landed) + || session + .artifacts + .iter() + .all(consensus::ChunkProgress::complete)) }) && self.consensus.state_transfer_stage() == consensus::StateTransferStage::Fetching { @@ -6731,6 +6798,16 @@ where } } + /// Whether the next offer round hashes against a pinned plan, instead of + /// returning at once to wait for the pin. + #[must_use] + pub fn transfer_plan_pinned(&self) -> bool { + matches!( + *self.transfer_plan.borrow(), + Some(TransferPlanSlot::Pinned(_)) + ) + } + /// The plan an offer round hashes against, or `None` while the flush that /// has to come before it runs. A mounted owner pins the plan in the I/O /// turn that finds every applied op in its segment, so commits landing @@ -7486,6 +7563,17 @@ where .await; return; } + if self.consumer_group_owner_admitted(&installation) { + Self::send_partition_deny_or_log( + consensus, + message.header(), + IggyError::TransientNotAccepted.as_code(), + "ownership installation is already applied or superseded", + reply.take(), + ) + .await; + return; + } Some(installation) } else { None @@ -8437,15 +8525,12 @@ where let journal_holds_op = self.log.journal().inner.holds_op(header.op); if journal_holds_op { - // Retransmit after downstream flap: durable here but commit - // hasn't caught up. Re-forward + re-ACK so primary's view of - // us is consistent. Both downstream and primary are idempotent - // on duplicate (replica, op). + // Re-ack retransmissions so a lost acknowledgement cannot stall commit. emit_partition_diag( tracing::Level::DEBUG, &PartitionDiagEvent::new( self.diag_ctx(), - "journal already holds prepare, re-forwarding + re-acking", + "journal already holds prepare, re-acking", ) .with_operation(header.operation) .with_op(header.op), @@ -8492,13 +8577,12 @@ where .map(|_| frozen_for_forward.clone()); let consensus = self.consensus(); if consensus.is_follower() && header.op > consensus.sequencer().current_sequence() { - consensus.sequencer().set_sequence(header.op); - consensus.set_last_prepare_checksum(header.checksum); + // Repair can hold this op above a hole, so the head moves only + // along the contiguous run. + self.advance_head_over_journaled_run(); consensus.observe_prepare_timestamp(header.timestamp); } - if let Err(error) = - replicate_frozen_to_next_in_chain(consensus, frozen_for_forward).await - { + if let Err(error) = replicate_frozen_prepare(consensus, frozen_for_forward).await { let is_transport_error = error.is_transport(); emit_partition_diag( if is_transport_error { @@ -8508,7 +8592,7 @@ where }, &PartitionDiagEvent::new( self.diag_ctx(), - "failed to re-forward retransmitted prepare to next in chain", + "failed to enqueue retransmitted prepare for replication", ) .with_operation(header.operation) .with_op(header.op) @@ -8540,32 +8624,28 @@ where .inner .repaired_window_shape(self.consensus().commit_min(), header.op - 1) .complete - && if header.op == 1 { - header.parent == 0 - } else { - self.log - .journal() - .inner - .repair_header(header.op - 1) - .is_some_and(|previous| previous.checksum == header.parent) - }; + && self.chains_to_journaled_predecessor(&header); if header.op != current_op + 1 && !fills_announced_gap { // `sequence` is what separates the two shapes this line covers: // a forward gap (op above the sequencer, the hole the repair // driver closes) and a retransmit of an op this replica already // sequenced. Without it they read identically. - emit_partition_diag( - tracing::Level::WARN, - &PartitionDiagEvent::new( - self.diag_ctx(), - "dropping out-of-order prepare (gap)", - ) - .with_operation(header.operation) - .with_op(header.op) - .with_sequence(current_op), + self.record_prepare_gap(&header, current_op, "dropping out-of-order prepare (gap)"); + // An op at or below the head is a retransmit, which the primary + // sends straight to every backup that has not acknowledged it. + return; + } + // While a transfer re-arm waits, an op above a hole only grows a + // journal that the install wipes. A `StartView` adopted in the wait + // lifts the head above the hole, so the next live op passes the gap + // check. The lowest missing op still lands, so the live stream can + // fill the hole. + if self.transfer_rearm.is_some() && header.op > self.held_through() + 1 { + self.record_prepare_gap( + &header, + current_op, + "dropping prepare above a hole while a transfer re-arm waits", ); - self.prepare_gap_drops - .set(self.prepare_gap_drops.get().saturating_add(1)); return; } } else { @@ -8588,7 +8668,7 @@ where // - above the sequencer: journaling an op the sequencer has not // assigned yet means the next local assignment would collide // with it. Unreachable today (view fences run first, one - // primary per view, the chain stops before the primary), so + // primary per view, fanout targets only backups), so // trip the invariant in debug; in release log loudly and drop // rather than crash a library or corrupt op assignment. if header.op > current_op { @@ -8622,9 +8702,7 @@ where ); } } - // Forward only after apply_replicated_operation journals the prepare. - // The journal and network share the frozen allocation, so the bytes - // retained for repair are exactly the bytes sent downstream. + // The journal and primary fanout share the frozen allocation. let replicated_result = if is_backup && header.operation == Operation::SendMessages { self.append_received_send_messages_to_journal(message).await } else { @@ -8705,10 +8783,18 @@ where if header.op >= current_op { consensus.sequencer().set_sequence(header.op); consensus.set_last_prepare_checksum(header.checksum); + // A filled hole joins the run that repair journaled above it. + if header + .op + .checked_add(1) + .is_some_and(|next| self.log.journal().inner.holds_op(next)) + { + self.advance_head_over_journaled_run(); + } } consensus.observe_prepare_timestamp(header.timestamp); } - if let Err(error) = replicate_frozen_to_next_in_chain(consensus, frozen_for_forward).await { + if let Err(error) = replicate_frozen_prepare(consensus, frozen_for_forward).await { let is_transport_error = error.is_transport(); emit_partition_diag( if is_transport_error { @@ -8718,7 +8804,7 @@ where }, &PartitionDiagEvent::new( self.diag_ctx(), - "failed to replicate prepare to next in chain", + "failed to enqueue prepare for replication", ) .with_operation(header.operation) .with_op(header.op) @@ -8747,6 +8833,65 @@ where self.send_prepare_ok(&header).await; } + /// Whether `header` chains onto the entry this replica holds one op below + /// it, resident or retained for repair. + fn chains_to_journaled_predecessor(&self, header: &PrepareHeader) -> bool { + if header.op == 1 { + header.parent == 0 + } else { + self.log + .journal() + .inner + .repair_header(header.op - 1) + .is_some_and(|previous| previous.checksum == header.parent) + } + } + + fn record_prepare_gap(&self, header: &PrepareHeader, sequence: u64, message: &'static str) { + self.prepare_gap_drops + .set(self.prepare_gap_drops.get().saturating_add(1)); + let now = self.consensus().message_bus().monotonic_micros(); + if now < self.prepare_gap_log_after.get() { + return; + } + self.prepare_gap_log_after + .set(now.saturating_add(PREPARE_GAP_LOG_INTERVAL_MICROS)); + emit_partition_diag( + tracing::Level::WARN, + &PartitionDiagEvent::new(self.diag_ctx(), message) + .with_operation(header.operation) + .with_op(header.op) + .with_sequence(sequence), + ); + } + + /// Move the head along the contiguous run the journal holds above it. + /// + /// DVC advertises the head and elections pick the highest, so a head above + /// a hole can win a view it cannot walk. A hole stops the head, and the op + /// that fills it, from repair or from the live stream, moves the head over + /// every op journaled above it meanwhile. The checksum comes from the + /// journaled header at the new head, so a lower backfill never rewinds the + /// parent the next prepare chains onto. No append observes the timestamps + /// of the passed ops: repair does not, and their live copies arrive below + /// the head. The head's timestamp therefore joins the floor here. + fn advance_head_over_journaled_run(&self) { + let consensus = self.consensus(); + let journal = &self.log.journal().inner; + let Some((head, checksum)) = + repaired_frontier_update(consensus.sequencer().current_sequence(), |op| { + journal.header_by_op(op) + }) + else { + return; + }; + consensus.sequencer().set_sequence(head); + consensus.set_last_prepare_checksum(checksum); + if let Some(header) = journal.header_by_op(head) { + consensus.observe_prepare_timestamp(header.timestamp); + } + } + #[allow(clippy::future_not_send)] pub async fn on_ack(&mut self, message: Message, config: &PartitionsConfig) { if self.fatal.is_some() { @@ -8834,6 +8979,7 @@ where /// replica ask its peers for it. #[allow(clippy::future_not_send)] pub async fn commit_journal(&mut self, config: &PartitionsConfig) { + self.refresh_proof_walk_memory(); if self.transition.is_some() || self.io_active.is_retiring() { self.notify_io(); return; @@ -9183,13 +9329,32 @@ where } else { 0 }; - if (validated.base_offset, validated.base_timestamp) != (expected_offset, header.timestamp) - { + if validated.base_timestamp != header.timestamp { return Err(IggyError::CannotAppendMessage); } - self.append_stamped_messages(message, validated) + if validated.base_offset == expected_offset { + return self + .append_stamped_messages(message, validated) + .await + .map(|journaled| journaled.prepare); + } + // Repair can journal ops above a hole and lift the dirty offset past + // it, so the live prepare that fills the hole starts below that offset. + // Chained to the entry below it, it carries the offsets the primary + // stamped, which repair would journal for this op as well. + let fills_hole = validated.base_offset < expected_offset + && self + .log + .journal() + .inner + .last_op() + .is_some_and(|last| last > header.op) + && self.chains_to_journaled_predecessor(&header); + if !fills_hole { + return Err(IggyError::CannotAppendMessage); + } + self.append_canonical_send_messages(message, validated) .await - .map(|journaled| journaled.prepare) } async fn append_stamped_messages( @@ -9307,6 +9472,32 @@ where .held_through(self.consensus.commit_min()) } + /// Progress follows the same per-operation durability contract as prepare acks. + #[must_use] + pub fn replication_progress(&self) -> Option { + if !self.consensus.is_normal() + || self.consensus.is_transferring() + || self.consensus.has_untrusted_log() + || self.view_fenced() + || self.read_history_is_changing() + { + return None; + } + let held = self.held_through(); + let mut received = self.consensus.commit_min(); + while received < held { + let next = received + 1; + let Some(header) = self.log.journal().inner.repair_header(next) else { + break; + }; + if !self.consensus.view_log_entry_proven(next) || !self.prepare_ack_ready(&header) { + break; + } + received = next; + } + Some(received) + } + /// Whether an anchored fence holds every entry of its window, so a proof /// now concludes instead of waiting on repair. #[must_use] @@ -9750,6 +9941,10 @@ where } self.offset_reservations_need_resync.set(true); self.truncated_since_scan.set(true); + drop(entries); + if self.log.journal().inner.is_empty() { + self.reset_proof_walk_memory(); + } Ok(removed) } @@ -9927,6 +10122,7 @@ where durable_offset: None, phase: MaterializationPhase::Inspect, save: None, + checksum: None, }); Ok(true) } @@ -10382,7 +10578,10 @@ where pending.evictable = pending.evictable.max(count); return; } - let retained = self.log.journal().inner.evict_prefix(count); + let persistence = self.persistence.as_deref(); + let retained = self.log.journal().inner.evict_prefix_located(count, |op| { + persistence?.segment_reference(op).map(BodyPosition::from) + }); // Eviction drains the ring and re-appends the retained tail, so the slots // a suffix snapshot describes move under it without the head or the // commit point changing. @@ -11757,7 +11956,14 @@ where index_position: index_writer.position(), index_writer, }; - let frozen_batches = if bodies_written.is_some() { + let checksum = if written == 0 { + self.take_segment_hasher(target.segment_size) + } else { + self.materialization + .as_mut() + .and_then(|pending| pending.checksum.take()) + }; + let frozen_batches = if bodies_written.is_some() && checksum.is_none() { Vec::new() } else { frozen_batches @@ -11775,6 +11981,7 @@ where bodies_written, written, completes, + checksum, }; Ok(Some(job)) } @@ -11839,6 +12046,19 @@ where } target.index_writer.advance(saved_indexes); self.log.segments_mut()[segment_index].size = IggyByteSize::from(size); + if saved == 0 { + if let Some(pending) = self.materialization.as_mut() { + pending.checksum = result.checksum; + } + } else if let Some(hasher) = result.checksum { + self.segment_checksum_cache.borrow_mut().insert( + target.segment_start, + crate::state_transfer::SegmentChecksumMemo::Hashing { + hashed_len: size, + hasher, + }, + ); + } Ok(()) } @@ -11959,6 +12179,7 @@ where self.log.index_writers_mut()[sealed_index] = None; self.log.indexes_mut()[sealed_index] = None; self.log.reset_read_state(sealed_index); + self.stamp_sealed_segment(target.segment_start); debug!( namespace_raw = self.namespace().inner(), sealed_end = target.segment_end, @@ -12091,8 +12312,15 @@ where fn capture_segment_removal(&mut self) -> Option> { let (segment, storage) = self.log.retire_front()?; + self.log + .journal() + .inner + .forget_segment(segment.start_offset); let (messages, indexes) = storage.segment_and_index_paths(); let anchor = self.anchor_cleanup_path(segment.start_offset); + self.segment_checksum_cache + .borrow_mut() + .remove(&segment.start_offset); self.maintenance.removing = Some(segment); self.clear_state_transfer_offer_cache(); Some(crate::PartitionIoJob::RemoveSegment { @@ -12509,14 +12737,63 @@ where } } + /// Bytes that repair journaled above `commit_min` while the view log was + /// unproven. + const fn proof_walk_bytes(&self) -> u64 { + let (commit_min, bytes) = self.proof_walk; + if commit_min == self.consensus().commit_min() { + bytes + } else { + 0 + } + } + + fn refresh_proof_walk_memory(&mut self) { + if !self.consensus.view_log_unproven() || self.proof_walk.0 != self.consensus.commit_min() { + self.reset_proof_walk_memory(); + } + } + + pub(crate) fn reset_proof_walk_memory(&mut self) { + if let Some(memory) = &mut self.proof_walk_memory { + memory.clear(); + } + self.proof_walk = (self.consensus.commit_min(), 0); + } + + /// Whether an unproven repair exceeded its WAL budget or exhausted the + /// shared allocation budget before accepting another frame. + /// + /// The walk fetches ops from the adopted head down, and nothing above + /// `commit_min` commits before it reaches `commit_min`, so every body it + /// fetches stays in memory until then. The WAL budget already caps the + /// prepares that a partition holds above its WAL checkpoint, so the walk + /// can hold as much, plus the one body that crosses it. Past that, state + /// transfer takes the window. Bodies that a later truncation drops still + /// count, so the bound errs toward transfer. The node's allocation budget + /// also applies without a WAL and never admits the body that exceeds it. + fn proof_walk_over_budget(&self) -> bool { + self.consensus().view_log_unproven() + && (self + .proof_walk_memory + .as_ref() + .is_some_and(crate::repair_memory::RepairMemoryReservation::exhausted) + || self + .persistence + .as_ref() + .is_some_and(|persistence| self.proof_walk_bytes() > persistence.capacity())) + } + /// Ingest one repaired prepare: journal + stage it exactly like a live /// replicated op, minus the view-number check, the gap check, and the ack (the /// op is already committed cluster-wide; there is nobody to ack to). The /// commit walk runs at `RepairDone`, after the floor is known. + #[allow(clippy::too_many_lines)] pub async fn apply_repaired_prepare( &mut self, message: Message, ) -> RepairIngest { + self.refresh_proof_walk_memory(); let header = *message.header(); if self.view_truncation_pending() || !self.prepare_history_available(&header) { return RepairIngest::Handled; @@ -12599,12 +12876,30 @@ where self.restart_repair_budget(); return RepairIngest::Handled; } + // A walk past its budget journals nothing more, and `complete_repair` + // hands the window to state transfer. + if self.proof_walk_over_budget() { + return RepairIngest::Handled; + } + let charge = if self.consensus.view_log_unproven() { + message.retained_allocation_size().unwrap_or(usize::MAX) + } else { + 0 + }; + if let Some(memory) = &mut self.proof_walk_memory + && !memory.try_add(charge) + { + return RepairIngest::Handled; + } let applied = if header.operation == Operation::SendMessages { self.append_repaired_send_messages(message).await } else { self.apply_replicated_operation(message).await.map(|_| ()) }; if let Err(error) = applied { + if let Some(memory) = &mut self.proof_walk_memory { + memory.release(charge); + } warn!( target: "iggy.partitions.diag", plane = "partitions", @@ -12619,27 +12914,16 @@ where // its peer serves it again every round, and a refund per serve would // pin the session to that peer. self.restart_repair_budget(); + if self.consensus().view_log_unproven() { + self.proof_walk = ( + self.consensus().commit_min(), + self.proof_walk_bytes() + u64::from(header.size), + ); + } self.consensus.note_repaired_view_header(header, evidence); + self.refresh_proof_walk_memory(); self.persist_repaired_prefix(); - // Advance the sequencer only along the CONTIGUOUS journaled - // frontier. DVC advertises `op = sequencer.current_sequence()` and - // elections pick the max, so bumping straight to a repaired op that - // sits above an unfilled hole would let this replica win a view it - // cannot walk. A dropped frame stalls the frontier here; the stall - // retry refills the hole and the next apply resumes the advance - // (walking over ops that were journaled out of order meanwhile). - // The checksum moves only with the frontier and is read from the - // journal header at the new head: a lower backfill must not rewind - // the parent the next prepare chains onto. - let previous_frontier = self.consensus().sequencer().current_sequence(); - let update = repaired_frontier_update(previous_frontier, |op| { - self.log.journal().inner.header_by_op(op) - }); - if let Some((frontier, frontier_checksum)) = update { - let consensus = self.consensus(); - consensus.sequencer().set_sequence(frontier); - consensus.set_last_prepare_checksum(frontier_checksum); - } + self.advance_head_over_journaled_run(); self.offset_reservations_need_resync.set(true); RepairIngest::Handled } @@ -12656,10 +12940,31 @@ where let Some(mut session) = self.repair else { return RepairConclusion::Done; }; - if !self.consensus().is_normal() || self.consensus().view() != session.view { + if !consensus::repair_session_live(self.consensus()) + || self.consensus().view() != session.view + { self.repair = None; return RepairConclusion::Done; } + if self.proof_walk_over_budget() { + let node_memory_exhausted = self + .proof_walk_memory + .as_ref() + .is_some_and(crate::repair_memory::RepairMemoryReservation::exhausted); + tracing::info!( + target: "iggy.partitions.diag", + plane = "partitions", + namespace_raw = self.namespace().inner(), + walk_bytes = self.proof_walk_bytes(), + node_memory_exhausted, + "unproven repair walk is past its memory budget, so state transfer takes the window" + ); + self.repair = None; + return RepairConclusion::FloorRefused { + floor: self.consensus().commit_min(), + to_op: session.commit_to_op, + }; + } let journal = &self.log.journal().inner; // Apply a resident prefix normally instead of substituting the peer's // floor for operations recovered after boot. @@ -12754,13 +13059,8 @@ where return conclusion; } } - if self - .persistence - .as_ref() - .is_some_and(|persistence| !persistence.is_durable_through(session.fetch_to_op)) - { - return RepairConclusion::InProgress; - } + // Apply verified prefixes before the whole window is durable so a + // checkpoint can reclaim WAL capacity for the remaining repair. let before = self.consensus().commit_min(); self.commit_journal(config).await; let commit_min = self.consensus().commit_min(); @@ -12890,6 +13190,25 @@ where Some(RepairConclusion::InProgress) } + /// Whether another bounded repair response can fit before checkpointing. + /// Unproven histories and unresolved floors must fetch before they can + /// apply a prefix, so WAL backpressure must not stop their proof walks. + #[must_use] + pub fn repair_fetch_ready(&self) -> bool { + self.view_fenced() + || self.consensus.has_untrusted_log() + || self.consensus.view_log_unproven() + || self.consensus.commit_min() >= self.consensus.commit_max() + || self.repair.is_some_and(|session| { + session + .floor + .is_some_and(|floor| floor > self.consensus.commit_min()) + }) + || self.persistence.as_ref().is_none_or(|persistence| { + persistence.has_capacity(journal::partition_journal::PREPARE_BYTES_MAX) + }) + } + /// Where the next pull of the repair window starts: past the resident /// prefix above a floor that still holds the walk, else past the held run /// above `commit_min`. `None` without a session. @@ -12929,6 +13248,33 @@ where Some(from_op) } + /// Highest op held below the hole that the proof walk waits on, for the + /// repair request about to go out, whose window starts above it. A donor + /// serves a window from its bottom up to a disk byte budget, so a window + /// that starts at held ops gets them again on every round and never + /// reaches the hole. Records the hole with it for + /// [`Self::next_proof_walk_pull`]. + pub fn note_proof_walk_request(&self) -> Option { + let hole = self.consensus().view_log_repair_op()?; + let held_below = self.log.journal().inner.held_below(hole); + self.proof_walk_requested.set((hole, held_below)); + held_below + } + + /// The unproven hole to pull again right after a `RepairDone` when the + /// walk gained ops below it since the last request, or `None`. The next + /// window then starts above them. A round that brought nothing leaves the + /// hole to the retry timer. + #[must_use] + pub fn next_proof_walk_pull(&self) -> Option { + let hole = self + .consensus() + .view_log_repair_op() + .filter(|&hole| !self.consensus().view_log_entry_proven(hole))?; + let walked = (hole, self.log.journal().inner.held_below(hole)); + (walked != self.proof_walk_requested.get()).then_some(hole) + } + /// The lowest resident `SendMessages` batch in `(floor, commit_to_op]`: /// where the repair window begins in offset space. /// @@ -13007,16 +13353,26 @@ where if !self.prepare_history_available(message.header()) { return Err(IggyError::HistoryUnavailable); } + let batch = decode_prepare_slice(message.as_slice())?.header; + self.append_canonical_send_messages(message, batch).await?; + Ok(()) + } - let (base_offset, base_timestamp, total_size, message_count) = { - let batch = decode_prepare_slice(message.as_slice())?; - ( - batch.header.base_offset, - batch.header.base_timestamp, - batch.header.total_size() as u64, - batch.message_count(), - ) - }; + /// Journal a `SendMessages` prepare at the offsets its batch carries: a + /// repaired op, or a live op that fills a hole below ops repair journaled. + /// The dirty offset only grows, so a fill below it leaves it in place. The + /// caller holds the write lock. + async fn append_canonical_send_messages( + &mut self, + message: Message, + batch: BatchHeader, + ) -> Result, IggyError> { + let (base_offset, base_timestamp, total_size, message_count) = ( + batch.base_offset, + batch.base_timestamp, + batch.total_size() as u64, + batch.message_count, + ); if message_count == 0 { return Err(IggyError::InvalidCommand); } @@ -13052,14 +13408,14 @@ where journal_info.max_timestamp = journal_info.max_timestamp.max(base_timestamp); let frozen = message.into_frozen(); - self.journal_append(frozen).await?; + self.journal_append(frozen.clone()).await?; self.note_append_live(); self.dirty_offset .store(dirty.max(last_offset), Ordering::Relaxed); self.log.segments_mut()[segment_index].current_position = next_position; self.log.journal_mut().info = journal_info; - Ok(()) + Ok(frozen) } /// Retirement requests name sessions rather than histories; their prepares @@ -13111,6 +13467,7 @@ where // consumer-offset ops (via `apply_replicated_operation`) append // to that journal before `send_prepare_ok` fires, so every op // that reaches here is journal-backed and ACKs as durable. + self.record_acknowledged_op(header.op); send_prepare_ok_common(self.consensus(), header, true).await } @@ -13269,7 +13626,7 @@ fn peek_op(entry: &Frozen<4096>) -> u64 { .op } -fn materialization_batch_meta( +pub fn materialization_batch_meta( entry: &Frozen<4096>, persisted_end: Option, ) -> Result, IggyError> { @@ -13620,8 +13977,8 @@ mod tests { use consensus::LocalPipeline; use iggy_binary_protocol::batch::BATCH_MESSAGE_HEADER_SIZE; use iggy_binary_protocol::{ - Command, CommitHeader, ConsensusHeader, ReplyHeader, StartViewHeader, WireConsumer, - WireEncode, WireName, + Command, CommitHeader, CommitReplyHeader, ConsensusHeader, HEADER_SIZE, ReplyHeader, + StartViewHeader, WireConsumer, WireEncode, WireName, }; use iggy_common::Aes256GcmEncryptor; use iggy_common::Durability::{Persisted, Replicated}; @@ -13846,6 +14203,72 @@ mod tests { assert_eq!(offsets.next_offset, 3); } + /// A tail whose bodies the ring released to segments does not keep a + /// plan pinned. The receiver would repair it from disk, which moves far + /// less per round than repair from memory, so the source re-pins once the + /// tail of the plan leaves memory. + #[compio::test] + async fn given_a_transfer_pin_whose_tail_left_memory_when_hashing_finishes_should_request_a_fresh_plan() + { + let root = tempfile::tempdir().unwrap(); + let (mut partition, _, config, mut parent) = + partition_with_committed_send(root.path(), iggy_common::Durability::Replicated, 3) + .await; + partition.set_io_notifier( + Rc::new(|_, _| {}), + crate::largest_legal_job_charge().unwrap(), + ); + let journal = &partition.log.journal().inner; + journal.set_repair_retention(true); + journal.set_ring_caps(crate::EVICTED_RING_CAPACITY, 1); + assert!(matches!( + partition.state_transfer_offer(&config).await, + Err(crate::state_transfer::PartitionTransferUnavailable::FlushPending) + )); + settle_partition_io(&mut partition, &config).await; + + let persistence = Rc::clone(partition.persistence.as_ref().unwrap()); + for op in 2..=3 { + let prepare = checksummed_segment_prepare(op, parent, op - 1, b"tail"); + let header = *prepare.header(); + parent = header.checksum; + persistence + .append(prepare.clone().into_frozen(), false) + .unwrap(); + assert!(persistence.start()); + Rc::clone(&persistence).run().await; + partition.consensus.with_pipeline_mut(|pipeline| { + pipeline.push(PipelineEntry::new(header)); + }); + partition + .append_repaired_send_messages(prepare) + .await + .unwrap(); + partition.consensus.advance_commit_max(op); + partition.commit_journal(&config).await; + settle_partition_io(&mut partition, &config).await; + } + assert_eq!(partition.consensus().commit_min(), 3); + assert!( + partition.log.journal().inner.evicted_header(2).is_some(), + "the byte cap released the body of op 2 to its segment" + ); + + let offer = partition.state_transfer_offer(&config).await; + assert!( + matches!( + offer, + Err(crate::state_transfer::PartitionTransferUnavailable::FlushPending) + ), + "{:?}", + offer.map(|offer| offer.commit_op) + ); + assert!(matches!( + *partition.transfer_plan.borrow(), + Some(TransferPlanSlot::Wanted) + )); + } + #[compio::test] async fn given_ops_committing_during_a_transfer_when_the_receiver_installs_should_land_on_the_pinned_op() { @@ -13901,7 +14324,7 @@ mod tests { let mut bytes = std::fs::read(&segment.log_path).unwrap(); bytes.truncate(usize::try_from(segment.entry.len).unwrap()); let staged = receiver - .spill_transfer_segment(&segment.entry, bytes) + .spill_transfer_segment(&segment.entry, bytes, None) .await .unwrap(); receiver @@ -13956,6 +14379,873 @@ mod tests { assert!(!partition.maintenance.transfer_flush); } + /// Removes every segment log of `partition`, then builds an offer. The + /// offer can only take each stamp from the checksum memo, and each stamp + /// must equal the manifest checksum of the bytes that the file held. + async fn assert_offer_reads_no_segment_file(partition: &mut IggyPartition) { + let config = repair_config(); + let expected = segment_artifacts_on_disk(partition); + let directory = partition.partition_dir().unwrap(); + for artifact in &expected { + std::fs::remove_file(format!("{directory}/{:020}.log", artifact.frontier)).unwrap(); + } + + assert!(matches!( + partition.state_transfer_offer(&config).await, + Err(PartitionTransferUnavailable::FlushPending) + )); + settle_partition_io(partition, &config).await; + let offer = partition.state_transfer_offer(&config).await.unwrap(); + + let offered: Vec<_> = offer.segments.iter().map(|segment| segment.entry).collect(); + assert_eq!(offered, expected); + } + + /// A save that the I/O byte cap splits into pieces feeds the memo the + /// bytes of every piece, in file order. + #[compio::test] + async fn given_saves_split_into_pieces_when_building_an_offer_should_not_read_the_segments() { + const OPS: u64 = 4; + let config = repair_config(); + let (directory, mut partition) = + Box::pin(disk_poll_partition_with_replicas(&config, 3)).await; + let job_charge_max = crate::largest_legal_job_charge().unwrap(); + partition.set_io_notifier(Rc::new(|_, _| {}), job_charge_max); + for op in 1..=OPS { + journal_send_batch(&mut partition, op).await; + } + let piece_charge = partition + .log + .journal() + .inner + .with_entries(|entries| { + crate::io::materialization_allocation_charge( + &entries[..1], + 1, + crate::iggy_index::IGGY_INDEX_SIZE, + ) + }) + .unwrap(); + partition.io_bytes_max.set(piece_charge); + partition.consensus().advance_commit_max(OPS); + settle_partition_io(&mut partition, &config).await; + partition.io_bytes_max.set(job_charge_max); + assert!(partition.fatal().is_none()); + assert_eq!( + std::fs::metadata(directory.path().join(format!("{:020}.index", 0))) + .unwrap() + .len(), + crate::iggy_index::IGGY_INDEX_SIZE as u64, + "one save must write every batch, so the cap split it into pieces" + ); + + Box::pin(assert_offer_reads_no_segment_file(&mut partition)).await; + } + + /// A save whose bodies the partition WAL already wrote feeds the memo the + /// same bytes that the segment file holds. The group has peers, so its + /// journal keeps the committed prepare that an offer carries. + #[compio::test] + async fn given_segment_bodies_written_by_the_wal_when_building_an_offer_should_not_read_them() { + const REPLICAS: u8 = 3; + for durability in [ + iggy_common::Durability::Replicated, + iggy_common::Durability::Persisted, + ] { + let root = tempfile::tempdir().unwrap(); + let (mut partition, _replies, _config, _parent) = + partition_with_committed_send(root.path(), durability, REPLICAS).await; + assert!( + partition + .log + .messages_writers() + .last() + .is_some_and(Option::is_none), + "{durability:?}: the segment must have no writer, so the WAL wrote its bodies" + ); + partition.set_io_notifier( + Rc::new(|_, _| {}), + crate::largest_legal_job_charge().unwrap(), + ); + + Box::pin(assert_offer_reads_no_segment_file(&mut partition)).await; + } + } + + /// One-message batches each segment of [`partition_with_sealed_segments`] + /// holds before it rolls. + const BATCHES_PER_SEGMENT: u64 = 2; + + /// `count` one-message records. Each call gives its messages new random + /// ids, so two calls give records of the same lengths but other bytes. + fn segment_records(count: u64) -> Vec> { + (0..count) + .map(|_| build_segment_record(IggyNamespace::new(1, 1, 0), 0)) + .collect() + } + + /// A primary on disk whose segments roll every [`BATCHES_PER_SEGMENT`] + /// batches, with one op per record committed and written to its segments. + async fn partition_with_sealed_segments( + records: &[Vec], + ) -> (tempfile::TempDir, IggyPartition) { + let config = repair_config(); + let (directory, mut partition) = + Box::pin(disk_poll_partition_with_replicas(&config, 3)).await; + let batch_len = build_segment_record(partition.namespace(), 0).len() as u64; + let segment_size = IggyByteSize::from(BATCHES_PER_SEGMENT * batch_len); + partition.runtime_options.segment_size = Some(segment_size); + partition.log.active_segment_mut().max_size = segment_size; + partition.set_io_notifier( + Rc::new(|_, _| {}), + crate::largest_legal_job_charge().unwrap(), + ); + Box::pin(commit_records(&mut partition, 1, records)).await; + assert_eq!(partition.consensus().commit_min(), records.len() as u64); + (directory, partition) + } + + /// Journals `records` as the ops from `first_op` on, and commits and + /// writes each one before the next. + async fn commit_records( + partition: &mut IggyPartition, + first_op: u64, + records: &[Vec], + ) { + let config = repair_config(); + for (op, record) in (first_op..).zip(records) { + journal_send_record(partition, op, record).await; + partition.consensus().advance_commit_max(op); + settle_partition_io(partition, &config).await; + } + } + + /// The offer `partition` serves once the I/O turn pinned its plan. + async fn serve_offer( + partition: &mut IggyPartition, + ) -> Rc { + let config = repair_config(); + assert!(matches!( + partition.state_transfer_offer(&config).await, + Err(PartitionTransferUnavailable::FlushPending) + )); + settle_partition_io(partition, &config).await; + partition.state_transfer_offer(&config).await.unwrap() + } + + /// The manifest entry of each segment of `partition`, read from its file. + fn segment_artifacts_on_disk( + partition: &IggyPartition, + ) -> Vec { + let directory = partition.partition_dir().unwrap(); + partition + .log + .segments() + .iter() + .map(|segment| { + let bytes = + std::fs::read(format!("{directory}/{:020}.log", segment.start_offset)).unwrap(); + assert_eq!(bytes.len() as u64, segment.size.as_bytes_u64()); + consensus::StateArtifact::for_bytes( + consensus::state_manifest::artifact_kind::SEGMENT_LOG, + segment.start_offset, + &bytes, + ) + }) + .collect() + } + + /// The append feeds the checksum memo of each segment it writes, so the + /// offer takes every stamp from memory and reads no segment file. The test + /// removes the files before the build to prove that. The stamps must still + /// equal the manifest checksum of the bytes that the files held. + #[compio::test] + async fn given_segments_written_by_this_process_when_building_an_offer_should_not_read_them() { + const OPS: u64 = 5; + let (directory, mut partition) = + Box::pin(partition_with_sealed_segments(&segment_records(OPS))).await; + let expected = segment_artifacts_on_disk(&partition); + assert_eq!(expected.len(), 3, "two sealed segments and the active one"); + for artifact in &expected { + std::fs::remove_file( + directory + .path() + .join(format!("{:020}.log", artifact.frontier)), + ) + .unwrap(); + } + + let offer = Box::pin(serve_offer(&mut partition)).await; + + assert_eq!(offer.commit_op, OPS); + let offered: Vec<_> = offer.segments.iter().map(|segment| segment.entry).collect(); + assert_eq!(offered, expected); + } + + /// Ops the receivers of the reuse tests commit: two sealed segments and + /// one batch in the active segment. + const RECEIVER_OPS: u64 = 5; + /// Ops the origin of the reuse tests commits: three sealed segments and + /// one batch in the active segment. + const ORIGIN_OPS: u64 = 7; + + /// Runs a reuse scan the way a session does: in a detached task, and then + /// accepted on the pump. Returns what the pump accepted. + async fn reuse_scan( + receiver: &mut IggyPartition, + manifest: &[consensus::StateArtifact], + ) -> Vec<(u32, crate::state_transfer::StagedSegmentMeta)> { + let scan = receiver.begin_reuse_scan(manifest); + let _ = receiver.staging_work_ended.take().unwrap().await; + let scanned = scan.result.borrow_mut().take().unwrap(); + receiver.accept_reuse_scan(scanned) + } + + /// Installs `offer` the way the shard does: the reuse scan first, then a + /// spill of each segment that the scan did not adopt. Returns the manifest + /// indexes that the scan adopted. + async fn install_offer_with_reuse( + receiver: &mut IggyPartition, + offer: &crate::state_transfer::PartitionStateTransferOffer, + ) -> Vec { + let mut staged = vec![None; offer.segments.len()]; + let mut adopted = Vec::new(); + for (index, meta) in reuse_scan(receiver, &offer.manifest()).await { + adopted.push(index); + staged[index as usize] = Some(meta); + } + for (slot, source) in staged.iter_mut().zip(&offer.segments) { + if slot.is_none() { + let mut bytes = std::fs::read(&source.log_path).unwrap(); + bytes.truncate(usize::try_from(source.entry.len).unwrap()); + *slot = Some( + receiver + .spill_transfer_segment(&source.entry, bytes, None) + .await + .unwrap(), + ); + } + } + receiver + .queue_state_transfer_install( + offer.commit_op, + staged.into_iter().map(Option::unwrap).collect(), + &offer.offsets.1, + 0, + ) + .unwrap(); + Box::pin(finish_install_io(receiver)).await.unwrap(); + adopted + } + + /// The offered bytes of each segment of `offer`. + fn offered_segment_bytes( + offer: &crate::state_transfer::PartitionStateTransferOffer, + ) -> Vec> { + offer + .segments + .iter() + .map(|source| { + let mut bytes = std::fs::read(&source.log_path).unwrap(); + bytes.truncate(usize::try_from(source.entry.len).unwrap()); + bytes + }) + .collect() + } + + /// The bytes of each segment file in the partition directory, in offset + /// order. + fn segment_file_bytes(directory: &Path) -> Vec> { + let mut paths: Vec<_> = std::fs::read_dir(directory) + .unwrap() + .map(|entry| entry.unwrap().path()) + .filter(|path| path.extension().is_some_and(|extension| extension == "log")) + .collect(); + paths.sort(); + paths + .iter() + .map(|path| std::fs::read(path).unwrap()) + .collect() + } + + #[cfg(target_os = "linux")] + fn log_inode(directory: &Path, start_offset: u64) -> u64 { + std::fs::metadata(directory.join(format!("{start_offset:020}.log"))) + .unwrap() + .ino() + } + + /// A receiver that already holds sealed segments of an offer links them to + /// the staging names instead of pulling them. The install then removes the + /// old names, sweeps the other staging files and renames the links, so the + /// installed sealed segments are the files that the receiver wrote. The + /// install also stamps every installed segment with its checksum, which + /// lets the next transfer reuse them too. + #[cfg(target_os = "linux")] + #[compio::test] + async fn given_receiver_holding_sealed_segments_when_installing_an_offer_should_reuse_them_without_a_copy() + { + let records = segment_records(ORIGIN_OPS); + let (_origin_directory, mut origin) = + Box::pin(partition_with_sealed_segments(&records)).await; + let held = &records[..usize::try_from(RECEIVER_OPS).unwrap()]; + let (directory, mut receiver) = Box::pin(partition_with_sealed_segments(held)).await; + let offer = Box::pin(serve_offer(&mut origin)).await; + let sealed = [0, BATCHES_PER_SEGMENT]; + let written = sealed.map(|start_offset| log_inode(directory.path(), start_offset)); + + let adopted = Box::pin(install_offer_with_reuse(&mut receiver, &offer)).await; + + assert_eq!( + adopted, + [0, 1], + "the two sealed segments that the receiver holds" + ); + assert_eq!( + sealed.map(|start_offset| log_inode(directory.path(), start_offset)), + written, + "the installed sealed segments are the files that the receiver wrote" + ); + assert_eq!( + segment_file_bytes(directory.path()), + offered_segment_bytes(&offer) + ); + assert!( + std::fs::read_dir(directory.path()) + .unwrap() + .all(|entry| !entry + .unwrap() + .path() + .to_string_lossy() + .ends_with(".staging")), + "the install renamed or swept every staging name" + ); + assert_eq!(receiver.consensus().commit_min(), ORIGIN_OPS); + let memos = receiver.segment_checksum_cache.borrow(); + for entry in offer.segments.iter().map(|source| source.entry) { + assert_eq!( + memos + .get(&entry.frontier) + .and_then(|memo| memo.checksum_of(entry.len)), + Some(entry.checksum), + "segment {} keeps its installed checksum", + entry.frontier + ); + } + } + + #[compio::test] + async fn given_cached_sealed_segment_corruption_when_installing_should_fetch_verified_bytes() { + let records = segment_records(ORIGIN_OPS); + let (_origin_directory, mut origin) = + Box::pin(partition_with_sealed_segments(&records)).await; + let (directory, mut receiver) = Box::pin(partition_with_sealed_segments( + &records[..usize::try_from(RECEIVER_OPS).unwrap()], + )) + .await; + let offer = Box::pin(serve_offer(&mut origin)).await; + let entry = offer.segments[0].entry; + let path = directory.path().join(format!("{:020}.log", entry.frontier)); + let mut corrupted = std::fs::read(&path).unwrap(); + *corrupted.last_mut().unwrap() ^= 1; + std::fs::write(&path, &corrupted).unwrap(); + assert_eq!(corrupted.len() as u64, entry.len); + assert_eq!( + receiver.segment_checksum_cache.borrow()[&entry.frontier].checksum_of(entry.len), + Some(entry.checksum), + "the append checksum still describes the original bytes" + ); + + let adopted = Box::pin(install_offer_with_reuse(&mut receiver, &offer)).await; + + assert_eq!(adopted, [1], "the corrupt sealed segment must be fetched"); + assert_eq!( + segment_file_bytes(directory.path()), + offered_segment_bytes(&offer) + ); + assert_eq!(receiver.consensus().commit_min(), ORIGIN_OPS); + } + + #[compio::test] + async fn given_corrupt_sealed_index_when_reusing_should_rebuild_from_verified_log() { + let records = segment_records(ORIGIN_OPS); + let (_origin_directory, mut origin) = + Box::pin(partition_with_sealed_segments(&records)).await; + let (directory, mut receiver) = Box::pin(partition_with_sealed_segments( + &records[..usize::try_from(RECEIVER_OPS).unwrap()], + )) + .await; + let offer = Box::pin(serve_offer(&mut origin)).await; + let source = &offer.segments[0]; + let expected = origin + .spill_transfer_segment( + &source.entry, + std::fs::read(&source.log_path).unwrap(), + None, + ) + .await + .unwrap(); + let expected_index = std::fs::read(&expected.index_staging).unwrap(); + let index_path = directory + .path() + .join(format!("{:020}.index", source.entry.frontier)); + let mut corrupted = std::fs::read(&index_path).unwrap(); + corrupted.fill(u8::MAX); + std::fs::write(&index_path, &corrupted).unwrap(); + + let adopted = Box::pin(install_offer_with_reuse(&mut receiver, &offer)).await; + + assert_eq!(adopted, [0, 1], "valid logs remain reusable"); + assert_eq!(std::fs::read(index_path).unwrap(), expected_index); + assert_eq!( + segment_file_bytes(directory.path()), + offered_segment_bytes(&offer) + ); + } + + #[compio::test] + async fn given_pending_reuse_scan_when_tearing_down_should_wait_for_its_task() { + let (_directory, mut receiver) = Box::pin(partition_with_sealed_segments( + &segment_records(RECEIVER_OPS), + )) + .await; + let manifest = Box::pin(serve_offer(&mut receiver)).await.manifest(); + let (release, previous) = futures::channel::oneshot::channel::<()>(); + receiver.staging_work_ended = Some(previous); + let scan = receiver.begin_reuse_scan(&manifest); + let mut drain = Box::pin(receiver.capture_teardown().drain()); + + assert!( + futures::poll!(&mut drain).is_pending(), + "teardown must wait even before the detached scan is first polled" + ); + assert!(scan.stop()); + assert!( + futures::poll!(&mut drain).is_pending(), + "cancellation does not prove that the file task has settled" + ); + drop(release); + drain.await.unwrap(); + assert!( + scan.result.borrow().is_some(), + "the scan finished before teardown" + ); + } + + #[compio::test] + async fn given_a_pending_spill_when_tearing_down_should_wait_for_its_task() { + let (directory, mut receiver) = Box::pin(partition_with_sealed_segments(&segment_records( + RECEIVER_OPS, + ))) + .await; + let offer = Box::pin(serve_offer(&mut receiver)).await; + let source = &offer.segments[0]; + let bytes = std::fs::read(&source.log_path).unwrap(); + let (release, previous) = futures::channel::oneshot::channel::<()>(); + receiver.staging_work_ended = Some(previous); + let spill = receiver.begin_transfer_segment_spill(&source.entry, bytes, None); + let mut drain = Box::pin(receiver.capture_teardown().drain()); + + assert!(futures::poll!(&mut drain).is_pending()); + assert!(spill.stop()); + assert!( + futures::poll!(&mut drain).is_pending(), + "cancellation must not release the staging writer before it settles" + ); + drop(release); + drain.await.unwrap(); + + assert!(matches!( + spill.result.borrow_mut().take().unwrap(), + Err(crate::state_transfer::SpillError::Cancelled) + )); + assert!( + !directory + .path() + .join("00000000000000000000.log.staging") + .exists(), + "the cancelled task must not write after teardown" + ); + } + + #[compio::test] + async fn given_an_abandoned_spill_when_a_reuse_scan_starts_should_wait_for_the_writer() { + let (_directory, mut receiver) = Box::pin(partition_with_sealed_segments( + &segment_records(RECEIVER_OPS), + )) + .await; + let offer = Box::pin(serve_offer(&mut receiver)).await; + let source = &offer.segments[0]; + let bytes = std::fs::read(&source.log_path).unwrap(); + let (release, previous) = futures::channel::oneshot::channel::<()>(); + receiver.staging_work_ended = Some(previous); + let spill = receiver.begin_transfer_segment_spill(&source.entry, bytes.clone(), None); + assert!(spill.stop()); + let scan = receiver.begin_reuse_scan(&offer.manifest()); + server_common::yield_to_reactor().await; + assert!( + !scan.landed(), + "a replacement scan must not overtake the abandoned staging writer" + ); + + drop(release); + let _ = receiver.staging_work_ended.take().unwrap().await; + + assert!(matches!( + spill.result.borrow_mut().take().unwrap(), + Err(crate::state_transfer::SpillError::Cancelled) + )); + let reused = receiver.accept_reuse_scan(scan.result.borrow_mut().take().unwrap()); + let staged = &reused.iter().find(|(index, _)| *index == 0).unwrap().1; + assert_eq!(std::fs::read(&staged.log_staging).unwrap(), bytes); + } + + #[compio::test] + async fn given_singleton_materialization_when_writing_should_skip_transfer_checksums() { + let config = repair_config(); + let (directory, mut partition) = Box::pin(disk_poll_partition(&config)).await; + let record = build_segment_record(partition.namespace(), 0); + Box::pin(commit_records( + &mut partition, + 1, + std::slice::from_ref(&record), + )) + .await; + + assert_eq!(partition.consensus().commit_min(), 1); + assert_eq!(segment_file_bytes(directory.path())[0].len(), record.len()); + assert!( + partition.segment_checksum_cache.borrow().is_empty(), + "a singleton has no state-transfer peer to serve" + ); + } + + /// A sealed segment takes no more appends, so its memo keeps only the + /// stamp, whether the appends or an offer read hashed it. The active + /// segment keeps its hasher for the bytes that it gains. + #[compio::test] + async fn given_sealed_segments_when_their_memos_cover_them_should_keep_no_hasher() { + let (_directory, mut partition) = Box::pin(partition_with_sealed_segments( + &segment_records(RECEIVER_OPS), + )) + .await; + let active = partition.log.active_segment().start_offset; + let hashing = |partition: &IggyPartition| -> Vec { + partition + .segment_checksum_cache + .borrow() + .iter() + .filter(|(_, memo)| { + matches!( + memo, + crate::state_transfer::SegmentChecksumMemo::Hashing { .. } + ) + }) + .map(|(start_offset, _)| *start_offset) + .collect() + }; + assert_eq!(hashing(&partition), [active], "after the appends"); + + partition.segment_checksum_cache.borrow_mut().clear(); + Box::pin(serve_offer(&mut partition)).await; + + assert_eq!(hashing(&partition), [active], "after an offer read"); + assert_eq!( + partition.segment_checksum_cache.borrow().len(), + partition.log.segments().len() + ); + } + + /// A receiver links every sealed segment of an offer that it holds, however + /// many it holds, so the transfer pulls none of them. + #[cfg(unix)] + #[compio::test] + async fn given_many_held_sealed_segments_when_installing_an_offer_should_link_every_one() { + const SEALED: u64 = 12; + let records = segment_records(SEALED * BATCHES_PER_SEGMENT + 1); + let (_origin_directory, mut origin) = + Box::pin(partition_with_sealed_segments(&records)).await; + let (_directory, mut receiver) = Box::pin(partition_with_sealed_segments(&records)).await; + let offer = Box::pin(serve_offer(&mut origin)).await; + + let adopted = Box::pin(install_offer_with_reuse(&mut receiver, &offer)).await; + + assert_eq!( + adopted, + (0..u32::try_from(SEALED).unwrap()).collect::>(), + "every held sealed segment" + ); + } + + #[cfg(target_os = "linux")] + #[compio::test] + async fn given_a_partial_reuse_scan_when_an_offer_advances_should_keep_verified_files() { + for spill_other in [false, true] { + let records = segment_records(ORIGIN_OPS + 1); + let (_origin_directory, mut origin) = + Box::pin(partition_with_sealed_segments(&records)).await; + let (directory, mut receiver) = Box::pin(partition_with_sealed_segments( + &records[..usize::try_from(RECEIVER_OPS).unwrap()], + )) + .await; + let previous = Box::pin(serve_offer(&mut receiver)).await.manifest(); + let scan = receiver.begin_reuse_scan(&previous); + let _ = receiver.staging_work_ended.take().unwrap().await; + let mut scanned = scan.result.borrow_mut().take().unwrap(); + assert!(scanned.reused.len() > 1); + // Model a deadline between artifacts using files verified by the real scan. + for (_, unused) in scanned.reused.split_off(1) { + std::fs::remove_file(unused.log_staging).unwrap(); + std::fs::remove_file(unused.index_staging).unwrap(); + } + let first = scanned.reused[0].1.clone(); + receiver.accept_reuse_scan(scanned); + let index = std::fs::File::open(&first.index_staging).unwrap(); + let verified_inode = index.metadata().unwrap().ino(); + let offer = Box::pin(serve_offer(&mut origin)).await; + let manifest = offer.manifest(); + assert_ne!(manifest, previous, "the source kept accepting writes"); + let stopped = receiver.begin_reuse_scan(&manifest); + assert!(stopped.stop()); + let _ = receiver.staging_work_ended.take().unwrap().await; + receiver.accept_reuse_scan(stopped.result.borrow_mut().take().unwrap()); + if spill_other { + let source = offer.segments.last().unwrap(); + let mut bytes = std::fs::read(&source.log_path).unwrap(); + bytes.truncate(usize::try_from(source.entry.len).unwrap()); + receiver + .spill_transfer_segment(&source.entry, bytes, None) + .await + .unwrap(); + } + + let reused = reuse_scan(&mut receiver, &manifest).await; + + assert!(reused.iter().any(|(position, _)| *position == 0)); + assert_eq!( + std::fs::metadata(&first.index_staging).unwrap().ino(), + verified_inode, + "spill_other={spill_other}: verified staging must not be rebuilt on every retry" + ); + drop(index); + Box::pin(install_offer_with_reuse(&mut receiver, &offer)).await; + assert_eq!( + segment_file_bytes(directory.path()), + offered_segment_bytes(&offer) + ); + } + } + + #[compio::test] + async fn given_verified_staging_when_a_spill_replaces_it_should_revalidate_for_an_older_offer() + { + let (_directory, mut receiver) = Box::pin(partition_with_sealed_segments( + &segment_records(RECEIVER_OPS), + )) + .await; + let own = Box::pin(serve_offer(&mut receiver)).await; + reuse_scan(&mut receiver, &own.manifest()).await; + let (_origin_directory, mut origin) = + Box::pin(partition_with_sealed_segments(&segment_records(ORIGIN_OPS))).await; + let offer = Box::pin(serve_offer(&mut origin)).await; + let source = &offer.segments[0]; + assert_ne!(source.entry.checksum, own.segments[0].entry.checksum); + receiver + .spill_transfer_segment( + &source.entry, + std::fs::read(&source.log_path).unwrap(), + None, + ) + .await + .unwrap(); + + let reused = reuse_scan(&mut receiver, &own.manifest()).await; + + let first = &reused.iter().find(|(index, _)| *index == 0).unwrap().1; + assert_eq!( + std::fs::read(&first.log_staging).unwrap(), + std::fs::read(&own.segments[0].log_path).unwrap(), + "an older offer must recover its verified bytes after a staging replacement" + ); + } + + /// A scan that is asked to stop before its first entry stages nothing and + /// leaves no memo, so the next scan of the same segment set runs whole. + #[cfg(unix)] + #[compio::test] + async fn given_a_stopped_reuse_scan_when_it_lands_should_reuse_nothing_and_leave_no_memo() { + let (_directory, mut receiver) = Box::pin(partition_with_sealed_segments( + &segment_records(RECEIVER_OPS), + )) + .await; + let manifest = Box::pin(serve_offer(&mut receiver)).await.manifest(); + let scan = receiver.begin_reuse_scan(&manifest); + assert!(scan.stop()); + + let _ = receiver.staging_work_ended.take().unwrap().await; + + let scanned = scan.result.borrow_mut().take().unwrap(); + assert_eq!(scanned.reused, Vec::new()); + assert!(scanned.memoized.is_empty(), "a stopped scan saw no entry"); + assert_eq!(receiver.accept_reuse_scan(scanned), Vec::new()); + assert!(receiver.reuse_scan_memo.borrow().is_none()); + assert_eq!( + reuse_scan(&mut receiver, &manifest).await.len(), + 2, + "the next scan links both held sealed segments" + ); + } + + /// A link that fails leaves the staging names as they were, so the staged + /// copy of an earlier attempt still serves the transfer. + #[cfg(target_os = "linux")] + #[compio::test] + async fn given_a_failing_link_when_a_staged_copy_exists_should_adopt_the_copy() { + let (directory, mut receiver) = Box::pin(partition_with_sealed_segments(&segment_records( + RECEIVER_OPS, + ))) + .await; + let offer = Box::pin(serve_offer(&mut receiver)).await; + let held = &offer.segments[0]; + receiver + .spill_transfer_segment(&held.entry, std::fs::read(&held.log_path).unwrap(), None) + .await + .unwrap(); + let staged = directory + .path() + .join(format!("{:020}.log.staging", held.entry.frontier)); + let copy = std::fs::metadata(&staged).unwrap().ino(); + // A directory at the scratch name fails the link. + std::fs::create_dir( + directory + .path() + .join(format!("{:020}.log.link.staging", held.entry.frontier)), + ) + .unwrap(); + + let adopted = reuse_scan(&mut receiver, &offer.manifest()).await; + + assert!( + adopted + .iter() + .any(|(_, meta)| meta.start_offset == held.entry.frontier && !meta.linked) + ); + assert_eq!( + std::fs::metadata(&staged).unwrap().ino(), + copy, + "the staging name still names the staged copy" + ); + } + + /// A spill gives a staging name that links a live segment a new file. + /// Writing through the link would change the live segment. + #[cfg(target_os = "linux")] + #[compio::test] + async fn given_a_linked_staging_name_when_spilling_other_bytes_should_keep_the_live_segment() { + let (directory, mut receiver) = Box::pin(partition_with_sealed_segments(&segment_records( + RECEIVER_OPS, + ))) + .await; + let own = Box::pin(serve_offer(&mut receiver)).await; + reuse_scan(&mut receiver, &own.manifest()).await; + let paths = ["log", "index"].map(|kind| directory.path().join(format!("{:020}.{kind}", 0))); + let held = paths.clone().map(|path| std::fs::read(path).unwrap()); + let staged_log = directory.path().join(format!("{:020}.log.staging", 0)); + assert_eq!( + std::fs::metadata(&staged_log).unwrap().ino(), + log_inode(directory.path(), 0), + "the scan linked the first segment" + ); + let (_origin_directory, mut origin) = + Box::pin(partition_with_sealed_segments(&segment_records(ORIGIN_OPS))).await; + let offer = Box::pin(serve_offer(&mut origin)).await; + let source = &offer.segments[0]; + + receiver + .spill_transfer_segment( + &source.entry, + std::fs::read(&source.log_path).unwrap(), + None, + ) + .await + .unwrap(); + + assert!( + paths.map(|path| std::fs::read(path).unwrap()) == held, + "the live segment keeps its bytes" + ); + assert_ne!( + std::fs::metadata(&staged_log).unwrap().ino(), + log_inode(directory.path(), 0) + ); + } + + /// A sealed segment with the base offset and the length of an offered one + /// but other bytes fails the checksum match. The receiver then pulls the + /// offered segment, and the install replaces the file. + #[compio::test] + async fn given_receiver_segment_with_other_bytes_when_installing_an_offer_should_copy_it() { + let (_origin_directory, mut origin) = + Box::pin(partition_with_sealed_segments(&segment_records(ORIGIN_OPS))).await; + let (directory, mut receiver) = Box::pin(partition_with_sealed_segments(&segment_records( + RECEIVER_OPS, + ))) + .await; + let offer = Box::pin(serve_offer(&mut origin)).await; + let held = segment_file_bytes(directory.path()); + let offered = offered_segment_bytes(&offer); + assert_eq!(held[0].len(), offered[0].len()); + assert_ne!(held[0], offered[0]); + + let adopted = Box::pin(install_offer_with_reuse(&mut receiver, &offer)).await; + + assert!( + adopted.is_empty(), + "no held segment matches an offered checksum" + ); + assert_eq!(segment_file_bytes(directory.path()), offered); + assert_eq!(receiver.consensus().commit_min(), ORIGIN_OPS); + } + + /// The segment that was active at an install keeps a hasher, so the + /// appends that seal it extend its checksum. The next transfer then links + /// it like any other sealed segment instead of pulling it again. + #[compio::test] + async fn given_installed_active_segment_when_appends_seal_it_should_reuse_it_in_the_next_transfer() + { + let records = segment_records(ORIGIN_OPS + BATCHES_PER_SEGMENT); + let (origin_held, appended) = records.split_at(usize::try_from(ORIGIN_OPS).unwrap()); + let (_origin_directory, mut origin) = + Box::pin(partition_with_sealed_segments(origin_held)).await; + let receiver_held = &records[..usize::try_from(RECEIVER_OPS).unwrap()]; + let (_directory, mut receiver) = + Box::pin(partition_with_sealed_segments(receiver_held)).await; + let first = Box::pin(serve_offer(&mut origin)).await; + Box::pin(install_offer_with_reuse(&mut receiver, &first)).await; + for partition in [&mut origin, &mut receiver] { + Box::pin(commit_records(partition, ORIGIN_OPS + 1, appended)).await; + } + let second = Box::pin(serve_offer(&mut origin)).await; + let active_at_install = first.segments.last().unwrap().entry.frontier; + + let adopted: Vec = reuse_scan(&mut receiver, &second.manifest()) + .await + .into_iter() + .map(|(_, meta)| meta.start_offset) + .collect(); + + assert_eq!( + adopted, + [ + 0, + BATCHES_PER_SEGMENT, + 2 * BATCHES_PER_SEGMENT, + active_at_install + ], + "every sealed segment, the one active at the install included" + ); + } + #[compio::test] async fn given_wal_lane_published_persisted_runs_when_idle_should_leave_the_tick_retry_set() { const RUNS: u64 = 4; @@ -14059,13 +15349,127 @@ mod tests { matches!( refused, Err( - crate::state_transfer::PartitionInstallError::StaleTransfer { + crate::state_transfer::PartitionInstallError::OfferErasesHeldOps { commit_op: 1, - commit_min: 3, + held_max: 3, } ) ), - "expected a stale refusal, got {refused:?}" + "expected a held-op refusal, got {refused:?}" + ); + } + + #[compio::test] + async fn given_a_no_wal_ack_above_commit_max_when_installing_should_preserve_the_held_op() { + const HELD_OP: u64 = 2; + for rebuilt in [false, true] { + let (_directory, mut receiver) = transfer_receiver(); + receiver.runtime_options.durability = iggy_common::Durability::Replicated; + let held = checksummed_segment_prepare(HELD_OP, 0, HELD_OP - 1, b"acknowledged"); + let header = *held.header(); + receiver + .log + .journal() + .inner + .append(held.into_frozen()) + .await + .unwrap(); + receiver.consensus.sequencer().set_sequence(HELD_OP); + if rebuilt { + assert!(receiver.register_rebuilt_ack(&header)); + } else { + assert!(receiver.send_prepare_ok(&header).await); + } + assert_eq!(receiver.consensus.commit_max(), 0); + let offsets = crate::state_transfer::ConsumerOffsetsWire { + dedup_capacity: consensus::CLIENTS_TABLE_MAX, + ..Default::default() + }; + let result = receiver + .install_state_transfer(&repair_config(), 1, Vec::new(), &offsets.encode()) + .await; + assert!( + matches!( + result, + Err( + crate::state_transfer::PartitionInstallError::OfferErasesHeldOps { + commit_op: 1, + held_max: HELD_OP + } + ) + ), + "acknowledged operation was not protected: {result:?}" + ); + assert_eq!( + receiver + .log + .journal() + .inner + .header_by_op(HELD_OP) + .unwrap() + .checksum, + header.checksum + ); + } + } + + #[compio::test] + async fn given_an_unacknowledged_no_wal_tail_when_installing_should_allow_repair_after_transfer() + { + let (_directory, mut receiver) = transfer_receiver(); + let tail = checksummed_segment_prepare(2, 0, 1, b"unacknowledged"); + receiver + .log + .journal() + .inner + .append(tail.into_frozen()) + .await + .unwrap(); + receiver.consensus.sequencer().set_sequence(2); + let offsets = crate::state_transfer::ConsumerOffsetsWire { + dedup_capacity: consensus::CLIENTS_TABLE_MAX, + ..Default::default() + }; + receiver + .install_state_transfer(&repair_config(), 1, Vec::new(), &offsets.encode()) + .await + .unwrap(); + assert!(receiver.log.journal().inner.is_empty()); + assert_eq!(receiver.consensus.commit_min(), 1); + } + + /// An install below the head keeps the head, and an offer at or below the + /// known commit keeps the commit point, so only the journal wipe can tell a + /// suffix snapshot taken before the install that the bodies it offers are + /// gone. + #[compio::test] + async fn given_a_suffix_snapshot_when_an_install_keeps_the_head_should_invalidate_it() { + const HEAD: u64 = 3; + let (_directory, mut receiver) = transfer_receiver(); + receiver.consensus().sequencer().set_sequence(HEAD); + receiver.consensus().advance_commit_max(HEAD); + receiver + .consensus() + .set_local_dvc_suffix(consensus::DvcSuffix::empty()); + assert!(!receiver.consensus().local_dvc_suffix_stale()); + let offsets = crate::state_transfer::ConsumerOffsetsWire { + dedup_capacity: consensus::CLIENTS_TABLE_MAX, + ..Default::default() + }; + + let installed = receiver + .install_state_transfer(&repair_config(), 1, Vec::new(), &offsets.encode()) + .await; + + assert!( + installed.is_ok(), + "the offer erases no held op, got {installed:?}" + ); + assert_eq!(receiver.consensus().sequencer().current_sequence(), HEAD); + assert!( + receiver.consensus().local_dvc_suffix_stale(), + "a suffix snapshot taken before the install still matches the log, so the \ + next DoViewChange offers bodies the install wiped" ); } @@ -14187,6 +15591,217 @@ mod tests { ); } + /// Two batches: the first is what a lagging replica holds, the second what + /// the primary appended after it. + fn held_and_pulled_batches(held: &[u8]) -> (Vec, usize) { + let first = checksummed_segment_prepare(1, 0, 0, held); + let second = checksummed_segment_prepare(2, first.header().checksum, 1, b"pulled"); + let mut body = first.as_slice()[size_of::()..].to_vec(); + let prefix_len = body.len(); + body.extend_from_slice(&second.as_slice()[size_of::()..]); + (body, prefix_len) + } + + #[compio::test] + async fn given_a_held_prefix_when_spilling_the_artifact_should_stage_what_a_whole_pull_stages() + { + let directory = tempfile::tempdir().unwrap(); + let (mut partition, _) = recording_partition_at(1, 3); + partition.set_partition_dir(directory.path().to_string_lossy().into_owned()); + let (body, prefix_len) = held_and_pulled_batches(b"held"); + let artifact = consensus::StateArtifact::for_bytes( + consensus::state_manifest::artifact_kind::SEGMENT_LOG, + 0, + &body, + ); + let held = directory.path().join("held.log"); + std::fs::write(&held, &body[..prefix_len]).unwrap(); + let (mut bytes, prefix) = crate::state_transfer::read_segment_prefix( + held.to_str().unwrap(), + artifact, + prefix_len as u64, + &Cell::new(false), + ) + .await + .unwrap(); + bytes.extend_from_slice(&body[prefix_len..]); + + let seeded = partition + .spill_transfer_segment(&artifact, bytes, Some(prefix)) + .await + .unwrap(); + let seeded_index = std::fs::read(&seeded.index_staging).unwrap(); + let pulled = partition + .spill_transfer_segment(&artifact, body, None) + .await + .unwrap(); + assert_eq!( + seeded, pulled, + "the seeded spill must stage the same segment" + ); + assert_eq!( + seeded_index, + std::fs::read(&pulled.index_staging).unwrap(), + "the walk resumed after the prefix must rebuild the same index" + ); + } + + #[compio::test] + async fn given_a_held_prefix_with_other_bytes_when_spilling_should_fail_the_checksum() { + let directory = tempfile::tempdir().unwrap(); + let (mut partition, _) = recording_partition_at(1, 3); + partition.set_partition_dir(directory.path().to_string_lossy().into_owned()); + let (body, prefix_len) = held_and_pulled_batches(b"held"); + let (other, other_len) = held_and_pulled_batches(b"HELD"); + assert_eq!(prefix_len, other_len); + let artifact = consensus::StateArtifact::for_bytes( + consensus::state_manifest::artifact_kind::SEGMENT_LOG, + 0, + &body, + ); + let held = directory.path().join("held.log"); + std::fs::write(&held, &other[..prefix_len]).unwrap(); + let (mut bytes, prefix) = crate::state_transfer::read_segment_prefix( + held.to_str().unwrap(), + artifact, + prefix_len as u64, + &Cell::new(false), + ) + .await + .unwrap(); + bytes.extend_from_slice(&body[prefix_len..]); + + let spilled = partition + .spill_transfer_segment(&artifact, bytes, Some(prefix)) + .await; + assert!( + matches!( + spilled, + Err(crate::state_transfer::SpillError::ManifestChecksum { frontier: 0 }) + ), + "a valid prefix with other bytes must fail the manifest checksum, got {spilled:?}" + ); + } + + #[compio::test] + async fn given_a_dropped_session_when_its_prefix_read_runs_should_stop_without_filling_a_buffer() + { + const REACTOR_TURNS_MAX: usize = 1024; + let directory = tempfile::tempdir().unwrap(); + let (mut partition, _) = recording_partition_at(1, 3); + let (body, prefix_len) = held_and_pulled_batches(b"held"); + let artifact = consensus::StateArtifact::for_bytes( + consensus::state_manifest::artifact_kind::SEGMENT_LOG, + 0, + &body, + ); + let held = directory.path().join("held.log"); + std::fs::write(&held, &body[..prefix_len]).unwrap(); + partition.transfer = Some(crate::state_transfer::PartitionTransferSession { + nonce: 1, + peer: 0, + commit_op: 0, + artifacts: vec![crate::state_transfer::TransferArtifact::Pending( + consensus::ArtifactProgress { + entry: artifact, + buf: Vec::new(), + }, + )], + target_accepted: true, + idle_ticks: 0, + spill: None, + scan: None, + seed: Some(crate::state_transfer::PrefixSeed::Planned { + artifact: 0, + log_path: held.to_str().unwrap().to_owned(), + len: prefix_len as u64, + }), + pulled_len: 0, + }); + assert!( + partition.drive_prefix_seed(), + "the pull must wait for the read" + ); + let Some(crate::state_transfer::PrefixSeed::Reading { read, .. }) = partition + .transfer + .as_ref() + .and_then(|session| session.seed.as_ref()) + else { + panic!("the read must start"); + }; + let landed = Rc::clone(&read.result); + + partition.transfer = None; + for _ in 0..REACTOR_TURNS_MAX { + if landed.borrow().is_some() { + break; + } + server_common::yield_to_reactor().await; + } + + let read = landed.borrow(); + assert!( + matches!( + read.as_ref(), + Some(Err(error)) if error.kind() == std::io::ErrorKind::Interrupted + ), + "a read whose session dropped must stop before it fills a buffer, got {read:?}" + ); + } + + /// A backup that fetches one segment artifact, whose prefix read landed. + fn fetching_with_a_landed_prefix_read() -> IggyPartition { + let (mut partition, _) = recording_partition_at(1, 3); + partition + .consensus() + .set_state_transfer_stage(consensus::StateTransferStage::AwaitingTarget); + partition + .consensus() + .set_state_transfer_stage(consensus::StateTransferStage::Fetching); + partition.transfer = Some(crate::state_transfer::PartitionTransferSession { + nonce: 1, + peer: 0, + commit_op: 0, + artifacts: vec![crate::state_transfer::TransferArtifact::Pending( + consensus::ArtifactProgress { + entry: consensus::StateArtifact::for_bytes( + consensus::state_manifest::artifact_kind::SEGMENT_LOG, + 0, + b"segment", + ), + buf: Vec::new(), + }, + )], + target_accepted: true, + idle_ticks: 0, + spill: None, + scan: None, + seed: Some(crate::state_transfer::PrefixSeed::Reading { + artifact: 0, + read: crate::state_transfer::LocalWork { + result: Rc::new(RefCell::new(Some(Err(std::io::Error::other( + "the read failed", + ))))), + ..crate::state_transfer::LocalWork::default() + }, + }), + pulled_len: 0, + }); + partition + } + + #[compio::test] + async fn given_a_landed_prefix_read_when_resuming_io_should_hand_the_pull_its_turn() { + let mut partition = fetching_with_a_landed_prefix_read(); + + let step = partition.resume_io(&repair_config()).await; + + assert!( + matches!(step, crate::PartitionIoStep::TransferReady), + "a landed read must resume the pull before its stall sweep" + ); + } + #[compio::test] async fn transfer_establishes_wal_body_ownership_without_losing_the_active_index_writer() { for durability in [ @@ -14225,7 +15840,7 @@ mod tests { ); staged.push( partition - .spill_transfer_segment(&artifact, body) + .spill_transfer_segment(&artifact, body, None) .await .unwrap(), ); @@ -14295,7 +15910,7 @@ mod tests { ); staged.push( partition - .spill_transfer_segment(&artifact, body) + .spill_transfer_segment(&artifact, body, None) .await .unwrap(), ); @@ -15801,6 +17416,109 @@ mod tests { writer.await; } + /// Past the byte cap the evicted ring releases the bodies that the WAL + /// wrote to segments and keeps their headers. Each body reads back from + /// its segment as the replicated prepare, after the WAL checkpoint too: + /// ops 1 and 2 in one read from the first segment, op 3 from the second. + /// Retention of a segment forgets the ops in it. + #[compio::test] + async fn given_bodies_past_the_ring_byte_cap_when_read_back_should_return_the_replicated_prepares() + { + const RECORDS_PER_SEGMENT: u64 = 2; + const OPS: u64 = 4; + const PAYLOAD: [u8; 4096] = [7; 4096]; + + let root = tempfile::tempdir().unwrap(); + let mut config = repair_config(); + config.path_layout.streams_root = root.path().to_string_lossy().into_owned(); + let directory = std::path::PathBuf::from(config.get_partition_path(1, 1, 0)); + std::fs::create_dir_all(&directory).unwrap(); + let (mut partition, _) = recording_partition_at(1, 3); + partition.set_partition_dir(directory.to_string_lossy().into_owned()); + partition.runtime_options.durability = iggy_common::Durability::Replicated; + partition.runtime_options.consumer_offset_durability = iggy_common::Durability::Replicated; + let prepare_len = checksummed_segment_prepare(1, 0, 0, &PAYLOAD) + .as_slice() + .len(); + let segment_size = IggyByteSize::from( + RECORDS_PER_SEGMENT * (prepare_len - size_of::()) as u64, + ); + partition.runtime_options.segment_size = Some(segment_size); + partition.log.active_segment_mut().max_size = segment_size; + partition.open_persistence().await.unwrap(); + partition.log.retire_front().unwrap(); + partition.install_empty_segment(&config, 0).await.unwrap(); + partition.set_io_notifier( + Rc::new(|_, _| {}), + crate::largest_legal_job_charge().unwrap(), + ); + // Room for one body, but not for two. + partition + .log + .journal() + .inner + .set_ring_caps(crate::EVICTED_RING_CAPACITY, (prepare_len * 3 / 2) as u64); + let persistence = Rc::clone(partition.persistence.as_ref().unwrap()); + + let mut prepares = Vec::new(); + let mut parent = 0; + let mut writer = hold_wal_writer(&persistence); + for offset in 0..OPS { + let op = offset + 1; + let prepare = checksummed_segment_prepare(op, parent, offset, &PAYLOAD); + parent = prepare.header().checksum; + prepares.push(prepare.clone()); + partition.on_replicate(prepare).await; + writer.await; + writer = hold_wal_writer(&persistence); + commit_on_heartbeat(&mut partition, &config, op).await; + settle_partition_io(&mut partition, &config).await; + } + assert_eq!( + partition.log.journal().inner.repair_retained_from(), + Some(1), + "repair serves every committed op" + ); + + persistence.request_checkpoint(); + partition.checkpoint_persistence(&config).await; + settle_partition_io(&mut partition, &config).await; + writer.await; + settle_partition_io(&mut partition, &config).await; + let released: Vec<_> = prepares[..prepares.len() - 1] + .iter() + .map(|prepare| { + partition + .log + .journal() + .inner + .evicted_header(prepare.header().op) + .expect("the byte cap released this body") + }) + .collect(); + let mut bodies = Vec::new(); + crate::read_released_bodies(&partition.log, &released, &mut bodies) + .await + .unwrap(); + assert_eq!(bodies.len(), released.len()); + for (body, prepare) in bodies.iter().zip(&prepares) { + assert_eq!(body.as_slice(), prepare.as_slice()); + } + + partition + .remove_sealed_segments_up_to(RECORDS_PER_SEGMENT - 1) + .await; + let crate::PartitionIoStep::Ready(plan) = partition.resume_io(&config).await else { + panic!("retention of the checkpointed segment is due"); + }; + execute_partition_io_plan(&mut partition, &config, plan).await; + assert_eq!( + partition.log.journal().inner.repair_retained_from(), + Some(RECORDS_PER_SEGMENT + 1), + "repair answers RangeEvicted for the ops of the removed segment" + ); + } + #[compio::test] async fn given_transfer_flush_when_tick_promotes_a_queued_send_should_ack_it_on_the_primary() { let root = tempfile::tempdir().unwrap(); @@ -16194,11 +17912,67 @@ mod tests { } pub(super) fn test_partition() -> IggyPartition { + test_partition_with_replicas(1) + } + + #[compio::test] + async fn given_snapshot_transfer_when_writes_evict_its_tail_should_release_only_confirmed_progress_and_expire_idle_peers() + { + const PEER: u8 = 1; + const NONCE: u128 = 1; + const BYTES: usize = 64 * 1024; + let mut partition = test_partition_with_replicas(3); + let budget = Arc::new(crate::RepairMemoryBudget::new(BYTES)); + partition.proof_walk_memory = Some(budget.reservation()); + let journal = &partition.log.journal().inner; + journal.set_ring_caps(1, u64::MAX); + assert!(partition.retain_transfer_tail(PEER, NONCE, 0)); + assert!(partition.transfer_tail_available(PEER, NONCE, 1)); + for op in 1..=8 { + let prepare = checksummed_segment_prepare(op, 0, op - 1, b"tail"); + journal.append(prepare.into_frozen()).await.unwrap(); + journal.evict_prefix(1); + } + assert_eq!(journal.repair_retained_from(), Some(2)); + partition.advance_transfer_tail(PEER, consensus::ReplicationProgress::Busy, 9); + assert_eq!(journal.repair_retained_from(), Some(2)); + partition.advance_transfer_tail( + PEER, + consensus::ReplicationProgress::Ready { received_op: 4 }, + 10, + ); + assert_eq!(journal.repair_retained_from(), Some(5)); + partition.expire_transfer_tails(19, 10); + assert!(partition.touch_transfer_tail(PEER, NONCE, 19)); + partition.renew_transfer_tail_for_repair(PEER, 28); + partition.expire_transfer_tails(29, 10); + assert_eq!( + journal.repair_retained_from(), + Some(5), + "a pull keeps the suffix without proving its prefix" + ); + partition.expire_transfer_tails(38, 10); + assert_eq!(journal.repair_retained_from(), Some(8)); + assert!(!partition.touch_transfer_tail(PEER, NONCE, 29)); + assert!( + !partition.retain_transfer_tail(PEER, NONCE, 29), + "an expired nonce must not rebuild a different manifest" + ); + assert!(partition.retain_transfer_tail(PEER, NONCE + 1, 29)); + assert!( + !partition.transfer_tail_available(PEER, NONCE + 1, 1), + "an old snapshot cannot promise already-missing operations" + ); + assert!(partition.transfer_tail_available(PEER, NONCE + 1, 8)); + assert!(budget.reservation().try_add(BYTES)); + } + + fn test_partition_with_replicas(replica_count: u8) -> IggyPartition { let namespace = IggyNamespace::new(1, 1, 0); let consensus = VsrConsensus::new( TEST_CLUSTER, 0, - 1, + replica_count, namespace.inner(), IggyMessageBus::new(0), LocalPipeline::new(), @@ -16231,9 +18005,16 @@ mod tests { /// Keep the returned directory alive until every read using those files ends. async fn disk_poll_partition( config: &PartitionsConfig, + ) -> (tempfile::TempDir, IggyPartition) { + Box::pin(disk_poll_partition_with_replicas(config, 1)).await + } + + async fn disk_poll_partition_with_replicas( + config: &PartitionsConfig, + replica_count: u8, ) -> (tempfile::TempDir, IggyPartition) { let directory = tempfile::tempdir().expect("create partition directory"); - let mut partition = test_partition(); + let mut partition = test_partition_with_replicas(replica_count); partition.set_partition_dir(directory.path().to_string_lossy().into_owned()); partition.log.retire_front().expect("retire empty segment"); partition @@ -17552,7 +19333,7 @@ mod tests { &body, ); let staged = backup - .spill_transfer_segment(&artifact, body) + .spill_transfer_segment(&artifact, body, None) .await .unwrap(); let offsets = crate::state_transfer::ConsumerOffsetsWire { @@ -17578,7 +19359,7 @@ mod tests { matches!( installed, Err( - crate::state_transfer::PartitionInstallError::StaleTransfer { + crate::state_transfer::PartitionInstallError::OfferErasesHeldOps { commit_op: 1, .. } @@ -17734,6 +19515,50 @@ mod tests { assert!(backup.pending_persisted_acks.borrow().is_empty()); } + #[compio::test] + async fn given_unflushed_wal_when_reporting_replication_progress_should_honor_topic_durability() + { + for durability in [ + iggy_common::Durability::Replicated, + iggy_common::Durability::Persisted, + ] { + let directory = tempfile::tempdir().unwrap(); + let (mut backup, _) = recording_partition_at(1, 3); + backup.set_partition_dir(directory.path().to_string_lossy().into_owned()); + backup.runtime_options.durability = durability; + backup.open_persistence().await.unwrap(); + let prepare = checksummed_segment_prepare(1, 0, 0, b"written"); + let header = *prepare.header(); + backup.consensus().sequencer().set_sequence(header.op); + backup + .log + .journal() + .inner + .append(prepare.clone().into_frozen()) + .await + .unwrap(); + let persistence = Rc::clone(backup.persistence.as_ref().unwrap()); + persistence.append(prepare.into_frozen(), false).unwrap(); + assert!(persistence.start()); + Rc::clone(&persistence).run().await; + assert!(persistence.is_written(&header)); + assert!(!persistence.is_durable(&header)); + + let expected = u64::from(!durability.is_persisted()); + assert_eq!( + backup.replication_progress(), + Some(expected), + "{durability:?}: progress must follow the prepare acknowledgment contract" + ); + backup.maintenance.flush_through = Some(header.op); + assert_eq!( + backup.replication_progress(), + Some(expected), + "{durability:?}: ordinary materialization does not replace the held history" + ); + } + } + #[compio::test] async fn given_unacked_repair_above_a_wal_hole_when_installing_should_allow_recovery() { let directory = tempfile::tempdir().unwrap(); @@ -17752,7 +19577,9 @@ mod tests { .await .unwrap(); backup.drive_persistence().await; - backup.consensus().advance_commit_max(1); + // Op 3 is committed but unacked here: the WAL hole at op 2 holds back + // its ack, so it must not block the install. + backup.consensus().advance_commit_max(3); let third = checksummed_segment_prepare(3, 0, 2, b"unacked"); let header = *third.header(); backup @@ -17772,7 +19599,7 @@ mod tests { &body, ); let staged = backup - .spill_transfer_segment(&artifact, body) + .spill_transfer_segment(&artifact, body, None) .await .unwrap(); let offsets = crate::state_transfer::ConsumerOffsetsWire { @@ -18000,6 +19827,84 @@ mod tests { } } + #[compio::test] + async fn given_a_backup_that_cannot_journal_a_prepare_when_it_arrives_should_neither_forward_nor_ack() + { + const MIDDLE: u8 = 1; + const TAIL: u8 = 2; + /// Led by replica 0 in a group of three, like view 0, so only the view + /// number distinguishes a prepare from another view. + const LATER_VIEW: u32 = 3; + let first = checksummed_segment_prepare(1, 0, 0, b"first"); + let above_gap = checksummed_segment_prepare(2, first.header().checksum, 1, b"second"); + let later_view = first + .clone() + .transmute_header(|original, header: &mut PrepareHeader| { + *header = original; + header.view = LATER_VIEW; + header.checksum = header.identity_checksum(); + }); + let cases = [ + (MIDDLE, true, &first), + (MIDDLE, false, &above_gap), + (MIDDLE, true, &later_view), + (TAIL, true, &first), + (TAIL, false, &above_gap), + ]; + for (replica, awaiting_target, prepare) in cases { + let (mut backup, _) = recording_partition_at(replica, 3); + let sent = backup.consensus().message_bus().sent_to_replicas.clone(); + if awaiting_target { + backup.consensus().begin_state_transfer_await(); + } + let header = *prepare.header(); + let prepare = prepare.clone(); + backup.on_replicate(prepare).await; + + let case = format!( + "replica {replica}, awaiting target {awaiting_target}, op {}, view {}", + header.op, header.view + ); + assert!(!backup.log.journal().inner.holds_op(header.op), "{case}"); + assert!( + sent.borrow().is_empty(), + "a backup never forwards or acknowledges refused data: {case}" + ); + } + } + + /// A backup that awaits a transfer target relays nothing outside `Normal` + /// status, where its view is not settled, and nothing at or below its + /// commit point, where the prepare is a late copy of a committed op. + #[compio::test] + async fn given_a_backup_outside_normal_status_or_at_its_commit_point_when_a_prepare_arrives_should_not_relay_it() + { + const MIDDLE: u8 = 1; + const OP: u64 = 1; + type Setup = fn(&IggyPartition); + let prepare = checksummed_segment_prepare(OP, 0, 0, b"first"); + let cases: [(&str, Setup); 2] = [ + ("outside normal status", |backup| { + backup.consensus().begin_view_probe(); + }), + ("at its commit point", |backup| { + backup.consensus().restore_commit_state(OP, OP); + }), + ]; + let mut relayed = Vec::new(); + for (case, setup) in cases { + let (mut backup, _) = recording_partition_at(MIDDLE, 3); + let sent = backup.consensus().message_bus().sent_to_replicas.clone(); + setup(&backup); + backup.consensus().begin_state_transfer_await(); + backup.on_replicate(prepare.clone()).await; + if !sent.borrow().is_empty() { + relayed.push(case); + } + } + assert!(relayed.is_empty(), "relayed a prepare {relayed:?}"); + } + async fn partition_with_pending_durable_ack() -> ( tempfile::TempDir, IggyPartition, @@ -18336,7 +20241,7 @@ mod tests { &segment_body, ); let staged = partition - .spill_transfer_segment(&artifact, segment_body) + .spill_transfer_segment(&artifact, segment_body, None) .await .unwrap(); let offsets = crate::state_transfer::ConsumerOffsetsWire { @@ -19198,7 +21103,12 @@ mod tests { None, None, ); - partition.on_replicate(prepare).await; + partition.apply_replicated_operation(prepare).await.unwrap(); + assert_eq!( + partition.acknowledged_op.get(), + installed_op, + "the successor may only be discarded if it was never acknowledged" + ); partition.consensus.set_view(1); partition.resynchronize_consumer_offset_reservations(); assert_eq!(partition.owner_install_barrier, installed_op + 1); @@ -19754,7 +21664,8 @@ mod tests { (installed, installed.owner), (higher, higher.owner), ] { - let op = partition.consensus.sequencer().current_sequence() + 1; + let op = partition.consensus.sequencer().current_sequence() + + u64::from(admitted.owner.generation > installed.owner.generation); partition .on_request(owner_install_request(&partition, &admitted), None) .await; @@ -19774,6 +21685,51 @@ mod tests { } } + #[compio::test] + async fn given_superseded_or_applied_owner_installs_when_resubmitted_should_not_consume_ops() { + const GROUP_ID: u64 = 7; + const FIRST_CLIENT: u128 = 42; + let mut partition = test_partition(); + install_test_owner(&mut partition, GROUP_ID, FIRST_CLIENT).await; + let stale = partition.consumer_group_owners[&GROUP_ID].installation; + let installed_op = install_test_owner(&mut partition, GROUP_ID, FIRST_CLIENT + 1).await; + let installed = partition.consumer_group_owners[&GROUP_ID].installation; + for installation in [stale, installed] { + let (sender, receiver) = consensus::oneshot_channel(); + partition + .on_request( + owner_install_request(&partition, &installation), + Some(sender), + ) + .await; + assert_eq!( + partition.consensus.sequencer().current_sequence(), + installed_op, + "an applied or superseded installation must not consume a log entry" + ); + assert_eq!( + receiver.await.unwrap().header().status, + IggyError::TransientNotAccepted.as_code() + ); + assert!(partition.consumer_group_owner_admitted(&installation)); + assert!(partition.pending_owner_install.get().is_none()); + assert_eq!( + partition.consumer_group_owner(GROUP_ID), + Some(installed.owner) + ); + } + let foreign = InstallConsumerGroupOwnerRequest { + incarnation: installed.incarnation + 1, + ..stale + }; + assert!(!partition.consumer_group_owner_admitted(&foreign)); + install_test_owner(&mut partition, GROUP_ID, FIRST_CLIENT + 2).await; + assert_eq!( + partition.consensus.sequencer().current_sequence(), + installed_op + 1 + ); + } + #[compio::test] async fn given_confirmed_group_deletion_when_retired_and_restarted_should_reject_late_owner() { const LIVE_GROUP_ID: u64 = 1; @@ -23368,7 +25324,7 @@ mod tests { } #[compio::test] - async fn wal_backpressure_on_an_intermediate_replica_does_not_stop_forwarding() { + async fn wal_backpressure_on_one_backup_does_not_stop_the_other_backup() { let (mut primary, _) = recording_partition_at(0, 3); let namespace = primary.namespace(); let request = checksumless_send_request(namespace, 1); @@ -23402,20 +25358,31 @@ mod tests { .exhaust_capacity_for_test(); let stalled_header = *message.header(); backup.on_replicate(message).await; - { - let sent = backup.consensus().message_bus().sent_to_replicas.borrow(); - let (_, forwarded) = sent + assert_eq!(backup.consensus().sequencer().current_sequence(), 1); + assert_eq!(backup.persistence.as_ref().unwrap().head(), 0); + assert!( + backup + .consensus() + .message_bus() + .sent_to_replicas + .borrow() + .is_empty() + ); + let message = + Message::::try_from(Owned::copy_from_slice(forwarded.as_slice())) + .unwrap(); + primary.on_replicate(message).await; + let healthy_copy = { + let sent = primary.consensus().message_bus().sent_to_replicas.borrow(); + let (_, frame) = sent .iter() .find(|(target, _)| *target == 2) - .expect("the downstream replica receives the prepare"); - let header = bytemuck::checked::from_bytes::( - &forwarded.as_slice()[..size_of::()], - ); - assert_eq!(header.command, Command::Prepare); - assert_eq!(backup.consensus().sequencer().current_sequence(), 1); - assert_eq!(backup.persistence.as_ref().unwrap().head(), 0); - assert!(!sent.iter().any(|(target, _)| *target == 0)); - } + .expect("the primary sends directly to the other backup"); + Message::::try_from(Owned::copy_from_slice(frame.as_slice())).unwrap() + }; + let (mut healthy, _) = recording_partition_at(2, 3); + healthy.on_replicate(healthy_copy).await; + assert!(healthy.log.journal().inner.holds_op(1)); assert!(!backup.register_rebuilt_ack(&stalled_header)); assert!(backup.pending_persisted_acks.borrow().contains_key(&1)); @@ -23877,8 +25844,8 @@ mod tests { } /// Runs the tick's superblock step between install steps, as the shard does. - async fn finish_install_io( - partition: &mut IggyPartition, + async fn finish_install_io( + partition: &mut IggyPartition, ) -> Result< crate::state_transfer::PartitionInstallOutcome, crate::state_transfer::PartitionInstallError, @@ -26145,6 +28112,132 @@ mod tests { assert!(!reopened.register_rebuilt_ack(&headers[0])); } + /// An install below the head of an adopted `StartView` leaves the WAL without + /// that view's log. A restart before repair must restore the adopted head, or + /// the replica votes for the view with a head at the install op. + #[compio::test] + async fn given_an_install_below_the_adopted_head_when_reopening_should_resume_the_adopted_view() + { + const BACKUP: u8 = 1; + const ADOPTED: u32 = 2; + const HELD: u64 = 1; + const HEAD: u64 = 3; + let directory = tempfile::tempdir().unwrap(); + let (config, mut partition) = replica_with_prepare_history(directory.path(), BACKUP).await; + let partition_dir = config.get_partition_path(1, 1, 0); + let store = Rc::new(PingPongSuperblock::open(&partition_dir).await.unwrap()); + partition.set_superblock(store.clone(), None); + let held = checksummed_segment_prepare(HELD, 0, 0, b"retained"); + partition.on_replicate(held.clone()).await; + let persistence = partition.persistence.as_ref().unwrap().clone(); + persistence + .drain_with_timeout(compio::runtime::time::sleep) + .await + .unwrap(); + let mut parent = *held.header(); + let mut headers = Vec::new(); + for op in HELD + 1..=HEAD { + let mut header = PrepareHeader { + op, + request: op, + timestamp: op, + view: ADOPTED, + parent: parent.checksum, + ..parent + }; + header.checksum = header.identity_checksum(); + parent = header; + headers.push(header); + } + headers.reverse(); + let start_view = Message::::new(size_of::< + iggy_binary_protocol::StartViewHeader, + >()) + .transmute_header(|_, header: &mut iggy_binary_protocol::StartViewHeader| { + header.command = Command::StartView; + header.cluster = TEST_CLUSTER; + header.group = partition.namespace().inner(); + header.view = ADOPTED; + header.replica = 2; + header.op = HEAD; + header.commit = HEAD - 1; + header.size = + u32::try_from(size_of::()).unwrap(); + }); + partition.consensus().handle_start_view( + consensus::PlaneKind::Partitions, + start_view.header(), + bytemuck::cast_slice(&headers), + ); + assert!(partition.persist_superblock_if_needed().await); + + let body = held.as_slice()[size_of::()..].to_vec(); + let artifact = consensus::StateArtifact::for_bytes( + consensus::state_manifest::artifact_kind::SEGMENT_LOG, + 0, + &body, + ); + let staged = partition + .spill_transfer_segment(&artifact, body, None) + .await + .unwrap(); + let offsets = crate::state_transfer::ConsumerOffsetsWire { + dedup_capacity: consensus::CLIENTS_TABLE_MAX, + prepare_checksum: Some(held.header().checksum), + checkpoint_prepare: held.as_slice().to_vec(), + next_offset: 1, + ..Default::default() + }; + partition.consensus().begin_state_transfer_await(); + partition + .install_state_transfer(&repair_config(), HELD, vec![staged], &offsets.encode()) + .await + .unwrap(); + assert_eq!(partition.consensus().sequencer().current_sequence(), HEAD); + assert!(partition.consensus().view_log_unproven()); + let certified_after_install = persistence.certified_log_view(); + let segment_size = partition.log.active_segment().max_size; + drop(persistence); + drop(partition); + drop(store); + + let (store, latest) = PingPongSuperblock::open_with_latest(&partition_dir) + .await + .unwrap(); + let journal::superblock::SuperblockContents::Present(bytes) = latest else { + panic!("adopted record must be durable"); + }; + let state = consensus::VsrState::try_from(bytes.as_slice()).unwrap(); + let (mut reopened, _) = recording_partition_at(BACKUP, 3); + reopened.consensus.set_view(state.view); + reopened.consensus.set_log_view(state.log_view); + reopened + .consensus + .mark_superblock_durable(state.view, state.log_view); + reopened.set_superblock(Rc::new(store), Some(&state)); + reopened.set_partition_dir(partition_dir); + reopened.log.active_segment_mut().max_size = segment_size; + reopened.open_persistence().await.unwrap(); + assert!( + !reopened.requires_state_transfer(), + "the certificate after the install is below the persisted log view" + ); + assert_eq!( + reopened.consensus().adopted_view_head(), + Some((ADOPTED, HEAD)) + ); + assert_eq!( + ( + reopened.consensus().sequencer().current_sequence(), + reopened.consensus().view_log_unproven(), + ), + (HEAD, true), + "the restart dropped the adopted head: certified view after the install \ + {certified_after_install:?}, state log_view {}", + state.log_view + ); + } + /// The WAL drops a divergent run before the journal does. The canonical /// tail cannot extend that WAL branch and exists only in the journal. /// A crash before refill leaves the committed prefix uncertified, @@ -28240,6 +30333,8 @@ mod tests { max_timestamp: 0, log_staging: directory.path().join("00000000000000000000.log.staging"), index_staging: directory.path().join("00000000000000000000.index.staging"), + checksum: 0, + linked: false, }; let offered_state = crate::state_transfer::ConsumerOffsetsWire { dedup_capacity: consensus::CLIENTS_TABLE_MAX, @@ -28535,6 +30630,8 @@ mod tests { max_timestamp: 0, log_staging: directory.path().join("00000000000000000000.log.staging"), index_staging: directory.path().join("00000000000000000000.index.staging"), + checksum: 0, + linked: false, }; let offered_state = crate::state_transfer::ConsumerOffsetsWire { owner_retirement: None, @@ -29105,6 +31202,34 @@ mod tests { (partition, replies, header) } + fn answer_partition_heartbeat( + partition: &IggyPartition, + action: &consensus::VsrAction, + ) { + if let consensus::VsrAction::SendCommit { + view, + group, + timestamp_monotonic, + .. + } = action + { + let reply = Message::::new(HEADER_SIZE).transmute_header( + |_, header: &mut CommitReplyHeader| { + header.command = Command::CommitReply; + header.size = u32::try_from(HEADER_SIZE).unwrap(); + header.cluster = partition.consensus().cluster(); + header.view = *view; + header.replica = 1; + header.group = *group; + header.partition_incarnation = partition.consensus().partition_incarnation(); + header.timestamp_monotonic = *timestamp_monotonic; + header.seal(); + }, + ); + partition.consensus().handle_commit_reply(reply.header()); + } + } + #[compio::test] async fn given_an_armed_self_ack_when_the_local_write_is_retried_should_not_send_a_second_prepare_ok() { @@ -29154,6 +31279,7 @@ mod tests { persistence.release_capacity_for_test(); } for action in partition.consensus().tick(PlaneKind::Partitions) { + answer_partition_heartbeat(&partition, &action); if let consensus::VsrAction::RetryLocalWrite { op } = action { partition.retry_local_write(op).await; retries += 1; @@ -29199,6 +31325,7 @@ mod tests { for _ in 0..consensus::TimeoutManager::PREPARE_TICKS * 40 { for action in partition.consensus().tick(PlaneKind::Partitions) { + answer_partition_heartbeat(&partition, &action); if let consensus::VsrAction::RetryLocalWrite { op } = action { partition.retry_local_write(op).await; } @@ -32155,10 +34282,115 @@ mod tests { ); } - /// While the WAL is not durable through `fetch_to_op`, `complete_repair` - /// waits and only the shard's walk backstop moves `commit_min`, so stall - /// rounds re-serve frames this journal holds. Modeled without a WAL, one - /// walk slice per round: the walk is progress, so the peer is kept. + #[compio::test] + async fn given_a_repair_larger_than_the_wal_when_a_prefix_is_durable_should_apply_it_before_fetching_the_rest() + { + const OPS: u64 = 3; + const PAYLOAD_BYTES: usize = 48 * 1024 * 1024; + let root = tempfile::tempdir().unwrap(); + let mut config = repair_config(); + config.segment_size = IggyByteSize::from(4 * PAYLOAD_BYTES as u64); + config.path_layout.streams_root = root.path().to_string_lossy().into_owned(); + let directory = std::path::PathBuf::from(config.get_partition_path(1, 1, 0)); + std::fs::create_dir_all(&directory).unwrap(); + let (mut backup, _) = recording_partition_at(1, 3); + backup.set_partition_dir(directory.to_string_lossy().into_owned()); + set_offset_dirs_under(&mut backup, &directory); + backup.runtime_options.durability = iggy_common::Durability::Persisted; + backup.runtime_options.segment_size = Some(config.segment_size); + let payload = vec![1; PAYLOAD_BYTES]; + let parents = segment_chain_parents(OPS, &payload); + let prepare = |op: u64| { + checksummed_segment_prepare( + op, + parents[usize::try_from(op - 1).unwrap()], + op - 1, + &payload, + ) + }; + let frame_bytes = prepare(1).total_len(); + backup.log.retire_front().unwrap(); + backup.install_empty_segment(&config, 0).await.unwrap(); + backup + .open_persistence_with_capacity( + journal::partition_journal::PARTITION_WAL_CAPACITY_MIN, + std::time::Duration::ZERO, + ) + .await + .unwrap(); + backup.consensus().advance_commit_max(OPS); + backup.repair = Some(RepairSession { + floor: None, + ..armed_session(OPS, 0) + }); + let persistence = Rc::clone(backup.persistence.as_ref().unwrap()); + assert!(persistence.segment_checkpoint().is_some()); + + persistence.append(prepare(1).into_frozen(), true).unwrap(); + assert!(persistence.start()); + backup.apply_repaired_prepare(prepare(1)).await; + Rc::clone(&persistence).run().await; + assert!(persistence.is_durable_through(1)); + persistence.append(prepare(2).into_frozen(), true).unwrap(); + assert!(persistence.start()); + backup.apply_repaired_prepare(prepare(2)).await; + assert!(!persistence.has_capacity(frame_bytes)); + assert!(!persistence.is_durable_through(2)); + assert!(!backup.repair_fetch_ready()); + + assert_eq!( + backup.complete_repair(&config).await, + RepairConclusion::InProgress + ); + assert_eq!( + backup.consensus().commit_min(), + 1, + "the verified durable prefix must apply before the rest of the window can fit" + ); + assert!(backup.repair.is_some()); + Rc::clone(&persistence).run().await; + settle_partition_io(&mut backup, &config).await; + assert_eq!( + backup.complete_repair(&config).await, + RepairConclusion::InProgress + ); + assert_eq!(backup.consensus().commit_min(), 2); + backup.checkpoint_persistence(&config).await; + settle_partition_io(&mut backup, &config).await; + persistence + .drain_with_timeout(|timeout| backup.consensus.message_bus().sleep(timeout)) + .await + .unwrap(); + settle_partition_io(&mut backup, &config).await; + assert!( + persistence.has_capacity(frame_bytes), + "applying and checkpointing the prefix must reclaim WAL capacity" + ); + assert!(backup.repair_fetch_ready()); + + backup.apply_repaired_prepare(prepare(3)).await; + backup.persist_repaired_prefix(); + persistence + .drain_with_timeout(|timeout| backup.consensus.message_bus().sleep(timeout)) + .await + .unwrap(); + settle_partition_io(&mut backup, &config).await; + assert_eq!( + backup.complete_repair(&config).await, + RepairConclusion::Done, + "commit={}, head={}, durable={}, sequence={}, fatal={:?}", + backup.consensus().commit_min(), + persistence.head(), + persistence.durable_op(), + backup.consensus().sequencer().current_sequence(), + backup.fatal(), + ); + assert_eq!(backup.consensus().commit_min(), OPS); + assert!(backup.fatal().is_none()); + } + + /// A bounded commit walk can leave already fetched frames for the next + /// round. Advancing the walk keeps the peer's repair budget intact. #[compio::test] async fn given_advancing_repair_walk_when_peer_re_serves_held_frames_should_keep_peer() { const ROUNDS: u32 = crate::types::REPAIR_MAX_STALL_RETRIES + 1; @@ -32355,6 +34587,10 @@ mod tests { artifacts: Vec::new(), target_accepted: false, idle_ticks: 0, + spill: None, + scan: None, + seed: None, + pulled_len: 0, }); } assert!(partition.burn_transfer_attempt(), "budget exhausts"); @@ -32524,6 +34760,185 @@ mod tests { assert!(backup.repair.is_none()); } + /// The repair walk of an unproven log runs from the adopted head down and + /// commits nothing before it reaches `commit_min`. A walk within the WAL + /// byte budget keeps repairing. Once it holds more, it journals nothing + /// more and hands the window to state transfer. + #[compio::test] + async fn given_an_unproven_walk_past_the_wal_byte_budget_when_completing_should_require_state_transfer() + { + const PAYLOAD_BYTES: usize = 4 * 1024 * 1024; + let budget = journal::partition_journal::PARTITION_WAL_CAPACITY_MIN; + let payload = vec![7_u8; PAYLOAD_BYTES]; + let prepare_bytes = u64::try_from( + checksummed_segment_prepare(1, 0, 0, &payload) + .as_slice() + .len(), + ) + .unwrap(); + let journaled = budget / prepare_bytes + 1; + let head = journaled + 2; + let parents = segment_chain_parents(head, &payload); + let prepare = |op: u64| { + let parent = parents[usize::try_from(op - 1).unwrap()]; + checksummed_segment_prepare(op, parent, op - 1, &payload) + }; + let root = tempfile::tempdir().unwrap(); + let (mut backup, _) = recording_partition_at(1, 3); + backup.set_partition_dir(root.path().to_string_lossy().into_owned()); + backup + .open_persistence_with_capacity(budget, std::time::Duration::ZERO) + .await + .unwrap(); + backup.consensus().advance_commit_max(head); + backup.consensus().sequencer().set_sequence(head); + backup.consensus().fence_view_log(head, head); + backup.repair = Some(floorless_session(head, head)); + + backup.apply_repaired_prepare(prepare(head)).await; + assert_eq!( + backup.complete_repair(&repair_config()).await, + RepairConclusion::InProgress, + "a walk within the budget keeps repairing" + ); + for op in (1..head).rev() { + backup.apply_repaired_prepare(prepare(op)).await; + } + + let journal = &backup.log.journal().inner; + let lowest_journaled = head - journaled + 1; + assert!( + journal.header_by_op(lowest_journaled).is_some(), + "the body that crosses the budget is journaled" + ); + assert!( + journal.header_by_op(lowest_journaled - 1).is_none(), + "the walk journals nothing past the budget" + ); + assert_eq!( + backup.complete_repair(&repair_config()).await, + RepairConclusion::FloorRefused { + floor: 0, + to_op: head + } + ); + assert_eq!(backup.consensus().commit_min(), 0); + assert!(backup.repair.is_none()); + } + + #[compio::test] + async fn given_walks_sharing_a_node_budget_when_it_is_full_should_transfer_and_release_discarded_bodies() + { + const HEAD: u64 = 3; + const PAYLOAD: &[u8] = b"repair body"; + let parents = segment_chain_parents(HEAD, PAYLOAD); + let prepare = |op: u64| { + checksummed_segment_prepare( + op, + parents[usize::try_from(op - 1).unwrap()], + op - 1, + PAYLOAD, + ) + }; + let capacity = prepare(HEAD).retained_allocation_size().unwrap(); + let budget = Arc::new(crate::RepairMemoryBudget::new(capacity)); + let backup = || { + let (mut partition, _) = recording_partition_at(1, 3); + partition.proof_walk_memory = Some(budget.reservation()); + partition.consensus().advance_commit_max(HEAD); + partition.consensus().sequencer().set_sequence(HEAD); + partition.consensus().fence_view_log(HEAD, HEAD); + partition.repair = Some(floorless_session(HEAD, HEAD)); + partition + }; + let mut first = backup(); + let mut second = backup(); + first.apply_repaired_prepare(prepare(HEAD)).await; + assert!(first.log.journal().inner.header_by_op(HEAD).is_some()); + assert!( + !budget.reservation().try_add(1), + "the first walk owns the available allocation" + ); + + second.apply_repaired_prepare(prepare(HEAD)).await; + assert!( + second.log.journal().inner.is_empty(), + "an over-budget body must not be journaled" + ); + assert_eq!( + second.complete_repair(&repair_config()).await, + RepairConclusion::FloorRefused { + floor: 0, + to_op: HEAD + }, + "memory pressure must hand this walk to state transfer" + ); + + first.truncate_uncommitted_from(1).await.unwrap(); + assert!( + budget.reservation().try_add(capacity), + "discarding the window releases its reservation" + ); + + let mut third = backup(); + third.apply_repaired_prepare(prepare(HEAD)).await; + assert!(third.log.journal().inner.header_by_op(HEAD).is_some()); + drop(third); + assert!( + budget.reservation().try_add(capacity), + "dropping a partition releases its reservation" + ); + } + + #[compio::test] + async fn given_a_primary_elect_when_repair_memory_is_exhausted_should_request_transfer() { + const VIEW: u32 = 4; + const HEAD: u64 = 3; + let (mut partition, _) = recording_partition_at(1, 3); + let prepare = checksummed_segment_prepare(HEAD, 0, HEAD - 1, b"repair"); + let body = &prepare.as_slice()[..size_of::()]; + let size = size_of::() + body.len(); + let mut dvc = Message::::new(size); + dvc.as_mut_slice()[size_of::()..] + .copy_from_slice(body); + let dvc = + dvc.transmute_header(|_, header: &mut iggy_binary_protocol::DoViewChangeHeader| { + header.command = Command::DoViewChange; + header.size = u32::try_from(size).unwrap(); + header.group = partition.consensus.group(); + header.cluster = TEST_CLUSTER; + header.view = VIEW; + header.log_view = 1; + header.replica = 0; + header.op = HEAD; + header.commit = HEAD; + header.present_bitset = 1; + }); + partition.consensus.handle_do_view_change( + consensus::PlaneKind::Partitions, + dvc.header(), + body, + ); + assert!(partition.consensus.is_primary_elect()); + partition.consensus.fence_view_log(HEAD, HEAD); + partition.repair = Some(RepairSession { + view: VIEW, + ..floorless_session(HEAD, HEAD) + }); + let budget = Arc::new(crate::RepairMemoryBudget::new(1)); + partition.proof_walk_memory = Some(budget.reservation()); + partition.apply_repaired_prepare(prepare).await; + assert!(partition.log.journal().inner.is_empty()); + assert_eq!( + partition.complete_repair(&repair_config()).await, + RepairConclusion::FloorRefused { + floor: 0, + to_op: HEAD + }, + "primary election must not discard the memory-pressure signal" + ); + } + /// The serving peer evicted every op below `RETAINED_FROM` and retains more /// than one repair chunk above it. Every request that starts below /// `RETAINED_FROM` gets `RangeEvicted` and the same first chunk, so the @@ -32828,12 +35243,23 @@ mod tests { /// replicated-apply path (which stamps offsets and re-checksums), with the /// sequencer advanced the way `on_replicate` does after a real append. async fn journal_send_batch(partition: &mut IggyPartition, op: u64) { + let record = build_segment_record(IggyNamespace::new(1, 1, 0), 0); + journal_send_record(partition, op, &record).await; + } + + /// [`journal_send_batch`] with the given `record` as the prepare body. The + /// record builder gives each message a random id, so partitions that must + /// write the same bytes share their records. + async fn journal_send_record( + partition: &mut IggyPartition, + op: u64, + record: &[u8], + ) { let namespace = IggyNamespace::new(1, 1, 0); - let record = build_segment_record(namespace, 0); let header_size = std::mem::size_of::(); let total = header_size + record.len(); let mut message = Message::::new(total); - message.as_mut_slice()[header_size..].copy_from_slice(&record); + message.as_mut_slice()[header_size..].copy_from_slice(record); let message = message.transmute_header(|_, header: &mut PrepareHeader| { header.command = Command::Prepare; header.partition_incarnation = partition.created_revision(); @@ -34184,6 +36610,10 @@ mod tests { impl PersistFixture { async fn new(log_path: &str, index_path: &str) -> Self { + Self::with_replicas(log_path, index_path, 1).await + } + + async fn with_replicas(log_path: &str, index_path: &str, replica_count: u8) -> Self { let log_cursor = Rc::new(AtomicU64::new(0)); let index_cursor = Rc::new(AtomicU64::new(0)); let messages_writer = @@ -34194,7 +36624,7 @@ mod tests { .await .expect("open segment index writer"); - let mut partition = test_partition(); + let mut partition = test_partition_with_replicas(replica_count); partition.log.retire_back(); partition.log.add_persisted_segment( Segment::new(0, IggyByteSize::from(1024 * 1024_u64)), @@ -34256,6 +36686,47 @@ mod tests { std::fs::metadata(path).expect("stat file").len() } + #[compio::test] + async fn given_large_saved_appends_when_accepted_should_keep_the_checksum_of_the_written_bytes() + { + const PAYLOAD_BYTES: usize = 12 * 1024 * 1024; + let directory = tempfile::tempdir().unwrap(); + let log_path = directory.path().join("segment.log"); + let index_path = directory.path().join("segment.index"); + let mut fixture = PersistFixture::with_replicas( + log_path.to_str().unwrap(), + index_path.to_str().unwrap(), + 3, + ) + .await; + fixture + .partition + .set_partition_dir(directory.path().to_str().unwrap().to_owned()); + let payload = vec![0x5a; PAYLOAD_BYTES]; + for offset in 0..2 { + let job = fixture.capture(&payload, offset).unwrap(); + assert!( + fixture.partition.segment_checksum_cache.borrow().is_empty(), + "the in-flight append owns its hasher" + ); + let result = job.execute().await; + assert!( + fixture.partition.segment_checksum_cache.borrow().is_empty(), + "file execution cannot publish a stamp before acceptance" + ); + fixture.partition.accept_materialization(result).unwrap(); + let written = std::fs::read(&log_path).unwrap(); + let memos = fixture.partition.segment_checksum_cache.borrow(); + assert_eq!( + memos + .get(&0) + .and_then(|memo| memo.checksum_of(written.len() as u64)), + Some(consensus::state_artifact_checksum(&written)), + "append {offset} must hash exactly the bytes accepted on disk" + ); + } + } + #[compio::test] async fn reference_wal_backlog_keeps_disk_poll_seek_bounded() { const RECORDS: u64 = 8; @@ -34917,7 +37388,7 @@ mod tests { ); staged.push( partition - .spill_transfer_segment(&artifact, body) + .spill_transfer_segment(&artifact, body, None) .await .unwrap(), ); diff --git a/core/partitions/src/iggy_partitions.rs b/core/partitions/src/iggy_partitions.rs index 9b0d176669..2f976b1bda 100644 --- a/core/partitions/src/iggy_partitions.rs +++ b/core/partitions/src/iggy_partitions.rs @@ -121,6 +121,7 @@ where fence_published: Rc>, persistence_notifier: RefCell>, io_notifier: RefCell>, + repair_memory: Option>, loopback_ready: Rc, /// Debug-only tripwire: counts live [`Self::with_partition`] borrows so /// `insert` / `remove` can assert the partitions vec is never mutated @@ -167,6 +168,7 @@ where fence_published: Rc::default(), persistence_notifier: RefCell::new(None), io_notifier: RefCell::new(None), + repair_memory: None, loopback_ready: Rc::default(), #[cfg(debug_assertions)] borrow_active: Cell::new(0), @@ -189,12 +191,26 @@ where fence_published: Rc::default(), persistence_notifier: RefCell::new(None), io_notifier: RefCell::new(None), + repair_memory: None, loopback_ready: Rc::default(), #[cfg(debug_assertions)] borrow_active: Cell::new(0), } } + /// Build an empty shard collection using the node's shared repair budget. + #[must_use] + pub fn with_repair_memory( + shard_id: ShardId, + config: PartitionsConfig, + capacity: usize, + budget: Arc, + ) -> Self { + let mut partitions = Self::with_capacity(shard_id, config, capacity); + partitions.repair_memory = Some(budget); + partitions + } + /// This node's key-encryption keys, for wrapping new stream keys. #[must_use] pub fn key_context(&self) -> Arc { @@ -394,6 +410,9 @@ where if let Some((notifier, bytes_max)) = self.io_notifier.borrow().as_ref() { partition.set_io_notifier(Rc::clone(notifier), *bytes_max); } + if let Some(budget) = &self.repair_memory { + partition.proof_walk_memory = Some(budget.reservation()); + } self.install_loopback_notifier(namespace, &partition); partition.publish_current_offset(); partition.set_consumer_group_offsets_reconcile_epoch(Rc::clone( diff --git a/core/partitions/src/install_backup.rs b/core/partitions/src/install_backup.rs index 4c669b8ce7..7037814c07 100644 --- a/core/partitions/src/install_backup.rs +++ b/core/partitions/src/install_backup.rs @@ -235,12 +235,16 @@ mod tests { ] { std::fs::write(root.join(name), name.as_bytes()).unwrap(); } + // A segment that the transfer reuses reaches its name as a link. + std::fs::hard_link(root.join("0.index"), root.join("0.index.staging")).unwrap(); begin(root, &BTreeSet::new()).await.unwrap(); // Model the install's unlink and atomic replacement operations. for name in ["0.log", "offsets/1", "prepares-1/frontier"] { std::fs::remove_file(root.join(name)).unwrap(); std::fs::write(root.join(name), b"replacement").unwrap(); } + std::fs::remove_file(root.join("0.index")).unwrap(); + std::fs::rename(root.join("0.index.staging"), root.join("0.index")).unwrap(); std::fs::write(root.join("99.log"), b"new segment").unwrap(); recover(root).await.unwrap(); for name in [ @@ -254,6 +258,7 @@ mod tests { assert_eq!(std::fs::read(root.join(name)).unwrap(), name.as_bytes()); } assert!(!root.join("99.log").exists()); + assert!(!root.join("0.index.staging").exists()); recover(root).await.unwrap(); } diff --git a/core/partitions/src/io.rs b/core/partitions/src/io.rs index 412c2f9c9b..82ed72f1cd 100644 --- a/core/partitions/src/io.rs +++ b/core/partitions/src/io.rs @@ -24,7 +24,7 @@ use std::task::{Poll, Waker}; use std::time::{Duration, Instant}; use compio::fs::File; -use consensus::VsrState; +use consensus::{StateArtifactHasher, VsrState}; use futures::{StreamExt, TryStreamExt}; use iggy_binary_protocol::PrepareHeader; use iggy_common::MAX_MESSAGE_SIZE_UPPER_BYTES; @@ -72,11 +72,13 @@ pub struct MaterializationIoJob { pub(crate) bodies_written: Option, pub(crate) written: u64, pub(crate) completes: bool, + pub(crate) checksum: Option>, } pub struct MaterializationIoResult { pub(crate) target: MaterializationTarget, pub(crate) outcome: Result<(u64, u64), IggyError>, + pub(crate) checksum: Option>, } pub enum PartitionIoJob { @@ -180,6 +182,7 @@ pub struct CapturedPartitionIo { #[derive(Default)] pub struct PartitionIoQuiescence { active: Cell>, + local_work: Cell, retiring: Cell, deleting: Cell, interrupted: Cell, @@ -208,10 +211,23 @@ impl PartitionIoQuiescence { pub(crate) fn set(&self, active: Option) { self.active.set(active); - if active.is_none() - && let Some(waiter) = self.waiter.borrow_mut().take() - { - waiter.wake(); + if active.is_none() { + self.wake_waiter(); + } + } + + pub(crate) fn begin_local_work(&self) -> bool { + if self.is_retiring() || self.is_interrupted() { + return false; + } + self.local_work.set(self.local_work.get() + 1); + true + } + + pub(crate) fn settle_local_work(&self) { + self.local_work.set(self.local_work.get() - 1); + if self.local_work.get() == 0 { + self.wake_waiter(); } } @@ -248,7 +264,7 @@ impl PartitionIoQuiescence { "partition writer was interrupted", ))); } - if self.active.get().is_none() { + if self.active.get().is_none() && self.local_work.get() == 0 { return Poll::Ready(Ok(())); } *self.waiter.borrow_mut() = Some(context.waker().clone()); @@ -256,6 +272,13 @@ impl PartitionIoQuiescence { }) .await } + + fn wake_waiter(&self) { + let waiter = self.waiter.borrow_mut().take(); + if let Some(waiter) = waiter { + waiter.wake(); + } + } } impl PartitionTeardown { @@ -541,8 +564,23 @@ impl MaterializationIoJob { bodies_written, written, completes, + mut checksum, .. } = self; + for batch in &mut batches { + if batch.len() < size_of::() { + return MaterializationIoResult { + target, + outcome: Err(IggyError::CannotWriteToFile), + checksum: None, + }; + } + *batch = batch.slice(size_of::()..); + } + if let Some(hasher) = checksum.as_mut() { + crate::state_transfer::hash_yielding(hasher, batches.iter().map(Frozen::as_slice)) + .await; + } let outcome = if let Some(saved) = bodies_written { target .index_writer @@ -556,11 +594,9 @@ impl MaterializationIoJob { return MaterializationIoResult { target, outcome: Err(IggyError::CannotWriteToFile), + checksum: None, }; }; - for batch in &mut batches { - *batch = batch.slice(size_of::()..); - } // Both halves must settle, including when either one fails. let (messages, indexes) = futures::future::join( writer.save_frozen_batches_at(&batches, position, completes), @@ -593,7 +629,11 @@ impl MaterializationIoJob { Ok((0, 0)) } }); - MaterializationIoResult { target, outcome } + MaterializationIoResult { + target, + outcome, + checksum, + } } } diff --git a/core/partitions/src/journal.rs b/core/partitions/src/journal.rs index 513f49f1ed..56e5aa99de 100644 --- a/core/partitions/src/journal.rs +++ b/core/partitions/src/journal.rs @@ -16,20 +16,29 @@ // under the License. use ahash::AHashMap; +use compio::buf::{IntoInner, IoBuf}; +use compio::io::AsyncReadAtExt; use iggy_binary_protocol::{Operation, PrepareHeader}; +use journal::partition_journal::SegmentReference; use journal::{Journal, Storage}; use server_common::{ - iobuf::{Frozen, Owned}, - send_messages::{self, BatchRef, COMMAND_HEADER_SIZE, decode_prepare_slice_trusted}, + iobuf::{Frozen, IOV_MAX, Owned}, + send_messages::{ + self, BatchRef, COMMAND_HEADER_SIZE, decode_prepare_slice, decode_prepare_slice_trusted, + }, }; use std::io; use std::{ cell::{Cell, RefCell, UnsafeCell}, - collections::{BTreeMap, HashMap}, + collections::{BTreeMap, HashMap, VecDeque}, + fmt::Debug, ops::RangeInclusive, + rc::{Rc, Weak}, }; use tracing::warn; +use crate::log::SegmentedLog; +use crate::repair_memory::RepairMemoryReservation; use crate::{Fragment, PollFragments, PollQueryResult}; const ZERO_LEN: usize = 0; @@ -57,6 +66,341 @@ pub struct RepairedWindowShape { pub complete: bool, } +/// Where a committed prepare's body sits in the log file of its segment. A +/// segment stores the body without the prepare header. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BodyPosition { + /// Start offset of the segment, which names its log file. + pub segment_start: u64, + pub position: u64, +} + +impl From for BodyPosition { + fn from(reference: SegmentReference) -> Self { + Self { + segment_start: reference.start_offset, + position: reference.position, + } + } +} + +/// A committed prepare the repair ring keeps without its body. The byte cap +/// released the body, which still sits in its segment, so repair reads it +/// back instead of answering `RangeEvicted`. +/// +/// Built only from a whole `SendMessages` prepare, so `header.size` covers the +/// prepare header and a batch. +#[derive(Debug, Clone, Copy)] +pub struct EvictedHeader { + header: PrepareHeader, + body: BodyPosition, + /// Checksum of the released batch. Bytes read back must carry it, so + /// another batch at the same position cannot pass for this op. + batch_checksum: u64, +} + +/// What one released entry costs the evicted ring: 288 bytes, so +/// [`EVICTED_HEADERS_MAX`] of them take 4.5 MiB per partition. +const EVICTED_HEADER_BYTES: u64 = size_of::() as u64; +const _: () = assert!(EVICTED_HEADER_BYTES == 288); + +impl EvictedHeader { + #[must_use] + pub const fn header(&self) -> &PrepareHeader { + &self.header + } + + /// Whether the body of `next` lies right after this one in the same + /// segment file. + const fn followed_by(&self, next: &Self) -> bool { + let body_len = (self.header.size as usize).saturating_sub(PREPARE_HEADER_SIZE); + self.body.segment_start == next.body.segment_start + && self.body.position + body_len as u64 == next.body.position + } + + /// `prepare`, read back for this entry, if its body is the released one. + fn verified(&self, prepare: Owned<4096>) -> io::Result { + let batch = decode_prepare_slice(prepare.as_slice()).map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("the body read back fails its checks: {error}"), + ) + })?; + if batch.header.batch_checksum != self.batch_checksum { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "another batch sits at the evicted body position", + )); + } + Ok(prepare.into()) + } + + /// Start offset of the segment whose log file holds the body. + #[must_use] + pub const fn segment_start(&self) -> u64 { + self.body.segment_start + } +} + +/// Reads the bodies of `released`, in op order, back from the log files of +/// their segments in `log`, and pushes each prepare to `bodies`. Bodies that +/// lie back to back in one file come back in one read. +/// +/// # Errors +/// Returns an error if a segment left `log`, if a body cannot be read, or if +/// the bytes there are not the released body: a torn or corrupt batch, or +/// another batch at the same position. Such bytes are never returned. The +/// bodies before the one that failed, `released[bodies.len()]`, stay in +/// `bodies`. +pub async fn read_released_bodies( + log: &SegmentedLog, + released: &[EvictedHeader], + bodies: &mut Vec, +) -> io::Result<()> { + let mut file: Option<(u64, compio::fs::File)> = None; + let runs = released + .chunk_by(EvictedHeader::followed_by) + .flat_map(|run| run.chunks(IOV_MAX)); + for run in runs { + let Some(first) = run.first() else { + continue; + }; + let segment = first.body.segment_start; + let file = match &mut file { + Some((start, file)) if *start == segment => file, + slot => { + let path = log.messages_path(segment).ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "the segment left the log") + })?; + &slot + .insert((segment, compio::fs::File::open(path).await?)) + .1 + } + }; + // A vectored file read in compio fills only the initialized bytes of + // each buffer, so the bodies start zeroed. + let buffers: Vec<_> = run + .iter() + .map(|evicted| { + let size = evicted.header.size as usize; + let mut prepare = Owned::<4096>::zeroed(size); + prepare.as_mut_slice()[..PREPARE_HEADER_SIZE] + .copy_from_slice(bytemuck::bytes_of(&evicted.header)); + prepare.slice(PREPARE_HEADER_SIZE..size) + }) + .collect(); + let compio::BufResult(read, buffers) = file + .read_vectored_exact_at(buffers, first.body.position) + .await; + for (evicted, body) in run.iter().zip(buffers) { + match evicted.verified(body.into_inner()) { + Ok(prepare) => bodies.push(prepare), + // Only the released body itself passes the checks, and a + // failed read leaves zeros where it stopped, so after a failed + // read the first body that fails is where the read stopped. + Err(error) => return Err(read.err().unwrap_or(error)), + } + } + } + Ok(()) +} + +/// A committed prepare the repair ring holds in memory, with the place of its +/// body in a segment when the WAL wrote it there. +struct RingEntry { + prepare: JournalBuffer, + body: Option, +} + +/// A snapshot's repair suffix. Advancing or dropping the last lease releases +/// its extra retention immediately. The ordinary ring keeps its own caps. +pub struct RepairTailLease { + state: Rc, + retained: Weak>, +} + +struct RepairTailState { + from_op: Cell, + valid: Cell, +} + +impl RepairTailLease { + pub(crate) fn is_valid(&self) -> bool { + self.state.valid.get() + } + + pub(crate) fn advance(&self, from_op: u64) { + self.state + .from_op + .set(self.state.from_op.get().max(from_op)); + if let Some(retained) = self.retained.upgrade() { + retained.borrow_mut().prune(); + } + } +} + +impl Drop for RepairTailLease { + fn drop(&mut self) { + self.state.valid.set(false); + if let Some(retained) = self.retained.upgrade() { + retained.borrow_mut().prune(); + } + } +} + +#[derive(Default)] +struct RepairTail { + leases: Vec>, + bodies: BTreeMap, + headers: BTreeMap, + memory: Option, +} + +impl RepairTail { + fn prune(&mut self) -> Option { + let mut from_op = None; + self.leases.retain(|lease| { + let Some(lease) = lease.upgrade().filter(|lease| lease.valid.get()) else { + return false; + }; + let floor = lease.from_op.get(); + from_op = Some(from_op.map_or(floor, |old: u64| old.min(floor))); + true + }); + let Some(from_op) = from_op else { + self.bodies.clear(); + self.headers.clear(); + self.memory = None; + return None; + }; + while let Some(entry) = self + .bodies + .first_entry() + .filter(|entry| *entry.key() < from_op) + { + let bytes = Self::body_memory_bytes(&entry.remove()); + if let Some(memory) = &mut self.memory { + memory.release(bytes); + } + } + while let Some(entry) = self + .headers + .first_entry() + .filter(|entry| *entry.key() < from_op) + { + entry.remove(); + if let Some(memory) = &mut self.memory { + memory.release(std::mem::size_of::<(u64, EvictedHeader)>()); + } + } + Some(from_op) + } + + fn invalidate(&mut self) { + for lease in &self.leases { + if let Some(lease) = lease.upgrade() { + lease.valid.set(false); + } + } + self.leases.clear(); + self.bodies.clear(); + self.headers.clear(); + self.memory = None; + } + + fn reserve(&mut self, bytes: usize) -> bool { + if self + .memory + .as_mut() + .is_some_and(|memory| memory.try_add(bytes)) + { + return true; + } + self.invalidate(); + false + } + + fn keep_header(&mut self, entry: &EvictedHeader, from_op: Option) { + let op = entry.header.op; + if from_op.is_none_or(|from_op| op < from_op) || self.headers.contains_key(&op) { + return; + } + if let Some(body) = self.bodies.remove(&op) + && let Some(memory) = &mut self.memory + { + memory.release(Self::body_memory_bytes(&body)); + } + if self.reserve(std::mem::size_of::<(u64, EvictedHeader)>()) { + self.headers.insert(op, *entry); + } + } + + fn keep_body(&mut self, op: u64, entry: RingEntry, from_op: Option) { + if from_op.is_none_or(|from_op| op < from_op) + || self.bodies.contains_key(&op) + || self.headers.contains_key(&op) + { + return; + } + if let Some(header) = entry.evicted_header() { + self.keep_header(&header, from_op); + } else if self.reserve(Self::body_memory_bytes(&entry.prepare)) { + self.bodies.insert(op, entry.prepare); + } + } + + fn body_memory_bytes(body: &JournalBuffer) -> usize { + JournalBuffer::allocation_size(body.backing_capacity()) + .and_then(|bytes| bytes.checked_add(std::mem::size_of::<(u64, JournalBuffer)>())) + .unwrap_or(usize::MAX) + } +} + +impl RingEntry { + /// What the ring keeps of this entry once its body leaves memory. `None` + /// when no segment holds the body. + fn evicted_header(&self) -> Option { + let body = self.body?; + let header = *bytemuck::checked::try_from_bytes::( + self.prepare.as_slice().get(..PREPARE_HEADER_SIZE)?, + ) + .ok()?; + if header.operation != Operation::SendMessages { + return None; + } + let batch = decode_prepare_slice_trusted(self.prepare.as_slice()).ok()?; + Some(EvictedHeader { + header, + body, + batch_checksum: batch.header.batch_checksum, + }) + } +} + +/// Drops the oldest entry of either tier of the evicted ring, so the ops it +/// keeps stay one run. Returns the body bytes it frees. +fn drop_oldest_evicted( + ring: &mut BTreeMap, + headers: &mut VecDeque, + retained: &mut RepairTail, + from_op: Option, +) -> u64 { + let oldest_held = ring.first_key_value().map(|(op, _)| *op); + match headers.front() { + Some(evicted) if oldest_held.is_none_or(|held| evicted.header.op < held) => { + if let Some(entry) = headers.pop_front() { + retained.keep_header(&entry, from_op); + } + 0 + } + _ => ring.pop_first().map_or(0, |(op, dropped)| { + let bytes = dropped.prepare.len() as u64; + retained.keep_body(op, dropped, from_op); + bytes + }), + } +} + /// Lookup key for querying messages from the journal. /// /// `ceiling` is the inclusive commit-frontier bound: the resident journal holds @@ -173,18 +517,27 @@ where inner: UnsafeCell>, /// Ring of recently evicted committed entries, keyed by op, retained so /// this replica can serve journal repair for rejoin windows after the - /// entries left the resident journal at flush. Bounded by + /// entries left the resident journal at flush. Bodies past the byte cap + /// move to `evicted_headers` when a segment holds them. Bounded by /// [`EVICTED_RING_CAPACITY`]; requests older than the ring answer /// `RangeEvicted` honestly. - evicted_ring: UnsafeCell>, - /// Running byte total of the buffers held by `evicted_ring`. + evicted_ring: UnsafeCell>, + /// Running byte total of the prepares held by `evicted_ring`. evicted_ring_bytes: Cell, + /// The ring's entries whose bodies the byte cap released to their + /// segments, in op order. At most [`EVICTED_HEADERS_MAX`]. + evicted_headers: UnsafeCell>, + repair_tail: Rc>, + /// Highest op the byte cap released or passed over, so it looks at each + /// entry once. An entry that lands below it later keeps its body until + /// the oldest entries go. + released_through: Cell, /// Entry-count ceiling for `evicted_ring`. Defaults to - /// [`EVICTED_RING_CAPACITY`]; the server overrides it from config at + /// [`EVICTED_RING_CAPACITY`], and the server overrides it from config at /// partition build. evicted_ring_capacity: Cell, /// Byte ceiling for `evicted_ring`. Defaults to - /// [`EVICTED_RING_BYTES_MAX`]; the server overrides it from config at + /// [`EVICTED_RING_BYTES_MAX`], and the server overrides it from config at /// partition build. evicted_ring_bytes_max: Cell, /// Single-replica groups have nobody to repair; retaining evicted @@ -209,40 +562,38 @@ where held_run: Cell<(u64, u64)>, } -/// How many evicted entries each partition retains for repair. Sized to -/// cover a few seconds of traffic around a node restart; anything older is -/// bulk-sync (phase 3) territory. +/// How many evicted entries each partition retains for repair with their +/// bodies in memory. The entries whose bodies went back to their segments +/// have their own cap, `EVICTED_HEADERS_MAX`. pub const EVICTED_RING_CAPACITY: usize = 65536; -/// Byte ceiling for the evicted ring. Every retained entry pins a full -/// batch, so an entry cap alone can consume too much memory across many -/// partitions. Whichever cap trips first evicts. +/// Byte ceiling for the bodies the evicted ring holds in memory. +/// +/// An entry that holds its body pins a full batch, so an entry cap alone can +/// consume too much memory across many partitions. Past this cap the oldest +/// bodies that a segment still holds leave memory, and the oldest entries go +/// when none can leave. The headers of released bodies do not count here. pub const EVICTED_RING_BYTES_MAX: u64 = 64 * 1024 * 1024; +/// How many entries whose bodies went back to their segments each partition +/// keeps for repair. Each one takes 288 bytes, so the cap holds 4.5 MiB per +/// partition, under 8% of the body window that [`EVICTED_RING_BYTES_MAX`] sets. +/// +/// The cap sets how far a backup can fall behind and still catch up by +/// repair: 128 repair rounds at the default chunk of 128 ops. Past it, the +/// backup gets `RangeEvicted` for the older ops and catches up by state +/// transfer, which reuses the sealed segments that the backup holds. +/// A power of two, so the deque that holds the headers, which makes room +/// before each push, never allocates past it. +pub const EVICTED_HEADERS_MAX: usize = 16384; +const _: () = assert!(EVICTED_HEADERS_MAX.is_power_of_two()); + impl Default for PartitionJournal where S: Storage + Default, { fn default() -> Self { - Self { - op_to_storage_offset: UnsafeCell::new(BTreeMap::new()), - offset_to_op: UnsafeCell::new(BTreeMap::new()), - timestamp_to_op: UnsafeCell::new(BTreeMap::new()), - headers: UnsafeCell::new(Vec::new()), - first_header_by_op: RefCell::new(AHashMap::new()), - inner: UnsafeCell::new(JournalInner { - storage: S::default(), - }), - evicted_ring: UnsafeCell::new(BTreeMap::new()), - evicted_ring_bytes: Cell::new(0), - evicted_ring_capacity: Cell::new(EVICTED_RING_CAPACITY), - evicted_ring_bytes_max: Cell::new(EVICTED_RING_BYTES_MAX), - repair_retention: Cell::new(true), - resident_control_ops: Cell::new(0), - evicted_through: Cell::new(0), - out_of_order: Cell::new(false), - held_run: Cell::new((0, 0)), - } + Self::with_storage(S::default()) } } @@ -333,8 +684,7 @@ impl PartitionJournal { let _ = inner.storage.drain(); } self.clear_resident_indexes(); - unsafe { &mut *self.evicted_ring.get() }.clear(); - self.evicted_ring_bytes.set(0); + self.clear_evicted(); self.evicted_through.set(0); } @@ -342,9 +692,36 @@ impl PartitionJournal { pub fn set_repair_retention(&self, enabled: bool) { self.repair_retention.set(enabled); if !enabled { - let ring = unsafe { &mut *self.evicted_ring.get() }; - ring.clear(); - self.evicted_ring_bytes.set(0); + self.clear_evicted(); + } + } + + fn clear_evicted(&self) { + self.repair_tail.borrow_mut().invalidate(); + unsafe { &mut *self.evicted_ring.get() }.clear(); + unsafe { &mut *self.evicted_headers.get() }.clear(); + self.evicted_ring_bytes.set(0); + self.released_through.set(0); + } + + pub(crate) fn retain_repair_from( + &self, + from_op: u64, + memory: RepairMemoryReservation, + ) -> RepairTailLease { + let state = Rc::new(RepairTailState { + from_op: Cell::new(from_op), + valid: Cell::new(true), + }); + let mut retained = self.repair_tail.borrow_mut(); + retained.prune(); + if retained.memory.is_none() { + retained.memory = Some(memory); + } + retained.leases.push(Rc::downgrade(&state)); + RepairTailLease { + state, + retained: Rc::downgrade(&self.repair_tail), } } @@ -357,13 +734,25 @@ impl PartitionJournal { self.evicted_ring_bytes_max.set(bytes_max); } - /// Entries and payload bytes the repair ring retains, for the shard sweep. + /// Entries the repair ring retains and the bytes they pin, for the shard + /// sweep: the prepare of each entry held in memory, and an + /// [`EvictedHeader`] for each entry whose body went back to its segment. /// /// Excludes unused allocation capacity, alignment and index overhead. /// The newest entry is retained even when it exceeds the byte budget. pub fn evicted_ring_occupancy(&self) -> (usize, u64) { let ring = unsafe { &*self.evicted_ring.get() }; - (ring.len(), self.evicted_ring_bytes.get()) + let headers = unsafe { &*self.evicted_headers.get() }; + let retained = self.repair_tail.borrow(); + ( + ring.len() + headers.len() + retained.bodies.len() + retained.headers.len(), + self.evicted_ring_bytes.get() + + headers.len() as u64 * EVICTED_HEADER_BYTES + + retained + .memory + .as_ref() + .map_or(0, |memory| memory.bytes() as u64), + ) } /// Committed entries already flushed out of the resident journal cannot be @@ -388,9 +777,15 @@ impl PartitionJournal { /// Restore the materialized commit point for elections and body repair. pub fn restore_checkpoint_prepare(&self, op: u64, prepare: JournalBuffer) { let ring = unsafe { &mut *self.evicted_ring.get() }; - debug_assert!(ring.is_empty()); + debug_assert!(ring.is_empty() && unsafe { &*self.evicted_headers.get() }.is_empty()); self.evicted_ring_bytes.set(prepare.len() as u64); - ring.insert(op, prepare); + ring.insert( + op, + RingEntry { + prepare, + body: None, + }, + ); } /// Entry bytes for `op`, from the resident journal or the evicted ring. @@ -405,7 +800,9 @@ impl PartitionJournal { } } let ring = unsafe { &*self.evicted_ring.get() }; - ring.get(&op).cloned() + ring.get(&op) + .map(|entry| entry.prepare.clone()) + .or_else(|| self.repair_tail.borrow().bodies.get(&op).cloned()) } /// The lowest resident `SendMessages` op in `ops` with its entry bytes, in @@ -460,8 +857,12 @@ impl PartitionJournal { return Some(header); } let ring = unsafe { &*self.evicted_ring.get() }; - let entry = ring.get(&op)?; - let header_bytes = entry.as_slice().get(..PREPARE_HEADER_SIZE)?; + let retained = self.repair_tail.borrow(); + let prepare = ring + .get(&op) + .map(|entry| &entry.prepare) + .or_else(|| retained.bodies.get(&op))?; + let header_bytes = prepare.as_slice().get(..PREPARE_HEADER_SIZE)?; bytemuck::checked::try_from_bytes::(header_bytes) .ok() .copied() @@ -486,32 +887,157 @@ impl PartitionJournal { } } let ring = unsafe { &*self.evicted_ring.get() }; - for (op, entry) in ring.range(ops) { + for (op, entry) in ring.range(ops.clone()) { if found.contains_key(op) { continue; } - let Some(header_bytes) = entry.as_slice().get(..PREPARE_HEADER_SIZE) else { + let Some(header_bytes) = entry.prepare.as_slice().get(..PREPARE_HEADER_SIZE) else { continue; }; if let Ok(header) = bytemuck::checked::try_from_bytes::(header_bytes) { found.insert(*op, *header); } } + for (op, entry) in self.repair_tail.borrow().bodies.range(ops) { + if found.contains_key(op) { + continue; + } + let Some(bytes) = entry.as_slice().get(..PREPARE_HEADER_SIZE) else { + continue; + }; + if let Ok(header) = bytemuck::checked::try_from_bytes::(bytes) { + found.insert(*op, *header); + } + } found } /// Oldest op this journal can still serve for repair, including resident - /// backfills older than the retained repair range. + /// backfills older than the retained repair range and bodies read back + /// from segments. pub fn repair_retained_from(&self) -> Option { + let headers = unsafe { &*self.evicted_headers.get() }; + self.oldest_in_memory() + .into_iter() + .chain(headers.front().map(|evicted| evicted.header.op)) + .chain( + self.repair_tail + .borrow() + .headers + .first_key_value() + .map(|(op, _)| *op), + ) + .min() + } + + /// Oldest op from which this journal holds every body up to its head in + /// memory, so repair serves that tail without a disk read. + pub fn memory_retained_from(&self) -> Option { + let headers = unsafe { &*self.evicted_headers.get() }; + let oldest = self.oldest_in_memory()?; + let newest_released = headers + .back() + .map(|entry| entry.header.op) + .into_iter() + .chain( + self.repair_tail + .borrow() + .headers + .last_key_value() + .map(|(op, _)| *op), + ) + .max(); + Some(newest_released.map_or(oldest, |op| oldest.max(op.saturating_add(1)))) + } + + fn oldest_in_memory(&self) -> Option { let ring = unsafe { &*self.evicted_ring.get() }; let resident = unsafe { &*self.op_to_storage_offset.get() }; ring.first_key_value() .map(|(op, _)| *op) .into_iter() .chain(resident.first_key_value().map(|(op, _)| *op)) + .chain( + self.repair_tail + .borrow() + .bodies + .first_key_value() + .map(|(op, _)| *op), + ) .min() } + /// The header the evicted ring keeps for `op` after the byte cap released + /// its body to a segment. [`Self::repair_entry`] and + /// [`Self::repair_header`] do not serve such an op: its body must be read + /// back with [`read_released_bodies`]. + #[must_use] + pub fn evicted_header(&self, op: u64) -> Option { + let headers = unsafe { &*self.evicted_headers.get() }; + headers + .binary_search_by_key(&op, |evicted| evicted.header.op) + .ok() + .and_then(|index| headers.get(index).copied()) + .or_else(|| self.repair_tail.borrow().headers.get(&op).copied()) + } + + /// Forgets the released entries whose bodies lie in segments up to the + /// one at `start_offset`, once retention deletes it, so repair answers + /// `RangeEvicted` for them. + pub fn forget_segment(&self, start_offset: u64) { + if self + .repair_tail + .borrow() + .headers + .values() + .any(|entry| entry.body.segment_start <= start_offset) + { + self.repair_tail.borrow_mut().invalidate(); + } + let headers = unsafe { &*self.evicted_headers.get() }; + let through = headers + .iter() + .take_while(|evicted| evicted.body.segment_start <= start_offset) + .last() + .map(|evicted| evicted.header.op); + if let Some(through) = through { + self.forget_released_through(through); + } + } + + /// Forgets every evicted entry at or below the released entry at `op`, + /// so repair answers `RangeEvicted` for them. A released entry is never + /// the newest one, so the commit point stays. + pub fn forget_released_through(&self, op: u64) { + let retained_affected = { + let retained = self.repair_tail.borrow(); + retained + .headers + .first_key_value() + .is_some_and(|(first, _)| *first <= op) + || retained + .bodies + .first_key_value() + .is_some_and(|(first, _)| *first <= op) + }; + if retained_affected { + self.repair_tail.borrow_mut().invalidate(); + } + let ring = unsafe { &mut *self.evicted_ring.get() }; + let headers = unsafe { &mut *self.evicted_headers.get() }; + let mut bytes = self.evicted_ring_bytes.get(); + while let Some(entry) = ring.first_entry().filter(|entry| *entry.key() <= op) { + bytes -= entry.remove().prepare.len() as u64; + } + while headers + .front() + .is_some_and(|evicted| evicted.header.op <= op) + { + headers.pop_front(); + } + self.evicted_ring_bytes.set(bytes); + } + /// Synchronous resident-range poll read. Never awaits (mem storage reads /// are pure memory copies), so a partition borrow held across it cannot span /// a scheduler yield. The poll path uses this; the disk tier, which does @@ -654,6 +1180,17 @@ impl PartitionJournal { /// Remove the persisted prefix and return the retained entries with their batch metadata. pub fn evict_prefix(&self, count: usize) -> Vec<(JournalBuffer, Option)> { + self.evict_prefix_located(count, |_| None) + } + + /// [`Self::evict_prefix`] for a journal whose bodies also sit in segments. + /// `locate` names where the body of an evicted op lies, so the evicted + /// ring can release that body from memory and still serve the op. + pub fn evict_prefix_located( + &self, + count: usize, + locate: impl Fn(u64) -> Option, + ) -> Vec<(JournalBuffer, Option)> { let all_entries = { let inner = unsafe { &*self.inner.get() }; inner.storage.drain() @@ -675,22 +1212,20 @@ impl PartitionJournal { let ring = unsafe { &mut *self.evicted_ring.get() }; let mut ring_bytes = self.evicted_ring_bytes.get(); for op in evicted_ops { - let Some(entry) = all_entries.next() else { + let Some(prepare) = all_entries.next() else { break; }; - ring_bytes += entry.len() as u64; + ring_bytes += prepare.len() as u64; + let entry = RingEntry { + prepare, + body: locate(op), + }; if let Some(previous) = ring.insert(op, entry) { - ring_bytes -= previous.len() as u64; - } - while ring.len() > self.evicted_ring_capacity.get() - || (ring_bytes > self.evicted_ring_bytes_max.get() && ring.len() > 1) - { - if let Some((_, dropped)) = ring.pop_first() { - ring_bytes -= dropped.len() as u64; - } + ring_bytes -= previous.prepare.len() as u64; } } self.evicted_ring_bytes.set(ring_bytes); + self.trim_evicted_ring(); } else { // Consume without retaining: the iterator itself must still // advance past the evicted prefix so the retained tail below is @@ -711,6 +1246,45 @@ impl PartitionJournal { result } + /// Keeps the evicted ring within its caps. Past the byte cap the oldest + /// bodies that a segment holds leave memory first, and the oldest entries + /// go when none can leave. The byte cap never takes the newest entry, so + /// the commit point stays servable for a view change. A released body + /// keeps its header, and the oldest entries make room for it first once + /// [`EVICTED_HEADERS_MAX`] headers are kept. + fn trim_evicted_ring(&self) { + let mut retained = self.repair_tail.borrow_mut(); + let retained_from = retained.prune(); + let ring = unsafe { &mut *self.evicted_ring.get() }; + let headers = unsafe { &mut *self.evicted_headers.get() }; + let mut bytes = self.evicted_ring_bytes.get(); + let bytes_max = self.evicted_ring_bytes_max.get(); + let newest = ring.last_key_value().map(|(op, _)| *op); + while bytes > bytes_max { + let Some((&op, entry)) = ring + .range(self.released_through.get().saturating_add(1)..) + .next() + .filter(|(op, _)| Some(**op) != newest) + else { + break; + }; + self.released_through.set(op); + if let Some(evicted) = entry.evicted_header() { + bytes -= entry.prepare.len() as u64; + ring.remove(&op); + while headers.len() >= EVICTED_HEADERS_MAX { + bytes -= drop_oldest_evicted(ring, headers, &mut retained, retained_from); + } + headers.push_back(evicted); + } + } + while ring.len() > self.evicted_ring_capacity.get() || (bytes > bytes_max && ring.len() > 1) + { + bytes -= drop_oldest_evicted(ring, headers, &mut retained, retained_from); + } + self.evicted_ring_bytes.set(bytes); + } + /// `append`, additionally returning the decoded `RetainedBatchMeta` for a /// `SendMessages` entry so the eviction path folds its accounting without a /// second decode of the same bytes. @@ -852,6 +1426,9 @@ where first_header_by_op: RefCell::new(AHashMap::new()), evicted_ring: UnsafeCell::new(BTreeMap::new()), evicted_ring_bytes: Cell::new(0), + evicted_headers: UnsafeCell::new(VecDeque::new()), + repair_tail: Rc::default(), + released_through: Cell::new(0), evicted_ring_capacity: Cell::new(EVICTED_RING_CAPACITY), evicted_ring_bytes_max: Cell::new(EVICTED_RING_BYTES_MAX), repair_retention: Cell::new(true), @@ -923,6 +1500,15 @@ where op_to_storage_offset.contains_key(&op) } + /// Highest resident op below `op`. + pub fn held_below(&self, op: u64) -> Option { + let op_to_storage_offset = unsafe { &*self.op_to_storage_offset.get() }; + op_to_storage_offset + .range(..op) + .next_back() + .map(|(held, _)| *held) + } + /// Highest op `h >= floor` such that every op in `floor + 1..=h` is resident. pub fn held_through(&self, floor: u64) -> u64 { self.held_through_until(floor, u64::MAX) @@ -1502,6 +2088,7 @@ pub fn select_resident( #[cfg(test)] mod tests { use super::*; + use crate::RepairMemoryBudget; use bytes::Bytes; use iggy_binary_protocol::{Command, HEADER_SIZE}; use journal::Journal; @@ -1511,6 +2098,151 @@ mod tests { decode_batch_slice, }; use server_common::sharding::IggyNamespace; + use std::sync::Arc; + + #[compio::test] + async fn given_a_small_snapshot_tail_with_a_large_backing_buffer_when_retained_should_charge_and_release_the_allocation() + { + const BACKING_CAPACITY: usize = 64 * 1024; + let mut backing = Owned::<4096>::with_capacity(BACKING_CAPACITY); + backing.extend_from_slice(build_prepare(1, HEADER_SIZE).as_slice()); + let prepare = Frozen::from(backing); + let charge = Frozen::<4096>::allocation_size(prepare.backing_capacity()).unwrap() + + size_of::<(u64, JournalBuffer)>(); + let budget = Arc::new(RepairMemoryBudget::new(charge)); + let journal = PartitionJournal::::default(); + journal.set_ring_caps(1, u64::MAX); + let lease = journal.retain_repair_from(1, budget.reservation()); + journal.append(prepare).await.unwrap(); + journal.evict_prefix(1); + journal + .append(build_prepare(2, HEADER_SIZE).into_frozen()) + .await + .unwrap(); + journal.evict_prefix(1); + + assert!(lease.is_valid()); + assert!(journal.repair_entry(1).is_some()); + assert!( + !budget.reservation().try_add(1), + "the retained frame must charge its complete backing allocation" + ); + lease.advance(2); + assert!(journal.repair_entry(1).is_none()); + assert!( + budget.reservation().try_add(charge), + "advancing the lease must refund the complete allocation charge" + ); + } + + #[compio::test] + async fn given_snapshot_leases_when_the_ring_evicts_should_retain_until_every_receiver_advances() + { + const OPS: u64 = 8; + let journal = PartitionJournal::::default(); + journal.set_ring_caps(1, u64::MAX); + let budget = Arc::new(RepairMemoryBudget::new(64 * 1024)); + let first = journal.retain_repair_from(2, budget.reservation()); + let second = journal.retain_repair_from(4, budget.reservation()); + for op in 1..=OPS { + journal + .append(build_prepare(op, HEADER_SIZE).into_frozen()) + .await + .unwrap(); + journal.evict_prefix(1); + } + assert!(journal.repair_entry(1).is_none()); + for op in 2..=OPS { + assert_eq!( + journal.repair_entry(op).unwrap().as_slice(), + build_prepare(op, HEADER_SIZE).as_slice() + ); + } + assert_eq!(journal.repair_retained_from(), Some(2)); + assert_eq!(journal.repair_headers_in(2..=OPS).len(), 7); + first.advance(6); + assert_eq!(journal.repair_retained_from(), Some(4)); + drop(second); + assert_eq!(journal.repair_retained_from(), Some(6)); + drop(first); + assert_eq!(journal.repair_retained_from(), Some(OPS)); + assert!( + budget.reservation().try_add(64 * 1024), + "all extra retention must release its budget" + ); + } + + #[compio::test] + async fn given_disk_backed_snapshot_tail_when_headers_leave_the_ring_should_keep_locations_without_bodies() + { + let journal = PartitionJournal::::default(); + journal.set_ring_caps(1, 1); + let budget = Arc::new(RepairMemoryBudget::new(64 * 1024)); + let lease = journal.retain_repair_from(1, budget.reservation()); + let ops = u64::try_from(EVICTED_HEADERS_MAX + 2).unwrap(); + for op in 1..=ops { + journal + .append(build_message_prepare(op, op - 1, 1, 1)) + .await + .unwrap(); + } + journal.evict_prefix_located(usize::try_from(ops).unwrap(), |op| { + Some(BodyPosition { + segment_start: 0, + position: op, + }) + }); + assert!(lease.is_valid()); + assert!(journal.repair_entry(1).is_none()); + assert_eq!(journal.evicted_header(1).unwrap().body.position, 1); + assert_eq!(journal.repair_retained_from(), Some(1)); + assert_eq!(journal.memory_retained_from(), Some(ops)); + journal.forget_segment(0); + assert!( + !lease.is_valid(), + "GC must invalidate the promise before its disk bodies disappear" + ); + assert!(journal.evicted_header(1).is_none()); + assert!(budget.reservation().try_add(64 * 1024)); + } + + #[compio::test] + async fn given_snapshot_tail_and_unproven_walk_when_the_shared_cap_is_full_should_invalidate_without_blocking_writes() + { + let charge = build_prepare(1, HEADER_SIZE) + .retained_allocation_size() + .unwrap() + + size_of::<(u64, JournalBuffer)>(); + let budget = Arc::new(RepairMemoryBudget::new(2 * charge)); + let mut walk = budget.reservation(); + assert!(walk.try_add(charge)); + let journal = PartitionJournal::::default(); + journal.set_ring_caps(1, u64::MAX); + let lease = journal.retain_repair_from(1, budget.reservation()); + for op in 1..=2 { + journal + .append(build_prepare(op, HEADER_SIZE).into_frozen()) + .await + .unwrap(); + journal.evict_prefix(1); + } + assert!(lease.is_valid()); + journal + .append(build_prepare(3, HEADER_SIZE).into_frozen()) + .await + .unwrap(); + journal.evict_prefix(1); + assert!(!lease.is_valid()); + assert_eq!(journal.repair_retained_from(), Some(3)); + assert_eq!(journal.evicted_ring_occupancy(), (1, HEADER_SIZE as u64)); + assert!(budget.reservation().try_add(charge)); + drop(walk); + let replacement = journal.retain_repair_from(3, budget.reservation()); + journal.clear_all(); + assert!(!replacement.is_valid()); + assert_eq!(journal.evicted_ring_occupancy(), (0, 0)); + assert!(budget.reservation().try_add(2 * charge)); + } fn build_prepare(op: u64, size: usize) -> Message { Message::::new(size).transmute_header(|_, h: &mut PrepareHeader| { @@ -1730,6 +2462,123 @@ mod tests { assert!(journal.repair_entry(8).is_none()); } + /// A body that no segment holds stays in memory while a newer body leaves + /// it, so the window that memory holds up to the head starts above the + /// newest released op. + #[compio::test] + async fn given_an_unlocated_body_below_a_released_one_when_asked_for_the_memory_window_should_start_above_the_release() + { + const PAYLOAD_LEN: usize = 64; + let journal = PartitionJournal::::default(); + let body_len = build_message_prepare(1, 0, 1, PAYLOAD_LEN).len() as u64; + journal.set_ring_caps(EVICTED_RING_CAPACITY, 2 * body_len); + for op in 1..=3 { + journal + .append(build_message_prepare(op, op - 1, 1, PAYLOAD_LEN)) + .await + .unwrap(); + } + journal.evict_prefix_located(3, |op| { + (op == 2).then_some(BodyPosition { + segment_start: 0, + position: 0, + }) + }); + assert!(journal.repair_entry(1).is_some()); + assert!(journal.evicted_header(2).is_some()); + assert_eq!(journal.repair_retained_from(), Some(1)); + assert_eq!(journal.memory_retained_from(), Some(3)); + } + + /// Released headers take no room from the bodies the byte cap keeps in + /// memory, and a newest body past the cap takes no header away. + #[compio::test] + async fn given_released_headers_when_bodies_pass_the_byte_cap_should_keep_both_windows() { + const OPS: u64 = 20; + const BODIES_KEPT: u64 = 3; + const PAYLOAD_LEN: usize = 64; + let located = |op| { + Some(BodyPosition { + segment_start: 0, + position: op, + }) + }; + let journal = PartitionJournal::::default(); + let body_len = build_message_prepare(1, 0, 1, PAYLOAD_LEN).len() as u64; + journal.set_ring_caps(EVICTED_RING_CAPACITY, BODIES_KEPT * body_len); + for op in 1..=OPS { + journal + .append(build_message_prepare(op, op - 1, 1, PAYLOAD_LEN)) + .await + .unwrap(); + } + journal.evict_prefix_located(usize::try_from(OPS).unwrap(), located); + for op in 1..=OPS - BODIES_KEPT { + assert!( + journal.evicted_header(op).is_some(), + "op {op} lost its header" + ); + } + for op in OPS - BODIES_KEPT + 1..=OPS { + assert!(journal.repair_entry(op).is_some(), "op {op} lost its body"); + } + + let large = build_message_prepare(OPS + 1, OPS, 1, PAYLOAD_LEN * 64); + assert!(large.len() as u64 > BODIES_KEPT * body_len); + journal.append(large).await.unwrap(); + journal.evict_prefix_located(1, located); + assert_eq!(journal.repair_retained_from(), Some(1)); + for op in 1..=OPS { + assert!( + journal.evicted_header(op).is_some(), + "op {op} lost its header" + ); + } + assert!(journal.repair_entry(OPS + 1).is_some()); + } + + /// Each release makes room in the header window first, so a full window + /// never grows its allocation past its cap. + #[compio::test] + async fn given_more_releases_than_the_header_cap_when_trimming_should_keep_the_deque_within_it() + { + const BATCH: u64 = 1024; + let headers_max = EVICTED_HEADERS_MAX; + let ops = u64::try_from(headers_max).unwrap() + 2; + let journal = PartitionJournal::::default(); + journal.set_ring_caps(EVICTED_RING_CAPACITY, 1); + let template = build_message_prepare(1, 0, 1, 1); + for first in (1..=ops).step_by(usize::try_from(BATCH).unwrap()) { + let last = (first + BATCH - 1).min(ops); + for op in first..=last { + let mut prepare = Message::::new(template.len()); + prepare.as_mut_slice().copy_from_slice(template.as_slice()); + let prepare = prepare.transmute_header(|header, stamped: &mut PrepareHeader| { + *stamped = header; + stamped.op = op; + }); + journal.append(prepare.into_frozen()).await.unwrap(); + } + journal.evict_prefix_located(usize::try_from(last - first + 1).unwrap(), |op| { + Some(BodyPosition { + segment_start: 0, + position: op, + }) + }); + } + let headers = unsafe { &*journal.evicted_headers.get() }; + assert!( + headers.capacity() <= headers_max, + "the header deque grew to {} slots", + headers.capacity() + ); + assert_eq!(headers.len(), headers_max); + assert_eq!( + journal.repair_retained_from(), + Some(ops - u64::try_from(headers_max).unwrap()) + ); + } + #[compio::test] async fn repair_headers_in_serves_the_commit_point_from_the_evicted_ring() { // Blank AT the commit point is the one slot a merge can neither adopt nor diff --git a/core/partitions/src/lib.rs b/core/partitions/src/lib.rs index 1ead193566..b1e14ccf08 100644 --- a/core/partitions/src/lib.rs +++ b/core/partitions/src/lib.rs @@ -33,11 +33,13 @@ pub mod offset_storage; mod partition_storage; mod persistence; mod poll_plan; +mod repair_memory; pub use persistence::FileSyncBarrier; pub(crate) use persistence::PersistenceDrain; pub use persistence::{ PartitionPersistence, PersistenceCompletion, PersistenceMetrics, PersistenceNotifier, }; +pub use repair_memory::RepairMemoryBudget; mod segment; pub mod segment_anchor; mod segment_recovery; @@ -70,7 +72,10 @@ pub use io::{ PartitionIoResult, PartitionIoStep, PartitionTeardown, PreparedPrimaryTransfer, largest_legal_job_charge, }; -pub use journal::{EVICTED_RING_BYTES_MAX, EVICTED_RING_CAPACITY}; +pub use journal::{ + BodyPosition, EVICTED_RING_BYTES_MAX, EVICTED_RING_CAPACITY, EvictedHeader, + read_released_bodies, +}; pub use view_fence::ViewWindowVerdict; /// Offsets a partition claims in its superblock ahead of the mint counter diff --git a/core/partitions/src/log.rs b/core/partitions/src/log.rs index da36addce0..ed8bb58755 100644 --- a/core/partitions/src/log.rs +++ b/core/partitions/src/log.rs @@ -351,6 +351,16 @@ where &self.storage } + /// Path of the log file of the segment that starts at `start_offset`. + #[must_use] + pub fn messages_path(&self, start_offset: u64) -> Option { + let slot = self + .segments + .binary_search_by_key(&start_offset, |segment| segment.start_offset) + .ok()?; + self.storage.get(slot)?.segment_and_index_paths().0 + } + pub const fn messages_writers(&self) -> &Vec>> { &self.messages_writers } diff --git a/core/partitions/src/persistence.rs b/core/partitions/src/persistence.rs index 064abbf86a..2d4f093b1d 100644 --- a/core/partitions/src/persistence.rs +++ b/core/partitions/src/persistence.rs @@ -686,6 +686,12 @@ impl PartitionPersistence { !self.retired.get() && self.failure.borrow().is_none() && op <= self.written_head.get() } + /// Where the WAL wrote the body of `op` in a segment, while the WAL still + /// holds the op. + pub fn segment_reference(&self, op: u64) -> Option { + self.segment_references.borrow().get(&op).copied() + } + pub const fn segment_checkpoint(&self) -> Option { self.segment_checkpoint.get() } @@ -1133,6 +1139,11 @@ impl PartitionPersistence { self.accepted_head.get() } + /// The WAL byte budget, `wal_bytes_max` in the server configuration. + pub const fn capacity(&self) -> u64 { + self.capacity + } + pub fn retire(&self) { self.retired.set(true); if let Some(lease) = &self.lease { diff --git a/core/partitions/src/repair_memory.rs b/core/partitions/src/repair_memory.rs new file mode 100644 index 0000000000..dd4dedb29e --- /dev/null +++ b/core/partitions/src/repair_memory.rs @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// Allocation budget shared by unproven repairs and snapshot repair tails. +/// +/// Reservations precede journaling and last until the proof finishes or the +/// retained window is discarded. Normal prepare queues have separate limits. +#[derive(Debug)] +pub struct RepairMemoryBudget { + bytes_max: usize, + used: AtomicUsize, +} + +impl RepairMemoryBudget { + #[must_use] + pub const fn new(bytes_max: usize) -> Self { + Self { + bytes_max, + used: AtomicUsize::new(0), + } + } + + pub(crate) fn reservation(self: &Arc) -> RepairMemoryReservation { + RepairMemoryReservation { + budget: Arc::clone(self), + bytes: 0, + exhausted: false, + } + } +} + +pub struct RepairMemoryReservation { + budget: Arc, + bytes: usize, + exhausted: bool, +} + +impl RepairMemoryReservation { + pub(crate) fn sibling(&self) -> Self { + self.budget.reservation() + } + + pub(crate) const fn bytes(&self) -> usize { + self.bytes + } + + pub(crate) fn try_add(&mut self, bytes: usize) -> bool { + if self.exhausted { + return false; + } + if bytes == 0 { + return true; + } + if self + .budget + .used + .try_update(Ordering::Relaxed, Ordering::Relaxed, |used| { + used.checked_add(bytes) + .filter(|next| *next <= self.budget.bytes_max) + }) + .is_err() + { + self.exhausted = true; + return false; + } + self.bytes += bytes; + true + } + + pub(crate) fn release(&mut self, bytes: usize) { + if bytes == 0 { + return; + } + self.bytes -= bytes; + self.budget.used.fetch_sub(bytes, Ordering::Relaxed); + } + + pub(crate) const fn exhausted(&self) -> bool { + self.exhausted + } + + pub(crate) fn clear(&mut self) { + self.release(self.bytes); + self.exhausted = false; + } +} + +impl Drop for RepairMemoryReservation { + fn drop(&mut self) { + self.clear(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Barrier; + use std::thread; + + #[test] + fn given_concurrent_shards_when_reserving_should_share_one_limit_and_release_on_drop() { + const SHARDS: usize = 8; + const CHARGE: usize = 1024; + const CAPACITY: usize = 3 * CHARGE; + let budget = Arc::new(RepairMemoryBudget::new(CAPACITY)); + let barrier = Arc::new(Barrier::new(SHARDS)); + let accepted = AtomicUsize::new(0); + thread::scope(|scope| { + for _ in 0..SHARDS { + scope.spawn(|| { + let mut reservation = budget.reservation(); + barrier.wait(); + if reservation.try_add(CHARGE) { + accepted.fetch_add(1, Ordering::Relaxed); + } + barrier.wait(); + }); + } + }); + assert_eq!(accepted.load(Ordering::Relaxed), CAPACITY / CHARGE); + assert_eq!(budget.used.load(Ordering::Relaxed), 0); + assert!(budget.reservation().try_add(CAPACITY)); + } +} diff --git a/core/partitions/src/state_transfer.rs b/core/partitions/src/state_transfer.rs index e19bc128c4..577232f8a8 100644 --- a/core/partitions/src/state_transfer.rs +++ b/core/partitions/src/state_transfer.rs @@ -29,6 +29,8 @@ use crate::IggyPartition; use crate::iggy_partition::{InstalledConsumerGroupOwner, InstalledPartitionHistory}; +use crate::io::PartitionIoQuiescence; +use crate::journal::RepairTailLease; use crate::offset_storage::{discard_offset_replacement, stage_offset_replacement}; use crate::partition_storage::{read_revision_record, write_revision_record}; use crate::segment_anchor::ANCHOR_SUFFIX; @@ -58,13 +60,18 @@ use server_common::iobuf::Owned; use server_common::send_messages::{decode_batch_slice, decode_prepare_slice}; use server_common::sharding::IggyNamespace; use server_common::yield_to_reactor; -use std::collections::{BTreeSet, HashSet}; +use std::cell::{Cell, RefCell}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::fmt; use std::future::Future; use std::mem::size_of; use std::path::{Path, PathBuf}; use std::rc::Rc; use std::sync::atomic::Ordering; +use std::task::Poll; + +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; /// Current state-transfer offsets format, including the prepare-chain anchor. pub(crate) const CONSUMER_OFFSETS_MAGIC: [u8; 4] = *b"ICO1"; @@ -135,9 +142,211 @@ pub struct PartitionTransferSession { /// Whether a descriptor has been accepted (an accepted EMPTY manifest is /// distinguishable from "still waiting"). pub target_accepted: bool, - /// Ticks with no frame progress; at the configured repair-retry - /// threshold the missing piece is re-requested. + /// Ticks since the last frame progress or chunk request. At the + /// configured repair-retry threshold, the missing piece is requested + /// again. pub idle_ticks: u32, + /// The reuse scan of the accepted manifest. The pull waits for it, + /// because the scan writes the staging names that the pull spills to. + pub scan: Option>, + /// The completed segment being validated and synced outside the pump. + pub spill: Option, + /// The leading bytes of one segment artifact that this replica already + /// holds, so the pull asks only for the rest. + pub seed: Option, + /// Chunk payload bytes that crossed the network in this session. + pub pulled_len: u64, +} + +/// The local copy of the leading bytes of one offered segment artifact. +/// +/// A lagging replica holds a prefix of the segment the primary is still +/// writing: the same base offset and fewer bytes. The pull then fetches only +/// the suffix. The spill checks the whole artifact against the manifest +/// checksum, so a prefix with other bytes costs one more pull of that +/// artifact and nothing else. +#[derive(Debug)] +pub enum PrefixSeed { + /// Chosen when the reuse scan lands. The read starts when the pull + /// reaches `artifact`, so the receiver holds one artifact buffer at a + /// time. + Planned { + artifact: u32, + log_path: String, + len: u64, + }, + /// A detached task reads, hashes and walks the prefix. + Reading { + artifact: u32, + read: LocalWork, + }, + /// The prefix bytes sit in the artifact buffer. + Seeded { artifact: u32, prefix: SeededPrefix }, +} + +/// Work that a detached task runs for a transfer session. +/// +/// A prefix read and a reuse scan are such work. The pump does not wait for +/// it: the task fills `result` and wakes the partition. Dropping this state +/// asks the task to stop. +#[derive(Debug)] +pub struct LocalWork { + pub result: Rc>>, + pub cancel: LocalWorkCancel, +} + +impl Default for LocalWork { + fn default() -> Self { + Self { + result: Rc::default(), + cancel: LocalWorkCancel::default(), + } + } +} + +impl LocalWork { + /// Whether the task finished, so the session can go on. + #[must_use] + pub fn landed(&self) -> bool { + self.result.borrow().is_some() + } + + /// Asks the task to stop at its next check. Returns `false` when it was + /// asked before. + #[must_use] + pub fn stop(&self) -> bool { + !self.cancel.0.replace(true) + } +} + +/// Asks local work to stop when the state that waits for it drops. +/// +/// That state drops with its session, or when its wait passes its bound. The +/// task checks the flag between its steps. A prefix read then frees its +/// buffer of up to one segment early, instead of holding it next to the +/// buffer of the next read. +#[derive(Debug, Default)] +pub struct LocalWorkCancel(Rc>); + +impl Drop for LocalWorkCancel { + fn drop(&mut self) { + self.0.set(true); + } +} + +/// What a reuse scan staged without a pull. +#[derive(Debug, Default)] +pub struct ReuseScanned { + /// The manifest index and the staged meta of each segment that the scan + /// linked or adopted. + pub reused: Vec<(u32, StagedSegmentMeta)>, + /// Verified files that a cancelled scan did not revisit. + pub(crate) memoized: BTreeMap, +} + +#[derive(Debug)] +/// Owns the completed artifact while the pull waits for its staging writer. +pub struct SegmentSpill { + pub artifact: usize, + pub seeded: bool, + pub work: LocalWork>, +} + +/// A durable staging pair and the checksum state needed if it becomes active. +pub struct SpilledSegment { + meta: StagedSegmentMeta, + hasher: SpilledHasher, +} + +impl fmt::Debug for SpilledSegment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SpilledSegment") + .field("meta", &self.meta) + .finish_non_exhaustive() + } +} + +/// What a landed reuse scan staged, for the log of the receiver. +#[derive(Debug, Default)] +pub struct ReusedSegments { + /// Staged copies of an earlier attempt. + pub adopted: usize, + pub adopted_len: u64, + /// Sealed segments that this replica held. + pub linked: usize, + pub linked_len: u64, + pub artifacts: usize, + pub total_len: u64, +} + +impl ReusedSegments { + /// `artifacts` are the slots of the session before the scan staged any. + fn new(artifacts: &[TransferArtifact], reused: &[(u32, StagedSegmentMeta)]) -> Self { + let mut segments = Self { + artifacts: artifacts.len(), + total_len: artifacts + .iter() + .filter_map(TransferArtifact::pending) + .map(|progress| progress.entry.len) + .sum(), + ..Self::default() + }; + for (_, meta) in reused { + if meta.linked { + segments.linked += 1; + segments.linked_len += meta.size; + } else { + segments.adopted += 1; + segments.adopted_len += meta.size; + } + } + segments + } +} + +impl PrefixSeed { + const fn artifact(&self) -> u32 { + match self { + Self::Planned { artifact, .. } + | Self::Reading { artifact, .. } + | Self::Seeded { artifact, .. } => *artifact, + } + } + + /// Whether the prefix read finished and the pull can continue. + #[must_use] + pub fn landed(&self) -> bool { + matches!(self, Self::Reading { read, .. } if read.landed()) + } +} + +/// The prefix bytes of a prefix read, with the hash and walk state after them. +type PrefixRead = std::io::Result<(Vec, SeededPrefix)>; + +/// The hash and walk state after the first `len` bytes of a segment artifact, +/// which the spill continues over the pulled suffix. +pub struct SeededPrefix { + len: u64, + hasher: Box, + /// `None` when the prefix failed its walk. The spill then walks the whole + /// artifact, once the checksum proved that the bytes are the offered ones. + walk: Option, +} + +impl fmt::Debug for SeededPrefix { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SeededPrefix") + .field("len", &self.len) + .finish_non_exhaustive() + } +} + +/// The checksum state of a spilled segment, which can extend as the segment +/// grows. +pub(crate) struct SpilledHasher { + start_offset: u64, + size: u64, + hasher: Box, } /// A scheduled transfer re-arm (see `IggyPartition::transfer_rearm`). @@ -238,14 +447,18 @@ pub struct StagedSegmentMeta { /// `.staging` extension is invisible to boot recovery, which filters on /// `extension == "log"`. pub log_staging: PathBuf, - /// The locally rebuilt sparse index for the staged log, one entry per - /// batch (denser than the origin's per-flush-chunk index; recovery is - /// sparse-tolerant either way). + /// The sparse index rebuilt from the verified log at one entry per + /// `INDEX_STRIDE_BYTES` of payload. pub index_staging: PathBuf, + /// The manifest checksum of the staged log, which the install keeps as + /// the stamp of the installed segment. + pub checksum: u64, + /// The staged log is a hard link to a sealed segment that this replica + /// held, so the transfer neither pulled nor rewrote its payload. + pub linked: bool, } -/// Memoized streaming checksum state for one segment file: the hasher fed -/// exactly `hashed_len` of its bytes, plus the stamp at that length. +/// Memoized checksum state for one segment file. /// /// Keyed per segment rather than per `(base offset, size)` pair so the ACTIVE /// segment extends its own hasher as it grows, instead of missing the memo on @@ -253,35 +466,51 @@ pub struct StagedSegmentMeta { /// new file at an existing base offset (install, converge) clears the whole /// map, which is what makes "same base offset, longer file, same leading bytes" /// hold. -pub(crate) struct SegmentChecksumMemo { - hashed_len: u64, - /// The stamp is NOT cached alongside: `StateArtifactHasher::finish` takes - /// `&self`, so it is a read of this hasher, and a second copy is just a - /// field that can drift. - hasher: StateArtifactHasher, +/// +/// Three paths fill it. The append feeds the bytes of every chunk it +/// materializes, so a segment that this process wrote from its first byte +/// needs no read. The install stamps each segment it installs with the +/// checksum it verified. The offer reads a file only for the bytes that no +/// memo covers, which after a restart is every segment of the first offer. +pub(crate) enum SegmentChecksumMemo { + /// The hasher fed exactly `hashed_len` bytes of the file. A read or an + /// append extends it. + Hashing { + hashed_len: u64, + /// The stamp is NOT cached alongside: `StateArtifactHasher::finish` + /// takes `&self`, so it is a read of this hasher, and a second copy + /// is just a field that can drift. Boxed, because the hasher state is + /// hundreds of bytes and every stamp in the map would pay that size. + hasher: Box, + }, + /// The checksum of the first `len` bytes, as an install verified them or + /// as the hasher of a sealed segment ended. Nothing can extend it, because + /// it holds no hasher. An append drops it, and a read starts again from + /// byte zero. + Stamped { len: u64, checksum: u64 }, } impl SegmentChecksumMemo { - fn new() -> Self { - Self { - hashed_len: 0, - hasher: StateArtifactHasher::new(), + /// The artifact stamp over the first `len` bytes, if the memo ends there. + pub(crate) fn checksum_of(&self, len: u64) -> Option { + match self { + Self::Hashing { hashed_len, hasher } => (*hashed_len == len).then(|| hasher.finish()), + Self::Stamped { + len: stamped_len, + checksum, + } => (*stamped_len == len).then_some(*checksum), } } } -/// The result of the last staged-segment reuse scan. +/// Verified staging files, keyed by segment start offset. /// -/// A scan reads every length-matching staged file whole, verifies it, and -/// re-walks every batch -- sequentially, on the pump. Peer rotation and stall -/// re-arms mint a fresh session against the same segment set, so without this -/// the full cost is re-paid per arm. `digest` covers every `SEGMENT_LOG` -/// manifest entry, so a hit means the new offer expects byte-identical staged -/// files; any write to a staging file, or any unlink of one, drops the memo, so -/// a hit can never describe bytes that were replaced meanwhile. +/// Partial scans retain their completed files across retries and advancing +/// offers. Each hit must match the offered length and checksum. Replacing a +/// staging file invalidates its entry; installs and convergence clear the map. +/// Live segments are verified and their indexes rebuilt before entering it. pub(crate) struct ReuseScanMemo { - digest: u64, - adopted: Vec<(u32, StagedSegmentMeta)>, + adopted: BTreeMap, } impl StagedSegmentMeta { @@ -305,6 +534,8 @@ impl StagedSegmentMeta { max_timestamp: stats.max_timestamp, log_staging, index_staging, + checksum: entry.checksum, + linked: false, } } } @@ -909,6 +1140,35 @@ mod tests { use super::*; use iggy_binary_protocol::primitives::partition_history::ConsumerGroupOwner; + #[test] + fn given_a_prefix_read_that_stops_without_a_result_should_land_an_error_and_wake() { + let result = Rc::new(RefCell::new(None)); + let woken = Rc::new(Cell::new(false)); + let io = Rc::new(PartitionIoQuiescence::default()); + assert!(io.begin_local_work()); + let landing = Landing { + io: Rc::clone(&io), + result: Rc::clone(&result), + wake: { + let woken = Rc::clone(&woken); + move || woken.set(true) + }, + stopped: prefix_read_stopped, + }; + + drop(landing); + + assert!( + matches!(result.borrow().as_ref(), Some(Err(_))), + "the pull must see a failed read and fetch the whole artifact" + ); + assert!(woken.get(), "the partition must resume the pull"); + assert!( + io.is_interrupted(), + "teardown must keep interrupted I/O fenced" + ); + } + #[test] fn given_segment_read_error_when_classifying_should_tell_local_faults_from_stale_offers() { const EIO: i32 = 5; @@ -1712,114 +1972,150 @@ pub(crate) async fn walk_segment_payload( base_offset: u64, bytes: &[u8], ) -> Result<(SegmentWalkStats, Vec), SegmentWalkError> { - let mut position = 0usize; - let mut next_offset = base_offset; - let mut stats: Option = None; - let mut index_bytes = Vec::new(); - let mut indexed_position: Option = None; - let mut since_yield = 0usize; - while position < bytes.len() { - // The walk re-hashes every message (`decode_batch_slice` verifies - // `batch_checksum`), so a multi-GiB artifact is a long CPU pass on the - // pump task. What these yields buy is NOT tick liveness: the consensus - // tick is a sibling `select_biased!` arm of this same task and arms are - // not polled while another arm's body awaits, so every group's tick and - // heartbeat on this shard stay frozen for the duration either way (see - // the tick-starvation TODO in `shard::router`). They buy the reactor: - // detached tasks and io_uring completions make progress instead of - // waiting out the whole pass. Moving the verify + walk off the pump is - // what would fix the tick, and the nonce re-check after the spill is - // already shaped for that. - if since_yield >= OFFER_HASH_CHUNK_LEN { - since_yield = 0; - yield_to_reactor().await; - } - let batch = - decode_batch_slice(&bytes[position..]).map_err(|source| SegmentWalkError::Batch { - position: position as u64, - source, - })?; - let header = batch.header; - if stats.is_none() && header.base_offset != base_offset { - return Err(SegmentWalkError::BaseOffsetMismatch { - expected: base_offset, - actual: header.base_offset, - }); - } - if header.base_offset != next_offset { - return Err(SegmentWalkError::NonContiguous { - expected: next_offset, - actual: header.base_offset, - }); - } - if header.message_count == 0 { - return Err(SegmentWalkError::Batch { - position: position as u64, - source: iggy_common::IggyError::InvalidMessagesCount, - }); + let mut walk = SegmentWalk::new(base_offset); + walk.advance(bytes).await?; + walk.finish() +} + +/// A walk that stops at the end of the bytes it was given and resumes there, +/// so a seeded prefix and the suffix that completes it are each walked once. +pub(crate) struct SegmentWalk { + base_offset: u64, + /// Payload bytes walked so far. + position: usize, + next_offset: u64, + stats: Option, + index_bytes: Vec, + indexed_position: Option, +} + +impl SegmentWalk { + pub(crate) const fn new(base_offset: u64) -> Self { + Self { + base_offset, + position: 0, + next_offset: base_offset, + stats: None, + index_bytes: Vec::new(), + indexed_position: None, } - // Peer-controlled operands under a reject-on-first-invalid contract: - // checked, not saturating -- a clamp would misdirect the diagnostic. - let Some(batch_end) = header - .base_offset - .checked_add(u64::from(header.message_count) - 1) - else { - return Err(SegmentWalkError::OffsetOverflow { - position: position as u64, - }); - }; - // The append-time canonical stamp, exactly what the flush path writes - // into index entries and segment bounds; `origin_timestamp` is - // client-supplied and would give the installed replica a divergent - // timestamp column (polls and retention keyed differently per node). - let timestamp = header.base_timestamp; - // STRIDED, not one entry per batch: the origin writes one entry per - // flush chunk, and a per-batch index is dense enough that a transferred - // segment never fits the sealed-index residency cap - // (`poll_plan::SEALED_INDEX_RESIDENT_MAX_BYTES`), so every sealed poll - // would fall back to binary-searching the file with single-entry preads - // -- a slow path `poll_plan` reserves for a - // `messages_required_to_save = 1` misconfiguration, which a transfer - // would otherwise produce unconditionally. Both consumers do lower-bound - // lookups and recovery walks forward from the last entry by design, so - // sparser is correct; the first batch always gets one. - let stride_reached = indexed_position - .is_none_or(|indexed| position.saturating_sub(indexed) >= INDEX_STRIDE_BYTES); - if stride_reached { - indexed_position = Some(position); - index_bytes.extend_from_slice(&header.base_offset.to_le_bytes()); - index_bytes.extend_from_slice(×tamp.to_le_bytes()); - index_bytes.extend_from_slice(&(position as u64).to_le_bytes()); - } - stats = Some(stats.map_or( - SegmentWalkStats { - end_offset: batch_end, - start_timestamp: timestamp, - end_timestamp: timestamp, - max_timestamp: timestamp, - }, - |previous| SegmentWalkStats { - end_offset: batch_end, - start_timestamp: previous.start_timestamp, - end_timestamp: timestamp, - max_timestamp: previous.max_timestamp.max(timestamp), - }, - )); - next_offset = batch_end - .checked_add(1) - .ok_or(SegmentWalkError::OffsetOverflow { - position: position as u64, + } + + /// Walk `bytes`, the payload from its first byte, from where the last + /// call stopped to its end. + async fn advance(&mut self, bytes: &[u8]) -> Result<(), SegmentWalkError> { + let mut since_yield = 0usize; + while self.position < bytes.len() { + // The walk re-hashes every message (`decode_batch_slice` verifies + // `batch_checksum`), so a multi-GiB artifact is a long CPU pass. On the + // pump task, what these yields buy is NOT tick liveness: the consensus + // tick is a sibling `select_biased!` arm of this same task and arms are + // not polled while another arm's body awaits, so every group's tick and + // heartbeat on this shard stay frozen for the duration either way (see + // the tick-starvation TODO in `shard::router`). They buy the reactor: + // detached tasks and io_uring completions make progress instead of + // waiting out the whole pass. In a detached task, such as the prefix + // seed read, the same yields let the pump run. + if since_yield >= OFFER_HASH_CHUNK_LEN { + since_yield = 0; + yield_to_reactor().await; + } + let batch = decode_batch_slice(&bytes[self.position..]).map_err(|source| { + SegmentWalkError::Batch { + position: self.position as u64, + source, + } })?; - // No trailing-bytes guard: the header decode floors `batch_length` - // at the 256-byte command header (no zero-step loop is possible) - // and `decode_batch_slice` already rejects a body shorter than - // `total_size()`. - position += header.total_size(); - since_yield += header.total_size(); - } - stats.map_or(Err(SegmentWalkError::Empty), |stats| { - Ok((stats, index_bytes)) - }) + let header = batch.header; + if self.stats.is_none() && header.base_offset != self.base_offset { + return Err(SegmentWalkError::BaseOffsetMismatch { + expected: self.base_offset, + actual: header.base_offset, + }); + } + if header.base_offset != self.next_offset { + return Err(SegmentWalkError::NonContiguous { + expected: self.next_offset, + actual: header.base_offset, + }); + } + if header.message_count == 0 { + return Err(SegmentWalkError::Batch { + position: self.position as u64, + source: iggy_common::IggyError::InvalidMessagesCount, + }); + } + // Peer-controlled operands under a reject-on-first-invalid contract: + // checked, not saturating -- a clamp would misdirect the diagnostic. + let Some(batch_end) = header + .base_offset + .checked_add(u64::from(header.message_count) - 1) + else { + return Err(SegmentWalkError::OffsetOverflow { + position: self.position as u64, + }); + }; + // The append-time canonical stamp, exactly what the flush path writes + // into index entries and segment bounds; `origin_timestamp` is + // client-supplied and would give the installed replica a divergent + // timestamp column (polls and retention keyed differently per node). + let timestamp = header.base_timestamp; + // STRIDED, not one entry per batch: the origin writes one entry per + // flush chunk, and a per-batch index is dense enough that a transferred + // segment never fits the sealed-index residency cap + // (`poll_plan::SEALED_INDEX_RESIDENT_MAX_BYTES`), so every sealed poll + // would fall back to binary-searching the file with single-entry preads + // -- a slow path `poll_plan` reserves for a + // `messages_required_to_save = 1` misconfiguration, which a transfer + // would otherwise produce unconditionally. Both consumers do lower-bound + // lookups and recovery walks forward from the last entry by design, so + // sparser is correct; the first batch always gets one. + let stride_reached = self + .indexed_position + .is_none_or(|indexed| self.position.saturating_sub(indexed) >= INDEX_STRIDE_BYTES); + if stride_reached { + self.indexed_position = Some(self.position); + self.index_bytes + .extend_from_slice(&header.base_offset.to_le_bytes()); + self.index_bytes.extend_from_slice(×tamp.to_le_bytes()); + self.index_bytes + .extend_from_slice(&(self.position as u64).to_le_bytes()); + } + self.stats = Some(self.stats.map_or( + SegmentWalkStats { + end_offset: batch_end, + start_timestamp: timestamp, + end_timestamp: timestamp, + max_timestamp: timestamp, + }, + |previous| SegmentWalkStats { + end_offset: batch_end, + start_timestamp: previous.start_timestamp, + end_timestamp: timestamp, + max_timestamp: previous.max_timestamp.max(timestamp), + }, + )); + self.next_offset = + batch_end + .checked_add(1) + .ok_or(SegmentWalkError::OffsetOverflow { + position: self.position as u64, + })?; + // No trailing-bytes guard: the header decode floors `batch_length` + // at the 256-byte command header (no zero-step loop is possible) + // and `decode_batch_slice` already rejects a body shorter than + // `total_size()`. + self.position += header.total_size(); + since_yield += header.total_size(); + } + Ok(()) + } + + fn finish(self) -> Result<(SegmentWalkStats, Vec), SegmentWalkError> { + self.stats.map_or(Err(SegmentWalkError::Empty), |stats| { + Ok((stats, self.index_bytes)) + }) + } } /// One offered segment: its manifest entry plus WHERE its bytes live. @@ -1900,11 +2196,23 @@ impl PartitionStateTransferOffer { /// these lengths, so hash rounds that span them still describe `commit_op`. pub(crate) struct TransferPlan { commit_op: u64, - /// `(start_offset, len, log_path)` of each non-empty segment. - segments: Vec<(u64, u64, String)>, + /// Each non-empty segment, in chain order. + segments: Vec, offsets: ConsumerOffsetsWire, } +/// One segment as a plan captured it. +pub(crate) struct PlannedSegment { + start_offset: u64, + len: u64, + log_path: String, + /// The artifact stamp over `len` bytes once known. The pin takes it from + /// the memo, or the round whose read completes the segment records it. + /// It is kept here because appends after the pin move the memo of the + /// active segment past `len`, so a later round cannot read it there. + checksum: Cell>, +} + /// How far a serving partition got toward the plan its offer rounds share. pub(crate) enum TransferPlanSlot { /// A request waits for the flush that has to come before the capture. @@ -1921,6 +2229,7 @@ pub(crate) enum TransferPlanSlot { #[derive(Debug)] pub enum PartitionTransferUnavailable { NotCaughtUpPrimary, + RepairTailExpired, /// In-memory / simulated partition: nothing on disk to serve. NoPartitionDir, RepairInProgress, @@ -1984,6 +2293,7 @@ impl PartitionTransferUnavailable { | Self::RepairInProgress | Self::NothingCommitted | Self::SegmentSetChanged + | Self::RepairTailExpired | Self::FlushPending | Self::OfferBuildInProgress { .. } => true, Self::NoPartitionDir @@ -2008,6 +2318,7 @@ impl fmt::Display for PartitionTransferUnavailable { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::NotCaughtUpPrimary => write!(f, "not the caught-up primary of this group"), + Self::RepairTailExpired => write!(f, "the snapshot's retained repair tail expired"), Self::MissingPrepareChecksum { op } => { write!(f, "partition has no checksum for committed op {op}") } @@ -2076,12 +2387,18 @@ pub enum PartitionInstallError { kind: ConsumerKind, }, /// `commit_op` fell below this replica's commit frontier; installing - /// would rewind `commit_min` (the anti-rewind assert, as a refusal) or - /// erase a committed op this replica holds. + /// would rewind `commit_min` (the anti-rewind assert, as a refusal). StaleTransfer { commit_op: u64, commit_min: u64, }, + /// The install wipes the journal, which holds an op above `commit_op` that + /// this replica may have acknowledged: one at or below `held_max`, its + /// accepted WAL head, or the known commit without a WAL. + OfferErasesHeldOps { + commit_op: u64, + held_max: u64, + }, /// The incoming frontier could not be made durable before the swap, so the /// install refuses rather than enter a window whose only durable witness /// would be the segments the failure path quarantines away. @@ -2171,6 +2488,14 @@ impl fmt::Display for PartitionInstallError { f, "transfer frontier {commit_op} is below the local commit frontier {commit_min}" ), + Self::OfferErasesHeldOps { + commit_op, + held_max, + } => write!( + f, + "transfer commit op {commit_op} would erase held ops up to {held_max} that \ + this replica may have acknowledged" + ), Self::FrontierNotDurable { frontier } => write!( f, "could not record the incoming offset frontier {frontier} before the swap" @@ -2231,6 +2556,11 @@ impl From for PartitionInstallError { /// `segment_recovery::sweep_scratch_files_and_collect_offsets`. pub const STAGING_SUFFIX: &str = ".staging"; +/// Extension of the scratch name that a link takes before its rename to the +/// staging name. It ends in `STAGING_SUFFIX`, so the staging sweeps and the +/// boot sweep remove a scratch name that a crash or a failed unlink left. +const LINK_SCRATCH_EXTENSION: &str = "link.staging"; + /// Staging-file names inside the partition directory. fn staging_paths(partition_dir: &str, start_offset: u64) -> (PathBuf, PathBuf) { ( @@ -2631,11 +2961,120 @@ pub(crate) async fn fsync_dir(partition_dir: &str) -> std::io::Result<()> { .await } +pub(crate) struct ServedRepairTail { + nonce: u128, + lease: Option, + last_active_tick: u64, +} + impl IggyPartition where B: MessageBus, SB: SuperblockStore, { + /// Pin before building the manifest: flushing and hashing can span several + /// rounds while writes keep evicting the suffix the receiver will need. + pub fn retain_transfer_tail(&self, peer: u8, nonce: u128, tick: u64) -> bool { + let Some(memory) = &self.proof_walk_memory else { + return true; + }; + let mut tails = self.served_repair_tails.borrow_mut(); + if let Some(tail) = tails.get_mut(&peer).filter(|tail| tail.nonce == nonce) { + tail.last_active_tick = tick; + return tail.lease.as_ref().is_some_and(RepairTailLease::is_valid); + } + let journal = &self.log.journal().inner; + let from_op = journal + .repair_retained_from() + .unwrap_or_else(|| self.consensus().commit_min().saturating_add(1)); + // The replacement must pin before the previous lease drops. + let lease = journal.retain_repair_from(from_op, memory.sibling()); + tails.insert( + peer, + ServedRepairTail { + nonce, + lease: Some(lease), + last_active_tick: tick, + }, + ); + true + } + + /// A descriptor is usable only while its complete post-snapshot suffix is + /// still retained. An exhausted or expired lease needs a new nonce. + pub fn transfer_tail_available(&self, peer: u8, nonce: u128, commit_op: u64) -> bool { + if self.proof_walk_memory.is_none() { + return true; + } + let tails = self.served_repair_tails.borrow(); + let Some(lease) = tails + .get(&peer) + .filter(|tail| tail.nonce == nonce) + .and_then(|tail| tail.lease.as_ref()) + .filter(|lease| lease.is_valid()) + else { + return false; + }; + let from_op = commit_op.saturating_add(1); + if self + .log + .journal() + .inner + .repair_retained_from() + .is_some_and(|oldest| oldest > from_op) + { + return false; + } + lease.advance(from_op); + true + } + + pub fn touch_transfer_tail(&self, peer: u8, nonce: u128, tick: u64) -> bool { + if self.proof_walk_memory.is_none() { + return true; + } + let mut tails = self.served_repair_tails.borrow_mut(); + let Some(tail) = tails.get_mut(&peer).filter(|tail| tail.nonce == nonce) else { + return false; + }; + tail.last_active_tick = tick; + tail.lease.as_ref().is_some_and(RepairTailLease::is_valid) + } + + /// Pull repair can precede a view-fenced progress reply. Its requests keep + /// the suffix alive without claiming that any prefix has been proven. + pub fn renew_transfer_tail_for_repair(&self, peer: u8, tick: u64) { + if let Some(tail) = self.served_repair_tails.borrow_mut().get_mut(&peer) { + tail.last_active_tick = tick; + } + } + + /// Only a correlated replication reply can release the acknowledged prefix. + /// Busy replies renew the lease while the receiver validates and installs. + pub fn advance_transfer_tail( + &self, + peer: u8, + progress: consensus::ReplicationProgress, + tick: u64, + ) { + if let Some(tail) = self.served_repair_tails.borrow_mut().get_mut(&peer) { + tail.last_active_tick = tick; + if let Some(lease) = &tail.lease + && let consensus::ReplicationProgress::Ready { received_op } = progress + { + lease.advance(received_op.saturating_add(1)); + } + } + } + + pub fn expire_transfer_tails(&self, tick: u64, idle_ticks_max: u64) { + for tail in self.served_repair_tails.borrow_mut().values_mut() { + if tick.saturating_sub(tail.last_active_tick) >= idle_ticks_max { + tail.lease = None; + } + } + } + fn plan_transfer_offset_writes( &self, offsets_wire: &ConsumerOffsetsWire, @@ -2696,21 +3135,26 @@ where offer } + /// Whether this replica serves its group's state transfer, which only the + /// primary does. An I/O owner pins the plan in a turn where the segments + /// end at `commit_min`, so a primary still applying what it committed can + /// serve. The inline flush runs through `commit_max` and needs the two + /// equal. + #[must_use] + pub fn serves_state_transfer(&self) -> bool { + if self.has_io_dispatcher() { + consensus::is_settled_primary(self.consensus()) + } else { + consensus::is_caught_up_primary(self.consensus()) + } + } + #[allow(clippy::too_many_lines)] async fn build_state_transfer_offer( &mut self, config: &PartitionsConfig, ) -> Result, PartitionTransferUnavailable> { - // An I/O owner pins the plan in a turn where the segments end at - // `commit_min`, so a primary still applying what it committed can - // serve. The inline flush runs through `commit_max` and needs the two - // equal. - let serving = if self.has_io_dispatcher() { - consensus::is_settled_primary(self.consensus()) - } else { - consensus::is_caught_up_primary(self.consensus()) - }; - if !serving { + if !self.serves_state_transfer() { return Err(PartitionTransferUnavailable::NotCaughtUpPrimary); } if self.partition_dir.is_none() { @@ -2787,37 +3231,44 @@ where // until the offer completes. let mut budget = OFFER_HASH_BUDGET_PER_ROUND_BYTES; let mut segments = Vec::with_capacity(planned.len()); - for (start_offset, size, log_path) in planned { - let Some(checksum) = self - .segment_checksum(*start_offset, *size, log_path, &mut budget) - .await? - else { - // CUMULATIVE across rounds, read back off the memo: per-round - // figures are constant by construction (a partial round always - // spends exactly the budget and always stops inside one - // segment), so they render identically on round 1 and round 30 - // and an operator cannot tell a converging pass from a wedged - // one. This is the only window onto a multi-round build. - let hashed = self.hashed_prefix_len(planned); - let total = planned.iter().map(|(_, size, _)| *size).sum::(); - // The completing round's sweep is skipped on this path, so - // prune here too: retention GC can unlink segments across a - // long build, and their memos would otherwise accumulate until - // some round finally runs the loop to the end. - self.retain_segment_checksum_memos(planned); - return Err(PartitionTransferUnavailable::OfferBuildInProgress { - hashed, - remaining: total.saturating_sub(hashed), - }); + for segment in planned { + let checksum = if let Some(checksum) = segment.checksum.get() { + checksum + } else { + let Some(checksum) = self + .segment_checksum( + segment.start_offset, + segment.len, + &segment.log_path, + &mut budget, + ) + .await? + else { + // CUMULATIVE across rounds, read back off the memo: + // per-round figures are constant by construction (a + // partial round always spends exactly the budget and + // always stops inside one segment), so they render + // identically on round 1 and round 30 and an operator + // cannot tell a converging pass from a wedged one. + // This is the only window onto a multi-round build. + let hashed = self.hashed_prefix_len(planned); + let total = planned.iter().map(|segment| segment.len).sum::(); + return Err(PartitionTransferUnavailable::OfferBuildInProgress { + hashed, + remaining: total.saturating_sub(hashed), + }); + }; + segment.checksum.set(Some(checksum)); + checksum }; segments.push(SegmentArtifactSource { entry: consensus::StateArtifact { kind: artifact_kind::SEGMENT_LOG, - frontier: *start_offset, - len: *size, + frontier: segment.start_offset, + len: segment.len, checksum, }, - log_path: log_path.clone(), + log_path: segment.log_path.clone(), }); } @@ -2837,28 +3288,25 @@ where .collect(); // Append-only within a segment instance, so a live size BELOW the // planned one means the file was replaced rather than extended. - let changed = planned.iter().any(|(start_offset, size, _)| { - live.get(start_offset) - .is_none_or(|live_size| live_size < size) + let changed = planned.iter().any(|segment| { + live.get(&segment.start_offset) + .is_none_or(|live_size| *live_size < segment.len) }); if changed { return Err(PartitionTransferUnavailable::SegmentSetChanged); } - // Sweep memo entries whose segment left the chain (GC), so the map - // tracks the live chain rather than growing with history. - self.segment_checksum_cache - .borrow_mut() - .retain(|start_offset, _| live.contains_key(start_offset)); } - // A completed hash can outlive the repair tail its snapshot needs. - // Re-pin while keeping the memos, so only newly appended bytes are hashed. + // A completed hash can outlive the repair tail its snapshot needs. A + // tail that only segments still hold counts as gone, because repair + // from disk is slow. Re-pin while keeping the memos, so only newly + // appended bytes are hashed. if self.consensus().commit_min() > plan.commit_op && self .log .journal() .inner - .repair_retained_from() + .memory_retained_from() .is_none_or(|retained_from| retained_from > plan.commit_op.saturating_add(1)) { *self.transfer_plan.borrow_mut() = Some(TransferPlanSlot::Wanted); @@ -2891,6 +3339,7 @@ where pub(crate) fn capture_transfer_plan( &self, ) -> Result { + let memos = self.segment_checksum_cache.borrow(); let mut segments = Vec::with_capacity(self.log.segments().len()); for (segment, storage) in self.log.segments().iter().zip(self.log.storages()) { let size = segment.size.as_bytes_u64(); @@ -2904,7 +3353,16 @@ where source: std::io::Error::other("segment holds bytes but no backing file"), }); }; - segments.push((segment.start_offset, size, log_path)); + segments.push(PlannedSegment { + start_offset: segment.start_offset, + len: size, + log_path, + checksum: Cell::new( + memos + .get(&segment.start_offset) + .and_then(|memo| memo.checksum_of(size)), + ), + }); } Ok(TransferPlan { commit_op: self.consensus().commit_min(), @@ -2913,37 +3371,51 @@ where }) } - /// Bytes of `planned` the memo already covers, clamped per segment to the - /// planned length so a memo carrying an active segment's later growth - /// cannot report more than this offer will hash. - fn hashed_prefix_len(&self, planned: &[(u64, u64, String)]) -> u64 { + /// Bytes of `planned` already hashed, clamped per segment to the planned + /// length so a memo carrying an active segment's later growth cannot + /// report more than this offer will hash. + fn hashed_prefix_len(&self, planned: &[PlannedSegment]) -> u64 { let memos = self.segment_checksum_cache.borrow(); planned .iter() - .map(|(start_offset, size, _)| { - memos - .get(start_offset) - .map_or(0, |memo| memo.hashed_len.min(*size)) + .map(|segment| { + if segment.checksum.get().is_some() { + return segment.len; + } + match memos.get(&segment.start_offset) { + Some(SegmentChecksumMemo::Hashing { hashed_len, .. }) => { + (*hashed_len).min(segment.len) + } + Some(SegmentChecksumMemo::Stamped { .. }) | None => 0, + } }) .sum() } - /// Drop memo entries whose segment is no longer in `planned`, which is the - /// live chain as of this round. - /// - /// Set-based rather than a scan per entry: this runs on every - /// budget-exhausted round, inside the frame body the budget exists to - /// bound, and `planned` is capped by `STATE_MANIFEST_ENTRIES_MAX` rather - /// than by anything an operator sized, so the quadratic form dominates the - /// hashing it was meant to make room for. - fn retain_segment_checksum_memos(&self, planned: &[(u64, u64, String)]) { - let live: HashSet = planned - .iter() - .map(|(start_offset, _, _)| *start_offset) - .collect(); - self.segment_checksum_cache - .borrow_mut() - .retain(|start_offset, _| live.contains(start_offset)); + /// Move the append's hasher into its file job. A shorter memo belongs to + /// an offer read and stays available for that read to finish. + pub(crate) fn take_segment_hasher(&self, chunk_start: u64) -> Option> { + if self.consensus().replica_count() == 1 { + return None; + } + self.partition_dir.as_ref()?; + let start_offset = self.log.active_segment().start_offset; + let mut memos = self.segment_checksum_cache.borrow_mut(); + match (chunk_start, memos.remove(&start_offset)) { + (0, _) => Some(Box::default()), + (_, Some(SegmentChecksumMemo::Hashing { hashed_len, hasher })) + if hashed_len == chunk_start => + { + Some(hasher) + } + (_, Some(memo @ SegmentChecksumMemo::Hashing { hashed_len, .. })) + if hashed_len < chunk_start => + { + memos.insert(start_offset, memo); + None + } + _ => None, + } } /// The artifact stamp over the first `size` bytes of a segment file, @@ -2975,12 +3447,16 @@ where .segment_checksum_cache .borrow_mut() .remove(&start_offset); - let mut memo = match memo { + let (hashed_len, mut hasher) = match memo { // `<=`, so the already-hashed case falls through to the shared tail: // `hash_segment_range` returns before opening the file when // `from == to`, and the finish + reinsert below is the same work the // separate arm did. - Some(memo) if memo.hashed_len <= size => memo, + Some(SegmentChecksumMemo::Hashing { hashed_len, hasher }) if hashed_len <= size => { + (hashed_len, hasher) + } + // A stamp holds no hasher to resume, so the read starts at byte 0. + Some(SegmentChecksumMemo::Stamped { .. }) => (0, Box::default()), // Segment bytes are append-only within one segment instance (the // failed-index-save path rewinds the writer cursor and returns // BEFORE the size increment), and every path that plants a fresh @@ -2991,34 +3467,60 @@ where shrunk.is_none(), "segment {start_offset} shrank to {size} bytes below its memo" ); - SegmentChecksumMemo::new() + (0, Box::default()) } }; // Clamped to the round's remaining budget, so a single multi-GiB // segment is split across rounds rather than being the granularity // floor. `finish` does not consume the hasher, so a partial pass is // simply a memo nobody stamps yet. - let target = size.min(memo.hashed_len.saturating_add(*budget)); - let hashed = target.saturating_sub(memo.hashed_len); + let target = size.min(hashed_len.saturating_add(*budget)); + let spent = target.saturating_sub(hashed_len); // Dropped on failure, not reinserted: the hasher is fed chunk by chunk // and a mid-range error leaves it holding bytes `hashed_len` does not // account for, so resuming from it would stamp a checksum over a // doubly-fed prefix. Losing the partial pass is the cheap side. - hash_segment_range(log_path, memo.hashed_len, target, &mut memo.hasher, None) + hash_segment_range(log_path, hashed_len, target, &mut hasher, None) .await .map_err(|source| PartitionTransferUnavailable::SegmentUnreadable { start_offset, source, })?; - memo.hashed_len = target; - let checksum = (target == size).then(|| memo.hasher.finish()); - self.segment_checksum_cache - .borrow_mut() - .insert(start_offset, memo); - *budget = budget.saturating_sub(hashed); + let checksum = (target == size).then(|| hasher.finish()); + self.segment_checksum_cache.borrow_mut().insert( + start_offset, + SegmentChecksumMemo::Hashing { + hashed_len: target, + hasher, + }, + ); + self.stamp_sealed_segment(start_offset); + *budget = budget.saturating_sub(spent); Ok(checksum) } + /// Replace the hasher of a sealed segment with its stamp once the memo + /// covers every byte. Nothing appends to a sealed segment, so its hasher + /// can never take another byte, and the stamp is 16 bytes where the + /// hasher is hundreds in two heap blocks. + pub(crate) fn stamp_sealed_segment(&self, start_offset: u64) { + let segments = self.log.segments(); + let Ok(position) = + segments.binary_search_by_key(&start_offset, |segment| segment.start_offset) + else { + return; + }; + let segment = &segments[position]; + let len = segment.size.as_bytes_u64(); + let mut memos = self.segment_checksum_cache.borrow_mut(); + if segment.sealed + && let Some(memo) = memos.get_mut(&start_offset) + && let Some(checksum) = memo.checksum_of(len) + { + *memo = SegmentChecksumMemo::Stamped { len, checksum }; + } + } + /// Release the cached offer once no requester holds one (the shard's /// offer-expiry sweep), and the plan of a build in progress with it: both /// name segments that retention may be about to unlink. @@ -3151,198 +3653,363 @@ where self.offsets_wire_snapshot().map(|wire| wire.consumers) } - /// Validate one completed `SEGMENT_LOG` artifact and spill it to staging - /// files, returning the walk metadata. Frees receiver memory as it goes: - /// after this the session drops the artifact's buffer. - /// - /// # Errors - /// `Err(walk error description)` when the payload fails validation (the - /// caller charges the decode budget), or a staging-write failure - /// description. - pub async fn spill_transfer_segment( + /// Validate and sync a completed segment without holding the shard pump. + /// Staging writers are serialized, including work of abandoned sessions. + pub fn begin_transfer_segment_spill( + &mut self, + entry: &consensus::StateArtifact, + bytes: Vec, + prefix: Option, + ) -> LocalWork> { + let job = self.plan_segment_spill(entry, bytes, prefix); + self.spawn_staging_work(segment_spill_stopped, move |cancelled| async move { + job?.run(&cancelled).await + }) + } + + /// Publish only the result owned by the current transfer session. + pub fn accept_transfer_segment_spill(&self, spilled: SpilledSegment) -> StagedSegmentMeta { + let SpilledSegment { meta, hasher } = spilled; + *self.spilled_hasher.borrow_mut() = Some(hasher); + self.reuse_scan_memo + .borrow_mut() + .get_or_insert_with(|| ReuseScanMemo { + adopted: BTreeMap::new(), + }) + .adopted + .insert(meta.start_offset, meta.clone()); + meta + } + + fn plan_segment_spill( + &self, + entry: &consensus::StateArtifact, + bytes: Vec, + prefix: Option, + ) -> Result { + let partition_dir = self + .partition_dir + .clone() + .ok_or(SpillError::NoPartitionDir)?; + // A replacement must invalidate the proof before any writer starts. + if let Some(memo) = self.reuse_scan_memo.borrow_mut().as_mut() { + memo.adopted.remove(&entry.frontier); + } + Ok(SegmentSpillJob { + partition_dir, + entry: *entry, + bytes, + prefix, + }) + } + + #[cfg(test)] + pub(crate) async fn spill_transfer_segment( &self, entry: &consensus::StateArtifact, bytes: Vec, + prefix: Option, ) -> Result { - let Some(partition_dir) = self.partition_dir.clone() else { - return Err(SpillError::NoPartitionDir); + let spilled = self + .plan_segment_spill(entry, bytes, prefix)? + .run(&Cell::new(false)) + .await?; + Ok(self.accept_transfer_segment_spill(spilled)) + } + + /// Start the reuse scan of `manifest` in a detached task, so that its + /// links, syncs and reads of staged copies do not hold the shard pump. + /// + /// The scan stages every segment of the manifest that this replica can + /// supply without a pull: a sealed live segment that holds the bytes of + /// an entry, through hard links, or else the staged copy of an earlier + /// attempt. It sweeps the staging strays that match no entry. Scans and + /// spills wait for the preceding writer, so an abandoned task cannot race + /// the next session on the same staging names. + pub fn begin_reuse_scan( + &mut self, + manifest: &[consensus::StateArtifact], + ) -> LocalWork { + let job = self.plan_reuse_scan(manifest); + self.spawn_staging_work(ReuseScanned::default, move |cancelled| async move { + match job { + Some(job) => job.run(&cancelled).await, + None => ReuseScanned::default(), + } + }) + } + + /// Hand a landed reuse scan to the session: stage the segments that it + /// reused, and plan the prefix seed among the rest. Returns `Pending` + /// while the scan runs, because the pull waits for it. The call that + /// finds the scan landed returns what it reused. + pub fn drive_reuse_scan(&mut self) -> Poll> { + let Some(scan) = self + .transfer + .as_ref() + .and_then(|session| session.scan.as_ref()) + else { + return Poll::Ready(None); }; - // This write replaces whatever the reuse memo recorded for this path, - // so the memo cannot outlive it: a later scan against an older offer - // must re-read the file rather than trust a walk of the old bytes. - self.reuse_scan_memo.borrow_mut().take(); - // Artifact-level integrity FIRST, exactly as the metadata plane - // verifies every artifact before decoding: the walk's per-batch - // checksums prove batch bodies, not that these are the bytes the - // manifest promised (length alone is implied by completion). - if !verify_state_artifact_yielding(entry, &bytes).await { - return Err(SpillError::ManifestChecksum { - frontier: entry.frontier, - }); + let Some(scanned) = scan.result.borrow_mut().take() else { + return Poll::Pending; + }; + let reused = self.accept_reuse_scan(scanned); + let Some(session) = self.transfer.as_mut() else { + return Poll::Ready(None); + }; + let segments = ReusedSegments::new(&session.artifacts, &reused); + session.scan = None; + for (index, meta) in reused { + session.artifacts[index as usize] = TransferArtifact::Staged(meta); } - let (stats, index_bytes) = walk_segment_payload(entry.frontier, &bytes) - .await - .map_err(SpillError::Walk)?; - let (log_staging, index_staging) = staging_paths(&partition_dir, entry.frontier); - let index_size = index_bytes.len() as u64; - // Two writes, not a loop: each moves its buffer into compio's - // owned-buffer API, so a segment-sized payload is never copied. - write_staging_file(&log_staging, bytes) - .await - .map_err(|source| SpillError::StagingIo { - path: log_staging.clone(), - source, - })?; - write_staging_file(&index_staging, index_bytes) - .await - .map_err(|source| SpillError::StagingIo { - path: index_staging.clone(), - source, - })?; - fsync_dir(&partition_dir) - .await - .map_err(|source| SpillError::StagingIo { - path: PathBuf::from(&partition_dir), - source, - })?; - Ok(StagedSegmentMeta::from_walk( - entry, - stats, - index_size, - log_staging, - index_staging, - )) + let seed = self + .transfer + .as_ref() + .and_then(|session| self.plan_prefix_seed(&session.artifacts)); + if let Some(session) = self.transfer.as_mut() { + session.seed = seed; + } + // The I/O turn that reports a landed scan leaves the ready queue ahead + // of any other I/O work that is due, so that work takes the next turn + // instead of the next notification. + self.notify_io(); + Poll::Ready(Some(segments)) } - /// Adopt an already-verified staged log without rewriting it: walk the - /// payload once (validation + a rebuilt sparse index), write only the - /// index sidecar, and return the walk metadata. The reuse scan calls - /// this after `verify_state_artifact` proved the bytes match the - /// manifest; rewriting the byte-identical log (and re-verifying a third - /// time) is exactly the work reuse exists to skip. The index write - /// stays: the scan never checks `.index.staging`, and a missing sidecar - /// would hand the install a missing rename source. - /// - /// No directory fsync here: every sidecar lands in the same directory, so - /// the caller fsyncs ONCE after its loop instead of once per adoption. - async fn adopt_staged_segment( - &self, - entry: &consensus::StateArtifact, - bytes: &[u8], - ) -> Result { - let Some(partition_dir) = self.partition_dir.clone() else { - return Err(SpillError::NoPartitionDir); + /// The reuse scan of `manifest`, planned on the pump. `None` without a + /// partition directory. + fn plan_reuse_scan(&self, manifest: &[consensus::StateArtifact]) -> Option { + let partition_dir = self.partition_dir.clone()?; + // The scan owns the memo while it can replace staging names. + let memoized = self + .reuse_scan_memo + .borrow_mut() + .take() + .map_or_else(BTreeMap::new, |memo| memo.adopted); + let entries = manifest + .iter() + .zip(0u32..) + .filter(|(entry, _)| entry.kind == artifact_kind::SEGMENT_LOG) + .map(|(entry, index)| (index, *entry, self.held_segment(entry))) + .collect(); + Some(ReuseScanJob { + partition_dir, + namespace_raw: self.consensus().group(), + memoized, + entries, + }) + } + + /// Select a sealed segment by its append checksum. The detached scan + /// verifies its on-disk bytes before reusing it. + fn held_segment(&self, entry: &consensus::StateArtifact) -> Option { + let position = self.sealed_segment_holding(entry.frontier, entry.len, entry.checksum)?; + let (Some(log_path), _) = self.log.storages()[position].segment_and_index_paths() else { + return None; }; - let (stats, index_bytes) = walk_segment_payload(entry.frontier, bytes) - .await - .map_err(SpillError::Walk)?; - let (log_staging, index_staging) = staging_paths(&partition_dir, entry.frontier); - let index_size = index_bytes.len() as u64; - write_staging_file(&index_staging, index_bytes) - .await - .map_err(|source| SpillError::StagingIo { - path: index_staging.clone(), - source, - })?; - Ok(StagedSegmentMeta::from_walk( - entry, - stats, - index_size, - log_staging, - index_staging, - )) + Some(HeldSegment { log_path }) } - /// Scan the partition directory for staging files left by an earlier - /// session and adopt every one that matches a manifest entry byte-for- - /// byte (length + artifact checksum + full re-walk). Sealed segments are - /// immutable, so on a retry or peer re-target typically only the active - /// segment and the offsets artifact re-pull. Staging strays matching no - /// entry are swept. + /// The position of the sealed live segment at `start_offset` whose memo + /// stamps its `len` bytes with `checksum`. + fn sealed_segment_holding(&self, start_offset: u64, len: u64, checksum: u64) -> Option { + let segments = self.log.segments(); + let position = segments + .binary_search_by_key(&start_offset, |segment| segment.start_offset) + .ok()?; + let segment = &segments[position]; + let stamp = self + .segment_checksum_cache + .borrow() + .get(&start_offset) + .and_then(|memo| memo.checksum_of(len)); + (segment.sealed && segment.size.as_bytes_u64() == len && stamp == Some(checksum)) + .then_some(position) + } + + /// Keep verified files, including progress made before a scan stopped. /// - /// A scan against a segment set this partition already scanned short- - /// circuits through `ReuseScanMemo`: rotating to another peer would - /// otherwise re-read and re-walk every staged file, up to 2 GiB each, - /// sequentially, on the pump. - pub async fn reuse_staged_segments( + /// The pump ran while the scan did, so a linked segment must still be + /// sealed and stamped as the manifest expects. Retention can have + /// removed it. An adopted copy needs no check, because only a scan, its + /// session and the install of that session write staging names, and the + /// session neither pulls nor installs before its scan lands. + pub(crate) fn accept_reuse_scan( &self, - manifest: &[consensus::StateArtifact], + mut scanned: ReuseScanned, ) -> Vec<(u32, StagedSegmentMeta)> { - let Some(partition_dir) = self.partition_dir.clone() else { - return Vec::new(); + let valid = |meta: &StagedSegmentMeta| { + !meta.linked + || self + .sealed_segment_holding(meta.start_offset, meta.size, meta.checksum) + .is_some() }; - let digest = segment_manifest_digest(manifest); - // Cloned out of the borrow: the file re-checks below await. - let memoized = self - .reuse_scan_memo - .borrow() - .as_ref() - .filter(|memo| memo.digest == digest) - .map(|memo| memo.adopted.clone()); - if let Some(adopted) = memoized { - // The memo proves the bytes were validated; only their continued - // presence needs re-checking (an install or a converge between the - // two scans unlinks them). - let mut intact = true; - for (_, meta) in &adopted { - let log_matches = matches!( - compio::fs::metadata(&meta.log_staging).await, - Ok(metadata) if metadata.len() == meta.size - ); - if !log_matches || compio::fs::metadata(&meta.index_staging).await.is_err() { - intact = false; - break; - } - } - if intact { - return adopted; - } - self.reuse_scan_memo.borrow_mut().take(); + let reused: Vec<_> = scanned + .reused + .into_iter() + .filter(|(_, meta)| valid(meta)) + .collect(); + scanned.memoized.retain(|_, meta| valid(meta)); + for (_, meta) in &reused { + scanned.memoized.insert(meta.start_offset, meta.clone()); } - let mut adopted = Vec::new(); - let mut matched_paths = Vec::new(); - for (index, entry) in manifest.iter().enumerate() { + *self.reuse_scan_memo.borrow_mut() = + (!scanned.memoized.is_empty()).then_some(ReuseScanMemo { + adopted: scanned.memoized, + }); + reused + } + + /// The segment artifact whose leading bytes this replica holds: a live + /// segment at the base offset of an entry that no reuse claimed, and + /// shorter than that entry. Such a segment is the one the primary kept + /// writing after this replica fell behind. The spill proves the prefix. + fn plan_prefix_seed(&self, artifacts: &[TransferArtifact]) -> Option { + let segments = self.log.segments(); + artifacts.iter().zip(0u32..).find_map(|(slot, artifact)| { + let entry = slot.pending()?.entry; if entry.kind != artifact_kind::SEGMENT_LOG { - continue; + return None; } - let (log_staging, _) = staging_paths(&partition_dir, entry.frontier); - // Length short-circuit BEFORE reading: length is the first - // conjunct of the artifact check anyway, and the common retry - // case is precisely an active-segment length mismatch -- no - // point reading up to 2 GiB just to discard it. - match compio::fs::metadata(&log_staging).await { - Ok(metadata) if metadata.len() == entry.len => {} - _ => continue, + let position = segments + .binary_search_by_key(&entry.frontier, |segment| segment.start_offset) + .ok()?; + let len = segments[position].size.as_bytes_u64(); + if len == 0 || len >= entry.len { + return None; } - let Ok(bytes) = compio::fs::read(&log_staging).await else { - continue; - }; - if !verify_state_artifact_yielding(entry, &bytes).await { - continue; + let (log_path, _) = self.log.storages()[position].segment_and_index_paths(); + Some(PrefixSeed::Planned { + artifact, + log_path: log_path?, + len, + }) + }) + } + + /// Start or finish the prefix read of the artifact that the pull reached. + /// + /// Returns `true` while the read runs. The next chunk request then waits, + /// because that request asks for the bytes that the read supplies. The read runs + /// in a detached task, so a read of up to a whole segment does not hold the + /// pump. A failed read leaves the artifact to a whole pull. + pub fn drive_prefix_seed(&mut self) -> bool { + let namespace_raw = self.namespace().inner(); + let Some(session) = self.transfer.as_mut() else { + return false; + }; + let next = session + .artifacts + .iter() + .position(|artifact| !consensus::ChunkProgress::complete(artifact)); + if session + .seed + .as_ref() + .is_none_or(|seed| Some(seed.artifact() as usize) != next) + { + return false; + } + match session.seed.take() { + Some(PrefixSeed::Planned { + artifact, + log_path, + len, + }) => { + let Some(entry) = session.artifacts[artifact as usize] + .pending() + .map(|progress| progress.entry) + else { + return false; + }; + let read = + self.spawn_local_work(prefix_read_stopped, move |cancelled| async move { + read_segment_prefix(&log_path, entry, len, &cancelled).await + }); + if let Some(session) = self.transfer.as_mut() { + session.seed = Some(PrefixSeed::Reading { artifact, read }); + } + true } - if let Ok(meta) = self.adopt_staged_segment(entry, &bytes).await { - matched_paths.push(meta.log_staging.clone()); - matched_paths.push(meta.index_staging.clone()); - #[allow(clippy::cast_possible_truncation)] - adopted.push((index as u32, meta)); + Some(PrefixSeed::Reading { artifact, read }) => { + let Some(landed) = read.result.borrow_mut().take() else { + session.seed = Some(PrefixSeed::Reading { artifact, read }); + return true; + }; + match (landed, session.artifacts[artifact as usize].pending_mut()) { + (Ok((bytes, prefix)), Some(progress)) if progress.buf.is_empty() => { + tracing::info!( + namespace_raw, + artifact, + seeded_len = prefix.len, + artifact_len = progress.entry.len, + "seeded a partition transfer artifact from the local segment" + ); + progress.buf = bytes; + session.seed = Some(PrefixSeed::Seeded { artifact, prefix }); + } + (Err(error), _) => tracing::info!( + namespace_raw, + artifact, + %error, + "cannot read the local segment prefix, so the pull fetches the \ + whole artifact" + ), + _ => {} + } + // The I/O turn that reports a landed read leaves the ready + // queue ahead of any other I/O work that is due, so that work + // takes the next turn instead of the next notification. + self.notify_io(); + false + } + seed => { + session.seed = seed; + false } } - // Every rebuilt sidecar landed in the same directory, so one fsync - // covers them all. Its failure discards EVERY adoption: the per-adopt - // filter above no longer sees a durability failure, and an undurable - // sidecar handed to the install is a missing rename source after a - // crash. - if !adopted.is_empty() && fsync_dir(&partition_dir).await.is_err() { - adopted.clear(); - matched_paths.clear(); + } + + fn spawn_staging_work + 'static>( + &mut self, + stopped: fn() -> T, + work: impl FnOnce(Rc>) -> F + 'static, + ) -> LocalWork { + let (ended, receiver) = futures::channel::oneshot::channel::<()>(); + let previous = self.staging_work_ended.replace(receiver); + self.spawn_local_work(stopped, move |cancelled| async move { + let _ended = ended; + if let Some(previous) = previous { + let _ = previous.await; + } + work(cancelled).await + }) + } + + /// Run `work` in a detached task, so that the pump does not wait for it. + /// `work` gets the flag that asks it to stop. + fn spawn_local_work + 'static>( + &self, + stopped: fn() -> T, + work: impl FnOnce(Rc>) -> F, + ) -> LocalWork { + let local = LocalWork::default(); + if !self.io_active.begin_local_work() { + *local.result.borrow_mut() = Some(stopped()); + self.notify_io(); + return local; } - // Sweep strays: anything staged that no adopted meta claims. - sweep_staging_except(&partition_dir, matched_paths.into_iter().collect()).await; - *self.reuse_scan_memo.borrow_mut() = Some(ReuseScanMemo { - digest, - adopted: adopted.clone(), + let landing = Landing { + io: Rc::clone(&self.io_active), + result: Rc::clone(&local.result), + wake: self.io_wake(), + stopped, + }; + let work = work(Rc::clone(&local.cancel.0)); + self.consensus().message_bus().spawn(async move { + landing.land(work.await); }); - adopted + local } /// Install a fully transferred partition state: swap the staged segment @@ -4159,7 +4826,9 @@ where fn clear_install_journal(&mut self) { self.log.journal().inner.clear_all(); + self.reset_proof_walk_memory(); self.log.journal_mut().info = crate::log::JournalInfo::default(); + self.consensus().note_journal_mutation(); } /// Clear the offset table for the installed one, which keeps `emptied_at`. @@ -4225,6 +4894,39 @@ where self.set_offset_space_used(install.next_offset > 0); self.recovered_durable_offset = install.staged.last().map(|meta| meta.end_offset); self.installed_frontier = (install.next_offset > 0).then_some(install.next_offset); + // Each staged log matched its manifest checksum, so a later offer or + // reuse scan can trust these stamps without reading the files. + let mut memos: std::collections::HashMap = install + .staged + .iter() + .map(|meta| { + ( + meta.start_offset, + SegmentChecksumMemo::Stamped { + len: meta.size, + checksum: meta.checksum, + }, + ) + }) + .collect(); + // The active segment grows after the install, and its first append + // drops a stamp, so the next transfer after it seals pulls it whole. + // The hasher of its spill extends with each append instead. + if let Some(spilled) = self.spilled_hasher.borrow_mut().take() + && let Some(active) = install.staged.last() + && spilled.start_offset == active.start_offset + && spilled.size == active.size + && spilled.hasher.finish() == active.checksum + { + memos.insert( + active.start_offset, + SegmentChecksumMemo::Hashing { + hashed_len: active.size, + hasher: spilled.hasher, + }, + ); + } + *self.segment_checksum_cache.borrow_mut() = memos; self.stats.zero_out_all(); self.stats.increment_segments_count( u32::try_from(self.log.segments().len()).expect("manifest segment count fits u32"), @@ -4247,17 +4949,10 @@ where install.commit_op, Owned::<4096>::copy_from_slice(&install.offsets_wire.checkpoint_prepare).into(), ); + self.consensus().note_journal_mutation(); } - let consensus = self.consensus(); - if install.commit_op > consensus.commit_min() { - consensus.set_commit_floor(install.commit_op); - } - consensus.sequencer().set_sequence(install.commit_op); - if let Some(checksum) = install.offsets_wire.prepare_checksum { - consensus.set_last_prepare_checksum(checksum); - } - consensus.clear_pipeline(); - consensus.advance_commit_max(install.commit_op); + self.consensus() + .install_checkpoint(install.commit_op, install.offsets_wire.prepare_checksum); // The cleared journal and pipeline held any unapplied install, and the // checkpoint already carries every owner installed up to commit_op. self.pending_owner_install.set(None); @@ -4318,25 +5013,24 @@ where commit_min, }); } - // The install wipes the journal and rewinds the sequencer to - // `commit_op`. The primary may have counted this replica's ack before - // its commit reached us. Acks wait for the contiguous WAL head, so - // protect held ops through that head as well as the known commit. - // Ops known only from heartbeats are not held: a transferring replica - // journals nothing, and repair fetches them after install. Refusing on - // `commit_max` alone would refuse every offer under steady writes: - // heartbeats carry the primary's `commit_min` past whatever op an offer - // pinned. + // The install wipes the journal. The primary may have counted this + // replica's ack before its commit reached us. Acks wait for the + // contiguous WAL head, so protect held ops through that head. Held ops + // above it were never acked, and repair fetches them again after + // install. Protecting them as well refuses every offer under steady + // writes: while the serving peer builds the offer, repair journals the + // committed tail past the op the offer pinned. Without a WAL, protect + // the recorded acknowledgement frontier and the known commit instead. let commit_max = self.consensus().commit_max(); - let held_max = self - .persistence - .as_ref() - .map_or(commit_max, |persistence| persistence.head().max(commit_max)); + let held_max = self.persistence.as_ref().map_or_else( + || commit_max.max(self.acknowledged_op.get()), + |persistence| persistence.head(), + ); let journal = &self.log.journal().inner; if commit_op < held_max && journal.holds_op_in(commit_op + 1..=held_max) { - return Err(PartitionInstallError::StaleTransfer { + return Err(PartitionInstallError::OfferErasesHeldOps { commit_op, - commit_min: held_max, + held_max, }); } let offsets_wire = ConsumerOffsetsWire::decode(offsets_bytes)?; @@ -4549,6 +5243,7 @@ where #[derive(Debug)] pub enum SpillError { NoPartitionDir, + Cancelled, /// The received bytes are not what the manifest promised. ManifestChecksum { frontier: u64, @@ -4566,6 +5261,7 @@ impl fmt::Display for SpillError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::NoPartitionDir => write!(f, "partition has no on-disk directory"), + Self::Cancelled => write!(f, "the transfer abandoned its segment spill"), Self::ManifestChecksum { frontier } => write!( f, "segment artifact at base offset {frontier} fails its manifest checksum" @@ -4614,6 +5310,12 @@ const OFFER_HASH_CHUNK_LEN: usize = 4 << 20; /// every storage class. const OFFER_HASH_BUDGET_PER_ROUND_BYTES: u64 = 256 * 1024 * 1024; +/// Bytes a prefix read reads between two checks of its cancel flag. A dropped +/// seed stops the read within one piece: 64 MiB, under a second on a +/// 125 MB/s cloud volume. Each piece opens the file once and allocates one +/// read chunk, which costs little next to 16 chunk reads. +const PREFIX_READ_PIECE_LEN: u64 = 16 * OFFER_HASH_CHUNK_LEN as u64; + /// Feed bytes `[from, to)` of `path` into `hasher`, read in /// [`OFFER_HASH_CHUNK_LEN`] chunks with one reactor yield per chunk, appending /// each chunk to `sink` when one is given. @@ -4666,11 +5368,8 @@ async fn hash_segment_range( /// against the manifest entry, chunked through `hash_segment_range` with one /// reactor yield per chunk. /// -/// The serving side runs this on the pump to answer a single chunk request, so -/// it reads and hashes in chunks rather than in one pass. The yields keep the -/// REACTOR moving (detached tasks, `io_uring` completions); they do not keep this -/// shard's consensus ticks alive, which are a sibling select arm of the same -/// task and stay frozen for the duration. +/// The serving side caches only artifacts that fit one chunk. Larger files +/// use bounded range reads and are verified by the receiver before install. /// /// The file may legitimately be LONGER than the entry (an active segment that /// kept appending after the offer was built); the artifact is the prefix. @@ -4695,6 +5394,379 @@ pub async fn load_verified_segment_artifact( Ok(bytes) } +/// Read a requested segment range without loading the rest of the file. +/// +/// The caller bounds `len` by the manifest and chunk limit. The receiver +/// verifies the complete artifact before installing it, including its prefix. +/// +/// # Errors +/// Returns the classified open or read error, including a short range. +pub async fn load_segment_chunk( + log_path: &str, + offset: u64, + len: usize, +) -> Result, SegmentLoadError> { + let file = compio::fs::File::open(log_path) + .await + .map_err(SegmentLoadError::classify)?; + let compio::BufResult(result, bytes) = file.read_exact_at(vec![0; len], offset).await; + result.map_err(SegmentLoadError::classify)?; + Ok(bytes) +} + +/// Read, hash and walk the first `len` bytes of a local segment file into a +/// buffer sized for the whole artifact, so the pulled suffix appends in place. +/// Once `cancelled` is set, the read stops before its next piece or the walk +/// with [`std::io::ErrorKind::Interrupted`] and frees the buffer. +pub(crate) async fn read_segment_prefix( + log_path: &str, + entry: consensus::StateArtifact, + len: u64, + cancelled: &Cell, +) -> std::io::Result<(Vec, SeededPrefix)> { + let capacity = usize::try_from(entry.len).map_err(std::io::Error::other)?; + let mut bytes = Vec::with_capacity(capacity); + let mut hasher = Box::::default(); + let mut position = 0; + loop { + if cancelled.get() { + return Err(std::io::Error::new( + std::io::ErrorKind::Interrupted, + "the transfer gave up this prefix read", + )); + } + if position == len { + break; + } + let end = len.min(position.saturating_add(PREFIX_READ_PIECE_LEN)); + hash_segment_range(log_path, position, end, &mut hasher, Some(&mut bytes)).await?; + position = end; + } + let mut walk = SegmentWalk::new(entry.frontier); + let walk = walk.advance(&bytes).await.is_ok().then_some(walk); + Ok((bytes, SeededPrefix { len, hasher, walk })) +} + +/// Lands the result of local work and wakes the partition. A task that stops +/// without a result, such as one that panicked, drops this guard with an +/// empty slot. The guard then lands `stopped()`, so the session does not wait +/// forever. +struct Landing { + io: Rc, + result: Rc>>, + wake: W, + stopped: fn() -> T, +} + +impl Landing { + fn land(self, result: T) { + *self.result.borrow_mut() = Some(result); + } +} + +impl Drop for Landing { + fn drop(&mut self) { + if self.result.borrow().is_none() { + self.io.interrupt(); + } + self.io.settle_local_work(); + self.result.borrow_mut().get_or_insert_with(self.stopped); + (self.wake)(); + } +} + +/// A prefix read that stopped without a result, so the artifact falls back to +/// a whole pull. +fn prefix_read_stopped() -> PrefixRead { + Err(std::io::Error::other( + "the prefix read stopped without a result", + )) +} + +const fn segment_spill_stopped() -> Result { + Err(SpillError::Cancelled) +} + +struct SegmentSpillJob { + partition_dir: String, + entry: consensus::StateArtifact, + bytes: Vec, + prefix: Option, +} + +impl SegmentSpillJob { + async fn run(self, cancelled: &Cell) -> Result { + if cancelled.get() { + return Err(SpillError::Cancelled); + } + let Self { + partition_dir, + entry, + bytes, + prefix, + } = self; + let (mut hasher, walk, hashed_len) = match prefix { + Some(prefix) => (prefix.hasher, prefix.walk, prefix.len), + None => (Box::default(), None, 0), + }; + // Artifact-level integrity FIRST, exactly as the metadata plane + // verifies every artifact before decoding: the walk's per-batch + // checksums prove batch bodies, not that these are the bytes the + // manifest promised (length alone is implied by completion). + let unhashed = usize::try_from(hashed_len) + .ok() + .and_then(|hashed_len| bytes.get(hashed_len..)); + let Some(unhashed) = unhashed.filter(|_| bytes.len() as u64 == entry.len) else { + return Err(SpillError::ManifestChecksum { + frontier: entry.frontier, + }); + }; + hash_yielding(&mut hasher, [unhashed]).await; + if cancelled.get() { + return Err(SpillError::Cancelled); + } + if hasher.finish() != entry.checksum { + return Err(SpillError::ManifestChecksum { + frontier: entry.frontier, + }); + } + let mut walk = walk.unwrap_or_else(|| SegmentWalk::new(entry.frontier)); + walk.advance(&bytes).await.map_err(SpillError::Walk)?; + let (stats, index_bytes) = walk.finish().map_err(SpillError::Walk)?; + if cancelled.get() { + return Err(SpillError::Cancelled); + } + let (log_staging, index_staging) = staging_paths(&partition_dir, entry.frontier); + let index_size = index_bytes.len() as u64; + // Two writes, not a loop: each moves its buffer into compio's + // owned-buffer API, so a segment-sized payload is never copied. + write_staging_file(&log_staging, bytes) + .await + .map_err(|source| SpillError::StagingIo { + path: log_staging.clone(), + source, + })?; + if cancelled.get() { + return Err(SpillError::Cancelled); + } + write_staging_file(&index_staging, index_bytes) + .await + .map_err(|source| SpillError::StagingIo { + path: index_staging.clone(), + source, + })?; + fsync_dir(&partition_dir) + .await + .map_err(|source| SpillError::StagingIo { + path: PathBuf::from(&partition_dir), + source, + })?; + if cancelled.get() { + return Err(SpillError::Cancelled); + } + Ok(SpilledSegment { + meta: StagedSegmentMeta::from_walk( + &entry, + stats, + index_size, + log_staging, + index_staging, + ), + hasher: SpilledHasher { + start_offset: entry.frontier, + size: entry.len, + hasher, + }, + }) + } +} + +/// A reuse scan that a detached task runs. It holds no partition state, so +/// the pump runs while it does. +struct ReuseScanJob { + partition_dir: String, + namespace_raw: u64, + memoized: BTreeMap, + /// The manifest index of each segment entry, the entry, and the live + /// segment that holds its bytes. + entries: Vec<(u32, consensus::StateArtifact, Option)>, +} + +impl ReuseScanJob { + /// Checks cancellation between files, retaining verified progress without + /// sweeping files that it did not reach. + async fn run(mut self, cancelled: &Cell) -> ReuseScanned { + let mut reused = Vec::new(); + let mut matched_paths = Vec::new(); + let mut complete = true; + for (index, entry, held) in &self.entries { + if cancelled.get() { + complete = false; + break; + } + let memoized = self + .memoized + .remove(&entry.frontier) + .filter(|meta| (meta.size, meta.checksum) == (entry.len, entry.checksum)); + let meta = if let Some(meta) = memoized + && matches!( + compio::fs::metadata(&meta.log_staging).await, + Ok(metadata) if metadata.len() == meta.size + ) + && matches!( + compio::fs::metadata(&meta.index_staging).await, + Ok(metadata) if metadata.len() == meta.index_size + ) { + Some(meta) + } else { + self.reuse(entry, held.as_ref()).await + }; + let Some(meta) = meta else { + continue; + }; + matched_paths.push(meta.log_staging.clone()); + matched_paths.push(meta.index_staging.clone()); + reused.push((*index, meta)); + } + // Every rebuilt sidecar and every link landed in the same directory, + // so one fsync covers them all. Its failure discards EVERY adoption: + // the per-adopt filter above no longer sees a durability failure, and + // an undurable sidecar handed to the install is a missing rename + // source after a crash. + if !reused.is_empty() && fsync_dir(&self.partition_dir).await.is_err() { + reused.clear(); + matched_paths.clear(); + } + // A stopped scan cannot tell a stray from a staged copy of an entry + // that it did not reach. + if complete { + // Sweep strays: anything staged that no adopted meta claims. + sweep_staging_except(&self.partition_dir, matched_paths.into_iter().collect()).await; + self.memoized.clear(); + } + ReuseScanned { + reused, + memoized: self.memoized, + } + } + + /// Prefer a verified live match, then try the staged copy of an earlier + /// attempt. Either path rebuilds the index from verified log bytes. + async fn reuse( + &self, + entry: &consensus::StateArtifact, + held: Option<&HeldSegment>, + ) -> Option { + if let Some(held) = held + && let Some(meta) = held + .link(&self.partition_dir, entry, self.namespace_raw) + .await + { + return Some(meta); + } + let (log_staging, _) = staging_paths(&self.partition_dir, entry.frontier); + // Length short-circuit BEFORE reading: length is the first + // conjunct of the artifact check anyway, and the common retry + // case is precisely an active-segment length mismatch -- no + // point reading up to 2 GiB just to discard it. + match compio::fs::metadata(&log_staging).await { + Ok(metadata) if metadata.len() == entry.len => {} + _ => return None, + } + let bytes = compio::fs::read(&log_staging).await.ok()?; + if !verify_state_artifact_yielding(entry, &bytes).await { + return None; + } + adopt_staged_segment(&self.partition_dir, entry, &bytes) + .await + .ok() + } +} + +/// A sealed live log selected by the pump for verification and reuse. +struct HeldSegment { + log_path: String, +} + +impl HeldSegment { + /// Verify and hard-link the log, rebuilding its index from the same bytes. + /// + /// Each link is synced before it takes the staging name, because an + /// installed segment must be as durable as a spilled one, and the live + /// writes of this segment did not have to sync. A staging name that + /// already names the live file is such a link from an earlier scan, so + /// it costs nothing. + async fn link( + &self, + partition_dir: &str, + entry: &consensus::StateArtifact, + namespace_raw: u64, + ) -> Option { + let log_metadata = compio::fs::metadata(&self.log_path).await.ok()?; + // The install opens the log at exactly the manifest length. + if log_metadata.len() != entry.len { + return None; + } + let bytes = compio::fs::read(&self.log_path).await.ok()?; + if !verify_state_artifact_yielding(entry, &bytes).await { + return None; + } + let (log_staging, _) = staging_paths(partition_dir, entry.frontier); + if !names_file_of(&log_metadata, &log_staging).await + && let Err(error) = link_durably(Path::new(&self.log_path), &log_staging).await + { + tracing::warn!( + %error, + namespace_raw, + path = %log_staging.display(), + "cannot link a held segment to its staging name" + ); + return None; + } + let mut meta = adopt_staged_segment(partition_dir, entry, &bytes) + .await + .ok()?; + meta.linked = true; + Some(meta) + } +} + +/// Adopt an already-verified staged log without rewriting it: walk the +/// payload once (validation + a rebuilt sparse index), write only the +/// index sidecar, and return the walk metadata. The reuse scan calls +/// this after `verify_state_artifact` proved the bytes match the +/// manifest; rewriting the byte-identical log (and re-verifying a third +/// time) is exactly the work reuse exists to skip. The index write +/// stays: the scan never checks `.index.staging`, and a missing sidecar +/// would hand the install a missing rename source. +/// +/// No directory fsync here: every sidecar lands in the same directory, so +/// the caller fsyncs ONCE after its loop instead of once per adoption. +async fn adopt_staged_segment( + partition_dir: &str, + entry: &consensus::StateArtifact, + bytes: &[u8], +) -> Result { + let (stats, index_bytes) = walk_segment_payload(entry.frontier, bytes) + .await + .map_err(SpillError::Walk)?; + let (log_staging, index_staging) = staging_paths(partition_dir, entry.frontier); + let index_size = index_bytes.len() as u64; + write_staging_file(&index_staging, index_bytes) + .await + .map_err(|source| SpillError::StagingIo { + path: index_staging.clone(), + source, + })?; + Ok(StagedSegmentMeta::from_walk( + entry, + stats, + index_size, + log_staging, + index_staging, + )) +} + /// Why a served segment could not be handed to a requester. /// /// The split is the whole point: a short read is what a concurrent GC @@ -4768,33 +5840,31 @@ async fn verify_state_artifact_yielding(entry: &consensus::StateArtifact, bytes: return false; } let mut hasher = StateArtifactHasher::new(); - for chunk in bytes.chunks(OFFER_HASH_CHUNK_LEN) { - hasher.update(chunk); - yield_to_reactor().await; - } + hash_yielding(&mut hasher, [bytes]).await; hasher.finish() == entry.checksum } -pub(crate) fn numeric_offset_id(path: &str) -> Option { - Path::new(path).file_name()?.to_str()?.parse().ok() +pub(crate) async fn hash_yielding<'a>( + hasher: &mut StateArtifactHasher, + slices: impl IntoIterator, +) { + let mut remaining = OFFER_HASH_CHUNK_LEN; + for mut bytes in slices { + while !bytes.is_empty() { + let (chunk, rest) = bytes.split_at(bytes.len().min(remaining)); + hasher.update(chunk); + bytes = rest; + remaining -= chunk.len(); + if remaining == 0 { + yield_to_reactor().await; + remaining = OFFER_HASH_CHUNK_LEN; + } + } + } } -/// Stamp over every `SEGMENT_LOG` entry of a manifest, keying -/// [`ReuseScanMemo`]. Equal digests mean the two offers expect byte-identical -/// staged files, so a scan already done for one answers the other; the offsets -/// artifact is excluded because the scan never looks at it (and it re-encodes -/// per build, so including it would defeat the memo on every rotation). -fn segment_manifest_digest(manifest: &[consensus::StateArtifact]) -> u64 { - let mut hasher = StateArtifactHasher::new(); - for entry in manifest - .iter() - .filter(|entry| entry.kind == artifact_kind::SEGMENT_LOG) - { - hasher.update(&entry.frontier.to_le_bytes()); - hasher.update(&entry.len.to_le_bytes()); - hasher.update(&entry.checksum.to_le_bytes()); - } - hasher.finish() +pub(crate) fn numeric_offset_id(path: &str) -> Option { + Path::new(path).file_name()?.to_str()?.parse().ok() } fn checked_retry_table( @@ -5054,8 +6124,15 @@ pub(crate) async fn reclaim_retry_checkpoint_files( DiskStorage.sync_directory(directory).await } +/// Write `payload` to a new file at `path`. A staging name can be a hard link +/// to a live segment that the reuse scan linked. Writing through that name in +/// place changes the live segment, so the old name goes first. async fn write_staging_file(path: &Path, payload: Vec) -> std::io::Result<()> { - let mut file = compio::fs::File::create(path) + remove_if_present(path).await?; + let mut file = compio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) .await .note_descriptor_exhaustion(|| format!("creating {}", path.display()))?; let (result, _) = file.write_all_at(payload, 0).await.into(); @@ -5063,3 +6140,52 @@ async fn write_staging_file(path: &Path, payload: Vec) -> std::io::Result<() file.sync_data().await?; Ok(()) } + +async fn remove_if_present(path: &Path) -> std::io::Result<()> { + match compio::fs::remove_file(path).await { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => Err(error), + _ => Ok(()), + } +} + +/// Hard-link `live` to `staged` through a scratch name, so that the staging +/// name only ever names a link whose data is durable, and a failure keeps +/// the staged copy that `staged` held. A rename does nothing when both names +/// are links to one file, so the caller must not link a file that `staged` +/// already names. +async fn link_durably(live: &Path, staged: &Path) -> std::io::Result<()> { + let scratch = staged.with_extension(LINK_SCRATCH_EXTENSION); + let linked = async { + remove_if_present(&scratch).await?; + compio::fs::hard_link(live, &scratch).await?; + compio::fs::File::open(&scratch) + .await + .note_descriptor_exhaustion(|| format!("opening {}", scratch.display()))? + .sync_data() + .await?; + compio::fs::rename(&scratch, staged).await + } + .await; + if linked.is_err() { + // A later sweep of the staging names removes a name that this cannot. + let _ = compio::fs::remove_file(&scratch).await; + } + linked +} + +/// Whether `staged` names the file that `live` describes. A link takes its +/// staging name only once its data is durable, so a scan that finds one +/// there keeps it without another link or sync. +#[cfg(unix)] +async fn names_file_of(live: &compio::fs::Metadata, staged: &Path) -> bool { + compio::fs::metadata(staged) + .await + .is_ok_and(|staged| staged.dev() == live.dev() && staged.ino() == live.ino()) +} + +/// Without inode numbers no scan can prove an earlier link, so each one +/// links again. +#[cfg(not(unix))] +async fn names_file_of(_live: &compio::fs::Metadata, _staged: &Path) -> bool { + false +} diff --git a/core/partitions/src/types.rs b/core/partitions/src/types.rs index 39e42fd079..68cecf19c3 100644 --- a/core/partitions/src/types.rs +++ b/core/partitions/src/types.rs @@ -356,8 +356,10 @@ pub enum RepairConclusion { /// The walk reached the requested frontier; the session was dropped. Done, /// The floor's continuity check failed: ops below it are neither locally - /// durable nor repaired. The session was dropped here -- state transfer - /// supersedes repair -- and the caller arms the transfer. + /// durable nor repaired. The repair walk of an unproven view log that + /// outgrew its byte budget also ends here, with `floor` at `commit_min`. + /// The session was dropped here -- state transfer supersedes repair -- and + /// the caller arms the transfer. FloorRefused { floor: u64, to_op: u64 }, } diff --git a/core/server/config.toml b/core/server/config.toml index 50b603d639..b9627fd952 100644 --- a/core/server/config.toml +++ b/core/server/config.toml @@ -497,8 +497,10 @@ name = "iggy-cluster" # whole ticks: values are rounded down to a multiple of 10ms, and anything under # 10ms is raised to a single tick rather than firing sooner. -# Backup-side liveness window for a consensus plane's primary (duration). -# A replica that sees no primary traffic for this long starts a view change. +# Liveness window for a consensus primary and its replication quorum (duration). +# Backups vote after primary silence. A primary without heartbeat replies from +# a quorum relinquishes leadership. Heartbeat replies require a coordinated +# node upgrade; mixed versions cannot sustain a primary's quorum checks. # Raise it on oversubscribed hosts where scheduling stalls fake primary # death. Must be at least "2s" and at least 4x commit_broadcast_interval: the # primary signals liveness through its commit broadcast, and the window must @@ -1035,8 +1037,19 @@ change_log_activity_capacity = 16384 # partition_wal_prepares_total over partition_wal_batches_total is the prepares per durable group, and # partition_wal_group_commit_waits_total counts the groups that actually took the optional wait # below, so a delay that never fires is distinguishable from one that fires and buys nothing. +# A backup whose log the adopted view has not proven yet repairs it from the +# head down and holds every batch it fetches in memory until the repair reaches +# its commit point. Once it holds more than this budget, it stops and catches up +# by state transfer instead. This bound is per partition, with the shared +# allocation cap below limiting concurrent walks across all shards. wal_bytes_max = "256 MiB" +# Node-wide allocation cap shared by unproven repair walks and snapshot repair +# tails. Exhausted walks fall back to state transfer; exhausted tails require +# a fresh snapshot. Normal prepare queues and transfer buffers have separate limits. +# Must be > 0. +repair_memory_bytes_max = "1 GiB" + # Bounded wait for more prepares before a persisted partition's WAL writer # starts its durability barrier, in microseconds. 0 disables it. Max 10000. # Spends up to this much acknowledgment latency to cut device writes: one @@ -1113,19 +1126,26 @@ consumer_offsets_max = 4096 # For superblock compatibility, see superblock_wedged_fatal_timeout in [cluster]. offset_reservation_lease = 65536 -# Entries the evicted ring retains per multi-replica partition for journal -# repair after a peer rejoins. Larger widens the window a restarting peer can be -# served from the ring before falling back to bulk sync, at the cost of pinned -# memory per partition. Must be > 0 and <= 65536. Single-replica partitions -# retain nothing regardless. +# Entries the evicted ring retains with their batches in memory per +# multi-replica partition for journal repair after a peer rejoins. Larger +# widens the window a restarting peer can be served from memory, at the cost of +# pinned memory per partition. Must be > 0 and <= 65536. Single-replica +# partitions retain nothing regardless. The ops whose batches the ring released +# to their segments do not count here. The ring keeps up to 16384 of them, 288 +# bytes each, and repair reads their batches back from disk. evicted_ring_capacity = 65536 -# Payload-byte ceiling per partition and replica; the first ring cap reached -# evicts the lowest ops, retaining the newest entry even if it alone exceeds -# this budget. Must be > 0 and <= "256 MiB". Allocation capacity, alignment and -# index overhead are additional memory, especially for small offset commits. -# There is no node-wide budget. partition_repair_ring_bytes reports payload -# bytes summed across a shard's partitions, not their allocated memory or RSS. +# Byte ceiling per partition and replica for the batches the ring holds in +# memory. Past the cap the oldest batches that a segment holds leave memory +# first, and repair reads them back from disk. When none can leave, the lowest +# ops go, but the newest entry stays even if it alone exceeds this budget. Must +# be > 0 and <= "256 MiB". The ops whose batches left memory take up to +# 4.5 MiB more. Allocation capacity, alignment and index overhead are +# additional memory, especially for small offset commits. There is no node-wide +# budget. +# partition_repair_ring_bytes reports these bytes plus 288 bytes for each +# released op, summed across a shard's partitions, not their allocated memory +# or RSS. evicted_ring_bytes_max = "64 MiB" # Byte budget for segment payloads a SERVING shard keeps resident to answer diff --git a/core/server/src/boot/mod.rs b/core/server/src/boot/mod.rs index 9a0f7e11a1..238260c882 100644 --- a/core/server/src/boot/mod.rs +++ b/core/server/src/boot/mod.rs @@ -345,6 +345,11 @@ pub fn bootstrap( &config, total_shards, )?)); + let repair_memory = Arc::new(partitions::RepairMemoryBudget::new( + usize::try_from(config.partition.repair_memory_bytes_max.as_bytes_u64()).map_err(|_| { + ServerError::Config(configs::ConfigurationError::InvalidConfigurationValue) + })?, + )); // Single-shot bundle handoff (see `MetadataHandoff`): shard 0 sends // one cloned `ServerMetadataBundle` per peer; each peer drains @@ -407,6 +412,7 @@ pub fn bootstrap( let config_for_shard = Arc::clone(&config); let shutdown_flag_for_shard = Arc::clone(&shutdown_flag); let owner_table_for_shard = Arc::clone(&owner_table); + let repair_memory_for_shard = Arc::clone(&repair_memory); let metadata_handoff_for_shard = if shard_id == 0 { MetadataHandoff::Owner { bundle_tx: metadata_bundle_tx.clone(), @@ -448,6 +454,7 @@ pub fn bootstrap( metadata_handoff_for_shard, barrier_for_shard, owner_table_for_shard, + repair_memory_for_shard, roster_cells_for_shard, applied_frontier_for_shard, shard_metrics_for_shard, @@ -522,6 +529,7 @@ async fn shard_main( metadata_handoff: MetadataHandoff, barrier: BootstrapBarrier, owner_table: Arc, + repair_memory: Arc, roster_cells: RosterCells, metadata_applied_frontier: Arc, shard_metrics_all: Vec, @@ -752,6 +760,7 @@ async fn shard_main( reply_inbox, shard_metrics, &roster_cells, + repair_memory, )) .await?; diff --git a/core/server/src/boot/recovery.rs b/core/server/src/boot/recovery.rs index dd1633873f..504479407f 100644 --- a/core/server/src/boot/recovery.rs +++ b/core/server/src/boot/recovery.rs @@ -107,6 +107,7 @@ pub(in crate::boot) async fn build_shard_for_thread( reply_inbox: ShardReceiver, metrics: ShardMetrics, roster_cells: &RosterCells, + repair_memory: Arc, ) -> Result { let shard_local_id = ShardId::new(shard_id); let retired_key: iggy_common::HeaderKey = @@ -172,7 +173,7 @@ pub(in crate::boot) async fn build_shard_for_thread( refuse_partitionless_stream_keys(metadata.mux_stm.streams(), &key_context, solo)?; } refuse_unreadable_topics(metadata.mux_stm.streams())?; - let partitions = IggyPartitions::with_capacity( + let partitions = IggyPartitions::with_repair_memory( shard_local_id, PartitionsConfig { messages_required_to_save: iggy_common::DEFAULT_MESSAGES_REQUIRED_TO_SAVE, @@ -188,6 +189,7 @@ pub(in crate::boot) async fn build_shard_for_thread( }, }, owned_partitions_capacity, + repair_memory, ); let shards_table = PapayaShardsTable::with_capacity(total_partitions); diff --git a/core/server/src/boot/threads.rs b/core/server/src/boot/threads.rs index a5e72f96a1..f6baf66fee 100644 --- a/core/server/src/boot/threads.rs +++ b/core/server/src/boot/threads.rs @@ -668,6 +668,7 @@ pub(in crate::boot) fn run_shard_thread( metadata_handoff: MetadataHandoff, barrier: BootstrapBarrier, owner_table: Arc, + repair_memory: Arc, roster_cells: RosterCells, metadata_applied_frontier: Arc, shard_metrics_all: Vec, @@ -718,6 +719,7 @@ pub(in crate::boot) fn run_shard_thread( metadata_handoff, barrier, owner_table, + repair_memory, roster_cells, metadata_applied_frontier, shard_metrics_all, diff --git a/core/server/src/dispatch/submit.rs b/core/server/src/dispatch/submit.rs index 1caff8d492..3fe30b03d5 100644 --- a/core/server/src/dispatch/submit.rs +++ b/core/server/src/dispatch/submit.rs @@ -646,7 +646,8 @@ mod tests { use iggy_binary_protocol::requests::consumer_groups::CreateConsumerGroupRequest; use iggy_binary_protocol::requests::topics::DeleteTopicRequest; use iggy_binary_protocol::{ - ForwardCompletionHeader, GenericHeader, Operation, WireEncode, WireIdentifier, WireName, + Command, ConsensusHeader, ForwardCompletionHeader, GenericHeader, HEADER_SIZE, Operation, + StartViewChangeHeader, WireEncode, WireIdentifier, WireName, }; use iggy_common::{IggyDuration, IggyError}; use journal::prepare_journal::PrepareJournal; @@ -1115,9 +1116,21 @@ mod tests { for _ in 0..TimeoutManager::NORMAL_HEARTBEAT_TICKS { consensus.tick(PlaneKind::Metadata); } + let vote = Message::::new(HEADER_SIZE).transmute_header( + |_, header: &mut StartViewChangeHeader| { + header.command = Command::StartViewChange; + header.cluster = consensus.cluster(); + header.group = consensus.group(); + header.view = 1; + header.replica = PRIMARY; + header.size = u32::try_from(HEADER_SIZE).unwrap(); + header.seal(); + }, + ); + consensus.handle_start_view_change(PlaneKind::Metadata, vote.header()); assert!( !consensus.is_normal(), - "a silent primary starts a view change" + "a quorum of votes starts a view change" ); assert_eq!( metadata diff --git a/core/server_common/src/consensus_message.rs b/core/server_common/src/consensus_message.rs index c981d4e373..cf73e993bb 100644 --- a/core/server_common/src/consensus_message.rs +++ b/core/server_common/src/consensus_message.rs @@ -20,10 +20,11 @@ use crate::sharding::METADATA_GROUP; use aligned_vec::{AVec, ConstAlign}; use futures::channel::oneshot; use iggy_binary_protocol::{ - Command, CommitHeader, ConsensusError, ConsensusHeader, ConsumerSessionHeartbeatHeader, - DoViewChangeHeader, ForwardCompletionHeader, ForwardLogoutHeader, ForwardLogoutResultHeader, - ForwardRegisterHeader, ForwardRegisterResultHeader, GenericHeader, HEADER_SIZE, Operation, - PrepareHeader, PrepareOkHeader, PrimaryRebalanceHeader, RepairPrepareHeader, + Command, CommitHeader, CommitReplyHeader, ConsensusError, ConsensusHeader, + ConsumerSessionHeartbeatHeader, DoViewChangeHeader, ForwardCompletionHeader, + ForwardLogoutHeader, ForwardLogoutResultHeader, ForwardRegisterHeader, + ForwardRegisterResultHeader, GenericHeader, HEADER_SIZE, Operation, PrepareHeader, + PrepareOkHeader, PrepareProgressHeader, PrimaryRebalanceHeader, RepairPrepareHeader, RepairRangeReplyHeader, RequestHeader, RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, RoutedRequestHeader, SegmentDeleteHeader, StartViewChangeHeader, StartViewHeader, StateChunkHeader, StateTransferTargetHeader, @@ -477,6 +478,13 @@ where self.backing.into_frozen() } + /// Allocation charge after freezing this frame into a retained journal entry. + /// Returns `None` if its allocation layout cannot be represented. + #[must_use] + pub fn retained_allocation_size(&self) -> Option { + Frozen::::allocation_size(self.backing.owned.backing_capacity()) + } + /// # Panics /// /// Panics if `H` and `T` have different sizes, or if the rewritten header @@ -657,6 +665,8 @@ pub enum MessageBag { DoViewChange(Message), StartView(Message), Commit(Message), + CommitReply(Message), + PrepareProgress(Message), RequestStartView(Message), RequestPrepares(Message), /// A journaled prepare re-sent verbatim for repair (command byte @@ -721,6 +731,8 @@ impl MessageBag { | Command::PrepareOk | Command::Reply | Command::Commit + | Command::CommitReply + | Command::PrepareProgress | Command::StartViewChange | Command::DoViewChange | Command::StartView @@ -784,6 +796,14 @@ impl MessageBag { let header = message.header(); (header.operation(), header.group) } + Self::CommitReply(message) => { + let header = message.header(); + (header.operation(), header.group) + } + Self::PrepareProgress(message) => { + let header = message.header(); + (header.operation(), header.group) + } Self::RequestStartView(message) => { let header = message.header(); (header.operation(), header.group) @@ -844,6 +864,8 @@ impl MessageBag { Self::DoViewChange(message) => message.into_generic(), Self::StartView(message) => message.into_generic(), Self::Commit(message) => message.into_generic(), + Self::CommitReply(message) => message.into_generic(), + Self::PrepareProgress(message) => message.into_generic(), Self::RequestStartView(message) => message.into_generic(), Self::RequestPrepares(message) => message.into_generic(), Self::RepairPrepare(message) => message.into_generic(), @@ -873,6 +895,8 @@ impl MessageBag { Self::DoViewChange(message) => message.header().command, Self::StartView(message) => message.header().command, Self::Commit(message) => message.header().command, + Self::CommitReply(message) => message.header().command, + Self::PrepareProgress(message) => message.header().command, Self::RequestStartView(message) => message.header().command, Self::RequestPrepares(message) => message.header().command, Self::RepairPrepare(message) => message.header().command(), @@ -902,6 +926,8 @@ impl MessageBag { Self::DoViewChange(message) => message.header().size(), Self::StartView(message) => message.header().size(), Self::Commit(message) => message.header().size(), + Self::CommitReply(message) => message.header().size(), + Self::PrepareProgress(message) => message.header().size(), Self::RequestStartView(message) => message.header().size(), Self::RequestPrepares(message) => message.header().size(), Self::RepairPrepare(message) => message.header().size(), @@ -931,6 +957,8 @@ impl MessageBag { Self::DoViewChange(message) => message.header().operation(), Self::StartView(message) => message.header().operation(), Self::Commit(message) => message.header().operation(), + Self::CommitReply(message) => message.header().operation(), + Self::PrepareProgress(message) => message.header().operation(), Self::RequestStartView(message) => message.header().operation(), Self::RequestPrepares(message) => message.header().operation(), Self::RepairPrepare(message) => message.header().operation(), @@ -1042,6 +1070,12 @@ where )), Command::StartView => Ok(Self::StartView(value.try_into_typed::()?)), Command::Commit => Ok(Self::Commit(value.try_into_typed::()?)), + Command::CommitReply => Ok(Self::CommitReply( + value.try_into_typed::()?, + )), + Command::PrepareProgress => Ok(Self::PrepareProgress( + value.try_into_typed::()?, + )), Command::RequestStartView => Ok(Self::RequestStartView( value.try_into_typed::()?, )), @@ -1108,6 +1142,7 @@ where #[cfg(test)] mod tests { use super::*; + use crate::sharding::IggyNamespace; use iggy_binary_protocol::{ ForwardLogoutHeader, ForwardLogoutOutcome, ForwardLogoutResultHeader, HEADER_SIZE, Operation, PrimaryRebalanceKind, ReplyHeader, RequestHeader, frame_checksum_bytes, @@ -1435,6 +1470,24 @@ mod tests { assert_ne!(bag.routing().1, METADATA_GROUP); } + #[test] + fn given_heartbeat_replies_when_classifying_links_should_preserve_their_group() { + for group in [METADATA_GROUP, IggyNamespace::new(1, 1, 0).inner()] { + let reply = Message::::new(HEADER_SIZE).transmute_header( + |_, header: &mut CommitReplyHeader| { + header.command = Command::CommitReply; + header.size = u32::try_from(HEADER_SIZE).unwrap(); + header.group = group; + header.timestamp_monotonic = 1; + header.seal(); + }, + ); + let bag = MessageBag::try_from(reply.into_generic()).unwrap(); + assert!(!MessageBag::routes_to_metadata(bag.command())); + assert_eq!(bag.routing().1, group); + } + } + fn primary_rebalance_frame(kind: PrimaryRebalanceKind) -> Message { Message::::new(HEADER_SIZE).transmute_header( |_, header: &mut PrimaryRebalanceHeader| { @@ -1721,6 +1774,7 @@ mod tests { Command::DoViewChange, Command::StartView, Command::Commit, + Command::CommitReply, ] { let bag = dispatch(cmd, 256).expect("dispatch"); assert_eq!(bag.command(), cmd, "round-trip for {cmd:?}"); diff --git a/core/server_common/src/iobuf.rs b/core/server_common/src/iobuf.rs index 0386938e2a..3d41ac45bb 100644 --- a/core/server_common/src/iobuf.rs +++ b/core/server_common/src/iobuf.rs @@ -118,6 +118,12 @@ impl Owned { &self.inner } + /// Capacity of the allocation retained when this buffer is frozen. + #[must_use] + pub fn backing_capacity(&self) -> usize { + self.inner.capacity() + } + pub fn as_mut_slice(&mut self) -> &mut [u8] { &mut self.inner } diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index 586ca01e03..7ef92129d6 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs @@ -55,14 +55,14 @@ use crossfire::AsyncRxTrait; use futures::FutureExt; use iggy_binary_protocol::responses::system::GetMetadataChangesResponse; use iggy_binary_protocol::{ - CHECKSUM_UNSEALED, Command, CommitHeader, ConsensusHeader, ConsumerSession, + CHECKSUM_UNSEALED, Command, CommitHeader, CommitReplyHeader, ConsensusHeader, ConsumerSession, ConsumerSessionHeartbeatHeader, DoViewChangeHeader, ForwardCompletionHeader, ForwardLogoutHeader, ForwardLogoutResultHeader, ForwardRegisterHeader, ForwardRegisterResultHeader, GenericHeader, Operation, PrepareHeader, PrepareOkHeader, - RepairPrepareHeader, RepairRangeReplyHeader, RequestPreparesHeader, RequestStartViewHeader, - RequestStateChunkHeader, RequestStateTransferHeader, RoutedRequestHeader, SegmentDeleteHeader, - SegmentDeleteKind, StartViewChangeHeader, StartViewHeader, StateChunkHeader, - StateTransferTargetHeader, WireDecode, + PrepareProgressHeader, RepairPrepareHeader, RepairRangeReplyHeader, RequestPreparesHeader, + RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, + RoutedRequestHeader, SegmentDeleteHeader, SegmentDeleteKind, StartViewChangeHeader, + StartViewHeader, StateChunkHeader, StateTransferTargetHeader, WireDecode, }; #[cfg(feature = "simulator")] use iggy_common::PartitionStats; @@ -82,7 +82,7 @@ use metadata::IggyMetadata; use metadata::impls::metadata::{PartitionPlacementView, StreamsFrontend, metadata_wal_head}; use metadata::stm::StateMachine; use metadata::{BoundSession, CommitWalkStop, MetadataSubmitError}; -use partitions::state_transfer::TransferArtifact; +use partitions::state_transfer::{PrefixSeed, TransferArtifact}; use partitions::{ FatalCommit, IggyPartition, IggyPartitions, PollFragments, PollingArgs, PollingConsumer, RepairIngest, @@ -95,8 +95,10 @@ use shards_table::ShardsTable; use std::cell::{Cell, RefCell}; use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; use std::future::Future; +use std::io; use std::rc::Rc; use std::sync::Arc; +use std::task::Poll; use std::time::{Duration, Instant}; pub type ShardPlane = @@ -1220,6 +1222,16 @@ impl ShardFrame { /// `[cluster] repair_chunk_max` at bootstrap. pub const REPAIR_CHUNK_MAX: u64 = 128; +/// Prepare bytes one partition `RequestPrepares` round serves. +/// +/// The reads run on the pump, so this shard's consensus ticks wait for them. +/// Bodies that lie back to back in a segment come back in one read. Resident +/// responses share the bound so receivers can reserve WAL space. 8 MiB +/// takes about 4 ms from a warm local SSD at 2 GB/s, under one 10 ms tick, +/// but about 67 ms from a block volume at 125 MB/s. The first body always +/// goes, so a batch above the budget still makes progress. +const REPAIR_BYTES_MAX: u64 = 8 * 1024 * 1024; + /// One in-flight metadata journal-repair stream (shard 0 only). #[derive(Debug, Clone, Copy)] struct MetadataRepairSession { @@ -1305,6 +1317,13 @@ const DEFAULT_BUS_MAX_MESSAGE_SIZE: usize = 64 * 1024 * 1024; /// protocol has no completion frame) or gave up. const STATE_TRANSFER_OFFER_EXPIRY_MULTIPLE: u32 = 10; +/// A parked partition transfer request waits for its plan pin for at most the +/// repair-retry interval divided by this. A requester re-sends a request that +/// one whole interval left unanswered, and that re-send burns a stall retry. +/// The refusal goes out with half the interval to spare for the network and +/// for the tick skew between the nodes. +const PARKED_TRANSFER_REQUEST_WAIT_DIVISOR: u32 = 2; + /// Lifetime of a FULLY SERVED offer, as a multiple of the repair-retry /// interval. It only has to outlive the receiver re-requesting a lost final /// chunk, but the receiver's stall re-request fires at exactly one such @@ -1313,6 +1332,16 @@ const STATE_TRANSFER_OFFER_EXPIRY_MULTIPLE: u32 = 10; /// drops the session with every byte already downloaded). const STATE_TRANSFER_SERVED_EXPIRY_MULTIPLE: u32 = 3; +/// A receiver waits for local work, a reuse scan or a prefix read, for at +/// most the serving offer's idle lifetime divided by this. The wait counts +/// from the last chunk request. A scan and a read can follow each other with +/// no request between them, so the bound covers both. The pull sends nothing +/// while it waits, so the offer ages the whole time. The other half of the +/// lifetime lets the first request after the wait reach the offer before it +/// expires. A scan that is asked to stop still finishes the file operation +/// that it runs. +const LOCAL_WORK_WAIT_DIVISOR: u32 = 2; + /// One in-flight metadata state transfer (shard 0 only): a cluster-restart /// rejoin replacing its snapshot-shaped state (metadata snapshot + client /// table) from the live primary before tail repair. @@ -1619,6 +1648,16 @@ impl<'a> TransferDescriptor<'a> { } } +/// A partition `RequestStateTransfer` that waits on the serving side for its +/// plan pin. The pin lands in an I/O turn after the request arrives. A refusal +/// costs the requester a whole re-arm interval, because a transient refusal +/// does not tell it that only the pin is missing. +struct ParkedTransferRequest { + header: RequestStateTransferHeader, + /// Ticks since the first arrival of this nonce. + waited_ticks: u32, +} + /// What `on_request_state_chunk` decided inside its offers borrow; the wire /// sends run after the borrow drops. enum ChunkReply { @@ -1715,6 +1754,7 @@ where /// `(namespace, requester replica id)`. Bounded by the replica count times /// the groups this shard serves; replaced per fresh nonce. state_transfer_offers: RefCell>, + partition_recovery: metadata::PartitionRecovery, /// Partition groups with an offer build under way but no offer yet, keyed /// by namespace and carrying ticks since the last request that advanced it. /// @@ -1724,6 +1764,16 @@ where /// clock as an idle offer, since a requester that walked away leaves /// nothing else to release the slot. partition_offer_builds: RefCell>, + /// Partition groups refused a served transfer for want of a slot, in the + /// order of their first refusal, each with the ticks since its last + /// refused request. Free slots go to the front of this queue, and a group + /// that stops asking ages out on the clock of an abandoned build. + partition_transfer_waiters: RefCell>, + /// Partition transfer requests that wait for their plan pin, keyed like + /// `state_transfer_offers`. The tick answers each one. [`BTreeMap`], not + /// `HashMap`: the tick answers in key order, so a replay stays + /// deterministic. + parked_partition_transfer_requests: RefCell>, /// See [`ServedSegmentCache`]. served_segment_cache: RefCell, @@ -1889,6 +1939,7 @@ where /// cursor the leading groups would spend the whole budget on every pass and /// the tail would never be reached. See [`rotate_sweep_to_cursor`]. partition_walk_cursor: Cell>, + replication_replay_cursor: Cell>, partition_placement_cursor: Cell>, /// Namespaces still owed a placement report in the current interval. partition_placement_owed: Cell, @@ -2092,6 +2143,7 @@ where let nonce_seed = boot_nonce(metadata.consensus.as_ref()); // Metadata admits a stream's wrap set with the keys its partitions open. let metadata = metadata.with_stream_key_context(partitions.key_context()); + let partition_recovery = metadata.applied_frontier().track_partition_recovery(); let plane = MuxPlane::new(variadic!(metadata, partitions)); let ShardIdentity { id, name } = identity; let poll_completions = @@ -2136,7 +2188,10 @@ where metadata_gap_ticks: Cell::new(0), metadata_walk_stuck_op: Cell::new(0), state_transfer_offers: RefCell::new(HashMap::new()), + partition_recovery, partition_offer_builds: RefCell::new(HashMap::new()), + partition_transfer_waiters: RefCell::new(Vec::new()), + parked_partition_transfer_requests: RefCell::new(BTreeMap::new()), served_segment_cache: RefCell::new(ServedSegmentCache::default()), register_forwards: RefCell::new(HashMap::new()), logout_forwards: RefCell::new(HashMap::new()), @@ -2149,6 +2204,7 @@ where repair_gap_debounce_ticks: Cell::new(partitions::REPAIR_RETRY_TICKS), partition_repairs_inflight: Cell::new(0), partition_walk_cursor: Cell::new(None), + replication_replay_cursor: Cell::new(None), partition_placement_cursor: Cell::new(None), partition_placement_owed: Cell::new(0), primary_rebalance: RefCell::default(), @@ -2977,6 +3033,7 @@ where let nonce_seed = boot_nonce(metadata.consensus.as_ref()); // Metadata admits a stream's wrap set with the keys its partitions open. let metadata = metadata.with_stream_key_context(partitions.key_context()); + let partition_recovery = metadata.applied_frontier().track_partition_recovery(); let plane = MuxPlane::new(variadic!(metadata, partitions)); let ShardIdentity { id, name } = identity; let host: Rc = Rc::new(NoopHost); @@ -3026,7 +3083,10 @@ where metadata_gap_ticks: Cell::new(0), metadata_walk_stuck_op: Cell::new(0), state_transfer_offers: RefCell::new(HashMap::new()), + partition_recovery, partition_offer_builds: RefCell::new(HashMap::new()), + partition_transfer_waiters: RefCell::new(Vec::new()), + parked_partition_transfer_requests: RefCell::new(BTreeMap::new()), served_segment_cache: RefCell::new(ServedSegmentCache::default()), register_forwards: RefCell::new(HashMap::new()), logout_forwards: RefCell::new(HashMap::new()), @@ -3039,6 +3099,7 @@ where repair_gap_debounce_ticks: Cell::new(partitions::REPAIR_RETRY_TICKS), partition_repairs_inflight: Cell::new(0), partition_walk_cursor: Cell::new(None), + replication_replay_cursor: Cell::new(None), partition_placement_cursor: Cell::new(None), partition_placement_owed: Cell::new(0), primary_rebalance: RefCell::default(), @@ -4048,6 +4109,8 @@ where MessageBag::DoViewChange(msg) => self.on_do_view_change(msg).await, MessageBag::StartView(msg) => self.on_start_view(msg).await, MessageBag::Commit(ref msg) => self.on_commit(msg).await, + MessageBag::CommitReply(ref msg) => self.on_commit_reply(msg), + MessageBag::PrepareProgress(ref msg) => self.on_prepare_progress(msg).await, MessageBag::RequestStartView(ref msg) => self.on_request_start_view(msg).await, MessageBag::RequestPrepares(ref msg) => self.on_request_prepares(msg).await, MessageBag::RepairPrepare(msg) => self.on_repair_prepare(msg).await, @@ -5813,7 +5876,11 @@ where // Ahead of `handle_commit`, which drops a lower-view heartbeat from // the real primary once this replica's view climbed alone. consensus.observe_primary(header.view, header.replica); - match consensus.handle_commit(&header) { + let outcome = consensus.handle_commit(&header); + if let Some(action) = outcome.reply(&header) { + dispatch_vsr_actions::(consensus, None, &[action]).await; + } + match outcome { CommitOutcome::Advanced => { // Mid-transfer the pre-install STM must not walk: the // snapshot being installed already contains those ops. @@ -5851,7 +5918,7 @@ where respond_start_view::(consensus).await; } } - CommitOutcome::Accepted => {} + CommitOutcome::Accepted | CommitOutcome::Ignored => {} } return; } @@ -5872,7 +5939,11 @@ where // elections nobody joined drops the real primary's lower-view // heartbeats there, and they are how it learns whom to repair from. consensus.observe_primary(header.view, header.replica); - match consensus.handle_commit(&header) { + let outcome = consensus.handle_commit(&header); + if let Some(action) = outcome.reply(&header) { + dispatch_vsr_actions::(consensus, None, &[action]).await; + } + match outcome { CommitOutcome::Advanced => { if !partition.consensus().is_transferring() { partition.commit_journal(config).await; @@ -5899,7 +5970,21 @@ where respond_start_view::(consensus).await; } } - CommitOutcome::Accepted => {} + CommitOutcome::Accepted | CommitOutcome::Ignored => {} + } + } + + fn on_commit_reply(&self, msg: &Message) { + let header = msg.header(); + let planes = self.plane.inner(); + if let Some(consensus) = &planes.0.consensus + && consensus.group() == header.group + { + consensus.handle_commit_reply(header); + return; + } + if let Some(partition) = planes.1.0.get_by_ns(&IggyNamespace::from_raw(header.group)) { + partition.consensus().handle_commit_reply(header); } } @@ -5967,23 +6052,235 @@ where dispatch_partition_wire_actions::(consensus, partition, wire_actions).await; } + #[allow(clippy::future_not_send)] + async fn on_prepare_progress(&self, msg: &Message) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: + Journal, Header = PrepareHeader>, + { + let header = *msg.header(); + let planes = self.plane.inner(); + let (consensus, progress) = if let Some(consensus) = &planes.0.consensus + && consensus.group() == header.group + { + if header.kind == PrepareProgressHeader::REPLY { + let _ = consensus.handle_prepare_progress(&header); + return; + } + let ready = consensus.is_normal() + && !consensus.is_transferring() + && !consensus.has_untrusted_log() + && consensus.history_certified(); + let mut received = consensus.commit_min(); + if ready && let Some(journal) = &planes.0.journal { + let journal = journal.handle(); + while received < consensus.sequencer().current_sequence() { + let next = received + 1; + let Ok(index) = usize::try_from(next) else { + break; + }; + if !consensus.view_log_entry_proven(next) || journal.header(index).is_none() { + break; + } + received = next; + } + } + (consensus, ready.then_some(received)) + } else { + let Some(partition) = planes.1.0.get_by_ns(&IggyNamespace::from_raw(header.group)) + else { + return; + }; + let consensus = partition.consensus(); + if header.kind == PrepareProgressHeader::REPLY { + if let Some(progress) = consensus.handle_prepare_progress(&header) { + partition.advance_transfer_tail(header.replica, progress, self.ticks.get()); + } + return; + } + (consensus, partition.replication_progress()) + }; + if header.kind != PrepareProgressHeader::PROBE + || header.nonce == 0 + || header.cluster != consensus.cluster() + || header.view != consensus.view() + || header.partition_incarnation != consensus.partition_incarnation() + || header.replica != consensus.primary_index(consensus.view()) + || header.replica == consensus.replica() + { + return; + } + if consensus.is_normal() && !consensus.is_transferring() { + consensus.advance_commit_max(header.commit_op); + } + let reply = Message::::new(size_of::()) + .transmute_header(|_, reply: &mut PrepareProgressHeader| { + *reply = header; + reply.replica = consensus.replica(); + reply.kind = PrepareProgressHeader::REPLY; + reply.ready = u8::from(progress.is_some()); + reply.received_op = progress.unwrap_or(0); + reply.seal(); + }); + let _ = self + .bus + .send_to_replica(header.replica, reply.into_generic().into_frozen()) + .await; + } + + #[allow(clippy::future_not_send)] + async fn drive_replication(&self, namespaces: &[IggyNamespace]) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: + Journal, Header = PrepareHeader>, + { + let metadata_group = self + .plane + .metadata() + .consensus + .as_ref() + .map(VsrConsensus::group); + let groups = metadata_group + .into_iter() + .chain(namespaces.iter().map(IggyNamespace::inner)); + let cursor = self.replication_replay_cursor.get(); + // Prefer keys after the cursor even when the commit-walk order rotates. + let rank = |key| (cursor.is_some_and(|last| key <= last), key); + let mut replay: Option<(u64, u8, u64)> = None; + for group in groups { + let Some(consensus) = self.replication_consensus(group) else { + continue; + }; + for replica in 0..consensus.replica_count() { + if replica == consensus.replica() { + continue; + } + match consensus.replication_work(replica) { + Some(consensus::ReplicationWork::Probe { nonce, through, .. }) => { + let header_size = size_of::(); + let probe = Message::::new(header_size) + .transmute_header(|_, header: &mut PrepareProgressHeader| { + header.command = Command::PrepareProgress; + header.size = + u32::try_from(header_size).expect("header size fits u32"); + header.cluster = consensus.cluster(); + header.view = consensus.view(); + header.replica = consensus.replica(); + header.group = group; + header.partition_incarnation = consensus.partition_incarnation(); + header.nonce = nonce; + header.observed_op = through; + header.commit_op = consensus.commit_min(); + header.kind = PrepareProgressHeader::PROBE; + header.seal(); + }); + let _ = self + .bus + .send_to_replica(replica, probe.into_generic().into_frozen()) + .await; + } + Some(consensus::ReplicationWork::Replay { from_op, .. }) + if replay.is_none_or(|(selected_group, selected_peer, _)| { + rank((group, replica)) < rank((selected_group, selected_peer)) + }) => + { + replay = Some((group, replica, from_op)); + } + Some(consensus::ReplicationWork::Replay { .. }) | None => {} + } + } + } + // One bounded range per tick keeps replay I/O independent of group count. + let Some((group, replica, from_op)) = replay else { + return; + }; + self.replication_replay_cursor.set(Some((group, replica))); + let request = { + let Some(consensus) = self.replication_consensus(group) else { + return; + }; + Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestPreparesHeader| { + header.command = Command::RequestPrepares; + header.size = u32::try_from(size_of::()) + .expect("header size fits u32"); + header.replica = replica; + header.group = group; + header.partition_incarnation = consensus.partition_incarnation(); + header.from_op = from_op; + header.to_op = consensus.sequencer().current_sequence(); + }) + }; + if self.serve_prepares(&request, true).await < from_op + && let Some(consensus) = self.replication_consensus(group) + { + consensus.defer_replication(replica); + } + } + + fn replication_consensus(&self, group: u64) -> Option<&VsrConsensus> + where + B: MessageBus, + { + let planes = self.plane.inner(); + if let Some(consensus) = &planes.0.consensus + && consensus.group() == group + { + return Some(consensus); + } + let partition = planes.1.0.get_by_ns(&IggyNamespace::from_raw(group))?; + (!partition.read_history_is_changing()).then(|| partition.consensus()) + } + + #[allow(clippy::future_not_send)] + async fn send_range_prepare( + &self, + consensus: &VsrConsensus, + target: u8, + op: u64, + entry: Frozen, + live: bool, + ) -> bool + where + B: MessageBus, + { + if !live { + return self.send_repair_prepare(target, entry).await; + } + send_replayed_prepare(consensus, target, op, entry).await + } + + #[allow(clippy::future_not_send)] + async fn on_request_prepares(&self, msg: &Message) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: + Journal, Header = PrepareHeader>, + { + self.serve_prepares(msg, false).await; + } + /// Serve a repair range from this replica's journal: stream /// `RepairPrepare` frames (stored prepares verbatim, command byte /// rewritten) in op order, prefixed by `RangeEvicted` when the front of /// the range is no longer retained, terminated by `RepairDone`. #[allow(clippy::future_not_send, clippy::too_many_lines)] - async fn on_request_prepares(&self, msg: &Message) + async fn serve_prepares(&self, msg: &Message, live: bool) -> u64 where B: MessageBus, MJ: JournalHandle, ::Target: Journal, Header = PrepareHeader>, - M: StreamsFrontend, { let header = *msg.header(); let target = header.replica; - // Snapshot the config-overridable chunk ceiling once; both plane - // branches below serve the same per-round window. + // Pull repair keeps its configured operation limit. Live partition + // replay uses the byte bound so small prepares can fill each turn. let repair_chunk_max = self.repair_chunk_max.get(); let planes = self.plane.inner(); if let Some(ref consensus) = planes.0.consensus @@ -5993,18 +6290,22 @@ where // Other requesters may receive only those peers' committed prefix. if consensus.is_transferring() { // Transfer can replace these entries. - return; + return header.from_op.saturating_sub(1); } if !matches!(consensus.status(), Status::Normal | Status::ViewChange) { - return; + return header.from_op.saturating_sub(1); } let Some(journal) = planes.0.journal.as_ref() else { - return; + return header.from_op.saturating_sub(1); }; let journal = journal.handle(); let cluster = consensus.cluster(); let self_id = consensus.replica(); - let to_op = repair_serve_ceiling(&header, consensus); + let to_op = if live { + header.to_op.min(consensus.sequencer().current_sequence()) + } else { + repair_serve_ceiling(&header, consensus) + }; // Skip the compacted prefix (below the snapshot floor) in one // RangeEvicted notice, then serve contiguously until the range // ends or the WAL runs out. @@ -6016,13 +6317,16 @@ where // servable anyway. The partition arm jumps to `retained_from` likewise. let mut from_op = header.from_op.max(journal.snapshot_op() + 1); if !consensus.view_log_entry_proven(from_op) { - return; + return header.from_op.saturating_sub(1); } #[allow(clippy::cast_possible_truncation)] while from_op <= to_op && journal.header(from_op as usize).is_none() { from_op += 1; } if from_op > to_op { + if live { + return header.from_op.saturating_sub(1); + } // Nothing in the requested range is retained. Answer the // eviction honestly: a bare `RepairDone(to_op)` here would // claim full coverage while serving zero prepares, and the @@ -6049,9 +6353,12 @@ where consensus.partition_incarnation(), ) .await; - return; + return header.from_op.saturating_sub(1); } if from_op > header.from_op { + if live { + return header.from_op.saturating_sub(1); + } self.send_repair_range_reply( cluster, self_id, @@ -6078,43 +6385,54 @@ where break; }; if !self - .send_repair_prepare(target, entry.into_generic().into_frozen()) + .send_range_prepare( + consensus, + target, + op, + entry.into_generic().into_frozen(), + live, + ) .await { break; } served_through = op; } - self.send_repair_range_reply( - cluster, - self_id, - target, - Command::RepairDone, - header.nonce, - served_through, - header.group, - consensus.partition_incarnation(), - ) - .await; - return; + if !live { + self.send_repair_range_reply( + cluster, + self_id, + target, + Command::RepairDone, + header.nonce, + served_through, + header.group, + consensus.partition_incarnation(), + ) + .await; + } + return served_through; } let namespace = IggyNamespace::from_raw(header.group); let Some(partition) = planes.1.0.get_mut_by_ns(&namespace) else { - return; + return header.from_op.saturating_sub(1); }; if !self.partition_incarnation_matches( partition, header.partition_incarnation, Command::RequestPrepares, ) { - return; + return header.from_op.saturating_sub(1); } if !matches!( partition.consensus().status(), Status::Normal | Status::ViewChange ) || partition.view_truncation_pending() { - return; + return header.from_op.saturating_sub(1); + } + if !live { + partition.renew_transfer_tail_for_repair(target, self.ticks.get()); } let cluster = partition.consensus().cluster(); let self_id = partition.consensus().replica(); @@ -6124,7 +6442,13 @@ where // until the body is journaled, and the primary's retransmit is dropped // by the backup gap check (adoption already advanced its sequencer to // the head), so repair is the only channel that can deliver them. - let to_op = repair_serve_ceiling(&header, partition.consensus()); + let to_op = if live { + header + .to_op + .min(partition.consensus().sequencer().current_sequence()) + } else { + repair_serve_ceiling(&header, partition.consensus()) + }; let fence_floor = partition.view_fenced().then(|| partition.fence_floor()); // `None` means the journal holds NOTHING, not "nothing was evicted": // the partition journal is memory-only and `clear_all` wipes the @@ -6145,9 +6469,12 @@ where .unwrap_or_else(|| partition.consensus().commit_min().saturating_add(1)); let mut from_op = header.from_op; if !partition.consensus().view_log_entry_proven(from_op) { - return; + return header.from_op.saturating_sub(1); } if retained_from > from_op { + if live { + return header.from_op.saturating_sub(1); + } self.send_repair_range_reply( cluster, self_id, @@ -6161,48 +6488,120 @@ where .await; from_op = retained_from; } - let chunk_end = to_op.min(from_op.saturating_add(repair_chunk_max - 1)); - let mut served_through = from_op.saturating_sub(1); + let chunk_ops_max = if live { + REPAIR_BYTES_MAX / size_of::() as u64 + } else { + repair_chunk_max + }; + let chunk_end = to_op.min(from_op.saturating_add(chunk_ops_max - 1)); + let journal = &partition.log.journal().inner; + // Every op is checked and taken before the first await, and the bodies + // the ring released come back from disk in one pass, so the bodies that + // lie back to back in a segment take one read. `None` stands for the + // next body read back. + let mut entries = Vec::new(); + let mut released = Vec::new(); + let mut response_bytes = 0; for op in from_op..=chunk_end { if !partition.consensus().view_log_entry_proven(op) { break; } - let Some(held) = partition.log.journal().inner.repair_header(op) else { - break; + let (held, evicted) = match journal.repair_header(op) { + Some(held) => (held, None), + None => match journal.evicted_header(op) { + Some(evicted) => (*evicted.header(), Some(evicted)), + None => break, + }, }; if !fenced_serve_admits(fence_floor, op, || { partition.consensus().adopted_view_vouches_for(&held) }) { break; } - let Some(entry) = partition.log.journal().inner.repair_entry(op) else { + if response_bytes > 0 && response_bytes + u64::from(held.size) > REPAIR_BYTES_MAX { + break; + } + response_bytes += u64::from(held.size); + if let Some(evicted) = evicted { + released.push(evicted); + entries.push(None); + } else { + let Some(entry) = journal.repair_entry(op) else { + break; + }; + entries.push(Some(entry)); + } + } + let mut bodies = Vec::with_capacity(released.len()); + let failed = + match partitions::read_released_bodies(&partition.log, &released, &mut bodies).await { + Ok(()) => None, + Err(error) => released + .get(bodies.len()) + .map(|evicted| (evicted.header().op, error)), + }; + let mut bodies = bodies.into_iter(); + let mut served_through = from_op.saturating_sub(1); + for (op, entry) in (from_op..).zip(entries) { + let Some(entry) = entry.or_else(|| bodies.next()) else { break; }; - if !self.send_repair_prepare(target, entry).await { + if !self + .send_range_prepare(partition.consensus(), target, op, entry, live) + .await + { break; } served_through = op; } - self.send_repair_range_reply( - cluster, - self_id, - target, - Command::RepairDone, - header.nonce, - served_through, - header.group, - partition.created_revision(), - ) - .await; - tracing::info!( - shard = self.id, - namespace_raw = header.group, - target, - from_op = header.from_op, - to_op, - served_through, - "served partition repair range" - ); + if let Some((op, error)) = failed { + // Bytes that are not the body and a body that left its file stay + // that way. Any other error can pass, so a later round reads again. + if matches!( + error.kind(), + io::ErrorKind::InvalidData | io::ErrorKind::UnexpectedEof | io::ErrorKind::NotFound + ) { + tracing::warn!( + shard = self.id, + namespace_raw = header.group, + op, + %error, + "cannot serve a repair body from its segment, so ops up to it answer RangeEvicted" + ); + journal.forget_released_through(op); + } else { + tracing::warn!( + shard = self.id, + namespace_raw = header.group, + op, + %error, + "cannot read a repair body from its segment, so a later round reads it again" + ); + } + } + if !live { + self.send_repair_range_reply( + cluster, + self_id, + target, + Command::RepairDone, + header.nonce, + served_through, + header.group, + partition.created_revision(), + ) + .await; + tracing::info!( + shard = self.id, + namespace_raw = header.group, + target, + from_op = header.from_op, + to_op, + served_through, + "served partition repair range" + ); + } + served_through } /// Ingest one repaired prepare. Metadata journals it into the WAL (the @@ -6565,6 +6964,7 @@ where session.to_op, header.group, consensus.partition_incarnation(), + None, ) .await; } @@ -6759,9 +7159,10 @@ where // delivery on a lossy bus). Proof moves backward through gaps, // then may need a higher canonical header's body. Either change // pulls the next chunk without waiting for the retry timer, and so - // do a held run that reached the chunk just requested and a - // floor-held resident prefix that grew. A shorter run is left to - // the retry timer. + // do a held run that reached the chunk just requested, a + // floor-held resident prefix that grew and an unproven walk that + // gained ops below its hole. A shorter run is left to the retry + // timer. if let partitions::RepairConclusion::FloorRefused { floor, to_op } = partition.complete_repair(&config).await { @@ -6835,8 +7236,13 @@ where ) }) }) + .or_else(|| partition.next_proof_walk_pull()) .or_else(|| partition.next_floor_pull()); let next = from_op.and_then(|from_op| { + if !partition.repair_fetch_ready() { + partition.repair.as_mut()?.requested_from = None; + return None; + } let live = partition.repair.as_mut()?; live.from_op = from_op; live.requested_from = Some(from_op); @@ -6851,6 +7257,7 @@ where to_op, header.group, partition.created_revision(), + partition.note_proof_walk_request(), ) .await; } @@ -6863,7 +7270,8 @@ where /// Ask `target` to stream its journaled prepares in `[from_op, to_op]` /// for `namespace`; answered by `RepairPrepare` frames terminated with /// `RepairDone` (prefixed by `RangeEvicted` when the front of the range - /// is no longer retained). + /// is no longer retained). `held_below_hole` is the highest op that a + /// partition holds below the hole of its unproven proof walk. #[allow(clippy::too_many_arguments)] #[allow(clippy::future_not_send, clippy::cast_possible_truncation)] async fn send_request_prepares

( @@ -6875,12 +7283,14 @@ where to_op: u64, namespace: u64, partition_incarnation: u64, + held_below_hole: Option, ) where B: MessageBus, P: Pipeline, { // Proof walks backward. End the chunk at its hole instead of re-serving - // the already held suffix for every missing predecessor. + // the already held suffix for every missing predecessor, and start it + // above the ops held below the hole, which the donor would serve first. let (from_op, to_op) = consensus .view_log_repair_op() .map_or((from_op, to_op), |missing| { @@ -6890,7 +7300,8 @@ where ( missing .saturating_sub(self.repair_chunk_max.get() - 1) - .max(consensus.commit_min() + 1), + .max(consensus.commit_min() + 1) + .max(held_below_hole.map_or(0, |held| held + 1)), missing, ) }); @@ -7235,6 +7646,7 @@ where to_op, consensus.group(), consensus.partition_incarnation(), + None, ) .await; } @@ -7379,6 +7791,7 @@ where to_op, consensus.group(), consensus.partition_incarnation(), + None, ) .await; } @@ -7579,6 +7992,7 @@ where to_op, consensus.group(), consensus.partition_incarnation(), + None, ) .await; } @@ -8749,6 +9163,11 @@ where self.ticks.set(self.ticks.get().wrapping_add(1)); let partitions = self.plane.partitions(); let repair_retry_ticks = self.repair_retry_ticks.get(); + // The serving side ages an idle offer on this clock, `max(1)` included. + let local_work_ticks_max = repair_retry_ticks + .max(1) + .saturating_mul(STATE_TRANSFER_OFFER_EXPIRY_MULTIPLE) + / LOCAL_WORK_WAIT_DIVISOR; self.partition_io.tick(); for identity in self.partition_io.interrupted() { if let Some(partition) = partitions.get_io_owner(&identity.namespace) { @@ -8787,6 +9206,11 @@ where let mut untrusted_logs = 0usize; for namespace in namespace_scratch.iter() { if let Some(partition) = partitions.get_mut_by_ns(namespace) { + partition.expire_transfer_tails( + self.ticks.get(), + u64::from(repair_retry_ticks.max(1)) + * u64::from(STATE_TRANSFER_OFFER_EXPIRY_MULTIPLE), + ); partition.drive_persistence().await; untrusted_logs += usize::from(partition.consensus().has_untrusted_log()); let (entries, bytes) = partition.repair_ring_occupancy(); @@ -8806,6 +9230,7 @@ where } } } + self.drive_replication(namespace_scratch).await; self.metrics.record_persistence(&persistence_metrics); self.partition_io.record_metrics(&self.metrics); self.metrics @@ -9015,8 +9440,9 @@ where continue; } let proof_op = partition.consensus().view_log_repair_op(); + let fetch_ready = partition.repair_fetch_ready(); let retry_due = partition.repair.as_mut().is_some_and(|session| { - if !session_live { + if !session_live || !fetch_ready { return false; } session.idle_ticks += 1; @@ -9180,6 +9606,7 @@ where to_op, namespace.inner(), partition.created_revision(), + partition.note_proof_walk_request(), ) .await; } @@ -9333,12 +9760,54 @@ where // the rejoin behind it) forever. Budget-bounded: a peer that died // mid-transfer is abandoned back to journal repair, which // re-targets the current primary. + let mut prefix_read_expired = false; + let mut reuse_scan_expired = false; + let mut local_work_landed = false; + let mut local_work_keepalive = None; let transfer_stalled = { let Some(partition) = partitions.get_mut_by_ns(&namespace) else { continue; }; partition.transfer.as_mut().and_then(|session| { session.idle_ticks += 1; + // A nonce-preserving retry renews the same pinned offer + // while this node validates or syncs its local files. + if session.target_accepted + && (session.scan.is_some() + || session.spill.is_some() + || matches!(session.seed, Some(PrefixSeed::Reading { .. }))) + && session.idle_ticks.is_multiple_of(repair_retry_ticks.max(1)) + { + local_work_keepalive = Some((session.peer, session.nonce)); + } + // Local work, not the peer, holds the pull, so a slow disk + // burns no retry of the peer. A prefix read that outlives + // its bound gives up the seed, and the artifact pulls whole. + // A reuse scan that outlives its bound is asked to stop, + // and the pull fetches what it did not reuse once it lands. + // Work that landed resumes the pull here when no I/O turn + // did. + if let Some(scan) = &session.scan { + if scan.landed() { + local_work_landed = true; + } else if session.idle_ticks >= local_work_ticks_max && scan.stop() { + reuse_scan_expired = true; + } + return None; + } + if let Some(spill) = &session.spill { + local_work_landed = spill.work.landed(); + return None; + } + if let Some(PrefixSeed::Reading { read, .. }) = &session.seed { + if read.landed() { + local_work_landed = true; + } else if session.idle_ticks >= local_work_ticks_max { + session.seed = None; + prefix_read_expired = true; + } + return None; + } if session.idle_ticks < repair_retry_ticks { return None; } @@ -9346,6 +9815,31 @@ where Some((session.peer, session.nonce, session.target_accepted)) }) }; + if reuse_scan_expired { + tracing::info!( + shard = self.id, + namespace_raw = namespace.inner(), + "the reuse scan outlived its bound, so the pull fetches the segments \ + that it did not reuse" + ); + } + if prefix_read_expired { + tracing::info!( + shard = self.id, + namespace_raw = namespace.inner(), + "the local segment prefix read outlived its bound, so the pull fetches \ + the whole artifact" + ); + } + if let Some((peer, nonce)) = local_work_keepalive + && let Some(partition) = partitions.get_by_ns(&namespace) + { + self.send_request_state_transfer(partition.consensus(), peer, nonce) + .await; + } + if prefix_read_expired || local_work_landed { + self.on_partition_transfer_progress(namespace.inner()).await; + } if let Some((peer, nonce, target_accepted)) = transfer_stalled { let Some(partition) = partitions.get_mut_by_ns(&namespace) else { continue; @@ -9359,8 +9853,8 @@ where ); // Staging files are KEPT: a later attempt adopts every // segment whose manifest entry still matches. The shared - // path charges the failure, rotates the peer, schedules - // the re-arm, and re-arms journal repair meanwhile. + // path charges the failure, rotates the peer and schedules + // the re-arm. self.abandon_or_rearm_partition_transfer(partition, peer) .await; } else if target_accepted { @@ -9375,13 +9869,21 @@ where // Scheduled transfer re-arm: count the backoff down and fire // once nothing else recovered the partition in the meantime (a // live session or a non-Idle stage owns the slot; the pending - // entry is then dropped as superseded). + // entry is then dropped as superseded). A journal that holds every + // op through `commit_max` is recovered as well, by the live stream + // or by repair, and the commit walk owns the rest. That re-arm + // stands down at once, not at the end of its backoff: while it + // waits, every repair route stays closed, so a gap that opens + // meanwhile heals only through a transfer. let rearm_peer = { let Some(partition) = partitions.get_mut_by_ns(&namespace) else { continue; }; + let journal_recovered = partition.transfer_rearm.is_some() + && !partition.requires_state_transfer() + && partition.held_through() >= partition.consensus().commit_max(); match partition.transfer_rearm.as_mut() { - Some(pending) if pending.after_ticks > 0 => { + Some(pending) if pending.after_ticks > 0 && !journal_recovered => { pending.after_ticks -= 1; None } @@ -9391,9 +9893,7 @@ where if partition.transfer.is_none() && partition.consensus().state_transfer_stage() == consensus::StateTransferStage::Idle - && (partition.requires_state_transfer() - || partition.consensus().commit_min() - < partition.consensus().commit_max()) + && !journal_recovered { Some(peer) } else { @@ -9427,18 +9927,25 @@ where // Count after every rotation, transfer abandonment and scheduled arm, // including paths that skip the rest of their partition's sweep. let mut repairs_live = 0usize; + let mut recovering = false; // A stranded key stays until its own store or delete succeeds, so a // gauge that never falls is the operator's only signal. let mut stranded = [0usize; ConsumerKind::COUNT]; for namespace in partitions.namespaces() { if let Some(partition) = partitions.get_by_ns(namespace) { repairs_live += usize::from(partition.repair.is_some()); + recovering |= partition.repair.is_some() + || partition.transfer.is_some() + || partition.transfer_rearm.is_some() + || partition.consensus().is_transferring() + || partition.consensus().has_untrusted_log(); for kind in ConsumerKind::ALL { stranded[kind.index()] += partition.stranded_consumer_offset_count(kind); } } } self.partition_repairs_inflight.set(repairs_live); + self.partition_recovery.set_active(recovering); for kind in ConsumerKind::ALL { self.metrics .set_consumer_offsets_stranded(kind, stranded[kind.index()]); @@ -9570,39 +10077,115 @@ where usize::try_from(slots).unwrap_or(usize::MAX).max(1) } - fn may_serve_another_partition_transfer(&self, namespace: u64) -> bool { + /// Admit a served transfer of `namespace`, or queue the group for a free + /// slot. A group with a build under way or an offer still being pulled + /// holds a slot, and its requesters share it. A fully served offer holds + /// none: nothing pulls its payload any more, so the cache evicts it ahead + /// of the payloads that live pulls keep using. Free slots go to refused + /// groups in the order of their first refusal, so a group whose transfer + /// just finished queues behind the groups that waited while it was served. + fn admit_partition_transfer(&self, namespace: u64) -> bool { let builds = self.partition_offer_builds.borrow(); - if builds.contains_key(&namespace) { - return true; - } let offers = self.state_transfer_offers.borrow(); - let mut served: Vec = offers - .iter() - .filter(|(_, served)| matches!(served.offer, ServedOffer::Partition(_))) - .map(|((offer_namespace, _), _)| *offer_namespace) - .collect(); - if served.contains(&namespace) { - return true; - } // Builds count too. A multi-round checksum pass holds no offer yet, so // counting only completed offers admitted every requester's whole // in-flight set at once and let each run its own pass: the frame bodies // stay bounded, but the pump carries N budgets per round-cycle and // every other frame, produce included, queues behind them. - served.extend(builds.keys().copied()); - served.sort_unstable(); - served.dedup(); - served.len() < self.partition_transfer_admission_cap() + let mut holders: Vec = offers + .iter() + .filter(|(_, served)| { + !served.fully_served && matches!(served.offer, ServedOffer::Partition(_)) + }) + .map(|((offer_namespace, _), _)| *offer_namespace) + .chain(builds.keys().copied()) + .collect(); + if holders.contains(&namespace) { + return true; + } + holders.sort_unstable(); + holders.dedup(); + let free = self + .partition_transfer_admission_cap() + .saturating_sub(holders.len()); + let mut waiters = self.partition_transfer_waiters.borrow_mut(); + let queued = waiters.iter().position(|(waiter, _)| *waiter == namespace); + if queued.unwrap_or(waiters.len()) < free { + if let Some(position) = queued { + waiters.remove(position); + } + return true; + } + match queued { + Some(position) => waiters[position].1 = 0, + None => waiters.push((namespace, 0)), + } + false } /// Serve one partition `RequestStateTransfer`: build (or re-serve) this /// group's offer and answer with the descriptor. - #[allow(clippy::future_not_send, clippy::too_many_lines)] + #[allow(clippy::future_not_send)] async fn on_partition_request_state_transfer(&self, msg: &Message) where B: MessageBus, { - let header = *msg.header(); + self.serve_partition_transfer_request(*msg.header(), None) + .await; + } + + /// Answer the parked partition transfer requests: with the offer once the + /// plan is pinned, or with a refusal once a request waited its bound. + /// + /// A tick starts at most one offer build, because a build reads and hashes + /// up to a whole round budget, and the ticks and heartbeats of every group + /// on this shard wait for the tick to end. The other pinned requests stay + /// parked, and their wait keeps counting. + /// + /// The bound refuses only a request that still waits for its pin, so a + /// pinned request builds on a later tick even past its bound. It is late + /// by at most one tick per other pinned request. Admission holds the + /// parked requests to its cap times the requesters of a group, 2 x 2 for 3 + /// replicas at the defaults, so the answer still arrives inside the half + /// retry interval that the bound leaves before the requester re-sends. + /// With a larger cap, a request can miss that window, and its re-send then + /// builds the offer in its own frame. + #[allow(clippy::future_not_send)] + async fn serve_parked_partition_transfer_requests(&self) + where + B: MessageBus, + { + let parked = std::mem::take(&mut *self.parked_partition_transfer_requests.borrow_mut()); + let mut build_started = false; + for (key, mut request) in parked { + request.waited_ticks += 1; + let builds = self + .plane + .partitions() + .get_by_ns(&IggyNamespace::from_raw(key.0)) + .is_some_and(IggyPartition::transfer_plan_pinned); + if builds && build_started { + self.parked_partition_transfer_requests + .borrow_mut() + .insert(key, request); + continue; + } + build_started |= builds; + self.serve_partition_transfer_request(request.header, Some(request.waited_ticks)) + .await; + } + } + + /// `parked_wait` holds the ticks that a parked request waited when the + /// tick answers it, and `None` for a request from the wire. + #[allow(clippy::future_not_send, clippy::too_many_lines)] + async fn serve_partition_transfer_request( + &self, + header: RequestStateTransferHeader, + parked_wait: Option, + ) where + B: MessageBus, + { // Before anything keyed by the requester: the offer map's documented // bound is the replica count, and an unvalidated id makes it 256 entries // per served group, each pinning an offer for its full expiry. @@ -9625,6 +10208,16 @@ where ) { return; } + // Only a request that passed the gates replaces a parked one. A re-send + // keeps the wait that its nonce already spent parked, so re-sends + // cannot hold the build slot past the bound. + let waited_ticks = parked_wait.unwrap_or_else(|| { + self.parked_partition_transfer_requests + .borrow_mut() + .remove(&(header.group, header.replica)) + .filter(|parked| parked.header.nonce == header.nonce) + .map_or(0, |parked| parked.waited_ticks) + }); let cluster = partition.consensus().cluster(); let self_id = partition.consensus().replica(); // First-wins per (requester, nonce), exactly as the metadata arm: a @@ -9644,39 +10237,27 @@ where served.idle_ticks = 0; Some(Rc::clone(offer)) }); + let serves_transfer = partition.serves_state_transfer(); + let admitted = + cached.is_some() || !serves_transfer || self.admit_partition_transfer(header.group); + let tail_available = !serves_transfer + || (admitted + && partition.retain_transfer_tail(header.replica, header.nonce, self.ticks.get())); // One resolve, one send: the three outcomes differ only in the // descriptor they produce, and duplicating the send made it possible for // them to drift on the progress they advertise. let offer = match cached { - Some(offer) => { - tracing::debug!( - shard = self.id, - namespace_raw = header.group, - requester = header.replica, - "re-answering a partition state transfer request from the offer \ - already served" - ); - Some(offer) - } - None if !self.may_serve_another_partition_transfer(header.group) => { - // Admission control, because the served-payload budget is a - // BYTE budget and the pulls that overrun it do not degrade - // gracefully. Each concurrent pull holds a different segment - // resident, so admitting more distinct groups than the budget - // has max-size slots makes them evict each other on every - // chunk: every request then re-reads and re-hashes a whole - // segment to serve one 256 KiB range, and a per-chunk serve - // that outruns the requester's stall interval exhausts its - // retry budget, so the pull rotates peers and never converges. - // Refusing the surplus is what makes the admitted ones finish. - tracing::info!( - shard = self.id, - namespace_raw = header.group, - requester = header.replica, - "already serving as many partition transfers as the served-payload \ - budget holds; refusing until one completes" - ); + _ if !tail_available => { + if !admitted { + tracing::info!( + shard = self.id, + namespace_raw = header.group, + requester = header.replica, + "already serving as many partition transfers as the served-payload \ + budget holds; refusing until a free slot reaches this group" + ); + } let (view, commit_max) = serving_progress(partition); self.send_state_transfer_target( cluster, @@ -9690,6 +10271,16 @@ where .await; return; } + Some(offer) => { + tracing::debug!( + shard = self.id, + namespace_raw = header.group, + requester = header.replica, + "re-answering a partition state transfer request from the offer \ + already served" + ); + Some(offer) + } None => { // Claim the admission slot for the whole build, not just for a // completed offer: the checksum pass runs over several rounds @@ -9697,7 +10288,15 @@ where self.partition_offer_builds .borrow_mut() .insert(header.group, 0); - match partition.state_transfer_offer(&config).await { + let built = partition.state_transfer_offer(&config).await.and_then(|offer| { + if partition.transfer_tail_available(header.replica, header.nonce, offer.commit_op) { + Ok(offer) + } else { + partition.clear_state_transfer_offer_cache(); + Err(partitions::state_transfer::PartitionTransferUnavailable::RepairTailExpired) + } + }); + match built { Ok(offer) => { self.partition_offer_builds .borrow_mut() @@ -9722,6 +10321,23 @@ where ); Some(offer) } + // Only the plan pin is missing, and an I/O turn pins it + // soon. The build slot stays claimed, as for any resumable + // refusal, and the tick answers the request. + Err(partitions::state_transfer::PartitionTransferUnavailable::FlushPending) + if waited_ticks + < self.repair_retry_ticks.get() + / PARKED_TRANSFER_REQUEST_WAIT_DIVISOR => + { + self.parked_partition_transfer_requests.borrow_mut().insert( + (header.group, header.replica), + ParkedTransferRequest { + header, + waited_ticks, + }, + ); + return; + } Err(reason) => { // The ACTUAL reason: "not the caught-up primary" is routine // (the requester re-targets), an unreadable segment is an @@ -9836,6 +10452,7 @@ where let serving_view = partition.consensus().view(); let serving_commit_max = partition.consensus().commit_max(); let chunk_len_max = self.state_chunk_len_max(); + let mut loaded_chunk = None; let reply = loop { let attempt = 'attempt: { let mut offers = self.state_transfer_offers.borrow_mut(); @@ -9847,6 +10464,11 @@ where transient: true, })); }; + if !partition.touch_transfer_tail(header.replica, header.nonce, self.ticks.get()) { + break 'attempt ChunkAttempt::Reply(Some(ChunkReply::Unavailable { + transient: true, + })); + } let ServedOffer::Partition(offer) = &served.offer else { break 'attempt ChunkAttempt::Reply(Some(ChunkReply::Unavailable { transient: true, @@ -9856,11 +10478,14 @@ where let artifact = header.artifact as usize; // Keeps a segment payload alive past the cache borrow below. let segment_payload: Rc>; - let artifact_bytes: &[u8] = match offer.artifact_at(artifact) { + let (artifact_bytes, start): (&[u8], usize) = match offer.artifact_at(artifact) { Some(partitions::state_transfer::PartitionArtifactSource::Offsets(bytes)) => { - bytes + (bytes, header.offset as usize) } Some(partitions::state_transfer::PartitionArtifactSource::Segment(source)) => { + if header.offset >= source.entry.len || header.len == 0 { + break 'attempt ChunkAttempt::Reply(None); + } match self .served_segment_cache .borrow_mut() @@ -9868,20 +10493,23 @@ where { Some(payload) => { segment_payload = payload; - &segment_payload + (&segment_payload, header.offset as usize) } None => { - break 'attempt ChunkAttempt::Load { - log_path: source.log_path.clone(), - entry: source.entry, - }; + if let Some(bytes) = loaded_chunk.as_deref() { + (bytes, 0) + } else { + break 'attempt ChunkAttempt::Load { + log_path: source.log_path.clone(), + entry: source.entry, + }; + } } } } // Index past the manifest: requester bug or stale frame. None => break 'attempt ChunkAttempt::Reply(None), }; - let start = header.offset as usize; // `start >= len` is the empty-chunk livelock refusal; see the // metadata arm for the full story. if start >= artifact_bytes.len() { @@ -9931,24 +10559,39 @@ where match attempt { ChunkAttempt::Reply(reply) => break reply, ChunkAttempt::Load { log_path, entry } => { - // Chunked read + incremental hash: this runs on the pump to - // answer ONE 256 KiB chunk request, so a whole-file read - // plus a single hash pass over up to a segment would be one - // long uninterruptible CPU+IO block. The chunking keeps the - // REACTOR moving; this shard's consensus ticks are a sibling - // select arm of the same task and stay frozen either way. - let loaded = partitions::state_transfer::load_verified_segment_artifact( - &log_path, &entry, - ) - .await; + // A whole-segment read on the pump can outlive the requester's + // retry budget. Cache only artifacts that fit one chunk; + // the receiver verifies larger artifacts before install. + let cache_whole = header.offset == 0 && entry.len <= chunk_len_max as u64; + let loaded = if cache_whole { + partitions::state_transfer::load_verified_segment_artifact( + &log_path, &entry, + ) + .await + } else { + let len = u64::from(header.len) + .min(chunk_len_max as u64) + .min(entry.len - header.offset) + as usize; + partitions::state_transfer::load_segment_chunk( + &log_path, + header.offset, + len, + ) + .await + }; let reason = match loaded { Ok(bytes) => { - self.served_segment_cache.borrow_mut().insert( - header.group, - entry.checksum, - Rc::new(bytes), - self.served_segment_cache_bytes_max.get(), - ); + if cache_whole { + self.served_segment_cache.borrow_mut().insert( + header.group, + entry.checksum, + Rc::new(bytes), + self.served_segment_cache_bytes_max.get(), + ); + } else { + loaded_chunk = Some(bytes); + } continue; } Err(reason) => reason, @@ -10232,14 +10875,13 @@ where /// Arm a fresh partition transfer session against `peer` and request its /// descriptor. Every partition arming site goes through here. Drops any - /// repair session (transfer supersedes repair; a transfer-unavailable - /// fallback re-arms repair fresh) and leaves the stage to its callers (they - /// own the `AwaitingTarget` transition). + /// repair session (transfer supersedes repair) and leaves the stage to its + /// callers (they own the `AwaitingTarget` transition). /// /// Refuses past [`Self::PARTITION_TRANSFERS_INFLIGHT_MAX`]: the arm /// converts to a scheduled re-arm (no failure charged -- the local slot - /// shortage is not the peer's fault) and the stage returns to Idle so - /// journal repair keeps the gap visible meanwhile. + /// shortage is not the peer's fault) and the stage returns to Idle, which + /// is the stage the re-arm fires from. /// /// `transfers_inflight` is computed by the caller BEFORE it formed its /// `&mut partition`: the counting scan takes shared borrows of every @@ -10292,6 +10934,10 @@ where artifacts: Vec::new(), target_accepted: false, idle_ticks: 0, + spill: None, + scan: None, + seed: None, + pulled_len: 0, }); self.send_request_state_transfer(partition.consensus(), peer, nonce) .await; @@ -10321,8 +10967,13 @@ where B: MessageBus + 'static, T: ShardsTable, { + // A scheduled transfer re-arm owns recovery, as at the gap detector and + // the floor-refused arm: it exists because repair could not fill the + // gap, and frames that a session journals above the gap meanwhile only + // wait for the install to wipe them. if !partition.consensus().is_normal() || partition.consensus().is_transferring() + || partition.transfer_rearm.is_some() || partition.repair.is_some() || partition.view_truncation_pending() { @@ -10455,6 +11106,7 @@ where fetch_to_op, partition.consensus().group(), partition.created_revision(), + partition.note_proof_walk_request(), ) .await; true @@ -10494,6 +11146,7 @@ where to_op, partition.consensus().group(), partition.created_revision(), + partition.note_proof_walk_request(), ) .await; } @@ -10545,6 +11198,7 @@ where to_op, partition.consensus().group(), partition.created_revision(), + partition.note_proof_walk_request(), ) .await; } @@ -10629,6 +11283,7 @@ where to_op, namespace, partition.created_revision(), + partition.note_proof_walk_request(), ) .await; } @@ -10824,36 +11479,20 @@ where // and re-run the staging scan for nothing. return; } - let reused = partition.reuse_staged_segments(&entries).await; - if !reused.is_empty() { - tracing::info!( - shard = self.id, - namespace_raw = header.group, - peer = header.replica, - adopted = reused.len(), - artifacts = entries.len(), - "adopted staged segments from an earlier transfer attempt" - ); - } - // Re-check the nonce AFTER the await: the staging scan yields, and a - // session re-minted underneath it must not get stamped with this - // (now stale) offer's commit_op and manifest. - let Some(session) = partition - .transfer - .as_mut() - .filter(|session| session.nonce == header.nonce) - else { + let scan = partition.begin_reuse_scan(&entries); + let Some(session) = partition.transfer.as_mut() else { return; }; session.target_accepted = true; session.commit_op = header.commit_op; + session.scan = Some(scan); // No reservation here: the manifest's total is bounded only by // `PARTITION_TRANSFER_TOTAL_LEN_MAX` (1 TiB), so reserving every // artifact up front is an eager address-space commit of the whole // manifest -- times the in-flight cap -- which turns fatal under strict // overcommit, `RLIMIT_AS`, or cgroup accounting, and contradicts the // session's own promise to bound receiver memory to ONE in-flight - // artifact. Artifacts adopted by the reuse scan below would also be + // artifact. Artifacts that the reuse scan adopts would also be // reserved and then overwritten with `Staged`, making the retry path's // reservation pure waste. `append_chunk` reserves the declared length on // an artifact's FIRST chunk instead, and only ever for the artifact the @@ -10868,9 +11507,6 @@ where }) .collect(); session.idle_ticks = 0; - for (index, meta) in reused { - session.artifacts[index as usize] = TransferArtifact::Staged(meta); - } let consensus = partition.consensus(); if consensus.state_transfer_stage() == consensus::StateTransferStage::AwaitingTarget { consensus.set_state_transfer_stage(consensus::StateTransferStage::Fetching); @@ -10921,6 +11557,9 @@ where if session.peer != header.replica { return; } + if session.spill.is_some() { + return; + } let payload = &msg.as_slice()[size_of::()..header.size as usize]; if !consensus::append_chunk( &mut session.artifacts, @@ -10931,6 +11570,7 @@ where return; } session.idle_ticks = 0; + session.pulled_len += payload.len() as u64; } partition.note_transfer_progress(); self.on_partition_transfer_progress(header.group).await; @@ -10944,7 +11584,6 @@ where where B: MessageBus + 'static, T: ShardsTable, - M: StreamsFrontend, { let planes = self.plane.inner(); let config = planes.1.0.config().clone(); @@ -10977,6 +11616,62 @@ where if !session.target_accepted { return; } + // The reuse scan writes the staging names that a spill writes, so + // nothing spills or installs before it lands. + if self.drive_reuse_scan(partition, namespace) { + return; + } + let landed_spill = partition.transfer.as_mut().and_then(|session| { + let spill = session.spill.as_ref()?; + let result = spill.work.result.borrow_mut().take()?; + let artifact = spill.artifact; + let seeded = spill.seeded; + session.spill = None; + Some((artifact, seeded, session.peer, result)) + }); + if let Some((index, seeded, peer, result)) = landed_spill { + match result { + Ok(spilled) => { + let meta = partition.accept_transfer_segment_spill(spilled); + if let Some(session) = partition.transfer.as_mut() { + session.artifacts[index] = TransferArtifact::Staged(meta); + } + } + // The local prefix is not a prefix of the offered segment. The + // spill took the buffer, so the artifact pulls again from its + // first byte, and nothing about the peer is charged. + Err(partitions::state_transfer::SpillError::ManifestChecksum { .. }) if seeded => { + tracing::warn!( + shard = self.id, + namespace_raw = namespace, + artifact = index, + "the local segment prefix does not match the offered artifact, \ + so the pull fetches the whole artifact" + ); + return Box::pin(self.on_partition_transfer_progress(namespace)).await; + } + Err(reason) => { + tracing::warn!( + shard = self.id, + namespace_raw = namespace, + artifact = index, + %reason, + "partition transfer segment failed validation at spill" + ); + self.abandon_or_rearm_partition_transfer(partition, peer) + .await; + return; + } + } + return Box::pin(self.on_partition_transfer_progress(namespace)).await; + } + let Some(session) = partition.transfer.as_ref() else { + return; + }; + + if session.spill.is_some() { + return; + } // Spill any segment artifact that just completed, freeing its buffer. let spill_candidate = session @@ -10993,45 +11688,38 @@ where .then_some(index) }); if let Some(index) = spill_candidate { - let (entry, bytes, peer, nonce) = { + let (entry, bytes, prefix) = { let Some(session) = partition.transfer.as_mut() else { return; }; + let prefix = match session.seed.take() { + Some(partitions::state_transfer::PrefixSeed::Seeded { artifact, prefix }) + if artifact as usize == index => + { + Some(prefix) + } + seed => { + session.seed = seed; + None + } + }; let Some(progress) = session.artifacts[index].pending_mut() else { return; }; let bytes = std::mem::take(&mut progress.buf); - (progress.entry, bytes, session.peer, session.nonce) + (progress.entry, bytes, prefix) }; - match partition.spill_transfer_segment(&entry, bytes).await { - Ok(meta) => { - // Nonce re-check across the spill await: a session - // re-minted underneath it has a fresh (possibly empty) - // artifact vec, and the stale index would panic. Pump- - // serial today, but nothing enforces that. - if let Some(session) = partition - .transfer - .as_mut() - .filter(|session| session.nonce == nonce) - { - session.artifacts[index] = TransferArtifact::Staged(meta); - } - } - Err(reason) => { - tracing::warn!( - shard = self.id, - namespace_raw = namespace, - artifact = index, - %reason, - "partition transfer segment failed validation at spill" - ); - self.abandon_or_rearm_partition_transfer(partition, peer) - .await; - return; - } + let seeded = prefix.is_some(); + let work = partition.begin_transfer_segment_spill(&entry, bytes, prefix); + if let Some(session) = partition.transfer.as_mut() { + session.idle_ticks = 0; + session.spill = Some(partitions::state_transfer::SegmentSpill { + artifact: index, + seeded, + work, + }); } - // Tail-call for the next candidate / chunk request. - return Box::pin(self.on_partition_transfer_progress(namespace)).await; + return; } let Some(session) = partition.transfer.as_ref() else { @@ -11052,6 +11740,13 @@ where let Some(session) = partition.transfer.take() else { return; }; + tracing::info!( + shard = self.id, + namespace_raw = namespace, + artifacts = session.artifacts.len(), + pulled_len = session.pulled_len, + "partition transfer pulled every artifact" + ); let commit_op = session.commit_op; let peer = session.peer; let mut offsets_bytes: Option> = None; @@ -11200,7 +11895,7 @@ where shard = self.id, namespace_raw = namespace, %error, - "partition state transfer install failed; falling back to journal repair" + "partition state transfer install failed; scheduling a re-arm" ); self.abandon_or_rearm_partition_transfer(partition, peer) .await; @@ -11214,8 +11909,8 @@ where /// undecodable artifact) re-ran the full pull -- including the serving /// primary's whole-segment reads -- at network round-trip rate, and the /// generation-keyed budget never exhausted on a committing cluster. - /// Journal repair is re-armed in the meantime so the gap stays visible - /// and anything repairable heals without waiting out the backoff. + /// Journal repair does not run meanwhile: the re-arm owns recovery (see + /// [`Self::maybe_request_partition_repair`]). #[allow(clippy::future_not_send)] async fn abandon_or_rearm_partition_transfer( &self, @@ -11309,7 +12004,6 @@ where rotate: bool, ) where B: MessageBus + 'static, - T: ShardsTable, { partition.transfer = None; let consensus = partition.consensus(); @@ -11346,7 +12040,6 @@ where }); let config = self.plane.partitions().config().clone(); partition.commit_journal(&config).await; - self.maybe_request_partition_repair(partition, peer).await; } /// Ask for the next missing partition chunk (first unspilled, incomplete @@ -11375,12 +12068,25 @@ where else { return; }; - let request = partition.transfer.as_ref().and_then(|session| { + if partition + .transfer + .as_ref() + .is_some_and(|session| session.spill.is_some()) + { + return; + } + if self.drive_reuse_scan(partition, namespace) || partition.drive_prefix_seed() { + return; + } + let request = partition.transfer.as_mut().and_then(|session| { if !session.target_accepted { return None; } let (index, offset, len) = consensus::next_pending_chunk(&session.artifacts, chunk_len_max)?; + // The serving offer ages from the last chunk that it served, and + // the bound on local work counts the same span. + session.idle_ticks = 0; Some((session.nonce, session.peer, index, offset, len)) }); let consensus_ids = { @@ -11403,6 +12109,47 @@ where } } + /// Hand a landed reuse scan to its session and log what it staged. + /// Returns `true` while the scan runs, because the pull waits for it. + fn drive_reuse_scan(&self, partition: &mut IggyPartition, namespace: u64) -> bool + where + B: MessageBus, + { + let Some(peer) = partition.transfer.as_ref().map(|session| session.peer) else { + return false; + }; + let Poll::Ready(reused) = partition.drive_reuse_scan() else { + return true; + }; + let Some(reused) = reused else { + return false; + }; + if reused.adopted > 0 { + tracing::info!( + shard = self.id, + namespace_raw = namespace, + peer, + adopted = reused.adopted, + adopted_len = reused.adopted_len, + artifacts = reused.artifacts, + "adopted staged segments from an earlier transfer attempt" + ); + } + if reused.linked > 0 { + tracing::info!( + shard = self.id, + namespace_raw = namespace, + peer, + reused = reused.linked, + reused_len = reused.linked_len, + artifacts = reused.artifacts, + total_len = reused.total_len, + "reused segments that this node already holds" + ); + } + false + } + /// Drop serving-side state-transfer offers that stopped being pulled. /// /// An offer pins its plane's payload for as long as it lives -- the metadata @@ -11446,6 +12193,15 @@ where } live }); + // A waiter that stopped asking (it installed from another peer, + // re-targeted or crashed) would otherwise keep the next free slot + // reserved for good. + self.partition_transfer_waiters + .borrow_mut() + .retain_mut(|(_, idle_ticks)| { + *idle_ticks += 1; + *idle_ticks < idle_expiry_ticks + }); let mut offers = self.state_transfer_offers.borrow_mut(); let namespaces_before: Vec = offers.keys().map(|(namespace, _)| *namespace).collect(); offers.retain(|(namespace, requester), served| { @@ -13760,7 +14516,9 @@ async fn dispatch_vsr_actions( continue; }; for replica in replicas { - send(*replica, frozen.clone()).await; + let _ = + send_replayed_prepare(consensus, *replica, header.op, frozen.clone()) + .await; } } } @@ -13785,6 +14543,26 @@ async fn dispatch_vsr_actions( // Handled by the caller, which owns the plane's journal. Metadata // ignores `RetryLocalWrite` and re-acks via `repair_primary_self_acks`. VsrAction::CommitJournal | VsrAction::RetryLocalWrite { .. } => {} + VsrAction::SendCommitReply { + view, + group, + timestamp_monotonic, + target, + } => { + let message = Message::::new(size_of::()) + .transmute_header(|_, header: &mut CommitReplyHeader| { + header.command = Command::CommitReply; + header.partition_incarnation = consensus.partition_incarnation(); + header.cluster = cluster; + header.replica = self_id; + header.view = *view; + header.group = *group; + header.timestamp_monotonic = *timestamp_monotonic; + header.size = size_of::() as u32; + header.seal(); + }); + send(*target, message.into_generic().into_frozen()).await; + } VsrAction::SendCommit { view, commit, @@ -13860,16 +14638,9 @@ async fn dispatch_partition_journal_actions( P: Pipeline, SB: SuperblockStore, { - let bus = consensus.message_bus(); let self_id = consensus.replica(); let journal = &partition.log.journal().inner; - let send = |target: u8, msg: Frozen| async move { - if let Err(e) = bus.send_to_replica(target, msg).await { - tracing::debug!(replica = self_id, target, "bus send failed: {e}"); - } - }; - // Same durable-before-send tripwire as `dispatch_vsr_actions`: this // dispatcher emits view-scoped `SendPrepareOk` too, and all callers are // persist-gated today -- assert it so a future bypass cannot slip @@ -13918,7 +14689,9 @@ async fn dispatch_partition_journal_actions( continue; }; for replica in replicas { - send(*replica, prepare.clone()).await; + let _ = + send_replayed_prepare(consensus, *replica, header.op, prepare.clone()) + .await; } } } @@ -13940,6 +14713,41 @@ async fn dispatch_partition_journal_actions( } } +#[allow(clippy::future_not_send)] +async fn send_replayed_prepare( + consensus: &VsrConsensus, + target: u8, + op: u64, + entry: Frozen, +) -> bool +where + B: MessageBus, + P: Pipeline, +{ + if !consensus.is_primary() || !consensus.is_normal() || consensus.is_transferring() { + return false; + } + let Some(entry) = restamp_prepare_view(entry, consensus.view()) else { + return false; + }; + let size = entry.len() as u64; + if !consensus.admit_replication(target, op, size, true) { + return false; + } + if consensus + .message_bus() + .send_to_replica(target, entry) + .await + .is_ok() + { + consensus.record_replication_send(target, op, size, true); + true + } else { + consensus.record_replication_failure(target, op); + false + } +} + #[cfg(test)] mod warn_pace_tests { use super::*; @@ -14917,15 +15725,14 @@ mod dvc_suffix_window_tests { #[cfg(test)] mod installed_transfer_dvc_tests { //! A lagging replica adopts a `StartView` and later installs a state - //! transfer below that view's head. The install rewinds the sequencer to - //! the transfer's op and keeps the adopted view. Each `DoViewChange` the - //! replica sends from then on carries the transfer op in its header but - //! starts its suffix at the adopted head, and its peers drop every one as - //! unreadable. With a third replica down, or back with an untrusted log, - //! no quorum of readable votes forms and the view change never ends. + //! transfer below that view's head. The install keeps that head, so each + //! `DoViewChange` the replica sends afterwards stays readable and + //! describes every uncommitted op of the adopted view. With a third + //! replica down, or back with an untrusted log, the view change has no + //! other quorum of readable votes. use super::*; - use consensus::{LocalPipeline, dvc_suffix_decode}; + use consensus::{LocalPipeline, TimeoutManager, dvc_suffix_decode}; use iggy_common::{IggyByteSize, PartitionStats}; use journal::prepare_journal::PrepareJournal; use message_bus::IggyMessageBus; @@ -14946,10 +15753,9 @@ mod installed_transfer_dvc_tests { const COMMIT: u64 = HEAD - 2; #[compio::test] - #[ignore = "known bug: a transfer installed below the adopted view head makes the replica's DoViewChange unreadable; run with --ignored"] async fn given_a_transfer_installed_below_the_adopted_head_when_a_view_change_follows_should_send_a_readable_do_view_change() { - // Inside the 128-entry window, so only the head disagrees. + // Inside the 128-entry window: only the header op can disagree with the suffix. let transfer = HEAD - 100; // Boxed: the setup future carries a whole partition. let (vote, message) = Box::pin(vote_after_install(transfer)).await; @@ -14976,7 +15782,6 @@ mod installed_transfer_dvc_tests { } #[compio::test] - #[ignore = "known bug: a transfer installed far below the adopted view head leaves uncommitted ops outside the DoViewChange window; run with --ignored"] async fn given_a_transfer_installed_far_below_the_adopted_head_when_a_view_change_follows_should_describe_every_uncommitted_op() { // Past the 128-entry window. @@ -14999,8 +15804,9 @@ mod installed_transfer_dvc_tests { } /// The lagging replica's vote for `NEXT_VIEW`, and the frame the shard sends - /// for it. Before the vote, the replica adopts the `ADOPTED_VIEW` `StartView` - /// and then installs a peer's state transfer at `transfer`. + /// for it. Before the vote, the replica adopts the `ADOPTED_VIEW` `StartView`, + /// installs a peer's state transfer at `transfer`, and then stops hearing the + /// primary of `ADOPTED_VIEW`, which is down. #[allow(clippy::future_not_send)] async fn vote_after_install(transfer: u64) -> (VsrAction, Message) { let bus = IggyMessageBus::new(0); @@ -15045,10 +15851,9 @@ mod installed_transfer_dvc_tests { // Repair finds the gap evicted at every peer, so a transfer follows. The // install cannot run here: its offsets artifact is a type private to - // `partitions`. So these are its journal wipe and then the steps of - // `publish_installed_consensus`, in its order, with a checkpoint prepare - // built here in place of the origin's. They rewind the sequencer to - // `transfer` and keep the adopted view. + // `partitions`. So these are its journal wipe and then the consensus step + // of `publish_installed_consensus`, with a checkpoint prepare built here in + // place of the origin's. let checkpoint = Message::::new(size_of::()) .transmute_header(|_, header: &mut PrepareHeader| { *header = prepare_header(transfer, 0); @@ -15058,12 +15863,19 @@ mod installed_transfer_dvc_tests { transfer, Owned::::copy_from_slice(checkpoint.as_slice()).into(), ); - let consensus = partition.consensus(); - consensus.set_commit_floor(transfer); - consensus.sequencer().set_sequence(transfer); - consensus.set_last_prepare_checksum(checkpoint.header().checksum); - consensus.clear_pipeline(); - consensus.advance_commit_max(transfer); + partition.consensus().note_journal_mutation(); + partition + .consensus() + .install_checkpoint(transfer, Some(checkpoint.header().checksum)); + + // The heartbeat timeout votes to leave the view, so the `StartViewChange` + // of replica 1 completes a quorum of votes. + let voted = (0..TimeoutManager::NORMAL_HEARTBEAT_TICKS) + .flat_map(|_| partition.consensus().tick(PlaneKind::Partitions)) + .any(|action| { + matches!(action, VsrAction::SendStartViewChange { view, .. } if view == NEXT_VIEW) + }); + assert!(voted, "the lagging replica votes to leave the view"); // `on_start_view_change`: snapshot the suffix, then vote. refresh_partition_dvc_suffix(&partition); @@ -15163,8 +15975,8 @@ mod adopted_view_retention_tests { use super::{adopted_view_headers, build_dvc_suffix, refresh_metadata_dvc_suffix}; use consensus::{ CLIENTS_TABLE_MAX, ClientTable, Consensus, JoinMode, LocalPipeline, PlaneKind, Project, - Sequencer, StateTransferStage, ViewLogProof, ViewRestore, VsrAction, VsrConsensus, - encode_prepare_headers, send_prepare_ok, view_log_proof, + Sequencer, StateTransferStage, TimeoutManager, ViewLogProof, ViewRestore, VsrAction, + VsrConsensus, encode_prepare_headers, send_prepare_ok, view_log_proof, }; use iggy_binary_protocol::{ Command, DoViewChangeHeader, Operation, PrepareHeader, PrepareOkHeader, @@ -15377,6 +16189,8 @@ mod adopted_view_retention_tests { }; lagging.fence_view_log(op, unproven_through); + vote_on_heartbeat_timeout(&restarted, NEXT_VIEW); + vote_on_heartbeat_timeout(&lagging, NEXT_VIEW); refresh_suffix(&restarted, journal); refresh_suffix(&lagging, lagging_journal); let vote = restarted @@ -15555,6 +16369,8 @@ mod adopted_view_retention_tests { }); reconcile(&lagging, lagging_journal); + vote_on_heartbeat_timeout(restarted, NEXT_VIEW); + vote_on_heartbeat_timeout(&lagging, NEXT_VIEW); refresh_metadata_dvc_suffix(restarted, metadata.journal.as_ref()); refresh_suffix(&lagging, lagging_journal); let vote = restarted @@ -15669,14 +16485,16 @@ mod adopted_view_retention_tests { ); next_primary.handle_commit(heartbeat.header()); next_primary.advance_commit_min(COMMITTED); - let mut replacement = original.clone(); + let replacement = original.clone(); let abandoned = projected_stream(&old_primary, "abandoned"); old_primary.pipeline_message(PlaneKind::Metadata, &abandoned); original.push(*abandoned.header()); old_primary.extend_history_certificate(HEAD, abandoned.header().checksum); // Replica 1 is isolated. The two remaining replicas decide view 11 - // through timer-produced SVCs and the DVCs their handlers emit. + // through the votes of their heartbeat timeouts and the DVCs their + // handlers emit. + vote_on_heartbeat_timeout(&lagging, ADOPTED_VIEW); elect_from_handlers( &next_primary, &lagging, @@ -15689,32 +16507,9 @@ mod adopted_view_retention_tests { assert_eq!(lagging.log_view_claim(), ADOPTED_VIEW); assert!(lagging.view_log_unproven()); let uncommitted = projected_stream(&next_primary, "replacement"); - next_primary.pipeline_message(PlaneKind::Metadata, &uncommitted); - replacement.push(*uncommitted.header()); - next_primary.extend_history_certificate(HEAD, uncommitted.header().checksum); assert_ne!(abandoned.header().checksum, uncommitted.header().checksum); - // Drop all repair traffic to replica 0 and the intermediate elections. - // Its timer eventually nominates replica 2 again. One body plus one nack - // retains the uncommitted replacement, which that primary announces. - elect_from_handlers( - &next_primary, - &lagging, - &replacement, - &lagging_journal, - LATER_VIEW, - ); - assert_eq!(next_primary.pending_view_log().unwrap().op_head, HEAD); - announce_from_handler(&next_primary, &lagging, &replacement, &lagging_journal); - assert_eq!(lagging.log_view_claim(), LATER_VIEW); - assert_eq!(lagging.commit_min(), COMMON); - assert_eq!( - lagging.pending_view_log().unwrap().headers[0].checksum, - uncommitted.header().checksum - ); - - let lagging = if restart { - let record = lagging.vsr_state(COMMON, 0); + let restore = |record: &consensus::VsrState| { let record = consensus::VsrState::try_from(record.to_bytes().as_slice()).unwrap(); let mut recovered = replica(BACKUP); @@ -15736,6 +16531,37 @@ mod adopted_view_retention_tests { recovered.resume_adopted_view(); recovered.init_as_backup(); recovered.begin_view_probe(); + recovered + }; + + // A restarted replica whose probes go unanswered enters the next view + // alone, so a replica that restarts while cut off can reach a view that + // no quorum voted for, and the merge of its stitched unproven headers + // must still hold. Replica 0 adopted the `LATER_VIEW` `StartView` of + // replica 2, which kept the uncommitted replacement above `COMMITTED`, + // and later restarted in the view before `target_view`. + let lagging = restore(&consensus::VsrState { + view: target_view - 1, + log_view: LATER_VIEW, + adopted_view: Some(consensus::AdoptedView { + view: LATER_VIEW, + group: METADATA_GROUP, + partition_incarnation: 0, + op_head: HEAD, + commit_max: COMMITTED, + headers: vec![*uncommitted.header(), *committed.header()], + }), + ..lagging.vsr_state(COMMON, 0) + }); + assert_eq!(lagging.log_view_claim(), LATER_VIEW); + assert_eq!(lagging.commit_min(), COMMON); + assert_eq!( + lagging.pending_view_log().unwrap().headers[0].checksum, + uncommitted.header().checksum + ); + + let lagging = if restart { + let recovered = restore(&lagging.vsr_state(COMMON, 0)); assert_eq!(recovered.adopted_view_head(), Some((LATER_VIEW, HEAD))); assert!(!recovered.history_certified()); recovered @@ -15978,6 +16804,22 @@ mod adopted_view_retention_tests { )); } + /// Tick `consensus` until its heartbeat timeout fires. The replica no longer + /// hears its primary, so it votes for `view`, and the `StartViewChange` of a + /// survivor that votes the same completes a quorum of votes. + fn vote_on_heartbeat_timeout(consensus: &VsrConsensus, view: u32) { + let voted = (0..TimeoutManager::NORMAL_HEARTBEAT_TICKS) + .flat_map(|_| consensus.tick(PlaneKind::Metadata)) + .any(|action| { + matches!(action, VsrAction::SendStartViewChange { view: next, .. } if next == view) + }); + assert!( + voted, + "replica {} votes for view {view}", + consensus.replica() + ); + } + /// The proof `reconcile_metadata_view_divergence` takes of an adopted view /// once the replica is not transferring, over a journal answering `held`. fn reconcile(consensus: &VsrConsensus, held: impl Fn(u64) -> Option) { @@ -17020,7 +17862,7 @@ mod partition_placement_tests { use std::mem::size_of; use std::sync::Arc; - type PlacementShard = IggyShard< + pub type PlacementShard = IggyShard< Rc, PrepareJournal, IggySnapshot, @@ -17029,20 +17871,30 @@ mod partition_placement_tests { >; pub fn owner(shard_id: u16) -> (PlacementShard, Receiver) { + owner_with_repair_memory(shard_id, None) + } + + pub fn owner_with_repair_memory( + shard_id: u16, + repair_memory: Option>, + ) -> (PlacementShard, Receiver) { let bus = Rc::new(IggyMessageBus::new(shard_id)); let segment_size = IggyByteSize::from(1_048_576_u64); - let partitions = IggyPartitions::new( - ShardId::new(shard_id), - PartitionsConfig { - messages_required_to_save: 100, - size_of_messages_required_to_save: segment_size, - validate_checksum: true, - segment_size, - preallocate_segments: false, - key_context: Arc::default(), - path_layout: PartitionPathLayout::default(), - }, - ); + let config = PartitionsConfig { + messages_required_to_save: 100, + size_of_messages_required_to_save: segment_size, + validate_checksum: true, + segment_size, + preallocate_segments: false, + key_context: Arc::default(), + path_layout: PartitionPathLayout::default(), + }; + let partitions = match repair_memory { + Some(budget) => { + IggyPartitions::with_repair_memory(ShardId::new(shard_id), config, 0, budget) + } + None => IggyPartitions::new(ShardId::new(shard_id), config), + }; let metadata = IggyMetadata::new( None, None, @@ -17065,7 +17917,7 @@ mod partition_placement_tests { (shard, target) } - fn insert(shard: &PlacementShard, id: usize, revision: u64, view: u32) -> IggyNamespace { + pub fn insert(shard: &PlacementShard, id: usize, revision: u64, view: u32) -> IggyNamespace { let namespace = IggyNamespace::new(0, 0, id); let mut consensus = VsrConsensus::new( 1, @@ -17978,3 +18830,345 @@ mod untrusted_do_view_change_tests { )); } } + +#[cfg(test)] +mod replication_flow_tests { + use super::*; + use crate::partition_placement_tests::{PlacementShard, insert, owner_with_repair_memory}; + use consensus::REPLICATION_BATCH_BYTES_MAX; + use journal::prepare_journal::PrepareJournal; + use message_bus::IggyMessageBus; + use server_common::iobuf::Owned; + use server_common::sharding::METADATA_GROUP; + use std::collections::BTreeSet; + use std::sync::Arc; + + type SentFrames = Rc)>>>; + + fn recording_shard() -> (PlacementShard, SentFrames) { + const REPAIR_MEMORY_BYTES_MAX: usize = 64 * 1024; + let (shard, _) = owner_with_repair_memory( + 0, + Some(Arc::new(partitions::RepairMemoryBudget::new( + REPAIR_MEMORY_BYTES_MAX, + ))), + ); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + shard + .bus + .set_replica_forward_fn(Box::new(move |replica, _, frame| { + captured.borrow_mut().push((replica, frame)); + Ok(()) + })); + for replica in 1..3 { + crate::test_support::connect_replica_on(&shard.bus, replica, 1); + } + (shard, sent) + } + + fn prepare(group: u64, op: u64) -> Message { + Message::::new(size_of::()).transmute_header( + |_, header: &mut PrepareHeader| { + header.command = Command::Prepare; + header.operation = Operation::StoreConsumerOffset; + header.cluster = 1; + header.group = group; + header.op = op; + header.timestamp = op; + header.size = u32::try_from(size_of::()).unwrap(); + header.checksum = header.identity_checksum(); + }, + ) + } + + fn ready_reply( + consensus: &VsrConsensus>, + replica: u8, + nonce: u64, + observed_op: u64, + received_op: u64, + ) -> Message { + Message::::new(size_of::()).transmute_header( + |_, header: &mut PrepareProgressHeader| { + header.command = Command::PrepareProgress; + header.cluster = consensus.cluster(); + header.group = consensus.group(); + header.view = consensus.view(); + header.partition_incarnation = consensus.partition_incarnation(); + header.replica = replica; + header.kind = PrepareProgressHeader::REPLY; + header.ready = 1; + header.nonce = nonce; + header.observed_op = observed_op; + header.received_op = received_op; + header.size = u32::try_from(size_of::()).unwrap(); + header.seal(); + }, + ) + } + + #[compio::test] + async fn given_a_full_transfer_admission_when_another_group_requests_should_not_pin_its_repair_tail() + { + const NONCE: u128 = 1; + let (shard, _) = recording_shard(); + let namespace = insert(&shard, 0, 0, 0); + let busy = insert(&shard, 1, 0, 0); + shard.served_segment_cache_bytes_max.set(0); + shard + .partition_offer_builds + .borrow_mut() + .insert(busy.inner(), 0); + let partition = shard.plane.partitions().get_by_ns(&namespace).unwrap(); + assert!(partition.serves_state_transfer()); + assert!(!partition.touch_transfer_tail(1, NONCE, 0)); + let request = + Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestStateTransferHeader| { + header.command = Command::RequestStateTransfer; + header.cluster = 1; + header.replica = 1; + header.group = namespace.inner(); + header.partition_incarnation = partition.consensus().partition_incarnation(); + header.nonce = NONCE; + header.size = u32::try_from(size_of::()).unwrap(); + header.seal(); + }); + + shard + .serve_partition_transfer_request(*request.header(), None) + .await; + + assert!( + !partition.touch_transfer_tail(1, NONCE, 0), + "a refused transfer must not retain a live repair-tail lease" + ); + assert_eq!( + shard.partition_transfer_waiters.borrow().as_slice(), + &[(namespace.inner(), 0)] + ); + } + + #[compio::test] + async fn given_a_delayed_progress_probe_when_prepares_retransmit_should_share_credit_and_resume_after_its_reply() + { + let (shard, sent) = recording_shard(); + let namespace = insert(&shard, 0, 0, 0); + let partition = shard.plane.partitions().get_by_ns(&namespace).unwrap(); + let entry = prepare(namespace.inner(), 1); + let header = *entry.header(); + partition + .log + .journal() + .inner + .append(entry.into_frozen()) + .await + .unwrap(); + let consensus = partition.consensus(); + consensus.sequencer().set_sequence(1); + assert!(consensus.admit_replication(1, 1, REPLICATION_BATCH_BYTES_MAX, false)); + consensus.record_replication_send(1, 1, REPLICATION_BATCH_BYTES_MAX, false); + consensus.record_replication_failure(1, 1); + let Some(consensus::ReplicationWork::Probe { nonce, through, .. }) = + consensus.replication_work(1) + else { + panic!("the outstanding batch must own a progress probe"); + }; + let retry = VsrAction::RetransmitPrepares { + targets: vec![(header, vec![1, 2])], + }; + + dispatch_partition_journal_actions(consensus, partition, &[retry]).await; + + assert_eq!( + sent.borrow() + .iter() + .map(|(replica, _)| *replica) + .collect::>(), + vec![2] + ); + assert!( + !consensus.admit_replication(2, 2, REPLICATION_BATCH_BYTES_MAX, false), + "a successful timeout retry must consume the same byte credit as replay" + ); + let reply = ready_reply(consensus, 1, nonce, through, 1); + assert!(consensus.handle_prepare_progress(reply.header()).is_some()); + assert_eq!( + consensus.commit_max(), + 0, + "progress is not a PrepareOk vote" + ); + sent.borrow_mut().clear(); + dispatch_partition_journal_actions( + consensus, + partition, + &[VsrAction::RetransmitPrepares { + targets: vec![(header, vec![1])], + }], + ) + .await; + assert_eq!( + sent.borrow().len(), + 1, + "the duplicate must still recover a lost PrepareOk" + ); + assert_eq!(sent.borrow()[0].0, 1); + } + + #[compio::test] + async fn given_a_metadata_progress_probe_when_prepares_retransmit_should_share_the_peer_window() + { + const JOURNAL_SLOTS: usize = 128; + let (shard, sent) = recording_shard(); + let consensus = VsrConsensus::new( + 1, + 0, + 3, + METADATA_GROUP, + Rc::clone(&shard.bus), + consensus::LocalPipeline::new(), + ); + consensus.init(); + let directory = tempfile::tempdir().unwrap(); + let journal = PrepareJournal::open_with_slots( + &directory.path().join("journal.wal"), + 0, + JOURNAL_SLOTS, + ) + .await + .unwrap(); + let entry = prepare(METADATA_GROUP, 1); + let header = *entry.header(); + journal.append(entry).await.unwrap(); + consensus.sequencer().set_sequence(1); + assert!(consensus.admit_replication(1, 1, REPLICATION_BATCH_BYTES_MAX, false)); + consensus.record_replication_send(1, 1, REPLICATION_BATCH_BYTES_MAX, false); + consensus.record_replication_failure(1, 1); + let Some(consensus::ReplicationWork::Probe { nonce, through, .. }) = + consensus.replication_work(1) + else { + panic!("the metadata batch must own a progress probe"); + }; + dispatch_vsr_actions( + &consensus, + Some(&journal), + &[VsrAction::RetransmitPrepares { + targets: vec![(header, vec![1, 2])], + }], + ) + .await; + assert_eq!( + sent.borrow() + .iter() + .map(|(replica, _)| *replica) + .collect::>(), + vec![2] + ); + assert!(!consensus.admit_replication(2, 2, REPLICATION_BATCH_BYTES_MAX, false)); + let reply = ready_reply(&consensus, 1, nonce, through, 1); + assert!(consensus.handle_prepare_progress(reply.header()).is_some()); + sent.borrow_mut().clear(); + dispatch_vsr_actions( + &consensus, + Some(&journal), + &[VsrAction::RetransmitPrepares { + targets: vec![(header, vec![1])], + }], + ) + .await; + assert_eq!( + sent.borrow().len(), + 1, + "metadata retries resume after the correlated reply" + ); + } + + #[compio::test] + async fn given_several_groups_and_peers_in_replay_when_ticks_advance_should_bound_each_turn_and_visit_every_pair() + { + const GROUPS: usize = 3; + const OPS: u64 = 3; + const PULL_ROUND_OPS_MAX: u64 = 1; + let (shard, sent) = recording_shard(); + shard.repair_chunk_max.set(PULL_ROUND_OPS_MAX); + let mut expected = BTreeSet::new(); + for id in 0..GROUPS { + let namespace = insert(&shard, id, 0, 0); + let partition = shard.plane.partitions().get_by_ns(&namespace).unwrap(); + for op in 1..=OPS { + partition + .log + .journal() + .inner + .append(prepare(namespace.inner(), op).into_frozen()) + .await + .unwrap(); + } + let consensus = partition.consensus(); + consensus.sequencer().set_sequence(OPS); + for replica in 1..3 { + assert!(consensus.admit_replication( + replica, + 1, + size_of::() as u64, + false + )); + consensus.record_replication_failure(replica, 1); + let Some(consensus::ReplicationWork::Probe { nonce, through, .. }) = + consensus.replication_work(replica) + else { + panic!("a missing prefix must request progress"); + }; + let reply = ready_reply(consensus, replica, nonce, through, 0); + assert!(consensus.handle_prepare_progress(reply.header()).is_some()); + expected.insert((namespace.inner(), replica)); + } + } + let mut observed = BTreeSet::new(); + for tick in 1..=expected.len() { + let op = OPS + u64::try_from(tick).unwrap(); + for id in 0..GROUPS { + let namespace = IggyNamespace::new(0, 0, id); + let partition = shard.plane.partitions().get_by_ns(&namespace).unwrap(); + partition + .log + .journal() + .inner + .append(prepare(namespace.inner(), op).into_frozen()) + .await + .unwrap(); + partition.consensus().sequencer().set_sequence(op); + } + sent.borrow_mut().clear(); + assert!(shard.tick_partitions(&mut Vec::new()).await.is_none()); + let frames = sent.borrow(); + let prepares: Vec<_> = frames + .iter() + .filter_map(|(replica, frame)| { + let message = Message::::try_from(Owned::copy_from_slice( + frame.as_slice(), + )) + .ok()?; + let header = message.header(); + (header.command == Command::Prepare).then_some((header.group, *replica)) + }) + .collect(); + assert!( + u64::try_from(prepares.len()).unwrap() > PULL_ROUND_OPS_MAX, + "live replay must not inherit the pull repair operation-count limit" + ); + let pairs: BTreeSet<_> = prepares.into_iter().collect(); + assert_eq!( + pairs.len(), + 1, + "one shard tick must drive only one bounded replay range" + ); + assert!( + observed.insert(*pairs.first().unwrap()), + "a busy pair must not starve another pair" + ); + } + assert_eq!(observed, expected); + } +} diff --git a/core/shard/src/partition_io.rs b/core/shard/src/partition_io.rs index 1c6841a652..c39e8a0723 100644 --- a/core/shard/src/partition_io.rs +++ b/core/shard/src/partition_io.rs @@ -828,7 +828,8 @@ mod tests { use metadata::stm::user::{Users, UsersInner}; use metadata::{IggyMetadata, MuxStateMachine}; use partitions::state_transfer::{ - PartitionArtifactSource, PartitionTransferSession, TransferArtifact, + LocalWork, PartitionArtifactSource, PartitionTransferSession, PrefixSeed, ReuseScanned, + StagedSegmentMeta, TransferArtifact, }; use partitions::{ IggyIndexWriter, IggyPartition, IggyPartitions, MessagesWriter, PartitionPathLayout, @@ -845,6 +846,7 @@ mod tests { use crate::host::NoopHost; use crate::metrics::ShardMetrics; use crate::shards_table::{PapayaShardsTable, ShardsTable}; + use crate::test_support::connect_replica_on; use crate::{ IggyShard, LifecycleFrame, PARTITION_REPAIRS_INFLIGHT_MAX, PartitionConsensusConfig, REPAIR_GAP_DEBOUNCE_TICKS_MIN, Receiver, ReconcileOp, ReplicaTopology, ShardFrame, @@ -1019,6 +1021,7 @@ mod tests { 1, group, consensus.partition_incarnation(), + None, ) .await; owner @@ -1124,6 +1127,375 @@ mod tests { assert_eq!(control_frame_incarnation(frame.as_slice()), INCARNATION); } + #[compio::test] + async fn given_a_served_snapshot_when_later_writes_overflow_the_ring_should_keep_its_repair_tail() + { + const NONCE: u128 = 1; + const OPS: u64 = 8; + const BUDGET: usize = 64 * 1024; + let bus = Rc::new(IggyMessageBus::new(0)); + let targets = record_transfer_targets(&bus); + connect_replica_on(&bus, TRANSFER_REQUESTER, 1); + let (mut owner, _sender) = test_owner_with_repair_memory( + &bus, + None, + Some(Arc::new(partitions::RepairMemoryBudget::new(BUDGET))), + ); + owner.partition_consensus.replica_count = TRANSFER_REQUESTER + 1; + let namespace = IggyNamespace::new(0, 0, 0); + let directory = tempfile::tempdir().unwrap(); + let partitions = owner.plane.partitions(); + let partition = partitions.get_mut_by_ns(&namespace).unwrap(); + partition.set_created_revision(SERVED_INCARNATION); + attach_segment_files(partition, directory.path(), false).await; + let journal = &partition.log.journal().inner; + journal.set_repair_retention(true); + journal.set_ring_caps(1, u64::MAX); + for op in 1..=OPS { + let request = send_request(namespace, op).transmute_header( + |old, header: &mut RoutedRequestHeader| { + *header = old; + header.partition_incarnation = SERVED_INCARNATION; + }, + ); + let (reply, replied) = consensus::oneshot_channel(); + partitions.on_request_with_reply(request, Some(reply)).await; + owner + .process_loopback(&mut crate::router::LoopbackRound::default()) + .await; + assert_eq!(replied.await.unwrap().header().status, 0); + if op == 1 { + owner + .on_message(request_state_transfer(namespace, NONCE, SERVED_INCARNATION)) + .await; + assert_eq!( + targets.borrow().last(), + Some(&(namespace.inner(), NONCE, OFFERED)) + ); + } + } + let partition = partitions.get_by_ns(&namespace).unwrap(); + // Flushing also exercises eviction after the offer's pinned cut. + partition + .log + .journal() + .inner + .evict_prefix(usize::try_from(OPS).unwrap()); + let journal = &partition.log.journal().inner; + assert_eq!(journal.repair_retained_from(), Some(2)); + for op in 2..=OPS { + assert!( + journal.repair_entry(op).is_some(), + "snapshot tail lost op {op}" + ); + } + owner.ticks.set(9); + let repair = Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestPreparesHeader| { + header.command = Command::RequestPrepares; + header.size = u32::try_from(size_of::()).unwrap(); + header.cluster = 1; + header.group = namespace.inner(); + header.replica = TRANSFER_REQUESTER; + header.partition_incarnation = SERVED_INCARNATION; + header.nonce = NONCE + 1; + header.from_op = 2; + header.to_op = OPS; + header.seal(); + }); + owner.on_message(MessageBag::RequestPrepares(repair)).await; + partition.expire_transfer_tails(10, 10); + assert_eq!( + journal.repair_retained_from(), + Some(2), + "active pull repair must renew the snapshot suffix" + ); + partition.expire_transfer_tails(19, 10); + assert_eq!(journal.repair_retained_from(), Some(OPS)); + owner + .on_message(request_state_transfer(namespace, NONCE, SERVED_INCARNATION)) + .await; + assert_eq!( + targets.borrow().last(), + Some(&(namespace.inner(), NONCE, REFUSED)) + ); + assert_eq!( + owner.state_transfer_offers.borrow()[&(namespace.inner(), TRANSFER_REQUESTER)].nonce, + NONCE + ); + } + + #[compio::test] + async fn given_a_large_transfer_artifact_when_requesting_a_range_should_read_only_that_range() { + const NONCE: u128 = 1; + const PREFIX: &[u8] = b"held prefix"; + const SUFFIX: &[u8] = b"remaining segment"; + const CHUNK_LEN: u32 = 4; + for offset in [0, PREFIX.len()] { + let bus = Rc::new(IggyMessageBus::new(0)); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + captured.borrow_mut().push(frame); + Ok(()) + })); + connect_replica_on(&bus, TRANSFER_REQUESTER, 1); + let (mut owner, _sender) = test_owner(&bus, None); + owner.partition_consensus.replica_count = TRANSFER_REQUESTER + 1; + let namespace = IggyNamespace::new(0, 0, 0); + owner + .plane + .partitions() + .get_mut_by_ns(&namespace) + .unwrap() + .set_created_revision(SERVED_INCARNATION); + let directory = tempfile::tempdir().unwrap(); + let log_path = directory.path().join("segment.log"); + let mut bytes = [PREFIX, SUFFIX].concat(); + bytes.resize(crate::STATE_CHUNK_LEN as usize + 1, 0); + let entry = StateArtifact::for_bytes(artifact_kind::SEGMENT_LOG, 0, &bytes); + let expected = bytes[offset..offset + CHUNK_LEN as usize].to_vec(); + // Unrequested bytes need not be read to answer this chunk. + bytes[..offset].fill(0); + bytes.truncate(offset + CHUNK_LEN as usize); + std::fs::write(&log_path, &bytes).unwrap(); + let offer = partitions::state_transfer::PartitionStateTransferOffer { + commit_op: 0, + segments: vec![partitions::state_transfer::SegmentArtifactSource { + entry, + log_path: log_path.to_str().unwrap().to_owned(), + }], + offsets: ( + StateArtifact::for_bytes(artifact_kind::CONSUMER_OFFSETS, 0, &[]), + Rc::new(Vec::new()), + ), + }; + owner.state_transfer_offers.borrow_mut().insert( + (namespace.inner(), TRANSFER_REQUESTER), + crate::ServedStateTransfer { + nonce: NONCE, + offer: crate::ServedOffer::Partition(Rc::new(offer)), + idle_ticks: 0, + fully_served: false, + }, + ); + let request = + Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestStateChunkHeader| { + header.command = Command::RequestStateChunk; + header.size = u32::try_from(size_of::()).unwrap(); + header.cluster = 1; + header.group = namespace.inner(); + header.replica = TRANSFER_REQUESTER; + header.partition_incarnation = SERVED_INCARNATION; + header.nonce = NONCE; + header.offset = offset as u64; + header.len = CHUNK_LEN; + header.seal(); + }); + owner + .on_message(MessageBag::RequestStateChunk(request)) + .await; + let frames = sent.borrow(); + assert_eq!(frames.len(), 1, "one reply for the requested chunk"); + assert_eq!( + frames[0].as_slice()[std::mem::offset_of!(GenericHeader, command)], + Command::StateChunk as u8, + "the held prefix and unread tail must not be loaded" + ); + let chunk = + Message::::try_from(Owned::copy_from_slice(frames[0].as_slice())) + .unwrap(); + assert_eq!(chunk.header().offset, offset as u64); + assert_eq!(chunk.body(), expected); + assert!( + !owner.state_transfer_offers.borrow()[&(namespace.inner(), TRANSFER_REQUESTER)] + .fully_served + ); + } + } + + /// A request frame that fails the gates, here one with a stale partition + /// incarnation, must leave the parked request of its requester in place, + /// so the tick answers that request once its plan pins. + #[compio::test] + async fn given_a_parked_transfer_request_when_a_stale_request_arrives_should_answer_the_parked_one() + { + const PARKED_NONCE: u128 = 1; + const STALE_NONCE: u128 = 2; + const STALE_INCARNATION: u64 = SERVED_INCARNATION - 1; + let bus = Rc::new(IggyMessageBus::new(0)); + let targets = record_transfer_targets(&bus); + let namespace = IggyNamespace::new(0, 0, 0); + let directory = tempfile::tempdir().unwrap(); + let (owner, _sender) = serving_shard(&bus, &[namespace], directory.path()).await; + owner + .on_message(request_state_transfer( + namespace, + PARKED_NONCE, + SERVED_INCARNATION, + )) + .await; + assert_eq!(parked_nonce(&owner, namespace), Some(PARKED_NONCE)); + + owner + .on_message(request_state_transfer( + namespace, + STALE_NONCE, + STALE_INCARNATION, + )) + .await; + + assert_eq!( + parked_nonce(&owner, namespace), + Some(PARKED_NONCE), + "a request that fails the gates must not replace the parked one" + ); + pin_transfer_plans(&owner, &[namespace]).await; + owner.serve_parked_partition_transfer_requests().await; + assert_eq!( + *targets.borrow(), + [(namespace.inner(), PARKED_NONCE, OFFERED)], + "the tick must answer the parked request with its offer" + ); + } + + /// Two parked requests whose plans pin together at their bound must start + /// their offer builds on two ticks. One build reads and hashes up to a + /// whole round budget, and every timer of the shard waits for the tick + /// that runs it. The bound refuses only a request that still waits for its + /// pin, so the request that misses the bound tick builds on the next one. + #[compio::test] + async fn given_two_parked_transfer_requests_when_their_plans_pin_at_the_bound_should_answer_both_on_two_ticks() + { + const NONCE: u128 = 1; + let bus = Rc::new(IggyMessageBus::new(0)); + let targets = record_transfer_targets(&bus); + let namespaces = [IggyNamespace::new(0, 0, 0), IggyNamespace::new(0, 0, 1)]; + let directory = tempfile::tempdir().unwrap(); + let (owner, _sender) = serving_shard(&bus, &namespaces, directory.path()).await; + for namespace in namespaces { + owner + .on_message(request_state_transfer(namespace, NONCE, SERVED_INCARNATION)) + .await; + } + for _ in 1..PARKED_WAIT_TICKS { + owner.serve_parked_partition_transfer_requests().await; + } + assert!(targets.borrow().is_empty()); + pin_transfer_plans(&owner, &namespaces).await; + + owner.serve_parked_partition_transfer_requests().await; + assert_eq!( + *targets.borrow(), + [(namespaces[0].inner(), NONCE, OFFERED)], + "the bound tick must start one offer build" + ); + assert_eq!(parked_nonce(&owner, namespaces[1]), Some(NONCE)); + owner.serve_parked_partition_transfer_requests().await; + + assert_eq!( + *targets.borrow(), + [ + (namespaces[0].inner(), NONCE, OFFERED), + (namespaces[1].inner(), NONCE, OFFERED), + ], + "the request that missed the bound tick must get its offer, not a refusal" + ); + } + + /// A parked request whose plan does not pin must get one transient refusal + /// at its bound, before the requester re-sends and burns a stall retry. + /// The build slot stays claimed, as for any resumable refusal. + #[compio::test] + async fn given_a_parked_transfer_request_when_its_plan_does_not_pin_should_refuse_it_at_the_bound() + { + const NONCE: u128 = 1; + let bus = Rc::new(IggyMessageBus::new(0)); + let targets = record_transfer_targets(&bus); + let namespace = IggyNamespace::new(0, 0, 0); + let directory = tempfile::tempdir().unwrap(); + let (owner, _sender) = serving_shard(&bus, &[namespace], directory.path()).await; + owner + .on_message(request_state_transfer(namespace, NONCE, SERVED_INCARNATION)) + .await; + assert_eq!(parked_nonce(&owner, namespace), Some(NONCE)); + assert!( + owner + .partition_offer_builds + .borrow() + .contains_key(&namespace.inner()) + ); + + for _ in 1..PARKED_WAIT_TICKS { + owner.serve_parked_partition_transfer_requests().await; + } + assert!( + targets.borrow().is_empty(), + "the request must wait its whole bound" + ); + owner.serve_parked_partition_transfer_requests().await; + + assert_eq!(*targets.borrow(), [(namespace.inner(), NONCE, REFUSED)]); + assert_eq!(parked_nonce(&owner, namespace), None); + assert!( + owner + .partition_offer_builds + .borrow() + .contains_key(&namespace.inner()), + "the build slot must stay claimed for the next request" + ); + } + + /// A re-send of a parked request keeps the wait of its nonce, so re-sends + /// cannot hold the build slot past the bound. A request with a new nonce + /// replaces the parked one and waits a whole bound of its own. + #[compio::test] + async fn given_a_parked_transfer_request_when_it_is_re_sent_should_refuse_it_at_the_bound_of_its_nonce() + { + const NONCE: u128 = 1; + const NEW_NONCE: u128 = 2; + const RE_SENT_AT: u32 = PARKED_WAIT_TICKS / 2; + for (re_sent, refused_at) in [ + (NONCE, PARKED_WAIT_TICKS), + (NEW_NONCE, RE_SENT_AT + PARKED_WAIT_TICKS), + ] { + let bus = Rc::new(IggyMessageBus::new(0)); + let targets = record_transfer_targets(&bus); + let namespace = IggyNamespace::new(0, 0, 0); + let directory = tempfile::tempdir().unwrap(); + let (owner, _sender) = serving_shard(&bus, &[namespace], directory.path()).await; + owner + .on_message(request_state_transfer(namespace, NONCE, SERVED_INCARNATION)) + .await; + for _ in 0..RE_SENT_AT { + owner.serve_parked_partition_transfer_requests().await; + } + owner + .on_message(request_state_transfer( + namespace, + re_sent, + SERVED_INCARNATION, + )) + .await; + + for _ in RE_SENT_AT + 1..refused_at { + owner.serve_parked_partition_transfer_requests().await; + } + assert!( + targets.borrow().is_empty(), + "nonce {re_sent} must wait until tick {refused_at}" + ); + owner.serve_parked_partition_transfer_requests().await; + + assert_eq!( + *targets.borrow(), + [(namespace.inner(), re_sent, REFUSED)], + "nonce {re_sent} must get its refusal at tick {refused_at}" + ); + } + } + #[compio::test] async fn given_a_partition_incarnation_when_foreign_repair_or_transfer_requests_arrive_should_not_reply() { @@ -1265,6 +1637,10 @@ mod tests { })], target_accepted: command == Command::StateChunk, idle_ticks: IDLE_TICKS, + spill: None, + scan: None, + seed: None, + pulled_len: 0, }); } partitions.insert(namespace, partition); @@ -2136,6 +2512,8 @@ mod tests { owner.partition_io.notifier(), owner.partition_io.limits.bytes_max(), ); + // A vote two views ahead shows that replica 1 left view 0, so even its + // primary joins the vote and enters the view change. let message = Message::::new(size_of::()) .transmute_header(|_, header: &mut StartViewChangeHeader| { header.command = Command::StartViewChange; @@ -2143,7 +2521,7 @@ mod tests { header.cluster = 1; header.group = namespace.inner(); header.replica = 1; - header.view = 1; + header.view = 2; header.seal(); }); owner.on_start_view_change(message).await; @@ -2383,6 +2761,10 @@ mod tests { })], target_accepted: true, idle_ticks: 0, + spill: None, + scan: None, + seed: None, + pulled_len: 0, }); owner @@ -3543,16 +3925,28 @@ mod tests { } } - /// A spent ring that cannot convert keeps rotating at the pace of the + /// A spent ring that cannot convert never arms a transfer. A primary has no + /// primary to transfer from, so it keeps rotating at the pace of the /// budget, one rotation per spent budget. A backup with a scheduled re-arm - /// waits for it, and a primary has no primary to transfer from. + /// leaves recovery to the re-arm, so its spent session ends without a + /// successor. #[compio::test] - async fn given_ring_that_cannot_convert_when_ring_is_spent_should_rotate_once_per_budget() { + async fn given_ring_that_cannot_convert_when_ring_is_spent_should_never_arm_a_transfer() { const REARM_BACKOFF_TICKS: u32 = 1024; let budget = REPAIR_MAX_STALL_RETRIES as usize; - for (case, replica, peer) in [ - ("a backup waiting to re-arm", REPAIR_BACKUP, REPAIR_PRIMARY), - ("a primary", REPAIR_PRIMARY, REPAIR_BACKUP), + for (case, replica, peer, successor) in [ + ( + "a backup waiting to re-arm", + REPAIR_BACKUP, + REPAIR_PRIMARY, + None, + ), + ( + "a primary", + REPAIR_PRIMARY, + REPAIR_BACKUP, + Some(REPAIR_NEXT_PEER), + ), ] { let bus = Rc::new(IggyMessageBus::new(0)); let (owner, _sender) = test_owner(&bus, None); @@ -3575,67 +3969,495 @@ mod tests { owner.plane.partitions().insert(namespace, partition); let mut expected = vec![(Some(peer), None); budget]; - expected.extend(vec![(Some(REPAIR_NEXT_PEER), None); budget + 1]); - expected.push((Some(peer), None)); + expected.extend(vec![(successor, None); budget + 1]); + expected.push((successor.map(|_| peer), None)); let observed = stall_rounds(&owner, namespace, expected.len()).await; assert_eq!( observed, expected, - "{case} must rotate once per spent budget and never arm a transfer" + "{case} must rotate only without a scheduled re-arm and never arm a transfer" ); } } - /// A spent ring rotates as before, here dropping the session of a group - /// of two, when a transfer cannot take over. A scheduled re-arm keeps its - /// backoff, a busy stage belongs to the transfer in flight, a session can - /// outlive its stage when an election clears the stage, and a transfer - /// carries no uncommitted suffix. + /// A prefix read that has not landed by its bound gives up the seed. The + /// pull then asks for the whole artifact while the serving offer still + /// lives and burns no retry of the peer. From then on the stall sweep runs + /// as for any pull. #[compio::test] - async fn given_transfer_that_cannot_take_over_when_ring_is_spent_should_rotate_instead_of_arming() + async fn given_a_prefix_read_that_does_not_land_when_its_bound_passes_should_pull_the_artifact_whole() { - const REARM_BACKOFF_TICKS: u32 = 1024; - const TRANSFER_NONCE: u128 = 7; - const SUFFIX_OPS: u64 = 2; - type Setup = fn(&mut IggyPartition, HeldSuperblock>); - let cases: [(&str, u64, Setup); 4] = [ - ("an uncommitted suffix", LAGGING_COMMIT_MIN, |partition| { - partition.repair = partition.repair.map(|session| RepairSession { - fetch_to_op: session.commit_to_op + SUFFIX_OPS, - ..session - }); - }), - ("a scheduled re-arm", LAGGING_COMMIT_MAX, |partition| { - partition.transfer_rearm = Some(partitions::state_transfer::PendingTransferRearm { - peer: REPAIR_PRIMARY, - after_ticks: REARM_BACKOFF_TICKS, - }); - }), - ("a transfer stage", LAGGING_COMMIT_MAX, |partition| { - partition.consensus().begin_state_transfer_await(); - }), - ("a transfer session", LAGGING_COMMIT_MAX, |partition| { - partition.transfer = Some(PartitionTransferSession { - nonce: TRANSFER_NONCE, - peer: REPAIR_PRIMARY, - commit_op: 0, - artifacts: Vec::new(), - target_accepted: false, - idle_ticks: 0, - }); + const INCARNATION: u64 = 7; + const NONCE: u128 = 7; + const RETRY_TICKS: u32 = 2; + const OFFSETS: &[u8] = b"offsets"; + let bus = Rc::new(IggyMessageBus::new(0)); + let requests = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&requests); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + let message = + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(); + if message.header().command == Command::RequestStateChunk { + let request = Message::::try_from(Owned::copy_from_slice( + frame.as_slice(), + )) + .unwrap(); + let header = request.header(); + captured + .borrow_mut() + .push((header.artifact, header.offset, header.len)); + } + Ok(()) + })); + connect_replica_on(&bus, REPAIR_PRIMARY, 1); + let (owner, _sender) = test_owner(&bus, None); + owner.set_repair_retry_ticks(RETRY_TICKS); + let namespace = IggyNamespace::new(0, 0, 0); + let directory = tempfile::tempdir().unwrap(); + let segment = send_request(namespace, 1).body().to_vec(); + let partitions = owner.plane.partitions(); + partitions.remove(&namespace).unwrap(); + let mut partition = + repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); + partition.repair = None; + partition.set_created_revision(INCARNATION); + partition.set_partition_dir(directory.path().to_str().unwrap().to_owned()); + partition + .consensus() + .set_state_transfer_stage(StateTransferStage::AwaitingTarget); + partition + .consensus() + .set_state_transfer_stage(StateTransferStage::Fetching); + partition.transfer = Some(PartitionTransferSession { + nonce: NONCE, + peer: REPAIR_PRIMARY, + commit_op: 0, + artifacts: vec![ + TransferArtifact::Pending(ArtifactProgress { + entry: StateArtifact::for_bytes(artifact_kind::SEGMENT_LOG, 0, &segment), + buf: Vec::new(), + }), + TransferArtifact::Pending(ArtifactProgress { + entry: StateArtifact::for_bytes(artifact_kind::CONSUMER_OFFSETS, 0, OFFSETS), + buf: Vec::new(), + }), + ], + target_accepted: true, + idle_ticks: 0, + spill: None, + scan: None, + seed: Some(PrefixSeed::Reading { + artifact: 0, + read: LocalWork::default(), }), - ]; - let transfer_state = |partition: &IggyPartition, HeldSuperblock>| { - ( - partition.transfer.as_ref().map(|session| session.nonce), - partition.consensus().state_transfer_stage(), - partition.transfer_rearm.is_some(), - ) - }; - for (case, commit_max, setup) in cases { - let bus = Rc::new(IggyMessageBus::new(0)); - let (owner, _sender) = test_owner(&bus, None); - owner.set_repair_retry_ticks(1); + pulled_len: 0, + }); + partitions.insert(namespace, partition); + let bound = RETRY_TICKS * crate::STATE_TRANSFER_OFFER_EXPIRY_MULTIPLE + / crate::LOCAL_WORK_WAIT_DIVISOR; + let whole = (0, 0, u32::try_from(segment.len()).unwrap()); + + stall_rounds(&owner, namespace, bound as usize - 1).await; + assert!( + requests.borrow().is_empty(), + "the pull must wait for its read" + ); + stall_rounds(&owner, namespace, 1).await; + assert_eq!( + *requests.borrow(), + [whole], + "the pull must ask for the whole artifact at the bound" + ); + assert_eq!( + partitions + .get_by_ns(&namespace) + .unwrap() + .transfer_attempts(), + 0, + "a slow local read must not burn a retry of the peer" + ); + stall_rounds(&owner, namespace, RETRY_TICKS as usize).await; + assert_eq!( + ( + requests.borrow().clone(), + partitions + .get_by_ns(&namespace) + .unwrap() + .transfer_attempts() + ), + (vec![whole, whole], 1), + "the stall sweep must re-request the whole artifact after the bound" + ); + + let size = size_of::() + segment.len(); + let mut chunk = Message::::new(size); + chunk.as_mut_slice()[size_of::()..].copy_from_slice(&segment); + let chunk = chunk.transmute_header(|_, header: &mut StateChunkHeader| { + header.command = Command::StateChunk; + header.size = u32::try_from(size).unwrap(); + header.cluster = 1; + header.group = namespace.inner(); + header.replica = REPAIR_PRIMARY; + header.partition_incarnation = INCARNATION; + header.nonce = NONCE; + header.seal(); + }); + owner.on_message(MessageBag::StateChunk(chunk)).await; + settle_transfer_spill(&owner, namespace).await; + + let session = partitions + .get_by_ns(&namespace) + .unwrap() + .transfer + .as_ref() + .unwrap(); + assert!( + matches!(session.artifacts[0], TransferArtifact::Staged(_)), + "the whole pull must stage the artifact" + ); + assert_eq!( + requests.borrow().last(), + Some(&(1, 0, u32::try_from(OFFSETS.len()).unwrap())), + "the session must go on to its next artifact" + ); + } + + /// The I/O turn that hands a landed prefix read to the pull leaves the + /// ready queue. The partition takes another turn, so I/O work that the + /// read went ahead of does not wait for the next notification. + #[compio::test] + async fn given_a_landed_prefix_read_when_its_turn_resumes_the_pull_should_keep_the_partition_ready() + { + let bus = Rc::new(IggyMessageBus::new(0)); + bus.set_replica_forward_fn(Box::new(|_, _, _| Ok(()))); + connect_replica_on(&bus, REPAIR_PRIMARY, 1); + let (owner, _sender) = test_owner(&bus, None); + let namespace = IggyNamespace::new(0, 0, 0); + let partitions = owner.plane.partitions(); + partitions.remove(&namespace).unwrap(); + let mut partition = + repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); + partition.repair = None; + partition + .consensus() + .set_state_transfer_stage(StateTransferStage::AwaitingTarget); + partition + .consensus() + .set_state_transfer_stage(StateTransferStage::Fetching); + partition.transfer = Some(PartitionTransferSession { + nonce: 1, + peer: REPAIR_PRIMARY, + commit_op: 0, + artifacts: vec![TransferArtifact::Pending(ArtifactProgress { + entry: StateArtifact::for_bytes(artifact_kind::SEGMENT_LOG, 0, b"segment"), + buf: Vec::new(), + })], + target_accepted: true, + idle_ticks: 0, + spill: None, + scan: None, + seed: Some(PrefixSeed::Reading { + artifact: 0, + read: LocalWork { + result: Rc::new(RefCell::new(Some(Err(io::Error::other("the read failed"))))), + ..LocalWork::default() + }, + }), + pulled_len: 0, + }); + partitions.insert(namespace, partition); + partitions.set_io_notifier( + owner.partition_io.notifier(), + owner.partition_io.limits.bytes_max(), + ); + while let Some((ready, incarnation)) = owner.partition_io.head() { + owner.partition_io.pop_ready(ready, incarnation); + } + let incarnation = partitions.get_io_owner(&namespace).unwrap().incarnation(); + partitions.get_io_owner(&namespace).unwrap().notify_io(); + + owner.service_partition_io().await; + + assert!( + partitions + .get_by_ns(&namespace) + .unwrap() + .transfer + .as_ref() + .unwrap() + .seed + .is_none(), + "the turn must hand the landed read to the pull" + ); + assert_eq!( + owner.partition_io.head(), + Some((namespace, incarnation)), + "the partition must keep a turn for the work that the read went ahead of" + ); + } + + /// A spent ring rotates as before, here dropping the session of a group + /// of two, when a transfer cannot take over. A scheduled re-arm keeps its + /// backoff, a busy stage belongs to the transfer in flight, a session can + /// outlive its stage when an election clears the stage, and a transfer + /// carries no uncommitted suffix. + /// A reuse scan that outlives its bound is asked to stop. The pull still + /// waits until the task lands, because the scan writes the staging names + /// that the pull spills to. It then fetches every segment that the scan did + /// not reuse and finishes. The wait burns no retry of the peer. + #[compio::test] + async fn given_a_reuse_scan_that_does_not_land_by_its_bound_when_it_lands_should_pull_the_rest_and_finish() + { + let receiver = scanning_receiver(); + let bound = SCANNING_RETRY_TICKS * crate::STATE_TRANSFER_OFFER_EXPIRY_MULTIPLE + / crate::LOCAL_WORK_WAIT_DIVISOR; + let partitions = receiver.owner.plane.partitions(); + let session = || { + partitions + .get_by_ns(&receiver.namespace) + .unwrap() + .transfer + .as_ref() + }; + + stall_rounds(&receiver.owner, receiver.namespace, bound as usize).await; + assert!( + receiver.requests.borrow().is_empty(), + "the pull must wait for its scan" + ); + let scan = session().unwrap().scan.as_ref().unwrap(); + assert!(!scan.stop(), "the bound must ask the scan to stop"); + *scan.result.borrow_mut() = Some(ReuseScanned::default()); + stall_rounds(&receiver.owner, receiver.namespace, 1).await; + assert_eq!( + *receiver.requests.borrow(), + [(0, 0, u32::try_from(receiver.segment.len()).unwrap())], + "the pull must fetch the segment that the scan did not reuse" + ); + receiver + .owner + .on_message(MessageBag::StateChunk(receiver.chunk(0, &receiver.segment))) + .await; + settle_transfer_spill(&receiver.owner, receiver.namespace).await; + receiver + .owner + .on_message(MessageBag::StateChunk(receiver.chunk(1, SCANNING_OFFSETS))) + .await; + + assert_eq!( + receiver.requests.borrow().last(), + Some(&(1, 0, u32::try_from(SCANNING_OFFSETS.len()).unwrap())), + ); + assert!( + session().is_none(), + "the session must finish its pull and hand it to the install" + ); + assert_eq!( + partitions + .get_by_ns(&receiver.namespace) + .unwrap() + .transfer_attempts(), + 0, + "a slow scan must not burn a retry of the peer" + ); + } + + #[compio::test] + async fn given_local_transfer_work_when_ticks_pass_should_refresh_the_same_offer() { + let receiver = scanning_receiver(); + stall_rounds( + &receiver.owner, + receiver.namespace, + 3 * SCANNING_RETRY_TICKS as usize, + ) + .await; + + assert_eq!( + *receiver.offer_requests.borrow(), + [( + receiver.namespace.inner(), + SCANNING_INCARNATION, + SCANNING_NONCE + ); 3], + "local work must renew the accepted offer instead of minting a new session" + ); + assert_eq!( + receiver + .owner + .plane + .partitions() + .get_by_ns(&receiver.namespace) + .unwrap() + .transfer_attempts(), + 0, + "waiting for local I/O must not spend the peer's retry budget" + ); + } + + #[compio::test] + async fn given_a_completed_artifact_when_receiving_its_last_chunk_should_queue_staging() { + let receiver = scanning_receiver(); + let partitions = receiver.owner.plane.partitions(); + *partitions + .get_by_ns(&receiver.namespace) + .unwrap() + .transfer + .as_ref() + .unwrap() + .scan + .as_ref() + .unwrap() + .result + .borrow_mut() = Some(ReuseScanned::default()); + receiver + .owner + .on_partition_transfer_progress(receiver.namespace.inner()) + .await; + + receiver + .owner + .on_message(MessageBag::StateChunk(receiver.chunk(0, &receiver.segment))) + .await; + + assert!( + !receiver + .directory + .path() + .join("00000000000000000000.log.staging") + .exists(), + "receiving a chunk must return before staging I/O runs on the shard pump" + ); + settle_transfer_spill(&receiver.owner, receiver.namespace).await; + assert!(matches!( + partitions + .get_by_ns(&receiver.namespace) + .unwrap() + .transfer + .as_ref() + .unwrap() + .artifacts[0], + TransferArtifact::Staged(_) + )); + } + + /// The pull neither requests nor spills while its reuse scan runs, and the + /// wait burns no retry of the peer. Once the scan lands, the pull fetches + /// only what the scan did not reuse. + #[compio::test] + async fn given_a_running_reuse_scan_when_the_pull_progresses_should_request_nothing_until_it_lands() + { + let receiver = scanning_receiver(); + let partitions = receiver.owner.plane.partitions(); + let session = || { + partitions + .get_by_ns(&receiver.namespace) + .unwrap() + .transfer + .as_ref() + .unwrap() + }; + + receiver + .owner + .on_partition_transfer_progress(receiver.namespace.inner()) + .await; + stall_rounds( + &receiver.owner, + receiver.namespace, + 2 * SCANNING_RETRY_TICKS as usize, + ) + .await; + assert!( + receiver.requests.borrow().is_empty(), + "the pull must wait for its scan" + ); + assert_eq!( + partitions + .get_by_ns(&receiver.namespace) + .unwrap() + .transfer_attempts(), + 0, + "a running scan must not burn a retry of the peer" + ); + let adopted = StagedSegmentMeta { + start_offset: 0, + end_offset: 0, + index_size: 0, + size: receiver.segment.len() as u64, + start_timestamp: 0, + end_timestamp: 0, + max_timestamp: 0, + log_staging: PathBuf::new(), + index_staging: PathBuf::new(), + checksum: 0, + linked: false, + }; + let mut scanned = ReuseScanned::default(); + scanned.reused.push((0, adopted)); + *session().scan.as_ref().unwrap().result.borrow_mut() = Some(scanned); + stall_rounds(&receiver.owner, receiver.namespace, 1).await; + + assert_eq!( + *receiver.requests.borrow(), + [(1, 0, u32::try_from(SCANNING_OFFSETS.len()).unwrap())], + "the pull must fetch only what the scan did not reuse" + ); + assert!(matches!( + session().artifacts[0], + TransferArtifact::Staged(_) + )); + } + + #[compio::test] + async fn given_transfer_that_cannot_take_over_when_ring_is_spent_should_rotate_instead_of_arming() + { + const REARM_BACKOFF_TICKS: u32 = 1024; + const TRANSFER_NONCE: u128 = 7; + const SUFFIX_OPS: u64 = 2; + type Setup = fn(&mut IggyPartition, HeldSuperblock>); + let cases: [(&str, u64, Setup); 4] = [ + ("an uncommitted suffix", LAGGING_COMMIT_MIN, |partition| { + partition.repair = partition.repair.map(|session| RepairSession { + fetch_to_op: session.commit_to_op + SUFFIX_OPS, + ..session + }); + }), + ("a scheduled re-arm", LAGGING_COMMIT_MAX, |partition| { + partition.transfer_rearm = Some(partitions::state_transfer::PendingTransferRearm { + peer: REPAIR_PRIMARY, + after_ticks: REARM_BACKOFF_TICKS, + }); + }), + ("a transfer stage", LAGGING_COMMIT_MAX, |partition| { + partition.consensus().begin_state_transfer_await(); + }), + ("a transfer session", LAGGING_COMMIT_MAX, |partition| { + partition.transfer = Some(PartitionTransferSession { + nonce: TRANSFER_NONCE, + peer: REPAIR_PRIMARY, + commit_op: 0, + artifacts: Vec::new(), + target_accepted: false, + idle_ticks: 0, + spill: None, + scan: None, + seed: None, + pulled_len: 0, + }); + }), + ]; + let transfer_state = |partition: &IggyPartition, HeldSuperblock>| { + ( + partition.transfer.as_ref().map(|session| session.nonce), + partition.consensus().state_transfer_stage(), + partition.transfer_rearm.is_some(), + ) + }; + for (case, commit_max, setup) in cases { + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = test_owner(&bus, None); + owner.set_repair_retry_ticks(1); let namespace = IggyNamespace::new(0, 0, 0); let mut backup = repairing_replica( &bus, @@ -3802,33 +4624,231 @@ mod tests { ); } - /// A rotation that the live-session ceiling refuses opens no session, so - /// it does not count toward the ring. The detector re-arms the primary - /// later, and the ring still reaches the other backup before it converts. + /// The order a lagging backup saw under load. Its repair peer had evicted + /// the bottom of the window, so the reply carried `RangeEvicted` and then + /// ops above a hole. The backup armed a transfer, the peer refused it as + /// transient, and the live stream then delivered the hole. The live + /// prepares must fill the hole and carry the head over the run that repair + /// journaled, and the scheduled re-arm must not start a transfer while the + /// commit walk catches up. #[compio::test] - async fn given_rotation_refused_at_session_ceiling_when_ring_resumes_should_ask_next_peer_before_transfer() - { + async fn given_repair_above_a_hole_when_the_live_stream_fills_it_should_need_no_transfer() { + /// Committed and applied before the repair round. + const APPLIED: u64 = 4; + /// The lowest op the repair peer still retains, so the hole + /// `APPLIED + 1..RETAINED_FROM` is 28 ops long. + const RETAINED_FROM: u64 = APPLIED + 29; + /// The end of the one chunk the peer serves from `RETAINED_FROM`. + const SERVED_THROUGH: u64 = RETAINED_FROM + crate::REPAIR_CHUNK_MAX - 1; + /// The group head and commit point: four bounded commit walks above + /// `APPLIED`, so the re-arm comes due while the walk still runs. + const HEAD: u64 = APPLIED + 4 * partitions::COMMIT_WALK_OPS_MAX as u64; + const NONCE: u128 = 1; let bus = Rc::new(IggyMessageBus::new(0)); - let (owner, _sender) = test_owner(&bus, None); + let (mut owner, _sender) = test_owner(&bus, None); + owner.partition_consensus.replica_count = 3; owner.set_repair_retry_ticks(1); - owner.set_repair_gap_debounce_ticks(REPAIR_GAP_DEBOUNCE_TICKS_MIN); let namespace = IggyNamespace::new(0, 0, 0); - owner.plane.partitions().insert( - namespace, - repairing_backup(&bus, namespace, LAGGING_COMMIT_MIN, LAGGING_COMMIT_MAX), - ); - let budget = REPAIR_MAX_STALL_RETRIES as usize; - let mut expected = vec![(Some(REPAIR_PRIMARY), None); budget]; - expected.extend(vec![(None, None); REPAIR_GAP_DEBOUNCE_TICKS_MIN as usize]); - expected.extend(vec![(Some(REPAIR_PRIMARY), None); budget + 1]); - expected.extend(vec![(Some(REPAIR_NEXT_PEER), None); budget + 1]); - expected.push((None, Some(REPAIR_PRIMARY))); + let prepares = primary_prepares(&bus, namespace, HEAD).await; + let applied = usize::try_from(APPLIED).unwrap(); + let mut backup = repairing_backup(&bus, namespace, 0, 0); + backup.repair = None; + for prepare in &prepares[..applied] { + backup.on_replicate(prepare.clone()).await; + } + backup.consensus().advance_commit_max(APPLIED); + backup + .commit_journal(owner.plane.partitions().config()) + .await; + assert_eq!(backup.consensus().commit_min(), APPLIED); + backup.consensus().advance_commit_max(HEAD); + backup.repair = Some(RepairSession { + nonce: NONCE, + view: REPAIR_VIEW, + from_op: APPLIED + 1, + commit_to_op: HEAD, + fetch_to_op: HEAD, + floor: None, + peer: REPAIR_PRIMARY, + idle_ticks: 0, + floor_pulled_from: 0, + requested_from: Some(APPLIED + 1), + }); + owner.plane.partitions().remove(&namespace).unwrap(); + owner.plane.partitions().insert(namespace, backup); - let mut observed = stall_rounds(&owner, namespace, budget).await; - // Other sessions fill the ceiling, for the sweep of the rotation only: - // the rotation hands the slot of its dropped session to the successor, - // and each sweep republishes the tally. - owner + let evicted = repair_done(namespace, REPAIR_PRIMARY, NONCE, RETAINED_FROM) + .transmute_header(|done, header: &mut RepairRangeReplyHeader| { + *header = done; + header.command = Command::RangeEvicted; + header.seal(); + }); + owner.on_repair_range_reply(&evicted).await; + for op in RETAINED_FROM..=SERVED_THROUGH { + let prepare = prepares[usize::try_from(op - 1).unwrap()].clone(); + owner.on_repair_prepare(as_repair_prepare(prepare)).await; + } + owner + .on_repair_range_reply(&repair_done( + namespace, + REPAIR_PRIMARY, + NONCE, + SERVED_THROUGH, + )) + .await; + let transfer_nonce = { + let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap(); + partition + .transfer + .as_ref() + .expect("the floor the peer evicted past arms a transfer") + .nonce + }; + let refusal = + Message::::new(size_of::()) + .transmute_header(|_, header: &mut StateTransferTargetHeader| { + header.command = Command::StateTransferTarget; + header.size = u32::try_from(size_of::()).unwrap(); + header.cluster = 1; + header.group = namespace.inner(); + header.replica = REPAIR_PRIMARY; + header.nonce = transfer_nonce; + header.unavailable_transient = 1; + header.seal(); + }); + owner + .on_message(MessageBag::StateTransferTarget(refusal)) + .await; + { + let partition = owner.plane.partitions().get_mut_by_ns(&namespace).unwrap(); + assert!(partition.transfer.is_none()); + assert!(partition.transfer_rearm.is_some()); + for prepare in &prepares[applied..] { + partition.on_replicate(prepare.clone()).await; + } + assert_eq!( + partition.held_through(), + HEAD, + "the live stream must fill the hole below the repaired run" + ); + assert_eq!(partition.consensus().sequencer().current_sequence(), HEAD); + } + + let rounds = usize::try_from(HEAD - APPLIED).unwrap() / partitions::COMMIT_WALK_OPS_MAX + 1; + let observed = stall_rounds(&owner, namespace, rounds).await; + + assert!( + observed.iter().all(|(_, transfer)| transfer.is_none()), + "a journal that holds every committed op needs no transfer: {observed:?}" + ); + let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap(); + assert_eq!(partition.consensus().commit_min(), HEAD); + assert!(partition.transfer_rearm.is_none()); + } + + /// A live copy of an op that repair journaled above a hole leaves the head + /// below the hole. A head above a hole can win a view change that this + /// replica cannot walk. + #[compio::test] + async fn given_repaired_op_above_a_hole_when_its_live_copy_arrives_should_keep_the_head_below_the_hole() + { + let bus = Rc::new(IggyMessageBus::new(0)); + let namespace = IggyNamespace::new(0, 0, 0); + let (mut backup, [below, _, above]) = + Box::pin(backup_with_repaired_op_above_a_hole(&bus, namespace)).await; + + backup.on_replicate(above).await; + + assert_eq!( + backup.consensus().sequencer().current_sequence(), + below.header().op, + "the head must stay below the hole" + ); + } + + /// A live prepare that fills a hole below a repaired op carries offsets + /// below the dirty offset, so only its link to the entry below it shows + /// that it belongs to this log. A copy with another parent stays out of the + /// journal, and the chained copy then fills the hole. + #[compio::test] + async fn given_hole_below_a_repaired_op_when_a_live_prepare_does_not_chain_to_it_should_not_journal_it() + { + let bus = Rc::new(IggyMessageBus::new(0)); + let namespace = IggyNamespace::new(0, 0, 0); + let (mut backup, [_, hole, _]) = + Box::pin(backup_with_repaired_op_above_a_hole(&bus, namespace)).await; + let op = hole.header().op; + let unchained = hole + .clone() + .transmute_header(|original, header: &mut PrepareHeader| { + *header = original; + header.parent = !original.parent; + header.checksum = header.identity_checksum(); + }); + + backup.on_replicate(unchained).await; + let unchained_journaled = backup.log.journal().inner.holds_op(op); + backup.on_replicate(hole).await; + + assert_eq!( + (unchained_journaled, backup.log.journal().inner.holds_op(op)), + (false, true), + "(unchained copy journaled, chained copy journaled)" + ); + } + + /// A backup that journaled the first of three ops from the live stream and + /// the third from repair, so the second is a hole below a repaired op. + /// Returned with the three prepares. + #[allow(clippy::future_not_send)] + async fn backup_with_repaired_op_above_a_hole( + bus: &Rc, + namespace: IggyNamespace, + ) -> ( + IggyPartition, HeldSuperblock>, + [Message; 3], + ) { + let mut prepares = primary_prepares(bus, namespace, 3).await; + let above = prepares.pop().unwrap(); + let hole = prepares.pop().unwrap(); + let below = prepares.pop().unwrap(); + let mut backup = repairing_backup(bus, namespace, 0, above.header().op); + backup.on_replicate(below.clone()).await; + backup.apply_repaired_prepare(above.clone()).await; + assert!( + backup.log.journal().inner.holds_op(above.header().op), + "repair journals the op above the hole" + ); + (backup, [below, hole, above]) + } + + /// A rotation that the live-session ceiling refuses opens no session, so + /// it does not count toward the ring. The detector re-arms the primary + /// later, and the ring still reaches the other backup before it converts. + #[compio::test] + async fn given_rotation_refused_at_session_ceiling_when_ring_resumes_should_ask_next_peer_before_transfer() + { + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = test_owner(&bus, None); + owner.set_repair_retry_ticks(1); + owner.set_repair_gap_debounce_ticks(REPAIR_GAP_DEBOUNCE_TICKS_MIN); + let namespace = IggyNamespace::new(0, 0, 0); + owner.plane.partitions().insert( + namespace, + repairing_backup(&bus, namespace, LAGGING_COMMIT_MIN, LAGGING_COMMIT_MAX), + ); + let budget = REPAIR_MAX_STALL_RETRIES as usize; + let mut expected = vec![(Some(REPAIR_PRIMARY), None); budget]; + expected.extend(vec![(None, None); REPAIR_GAP_DEBOUNCE_TICKS_MIN as usize]); + expected.extend(vec![(Some(REPAIR_PRIMARY), None); budget + 1]); + expected.extend(vec![(Some(REPAIR_NEXT_PEER), None); budget + 1]); + expected.push((None, Some(REPAIR_PRIMARY))); + + let mut observed = stall_rounds(&owner, namespace, budget).await; + // Other sessions fill the ceiling, for the sweep of the rotation only: + // the rotation hands the slot of its dropped session to the successor, + // and each sweep republishes the tally. + owner .partition_repairs_inflight .set(PARTITION_REPAIRS_INFLIGHT_MAX + 1); observed.extend(stall_rounds(&owner, namespace, expected.len() - budget).await); @@ -3965,193 +4985,685 @@ mod tests { partitions.insert(namespace, backup); } owner - .partition_repairs_inflight - .set(PARTITION_REPAIRS_INFLIGHT_MAX); - let waiting = namespaces[PARTITION_REPAIRS_INFLIGHT_MAX]; - let mut waiting_armed = false; - for sweep in 1..=2 * budget + 1 { - assert!(owner.tick_partitions(&mut Vec::new()).await.is_none()); - let live = namespaces - .iter() - .filter(|namespace| partitions.get_by_ns(namespace).unwrap().repair.is_some()) - .count(); - assert_eq!( - owner.partition_repairs_inflight.get(), - live, - "sweep {sweep}" - ); - assert!( - live <= PARTITION_REPAIRS_INFLIGHT_MAX, - "sweep {sweep}: {live}" - ); - waiting_armed |= partitions.get_by_ns(&waiting).unwrap().repair.is_some(); - } - assert!( - waiting_armed, - "spent rings must release capacity for a waiting group" + .partition_repairs_inflight + .set(PARTITION_REPAIRS_INFLIGHT_MAX); + let waiting = namespaces[PARTITION_REPAIRS_INFLIGHT_MAX]; + let mut waiting_armed = false; + for sweep in 1..=2 * budget + 1 { + assert!(owner.tick_partitions(&mut Vec::new()).await.is_none()); + let live = namespaces + .iter() + .filter(|namespace| partitions.get_by_ns(namespace).unwrap().repair.is_some()) + .count(); + assert_eq!( + owner.partition_repairs_inflight.get(), + live, + "sweep {sweep}" + ); + assert!( + live <= PARTITION_REPAIRS_INFLIGHT_MAX, + "sweep {sweep}: {live}" + ); + waiting_armed |= partitions.get_by_ns(&waiting).unwrap().repair.is_some(); + } + assert!( + waiting_armed, + "spent rings must release capacity for a waiting group" + ); + } + + #[compio::test] + async fn given_staggered_sessions_at_ceiling_when_rotating_should_preserve_each_slot() { + const WAITING_GROUPS: usize = 4; + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = test_owner(&bus, None); + owner.set_repair_retry_ticks(1); + owner.set_repair_gap_debounce_ticks(REPAIR_GAP_DEBOUNCE_TICKS_MIN); + let partitions = owner.plane.partitions(); + let budget = REPAIR_MAX_STALL_RETRIES as usize + 1; + let namespaces: Vec<_> = (0..PARTITION_REPAIRS_INFLIGHT_MAX + WAITING_GROUPS) + .map(|partition_id| IggyNamespace::new(0, 0, partition_id)) + .collect(); + for (index, &namespace) in namespaces.iter().enumerate() { + let mut backup = + repairing_backup(&bus, namespace, LAGGING_COMMIT_MIN, LAGGING_COMMIT_MAX); + if index < PARTITION_REPAIRS_INFLIGHT_MAX { + for _ in 0..index % budget { + assert!(!backup.burn_repair_attempt()); + } + } else { + backup.repair = None; + backup.gap_ticks.set(REPAIR_GAP_DEBOUNCE_TICKS_MIN); + } + partitions.insert(namespace, backup); + } + owner + .partition_repairs_inflight + .set(PARTITION_REPAIRS_INFLIGHT_MAX); + for sweep in 1..=budget { + assert!(owner.tick_partitions(&mut Vec::new()).await.is_none()); + let live = namespaces + .iter() + .filter(|namespace| partitions.get_by_ns(namespace).unwrap().repair.is_some()) + .count(); + assert_eq!(live, PARTITION_REPAIRS_INFLIGHT_MAX, "sweep {sweep}"); + assert_eq!( + owner.partition_repairs_inflight.get(), + live, + "sweep {sweep}" + ); + for (index, namespace) in namespaces.iter().enumerate() { + let expected = if index >= PARTITION_REPAIRS_INFLIGHT_MAX { + None + } else if sweep >= budget - index % budget { + Some(REPAIR_NEXT_PEER) + } else { + Some(REPAIR_PRIMARY) + }; + let peer = partitions + .get_by_ns(namespace) + .unwrap() + .repair + .map(|session| session.peer); + assert_eq!(peer, expected, "sweep {sweep}, group {index}"); + } + } + } + + /// A transfer abandoned at the session ceiling schedules a re-arm and opens + /// no repair session while it waits, so its slot goes to a waiting group. + #[compio::test] + async fn given_transfer_abandoned_at_ceiling_when_its_re_arm_waits_should_give_the_slot_to_a_waiting_group() + { + const OTHER_SESSIONS: usize = PARTITION_REPAIRS_INFLIGHT_MAX - 1; + const SWEEPS: usize = 2; + for retry_ticks in [1, partitions::REPAIR_RETRY_TICKS] { + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = test_owner(&bus, None); + owner.set_repair_retry_ticks(retry_ticks); + owner.set_repair_gap_debounce_ticks(REPAIR_GAP_DEBOUNCE_TICKS_MIN); + let partitions = owner.plane.partitions(); + let namespaces: Vec<_> = (0..=PARTITION_REPAIRS_INFLIGHT_MAX) + .map(|partition_id| IggyNamespace::new(0, 0, partition_id)) + .collect(); + for &namespace in &namespaces[..OTHER_SESSIONS] { + partitions.insert( + namespace, + repairing_backup(&bus, namespace, LAGGING_COMMIT_MIN, LAGGING_COMMIT_MAX), + ); + } + let transferring = namespaces[OTHER_SESSIONS]; + let mut backup = + repairing_backup(&bus, transferring, LAGGING_COMMIT_MIN, LAGGING_COMMIT_MAX); + backup.consensus().begin_state_transfer_await(); + assert!( + owner + .arm_partition_transfer(&mut backup, REPAIR_PRIMARY, 0) + .await + ); + for _ in 0..consensus::STATE_TRANSFER_MAX_STALL_RETRIES { + assert!(!backup.burn_transfer_attempt()); + } + backup.transfer.as_mut().unwrap().idle_ticks = retry_ticks - 1; + partitions.insert(transferring, backup); + let waiting = namespaces[PARTITION_REPAIRS_INFLIGHT_MAX]; + let mut late = repairing_backup(&bus, waiting, LAGGING_COMMIT_MIN, LAGGING_COMMIT_MAX); + late.repair = None; + late.gap_ticks.set(REPAIR_GAP_DEBOUNCE_TICKS_MIN); + partitions.insert(waiting, late); + owner.partition_repairs_inflight.set(OTHER_SESSIONS); + + for sweep in 1..=SWEEPS { + assert!(owner.tick_partitions(&mut Vec::new()).await.is_none()); + let live = namespaces + .iter() + .filter(|namespace| partitions.get_by_ns(namespace).unwrap().repair.is_some()) + .count(); + assert_eq!( + live, PARTITION_REPAIRS_INFLIGHT_MAX, + "interval {retry_ticks}, sweep {sweep}" + ); + assert_eq!( + owner.partition_repairs_inflight.get(), + live, + "interval {retry_ticks}, sweep {sweep}" + ); + let abandoned = partitions.get_by_ns(&transferring).unwrap(); + assert_eq!( + ( + abandoned.repair.is_some(), + abandoned.transfer_rearm.is_some(), + partitions.get_by_ns(&waiting).unwrap().repair.is_some(), + ), + (false, true, true), + "(abandoned group repairs, its re-arm waits, waiting group repairs) at \ + interval {retry_ticks}, sweep {sweep}" + ); + } + } + } + + /// A transfer that supersedes a partly spent repair restarts the repair + /// budget. Once the transfer is abandoned, no repair session opens while + /// its re-arm waits, and the re-arm then arms the transfer again. + #[compio::test] + async fn given_superseded_repair_when_its_transfer_is_abandoned_should_open_no_repair_until_the_re_arm() + { + /// Long enough for a repair session to spend a whole budget and rotate + /// while the re-arm waits. + const REARM_BACKOFF_TICKS: u32 = 2 * (REPAIR_MAX_STALL_RETRIES + 1); + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = test_owner(&bus, None); + owner.set_repair_retry_ticks(1); + let namespace = IggyNamespace::new(0, 0, 0); + let mut backup = repairing_backup(&bus, namespace, LAGGING_COMMIT_MIN, LAGGING_COMMIT_MAX); + for _ in 0..REPAIR_MAX_STALL_RETRIES { + assert!(!backup.burn_repair_attempt()); + } + backup.note_repair_rotation(); + backup.repair.as_mut().unwrap().peer = REPAIR_NEXT_PEER; + backup.consensus().begin_state_transfer_await(); + + assert!( + owner + .arm_partition_transfer(&mut backup, REPAIR_PRIMARY, 0) + .await + ); + assert!(backup.repair.is_none()); + assert_eq!(backup.repair_rotations(), 0); + owner + .abandon_or_rearm_partition_transfer(&mut backup, REPAIR_PRIMARY) + .await; + assert!(backup.repair.is_none()); + let rearm = backup.transfer_rearm.as_mut().unwrap(); + rearm.after_ticks = REARM_BACKOFF_TICKS; + let rearm_peer = rearm.peer; + owner.plane.partitions().insert(namespace, backup); + let waiting = REARM_BACKOFF_TICKS as usize; + let observed = stall_rounds(&owner, namespace, waiting + 1).await; + let mut expected = vec![(None, None); waiting]; + expected.push((None, Some(rearm_peer))); + assert_eq!( + observed, expected, + "a repair session must not open while the re-arm waits" + ); + } + + /// A lagging backup whose peers evicted the ops it misses converts its + /// repair to a state transfer. Each transient refusal schedules a re-arm, + /// and no repair session opens while it waits. The frames that peers + /// serve above their floor and the live prepares above the hole therefore + /// stay out of the journal until the transfer installs. + #[compio::test] + async fn given_peers_serving_above_their_floor_when_the_transfer_is_refused_repeatedly_should_not_grow_the_journal() + { + const FIRST_FLOOR: u64 = 4; + const REFUSALS: u64 = 3; + /// Room for one more served reply per refusal above the first one. + const COMMIT_MAX: u64 = FIRST_FLOOR + (REFUSALS + 1) * FRAMES_ABOVE_FLOOR; + /// A flat re-arm fires on the second tick at a one-tick retry interval, + /// so four ticks leave slack. + const REARM_TICKS_MAX: usize = 4; + let bus = Rc::new(IggyMessageBus::new(0)); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + bus.set_replica_forward_fn(Box::new(move |peer, _, frame| { + captured.borrow_mut().push((peer, frame)); + Ok(()) + })); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = primary_prepares(&bus, namespace, COMMIT_MAX + REFUSALS).await; + let mut floor = FIRST_FLOOR; + let (owner, _sender) = floor_converted_backup( + &bus, + namespace, + floor, + &prepares[..usize::try_from(COMMIT_MAX).unwrap()], + ) + .await; + let partitions = owner.plane.partitions(); + let recovery = || { + let partition = partitions.get_by_ns(&namespace).unwrap(); + let journal = &partition.log.journal().inner; + ( + journal.resident_count(), + journal.last_op(), + partition.transfer.is_some(), + ) + }; + let converted = recovery(); + assert_eq!( + converted, + ( + usize::try_from(FRAMES_ABOVE_FLOOR).unwrap(), + Some(FIRST_FLOOR + FRAMES_ABOVE_FLOOR), + true + ), + "the first conversion journals the frames served above the floor and arms a transfer" + ); + sent.borrow_mut().clear(); + + for refusal in 1..=REFUSALS { + let (peer, nonce) = { + let partition = partitions.get_by_ns(&namespace).unwrap(); + let transfer = partition.transfer.as_ref().unwrap(); + (transfer.peer, transfer.nonce) + }; + owner + .on_message(transient_transfer_refusal(namespace, (peer, nonce))) + .await; + let live = prepares[usize::try_from(COMMIT_MAX + refusal - 1).unwrap()].clone(); + partitions + .get_mut_by_ns(&namespace) + .unwrap() + .on_replicate(live) + .await; + for _ in 0..REARM_TICKS_MAX { + let requests: Vec<_> = sent + .borrow_mut() + .drain(..) + .filter_map(|(peer, frame)| { + let request = Message::::try_from( + Owned::copy_from_slice(frame.as_slice()), + ) + .ok()?; + (request.header().command == Command::RequestPrepares) + .then(|| (peer, request.header().nonce)) + }) + .collect(); + for request in requests { + floor += FRAMES_ABOVE_FLOOR; + serve_above_floor(&owner, namespace, request, floor, &prepares).await; + } + if partitions.get_by_ns(&namespace).unwrap().transfer.is_some() { + break; + } + assert!(owner.tick_partitions(&mut Vec::new()).await.is_none()); + } + assert_eq!( + recovery(), + converted, + "(resident entries, last op, transfer re-armed) after refusal {refusal}" + ); + } + } + + /// A backup adopts a `StartView` at the head it holds while a transfer + /// re-arm waits, so the next live prepare passes the gap check above its + /// hole. The backup journals and acknowledges nothing, because the + /// install wipes the journal. The primary sends to the other backups. + #[compio::test] + async fn given_start_view_adopted_while_a_transfer_re_arm_waits_when_a_live_prepare_arrives_should_drop_it_without_growing_the_journal() + { + const FLOOR: u64 = 4; + /// The backup holds every op above the floor through the group head. + const HEAD: u64 = FLOOR + FRAMES_ABOVE_FLOOR; + let bus = Rc::new(IggyMessageBus::new(0)); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + bus.set_replica_forward_fn(Box::new(move |replica, _, frame| { + let message = + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(); + captured + .borrow_mut() + .push((replica, message.header().command)); + Ok(()) + })); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = primary_prepares(&bus, namespace, HEAD + 1).await; + let head = usize::try_from(HEAD).unwrap(); + let (owner, _sender) = + floor_converted_backup(&bus, namespace, FLOOR, &prepares[..head]).await; + let partitions = owner.plane.partitions(); + let transfer = { + let partition = partitions.get_by_ns(&namespace).unwrap(); + let transfer = partition.transfer.as_ref().unwrap(); + (transfer.peer, transfer.nonce) + }; + owner + .on_message(transient_transfer_refusal(namespace, transfer)) + .await; + owner + .on_start_view(start_view_suffix( + namespace, + REPAIR_VIEW, + HEAD, + HEAD, + &[*prepares[head - 1].header()], + )) + .await; + let journal_shape = || { + let journal = &partitions + .get_by_ns(&namespace) + .unwrap() + .log + .journal() + .inner; + (journal.resident_count(), journal.last_op()) + }; + let adopted = journal_shape(); + { + let partition = partitions.get_by_ns(&namespace).unwrap(); + assert!(partition.transfer_rearm.is_some(), "the re-arm still waits"); + assert_eq!(partition.consensus().sequencer().current_sequence(), HEAD); + } + sent.borrow_mut().clear(); + + partitions + .get_mut_by_ns(&namespace) + .unwrap() + .on_replicate(prepares[head].clone()) + .await; + + assert_eq!(journal_shape(), adopted, "(resident entries, last op)"); + assert!( + sent.borrow().is_empty(), + "refused data is neither forwarded nor acknowledged" + ); + } + + /// The live stream fills the hole of a backup whose transfer re-arm waits + /// out a long backoff. The re-arm stands down at once, so a gap that opens + /// later reaches the gap detector and is repaired, not transferred. + #[compio::test] + async fn given_hole_filled_while_a_long_re_arm_waits_when_a_new_gap_opens_should_repair_it() { + /// The ops that the live stream delivers into the hole. + const HEAD: u64 = 3; + /// Far past the rounds below, so only a stand-down lets repair run. + const REARM_BACKOFF_TICKS: u32 = 1024; + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = test_owner(&bus, None); + owner.set_repair_retry_ticks(1); + owner.set_repair_gap_debounce_ticks(REPAIR_GAP_DEBOUNCE_TICKS_MIN); + let namespace = IggyNamespace::new(0, 0, 0); + let mut backup = repairing_backup(&bus, namespace, 0, HEAD); + backup.repair = None; + backup.transfer_rearm = Some(partitions::state_transfer::PendingTransferRearm { + peer: REPAIR_PRIMARY, + after_ticks: REARM_BACKOFF_TICKS, + }); + for prepare in primary_prepares(&bus, namespace, HEAD).await { + backup.on_replicate(prepare).await; + } + assert_eq!(backup.held_through(), HEAD); + owner.plane.partitions().insert(namespace, backup); + + assert!(owner.tick_partitions(&mut Vec::new()).await.is_none()); + { + let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap(); + assert!( + partition.transfer_rearm.is_none(), + "a journal that holds every op through commit_max must stand the re-arm down" + ); + partition.consensus().advance_commit_max(HEAD + 1); + } + let debounce = REPAIR_GAP_DEBOUNCE_TICKS_MIN as usize; + let observed = stall_rounds(&owner, namespace, debounce).await; + + let mut expected = vec![(None, None); debounce - 1]; + expected.push((Some(REPAIR_PRIMARY), None)); + assert_eq!( + observed, expected, + "the new gap must open a repair session, never a transfer" + ); + } + + /// A backup lagging from op 1 to the last of `prepares` whose first repair + /// met a peer that evicted every op up to `floor`. The backup journaled the + /// prepares served above the floor and armed a transfer. + #[allow(clippy::future_not_send)] + async fn floor_converted_backup( + bus: &Rc, + namespace: IggyNamespace, + floor: u64, + prepares: &[Message], + ) -> (IoTestShard, TaggedSender) { + for peer in [REPAIR_NEXT_PEER, REPAIR_PRIMARY] { + connect_replica_on(bus, peer, 1); + } + let (mut owner, sender) = test_owner(bus, None); + owner.partition_consensus.replica_count = 3; + owner.set_repair_retry_ticks(1); + let commit_max = u64::try_from(prepares.len()).unwrap(); + let partitions = owner.plane.partitions(); + partitions.remove(&namespace).unwrap(); + partitions.insert(namespace, repairing_backup(bus, namespace, 0, commit_max)); + let session = partitions.get_by_ns(&namespace).unwrap().repair.unwrap(); + serve_above_floor( + &owner, + namespace, + (session.peer, session.nonce), + floor, + prepares, + ) + .await; + assert!( + partitions.get_by_ns(&namespace).unwrap().transfer.is_some(), + "the evicted floor arms a transfer" + ); + (owner, sender) + } + + /// The answer of the transfer peer that it cannot serve the session + /// `(peer, nonce)` yet. + fn transient_transfer_refusal( + namespace: IggyNamespace, + (peer, nonce): (u8, u128), + ) -> MessageBag { + MessageBag::StateTransferTarget( + Message::::new(size_of::()) + .transmute_header(|_, header: &mut StateTransferTargetHeader| { + header.command = Command::StateTransferTarget; + header.size = u32::try_from(size_of::()).unwrap(); + header.cluster = 1; + header.group = namespace.inner(); + header.replica = peer; + header.nonce = nonce; + header.unavailable_transient = 1; + header.seal(); + }), + ) + } + + /// Prepares a peer serves above its eviction floor in one repair reply. + const FRAMES_ABOVE_FLOOR: u64 = 4; + + /// Answer the repair request `(peer, nonce)` as a peer that evicted every + /// op up to `floor`: `RangeEvicted`, the next [`FRAMES_ABOVE_FLOOR`] + /// prepares, then `RepairDone`. + #[allow(clippy::future_not_send)] + async fn serve_above_floor( + owner: &IoTestShard, + namespace: IggyNamespace, + (peer, nonce): (u8, u128), + floor: u64, + prepares: &[Message], + ) { + let last = floor + FRAMES_ABOVE_FLOOR; + let evicted = repair_done(namespace, peer, nonce, floor + 1).transmute_header( + |done, evicted: &mut RepairRangeReplyHeader| { + *evicted = done; + evicted.command = Command::RangeEvicted; + evicted.seal(); + }, + ); + owner.on_repair_range_reply(&evicted).await; + let served = usize::try_from(floor).unwrap()..usize::try_from(last).unwrap(); + for prepare in &prepares[served] { + owner + .on_repair_prepare(as_repair_prepare(prepare.clone())) + .await; + } + owner + .on_repair_range_reply(&repair_done(namespace, peer, nonce, last)) + .await; + } + + /// A group whose served transfer finished asks again at once. The slot + /// goes to the group that waited while it was served. + #[compio::test] + async fn given_fully_served_group_when_it_asks_again_should_leave_its_slot_to_the_waiting_group() + { + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = one_transfer_slot_owner(&bus); + let [served, waiting] = + [0, 1].map(|partition_id| IggyNamespace::new(0, 0, partition_id).inner()); + serve_partition_offer(&owner, served, false); + assert!(!owner.admit_partition_transfer(waiting)); + + serve_partition_offer(&owner, served, true); + + assert_eq!( + ( + owner.admit_partition_transfer(served), + owner.admit_partition_transfer(waiting) + ), + (false, true), + "(finished group admitted, waiting group admitted)" ); } + /// Refused groups take a freed slot in the order of their first refusal, + /// whichever of them asks first. #[compio::test] - async fn given_staggered_sessions_at_ceiling_when_rotating_should_preserve_each_slot() { - const WAITING_GROUPS: usize = 4; + async fn given_two_waiting_groups_when_a_slot_frees_should_admit_the_oldest_waiter() { let bus = Rc::new(IggyMessageBus::new(0)); - let (owner, _sender) = test_owner(&bus, None); - owner.set_repair_retry_ticks(1); - owner.set_repair_gap_debounce_ticks(REPAIR_GAP_DEBOUNCE_TICKS_MIN); - let partitions = owner.plane.partitions(); - let budget = REPAIR_MAX_STALL_RETRIES as usize + 1; - let namespaces: Vec<_> = (0..PARTITION_REPAIRS_INFLIGHT_MAX + WAITING_GROUPS) - .map(|partition_id| IggyNamespace::new(0, 0, partition_id)) - .collect(); - for (index, &namespace) in namespaces.iter().enumerate() { - let mut backup = - repairing_backup(&bus, namespace, LAGGING_COMMIT_MIN, LAGGING_COMMIT_MAX); - if index < PARTITION_REPAIRS_INFLIGHT_MAX { - for _ in 0..index % budget { - assert!(!backup.burn_repair_attempt()); - } - } else { - backup.repair = None; - backup.gap_ticks.set(REPAIR_GAP_DEBOUNCE_TICKS_MIN); - } - partitions.insert(namespace, backup); - } - owner - .partition_repairs_inflight - .set(PARTITION_REPAIRS_INFLIGHT_MAX); - for sweep in 1..=budget { - assert!(owner.tick_partitions(&mut Vec::new()).await.is_none()); - let live = namespaces - .iter() - .filter(|namespace| partitions.get_by_ns(namespace).unwrap().repair.is_some()) - .count(); - assert_eq!(live, PARTITION_REPAIRS_INFLIGHT_MAX, "sweep {sweep}"); - assert_eq!( - owner.partition_repairs_inflight.get(), - live, - "sweep {sweep}" - ); - for (index, namespace) in namespaces.iter().enumerate() { - let expected = if index >= PARTITION_REPAIRS_INFLIGHT_MAX { - None - } else if sweep >= budget - index % budget { - Some(REPAIR_NEXT_PEER) - } else { - Some(REPAIR_PRIMARY) - }; - let peer = partitions - .get_by_ns(namespace) - .unwrap() - .repair - .map(|session| session.peer); - assert_eq!(peer, expected, "sweep {sweep}, group {index}"); - } - } + let (owner, _sender) = one_transfer_slot_owner(&bus); + let [served, oldest, newest] = + [0, 1, 2].map(|partition_id| IggyNamespace::new(0, 0, partition_id).inner()); + serve_partition_offer(&owner, served, false); + assert!(!owner.admit_partition_transfer(oldest)); + assert!(!owner.admit_partition_transfer(newest)); + + owner.state_transfer_offers.borrow_mut().clear(); + + assert_eq!( + ( + owner.admit_partition_transfer(newest), + owner.admit_partition_transfer(oldest) + ), + (false, true), + "(newest waiter admitted, oldest waiter admitted)" + ); } + /// A waiter that stops asking ages out on the clock of an abandoned build, + /// so the free slot it held reaches the waiter behind it. #[compio::test] - async fn given_transfer_abandoned_at_ceiling_when_repair_rearms_should_count_new_session() { - const OTHER_SESSIONS: usize = PARTITION_REPAIRS_INFLIGHT_MAX - 1; - const SWEEPS: usize = 2; - for retry_ticks in [1, partitions::REPAIR_RETRY_TICKS] { - let bus = Rc::new(IggyMessageBus::new(0)); - let (owner, _sender) = test_owner(&bus, None); - owner.set_repair_retry_ticks(retry_ticks); - owner.set_repair_gap_debounce_ticks(REPAIR_GAP_DEBOUNCE_TICKS_MIN); - let partitions = owner.plane.partitions(); - let namespaces: Vec<_> = (0..=PARTITION_REPAIRS_INFLIGHT_MAX) - .map(|partition_id| IggyNamespace::new(0, 0, partition_id)) - .collect(); - for &namespace in &namespaces[..OTHER_SESSIONS] { - partitions.insert( - namespace, - repairing_backup(&bus, namespace, LAGGING_COMMIT_MIN, LAGGING_COMMIT_MAX), - ); - } - let transferring = namespaces[OTHER_SESSIONS]; - let mut backup = - repairing_backup(&bus, transferring, LAGGING_COMMIT_MIN, LAGGING_COMMIT_MAX); - backup.consensus().begin_state_transfer_await(); + async fn given_waiter_that_stops_asking_when_its_idle_window_passes_should_free_its_turn() { + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = one_transfer_slot_owner(&bus); + owner.set_repair_retry_ticks(1); + let [served, silent, asking] = + [0, 1, 2].map(|partition_id| IggyNamespace::new(0, 0, partition_id).inner()); + serve_partition_offer(&owner, served, false); + assert!(!owner.admit_partition_transfer(silent)); + assert!(!owner.admit_partition_transfer(asking)); + owner.state_transfer_offers.borrow_mut().clear(); + + for sweep in 0..crate::STATE_TRANSFER_OFFER_EXPIRY_MULTIPLE { assert!( - owner - .arm_partition_transfer(&mut backup, REPAIR_PRIMARY, 0) - .await + !owner.admit_partition_transfer(asking), + "the silent waiter keeps its turn through sweep {sweep}" ); - for _ in 0..consensus::STATE_TRANSFER_MAX_STALL_RETRIES { - assert!(!backup.burn_transfer_attempt()); - } - backup.transfer.as_mut().unwrap().idle_ticks = retry_ticks - 1; - partitions.insert(transferring, backup); - let waiting = namespaces[PARTITION_REPAIRS_INFLIGHT_MAX]; - let mut late = repairing_backup(&bus, waiting, LAGGING_COMMIT_MIN, LAGGING_COMMIT_MAX); - late.repair = None; - late.gap_ticks.set(REPAIR_GAP_DEBOUNCE_TICKS_MIN); - partitions.insert(waiting, late); - owner.partition_repairs_inflight.set(OTHER_SESSIONS); - - for sweep in 1..=SWEEPS { - assert!(owner.tick_partitions(&mut Vec::new()).await.is_none()); - let live = namespaces - .iter() - .filter(|namespace| partitions.get_by_ns(namespace).unwrap().repair.is_some()) - .count(); - assert_eq!( - live, PARTITION_REPAIRS_INFLIGHT_MAX, - "interval {retry_ticks}, sweep {sweep}" - ); - assert_eq!( - owner.partition_repairs_inflight.get(), - live, - "interval {retry_ticks}, sweep {sweep}" - ); - assert!( - partitions - .get_by_ns(&transferring) - .unwrap() - .repair - .is_some() - ); - assert!(partitions.get_by_ns(&waiting).unwrap().repair.is_none()); - } + owner.expire_idle_state_transfer_offers(); } + + assert!(owner.admit_partition_transfer(asking)); } + /// A replica that is not the primary of a group refuses a transfer request + /// for it without queueing the group, so a freed slot goes to the next + /// group that asks. #[compio::test] - async fn given_partly_spent_repair_when_transfer_supersedes_it_should_restart_fallback_budget() - { - const REARM_BACKOFF_TICKS: u32 = 2 * (REPAIR_MAX_STALL_RETRIES + 1); + async fn given_backup_of_a_group_when_its_transfer_is_requested_should_refuse_without_queueing_it() + { + const REQUESTER: u8 = 0; + const NONCE: u128 = 1; let bus = Rc::new(IggyMessageBus::new(0)); - let (owner, _sender) = test_owner(&bus, None); - owner.set_repair_retry_ticks(1); - let namespace = IggyNamespace::new(0, 0, 0); - let mut backup = repairing_backup(&bus, namespace, LAGGING_COMMIT_MIN, LAGGING_COMMIT_MAX); - for _ in 0..REPAIR_MAX_STALL_RETRIES { - assert!(!backup.burn_repair_attempt()); - } - backup.note_repair_rotation(); - backup.repair.as_mut().unwrap().peer = REPAIR_NEXT_PEER; - backup.consensus().begin_state_transfer_await(); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + captured.borrow_mut().push(frame); + Ok(()) + })); + connect_replica_on(&bus, REQUESTER, 1); + let (mut owner, _sender) = one_transfer_slot_owner(&bus); + owner.partition_consensus.replica_count = 3; + let [served, backed_up, next] = + [0, 1, 2].map(|partition_id| IggyNamespace::new(0, 0, partition_id)); + let mut backup = repairing_backup(&bus, backed_up, 0, 0); + backup.repair = None; + owner.plane.partitions().insert(backed_up, backup); + serve_partition_offer(&owner, served.inner(), false); + let request = + Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestStateTransferHeader| { + header.command = Command::RequestStateTransfer; + header.size = u32::try_from(size_of::()).unwrap(); + header.cluster = 1; + header.group = backed_up.inner(); + header.replica = REQUESTER; + header.nonce = NONCE; + header.seal(); + }); - assert!( - owner - .arm_partition_transfer(&mut backup, REPAIR_PRIMARY, 0) - .await - ); - assert!(backup.repair.is_none()); - assert_eq!(backup.repair_rotations(), 0); - owner - .abandon_or_rearm_partition_transfer(&mut backup, REPAIR_PRIMARY) - .await; - backup.transfer_rearm.as_mut().unwrap().after_ticks = REARM_BACKOFF_TICKS; - owner.plane.partitions().insert(namespace, backup); - let budget = REPAIR_MAX_STALL_RETRIES as usize; - let observed = stall_rounds(&owner, namespace, budget + 1).await; - let mut expected = vec![(Some(REPAIR_PRIMARY), None); budget]; - expected.push((Some(REPAIR_NEXT_PEER), None)); + owner.on_partition_request_state_transfer(&request).await; + owner.state_transfer_offers.borrow_mut().clear(); + + let refusals: Vec<_> = sent + .borrow() + .iter() + .map(|frame| { + let reply = Message::::try_from(Owned::copy_from_slice( + frame.as_slice(), + )) + .unwrap(); + ( + reply.header().available, + reply.header().unavailable_transient, + ) + }) + .collect(); + let queued = owner.partition_transfer_waiters.borrow().len(); assert_eq!( - observed, expected, - "fallback repair must get the whole stall budget" + ( + refusals, + queued, + owner.admit_partition_transfer(next.inner()) + ), + (vec![(0, 1)], 0, true), + "((available, transient) per reply, queued groups, next group admitted)" + ); + } + + /// A test owner whose served-payload budget holds one transfer slot. + fn one_transfer_slot_owner(bus: &Rc) -> (IoTestShard, TaggedSender) { + let (owner, sender) = test_owner(bus, None); + owner.set_served_segment_cache_bytes_max(crate::PARTITION_ARTIFACT_LEN_DEFAULT); + assert_eq!(owner.partition_transfer_admission_cap(), 1); + (owner, sender) + } + + /// Record an offer of the partition `namespace` served to [`REPAIR_BACKUP`]. + fn serve_partition_offer(owner: &IoTestShard, namespace: u64, fully_served: bool) { + let offer = partitions::state_transfer::PartitionStateTransferOffer { + commit_op: 0, + segments: Vec::new(), + offsets: ( + StateArtifact::for_bytes(artifact_kind::CONSUMER_OFFSETS, 0, &[]), + Rc::new(Vec::new()), + ), + }; + owner.state_transfer_offers.borrow_mut().insert( + (namespace, REPAIR_BACKUP), + crate::ServedStateTransfer { + nonce: 0, + offer: crate::ServedOffer::Partition(Rc::new(offer)), + idle_ticks: 0, + fully_served, + }, ); } @@ -5268,7 +6780,7 @@ mod tests { let expected_from = if partition.consensus().view_log_entry_proven(missing) { missing } else { - missing.saturating_sub(CHUNK - 1).max(APPLIED + 1) + missing.saturating_sub(CHUNK - 1).max(LOCAL_HEAD + 1) }; assert_eq!(request.from_op, expected_from); assert_eq!(request.to_op, missing); @@ -5335,7 +6847,7 @@ mod tests { owner.on_repair_range_reply(&reply).await; } assert_eq!(rounds, (NEW_HEAD - LOCAL_HEAD).div_ceil(CHUNK) + 1); - assert_eq!(bodies, NEW_HEAD - APPLIED); + assert_eq!(bodies, NEW_HEAD - LOCAL_HEAD); assert!(requests.borrow().is_empty()); let partition = owner.plane.partitions().get_by_ns(&namespace).unwrap(); assert!(!partition.consensus().view_log_unproven()); @@ -5462,11 +6974,12 @@ mod tests { } } - /// The candidate starts [`HEADLESS_NEXT_VIEW`] while `backup` holds the proven + /// Both live replicas vote to leave [`HEADLESS_VIEW`]. The vote of the + /// candidate starts [`HEADLESS_NEXT_VIEW`] at `backup`, which holds the proven /// log of [`HEADLESS_VIEW`] and has run nothing since its proof settled. The - /// backup votes only after the held write of `store` persists the new view, - /// and the candidate wins the view with that vote. - #[allow(clippy::future_not_send)] + /// backup sends its view change messages only after the held write of `store` + /// persists the new view, and the candidate wins the view with them. + #[allow(clippy::future_not_send, clippy::too_many_lines)] async fn elect_beside_proven_backup( namespace: IggyNamespace, backup: &HeadlessReplica, @@ -5480,14 +6993,19 @@ mod tests { .into_iter() .all(|command| backup.sent_to(HEADLESS_CANDIDATE, command, HEADLESS_NEXT_VIEW)) }; + let candidate_voted = || { + candidate.sent_to( + HEADLESS_BACKUP, + Command::StartViewChange, + HEADLESS_NEXT_VIEW, + ) + }; assert!(backup_consensus().is_normal()); assert!(!backup_consensus().view_log_unproven()); - // Only the candidate ticks: its heartbeat timeout starts the next view - // while the backup runs nothing after its proof settled. let mut scratch = Vec::new(); for _ in 0..2 * TimeoutManager::NORMAL_HEARTBEAT_TICKS { - if candidate_consensus().view() == HEADLESS_NEXT_VIEW { + if candidate_voted() { break; } assert!( @@ -5498,7 +7016,25 @@ mod tests { .is_none() ); } - assert_eq!(candidate_consensus().view(), HEADLESS_NEXT_VIEW); + assert!( + candidate_voted(), + "the candidate's heartbeat timeout votes to leave the view" + ); + // Only the consensus of the backup ticks, so no persist sweep runs after its + // proof settled. Its own vote stays unsent, and the vote of the candidate + // completes its quorum. + let backup_heartbeat_voted = (0..TimeoutManager::NORMAL_HEARTBEAT_TICKS) + .flat_map(|_| backup_consensus().tick(consensus::PlaneKind::Partitions)) + .any(|action| { + matches!( + action, + VsrAction::SendStartViewChange { view, .. } if view == HEADLESS_NEXT_VIEW + ) + }); + assert!( + backup_heartbeat_voted, + "the backup's heartbeat timeout votes to leave the view" + ); let (entered, mut started) = oneshot::channel(); let (release, held) = oneshot::channel(); *store.entered.borrow_mut() = Some(entered); @@ -5723,7 +7259,7 @@ mod tests { /// backup. It truncates the op that the merged log replaces, which clears the /// WAL certificate of the view it replicated in, and proves the merge through /// a repair. Returns the backup and the elect, whose view has not started. - #[allow(clippy::future_not_send)] + #[allow(clippy::future_not_send, clippy::too_many_lines)] async fn elect_with_settled_merge( namespace: IggyNamespace, directory: &Path, @@ -5777,14 +7313,20 @@ mod tests { assert!(!backup_consensus().view_log_unproven()); assert_eq!(elect_partition().log.journal().inner.last_op(), Some(HEAD)); + // The heartbeat timeout of the backup votes to leave the view. The elect + // never left `REPAIR_VIEW`, so that vote is more than one view ahead of it, + // and the elect joins it. let mut scratch = Vec::new(); for _ in 0..2 * TimeoutManager::NORMAL_HEARTBEAT_TICKS { - if backup_consensus().view() == HEADLESS_NEXT_VIEW { + if backup.sent_to( + HEADLESS_CANDIDATE, + Command::StartViewChange, + HEADLESS_NEXT_VIEW, + ) { break; } assert!(backup.owner.tick_partitions(&mut scratch).await.is_none()); } - assert_eq!(backup_consensus().view(), HEADLESS_NEXT_VIEW); elect .owner .on_message(backup.take(HEADLESS_CANDIDATE, Command::StartViewChange)) @@ -5805,6 +7347,7 @@ mod tests { .owner .on_message(elect.take(HEADLESS_BACKUP, Command::StartViewChange)) .await; + assert_eq!(backup_consensus().view(), HEADLESS_NEXT_VIEW); elect .owner .on_message(backup.take(HEADLESS_CANDIDATE, Command::DoViewChange)) @@ -5916,16 +7459,20 @@ mod tests { crate::refresh_partition_dvc_suffix(elect.partition(namespace)); let settled = elect_consensus().local_dvc_suffix(); - // Only the backup ticks: its status timeout supersedes the elect, which runs - // nothing after its proof settled. + // Only the backup ticks: its status timeout votes to supersede the elect, + // which runs nothing after its proof settled. The elect joins that vote + // from its view change. let mut scratch = Vec::new(); for _ in 0..2 * TimeoutManager::VIEW_CHANGE_STATUS_TICKS { - if backup_consensus().view() == SUPERSEDING_VIEW { + if backup.sent_to( + HEADLESS_CANDIDATE, + Command::StartViewChange, + SUPERSEDING_VIEW, + ) { break; } assert!(backup.owner.tick_partitions(&mut scratch).await.is_none()); } - assert_eq!(backup_consensus().view(), SUPERSEDING_VIEW); // `take` returns the first match, so drop the frames of earlier views, the // backup's retransmits among them. for replica in [&backup, &elect] { @@ -5991,6 +7538,7 @@ mod tests { .owner .on_message(elect.take(SUPERSEDING_PRIMARY, Command::StartViewChange)) .await; + assert_eq!(backup_consensus().view(), SUPERSEDING_VIEW); backup .owner .on_message(MessageBag::DoViewChange(vote)) @@ -6080,14 +7628,12 @@ mod tests { compio::time::timeout(REPLY_DEADLINE, Box::pin(certified)) .await .expect("the elect's WAL must certify the view it adopted"); - for _ in 0..2 * TimeoutManager::NORMAL_HEARTBEAT_TICKS { - if elect_consensus().view() == HEADLESS_NEXT_VIEW { - break; - } + // The third replica goes silent, so the heartbeat timeout of the elect votes + // to leave the view. + for _ in 0..TimeoutManager::NORMAL_HEARTBEAT_TICKS { assert!(elect.owner.tick_partitions(&mut scratch).await.is_none()); } - assert_eq!(elect_consensus().view(), HEADLESS_NEXT_VIEW); - let announced = async { + let voted = async { while !elect.sent_to( HEADLESS_BACKUP, Command::StartViewChange, @@ -6096,9 +7642,11 @@ mod tests { elect.step(&mut scratch).await; } }; - compio::time::timeout(REPLY_DEADLINE, Box::pin(announced)) + compio::time::timeout(REPLY_DEADLINE, Box::pin(voted)) .await - .expect("the elect must announce the view change"); + .expect("the elect must vote to leave the view"); + // The backup never left `REPAIR_VIEW`, so that vote is more than one view + // ahead of it, and the backup joins it. backup .owner .on_message(elect.take(HEADLESS_BACKUP, Command::StartViewChange)) @@ -6107,8 +7655,21 @@ mod tests { .owner .on_message(backup.take(HEADLESS_CANDIDATE, Command::StartViewChange)) .await; - // The backup's own vote never arrives, so the quorum is the elect's and the - // third replica's, which offers a body for every op. + assert_eq!(elect_consensus().view(), HEADLESS_NEXT_VIEW); + let announced = async { + while !elect.sent_to( + HEADLESS_BACKUP, + Command::StartViewChange, + HEADLESS_NEXT_VIEW, + ) { + elect.step(&mut scratch).await; + } + }; + compio::time::timeout(REPLY_DEADLINE, Box::pin(announced)) + .await + .expect("the elect must announce the view change"); + // The backup's own DoViewChange never arrives, so the quorum is the elect's + // and the third replica's, which offers a body for every op. let third_replica = DvcSuffix::new( [&lagged, &prepares[1], &prepares[0]] .map(|prepare| *prepare.header()) @@ -6149,15 +7710,19 @@ mod tests { assert!(elect_consensus().is_primary_elect()); assert!(elect_partition().wal_keeps_log_view()); - // Only the backup ticks: its status timeout supersedes the elect, which runs - // nothing after its proof settled. + // Only the backup ticks: its status timeout votes to supersede the elect, + // which runs nothing after its proof settled. The elect joins that vote + // from its view change. for _ in 0..2 * TimeoutManager::VIEW_CHANGE_STATUS_TICKS { - if backup_consensus().view() == SUPERSEDING_VIEW { + if backup.sent_to( + HEADLESS_CANDIDATE, + Command::StartViewChange, + SUPERSEDING_VIEW, + ) { break; } assert!(backup.owner.tick_partitions(&mut scratch).await.is_none()); } - assert_eq!(backup_consensus().view(), SUPERSEDING_VIEW); for replica in [&backup, &elect] { replica .sent @@ -6191,6 +7756,7 @@ mod tests { .owner .on_message(elect.take(SUPERSEDING_PRIMARY, Command::StartViewChange)) .await; + assert_eq!(backup_consensus().view(), SUPERSEDING_VIEW); backup .owner .on_message(MessageBag::DoViewChange(vote)) @@ -7141,61 +8707,630 @@ mod tests { } Ok(()) })); - for requester in [BACKUP, NEXT_PRIMARY] { - crate::test_support::connect_replica_on(&bus, requester, 1); - } - let (owner, _sender) = test_owner(&bus, None); - let namespace = IggyNamespace::new(0, 0, 0); - let prepares = primary_prepares(&bus, namespace, HEAD).await; - let mut donor = repairing_replica(&bus, namespace, (DONOR, REPLICAS), REPAIR_PRIMARY, 0, 0); - donor.repair = None; - for prepare in prepares { - donor.on_replicate(prepare).await; + for requester in [BACKUP, NEXT_PRIMARY] { + crate::test_support::connect_replica_on(&bus, requester, 1); + } + let (owner, _sender) = test_owner(&bus, None); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = primary_prepares(&bus, namespace, HEAD).await; + let mut donor = repairing_replica(&bus, namespace, (DONOR, REPLICAS), REPAIR_PRIMARY, 0, 0); + donor.repair = None; + for prepare in prepares { + donor.on_replicate(prepare).await; + } + donor.consensus().advance_commit_max(COMMITTED); + donor + .commit_journal(owner.plane.partitions().config()) + .await; + for voter in [REPAIR_PRIMARY, BACKUP] { + let change = + view_control_message(namespace, Command::StartViewChange, NEXT_VIEW, voter); + let MessageBag::StartViewChange(change) = change else { + unreachable!() + }; + donor + .consensus() + .handle_start_view_change(consensus::PlaneKind::Partitions, change.header()); + } + assert_eq!(donor.consensus().status(), consensus::Status::ViewChange); + owner.plane.partitions().remove(&namespace).unwrap(); + owner.plane.partitions().insert(namespace, donor); + + for (requester, view, through) in [ + (BACKUP, NEXT_VIEW, COMMITTED), + (NEXT_PRIMARY, NEXT_VIEW, HEAD), + (NEXT_PRIMARY, REPAIR_VIEW, COMMITTED), + ] { + let request = Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestPreparesHeader| { + header.command = Command::RequestPrepares; + header.cluster = 1; + header.group = namespace.inner(); + header.replica = requester; + header.view = view; + header.from_op = 1; + header.to_op = HEAD; + header.size = u32::try_from(size_of::()).unwrap(); + header.seal(); + }); + sent.borrow_mut().clear(); + owner.on_request_prepares(&request).await; + let served: Vec<_> = sent.borrow().iter().map(|header| header.op).collect(); + assert_eq!( + served, + (1..=through).collect::>(), + "requester {requester} in view {view}" + ); + } + } + + /// A donor whose evicted ring released the bodies of ops 1..=3 to their + /// segment serves them from the segment, byte for byte. It never serves a + /// corrupt or a torn body: the round stops there, and from then on the + /// ops up to it answer `RangeEvicted`. + #[compio::test] + async fn given_bodies_released_to_a_segment_when_a_backup_requests_them_should_serve_only_intact_ones() + { + let donor = Box::pin(SegmentDonor::new(4)).await; + let SegmentDonor { + prepares, + log_path, + positions, + .. + } = &donor; + + assert_eq!( + donor.answers(1).await, + [ + (Command::RepairPrepare, 1), + (Command::RepairPrepare, 2), + (Command::RepairPrepare, 3), + (Command::RepairPrepare, 4), + (Command::RepairDone, 4), + ] + ); + + // A flipped byte at the end of the body of op 2, and op 3 cut in half. + let mut segment = std::fs::read(log_path).unwrap(); + let op_3_start = usize::try_from(positions[&3]).unwrap(); + segment[op_3_start - 1] ^= 0xFF; + segment + .truncate(op_3_start + (prepares[2].as_slice().len() - size_of::()) / 2); + std::fs::write(log_path, &segment).unwrap(); + assert_eq!( + donor.answers(1).await, + [(Command::RepairPrepare, 1), (Command::RepairDone, 1)], + "the corrupt body of op 2 is not served" + ); + assert_eq!( + donor.answers(2).await, + [(Command::RangeEvicted, 3), (Command::RepairDone, 2)], + "the torn body of op 3 is not served" + ); + assert_eq!( + donor.answers(3).await, + [ + (Command::RangeEvicted, 4), + (Command::RepairPrepare, 4), + (Command::RepairDone, 4), + ] + ); + } + + /// A log file that cannot be opened for now fails the read with an error + /// that can pass, like a file descriptor limit. A link that points at + /// itself stands in for it, because a lower limit would reach the other + /// tests in the process. The round answers without the released ops and + /// keeps them, so a later round serves them. + #[compio::test] + async fn given_a_log_file_that_cannot_be_opened_when_a_backup_requests_released_bodies_should_serve_them_once_it_opens() + { + let donor = Box::pin(SegmentDonor::new(3)).await; + let segment = std::fs::read(&donor.log_path).unwrap(); + std::fs::remove_file(&donor.log_path).unwrap(); + std::os::unix::fs::symlink(&donor.log_path, &donor.log_path).unwrap(); + assert_eq!(donor.answers(1).await, [(Command::RepairDone, 0)]); + + std::fs::remove_file(&donor.log_path).unwrap(); + std::fs::write(&donor.log_path, &segment).unwrap(); + assert_eq!( + donor.answers(1).await, + [ + (Command::RepairPrepare, 1), + (Command::RepairPrepare, 2), + (Command::RepairPrepare, 3), + (Command::RepairDone, 3), + ] + ); + } + + /// A whole batch that passes its own checks, but sits where the ring + /// expects another body, is never served: here the body of op 3 written + /// over the body of op 2, which has the same length. + #[compio::test] + async fn given_another_batch_at_a_released_body_position_when_a_backup_requests_it_should_not_serve_it() + { + let donor = Box::pin(SegmentDonor::new(4)).await; + let body = |op: usize| &donor.prepares[op - 1].as_slice()[size_of::()..]; + assert_eq!(body(2).len(), body(3).len()); + let mut segment = std::fs::read(&donor.log_path).unwrap(); + let op_2_start = usize::try_from(donor.positions[&2]).unwrap(); + segment[op_2_start..op_2_start + body(3).len()].copy_from_slice(body(3)); + std::fs::write(&donor.log_path, &segment).unwrap(); + + assert_eq!( + donor.answers(1).await, + [(Command::RepairPrepare, 1), (Command::RepairDone, 1)], + "the batch of op 3 is not served as op 2" + ); + assert_eq!( + donor.answers(2).await, + [ + (Command::RangeEvicted, 3), + (Command::RepairPrepare, 3), + (Command::RepairPrepare, 4), + (Command::RepairDone, 4), + ] + ); + } + + /// Released bodies that lie back to back in a segment file come back in + /// one read. A run ends at a gap in the file and at the end of its + /// segment, also where the next body starts at the same position in the + /// next file. Every body reaches the backup byte for byte. + #[compio::test] + async fn given_released_bodies_in_runs_across_two_segments_when_a_backup_requests_them_should_serve_each_one() + { + const HEAD: u64 = 6; + const GAP: usize = 64; + const SECOND_SEGMENT_START: u64 = 1000; + let bus = Rc::new(IggyMessageBus::new(0)); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + captured.borrow_mut().push( + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(), + ); + Ok(()) + })); + connect_replica_on(&bus, REPAIR_PRIMARY, 1); + let (owner, _sender) = test_owner(&bus, None); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = primary_prepares(&bus, namespace, HEAD).await; + let mut donor = + repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); + donor.repair = None; + let directory = tempfile::tempdir().unwrap(); + let (first_path, _) = attach_segment_files(&mut donor, directory.path(), false).await; + for prepare in &prepares { + donor.on_replicate(prepare.clone()).await; + } + let second_path = + attach_next_segment_file(&mut donor, directory.path(), SECOND_SEGMENT_START).await; + + // Ops 1 and 2 back to back, a gap, then op 3 in the first file. Ops 4 + // and 5 back to back in the second file, op 4 where op 3 ends. + let body = |op: u64| { + &prepares[usize::try_from(op - 1).unwrap()].as_slice()[size_of::()..] + }; + let mut first = Vec::new(); + let mut second = Vec::new(); + let mut positions = BTreeMap::new(); + for op in 1..=2 { + positions.insert(op, (0, first.len())); + first.extend_from_slice(body(op)); + } + first.resize(first.len() + GAP, u8::MAX); + positions.insert(3, (0, first.len())); + first.extend_from_slice(body(3)); + second.resize(first.len(), 0); + for op in 4..=5 { + positions.insert(op, (SECOND_SEGMENT_START, second.len())); + second.extend_from_slice(body(op)); + } + std::fs::write(&first_path, &first).unwrap(); + std::fs::write(&second_path, &second).unwrap(); + release_bodies(&donor, &prepares, |op| { + positions + .get(&op) + .map(|&(segment_start, position)| partitions::BodyPosition { + segment_start, + position: u64::try_from(position).unwrap(), + }) + }); + owner.plane.partitions().remove(&namespace).unwrap(); + owner.plane.partitions().insert(namespace, donor); + + assert_eq!( + repair_answers(&owner, &sent, namespace, 1, &prepares).await, + (1..=HEAD) + .map(|op| (Command::RepairPrepare, op)) + .chain([(Command::RepairDone, HEAD)]) + .collect::>() + ); + } + + /// Asks `owner` for ops `from_op..` of `prepares` as the primary of + /// [`REPAIR_VIEW`], and returns the command and op of each frame it sent. + /// Each `RepairPrepare` must carry its prepare from `prepares` verbatim. + #[allow(clippy::future_not_send)] + async fn repair_answers( + owner: &IoTestShard, + sent: &RefCell>>, + namespace: IggyNamespace, + from_op: u64, + prepares: &[Message], + ) -> Vec<(Command, u64)> { + const COMMAND_OFFSET: usize = std::mem::offset_of!(GenericHeader, command); + let request = Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestPreparesHeader| { + header.command = Command::RequestPrepares; + header.cluster = 1; + header.group = namespace.inner(); + header.replica = REPAIR_PRIMARY; + header.view = REPAIR_VIEW; + header.from_op = from_op; + header.to_op = u64::try_from(prepares.len()).unwrap(); + header.size = u32::try_from(size_of::()).unwrap(); + header.seal(); + }); + sent.borrow_mut().clear(); + owner.on_request_prepares(&request).await; + sent.borrow_mut() + .drain(..) + .map(|message| { + let command = message.header().command; + if command != Command::RepairPrepare { + let reply: Message = message.try_into_typed().unwrap(); + return (command, reply.header().op); + } + let prepare: Message = + message.try_into_typed().unwrap(); + let op = prepare.header().0.op; + let mut expected = prepares[usize::try_from(op).unwrap() - 1] + .as_slice() + .to_vec(); + expected[COMMAND_OFFSET] = Command::RepairPrepare as u8; + assert_eq!( + prepare.as_slice(), + expected, + "op {op} reaches the backup as journaled" + ); + (command, op) + }) + .collect() + } + + #[compio::test] + async fn given_large_resident_repairs_when_serving_should_bound_bytes_and_allow_one_oversized_frame() + { + const HEAD: u64 = 3; + for payload_bytes in [4 * 1024 * 1024, 12 * 1024 * 1024] { + let bus = Rc::new(IggyMessageBus::new(0)); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + captured.borrow_mut().push( + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(), + ); + Ok(()) + })); + connect_replica_on(&bus, REPAIR_PRIMARY, 1); + let (owner, _sender) = test_owner(&bus, None); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = sized_primary_prepares(&bus, namespace, HEAD, payload_bytes).await; + let mut donor = + repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); + donor.repair = None; + for prepare in &prepares { + donor.on_replicate(prepare.clone()).await; + } + owner.plane.partitions().remove(&namespace).unwrap(); + owner.plane.partitions().insert(namespace, donor); + + let mut from_op = 1; + while from_op <= HEAD { + let answers = repair_answers(&owner, &sent, namespace, from_op, &prepares).await; + let served: Vec<_> = answers + .iter() + .filter(|(command, _)| *command == Command::RepairPrepare) + .map(|(_, op)| *op) + .collect(); + assert!( + !served.is_empty(), + "a legal oversized frame must make progress" + ); + let bytes: usize = served + .iter() + .map(|op| prepares[usize::try_from(op - 1).unwrap()].total_len()) + .sum(); + assert!( + bytes <= usize::try_from(crate::REPAIR_BYTES_MAX).unwrap() || served.len() == 1, + "resident response exceeded the byte budget: {bytes} bytes, ops {served:?}" + ); + let through = *served.last().unwrap(); + assert_eq!(answers.last(), Some(&(Command::RepairDone, through))); + assert_eq!(served, (from_op..=through).collect::>()); + from_op = through + 1; + } + } + } + + /// A primary whose evicted ring released bodies of 1 MiB to their segment + /// reads at most `REPAIR_BYTES_MAX` of them per round, from the + /// bottom of the window. A backup that adopted the view walks its unproven + /// log down from the hole below the adopted head. Each window starts above + /// the bodies the backup holds below the hole, and a round that brought + /// such bodies pulls the next window at once, so the walk reaches the hole + /// and proves the log without a state transfer. + #[compio::test] + async fn given_released_bodies_past_the_disk_budget_when_a_backup_walks_down_should_reach_the_hole() + { + const HEAD: u64 = 40; + const BATCH_BYTES: usize = 1024 * 1024; + const NEW_VIEW: u32 = REPAIR_VIEW + 3; + const EXCHANGES_MAX: usize = 32; + let primary_bus = Rc::new(IggyMessageBus::new(0)); + let backup_bus = Rc::new(IggyMessageBus::new(0)); + let served = capture_replica_frames(&primary_bus); + let requested = capture_replica_frames(&backup_bus); + connect_replica_on(&primary_bus, REPAIR_BACKUP, 1); + connect_replica_on(&backup_bus, REPAIR_PRIMARY, 1); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = sized_primary_prepares(&primary_bus, namespace, HEAD, BATCH_BYTES).await; + served.borrow_mut().clear(); + + let mut consensus = VsrConsensus::new( + 1, + REPAIR_PRIMARY, + 3, + namespace.inner(), + Rc::clone(&primary_bus), + LocalPipeline::new(), + ); + consensus.set_view(NEW_VIEW); + consensus.set_log_view(NEW_VIEW); + consensus.init(); + consensus.advance_commit_max(HEAD); + let mut primary = IggyPartition::with_in_memory_storage( + Arc::new(PartitionStats::default()), + consensus, + IggyByteSize::from(SEGMENT_BYTES), + ); + let directory = tempfile::tempdir().unwrap(); + let (log_path, _) = attach_segment_files(&mut primary, directory.path(), false).await; + for prepare in &prepares { + primary + .log + .journal() + .inner + .append(prepare.clone().into_frozen()) + .await + .unwrap(); + } + release_bodies_to_segment(&primary, &log_path, &prepares); + let (primary_owner, _primary_sender) = test_owner(&primary_bus, None); + primary_owner.plane.partitions().remove(&namespace).unwrap(); + primary_owner.plane.partitions().insert(namespace, primary); + + let (backup_owner, _backup_sender) = test_owner(&backup_bus, None); + backup_owner.set_repair_retry_ticks(1); + let mut backup = repairing_replica( + &backup_bus, + namespace, + (REPAIR_BACKUP, 3), + REPAIR_PRIMARY, + 0, + 0, + ); + backup.repair = None; + let partitions = backup_owner.plane.partitions(); + partitions.remove(&namespace).unwrap(); + partitions.insert(namespace, backup); + backup_owner + .on_start_view(start_view_suffix( + namespace, + NEW_VIEW, + HEAD, + HEAD, + &[*prepares.last().unwrap().header()], + )) + .await; + let backup = partitions.get_by_ns(&namespace).unwrap(); + let hole = backup.consensus().view_log_repair_op().unwrap(); + assert!(!backup.consensus().view_log_entry_proven(hole)); + + let mut stall_ticks = 0; + for _ in 0..EXCHANGES_MAX { + if !backup.consensus().view_log_unproven() { + break; + } + let requests: Vec<_> = requested + .borrow_mut() + .drain(..) + .filter(|(_, frame)| frame.header().command == Command::RequestPrepares) + .collect(); + if requests.is_empty() { + stall_ticks += 1; + assert!( + backup_owner + .tick_partitions(&mut Vec::new()) + .await + .is_none() + ); + continue; + } + for (peer, request) in requests { + assert_eq!(peer, REPAIR_PRIMARY, "the walk left the primary"); + primary_owner + .on_request_prepares(&request.try_into_typed().unwrap()) + .await; + } + let frames: Vec<_> = served.borrow_mut().drain(..).collect(); + for (_, frame) in frames { + match frame.header().command { + Command::RepairPrepare => { + backup_owner + .on_repair_prepare(frame.try_into_typed().unwrap()) + .await; + } + command => { + assert_ne!(command, Command::RangeEvicted, "the primary holds every op"); + backup_owner + .on_repair_range_reply(&frame.try_into_typed().unwrap()) + .await; + } + } + } + } + assert!( + !backup.consensus().view_log_unproven(), + "the walk never reached its hole at op {hole}" + ); + assert_eq!( + stall_ticks, 0, + "a round that brought ops below the hole pulls the next window at once" + ); + assert!(backup.log.journal().inner.holds_op(hole)); + assert!(backup.transfer.is_none()); + assert_eq!( + backup.consensus().state_transfer_stage(), + consensus::StateTransferStage::Idle + ); + } + + /// Frames a shard sent to claimed replicas, each with its replica. + type ReplicaFrames = Rc)>>>; + + /// Every frame the shard on `bus` sends to a claimed replica, with that + /// replica. + fn capture_replica_frames(bus: &IggyMessageBus) -> ReplicaFrames { + let frames = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&frames); + bus.set_replica_forward_fn(Box::new(move |replica, _, frame| { + captured.borrow_mut().push(( + replica, + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(), + )); + Ok(()) + })); + frames + } + + /// A primary of [`REPAIR_VIEW`] that journaled ops `1..=head` and released + /// every body but the newest one to the log file of its first segment. + struct SegmentDonor { + owner: IoTestShard, + sent: Rc>>>, + namespace: IggyNamespace, + prepares: Vec>, + log_path: PathBuf, + /// The position of each released body in the log file. + positions: BTreeMap, + _sender: TaggedSender, + _directory: tempfile::TempDir, + } + + impl SegmentDonor { + #[allow(clippy::future_not_send)] + async fn new(head: u64) -> Self { + let bus = Rc::new(IggyMessageBus::new(0)); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + captured.borrow_mut().push( + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(), + ); + Ok(()) + })); + connect_replica_on(&bus, REPAIR_PRIMARY, 1); + let (owner, sender) = test_owner(&bus, None); + let namespace = IggyNamespace::new(0, 0, 0); + let prepares = primary_prepares(&bus, namespace, head).await; + let mut donor = + repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); + donor.repair = None; + let directory = tempfile::tempdir().unwrap(); + let (log_path, _) = attach_segment_files(&mut donor, directory.path(), false).await; + for prepare in &prepares { + donor.on_replicate(prepare.clone()).await; + } + let positions = release_bodies_to_segment(&donor, &log_path, &prepares); + owner.plane.partitions().remove(&namespace).unwrap(); + owner.plane.partitions().insert(namespace, donor); + Self { + owner, + sent, + namespace, + prepares, + log_path, + positions, + _sender: sender, + _directory: directory, + } } - donor.consensus().advance_commit_max(COMMITTED); - donor - .commit_journal(owner.plane.partitions().config()) - .await; - let change = - view_control_message(namespace, Command::StartViewChange, NEXT_VIEW, NEXT_PRIMARY); - let MessageBag::StartViewChange(change) = change else { - unreachable!() - }; - donor - .consensus() - .handle_start_view_change(consensus::PlaneKind::Partitions, change.header()); - assert_eq!(donor.consensus().status(), consensus::Status::ViewChange); - owner.plane.partitions().remove(&namespace).unwrap(); - owner.plane.partitions().insert(namespace, donor); - for (requester, view, through) in [ - (BACKUP, NEXT_VIEW, COMMITTED), - (NEXT_PRIMARY, NEXT_VIEW, HEAD), - (NEXT_PRIMARY, REPAIR_VIEW, COMMITTED), - ] { - let request = Message::::new(size_of::()) - .transmute_header(|_, header: &mut RequestPreparesHeader| { - header.command = Command::RequestPrepares; - header.cluster = 1; - header.group = namespace.inner(); - header.replica = requester; - header.view = view; - header.from_op = 1; - header.to_op = HEAD; - header.size = u32::try_from(size_of::()).unwrap(); - header.seal(); - }); - sent.borrow_mut().clear(); - owner.on_request_prepares(&request).await; - let served: Vec<_> = sent.borrow().iter().map(|header| header.op).collect(); - assert_eq!( - served, - (1..=through).collect::>(), - "requester {requester} in view {view}" - ); + /// See [`repair_answers`]. + #[allow(clippy::future_not_send)] + async fn answers(&self, from_op: u64) -> Vec<(Command, u64)> { + repair_answers( + &self.owner, + &self.sent, + self.namespace, + from_op, + &self.prepares, + ) + .await } } + /// Writes the bodies of `prepares`, which `donor` journaled, back to back + /// into the log file of the first segment at `log_path`, and releases + /// them there with [`release_bodies`]. Returns the position of each body + /// in the file. + fn release_bodies_to_segment( + donor: &IggyPartition, HeldSuperblock>, + log_path: &Path, + prepares: &[Message], + ) -> BTreeMap { + let mut segment = Vec::new(); + let mut positions = BTreeMap::new(); + for prepare in prepares { + positions.insert(prepare.header().op, u64::try_from(segment.len()).unwrap()); + segment.extend_from_slice(&prepare.as_slice()[size_of::()..]); + } + std::fs::write(log_path, &segment).unwrap(); + release_bodies(donor, prepares, |op| { + positions + .get(&op) + .map(|&position| partitions::BodyPosition { + segment_start: 0, + position, + }) + }); + positions + } + + /// Moves `prepares`, which `donor` journaled, into its evicted ring with + /// every body but the newest one released to the place `locate` names. + fn release_bodies( + donor: &IggyPartition, HeldSuperblock>, + prepares: &[Message], + locate: impl Fn(u64) -> Option, + ) { + let journal = &donor.log.journal().inner; + // Room for the body of the newest op only. + journal.set_ring_caps( + partitions::EVICTED_RING_CAPACITY, + u64::try_from(prepares.last().unwrap().as_slice().len()).unwrap(), + ); + journal.evict_prefix_located(prepares.len(), locate); + } + /// Seven replicas: the requester committed ops 1..=2 in view 1, adopted view 2 /// at op 2 and learned commit 3 from view 2's primary. A stale donor serves the /// op 3 view 1 prepared, which view 2 replaced. Nothing the requester holds @@ -7386,6 +9521,7 @@ mod tests { let session = partition.repair.unwrap(); assert_eq!(session.peer, PRIMARY); assert_ne!(session.nonce, OLD_NONCE); + // The branch's op 4 is held below the hole, so the window starts above it. let request = requests.borrow_mut().pop().expect("a re-request"); assert_eq!( ( @@ -7394,7 +9530,7 @@ mod tests { request.1.from_op, request.1.to_op ), - (PRIMARY, session.nonce, FORK, HEAD - 1) + (PRIMARY, session.nonce, HEAD - 1, HEAD - 1) ); owner .on_repair_range_reply(&repair_done(namespace, OLD_PEER, OLD_NONCE, HEAD - 1)) @@ -7961,6 +10097,14 @@ mod tests { fn test_owner( bus: &B, store: Option<&Rc>, + ) -> (IoTestShard, TaggedSender) { + test_owner_with_repair_memory(bus, store, None) + } + + fn test_owner_with_repair_memory( + bus: &B, + store: Option<&Rc>, + repair_memory: Option>, ) -> (IoTestShard, TaggedSender) { let shard_id = ShardId::new(0); let config = PartitionsConfig { @@ -7972,7 +10116,10 @@ mod tests { key_context: Arc::default(), path_layout: PartitionPathLayout::default(), }; - let partitions = IggyPartitions::new(shard_id, config); + let partitions = match repair_memory { + Some(budget) => IggyPartitions::with_repair_memory(shard_id, config, 0, budget), + None => IggyPartitions::new(shard_id, config), + }; let routes = PapayaShardsTable::new(); let mut inner = StreamsInner::default(); let mut stream = Stream::default(); @@ -8102,6 +10249,18 @@ mod tests { bus: &Rc, namespace: IggyNamespace, ops: u64, + ) -> Vec> { + sized_primary_prepares(bus, namespace, ops, 1).await + } + + /// [`primary_prepares`] whose one message per send carries `payload_len` + /// bytes. + #[allow(clippy::future_not_send)] + async fn sized_primary_prepares( + bus: &Rc, + namespace: IggyNamespace, + ops: u64, + payload_len: usize, ) -> Vec> { let mut consensus = VsrConsensus::new( 1, @@ -8121,7 +10280,7 @@ mod tests { ); let mut prepares = Vec::new(); for op in 1..=ops { - origin.on_request(send_request(namespace, op), None).await; + Box::pin(origin.on_request(sized_send_request(namespace, op, payload_len), None)).await; let frozen = origin.log.journal().inner.repair_entry(op).unwrap(); let mut prepare = Message::::new(frozen.len()); prepare.as_mut_slice().copy_from_slice(frozen.as_slice()); @@ -8133,6 +10292,153 @@ mod tests { prepares } + const SCANNING_INCARNATION: u64 = 7; + const SCANNING_NONCE: u128 = 7; + const SCANNING_RETRY_TICKS: u32 = 2; + const SCANNING_OFFSETS: &[u8] = b"offsets"; + + /// A receiver whose session waits for a reuse scan before it pulls a + /// segment and an offsets artifact from `REPAIR_PRIMARY`. + struct ScanningReceiver { + owner: IoTestShard, + _sender: TaggedSender, + /// `(artifact, offset, len)` of each chunk request that it sends. + requests: Rc>>, + offer_requests: Rc>>, + namespace: IggyNamespace, + segment: Vec, + directory: tempfile::TempDir, + } + + impl ScanningReceiver { + fn chunk(&self, artifact: u32, payload: &[u8]) -> Message { + let size = size_of::() + payload.len(); + let mut chunk = Message::::new(size); + chunk.as_mut_slice()[size_of::()..].copy_from_slice(payload); + chunk.transmute_header(|_, header: &mut StateChunkHeader| { + header.command = Command::StateChunk; + header.size = u32::try_from(size).unwrap(); + header.cluster = 1; + header.group = self.namespace.inner(); + header.replica = REPAIR_PRIMARY; + header.partition_incarnation = SCANNING_INCARNATION; + header.nonce = SCANNING_NONCE; + header.artifact = artifact; + header.seal(); + }) + } + } + + fn scanning_receiver() -> ScanningReceiver { + let bus = Rc::new(IggyMessageBus::new(0)); + let requests = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&requests); + let offer_requests = Rc::new(RefCell::new(Vec::new())); + let captured_offers = Rc::clone(&offer_requests); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + let message = + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(); + if message.header().command == Command::RequestStateChunk { + let request = Message::::try_from(Owned::copy_from_slice( + frame.as_slice(), + )) + .unwrap(); + let header = request.header(); + captured + .borrow_mut() + .push((header.artifact, header.offset, header.len)); + } else if message.header().command == Command::RequestStateTransfer { + let request = Message::::try_from( + Owned::copy_from_slice(frame.as_slice()), + ) + .unwrap(); + let header = request.header(); + captured_offers.borrow_mut().push(( + header.group, + header.partition_incarnation, + header.nonce, + )); + } + Ok(()) + })); + connect_replica_on(&bus, REPAIR_PRIMARY, 1); + let (owner, sender) = test_owner(&bus, None); + owner.set_repair_retry_ticks(SCANNING_RETRY_TICKS); + let namespace = IggyNamespace::new(0, 0, 0); + let directory = tempfile::tempdir().unwrap(); + let segment = send_request(namespace, 1).body().to_vec(); + let partitions = owner.plane.partitions(); + partitions.remove(&namespace).unwrap(); + let mut partition = + repairing_replica(&bus, namespace, (REPAIR_BACKUP, 3), REPAIR_PRIMARY, 0, 0); + partition.repair = None; + partition.set_created_revision(SCANNING_INCARNATION); + partition.set_partition_dir(directory.path().to_str().unwrap().to_owned()); + partition + .consensus() + .set_state_transfer_stage(StateTransferStage::AwaitingTarget); + partition + .consensus() + .set_state_transfer_stage(StateTransferStage::Fetching); + partition.transfer = Some(PartitionTransferSession { + nonce: SCANNING_NONCE, + peer: REPAIR_PRIMARY, + commit_op: 0, + artifacts: vec![ + TransferArtifact::Pending(ArtifactProgress { + entry: StateArtifact::for_bytes(artifact_kind::SEGMENT_LOG, 0, &segment), + buf: Vec::new(), + }), + TransferArtifact::Pending(ArtifactProgress { + entry: StateArtifact::for_bytes( + artifact_kind::CONSUMER_OFFSETS, + 0, + SCANNING_OFFSETS, + ), + buf: Vec::new(), + }), + ], + target_accepted: true, + idle_ticks: 0, + spill: None, + scan: Some(LocalWork::default()), + seed: None, + pulled_len: 0, + }); + partitions.insert(namespace, partition); + ScanningReceiver { + owner, + _sender: sender, + requests, + offer_requests, + namespace, + segment, + directory, + } + } + + #[allow(clippy::future_not_send)] + async fn settle_transfer_spill(owner: &IoTestShard, namespace: IggyNamespace) { + const TURNS_MAX: usize = 1024; + for _ in 0..TURNS_MAX { + let pending = owner + .plane + .partitions() + .get_by_ns(&namespace) + .and_then(|partition| partition.transfer.as_ref()) + .is_some_and(|session| session.spill.is_some()); + if !pending { + return; + } + compio::runtime::time::sleep(Duration::from_millis(1)).await; + owner + .on_partition_transfer_progress(namespace.inner()) + .await; + } + panic!("the queued spill did not settle"); + } + /// Run `rounds` stall rounds, one per sweep, and record after each the /// peer of the repair session and of the transfer session. #[allow(clippy::future_not_send)] @@ -8200,6 +10506,35 @@ mod tests { (log_path, index_path) } + /// Adds a segment that starts at `start_offset` behind the ones of + /// `partition`, backed by a fresh log file in `directory`, and returns the + /// path of that file. + #[allow(clippy::future_not_send)] + async fn attach_next_segment_file( + partition: &mut IggyPartition, HeldSuperblock>, + directory: &Path, + start_offset: u64, + ) -> PathBuf { + let log_path = directory.join(format!("{start_offset:020}.log")); + let index_path = directory.join(format!("{start_offset:020}.index")); + let storage = SegmentStorage::new( + log_path.to_str().unwrap(), + index_path.to_str().unwrap(), + 0, + 0, + false, + ) + .await + .unwrap(); + partition.log.add_persisted_segment( + Segment::new(start_offset, IggyByteSize::from(SEGMENT_BYTES)), + storage, + None, + None, + ); + log_path + } + fn submit( sender: &TaggedSender, namespace: IggyNamespace, @@ -8218,14 +10553,23 @@ mod tests { /// Each request comes from its own client, because a client has one request /// in flight per partition. fn send_request(namespace: IggyNamespace, request: u64) -> Message { + sized_send_request(namespace, request, 1) + } + + /// [`send_request`] whose message carries `payload_len` bytes. + fn sized_send_request( + namespace: IggyNamespace, + request: u64, + payload_len: usize, + ) -> Message { let mut messages = IggyMessages::with_capacity(1); messages.push(IggyMessage { header: IggyMessageHeader { id: u128::from(request), - payload_length: 1, + payload_length: u32::try_from(payload_len).unwrap(), ..Default::default() }, - payload: vec![1].into(), + payload: vec![1; payload_len].into(), user_headers: None, }); SendMessagesOwned::from_messages(namespace, &messages) @@ -8242,6 +10586,144 @@ mod tests { .unwrap() } + /// Incarnation of the groups that a [`serving_shard`] hosts. + const SERVED_INCARNATION: u64 = 7; + /// The replica that asks a [`serving_shard`] for every transfer. + const TRANSFER_REQUESTER: u8 = 1; + + /// A shard whose primaries of `namespaces` each hold one committed op in + /// segment files under `directory`. Its I/O owner pins their transfer + /// plans, so a transfer request parks until an I/O turn pins its plan. + #[allow(clippy::future_not_send)] + async fn serving_shard( + bus: &Rc, + namespaces: &[IggyNamespace], + directory: &Path, + ) -> (IoTestShard, TaggedSender) { + connect_replica_on(bus, TRANSFER_REQUESTER, 1); + let (mut owner, sender) = test_owner(bus, None); + owner.partition_consensus.replica_count = TRANSFER_REQUESTER + 1; + let partitions = owner.plane.partitions(); + for namespace in namespaces { + let directory = directory.join(namespace.inner().to_string()); + std::fs::create_dir(&directory).unwrap(); + let partition = partitions.get_mut_by_ns(namespace).unwrap(); + partition.set_created_revision(SERVED_INCARNATION); + attach_segment_files(partition, &directory, false).await; + let request = send_request(*namespace, 1).transmute_header( + |old, header: &mut RoutedRequestHeader| { + *header = old; + header.partition_incarnation = SERVED_INCARNATION; + }, + ); + let (reply, replied) = consensus::oneshot_channel(); + partitions.on_request_with_reply(request, Some(reply)).await; + owner + .process_loopback(&mut crate::router::LoopbackRound::default()) + .await; + assert_eq!(replied.await.unwrap().header().status, 0); + } + partitions.set_io_notifier( + owner.partition_io.notifier(), + owner.partition_io.limits.bytes_max(), + ); + (owner, sender) + } + + /// A `RequestStateTransfer` for `namespace` from [`TRANSFER_REQUESTER`]. + fn request_state_transfer( + namespace: IggyNamespace, + nonce: u128, + partition_incarnation: u64, + ) -> MessageBag { + MessageBag::RequestStateTransfer( + Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestStateTransferHeader| { + header.command = Command::RequestStateTransfer; + header.size = u32::try_from(size_of::()).unwrap(); + header.cluster = 1; + header.group = namespace.inner(); + header.replica = TRANSFER_REQUESTER; + header.partition_incarnation = partition_incarnation; + header.nonce = nonce; + header.seal(); + }), + ) + } + + /// `available` and `unavailable_transient` of a transfer target that + /// carries an offer. + const OFFERED: (u8, u8) = (1, 0); + /// `available` and `unavailable_transient` of a transient refusal. + const REFUSED: (u8, u8) = (0, 1); + /// Ticks that a parked request waits for its plan pin at the default + /// repair-retry interval. + const PARKED_WAIT_TICKS: u32 = + partitions::REPAIR_RETRY_TICKS / crate::PARKED_TRANSFER_REQUEST_WAIT_DIVISOR; + + /// `(group, nonce, (available, unavailable_transient))` of each + /// `StateTransferTarget` that a bus forwarded. + type RecordedTargets = Rc>>; + + /// Records each `StateTransferTarget` that `bus` forwards. + fn record_transfer_targets(bus: &IggyMessageBus) -> RecordedTargets { + let targets = Rc::new(RefCell::new(Vec::new())); + let recorded = Rc::clone(&targets); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + let message = + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(); + if message.header().command == Command::StateTransferTarget { + let target = Message::::try_from( + Owned::copy_from_slice(frame.as_slice()), + ) + .unwrap(); + let header = target.header(); + recorded.borrow_mut().push(( + header.group, + header.nonce, + (header.available, header.unavailable_transient), + )); + } + Ok(()) + })); + targets + } + + fn parked_nonce(owner: &IoTestShard, namespace: IggyNamespace) -> Option { + owner + .parked_partition_transfer_requests + .borrow() + .get(&(namespace.inner(), TRANSFER_REQUESTER)) + .map(|parked| parked.header.nonce) + } + + /// Runs I/O turns until the transfer plans of `namespaces` are pinned. A + /// pin waits for one flush, so it gets the [`REPLY_DEADLINE`] that these + /// tests give one write, also on a loaded host. + #[allow(clippy::future_not_send)] + async fn pin_transfer_plans(owner: &IoTestShard, namespaces: &[IggyNamespace]) { + let pinned = || { + namespaces.iter().all(|namespace| { + owner + .plane + .partitions() + .get_by_ns(namespace) + .is_some_and(IggyPartition::transfer_plan_pinned) + }) + }; + let pin = async { + while !pinned() { + owner.service_partition_io().await; + compio::time::sleep(TASK_POLL_INTERVAL).await; + } + }; + assert!( + compio::time::timeout(REPLY_DEADLINE, pin).await.is_ok(), + "the I/O turns must pin every transfer plan" + ); + } + fn view_control_message( namespace: IggyNamespace, command: Command, diff --git a/core/shard/src/poll/completion_tests.rs b/core/shard/src/poll/completion_tests.rs index 22c371c572..f4e4807d21 100644 --- a/core/shard/src/poll/completion_tests.rs +++ b/core/shard/src/poll/completion_tests.rs @@ -2173,6 +2173,10 @@ where artifacts: Vec::new(), target_accepted: false, idle_ticks: 0, + spill: None, + scan: None, + seed: None, + pulled_len: 0, }); } diff --git a/core/shard/src/primary_rebalance.rs b/core/shard/src/primary_rebalance.rs index f594ffee55..3f503bee2a 100644 --- a/core/shard/src/primary_rebalance.rs +++ b/core/shard/src/primary_rebalance.rs @@ -53,6 +53,14 @@ const STABLE_INTERVAL: Duration = Duration::from_secs(5); const GRANT_TIMEOUT: Duration = Duration::from_secs(1); const TRANSFER_TIMEOUT: Duration = Duration::from_secs(30); const PROBES_PER_TICK: usize = 8; +/// A moved primary that keeps its partition this long counts as held. A backup +/// that cannot follow the target loses the view inside it: one view-change +/// status timeout to give up on the view, one more to escalate past it. Sized +/// for the default 5 s timeouts. Also the first wait after a failed move. +const MOVE_HOLD_INTERVAL: Duration = Duration::from_secs(30); +/// Caps the wait after repeated failed moves, so a target that recovered gets +/// primaries back within minutes. +const TARGET_BACKOFF_MAX: Duration = Duration::from_secs(240); #[cfg(test)] mod tests; @@ -70,6 +78,40 @@ pub struct PrimaryRebalance { /// When this node's partition admission last opened under this authority. /// A probe sent before then carried no fresh quorum evidence. admission_opened: Option, + /// Moves a source carried out, watched until the target holds the + /// partition for `MOVE_HOLD_INTERVAL` or loses it to a view change. + moves: Vec, + /// Per replica. A target ready for a move can still be one that a backup + /// cannot hear in time, and only a lost view shows it. Moving to it again + /// at once would repeat the view changes forever. + backoffs: Vec, +} + +struct MovedPrimary { + plan: PartitionPrimaryTransfer, + since: Instant, +} + +#[derive(Clone, Copy, Default)] +struct TargetBackoff { + until: Option, + wait: Duration, +} + +impl TargetBackoff { + fn blocks(self, now: Instant) -> bool { + self.until.is_some_and(|until| now < until) + } + + /// Moves watched together often fail together, for example when the target + /// restarts, so a failure inside a running wait counts once. + fn record_failure(&mut self, now: Instant) { + if self.blocks(now) { + return; + } + self.wait = (self.wait * 2).clamp(MOVE_HOLD_INTERVAL, TARGET_BACKOFF_MAX); + self.until = Some(now + self.wait); + } } #[derive(Clone, Default)] @@ -230,6 +272,8 @@ impl Default for PrimaryRebalance { pending: None, received_probe: None, admission_opened: None, + moves: Vec::new(), + backoffs: Vec::new(), } } } @@ -299,6 +343,7 @@ where replicas ], eligible: vec![false; replicas], + backoffs: vec![TargetBackoff::default(); replicas], stable_since: now, next_move: now, received_probe: state.received_probe, @@ -309,10 +354,13 @@ where let admission_opened = state.admission_opened; for replica in 0..consensus.replica_count() { let index = usize::from(replica); - let ready = replica == consensus.replica() - || state.peers[index] + let ready = if replica == consensus.replica() { + !metadata.applied_frontier().has_partition_recovery() + } else { + state.peers[index] .ready_at - .is_some_and(|ready| now.duration_since(ready) <= PROBE_TIMEOUT); + .is_some_and(|ready| now.duration_since(ready) <= PROBE_TIMEOUT) + }; if state.eligible[index] != ready { state.eligible[index] = ready; changed = true; @@ -363,16 +411,61 @@ where MovePhase::Transfer => TRANSFER_TIMEOUT, }; if invalid || expired { + // The planned view ended after the grant, so watch whether + // the target keeps the partition. + let moved = + (invalid && current.is_some() && pending.phase == MovePhase::Transfer) + .then_some(pending.plan); state.pending = None; + if let Some(plan) = moved { + state.moves.push(MovedPrimary { plan, since: now }); + } } } + let PrimaryRebalance { + moves, backoffs, .. + } = &mut *state; + moves.retain(|moved| { + let target = moved.plan.target; + let (Some(current), Some(backoff)) = ( + metadata.partition_placement_view( + moved.plan.namespace, + moved.plan.created_revision, + ), + backoffs.get_mut(usize::from(target)), + ) else { + return false; + }; + if consensus.primary_index(current.view) != target { + backoff.record_failure(now); + return false; + } + if now.duration_since(moved.since) >= MOVE_HOLD_INTERVAL { + *backoff = TargetBackoff::default(); + return false; + } + true + }); if state.pending.is_none() && now >= state.next_move && now.duration_since(state.stable_since) >= STABLE_INTERVAL { state.next_move = now + PROBE_INTERVAL; - if let Some(plan) = - metadata.primary_rebalance_candidate(&state.eligible, state.cursor) + // After its wait, a backed-off target gets one move at a time + // until one holds. It also sits out as a source: it lost the + // primaries moved to it, so it rarely has any to give. + let mut candidates = state.eligible.clone(); + for ((replica, eligible), backoff) in + candidates.iter_mut().enumerate().zip(&state.backoffs) + { + let probing = backoff.wait > Duration::ZERO + && state + .moves + .iter() + .any(|moved| usize::from(moved.plan.target) == replica); + *eligible &= !backoff.blocks(now) && !probing; + } + if let Some(plan) = metadata.primary_rebalance_candidate(&candidates, state.cursor) { let mut probe = probe_header(consensus, self.next_forward_nonce(consensus.replica())); @@ -452,7 +545,12 @@ where }; reply.ready = u8::from( header.kind == PrimaryRebalanceKind::MetadataProbe - && consensus_ready(consensus), + && consensus_ready(consensus) + && !self + .plane + .metadata() + .applied_frontier() + .has_partition_recovery(), ); self.send_rebalance(header.replica, reply).await; } @@ -570,6 +668,11 @@ where && partition.consensus().is_primary() } else { partition_ready(partition, &header) + && !self + .plane + .metadata() + .applied_frontier() + .has_partition_recovery() } })); if let Some(partition) = partition { @@ -796,6 +899,7 @@ fn partition_ready( probe: &PrimaryRebalanceHeader, ) -> bool { partition_history_ready(partition, probe) + && partition.repair.is_none() && partition.consensus().commit_min() >= probe.commit && partition.consensus().commit_min() == partition.consensus().commit_max() && !partition.history_is_busy() diff --git a/core/shard/src/primary_rebalance/tests.rs b/core/shard/src/primary_rebalance/tests.rs index 2d78f4e256..16630b7b76 100644 --- a/core/shard/src/primary_rebalance/tests.rs +++ b/core/shard/src/primary_rebalance/tests.rs @@ -189,6 +189,7 @@ fn arm(shard: &TestShard, phase: MovePhase) -> PrimaryR view: Some(METADATA_VIEW), peers: vec![Peer::default(); usize::from(REPLICAS)], eligible: vec![true; usize::from(REPLICAS)], + backoffs: vec![TargetBackoff::default(); usize::from(REPLICAS)], pending: Some(PendingMove { plan: PartitionPrimaryTransfer { namespace, @@ -480,6 +481,87 @@ async fn given_a_sent_grant_when_the_new_view_is_unseen_should_hold_the_move_unt } } +#[test] +fn given_failed_moves_to_a_target_when_backing_off_should_double_each_wait_once_up_to_the_cap() { + let mut now = Instant::now(); + let mut backoff = TargetBackoff::default(); + for wait in [ + MOVE_HOLD_INTERVAL, + MOVE_HOLD_INTERVAL * 2, + MOVE_HOLD_INTERVAL * 4, + TARGET_BACKOFF_MAX, + TARGET_BACKOFF_MAX, + ] { + backoff.record_failure(now); + // Another watched move fails inside the same wait. + backoff.record_failure(now + wait / 2); + assert_eq!(backoff.wait, wait); + assert!(backoff.blocks(now + wait / 2)); + now += wait; + assert!(!backoff.blocks(now)); + } +} + +#[compio::test] +async fn given_a_backed_off_target_when_its_probe_move_is_lost_or_held_should_double_or_reset_the_wait() + { + let target_view = PARTITION_VIEW + 1; + for held in [false, true] { + let (shard, _inbox) = owner(); + arm(&shard, MovePhase::Transfer); + let metadata = shard.plane.metadata(); + assert_eq!( + metadata + .consensus + .as_ref() + .unwrap() + .primary_index(target_view), + TARGET + ); + let namespace = IggyNamespace::new(1, 1, 0); + let streams = metadata.mux_stm.streams(); + streams.seed_namespace(namespace, namespace.inner(), PARTITION_VIEW); + let created_revision = streams.created_revision_for_namespace(namespace).unwrap(); + let observe = |view| { + metadata.observe_partition_views(&[PartitionPlacementView { + namespace, + created_revision, + view, + }]); + }; + observe(PARTITION_VIEW); + let moved_at = shard.bus.monotonic_now(); + { + let mut state = shard.primary_rebalance.borrow_mut(); + let pending = state.pending.as_mut().unwrap(); + pending.plan.created_revision = created_revision; + pending.since = moved_at; + // The wait after one failed move ran out, so this move probes the target. + state.backoffs[usize::from(TARGET)] = TargetBackoff { + until: Some(moved_at), + wait: MOVE_HOLD_INTERVAL, + }; + } + + observe(target_view); + tick_with_ready_peers(&shard, moved_at).await; + if held { + tick_with_ready_peers(&shard, moved_at + MOVE_HOLD_INTERVAL).await; + } else { + observe(target_view + 1); + tick_with_ready_peers(&shard, moved_at + PROBE_INTERVAL).await; + } + + let wait = shard.primary_rebalance.borrow().backoffs[usize::from(TARGET)].wait; + let expected = if held { + Duration::ZERO + } else { + MOVE_HOLD_INTERVAL * 2 + }; + assert_eq!(wait, expected, "held={held}"); + } +} + #[compio::test] async fn given_an_unsent_grant_when_the_coordinator_ticks_should_plan_again_before_the_transfer_timeout() { @@ -762,6 +844,7 @@ fn given_partition_probe_when_local_history_is_not_ready_should_refuse() { "recovering", "unproved", "transfer", + "repair", "ceded", "recovery barrier", "transition", @@ -782,6 +865,20 @@ fn given_partition_probe_when_local_history_is_not_ready_should_refuse() { "transfer" => partition .consensus() .set_state_transfer_stage(StateTransferStage::AwaitingTarget), + "repair" => { + partition.repair = Some(partitions::RepairSession { + nonce: NONCE, + view: PARTITION_VIEW, + from_op: COMMITTED + 1, + commit_to_op: COMMITTED + 1, + fetch_to_op: COMMITTED + 1, + floor: None, + peer: SOURCE, + idle_ticks: 0, + floor_pulled_from: 0, + requested_from: None, + }); + } "ceded" => partition.consensus().init_as_backup(), "recovery barrier" => partition.consensus().set_recovery_barrier(COMMITTED + 1), "transition" => { @@ -812,6 +909,45 @@ fn given_busy_source_history_when_capturing_frontier_should_allow_drain_but_not_ assert!(!partition_ready(&partition, &probe)); } +#[compio::test] +async fn given_another_shard_recovering_when_metadata_is_probed_should_refuse_primary_moves_without_blocking_admission() + { + let (shard, _inbox) = owner_at::(None, 1, 3, 0); + let consensus = shard.plane.metadata().consensus.as_ref().unwrap(); + let frontier = shard.plane.metadata().applied_frontier(); + let other_shard = frontier.track_partition_recovery(); + other_shard.set_active(true); + let mut probe = probe_header(consensus, NONCE); + probe.replica = 0; + probe.quorum_age_ns = 0; + shard.on_primary_rebalance(probe).await; + probe.nonce += 1; + shard.on_primary_rebalance(probe).await; + assert_eq!(shard.bus.sent.borrow().last().unwrap().1.ready, 0); + assert!(frontier.partition_requests_allowed(shard.bus.monotonic_now())); + drop(other_shard); + probe.nonce += 1; + shard.on_primary_rebalance(probe).await; + assert_eq!(shard.bus.sent.borrow().last().unwrap().1.ready, 1); +} + +#[compio::test] +async fn given_local_partition_recovery_when_the_coordinator_probes_should_exclude_itself_until_recovery_finishes() + { + let (shard, _inbox) = owner_at::(None, 0, 3, 0); + let recovery = shard + .plane + .metadata() + .applied_frontier() + .track_partition_recovery(); + recovery.set_active(true); + shard.tick_primary_rebalance().await; + assert!(!shard.primary_rebalance.borrow().eligible[0]); + recovery.set_active(false); + shard.tick_primary_rebalance().await; + assert!(shard.primary_rebalance.borrow().eligible[0]); +} + #[compio::test] async fn given_three_or_five_replicas_when_a_quorum_answers_not_ready_should_credit_probe_send_times() { diff --git a/core/shard/src/router.rs b/core/shard/src/router.rs index 6fb87ff1b6..bbb7e5266a 100644 --- a/core/shard/src/router.rs +++ b/core/shard/src/router.rs @@ -420,6 +420,7 @@ where // offers live on every shard that hosts a serving group -- // parked behind the shard-0 gate they would never expire. self.expire_idle_state_transfer_offers(); + self.serve_parked_partition_transfer_requests().await; // A drain completes without an event, so a pending // revocation wakes the reconciler each tick. A published // fence wakes it once, and retries of an unpublished one diff --git a/core/simulator/README.md b/core/simulator/README.md index acd51282b1..a7e34edbf4 100644 --- a/core/simulator/README.md +++ b/core/simulator/README.md @@ -202,7 +202,7 @@ replica 2: live | metadata ... commit=1407..1414 barrier=1359 transferring=false The suite lives at the end of `src/lib.rs`, with unit tests next to the modules they cover. Names follow `given_X_when_Y_should_Z` where there is a meaningful given; older behavioural names remain where there is not. Every test that touches replies initialises `MemoryPool` first, as above. -What the suite pins, by example: `workload_replay_is_deterministic` and `multi_shard_replay_is_deterministic` (the seed contract), `committed_metadata_agrees_across_replicas`, `given_advanced_view_when_metadata_replica_restarts_should_recover_view_from_superblock`, `given_superblock_write_fails_when_primary_crashes_should_withhold_votes_and_not_elect`, `checkpointing_cluster_serves_a_chunked_state_transfer`, `shell_detects_partition_borrow_held_across_await`. +What the suite pins, by example: `workload_replay_is_deterministic` and `multi_shard_replay_is_deterministic` (the seed contract), `committed_metadata_agrees_across_replicas`, `given_advanced_view_when_metadata_replica_restarts_should_recover_view_from_superblock`, `given_superblock_write_fails_when_primary_crashes_should_withhold_the_new_view_from_peers`, `checkpointing_cluster_serves_a_chunked_state_transfer`, `shell_detects_partition_borrow_held_across_await`. Shell simulations run the production heartbeat verifier on every shard and the production consumer-group liveness loop on shard 0, using the virtual monotonic clock. Crash stops those tasks; restart rebuilds them. Silent or disconnected clients lose their lease through the same ordered cleanup as the server. Each simulator owns a temporary system directory for liveness fences, retained across replica restarts and removed when the simulator drops. Checkpoint simulations reuse their configured directory without changing its lifetime. diff --git a/core/simulator/src/lib.rs b/core/simulator/src/lib.rs index 4e0c45aff8..7d264052ee 100644 --- a/core/simulator/src/lib.rs +++ b/core/simulator/src/lib.rs @@ -40,7 +40,7 @@ use executor::{DetExecutor, RunOutcome, TaskId}; use iggy_binary_protocol::primitives::partition_history::{ConsumerGroupOwner, PartitionContext}; use iggy_binary_protocol::{ Command, CommitHeader, DoViewChangeHeader, GenericHeader, PrepareHeader, ReplyHeader, - RequestHeader, RoutedRequestHeader, StartViewChangeHeader, StartViewHeader, + RequestHeader, RoutedRequestHeader, StartViewHeader, }; use iggy_common::{Durability, IggyError}; use message_bus::MessageBus; @@ -918,14 +918,17 @@ impl Simulator { // left `status` at 0, so it reads as a completed handshake and hands // back a session the cluster never granted, silently, whenever that // commit point is nonzero. - if answer.is_none() - && header.client == client_id + if header.client == client_id && header.operation == expected_operation && header.request == request - && !setup_reply_is_transient(&reply) { - answer = Some(reply); - continue; + if setup_reply_is_transient(&reply) { + continue; + } + if answer.is_none() { + answer = Some(reply); + continue; + } } self.deferred_client_replies.push((from, reply)); } @@ -2153,7 +2156,8 @@ fn materialise_partition( /// The partition group, incarnation and view of a message the persist gate /// holds until the replica's view is durable, or `None` for any other message. -/// `PrepareOk` names no incarnation, so it is left out. +/// `PrepareOk` names no incarnation, so it is left out. A `StartViewChange` is +/// left out too, because a replica votes for a view before it enters it. fn view_scoped_partition_frame( message: &Message, ) -> Option<(IggyNamespace, u64, u32)> { @@ -2166,13 +2170,6 @@ fn view_scoped_partition_frame( let header = message.try_as_typed::().ok()?.header(); (header.group, header.partition_incarnation) } - Command::StartViewChange => { - let header = message - .try_as_typed::() - .ok()? - .header(); - (header.group, header.partition_incarnation) - } Command::DoViewChange => { let header = message.try_as_typed::().ok()?.header(); (header.group, header.partition_incarnation) @@ -3017,7 +3014,16 @@ mod tests { fn given_no_commit_heartbeat_when_a_partition_fence_commits_should_reach_every_backup_within_a_few_ticks() { const FENCE_STEPS: usize = 400; - let (mut sim, client) = cluster(0x4645_4E43); + let network = packet::PacketSimulatorOptions { + node_count: 3, + client_count: 1, + seed: 0x4645_4E43, + one_way_delay_min: 1, + one_way_delay_mean: 1, + ..packet::PacketSimulatorOptions::default() + }; + let mut sim = Simulator::new(3, std::iter::once(1), network); + let client = SimClient::new(1); let namespace = IggyNamespace::new(0, 0, 0); sim.init_partition(namespace); sim.register_client_with_primary(&client); @@ -3071,7 +3077,28 @@ mod tests { commit_max(&sim, replica).is_some_and(|commit| commit >= fence_op) }) }), - "every backup must learn the fence commit {fence_op} within {EAGER_COMMIT_TICKS} ticks" + "every backup must learn the fence commit {fence_op} within {EAGER_COMMIT_TICKS} ticks: {:?}", + (0..sim.replica_count) + .map(|replica| { + let partition = sim.replicas[usize::from(replica)] + .partition_shard(namespace) + .plane + .partitions() + .get_by_ns(&namespace) + .unwrap(); + let consensus = partition.consensus(); + ( + consensus.status(), + consensus.view(), + consensus.sequencer().current_sequence(), + consensus.commit_min(), + consensus.commit_max(), + partition.log.journal().inner.holds_op(fence_op), + partition.history_deleted(), + partition.repair, + ) + }) + .collect::>() ); } @@ -3438,7 +3465,13 @@ mod tests { for index in 0..SENDS { let send = client.send_messages(fixture, &[Bytes::from(format!("deleted-{index}"))]); let reply = submit_and_wait_for_reply(&mut sim, client.client_id(), 0, send); - assert_eq!(reply.header().status, 0, "send {index} must commit"); + assert_eq!( + reply.header().status, + 0, + "send {index} must commit: {:?}, body {:?}", + reply.header(), + String::from_utf8_lossy(reply.body()) + ); } complete_metadata_op(&mut sim, &client, client.delete_topic(&stream, &topic)); // A later prepare carries the completion's commit point, so the restarted @@ -3618,7 +3651,13 @@ mod tests { for index in 0..SENDS { let send = client.send_messages(fixture, &[Bytes::from(format!("recreated-{index}"))]); let reply = submit_and_wait_for_reply(&mut sim, client.client_id(), 0, send); - assert_eq!(reply.header().status, 0, "send {index} must commit"); + assert_eq!( + reply.header().status, + 0, + "send {index} must commit: {:?}, body {:?}", + reply.header(), + String::from_utf8_lossy(reply.body()) + ); } let committed = |sim: &Simulator, replica_idx: usize| { sim.offsets(replica_idx, fixture) @@ -3964,7 +4003,13 @@ mod tests { for index in 0..SENDS { let send = client.send_messages(fixture, &[Bytes::from(format!("deleted-{index}"))]); let reply = submit_and_wait_for_reply(&mut sim, client.client_id(), 0, send); - assert_eq!(reply.header().status, 0, "send {index} must commit"); + assert_eq!( + reply.header().status, + 0, + "send {index} must commit: {:?}, body {:?}", + reply.header(), + String::from_utf8_lossy(reply.body()) + ); } assert!( (0..PROGRESS_STEPS).any(|_| { @@ -4190,8 +4235,22 @@ mod tests { let (mut sim, client, _) = consumer_group_disconnect_cluster(3, session_count + 2); for partition in 0..PARTITIONS { let namespace = IggyNamespace::new(0, 0, partition); - sim.init_partition(namespace); - sim.seed_stream_topic_partition(namespace); + if sim.replicas[0] + .partition_shard(namespace) + .plane + .partitions() + .get_by_ns(&namespace) + .is_none() + { + sim.init_partition(namespace); + sim.seed_stream_topic_partition(namespace); + } + } + for replica in &sim.replicas { + replica.shards[0] + .plane + .metadata() + .seed_applied_frontier_from_consensus(); } sim.discover_partition_contexts(&client); for partition in 0..PARTITIONS { @@ -4667,11 +4726,11 @@ mod tests { ); } - /// Without a quorum the pending owner never installs, so settling stops at - /// the stale revocation bound and the quiesce check fails the revocation. + /// Without a quorum the primary steps down, so settlement must fail before + /// the convergence oracle can choose a leader to inspect. #[test] - #[should_panic(expected = "pending revocation")] - fn given_a_pending_owner_without_quorum_when_settled_should_fail_the_quiesce_check() { + #[should_panic(expected = "the views never settled")] + fn given_a_pending_owner_without_quorum_when_settling_should_refuse_convergence() { let mut sim = pending_leave_fixture(); sim.replica_crash(1); sim.replica_crash(2); @@ -4916,8 +4975,20 @@ mod tests { const PROGRESS_STEPS: usize = 4_000; for index in 0..count { let send = client.send_messages(namespace, &[Bytes::from(format!("message-{index}"))]); - let reply = submit_and_wait_for_reply(sim, client.client_id(), 0, send); - assert_eq!(reply.header().status, 0, "send {index} must commit"); + let reply = sim.await_setup_reply( + client, + 0, + &send.into_generic(), + Operation::SendMessages, + "commit_sends_everywhere", + ); + assert_eq!( + reply.header().status, + 0, + "send {index} must commit: {:?}, body {:?}", + reply.header(), + String::from_utf8_lossy(reply.body()) + ); } let last = u64::try_from(count - 1).expect("a send count fits u64"); assert!( @@ -5242,7 +5313,26 @@ mod tests { return reply; } } - panic!("request {request_id} did not receive a reply"); + panic!( + "request {request_id} did not receive a reply; metadata {:?}", + sim.replicas + .iter() + .enumerate() + .map(|(node, replica)| ( + node, + sim.metadata_consensus(node).map(|consensus| ( + consensus.status(), + consensus.view(), + consensus.commit_min(), + consensus.commit_max(), + consensus.sequencer().current_sequence(), + consensus.pipeline_head_header() + )), + replica.metadata_journal.last_op(), + replica.shards[0].metadata_repair_window() + )) + .collect::>() + ); } /// `(stream id, created op)` of `name` on `replica`. @@ -6123,6 +6213,10 @@ mod tests { .unwrap() .vsr_state(0, 0) .with_history_certificate(None); + let view_before_crash = sim + .metadata_consensus(usize::from(PEER)) + .expect("shard 0 hosts the metadata plane") + .view(); sim.replica_crash(0); sim.replica_crash(LEGACY); @@ -6145,10 +6239,14 @@ mod tests { Status::Recovering, "an unanswered probe let an unproven log vote" ); - assert_ne!( - with_consensus(&sim, PEER).0, - Status::Normal, - "one peer started a view without a quorum" + let peer = sim + .metadata_consensus(usize::from(PEER)) + .expect("shard 0 hosts the metadata plane"); + // The peer votes to replace the crashed primary but stays in its view. + assert!( + peer.view() == view_before_crash || peer.status() != Status::Normal, + "one peer started view {} without a quorum", + peer.view() ); sim.replica_restart(0); @@ -7433,15 +7531,16 @@ mod tests { } #[test] - fn given_superblock_write_fails_when_primary_crashes_should_withhold_votes_and_not_elect() { - // The gate under test: a view-scoped send (SVC, DVC, StartView) must not go - // out until the new view is durable. Failing one survivor's superblock writes + fn given_superblock_write_fails_when_primary_crashes_should_withhold_the_new_view_from_peers() { + // The gate under test: a view-scoped send (DVC, StartView) must not go out + // until the new view is durable. Failing one survivor's superblock writes // makes its persist gate return false, advancing its view in-memory while - // withholding every view-scoped send. Quorum 2 of 3, so crashing the primary - // leaves one working survivor whose lone vote cannot reach quorum and NO new - // primary is elected. Sending before persisting would instead elect a primary - // with no durable record of the new view, reintroducing split-brain on its - // restart. Positive control: + // withholding every view-scoped send. A StartViewChange is only a vote and + // changes no view, so both survivors enter view 1. The faulted survivor leads + // view 1 and can win it in memory, but its StartView never goes out, so the + // working survivor never starts the view. Sending before persisting would + // instead start a view whose primary has no durable record of it, + // reintroducing split-brain on its restart. Positive control: // `given_advanced_view_when_metadata_replica_restarts_...`, same crash with // healthy superblocks. server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { @@ -7464,8 +7563,8 @@ mod tests { ); // Replica 1 survives and is primary-by-index for view 1. Failing its - // superblock keeps it from durably taking any new view, so it never emits a - // view-scoped vote. + // superblock keeps it from durably taking any new view, so it never sends a + // DoViewChange or a StartView. sim.replicas[1].superblock.set_fail_writes(); sim.replica_crash(0); @@ -7473,24 +7572,18 @@ mod tests { sim.step(); } - for replica in 1..replica_count { - let elected = sim.replicas[replica as usize].shards[0] - .plane - .metadata() - .consensus - .as_ref() - .is_some_and(|consensus| { - consensus.view() > 0 - && consensus.status() == Status::Normal - && consensus.is_primary() - }); - assert!( - !elected, - "replica {replica} must not be elected primary while a survivor's \ - superblock write fails: the persist gate withholds its view-scoped \ - votes, so quorum cannot form" - ); - } + let working = sim.replicas[2].shards[0] + .plane + .metadata() + .consensus + .as_ref() + .expect("replica 2 owns metadata consensus"); + assert!( + working.view() == 0 || working.status() != Status::Normal, + "replica 2 started view {} while the superblock write of its primary fails: \ + the persist gate must withhold the StartView", + working.view() + ); // The faulted replica never durably recorded a new view, so a restart recovers // its old view, never one it might have voted in. @@ -8567,6 +8660,94 @@ mod tests { full_restart_awaiting_transfer_starts_a_view(&[1, 2]); } + /// A backup restarts with its partition files lost while it cannot reach its + /// peers, so nobody answers its view probe. Once the link heals it must adopt + /// the view of the primary that kept leading: only that primary can name the + /// state transfer it waits for. + #[test] + fn given_a_backup_awaiting_a_state_transfer_when_its_probes_go_unanswered_should_adopt_the_settled_view_after_the_heal() + { + const PROGRESS_STEPS: usize = 4_000; + // Three exhausted probes, each one lone view change for a replica + // allowed to elect on its recovered log. + const ISOLATION_STEPS: u64 = 3 + * consensus::TimeoutManager::REQUEST_START_VIEW_MESSAGE_TICKS + * consensus::PROBE_ATTEMPTS_MAX as u64; + let (mut sim, client) = cluster(packet::PacketSimulatorOptions::default().seed); + sim.register_client_with_primary(&client); + let namespace = IggyNamespace::new(1, 1, 0); + sim.seed_stream_topic_partition(namespace); + for replica in &sim.replicas { + materialise_partition(replica, namespace, sim.consumer_offsets_max); + } + let primary = (0..PROGRESS_STEPS) + .find_map(|_| { + sim.step(); + (0..3u8).find(|replica| { + sim.partition_consensus_state(usize::from(*replica), namespace) + .is_some_and(|state| state.status == Status::Normal && state.is_primary) + }) + }) + .expect("the fresh partition elects a primary"); + let awaiting = (primary + 1) % 3; + let set_links = |sim: &mut Simulator, enabled: bool| { + for peer in (0..3u8).filter(|peer| *peer != awaiting) { + sim.network.set_link_filter( + packet::ProcessId::Replica(awaiting), + packet::ProcessId::Replica(peer), + enabled, + ); + sim.network.set_link_filter( + packet::ProcessId::Replica(peer), + packet::ProcessId::Replica(awaiting), + enabled, + ); + } + }; + + sim.replica_crash(awaiting); + set_links(&mut sim, false); + sim.replica_restart(awaiting); + { + let partitions = sim.replicas[usize::from(awaiting)] + .partition_shard(namespace) + .plane + .partitions(); + let partition = partitions + .get_mut_by_ns(&namespace) + .expect("a restarted replica rebuilds the partition"); + partition.force_requires_state_transfer_for_test(); + assert!(partition.requires_state_transfer()); + } + for _ in 0..ISOLATION_STEPS { + sim.step(); + } + let isolated_view = sim + .partition_consensus_state(usize::from(awaiting), namespace) + .expect("the awaiting replica hosts the partition") + .view; + set_links(&mut sim, true); + + let settled = |sim: &Simulator| { + let leader = sim + .partition_consensus_state(usize::from(primary), namespace) + .expect("the primary hosts the partition"); + sim.partition_consensus_state(usize::from(awaiting), namespace) + .is_some_and(|state| state.view == leader.view) + }; + assert!( + (0..PROGRESS_STEPS).any(|_| { + sim.step(); + settled(&sim) + }), + "replica {awaiting} awaits a state transfer and never adopted the view of primary \ + {primary}: it reached view {isolated_view} while cut off, states={:?}", + (0..3) + .map(|replica| sim.partition_consensus_state(replica, namespace)) + .collect::>() + ); + } + fn successful_send_reply_count(replies: &[Message]) -> usize { replies .iter() @@ -12201,6 +12382,9 @@ mod view_change_data_loss_tests { /// the header distinguishes it from a commit and the code is the only signal. pub(crate) fn setup_reply_is_transient(reply: &Message) -> bool { let header = reply.header(); + if workload::TransientRejection::from_code(header.status).is_some() { + return true; + } let Some(body) = reply .as_slice() .get(size_of::()..header.size as usize) @@ -12465,10 +12649,28 @@ mod partition_repair_driver_tests { /// are still declared in each test, where its fault setup is read. macro_rules! withhold_one_prepare { ($namespace:ident, $withheld_op:ident) => { + static MIN_WITHHELD_OP: AtomicU64 = AtomicU64::new(0); + fn arm_withheld_prepare(sim: &Simulator) { + let namespace = IggyNamespace::from_raw($namespace.load(Ordering::Relaxed)); + let primary = sim.primary_index(namespace).unwrap(); + let partition = sim.replicas[usize::from(primary)] + .partition_shard(namespace) + .plane + .partitions() + .get_by_ns(&namespace) + .unwrap(); + MIN_WITHHELD_OP.store( + partition.consensus().sequencer().current_sequence() + 1, + Ordering::Relaxed, + ); + } fn withhold_one_prepare(packet: &Packet) -> bool { let Some(header) = prepare_for(packet, $namespace.load(Ordering::Relaxed)) else { return false; }; + if header.op < MIN_WITHHELD_OP.load(Ordering::Relaxed) { + return false; + } $withheld_op .compare_exchange(0, header.op, Ordering::Relaxed, Ordering::Relaxed) .is_ok() @@ -12862,6 +13064,9 @@ mod partition_repair_driver_tests { /// alone. Without it the retry timer could heal the gap and the test /// would pass with no repair driver at all. fn starve_commit_edge(packet: &Packet) -> bool { + if withhold_one_prepare(packet) { + return true; + } let group = GAP_NS.load(Ordering::Relaxed); if let Some(header) = prepare_for(packet, group) { return header.op == WITHHELD_OP.load(Ordering::Relaxed); @@ -12877,6 +13082,7 @@ mod partition_repair_driver_tests { WITHHELD_OP.store(0, Ordering::Relaxed); produce(&mut sim, &client, namespace, WARMUP_SENDS, "warmup"); + arm_withheld_prepare(&sim); let (_, _, warm_commit_min, _) = group_state(&sim, LAGGING, namespace); assert!( warm_commit_min > 0, @@ -12884,9 +13090,6 @@ mod partition_repair_driver_tests { below would open at the group's first op" ); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(starve_commit_edge); @@ -12896,7 +13099,7 @@ mod partition_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no partition prepare crossed the chain link, so the fault never armed" + "no partition prepare crossed the primary link, so the fault never armed" ); assert!( gap_drops(&sim, LAGGING, namespace) > 0, @@ -12960,6 +13163,9 @@ mod partition_repair_driver_tests { withhold_one_prepare!(GAP_NS, WITHHELD_OP); fn starve_commit_edge(packet: &Packet) -> bool { + if withhold_one_prepare(packet) { + return true; + } let group = GAP_NS.load(Ordering::Relaxed); if let Some(header) = prepare_for(packet, group) { return header.op == WITHHELD_OP.load(Ordering::Relaxed); @@ -12975,10 +13181,8 @@ mod partition_repair_driver_tests { WITHHELD_OP.store(0, Ordering::Relaxed); produce(&mut sim, &client, namespace, WARMUP_SENDS, "warmup"); + arm_withheld_prepare(&sim); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(starve_commit_edge); @@ -12987,7 +13191,7 @@ mod partition_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no partition prepare crossed the chain link, so the fault never armed" + "no partition prepare crossed the primary link, so the fault never armed" ); let (_, _, commit_min, commit_max) = group_state(&sim, LAGGING, namespace); assert!( @@ -13168,7 +13372,7 @@ mod partition_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert!( WITHHELD_COPIES.load(Ordering::Relaxed) > 0, - "no partition prepare crossed the chain link, so the fault never armed" + "no partition prepare crossed the primary link, so the fault never armed" ); assert!( gap_drops(&sim, LAGGING_OF_FIVE, namespace) > 0, @@ -13336,16 +13540,18 @@ mod partition_repair_driver_tests { withhold_one_prepare!(GAP_NS, WITHHELD_OP); - /// Primary -> 2: withhold every direct prepare (live ones ride the - /// chain, so this starves only retransmit heals), the group's commit - /// heartbeats, and its repair terminators. The repaired ops themselves + /// Withhold one live prepare and its retries, commit heartbeats, and + /// repair terminators. Later prepares carry the committed frontier. The repaired ops themselves /// pass, so the window lands resident while `complete_repair`, the walk /// the terminator would run, never fires. fn starve_walk_edges(packet: &Packet) -> bool { - let group = GAP_NS.load(Ordering::Relaxed); - if prepare_for(packet, group).is_some() { + if withhold_one_prepare(packet) { return true; } + let group = GAP_NS.load(Ordering::Relaxed); + if let Some(header) = prepare_for(packet, group) { + return header.op == WITHHELD_OP.load(Ordering::Relaxed); + } if is_repair_done_for(packet, group) { WITHHELD_DONES.fetch_add(1, Ordering::Relaxed); return true; @@ -13353,7 +13559,7 @@ mod partition_repair_driver_tests { is_commit_for(packet, group) } - let (mut sim, client) = cluster(0x5EED_0235); + let (mut sim, client) = cluster(0x5EED_0237); let namespace = IggyNamespace::new(1, 1, 0); sim.init_partition(namespace); sim.register_client_with_primary(&client); @@ -13362,6 +13568,7 @@ mod partition_repair_driver_tests { WITHHELD_DONES.store(0, Ordering::Relaxed); produce(&mut sim, &client, namespace, WARMUP_SENDS, "warmup"); + arm_withheld_prepare(&sim); let (_, _, warm_commit_min, _) = group_state(&sim, LAGGING, namespace); assert!( warm_commit_min > 0, @@ -13369,9 +13576,6 @@ mod partition_repair_driver_tests { below would open at the group's first op" ); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(starve_walk_edges); @@ -13381,7 +13585,7 @@ mod partition_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no partition prepare crossed the chain link, so the fault never armed" + "no partition prepare crossed the primary link, so the fault never armed" ); for _ in 0..STRAND_QUIET_STEPS { @@ -13438,6 +13642,9 @@ mod partition_repair_driver_tests { /// Primary -> 2: withhold this group's commit heartbeats, so the /// edge-triggered backstop cannot be what arms the repair below. fn starve_commit_edge(packet: &Packet) -> bool { + if withhold_one_prepare(packet) { + return true; + } let group = GAP_NS.load(Ordering::Relaxed); if let Some(header) = prepare_for(packet, group) { return header.op == WITHHELD_OP.load(Ordering::Relaxed); @@ -13453,9 +13660,7 @@ mod partition_repair_driver_tests { WITHHELD_OP.store(0, Ordering::Relaxed); produce(&mut sim, &client, namespace, WARMUP_SENDS, "warmup"); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); + arm_withheld_prepare(&sim); *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(starve_commit_edge); @@ -13466,7 +13671,7 @@ mod partition_repair_driver_tests { assert_ne!( WITHHELD_OP.load(Ordering::Relaxed), 0, - "no partition prepare crossed the chain link, so the fault never armed" + "no partition prepare crossed the primary link, so the fault never armed" ); let (_, _, commit_min, commit_max) = group_state(&sim, LAGGING, namespace); assert!( @@ -13595,8 +13800,10 @@ mod partition_repair_driver_tests { WITHHELD_OP.store(0, Ordering::Relaxed); produce(&mut sim, &client, namespace, WARMUP_SENDS, "warmup"); + arm_withheld_prepare(&sim); + *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = + .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(withhold_one_prepare); // Stepped one at a time and stopped on the step that buffered a drop: @@ -13773,10 +13980,25 @@ mod metadata_repair_driver_tests { /// per-test or the siblings in this binary would share one fault. macro_rules! withhold_one_metadata_prepare { ($withheld_op:ident) => { + static MIN_WITHHELD_OP: AtomicU64 = AtomicU64::new(0); + fn arm_withheld_prepare(sim: &Simulator) { + let primary = sim.metadata_primary_index().unwrap(); + MIN_WITHHELD_OP.store( + sim.metadata_consensus(usize::from(primary)) + .unwrap() + .sequencer() + .current_sequence() + + 1, + Ordering::Relaxed, + ); + } fn withhold_one_prepare(packet: &Packet) -> bool { let Some(header) = prepare_for(packet, METADATA_GROUP) else { return false; }; + if header.op < MIN_WITHHELD_OP.load(Ordering::Relaxed) { + return false; + } $withheld_op .compare_exchange(0, header.op, Ordering::Relaxed, Ordering::Relaxed) .is_ok() @@ -13794,6 +14016,9 @@ mod metadata_repair_driver_tests { macro_rules! starve_commit_edge { ($withheld_op:ident) => { fn starve_commit_edge(packet: &Packet) -> bool { + if withhold_one_prepare(packet) { + return true; + } if let Some(header) = prepare_for(packet, METADATA_GROUP) { return header.op == $withheld_op.load(Ordering::Relaxed); } @@ -13852,11 +14077,22 @@ mod metadata_repair_driver_tests { /// Submit `sends` stream creations to the primary, stepping between each. fn create_streams(sim: &mut Simulator, client: &SimClient, sends: usize, tag: &str) { for index in 0..sends { - let msg = client.create_stream(&format!("{tag}-{index}")); - sim.submit_request(client.client_id(), 0, msg.into_generic()); - for _ in 0..STEPS_PER_SEND { - sim.step(); - } + let request = client + .create_stream(&format!("{tag}-{index}")) + .into_generic(); + let reply = sim.await_setup_reply( + client, + 0, + &request, + iggy_binary_protocol::Operation::CreateStream, + "create_streams", + ); + assert_eq!( + reply.header().status, + 0, + "metadata setup {tag}-{index} must commit" + ); + assert_eq!(iggy_binary_protocol::result_code(reply.body()), Some(0)); } } @@ -14040,6 +14276,7 @@ mod metadata_repair_driver_tests { WITHHELD_OP.store(0, Ordering::Relaxed); create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm"); + arm_withheld_prepare(&sim); let (_, _, warm_commit_min, _) = metadata_state(&sim, LAGGING); assert!( warm_commit_min > 0, @@ -14047,9 +14284,6 @@ mod metadata_repair_driver_tests { below would open at the group's first op" ); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(starve_commit_edge); @@ -14059,7 +14293,7 @@ mod metadata_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no metadata prepare crossed the chain link, so the fault never armed" + "no metadata prepare crossed the primary link, so the fault never armed" ); assert!( gap_drops(&sim, LAGGING) > 0, @@ -14123,10 +14357,8 @@ mod metadata_repair_driver_tests { WITHHELD_OP.store(0, Ordering::Relaxed); create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm"); + arm_withheld_prepare(&sim); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(starve_commit_edge); @@ -14135,7 +14367,7 @@ mod metadata_repair_driver_tests { assert_ne!( WITHHELD_OP.load(Ordering::Relaxed), 0, - "no metadata prepare crossed the chain link, so the fault never armed" + "no metadata prepare crossed the primary link, so the fault never armed" ); // Move the primary's retention floor past the whole gap window before @@ -14171,15 +14403,17 @@ mod metadata_repair_driver_tests { static WITHHELD_DONES: AtomicU64 = AtomicU64::new(0); withhold_one_metadata_prepare!(WITHHELD_OP); - /// Primary -> 2: withhold every direct prepare (live ones ride the - /// chain, so this starves only retransmit heals), the group's commit - /// heartbeats, and its repair terminators. The repaired ops themselves + /// Withhold one live prepare and its retries, commit heartbeats, and + /// repair terminators. Later prepares carry the committed frontier. The repaired ops themselves /// pass, so the window lands resident while the `RepairDone` that would /// run the walk never fires. fn starve_walk_edges(packet: &Packet) -> bool { - if prepare_for(packet, METADATA_GROUP).is_some() { + if withhold_one_prepare(packet) { return true; } + if let Some(header) = prepare_for(packet, METADATA_GROUP) { + return header.op == WITHHELD_OP.load(Ordering::Relaxed); + } if is_repair_done_for(packet, METADATA_GROUP) { WITHHELD_DONES.fetch_add(1, Ordering::Relaxed); return true; @@ -14187,12 +14421,13 @@ mod metadata_repair_driver_tests { is_commit_for(packet, METADATA_GROUP) } - let (mut sim, client) = cluster(0x5EED_0242); + let (mut sim, client) = cluster(0x5EED_0243); sim.register_client_with_primary(&client); WITHHELD_OP.store(0, Ordering::Relaxed); WITHHELD_DONES.store(0, Ordering::Relaxed); create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm"); + arm_withheld_prepare(&sim); let (_, _, warm_commit_min, _) = metadata_state(&sim, LAGGING); assert!( warm_commit_min > 0, @@ -14200,9 +14435,6 @@ mod metadata_repair_driver_tests { below would open at the group's first op" ); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(starve_walk_edges); @@ -14212,7 +14444,7 @@ mod metadata_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no metadata prepare crossed the chain link, so the fault never armed" + "no metadata prepare crossed the primary link, so the fault never armed" ); for _ in 0..STRAND_QUIET_STEPS { @@ -14259,16 +14491,14 @@ mod metadata_repair_driver_tests { withhold_one_metadata_prepare!(WITHHELD_OP); starve_commit_edge!(WITHHELD_OP); - let (mut sim, client) = cluster(0x5EED_0245); + let (mut sim, client) = cluster(0x5EED_0246); sim.register_client_with_primary(&client); WITHHELD_OP.store(0, Ordering::Relaxed); sim.replicas[LAGGING as usize].shards[0].set_repair_retry_ticks(ROTATE_RETRY_TICKS); create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm"); + arm_withheld_prepare(&sim); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(starve_commit_edge); @@ -14277,7 +14507,7 @@ mod metadata_repair_driver_tests { assert_ne!( WITHHELD_OP.load(Ordering::Relaxed), 0, - "no metadata prepare crossed the chain link, so the fault never armed" + "no metadata prepare crossed the primary link, so the fault never armed" ); // Corrupt the lowest op repair has put back above the commit point, @@ -14350,7 +14580,14 @@ mod metadata_repair_driver_tests { /// The peer `gap_repair_peer` names for a gap-stopped backup is the /// primary, so the session can only heal by rotating off it. - fn blackhole(_packet: &Packet) -> bool { + fn blackhole(packet: &Packet) -> bool { + if packet.message.header().command == Command::PrepareProgress { + return false; + } + if let Some(header) = prepare_for(packet, METADATA_GROUP) { + return withhold_one_prepare(packet) + || header.op == WITHHELD_OP.load(Ordering::Relaxed); + } true } @@ -14376,6 +14613,7 @@ mod metadata_repair_driver_tests { sim.replicas[LAGGING as usize].shards[0].set_repair_retry_ticks(ROTATE_RETRY_TICKS); create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm"); + arm_withheld_prepare(&sim); let (_, _, warm_commit_min, _) = metadata_state(&sim, LAGGING); assert!( warm_commit_min > 0, @@ -14383,13 +14621,9 @@ mod metadata_repair_driver_tests { below would open at the group's first op" ); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); // Everything, not just this group's commit edge: the primary must be // unable to answer the repair it is about to be asked for, while the - // chain (0 -> 1 -> 2) keeps carrying the prepares that advance - // `commit_max` past the hole. + // later direct prepares keep advancing `commit_max` past the hole. *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(blackhole); @@ -14404,7 +14638,7 @@ mod metadata_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no metadata prepare crossed the chain link, so the fault never armed" + "no metadata prepare crossed the primary link, so the fault never armed" ); for _ in 0..ROTATE_QUIET_STEPS { @@ -14453,6 +14687,7 @@ mod metadata_repair_driver_tests { /// direct reply, so no stall clock fires and no budget is charged: the /// replica never leaves that peer, even once another one can serve the hole. #[test] + #[allow(clippy::too_many_lines)] fn given_a_repair_peer_without_snapshot_when_it_answers_range_evicted_should_rotate_instead_of_transferring() { /// Where the session rotates off the primary (`next_transfer_peer`), and @@ -14463,7 +14698,14 @@ mod metadata_repair_driver_tests { static TRANSFER_REQUESTS: AtomicU64 = AtomicU64::new(0); withhold_one_metadata_prepare!(WITHHELD_OP); - fn blackhole(_packet: &Packet) -> bool { + fn blackhole(packet: &Packet) -> bool { + if packet.message.header().command == Command::PrepareProgress { + return false; + } + if let Some(header) = prepare_for(packet, METADATA_GROUP) { + return withhold_one_prepare(packet) + || header.op == WITHHELD_OP.load(Ordering::Relaxed); + } true } @@ -14475,13 +14717,14 @@ mod metadata_repair_driver_tests { false } - let (mut sim, client) = cluster(0x5EED_0246); + let (mut sim, client) = cluster(0x5EED_0247); sim.register_client_with_primary(&client); WITHHELD_OP.store(0, Ordering::Relaxed); TRANSFER_REQUESTS.store(0, Ordering::Relaxed); sim.replicas[LAGGING as usize].shards[0].set_repair_retry_ticks(ROTATE_RETRY_TICKS); create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm"); + arm_withheld_prepare(&sim); *sim.network .link_drop_packet_fn(ProcessId::Replica(SUCCESSOR), ProcessId::Replica(LAGGING)) = @@ -14498,7 +14741,7 @@ mod metadata_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no metadata prepare crossed the chain link, so the fault never armed" + "no metadata prepare crossed the primary link, so the fault never armed" ); // The hole a lost repair frame leaves (see `SimJournal::forget_op`). The @@ -14602,10 +14845,8 @@ mod metadata_repair_driver_tests { TRANSFER_REQUESTS.store(0, Ordering::Relaxed); create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm"); + arm_withheld_prepare(&sim); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); // Starved so no heartbeat arms the repair before the primary's hole exists. *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = @@ -14618,7 +14859,7 @@ mod metadata_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no metadata prepare crossed the chain link, so the fault never armed" + "no metadata prepare crossed the primary link, so the fault never armed" ); let (_, _, primary_commit_min, _) = metadata_state(&sim, 0); assert!( @@ -14703,10 +14944,8 @@ mod metadata_repair_driver_tests { TRANSFER_REQUESTS.store(0, Ordering::Relaxed); create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm"); + arm_withheld_prepare(&sim); - *sim.network - .link_drop_packet_fn(ProcessId::Replica(1), ProcessId::Replica(LAGGING)) = - Some(withhold_one_prepare); *sim.network .link_drop_packet_fn(ProcessId::Replica(0), ProcessId::Replica(LAGGING)) = Some(starve_commit_edge); @@ -14720,7 +14959,7 @@ mod metadata_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no metadata prepare crossed the chain link, so the fault never armed" + "no metadata prepare crossed the primary link, so the fault never armed" ); for peer in [0, 1] { let (_, _, peer_commit_min, _) = metadata_state(&sim, peer); @@ -14805,6 +15044,7 @@ mod metadata_repair_driver_tests { .set_repair_retry_ticks(ROTATE_RETRY_TICKS); create_streams(&mut sim, &client, WARMUP_SENDS, "md-warm"); + arm_withheld_prepare(&sim); *sim.network.link_drop_packet_fn( ProcessId::Replica(LAGGING_OF_FIVE - 1), @@ -14818,7 +15058,7 @@ mod metadata_repair_driver_tests { let withheld = WITHHELD_OP.load(Ordering::Relaxed); assert_ne!( withheld, 0, - "no metadata prepare crossed the chain link, so the fault never armed" + "no metadata prepare crossed the primary link, so the fault never armed" ); // A peer can learn the last gap commit from the primary's next `Commit`, // after the sends end. Waiting one more send interval keeps the gap phase @@ -15750,8 +15990,8 @@ mod metadata_read_frontier_tests { ); assert_eq!( before.header().status, - 0, - "the old primary may serve until its evidence expires: partition {:?}, fresh {}", + IggyError::TransientNotAccepted.as_code(), + "quorum loss must fence reads before metadata evidence expires: partition {:?}, fresh {}", sim.partition_consensus_state(0, namespace), partition_fresh(&sim, 0) ); @@ -15782,11 +16022,11 @@ mod metadata_read_frontier_tests { .consensus .as_ref() .unwrap(); - assert!( - old.is_primary(), - "the isolated node still believes it is primary" + assert_ne!( + old.status(), + Status::Normal, + "the minority cannot regain leadership" ); - assert_eq!(old.view(), 0); let convergence_start = sim.network.current_tick(); for _ in 0..SETUP_TOTAL_STEPS { if [2, 3, 4] @@ -16419,6 +16659,10 @@ mod applied_divergence_resync_tests { artifacts: Vec::new(), target_accepted: false, idle_ticks: 0, + spill: None, + scan: None, + seed: None, + pulled_len: 0, }); let consensus = partition.consensus(); consensus.set_state_transfer_stage(StateTransferStage::AwaitingTarget); @@ -16528,13 +16772,16 @@ mod metadata_walk_stuck_detector_tests { /// Swarm seed whose network elects replica 1 primary of view 1 with a /// committed pipeline head it has not yet walked. - const SEED: u64 = 1723; + const SEED: u64 = 70; const REPLICAS: u8 = 3; const CLIENTS: [u128; 3] = [1, 2, 3]; const VIEW_0_PRIMARY: u8 = 0; /// The replica the faults leave to win view 1. - const PRIMARY_ELECT: usize = 1; + const PRIMARY_ELECT: u8 = 1; + /// Holds the last registration as committed. Its `DoViewChange` would end + /// the view-1 change with that op committed and the pipeline empty. + const COMMITTED_PEER: u8 = 2; /// Replica faults as `(step, replica, crash)`. The view-0 primary goes /// down, which puts the view-1 change on the primary-elect, and comes back @@ -16543,7 +16790,7 @@ mod metadata_walk_stuck_detector_tests { [(130, VIEW_0_PRIMARY, true), (250, VIEW_0_PRIMARY, false)]; /// Past the inherited-head window, then the drain heals the cluster. - const FAULT_STEPS: usize = 800; + const FAULT_STEPS: usize = 1_000; const DRAIN_STEPS: usize = 4_000; /// Only the three `Register`s go through the metadata plane: no client @@ -16553,6 +16800,9 @@ mod metadata_walk_stuck_detector_tests { #[test] fn given_primary_elect_inheriting_a_committed_head_when_the_tick_probes_the_walk_should_keep_its_prepare() { + fn drop_do_view_change(packet: &packet::Packet) -> bool { + packet.message.header().command == Command::DoViewChange + } server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { enabled: false, size: iggy_common::IggyByteSize::from(0u64), @@ -16567,6 +16817,16 @@ mod metadata_walk_stuck_detector_tests { sim.register_client_with_primary(&SimClient::new(id)); } + // A view change needs a quorum of votes, so the peer that also lost the + // view-0 primary takes part in the view-1 change, and its + // `DoViewChange` would commit the last registration. Without it the + // primary-elect merges the log of the restarted view-0 primary, which + // holds that op above its commit point. + *sim.network.link_drop_packet_fn( + ProcessId::Replica(COMMITTED_PEER), + ProcessId::Replica(PRIMARY_ELECT), + ) = Some(drop_do_view_change); + let mut inherited_head_seen = false; for step in 0..FAULT_STEPS { for &(at, replica, crash) in &FAULTS { @@ -16581,6 +16841,10 @@ mod metadata_walk_stuck_detector_tests { sim.step(); inherited_head_seen |= primary_elect_holds_inherited_head(&sim); } + *sim.network.link_drop_packet_fn( + ProcessId::Replica(COMMITTED_PEER), + ProcessId::Replica(PRIMARY_ELECT), + ) = None; // Without this a drifted seed that never builds the shape passes the // drain assertion below without testing anything. assert!( @@ -16613,7 +16877,7 @@ mod metadata_walk_stuck_detector_tests { /// does not count. fn primary_elect_holds_inherited_head(sim: &Simulator) -> bool { let consensus = sim - .metadata_consensus(PRIMARY_ELECT) + .metadata_consensus(usize::from(PRIMARY_ELECT)) .expect("shard 0 hosts the metadata plane"); consensus.is_primary() && consensus.is_normal() diff --git a/core/simulator/src/packet.rs b/core/simulator/src/packet.rs index 30417e2004..0521dd0183 100644 --- a/core/simulator/src/packet.rs +++ b/core/simulator/src/packet.rs @@ -341,7 +341,7 @@ pub struct PacketSimulator { /// One past the highest [`Command`] discriminant, sizing [`COMMAND_LABELS`] and /// the delivery counters. Raising it is part of adding a command. -pub const COMMAND_COUNT_MAX: usize = 35; +pub const COMMAND_COUNT_MAX: usize = 37; /// Names for each [`Command`] discriminant, so a coverage report reads as /// protocol rather than as integers. Indexed by discriminant; the trailing @@ -382,6 +382,8 @@ pub const COMMAND_LABELS: [&str; COMMAND_COUNT_MAX] = [ "ForwardCompletion", "PrimaryRebalance", "SegmentDelete", + "CommitReply", + "PrepareProgress", ]; const _: () = { diff --git a/core/simulator/src/slow_replica_tests.rs b/core/simulator/src/slow_replica_tests.rs index 0c22f42e33..cb5b60d900 100644 --- a/core/simulator/src/slow_replica_tests.rs +++ b/core/simulator/src/slow_replica_tests.rs @@ -19,17 +19,33 @@ //! //! Replica 2 is cut off for a moment, long enough to start a view change of its //! own, and every message it sends from then on takes longer than the -//! view-change status timeout to arrive. Its `StartViewChange` for the next -//! view therefore lands after it already timed out into the view after that. -//! Replicas 0 and 1 are Normal and commit together, yet each such -//! `StartViewChange` takes them out of their working view, and the primary -//! keeps moving between them. Nothing but latency is injected after the cut -//! heals: no packet is dropped. +//! view-change status timeout to arrive. A replica that times out into the +//! next view before its `StartViewChange` lands stays one view ahead of every +//! vote it sends, and each vote that lands takes replicas 0 and 1 out of their +//! working view while they commit together. Nothing but latency is injected +//! after the cut heals: no packet is dropped. +//! +//! Replica 2 can also hear a single peer late while every other link stays +//! fast. On a real network a slow egress of replica 2 delays the TCP +//! acknowledgements of the prepares replica 1 forwards to it down the chain, so +//! they queue on that link, and every frame replica 1 sends after them waits in +//! the same queue. Replica 2 then gives up on each view replica 1 leads before +//! its `StartView` arrives, and the rebalancer must not keep handing primaries +//! back to replica 1. +//! +//! A replica that stops hearing a primary its peers still hear must not depose +//! it, on either plane, while a primary that crashed or was cut off is still +//! replaced as fast as before. +use bytes::Bytes; +use consensus::{PartitionsHandle, Status, TimeoutManager}; +use iggy_binary_protocol::{Command, Operation, PrepareHeader, RequestPreparesHeader}; use server_common::sharding::IggyNamespace; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use crate::Simulator; -use crate::packet::ProcessId; +use crate::client::SimClient; +use crate::packet::{Packet, ProcessId}; use crate::partition_repair_driver_tests::cluster; use crate::workload::{ self, Workload, @@ -39,11 +55,30 @@ use crate::workload::{ const SEED: u64 = 0x5052_494D; const SLOW: u8 = 2; +/// Leads the single group of [`Run::single`] and the metadata plane. +const LEADER: u8 = 0; +/// The peer replica 2 hears late: its predecessor in the chain of every group +/// that replica 0 or 1 leads. +const LAGGING_PEER: u8 = 1; /// Groups the rebalancer spreads across the replicas, so each replica leads /// one before the fault. const GROUPS: usize = 3; /// Ticks for the rebalancer to spread the primaries. const SPREAD_TICKS: u64 = 10_000; +/// Ticks for a single group to settle under the workload before a fault, as +/// long as the spread, so the rebalancer had the same time to act. +const SETTLE_TICKS: u64 = SPREAD_TICKS; +/// Ten view-change status timeouts: a replica cut off this long times out +/// again and again. +const PARTITION_TICKS: u64 = 10 * TimeoutManager::VIEW_CHANGE_STATUS_TICKS; +/// A backup times out a heartbeat timeout after the last heartbeat it heard, +/// and that heartbeat left the primary at most one heartbeat interval before +/// the fault. Every backup heard one in that interval, so waiting for the +/// backup that times out last adds nothing to the bound. The view change +/// itself must finish within one retransmit interval of its votes. +const ELECTION_TICKS_MAX: u64 = TimeoutManager::NORMAL_HEARTBEAT_TICKS + + TimeoutManager::COMMIT_MESSAGE_TICKS + + TimeoutManager::START_VIEW_CHANGE_MESSAGE_TICKS; /// Past `NORMAL_HEARTBEAT_TICKS` (500): the cut lasts long enough that the /// group replica 2 led fails over and replica 2 starts a view change itself. const CUT_TICKS: u64 = 600; @@ -51,89 +86,441 @@ const CUT_TICKS: u64 = 600; /// `VIEW_CHANGE_STATUS_TICKS` (500). Below that bound the healthy pair adopts /// replica 2's view before it times out again, and the views settle. const EGRESS_DELAY_TICKS: u64 = 520; -/// Measured after the heal: 300 s of virtual time at 10 ms per tick. +/// One-way delay from the lagging peer to replica 2: 20 s, inside the 15 to +/// 55 s the queue took on a real cluster. Any delay past +/// `VIEW_CHANGE_STATUS_TICKS` (500) loses the `StartView` of that peer. +const LAGGING_PEER_DELAY_TICKS: u64 = 2_000; +/// Measured after the fault starts: 300 s of virtual time at 10 ms per tick. const MEASURE_TICKS: u64 = 30_000; -/// The heal itself may cost a few view changes while replica 2 rejoins: 450 -/// ticks of delay costs up to 3 here, 520 ticks costs about 60. +/// The heal itself may cost a few view changes while replica 2 rejoins: up to 2 +/// per group here with 450, 520 or 1200 ticks of delay. const VIEW_CHANGES_MAX: u32 = 5; +/// Summed over the groups, because the rebalancer backs off per target: one +/// view change when replica 2 leaves the view replica 1 leads at the start, +/// then two for each primary moved to replica 1 that replica 2 cannot follow. +/// The waits after the failed moves, 30, 60 and 120 s, leave room for four such +/// moves in the window, and a fifth would need 450 s. Without the backoff a +/// move fails every few seconds. +const LAGGING_PEER_VIEW_CHANGES_MAX: u32 = 9; #[test] -#[ignore = "known bug: a replica slower than the view-change status timeout keeps the healthy pair leaving its working view; run with --ignored"] -fn given_a_replica_slower_than_the_view_change_timeout_when_it_rejoins_should_not_keep_the_healthy_pair_changing_views() +#[allow(clippy::too_many_lines)] +fn given_a_missed_committed_prepare_when_pull_repair_is_unavailable_should_rejoin_live_delivery_under_load() { - let (mut sim, client) = cluster(SEED); - let namespaces: Vec<_> = (0..GROUPS) - .map(|partition| IggyNamespace::new(1, 1, partition)) - .collect(); - for &namespace in &namespaces { - sim.init_partition(namespace); + const WRITES: usize = 24; + const TICKS_PER_WRITE: usize = 250; + static GROUP: AtomicU64 = AtomicU64::new(0); + static MISSING: AtomicU64 = AtomicU64::new(0); + static WITHHOLD: AtomicBool = AtomicBool::new(true); + static DROPPED: AtomicU64 = AtomicU64::new(0); + + fn withhold_prepare(packet: &Packet) -> bool { + if !WITHHOLD.load(Ordering::Relaxed) || packet.message.header().command != Command::Prepare + { + return false; + } + let message = packet.message.try_as_typed::().unwrap(); + let header = message.header(); + if header.group == GROUP.load(Ordering::Relaxed) + && header.op == MISSING.load(Ordering::Relaxed) + { + DROPPED.fetch_add(1, Ordering::Relaxed); + return true; + } + false + } + + fn block_pull_repair(packet: &Packet) -> bool { + packet.message.header().command == Command::RequestPrepares + && packet + .message + .try_as_typed::() + .unwrap() + .header() + .group + == GROUP.load(Ordering::Relaxed) } + + let (mut sim, client) = cluster(SEED); + let namespace = IggyNamespace::new(1, 1, 0); + sim.init_partition(namespace); sim.register_client_with_primary(&client); - let mut options = WorkloadOptions::new(SEED, sim.replica_count, namespaces.clone()); - options.weights = ActionWeights::partition_only(); - let mut workload = Workload::new(options); - let clients = [client]; - let mut invariants = Invariants::new(); - let mut tick = |sim: &mut Simulator| { - workload::run(sim, &mut workload, &clients, 1, u64::MAX, &mut invariants) - }; - - for _ in 0..SPREAD_TICKS { - tick(&mut sim); - } - set_peer_links(&mut sim, false); + let warmup = client + .send_messages(namespace, &[Bytes::from_static(b"warmup")]) + .into_generic(); + let reply = sim.await_setup_reply( + &client, + LEADER, + &warmup, + Operation::SendMessages, + "warmup prefix", + ); + assert_eq!(reply.header().status, 0); + assert!((0..500).any(|_| { + sim.step(); + sim.partition_consensus_state(usize::from(SLOW), namespace) + .is_some_and(|state| state.commit_min >= 1) + })); + GROUP.store(namespace.inner(), Ordering::Relaxed); + MISSING.store(2, Ordering::Relaxed); + WITHHOLD.store(true, Ordering::Relaxed); + DROPPED.store(0, Ordering::Relaxed); + *sim.network + .link_drop_packet_fn(ProcessId::Replica(LEADER), ProcessId::Replica(SLOW)) = + Some(withhold_prepare); + for peer in [0, 1] { + *sim.network + .link_drop_packet_fn(ProcessId::Replica(SLOW), ProcessId::Replica(peer)) = + Some(block_pull_repair); + } + let repair_prepares = sim.network.command_counts()[Command::RepairPrepare as usize]; + let request = client + .send_messages(namespace, &[Bytes::from_static(b"missed")]) + .into_generic(); + let reply = sim.await_setup_reply( + &client, + LEADER, + &request, + Operation::SendMessages, + "missed prepare setup", + ); + assert_eq!(reply.header().status, 0); + assert!(DROPPED.load(Ordering::Relaxed) > 0); + assert!( + sim.replicas[usize::from(SLOW)] + .partition_shard(namespace) + .plane + .partitions() + .get_by_ns(&namespace) + .unwrap() + .consensus() + .commit_min() + < 2 + ); + WITHHOLD.store(false, Ordering::Relaxed); + + let mut caught_under_load = false; + for index in 0..WRITES { + let request = client + .send_messages(namespace, &[Bytes::from(format!("live-{index}"))]) + .into_generic(); + let reply = sim.await_setup_reply( + &client, + LEADER, + &request, + Operation::SendMessages, + "continued load", + ); + assert_eq!(reply.header().status, 0); + for _ in 0..TICKS_PER_WRITE { + sim.step(); + } + let state = sim + .partition_consensus_state(usize::from(SLOW), namespace) + .unwrap(); + caught_under_load |= index < WRITES - 1 && state.commit_min >= reply.header().op; + assert_eq!((state.status, state.view), (Status::Normal, 0)); + } + assert!( + caught_under_load, + "the replica must catch up before writes stop: source {:?}, backup {:?}, progress {:?}", + sim.partition_consensus_state(usize::from(LEADER), namespace), + sim.partition_consensus_state(usize::from(SLOW), namespace), + sim.replicas[usize::from(SLOW)] + .partition_shard(namespace) + .plane + .partitions() + .get_by_ns(&namespace) + .map(|partition| ( + partition.held_through(), + partition.replication_progress(), + partition.view_fenced(), + partition.repair + )), + ); + assert!(sim.network.delivered_any(Command::PrepareProgress)); + assert_eq!( + sim.network.command_counts()[Command::RepairPrepare as usize], + repair_prepares, + "the committed hole must heal through ordered live replay while pull repair is blocked" + ); +} + +#[test] +fn given_a_replica_slower_than_the_view_change_timeout_when_it_rejoins_should_not_keep_the_healthy_pair_changing_views() + { + let mut run = Run::spread(); + set_peer_links(&mut run.sim, SLOW, false); for _ in 0..CUT_TICKS { - tick(&mut sim); + run.tick(); } - set_peer_links(&mut sim, true); - for peer in (0..sim.replica_count).filter(|&peer| peer != SLOW) { - *sim.network + set_peer_links(&mut run.sim, SLOW, true); + for peer in (0..run.sim.replica_count).filter(|&peer| peer != SLOW) { + *run.sim + .network .link_extra_delay(ProcessId::Replica(SLOW), ProcessId::Replica(peer)) = EGRESS_DELAY_TICKS; } + let changes = run.measure_view_changes(); + for (namespace, &count) in run.namespaces.iter().zip(&changes) { + assert!( + count <= VIEW_CHANGES_MAX, + "replicas 0 and 1 changed view {count} times for {namespace:?} in {MEASURE_TICKS} \ + ticks, while only replica 2 was slow: one replica whose messages arrive later \ + than the view-change status timeout keeps the healthy pair leaving its working \ + view (changes per group {changes:?})" + ); + } +} - let first_views: Vec = namespaces - .iter() - .map(|&namespace| healthy_view(&sim, namespace)) - .collect(); - let mut views = first_views.clone(); - let mut changes = vec![0_u32; GROUPS]; - let mut replies = 0; - for _ in 0..MEASURE_TICKS { - replies += tick(&mut sim); - for (index, &namespace) in namespaces.iter().enumerate() { - let view = healthy_view(&sim, namespace); - if view != views[index] { - changes[index] += 1; - views[index] = view; - } - } +#[test] +fn given_a_replica_that_hears_one_peer_late_when_primaries_are_rebalanced_should_not_keep_the_healthy_pair_changing_views() + { + let mut run = Run::spread(); + *run.sim + .network + .link_extra_delay(ProcessId::Replica(LAGGING_PEER), ProcessId::Replica(SLOW)) = + LAGGING_PEER_DELAY_TICKS; + let changes = run.measure_view_changes(); + let total: u32 = changes.iter().sum(); + assert!( + total <= LAGGING_PEER_VIEW_CHANGES_MAX, + "replicas 0 and 1 changed view {total} times in {MEASURE_TICKS} ticks, while replica 2 \ + heard replica 1 late: replica 2 leaves every view replica 1 leads before it hears of \ + it, and each primary moved back to replica 1 starts the view changes again (changes \ + per group {changes:?})" + ); +} + +#[test] +fn given_a_replica_that_hears_the_primary_late_should_not_depose_it() { + let mut run = Run::single(); + let metadata_view = healthy_metadata_view(&run.sim); + *run.sim + .network + .link_extra_delay(ProcessId::Replica(LEADER), ProcessId::Replica(SLOW)) = + LAGGING_PEER_DELAY_TICKS; + + let changes = run.measure_view_changes(); + + assert_eq!( + (changes[0], healthy_metadata_view(&run.sim)), + (0, metadata_view), + "replica 2 heard replica 0 {LAGGING_PEER_DELAY_TICKS} ticks late, and replicas 0 and 1, \ + which hear it in time, followed replica 2 out of its view: (partition view changes, \ + metadata view)" + ); +} + +#[test] +fn given_a_backup_cut_off_for_many_view_change_timeouts_when_it_heals_should_not_change_the_view() { + let mut run = Run::single(); + let metadata_view = healthy_metadata_view(&run.sim); + let view = healthy_view(&run.sim, run.namespaces[0]); + set_peer_links(&mut run.sim, SLOW, false); + for _ in 0..PARTITION_TICKS { + run.tick(); } + set_peer_links(&mut run.sim, SLOW, true); + + let changes = run.measure_view_changes(); + + assert_eq!( + ( + healthy_view(&run.sim, run.namespaces[0]), + changes[0], + healthy_metadata_view(&run.sim) + ), + (view, 0, metadata_view), + "replica 2 was cut off for {PARTITION_TICKS} ticks, and after the heal replicas 0 and 1 \ + left the view of a primary they never stopped hearing: (partition view, partition view \ + changes after the heal, metadata view)" + ); +} + +#[test] +fn given_a_crashed_primary_should_elect_a_new_one_within_the_election_bound() { + let mut run = Run::single(); + run.sim.replica_crash(LEADER); + + assert!( + run.others_lead_within_election_bound(LEADER), + "replicas 1 and 2 did not elect a partition and a metadata primary within \ + {ELECTION_TICKS_MAX} ticks of the crash of replica 0" + ); +} + +#[test] +fn given_a_primary_cut_off_should_be_replaced_and_rejoin_as_a_backup() { + let mut run = Run::single(); + let namespace = run.namespaces[0]; + set_peer_links(&mut run.sim, LEADER, false); + assert!( - replies > 0, - "the workload must keep committing after the heal" + run.others_lead_within_election_bound(LEADER), + "replicas 1 and 2 did not elect a partition and a metadata primary within \ + {ELECTION_TICKS_MAX} ticks of cutting off replica 0" + ); + let views = run.views_of(LEADER + 1, namespace); + set_peer_links(&mut run.sim, LEADER, true); + for _ in 0..PARTITION_TICKS { + run.tick(); + } + + assert_eq!( + ( + run.views_of(LEADER + 1, namespace), + run.views_of(LEADER, namespace) + ), + (views, views), + "after the heal replica 0 must adopt the views replicas 1 and 2 elected without \ + another view change: ((partition, metadata) of replica 1, of replica 0)" ); - for (index, &namespace) in namespaces.iter().enumerate() { + let rejoined = run + .sim + .partition_consensus_state(usize::from(LEADER), namespace) + .is_some_and(|state| state.status == Status::Normal && !state.is_primary); + assert!(rejoined, "replica 0 must rejoin the partition as a backup"); +} + +/// The groups after the rebalancer spread them, with the workload that keeps +/// them committing. +struct Run { + sim: Simulator, + workload: Workload, + clients: [SimClient; 1], + invariants: Invariants, + namespaces: Vec, +} + +impl Run { + fn spread() -> Self { + Self::new(GROUPS, SPREAD_TICKS) + } + + /// One group that replica 0 leads, like the metadata plane. + fn single() -> Self { + let run = Self::new(1, SETTLE_TICKS); + assert_eq!( + ( + run.sim.primary_index(run.namespaces[0]), + run.sim.metadata_primary_index() + ), + (Some(LEADER), Some(LEADER)), + "replica 0 must lead the group and the metadata plane before the fault" + ); + run + } + + fn new(groups: usize, settle_ticks: u64) -> Self { + let (mut sim, client) = cluster(SEED); + let namespaces: Vec<_> = (0..groups) + .map(|partition| IggyNamespace::new(1, 1, partition)) + .collect(); + for &namespace in &namespaces { + sim.init_partition(namespace); + } + sim.register_client_with_primary(&client); + let mut options = WorkloadOptions::new(SEED, sim.replica_count, namespaces.clone()); + options.weights = ActionWeights::partition_only(); + let mut run = Self { + sim, + workload: Workload::new(options), + clients: [client], + invariants: Invariants::new(), + namespaces, + }; + for _ in 0..settle_ticks { + run.tick(); + } + run + } + + fn tick(&mut self) -> u64 { + workload::run( + &mut self.sim, + &mut self.workload, + &self.clients, + 1, + u64::MAX, + &mut self.invariants, + ) + } + + /// Runs `MEASURE_TICKS` under the fault already in place and returns how + /// often replicas 0 and 1 changed view in each group. + fn measure_view_changes(&mut self) -> Vec { + let mut views: Vec = self + .namespaces + .iter() + .map(|&namespace| healthy_view(&self.sim, namespace)) + .collect(); + let mut changes = vec![0_u32; self.namespaces.len()]; + let mut replies = 0; + for _ in 0..MEASURE_TICKS { + replies += self.tick(); + for (index, &namespace) in self.namespaces.iter().enumerate() { + let view = healthy_view(&self.sim, namespace); + if view != views[index] { + changes[index] += 1; + views[index] = view; + } + } + } assert!( - changes[index] <= VIEW_CHANGES_MAX, - "replicas 0 and 1 changed view {} times for {namespace:?} in {MEASURE_TICKS} ticks \ - (view {} -> {}), while only replica 2 was slow: one replica whose messages \ - arrive later than the view-change status timeout keeps the healthy pair leaving \ - its working view (changes per group {changes:?})", - changes[index], - first_views[index], - views[index], + replies > 0, + "the workload must keep committing under the fault" ); + changes + } + + /// Whether a replica other than `failed` becomes a `Normal` primary of the + /// group and of the metadata plane within `ELECTION_TICKS_MAX`. + fn others_lead_within_election_bound(&mut self, failed: u8) -> bool { + (0..ELECTION_TICKS_MAX).any(|_| { + self.tick(); + let mut others = (0..self.sim.replica_count).filter(|&replica| replica != failed); + let namespace = self.namespaces[0]; + others.clone().any(|replica| { + self.sim + .partition_consensus_state(usize::from(replica), namespace) + .is_some_and(|state| state.is_primary && state.status == Status::Normal) + }) && others.any(|replica| { + self.sim + .metadata_consensus(usize::from(replica)) + .is_some_and(|consensus| { + consensus.is_primary() && consensus.status() == Status::Normal + }) + }) + }) + } + + /// The partition and metadata views of `replica`. + fn views_of(&self, replica: u8, namespace: IggyNamespace) -> (Option, Option) { + ( + self.sim + .partition_consensus_state(usize::from(replica), namespace) + .map(|state| state.view), + self.sim + .metadata_consensus(usize::from(replica)) + .map(consensus::VsrConsensus::view), + ) } } -/// Cuts or restores every link between the slow replica and its peers. -fn set_peer_links(sim: &mut Simulator, enabled: bool) { - for peer in (0..sim.replica_count).filter(|&peer| peer != SLOW) { - sim.network - .set_link_filter(ProcessId::Replica(SLOW), ProcessId::Replica(peer), enabled); - sim.network - .set_link_filter(ProcessId::Replica(peer), ProcessId::Replica(SLOW), enabled); +/// Cuts or restores every link between `replica` and its peers. +fn set_peer_links(sim: &mut Simulator, replica: u8, enabled: bool) { + for peer in (0..sim.replica_count).filter(|&peer| peer != replica) { + sim.network.set_link_filter( + ProcessId::Replica(replica), + ProcessId::Replica(peer), + enabled, + ); + sim.network.set_link_filter( + ProcessId::Replica(peer), + ProcessId::Replica(replica), + enabled, + ); } } @@ -146,3 +533,13 @@ fn healthy_view(sim: &Simulator, namespace: IggyNamespace) -> u32 { .max() .unwrap_or(0) } + +/// The highest metadata view replicas 0 and 1 hold. +fn healthy_metadata_view(sim: &Simulator) -> u32 { + (0..sim.replica_count) + .filter(|&replica| replica != SLOW) + .filter_map(|replica| sim.metadata_consensus(usize::from(replica))) + .map(consensus::VsrConsensus::view) + .max() + .unwrap_or(0) +} diff --git a/core/simulator/src/stale_prepare_commit_tests.rs b/core/simulator/src/stale_prepare_commit_tests.rs index 3e92fb5213..dd8302d2f5 100644 --- a/core/simulator/src/stale_prepare_commit_tests.rs +++ b/core/simulator/src/stale_prepare_commit_tests.rs @@ -26,6 +26,7 @@ use bytes::Bytes; use consensus::{PartitionsHandle, Status}; use futures::FutureExt; +use iggy_binary_protocol::Command; use iggy_common::PollingStrategy; use partitions::{PollingArgs, PollingConsumer}; use server_common::send_messages::{BatchIntegrity, decode_batch_slice_with}; @@ -33,7 +34,7 @@ use server_common::sharding::IggyNamespace; use crate::Simulator; use crate::client::SimClient; -use crate::packet::ProcessId; +use crate::packet::{Packet, ProcessId}; use crate::partition_repair_driver_tests::{cluster, group_state}; const REPLICAS: u8 = 3; @@ -42,8 +43,6 @@ const STEPS_PER_SEND: usize = 12; /// Well past `NORMAL_HEARTBEAT_TICKS` (500), so the majority elects. const ELECTION_STEPS: usize = 5_000; const SETTLE_STEPS: usize = 5_000; -/// Long enough for a lone survivor's view changes to rotate onto the old primary. -const REJOIN_STEPS: usize = 40_000; fn send(sim: &mut Simulator, client: &SimClient, namespace: IggyNamespace, to: u8, payload: &str) { let request = client.send_messages(namespace, &[Bytes::from(payload.to_owned())]); @@ -152,13 +151,17 @@ fn given_an_old_primary_holding_an_uncommitted_prepare_when_it_rejoins_a_view_th // The new view commits different prepares at op 4 and beyond. for index in 0..4 { - send( - &mut sim, + let request = client + .send_messages(namespace, &[Bytes::from(format!("fresh-{index}"))]) + .into_generic(); + let reply = sim.await_setup_reply( &client, - namespace, new_primary, - &format!("fresh-{index}"), + &request, + iggy_binary_protocol::Operation::SendMessages, + "fresh view setup", ); + assert_eq!(reply.header().status, 0); } let (_, _, majority_commit, _) = group_state(&sim, new_primary, namespace); assert!( @@ -218,13 +221,17 @@ fn given_an_old_primary_holding_an_uncommitted_prepare_when_it_is_elected_over_a .expect("replicas 1 and 2 never elected a new view"); let new_primary = primary_of(&sim, 1, namespace, new_view); for index in 0..4 { - send( - &mut sim, + let request = client + .send_messages(namespace, &[Bytes::from(format!("fresh-{index}"))]) + .into_generic(); + let reply = sim.await_setup_reply( &client, - namespace, new_primary, - &format!("fresh-{index}"), + &request, + iggy_binary_protocol::Operation::SendMessages, + "fresh view setup", ); + assert_eq!(reply.header().status, 0); } let (_, _, majority_commit, _) = group_state(&sim, new_primary, namespace); assert!( @@ -232,19 +239,17 @@ fn given_an_old_primary_holding_an_uncommitted_prepare_when_it_is_elected_over_a "the new view committed only through {majority_commit}" ); - // Cut the new primary off and keep the old one apart until the survivor's view - // changes reach a view the old primary leads; then let those two meet. + // Cut the new primary off and let the old primary meet the survivor. The old + // primary's DoViewChange is lost on the way, so the view the survivor leads + // cannot start, and the next view, which the old primary leads, starts. let survivor = (0..REPLICAS) .find(|replica| *replica != OLD_PRIMARY && *replica != new_primary) .expect("three replicas"); isolate(&mut sim, new_primary, true); - (0..REJOIN_STEPS) - .find(|_| { - sim.step(); - let (_, view, _, _) = group_state(&sim, survivor, namespace); - primary_of(&sim, survivor, namespace, view) == OLD_PRIMARY - }) - .expect("the survivor never reached a view the old primary leads"); + *sim.network.link_drop_packet_fn( + ProcessId::Replica(OLD_PRIMARY), + ProcessId::Replica(survivor), + ) = Some(is_do_view_change); connect(&mut sim, OLD_PRIMARY, survivor); let led_view = (0..SETTLE_STEPS) .find_map(|_| { @@ -256,6 +261,10 @@ fn given_an_old_primary_holding_an_uncommitted_prepare_when_it_is_elected_over_a .then_some(view) }) .expect("the old primary never led a view: the primary-elect path did not run"); + *sim.network.link_drop_packet_fn( + ProcessId::Replica(OLD_PRIMARY), + ProcessId::Replica(survivor), + ) = None; isolate(&mut sim, new_primary, false); for _ in 0..SETTLE_STEPS { @@ -308,12 +317,18 @@ fn given_a_fenced_window_wider_than_a_repair_chunk_when_the_old_primary_rejoins_ .expect("replicas 1 and 2 never elected a new view"); let new_primary = primary_of(&sim, 1, namespace, new_view); for index in 0..12 { - send( - &mut sim, + let request = client.send_messages(namespace, &[Bytes::from(format!("fresh-{index}"))]); + let reply = sim.await_setup_reply( &client, - namespace, new_primary, - &format!("fresh-{index}"), + &request.into_generic(), + iggy_binary_protocol::Operation::SendMessages, + "fresh rejoin setup", + ); + assert_eq!( + reply.header().status, + 0, + "fresh operation {index} must commit before rejoining" ); } let (_, _, majority_commit, _) = group_state(&sim, new_primary, namespace); @@ -395,13 +410,17 @@ fn cut_to_catch_up(seed: u64) -> Option { .expect("replicas 1 and 2 never elected a new view"); let new_primary = primary_of(&sim, 1, namespace, new_view); for index in 0..6 { - send( - &mut sim, + let request = client + .send_messages(namespace, &[Bytes::from(format!("fresh-{index}"))]) + .into_generic(); + let reply = sim.await_setup_reply( &client, - namespace, new_primary, - &format!("fresh-{index}"), + &request, + iggy_binary_protocol::Operation::SendMessages, + "fresh view setup", ); + assert_eq!(reply.header().status, 0); } let (_, _, majority_commit, _) = group_state(&sim, new_primary, namespace); @@ -497,6 +516,10 @@ fn primary_of(sim: &Simulator, replica: u8, namespace: IggyNamespace, view: u32) .primary_index(view) } +fn is_do_view_change(packet: &Packet) -> bool { + packet.message.header().command == Command::DoViewChange +} + fn connect(sim: &mut Simulator, left: u8, right: u8) { sim.network .set_link_filter(ProcessId::Replica(left), ProcessId::Replica(right), true); diff --git a/core/simulator/src/workload/state_checker.rs b/core/simulator/src/workload/state_checker.rs index f134a9a2a2..b141471cbe 100644 --- a/core/simulator/src/workload/state_checker.rs +++ b/core/simulator/src/workload/state_checker.rs @@ -432,11 +432,17 @@ fn check_partition_owner( if header.operation == Operation::InstallConsumerGroupOwner { let install = InstallConsumerGroupOwnerRequest::decode_from(body) .expect("a committed owner installation decodes"); - if !owners + let retired = owners .retirement .as_ref() - .is_some_and(|catalog| catalog.retires(install.group_id)) - { + .is_some_and(|catalog| catalog.retires(install.group_id)); + // The partition keeps the higher generation, so a superseded install + // that commits after its successor changes no owner. + let superseded = owners + .installed + .get(&install.group_id) + .is_some_and(|installed| install.owner.generation <= installed.generation); + if !retired && !superseded { owners.installed.insert(install.group_id, install.owner); } return; @@ -908,6 +914,50 @@ mod tests { ); } + /// The partition keeps the higher generation when a superseded owner install + /// commits after its successor (`apply_consumer_group_owner` returns early), + /// so the successor still owns the group and its offset is fenced correctly. + #[test] + fn given_a_superseded_install_committed_after_its_successor_when_the_successor_commits_should_pass_the_oracle() + { + let superseded = ConsumerGroupOwner { + client_id: 0, + session: 0, + generation: 85, + }; + let successor = ConsumerGroupOwner { + client_id: 2, + session: 2, + generation: 88, + }; + let mut owners = PartitionOwners::default(); + for (op, owner) in [(19, successor), (24, superseded)] { + let install = InstallConsumerGroupOwnerRequest { + incarnation: 1, + group_id: 0, + owner, + metadata_op: owner.generation, + }; + check_partition_owner( + &mut owners, + &PrepareHeader { + operation: Operation::InstallConsumerGroupOwner, + op, + ..Default::default() + }, + &install.to_bytes(), + SEED, + ); + } + check_offset( + &mut owners, + successor.client_id, + successor.session, + successor.generation, + Operation::StoreConsumerOffset, + ); + } + fn installed_owner() -> PartitionOwners { PartitionOwners { installed: BTreeMap::from([( From 80fcae34d8767cd54c3a1b9d8ef8f0cb42657308 Mon Sep 17 00:00:00 2001 From: Hubert Gruszecki Date: Thu, 29 Oct 2026 13:46:27 +0100 Subject: [PATCH 02/10] test(consensus): expose produce stalls in replication flow control The primary stops sending prepares to a backup when its send window fills or while a progress probe is out, and resumes only on the next 10 ms consensus tick. The default simulated network reorders packets, so gap repair hides this stall. Add an ordered network mode, off by default, that delivers each link in send order like a TCP link. Two tests send one produce per tick and require each answer within its four hops plus one probe round trip. Both fail here and pass on c92d2bd90. --- core/simulator/src/lib.rs | 3 + core/simulator/src/packet.rs | 49 ++++++- core/simulator/src/replication_flow_tests.rs | 137 +++++++++++++++++++ 3 files changed, 184 insertions(+), 5 deletions(-) create mode 100644 core/simulator/src/replication_flow_tests.rs diff --git a/core/simulator/src/lib.rs b/core/simulator/src/lib.rs index 7d264052ee..75542be834 100644 --- a/core/simulator/src/lib.rs +++ b/core/simulator/src/lib.rs @@ -2226,6 +2226,9 @@ mod slow_replica_tests; #[cfg(test)] mod stale_prepare_commit_tests; +#[cfg(test)] +mod replication_flow_tests; + #[cfg(test)] mod tests { use super::partition_repair_driver_tests::{ diff --git a/core/simulator/src/packet.rs b/core/simulator/src/packet.rs index 0521dd0183..05e909e952 100644 --- a/core/simulator/src/packet.rs +++ b/core/simulator/src/packet.rs @@ -77,6 +77,9 @@ pub struct Packet { pub message: Message, /// Tick at which this packet becomes deliverable. pub ready_at: u64, + /// Send order. With [`PacketSimulatorOptions::ordered`], each link + /// delivers in this order. + pub seq: u64, } impl Clone for Packet { @@ -86,6 +89,7 @@ impl Clone for Packet { to: self.to, message: self.message.deep_copy(), ready_at: self.ready_at, + seq: self.seq, } } } @@ -131,6 +135,10 @@ pub struct PacketSimulatorOptions { pub client_count: u8, /// PRNG seed for deterministic behavior. pub seed: u64, + /// Deliver each link in send order, `one_way_delay_min` ticks plus the + /// link's extra delay after the send, and never evict: one TCP connection + /// on an idle network. `one_way_delay_mean` and `link_capacity` do not apply. + pub ordered: bool, } impl Default for PacketSimulatorOptions { @@ -152,6 +160,7 @@ impl Default for PacketSimulatorOptions { node_count: 1, client_count: 0, seed: 0, + ordered: false, } } } @@ -214,6 +223,20 @@ impl PacketSimulatorOptions { } } +/// The oldest packet once it is ready, so a later packet never passes an +/// earlier one, even when the link delay changes between the two sends. +fn remove_oldest_ready(packets: &mut ReadyQueue, tick: u64) -> Option { + let (index, oldest) = packets + .as_slice() + .iter() + .enumerate() + .min_by_key(|(_, packet)| packet.seq)?; + if oldest.ready_at > tick { + return None; + } + Some(packets.remove_at(index)) +} + /// Uniform draw over an enum's variants. /// /// Generic rather than a `match` on a drawn index: a match would silently keep @@ -337,6 +360,8 @@ pub struct PacketSimulator { /// protocol it exercised, which is the difference between covering a path and /// merely compiling it. command_counts: [u64; COMMAND_COUNT_MAX], + /// [`Packet::seq`] of the next submitted packet. + next_seq: u64, } /// One past the highest [`Command`] discriminant, sizing [`COMMAND_LABELS`] and @@ -477,6 +502,7 @@ impl PacketSimulator { auto_partition_nodes: (0..node_count).collect(), delivered: Vec::new(), command_counts: [0; COMMAND_COUNT_MAX], + next_seq: 0, } } @@ -533,11 +559,15 @@ impl PacketSimulator { /// /// Always enqueues the packet. Loss and replay checks happen at delivery /// time in `step()`. If the link is at capacity, a random existing packet - /// is evicted to make room. + /// is evicted to make room, unless the network is ordered. pub fn submit(&mut self, from: ProcessId, to: ProcessId, message: Message) { let idx = self.link_index(from, to); - let delay = Self::calculate_delay(&mut self.prng, &self.options) - .saturating_add(self.links[idx].extra_delay); + let delay = if self.options.ordered { + self.options.one_way_delay_min + } else { + Self::calculate_delay(&mut self.prng, &self.options) + } + .saturating_add(self.links[idx].extra_delay); let ready_at = self.current_tick.saturating_add(delay); let packet = Packet { @@ -545,10 +575,14 @@ impl PacketSimulator { to, message, ready_at, + seq: self.next_seq, }; + self.next_seq += 1; // If at capacity, evict a random existing packet - if self.links[idx].packets.len() >= self.options.link_capacity as usize { + if !self.options.ordered + && self.links[idx].packets.len() >= self.options.link_capacity as usize + { self.links[idx].packets.remove_random(&mut self.prng); tracing::trace!(?from, ?to, "evicted random packet (link at capacity)"); } @@ -720,7 +754,12 @@ impl PacketSimulator { } loop { - let Some(packet) = link.packets.remove_ready(prng, *current_tick) else { + let next = if options.ordered { + remove_oldest_ready(&mut link.packets, *current_tick) + } else { + link.packets.remove_ready(prng, *current_tick) + }; + let Some(packet) = next else { break; }; diff --git a/core/simulator/src/replication_flow_tests.rs b/core/simulator/src/replication_flow_tests.rs new file mode 100644 index 0000000000..9737cdaf89 --- /dev/null +++ b/core/simulator/src/replication_flow_tests.rs @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Flow control to a backup must not hold produces for a consensus tick. +//! +//! The primary stops sending prepares to a backup while a progress probe is +//! out, and when the bytes sent since the last probe reply reach +//! `REPLICATION_BATCH_BYTES_MAX`. A timer also probes each backup about once +//! per second. A stop that ends one probe round trip later costs little on a +//! real network. A stop that waits for the next tick, to send the probe or to +//! resume the prepares, costs up to 10 ms per wait. Steady produce repeats the +//! stops, so these waits show in the publish tail. +//! +//! The network is ordered, like the TCP link between two replicas, and each hop +//! takes one tick. A produce takes four hops: the request, the prepare, the +//! prepare ok and the reply. A probe round trip adds two hops. A produce that +//! takes longer waited for a tick. + +use bytes::Bytes; +use consensus::REPLICATION_BATCH_BYTES_MAX; +use server_common::sharding::IggyNamespace; + +use crate::Simulator; +use crate::client::SimClient; +use crate::packet::PacketSimulatorOptions; + +const SEED: u64 = 0x5245_504C; +const REPLICAS: u8 = 3; +/// Leads view 0. +const PRIMARY: u8 = 0; +const ONE_WAY_DELAY_TICKS: u64 = 1; +/// The request, the prepare, the prepare ok and the reply. +const PRODUCE_HOPS: u64 = 4; +/// A probe and its reply. +const PROBE_HOPS: u64 = 2; +const LATENCY_TICKS_MAX: u64 = (PRODUCE_HOPS + PROBE_HOPS) * ONE_WAY_DELAY_TICKS; +/// A client has one produce in flight, so one produce per tick needs more +/// clients than the slowest produce takes ticks. +const CLIENTS: u8 = 16; + +#[test] +fn given_steady_produce_when_a_backup_window_fills_should_answer_within_one_probe_round_trip() { + const REQUEST_BYTES: u64 = 512 * 1024; + /// Bytes sent to each backup, counted in windows. + const WINDOWS: u64 = 3; + assert_produce_latency( + REQUEST_BYTES, + WINDOWS * REPLICATION_BATCH_BYTES_MAX / REQUEST_BYTES, + ); +} + +#[test] +fn given_steady_produce_when_the_progress_timer_probes_a_backup_should_answer_within_one_probe_round_trip() + { + /// Never fills a window, so only the timer sends probes. + const REQUEST_BYTES: u64 = 1024; + /// A backup is probed 1 s (`PROGRESS_INTERVAL_MICROS`) after the first + /// prepare that follows the last probe reply: 100 ticks of 10 ms. + const PROBE_INTERVAL_TICKS: u64 = 100; + /// Four intervals, so the timer probes each backup more than once. + const TICKS: u64 = 4 * PROBE_INTERVAL_TICKS; + assert_produce_latency(REQUEST_BYTES, TICKS); +} + +/// Sends one produce per tick for `ticks` ticks, like a producer at a fixed +/// rate, and asserts that each one is answered within [`LATENCY_TICKS_MAX`]. +fn assert_produce_latency(request_bytes: u64, ticks: u64) { + server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { + enabled: false, + size: iggy_common::IggyByteSize::from(0u64), + bucket_capacity: 1, + }); + let network = PacketSimulatorOptions { + node_count: REPLICAS, + client_count: CLIENTS, + seed: SEED, + one_way_delay_min: ONE_WAY_DELAY_TICKS, + ordered: true, + ..PacketSimulatorOptions::default() + }; + let client_ids = 1..=u128::from(CLIENTS); + let mut sim = Simulator::new(usize::from(REPLICAS), client_ids.clone(), network); + let namespace = IggyNamespace::new(1, 1, 0); + sim.init_partition(namespace); + let clients: Vec = client_ids.map(SimClient::new).collect(); + for client in &clients { + sim.register_client_with_primary(client); + } + + let payload = Bytes::from(vec![0; usize::try_from(request_bytes).unwrap()]); + let mut sent_at: Vec> = vec![None; clients.len()]; + let mut late = Vec::new(); + // The produces sent last get the whole bound to finish. + for tick in 0..ticks + LATENCY_TICKS_MAX { + if tick < ticks + && let Some(idle) = sent_at.iter().position(Option::is_none) + { + let request = clients[idle].send_messages(namespace, std::slice::from_ref(&payload)); + sim.submit_request(clients[idle].client_id(), PRIMARY, request.into_generic()); + sent_at[idle] = Some(tick); + } + for (_, reply) in sim.step_with_senders() { + let header = reply.header(); + assert_eq!(header.status, 0, "produce failed at tick {tick}"); + let client = clients + .iter() + .position(|client| client.client_id() == header.client) + .expect("reply to a test client"); + let sent = sent_at[client].take().expect("one reply per produce"); + if tick - sent > LATENCY_TICKS_MAX { + late.push((sent, Some(tick - sent))); + } + } + } + late.extend(sent_at.into_iter().flatten().map(|sent| (sent, None))); + late.sort_unstable(); + assert!( + late.is_empty(), + "every produce must be answered within {LATENCY_TICKS_MAX} ticks: {PRODUCE_HOPS} hops \ + and one probe round trip of {PROBE_HOPS}, so a slower one waited for a timer tick. \ + (tick sent, ticks to the reply or None without one): {late:?}" + ); +} From 448f46fcb4e2f011ce913b24d9e293445d0d3347 Mon Sep 17 00:00:00 2001 From: Hubert Gruszecki Date: Thu, 29 Oct 2026 17:12:05 +0100 Subject: [PATCH 03/10] fix(consensus): stop replication flow control from stalling produces The window to each backup was stop-and-wait: sends paused while a progress probe was out and resumed on the next 10 ms tick. Both backup windows filled at the same op, so fault-free produce p99 rose from 2.25 to 11.34 ms. The flow now pipelines credit. A checkpoint probe follows the frame that fills half the window on the same FIFO link. Its reply credits only the bytes sent before it, so no send waits. Replies carry a new held_op, taken from reserved header bytes, so an unflushed persisted WAL no longer reads as a gap. A retransmit no longer starts a replay. Counters for refused and replayed prepares stay at zero while backups keep up, and the simulator tests now require that. --- core/binary_protocol/src/consensus/header.rs | 20 +- core/consensus/src/lib.rs | 5 +- core/consensus/src/plane_helpers.rs | 26 +- core/consensus/src/replication.rs | 950 +++++++++++++++---- core/partitions/src/iggy_partition.rs | 38 +- core/shard/src/lib.rs | 141 +-- core/shard/src/metrics.rs | 40 + core/simulator/src/replication_flow_tests.rs | 57 +- 8 files changed, 986 insertions(+), 291 deletions(-) diff --git a/core/binary_protocol/src/consensus/header.rs b/core/binary_protocol/src/consensus/header.rs index 7f4fee0ecf..81dc625db3 100644 --- a/core/binary_protocol/src/consensus/header.rs +++ b/core/binary_protocol/src/consensus/header.rs @@ -1567,12 +1567,17 @@ impl ConsensusHeader for CommitReplyHeader { } } -/// Correlates a replication batch with the backup's contiguous acknowledged prefix. +/// Correlates a replication checkpoint with the backup's contiguous prefixes. /// -/// Progress releases transport credit and never counts toward a commit quorum. -/// Replies echo the nonce and observed op; `ready` certifies the received prefix. -/// Probes advertise the primary's applied frontier for retention fallback. -/// Probes must stay ordered with prepares on the sender's queue. +/// Probes must stay ordered with prepares on the sender's queue, so a reply +/// proves that the backup consumed every frame sent before its probe. That FIFO +/// position, not an op, releases transport credit. Progress never counts toward +/// a commit quorum. +/// Replies echo the nonce and observed op. `held_op` is the end of the prefix +/// that the backup holds with proof in the current view, so the primary finds +/// replication gaps from it. `received_op <= held_op` is the part of that +/// prefix that meets the prepare acknowledgment rules, and `ready` certifies it. Probes advertise the primary's applied frontier for retention +/// fallback. /// Nodes using this message require a coordinated cluster upgrade. #[repr(C)] #[derive(Debug, Clone, Copy, CheckedBitPattern, NoUninit)] @@ -1591,17 +1596,18 @@ pub struct PrepareProgressHeader { pub partition_incarnation: u64, pub observed_op: u64, pub received_op: u64, + pub held_op: u64, pub commit_op: u64, pub kind: u8, pub ready: u8, - pub reserved: [u8; 78], + pub reserved: [u8; 70], } const _: () = { assert!(size_of::() == HEADER_SIZE); assert!(offset_of!(PrepareProgressHeader, nonce) == 128); assert!(offset_of!(PrepareProgressHeader, partition_incarnation) == 144); - assert!(offset_of!(PrepareProgressHeader, reserved) + size_of::<[u8; 78]>() == HEADER_SIZE); + assert!(offset_of!(PrepareProgressHeader, reserved) + size_of::<[u8; 70]>() == HEADER_SIZE); }; impl ConsensusHeader for PrepareProgressHeader { diff --git a/core/consensus/src/lib.rs b/core/consensus/src/lib.rs index ecc5080caa..6df0a8d203 100644 --- a/core/consensus/src/lib.rs +++ b/core/consensus/src/lib.rs @@ -199,7 +199,10 @@ pub use impls::*; mod plane_mux; pub use plane_mux::*; mod replication; -pub use replication::{REPLICATION_BATCH_BYTES_MAX, ReplicationProgress, ReplicationWork}; +pub use replication::{ + BackupPrefix, REPLICATION_BATCH_BYTES_MAX, ReplicationProgress, ReplicationSend, + ReplicationStats, ReplicationWork, +}; mod plane_helpers; pub use plane_helpers::*; mod metadata_helpers; diff --git a/core/consensus/src/plane_helpers.rs b/core/consensus/src/plane_helpers.rs index f5081e1c6b..19a1051a68 100644 --- a/core/consensus/src/plane_helpers.rs +++ b/core/consensus/src/plane_helpers.rs @@ -17,7 +17,7 @@ use crate::{ Consensus, DvcHeaderKind, IgnoreReason, MergedLog, Pipeline, PipelineEntry, PlaneKind, - PrepareOkOutcome, Sequencer, Status, VsrConsensus, dvc_header_kind, + PrepareOkOutcome, ReplicationSend, Sequencer, Status, VsrConsensus, dvc_header_kind, }; use iggy_binary_protocol::consensus::{REJECTION_SECTION_LEN, write_rejection_section}; use iggy_binary_protocol::{ @@ -176,32 +176,18 @@ where B: MessageBus, P: Pipeline, { - let (op, size) = { - let header = frozen_prepare_header(&message)?; - (header.op, u64::from(header.size)) - }; + let op = frozen_prepare_header(&message)?.op; let mut first_error = None; for replica in 0..consensus.replica_count() { if replica == consensus.replica() { continue; } - if !consensus.admit_replication(replica, op, size, false) { - continue; - } - match consensus - .message_bus() - .send_to_replica(replica, message.clone()) + if let Err(error) = consensus + .send_replicated_prepare(replica, op, message.clone(), ReplicationSend::Live) .await + && first_error.is_none() { - Ok(()) => { - consensus.record_replication_send(replica, op, size, false); - } - Err(error) => { - consensus.record_replication_failure(replica, op); - if first_error.is_none() { - first_error = Some(error); - } - } + first_error = Some(error); } } first_error.map_or(Ok(()), |error| Err(error.into())) diff --git a/core/consensus/src/replication.rs b/core/consensus/src/replication.rs index e499c3fed6..45bd02f20c 100644 --- a/core/consensus/src/replication.rs +++ b/core/consensus/src/replication.rs @@ -16,32 +16,69 @@ // under the License. use crate::{Consensus, Pipeline, PipelineEntry, Sequencer, VsrConsensus}; -use iggy_binary_protocol::PrepareProgressHeader; -use message_bus::MessageBus; +use iggy_binary_protocol::{Command, ConsensusHeader, HEADER_SIZE, PrepareProgressHeader}; +use message_bus::{MessageBus, SendError}; +use server_common::{MESSAGE_ALIGN, Message, iobuf::Frozen}; +use std::mem::take; pub const REPLICATION_BATCH_BYTES_MAX: u64 = 32 * 1024 * 1024; +/// A checkpoint at half the window returns its credit before the window fills, +/// as long as the backup consumes half a window within one round trip. +const CHECKPOINT_BYTES: u64 = REPLICATION_BATCH_BYTES_MAX / 2; const PROGRESS_INTERVAL_MICROS: u64 = 1_000_000; +/// Probes and their replies are header-only frames. +#[allow(clippy::cast_possible_truncation)] +const PROGRESS_FRAME_BYTES: u32 = HEADER_SIZE as u32; -/// A probe follows every prepare in its batch on the same FIFO link. No new -/// prepares are sent until its reply, so a reply also drains the batch's credit. -/// Payloads remain in the journal, never in this per-peer state. +/// Bounds the prepare bytes in flight to each backup by the credit that +/// checkpoints return. +/// +/// A checkpoint is a progress probe on the same FIFO link as the prepares, so +/// its reply proves that the backup consumed every frame sent before it. The +/// reply credits only that prefix, and no send waits for it. Payloads remain in +/// the journal, never in this per-peer state. #[derive(Debug)] pub struct ReplicationFlow { epoch: Option<(u32, u64)>, nonce: u64, peers: Vec, + stats: ReplicationStats, } #[derive(Debug, Clone, Copy, Default)] struct PeerReplication { + /// Every copy sent in this epoch, retransmits included. + sent_bytes: u64, + /// The prefix of `sent_bytes` that a checkpoint reply proved consumed. + credited_bytes: u64, sent_through: u64, - bytes: u64, + /// The next op of the ordered resend. Live sends wait until it reaches them. replay_from: Option, - probe: Option<(u64, u64)>, + checkpoint: Option, + /// The backup drops or cannot acknowledge prepares, so nothing is sent. + busy: bool, + /// When the tick sends a new checkpoint or resends the outstanding one. due: u64, + /// Replay waits until then after history was unavailable. retry_at: u64, } +impl PeerReplication { + const fn outstanding(&self) -> u64 { + self.sent_bytes.saturating_sub(self.credited_bytes) + } +} + +#[derive(Debug, Clone, Copy)] +struct Checkpoint { + nonce: u64, + through: u64, + bytes: u64, + /// A gap at or above the cursor of issue time was still waiting for replay, + /// so the reply cannot move the cursor back to it. + replay_from: Option, +} + #[derive(Debug, Clone, Copy)] pub enum ReplicationWork { Probe { @@ -61,12 +98,43 @@ pub enum ReplicationProgress { Ready { received_op: u64 }, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReplicationSend { + /// A new op from the primary's prepare path. + Live, + /// An ordered resend from the replay cursor. + Replay, + /// A prepare-timeout resend of one op. It never moves the cursor, so a + /// healthy backup gets no replay. + Retransmit, +} + +/// Replication events since the previous [`VsrConsensus::take_replication_stats`]. +/// Both stay at zero while every backup keeps up. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct ReplicationStats { + /// Live prepares that the window, the replay cursor or a busy backup refused. + pub live_refusals: u64, + /// Prepares sent by replay. + pub replayed: u64, +} + +/// A backup's answer to a checkpoint. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct BackupPrefix { + /// Every op through it is in the journal with proof in the current view. + pub held_op: u64, + /// The part of the held prefix that also meets the `PrepareOk` rules. + pub received_op: u64, +} + impl ReplicationFlow { pub fn new(replica_count: u8) -> Self { Self { epoch: None, nonce: 0, peers: vec![PeerReplication::default(); usize::from(replica_count)], + stats: ReplicationStats::default(), } } @@ -77,33 +145,53 @@ impl ReplicationFlow { } } - fn admit(&mut self, replica: u8, op: u64, size: u64, replay: bool) -> bool { + fn admit(&mut self, replica: u8, op: u64, size: u64, send: ReplicationSend) -> bool { let peer = &mut self.peers[usize::from(replica)]; - if peer.probe.is_some() { - return false; + let outstanding = peer.outstanding(); + // One legal frame larger than the window must still make progress. + let window_full = + outstanding > 0 && outstanding.saturating_add(size) > REPLICATION_BATCH_BYTES_MAX; + let out_of_order = match send { + ReplicationSend::Live => peer.replay_from.is_some_and(|from| from != op), + ReplicationSend::Replay => peer.replay_from != Some(op), + ReplicationSend::Retransmit => false, + }; + if !peer.busy && !window_full && !out_of_order { + return true; } - if !replay && peer.replay_from.is_some_and(|from| from != op) { - return false; + if window_full && peer.checkpoint.is_none() { + peer.due = 0; } - // One legal frame larger than the window must still make progress. - if peer.bytes > 0 && peer.bytes.saturating_add(size) > REPLICATION_BATCH_BYTES_MAX { + if send == ReplicationSend::Live { peer.replay_from.get_or_insert(op); - peer.due = 0; - return false; + self.stats.live_refusals += 1; } - true + false } - fn sent(&mut self, replica: u8, op: u64, size: u64, replay: bool, now: u64) { + /// Returns the checkpoint that must follow this frame on its link. + fn sent( + &mut self, + replica: u8, + op: u64, + size: u64, + send: ReplicationSend, + now: u64, + ) -> Option { let peer = &mut self.peers[usize::from(replica)]; - if peer.bytes == 0 { + if peer.outstanding() == 0 && peer.checkpoint.is_none() { peer.due = now.saturating_add(PROGRESS_INTERVAL_MICROS); } - peer.bytes = peer.bytes.saturating_add(size); + peer.sent_bytes = peer.sent_bytes.saturating_add(size); peer.sent_through = peer.sent_through.max(op); - if replay || peer.replay_from.is_some() { + if send != ReplicationSend::Retransmit && peer.replay_from == Some(op) { peer.replay_from = Some(op.saturating_add(1)); } + if send == ReplicationSend::Replay { + self.stats.replayed += 1; + } + let due = peer.checkpoint.is_none() && peer.outstanding() >= CHECKPOINT_BYTES; + due.then(|| self.checkpoint(replica, now)) } fn failed(&mut self, replica: u8, op: u64) { @@ -112,58 +200,104 @@ impl ReplicationFlow { peer.due = 0; } + /// A checkpoint counts only once its frame is queued. Dropping one that the + /// peer queue refused lets the next send issue a new one, instead of the + /// window waiting for the interval to end. + fn probe_failed(&mut self, replica: u8, nonce: u64) { + let peer = &mut self.peers[usize::from(replica)]; + if peer + .checkpoint + .is_some_and(|checkpoint| checkpoint.nonce == nonce) + { + peer.checkpoint = None; + } + } + + fn checkpoint(&mut self, replica: u8, now: u64) -> ReplicationWork { + self.nonce = self.nonce.wrapping_add(1).max(1); + let peer = &mut self.peers[usize::from(replica)]; + peer.checkpoint = Some(Checkpoint { + nonce: self.nonce, + through: peer.sent_through, + bytes: peer.sent_bytes, + replay_from: peer.replay_from, + }); + peer.due = now.saturating_add(PROGRESS_INTERVAL_MICROS); + ReplicationWork::Probe { + replica, + nonce: self.nonce, + through: peer.sent_through, + } + } + fn work(&mut self, replica: u8, head: u64, now: u64) -> Option { let peer = &mut self.peers[usize::from(replica)]; - if now < peer.retry_at && peer.bytes == 0 { + let backoff = now < peer.retry_at; + if backoff && peer.outstanding() == 0 { return None; } - if let Some((nonce, through)) = peer.probe { - if now < peer.due { - return None; - } - peer.due = now.saturating_add(PROGRESS_INTERVAL_MICROS); - return Some(ReplicationWork::Probe { - replica, - nonce, - through, - }); - } if peer.replay_from.is_some_and(|from| from > head) { peer.replay_from = None; } - if now >= peer.due && (peer.bytes > 0 || peer.replay_from.is_some()) { - self.nonce = self.nonce.wrapping_add(1).max(1); - let through = peer.sent_through; - peer.probe = Some((self.nonce, through)); - peer.due = now.saturating_add(PROGRESS_INTERVAL_MICROS); - return Some(ReplicationWork::Probe { - replica, - nonce: self.nonce, - through, - }); + if now >= peer.due { + if let Some(checkpoint) = peer.checkpoint { + // A lost checkpoint goes out again with its nonce and prefix. + peer.due = now.saturating_add(PROGRESS_INTERVAL_MICROS); + return Some(ReplicationWork::Probe { + replica, + nonce: checkpoint.nonce, + through: checkpoint.through, + }); + } + if peer.outstanding() > 0 || peer.replay_from.is_some() || peer.busy { + return Some(self.checkpoint(replica, now)); + } + } + if backoff || peer.busy { + return None; } peer.replay_from .map(|from_op| ReplicationWork::Replay { replica, from_op }) } - fn reply(&mut self, header: &PrepareProgressHeader, head: u64) -> Option { + fn reply( + &mut self, + header: &PrepareProgressHeader, + head: u64, + now: u64, + ) -> Option { let peer = &mut self.peers[usize::from(header.replica)]; - if peer.probe != Some((header.nonce, header.observed_op)) { + let checkpoint = peer.checkpoint?; + let ready = header.ready != 0; + if (checkpoint.nonce, checkpoint.through) != (header.nonce, header.observed_op) + || (ready && header.received_op > header.held_op) + { return None; } - if header.ready == 0 { - // Readiness can change before the retry interval for a lost probe expires. + peer.checkpoint = None; + // The reply proves FIFO consumption, not readiness, so Busy credits too. + peer.credited_bytes = peer.credited_bytes.max(checkpoint.bytes); + if !ready { + peer.busy = true; + let next = peer.sent_through.saturating_add(1); + peer.replay_from.get_or_insert(next); + // Readiness can change before the interval of a lost probe ends. peer.due = 0; return Some(ReplicationProgress::Busy); } - peer.probe = None; - peer.bytes = 0; + peer.busy = false; peer.retry_at = 0; - peer.sent_through = header.received_op.min(head); - peer.replay_from = (peer.sent_through < head).then(|| peer.sent_through + 1); - peer.due = u64::MAX; + peer.due = now.saturating_add(PROGRESS_INTERVAL_MICROS); + // A newer head alone never starts a replay. + let held_op = header.held_op.min(head); + if held_op < checkpoint.through { + let gap = held_op + 1; + if checkpoint.replay_from.is_none_or(|from| gap < from) { + peer.replay_from = Some(peer.replay_from.map_or(gap, |from| from.min(gap))); + } + } Some(ReplicationProgress::Ready { - received_op: peer.sent_through, + received_op: header.received_op.min(head), }) } } @@ -173,23 +307,69 @@ where B: MessageBus, P: Pipeline, { - pub fn admit_replication(&self, replica: u8, op: u64, size: u64, replay: bool) -> bool { + /// Send one prepare copy to a backup under its replication window. Once + /// half the window is outstanding, a checkpoint follows the copy. + /// + /// Returns `Ok(false)` when the flow refuses the copy. + /// + /// # Errors + /// + /// Returns the bus error when the copy cannot be enqueued. + #[allow(clippy::future_not_send)] + pub async fn send_replicated_prepare( + &self, + replica: u8, + op: u64, + prepare: Frozen, + send: ReplicationSend, + ) -> Result { + let size = prepare.len() as u64; + if !self.admit_replication(replica, op, size, send) { + return Ok(false); + } + if let Err(error) = self.message_bus().send_to_replica(replica, prepare).await { + self.record_replication_failure(replica, op); + return Err(error); + } + if let Some(ReplicationWork::Probe { nonce, through, .. }) = + self.record_replication_send(replica, op, size, send) + { + self.send_replication_probe(replica, nonce, through).await; + } + Ok(true) + } + + pub fn admit_replication( + &self, + replica: u8, + op: u64, + size: u64, + send: ReplicationSend, + ) -> bool { if !self.is_acting_primary() || self.has_observed_newer_view() || self.is_transferring() { return false; } let mut flow = self.replication.borrow_mut(); flow.in_epoch(self.view(), self.partition_incarnation()); - flow.admit(replica, op, size, replay) + flow.admit(replica, op, size, send) } - pub fn record_replication_send(&self, replica: u8, op: u64, size: u64, replay: bool) { + /// Returns the checkpoint that must follow the sent copy. + #[must_use] + pub fn record_replication_send( + &self, + replica: u8, + op: u64, + size: u64, + send: ReplicationSend, + ) -> Option { self.replication.borrow_mut().sent( replica, op, size, - replay, + send, self.message_bus().monotonic_micros(), - ); + ) } pub fn record_replication_failure(&self, replica: u8, op: u64) { @@ -223,6 +403,36 @@ where ) } + /// Send the probe of a [`ReplicationWork::Probe`] behind every frame + /// already queued for `replica`. + #[allow(clippy::future_not_send)] + pub async fn send_replication_probe(&self, replica: u8, nonce: u64, through: u64) { + let probe = Message::::new(HEADER_SIZE).transmute_header( + |_, header: &mut PrepareProgressHeader| { + header.command = Command::PrepareProgress; + header.size = PROGRESS_FRAME_BYTES; + header.cluster = self.cluster(); + header.view = self.view(); + header.replica = self.replica(); + header.group = self.group(); + header.partition_incarnation = self.partition_incarnation(); + header.nonce = nonce; + header.observed_op = through; + header.commit_op = self.commit_min(); + header.kind = PrepareProgressHeader::PROBE; + header.seal(); + }, + ); + if self + .message_bus() + .send_to_replica(replica, probe.into_generic().into_frozen()) + .await + .is_err() + { + self.replication.borrow_mut().probe_failed(replica, nonce); + } + } + pub fn handle_prepare_progress( &self, header: &PrepareProgressHeader, @@ -242,183 +452,381 @@ where } let mut flow = self.replication.borrow_mut(); flow.in_epoch(self.view(), self.partition_incarnation()); - let progress = flow.reply(header, self.sequencer().current_sequence())?; + let progress = flow.reply( + header, + self.sequencer().current_sequence(), + self.message_bus().monotonic_micros(), + )?; if let ReplicationProgress::Ready { received_op } = progress { self.note_backup_ack(header.replica, received_op); } Some(progress) } + + pub fn take_replication_stats(&self) -> ReplicationStats { + take(&mut self.replication.borrow_mut().stats) + } } #[cfg(test)] mod tests { use super::{ - PROGRESS_INTERVAL_MICROS, REPLICATION_BATCH_BYTES_MAX, ReplicationFlow, ReplicationWork, + CHECKPOINT_BYTES, PROGRESS_INTERVAL_MICROS, REPLICATION_BATCH_BYTES_MAX, ReplicationFlow, + ReplicationProgress, ReplicationSend, ReplicationStats, ReplicationWork, }; use crate::{LocalPipeline, Sequencer, VsrConsensus, impls::test_bus::NoopBus}; use iggy_binary_protocol::{Command, ConsensusHeader, HEADER_SIZE, PrepareProgressHeader}; use server_common::Message; + use std::collections::VecDeque; + + const MIB: u64 = 1024 * 1024; - fn reply(nonce: u64, observed_op: u64, received_op: u64) -> Message { + fn ready( + replica: u8, + nonce: u64, + observed_op: u64, + held_op: u64, + received_op: u64, + ) -> Message { Message::::new(HEADER_SIZE).transmute_header( |_, header: &mut PrepareProgressHeader| { header.command = Command::PrepareProgress; header.cluster = 1; header.size = u32::try_from(HEADER_SIZE).unwrap(); - header.replica = 1; + header.replica = replica; header.kind = PrepareProgressHeader::REPLY; header.ready = 1; header.nonce = nonce; header.observed_op = observed_op; + header.held_op = held_op; header.received_op = received_op; header.seal(); }, ) } + fn busy(replica: u8, nonce: u64, observed_op: u64) -> Message { + ready(replica, nonce, observed_op, 0, 0).transmute_header( + |old, header: &mut PrepareProgressHeader| { + *header = old; + header.ready = 0; + }, + ) + } + + /// Sends a copy the flow must admit, and returns the checkpoint behind it. + fn send( + flow: &mut ReplicationFlow, + replica: u8, + op: u64, + size: u64, + send: ReplicationSend, + now: u64, + ) -> Option { + assert!( + flow.admit(replica, op, size, send), + "{send:?} op {op} to replica {replica} was refused" + ); + flow.sent(replica, op, size, send, now) + } + + /// Replica 2 answers like a persisted topic whose WAL write of the last op + /// is not durable yet. Replies arrive two ops after their checkpoint, so + /// each checkpoint is out while the stream continues. #[test] - fn given_a_full_replay_batch_when_deferred_should_drain_credit_before_history_backoff() { - let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); - consensus.init(); - consensus.sequencer().set_sequence(2); - assert!(consensus.admit_replication(1, 1, REPLICATION_BATCH_BYTES_MAX, true)); - consensus.record_replication_send(1, 1, REPLICATION_BATCH_BYTES_MAX, true); - assert!(!consensus.admit_replication(1, 2, 1, true)); - consensus.defer_replication(1); - let Some(ReplicationWork::Probe { nonce, through, .. }) = consensus.replication_work(1) + fn given_healthy_backups_when_the_stream_passes_many_windows_should_never_refuse_or_replay() { + const SIZE: u64 = MIB; + const OPS: u64 = 10 * REPLICATION_BATCH_BYTES_MAX / SIZE; + const REPLY_DELAY_OPS: u64 = 2; + const OP_INTERVAL_MICROS: u64 = 10_000; + let mut flow = ReplicationFlow::new(3); + flow.in_epoch(0, 1); + let mut replies = VecDeque::new(); + for op in 1..=OPS { + let now = op * OP_INTERVAL_MICROS; + for replica in 1..3 { + let checkpoint = send(&mut flow, replica, op, SIZE, ReplicationSend::Live, now); + for work in [checkpoint, flow.work(replica, op, now)] + .into_iter() + .flatten() + { + let ReplicationWork::Probe { nonce, through, .. } = work else { + panic!("replica {replica} replays at op {op}: {work:?}"); + }; + replies.push_back((op + REPLY_DELAY_OPS, replica, nonce, through)); + } + } + while let Some(&(at, replica, nonce, through)) = replies.front() + && at <= op + { + replies.pop_front(); + let received = if replica == 2 { through - 1 } else { through }; + let reply = ready(replica, nonce, through, through, received); + assert!(matches!( + flow.reply(reply.header(), op, now), + Some(ReplicationProgress::Ready { .. }) + )); + } + } + assert_eq!(flow.stats, ReplicationStats::default()); + } + + #[test] + fn given_bytes_sent_after_a_checkpoint_when_its_reply_arrives_should_credit_only_that_prefix() { + const SIZE: u64 = CHECKPOINT_BYTES / 2; + let mut flow = ReplicationFlow::new(3); + flow.in_epoch(0, 1); + assert!(send(&mut flow, 1, 1, SIZE, ReplicationSend::Live, 0).is_none()); + let Some(ReplicationWork::Probe { nonce, through, .. }) = + send(&mut flow, 1, 2, SIZE, ReplicationSend::Live, 0) else { - panic!("a full batch must drain without the unavailable-history delay"); + panic!("half a window outstanding must send a checkpoint"); }; - assert!( - consensus - .handle_prepare_progress(reply(nonce, through, 1).header()) - .is_some() - ); + assert!(send(&mut flow, 1, 3, SIZE, ReplicationSend::Live, 0).is_none()); + assert!(send(&mut flow, 1, 4, SIZE, ReplicationSend::Live, 0).is_none()); + assert!(!flow.admit(1, 4, 1, ReplicationSend::Retransmit)); assert!( matches!( - consensus.replication_work(1), - Some(ReplicationWork::Replay { from_op: 2, .. }) + flow.work(1, 4, PROGRESS_INTERVAL_MICROS), + Some(ReplicationWork::Probe { nonce: resent, through: 2, .. }) if resent == nonce ), - "a drained batch must resume replay without the stale history delay" + "a lost checkpoint goes out again with its nonce and prefix" ); - - assert!(consensus.admit_replication(2, 1, 1, true)); - consensus.record_replication_failure(2, 1); - consensus.defer_replication(2); + for stale in [ + ready(1, nonce + 1, through, through, through), + ready(1, nonce, through + 1, through, through), + ready(1, nonce, through, through - 1, through), + ] { + assert_eq!(flow.reply(stale.header(), 4, 0), None); + } + let reply = ready(1, nonce, through, through, through); + assert!(flow.reply(reply.header(), 4, 0).is_some()); + assert_eq!( + flow.reply(reply.header(), 4, 0), + None, + "a duplicate reply credits nothing" + ); + assert!(flow.admit(1, 4, CHECKPOINT_BYTES, ReplicationSend::Retransmit)); assert!( - consensus.replication_work(2).is_none(), - "unavailable history without outstanding bytes must still back off" + !flow.admit(1, 4, CHECKPOINT_BYTES + 1, ReplicationSend::Retransmit), + "bytes sent after the checkpoint stay charged" ); } #[test] - fn given_a_full_peer_window_when_its_probe_is_delayed_should_bound_bytes_and_keep_other_peers_sending() - { + fn given_a_checkpoint_the_peer_queue_refused_when_a_copy_follows_should_send_a_new_one() { let mut flow = ReplicationFlow::new(3); flow.in_epoch(0, 1); - assert!(flow.admit(1, 1, REPLICATION_BATCH_BYTES_MAX, false)); - flow.sent(1, 1, REPLICATION_BATCH_BYTES_MAX, false, 0); - assert!(!flow.admit(1, 2, 1, false)); - let Some(ReplicationWork::Probe { nonce, through, .. }) = flow.work(1, 2, 0) else { - panic!("full window must probe"); + let Some(ReplicationWork::Probe { nonce: refused, .. }) = + send(&mut flow, 1, 1, CHECKPOINT_BYTES, ReplicationSend::Live, 0) + else { + panic!("half a window outstanding must send a checkpoint"); }; - assert!(!flow.admit(1, 2, 1, true)); - assert!( - flow.admit(2, 2, 1, false), - "a delayed peer must not stop another peer" - ); + flow.probe_failed(1, refused); + let Some(ReplicationWork::Probe { + nonce, through: 2, .. + }) = send(&mut flow, 1, 2, 1, ReplicationSend::Live, 0) + else { + panic!("the next copy must carry a new checkpoint, not wait for the interval"); + }; + flow.probe_failed(1, refused); + assert_eq!(flow.reply(ready(1, refused, 1, 1, 1).header(), 2, 0), None); assert!( - matches!(flow.work(1, 3, PROGRESS_INTERVAL_MICROS), Some(ReplicationWork::Probe { nonce: repeated, through: 1, .. }) if repeated == nonce) + flow.reply(ready(1, nonce, 2, 2, 2).header(), 2, 0) + .is_some(), + "a failure of an older checkpoint must keep the new one" ); - let _ = flow.reply(reply(nonce, through, 0).header(), 3); - assert!(matches!( - flow.work(1, 3, PROGRESS_INTERVAL_MICROS), - Some(ReplicationWork::Replay { from_op: 1, .. }) - )); + assert_eq!(flow.peers[1].outstanding(), 0); + } + + #[test] + fn given_a_lost_prepare_when_a_checkpoint_finds_the_gap_should_replay_it_before_live_sends() { + const SIZE: u64 = 100; + let mut flow = ReplicationFlow::new(3); + flow.in_epoch(0, 1); + for op in 1..=3 { + assert!(send(&mut flow, 1, op, SIZE, ReplicationSend::Live, 0).is_none()); + } + let mut now = PROGRESS_INTERVAL_MICROS; + let Some(ReplicationWork::Probe { + nonce, through: 3, .. + }) = flow.work(1, 3, now) + else { + panic!("the timer must checkpoint outstanding bytes"); + }; + let lost_second = ready(1, nonce, 3, 1, 1); + assert!(flow.reply(lost_second.header(), 3, now).is_some()); + assert!(!flow.admit(1, 4, SIZE, ReplicationSend::Live)); + now += PROGRESS_INTERVAL_MICROS; + let Some(ReplicationWork::Probe { + nonce: early, + through: 3, + .. + }) = flow.work(1, 4, now) + else { + panic!("the timer must checkpoint a waiting replay"); + }; assert!( - !flow.admit(1, 4, 1, false), - "live traffic cannot overtake replay" + matches!( + flow.work(1, 4, now), + Some(ReplicationWork::Replay { from_op: 2, .. }) + ), + "replay must not wait for the checkpoint" ); - for op in 1..=4 { - assert!(flow.admit(1, op, 1, true)); - flow.sent(1, op, 1, true, PROGRESS_INTERVAL_MICROS); + for op in 2..=4 { + assert!(send(&mut flow, 1, op, SIZE, ReplicationSend::Replay, now).is_none()); } - assert!(flow.work(1, 4, PROGRESS_INTERVAL_MICROS).is_none()); + let before_replay = ready(1, early, 3, 1, 1); + assert!(flow.reply(before_replay.header(), 4, now).is_some()); assert!( - flow.admit(1, 5, 1, false), - "enqueueing through the moving head restores live delivery" + flow.admit(1, 5, SIZE, ReplicationSend::Live), + "a checkpoint sent before the replay must not move the cursor back" + ); + assert_eq!( + flow.stats, + ReplicationStats { + live_refusals: 1, + replayed: 3 + } ); } #[test] - fn given_an_inflight_local_prepare_when_a_probe_finishes_should_join_replay_without_waiting_for_a_tick() - { + fn given_a_healthy_backup_when_a_prepare_retransmits_should_charge_it_without_a_replay() { + const SIZE: u64 = 100; let mut flow = ReplicationFlow::new(3); flow.in_epoch(0, 1); - flow.sent(1, 1, 100, false, 0); - let Some(ReplicationWork::Probe { nonce, through, .. }) = - flow.work(1, 2, PROGRESS_INTERVAL_MICROS) + for op in 1..=2 { + assert!(send(&mut flow, 1, op, SIZE, ReplicationSend::Live, 0).is_none()); + } + assert!(send(&mut flow, 1, 1, SIZE, ReplicationSend::Retransmit, 0).is_none()); + assert_eq!(flow.peers[1].outstanding(), 3 * SIZE); + assert!(flow.admit(1, 3, SIZE, ReplicationSend::Live)); + assert!(flow.work(1, 3, 0).is_none()); + assert_eq!(flow.stats, ReplicationStats::default()); + } + + #[test] + fn given_a_busy_backup_when_prepares_arrive_should_send_nothing_until_a_ready_reply() { + const SIZE: u64 = 100; + let mut flow = ReplicationFlow::new(3); + flow.in_epoch(0, 1); + for op in 1..=2 { + assert!(send(&mut flow, 1, op, SIZE, ReplicationSend::Live, 0).is_none()); + } + let now = PROGRESS_INTERVAL_MICROS; + let Some(ReplicationWork::Probe { nonce, through, .. }) = flow.work(1, 2, now) else { + panic!("the timer must checkpoint outstanding bytes"); + }; + assert_eq!( + flow.reply(busy(1, nonce, through).header(), 2, now), + Some(ReplicationProgress::Busy) + ); + assert_eq!( + flow.peers[1].outstanding(), + 0, + "a busy reply still proves FIFO consumption" + ); + assert!( + !flow.admit(1, 3, SIZE, ReplicationSend::Live), + "a live op equal to the cursor waits too" + ); + assert!(!flow.admit(1, 2, SIZE, ReplicationSend::Retransmit)); + let Some(ReplicationWork::Probe { + nonce: recheck, + through: 2, + .. + }) = flow.work(1, 3, now) else { - panic!("the sent batch must be probed"); + panic!("the next tick must ask a busy backup again"); }; - let _ = flow.reply(reply(nonce, through, 1).header(), 2); assert!( - !flow.admit(1, 3, 100, false), - "a fresh op cannot skip replay" + flow.work(1, 3, now).is_none(), + "a busy backup gets no replay" ); assert!( - flow.admit(1, 2, 100, false), - "the next queued local prepare must join the ordered stream immediately" + flow.reply(ready(1, recheck, 2, 2, 2).header(), 3, now) + .is_some() ); - flow.sent(1, 2, 100, false, PROGRESS_INTERVAL_MICROS); assert!( - flow.admit(1, 3, 100, false), - "continuous live traffic must advance the replay cursor without a timer" + flow.admit(1, 3, SIZE, ReplicationSend::Live), + "a ready reply resumes after the held prefix" ); } #[test] - fn given_an_enqueue_failure_when_a_correlated_probe_finds_a_gap_should_replay_the_committed_suffix() + fn given_a_silent_backup_when_sends_continue_should_bound_its_bytes_and_keep_other_peers_sending() { + const SIZE: u64 = MIB; let mut flow = ReplicationFlow::new(3); flow.in_epoch(0, 1); - flow.sent(1, 3, 100, false, 0); - flow.failed(1, 4); - let Some(ReplicationWork::Probe { nonce, through, .. }) = flow.work(1, 6, 0) else { - panic!("failed enqueue must probe"); - }; - let _ = flow.reply(reply(nonce + 1, through, 6).header(), 6); + let mut op = 1; + while flow.admit(1, op, SIZE, ReplicationSend::Live) { + let _ = flow.sent(1, op, SIZE, ReplicationSend::Live, 0); + op += 1; + } + assert_eq!(flow.peers[1].outstanding(), REPLICATION_BATCH_BYTES_MAX); assert!( - !flow.admit(1, 7, 1, false), - "an uncorrelated reply releases no credit" + send( + &mut flow, + 2, + 1, + REPLICATION_BATCH_BYTES_MAX + 1, + ReplicationSend::Live, + 0 + ) + .is_some(), + "a silent peer must not stop another peer, and one frame larger than the window passes" ); - let busy = - reply(nonce, through, 6).transmute_header(|old, header: &mut PrepareProgressHeader| { - *header = old; - header.ready = 0; - }); - assert_eq!( - flow.reply(busy.header(), 6), - Some(super::ReplicationProgress::Busy) + assert!(!flow.admit(2, 2, 1, ReplicationSend::Live)); + } + + #[test] + fn given_a_full_replay_window_when_deferred_should_drain_credit_before_history_backoff() { + let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); + consensus.init(); + consensus.sequencer().set_sequence(2); + assert!(consensus.admit_replication(1, 1, 1, ReplicationSend::Live)); + consensus.record_replication_failure(1, 1); + assert!(consensus.admit_replication( + 1, + 1, + REPLICATION_BATCH_BYTES_MAX, + ReplicationSend::Replay + )); + let Some(ReplicationWork::Probe { nonce, through, .. }) = consensus + .record_replication_send(1, 1, REPLICATION_BATCH_BYTES_MAX, ReplicationSend::Replay) + else { + panic!("a full window must send a checkpoint behind its last frame"); + }; + assert!(!consensus.admit_replication(1, 2, 1, ReplicationSend::Replay)); + consensus.defer_replication(1); + assert!( + matches!( + consensus.replication_work(1), + Some(ReplicationWork::Probe { nonce: resent, .. }) if resent == nonce + ), + "a full window must drain without the unavailable-history delay" ); assert!( - !flow.admit(1, 7, 1, false), - "a busy replica does not establish progress" + consensus + .handle_prepare_progress(ready(1, nonce, through, 1, 1).header()) + .is_some() ); assert!( - matches!(flow.work(1, 6, 1), Some(ReplicationWork::Probe { nonce: repeated, through: observed, .. }) if repeated == nonce && observed == through), - "a busy reply must recheck readiness on the next tick without releasing its batch credit" + matches!( + consensus.replication_work(1), + Some(ReplicationWork::Replay { from_op: 2, .. }) + ), + "a drained window must resume replay without the stale history delay" ); - let _ = flow.reply(reply(nonce, through, 2).header(), 6); - assert!(matches!( - flow.work(1, 6, 1), - Some(ReplicationWork::Replay { from_op: 3, .. }) - )); - flow.in_epoch(1, 1); - let _ = flow.reply(reply(nonce, through, 2).header(), 6); + + assert!(consensus.admit_replication(2, 1, 1, ReplicationSend::Live)); + consensus.record_replication_failure(2, 1); + consensus.defer_replication(2); assert!( - flow.admit(1, 7, 1, false), - "a new view discards the previous sender window" + consensus.replication_work(2).is_none(), + "unavailable history without outstanding bytes must still back off" ); } @@ -428,15 +836,19 @@ mod tests { let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); consensus.init(); consensus.sequencer().set_sequence(2); - assert!(consensus.admit_replication(1, 1, 1, false)); - consensus.record_replication_send(1, 1, 1, false); - consensus.record_replication_failure(1, 2); - let Some(ReplicationWork::Probe { nonce, through, .. }) = consensus.replication_work(1) + assert!(consensus.admit_replication( + 1, + 1, + REPLICATION_BATCH_BYTES_MAX, + ReplicationSend::Live + )); + let Some(ReplicationWork::Probe { nonce, through, .. }) = consensus + .record_replication_send(1, 1, REPLICATION_BATCH_BYTES_MAX, ReplicationSend::Live) else { - panic!("failed enqueue must probe"); + panic!("a full window must send a checkpoint behind its last frame"); }; for stale in ["view", "incarnation", "nonce", "peer", "cluster", "group"] { - let message = reply(nonce, through, 2).transmute_header( + let message = ready(1, nonce, through, 1, 1).transmute_header( |old, header: &mut PrepareProgressHeader| { *header = old; match stale { @@ -452,13 +864,221 @@ mod tests { ); assert_eq!(consensus.handle_prepare_progress(message.header()), None); assert!( - !consensus.admit_replication(1, 3, 1, false), + !consensus.admit_replication(1, 2, 1, ReplicationSend::Retransmit), "accepted stale {stale}" ); } - let _ = consensus.handle_prepare_progress(reply(nonce, through, 2).header()); - assert!(consensus.admit_replication(1, 3, 1, false)); + let _ = consensus.handle_prepare_progress(ready(1, nonce, through, 1, 1).header()); + assert!(consensus.admit_replication(1, 2, 1, ReplicationSend::Live)); assert_eq!(consensus.commit_min(), 0); assert_eq!(consensus.commit_max(), 0); } + + const MODEL_BACKUP: u8 = 1; + + enum Frame { + Prepare { op: u64, size: u64 }, + Probe { nonce: u64, through: u64 }, + } + + /// A primary that sends to one backup over a FIFO link. While faults are + /// on, the backup drops frames, sends fail, replies get lost, and the + /// backup answers Busy. + struct Model { + flow: ReplicationFlow, + random: u64, + faults: bool, + now: u64, + head: u64, + sizes: Vec, + link: VecDeque, + replies: VecDeque>, + pushed_bytes: u64, + consumed_bytes: u64, + held: u64, + } + + impl Model { + const TICK_MICROS: u64 = 10_000; + const SIZE_MAX: u64 = 4 * MIB; + const REPLAY_OPS_MAX: u64 = 8; + const FAULT_ONE_IN: u64 = 10; + + fn new(seed: u64) -> Self { + let mut flow = ReplicationFlow::new(2); + flow.in_epoch(0, 1); + Self { + flow, + random: seed, + faults: true, + now: 0, + head: 0, + sizes: Vec::new(), + link: VecDeque::new(), + replies: VecDeque::new(), + pushed_bytes: 0, + consumed_bytes: 0, + held: 0, + } + } + + /// Xorshift, so the run needs no new dependency. + fn below(&mut self, bound: u64) -> u64 { + self.random ^= self.random << 13; + self.random ^= self.random >> 7; + self.random ^= self.random << 17; + self.random % bound + } + + fn fault(&mut self) -> bool { + self.faults && self.below(Self::FAULT_ONE_IN) == 0 + } + + fn step(&mut self) { + match self.below(5) { + 0 => { + self.head += 1; + let size = 1 + self.below(Self::SIZE_MAX); + self.sizes.push(size); + self.send(self.head, ReplicationSend::Live); + } + 1 if self.head > 0 => { + let op = 1 + self.below(self.head); + self.send(op, ReplicationSend::Retransmit); + } + 2 => self.tick(), + 3 => self.deliver_frame(), + _ => self.deliver_reply(), + } + self.check(); + } + + fn tick(&mut self) { + self.now += Self::TICK_MICROS; + if let Some(work) = self.flow.work(MODEL_BACKUP, self.head, self.now) { + self.run(work); + } + } + + fn run(&mut self, work: ReplicationWork) { + match work { + ReplicationWork::Probe { nonce, through, .. } => { + if self.fault() { + self.flow.probe_failed(MODEL_BACKUP, nonce); + } else { + self.link.push_back(Frame::Probe { nonce, through }); + } + } + ReplicationWork::Replay { from_op, .. } => { + let to_op = self.head.min(from_op + Self::REPLAY_OPS_MAX - 1); + for op in from_op..=to_op { + if !self.send(op, ReplicationSend::Replay) { + break; + } + } + } + } + } + + fn send(&mut self, op: u64, send: ReplicationSend) -> bool { + let size = self.sizes[usize::try_from(op - 1).unwrap()]; + if !self.flow.admit(MODEL_BACKUP, op, size, send) { + return false; + } + if self.fault() { + self.flow.failed(MODEL_BACKUP, op); + return false; + } + self.link.push_back(Frame::Prepare { op, size }); + self.pushed_bytes += size; + if let Some(work) = self.flow.sent(MODEL_BACKUP, op, size, send, self.now) { + self.run(work); + } + true + } + + fn deliver_frame(&mut self) { + let Some(frame) = self.link.pop_front() else { + return; + }; + let dropped = self.fault(); + match frame { + Frame::Prepare { op, size } => { + self.consumed_bytes += size; + if !dropped && op == self.held + 1 { + self.held = op; + } + } + Frame::Probe { nonce, through } if !dropped => { + let reply = if self.fault() { + busy(MODEL_BACKUP, nonce, through) + } else { + ready(MODEL_BACKUP, nonce, through, self.held, self.held) + }; + self.replies.push_back(reply); + } + Frame::Probe { .. } => {} + } + } + + fn deliver_reply(&mut self) { + let Some(reply) = self.replies.pop_front() else { + return; + }; + if !self.fault() { + let _ = self.flow.reply(reply.header(), self.head, self.now); + } + } + + fn check(&self) { + let peer = &self.flow.peers[usize::from(MODEL_BACKUP)]; + assert_eq!(peer.sent_bytes, self.pushed_bytes); + assert!(peer.credited_bytes <= peer.sent_bytes); + assert!( + peer.credited_bytes <= self.consumed_bytes, + "credit {} passed the {} bytes the backup consumed", + peer.credited_bytes, + self.consumed_bytes + ); + assert!(peer.outstanding() <= REPLICATION_BATCH_BYTES_MAX); + } + } + + #[test] + fn given_random_losses_failures_and_busy_answers_when_the_flow_runs_should_keep_its_invariants_and_deliver_every_op() + { + const SEEDS: u64 = 16; + const STEPS: u32 = 20_000; + const SETTLE_TICKS: u32 = 2_000; + for seed in 1..=SEEDS { + let mut model = Model::new(seed); + for _ in 0..STEPS { + model.step(); + } + let stats = model.flow.stats; + assert!( + stats.live_refusals > 0 && stats.replayed > 0, + "seed {seed}: the faults must force refusals and replays, or the run proves nothing: \ + {stats:?}" + ); + model.faults = false; + for _ in 0..SETTLE_TICKS { + if model.held == model.head { + break; + } + model.tick(); + while !model.link.is_empty() { + model.deliver_frame(); + } + while !model.replies.is_empty() { + model.deliver_reply(); + } + model.check(); + } + assert_eq!( + model.held, model.head, + "seed {seed}: the backup must hold every op once the faults stop" + ); + } + } } diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs index cc79fd3e53..66785fcb2d 100644 --- a/core/partitions/src/iggy_partition.rs +++ b/core/partitions/src/iggy_partition.rs @@ -9472,9 +9472,10 @@ where .held_through(self.consensus.commit_min()) } - /// Progress follows the same per-operation durability contract as prepare acks. + /// The held prefix finds replication gaps. The received prefix inside it + /// follows the same per-operation durability contract as prepare acks. #[must_use] - pub fn replication_progress(&self) -> Option { + pub fn replication_progress(&self) -> Option { if !self.consensus.is_normal() || self.consensus.is_transferring() || self.consensus.has_untrusted_log() @@ -9483,19 +9484,26 @@ where { return None; } - let held = self.held_through(); - let mut received = self.consensus.commit_min(); - while received < held { - let next = received + 1; + let held_through = self.held_through(); + let mut held = self.consensus.commit_min(); + let mut received = None; + while held < held_through { + let next = held + 1; let Some(header) = self.log.journal().inner.repair_header(next) else { break; }; - if !self.consensus.view_log_entry_proven(next) || !self.prepare_ack_ready(&header) { + if !self.consensus.view_log_entry_proven(next) { break; } - received = next; + if received.is_none() && !self.prepare_ack_ready(&header) { + received = Some(held); + } + held = next; } - Some(received) + Some(consensus::BackupPrefix { + held_op: held, + received_op: received.unwrap_or(held), + }) } /// Whether an anchored fence holds every entry of its window, so a proof @@ -19544,16 +19552,20 @@ mod tests { assert!(persistence.is_written(&header)); assert!(!persistence.is_durable(&header)); - let expected = u64::from(!durability.is_persisted()); + let expected = Some(consensus::BackupPrefix { + held_op: header.op, + received_op: u64::from(!durability.is_persisted()), + }); assert_eq!( backup.replication_progress(), - Some(expected), - "{durability:?}: progress must follow the prepare acknowledgment contract" + expected, + "{durability:?}: the held prefix finds gaps, and the received prefix follows \ + the prepare acknowledgment contract" ); backup.maintenance.flush_through = Some(header.op); assert_eq!( backup.replication_progress(), - Some(expected), + expected, "{durability:?}: ordinary materialization does not replace the held history" ); } diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index 7ef92129d6..4df96c31d2 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs @@ -42,10 +42,11 @@ pub use router::CONSENSUS_TICK_INTERVAL; #[cfg(feature = "simulator")] use consensus::LocalPipeline; use consensus::{ - AdoptedView, ChunkProgress, CommitOutcome, Consensus, ConsensusClock, DVC_HEADERS_MAX, - DvcHeaderKind, DvcSuffix, FatalReason, MergedLog, MetadataHandle, MuxPlane, PartitionsHandle, - Pipeline, Plane, PlaneKind, RepairEvidence, STATE_TRANSFER_MAX_DECODE_RETRIES, - STATE_TRANSFER_MAX_STALL_RETRIES, Sequencer, StartViewRefusal, Status, VsrAction, VsrConsensus, + AdoptedView, BackupPrefix, ChunkProgress, CommitOutcome, Consensus, ConsensusClock, + DVC_HEADERS_MAX, DvcHeaderKind, DvcSuffix, FatalReason, MergedLog, MetadataHandle, MuxPlane, + PartitionsHandle, Pipeline, Plane, PlaneKind, RepairEvidence, ReplicationSend, + ReplicationStats, STATE_TRANSFER_MAX_DECODE_RETRIES, STATE_TRANSFER_MAX_STALL_RETRIES, + Sequencer, StartViewRefusal, Status, VsrAction, VsrConsensus, build_deny_reply_from_request_header, dvc_blank, dvc_header_kind, encode_prepare_headers, fatal, repaired_frontier_update, repaired_prepare_evidence, restamp_prepare_view, verify_prepare_integrity, @@ -6087,7 +6088,12 @@ where received = next; } } - (consensus, ready.then_some(received)) + // The metadata walk has no durability test, so one value is both prefixes. + let prefix = BackupPrefix { + held_op: received, + received_op: received, + }; + (consensus, ready.then_some(prefix)) } else { let Some(partition) = planes.1.0.get_by_ns(&IggyNamespace::from_raw(header.group)) else { @@ -6121,7 +6127,9 @@ where reply.replica = consensus.replica(); reply.kind = PrepareProgressHeader::REPLY; reply.ready = u8::from(progress.is_some()); - reply.received_op = progress.unwrap_or(0); + let prefix = progress.unwrap_or_default(); + reply.held_op = prefix.held_op; + reply.received_op = prefix.received_op; reply.seal(); }); let _ = self @@ -6151,36 +6159,22 @@ where // Prefer keys after the cursor even when the commit-walk order rotates. let rank = |key| (cursor.is_some_and(|last| key <= last), key); let mut replay: Option<(u64, u8, u64)> = None; + let mut stats = ReplicationStats::default(); for group in groups { let Some(consensus) = self.replication_consensus(group) else { continue; }; + let taken = consensus.take_replication_stats(); + stats.live_refusals += taken.live_refusals; + stats.replayed += taken.replayed; for replica in 0..consensus.replica_count() { if replica == consensus.replica() { continue; } match consensus.replication_work(replica) { Some(consensus::ReplicationWork::Probe { nonce, through, .. }) => { - let header_size = size_of::(); - let probe = Message::::new(header_size) - .transmute_header(|_, header: &mut PrepareProgressHeader| { - header.command = Command::PrepareProgress; - header.size = - u32::try_from(header_size).expect("header size fits u32"); - header.cluster = consensus.cluster(); - header.view = consensus.view(); - header.replica = consensus.replica(); - header.group = group; - header.partition_incarnation = consensus.partition_incarnation(); - header.nonce = nonce; - header.observed_op = through; - header.commit_op = consensus.commit_min(); - header.kind = PrepareProgressHeader::PROBE; - header.seal(); - }); - let _ = self - .bus - .send_to_replica(replica, probe.into_generic().into_frozen()) + consensus + .send_replication_probe(replica, nonce, through) .await; } Some(consensus::ReplicationWork::Replay { from_op, .. }) @@ -6194,6 +6188,7 @@ where } } } + self.metrics.record_replication(stats); // One bounded range per tick keeps replay I/O independent of group count. let Some((group, replica, from_op)) = replay else { return; @@ -6251,7 +6246,7 @@ where if !live { return self.send_repair_prepare(target, entry).await; } - send_replayed_prepare(consensus, target, op, entry).await + send_replayed_prepare(consensus, target, op, entry, ReplicationSend::Replay).await } #[allow(clippy::future_not_send)] @@ -14516,9 +14511,14 @@ async fn dispatch_vsr_actions( continue; }; for replica in replicas { - let _ = - send_replayed_prepare(consensus, *replica, header.op, frozen.clone()) - .await; + let _ = send_replayed_prepare( + consensus, + *replica, + header.op, + frozen.clone(), + ReplicationSend::Retransmit, + ) + .await; } } } @@ -14689,9 +14689,14 @@ async fn dispatch_partition_journal_actions( continue; }; for replica in replicas { - let _ = - send_replayed_prepare(consensus, *replica, header.op, prepare.clone()) - .await; + let _ = send_replayed_prepare( + consensus, + *replica, + header.op, + prepare.clone(), + ReplicationSend::Retransmit, + ) + .await; } } } @@ -14719,6 +14724,7 @@ async fn send_replayed_prepare( target: u8, op: u64, entry: Frozen, + send: ReplicationSend, ) -> bool where B: MessageBus, @@ -14730,22 +14736,12 @@ where let Some(entry) = restamp_prepare_view(entry, consensus.view()) else { return false; }; - let size = entry.len() as u64; - if !consensus.admit_replication(target, op, size, true) { - return false; - } - if consensus - .message_bus() - .send_to_replica(target, entry) - .await - .is_ok() - { - consensus.record_replication_send(target, op, size, true); - true - } else { - consensus.record_replication_failure(target, op); - false - } + matches!( + consensus + .send_replicated_prepare(target, op, entry, send) + .await, + Ok(true) + ) } #[cfg(test)] @@ -18901,6 +18897,7 @@ mod replication_flow_tests { header.ready = 1; header.nonce = nonce; header.observed_op = observed_op; + header.held_op = received_op; header.received_op = received_op; header.size = u32::try_from(size_of::()).unwrap(); header.seal(); @@ -18967,13 +18964,16 @@ mod replication_flow_tests { .unwrap(); let consensus = partition.consensus(); consensus.sequencer().set_sequence(1); - assert!(consensus.admit_replication(1, 1, REPLICATION_BATCH_BYTES_MAX, false)); - consensus.record_replication_send(1, 1, REPLICATION_BATCH_BYTES_MAX, false); - consensus.record_replication_failure(1, 1); - let Some(consensus::ReplicationWork::Probe { nonce, through, .. }) = - consensus.replication_work(1) + assert!(consensus.admit_replication( + 1, + 1, + REPLICATION_BATCH_BYTES_MAX, + ReplicationSend::Live + )); + let Some(consensus::ReplicationWork::Probe { nonce, through, .. }) = consensus + .record_replication_send(1, 1, REPLICATION_BATCH_BYTES_MAX, ReplicationSend::Live) else { - panic!("the outstanding batch must own a progress probe"); + panic!("a full window must send a checkpoint behind its last frame"); }; let retry = VsrAction::RetransmitPrepares { targets: vec![(header, vec![1, 2])], @@ -18989,7 +18989,12 @@ mod replication_flow_tests { vec![2] ); assert!( - !consensus.admit_replication(2, 2, REPLICATION_BATCH_BYTES_MAX, false), + !consensus.admit_replication( + 2, + 2, + REPLICATION_BATCH_BYTES_MAX, + ReplicationSend::Retransmit + ), "a successful timeout retry must consume the same byte credit as replay" ); let reply = ready_reply(consensus, 1, nonce, through, 1); @@ -19042,13 +19047,16 @@ mod replication_flow_tests { let header = *entry.header(); journal.append(entry).await.unwrap(); consensus.sequencer().set_sequence(1); - assert!(consensus.admit_replication(1, 1, REPLICATION_BATCH_BYTES_MAX, false)); - consensus.record_replication_send(1, 1, REPLICATION_BATCH_BYTES_MAX, false); - consensus.record_replication_failure(1, 1); - let Some(consensus::ReplicationWork::Probe { nonce, through, .. }) = - consensus.replication_work(1) + assert!(consensus.admit_replication( + 1, + 1, + REPLICATION_BATCH_BYTES_MAX, + ReplicationSend::Live + )); + let Some(consensus::ReplicationWork::Probe { nonce, through, .. }) = consensus + .record_replication_send(1, 1, REPLICATION_BATCH_BYTES_MAX, ReplicationSend::Live) else { - panic!("the metadata batch must own a progress probe"); + panic!("a full metadata window must send a checkpoint behind its last frame"); }; dispatch_vsr_actions( &consensus, @@ -19065,7 +19073,12 @@ mod replication_flow_tests { .collect::>(), vec![2] ); - assert!(!consensus.admit_replication(2, 2, REPLICATION_BATCH_BYTES_MAX, false)); + assert!(!consensus.admit_replication( + 2, + 2, + REPLICATION_BATCH_BYTES_MAX, + ReplicationSend::Retransmit + )); let reply = ready_reply(&consensus, 1, nonce, through, 1); assert!(consensus.handle_prepare_progress(reply.header()).is_some()); sent.borrow_mut().clear(); @@ -19112,7 +19125,7 @@ mod replication_flow_tests { replica, 1, size_of::() as u64, - false + ReplicationSend::Live )); consensus.record_replication_failure(replica, 1); let Some(consensus::ReplicationWork::Probe { nonce, through, .. }) = diff --git a/core/shard/src/metrics.rs b/core/shard/src/metrics.rs index 6a25f91ec7..d536227758 100644 --- a/core/shard/src/metrics.rs +++ b/core/shard/src/metrics.rs @@ -47,6 +47,7 @@ use std::hash::{Hash, Hasher}; use std::sync::{Arc, OnceLock}; use std::time::Duration; +use consensus::ReplicationStats; use iggy_binary_protocol::{COMMAND_TABLE, Command, command_index}; use iggy_common::ConsumerKind; use message_bus::{ReplicaLinkMetrics, ReplicaReadMetrics}; @@ -376,6 +377,8 @@ pub struct ShardMetrics { replica_socket_reads_total: Counter, replica_inbound_frames_total: Counter, replica_outbound_drops_total: Counter, + replication_live_refusals_total: Counter, + replication_replayed_prepares_total: Counter, replica_bundle_teardowns_total: Counter, replica_preamble_refusals_total: Counter, replica_handshake_failures_total: Counter, @@ -466,6 +469,8 @@ impl ShardMetrics { replica_socket_reads_total: Counter::default(), replica_inbound_frames_total: Counter::default(), replica_outbound_drops_total: Counter::default(), + replication_live_refusals_total: Counter::default(), + replication_replayed_prepares_total: Counter::default(), replica_bundle_teardowns_total: Counter::default(), replica_preamble_refusals_total: Counter::default(), replica_handshake_failures_total: Counter::default(), @@ -544,6 +549,31 @@ impl ShardMetrics { .set(i64::try_from(stats.bundles_up).unwrap_or(i64::MAX)); } + /// Fold one sweep's replication flow events in, drained by + /// `VsrConsensus::take_replication_stats`. Both stay at zero while every + /// backup keeps up, so a nonzero value on a run with no fault means that + /// flow control held prepares back. + pub fn record_replication(&self, stats: ReplicationStats) { + self.replication_live_refusals_total + .inc_by(stats.live_refusals); + self.replication_replayed_prepares_total + .inc_by(stats.replayed); + } + + /// Snapshot of `replication_live_refusals_total`. Test/simulator accessor. + #[cfg(any(test, feature = "simulator"))] + #[must_use] + pub fn replication_live_refusals_value(&self) -> u64 { + self.replication_live_refusals_total.get() + } + + /// Snapshot of `replication_replayed_prepares_total`. Test/simulator accessor. + #[cfg(any(test, feature = "simulator"))] + #[must_use] + pub fn replication_replayed_prepares_value(&self) -> u64 { + self.replication_replayed_prepares_total.get() + } + fn register_persistence(&self, registry: &mut Registry) { registry.register( "partition_wal_disk_bytes", @@ -1209,6 +1239,16 @@ impl ShardMetrics { "replica frames dropped because a peer send queue was full", self.replica_outbound_drops_total.clone(), ); + registry.register( + "replication_live_refusals", + "live prepares that a backup's replication window, replay cursor or busy state refused", + self.replication_live_refusals_total.clone(), + ); + registry.register( + "replication_replayed_prepares", + "prepares resent to a backup by ordered replay", + self.replication_replayed_prepares_total.clone(), + ); registry.register( "replica_bundle_teardowns", "replica bundles torn down because a link closed or the bundle stayed incomplete", diff --git a/core/simulator/src/replication_flow_tests.rs b/core/simulator/src/replication_flow_tests.rs index 9737cdaf89..32e06d4fba 100644 --- a/core/simulator/src/replication_flow_tests.rs +++ b/core/simulator/src/replication_flow_tests.rs @@ -15,20 +15,20 @@ // specific language governing permissions and limitations // under the License. -//! Flow control to a backup must not hold produces for a consensus tick. +//! Flow control must not hold back the prepares of a backup that keeps up. //! -//! The primary stops sending prepares to a backup while a progress probe is -//! out, and when the bytes sent since the last probe reply reach -//! `REPLICATION_BATCH_BYTES_MAX`. A timer also probes each backup about once -//! per second. A stop that ends one probe round trip later costs little on a -//! real network. A stop that waits for the next tick, to send the probe or to -//! resume the prepares, costs up to 10 ms per wait. Steady produce repeats the -//! stops, so these waits show in the publish tail. +//! The primary charges each prepare it sends to a backup against a window of +//! `REPLICATION_BATCH_BYTES_MAX` bytes. A progress probe follows the prepares on +//! the same ordered link, and its reply returns the credit of the bytes sent +//! before it. Probes go out at half a window and about once per second, and no +//! prepare waits for a probe reply. So a backup that keeps up never fills its +//! window, and no prepare goes out late or by replay. //! //! The network is ordered, like the TCP link between two replicas, and each hop //! takes one tick. A produce takes four hops: the request, the prepare, the -//! prepare ok and the reply. A probe round trip adds two hops. A produce that -//! takes longer waited for a tick. +//! prepare ok and the reply. A produce that takes longer waited for flow +//! control, and the other backup can hide that wait. So the flow counters must +//! also stay at zero. use bytes::Bytes; use consensus::REPLICATION_BATCH_BYTES_MAX; @@ -45,15 +45,13 @@ const PRIMARY: u8 = 0; const ONE_WAY_DELAY_TICKS: u64 = 1; /// The request, the prepare, the prepare ok and the reply. const PRODUCE_HOPS: u64 = 4; -/// A probe and its reply. -const PROBE_HOPS: u64 = 2; -const LATENCY_TICKS_MAX: u64 = (PRODUCE_HOPS + PROBE_HOPS) * ONE_WAY_DELAY_TICKS; +const LATENCY_TICKS_MAX: u64 = PRODUCE_HOPS * ONE_WAY_DELAY_TICKS; /// A client has one produce in flight, so one produce per tick needs more /// clients than the slowest produce takes ticks. const CLIENTS: u8 = 16; #[test] -fn given_steady_produce_when_a_backup_window_fills_should_answer_within_one_probe_round_trip() { +fn given_steady_produce_when_it_passes_several_windows_should_answer_without_flow_control_waits() { const REQUEST_BYTES: u64 = 512 * 1024; /// Bytes sent to each backup, counted in windows. const WINDOWS: u64 = 3; @@ -64,12 +62,12 @@ fn given_steady_produce_when_a_backup_window_fills_should_answer_within_one_prob } #[test] -fn given_steady_produce_when_the_progress_timer_probes_a_backup_should_answer_within_one_probe_round_trip() +fn given_steady_produce_when_the_progress_timer_probes_a_backup_should_answer_without_flow_control_waits() { /// Never fills a window, so only the timer sends probes. const REQUEST_BYTES: u64 = 1024; - /// A backup is probed 1 s (`PROGRESS_INTERVAL_MICROS`) after the first - /// prepare that follows the last probe reply: 100 ticks of 10 ms. + /// The timer probes a backup 1 s (`PROGRESS_INTERVAL_MICROS`) after its + /// last probe reply: 100 ticks of 10 ms. const PROBE_INTERVAL_TICKS: u64 = 100; /// Four intervals, so the timer probes each backup more than once. const TICKS: u64 = 4 * PROBE_INTERVAL_TICKS; @@ -77,7 +75,8 @@ fn given_steady_produce_when_the_progress_timer_probes_a_backup_should_answer_wi } /// Sends one produce per tick for `ticks` ticks, like a producer at a fixed -/// rate, and asserts that each one is answered within [`LATENCY_TICKS_MAX`]. +/// rate. Asserts that each one is answered within [`LATENCY_TICKS_MAX`], and +/// that flow control refused no live prepare and replayed none. fn assert_produce_latency(request_bytes: u64, ticks: u64) { server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { enabled: false, @@ -130,8 +129,24 @@ fn assert_produce_latency(request_bytes: u64, ticks: u64) { late.sort_unstable(); assert!( late.is_empty(), - "every produce must be answered within {LATENCY_TICKS_MAX} ticks: {PRODUCE_HOPS} hops \ - and one probe round trip of {PROBE_HOPS}, so a slower one waited for a timer tick. \ - (tick sent, ticks to the reply or None without one): {late:?}" + "every produce must be answered within its {PRODUCE_HOPS} hops, so a slower one waited \ + for flow control. (tick sent, ticks to the reply or None without one): {late:?}" + ); + let shards = sim + .replicas + .iter() + .flat_map(|replica| replica.shards.iter()); + let (refusals, replayed) = shards.fold((0, 0), |(refusals, replayed), shard| { + let metrics = shard.metrics(); + ( + refusals + metrics.replication_live_refusals_value(), + replayed + metrics.replication_replayed_prepares_value(), + ) + }); + assert_eq!( + (refusals, replayed), + (0, 0), + "flow control must not refuse or replay prepares while every backup keeps up: \ + (live refusals, replayed prepares)" ); } From cbd4838ecb994696e01d86b1481f625fc7f44737 Mon Sep 17 00:00:00 2001 From: Hubert Gruszecki Date: Thu, 29 Oct 2026 17:34:51 +0100 Subject: [PATCH 04/10] test(cluster): ignore the slow rebalance test after a replica returns It runs for about 125 s locally and 127 to 131 s in CI, the only cluster test over 60 s. Run it with --run-ignored only. --- core/integration/tests/cluster/partition_primary_routing.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/integration/tests/cluster/partition_primary_routing.rs b/core/integration/tests/cluster/partition_primary_routing.rs index 9806bade41..367216ce13 100644 --- a/core/integration/tests/cluster/partition_primary_routing.rs +++ b/core/integration/tests/cluster/partition_primary_routing.rs @@ -731,6 +731,7 @@ async fn given_partitions_on_different_primaries_when_sending_round_robin_should } #[iggy_harness(cluster_nodes = 5, server(sharding.cpu_allocation = "2"))] +#[ignore = "slow: about two minutes against a five-node cluster, run with --run-ignored only"] async fn given_balanced_primaries_when_a_replica_fails_and_returns_should_rebalance_without_data_loss( harness: &mut TestHarness, ) { From 8004da78a2b8b4ea9cc93fb88809e77c0c9d9c9b Mon Sep 17 00:00:00 2001 From: Hubert Gruszecki Date: Thu, 29 Oct 2026 21:40:24 +0100 Subject: [PATCH 05/10] fix(cluster): resume live replication after a state transfer A ready progress reply lowered the replay cursor but never raised it. Busy replies during a state transfer parked the cursor on an op that the primary later dropped. After the backup caught up, the replay from that op failed on every try, and the cursor refused every live prepare. Under load, the backup stayed on pull repair and never got its primaries back. If the backup holds the op at the cursor, a ready reply now moves the cursor to the op after the held prefix. A gap already moved it there. The seeded flow model now drops old ops and repairs the backup. A live replay also read up to 8 MiB of bodies before the window admitted the first frame. It now asks the window before each read. A prefix read that outlives its bound is now asked to stop and kept until it lands, so a transfer never holds two artifact buffers. --- core/consensus/src/plane_helpers.rs | 3 +- core/consensus/src/replication.rs | 169 ++++++++++++++++++++++---- core/partitions/src/state_transfer.rs | 6 +- core/shard/src/lib.rs | 39 ++++-- core/shard/src/partition_io.rs | 144 ++++++++++++++++++++-- 5 files changed, 310 insertions(+), 51 deletions(-) diff --git a/core/consensus/src/plane_helpers.rs b/core/consensus/src/plane_helpers.rs index 19a1051a68..0fe8620c3d 100644 --- a/core/consensus/src/plane_helpers.rs +++ b/core/consensus/src/plane_helpers.rs @@ -238,8 +238,7 @@ where Ok(consensus.is_acting_primary() && !consensus.has_observed_newer_view() && !consensus.is_transferring() - && header.view == consensus.view() - && consensus.replica() == consensus.primary_index(header.view)) + && header.view == consensus.view()) } /// Re-stamp a stored prepare with the current view before retransmission. diff --git a/core/consensus/src/replication.rs b/core/consensus/src/replication.rs index 45bd02f20c..c6576a672a 100644 --- a/core/consensus/src/replication.rs +++ b/core/consensus/src/replication.rs @@ -67,6 +67,14 @@ impl PeerReplication { const fn outstanding(&self) -> u64 { self.sent_bytes.saturating_sub(self.credited_bytes) } + + /// Whether the window takes a frame of `size` once `queued` more bytes are + /// sent. An empty window takes any frame, so one legal frame larger than + /// the window still makes progress. + const fn window_takes(&self, queued: u64, size: u64) -> bool { + let outstanding = self.outstanding().saturating_add(queued); + outstanding == 0 || outstanding.saturating_add(size) <= REPLICATION_BATCH_BYTES_MAX + } } #[derive(Debug, Clone, Copy)] @@ -122,7 +130,8 @@ pub struct ReplicationStats { /// A backup's answer to a checkpoint. #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] pub struct BackupPrefix { - /// Every op through it is in the journal with proof in the current view. + /// Every op through it is applied, or is in the journal with proof in the + /// current view. An op above the commit need not be durable yet. pub held_op: u64, /// The part of the held prefix that also meets the `PrepareOk` rules. pub received_op: u64, @@ -147,10 +156,7 @@ impl ReplicationFlow { fn admit(&mut self, replica: u8, op: u64, size: u64, send: ReplicationSend) -> bool { let peer = &mut self.peers[usize::from(replica)]; - let outstanding = peer.outstanding(); - // One legal frame larger than the window must still make progress. - let window_full = - outstanding > 0 && outstanding.saturating_add(size) > REPLICATION_BATCH_BYTES_MAX; + let window_full = !peer.window_takes(0, size); let out_of_order = match send { ReplicationSend::Live => peer.replay_from.is_some_and(|from| from != op), ReplicationSend::Replay => peer.replay_from != Some(op), @@ -200,6 +206,12 @@ impl ReplicationFlow { peer.due = 0; } + fn defer(&mut self, replica: u8, now: u64) { + let peer = &mut self.peers[usize::from(replica)]; + peer.due = 0; + peer.retry_at = now.saturating_add(PROGRESS_INTERVAL_MICROS); + } + /// A checkpoint counts only once its frame is queued. Dropping one that the /// peer queue refused lets the next send issue a new one, instead of the /// window waiting for the interval to end. @@ -288,13 +300,15 @@ impl ReplicationFlow { peer.busy = false; peer.retry_at = 0; peer.due = now.saturating_add(PROGRESS_INTERVAL_MICROS); - // A newer head alone never starts a replay. let held_op = header.held_op.min(head); - if held_op < checkpoint.through { - let gap = held_op + 1; - if checkpoint.replay_from.is_none_or(|from| gap < from) { - peer.replay_from = Some(peer.replay_from.map_or(gap, |from| from.min(gap))); - } + // A newer head alone never starts a replay. + let gap = held_op < checkpoint.through + && checkpoint.replay_from.is_none_or(|from| held_op + 1 < from); + // Repair or a state transfer can carry the backup past the cursor. The + // primary can drop the op at the cursor by then, and a replay from it + // would block live sends for good. + if gap || peer.replay_from.is_some_and(|from| from <= held_op) { + peer.replay_from = Some(held_op + 1); } Some(ReplicationProgress::Ready { received_op: header.received_op.min(head), @@ -354,6 +368,14 @@ where flow.admit(replica, op, size, send) } + /// Whether the window of `replica` takes a frame of `size` behind `queued` + /// bytes that are not sent yet. A replay asks before it reads a body, so + /// it reads no body that the window of [`Self::send_replicated_prepare`] + /// then refuses. + pub fn replication_window_takes(&self, replica: u8, queued: u64, size: u64) -> bool { + self.replication.borrow().peers[usize::from(replica)].window_takes(queued, size) + } + /// Returns the checkpoint that must follow the sent copy. #[must_use] pub fn record_replication_send( @@ -377,13 +399,8 @@ where } pub fn defer_replication(&self, replica: u8) { - let mut flow = self.replication.borrow_mut(); - let peer = &mut flow.peers[usize::from(replica)]; - peer.due = 0; - peer.retry_at = self - .message_bus() - .monotonic_micros() - .saturating_add(PROGRESS_INTERVAL_MICROS); + let now = self.message_bus().monotonic_micros(); + self.replication.borrow_mut().defer(replica, now); } pub fn replication_work(&self, replica: u8) -> Option { @@ -754,6 +771,94 @@ mod tests { ); } + /// Busy replies during a state transfer park the cursor on an op that the + /// primary later drops from memory. A replay from there never succeeds, so + /// the cursor would refuse every live send. + #[test] + fn given_a_backup_that_repair_moved_past_the_cursor_when_it_answers_ready_should_replay_after_its_held_prefix() + { + const SIZE: u64 = 100; + let mut flow = ReplicationFlow::new(3); + flow.in_epoch(0, 1); + for op in 1..=3 { + assert!(send(&mut flow, 1, op, SIZE, ReplicationSend::Live, 0).is_none()); + } + let mut now = PROGRESS_INTERVAL_MICROS; + let Some(ReplicationWork::Probe { nonce, through, .. }) = flow.work(1, 3, now) else { + panic!("the timer must checkpoint outstanding bytes"); + }; + assert_eq!( + flow.reply(busy(1, nonce, through).header(), 3, now), + Some(ReplicationProgress::Busy) + ); + assert!(!flow.admit(1, 4, SIZE, ReplicationSend::Live)); + let Some(ReplicationWork::Probe { + nonce: repaired, + through: 3, + .. + }) = flow.work(1, 10, now) + else { + panic!("the next tick must ask a busy backup again"); + }; + assert_eq!( + flow.reply(ready(1, repaired, 3, 8, 8).header(), 10, now), + Some(ReplicationProgress::Ready { received_op: 8 }) + ); + assert!( + matches!( + flow.work(1, 10, now), + Some(ReplicationWork::Replay { from_op: 9, .. }) + ), + "replay must resume after the held prefix, not at the parked cursor" + ); + for op in 9..=10 { + assert!(send(&mut flow, 1, op, SIZE, ReplicationSend::Replay, now).is_none()); + } + assert!(send(&mut flow, 1, 11, SIZE, ReplicationSend::Live, now).is_none()); + + now += PROGRESS_INTERVAL_MICROS; + let Some(ReplicationWork::Probe { + nonce: installed, + through: 11, + .. + }) = flow.work(1, 11, now) + else { + panic!("the timer must checkpoint outstanding bytes"); + }; + assert!( + flow.reply(ready(1, installed, 11, 5, 5).header(), 11, now) + .is_some() + ); + assert!( + matches!( + flow.work(1, 11, now), + Some(ReplicationWork::Replay { from_op: 6, .. }) + ), + "an install that drops held ops must restart replay at the first lost op" + ); + + now += PROGRESS_INTERVAL_MICROS; + let Some(ReplicationWork::Probe { + nonce: caught_up, + through: 11, + .. + }) = flow.work(1, 11, now) + else { + panic!("the timer must checkpoint a waiting replay"); + }; + assert!( + flow.reply(ready(1, caught_up, 11, 6, 6).header(), 11, now) + .is_some() + ); + assert!( + matches!( + flow.work(1, 11, now), + Some(ReplicationWork::Replay { from_op: 7, .. }) + ), + "a backup that holds the op at the cursor must not get it again" + ); + } + #[test] fn given_a_silent_backup_when_sends_continue_should_bound_its_bytes_and_keep_other_peers_sending() { @@ -881,9 +986,10 @@ mod tests { Probe { nonce: u64, through: u64 }, } - /// A primary that sends to one backup over a FIFO link. While faults are - /// on, the backup drops frames, sends fail, replies get lost, and the - /// backup answers Busy. + /// A primary that sends to one backup over a FIFO link, and replays only + /// its newest ops. While faults are on, the backup drops frames, sends + /// fail, replies get lost, the backup answers Busy, and repair from another + /// replica moves the backup forward. struct Model { flow: ReplicationFlow, random: u64, @@ -902,6 +1008,7 @@ mod tests { const TICK_MICROS: u64 = 10_000; const SIZE_MAX: u64 = 4 * MIB; const REPLAY_OPS_MAX: u64 = 8; + const RETAINED_OPS: u64 = 16; const FAULT_ONE_IN: u64 = 10; fn new(seed: u64) -> Self { @@ -935,7 +1042,7 @@ mod tests { } fn step(&mut self) { - match self.below(5) { + match self.below(6) { 0 => { self.head += 1; let size = 1 + self.below(Self::SIZE_MAX); @@ -943,16 +1050,25 @@ mod tests { self.send(self.head, ReplicationSend::Live); } 1 if self.head > 0 => { - let op = 1 + self.below(self.head); + let op = self.head - self.below(self.head.min(Self::RETAINED_OPS)); self.send(op, ReplicationSend::Retransmit); } 2 => self.tick(), 3 => self.deliver_frame(), + 4 if self.fault() => { + let behind = self.below(Self::RETAINED_OPS); + self.repair(behind); + } _ => self.deliver_reply(), } self.check(); } + /// The backup holds every op up to `behind` ops before the head. + fn repair(&mut self, behind: u64) { + self.held = self.held.max(self.head.saturating_sub(behind)); + } + fn tick(&mut self) { self.now += Self::TICK_MICROS; if let Some(work) = self.flow.work(MODEL_BACKUP, self.head, self.now) { @@ -969,6 +1085,11 @@ mod tests { self.link.push_back(Frame::Probe { nonce, through }); } } + ReplicationWork::Replay { from_op, .. } + if from_op + Self::RETAINED_OPS <= self.head => + { + self.flow.defer(MODEL_BACKUP, self.now); + } ReplicationWork::Replay { from_op, .. } => { let to_op = self.head.min(from_op + Self::REPLAY_OPS_MAX - 1); for op in from_op..=to_op { @@ -1062,6 +1183,8 @@ mod tests { {stats:?}" ); model.faults = false; + // Only repair delivers the ops that the primary no longer holds. + model.repair(Model::RETAINED_OPS); for _ in 0..SETTLE_TICKS { if model.held == model.head { break; diff --git a/core/partitions/src/state_transfer.rs b/core/partitions/src/state_transfer.rs index 577232f8a8..bed2c2ad14 100644 --- a/core/partitions/src/state_transfer.rs +++ b/core/partitions/src/state_transfer.rs @@ -221,10 +221,10 @@ impl LocalWork { /// Asks local work to stop when the state that waits for it drops. /// -/// That state drops with its session, or when its wait passes its bound. The +/// That state drops with its session. A wait that passes its bound asks with +/// [`LocalWork::stop`] instead, and keeps the state until the work lands. The /// task checks the flag between its steps. A prefix read then frees its -/// buffer of up to one segment early, instead of holding it next to the -/// buffer of the next read. +/// buffer of up to one segment early. #[derive(Debug, Default)] pub struct LocalWorkCancel(Rc>); diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index 4df96c31d2..b3715da850 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs @@ -6376,6 +6376,12 @@ where let Some(entry_header) = journal.header(op as usize).map(|h| *h) else { break; }; + // Each earlier frame of this loop is already sent. + if live + && !consensus.replication_window_takes(target, 0, u64::from(entry_header.size)) + { + break; + } let Some(entry) = journal.entry(&entry_header).await else { break; }; @@ -6513,10 +6519,18 @@ where }) { break; } - if response_bytes > 0 && response_bytes + u64::from(held.size) > REPAIR_BYTES_MAX { + let size = u64::from(held.size); + if response_bytes > 0 && response_bytes + size > REPAIR_BYTES_MAX { + break; + } + if live + && !partition + .consensus() + .replication_window_takes(target, response_bytes, size) + { break; } - response_bytes += u64::from(held.size); + response_bytes += size; if let Some(evicted) = evicted { released.push(evicted); entries.push(None); @@ -9776,12 +9790,12 @@ where local_work_keepalive = Some((session.peer, session.nonce)); } // Local work, not the peer, holds the pull, so a slow disk - // burns no retry of the peer. A prefix read that outlives - // its bound gives up the seed, and the artifact pulls whole. - // A reuse scan that outlives its bound is asked to stop, - // and the pull fetches what it did not reuse once it lands. - // Work that landed resumes the pull here when no I/O turn - // did. + // burns no retry of the peer. Work that outlives its bound + // is asked to stop, and the pull goes on once it lands. A + // stopped prefix read thus frees its artifact buffer before + // the pull fetches the whole artifact into a new one, and a + // stopped reuse scan leaves the rest to the pull. Work that + // landed resumes the pull here when no I/O turn did. if let Some(scan) = &session.scan { if scan.landed() { local_work_landed = true; @@ -9797,8 +9811,7 @@ where if let Some(PrefixSeed::Reading { read, .. }) = &session.seed { if read.landed() { local_work_landed = true; - } else if session.idle_ticks >= local_work_ticks_max { - session.seed = None; + } else if session.idle_ticks >= local_work_ticks_max && read.stop() { prefix_read_expired = true; } return None; @@ -9822,8 +9835,8 @@ where tracing::info!( shard = self.id, namespace_raw = namespace.inner(), - "the local segment prefix read outlived its bound, so the pull fetches \ - the whole artifact" + "the local segment prefix read outlived its bound, so it is asked to \ + stop before the pull goes on" ); } if let Some((peer, nonce)) = local_work_keepalive @@ -9832,7 +9845,7 @@ where self.send_request_state_transfer(partition.consensus(), peer, nonce) .await; } - if prefix_read_expired || local_work_landed { + if local_work_landed { self.on_partition_transfer_progress(namespace.inner()).await; } if let Some((peer, nonce, target_accepted)) = transfer_stalled { diff --git a/core/shard/src/partition_io.rs b/core/shard/src/partition_io.rs index c39e8a0723..e853289cad 100644 --- a/core/shard/src/partition_io.rs +++ b/core/shard/src/partition_io.rs @@ -797,8 +797,8 @@ mod tests { use consensus::{ ArtifactProgress, Consensus, DvcSuffix, LocalPipeline, PartitionsHandle, Pipeline, Plane, - Sequencer, StateArtifact, StateTransferStage, TimeoutManager, VsrAction, VsrConsensus, - artifact_kind, + REPLICATION_BATCH_BYTES_MAX, ReplicationSend, Sequencer, StateArtifact, StateTransferStage, + TimeoutManager, VsrAction, VsrConsensus, artifact_kind, }; use futures::FutureExt; use futures::channel::oneshot; @@ -3980,12 +3980,14 @@ mod tests { } } - /// A prefix read that has not landed by its bound gives up the seed. The - /// pull then asks for the whole artifact while the serving offer still - /// lives and burns no retry of the peer. From then on the stall sweep runs - /// as for any pull. + /// A prefix read that has not landed by its bound is asked to stop. The + /// pull waits until the read lands, so the read frees its artifact buffer + /// before the pull fetches the whole artifact into a new one. The pull + /// then asks for the whole artifact while the serving offer still lives + /// and burns no retry of the peer. From then on the stall sweep runs as + /// for any pull. #[compio::test] - async fn given_a_prefix_read_that_does_not_land_when_its_bound_passes_should_pull_the_artifact_whole() + async fn given_a_prefix_read_that_does_not_land_by_its_bound_when_it_lands_should_pull_the_artifact_whole() { const INCARNATION: u64 = 7; const NONCE: u128 = 7; @@ -4058,16 +4060,39 @@ mod tests { / crate::LOCAL_WORK_WAIT_DIVISOR; let whole = (0, 0, u32::try_from(segment.len()).unwrap()); - stall_rounds(&owner, namespace, bound as usize - 1).await; + stall_rounds(&owner, namespace, bound as usize).await; assert!( requests.borrow().is_empty(), "the pull must wait for its read" ); + let read = || { + let Some(PrefixSeed::Reading { read, .. }) = &partitions + .get_by_ns(&namespace) + .unwrap() + .transfer + .as_ref() + .unwrap() + .seed + else { + panic!("the session must keep the read until it lands"); + }; + read + }; + assert!(!read().stop(), "the bound must ask the read to stop"); + stall_rounds(&owner, namespace, RETRY_TICKS as usize).await; + assert!( + requests.borrow().is_empty(), + "the pull must not fetch the artifact while the stopped read holds its buffer" + ); + *read().result.borrow_mut() = Some(Err(io::Error::new( + io::ErrorKind::Interrupted, + "the transfer gave up this prefix read", + ))); stall_rounds(&owner, namespace, 1).await; assert_eq!( *requests.borrow(), [whole], - "the pull must ask for the whole artifact at the bound" + "the pull must ask for the whole artifact once the read lands" ); assert_eq!( partitions @@ -8817,6 +8842,90 @@ mod tests { ); } + /// A replay to a backup reads no released body that the replication + /// window refuses. A window with room for op 1 reads op 1 only, and a full + /// window reads nothing. The body of op 2 is corrupt, so a read of it + /// would drop ops 1..=2 from the ring. + #[compio::test] + async fn given_released_bodies_when_the_replication_window_refuses_them_should_not_read_them() { + const HEAD: u64 = 3; + let bus = Rc::new(IggyMessageBus::new(0)); + let replayed = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&replayed); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + let message = + Message::::try_from(Owned::copy_from_slice(frame.as_slice())) + .unwrap(); + if message.header().command == Command::Prepare { + let prepare: Message = message.try_into_typed().unwrap(); + captured.borrow_mut().push(prepare.header().op); + } + Ok(()) + })); + let (owner, _sender) = test_owner(&bus, None); + let namespace = IggyNamespace::new(0, 0, 0); + let (mut primary, prepares) = journaling_primary(&bus, namespace, HEAD, 1).await; + let directory = tempfile::tempdir().unwrap(); + let (log_path, _) = attach_segment_files(&mut primary, directory.path(), false).await; + let positions = release_bodies_to_segment(&primary, &log_path, &prepares); + let mut segment = std::fs::read(&log_path).unwrap(); + segment[usize::try_from(positions[&2]).unwrap()] ^= 0xFF; + std::fs::write(&log_path, &segment).unwrap(); + let partitions = owner.plane.partitions(); + partitions.remove(&namespace).unwrap(); + partitions.insert(namespace, primary); + connect_replica_on(&bus, REPAIR_BACKUP, 1); + let first = u64::from(prepares[0].header().size); + let incarnation = { + let consensus = partitions.get_by_ns(&namespace).unwrap().consensus(); + assert!(consensus.admit_replication(REPAIR_BACKUP, 1, first, ReplicationSend::Live)); + consensus.record_replication_failure(REPAIR_BACKUP, 1); + let _checkpoint = consensus.record_replication_send( + REPAIR_BACKUP, + 1, + REPLICATION_BATCH_BYTES_MAX - first, + ReplicationSend::Retransmit, + ); + consensus.partition_incarnation() + }; + let replay = |from_op| { + Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestPreparesHeader| { + header.command = Command::RequestPrepares; + header.size = u32::try_from(size_of::()).unwrap(); + header.replica = REPAIR_BACKUP; + header.group = namespace.inner(); + header.partition_incarnation = incarnation; + header.from_op = from_op; + header.to_op = HEAD; + }) + }; + let retained_from = || { + partitions + .get_by_ns(&namespace) + .unwrap() + .log + .journal() + .inner + .repair_retained_from() + }; + + assert_eq!(owner.serve_prepares(&replay(1), true).await, 1); + assert_eq!(*replayed.borrow(), [1], "the window must take op 1 only"); + assert_eq!( + retained_from(), + Some(1), + "the replay must not read the body of op 2 that the window refuses" + ); + assert_eq!(owner.serve_prepares(&replay(2), true).await, 1); + assert_eq!(*replayed.borrow(), [1], "a full window must take nothing"); + assert_eq!( + retained_from(), + Some(1), + "a full window must not read a body" + ); + } + /// A log file that cannot be opened for now fails the read with an error /// that can pass, like a file descriptor limit. A link that points at /// itself stands in for it, because a lower limit would reach the other @@ -10262,6 +10371,21 @@ mod tests { ops: u64, payload_len: usize, ) -> Vec> { + journaling_primary(bus, namespace, ops, payload_len).await.1 + } + + /// The primary of [`REPAIR_VIEW`] after it journaled the first `ops` sends + /// of `payload_len` bytes each, with a copy of each prepare. + #[allow(clippy::future_not_send)] + async fn journaling_primary( + bus: &Rc, + namespace: IggyNamespace, + ops: u64, + payload_len: usize, + ) -> ( + IggyPartition, HeldSuperblock>, + Vec>, + ) { let mut consensus = VsrConsensus::new( 1, REPAIR_PRIMARY, @@ -10289,7 +10413,7 @@ mod tests { origin.consensus().clear_pipeline(); origin.consensus().drain_loopback_into(&mut Vec::new()); } - prepares + (origin, prepares) } const SCANNING_INCARNATION: u64 = 7; From c55e8d0923904b270ca4ad8c6b4edf7b962aa786 Mon Sep 17 00:00:00 2001 From: Hubert Gruszecki Date: Fri, 30 Oct 2026 02:05:15 +0100 Subject: [PATCH 06/10] fix(partitions): stop the commit walk from scanning the whole journal A lagging backup can hold hundreds of thousands of unapplied entries. On every tick, committed_headers_from looped over all of them to pick at most 64 ops. One shard then spent most of its CPU in this walk and applied entries slower than new prepares arrived. Its journal grew without limit, and the replica never caught up. Look up each op in the first-occurrence index instead. A walk now costs one lookup per returned header. --- core/partitions/src/journal.rs | 68 +++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/core/partitions/src/journal.rs b/core/partitions/src/journal.rs index 56e5aa99de..2cbc5537e3 100644 --- a/core/partitions/src/journal.rs +++ b/core/partitions/src/journal.rs @@ -1619,6 +1619,10 @@ where /// `limit` bounds what one caller commits in a single pass. The run is /// re-derived from the caller's own `commit_min` every time, so a truncated /// answer is resumed, not lost. + /// + /// Costs one index lookup per returned header, however many headers are + /// resident: a lagging backup holds hundreds of thousands, and the commit + /// walk calls this on every tick. pub fn committed_headers_from( &self, from_op: u64, @@ -1633,37 +1637,20 @@ where if from_op > commit_max { return Vec::new(); } - let headers = unsafe { &*self.headers.get() }; - // Sized off the RUN the caller asked for, capped by the headers that - // could possibly cover it. Sizing off `headers.len()` alone would make - // a one-op run allocate a slot per resident header, and `commit_max` is - // a cluster frontier this replica may be arbitrarily far below, so - // neither bound can be dropped. `saturating_add` because `commit_max` - // is `u64::MAX` on a saturated frontier, where `+ 1` would panic in - // debug and wrap to an empty span (a walk that never resumes) in - // release. - let span = (commit_max - from_op) - .saturating_add(1) - .min(limit as u64) - .min(headers.len() as u64); - // `span` is a `min` of two `usize`-derived values on a 64-bit target, - // so the narrowing is lossless; `slot` is then compared against it as - // a `u64` BEFORE any cast, so nothing depends on the cast to bound it. - #[allow(clippy::cast_possible_truncation)] - let span = span as usize; - let mut slots: Vec> = vec![None; span]; - for header in headers { - if header.op < from_op || header.op - from_op >= span as u64 { - continue; - } - #[allow(clippy::cast_possible_truncation)] - let slot = (header.op - from_op) as usize; - // Keep the first resident occurrence, matching `header_by_op`. - slots[slot].get_or_insert(*header); - } + // `saturating_add` because `commit_max` is `u64::MAX` on a saturated + // frontier, where `+ 1` would panic in debug and wrap to an empty span + // (a walk that never resumes) in release. + let span = (commit_max - from_op).saturating_add(1).min(limit as u64); // Stops at the first hole: a replication gap must not be skipped, or - // `advance_commit_min`'s sequential contract breaks. - slots.into_iter().map_while(|header| header).collect() + // `advance_commit_min`'s sequential contract breaks. `header_by_op` + // returns the first resident occurrence of an op. + (0..span) + .map_while(|index| { + from_op + .checked_add(index) + .and_then(|op| self.header_by_op(op)) + }) + .collect() } /// Oldest message offset still resident in the in-memory journal, if @@ -2847,6 +2834,27 @@ mod tests { ); } + #[compio::test] + async fn given_reordered_duplicates_when_walking_should_follow_op_order_and_first_copy() { + let journal = PartitionJournal::::default(); + for (op, checksum) in [(3, 30), (1, 10), (4, 40), (2, 20), (1, 11), (2, 21)] { + let entry = build_prepare(op, HEADER_SIZE).transmute_header( + |mut old, header: &mut PrepareHeader| { + old.checksum = checksum; + *header = old; + }, + ); + journal.append(entry.into_frozen()).await.unwrap(); + } + + let run: Vec<_> = journal + .committed_headers_from(1, u64::MAX, usize::MAX) + .iter() + .map(|header| (header.op, header.checksum)) + .collect(); + assert_eq!(run, vec![(1, 10), (2, 20), (3, 30), (4, 40)]); + } + /// Three-message batch with the broker append time (`base_timestamp`) /// deliberately AFTER every producer stamp (`origin_timestamp` + deltas), /// the layout every real batch has (the broker stamps later than the From 110b6cf6e966e7d743e005e84f6e52f32e2e0461 Mon Sep 17 00:00:00 2001 From: Hubert Gruszecki Date: Fri, 30 Oct 2026 03:27:38 +0100 Subject: [PATCH 07/10] fix(partitions): evict the journal prefix without re-appending the tail A lagging backup holds hundreds of thousands of journal entries and flushes after almost every commit walk. Each eviction drained the journal and appended every retained entry again, which decoded each batch and rebuilt the op, offset and timestamp indexes. A shard with three catching-up partitions stayed at 100% CPU and the backup did not catch up while producers kept writing. The entries and headers now live in deques with a front offset, so an eviction pops the prefix and removes its index keys in O(evicted). The journal keeps running message and byte totals for journal.info. A journal out of op order, or a prefix op with a copy in the retained tail, still rebuilds the tail. Both shapes are rare. --- core/partitions/src/iggy_partition.rs | 59 ++- core/partitions/src/journal.rs | 583 +++++++++++++++++++------- core/shard/src/partition_io.rs | 6 +- 3 files changed, 454 insertions(+), 194 deletions(-) diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs index 66785fcb2d..34ab4184b1 100644 --- a/core/partitions/src/iggy_partition.rs +++ b/core/partitions/src/iggy_partition.rs @@ -6222,8 +6222,8 @@ where .journal() .inner .with_entries(|entries| entries.len().saturating_sub(prefix)); - // Eviction rebuilds the retained tail. Amortize partial flushes so - // a slow writer cannot make us rebuild a large tail after each write. + // A partial flush waits until it frees at least as many entries as + // it keeps, so a slow writer gets fewer and larger flushes. if (ready_through == through || prefix >= retained) && self.begin_materialization( ready_through, @@ -10326,8 +10326,8 @@ where .map(|pending| pending.index_span_limit) }); self.log.journal().inner.with_entries(|entries| { - let entries = entries - .get(first_entry..prefix_len) + let entries = (first_entry <= prefix_len && prefix_len <= entries.len()) + .then(|| entries.range(first_entry..prefix_len)) .ok_or(IggyError::CannotWriteToFile)?; let mut backing = 0_usize; let mut file_position = position; @@ -10409,9 +10409,10 @@ where .ok_or(IggyError::CannotWriteToFile)?; let completes = chunk.end_entry == save.chunk.end_entry; let batches = self.log.journal().inner.with_entries(|entries| { - let entries = entries - .get(chunk.first_entry..chunk.end_entry) - .ok_or(IggyError::CannotWriteToFile)?; + let entries = (chunk.first_entry <= chunk.end_entry + && chunk.end_entry <= entries.len()) + .then(|| entries.range(chunk.first_entry..chunk.end_entry)) + .ok_or(IggyError::CannotWriteToFile)?; let mut batches = Vec::with_capacity(chunk.buffers); for entry in entries { if materialization_batch_meta(entry, chunk.persisted_end)?.is_some() { @@ -10572,12 +10573,10 @@ where } /// Evict the committed prefix (the `count` front entries read by - /// `committed_prefix_len`) and reset `journal.info` to reflect only the - /// uncommitted tail left resident, so the next persist threshold counts that - /// tail alone. Call once the prefix is durable, or when there is nothing to - /// persist. The retained tail's accounting is folded from the meta - /// `evict_prefix` surfaced during its re-append, so the tail is not decoded - /// a second time. + /// `committed_prefix_len`) and set the message count and size in + /// `journal.info` to the batches left resident, so the next persist + /// threshold counts that tail alone. Call once the prefix is durable, or + /// when there is nothing to persist. fn evict_committed_prefix(&mut self, count: usize) { if count == 0 { return; @@ -10587,26 +10586,20 @@ where return; } let persistence = self.persistence.as_deref(); - let retained = self.log.journal().inner.evict_prefix_located(count, |op| { + let journal = &self.log.journal().inner; + journal.evict_prefix_located(count, |op| { persistence?.segment_reference(op).map(BodyPosition::from) }); - // Eviction drains the ring and re-appends the retained tail, so the slots - // a suffix snapshot describes move under it without the head or the - // commit point changing. + // Every retained entry moves `count` slots towards the front, so the + // slots a suffix snapshot describes move under it without the head or + // the commit point changing. self.consensus.note_journal_mutation(); - let mut retained_info = JournalInfo::default(); - for (_, meta) in &retained { - if let Some(meta) = meta { - accumulate_committed_info( - &mut retained_info, - meta.base_offset, - meta.base_timestamp, - meta.total_size, - meta.message_count, - ); - } - } - self.log.journal_mut().info = retained_info; + // Appends reject a batch that overflows the count, so it fits. + let messages_count = u32::try_from(journal.resident_messages()).unwrap_or(u32::MAX); + let size = IggyByteSize::from(journal.resident_message_size()); + let info = &mut self.log.journal_mut().info; + info.messages_count = messages_count; + info.size = size; } async fn handle_committed_entries( @@ -14428,7 +14421,7 @@ mod tests { .inner .with_entries(|entries| { crate::io::materialization_allocation_charge( - &entries[..1], + std::slice::from_ref(&entries[0]), 1, crate::iggy_index::IGGY_INDEX_SIZE, ) @@ -35349,7 +35342,7 @@ mod tests { partition.consensus().advance_commit_max(3); partition.repair = Some(armed_fetch_session(2, 3, 0)); partition.commit_journal(&repair_config()).await; - let _ = partition.log.journal().inner.evict_prefix(3); + partition.log.journal().inner.evict_prefix(3); let conclusion = partition.complete_repair(&repair_config()).await; @@ -36963,7 +36956,7 @@ mod tests { .log .journal() .inner - .with_entries(<[_]>::len), + .with_entries(VecDeque::len), usize::try_from(RECORDS).unwrap() ); } diff --git a/core/partitions/src/journal.rs b/core/partitions/src/journal.rs index 2cbc5537e3..4ab6984fa0 100644 --- a/core/partitions/src/journal.rs +++ b/core/partitions/src/journal.rs @@ -30,7 +30,7 @@ use server_common::{ use std::io; use std::{ cell::{Cell, RefCell, UnsafeCell}, - collections::{BTreeMap, HashMap, VecDeque}, + collections::{BTreeMap, VecDeque}, fmt::Debug, ops::RangeInclusive, rc::{Rc, Weak}, @@ -45,11 +45,11 @@ const ZERO_LEN: usize = 0; const PREPARE_HEADER_SIZE: usize = std::mem::size_of::(); type JournalBuffer = Frozen<4096>; -/// Decoded `SendMessages` header fields surfaced from a journal (re-)append so a -/// caller can fold segment accounting without a second decode of the same bytes. -/// Raw header values only: the journal stays agnostic of partition-layer -/// accounting types (`JournalInfo` lives in the log layer). `None` is surfaced -/// for non-`SendMessages` ops, which carry no segment bytes. +/// Decoded `SendMessages` header fields: what a resident batch adds to the +/// journal's offset and timestamp indexes and message totals, and what a flush +/// folds into segment accounting. Raw header values only: the journal stays +/// agnostic of partition-layer accounting types (`JournalInfo` lives in the +/// log layer). #[derive(Clone, Copy)] pub struct RetainedBatchMeta { pub base_offset: u64, @@ -58,6 +58,29 @@ pub struct RetainedBatchMeta { pub message_count: u32, } +impl RetainedBatchMeta { + /// `None` for an op that is not `SendMessages` or that carries no + /// messages: such an entry has no offset to index and no segment bytes. + /// + /// Trusted (no batch-hash): every entry here was just stamped by + /// `stamp_prepare_for_persistence` (its checksum recomputed over this exact + /// blob) or was validated by such an append before, so re-hashing the + /// ~1 MiB blob only to read the header is waste. + fn of(header: &PrepareHeader, entry: &JournalBuffer) -> Option { + if header.operation != Operation::SendMessages { + return None; + } + let batch = decode_prepare_slice_trusted(entry.as_slice()).ok()?; + let message_count = batch.message_count(); + (message_count != 0).then(|| Self { + base_offset: batch.header.base_offset, + base_timestamp: batch.header.base_timestamp, + total_size: batch.header.total_size() as u64, + message_count, + }) + } +} + /// What one pass over the journal headers found for a repair window. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct RepairedWindowShape { @@ -449,7 +472,7 @@ pub struct SelectedBatchSlice { /// /// # Warning — development storage only /// -/// This storage backs the `Journal` trait with a plain `Vec` +/// This storage backs the `Journal` trait with a plain `VecDeque` /// inside an `UnsafeCell`. Writes never hit disk, nothing is `fsync`ed, and /// every entry is lost on process exit. /// @@ -470,11 +493,10 @@ pub struct SelectedBatchSlice { /// production cluster traffic. #[derive(Debug, Default)] pub struct PartitionJournalMemStorage { - entries: UnsafeCell>, - /// Maps byte offset (as if disk-backed) to index in entries Vec - offset_to_index: UnsafeCell>, - /// Current write position (cumulative byte offset) - current_offset: UnsafeCell, + entries: UnsafeCell>, + /// Offset of `entries[0]`. An offset counts the entries written since the + /// last drain, so popping the front leaves every other offset valid. + front_offset: Cell, } impl Storage for PartitionJournalMemStorage { @@ -485,15 +507,8 @@ impl Storage for PartitionJournalMemStorage { } async fn read_at(&self, offset: usize, _buffer: Self::Buffer) -> io::Result { - let offset_to_index = unsafe { &*self.offset_to_index.get() }; - let Some(&index) = offset_to_index.get(&offset) else { - return Ok(Owned::<4096>::zeroed(0).into()); - }; - - let entries = unsafe { &*self.entries.get() }; - Ok(entries - .get(index) - .cloned() + Ok(self + .read_at_sync(offset) .unwrap_or_else(|| Owned::<4096>::zeroed(0).into())) } } @@ -511,9 +526,16 @@ where /// Keeping `op` in the key preserves duplicate timestamps while still /// letting us seek to the closest batch for timestamp-based polling. timestamp_to_op: UnsafeCell>, - headers: UnsafeCell>, + headers: UnsafeCell>, + /// Position of `headers[0]`, counted from the last clear, so the positions + /// in `first_header_by_op` stay valid when the front leaves. + front_position: Cell, /// Commit validation uses the first resident occurrence, including duplicates. first_header_by_op: RefCell>, + /// Messages in the resident `SendMessages` batches. + resident_messages: Cell, + /// Segment bytes of the resident `SendMessages` batches. + resident_message_size: Cell, inner: UnsafeCell>, /// Ring of recently evicted committed entries, keyed by op, retained so /// this replica can serve journal repair for rejoin windows after the @@ -615,30 +637,33 @@ where impl PartitionJournalMemStorage { /// Synchronous mirror of [`Storage::read_at`] for the poll path. Mem - /// storage never hits the reactor (it copies from an in-memory `Vec`), so - /// the read can run under a partition borrow without crossing an `.await` - /// - the property that keeps poll-read sound. + /// storage never hits the reactor (it copies from an in-memory `VecDeque`), + /// so the read can run under a partition borrow without crossing an + /// `.await` - the property that keeps poll-read sound. fn read_at_sync(&self, offset: usize) -> Option { - let offset_to_index = unsafe { &*self.offset_to_index.get() }; - let index = *offset_to_index.get(&offset)?; let entries = unsafe { &*self.entries.get() }; - entries.get(index).cloned() + entries + .get(offset.checked_sub(self.front_offset.get())?) + .cloned() } fn entries(&self) -> Vec { let entries = unsafe { &*self.entries.get() }; - entries.clone() + entries.iter().cloned().collect() } fn drain(&self) -> Vec { let entries = unsafe { &mut *self.entries.get() }; - let offset_to_index = unsafe { &mut *self.offset_to_index.get() }; - let current_offset = unsafe { &mut *self.current_offset.get() }; - - offset_to_index.clear(); - *current_offset = 0; + self.front_offset.set(0); + std::mem::take(entries).into() + } - std::mem::take(entries) + /// Removes the front entry and keeps the offsets of the others. + fn pop_front(&self) -> Option { + let entries = unsafe { &mut *self.entries.get() }; + let entry = entries.pop_front()?; + self.front_offset.set(self.front_offset.get() + 1); + Some(entry) } fn is_empty(&self) -> bool { @@ -647,21 +672,14 @@ impl PartitionJournalMemStorage { } fn current_offset(&self) -> usize { - let current_offset = unsafe { &*self.current_offset.get() }; - *current_offset + let entries = unsafe { &*self.entries.get() }; + self.front_offset.get() + entries.len() } fn write_at_sync(&self, buf: JournalBuffer) -> usize { let len = buf.len(); let entries = unsafe { &mut *self.entries.get() }; - let offset_to_index = unsafe { &mut *self.offset_to_index.get() }; - let current_offset = unsafe { &mut *self.current_offset.get() }; - - let index = entries.len(); - offset_to_index.insert(*current_offset, index); - entries.push(buf); - *current_offset += len; - + entries.push_back(buf); len } } @@ -774,6 +792,18 @@ impl PartitionJournal { self.resident_control_ops.get() } + /// Messages in the resident `SendMessages` batches, which the partition's + /// message-count flush threshold reads. + pub const fn resident_messages(&self) -> u64 { + self.resident_messages.get() + } + + /// Segment bytes of the resident `SendMessages` batches, which the + /// partition's size flush threshold reads. + pub const fn resident_message_size(&self) -> u64 { + self.resident_message_size.get() + } + /// Restore the materialized commit point for elections and body repair. pub fn restore_checkpoint_prepare(&self, op: u64, prepare: JournalBuffer) { let ring = unsafe { &mut *self.evicted_ring.get() }; @@ -1115,8 +1145,8 @@ impl PartitionJournal { } /// Inspect resident allocations without pinning extra payload references. - /// The closure cannot retain the borrowed entry vector across owner mutations. - pub(crate) fn with_entries(&self, inspect: impl FnOnce(&[JournalBuffer]) -> R) -> R { + /// The closure cannot retain the borrowed entry deque across owner mutations. + pub(crate) fn with_entries(&self, inspect: impl FnOnce(&VecDeque) -> R) -> R { let inner = unsafe { &*self.inner.get() }; let entries = unsafe { &*inner.storage.entries.get() }; inspect(entries) @@ -1173,77 +1203,139 @@ impl PartitionJournal { ordered.sort_by_key(|(op, _)| *op); self.clear_resident_indexes(); for (_, entry) in ordered { - self.append_with_meta_sync(entry); + self.append_sync(entry); } true } - /// Remove the persisted prefix and return the retained entries with their batch metadata. - pub fn evict_prefix(&self, count: usize) -> Vec<(JournalBuffer, Option)> { - self.evict_prefix_located(count, |_| None) + /// Remove the persisted prefix: the `count` front entries. + pub fn evict_prefix(&self, count: usize) { + self.evict_prefix_located(count, |_| None); } /// [`Self::evict_prefix`] for a journal whose bodies also sit in segments. /// `locate` names where the body of an evicted op lies, so the evicted /// ring can release that body from memory and still serve the op. - pub fn evict_prefix_located( - &self, - count: usize, - locate: impl Fn(u64) -> Option, - ) -> Vec<(JournalBuffer, Option)> { - let all_entries = { - let inner = unsafe { &*self.inner.get() }; - inner.storage.drain() - }; - // Ops are positional against `headers` until the clear below; capture - // the evicted prefix's ops first so the ring stays op-addressable. - let evicted_ops: Vec = { - let headers = unsafe { &*self.headers.get() }; - headers.iter().take(count).map(|header| header.op).collect() + /// + /// Costs O(`count`), however many entries stay resident: a lagging backup + /// holds hundreds of thousands and evicts after every flush. Two rare + /// shapes rebuild the retained tail instead, because a popped entry would + /// leave a stale index behind: a journal out of op order, and a prefix op + /// that has another copy in the retained tail. + pub fn evict_prefix_located(&self, count: usize, locate: impl Fn(u64) -> Option) { + let evicted = if self.out_of_order.get() || self.retains_copy_of_prefix(count) { + self.rebuild_without_prefix(count) + } else { + self.pop_prefix(count) }; - if let Some(&highest) = evicted_ops.iter().max() { + if let Some(highest) = evicted.iter().map(|(op, _)| *op).max() { self.evicted_through .set(self.evicted_through.get().max(highest)); } - self.clear_resident_indexes(); + if !self.repair_retention.get() { + return; + } + let ring = unsafe { &mut *self.evicted_ring.get() }; + let mut ring_bytes = self.evicted_ring_bytes.get(); + for (op, prepare) in evicted { + ring_bytes += prepare.len() as u64; + let entry = RingEntry { + prepare, + body: locate(op), + }; + if let Some(previous) = ring.insert(op, entry) { + ring_bytes -= previous.prepare.len() as u64; + } + } + self.evicted_ring_bytes.set(ring_bytes); + self.trim_evicted_ring(); + } - let mut all_entries = all_entries.into_iter(); - if self.repair_retention.get() { - let ring = unsafe { &mut *self.evicted_ring.get() }; - let mut ring_bytes = self.evicted_ring_bytes.get(); - for op in evicted_ops { - let Some(prepare) = all_entries.next() else { - break; - }; - ring_bytes += prepare.len() as u64; - let entry = RingEntry { - prepare, - body: locate(op), - }; - if let Some(previous) = ring.insert(op, entry) { - ring_bytes -= previous.prepare.len() as u64; - } + /// Whether an op in the `count` front entries has a copy behind them. The + /// op index names the last copy, so one lookup per op answers it. + fn retains_copy_of_prefix(&self, count: usize) -> bool { + let headers = unsafe { &*self.headers.get() }; + let op_to_storage_offset = unsafe { &*self.op_to_storage_offset.get() }; + let inner = unsafe { &*self.inner.get() }; + let retained_from = inner.storage.current_offset() - headers.len().saturating_sub(count); + headers.iter().take(count).any(|header| { + op_to_storage_offset + .get(&header.op) + .is_some_and(|&storage_offset| storage_offset >= retained_from) + }) + } + + /// Pops the `count` front entries and removes them from every index. The + /// caller makes sure that no evicted op has a retained copy, so each index + /// ends as a rebuild of the retained tail would leave it. + fn pop_prefix(&self, count: usize) -> Vec<(u64, JournalBuffer)> { + let headers = unsafe { &mut *self.headers.get() }; + let inner = unsafe { &*self.inner.get() }; + let op_to_storage_offset = unsafe { &mut *self.op_to_storage_offset.get() }; + let offset_to_op = unsafe { &mut *self.offset_to_op.get() }; + let timestamp_to_op = unsafe { &mut *self.timestamp_to_op.get() }; + let mut first_header_by_op = self.first_header_by_op.borrow_mut(); + let mut evicted = Vec::with_capacity(count.min(headers.len())); + while evicted.len() < count { + // The length-lock pairs every header with one stored entry. + let (Some(header), Some(entry)) = (headers.pop_front(), inner.storage.pop_front()) + else { + break; + }; + self.front_position.set(self.front_position.get() + 1); + let op = header.op; + op_to_storage_offset.remove(&op); + first_header_by_op.remove(&op); + if header.operation != Operation::SendMessages { + self.resident_control_ops + .set(self.resident_control_ops.get() - 1); } - self.evicted_ring_bytes.set(ring_bytes); - self.trim_evicted_ring(); - } else { - // Consume without retaining: the iterator itself must still - // advance past the evicted prefix so the retained tail below is - // aligned. - for _ in &evicted_ops { - if all_entries.next().is_none() { - break; + if let Some(meta) = RetainedBatchMeta::of(&header, &entry) { + // A later batch at the same offset keeps the key. + if offset_to_op.get(&meta.base_offset) == Some(&op) { + offset_to_op.remove(&meta.base_offset); } + timestamp_to_op.remove(&(meta.base_timestamp, op)); + self.resident_messages + .set(self.resident_messages.get() - u64::from(meta.message_count)); + self.resident_message_size + .set(self.resident_message_size.get() - meta.total_size); } + evicted.push((op, entry)); + } + // Every op above the highest evicted one is still resident, so the + // cached run keeps that part instead of a full walk on the next probe. + if let Some(highest) = evicted.iter().map(|(op, _)| *op).max() { + let (run_floor, run_through) = self.held_run.get(); + let floor = run_floor.max(highest); + self.held_run.set(if floor < run_through { + (floor, run_through) + } else { + (0, 0) + }); } - let retained: Vec = all_entries.collect(); - let mut result = Vec::with_capacity(retained.len()); - for entry in retained { - let meta = self.append_with_meta_sync(entry.clone()); - result.push((entry, meta)); - } + evicted + } - result + /// Evicts the `count` front entries by draining the journal and appending + /// the retained tail again. + fn rebuild_without_prefix(&self, count: usize) -> Vec<(u64, JournalBuffer)> { + let entries = { + let inner = unsafe { &*self.inner.get() }; + inner.storage.drain() + }; + // Positional against `headers` until the clear below. + let ops: Vec = { + let headers = unsafe { &*self.headers.get() }; + headers.iter().map(|header| header.op).collect() + }; + self.clear_resident_indexes(); + let mut entries = ops.into_iter().zip(entries); + let evicted = entries.by_ref().take(count).collect(); + for (_, entry) in entries { + self.append_sync(entry); + } + evicted } /// Keeps the evicted ring within its caps. Past the byte cap the oldest @@ -1285,63 +1377,32 @@ impl PartitionJournal { self.evicted_ring_bytes.set(bytes); } - /// `append`, additionally returning the decoded `RetainedBatchMeta` for a - /// `SendMessages` entry so the eviction path folds its accounting without a - /// second decode of the same bytes. - /// /// INVARIANT (length-lock): the header is pushed before `storage.write_at`, /// so `headers[i]` and the entry at storage index `i` stay positionally /// paired - materialization reads entries by that position. `MemStorage::write_at` /// is infallible, so the push never runs ahead of a failed write. A future /// fallible `Storage` MUST roll the header push back on a write error (or /// write before pushing the header) or the zip desyncs. - fn append_with_meta_sync(&self, entry: JournalBuffer) -> Option { + fn append_sync(&self, entry: JournalBuffer) { let header_bytes = &entry[..PREPARE_HEADER_SIZE]; let header = *bytemuck::checked::try_from_bytes::(header_bytes) .expect("partition journal append expects a valid prepare header"); let op = header.op; - // One decode feeds both the offset/timestamp index and the surfaced - // accounting meta. Both are keyed on `base_timestamp`, the broker - // append time stamped into replies: the seek hint must live on the - // same clock as `select_batch_slice`'s filter or timestamp polls seek - // to the wrong resident entry. - // Trusted (no batch-hash): every entry reaching append was just stamped - // by `stamp_prepare_for_persistence` (its checksum recomputed over this - // exact blob) or re-appended from an already-validated resident entry, - // so re-hashing the ~1 MiB blob here only to read the header is waste. - let (index_offset_timestamp, meta) = if header.operation == Operation::SendMessages { - match decode_prepare_slice_trusted(entry.as_slice()) { - Ok(batch) if batch.message_count() != 0 => { - let message_count = batch.message_count(); - let meta = RetainedBatchMeta { - base_offset: batch.header.base_offset, - base_timestamp: batch.header.base_timestamp, - total_size: batch.header.total_size() as u64, - message_count, - }; - ( - Some((batch.header.base_offset, batch.header.base_timestamp)), - Some(meta), - ) - } - _ => (None, None), - } - } else { - (None, None) - }; + let meta = RetainedBatchMeta::of(&header, &entry); { let headers = unsafe { &mut *self.headers.get() }; - if headers.last().is_some_and(|last| last.op > op) { + if headers.back().is_some_and(|last| last.op > op) { self.out_of_order.set(true); } if !headers.is_empty() { + let position = self.front_position.get() + headers.len(); self.first_header_by_op .borrow_mut() .entry(op) - .or_insert(headers.len()); + .or_insert(position); } - headers.push(header); + headers.push_back(header); }; let storage_offset = { @@ -1360,15 +1421,22 @@ impl PartitionJournal { .set(self.resident_control_ops.get() + 1); } - if let Some((offset, timestamp)) = index_offset_timestamp { + if let Some(meta) = meta { let offset_to_op = unsafe { &mut *self.offset_to_op.get() }; - offset_to_op.insert(offset, op); + offset_to_op.insert(meta.base_offset, op); + // Keyed on `base_timestamp`, the broker append time stamped into + // replies: the seek hint must live on the same clock as + // `select_batch_slice`'s filter or timestamp polls seek to the + // wrong resident entry. let timestamp_to_op = unsafe { &mut *self.timestamp_to_op.get() }; - timestamp_to_op.insert((timestamp, op), op); - } + timestamp_to_op.insert((meta.base_timestamp, op), op); - meta + self.resident_messages + .set(self.resident_messages.get() + u64::from(meta.message_count)); + self.resident_message_size + .set(self.resident_message_size.get() + meta.total_size); + } } pub fn is_empty(&self) -> bool { @@ -1421,9 +1489,12 @@ where op_to_storage_offset: UnsafeCell::new(BTreeMap::new()), offset_to_op: UnsafeCell::new(BTreeMap::new()), timestamp_to_op: UnsafeCell::new(BTreeMap::new()), - headers: UnsafeCell::new(Vec::new()), + headers: UnsafeCell::new(VecDeque::new()), + front_position: Cell::new(0), inner: UnsafeCell::new(JournalInner { storage }), first_header_by_op: RefCell::new(AHashMap::new()), + resident_messages: Cell::new(0), + resident_message_size: Cell::new(0), evicted_ring: UnsafeCell::new(BTreeMap::new()), evicted_ring_bytes: Cell::new(0), evicted_headers: UnsafeCell::new(VecDeque::new()), @@ -1442,22 +1513,27 @@ where /// Forget every resident header and index. The caller drained the entries. fn clear_resident_indexes(&self) { unsafe { &mut *self.headers.get() }.clear(); + self.front_position.set(0); self.first_header_by_op.borrow_mut().clear(); unsafe { &mut *self.op_to_storage_offset.get() }.clear(); unsafe { &mut *self.offset_to_op.get() }.clear(); self.held_run.set((0, 0)); unsafe { &mut *self.timestamp_to_op.get() }.clear(); self.resident_control_ops.set(0); + self.resident_messages.set(0); + self.resident_message_size.set(0); self.out_of_order.set(false); } pub fn header_by_op(&self, op: u64) -> Option { let headers = unsafe { &*self.headers.get() }; - if headers.first().is_some_and(|header| header.op == op) { - return headers.first().copied(); + if headers.front().is_some_and(|header| header.op == op) { + return headers.front().copied(); } - let index = *self.first_header_by_op.borrow().get(&op)?; - let header = headers.get(index); + let position = *self.first_header_by_op.borrow().get(&op)?; + let header = position + .checked_sub(self.front_position.get()) + .and_then(|index| headers.get(index)); debug_assert!(header.is_some_and(|header| header.op == op)); header.copied() } @@ -1747,7 +1823,7 @@ impl Journal for PartitionJournal { } async fn append(&self, entry: Self::Entry) -> io::Result<()> { - self.append_with_meta_sync(entry); + self.append_sync(entry); Ok(()) } @@ -1775,7 +1851,7 @@ impl Journal for PartitionJournal { inner.storage.drain() }; // Positional against `headers` until the clear below (see the length-lock - // invariant on `append_with_meta_sync`), so the ops are captured first. + // invariant on `append_sync`), so the ops are captured first. let entry_ops: Vec = { let headers = unsafe { &*self.headers.get() }; headers.iter().map(|header| header.op).collect() @@ -1789,7 +1865,7 @@ impl Journal for PartitionJournal { continue; } // Replays bytes this journal already accepted once, so it cannot fail. - self.append_with_meta_sync(entry); + self.append_sync(entry); } Ok(removed) } @@ -2675,9 +2751,9 @@ mod tests { "read must not evict op 1" ); - let retained = journal.evict_prefix(committed.len()); + journal.evict_prefix(committed.len()); assert_eq!( - retained.len(), + journal.resident_entries().len(), 2, "ops 3 and 4 stay resident after eviction" ); @@ -2688,7 +2764,7 @@ mod tests { let header3 = journal.header_by_op(3).expect("op 3 must be retained"); let header4 = journal.header_by_op(4).expect("op 4 must be retained"); - // Retained entries are still byte-readable after the storage rebuild. + // Retained entries are still byte-readable after the eviction. for header in [header3, header4] { let entry = journal .entry(&header) @@ -2703,8 +2779,7 @@ mod tests { // Advancing the frontier flushes the rest with no gap. let committed = journal.committed_prefix(4); - let rest = journal.evict_prefix(committed.len()); - assert!(rest.is_empty(), "ops 3 and 4 flush on the next evict"); + journal.evict_prefix(committed.len()); assert!(journal.is_empty(), "journal is empty once all ops flushed"); } @@ -2735,8 +2810,12 @@ mod tests { .collect(); assert_eq!(ops, vec![1, 2], "prefix stops before the op 3 gap"); - let retained = journal.evict_prefix(committed.len()); - assert_eq!(retained.len(), 1, "op 4 stays retained past the gap"); + journal.evict_prefix(committed.len()); + assert_eq!( + journal.resident_entries().len(), + 1, + "op 4 stays retained past the gap" + ); assert!(journal.header_by_op(4).is_some(), "op 4 still resident"); } @@ -2973,7 +3052,7 @@ mod tests { ); assert_eq!(journal.resident_control_ops(), 3); - // Eviction rebuilds the poll view from the retained tail. + // Eviction drops the evicted batches from the poll view. journal.evict_prefix(2); assert_eq!(entry_ops(&journal.resident_message_entries()), vec![3, 5]); assert_eq!(journal.resident_control_ops(), 2); @@ -3011,6 +3090,196 @@ mod tests { assert_eq!(journal.resident_control_ops(), 0); } + /// What readers see of the resident journal. The storage offsets of an + /// eviction and of a rebuild differ, so entries are read back through them + /// instead of compared. Probing `held_through` also seeds its cached run. + #[derive(Debug, PartialEq)] + struct ResidentView { + entry_ops: Vec, + headers: Vec>, + reads: Vec<(u64, Option)>, + offsets: BTreeMap, + timestamps: BTreeMap<(u64, u64), u64>, + held_through: Vec, + control_ops: usize, + totals: (u64, u64), + out_of_order: bool, + } + + fn resident_view( + journal: &PartitionJournal, + top_op: u64, + ) -> ResidentView { + let inner = unsafe { &*journal.inner.get() }; + let entries = journal.resident_entries(); + let folded = entries.iter().fold((0, 0), |(messages, size), entry| { + let header = + bytemuck::checked::try_from_bytes::(&entry[..PREPARE_HEADER_SIZE]) + .expect("entry holds a valid prepare header"); + RetainedBatchMeta::of(header, entry).map_or((messages, size), |meta| { + ( + messages + u64::from(meta.message_count), + size + meta.total_size, + ) + }) + }); + let totals = (journal.resident_messages(), journal.resident_message_size()); + assert_eq!(totals, folded, "the totals must count the resident batches"); + ResidentView { + entry_ops: entry_ops(&entries), + headers: (0..=top_op).map(|op| journal.header_by_op(op)).collect(), + reads: unsafe { &*journal.op_to_storage_offset.get() } + .iter() + .map(|(&op, &storage_offset)| { + let entry = inner.storage.read_at_sync(storage_offset); + (op, entry.map(|entry| entry_ops(&[entry])[0])) + }) + .collect(), + offsets: unsafe { &*journal.offset_to_op.get() }.clone(), + timestamps: unsafe { &*journal.timestamp_to_op.get() }.clone(), + held_through: (0..=top_op) + .map(|floor| journal.held_through(floor)) + .collect(), + control_ops: journal.resident_control_ops(), + totals, + out_of_order: journal.out_of_order.get(), + } + } + + async fn journal_with( + entries: &[Frozen<4096>], + ) -> PartitionJournal { + let journal = PartitionJournal::::default(); + for entry in entries { + journal.append(entry.clone()).await.expect("append"); + } + journal + } + + #[compio::test] + async fn given_journal_shapes_when_evicting_a_prefix_should_leave_what_a_rebuild_leaves() { + // `(op, base offset)` per entry, `None` for an offset op. The shapes: + // batches between offset ops, two batches at one base offset, and a + // duplicated op that a prefix either takes whole or splits. + let shapes: [&[(u64, Option)]; 3] = [ + &[ + (1, Some(0)), + (2, None), + (3, Some(3)), + (4, None), + (5, Some(6)), + (6, Some(9)), + (7, None), + ], + &[(1, Some(0)), (2, Some(3)), (3, Some(3)), (4, Some(6))], + &[(1, Some(0)), (2, None), (2, None), (3, Some(3)), (4, None)], + ]; + for shape in shapes { + let entries: Vec<_> = shape + .iter() + .zip(0u128..) + .map(|(&(op, base_offset), checksum)| { + base_offset.map_or_else( + || { + build_prepare(op, HEADER_SIZE + 16) + .transmute_header(|mut old, header: &mut PrepareHeader| { + old.checksum = checksum; + *header = old; + }) + .into_frozen() + }, + |base_offset| build_message_prepare(op, base_offset, 3, 8), + ) + }) + .collect(); + let next_op = shape.iter().map(|(op, _)| op).max().unwrap() + 1; + let next = build_message_prepare(next_op, 100, 3, 8); + for count in 0..=entries.len() + 1 { + let popped = journal_with(&entries).await; + let rebuilt = journal_with(&entries).await; + resident_view(&popped, next_op); + resident_view(&rebuilt, next_op); + + popped.evict_prefix(count); + rebuilt.rebuild_without_prefix(count); + assert_eq!( + resident_view(&popped, next_op), + resident_view(&rebuilt, next_op), + "shape {shape:?}, evicting {count}" + ); + + // Positions taken after the eviction must stay right too. + for journal in [&popped, &rebuilt] { + journal.append(next.clone()).await.expect("append"); + } + popped.evict_prefix(1); + rebuilt.rebuild_without_prefix(1); + assert_eq!( + resident_view(&popped, next_op), + resident_view(&rebuilt, next_op), + "shape {shape:?}, evicting {count} and then 1" + ); + } + } + } + + #[compio::test] + async fn given_a_backfill_inside_the_prefix_when_evicting_should_flush_the_tail_in_order() { + let journal = PartitionJournal::::default(); + for op in [1u64, 2, 5, 3, 4] { + journal + .append(build_control_prepare(op)) + .await + .expect("append"); + } + assert_eq!(journal.committed_prefix_len(0, u64::MAX), 0); + + journal.evict_prefix(3); + + assert_eq!(entry_ops(&journal.resident_entries()), vec![3, 4]); + assert_eq!( + journal.committed_prefix_len(0, u64::MAX), + 2, + "the out-of-order pair left with the prefix" + ); + } + + #[compio::test] + async fn given_every_rebuild_when_counting_resident_messages_should_count_the_batches() { + let journal = PartitionJournal::::default(); + for (op, base_offset) in [(1u64, 0u64), (3, 6), (4, 9), (2, 3)] { + journal + .append(build_message_prepare(op, base_offset, 3, 8)) + .await + .expect("append"); + } + let batch_size = journal.resident_message_size() / 4; + assert_eq!(journal.resident_messages(), 12); + + assert!(journal.sort_by_op()); + assert_eq!(journal.resident_messages(), 12, "sorting keeps every batch"); + + journal.evict_prefix(1); + assert_eq!(journal.resident_messages(), 9); + assert_eq!(journal.resident_message_size(), 3 * batch_size); + + journal.truncate_from(4).await.expect("truncate"); + assert_eq!(journal.resident_messages(), 6, "ops 2 and 3 stay"); + + journal.commit(); + assert_eq!(journal.resident_messages(), 0); + + journal + .append(build_message_prepare(5, 12, 3, 8)) + .await + .expect("append"); + journal.clear_all(); + assert_eq!( + (journal.resident_messages(), journal.resident_message_size()), + (0, 0) + ); + } + fn offset_lookup(offset: u64, count: u32) -> MessageLookup { MessageLookup::Offset { offset, diff --git a/core/shard/src/partition_io.rs b/core/shard/src/partition_io.rs index e853289cad..196530e786 100644 --- a/core/shard/src/partition_io.rs +++ b/core/shard/src/partition_io.rs @@ -7239,10 +7239,8 @@ mod tests { .restore_commit_state(commit, commit); } let journal = &backup.partition(namespace).log.journal().inner; - assert_eq!( - journal.evict_prefix(adopted).len(), - prepares.len() - adopted - ); + journal.evict_prefix(adopted); + assert_eq!(journal.resident_entries().len(), prepares.len() - adopted); assert!(journal.header_by_op(ADOPTED_HEAD).is_none()); assert!(journal.repair_header(ADOPTED_HEAD).is_some()); } From c0037274e09fce5600557cf878a0b671c0003c91 Mon Sep 17 00:00:00 2001 From: Hubert Gruszecki Date: Fri, 30 Oct 2026 10:11:26 +0100 Subject: [PATCH 08/10] fix(integration): remove the duplicate fast consumer shutdown The rebase onto #4687 kept a second shutdown of the fast consumer after master already shuts it down and drops it, so the integration tests did not compile. --- .../tests/server/scenarios/delete_segments_scenario.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/integration/tests/server/scenarios/delete_segments_scenario.rs b/core/integration/tests/server/scenarios/delete_segments_scenario.rs index 3f0cabb800..09959c71fa 100644 --- a/core/integration/tests/server/scenarios/delete_segments_scenario.rs +++ b/core/integration/tests/server/scenarios/delete_segments_scenario.rs @@ -775,8 +775,6 @@ pub async fn run_multi_consumer_barrier(harness: &mut TestHarness, restart_serve (TOTAL_MESSAGES - 1) as u64, ) .await; - // Stop background polling before the deliberate server restarts. - fast_consumer.shutdown().await.unwrap(); // --- Set up slow standalone consumer: store offset at 0 --- let slow_consumer = Consumer { From f608e66b78afdbcd60d3a73351c692bb83be851f Mon Sep 17 00:00:00 2001 From: Hubert Gruszecki Date: Fri, 30 Oct 2026 10:15:16 +0100 Subject: [PATCH 09/10] fix(cluster): read repair bodies and transfer chunks off the shard pump A serving shard read released prepare bodies and transfer chunks inline on its message pump. Each read that went to disk stalled the frames, ticks and replies of every group on that shard. A replica that catches up after a long outage can need gigabytes of such reads from its peers. The pump now decides what to serve, and a detached task reads the bytes. When the bytes land, the pump serves the request again against the current state. Reads run one at a time per shard, and the next starts only after the pump took the last result. --- core/partitions/src/iggy_partition.rs | 13 +- core/partitions/src/journal.rs | 197 ++++++++++----- core/partitions/src/lib.rs | 4 +- core/shard/src/lib.rs | 300 +++++++++++++++++------ core/shard/src/partition_io.rs | 333 +++++++++++++++++++++++++- core/shard/src/router.rs | 9 +- core/shard/src/served_reads.rs | 218 +++++++++++++++++ 7 files changed, 919 insertions(+), 155 deletions(-) create mode 100644 core/shard/src/served_reads.rs diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs index 34ab4184b1..35b02c6c91 100644 --- a/core/partitions/src/iggy_partition.rs +++ b/core/partitions/src/iggy_partition.rs @@ -17497,12 +17497,13 @@ mod tests { .expect("the byte cap released this body") }) .collect(); - let mut bodies = Vec::new(); - crate::read_released_bodies(&partition.log, &released, &mut bodies) - .await - .unwrap(); - assert_eq!(bodies.len(), released.len()); - for (body, prepare) in bodies.iter().zip(&prepares) { + let mut bodies = crate::ReleasedRead::new(&partition.log, released.clone()) + .read() + .await; + for (evicted, prepare) in released.iter().zip(&prepares) { + let body = bodies + .take(evicted) + .expect("the read brings every body back"); assert_eq!(body.as_slice(), prepare.as_slice()); } diff --git a/core/partitions/src/journal.rs b/core/partitions/src/journal.rs index 4ab6984fa0..b94018158d 100644 --- a/core/partitions/src/journal.rs +++ b/core/partitions/src/journal.rs @@ -163,69 +163,152 @@ impl EvictedHeader { pub const fn segment_start(&self) -> u64 { self.body.segment_start } + + const fn same_prepare(&self, other: &Self) -> bool { + self.header.op == other.header.op && self.header.checksum == other.header.checksum + } } -/// Reads the bodies of `released`, in op order, back from the log files of -/// their segments in `log`, and pushes each prepare to `bodies`. Bodies that -/// lie back to back in one file come back in one read. -/// -/// # Errors -/// Returns an error if a segment left `log`, if a body cannot be read, or if -/// the bytes there are not the released body: a torn or corrupt batch, or -/// another batch at the same position. Such bytes are never returned. The -/// bodies before the one that failed, `released[bodies.len()]`, stay in -/// `bodies`. -pub async fn read_released_bodies( - log: &SegmentedLog, - released: &[EvictedHeader], - bodies: &mut Vec, -) -> io::Result<()> { - let mut file: Option<(u64, compio::fs::File)> = None; - let runs = released - .chunk_by(EvictedHeader::followed_by) - .flat_map(|run| run.chunks(IOV_MAX)); - for run in runs { - let Some(first) = run.first() else { - continue; - }; - let segment = first.body.segment_start; - let file = match &mut file { - Some((start, file)) if *start == segment => file, - slot => { - let path = log.messages_path(segment).ok_or_else(|| { - io::Error::new(io::ErrorKind::NotFound, "the segment left the log") - })?; - &slot - .insert((segment, compio::fs::File::open(path).await?)) - .1 +/// The released bodies that one serve reads back, with the log file of each +/// segment that holds one. It owns what it reads, so a task that runs beside +/// the pump can read the bodies while the log changes. +#[derive(Debug)] +pub struct ReleasedRead { + released: Vec, + /// The log file of each segment by its start offset, `None` when the + /// segment left the log. + files: Vec<(u64, Option)>, +} + +impl ReleasedRead { + #[must_use] + pub fn new(log: &SegmentedLog, released: Vec) -> Self { + let mut files: Vec<(u64, Option)> = Vec::new(); + for evicted in &released { + let segment = evicted.body.segment_start; + if files.iter().all(|(start, _)| *start != segment) { + files.push((segment, log.messages_path(segment))); } + } + Self { released, files } + } + + /// Reads the bodies back in op order. Bodies that lie back to back in one + /// file come back in one read. + /// + /// The read stops at the first body that it cannot serve: the segment + /// left the log, the read failed, or the bytes there are not the released + /// body, such as a torn or corrupt batch or another batch at the same + /// position. Such bytes are never returned. + pub async fn read(self) -> ReleasedBodies { + let mut bodies = ReleasedBodies { + read: VecDeque::with_capacity(self.released.len()), + failed: None, }; - // A vectored file read in compio fills only the initialized bytes of - // each buffer, so the bodies start zeroed. - let buffers: Vec<_> = run - .iter() - .map(|evicted| { - let size = evicted.header.size as usize; - let mut prepare = Owned::<4096>::zeroed(size); - prepare.as_mut_slice()[..PREPARE_HEADER_SIZE] - .copy_from_slice(bytemuck::bytes_of(&evicted.header)); - prepare.slice(PREPARE_HEADER_SIZE..size) - }) - .collect(); - let compio::BufResult(read, buffers) = file - .read_vectored_exact_at(buffers, first.body.position) - .await; - for (evicted, body) in run.iter().zip(buffers) { - match evicted.verified(body.into_inner()) { - Ok(prepare) => bodies.push(prepare), - // Only the released body itself passes the checks, and a - // failed read leaves zeros where it stopped, so after a failed - // read the first body that fails is where the read stopped. - Err(error) => return Err(read.err().unwrap_or(error)), + if let Err(error) = self.read_runs(&mut bodies.read).await { + bodies.failed = self + .released + .get(bodies.read.len()) + .map(|evicted| (*evicted, error)); + } + bodies + } + + async fn read_runs( + &self, + read: &mut VecDeque<(EvictedHeader, JournalBuffer)>, + ) -> io::Result<()> { + let mut file: Option<(u64, compio::fs::File)> = None; + let runs = self + .released + .chunk_by(EvictedHeader::followed_by) + .flat_map(|run| run.chunks(IOV_MAX)); + for run in runs { + let Some(first) = run.first() else { + continue; + }; + let segment = first.body.segment_start; + let file = match &mut file { + Some((start, file)) if *start == segment => file, + slot => { + let path = self + .files + .iter() + .find(|(start, _)| *start == segment) + .and_then(|(_, path)| path.as_deref()) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "the segment left the log") + })?; + &slot + .insert((segment, compio::fs::File::open(path).await?)) + .1 + } + }; + // A vectored file read in compio fills only the initialized bytes of + // each buffer, so the bodies start zeroed. + let buffers: Vec<_> = run + .iter() + .map(|evicted| { + let size = evicted.header.size as usize; + let mut prepare = Owned::<4096>::zeroed(size); + prepare.as_mut_slice()[..PREPARE_HEADER_SIZE] + .copy_from_slice(bytemuck::bytes_of(&evicted.header)); + prepare.slice(PREPARE_HEADER_SIZE..size) + }) + .collect(); + let compio::BufResult(result, buffers) = file + .read_vectored_exact_at(buffers, first.body.position) + .await; + for (evicted, body) in run.iter().zip(buffers) { + match evicted.verified(body.into_inner()) { + Ok(prepare) => read.push_back((*evicted, prepare)), + // Only the released body itself passes the checks, and a + // failed read leaves zeros where it stopped, so after a failed + // read the first body that fails is where the read stopped. + Err(error) => return Err(result.err().unwrap_or(error)), + } } } + Ok(()) + } +} + +/// The bodies that a [`ReleasedRead`] brought back, in op order. +#[derive(Debug)] +pub struct ReleasedBodies { + read: VecDeque<(EvictedHeader, JournalBuffer)>, + /// The body that the read could not serve, and why. The read stopped + /// there. + failed: Option<(EvictedHeader, io::Error)>, +} + +impl ReleasedBodies { + /// The body read back for `evicted`. Callers take ops in ascending order, + /// so the bodies of lower ops are dropped. + pub fn take(&mut self, evicted: &EvictedHeader) -> Option { + while self + .read + .front() + .is_some_and(|(front, _)| front.header.op < evicted.header.op) + { + self.read.pop_front(); + } + if !self + .read + .front() + .is_some_and(|(front, _)| front.same_prepare(evicted)) + { + return None; + } + self.read.pop_front().map(|(_, body)| body) + } + + /// Why the read could not serve `evicted`, if it stopped there. + pub fn take_failure(&mut self, evicted: &EvictedHeader) -> Option { + self.failed + .take_if(|(failed, _)| failed.same_prepare(evicted)) + .map(|(_, error)| error) } - Ok(()) } /// A committed prepare the repair ring holds in memory, with the place of its @@ -1000,7 +1083,7 @@ impl PartitionJournal { /// The header the evicted ring keeps for `op` after the byte cap released /// its body to a segment. [`Self::repair_entry`] and /// [`Self::repair_header`] do not serve such an op: its body must be read - /// back with [`read_released_bodies`]. + /// back with a [`ReleasedRead`]. #[must_use] pub fn evicted_header(&self, op: u64) -> Option { let headers = unsafe { &*self.evicted_headers.get() }; diff --git a/core/partitions/src/lib.rs b/core/partitions/src/lib.rs index b1e14ccf08..085d06f0c2 100644 --- a/core/partitions/src/lib.rs +++ b/core/partitions/src/lib.rs @@ -73,8 +73,8 @@ pub use io::{ largest_legal_job_charge, }; pub use journal::{ - BodyPosition, EVICTED_RING_BYTES_MAX, EVICTED_RING_CAPACITY, EvictedHeader, - read_released_bodies, + BodyPosition, EVICTED_RING_BYTES_MAX, EVICTED_RING_CAPACITY, EvictedHeader, ReleasedBodies, + ReleasedRead, }; pub use view_fence::ViewWindowVerdict; diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index b3715da850..f2717de183 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs @@ -25,6 +25,7 @@ pub use partition_io::{PartitionIoLimits, PartitionIoLimitsError}; mod poll; mod primary_rebalance; mod router; +mod served_reads; pub mod shards_table; #[cfg(test)] mod test_support; @@ -88,6 +89,7 @@ use partitions::{ FatalCommit, IggyPartition, IggyPartitions, PollFragments, PollingArgs, PollingConsumer, RepairIngest, }; +use served_reads::{LandedRead, ServedReadKey}; use server_common::sharding::{ IggyNamespace, LIST_CLIENTS_GATHER_TIMEOUT, PartitionLocation, ShardId, }; @@ -1225,12 +1227,12 @@ pub const REPAIR_CHUNK_MAX: u64 = 128; /// Prepare bytes one partition `RequestPrepares` round serves. /// -/// The reads run on the pump, so this shard's consensus ticks wait for them. -/// Bodies that lie back to back in a segment come back in one read. Resident -/// responses share the bound so receivers can reserve WAL space. 8 MiB -/// takes about 4 ms from a warm local SSD at 2 GB/s, under one 10 ms tick, -/// but about 67 ms from a block volume at 125 MB/s. The first body always -/// goes, so a batch above the budget still makes progress. +/// Bodies that the ring released come back in one detached read per round, +/// and a shard runs one such read at a time, so this bounds the bytes that +/// read holds. Bodies that lie back to back in a segment come back in one +/// read. Resident responses share the bound so receivers can reserve WAL +/// space. The first body always goes, so a batch above the budget still +/// makes progress. const REPAIR_BYTES_MAX: u64 = 8 * 1024 * 1024; /// One in-flight metadata journal-repair stream (shard 0 only). @@ -1830,6 +1832,8 @@ where /// displace their results. Only the owner pump validates completions. poll_completions: poll::completion::PollCompletionLane, partition_io: partition_io::PartitionIoLane, + /// Disk reads that answer peers, which the pump serves once they land. + served_reads: served_reads::ServedReads, cooperative_events: Cell, deferred_polls: poll::deferred::DeferredPolls, @@ -2170,6 +2174,7 @@ where reply_inbox, poll_completions, partition_io, + served_reads: served_reads::ServedReads::default(), cooperative_events: Cell::new(0), deferred_polls: poll::deferred::DeferredPolls::new(metrics.deferred_polls.clone()), shards_table, @@ -3067,6 +3072,7 @@ where .expect("simulator supports the production partition I/O limits"), &metrics, ), + served_reads: served_reads::ServedReads::default(), cooperative_events: Cell::new(0), shards_table, partition_consensus, @@ -6178,7 +6184,11 @@ where .await; } Some(consensus::ReplicationWork::Replay { from_op, .. }) - if replay.is_none_or(|(selected_group, selected_peer, _)| { + if !self.served_reads.reading(ServedReadKey::Prepares { + group, + target: replica, + live: true, + }) && replay.is_none_or(|(selected_group, selected_peer, _)| { rank((group, replica)) < rank((selected_group, selected_peer)) }) => { @@ -6210,13 +6220,78 @@ where header.to_op = consensus.sequencer().current_sequence(); }) }; - if self.serve_prepares(&request, true).await < from_op + let served = self.serve_prepares(&request, true, None).await; + self.defer_unserved_replay(group, replica, from_op, served); + } + + /// A replay that served nothing waits before the next one, so a body or a + /// peer that is not there is not asked for on every tick. + fn defer_unserved_replay(&self, group: u64, replica: u8, from_op: u64, served: Option) + where + B: MessageBus, + { + if served.is_some_and(|served| served < from_op) && let Some(consensus) = self.replication_consensus(group) { consensus.defer_replication(replica); } } + /// Serve the oldest request whose disk read landed, against the state of + /// now, as the pump serves any request. + #[allow(clippy::future_not_send)] + pub(crate) async fn serve_landed_read(&self) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: + Journal, Header = PrepareHeader>, + { + match self.served_reads.take_landed() { + Some(LandedRead::Prepares { + request, + live, + mut bodies, + }) => { + let served = self.serve_prepares(&request, live, Some(&mut bodies)).await; + if live { + let header = request.header(); + self.defer_unserved_replay( + header.group, + header.replica, + header.from_op, + served, + ); + } + } + Some(LandedRead::Chunk { + header, + artifact, + load, + }) => { + self.serve_partition_chunk(header, Some((artifact, load))) + .await; + } + None => {} + } + } + + /// Serve landed reads, as the pump does, until no read runs or waits. + #[cfg(test)] + #[allow(clippy::future_not_send)] + pub(crate) async fn settle_served_reads(&self) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: + Journal, Header = PrepareHeader>, + { + while !self.served_reads.is_empty() { + self.served_reads.landed().await; + self.serve_landed_read().await; + } + } + fn replication_consensus(&self, group: u64) -> Option<&VsrConsensus> where B: MessageBus, @@ -6257,15 +6332,25 @@ where ::Target: Journal, Header = PrepareHeader>, { - self.serve_prepares(msg, false).await; + self.serve_prepares(msg, false, None).await; } /// Serve a repair range from this replica's journal: stream /// `RepairPrepare` frames (stored prepares verbatim, command byte /// rewritten) in op order, prefixed by `RangeEvicted` when the front of /// the range is no longer retained, terminated by `RepairDone`. + /// + /// Returns the last op served. A partition range that needs bodies the + /// ring released sends nothing and returns `None`: a detached read brings + /// the bodies back, and the pump serves the request again with them in + /// `landed`. #[allow(clippy::future_not_send, clippy::too_many_lines)] - async fn serve_prepares(&self, msg: &Message, live: bool) -> u64 + async fn serve_prepares( + &self, + msg: &Message, + live: bool, + mut landed: Option<&mut partitions::ReleasedBodies>, + ) -> Option where B: MessageBus, MJ: JournalHandle, @@ -6285,13 +6370,13 @@ where // Other requesters may receive only those peers' committed prefix. if consensus.is_transferring() { // Transfer can replace these entries. - return header.from_op.saturating_sub(1); + return Some(header.from_op.saturating_sub(1)); } if !matches!(consensus.status(), Status::Normal | Status::ViewChange) { - return header.from_op.saturating_sub(1); + return Some(header.from_op.saturating_sub(1)); } let Some(journal) = planes.0.journal.as_ref() else { - return header.from_op.saturating_sub(1); + return Some(header.from_op.saturating_sub(1)); }; let journal = journal.handle(); let cluster = consensus.cluster(); @@ -6312,7 +6397,7 @@ where // servable anyway. The partition arm jumps to `retained_from` likewise. let mut from_op = header.from_op.max(journal.snapshot_op() + 1); if !consensus.view_log_entry_proven(from_op) { - return header.from_op.saturating_sub(1); + return Some(header.from_op.saturating_sub(1)); } #[allow(clippy::cast_possible_truncation)] while from_op <= to_op && journal.header(from_op as usize).is_none() { @@ -6320,7 +6405,7 @@ where } if from_op > to_op { if live { - return header.from_op.saturating_sub(1); + return Some(header.from_op.saturating_sub(1)); } // Nothing in the requested range is retained. Answer the // eviction honestly: a bare `RepairDone(to_op)` here would @@ -6348,11 +6433,11 @@ where consensus.partition_incarnation(), ) .await; - return header.from_op.saturating_sub(1); + return Some(header.from_op.saturating_sub(1)); } if from_op > header.from_op { if live { - return header.from_op.saturating_sub(1); + return Some(header.from_op.saturating_sub(1)); } self.send_repair_range_reply( cluster, @@ -6412,25 +6497,33 @@ where ) .await; } - return served_through; + return Some(served_through); } let namespace = IggyNamespace::from_raw(header.group); let Some(partition) = planes.1.0.get_mut_by_ns(&namespace) else { - return header.from_op.saturating_sub(1); + return Some(header.from_op.saturating_sub(1)); }; if !self.partition_incarnation_matches( partition, header.partition_incarnation, Command::RequestPrepares, ) { - return header.from_op.saturating_sub(1); + return Some(header.from_op.saturating_sub(1)); } if !matches!( partition.consensus().status(), Status::Normal | Status::ViewChange ) || partition.view_truncation_pending() { - return header.from_op.saturating_sub(1); + return Some(header.from_op.saturating_sub(1)); + } + let read_key = ServedReadKey::Prepares { + group: header.group, + target, + live, + }; + if landed.is_none() && self.served_reads.reading(read_key) { + return None; } if !live { partition.renew_transfer_tail_for_repair(target, self.ticks.get()); @@ -6470,23 +6563,13 @@ where .unwrap_or_else(|| partition.consensus().commit_min().saturating_add(1)); let mut from_op = header.from_op; if !partition.consensus().view_log_entry_proven(from_op) { - return header.from_op.saturating_sub(1); + return Some(header.from_op.saturating_sub(1)); } - if retained_from > from_op { + let range_evicted = retained_from > from_op; + if range_evicted { if live { - return header.from_op.saturating_sub(1); + return Some(header.from_op.saturating_sub(1)); } - self.send_repair_range_reply( - cluster, - self_id, - target, - Command::RangeEvicted, - header.nonce, - retained_from, - header.group, - partition.created_revision(), - ) - .await; from_op = retained_from; } let chunk_ops_max = if live { @@ -6496,10 +6579,9 @@ where }; let chunk_end = to_op.min(from_op.saturating_add(chunk_ops_max - 1)); let journal = &partition.log.journal().inner; - // Every op is checked and taken before the first await, and the bodies - // the ring released come back from disk in one pass, so the bodies that - // lie back to back in a segment take one read. `None` stands for the - // next body read back. + // Every op is checked and taken before the first await. A body that the + // ring released comes from `landed`, and `None` stands for one that no + // read brought back yet. let mut entries = Vec::new(); let mut released = Vec::new(); let mut response_bytes = 0; @@ -6532,8 +6614,13 @@ where } response_bytes += size; if let Some(evicted) = evicted { - released.push(evicted); - entries.push(None); + let body = landed + .as_deref_mut() + .and_then(|landed| landed.take(&evicted)); + if body.is_none() { + released.push(evicted); + } + entries.push(body); } else { let Some(entry) = journal.repair_entry(op) else { break; @@ -6541,18 +6628,41 @@ where entries.push(Some(entry)); } } - let mut bodies = Vec::with_capacity(released.len()); - let failed = - match partitions::read_released_bodies(&partition.log, &released, &mut bodies).await { - Ok(()) => None, - Err(error) => released - .get(bodies.len()) - .map(|evicted| (evicted.header().op, error)), - }; - let mut bodies = bodies.into_iter(); + if landed.is_none() && !released.is_empty() { + // Bodies that lie back to back in a segment take one read. Nothing + // goes out before they land, so the frames keep op order. + let read = partitions::ReleasedRead::new(&partition.log, released); + let request = msg.clone(); + self.served_reads.start(&self.bus, read_key, async move { + LandedRead::Prepares { + request, + live, + bodies: read.read().await, + } + }); + return None; + } + if range_evicted { + self.send_repair_range_reply( + cluster, + self_id, + target, + Command::RangeEvicted, + header.nonce, + retained_from, + header.group, + partition.created_revision(), + ) + .await; + } + // The serve ends at the first body that no read brought back. + let failed = released.first().and_then(|unread| { + let error = landed.as_mut()?.take_failure(unread)?; + Some((unread.header().op, error)) + }); let mut served_through = from_op.saturating_sub(1); for (op, entry) in (from_op..).zip(entries) { - let Some(entry) = entry.or_else(|| bodies.next()) else { + let Some(entry) = entry else { break; }; if !self @@ -6610,7 +6720,7 @@ where "served partition repair range" ); } - served_through + Some(served_through) } /// Ingest one repaired prepare. Metadata journals it into the WAL (the @@ -10410,27 +10520,44 @@ where .await; } + #[allow(clippy::future_not_send)] + async fn on_partition_request_state_chunk(&self, msg: &Message) + where + B: MessageBus, + { + self.serve_partition_chunk(*msg.header(), None).await; + } + /// Serve one partition chunk. Segment payloads are loaded from disk on /// demand into the shard-wide [`ServedSegmentCache`] (content-addressed, /// so simultaneous rejoiners share one resident copy); a load or /// re-verification failure (GC unlinked the file, bytes changed) evicts /// the offer and tells the requester to restart with a fresh one -- /// which then reflects the current segment set, so the retry converges. + /// + /// A detached read loads the segment bytes, and the pump serves the + /// request again with them in `loaded`, beside the artifact they came from. #[allow( clippy::future_not_send, clippy::cast_possible_truncation, clippy::too_many_lines )] - async fn on_partition_request_state_chunk(&self, msg: &Message) - where + async fn serve_partition_chunk( + &self, + header: RequestStateChunkHeader, + mut loaded: Option<( + consensus::StateArtifact, + Result, partitions::state_transfer::SegmentLoadError>, + )>, + ) where B: MessageBus, { - // TODO: Move partition offer hashing, segment loads and receiver spills - // into bounded file-lane jobs so they cannot stall consensus ticks. + // TODO: Move partition offer hashing off the pump, so it cannot stall + // consensus ticks. // Pass 1 inside the borrow decides; a segment artifact that is not - // resident exits with its path and is loaded OUTSIDE the borrow (a - // RefCell borrow must not be held across an await), then pass 2 - // stores and serves it. + // resident exits with its path and a detached read loads it (a + // RefCell borrow must not be held across an await), then pass 2 of + // the serve that gets the bytes stores and serves them. enum ChunkAttempt { Reply(Option), Load { @@ -10439,11 +10566,17 @@ where }, } - let header = *msg.header(); // The requester id keys the offer map, whose bound is the replica count. if !self.peer_is_known(header.replica, "RequestStateChunk") { return; } + let read_key = ServedReadKey::Chunk { + group: header.group, + requester: header.replica, + }; + if loaded.is_none() && self.served_reads.reading(read_key) { + return; + } let planes = self.plane.inner(); let Some(partition) = planes.1.0.get_by_ns(&IggyNamespace::from_raw(header.group)) else { return; @@ -10567,28 +10700,43 @@ where match attempt { ChunkAttempt::Reply(reply) => break reply, ChunkAttempt::Load { log_path, entry } => { - // A whole-segment read on the pump can outlive the requester's - // retry budget. Cache only artifacts that fit one chunk; - // the receiver verifies larger artifacts before install. + // A whole-segment read can outlive the requester's retry + // budget. Cache only artifacts that fit one chunk; the + // receiver verifies larger artifacts before install. let cache_whole = header.offset == 0 && entry.len <= chunk_len_max as u64; - let loaded = if cache_whole { - partitions::state_transfer::load_verified_segment_artifact( - &log_path, &entry, - ) - .await - } else { + // The offer can change while a read runs. Bytes of another + // artifact must never serve, or fill the cache for, this one. + let Some(load) = loaded + .take() + .and_then(|(artifact, load)| (artifact == entry).then_some(load)) + else { let len = u64::from(header.len) .min(chunk_len_max as u64) .min(entry.len - header.offset) as usize; - partitions::state_transfer::load_segment_chunk( - &log_path, - header.offset, - len, - ) - .await + self.served_reads.start(&self.bus, read_key, async move { + let load = if cache_whole { + partitions::state_transfer::load_verified_segment_artifact( + &log_path, &entry, + ) + .await + } else { + partitions::state_transfer::load_segment_chunk( + &log_path, + header.offset, + len, + ) + .await + }; + LandedRead::Chunk { + header, + artifact: entry, + load, + } + }); + return; }; - let reason = match loaded { + let reason = match load { Ok(bytes) => { if cache_whole { self.served_segment_cache.borrow_mut().insert( diff --git a/core/shard/src/partition_io.rs b/core/shard/src/partition_io.rs index 196530e786..d920751188 100644 --- a/core/shard/src/partition_io.rs +++ b/core/shard/src/partition_io.rs @@ -1296,6 +1296,11 @@ mod tests { owner .on_message(MessageBag::RequestStateChunk(request)) .await; + assert!( + sent.borrow().is_empty(), + "the chunk goes out once its read lands" + ); + owner.settle_served_reads().await; let frames = sent.borrow(); assert_eq!(frames.len(), 1, "one reply for the requested chunk"); assert_eq!( @@ -1315,6 +1320,90 @@ mod tests { } } + /// The offer can change while a chunk read runs. Bytes read for the + /// artifact of the old offer never answer the request or fill the cache: + /// the serve reads the artifact of the offer of now. + #[compio::test] + async fn given_a_chunk_read_when_the_offer_changes_before_it_lands_should_serve_the_new_artifact() + { + const NONCE: u128 = 1; + const OLD: &[u8] = b"old segment bytes"; + const NEW: &[u8] = b"new segment bytes"; + const CHUNK_LEN: u32 = 8; + let bus = Rc::new(IggyMessageBus::new(0)); + let sent = Rc::new(RefCell::new(Vec::new())); + let captured = Rc::clone(&sent); + bus.set_replica_forward_fn(Box::new(move |_, _, frame| { + captured.borrow_mut().push(frame); + Ok(()) + })); + connect_replica_on(&bus, TRANSFER_REQUESTER, 1); + let (mut owner, _sender) = test_owner(&bus, None); + owner.partition_consensus.replica_count = TRANSFER_REQUESTER + 1; + let namespace = IggyNamespace::new(0, 0, 0); + owner + .plane + .partitions() + .get_mut_by_ns(&namespace) + .unwrap() + .set_created_revision(SERVED_INCARNATION); + let directory = tempfile::tempdir().unwrap(); + let offer_of = |name: &str, bytes: &[u8]| { + let log_path = directory.path().join(name); + std::fs::write(&log_path, bytes).unwrap(); + let offer = partitions::state_transfer::PartitionStateTransferOffer { + commit_op: 0, + segments: vec![partitions::state_transfer::SegmentArtifactSource { + entry: StateArtifact::for_bytes(artifact_kind::SEGMENT_LOG, 0, bytes), + log_path: log_path.to_str().unwrap().to_owned(), + }], + offsets: ( + StateArtifact::for_bytes(artifact_kind::CONSUMER_OFFSETS, 0, &[]), + Rc::new(Vec::new()), + ), + }; + crate::ServedStateTransfer { + nonce: NONCE, + offer: crate::ServedOffer::Partition(Rc::new(offer)), + idle_ticks: 0, + fully_served: false, + } + }; + let offer_key = (namespace.inner(), TRANSFER_REQUESTER); + owner + .state_transfer_offers + .borrow_mut() + .insert(offer_key, offer_of("old.log", OLD)); + let request = Message::::new(size_of::()) + .transmute_header(|_, header: &mut RequestStateChunkHeader| { + header.command = Command::RequestStateChunk; + header.size = u32::try_from(size_of::()).unwrap(); + header.cluster = 1; + header.group = namespace.inner(); + header.replica = TRANSFER_REQUESTER; + header.partition_incarnation = SERVED_INCARNATION; + header.nonce = NONCE; + header.len = CHUNK_LEN; + header.seal(); + }); + + owner + .on_message(MessageBag::RequestStateChunk(request)) + .await; + owner + .state_transfer_offers + .borrow_mut() + .insert(offer_key, offer_of("new.log", NEW)); + owner.settle_served_reads().await; + + let frames = sent.borrow(); + assert_eq!(frames.len(), 1, "one reply for the requested chunk"); + let chunk = + Message::::try_from(Owned::copy_from_slice(frames[0].as_slice())) + .unwrap(); + assert_eq!(chunk.body(), &NEW[..CHUNK_LEN as usize]); + } + /// A request frame that fails the gates, here one with a stale partition /// incarnation, must leave the parked request of its requester in place, /// so the tick answers that request once its plan pins. @@ -8840,6 +8929,196 @@ mod tests { ); } + /// A round that needs released bodies sends nothing until a detached read + /// brings them back, so the pump goes on while the disk works: another + /// requester's round of a resident op goes out first. A request that + /// repeats the round waits for its read. Once the bodies land, the round + /// goes out in op order and `RepairDone` ends it. + #[compio::test] + async fn given_a_read_of_released_bodies_when_other_requests_arrive_should_serve_them_meanwhile() + { + const OTHER_REQUESTER: u8 = 0; + let donor = Box::pin(SegmentDonor::new(4)).await; + connect_replica_on(&donor.owner.bus, OTHER_REQUESTER, 1); + donor.sent.borrow_mut().clear(); + let request = repair_request(donor.namespace, 1, donor.prepares.len()); + let resident = repair_request(donor.namespace, 4, donor.prepares.len()).transmute_header( + |old, header: &mut RequestPreparesHeader| { + *header = old; + header.replica = OTHER_REQUESTER; + }, + ); + + let held = donor.owner.served_reads.hold(); + donor.owner.on_request_prepares(&request).await; + donor.owner.on_request_prepares(&request).await; + donor.owner.on_request_prepares(&resident).await; + assert_eq!( + sent_answers(&donor.sent, &donor.prepares), + [(Command::RepairPrepare, 4), (Command::RepairDone, 4)], + "only the resident round goes out while the read waits" + ); + + drop(held); + donor.owner.settle_served_reads().await; + assert_eq!( + sent_answers(&donor.sent, &donor.prepares), + [ + (Command::RepairPrepare, 1), + (Command::RepairPrepare, 2), + (Command::RepairPrepare, 3), + (Command::RepairPrepare, 4), + (Command::RepairDone, 4), + ], + "the repeated request is answered once" + ); + } + + /// The running pump serves other frames while a round waits for its + /// released bodies, and answers that round once the bodies land. + #[compio::test] + async fn given_a_running_pump_when_a_round_waits_for_released_bodies_should_serve_other_frames() + { + const OTHER_REQUESTER: u8 = 0; + const PUMP_WAITS_MAX: usize = 1000; + let donor = &Box::pin(SegmentDonor::new(4)).await; + connect_replica_on(&donor.owner.bus, OTHER_REQUESTER, 1); + donor.sent.borrow_mut().clear(); + let deliver = |request: Message| { + donor + .sender + .try_send(ShardFrame::consensus( + donor.owner.id, + MessageBag::RequestPrepares(request), + )) + .expect("fixture request fits the inbox"); + }; + let sent_frames = |count: usize| async move { + for _ in 0..PUMP_WAITS_MAX { + if donor.sent.borrow().len() >= count { + return; + } + donor.owner.bus.sleep(Duration::from_millis(1)).await; + } + panic!( + "the pump sent {} of {count} frames", + donor.sent.borrow().len() + ); + }; + let (stop, stopped) = channel(1); + let pump = donor + .owner + .run_message_pump(stopped, Arc::new(AtomicBool::new(false))); + let exercise = async { + let held = donor.owner.served_reads.hold(); + deliver(repair_request(donor.namespace, 1, donor.prepares.len())); + deliver( + repair_request(donor.namespace, 4, donor.prepares.len()).transmute_header( + |old, header: &mut RequestPreparesHeader| { + *header = old; + header.replica = OTHER_REQUESTER; + }, + ), + ); + sent_frames(2).await; + assert_eq!( + sent_answers(&donor.sent, &donor.prepares), + [(Command::RepairPrepare, 4), (Command::RepairDone, 4)] + ); + + drop(held); + sent_frames(5).await; + assert_eq!( + sent_answers(&donor.sent, &donor.prepares), + [ + (Command::RepairPrepare, 1), + (Command::RepairPrepare, 2), + (Command::RepairPrepare, 3), + (Command::RepairPrepare, 4), + (Command::RepairDone, 4), + ] + ); + stop.try_send(()).unwrap(); + }; + let (fault, ()) = futures::join!(pump, exercise); + assert!(fault.is_none()); + } + + /// The bodies can land after the partition left the shard. The serve + /// checks the state again, so it sends nothing, and a later request reads + /// again. + #[compio::test] + async fn given_a_read_of_released_bodies_when_the_partition_leaves_should_send_nothing() { + let donor = Box::pin(SegmentDonor::new(4)).await; + let partitions = donor.owner.plane.partitions(); + donor.sent.borrow_mut().clear(); + donor + .owner + .on_request_prepares(&repair_request(donor.namespace, 1, donor.prepares.len())) + .await; + let partition = partitions.remove(&donor.namespace).unwrap(); + donor.owner.settle_served_reads().await; + assert!(donor.sent.borrow().is_empty()); + + partitions.insert(donor.namespace, partition); + assert_eq!( + donor.answers(1).await, + [ + (Command::RepairPrepare, 1), + (Command::RepairPrepare, 2), + (Command::RepairPrepare, 3), + (Command::RepairPrepare, 4), + (Command::RepairDone, 4), + ] + ); + } + + /// A read starts only after the pump took the bytes of the read before + /// it, so a busy pump holds the bodies of one round, not of every round + /// that waits. + #[compio::test] + async fn given_a_landed_read_when_the_pump_has_not_taken_it_should_start_no_other_read() { + const OTHER_REQUESTER: u8 = 0; + // Ample time for a second read of the fixture segment to land, if it + // could start before the take. + const SECOND_READ_WINDOW: Duration = Duration::from_millis(10); + let donor = Box::pin(SegmentDonor::new(4)).await; + connect_replica_on(&donor.owner.bus, OTHER_REQUESTER, 1); + donor.sent.borrow_mut().clear(); + let round = [ + (Command::RepairPrepare, 1), + (Command::RepairPrepare, 2), + (Command::RepairPrepare, 3), + (Command::RepairPrepare, 4), + (Command::RepairDone, 4), + ]; + let other = repair_request(donor.namespace, 1, donor.prepares.len()).transmute_header( + |old, header: &mut RequestPreparesHeader| { + *header = old; + header.replica = OTHER_REQUESTER; + }, + ); + donor + .owner + .on_request_prepares(&repair_request(donor.namespace, 1, donor.prepares.len())) + .await; + donor.owner.on_request_prepares(&other).await; + + donor.owner.served_reads.landed().await; + donor.owner.bus.sleep(SECOND_READ_WINDOW).await; + assert_eq!( + donor.owner.served_reads.landed_count(), + 1, + "the second read waits for the pump to take the first" + ); + + donor.owner.settle_served_reads().await; + assert_eq!( + sent_answers(&donor.sent, &donor.prepares), + [round, round].concat() + ); + } + /// A replay to a backup reads no released body that the replication /// window refuses. A window with room for op 1 reads op 1 only, and a full /// window reads nothing. The body of op 2 is corrupt, so a read of it @@ -8908,14 +9187,19 @@ mod tests { .repair_retained_from() }; - assert_eq!(owner.serve_prepares(&replay(1), true).await, 1); + assert_eq!( + owner.serve_prepares(&replay(1), true, None).await, + None, + "the body of op 1 is on disk" + ); + owner.settle_served_reads().await; assert_eq!(*replayed.borrow(), [1], "the window must take op 1 only"); assert_eq!( retained_from(), Some(1), "the replay must not read the body of op 2 that the window refuses" ); - assert_eq!(owner.serve_prepares(&replay(2), true).await, 1); + assert_eq!(owner.serve_prepares(&replay(2), true, None).await, Some(1)); assert_eq!(*replayed.borrow(), [1], "a full window must take nothing"); assert_eq!( retained_from(), @@ -9059,8 +9343,8 @@ mod tests { } /// Asks `owner` for ops `from_op..` of `prepares` as the primary of - /// [`REPAIR_VIEW`], and returns the command and op of each frame it sent. - /// Each `RepairPrepare` must carry its prepare from `prepares` verbatim. + /// [`REPAIR_VIEW`], serves the reads that land, and returns what + /// [`sent_answers`] finds. #[allow(clippy::future_not_send)] async fn repair_answers( owner: &IoTestShard, @@ -9069,21 +9353,43 @@ mod tests { from_op: u64, prepares: &[Message], ) -> Vec<(Command, u64)> { - const COMMAND_OFFSET: usize = std::mem::offset_of!(GenericHeader, command); - let request = Message::::new(size_of::()) - .transmute_header(|_, header: &mut RequestPreparesHeader| { + sent.borrow_mut().clear(); + owner + .on_request_prepares(&repair_request(namespace, from_op, prepares.len())) + .await; + owner.settle_served_reads().await; + sent_answers(sent, prepares) + } + + /// A request of [`REPAIR_PRIMARY`] in [`REPAIR_VIEW`] for ops + /// `from_op..=head`. + fn repair_request( + namespace: IggyNamespace, + from_op: u64, + head: usize, + ) -> Message { + Message::::new(size_of::()).transmute_header( + |_, header: &mut RequestPreparesHeader| { header.command = Command::RequestPrepares; header.cluster = 1; header.group = namespace.inner(); header.replica = REPAIR_PRIMARY; header.view = REPAIR_VIEW; header.from_op = from_op; - header.to_op = u64::try_from(prepares.len()).unwrap(); + header.to_op = u64::try_from(head).unwrap(); header.size = u32::try_from(size_of::()).unwrap(); header.seal(); - }); - sent.borrow_mut().clear(); - owner.on_request_prepares(&request).await; + }, + ) + } + + /// Takes the frames in `sent`, and returns the command and op of each. + /// Each `RepairPrepare` must carry its prepare from `prepares` verbatim. + fn sent_answers( + sent: &RefCell>>, + prepares: &[Message], + ) -> Vec<(Command, u64)> { + const COMMAND_OFFSET: usize = std::mem::offset_of!(GenericHeader, command); sent.borrow_mut() .drain(..) .map(|message| { @@ -9274,6 +9580,7 @@ mod tests { primary_owner .on_request_prepares(&request.try_into_typed().unwrap()) .await; + primary_owner.settle_served_reads().await; } let frames: Vec<_> = served.borrow_mut().drain(..).collect(); for (_, frame) in frames { @@ -9337,7 +9644,7 @@ mod tests { log_path: PathBuf, /// The position of each released body in the log file. positions: BTreeMap, - _sender: TaggedSender, + sender: TaggedSender, _directory: tempfile::TempDir, } @@ -9376,7 +9683,7 @@ mod tests { prepares, log_path, positions, - _sender: sender, + sender, _directory: directory, } } diff --git a/core/shard/src/router.rs b/core/shard/src/router.rs index bbb7e5266a..95d3e47335 100644 --- a/core/shard/src/router.rs +++ b/core/shard/src/router.rs @@ -529,7 +529,11 @@ where () = poll_fn(|context| { self.plane.partitions().register_loopback_waker(context.waker()); self.partition_io.register_waker(context.waker()); - if self.has_pending_loopback(&loopback_buf) || self.partition_io.has_ready() { + self.served_reads.register_waker(context.waker()); + if self.has_pending_loopback(&loopback_buf) + || self.partition_io.has_ready() + || self.served_reads.has_landed() + { Poll::Ready(()) } else { Poll::Pending @@ -540,6 +544,9 @@ where } } self.service_partition_io().await; + // After every event, as partition I/O, so a busy inbox cannot + // starve the serves that wait for a disk read. + self.serve_landed_read().await; } self.poll_completions.close(); diff --git a/core/shard/src/served_reads.rs b/core/shard/src/served_reads.rs new file mode 100644 index 0000000000..c096365829 --- /dev/null +++ b/core/shard/src/served_reads.rs @@ -0,0 +1,218 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Disk reads that answer a peer run beside the pump. +//! +//! A repair or a replay can need prepare bodies that the journal released to +//! their segment, and a transfer chunk can need segment bytes. A read on the +//! pump would hold the frames, ticks and replies of every group on the shard +//! until the disk answers. So the pump decides what to serve, a detached task +//! reads the bytes, and the pump serves the request again once they land. That +//! serve checks the state again, so the bytes never answer a request that the +//! state no longer admits. +//! +//! The reads run one at a time per shard, as they did on the pump, and a read +//! starts only after the pump took the bytes of the reads before it. So a +//! shard holds the bytes of two reads at most: the one that the pump serves +//! and the one that runs. A key has at most one read, and a shard holds at +//! most [`SERVED_READS_MAX`] keys. The pump drops a request past either bound, +//! and its requester asks again. + +use std::cell::RefCell; +use std::collections::{HashSet, VecDeque}; +use std::future::poll_fn; +use std::rc::Rc; +use std::task::{Context, Poll, Waker}; + +use consensus::StateArtifact; +use iggy_binary_protocol::{RequestPreparesHeader, RequestStateChunkHeader}; +use journal::local_gate::LocalGate; +use message_bus::MessageBus; +use partitions::ReleasedBodies; +use partitions::state_transfer::SegmentLoadError; +use server_common::Message; + +/// Keys that one shard reads or waits to read for. A waiting key holds its +/// request and the headers of one round, but no bytes. +pub const SERVED_READS_MAX: usize = 64; + +/// What a read serves. A request whose key reads waits for that read. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ServedReadKey { + /// Released bodies that a repair, or a replay when `live`, sends `target`. + Prepares { group: u64, target: u8, live: bool }, + /// Segment bytes of the transfer chunk that `requester` pulls. + Chunk { group: u64, requester: u8 }, +} + +/// A read that landed, with the request that it serves. +pub enum LandedRead { + Prepares { + request: Message, + live: bool, + bodies: ReleasedBodies, + }, + Chunk { + header: RequestStateChunkHeader, + /// The offered artifact that `load` read. + artifact: StateArtifact, + load: Result, SegmentLoadError>, + }, +} + +/// The pump and the read tasks share one state, so a shard holds one pointer. +#[derive(Default)] +pub struct ServedReads { + shared: Rc, +} + +#[derive(Default)] +struct Shared { + keys: RefCell>, + landed: RefCell)>>, + pump: RefCell>, + /// The read whose turn came while the pump had not taken every result. + taken: RefCell>, + gate: LocalGate, +} + +impl Shared { + /// Ready once the pump took every read that landed. + fn poll_taken(&self, context: &Context<'_>) -> Poll<()> { + if self.landed.borrow().is_empty() { + return Poll::Ready(()); + } + self.taken.replace(Some(context.waker().clone())); + Poll::Pending + } +} + +impl ServedReads { + pub(crate) fn reading(&self, key: ServedReadKey) -> bool { + self.shared.keys.borrow().contains(&key) + } + + /// Runs `read` in a detached task once no other read of this shard runs + /// and the pump took every result, and holds `key` until the pump takes + /// this one. Runs nothing when `key` already reads or the shard holds + /// [`SERVED_READS_MAX`] keys. + pub(crate) fn start( + &self, + bus: &B, + key: ServedReadKey, + read: impl Future + 'static, + ) { + { + let mut keys = self.shared.keys.borrow_mut(); + if keys.len() >= SERVED_READS_MAX || !keys.insert(key) { + return; + } + } + let landing = Landing { + shared: Rc::clone(&self.shared), + key, + read: None, + }; + let shared = Rc::clone(&self.shared); + bus.spawn(async move { + let _turn = shared.gate.acquire().await; + poll_fn(|context| shared.poll_taken(context)).await; + landing.land(read.await); + }); + } + + pub(crate) fn register_waker(&self, waker: &Waker) { + let mut current = self.shared.pump.borrow_mut(); + if current + .as_ref() + .is_none_or(|current| !current.will_wake(waker)) + { + *current = Some(waker.clone()); + } + } + + pub(crate) fn has_landed(&self) -> bool { + !self.shared.landed.borrow().is_empty() + } + + /// Takes the oldest landed read and frees its key. A read that stopped + /// without a result frees its key and returns `None`. + pub(crate) fn take_landed(&self) -> Option { + let (key, read) = self.shared.landed.borrow_mut().pop_front()?; + self.shared.keys.borrow_mut().remove(&key); + if let Some(taken) = self.shared.taken.take() { + taken.wake(); + } + read + } + + #[cfg(test)] + pub(crate) fn is_empty(&self) -> bool { + self.shared.keys.borrow().is_empty() + } + + #[cfg(test)] + pub(crate) fn landed_count(&self) -> usize { + self.shared.landed.borrow().len() + } + + #[cfg(test)] + #[allow(clippy::future_not_send)] + pub(crate) async fn landed(&self) { + poll_fn(|context| { + self.register_waker(context.waker()); + if self.has_landed() { + Poll::Ready(()) + } else { + Poll::Pending + } + }) + .await; + } + + /// Holds the turn of this shard's reads, so none runs until the guard drops. + #[cfg(test)] + pub(crate) fn hold(&self) -> journal::local_gate::LocalGateGuard<'_> { + self.shared.gate.try_acquire().expect("no read runs") + } +} + +/// Hands the result of a read to the pump. A task that stops without one, such +/// as one that panicked, hands over nothing, so its key frees either way. +struct Landing { + shared: Rc, + key: ServedReadKey, + read: Option, +} + +impl Landing { + fn land(mut self, read: LandedRead) { + self.read = Some(read); + } +} + +impl Drop for Landing { + fn drop(&mut self) { + self.shared + .landed + .borrow_mut() + .push_back((self.key, self.read.take())); + if let Some(waker) = self.shared.pump.borrow_mut().take() { + waker.wake(); + } + } +} From d8ccfa84490be6a00f0d9578cf4b0b53ab1a0b7e Mon Sep 17 00:00:00 2001 From: Hubert Gruszecki Date: Sat, 31 Oct 2026 11:54:03 +0100 Subject: [PATCH 10/10] test(integration): fix external auth and Swift backlog test flakes The callout cap test held its slow callouts for 30 s, which is also the operation deadline of the overflow login. When a late retry found a free slot, it reached the mock service and the test counted 3 requests, not 2. Hold the callouts for twice that deadline. The Swift backlog test allowed four reads for the message on the other partition, but TLS runs needed up to seven. Each fetch in flight takes a data connection of its own, so the backlog can answer while the other fetch opens one. Read until the message arrives, and fail if half of the backlog came first. --- core/integration/tests/server/external_auth.rs | 5 ++++- foreign/swift/Tests/IggyE2ETests/DeferredPollE2ETests.swift | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/core/integration/tests/server/external_auth.rs b/core/integration/tests/server/external_auth.rs index 1a788b0007..a6688b88be 100644 --- a/core/integration/tests/server/external_auth.rs +++ b/core/integration/tests/server/external_auth.rs @@ -25,6 +25,7 @@ use std::collections::HashMap; use std::time::Duration; use iggy::prelude::*; +use iggy_common::BINARY_OPERATION_TIMEOUT; use integration::harness::ext_auth_server::{ EXT_AUTH_IGGY_USERNAME, EXT_AUTH_PASSWORD, EXT_AUTH_USERNAME, EXT_DENIED_PASSWORD, EXT_DENIED_USERNAME, ExtAuthServer, @@ -359,7 +360,9 @@ mod serialized { #[tokio::test] async fn given_ext_auth_when_callout_cap_saturated_should_return_retryable_error() { - let ext_server = ExtAuthServer::start_slow(Duration::from_secs(30)).await; + // The overflow login retries until its operation timeout, so a callout + // that ends before it frees a slot and lets a retry reach the service. + let ext_server = ExtAuthServer::start_slow(BINARY_OPERATION_TIMEOUT * 2).await; let mut envs = ext_auth_envs(&ext_server.url()); envs.insert("IGGY_EXTERNAL_AUTH_TIMEOUT".to_owned(), "60 s".to_owned()); envs.insert( diff --git a/foreign/swift/Tests/IggyE2ETests/DeferredPollE2ETests.swift b/foreign/swift/Tests/IggyE2ETests/DeferredPollE2ETests.swift index 63834ddf79..022fa80443 100644 --- a/foreign/swift/Tests/IggyE2ETests/DeferredPollE2ETests.swift +++ b/foreign/swift/Tests/IggyE2ETests/DeferredPollE2ETests.swift @@ -71,10 +71,12 @@ struct DeferredPollE2ETests { streamID: scratch.stream, topicID: scratch.topic, consumer: .consumer(try Identifier(named: E2EEnvironment.uniqueName("swift-e2e-backlog"))), configuration: ReaderConfiguration(partitions: [0, 1], start: .first, options: PollOptions(count: 1, maxWait: .seconds(1)))) var payloads: [String] = [] - for _ in 0..<4 { + // Each fetch in flight takes a data connection of its own, so the + // backlog can answer a few times while the other fetch opens one. + while !payloads.contains("late"), payloads.count < backlog.count / 2 { payloads += try #require(try await reader.next()).messages.map(\.payloadString) } - #expect(payloads.contains("late"), "\(payloads)") + #expect(payloads.contains("late"), "\(payloads.count) messages came without it") try await reader.close() } }