From 5a4597cbe188d968e0d22425ec734a110a30c98a Mon Sep 17 00:00:00 2001 From: Gunther Xing Date: Sat, 15 Aug 2026 15:06:18 +0800 Subject: [PATCH 01/15] feat(connectors): Add Iggy source connector Replicates a topic from an upstream Iggy cluster with per-partition offset tracking. --- Cargo.lock | 17 + Cargo.toml | 1 + .../connectors/iggy_source.toml | 46 ++ core/connectors/sources/README.md | 1 + .../connectors/sources/iggy_source/Cargo.toml | 51 ++ .../sources/iggy_source/config.toml | 42 + .../connectors/sources/iggy_source/src/lib.rs | 745 ++++++++++++++++++ .../tests/connectors/iggy_source/mod.rs | 18 + .../connectors/iggy_source/replication.rs | 363 +++++++++ .../tests/connectors/iggy_source/source.toml | 20 + core/integration/tests/connectors/mod.rs | 1 + 11 files changed, 1305 insertions(+) create mode 100644 core/connectors/runtime/example_config/connectors/iggy_source.toml create mode 100644 core/connectors/sources/iggy_source/Cargo.toml create mode 100644 core/connectors/sources/iggy_source/config.toml create mode 100644 core/connectors/sources/iggy_source/src/lib.rs create mode 100644 core/integration/tests/connectors/iggy_source/mod.rs create mode 100644 core/integration/tests/connectors/iggy_source/replication.rs create mode 100644 core/integration/tests/connectors/iggy_source/source.toml diff --git a/Cargo.lock b/Cargo.lock index 5e2e889c4e..79fca6127d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7601,6 +7601,23 @@ dependencies = [ "uuid", ] +[[package]] +name = "iggy_connector_iggy_source" +version = "0.5.0-edge.1" +dependencies = [ + "async-trait", + "dashmap", + "iggy", + "iggy_common", + "iggy_connector_sdk", + "rmp-serde", + "secrecy", + "serde", + "serde_json", + "tokio", + "tracing", +] + [[package]] name = "iggy_connector_influxdb_sink" version = "0.5.1-edge.1" diff --git a/Cargo.toml b/Cargo.toml index 29542a9f28..39ed07f690 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,7 @@ members = [ "core/connectors/sinks/surrealdb_sink", "core/connectors/sources/elasticsearch_source", "core/connectors/sources/http_source", + "core/connectors/sources/iggy_source", "core/connectors/sources/influxdb_source", "core/connectors/sources/mqtt_source", "core/connectors/sources/postgres_source", diff --git a/core/connectors/runtime/example_config/connectors/iggy_source.toml b/core/connectors/runtime/example_config/connectors/iggy_source.toml new file mode 100644 index 0000000000..887d1b1177 --- /dev/null +++ b/core/connectors/runtime/example_config/connectors/iggy_source.toml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +type = "source" +key = "iggy" +enabled = true +version = 0 +name = "Iggy source" +path = "target/release/libiggy_connector_iggy_source" +verbose = false # Log message processing at info level instead of debug (default: false) + +[[streams]] +stream = "downstream_stream" +topic = "downstream_topic" +schema = "raw" +batch_length = 100 +linger_time = "5ms" + +[plugin_config] +# Connection string of the UPSTREAM Iggy cluster to replicate from +connection_string = "iggy+tcp://iggy:iggy@127.0.0.1:8090" +upstream_stream = "upstream_stream" +upstream_topic = "upstream_topic" +poll_interval = "1s" +batch_size = 100 +# "earliest", "latest", or a numeric offset; used only for partitions without +# a saved offset in the persisted connector state +initial_offset = "earliest" +include_user_headers = true +retry_interval = "1s" +max_retry_interval = "60s" +verbose_logging = false diff --git a/core/connectors/sources/README.md b/core/connectors/sources/README.md index d7e53a34e8..09f19f9a95 100644 --- a/core/connectors/sources/README.md +++ b/core/connectors/sources/README.md @@ -10,6 +10,7 @@ Source connectors are responsible for ingesting data from external sources into | ------ | ----------- | | **elasticsearch_source** | Polls documents from Elasticsearch indices with timestamp-based tracking | | **http_source** | Webhook gateway: an embedded HTTP server shared by every instance, with per-endpoint bearer/HMAC auth and a management API for endpoints registered at runtime | +| **iggy_source** | Replicates a topic from an upstream Iggy cluster with per-partition offset tracking | | **influxdb_source** | Polls InfluxDB with cursor-based timestamp tracking; supports V2 (Flux, annotated CSV) and V3 (SQL, JSONL) | | **mqtt_source** | Subscribes to MQTT 3.1.1 or MQTT 5 brokers with QoS 0, 1, or 2 | | **postgres_source** | Reads rows from PostgreSQL tables with multiple strategies: delete after read, mark as processed, or timestamp tracking | diff --git a/core/connectors/sources/iggy_source/Cargo.toml b/core/connectors/sources/iggy_source/Cargo.toml new file mode 100644 index 0000000000..8b45f7c7da --- /dev/null +++ b/core/connectors/sources/iggy_source/Cargo.toml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "iggy_connector_iggy_source" +version = "0.5.0-edge.1" +description = "Iggy source connector replicating a topic from an upstream Apache Iggy cluster" +edition = "2024" +license = "Apache-2.0" +keywords = ["iggy", "messaging", "streaming", "replication"] +categories = ["command-line-utilities", "database", "network-programming"] +homepage = "https://iggy.apache.org" +documentation = "https://iggy.apache.org/docs" +repository = "https://github.com/apache/iggy" +readme = "../../README.md" +publish = false + +[package.metadata.cargo-machete] +ignored = ["dashmap"] + +[lib] +crate-type = ["cdylib", "lib"] + +[dependencies] +async-trait = { workspace = true } +dashmap = { workspace = true } +iggy = { workspace = true } +iggy_common = { workspace = true } +iggy_connector_sdk = { workspace = true } +rmp-serde = { workspace = true } +secrecy = { workspace = true } +serde = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +serde_json = { workspace = true } diff --git a/core/connectors/sources/iggy_source/config.toml b/core/connectors/sources/iggy_source/config.toml new file mode 100644 index 0000000000..2f4d3bc144 --- /dev/null +++ b/core/connectors/sources/iggy_source/config.toml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +type = "source" +key = "iggy" +enabled = true +version = 0 +name = "Iggy source" +path = "../../target/release/libiggy_connector_iggy_source" +verbose = false + +[[streams]] +stream = "downstream_stream" +topic = "downstream_topic" +schema = "raw" +batch_length = 100 + +[plugin_config] +connection_string = "iggy+tcp://iggy:iggy@127.0.0.1:8090" +upstream_stream = "upstream_stream" +upstream_topic = "upstream_topic" +poll_interval = "1s" +batch_size = 100 +initial_offset = "earliest" +include_user_headers = true +retry_interval = "1s" +max_retry_interval = "60s" +verbose_logging = false diff --git a/core/connectors/sources/iggy_source/src/lib.rs b/core/connectors/sources/iggy_source/src/lib.rs new file mode 100644 index 0000000000..a9c818290b --- /dev/null +++ b/core/connectors/sources/iggy_source/src/lib.rs @@ -0,0 +1,745 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use async_trait::async_trait; +use iggy::prelude::{ + Client, CompressionAlgorithm, Consumer, Identifier, IggyClient, IggyError, IggyMessage, + MessageClient, PollingStrategy, StreamClient, TopicClient, TopicCreateOptions, +}; +use iggy_connector_sdk::{ + ConnectorState, Error, ProducedMessage, ProducedMessages, Schema, Source, + retry::{exponential_backoff, jitter, parse_duration}, + source_connector, +}; +use secrecy::{ExposeSecret, SecretString}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::HashMap, + str::FromStr, + sync::atomic::{AtomicU64, Ordering}, + time::Duration, +}; +use tokio::{sync::Mutex, time::sleep}; +use tracing::{debug, error, info, warn}; + +source_connector!(IggySource); + +const CONNECTOR_NAME: &str = "Iggy source"; +const DEFAULT_POLL_INTERVAL: &str = "2s"; +const DEFAULT_RETRY_INTERVAL: &str = "1s"; +const DEFAULT_MAX_RETRY_INTERVAL: &str = "60s"; +const DEFAULT_BATCH_SIZE: u32 = 100; +const AUTO_CREATED_PARTITIONS_COUNT: u32 = 1; + +/// Configuration for the Iggy source connector, replicating a topic from an +/// upstream Iggy cluster. `connection_string` points at the upstream cluster. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IggySourceConfig { + #[serde(serialize_with = "iggy_common::serde_secret::serialize_secret")] + pub connection_string: SecretString, + pub upstream_stream: String, + pub upstream_topic: String, + pub poll_interval: Option, + pub batch_size: Option, + pub initial_offset: Option, + pub include_user_headers: Option, + pub retry_interval: Option, + pub max_retry_interval: Option, + pub verbose_logging: Option, +} + +/// Starting point for a partition that has no saved offset in the state yet. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InitialOffset { + Earliest, + Latest, + Offset(u64), +} + +impl FromStr for InitialOffset { + type Err = (); + + fn from_str(s: &str) -> Result { + match s.trim().to_lowercase().as_str() { + "earliest" => Ok(Self::Earliest), + "latest" => Ok(Self::Latest), + other => other.parse::().map(Self::Offset).map_err(|_| ()), + } + } +} + +/// Persisted state. `offsets` maps each upstream partition to the offset of +/// the last message that was handed to the runtime and therefore successfully +/// produced to the downstream cluster; the runtime persists the state only +/// after a successful downstream send, so a restart resumes from `offset + 1`. +#[derive(Debug, Serialize, Deserialize)] +struct State { + offsets: HashMap, + messages_synced: u64, + errors_count: u64, +} + +#[derive(Debug)] +pub struct IggySource { + id: u32, + config: IggySourceConfig, + client: Option, + state: Mutex, + partitions: Vec, + stream_id: Option, + topic_id: Option, + poll_interval: Duration, + retry_interval: Duration, + max_retry_interval: Duration, + consecutive_failures: AtomicU64, + initial_offset: InitialOffset, + batch_size: u32, + include_user_headers: bool, + verbose: bool, +} + +impl IggySource { + pub fn new(id: u32, config: IggySourceConfig, state: Option) -> Self { + let verbose = config.verbose_logging.unwrap_or(false); + let restored_state = state + .and_then(|s| s.deserialize::(CONNECTOR_NAME, id)) + .inspect(|s| { + info!( + "Restored state for {CONNECTOR_NAME} connector ID: {id}. \ + Offsets: {:?}, messages synced: {}, errors: {}", + s.offsets, s.messages_synced, s.errors_count + ); + }); + + let poll_interval = parse_duration(config.poll_interval.as_deref(), DEFAULT_POLL_INTERVAL); + let retry_interval = + parse_duration(config.retry_interval.as_deref(), DEFAULT_RETRY_INTERVAL); + let max_retry_interval = parse_duration( + config.max_retry_interval.as_deref(), + DEFAULT_MAX_RETRY_INTERVAL, + ); + + let initial_offset = config + .initial_offset + .as_deref() + .map(InitialOffset::from_str) + .and_then(Result::ok) + .unwrap_or_else(|| { + warn!( + "Invalid initial offset {:?} for {CONNECTOR_NAME} connector ID: {id}, \ + defaulting to earliest", + config.initial_offset + ); + InitialOffset::Earliest + }); + + let batch_size = config.batch_size.unwrap_or(DEFAULT_BATCH_SIZE); + let include_user_headers = config.include_user_headers.unwrap_or(true); + + IggySource { + id, + config, + client: None, + state: Mutex::new(restored_state.unwrap_or(State { + offsets: HashMap::new(), + messages_synced: 0, + errors_count: 0, + })), + partitions: Vec::new(), + stream_id: None, + topic_id: None, + poll_interval, + retry_interval, + max_retry_interval, + consecutive_failures: AtomicU64::new(0), + initial_offset, + batch_size, + include_user_headers, + verbose, + } + } + + fn serialize_state(&self, state: &State) -> Option { + ConnectorState::serialize(state, CONNECTOR_NAME, self.id) + } + + async fn ensure_stream_and_topic( + &self, + client: &IggyClient, + stream_id: &Identifier, + topic_id: &Identifier, + ) -> Result<(), Error> { + match client.get_stream(stream_id).await { + Ok(Some(_)) => {} + Ok(None) => { + warn!( + "Upstream stream '{}' does not exist, creating it for {CONNECTOR_NAME} \ + connector ID: {}", + self.config.upstream_stream, self.id + ); + client + .create_stream(&self.config.upstream_stream) + .await + .map_err(|e| { + Error::InitError(format!( + "Failed to create upstream stream '{}': {e}", + self.config.upstream_stream + )) + })?; + } + Err(e) => { + return Err(Error::InitError(format!( + "Failed to check upstream stream '{}': {e}", + self.config.upstream_stream + ))); + } + } + + match client.get_topic(stream_id, topic_id).await { + Ok(Some(_)) => Ok(()), + Ok(None) => { + warn!( + "Upstream topic '{}' does not exist, creating it with \ + {AUTO_CREATED_PARTITIONS_COUNT} partition(s) for {CONNECTOR_NAME} \ + connector ID: {}", + self.config.upstream_topic, self.id + ); + client + .create_topic( + stream_id, + &self.config.upstream_topic, + &TopicCreateOptions { + partitions_count: Some(AUTO_CREATED_PARTITIONS_COUNT), + compression_algorithm: Some(CompressionAlgorithm::None), + ..TopicCreateOptions::default() + }, + ) + .await + .map_err(|e| { + Error::InitError(format!( + "Failed to create upstream topic '{}': {e}", + self.config.upstream_topic + )) + })?; + Ok(()) + } + Err(e) => Err(Error::InitError(format!( + "Failed to check upstream topic '{}': {e}", + self.config.upstream_topic + ))), + } + } +} + +#[async_trait] +impl Source for IggySource { + async fn open(&mut self) -> Result<(), Error> { + let redacted = redact_connection_string(self.config.connection_string.expose_secret()); + info!( + "Opening {CONNECTOR_NAME} connector ID: {}, upstream: {}/{} at {}", + self.id, self.config.upstream_stream, self.config.upstream_topic, redacted + ); + + let client = + IggyClient::from_connection_string(self.config.connection_string.expose_secret()) + .map_err(|e| { + Error::InitError(format!("Failed to parse upstream connection string: {e}")) + })?; + + client.connect().await.map_err(|e| { + Error::InitError(format!("Failed to connect to upstream Iggy cluster: {e}")) + })?; + + let stream_id = Identifier::named(&self.config.upstream_stream).map_err(|_| { + Error::InvalidConfigValue(format!( + "Invalid upstream stream name '{}'", + self.config.upstream_stream + )) + })?; + let topic_id = Identifier::named(&self.config.upstream_topic).map_err(|_| { + Error::InvalidConfigValue(format!( + "Invalid upstream topic name '{}'", + self.config.upstream_topic + )) + })?; + + self.stream_id = Some(stream_id.clone()); + self.topic_id = Some(topic_id.clone()); + + self.ensure_stream_and_topic(&client, &stream_id, &topic_id) + .await?; + + let topic = client + .get_topic(&stream_id, &topic_id) + .await + .map_err(|e| Error::InitError(format!("Failed to fetch upstream topic details: {e}")))? + .ok_or_else(|| { + Error::InitError(format!( + "Upstream topic '{}/{}' not found after creation", + self.config.upstream_stream, self.config.upstream_topic + )) + })?; + self.partitions = topic + .partitions + .iter() + .map(|partition| partition.id) + .collect(); + + self.client = Some(client); + info!( + "Opened {CONNECTOR_NAME} connector ID: {}, partitions: {}, initial offset: {:?}, \ + poll interval: {:?}, batch size: {}", + self.id, + self.partitions.len(), + self.initial_offset, + self.poll_interval, + self.batch_size + ); + Ok(()) + } + + async fn poll(&self) -> Result { + sleep(self.poll_interval).await; + + let client = self + .client + .as_ref() + .ok_or_else(|| Error::InitError("Upstream client not connected".to_string()))?; + let stream_id = self + .stream_id + .as_ref() + .ok_or_else(|| Error::InitError("Upstream stream not initialized".to_string()))?; + let topic_id = self + .topic_id + .as_ref() + .ok_or_else(|| Error::InitError("Upstream topic not initialized".to_string()))?; + + let failures = self.consecutive_failures.load(Ordering::Relaxed); + if failures > 0 { + let delay = jitter(exponential_backoff( + self.retry_interval, + failures as u32, + self.max_retry_interval, + )); + debug!( + "Backing off for {delay:?} after {failures} consecutive failures for \ + {CONNECTOR_NAME} connector ID: {}", + self.id + ); + sleep(delay).await; + } + + let consumer = Consumer::default(); + + let saved_offsets: HashMap = { + let state = self.state.lock().await; + state.offsets.clone() + }; + + let mut messages = Vec::with_capacity(self.partitions.len() * self.batch_size as usize); + let mut new_offsets: HashMap = HashMap::new(); + let mut reset_offsets: Vec = Vec::new(); + let mut errors_in_cycle: u64 = 0; + let mut any_success = false; + let mut connection_failure = false; + + for &partition_id in &self.partitions { + let strategy = next_strategy( + self.initial_offset, + saved_offsets.get(&partition_id).copied(), + ); + let polled = client + .poll_messages( + stream_id, + topic_id, + Some(partition_id), + &consumer, + &strategy, + self.batch_size, + false, + ) + .await; + + match polled { + Ok(polled) => { + any_success = true; + if polled.messages.is_empty() { + continue; + } + let last_offset = polled + .messages + .last() + .map(|message| message.header.offset) + .unwrap_or_default(); + for message in &polled.messages { + match build_produced_message(message, self.include_user_headers) { + Ok(produced) => messages.push(produced), + Err(e) => { + error!( + "Failed to convert upstream message at offset {} on \ + partition {partition_id} for {CONNECTOR_NAME} connector \ + ID: {}: {e}", + message.header.offset, self.id + ); + errors_in_cycle += 1; + } + } + } + new_offsets.insert(partition_id, last_offset); + } + Err(IggyError::InvalidOffset(offset)) => { + warn!( + "Saved offset {offset} for partition {partition_id} no longer valid \ + for {CONNECTOR_NAME} connector ID: {}, resetting to initial offset", + self.id + ); + reset_offsets.push(partition_id); + errors_in_cycle += 1; + } + Err(e) => { + error!( + "Failed to poll partition {partition_id} for {CONNECTOR_NAME} \ + connector ID: {}: {e}", + self.id + ); + errors_in_cycle += 1; + connection_failure = true; + break; + } + } + } + + let (persisted_state, total_synced) = { + let mut state = self.state.lock().await; + for (partition_id, offset) in new_offsets { + state.offsets.insert(partition_id, offset); + } + for partition_id in reset_offsets { + state.offsets.remove(&partition_id); + } + state.messages_synced += messages.len() as u64; + state.errors_count += errors_in_cycle; + let persisted = self.serialize_state(&state); + (persisted, state.messages_synced) + }; + + if connection_failure && !any_success { + self.consecutive_failures.fetch_add(1, Ordering::Relaxed); + } else { + self.consecutive_failures.store(0, Ordering::Relaxed); + } + + if self.verbose { + info!( + "{CONNECTOR_NAME} connector ID: {} polled {} messages from {} partition(s). \ + Total synced: {}, errors in cycle: {}", + self.id, + messages.len(), + self.partitions.len(), + total_synced, + errors_in_cycle + ); + } else { + debug!( + "{CONNECTOR_NAME} connector ID: {} polled {} messages from {} partition(s). \ + Total synced: {}, errors in cycle: {}", + self.id, + messages.len(), + self.partitions.len(), + total_synced, + errors_in_cycle + ); + } + + Ok(ProducedMessages { + schema: Schema::Raw, + messages, + state: persisted_state, + }) + } + + async fn close(&mut self) -> Result<(), Error> { + if let Some(client) = self.client.take() + && let Err(e) = client.disconnect().await + { + warn!( + "Failed to disconnect from upstream cluster for {CONNECTOR_NAME} \ + connector ID: {}: {e}", + self.id + ); + } + + let state = self.state.lock().await; + info!( + "{CONNECTOR_NAME} connector ID: {} closed. Total messages synced: {}, total errors: {}", + self.id, state.messages_synced, state.errors_count + ); + Ok(()) + } +} + +fn next_strategy(initial: InitialOffset, saved_offset: Option) -> PollingStrategy { + match saved_offset { + Some(offset) => PollingStrategy::offset(offset.saturating_add(1)), + None => match initial { + InitialOffset::Earliest => PollingStrategy::first(), + InitialOffset::Latest => PollingStrategy::last(), + InitialOffset::Offset(offset) => PollingStrategy::offset(offset), + }, + } +} + +fn build_produced_message( + message: &IggyMessage, + include_user_headers: bool, +) -> Result { + let headers = if include_user_headers { + message.user_headers_map().map_err(|e| { + Error::InvalidRecordValue(format!("Failed to parse upstream user headers: {e}")) + })? + } else { + None + }; + + Ok(ProducedMessage { + id: (message.header.id != 0).then_some(message.header.id), + headers, + checksum: None, + timestamp: None, + origin_timestamp: (message.header.origin_timestamp != 0) + .then_some(message.header.origin_timestamp), + payload: message.payload.to_vec(), + }) +} + +fn redact_connection_string(connection_string: &str) -> String { + let Some(scheme_end) = connection_string.find("://") else { + return "***".to_string(); + }; + let Some(rest) = connection_string.get(scheme_end + 3..) else { + return "***".to_string(); + }; + let Some(at_offset) = rest.find('@') else { + return "***".to_string(); + }; + let userinfo_end = scheme_end + 3 + at_offset; + let userinfo = &connection_string[scheme_end + 3..userinfo_end]; + let redact_from = match userinfo.find(':') { + Some(colon) => scheme_end + 3 + colon + 1, + None => scheme_end + 3, + }; + let mut redacted = connection_string.to_string(); + redacted.replace_range(redact_from..userinfo_end, "***"); + redacted +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_config() -> IggySourceConfig { + IggySourceConfig { + connection_string: SecretString::from( + "iggy+tcp://iggy:iggy@127.0.0.1:8090".to_string(), + ), + upstream_stream: "upstream_stream".to_string(), + upstream_topic: "upstream_topic".to_string(), + poll_interval: Some("100ms".to_string()), + batch_size: Some(50), + initial_offset: Some("0".to_string()), + include_user_headers: Some(true), + retry_interval: Some("100ms".to_string()), + max_retry_interval: Some("5s".to_string()), + verbose_logging: Some(false), + } + } + + #[test] + fn given_persisted_state_should_restore_offsets_and_counts() { + let state = State { + offsets: HashMap::from([(0, 42), (1, 7)]), + messages_synced: 500, + errors_count: 3, + }; + + let serialized = rmp_serde::to_vec(&state).expect("Failed to serialize state"); + let connector_state = ConnectorState(serialized); + + let source = IggySource::new(1, test_config(), Some(connector_state)); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let restored = source.state.lock().await; + assert_eq!(restored.offsets.get(&0), Some(&42)); + assert_eq!(restored.offsets.get(&1), Some(&7)); + assert_eq!(restored.messages_synced, 500); + assert_eq!(restored.errors_count, 3); + }); + } + + #[test] + fn given_no_state_should_start_fresh() { + let source = IggySource::new(1, test_config(), None); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let state = source.state.lock().await; + assert!(state.offsets.is_empty()); + assert_eq!(state.messages_synced, 0); + assert_eq!(state.errors_count, 0); + }); + } + + #[test] + fn given_invalid_state_should_start_fresh() { + let invalid_state = ConnectorState(b"not valid msgpack".to_vec()); + let source = IggySource::new(1, test_config(), Some(invalid_state)); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let state = source.state.lock().await; + assert!(state.offsets.is_empty()); + assert_eq!(state.messages_synced, 0); + assert_eq!(state.errors_count, 0); + }); + } + + #[test] + fn state_should_be_serializable_and_deserializable() { + let original = State { + offsets: HashMap::from([(2, 1000)]), + messages_synced: 1000, + errors_count: 5, + }; + + let serialized = rmp_serde::to_vec(&original).expect("Failed to serialize"); + let deserialized: State = + rmp_serde::from_slice(&serialized).expect("Failed to deserialize"); + + assert_eq!(original.offsets, deserialized.offsets); + assert_eq!(original.messages_synced, deserialized.messages_synced); + assert_eq!(original.errors_count, deserialized.errors_count); + } + + #[test] + fn serialize_state_helper_should_produce_valid_connector_state() { + let source = IggySource::new(1, test_config(), None); + let state = State { + offsets: HashMap::from([(0, 42)]), + messages_synced: 42, + errors_count: 0, + }; + + let connector_state = source.serialize_state(&state); + assert!(connector_state.is_some()); + + let bytes = connector_state.unwrap().0; + let restored: State = rmp_serde::from_slice(&bytes).expect("Failed to deserialize state"); + assert_eq!(restored.messages_synced, 42); + } + + #[test] + fn config_should_deserialize_from_json_with_defaults() { + let json = r#"{ + "connection_string": "iggy+tcp://iggy:iggy@127.0.0.1:8090", + "upstream_stream": "stream", + "upstream_topic": "topic" + }"#; + + let config: IggySourceConfig = serde_json::from_str(json).expect("Failed to parse config"); + + assert_eq!(config.upstream_stream, "stream"); + assert_eq!(config.upstream_topic, "topic"); + assert!(config.poll_interval.is_none()); + assert!(config.initial_offset.is_none()); + } + + #[test] + fn config_should_apply_defaults_in_new() { + let json = r#"{ + "connection_string": "iggy+tcp://iggy:iggy@127.0.0.1:8090", + "upstream_stream": "stream", + "upstream_topic": "topic" + }"#; + + let config: IggySourceConfig = serde_json::from_str(json).expect("Failed to parse config"); + let source = IggySource::new(1, config, None); + + assert_eq!(source.poll_interval, Duration::from_secs(2)); + assert_eq!(source.retry_interval, Duration::from_secs(1)); + assert_eq!(source.max_retry_interval, Duration::from_secs(60)); + assert_eq!(source.batch_size, DEFAULT_BATCH_SIZE); + assert!(source.include_user_headers); + assert_eq!(source.initial_offset, InitialOffset::Earliest); + assert!(!source.verbose); + } + + #[test] + fn given_invalid_initial_offset_should_default_to_earliest() { + let config = IggySourceConfig { + initial_offset: Some("not-an-offset".to_string()), + ..test_config() + }; + + let source = IggySource::new(1, config, None); + assert_eq!(source.initial_offset, InitialOffset::Earliest); + } + + #[test] + fn initial_offset_should_parse_semantics() { + assert_eq!( + "earliest".parse::(), + Ok(InitialOffset::Earliest) + ); + assert_eq!("Latest".parse::(), Ok(InitialOffset::Latest)); + assert_eq!("42".parse::(), Ok(InitialOffset::Offset(42))); + assert!("invalid".parse::().is_err()); + } + + #[test] + fn next_strategy_should_jump_after_saved_offset() { + assert_eq!( + next_strategy(InitialOffset::Earliest, Some(41)), + PollingStrategy::offset(42) + ); + assert_eq!( + next_strategy(InitialOffset::Earliest, None), + PollingStrategy::first() + ); + assert_eq!( + next_strategy(InitialOffset::Latest, None), + PollingStrategy::last() + ); + assert_eq!( + next_strategy(InitialOffset::Offset(7), None), + PollingStrategy::offset(7) + ); + } + + #[test] + fn redact_connection_string_should_hide_credentials() { + let redacted = redact_connection_string("iggy+tcp://iggy:password@127.0.0.1:8090"); + assert_eq!(redacted, "iggy+tcp://iggy:***@127.0.0.1:8090"); + assert!(!redacted.contains("password")); + + let token_redacted = redact_connection_string("iggy+tcp://iggypat-abc123@127.0.0.1:8090"); + assert_eq!(token_redacted, "iggy+tcp://***@127.0.0.1:8090"); + assert!(!token_redacted.contains("abc123")); + + assert_eq!(redact_connection_string("not-a-url"), "***"); + } +} diff --git a/core/integration/tests/connectors/iggy_source/mod.rs b/core/integration/tests/connectors/iggy_source/mod.rs new file mode 100644 index 0000000000..8b79a333a8 --- /dev/null +++ b/core/integration/tests/connectors/iggy_source/mod.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod replication; diff --git a/core/integration/tests/connectors/iggy_source/replication.rs b/core/integration/tests/connectors/iggy_source/replication.rs new file mode 100644 index 0000000000..e42d311ec8 --- /dev/null +++ b/core/integration/tests/connectors/iggy_source/replication.rs @@ -0,0 +1,363 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use async_trait::async_trait; +use bytes::Bytes; +use iggy::prelude::{ + DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME, HeaderKey, HeaderValue, IggyClient, IggyMessage, + Partitioning, +}; +use iggy_common::{ + Consumer, Identifier, MessageClient, PollingStrategy, StreamClient, TopicClient, +}; +use integration::harness::{ + ServerHandle, TestBinary, TestBinaryError, TestContext, TestFixture, TestHarness, + TestServerConfig, seeds, +}; +use integration::iggy_harness; +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; +use std::time::Duration; +use tokio::time::sleep; + +const UPSTREAM_STREAM: &str = "upstream_stream"; +const UPSTREAM_TOPIC: &str = "upstream_topic"; +const TEST_MESSAGE_COUNT: usize = 5; +const POLL_ATTEMPTS: usize = 100; +const POLL_INTERVAL_MS: u64 = 50; +const POLL_BATCH: u32 = 100; +const HEADER_PRODUCER_KEY: &str = "producer"; +const HEADER_PRODUCER_VALUE: &str = "integration-test"; +const HEADER_SEQ_KEY: &str = "seq"; + +/// Boots a second `iggy-server` that acts as the upstream cluster the +/// `iggy_source` connector replicates from. The connector config's +/// `connection_string` is injected through the runtime env override. +pub struct IggySourceUpstreamFixture { + upstream: ServerHandle, +} + +#[async_trait] +impl TestFixture for IggySourceUpstreamFixture { + async fn setup() -> Result { + let mut context = TestContext::new(None, true)?; + context.ensure_created()?; + let mut upstream = ServerHandle::with_config( + TestServerConfig::builder() + .quic_enabled(false) + .websocket_enabled(false) + .http_enabled(false) + .build(), + Arc::new(context), + ); + upstream.start()?; + Ok(Self { upstream }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + let tcp_addr = self + .upstream + .tcp_addr() + .expect("upstream server TCP address"); + HashMap::from([ + ( + "IGGY_CONNECTORS_SOURCE_IGGY_PLUGIN_CONFIG_CONNECTION_STRING".to_string(), + format!("iggy+tcp://{DEFAULT_ROOT_USERNAME}:{DEFAULT_ROOT_PASSWORD}@{tcp_addr}"), + ), + ( + "IGGY_CONNECTORS_SOURCE_IGGY_PLUGIN_CONFIG_UPSTREAM_STREAM".to_string(), + UPSTREAM_STREAM.to_string(), + ), + ( + "IGGY_CONNECTORS_SOURCE_IGGY_PLUGIN_CONFIG_UPSTREAM_TOPIC".to_string(), + UPSTREAM_TOPIC.to_string(), + ), + ( + "IGGY_CONNECTORS_SOURCE_IGGY_PLUGIN_CONFIG_POLL_INTERVAL".to_string(), + "100ms".to_string(), + ), + ( + "IGGY_CONNECTORS_SOURCE_IGGY_PLUGIN_CONFIG_INITIAL_OFFSET".to_string(), + "earliest".to_string(), + ), + ( + "IGGY_CONNECTORS_SOURCE_IGGY_STREAMS_0_STREAM".to_string(), + seeds::names::STREAM.to_string(), + ), + ( + "IGGY_CONNECTORS_SOURCE_IGGY_STREAMS_0_TOPIC".to_string(), + seeds::names::TOPIC.to_string(), + ), + ( + "IGGY_CONNECTORS_SOURCE_IGGY_STREAMS_0_SCHEMA".to_string(), + "raw".to_string(), + ), + ( + "IGGY_CONNECTORS_SOURCE_IGGY_PATH".to_string(), + "../../target/debug/libiggy_connector_iggy_source".to_string(), + ), + ]) + } +} + +impl IggySourceUpstreamFixture { + pub async fn client(&self) -> Result { + self.upstream + .tcp_client()? + .with_root_login() + .connect() + .await + } + + /// The connector auto-creates the upstream stream/topic in `open()`; wait + /// until both exist before producing so no message races topic creation. + pub async fn ensure_upstream_topic(&self, client: &IggyClient) { + let stream_id = Identifier::named(UPSTREAM_STREAM).expect("valid stream name"); + let topic_id = Identifier::named(UPSTREAM_TOPIC).expect("valid topic name"); + for _ in 0..POLL_ATTEMPTS { + if client + .get_stream(&stream_id) + .await + .is_ok_and(|stream| stream.is_some()) + && client + .get_topic(&stream_id, &topic_id) + .await + .is_ok_and(|topic| topic.is_some()) + { + return; + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + panic!("Upstream stream/topic were not created within the poll window"); + } + + pub async fn produce_messages(&self, client: &IggyClient, payloads: &[String]) { + let mut messages = payloads + .iter() + .enumerate() + .map(|(i, payload)| { + let mut headers = BTreeMap::new(); + headers.insert( + HeaderKey::try_from(HEADER_PRODUCER_KEY).expect("valid header key"), + HeaderValue::try_from(HEADER_PRODUCER_VALUE).expect("valid header value"), + ); + headers.insert( + HeaderKey::try_from(HEADER_SEQ_KEY).expect("valid header key"), + (i as u64).into(), + ); + IggyMessage::builder() + .id((i as u128) + 1) + .payload(Bytes::from(payload.clone())) + .user_headers(headers) + .build() + .expect("Failed to build upstream message") + }) + .collect::>(); + client + .send_messages( + &Identifier::named(UPSTREAM_STREAM).expect("valid stream name"), + &Identifier::named(UPSTREAM_TOPIC).expect("valid topic name"), + &Partitioning::partition_id(0), + &mut messages, + ) + .await + .expect("Failed to send messages upstream"); + } +} + +/// Drains the downstream test topic with a fresh consumer, returning every +/// message exactly once. Waits until at least `min_expected` messages arrive +/// (the connector syncs asynchronously), then ends the drain after two +/// consecutive empty polls, which also covers the connector's in-flight +/// batches. +async fn drain_downstream_topic( + harness: &TestHarness, + consumer_name: &str, + min_expected: usize, +) -> Vec { + let client = harness.root_client().await.expect("root client"); + let stream_id: Identifier = seeds::names::STREAM.try_into().expect("valid stream name"); + let topic_id: Identifier = seeds::names::TOPIC.try_into().expect("valid topic name"); + let consumer = Consumer::new(Identifier::named(consumer_name).expect("valid consumer name")); + + let mut received: Vec = Vec::new(); + let mut empty_polls = 0usize; + for _ in 0..POLL_ATTEMPTS { + let polled = client + .poll_messages( + &stream_id, + &topic_id, + None, + &consumer, + &PollingStrategy::next(), + POLL_BATCH, + true, + ) + .await + .expect("Failed to poll downstream topic"); + if polled.messages.is_empty() { + // Two consecutive empty polls only end the drain once the + // expected messages have been observed; before that, an empty + // topic just means the connector has not synced yet. + if received.len() >= min_expected { + empty_polls += 1; + if empty_polls >= 2 { + break; + } + } + } else { + empty_polls = 0; + received.extend(polled.messages); + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + received +} + +fn assert_headers(message: &IggyMessage, expected_seq: u64) { + let headers = message + .user_headers_map() + .expect("Failed to parse user headers") + .expect("User headers missing"); + assert_eq!( + headers + .get(&HeaderKey::try_from(HEADER_PRODUCER_KEY).expect("valid header key")) + .map(HeaderValue::as_str) + .and_then(Result::ok), + Some(HEADER_PRODUCER_VALUE), + "Producer header mismatch" + ); + assert_eq!( + headers + .get(&HeaderKey::try_from(HEADER_SEQ_KEY).expect("valid header key")) + .map(HeaderValue::as_uint64) + .and_then(Result::ok), + Some(expected_seq), + "Seq header mismatch" + ); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), + seed = seeds::connector_stream +)] +async fn iggy_source_replicates_messages_with_headers( + harness: &TestHarness, + fixture: IggySourceUpstreamFixture, +) { + let upstream_client = fixture.client().await.expect("upstream client"); + fixture.ensure_upstream_topic(&upstream_client).await; + + let payloads: Vec = (0..TEST_MESSAGE_COUNT) + .map(|i| format!("upstream-message-{i}")) + .collect(); + fixture.produce_messages(&upstream_client, &payloads).await; + + let received = + drain_downstream_topic(harness, "iggy_source_headers_consumer", TEST_MESSAGE_COUNT).await; + assert_eq!( + received.len(), + TEST_MESSAGE_COUNT, + "Expected {TEST_MESSAGE_COUNT} synced messages, got {}", + received.len() + ); + + for (i, message) in received.iter().enumerate() { + assert_eq!( + String::from_utf8_lossy(&message.payload), + payloads[i], + "Payload mismatch at index {i}" + ); + assert_headers(message, i as u64); + } +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), + seed = seeds::connector_stream +)] +async fn state_persists_across_connector_restart( + harness: &mut TestHarness, + fixture: IggySourceUpstreamFixture, +) { + let upstream_client = fixture.client().await.expect("upstream client"); + fixture.ensure_upstream_topic(&upstream_client).await; + + let first_batch: Vec = (0..TEST_MESSAGE_COUNT) + .map(|i| format!("first-batch-{i}")) + .collect(); + fixture + .produce_messages(&upstream_client, &first_batch) + .await; + + let received_before = + drain_downstream_topic(harness, "iggy_source_restart_consumer", TEST_MESSAGE_COUNT).await; + assert_eq!( + received_before.len(), + TEST_MESSAGE_COUNT, + "Expected {TEST_MESSAGE_COUNT} messages before restart, got {}", + received_before.len() + ); + + harness + .server_mut() + .stop_dependents() + .expect("Failed to stop connectors runtime"); + + let second_batch: Vec = (0..TEST_MESSAGE_COUNT) + .map(|i| format!("second-batch-{i}")) + .collect(); + fixture + .produce_messages(&upstream_client, &second_batch) + .await; + + harness + .server_mut() + .start_dependents() + .await + .expect("Failed to restart connectors runtime"); + + // A fresh consumer drains the whole downstream topic exactly once. If the + // connector replayed the first batch after restart (stale state) or + // skipped the second batch (offset jumped too far), the sequence below + // would differ. + let received_after = drain_downstream_topic( + harness, + "iggy_source_restart_consumer_after", + TEST_MESSAGE_COUNT * 2, + ) + .await; + let expected: Vec = first_batch + .iter() + .chain(second_batch.iter()) + .cloned() + .collect(); + assert_eq!( + received_after.len(), + expected.len(), + "Expected exactly {} messages after restart (no duplicates, no loss), got {}", + expected.len(), + received_after.len() + ); + for (i, message) in received_after.iter().enumerate() { + assert_eq!( + String::from_utf8_lossy(&message.payload), + expected[i], + "Sequence mismatch at index {i}" + ); + } +} diff --git a/core/integration/tests/connectors/iggy_source/source.toml b/core/integration/tests/connectors/iggy_source/source.toml new file mode 100644 index 0000000000..e25cd79246 --- /dev/null +++ b/core/integration/tests/connectors/iggy_source/source.toml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[connectors] +config_type = "local" +config_dir = "../connectors/sources/iggy_source" diff --git a/core/integration/tests/connectors/mod.rs b/core/integration/tests/connectors/mod.rs index c5a9a13e06..76caafece1 100644 --- a/core/integration/tests/connectors/mod.rs +++ b/core/integration/tests/connectors/mod.rs @@ -24,6 +24,7 @@ mod fixtures; mod http; mod http_config_provider; mod iceberg; +mod iggy_source; mod influxdb; mod meilisearch; mod mongodb; From a5ddffbacca4c5ddcdf78a0eaecd5f9beb348d8b Mon Sep 17 00:00:00 2001 From: Gunther Xing Date: Thu, 27 Aug 2026 20:24:52 +0800 Subject: [PATCH 02/15] feat(connectors): commit Iggy source state after batch ACK --- .../connectors/sources/iggy_source/src/lib.rs | 117 ++++++++++++---- .../src/harness/handle/connectors_runtime.rs | 6 + .../connectors/iggy_source/replication.rs | 126 +++++++++++++++++- 3 files changed, 219 insertions(+), 30 deletions(-) diff --git a/core/connectors/sources/iggy_source/src/lib.rs b/core/connectors/sources/iggy_source/src/lib.rs index a9c818290b..0abcd1ce16 100644 --- a/core/connectors/sources/iggy_source/src/lib.rs +++ b/core/connectors/sources/iggy_source/src/lib.rs @@ -23,6 +23,7 @@ use iggy::prelude::{ use iggy_connector_sdk::{ ConnectorState, Error, ProducedMessage, ProducedMessages, Schema, Source, retry::{exponential_backoff, jitter, parse_duration}, + source::SourceBatchResult, source_connector, }; use secrecy::{ExposeSecret, SecretString}; @@ -82,11 +83,10 @@ impl FromStr for InitialOffset { } } -/// Persisted state. `offsets` maps each upstream partition to the offset of -/// the last message that was handed to the runtime and therefore successfully -/// produced to the downstream cluster; the runtime persists the state only -/// after a successful downstream send, so a restart resumes from `offset + 1`. -#[derive(Debug, Serialize, Deserialize)] +/// Committed state. `offsets` maps each upstream partition to the offset of +/// the last message confirmed by the runtime after both the downstream send +/// and checkpoint save succeeded. +#[derive(Debug, Clone, Serialize, Deserialize)] struct State { offsets: HashMap, messages_synced: u64, @@ -99,6 +99,7 @@ pub struct IggySource { config: IggySourceConfig, client: Option, state: Mutex, + pending_state: Mutex>, partitions: Vec, stream_id: Option, topic_id: Option, @@ -159,6 +160,7 @@ impl IggySource { messages_synced: 0, errors_count: 0, })), + pending_state: Mutex::new(None), partitions: Vec::new(), stream_id: None, topic_id: None, @@ -345,14 +347,9 @@ impl Source for IggySource { let consumer = Consumer::default(); - let saved_offsets: HashMap = { - let state = self.state.lock().await; - state.offsets.clone() - }; + let mut candidate_state = self.state.lock().await.clone(); let mut messages = Vec::with_capacity(self.partitions.len() * self.batch_size as usize); - let mut new_offsets: HashMap = HashMap::new(); - let mut reset_offsets: Vec = Vec::new(); let mut errors_in_cycle: u64 = 0; let mut any_success = false; let mut connection_failure = false; @@ -360,7 +357,7 @@ impl Source for IggySource { for &partition_id in &self.partitions { let strategy = next_strategy( self.initial_offset, - saved_offsets.get(&partition_id).copied(), + candidate_state.offsets.get(&partition_id).copied(), ); let polled = client .poll_messages( @@ -399,7 +396,7 @@ impl Source for IggySource { } } } - new_offsets.insert(partition_id, last_offset); + candidate_state.offsets.insert(partition_id, last_offset); } Err(IggyError::InvalidOffset(offset)) => { warn!( @@ -407,7 +404,7 @@ impl Source for IggySource { for {CONNECTOR_NAME} connector ID: {}, resetting to initial offset", self.id ); - reset_offsets.push(partition_id); + candidate_state.offsets.remove(&partition_id); errors_in_cycle += 1; } Err(e) => { @@ -423,19 +420,13 @@ impl Source for IggySource { } } - let (persisted_state, total_synced) = { - let mut state = self.state.lock().await; - for (partition_id, offset) in new_offsets { - state.offsets.insert(partition_id, offset); - } - for partition_id in reset_offsets { - state.offsets.remove(&partition_id); - } - state.messages_synced += messages.len() as u64; - state.errors_count += errors_in_cycle; - let persisted = self.serialize_state(&state); - (persisted, state.messages_synced) - }; + candidate_state.messages_synced += messages.len() as u64; + candidate_state.errors_count += errors_in_cycle; + let total_synced = candidate_state.messages_synced; + let persisted_state = self.serialize_state(&candidate_state).ok_or_else(|| { + Error::Serialization("failed to serialize Iggy source state".to_string()) + })?; + *self.pending_state.lock().await = Some(candidate_state); if connection_failure && !any_success { self.consecutive_failures.fetch_add(1, Ordering::Relaxed); @@ -468,10 +459,20 @@ impl Source for IggySource { Ok(ProducedMessages { schema: Schema::Raw, messages, - state: persisted_state, + state: Some(persisted_state), }) } + async fn on_batch_result(&self, result: SourceBatchResult) -> Result<(), Error> { + let candidate_state = self.pending_state.lock().await.take(); + if result == SourceBatchResult::Ack + && let Some(candidate_state) = candidate_state + { + *self.state.lock().await = candidate_state; + } + Ok(()) + } + async fn close(&mut self) -> Result<(), Error> { if let Some(client) = self.client.take() && let Err(e) = client.disconnect().await @@ -652,6 +653,66 @@ mod tests { assert_eq!(restored.messages_synced, 42); } + #[test] + fn given_nack_when_state_is_staged_should_keep_committed_state() { + let source = IggySource::new(1, test_config(), None); + let runtime = tokio::runtime::Runtime::new().expect("failed to create test runtime"); + runtime.block_on(async { + *source.state.lock().await = State { + offsets: HashMap::from([(0, 9)]), + messages_synced: 10, + errors_count: 1, + }; + *source.pending_state.lock().await = Some(State { + offsets: HashMap::from([(0, 19)]), + messages_synced: 20, + errors_count: 2, + }); + + source + .on_batch_result(SourceBatchResult::Nack) + .await + .expect("NACK should be applied"); + + let committed = source.state.lock().await; + assert_eq!(committed.offsets, HashMap::from([(0, 9)])); + assert_eq!(committed.messages_synced, 10); + assert_eq!(committed.errors_count, 1); + drop(committed); + assert!(source.pending_state.lock().await.is_none()); + }); + } + + #[test] + fn given_ack_when_state_is_staged_should_commit_candidate_state() { + let source = IggySource::new(1, test_config(), None); + let runtime = tokio::runtime::Runtime::new().expect("failed to create test runtime"); + runtime.block_on(async { + *source.state.lock().await = State { + offsets: HashMap::from([(0, 9)]), + messages_synced: 10, + errors_count: 1, + }; + *source.pending_state.lock().await = Some(State { + offsets: HashMap::from([(0, 19)]), + messages_synced: 20, + errors_count: 2, + }); + + source + .on_batch_result(SourceBatchResult::Ack) + .await + .expect("ACK should be applied"); + + let committed = source.state.lock().await; + assert_eq!(committed.offsets, HashMap::from([(0, 19)])); + assert_eq!(committed.messages_synced, 20); + assert_eq!(committed.errors_count, 2); + drop(committed); + assert!(source.pending_state.lock().await.is_none()); + }); + } + #[test] fn config_should_deserialize_from_json_with_defaults() { let json = r#"{ diff --git a/core/integration/src/harness/handle/connectors_runtime.rs b/core/integration/src/harness/handle/connectors_runtime.rs index a15fe7eb36..7f5b8fb874 100644 --- a/core/integration/src/harness/handle/connectors_runtime.rs +++ b/core/integration/src/harness/handle/connectors_runtime.rs @@ -77,6 +77,12 @@ impl ConnectorsRuntimeHandle { self.context.connectors_runtime_state_path(self.server_id) } + /// Add an environment variable to the connectors runtime config. + /// Must be called before starting the runtime. + pub fn add_env(&mut self, key: impl Into, value: impl Into) { + self.config.extra_envs.insert(key.into(), value.into()); + } + pub fn collect_logs(&self) -> (String, String) { common::collect_logs(&self.stdout_path, &self.stderr_path) } diff --git a/core/integration/tests/connectors/iggy_source/replication.rs b/core/integration/tests/connectors/iggy_source/replication.rs index e42d311ec8..9325e26411 100644 --- a/core/integration/tests/connectors/iggy_source/replication.rs +++ b/core/integration/tests/connectors/iggy_source/replication.rs @@ -24,22 +24,27 @@ use iggy::prelude::{ use iggy_common::{ Consumer, Identifier, MessageClient, PollingStrategy, StreamClient, TopicClient, }; +use iggy_connector_sdk::api::ConnectorRuntimeStats; use integration::harness::{ ServerHandle, TestBinary, TestBinaryError, TestContext, TestFixture, TestHarness, TestServerConfig, seeds, }; use integration::iggy_harness; +use reqwest::Client; use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use std::time::Duration; -use tokio::time::sleep; +use tokio::time::{sleep, timeout}; +const API_KEY: &str = "test-api-key"; +const SOURCE_KEY: &str = "iggy"; const UPSTREAM_STREAM: &str = "upstream_stream"; const UPSTREAM_TOPIC: &str = "upstream_topic"; const TEST_MESSAGE_COUNT: usize = 5; -const POLL_ATTEMPTS: usize = 100; +const POLL_ATTEMPTS: usize = 200; const POLL_INTERVAL_MS: u64 = 50; const POLL_BATCH: u32 = 100; +const WAIT_TIMEOUT: Duration = Duration::from_secs(15); const HEADER_PRODUCER_KEY: &str = "producer"; const HEADER_PRODUCER_VALUE: &str = "integration-test"; const HEADER_SEQ_KEY: &str = "seq"; @@ -251,7 +256,50 @@ fn assert_headers(message: &IggyMessage, expected_seq: u64) { ); } +async fn source_errors(http: &Client, api_url: &str) -> u64 { + let stats = http + .get(format!("{api_url}/stats")) + .header("api-key", API_KEY) + .send() + .await + .expect("runtime stats should be available") + .json::() + .await + .expect("runtime stats should be valid"); + stats + .connectors + .iter() + .find(|connector| connector.key == SOURCE_KEY) + .expect("Iggy source stats should be present") + .errors +} + +async fn wait_for_source_error_after(http: &Client, api_url: &str, previous_errors: u64) { + timeout(WAIT_TIMEOUT, async { + loop { + if let Ok(response) = http + .get(format!("{api_url}/stats")) + .header("api-key", API_KEY) + .send() + .await + && let Ok(stats) = response.json::().await + && let Some(source) = stats + .connectors + .iter() + .find(|connector| connector.key == SOURCE_KEY) + && source.errors > previous_errors + { + break; + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + }) + .await + .expect("Iggy source did not report a downstream send failure"); +} + #[iggy_harness( + cluster_nodes = 1, server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), seed = seeds::connector_stream )] @@ -287,6 +335,80 @@ async fn iggy_source_replicates_messages_with_headers( } #[iggy_harness( + cluster_nodes = 1, + server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), + seed = seeds::connector_stream +)] +async fn downstream_outage_replays_nacked_batch_after_recovery( + harness: &mut TestHarness, + fixture: IggySourceUpstreamFixture, +) { + let upstream_client = fixture.client().await.expect("upstream client"); + fixture.ensure_upstream_topic(&upstream_client).await; + + let downstream_address = harness + .server() + .tcp_addr() + .expect("downstream server TCP address"); + harness + .server_mut() + .stop_dependents() + .expect("Failed to stop connectors runtime"); + harness + .server_mut() + .connectors_runtime_mut() + .expect("connectors runtime") + .add_env( + "IGGY_CONNECTORS_IGGY_ADDRESS", + format!("{downstream_address}?reconnection_retries=1&reconnection_interval=100ms"), + ); + harness + .server_mut() + .start_dependents() + .await + .expect("Failed to restart connectors runtime with finite reconnection retries"); + + let api_url = harness + .connectors_runtime() + .expect("connectors runtime") + .http_url(); + let http = Client::new(); + let errors_before_outage = source_errors(&http, &api_url).await; + + harness + .server_mut() + .stop() + .expect("Failed to stop downstream server"); + + let payloads: Vec = (0..TEST_MESSAGE_COUNT) + .map(|i| format!("outage-message-{i}")) + .collect(); + fixture.produce_messages(&upstream_client, &payloads).await; + wait_for_source_error_after(&http, &api_url, errors_before_outage).await; + + harness + .server_mut() + .start() + .expect("Failed to restart downstream server"); + + let received = + drain_downstream_topic(harness, "iggy_source_outage_consumer", TEST_MESSAGE_COUNT).await; + assert_eq!( + received.len(), + TEST_MESSAGE_COUNT, + "Expected the NACKed batch to be replayed after downstream recovery" + ); + for (index, message) in received.iter().enumerate() { + assert_eq!( + String::from_utf8_lossy(&message.payload), + payloads[index], + "Payload mismatch at index {index}" + ); + } +} + +#[iggy_harness( + cluster_nodes = 1, server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), seed = seeds::connector_stream )] From 53c38d6e441bc6daded7eab098c6da3f8dbdb3e7 Mon Sep 17 00:00:00 2001 From: Gunther Xing Date: Mon, 7 Sep 2026 21:45:15 +0800 Subject: [PATCH 03/15] fix(connectors): reject malformed Iggy source batches --- .../connectors/sources/iggy_source/src/lib.rs | 114 ++++++++++++------ .../connectors/iggy_source/replication.rs | 105 +++++++++++++++- 2 files changed, 184 insertions(+), 35 deletions(-) diff --git a/core/connectors/sources/iggy_source/src/lib.rs b/core/connectors/sources/iggy_source/src/lib.rs index 0abcd1ce16..2c7dbd1446 100644 --- a/core/connectors/sources/iggy_source/src/lib.rs +++ b/core/connectors/sources/iggy_source/src/lib.rs @@ -377,25 +377,18 @@ impl Source for IggySource { if polled.messages.is_empty() { continue; } + let partition_messages = build_produced_messages( + &polled.messages, + self.include_user_headers, + partition_id, + self.id, + )?; let last_offset = polled .messages .last() .map(|message| message.header.offset) .unwrap_or_default(); - for message in &polled.messages { - match build_produced_message(message, self.include_user_headers) { - Ok(produced) => messages.push(produced), - Err(e) => { - error!( - "Failed to convert upstream message at offset {} on \ - partition {partition_id} for {CONNECTOR_NAME} connector \ - ID: {}: {e}", - message.header.offset, self.id - ); - errors_in_cycle += 1; - } - } - } + messages.extend(partition_messages); candidate_state.offsets.insert(partition_id, last_offset); } Err(IggyError::InvalidOffset(offset)) => { @@ -504,27 +497,43 @@ fn next_strategy(initial: InitialOffset, saved_offset: Option) -> PollingSt } } -fn build_produced_message( - message: &IggyMessage, +fn build_produced_messages( + messages: &[IggyMessage], include_user_headers: bool, -) -> Result { - let headers = if include_user_headers { - message.user_headers_map().map_err(|e| { - Error::InvalidRecordValue(format!("Failed to parse upstream user headers: {e}")) - })? - } else { - None - }; - - Ok(ProducedMessage { - id: (message.header.id != 0).then_some(message.header.id), - headers, - checksum: None, - timestamp: None, - origin_timestamp: (message.header.origin_timestamp != 0) - .then_some(message.header.origin_timestamp), - payload: message.payload.to_vec(), - }) + partition_id: u32, + connector_id: u32, +) -> Result, Error> { + let mut produced_messages = Vec::with_capacity(messages.len()); + for message in messages { + let headers = if include_user_headers { + message + .user_headers_map() + .map_err(|error| { + Error::InvalidRecordValue(format!( + "Failed to parse upstream user headers: {error}" + )) + }) + .inspect_err(|error| { + error!( + "Failed to convert upstream message at offset {} on partition \ + {partition_id} for {CONNECTOR_NAME} connector ID: {connector_id}: {error}", + message.header.offset + ); + })? + } else { + None + }; + produced_messages.push(ProducedMessage { + id: (message.header.id != 0).then_some(message.header.id), + headers, + checksum: None, + timestamp: None, + origin_timestamp: (message.header.origin_timestamp != 0) + .then_some(message.header.origin_timestamp), + payload: message.payload.to_vec(), + }); + } + Ok(produced_messages) } fn redact_connection_string(connection_string: &str) -> String { @@ -569,6 +578,30 @@ mod tests { } } + fn test_message(offset: u64) -> IggyMessage { + let mut message = IggyMessage::builder() + .payload("test payload".into()) + .build() + .expect("test message should be valid"); + message.header.offset = offset; + message + } + + fn test_message_with_unknown_header_kind(offset: u64) -> IggyMessage { + let mut user_headers = vec![0xFF]; + user_headers.extend_from_slice(&3u32.to_le_bytes()); + user_headers.extend_from_slice(b"key"); + user_headers.push(0xFF); + user_headers.extend_from_slice(&5u32.to_le_bytes()); + user_headers.extend_from_slice(b"value"); + + let mut message = test_message(offset); + message.header.user_headers_length = + u32::try_from(user_headers.len()).expect("test user headers should fit in u32"); + message.user_headers = Some(user_headers.into()); + message + } + #[test] fn given_persisted_state_should_restore_offsets_and_counts() { let state = State { @@ -791,6 +824,19 @@ mod tests { ); } + #[test] + fn given_malformed_message_when_building_polled_batch_should_reject_entire_batch() { + let messages = [ + test_message(10), + test_message_with_unknown_header_kind(11), + test_message(12), + ]; + + let result = build_produced_messages(&messages, true, 0, 1); + + assert!(matches!(result, Err(Error::InvalidRecordValue(_)))); + } + #[test] fn redact_connection_string_should_hide_credentials() { let redacted = redact_connection_string("iggy+tcp://iggy:password@127.0.0.1:8090"); diff --git a/core/integration/tests/connectors/iggy_source/replication.rs b/core/integration/tests/connectors/iggy_source/replication.rs index 9325e26411..5f36c17e42 100644 --- a/core/integration/tests/connectors/iggy_source/replication.rs +++ b/core/integration/tests/connectors/iggy_source/replication.rs @@ -48,6 +48,8 @@ const WAIT_TIMEOUT: Duration = Duration::from_secs(15); const HEADER_PRODUCER_KEY: &str = "producer"; const HEADER_PRODUCER_VALUE: &str = "integration-test"; const HEADER_SEQ_KEY: &str = "seq"; +const INCLUDE_USER_HEADERS_ENV: &str = + "IGGY_CONNECTORS_SOURCE_IGGY_PLUGIN_CONFIG_INCLUDE_USER_HEADERS"; /// Boots a second `iggy-server` that acts as the upstream cluster the /// `iggy_source` connector replicates from. The connector config's @@ -172,18 +174,40 @@ impl IggySourceUpstreamFixture { .expect("Failed to build upstream message") }) .collect::>(); + self.produce_iggy_messages(client, &mut messages).await; + } + + async fn produce_iggy_messages(&self, client: &IggyClient, messages: &mut [IggyMessage]) { client .send_messages( &Identifier::named(UPSTREAM_STREAM).expect("valid stream name"), &Identifier::named(UPSTREAM_TOPIC).expect("valid topic name"), &Partitioning::partition_id(0), - &mut messages, + messages, ) .await .expect("Failed to send messages upstream"); } } +fn message_with_unknown_header_kind(payload: &str) -> IggyMessage { + let mut user_headers = vec![0xFF]; + user_headers.extend_from_slice(&3u32.to_le_bytes()); + user_headers.extend_from_slice(b"key"); + user_headers.push(0xFF); + user_headers.extend_from_slice(&5u32.to_le_bytes()); + user_headers.extend_from_slice(b"value"); + + let mut message = IggyMessage::builder() + .payload(Bytes::copy_from_slice(payload.as_bytes())) + .build() + .expect("test message should be valid"); + message.header.user_headers_length = + u32::try_from(user_headers.len()).expect("test user headers should fit in u32"); + message.user_headers = Some(user_headers.into()); + message +} + /// Drains the downstream test topic with a fresh consumer, returning every /// message exactly once. Waits until at least `min_expected` messages arrive /// (the connector syncs asynchronously), then ends the drain after two @@ -298,6 +322,23 @@ async fn wait_for_source_error_after(http: &Client, api_url: &str, previous_erro .expect("Iggy source did not report a downstream send failure"); } +async fn wait_for_connector_log(harness: &TestHarness, marker: &str) { + timeout(WAIT_TIMEOUT, async { + loop { + let (stdout, stderr) = harness + .connectors_runtime() + .expect("connectors runtime") + .collect_logs(); + if stdout.contains(marker) || stderr.contains(marker) { + break; + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + }) + .await + .expect("Iggy source did not log the expected message build failure"); +} + #[iggy_harness( cluster_nodes = 1, server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), @@ -334,6 +375,68 @@ async fn iggy_source_replicates_messages_with_headers( } } +#[iggy_harness( + cluster_nodes = 1, + server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), + seed = seeds::connector_stream +)] +async fn malformed_user_headers_do_not_advance_source_offset( + harness: &mut TestHarness, + fixture: IggySourceUpstreamFixture, +) { + let upstream_client = fixture.client().await.expect("upstream client"); + fixture.ensure_upstream_topic(&upstream_client).await; + + let first_payload = vec!["before-malformed-headers".to_string()]; + fixture + .produce_messages(&upstream_client, &first_payload) + .await; + let first_received = drain_downstream_topic(harness, "iggy_source_before_malformed", 1).await; + assert_eq!(first_received.len(), 1, "Expected the first offset to sync"); + + let mut blocked_batch = vec![ + message_with_unknown_header_kind("malformed-headers"), + IggyMessage::builder() + .payload(Bytes::from_static(b"after-malformed-headers")) + .build() + .expect("test message should be valid"), + ]; + fixture + .produce_iggy_messages(&upstream_client, &mut blocked_batch) + .await; + wait_for_connector_log(harness, "Failed to convert upstream message at offset 1").await; + + harness + .server_mut() + .stop_dependents() + .expect("Failed to stop connectors runtime"); + harness + .server_mut() + .connectors_runtime_mut() + .expect("connectors runtime") + .add_env(INCLUDE_USER_HEADERS_ENV, "false"); + harness + .server_mut() + .start_dependents() + .await + .expect("Failed to restart connectors runtime without user headers"); + + let received = drain_downstream_topic(harness, "iggy_source_after_malformed", 3).await; + let payloads = received + .iter() + .map(|message| String::from_utf8_lossy(&message.payload).into_owned()) + .collect::>(); + assert_eq!( + payloads, + [ + "before-malformed-headers", + "malformed-headers", + "after-malformed-headers", + ], + "The malformed offset and its tail should be replayed after restart" + ); +} + #[iggy_harness( cluster_nodes = 1, server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), From 8a0217a3f18d252ca185fbcc8addb3492cdbd8cf Mon Sep 17 00:00:00 2001 From: Gunther Xing Date: Tue, 8 Sep 2026 11:33:14 +0800 Subject: [PATCH 04/15] fix(connectors): back off after invalid source offsets --- core/connectors/sources/iggy_source/src/lib.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/core/connectors/sources/iggy_source/src/lib.rs b/core/connectors/sources/iggy_source/src/lib.rs index 2c7dbd1446..125644b4ff 100644 --- a/core/connectors/sources/iggy_source/src/lib.rs +++ b/core/connectors/sources/iggy_source/src/lib.rs @@ -338,7 +338,7 @@ impl Source for IggySource { self.max_retry_interval, )); debug!( - "Backing off for {delay:?} after {failures} consecutive failures for \ + "Backing off for {delay:?} after {failures} consecutive poll failures for \ {CONNECTOR_NAME} connector ID: {}", self.id ); @@ -351,8 +351,6 @@ impl Source for IggySource { let mut messages = Vec::with_capacity(self.partitions.len() * self.batch_size as usize); let mut errors_in_cycle: u64 = 0; - let mut any_success = false; - let mut connection_failure = false; for &partition_id in &self.partitions { let strategy = next_strategy( @@ -373,7 +371,6 @@ impl Source for IggySource { match polled { Ok(polled) => { - any_success = true; if polled.messages.is_empty() { continue; } @@ -407,7 +404,6 @@ impl Source for IggySource { self.id ); errors_in_cycle += 1; - connection_failure = true; break; } } @@ -421,7 +417,7 @@ impl Source for IggySource { })?; *self.pending_state.lock().await = Some(candidate_state); - if connection_failure && !any_success { + if errors_in_cycle > 0 { self.consecutive_failures.fetch_add(1, Ordering::Relaxed); } else { self.consecutive_failures.store(0, Ordering::Relaxed); From 1546b9ba4532fe3ec707d450b7ed564f6d211ccb Mon Sep 17 00:00:00 2001 From: Gunther Xing Date: Tue, 8 Sep 2026 15:59:17 +0800 Subject: [PATCH 05/15] docs(connectors): clarify latest offset retry risk --- core/connectors/sources/iggy_source/config.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/connectors/sources/iggy_source/config.toml b/core/connectors/sources/iggy_source/config.toml index 2f4d3bc144..0a27c336bb 100644 --- a/core/connectors/sources/iggy_source/config.toml +++ b/core/connectors/sources/iggy_source/config.toml @@ -35,6 +35,11 @@ upstream_stream = "upstream_stream" upstream_topic = "upstream_topic" poll_interval = "1s" batch_size = 100 +# Starting position used only when a partition has no committed offset: +# "earliest", "latest", or an absolute numeric offset. +# With "latest", if the first batch is NACKed before an offset is committed, +# retrying recalculates the moving tail and may skip messages from the failed +# batch when the upstream topic advances. initial_offset = "earliest" include_user_headers = true retry_interval = "1s" From a8735f20f215dd265d770084143543ea1f4bab2e Mon Sep 17 00:00:00 2001 From: Gunther Xing Date: Tue, 8 Sep 2026 16:46:29 +0800 Subject: [PATCH 06/15] test(connectors): expand Iggy source coverage --- Cargo.lock | 1 + core/connectors/sources/iggy_source/README.md | 119 ++++++++++ .../connectors/sources/iggy_source/src/lib.rs | 92 ++++++-- core/integration/Cargo.toml | 1 + .../connectors/iggy_source/replication.rs | 206 +++++++++++++++++- 5 files changed, 396 insertions(+), 23 deletions(-) create mode 100644 core/connectors/sources/iggy_source/README.md diff --git a/Cargo.lock b/Cargo.lock index 79fca6127d..81c1fb61ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8164,6 +8164,7 @@ dependencies = [ "reqwest-retry", "ring", "rmcp", + "rmp-serde", "rumqttc-v4-next", "rumqttc-v5-next", "rust-s3", diff --git a/core/connectors/sources/iggy_source/README.md b/core/connectors/sources/iggy_source/README.md new file mode 100644 index 0000000000..85ba06883a --- /dev/null +++ b/core/connectors/sources/iggy_source/README.md @@ -0,0 +1,119 @@ +# Iggy Source Connector + +The Iggy source connector replicates a topic from an upstream Apache Iggy +cluster into a stream and topic managed by the connectors runtime. It polls +every upstream partition and persists the last acknowledged offset for each +partition so replication can resume after a restart. + +## Features + +- Polls every partition of one upstream topic. +- Preserves payload bytes and non-zero message IDs. +- Preserves user headers by default, with an option to omit them. +- Stores per-partition offsets in the connector runtime's state storage. +- Retries polling failures with exponential backoff and jitter. +- Creates the configured upstream stream or topic when it does not exist. + +## Configuration + +```toml +type = "source" +key = "iggy" +enabled = true +version = 0 +name = "Iggy source" +path = "target/release/libiggy_connector_iggy_source" +verbose = false +benchmark = false + +[[streams]] +stream = "downstream_stream" +topic = "downstream_topic" +schema = "raw" +batch_length = 100 +linger_time = "5ms" + +[plugin_config] +connection_string = "iggy+tcp://iggy:iggy@127.0.0.1:8090" +upstream_stream = "upstream_stream" +upstream_topic = "upstream_topic" +poll_interval = "1s" +batch_size = 100 +initial_offset = "earliest" +include_user_headers = true +retry_interval = "1s" +max_retry_interval = "60s" +verbose_logging = false +``` + +Use `schema = "raw"` on the downstream stream to replicate payload bytes +without decoding or re-encoding them. The runtime-level `batch_length` controls +downstream producer batching, while `plugin_config.batch_size` controls how many +messages are requested from each upstream partition in one poll cycle. + +### Plugin Fields + +| Field | Required | Default | Description | +| --- | --- | --- | --- | +| `connection_string` | yes | none | Connection string for the upstream Iggy cluster. | +| `upstream_stream` | yes | none | Name of the stream to replicate from. | +| `upstream_topic` | yes | none | Name of the topic to replicate from. | +| `poll_interval` | no | `2s` | Delay before each upstream poll cycle. | +| `batch_size` | no | `100` | Maximum messages requested from each upstream partition per poll cycle. | +| `initial_offset` | no | `earliest` | Starting position for a partition without a saved offset. Accepts `earliest`, `latest`, or an absolute numeric offset. | +| `include_user_headers` | no | `true` | Copy user headers to downstream messages. | +| `retry_interval` | no | `1s` | Base delay used for exponential backoff after a failed poll cycle. | +| `max_retry_interval` | no | `60s` | Maximum delay between poll retries. | +| `verbose_logging` | no | `false` | Log per-cycle connector details at info level instead of debug level. | + +The `connection_string` can use any transport supported by the Rust client, +including TCP, QUIC, HTTP, and WebSocket. It may contain credentials, so keep +the connector configuration private or supply the value through the +`IGGY_CONNECTORS_SOURCE_IGGY_PLUGIN_CONFIG_CONNECTION_STRING` environment +variable. + +## Offset and Delivery Semantics + +For each partition, the connector resumes at one offset after the last saved +offset. `initial_offset` is used only when that partition has no saved offset: + +- `earliest` starts with the oldest available message. +- `latest` requests up to `batch_size` of the most recent messages. +- A numeric value starts at that absolute offset. + +The connector stages new offsets while polling. It commits them only after the +runtime sends the complete downstream batch and saves the connector state. If +the batch is rejected, the previous offsets remain committed and the messages +are eligible for replay. Consumers should therefore tolerate duplicate +delivery after failures or crashes. + +> When `initial_offset = "latest"` and no offset has been committed yet, each +> retry recalculates the current tail of the upstream partition. If a first +> batch is rejected while new messages arrive, messages from that failed batch +> can be skipped. Use `earliest` or a numeric offset when replay continuity is +> required. + +If retention makes a saved offset invalid, the connector clears that +partition's saved position and applies `initial_offset` again. + +## Message Mapping + +| Upstream field | Downstream behavior | +| --- | --- | +| Payload | Copied without modification when the downstream schema is `raw`. | +| Message ID | Copied when the upstream ID is non-zero. | +| User headers | Copied when `include_user_headers = true`. | +| Offset and partition | Reassigned by the downstream Iggy topic. | +| Timestamp and checksum | Generated for the downstream message. | + +## Operational Notes + +- The connector discovers upstream partitions when it opens. Restart it after + adding partitions to the upstream topic. +- If the upstream stream is missing, the connector creates it. If the topic is + missing, it creates the topic with one partition and no compression. +- Invalid `initial_offset` values produce a warning and fall back to + `earliest`. +- If user headers cannot be decoded, the connector rejects the complete polled + batch so its offsets are not advanced. Set `include_user_headers = false` to + replicate payloads without parsing upstream headers. diff --git a/core/connectors/sources/iggy_source/src/lib.rs b/core/connectors/sources/iggy_source/src/lib.rs index 125644b4ff..23dea2b3e0 100644 --- a/core/connectors/sources/iggy_source/src/lib.rs +++ b/core/connectors/sources/iggy_source/src/lib.rs @@ -179,6 +179,24 @@ impl IggySource { ConnectorState::serialize(state, CONNECTOR_NAME, self.id) } + fn try_recover_invalid_offset( + &self, + state: &mut State, + partition_id: u32, + poll_error: IggyError, + ) -> Result<(), IggyError> { + let IggyError::InvalidOffset(offset) = poll_error else { + return Err(poll_error); + }; + warn!( + "Saved offset {offset} for partition {partition_id} no longer valid \ + for {CONNECTOR_NAME} connector ID: {}, resetting to initial offset", + self.id + ); + state.offsets.remove(&partition_id); + Ok(()) + } + async fn ensure_stream_and_topic( &self, client: &IggyClient, @@ -388,23 +406,20 @@ impl Source for IggySource { messages.extend(partition_messages); candidate_state.offsets.insert(partition_id, last_offset); } - Err(IggyError::InvalidOffset(offset)) => { - warn!( - "Saved offset {offset} for partition {partition_id} no longer valid \ - for {CONNECTOR_NAME} connector ID: {}, resetting to initial offset", - self.id - ); - candidate_state.offsets.remove(&partition_id); - errors_in_cycle += 1; - } - Err(e) => { - error!( - "Failed to poll partition {partition_id} for {CONNECTOR_NAME} \ - connector ID: {}: {e}", - self.id - ); + Err(poll_error) => { errors_in_cycle += 1; - break; + if let Err(poll_error) = self.try_recover_invalid_offset( + &mut candidate_state, + partition_id, + poll_error, + ) { + error!( + "Failed to poll partition {partition_id} for {CONNECTOR_NAME} \ + connector ID: {}: {poll_error}", + self.id + ); + break; + } } } } @@ -820,6 +835,51 @@ mod tests { ); } + #[test] + fn given_invalid_offset_should_reset_only_affected_partition_to_initial_strategy() { + let config = IggySourceConfig { + initial_offset: Some("earliest".to_string()), + ..test_config() + }; + let source = IggySource::new(1, config, None); + let mut state = State { + offsets: HashMap::from([(0, 42), (1, 7)]), + messages_synced: 50, + errors_count: 3, + }; + + source + .try_recover_invalid_offset(&mut state, 0, IggyError::InvalidOffset(43)) + .expect("invalid offset should be recoverable"); + + assert_eq!(state.offsets, HashMap::from([(1, 7)])); + assert_eq!(state.messages_synced, 50); + assert_eq!(state.errors_count, 3); + assert_eq!( + next_strategy(source.initial_offset, state.offsets.get(&0).copied()), + PollingStrategy::first() + ); + assert_eq!( + next_strategy(source.initial_offset, state.offsets.get(&1).copied()), + PollingStrategy::offset(8) + ); + } + + #[test] + fn given_other_poll_error_should_preserve_partition_offsets() { + let source = IggySource::new(1, test_config(), None); + let mut state = State { + offsets: HashMap::from([(0, 42), (1, 7)]), + messages_synced: 50, + errors_count: 3, + }; + + let result = source.try_recover_invalid_offset(&mut state, 0, IggyError::Error); + + assert!(matches!(result, Err(IggyError::Error))); + assert_eq!(state.offsets, HashMap::from([(0, 42), (1, 7)])); + } + #[test] fn given_malformed_message_when_building_polled_batch_should_reject_entire_batch() { let messages = [ diff --git a/core/integration/Cargo.toml b/core/integration/Cargo.toml index 22a77873a2..0dd3b974a7 100644 --- a/core/integration/Cargo.toml +++ b/core/integration/Cargo.toml @@ -94,6 +94,7 @@ rmcp = { workspace = true, features = [ "transport-streamable-http-client", "transport-streamable-http-client-reqwest", ] } +rmp-serde = { workspace = true } rumqttc-v4 = { workspace = true } rumqttc-v5 = { workspace = true } rust-s3 = { workspace = true } diff --git a/core/integration/tests/connectors/iggy_source/replication.rs b/core/integration/tests/connectors/iggy_source/replication.rs index 5f36c17e42..36945800f2 100644 --- a/core/integration/tests/connectors/iggy_source/replication.rs +++ b/core/integration/tests/connectors/iggy_source/replication.rs @@ -22,7 +22,8 @@ use iggy::prelude::{ Partitioning, }; use iggy_common::{ - Consumer, Identifier, MessageClient, PollingStrategy, StreamClient, TopicClient, + Consumer, Identifier, MessageClient, PartitionClient, PollingStrategy, StreamClient, + TopicClient, }; use iggy_connector_sdk::api::ConnectorRuntimeStats; use integration::harness::{ @@ -31,6 +32,7 @@ use integration::harness::{ }; use integration::iggy_harness; use reqwest::Client; +use serde::Deserialize; use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use std::time::Duration; @@ -40,6 +42,7 @@ const API_KEY: &str = "test-api-key"; const SOURCE_KEY: &str = "iggy"; const UPSTREAM_STREAM: &str = "upstream_stream"; const UPSTREAM_TOPIC: &str = "upstream_topic"; +const MULTI_PARTITION_COUNT: u32 = 3; const TEST_MESSAGE_COUNT: usize = 5; const POLL_ATTEMPTS: usize = 200; const POLL_INTERVAL_MS: u64 = 50; @@ -58,6 +61,13 @@ pub struct IggySourceUpstreamFixture { upstream: ServerHandle, } +#[derive(Debug, Deserialize)] +struct PersistedIggySourceState { + offsets: HashMap, + messages_synced: u64, + errors_count: u64, +} + #[async_trait] impl TestFixture for IggySourceUpstreamFixture { async fn setup() -> Result { @@ -153,10 +163,22 @@ impl IggySourceUpstreamFixture { } pub async fn produce_messages(&self, client: &IggyClient, payloads: &[String]) { + self.produce_messages_to_partition(client, 0, 0, payloads) + .await; + } + + pub async fn produce_messages_to_partition( + &self, + client: &IggyClient, + partition_id: u32, + sequence_start: u64, + payloads: &[String], + ) { let mut messages = payloads .iter() .enumerate() .map(|(i, payload)| { + let sequence = sequence_start + i as u64; let mut headers = BTreeMap::new(); headers.insert( HeaderKey::try_from(HEADER_PRODUCER_KEY).expect("valid header key"), @@ -164,25 +186,31 @@ impl IggySourceUpstreamFixture { ); headers.insert( HeaderKey::try_from(HEADER_SEQ_KEY).expect("valid header key"), - (i as u64).into(), + sequence.into(), ); IggyMessage::builder() - .id((i as u128) + 1) + .id(u128::from(sequence) + 1) .payload(Bytes::from(payload.clone())) .user_headers(headers) .build() .expect("Failed to build upstream message") }) .collect::>(); - self.produce_iggy_messages(client, &mut messages).await; + self.produce_iggy_messages(client, partition_id, &mut messages) + .await; } - async fn produce_iggy_messages(&self, client: &IggyClient, messages: &mut [IggyMessage]) { + async fn produce_iggy_messages( + &self, + client: &IggyClient, + partition_id: u32, + messages: &mut [IggyMessage], + ) { client .send_messages( &Identifier::named(UPSTREAM_STREAM).expect("valid stream name"), &Identifier::named(UPSTREAM_TOPIC).expect("valid topic name"), - &Partitioning::partition_id(0), + &Partitioning::partition_id(partition_id), messages, ) .await @@ -339,6 +367,40 @@ async fn wait_for_connector_log(harness: &TestHarness, marker: &str) { .expect("Iggy source did not log the expected message build failure"); } +async fn wait_for_source_state( + harness: &TestHarness, + expected_offsets: &HashMap, + expected_messages_synced: u64, + minimum_errors: u64, +) -> PersistedIggySourceState { + let state_path = harness + .connectors_runtime() + .expect("connectors runtime") + .state_path() + .join("source_iggy.state"); + + timeout(WAIT_TIMEOUT, async { + loop { + if let Ok(bytes) = tokio::fs::read(&state_path).await + && let Ok(state) = rmp_serde::from_slice::(&bytes) + && state.offsets == *expected_offsets + && state.messages_synced == expected_messages_synced + && state.errors_count >= minimum_errors + { + break state; + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + }) + .await + .unwrap_or_else(|_| { + panic!( + "Iggy source state did not reach offsets {expected_offsets:?}, synced count \ + {expected_messages_synced}, and at least {minimum_errors} error(s)" + ) + }) +} + #[iggy_harness( cluster_nodes = 1, server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), @@ -402,7 +464,7 @@ async fn malformed_user_headers_do_not_advance_source_offset( .expect("test message should be valid"), ]; fixture - .produce_iggy_messages(&upstream_client, &mut blocked_batch) + .produce_iggy_messages(&upstream_client, 0, &mut blocked_batch) .await; wait_for_connector_log(harness, "Failed to convert upstream message at offset 1").await; @@ -437,6 +499,136 @@ async fn malformed_user_headers_do_not_advance_source_offset( ); } +#[iggy_harness( + cluster_nodes = 1, + server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), + seed = seeds::connector_stream +)] +async fn multiple_upstream_partitions_resume_from_independent_offsets( + harness: &mut TestHarness, + fixture: IggySourceUpstreamFixture, +) { + let upstream_client = fixture.client().await.expect("upstream client"); + fixture.ensure_upstream_topic(&upstream_client).await; + + harness + .server_mut() + .stop_dependents() + .expect("Failed to stop connectors runtime"); + let stream_id = Identifier::named(UPSTREAM_STREAM).expect("valid stream name"); + let topic_id = Identifier::named(UPSTREAM_TOPIC).expect("valid topic name"); + upstream_client + .create_partitions( + &stream_id, + &topic_id, + MULTI_PARTITION_COUNT.saturating_sub(1), + ) + .await + .expect("Failed to create upstream partitions"); + let topic = upstream_client + .get_topic(&stream_id, &topic_id) + .await + .expect("Failed to fetch upstream topic") + .expect("Upstream topic should exist"); + assert_eq!(topic.partitions_count, MULTI_PARTITION_COUNT); + harness + .server_mut() + .start_dependents() + .await + .expect("Failed to restart connectors runtime"); + + let partition_payloads = [ + vec!["partition-0-message-0".to_string()], + vec![ + "partition-1-message-0".to_string(), + "partition-1-message-1".to_string(), + ], + vec![ + "partition-2-message-0".to_string(), + "partition-2-message-1".to_string(), + "partition-2-message-2".to_string(), + ], + ]; + for (partition_id, payloads) in partition_payloads.iter().enumerate() { + fixture + .produce_messages_to_partition( + &upstream_client, + partition_id as u32, + (partition_id as u64) * 100, + payloads, + ) + .await; + } + + let initial_message_count = partition_payloads.iter().map(Vec::len).sum::(); + let initial_received = drain_downstream_topic( + harness, + "iggy_source_multiple_partitions_before_restart", + initial_message_count, + ) + .await; + let mut initial_actual = initial_received + .iter() + .map(|message| String::from_utf8_lossy(&message.payload).into_owned()) + .collect::>(); + let mut initial_expected = partition_payloads + .iter() + .flatten() + .cloned() + .collect::>(); + initial_actual.sort(); + initial_expected.sort(); + assert_eq!(initial_actual, initial_expected); + + let initial_offsets = HashMap::from([(0, 0), (1, 1), (2, 2)]); + wait_for_source_state(harness, &initial_offsets, initial_message_count as u64, 0).await; + + harness + .server_mut() + .stop_dependents() + .expect("Failed to stop connectors runtime"); + let resumed_payloads = [ + "partition-0-after-restart".to_string(), + "partition-1-after-restart".to_string(), + "partition-2-after-restart".to_string(), + ]; + for (partition_id, payload) in resumed_payloads.iter().enumerate() { + fixture + .produce_messages_to_partition( + &upstream_client, + partition_id as u32, + 1_000 + partition_id as u64, + std::slice::from_ref(payload), + ) + .await; + } + harness + .server_mut() + .start_dependents() + .await + .expect("Failed to restart connectors runtime"); + + let total_message_count = initial_message_count + resumed_payloads.len(); + let received_after_restart = drain_downstream_topic( + harness, + "iggy_source_multiple_partitions_after_restart", + total_message_count, + ) + .await; + let mut actual_after_restart = received_after_restart + .iter() + .map(|message| String::from_utf8_lossy(&message.payload).into_owned()) + .collect::>(); + let mut expected_after_restart = initial_expected; + expected_after_restart.extend(resumed_payloads); + actual_after_restart.sort(); + expected_after_restart.sort(); + assert_eq!(actual_after_restart, expected_after_restart); + + let resumed_offsets = HashMap::from([(0, 1), (1, 2), (2, 3)]); + wait_for_source_state(harness, &resumed_offsets, total_message_count as u64, 0).await; +} + #[iggy_harness( cluster_nodes = 1, server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), From ffcb38b9cc2512268b21a49d2e6b8d5f42e97231 Mon Sep 17 00:00:00 2001 From: Gunther Xing Date: Wed, 9 Sep 2026 14:59:42 +0800 Subject: [PATCH 07/15] build(connectors): include Iggy source in release artifacts --- .github/workflows/_build_rust_artifacts.yml | 2 +- .github/workflows/edge-release.yml | 1 + scripts/bump-version.sh | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_build_rust_artifacts.yml b/.github/workflows/_build_rust_artifacts.yml index df77bba9cc..fd9e51cb29 100644 --- a/.github/workflows/_build_rust_artifacts.yml +++ b/.github/workflows/_build_rust_artifacts.yml @@ -46,7 +46,7 @@ on: connector_plugins: type: string required: false - default: "iggy_connector_elasticsearch_sink,iggy_connector_elasticsearch_source,iggy_connector_iceberg_sink,iggy_connector_postgres_sink,iggy_connector_postgres_source,iggy_connector_quickwit_sink,iggy_connector_random_source,iggy_connector_s3_sink,iggy_connector_stdout_sink,iggy_connector_surrealdb_sink, iggy_connector_rabbitmq_sink" + default: "iggy_connector_elasticsearch_sink,iggy_connector_elasticsearch_source,iggy_connector_iceberg_sink,iggy_connector_iggy_source,iggy_connector_postgres_sink,iggy_connector_postgres_source,iggy_connector_quickwit_sink,iggy_connector_random_source,iggy_connector_s3_sink,iggy_connector_stdout_sink,iggy_connector_surrealdb_sink, iggy_connector_rabbitmq_sink" description: "Comma-separated list of connector plugin crates to build as shared libraries" outputs: artifact_name: diff --git a/.github/workflows/edge-release.yml b/.github/workflows/edge-release.yml index f464a53f9b..da85f2cb6f 100644 --- a/.github/workflows/edge-release.yml +++ b/.github/workflows/edge-release.yml @@ -98,6 +98,7 @@ jobs: - `iggy_connector_elasticsearch_sink` - `iggy_connector_elasticsearch_source` - `iggy_connector_iceberg_sink` + - `iggy_connector_iggy_source` - `iggy_connector_postgres_sink` - `iggy_connector_postgres_source` - `iggy_connector_quickwit_sink` diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index 0ef47a5d94..faf795d128 100755 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -89,7 +89,7 @@ EOF RUST_COMPONENTS="rust-sdk rust-common rust-binary-protocol rust-server rust-cli rust-connector-sdk rust-mcp rust-kafka-gateway rust-bench rust-bench-dashboard-frontend rust-bench-dashboard-server rust-bench-report" CONNECTOR_SINK_COMPONENTS="rust-connector-clickhouse-sink rust-connector-delta-sink rust-connector-doris-sink rust-connector-elasticsearch-sink rust-connector-http-sink rust-connector-iceberg-sink rust-connector-influxdb-sink rust-connector-meilisearch-sink rust-connector-mongodb-sink rust-connector-postgres-sink rust-connector-quickwit-sink rust-connector-rabbitmq-sink rust-connector-redshift-sink rust-connector-s3-sink rust-connector-stdout-sink rust-connector-surrealdb-sink" -CONNECTOR_SOURCE_COMPONENTS="rust-connector-elasticsearch-source rust-connector-http-source rust-connector-influxdb-source rust-connector-postgres-source rust-connector-random-source" +CONNECTOR_SOURCE_COMPONENTS="rust-connector-elasticsearch-source rust-connector-http-source rust-connector-iggy-source rust-connector-influxdb-source rust-connector-postgres-source rust-connector-random-source" CONNECTOR_COMPONENTS="rust-connector-runtime ${CONNECTOR_SINK_COMPONENTS} ${CONNECTOR_SOURCE_COMPONENTS}" SDK_COMPONENTS="sdk-python sdk-node sdk-go sdk-csharp sdk-java" ALL_COMPONENTS="${RUST_COMPONENTS} ${CONNECTOR_COMPONENTS} ${SDK_COMPONENTS} web-ui" From cb2aacd98462d2dbc13994b292ec6e3bdbf01f8d Mon Sep 17 00:00:00 2001 From: Gunther Xing Date: Wed, 9 Sep 2026 15:07:43 +0800 Subject: [PATCH 08/15] fix(connectors): isolate Iggy source poll failures --- .../connectors/iggy_source.toml | 3 + core/connectors/sources/iggy_source/README.md | 20 +- .../sources/iggy_source/config.toml | 3 + .../connectors/sources/iggy_source/src/lib.rs | 365 ++++++++++++------ .../connectors/iggy_source/replication.rs | 232 ++++++++--- 5 files changed, 451 insertions(+), 172 deletions(-) diff --git a/core/connectors/runtime/example_config/connectors/iggy_source.toml b/core/connectors/runtime/example_config/connectors/iggy_source.toml index 887d1b1177..a1b9b1ab38 100644 --- a/core/connectors/runtime/example_config/connectors/iggy_source.toml +++ b/core/connectors/runtime/example_config/connectors/iggy_source.toml @@ -41,6 +41,9 @@ batch_size = 100 # a saved offset in the persisted connector state initial_offset = "earliest" include_user_headers = true +# "block" preserves malformed messages for retry; "drop_headers" forwards +# their payload without user headers. +malformed_message_policy = "block" retry_interval = "1s" max_retry_interval = "60s" verbose_logging = false diff --git a/core/connectors/sources/iggy_source/README.md b/core/connectors/sources/iggy_source/README.md index 85ba06883a..6d73d64d74 100644 --- a/core/connectors/sources/iggy_source/README.md +++ b/core/connectors/sources/iggy_source/README.md @@ -11,7 +11,7 @@ partition so replication can resume after a restart. - Preserves payload bytes and non-zero message IDs. - Preserves user headers by default, with an option to omit them. - Stores per-partition offsets in the connector runtime's state storage. -- Retries polling failures with exponential backoff and jitter. +- Retries fully failed polling cycles with exponential backoff and jitter. - Creates the configured upstream stream or topic when it does not exist. ## Configuration @@ -41,6 +41,7 @@ poll_interval = "1s" batch_size = 100 initial_offset = "earliest" include_user_headers = true +malformed_message_policy = "block" retry_interval = "1s" max_retry_interval = "60s" verbose_logging = false @@ -62,7 +63,8 @@ messages are requested from each upstream partition in one poll cycle. | `batch_size` | no | `100` | Maximum messages requested from each upstream partition per poll cycle. | | `initial_offset` | no | `earliest` | Starting position for a partition without a saved offset. Accepts `earliest`, `latest`, or an absolute numeric offset. | | `include_user_headers` | no | `true` | Copy user headers to downstream messages. | -| `retry_interval` | no | `1s` | Base delay used for exponential backoff after a failed poll cycle. | +| `malformed_message_policy` | no | `block` | Handling for messages with unparsable user headers: `block` retries from the failed offset; `drop_headers` forwards the payload without those headers. | +| `retry_interval` | no | `1s` | Base delay used for exponential backoff when every partition poll in a cycle fails. | | `max_retry_interval` | no | `60s` | Maximum delay between poll retries. | | `verbose_logging` | no | `false` | Log per-cycle connector details at info level instead of debug level. | @@ -114,6 +116,14 @@ partition's saved position and applies `initial_offset` again. missing, it creates the topic with one partition and no compression. - Invalid `initial_offset` values produce a warning and fall back to `earliest`. -- If user headers cannot be decoded, the connector rejects the complete polled - batch so its offsets are not advanced. Set `include_user_headers = false` to - replicate payloads without parsing upstream headers. +- If user headers cannot be decoded, `malformed_message_policy = "block"` + forwards the valid prefix before the malformed message, leaves that + partition at the failed offset, and continues polling other partitions at + the normal `poll_interval`. Record conversion errors do not activate the + connector-wide polling backoff. +- `malformed_message_policy = "drop_headers"` forwards a malformed message's + payload without its user headers and advances the partition offset. Setting + `include_user_headers = false` bypasses header parsing for every message. +- A polling error on one partition is logged and counted without skipping the + remaining partitions. Connector-wide backoff is activated only when no + partition poll succeeds during the cycle. diff --git a/core/connectors/sources/iggy_source/config.toml b/core/connectors/sources/iggy_source/config.toml index 0a27c336bb..46c886c1f2 100644 --- a/core/connectors/sources/iggy_source/config.toml +++ b/core/connectors/sources/iggy_source/config.toml @@ -42,6 +42,9 @@ batch_size = 100 # batch when the upstream topic advances. initial_offset = "earliest" include_user_headers = true +# "block" preserves malformed messages for retry; "drop_headers" forwards +# their payload without user headers. +malformed_message_policy = "block" retry_interval = "1s" max_retry_interval = "60s" verbose_logging = false diff --git a/core/connectors/sources/iggy_source/src/lib.rs b/core/connectors/sources/iggy_source/src/lib.rs index 23dea2b3e0..c1e79e006f 100644 --- a/core/connectors/sources/iggy_source/src/lib.rs +++ b/core/connectors/sources/iggy_source/src/lib.rs @@ -30,6 +30,7 @@ use secrecy::{ExposeSecret, SecretString}; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, + fmt, str::FromStr, sync::atomic::{AtomicU64, Ordering}, time::Duration, @@ -58,11 +59,30 @@ pub struct IggySourceConfig { pub batch_size: Option, pub initial_offset: Option, pub include_user_headers: Option, + #[serde(default)] + pub malformed_message_policy: MalformedMessagePolicy, pub retry_interval: Option, pub max_retry_interval: Option, pub verbose_logging: Option, } +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MalformedMessagePolicy { + #[default] + Block, + DropHeaders, +} + +impl fmt::Display for MalformedMessagePolicy { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Block => formatter.write_str("block"), + Self::DropHeaders => formatter.write_str("drop_headers"), + } + } +} + /// Starting point for a partition that has no saved offset in the state yet. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum InitialOffset { @@ -93,6 +113,23 @@ struct State { errors_count: u64, } +#[derive(Debug, Default)] +struct PollCycleCounts { + successful_upstream_polls: u64, + upstream_poll_errors: u64, + conversion_errors: u64, +} + +impl PollCycleCounts { + fn total_errors(&self) -> u64 { + self.upstream_poll_errors + self.conversion_errors + } + + fn requires_backoff(&self) -> bool { + self.upstream_poll_errors > 0 && self.successful_upstream_polls == 0 + } +} + #[derive(Debug)] pub struct IggySource { id: u32, @@ -106,10 +143,11 @@ pub struct IggySource { poll_interval: Duration, retry_interval: Duration, max_retry_interval: Duration, - consecutive_failures: AtomicU64, + consecutive_failed_poll_cycles: AtomicU64, initial_offset: InitialOffset, batch_size: u32, include_user_headers: bool, + malformed_message_policy: MalformedMessagePolicy, verbose: bool, } @@ -150,6 +188,7 @@ impl IggySource { let batch_size = config.batch_size.unwrap_or(DEFAULT_BATCH_SIZE); let include_user_headers = config.include_user_headers.unwrap_or(true); + let malformed_message_policy = config.malformed_message_policy; IggySource { id, @@ -167,10 +206,11 @@ impl IggySource { poll_interval, retry_interval, max_retry_interval, - consecutive_failures: AtomicU64::new(0), + consecutive_failed_poll_cycles: AtomicU64::new(0), initial_offset, batch_size, include_user_headers, + malformed_message_policy, verbose, } } @@ -179,24 +219,6 @@ impl IggySource { ConnectorState::serialize(state, CONNECTOR_NAME, self.id) } - fn try_recover_invalid_offset( - &self, - state: &mut State, - partition_id: u32, - poll_error: IggyError, - ) -> Result<(), IggyError> { - let IggyError::InvalidOffset(offset) = poll_error else { - return Err(poll_error); - }; - warn!( - "Saved offset {offset} for partition {partition_id} no longer valid \ - for {CONNECTOR_NAME} connector ID: {}, resetting to initial offset", - self.id - ); - state.offsets.remove(&partition_id); - Ok(()) - } - async fn ensure_stream_and_topic( &self, client: &IggyClient, @@ -322,12 +344,13 @@ impl Source for IggySource { self.client = Some(client); info!( "Opened {CONNECTOR_NAME} connector ID: {}, partitions: {}, initial offset: {:?}, \ - poll interval: {:?}, batch size: {}", + poll interval: {:?}, batch size: {}, malformed message policy: {}", self.id, self.partitions.len(), self.initial_offset, self.poll_interval, - self.batch_size + self.batch_size, + self.malformed_message_policy ); Ok(()) } @@ -348,15 +371,16 @@ impl Source for IggySource { .as_ref() .ok_or_else(|| Error::InitError("Upstream topic not initialized".to_string()))?; - let failures = self.consecutive_failures.load(Ordering::Relaxed); - if failures > 0 { + let failed_cycles = self.consecutive_failed_poll_cycles.load(Ordering::Relaxed); + if failed_cycles > 0 { let delay = jitter(exponential_backoff( self.retry_interval, - failures as u32, + failed_cycles as u32, self.max_retry_interval, )); debug!( - "Backing off for {delay:?} after {failures} consecutive poll failures for \ + "Backing off for {delay:?} after {failed_cycles} consecutive poll cycles with no \ + successful partitions for \ {CONNECTOR_NAME} connector ID: {}", self.id ); @@ -368,7 +392,7 @@ impl Source for IggySource { let mut candidate_state = self.state.lock().await.clone(); let mut messages = Vec::with_capacity(self.partitions.len() * self.batch_size as usize); - let mut errors_in_cycle: u64 = 0; + let mut cycle_counts = PollCycleCounts::default(); for &partition_id in &self.partitions { let strategy = next_strategy( @@ -389,74 +413,97 @@ impl Source for IggySource { match polled { Ok(polled) => { + cycle_counts.successful_upstream_polls += 1; if polled.messages.is_empty() { continue; } - let partition_messages = build_produced_messages( + let PartitionBuildOutcome { + messages: partition_messages, + last_produced_offset, + errors: conversion_errors, + } = build_produced_messages( &polled.messages, self.include_user_headers, - partition_id, - self.id, - )?; - let last_offset = polled - .messages - .last() - .map(|message| message.header.offset) - .unwrap_or_default(); - messages.extend(partition_messages); - candidate_state.offsets.insert(partition_id, last_offset); - } - Err(poll_error) => { - errors_in_cycle += 1; - if let Err(poll_error) = self.try_recover_invalid_offset( - &mut candidate_state, - partition_id, - poll_error, - ) { + self.malformed_message_policy, + ); + + cycle_counts.conversion_errors += conversion_errors.len() as u64; + for conversion_error in conversion_errors { error!( - "Failed to poll partition {partition_id} for {CONNECTOR_NAME} \ - connector ID: {}: {poll_error}", - self.id + "Failed to convert upstream message at offset {} on partition \ + {partition_id} for {CONNECTOR_NAME} connector ID: {}. \ + Malformed message policy: {}. {}", + conversion_error.offset, + self.id, + self.malformed_message_policy, + conversion_error.error ); - break; + } + + messages.extend(partition_messages); + if let Some(last_produced_offset) = last_produced_offset { + candidate_state + .offsets + .insert(partition_id, last_produced_offset); } } + Err(IggyError::InvalidOffset(offset)) => { + cycle_counts.upstream_poll_errors += 1; + warn!( + "Saved offset {offset} for partition {partition_id} no longer valid \ + for {CONNECTOR_NAME} connector ID: {}, resetting to initial offset", + self.id + ); + candidate_state.offsets.remove(&partition_id); + } + Err(poll_error) => { + cycle_counts.upstream_poll_errors += 1; + error!( + "Failed to poll partition {partition_id} for {CONNECTOR_NAME} \ + connector ID: {}: {poll_error}", + self.id + ); + } } } candidate_state.messages_synced += messages.len() as u64; - candidate_state.errors_count += errors_in_cycle; + candidate_state.errors_count += cycle_counts.total_errors(); let total_synced = candidate_state.messages_synced; let persisted_state = self.serialize_state(&candidate_state).ok_or_else(|| { Error::Serialization("failed to serialize Iggy source state".to_string()) })?; *self.pending_state.lock().await = Some(candidate_state); - if errors_in_cycle > 0 { - self.consecutive_failures.fetch_add(1, Ordering::Relaxed); + if cycle_counts.requires_backoff() { + self.consecutive_failed_poll_cycles + .fetch_add(1, Ordering::Relaxed); } else { - self.consecutive_failures.store(0, Ordering::Relaxed); + self.consecutive_failed_poll_cycles + .store(0, Ordering::Relaxed); } if self.verbose { info!( "{CONNECTOR_NAME} connector ID: {} polled {} messages from {} partition(s). \ - Total synced: {}, errors in cycle: {}", + Total synced: {}, poll errors in cycle: {}, conversion errors in cycle: {}", self.id, messages.len(), self.partitions.len(), total_synced, - errors_in_cycle + cycle_counts.upstream_poll_errors, + cycle_counts.conversion_errors ); } else { debug!( "{CONNECTOR_NAME} connector ID: {} polled {} messages from {} partition(s). \ - Total synced: {}, errors in cycle: {}", + Total synced: {}, poll errors in cycle: {}, conversion errors in cycle: {}", self.id, messages.len(), self.partitions.len(), total_synced, - errors_in_cycle + cycle_counts.upstream_poll_errors, + cycle_counts.conversion_errors ); } @@ -508,32 +555,49 @@ fn next_strategy(initial: InitialOffset, saved_offset: Option) -> PollingSt } } +#[derive(Debug)] +struct PartitionBuildOutcome { + messages: Vec, + last_produced_offset: Option, + errors: Vec, +} + +#[derive(Debug)] +struct MessageBuildError { + offset: u64, + error: Error, +} + fn build_produced_messages( messages: &[IggyMessage], include_user_headers: bool, - partition_id: u32, - connector_id: u32, -) -> Result, Error> { + malformed_message_policy: MalformedMessagePolicy, +) -> PartitionBuildOutcome { let mut produced_messages = Vec::with_capacity(messages.len()); + let mut last_produced_offset = None; + let mut errors = Vec::new(); + for message in messages { let headers = if include_user_headers { - message - .user_headers_map() - .map_err(|error| { - Error::InvalidRecordValue(format!( - "Failed to parse upstream user headers: {error}" - )) - }) - .inspect_err(|error| { - error!( - "Failed to convert upstream message at offset {} on partition \ - {partition_id} for {CONNECTOR_NAME} connector ID: {connector_id}: {error}", - message.header.offset - ); - })? + match message.user_headers_map() { + Ok(headers) => headers, + Err(error) => { + errors.push(MessageBuildError { + offset: message.header.offset, + error: Error::InvalidRecordValue(format!( + "Failed to parse upstream user headers: {error}" + )), + }); + if malformed_message_policy == MalformedMessagePolicy::Block { + break; + } + None + } + } } else { None }; + last_produced_offset = Some(message.header.offset); produced_messages.push(ProducedMessage { id: (message.header.id != 0).then_some(message.header.id), headers, @@ -544,7 +608,12 @@ fn build_produced_messages( payload: message.payload.to_vec(), }); } - Ok(produced_messages) + + PartitionBuildOutcome { + messages: produced_messages, + last_produced_offset, + errors, + } } fn redact_connection_string(connection_string: &str) -> String { @@ -570,6 +639,10 @@ fn redact_connection_string(connection_string: &str) -> String { #[cfg(test)] mod tests { + use std::collections::BTreeMap; + + use iggy::prelude::{HeaderKey, HeaderValue}; + use super::*; fn test_config() -> IggySourceConfig { @@ -583,6 +656,7 @@ mod tests { batch_size: Some(50), initial_offset: Some("0".to_string()), include_user_headers: Some(true), + malformed_message_policy: MalformedMessagePolicy::Block, retry_interval: Some("100ms".to_string()), max_retry_interval: Some("5s".to_string()), verbose_logging: Some(false), @@ -590,8 +664,14 @@ mod tests { } fn test_message(offset: u64) -> IggyMessage { + let mut headers = BTreeMap::new(); + headers.insert( + HeaderKey::try_from("key").expect("test header key should be valid"), + HeaderValue::try_from("value").expect("test header value should be valid"), + ); let mut message = IggyMessage::builder() .payload("test payload".into()) + .user_headers(headers) .build() .expect("test message should be valid"); message.header.offset = offset; @@ -680,6 +760,40 @@ mod tests { assert_eq!(original.errors_count, deserialized.errors_count); } + #[test] + fn given_only_conversion_errors_should_count_without_requiring_backoff() { + let cycle_counts = PollCycleCounts { + conversion_errors: 2, + ..PollCycleCounts::default() + }; + + assert_eq!(cycle_counts.total_errors(), 2); + assert!(!cycle_counts.requires_backoff()); + } + + #[test] + fn given_all_upstream_polls_failed_should_require_backoff() { + let cycle_counts = PollCycleCounts { + upstream_poll_errors: 1, + ..PollCycleCounts::default() + }; + + assert_eq!(cycle_counts.total_errors(), 1); + assert!(cycle_counts.requires_backoff()); + } + + #[test] + fn given_partial_upstream_poll_success_should_not_require_backoff() { + let cycle_counts = PollCycleCounts { + successful_upstream_polls: 2, + upstream_poll_errors: 1, + conversion_errors: 1, + }; + + assert_eq!(cycle_counts.total_errors(), 2); + assert!(!cycle_counts.requires_backoff()); + } + #[test] fn serialize_state_helper_should_produce_valid_connector_state() { let source = IggySource::new(1, test_config(), None); @@ -771,6 +885,10 @@ mod tests { assert_eq!(config.upstream_topic, "topic"); assert!(config.poll_interval.is_none()); assert!(config.initial_offset.is_none()); + assert_eq!( + config.malformed_message_policy, + MalformedMessagePolicy::Block + ); } #[test] @@ -789,10 +907,32 @@ mod tests { assert_eq!(source.max_retry_interval, Duration::from_secs(60)); assert_eq!(source.batch_size, DEFAULT_BATCH_SIZE); assert!(source.include_user_headers); + assert_eq!( + source.malformed_message_policy, + MalformedMessagePolicy::Block + ); assert_eq!(source.initial_offset, InitialOffset::Earliest); assert!(!source.verbose); } + #[test] + fn configured_drop_headers_policy_should_be_applied() { + let json = r#"{ + "connection_string": "iggy+tcp://iggy:iggy@127.0.0.1:8090", + "upstream_stream": "stream", + "upstream_topic": "topic", + "malformed_message_policy": "drop_headers" + }"#; + + let config: IggySourceConfig = serde_json::from_str(json).expect("Failed to parse config"); + let source = IggySource::new(1, config, None); + + assert_eq!( + source.malformed_message_policy, + MalformedMessagePolicy::DropHeaders + ); + } + #[test] fn given_invalid_initial_offset_should_default_to_earliest() { let config = IggySourceConfig { @@ -836,61 +976,46 @@ mod tests { } #[test] - fn given_invalid_offset_should_reset_only_affected_partition_to_initial_strategy() { - let config = IggySourceConfig { - initial_offset: Some("earliest".to_string()), - ..test_config() - }; - let source = IggySource::new(1, config, None); - let mut state = State { - offsets: HashMap::from([(0, 42), (1, 7)]), - messages_synced: 50, - errors_count: 3, - }; - - source - .try_recover_invalid_offset(&mut state, 0, IggyError::InvalidOffset(43)) - .expect("invalid offset should be recoverable"); - - assert_eq!(state.offsets, HashMap::from([(1, 7)])); - assert_eq!(state.messages_synced, 50); - assert_eq!(state.errors_count, 3); - assert_eq!( - next_strategy(source.initial_offset, state.offsets.get(&0).copied()), - PollingStrategy::first() - ); - assert_eq!( - next_strategy(source.initial_offset, state.offsets.get(&1).copied()), - PollingStrategy::offset(8) - ); - } - - #[test] - fn given_other_poll_error_should_preserve_partition_offsets() { - let source = IggySource::new(1, test_config(), None); - let mut state = State { - offsets: HashMap::from([(0, 42), (1, 7)]), - messages_synced: 50, - errors_count: 3, - }; + fn given_malformed_message_with_block_policy_should_return_successful_prefix() { + let messages = [ + test_message(10), + test_message_with_unknown_header_kind(11), + test_message(12), + ]; - let result = source.try_recover_invalid_offset(&mut state, 0, IggyError::Error); + let result = build_produced_messages(&messages, true, MalformedMessagePolicy::Block); - assert!(matches!(result, Err(IggyError::Error))); - assert_eq!(state.offsets, HashMap::from([(0, 42), (1, 7)])); + assert_eq!(result.messages.len(), 1); + assert_eq!(result.last_produced_offset, Some(10)); + assert_eq!(result.errors.len(), 1); + assert_eq!(result.errors[0].offset, 11); + assert!(matches!( + &result.errors[0].error, + Error::InvalidRecordValue(_) + )); } #[test] - fn given_malformed_message_when_building_polled_batch_should_reject_entire_batch() { + fn given_malformed_message_with_drop_headers_policy_should_forward_entire_batch() { let messages = [ test_message(10), test_message_with_unknown_header_kind(11), test_message(12), ]; - let result = build_produced_messages(&messages, true, 0, 1); - - assert!(matches!(result, Err(Error::InvalidRecordValue(_)))); + let result = build_produced_messages(&messages, true, MalformedMessagePolicy::DropHeaders); + + assert_eq!(result.messages.len(), 3); + assert_eq!(result.last_produced_offset, Some(12)); + assert!(result.messages[0].headers.is_some()); + assert!(result.messages[1].headers.is_none()); + assert!(result.messages[2].headers.is_some()); + assert_eq!(result.errors.len(), 1); + assert_eq!(result.errors[0].offset, 11); + assert!(matches!( + &result.errors[0].error, + Error::InvalidRecordValue(_) + )); } #[test] diff --git a/core/integration/tests/connectors/iggy_source/replication.rs b/core/integration/tests/connectors/iggy_source/replication.rs index 36945800f2..f7e515b086 100644 --- a/core/integration/tests/connectors/iggy_source/replication.rs +++ b/core/integration/tests/connectors/iggy_source/replication.rs @@ -51,8 +51,11 @@ const WAIT_TIMEOUT: Duration = Duration::from_secs(15); const HEADER_PRODUCER_KEY: &str = "producer"; const HEADER_PRODUCER_VALUE: &str = "integration-test"; const HEADER_SEQ_KEY: &str = "seq"; -const INCLUDE_USER_HEADERS_ENV: &str = - "IGGY_CONNECTORS_SOURCE_IGGY_PLUGIN_CONFIG_INCLUDE_USER_HEADERS"; +const MALFORMED_MESSAGE_POLICY_ENV: &str = + "IGGY_CONNECTORS_SOURCE_IGGY_PLUGIN_CONFIG_MALFORMED_MESSAGE_POLICY"; +const POLL_INTERVAL_ENV: &str = "IGGY_CONNECTORS_SOURCE_IGGY_PLUGIN_CONFIG_POLL_INTERVAL"; +const RETRY_INTERVAL_ENV: &str = "IGGY_CONNECTORS_SOURCE_IGGY_PLUGIN_CONFIG_RETRY_INTERVAL"; +const MAX_RETRY_INTERVAL_ENV: &str = "IGGY_CONNECTORS_SOURCE_IGGY_PLUGIN_CONFIG_MAX_RETRY_INTERVAL"; /// Boots a second `iggy-server` that acts as the upstream cluster the /// `iggy_source` connector replicates from. The connector config's @@ -350,23 +353,6 @@ async fn wait_for_source_error_after(http: &Client, api_url: &str, previous_erro .expect("Iggy source did not report a downstream send failure"); } -async fn wait_for_connector_log(harness: &TestHarness, marker: &str) { - timeout(WAIT_TIMEOUT, async { - loop { - let (stdout, stderr) = harness - .connectors_runtime() - .expect("connectors runtime") - .collect_logs(); - if stdout.contains(marker) || stderr.contains(marker) { - break; - } - sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; - } - }) - .await - .expect("Iggy source did not log the expected message build failure"); -} - async fn wait_for_source_state( harness: &TestHarness, expected_offsets: &HashMap, @@ -401,6 +387,45 @@ async fn wait_for_source_state( }) } +async fn restart_with_upstream_partition_count( + harness: &mut TestHarness, + upstream_client: &IggyClient, + expected_partitions: u32, +) { + harness + .server_mut() + .stop_dependents() + .expect("Failed to stop connectors runtime"); + + let stream_id = Identifier::named(UPSTREAM_STREAM).expect("valid stream name"); + let topic_id = Identifier::named(UPSTREAM_TOPIC).expect("valid topic name"); + let topic = upstream_client + .get_topic(&stream_id, &topic_id) + .await + .expect("Failed to fetch upstream topic") + .expect("Upstream topic should exist"); + let additional_partitions = expected_partitions.saturating_sub(topic.partitions_count); + if additional_partitions > 0 { + upstream_client + .create_partitions(&stream_id, &topic_id, additional_partitions) + .await + .expect("Failed to create upstream partitions"); + } + + let topic = upstream_client + .get_topic(&stream_id, &topic_id) + .await + .expect("Failed to fetch upstream topic") + .expect("Upstream topic should exist"); + assert_eq!(topic.partitions_count, expected_partitions); + + harness + .server_mut() + .start_dependents() + .await + .expect("Failed to restart connectors runtime"); +} + #[iggy_harness( cluster_nodes = 1, server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), @@ -442,7 +467,7 @@ async fn iggy_source_replicates_messages_with_headers( server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), seed = seeds::connector_stream )] -async fn malformed_user_headers_do_not_advance_source_offset( +async fn block_policy_commits_prefix_and_drop_headers_resumes_partition( harness: &mut TestHarness, fixture: IggySourceUpstreamFixture, ) { @@ -457,6 +482,10 @@ async fn malformed_user_headers_do_not_advance_source_offset( assert_eq!(first_received.len(), 1, "Expected the first offset to sync"); let mut blocked_batch = vec![ + IggyMessage::builder() + .payload(Bytes::from_static(b"valid-prefix")) + .build() + .expect("test message should be valid"), message_with_unknown_header_kind("malformed-headers"), IggyMessage::builder() .payload(Bytes::from_static(b"after-malformed-headers")) @@ -466,7 +495,21 @@ async fn malformed_user_headers_do_not_advance_source_offset( fixture .produce_iggy_messages(&upstream_client, 0, &mut blocked_batch) .await; - wait_for_connector_log(harness, "Failed to convert upstream message at offset 1").await; + + let blocked_offsets = HashMap::from([(0, 1)]); + wait_for_source_state(harness, &blocked_offsets, 2, 1).await; + + let received_while_blocked = + drain_downstream_topic(harness, "iggy_source_while_blocked", 2).await; + let blocked_payloads = received_while_blocked + .iter() + .map(|message| String::from_utf8_lossy(&message.payload).into_owned()) + .collect::>(); + assert_eq!( + blocked_payloads, + ["before-malformed-headers", "valid-prefix"], + "Only the valid prefix should be committed while the malformed offset is blocked" + ); harness .server_mut() @@ -476,14 +519,14 @@ async fn malformed_user_headers_do_not_advance_source_offset( .server_mut() .connectors_runtime_mut() .expect("connectors runtime") - .add_env(INCLUDE_USER_HEADERS_ENV, "false"); + .add_env(MALFORMED_MESSAGE_POLICY_ENV, "drop_headers"); harness .server_mut() .start_dependents() .await - .expect("Failed to restart connectors runtime without user headers"); + .expect("Failed to restart connectors runtime with drop_headers policy"); - let received = drain_downstream_topic(harness, "iggy_source_after_malformed", 3).await; + let received = drain_downstream_topic(harness, "iggy_source_after_malformed", 4).await; let payloads = received .iter() .map(|message| String::from_utf8_lossy(&message.payload).into_owned()) @@ -492,11 +535,27 @@ async fn malformed_user_headers_do_not_advance_source_offset( payloads, [ "before-malformed-headers", + "valid-prefix", "malformed-headers", "after-malformed-headers", ], - "The malformed offset and its tail should be replayed after restart" + "The malformed offset and its tail should resume without replaying the committed prefix" + ); + + let malformed = received + .iter() + .find(|message| message.payload.as_ref() == b"malformed-headers") + .expect("malformed message should be forwarded"); + assert!( + malformed + .user_headers_map() + .expect("downstream user headers should be valid") + .is_none(), + "drop_headers should remove only the unparsable user headers" ); + + let resumed_offsets = HashMap::from([(0, 3)]); + wait_for_source_state(harness, &resumed_offsets, 4, 2).await; } #[iggy_harness( @@ -504,38 +563,117 @@ async fn malformed_user_headers_do_not_advance_source_offset( server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), seed = seeds::connector_stream )] -async fn multiple_upstream_partitions_resume_from_independent_offsets( +async fn malformed_user_headers_do_not_back_off_healthy_partitions( harness: &mut TestHarness, fixture: IggySourceUpstreamFixture, ) { let upstream_client = fixture.client().await.expect("upstream client"); fixture.ensure_upstream_topic(&upstream_client).await; + let connectors_runtime = harness + .server_mut() + .connectors_runtime_mut() + .expect("connectors runtime"); + connectors_runtime.add_env(RETRY_INTERVAL_ENV, "30s"); + connectors_runtime.add_env(MAX_RETRY_INTERVAL_ENV, "30s"); + restart_with_upstream_partition_count(harness, &upstream_client, MULTI_PARTITION_COUNT).await; - harness + fixture + .produce_messages_to_partition(&upstream_client, 0, 0, &["healthy-partition-0".to_string()]) + .await; + let mut malformed = [message_with_unknown_header_kind("blocked-partition-1")]; + fixture + .produce_iggy_messages(&upstream_client, 1, &mut malformed) + .await; + fixture + .produce_messages_to_partition(&upstream_client, 2, 0, &["healthy-partition-2".to_string()]) + .await; + + let expected_offsets = HashMap::from([(0, 0), (2, 0)]); + wait_for_source_state(harness, &expected_offsets, 2, 1).await; + + fixture + .produce_messages_to_partition( + &upstream_client, + 2, + 1, + &["healthy-after-conversion-error".to_string()], + ) + .await; + + let expected_offsets = HashMap::from([(0, 0), (2, 1)]); + wait_for_source_state(harness, &expected_offsets, 3, 2).await; + + let received = drain_downstream_topic(harness, "iggy_source_partition_isolation", 3).await; + let mut payloads = received + .iter() + .map(|message| String::from_utf8_lossy(&message.payload).into_owned()) + .collect::>(); + payloads.sort(); + assert_eq!( + payloads, + [ + "healthy-after-conversion-error", + "healthy-partition-0", + "healthy-partition-2", + ] + ); +} + +#[iggy_harness( + cluster_nodes = 1, + server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), + seed = seeds::connector_stream +)] +async fn non_recoverable_poll_error_does_not_skip_remaining_partitions( + harness: &mut TestHarness, + fixture: IggySourceUpstreamFixture, +) { + let upstream_client = fixture.client().await.expect("upstream client"); + fixture.ensure_upstream_topic(&upstream_client).await; + let connectors_runtime = harness .server_mut() - .stop_dependents() - .expect("Failed to stop connectors runtime"); - let stream_id = Identifier::named(UPSTREAM_STREAM).expect("valid stream name"); - let topic_id = Identifier::named(UPSTREAM_TOPIC).expect("valid topic name"); + .connectors_runtime_mut() + .expect("connectors runtime"); + connectors_runtime.add_env(POLL_INTERVAL_ENV, "5s"); + connectors_runtime.add_env(RETRY_INTERVAL_ENV, "30s"); + connectors_runtime.add_env(MAX_RETRY_INTERVAL_ENV, "30s"); + restart_with_upstream_partition_count(harness, &upstream_client, MULTI_PARTITION_COUNT).await; + + fixture + .produce_messages_to_partition(&upstream_client, 0, 0, &["before-topic-delete".to_string()]) + .await; + let initial_offsets = HashMap::from([(0, 0)]); + let initial_state = wait_for_source_state(harness, &initial_offsets, 1, 0).await; + assert_eq!(initial_state.errors_count, 0); + upstream_client - .create_partitions( - &stream_id, - &topic_id, - MULTI_PARTITION_COUNT.saturating_sub(1), + .delete_topic( + &Identifier::named(UPSTREAM_STREAM).expect("valid stream name"), + &Identifier::named(UPSTREAM_TOPIC).expect("valid topic name"), ) .await - .expect("Failed to create upstream partitions"); - let topic = upstream_client - .get_topic(&stream_id, &topic_id) - .await - .expect("Failed to fetch upstream topic") - .expect("Upstream topic should exist"); - assert_eq!(topic.partitions_count, MULTI_PARTITION_COUNT); - harness - .server_mut() - .start_dependents() - .await - .expect("Failed to restart connectors runtime"); + .expect("Failed to delete upstream topic"); + + let failed_state = wait_for_source_state(harness, &initial_offsets, 1, 1).await; + assert_eq!( + failed_state.errors_count, + u64::from(MULTI_PARTITION_COUNT), + "Every discovered partition should be attempted before connector-wide backoff" + ); +} + +#[iggy_harness( + cluster_nodes = 1, + server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), + seed = seeds::connector_stream +)] +async fn multiple_upstream_partitions_resume_from_independent_offsets( + harness: &mut TestHarness, + fixture: IggySourceUpstreamFixture, +) { + let upstream_client = fixture.client().await.expect("upstream client"); + fixture.ensure_upstream_topic(&upstream_client).await; + restart_with_upstream_partition_count(harness, &upstream_client, MULTI_PARTITION_COUNT).await; let partition_payloads = [ vec!["partition-0-message-0".to_string()], From 2f03c15a487fa400a88aec908bc299c7a56d1d27 Mon Sep 17 00:00:00 2001 From: Gunther Xing Date: Wed, 9 Sep 2026 15:16:46 +0800 Subject: [PATCH 09/15] fix(connectors): validate Iggy source batch size --- core/connectors/sources/iggy_source/README.md | 3 +- .../connectors/sources/iggy_source/src/lib.rs | 41 ++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/core/connectors/sources/iggy_source/README.md b/core/connectors/sources/iggy_source/README.md index 6d73d64d74..f5fc5790b9 100644 --- a/core/connectors/sources/iggy_source/README.md +++ b/core/connectors/sources/iggy_source/README.md @@ -60,7 +60,7 @@ messages are requested from each upstream partition in one poll cycle. | `upstream_stream` | yes | none | Name of the stream to replicate from. | | `upstream_topic` | yes | none | Name of the topic to replicate from. | | `poll_interval` | no | `2s` | Delay before each upstream poll cycle. | -| `batch_size` | no | `100` | Maximum messages requested from each upstream partition per poll cycle. | +| `batch_size` | no | `100` | Maximum messages requested from each upstream partition per poll cycle. Must be between `1` and `10,000`. | | `initial_offset` | no | `earliest` | Starting position for a partition without a saved offset. Accepts `earliest`, `latest`, or an absolute numeric offset. | | `include_user_headers` | no | `true` | Copy user headers to downstream messages. | | `malformed_message_policy` | no | `block` | Handling for messages with unparsable user headers: `block` retries from the failed offset; `drop_headers` forwards the payload without those headers. | @@ -112,6 +112,7 @@ partition's saved position and applies `initial_offset` again. - The connector discovers upstream partitions when it opens. Restart it after adding partitions to the upstream topic. +- A `batch_size` outside `1..=10,000` prevents the connector from starting. - If the upstream stream is missing, the connector creates it. If the topic is missing, it creates the topic with one partition and no compression. - Invalid `initial_offset` values produce a warning and fall back to diff --git a/core/connectors/sources/iggy_source/src/lib.rs b/core/connectors/sources/iggy_source/src/lib.rs index c1e79e006f..c0ba49b278 100644 --- a/core/connectors/sources/iggy_source/src/lib.rs +++ b/core/connectors/sources/iggy_source/src/lib.rs @@ -45,6 +45,7 @@ const DEFAULT_POLL_INTERVAL: &str = "2s"; const DEFAULT_RETRY_INTERVAL: &str = "1s"; const DEFAULT_MAX_RETRY_INTERVAL: &str = "60s"; const DEFAULT_BATCH_SIZE: u32 = 100; +const MAX_BATCH_SIZE: u32 = 10_000; const AUTO_CREATED_PARTITIONS_COUNT: u32 = 1; /// Configuration for the Iggy source connector, replicating a topic from an @@ -290,6 +291,8 @@ impl IggySource { #[async_trait] impl Source for IggySource { async fn open(&mut self) -> Result<(), Error> { + validate_batch_size(self.batch_size)?; + let redacted = redact_connection_string(self.config.connection_string.expose_secret()); info!( "Opening {CONNECTOR_NAME} connector ID: {}, upstream: {}/{} at {}", @@ -391,7 +394,7 @@ impl Source for IggySource { let mut candidate_state = self.state.lock().await.clone(); - let mut messages = Vec::with_capacity(self.partitions.len() * self.batch_size as usize); + let mut messages = Vec::with_capacity(self.batch_size as usize); let mut cycle_counts = PollCycleCounts::default(); for &partition_id in &self.partitions { @@ -544,6 +547,15 @@ impl Source for IggySource { } } +fn validate_batch_size(batch_size: u32) -> Result<(), Error> { + if !(1..=MAX_BATCH_SIZE).contains(&batch_size) { + return Err(Error::InvalidConfigValue(format!( + "batch_size must be between 1 and {MAX_BATCH_SIZE}, got {batch_size}" + ))); + } + Ok(()) +} + fn next_strategy(initial: InitialOffset, saved_offset: Option) -> PollingStrategy { match saved_offset { Some(offset) => PollingStrategy::offset(offset.saturating_add(1)), @@ -944,6 +956,33 @@ mod tests { assert_eq!(source.initial_offset, InitialOffset::Earliest); } + #[test] + fn given_out_of_range_batch_size_when_opening_should_reject_config() { + let runtime = tokio::runtime::Runtime::new().expect("failed to create test runtime"); + runtime.block_on(async { + for batch_size in [0, MAX_BATCH_SIZE + 1, u32::MAX] { + let config = IggySourceConfig { + batch_size: Some(batch_size), + ..test_config() + }; + let mut source = IggySource::new(1, config, None); + + let error = source + .open() + .await + .expect_err("out-of-range batch_size should be rejected"); + + assert!(matches!(error, Error::InvalidConfigValue(_))); + } + }); + } + + #[test] + fn batch_size_at_supported_boundaries_should_be_valid() { + assert!(validate_batch_size(1).is_ok()); + assert!(validate_batch_size(MAX_BATCH_SIZE).is_ok()); + } + #[test] fn initial_offset_should_parse_semantics() { assert_eq!( From 4bd2c5036deb39e9ae7a7b60f1ad709d3e3509a6 Mon Sep 17 00:00:00 2001 From: Gunther Xing Date: Mon, 14 Sep 2026 11:48:47 +0800 Subject: [PATCH 10/15] fix(connectors): adapt Iggy source retry backoff --- core/connectors/sources/iggy_source/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/connectors/sources/iggy_source/src/lib.rs b/core/connectors/sources/iggy_source/src/lib.rs index c0ba49b278..bdfaa79e91 100644 --- a/core/connectors/sources/iggy_source/src/lib.rs +++ b/core/connectors/sources/iggy_source/src/lib.rs @@ -22,7 +22,7 @@ use iggy::prelude::{ }; use iggy_connector_sdk::{ ConnectorState, Error, ProducedMessage, ProducedMessages, Schema, Source, - retry::{exponential_backoff, jitter, parse_duration}, + retry::{parse_duration, retry_backoff}, source::SourceBatchResult, source_connector, }; @@ -376,11 +376,11 @@ impl Source for IggySource { let failed_cycles = self.consecutive_failed_poll_cycles.load(Ordering::Relaxed); if failed_cycles > 0 { - let delay = jitter(exponential_backoff( + let delay = retry_backoff( self.retry_interval, failed_cycles as u32, self.max_retry_interval, - )); + ); debug!( "Backing off for {delay:?} after {failed_cycles} consecutive poll cycles with no \ successful partitions for \ From 6a9184ca76643f534f7536bd510f445de3e0f9fa Mon Sep 17 00:00:00 2001 From: Gunther Xing Date: Fri, 25 Sep 2026 00:22:50 +0800 Subject: [PATCH 11/15] fix(connectors): reset source offsets after topic recreation A recreated upstream topic can have a frontier below saved offsets, causing empty polls to skip its history. Track the topic creation time and apply initial_offset when it changes. Cover recreation in an integration test and document the configuration behavior. --- .../connectors/iggy_source.toml | 4 +- core/connectors/sources/iggy_source/README.md | 5 +- .../sources/iggy_source/config.toml | 3 +- .../connectors/sources/iggy_source/src/lib.rs | 80 +++++++++++++++++-- .../connectors/iggy_source/replication.rs | 61 +++++++++++++- 5 files changed, 143 insertions(+), 10 deletions(-) diff --git a/core/connectors/runtime/example_config/connectors/iggy_source.toml b/core/connectors/runtime/example_config/connectors/iggy_source.toml index a1b9b1ab38..88d372ffd8 100644 --- a/core/connectors/runtime/example_config/connectors/iggy_source.toml +++ b/core/connectors/runtime/example_config/connectors/iggy_source.toml @@ -37,8 +37,8 @@ upstream_stream = "upstream_stream" upstream_topic = "upstream_topic" poll_interval = "1s" batch_size = 100 -# "earliest", "latest", or a numeric offset; used only for partitions without -# a saved offset in the persisted connector state +# "earliest", "latest", or a numeric offset; used for partitions without a +# saved offset, including after the upstream topic is recreated with the same name. initial_offset = "earliest" include_user_headers = true # "block" preserves malformed messages for retry; "drop_headers" forwards diff --git a/core/connectors/sources/iggy_source/README.md b/core/connectors/sources/iggy_source/README.md index f5fc5790b9..28268748f0 100644 --- a/core/connectors/sources/iggy_source/README.md +++ b/core/connectors/sources/iggy_source/README.md @@ -61,7 +61,7 @@ messages are requested from each upstream partition in one poll cycle. | `upstream_topic` | yes | none | Name of the topic to replicate from. | | `poll_interval` | no | `2s` | Delay before each upstream poll cycle. | | `batch_size` | no | `100` | Maximum messages requested from each upstream partition per poll cycle. Must be between `1` and `10,000`. | -| `initial_offset` | no | `earliest` | Starting position for a partition without a saved offset. Accepts `earliest`, `latest`, or an absolute numeric offset. | +| `initial_offset` | no | `earliest` | Starting position for a partition without a saved offset, including after the upstream topic is recreated with the same name. Accepts `earliest`, `latest`, or an absolute numeric offset. | | `include_user_headers` | no | `true` | Copy user headers to downstream messages. | | `malformed_message_policy` | no | `block` | Handling for messages with unparsable user headers: `block` retries from the failed offset; `drop_headers` forwards the payload without those headers. | | `retry_interval` | no | `1s` | Base delay used for exponential backoff when every partition poll in a cycle fails. | @@ -83,6 +83,9 @@ offset. `initial_offset` is used only when that partition has no saved offset: - `latest` requests up to `batch_size` of the most recent messages. - A numeric value starts at that absolute offset. +If the upstream topic is recreated with the same name, the connector clears +its saved offsets and uses `initial_offset` for the new topic. + The connector stages new offsets while polling. It commits them only after the runtime sends the complete downstream batch and saves the connector state. If the batch is rejected, the previous offsets remain committed and the messages diff --git a/core/connectors/sources/iggy_source/config.toml b/core/connectors/sources/iggy_source/config.toml index 46c886c1f2..13a8d2af47 100644 --- a/core/connectors/sources/iggy_source/config.toml +++ b/core/connectors/sources/iggy_source/config.toml @@ -35,7 +35,8 @@ upstream_stream = "upstream_stream" upstream_topic = "upstream_topic" poll_interval = "1s" batch_size = 100 -# Starting position used only when a partition has no committed offset: +# Starting position when a partition has no committed offset, including after +# the upstream topic is recreated with the same name: # "earliest", "latest", or an absolute numeric offset. # With "latest", if the first batch is NACKed before an offset is committed, # retrying recalculates the moving tail and may skip messages from the failed diff --git a/core/connectors/sources/iggy_source/src/lib.rs b/core/connectors/sources/iggy_source/src/lib.rs index bdfaa79e91..f59f792391 100644 --- a/core/connectors/sources/iggy_source/src/lib.rs +++ b/core/connectors/sources/iggy_source/src/lib.rs @@ -112,6 +112,22 @@ struct State { offsets: HashMap, messages_synced: u64, errors_count: u64, + topic_created_at: Option, +} + +impl State { + fn reset_offsets_if_topic_changed(&mut self, created_at: u64) -> bool { + if self.topic_created_at == Some(created_at) { + return false; + } + + // Old offsets can be past the new topic's frontier, where polls return + // empty batches and silently skip its history. + let reset_offsets = !self.offsets.is_empty(); + self.offsets.clear(); + self.topic_created_at = Some(created_at); + reset_offsets + } } #[derive(Debug, Default)] @@ -199,6 +215,7 @@ impl IggySource { offsets: HashMap::new(), messages_synced: 0, errors_count: 0, + topic_created_at: None, })), pending_state: Mutex::new(None), partitions: Vec::new(), @@ -390,14 +407,34 @@ impl Source for IggySource { sleep(delay).await; } - let consumer = Consumer::default(); - let mut candidate_state = self.state.lock().await.clone(); + let topic = client + .get_topic(stream_id, topic_id) + .await + .map_err(|error| { + Error::Connection(format!("Failed to fetch upstream topic details: {error}")) + })?; + let current_partitions = topic.as_ref().map(|topic| { + if candidate_state.reset_offsets_if_topic_changed(topic.created_at.as_micros()) { + warn!( + "Upstream topic '{}/{}' changed for {CONNECTOR_NAME} connector ID: {}, \ + resetting saved offsets to initial offset", + self.config.upstream_stream, self.config.upstream_topic, self.id + ); + } + topic + .partitions + .iter() + .map(|partition| partition.id) + .collect::>() + }); + let partitions = current_partitions.as_deref().unwrap_or(&self.partitions); + let consumer = Consumer::default(); let mut messages = Vec::with_capacity(self.batch_size as usize); let mut cycle_counts = PollCycleCounts::default(); - for &partition_id in &self.partitions { + for &partition_id in partitions { let strategy = next_strategy( self.initial_offset, candidate_state.offsets.get(&partition_id).copied(), @@ -492,7 +529,7 @@ impl Source for IggySource { Total synced: {}, poll errors in cycle: {}, conversion errors in cycle: {}", self.id, messages.len(), - self.partitions.len(), + partitions.len(), total_synced, cycle_counts.upstream_poll_errors, cycle_counts.conversion_errors @@ -503,7 +540,7 @@ impl Source for IggySource { Total synced: {}, poll errors in cycle: {}, conversion errors in cycle: {}", self.id, messages.len(), - self.partitions.len(), + partitions.len(), total_synced, cycle_counts.upstream_poll_errors, cycle_counts.conversion_errors @@ -711,6 +748,7 @@ mod tests { offsets: HashMap::from([(0, 42), (1, 7)]), messages_synced: 500, errors_count: 3, + topic_created_at: Some(100), }; let serialized = rmp_serde::to_vec(&state).expect("Failed to serialize state"); @@ -725,6 +763,7 @@ mod tests { assert_eq!(restored.offsets.get(&1), Some(&7)); assert_eq!(restored.messages_synced, 500); assert_eq!(restored.errors_count, 3); + assert_eq!(restored.topic_created_at, Some(100)); }); } @@ -738,6 +777,7 @@ mod tests { assert!(state.offsets.is_empty()); assert_eq!(state.messages_synced, 0); assert_eq!(state.errors_count, 0); + assert_eq!(state.topic_created_at, None); }); } @@ -752,6 +792,7 @@ mod tests { assert!(state.offsets.is_empty()); assert_eq!(state.messages_synced, 0); assert_eq!(state.errors_count, 0); + assert_eq!(state.topic_created_at, None); }); } @@ -761,6 +802,7 @@ mod tests { offsets: HashMap::from([(2, 1000)]), messages_synced: 1000, errors_count: 5, + topic_created_at: Some(100), }; let serialized = rmp_serde::to_vec(&original).expect("Failed to serialize"); @@ -770,6 +812,29 @@ mod tests { assert_eq!(original.offsets, deserialized.offsets); assert_eq!(original.messages_synced, deserialized.messages_synced); assert_eq!(original.errors_count, deserialized.errors_count); + assert_eq!(original.topic_created_at, deserialized.topic_created_at); + } + + #[test] + fn given_changed_topic_should_reset_offsets_only() { + let mut state = State { + offsets: HashMap::from([(0, 42), (1, 7)]), + messages_synced: 500, + errors_count: 3, + topic_created_at: Some(100), + }; + + assert!(!state.reset_offsets_if_topic_changed(100)); + assert_eq!(state.offsets.len(), 2); + assert!(state.reset_offsets_if_topic_changed(101)); + assert!(state.offsets.is_empty()); + assert_eq!( + next_strategy(InitialOffset::Earliest, state.offsets.get(&0).copied()), + PollingStrategy::first() + ); + assert_eq!(state.topic_created_at, Some(101)); + assert_eq!(state.messages_synced, 500); + assert_eq!(state.errors_count, 3); } #[test] @@ -813,6 +878,7 @@ mod tests { offsets: HashMap::from([(0, 42)]), messages_synced: 42, errors_count: 0, + topic_created_at: Some(100), }; let connector_state = source.serialize_state(&state); @@ -832,11 +898,13 @@ mod tests { offsets: HashMap::from([(0, 9)]), messages_synced: 10, errors_count: 1, + topic_created_at: Some(100), }; *source.pending_state.lock().await = Some(State { offsets: HashMap::from([(0, 19)]), messages_synced: 20, errors_count: 2, + topic_created_at: Some(100), }); source @@ -862,11 +930,13 @@ mod tests { offsets: HashMap::from([(0, 9)]), messages_synced: 10, errors_count: 1, + topic_created_at: Some(100), }; *source.pending_state.lock().await = Some(State { offsets: HashMap::from([(0, 19)]), messages_synced: 20, errors_count: 2, + topic_created_at: Some(100), }); source diff --git a/core/integration/tests/connectors/iggy_source/replication.rs b/core/integration/tests/connectors/iggy_source/replication.rs index f7e515b086..6d81253690 100644 --- a/core/integration/tests/connectors/iggy_source/replication.rs +++ b/core/integration/tests/connectors/iggy_source/replication.rs @@ -19,7 +19,7 @@ use async_trait::async_trait; use bytes::Bytes; use iggy::prelude::{ DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME, HeaderKey, HeaderValue, IggyClient, IggyMessage, - Partitioning, + Partitioning, TopicCreateOptions, }; use iggy_common::{ Consumer, Identifier, MessageClient, PartitionClient, PollingStrategy, StreamClient, @@ -69,6 +69,7 @@ struct PersistedIggySourceState { offsets: HashMap, messages_synced: u64, errors_count: u64, + topic_created_at: Option, } #[async_trait] @@ -916,3 +917,61 @@ async fn state_persists_across_connector_restart( ); } } + +#[iggy_harness( + cluster_nodes = 1, + server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), + seed = seeds::connector_stream +)] +async fn recreated_upstream_topic_replicates_from_initial_offset( + harness: &TestHarness, + fixture: IggySourceUpstreamFixture, +) { + let upstream_client = fixture.client().await.expect("upstream client"); + fixture.ensure_upstream_topic(&upstream_client).await; + + let original = (0..TEST_MESSAGE_COUNT) + .map(|index| format!("original-{index}")) + .collect::>(); + fixture.produce_messages(&upstream_client, &original).await; + let original_state = wait_for_source_state(harness, &HashMap::from([(0, 4)]), 5, 0).await; + assert!(original_state.topic_created_at.is_some()); + + let stream_id = Identifier::named(UPSTREAM_STREAM).expect("valid stream name"); + let topic_id = Identifier::named(UPSTREAM_TOPIC).expect("valid topic name"); + upstream_client + .delete_topic(&stream_id, &topic_id) + .await + .expect("Failed to delete upstream topic"); + upstream_client + .create_topic( + &stream_id, + UPSTREAM_TOPIC, + &TopicCreateOptions { + partitions_count: Some(1), + ..TopicCreateOptions::default() + }, + ) + .await + .expect("Failed to recreate upstream topic"); + + let replacement = ["replacement-0".to_string(), "replacement-1".to_string()]; + fixture + .produce_messages(&upstream_client, &replacement) + .await; + + let replacement_state = wait_for_source_state(harness, &HashMap::from([(0, 1)]), 7, 0).await; + assert_ne!( + replacement_state.topic_created_at, + original_state.topic_created_at + ); + let received = drain_downstream_topic(harness, "iggy_source_recreated_topic", 7).await; + let payloads = received + .iter() + .map(|message| String::from_utf8_lossy(&message.payload).into_owned()) + .collect::>(); + assert_eq!( + payloads, + original.into_iter().chain(replacement).collect::>() + ); +} From 1da966626639b81b512adb9d741bf4442a1e3e09 Mon Sep 17 00:00:00 2001 From: Gunther Xing Date: Fri, 25 Sep 2026 00:25:54 +0800 Subject: [PATCH 12/15] fix(connectors): harden Iggy source startup and polling Avoid startup failure after stream or topic creation races and prevent malformed header bytes from advancing offsets under the block policy. Persist state only when it changes, remove the unreachable invalid-offset reset, and document the restart required after repeated downstream failures. --- core/connectors/sources/iggy_source/README.md | 7 +- .../connectors/sources/iggy_source/src/lib.rs | 109 +++++++++++++----- 2 files changed, 86 insertions(+), 30 deletions(-) diff --git a/core/connectors/sources/iggy_source/README.md b/core/connectors/sources/iggy_source/README.md index 28268748f0..0a1b4d602b 100644 --- a/core/connectors/sources/iggy_source/README.md +++ b/core/connectors/sources/iggy_source/README.md @@ -92,15 +92,16 @@ the batch is rejected, the previous offsets remain committed and the messages are eligible for replay. Consumers should therefore tolerate duplicate delivery after failures or crashes. +After five consecutive downstream send or state-save failures, the source +stops. An operator must restart it before it can replay the unacknowledged +messages. + > When `initial_offset = "latest"` and no offset has been committed yet, each > retry recalculates the current tail of the upstream partition. If a first > batch is rejected while new messages arrive, messages from that failed batch > can be skipped. Use `earliest` or a numeric offset when replay continuity is > required. -If retention makes a saved offset invalid, the connector clears that -partition's saved position and applies `initial_offset` again. - ## Message Mapping | Upstream field | Downstream behavior | diff --git a/core/connectors/sources/iggy_source/src/lib.rs b/core/connectors/sources/iggy_source/src/lib.rs index f59f792391..473bfc4973 100644 --- a/core/connectors/sources/iggy_source/src/lib.rs +++ b/core/connectors/sources/iggy_source/src/lib.rs @@ -107,7 +107,7 @@ impl FromStr for InitialOffset { /// Committed state. `offsets` maps each upstream partition to the offset of /// the last message confirmed by the runtime after both the downstream send /// and checkpoint save succeeded. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] struct State { offsets: HashMap, messages_synced: u64, @@ -251,15 +251,15 @@ impl IggySource { connector ID: {}", self.config.upstream_stream, self.id ); - client - .create_stream(&self.config.upstream_stream) - .await - .map_err(|e| { - Error::InitError(format!( + match client.create_stream(&self.config.upstream_stream).await { + Ok(_) | Err(IggyError::StreamNameAlreadyExists(_)) => {} + Err(e) => { + return Err(Error::InitError(format!( "Failed to create upstream stream '{}': {e}", self.config.upstream_stream - )) - })?; + ))); + } + } } Err(e) => { return Err(Error::InitError(format!( @@ -278,7 +278,7 @@ impl IggySource { connector ID: {}", self.config.upstream_topic, self.id ); - client + match client .create_topic( stream_id, &self.config.upstream_topic, @@ -289,12 +289,15 @@ impl IggySource { }, ) .await - .map_err(|e| { - Error::InitError(format!( + { + Ok(_) | Err(IggyError::TopicNameAlreadyExists(_, _)) => {} + Err(e) => { + return Err(Error::InitError(format!( "Failed to create upstream topic '{}': {e}", self.config.upstream_topic - )) - })?; + ))); + } + } Ok(()) } Err(e) => Err(Error::InitError(format!( @@ -487,15 +490,6 @@ impl Source for IggySource { .insert(partition_id, last_produced_offset); } } - Err(IggyError::InvalidOffset(offset)) => { - cycle_counts.upstream_poll_errors += 1; - warn!( - "Saved offset {offset} for partition {partition_id} no longer valid \ - for {CONNECTOR_NAME} connector ID: {}, resetting to initial offset", - self.id - ); - candidate_state.offsets.remove(&partition_id); - } Err(poll_error) => { cycle_counts.upstream_poll_errors += 1; error!( @@ -510,10 +504,16 @@ impl Source for IggySource { candidate_state.messages_synced += messages.len() as u64; candidate_state.errors_count += cycle_counts.total_errors(); let total_synced = candidate_state.messages_synced; - let persisted_state = self.serialize_state(&candidate_state).ok_or_else(|| { - Error::Serialization("failed to serialize Iggy source state".to_string()) - })?; - *self.pending_state.lock().await = Some(candidate_state); + let state_changed = *self.state.lock().await != candidate_state; + let persisted_state = if state_changed { + let persisted_state = self.serialize_state(&candidate_state).ok_or_else(|| { + Error::Serialization("failed to serialize Iggy source state".to_string()) + })?; + *self.pending_state.lock().await = Some(candidate_state); + Some(persisted_state) + } else { + None + }; if cycle_counts.requires_backoff() { self.consecutive_failed_poll_cycles @@ -550,7 +550,7 @@ impl Source for IggySource { Ok(ProducedMessages { schema: Schema::Raw, messages, - state: Some(persisted_state), + state: persisted_state, }) } @@ -629,6 +629,18 @@ fn build_produced_messages( for message in messages { let headers = if include_user_headers { match message.user_headers_map() { + Ok(None) if message.user_headers.is_some() => { + errors.push(MessageBuildError { + offset: message.header.offset, + error: Error::InvalidRecordValue( + "Failed to parse upstream user headers".to_string(), + ), + }); + if malformed_message_policy == MalformedMessagePolicy::Block { + break; + } + None + } Ok(headers) => headers, Err(error) => { errors.push(MessageBuildError { @@ -742,6 +754,14 @@ mod tests { message } + fn test_message_with_invalid_header_bytes(offset: u64) -> IggyMessage { + let mut message = test_message(offset); + message.header.user_headers_length = 1; + message.user_headers = Some(vec![0xFF].into()); + assert!(message.user_headers_map().unwrap().is_none()); + message + } + #[test] fn given_persisted_state_should_restore_offsets_and_counts() { let state = State { @@ -1104,6 +1124,22 @@ mod tests { )); } + #[test] + fn given_invalid_header_bytes_with_block_policy_should_return_successful_prefix() { + let messages = [ + test_message(10), + test_message_with_invalid_header_bytes(11), + test_message(12), + ]; + + let result = build_produced_messages(&messages, true, MalformedMessagePolicy::Block); + + assert_eq!(result.messages.len(), 1); + assert_eq!(result.last_produced_offset, Some(10)); + assert_eq!(result.errors.len(), 1); + assert_eq!(result.errors[0].offset, 11); + } + #[test] fn given_malformed_message_with_drop_headers_policy_should_forward_entire_batch() { let messages = [ @@ -1127,6 +1163,25 @@ mod tests { )); } + #[test] + fn given_invalid_header_bytes_with_drop_headers_policy_should_count_error() { + let messages = [ + test_message(10), + test_message_with_invalid_header_bytes(11), + test_message(12), + ]; + + let result = build_produced_messages(&messages, true, MalformedMessagePolicy::DropHeaders); + + assert_eq!(result.messages.len(), 3); + assert_eq!(result.last_produced_offset, Some(12)); + assert!(result.messages[0].headers.is_some()); + assert!(result.messages[1].headers.is_none()); + assert!(result.messages[2].headers.is_some()); + assert_eq!(result.errors.len(), 1); + assert_eq!(result.errors[0].offset, 11); + } + #[test] fn redact_connection_string_should_hide_credentials() { let redacted = redact_connection_string("iggy+tcp://iggy:password@127.0.0.1:8090"); From ca597974904f21ae68ee1237c0099ff9fd1a9b8d Mon Sep 17 00:00:00 2001 From: Gunther Xing Date: Tue, 29 Sep 2026 21:36:20 +0800 Subject: [PATCH 13/15] fix(connectors): address Iggy source review feedback Correct startup defaults, credential redaction, and auto-created topic durability. Align the integration suite name with CI selection and document automatic partition discovery. --- .../connectors/sources/iggy_source/Cargo.toml | 3 ++ core/connectors/sources/iggy_source/README.md | 6 ++-- .../connectors/sources/iggy_source/src/lib.rs | 24 ++++++++------ .../connectors/{iggy_source => iggy}/mod.rs | 0 .../{iggy_source => iggy}/replication.rs | 32 +++++++++++++------ .../{iggy_source => iggy}/source.toml | 0 core/integration/tests/connectors/mod.rs | 2 +- 7 files changed, 44 insertions(+), 23 deletions(-) rename core/integration/tests/connectors/{iggy_source => iggy}/mod.rs (100%) rename core/integration/tests/connectors/{iggy_source => iggy}/replication.rs (97%) rename core/integration/tests/connectors/{iggy_source => iggy}/source.toml (100%) diff --git a/core/connectors/sources/iggy_source/Cargo.toml b/core/connectors/sources/iggy_source/Cargo.toml index 8b45f7c7da..baf6d5e594 100644 --- a/core/connectors/sources/iggy_source/Cargo.toml +++ b/core/connectors/sources/iggy_source/Cargo.toml @@ -49,3 +49,6 @@ tracing = { workspace = true } [dev-dependencies] serde_json = { workspace = true } + +[lints] +workspace = true diff --git a/core/connectors/sources/iggy_source/README.md b/core/connectors/sources/iggy_source/README.md index 0a1b4d602b..1b53775e56 100644 --- a/core/connectors/sources/iggy_source/README.md +++ b/core/connectors/sources/iggy_source/README.md @@ -114,11 +114,11 @@ messages. ## Operational Notes -- The connector discovers upstream partitions when it opens. Restart it after - adding partitions to the upstream topic. +- The connector discovers new upstream partitions automatically on each poll cycle. - A `batch_size` outside `1..=10,000` prevents the connector from starting. - If the upstream stream is missing, the connector creates it. If the topic is - missing, it creates the topic with one partition and no compression. + missing, it creates the topic with one partition, no compression, persisted + durability, and a one-message save threshold. - Invalid `initial_offset` values produce a warning and fall back to `earliest`. - If user headers cannot be decoded, `malformed_message_policy = "block"` diff --git a/core/connectors/sources/iggy_source/src/lib.rs b/core/connectors/sources/iggy_source/src/lib.rs index 473bfc4973..fdadbe74b2 100644 --- a/core/connectors/sources/iggy_source/src/lib.rs +++ b/core/connectors/sources/iggy_source/src/lib.rs @@ -20,6 +20,7 @@ use iggy::prelude::{ Client, CompressionAlgorithm, Consumer, Identifier, IggyClient, IggyError, IggyMessage, MessageClient, PollingStrategy, StreamClient, TopicClient, TopicCreateOptions, }; +use iggy_common::Durability; use iggy_connector_sdk::{ ConnectorState, Error, ProducedMessage, ProducedMessages, Schema, Source, retry::{parse_duration, retry_backoff}, @@ -189,19 +190,17 @@ impl IggySource { DEFAULT_MAX_RETRY_INTERVAL, ); - let initial_offset = config - .initial_offset - .as_deref() - .map(InitialOffset::from_str) - .and_then(Result::ok) - .unwrap_or_else(|| { + let initial_offset = match config.initial_offset.as_deref() { + None => InitialOffset::Earliest, + Some(value) => value.parse().unwrap_or_else(|_| { warn!( "Invalid initial offset {:?} for {CONNECTOR_NAME} connector ID: {id}, \ defaulting to earliest", - config.initial_offset + value ); InitialOffset::Earliest - }); + }), + }; let batch_size = config.batch_size.unwrap_or(DEFAULT_BATCH_SIZE); let include_user_headers = config.include_user_headers.unwrap_or(true); @@ -285,6 +284,8 @@ impl IggySource { &TopicCreateOptions { partitions_count: Some(AUTO_CREATED_PARTITIONS_COUNT), compression_algorithm: Some(CompressionAlgorithm::None), + durability: Durability::Persisted, + messages_required_to_save: Some(1), ..TopicCreateOptions::default() }, ) @@ -684,7 +685,7 @@ fn redact_connection_string(connection_string: &str) -> String { let Some(rest) = connection_string.get(scheme_end + 3..) else { return "***".to_string(); }; - let Some(at_offset) = rest.find('@') else { + let Some(at_offset) = rest.rfind('@') else { return "***".to_string(); }; let userinfo_end = scheme_end + 3 + at_offset; @@ -1192,6 +1193,11 @@ mod tests { assert_eq!(token_redacted, "iggy+tcp://***@127.0.0.1:8090"); assert!(!token_redacted.contains("abc123")); + let password_with_at = + redact_connection_string("iggy+tcp://iggy:secret@fragment@127.0.0.1:8090"); + assert_eq!(password_with_at, "iggy+tcp://iggy:***@127.0.0.1:8090"); + assert!(!password_with_at.contains("fragment")); + assert_eq!(redact_connection_string("not-a-url"), "***"); } } diff --git a/core/integration/tests/connectors/iggy_source/mod.rs b/core/integration/tests/connectors/iggy/mod.rs similarity index 100% rename from core/integration/tests/connectors/iggy_source/mod.rs rename to core/integration/tests/connectors/iggy/mod.rs diff --git a/core/integration/tests/connectors/iggy_source/replication.rs b/core/integration/tests/connectors/iggy/replication.rs similarity index 97% rename from core/integration/tests/connectors/iggy_source/replication.rs rename to core/integration/tests/connectors/iggy/replication.rs index 6d81253690..ff32dfb272 100644 --- a/core/integration/tests/connectors/iggy_source/replication.rs +++ b/core/integration/tests/connectors/iggy/replication.rs @@ -22,8 +22,8 @@ use iggy::prelude::{ Partitioning, TopicCreateOptions, }; use iggy_common::{ - Consumer, Identifier, MessageClient, PartitionClient, PollingStrategy, StreamClient, - TopicClient, + Consumer, Durability, Identifier, MessageClient, PartitionClient, PollingStrategy, + StreamClient, TopicClient, TopicRuntimeOptions, }; use iggy_connector_sdk::api::ConnectorRuntimeStats; use integration::harness::{ @@ -429,7 +429,7 @@ async fn restart_with_upstream_partition_count( #[iggy_harness( cluster_nodes = 1, - server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), + server(connectors_runtime(config_path = "tests/connectors/iggy/source.toml")), seed = seeds::connector_stream )] async fn iggy_source_replicates_messages_with_headers( @@ -439,6 +439,18 @@ async fn iggy_source_replicates_messages_with_headers( let upstream_client = fixture.client().await.expect("upstream client"); fixture.ensure_upstream_topic(&upstream_client).await; + let topic = upstream_client + .get_topic( + &Identifier::named(UPSTREAM_STREAM).expect("valid stream name"), + &Identifier::named(UPSTREAM_TOPIC).expect("valid topic name"), + ) + .await + .expect("upstream topic request") + .expect("upstream topic"); + let options = TopicRuntimeOptions::from_resource_options(&topic.options); + assert_eq!(options.durability, Durability::Persisted); + assert_eq!(options.messages_required_to_save, Some(1)); + let payloads: Vec = (0..TEST_MESSAGE_COUNT) .map(|i| format!("upstream-message-{i}")) .collect(); @@ -465,7 +477,7 @@ async fn iggy_source_replicates_messages_with_headers( #[iggy_harness( cluster_nodes = 1, - server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), + server(connectors_runtime(config_path = "tests/connectors/iggy/source.toml")), seed = seeds::connector_stream )] async fn block_policy_commits_prefix_and_drop_headers_resumes_partition( @@ -561,7 +573,7 @@ async fn block_policy_commits_prefix_and_drop_headers_resumes_partition( #[iggy_harness( cluster_nodes = 1, - server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), + server(connectors_runtime(config_path = "tests/connectors/iggy/source.toml")), seed = seeds::connector_stream )] async fn malformed_user_headers_do_not_back_off_healthy_partitions( @@ -622,7 +634,7 @@ async fn malformed_user_headers_do_not_back_off_healthy_partitions( #[iggy_harness( cluster_nodes = 1, - server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), + server(connectors_runtime(config_path = "tests/connectors/iggy/source.toml")), seed = seeds::connector_stream )] async fn non_recoverable_poll_error_does_not_skip_remaining_partitions( @@ -665,7 +677,7 @@ async fn non_recoverable_poll_error_does_not_skip_remaining_partitions( #[iggy_harness( cluster_nodes = 1, - server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), + server(connectors_runtime(config_path = "tests/connectors/iggy/source.toml")), seed = seeds::connector_stream )] async fn multiple_upstream_partitions_resume_from_independent_offsets( @@ -770,7 +782,7 @@ async fn multiple_upstream_partitions_resume_from_independent_offsets( #[iggy_harness( cluster_nodes = 1, - server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), + server(connectors_runtime(config_path = "tests/connectors/iggy/source.toml")), seed = seeds::connector_stream )] async fn downstream_outage_replays_nacked_batch_after_recovery( @@ -843,7 +855,7 @@ async fn downstream_outage_replays_nacked_batch_after_recovery( #[iggy_harness( cluster_nodes = 1, - server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), + server(connectors_runtime(config_path = "tests/connectors/iggy/source.toml")), seed = seeds::connector_stream )] async fn state_persists_across_connector_restart( @@ -920,7 +932,7 @@ async fn state_persists_across_connector_restart( #[iggy_harness( cluster_nodes = 1, - server(connectors_runtime(config_path = "tests/connectors/iggy_source/source.toml")), + server(connectors_runtime(config_path = "tests/connectors/iggy/source.toml")), seed = seeds::connector_stream )] async fn recreated_upstream_topic_replicates_from_initial_offset( diff --git a/core/integration/tests/connectors/iggy_source/source.toml b/core/integration/tests/connectors/iggy/source.toml similarity index 100% rename from core/integration/tests/connectors/iggy_source/source.toml rename to core/integration/tests/connectors/iggy/source.toml diff --git a/core/integration/tests/connectors/mod.rs b/core/integration/tests/connectors/mod.rs index 76caafece1..238d211a49 100644 --- a/core/integration/tests/connectors/mod.rs +++ b/core/integration/tests/connectors/mod.rs @@ -24,7 +24,7 @@ mod fixtures; mod http; mod http_config_provider; mod iceberg; -mod iggy_source; +mod iggy; mod influxdb; mod meilisearch; mod mongodb; From 8f536d7c7f44b0bc5211342a3e42db77ee95d069 Mon Sep 17 00:00:00 2001 From: Gunther Xing Date: Tue, 29 Sep 2026 21:37:42 +0800 Subject: [PATCH 14/15] fix(connectors): cap Iggy source poll cycles Wide upstream topics could accumulate one full batch from every partition in a single FFI transfer. Limit each cycle to 10,000 messages while retaining batch_size as the per-partition request limit, then resume from the first unpolled partition. --- core/connectors/sources/iggy_source/README.md | 11 ++-- .../connectors/sources/iggy_source/src/lib.rs | 64 +++++++++++++++++-- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/core/connectors/sources/iggy_source/README.md b/core/connectors/sources/iggy_source/README.md index 1b53775e56..2c5541250b 100644 --- a/core/connectors/sources/iggy_source/README.md +++ b/core/connectors/sources/iggy_source/README.md @@ -7,7 +7,7 @@ partition so replication can resume after a restart. ## Features -- Polls every partition of one upstream topic. +- Polls every partition of one upstream topic over successive cycles. - Preserves payload bytes and non-zero message IDs. - Preserves user headers by default, with an option to omit them. - Stores per-partition offsets in the connector runtime's state storage. @@ -49,8 +49,9 @@ verbose_logging = false Use `schema = "raw"` on the downstream stream to replicate payload bytes without decoding or re-encoding them. The runtime-level `batch_length` controls -downstream producer batching, while `plugin_config.batch_size` controls how many -messages are requested from each upstream partition in one poll cycle. +downstream producer batching, while `plugin_config.batch_size` limits each +upstream partition poll. A poll cycle forwards at most 10,000 messages across +all partitions and resumes with another partition on the next cycle. ### Plugin Fields @@ -60,7 +61,7 @@ messages are requested from each upstream partition in one poll cycle. | `upstream_stream` | yes | none | Name of the stream to replicate from. | | `upstream_topic` | yes | none | Name of the topic to replicate from. | | `poll_interval` | no | `2s` | Delay before each upstream poll cycle. | -| `batch_size` | no | `100` | Maximum messages requested from each upstream partition per poll cycle. Must be between `1` and `10,000`. | +| `batch_size` | no | `100` | Maximum messages requested from each upstream partition per poll. Must be between `1` and `10,000`. | | `initial_offset` | no | `earliest` | Starting position for a partition without a saved offset, including after the upstream topic is recreated with the same name. Accepts `earliest`, `latest`, or an absolute numeric offset. | | `include_user_headers` | no | `true` | Copy user headers to downstream messages. | | `malformed_message_policy` | no | `block` | Handling for messages with unparsable user headers: `block` retries from the failed offset; `drop_headers` forwards the payload without those headers. | @@ -116,6 +117,8 @@ messages. - The connector discovers new upstream partitions automatically on each poll cycle. - A `batch_size` outside `1..=10,000` prevents the connector from starting. +- The 10,000-message cycle cap bounds message count, not payload bytes. Large + messages can still use substantial memory while the batch crosses the runtime. - If the upstream stream is missing, the connector creates it. If the topic is missing, it creates the topic with one partition, no compression, persisted durability, and a one-message save threshold. diff --git a/core/connectors/sources/iggy_source/src/lib.rs b/core/connectors/sources/iggy_source/src/lib.rs index fdadbe74b2..687f563159 100644 --- a/core/connectors/sources/iggy_source/src/lib.rs +++ b/core/connectors/sources/iggy_source/src/lib.rs @@ -47,6 +47,8 @@ const DEFAULT_RETRY_INTERVAL: &str = "1s"; const DEFAULT_MAX_RETRY_INTERVAL: &str = "60s"; const DEFAULT_BATCH_SIZE: u32 = 100; const MAX_BATCH_SIZE: u32 = 10_000; +// A cycle must not accumulate more messages than one legal partition poll. +const MAX_POLL_CYCLE_MESSAGES: u32 = MAX_BATCH_SIZE; const AUTO_CREATED_PARTITIONS_COUNT: u32 = 1; /// Configuration for the Iggy source connector, replicating a topic from an @@ -162,6 +164,7 @@ pub struct IggySource { retry_interval: Duration, max_retry_interval: Duration, consecutive_failed_poll_cycles: AtomicU64, + next_partition: AtomicU64, initial_offset: InitialOffset, batch_size: u32, include_user_headers: bool, @@ -224,6 +227,7 @@ impl IggySource { retry_interval, max_retry_interval, consecutive_failed_poll_cycles: AtomicU64::new(0), + next_partition: AtomicU64::new(0), initial_offset, batch_size, include_user_headers, @@ -438,7 +442,18 @@ impl Source for IggySource { let mut messages = Vec::with_capacity(self.batch_size as usize); let mut cycle_counts = PollCycleCounts::default(); - for &partition_id in partitions { + let first_partition = next_partition_index( + self.next_partition.load(Ordering::Relaxed) as usize, + 0, + partitions.len(), + ); + for &partition_id in partitions[first_partition..] + .iter() + .chain(&partitions[..first_partition]) + { + let Some(poll_count) = next_poll_count(self.batch_size, messages.len()) else { + break; + }; let strategy = next_strategy( self.initial_offset, candidate_state.offsets.get(&partition_id).copied(), @@ -450,7 +465,7 @@ impl Source for IggySource { Some(partition_id), &consumer, &strategy, - self.batch_size, + poll_count, false, ) .await; @@ -502,6 +517,13 @@ impl Source for IggySource { } } + let attempted_partitions = + (cycle_counts.successful_upstream_polls + cycle_counts.upstream_poll_errors) as usize; + self.next_partition.store( + next_partition_index(first_partition, attempted_partitions, partitions.len()) as u64, + Ordering::Relaxed, + ); + candidate_state.messages_synced += messages.len() as u64; candidate_state.errors_count += cycle_counts.total_errors(); let total_synced = candidate_state.messages_synced; @@ -530,7 +552,7 @@ impl Source for IggySource { Total synced: {}, poll errors in cycle: {}, conversion errors in cycle: {}", self.id, messages.len(), - partitions.len(), + attempted_partitions, total_synced, cycle_counts.upstream_poll_errors, cycle_counts.conversion_errors @@ -541,7 +563,7 @@ impl Source for IggySource { Total synced: {}, poll errors in cycle: {}, conversion errors in cycle: {}", self.id, messages.len(), - partitions.len(), + attempted_partitions, total_synced, cycle_counts.upstream_poll_errors, cycle_counts.conversion_errors @@ -594,6 +616,23 @@ fn validate_batch_size(batch_size: u32) -> Result<(), Error> { Ok(()) } +fn next_poll_count(batch_size: u32, produced_count: usize) -> Option { + let remaining = MAX_POLL_CYCLE_MESSAGES.saturating_sub(produced_count as u32); + (remaining > 0).then_some(batch_size.min(remaining)) +} + +fn next_partition_index( + first_partition: usize, + attempted_partitions: usize, + partitions_count: usize, +) -> usize { + if partitions_count == 0 { + 0 + } else { + (first_partition + attempted_partitions) % partitions_count + } +} + fn next_strategy(initial: InitialOffset, saved_offset: Option) -> PollingStrategy { match saved_offset { Some(offset) => PollingStrategy::offset(offset.saturating_add(1)), @@ -1074,6 +1113,23 @@ mod tests { assert!(validate_batch_size(MAX_BATCH_SIZE).is_ok()); } + #[test] + fn given_cycle_capacity_when_polling_should_preserve_partition_limit() { + assert_eq!(next_poll_count(100, 0), Some(100)); + assert_eq!(next_poll_count(MAX_BATCH_SIZE, 0), Some(MAX_BATCH_SIZE)); + assert_eq!(next_poll_count(100, 9_950), Some(50)); + assert_eq!(next_poll_count(100, 10_000), None); + } + + #[test] + fn given_cycle_cap_when_advancing_partition_should_resume_after_attempted_partitions() { + assert_eq!(next_partition_index(0, 100, 1_000), 100); + assert_eq!(next_partition_index(100, 100, 1_000), 200); + assert_eq!(next_partition_index(900, 100, 1_000), 0); + assert_eq!(next_partition_index(4, 5, 5), 4); + assert_eq!(next_partition_index(0, 0, 0), 0); + } + #[test] fn initial_offset_should_parse_semantics() { assert_eq!( From b6baf62b6cbf26cf3ff434ad455631d00c843c2c Mon Sep 17 00:00:00 2001 From: Gunther Xing Date: Tue, 27 Oct 2026 15:21:54 +0800 Subject: [PATCH 15/15] fix(connectors): pass stream options in Iggy source The stream creation API requires options, so the Iggy source failed to compile in CI. Pass default options when creating the upstream stream. --- core/connectors/sources/iggy_source/src/lib.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/connectors/sources/iggy_source/src/lib.rs b/core/connectors/sources/iggy_source/src/lib.rs index 687f563159..8976e860ce 100644 --- a/core/connectors/sources/iggy_source/src/lib.rs +++ b/core/connectors/sources/iggy_source/src/lib.rs @@ -18,7 +18,8 @@ use async_trait::async_trait; use iggy::prelude::{ Client, CompressionAlgorithm, Consumer, Identifier, IggyClient, IggyError, IggyMessage, - MessageClient, PollingStrategy, StreamClient, TopicClient, TopicCreateOptions, + MessageClient, PollingStrategy, StreamClient, StreamCreateOptions, TopicClient, + TopicCreateOptions, }; use iggy_common::Durability; use iggy_connector_sdk::{ @@ -254,7 +255,13 @@ impl IggySource { connector ID: {}", self.config.upstream_stream, self.id ); - match client.create_stream(&self.config.upstream_stream).await { + match client + .create_stream( + &self.config.upstream_stream, + &StreamCreateOptions::default(), + ) + .await + { Ok(_) | Err(IggyError::StreamNameAlreadyExists(_)) => {} Err(e) => { return Err(Error::InitError(format!(