From 4937548223ead32b01372dbd102e711b88ea6791 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 10 Oct 2026 10:26:53 -0400 Subject: [PATCH 1/5] Initial version of connectors template --- .claude/skills/connector-pr-review/SKILL.md | 248 +++++++ .claude/skills/connector-runtime/SKILL.md | 4 +- .claude/skills/connector-sink/SKILL.md | 3 +- .claude/skills/connector-sink/TEMPLATE.md | 335 +++++++-- .claude/skills/connector-source/SKILL.md | 125 ++-- .claude/skills/connector-source/TEMPLATE.md | 391 +++++++++-- .claude/skills/connectors-overview/SKILL.md | 72 +- Cargo.lock | 35 + Cargo.toml | 2 + core/connectors/BLOG_POST.md | 104 +++ core/connectors/README.md | 4 + .../docs/authoring-sinks-and-sources.md | 196 ++++++ .../connectors/sink_template.toml | 48 ++ .../connectors/source_template.toml | 47 ++ core/connectors/sinks/README.md | 1 + .../connectors/sinks/sink_template/Cargo.toml | 57 ++ core/connectors/sinks/sink_template/README.md | 78 +++ .../sinks/sink_template/config.toml | 49 ++ .../connectors/sinks/sink_template/src/lib.rs | 608 +++++++++++++++++ core/connectors/sources/README.md | 1 + .../sources/source_template/Cargo.toml | 60 ++ .../sources/source_template/README.md | 70 ++ .../sources/source_template/config.toml | 48 ++ .../sources/source_template/src/lib.rs | 639 ++++++++++++++++++ 24 files changed, 3028 insertions(+), 197 deletions(-) create mode 100644 .claude/skills/connector-pr-review/SKILL.md create mode 100644 core/connectors/BLOG_POST.md create mode 100644 core/connectors/docs/authoring-sinks-and-sources.md create mode 100644 core/connectors/runtime/example_config/connectors/sink_template.toml create mode 100644 core/connectors/runtime/example_config/connectors/source_template.toml create mode 100644 core/connectors/sinks/sink_template/Cargo.toml create mode 100644 core/connectors/sinks/sink_template/README.md create mode 100644 core/connectors/sinks/sink_template/config.toml create mode 100644 core/connectors/sinks/sink_template/src/lib.rs create mode 100644 core/connectors/sources/source_template/Cargo.toml create mode 100644 core/connectors/sources/source_template/README.md create mode 100644 core/connectors/sources/source_template/config.toml create mode 100644 core/connectors/sources/source_template/src/lib.rs diff --git a/.claude/skills/connector-pr-review/SKILL.md b/.claude/skills/connector-pr-review/SKILL.md new file mode 100644 index 0000000000..4725fcd6ee --- /dev/null +++ b/.claude/skills/connector-pr-review/SKILL.md @@ -0,0 +1,248 @@ +--- +name: connector-pr-review +description: Review checklist for Apache Iggy connector sink/source PRs. Load when reviewing a connectors plugin PR, when authoring a new sink/source and wanting to pre-flight against common review blockers, or when diagnosing why a connectors PR is stuck in review. Encodes recurring review patterns mined from real apache/iggy connector PRs. NOT for runtime/SDK internals (use connector-runtime / connector-sdk). +--- + +# Connector PR review checklist + +> Universal rules live in [connectors-overview](../connectors-overview/SKILL.md). +> Authoring skills: [connector-sink](../connector-sink/SKILL.md), +> [connector-source](../connector-source/SKILL.md), +> [connector-testing](../connector-testing/SKILL.md). +> Fill-in-the-blank kits: those skills' `TEMPLATE.md` files. + +Use this skill to **catch the issues that repeatedly burn review cycles** +before asking for a human re-review. Cite symbols/paths, not stale line numbers. + +## Contents + +- [How to use](#how-to-use) +- [Blockers (must fix before merge)](#blockers-must-fix-before-merge) +- [High-frequency convention nits](#high-frequency-convention-nits) +- [Delivery semantics (document honestly)](#delivery-semantics-document-honestly) +- [PR / CI hygiene](#pr--ci-hygiene) +- [Pre-flight author checklist](#pre-flight-author-checklist) +- [Evidence base](#evidence-base) + +## How to use + +1. Load this skill for any PR under `core/connectors/sinks/` or `core/connectors/sources/`. +2. Walk **Blockers** first. Any hit is CHANGES_REQUESTED. +3. Then **Convention nits** and **Delivery semantics**. +4. End with **PR / CI hygiene** (cheap passes that still delay first review). +5. Prefer "copy the closest exemplar" over inventing new knobs. + +## Blockers (must fix before merge) + +### B1. Secrets + +- [ ] Every credential field is `secrecy::SecretString` with + `#[serde(serialize_with = "iggy_common::serde_secret::serialize_secret")]`. +- [ ] No connection string / API key / token in `info!`/`debug!`/`error!` / + `format!` into SQL / file-state metadata. +- [ ] Source state must not persist URL userinfo or bearer tokens. + +Plain `String` for a credential is a review-blocker. Pattern: +`PostgresSinkConfig::connection_string` in `sinks/postgres_sink`. + +### B2. Swallowing `Err` while offsets advance (sinks) + +- [ ] `consume()` must **not** catch a batch failure and return `Ok(())`. +- [ ] Prefer `last_err` pattern: process all batches, return the last transient + error (see `connector-sink` Hard rules / TEMPLATE). +- [ ] README must not claim "no data loss" / strong idempotency unless the + backend + runtime path actually enforce it. + +Today the runtime can commit consumer offsets even when plugin errors are +poorly surfaced (#2927 / #2928 class issues). Authors must be honest about +loss windows instead of overselling. + +### B3. Source cursor / side-effects staged for `on_batch_result`, not committed in `poll()` + +Since #3855, this is an SDK-enforced contract, not just a convention: +`poll()` returns *candidate* state; the runtime sends the batch, saves +that state on success, then calls `on_batch_result(Ack | Nack)`. Only +`on_batch_result` may commit. + +- [ ] `poll()` stages cursor changes and destructive work (delete-after-read / + mark-processed / ACK upstream) - it does not mutate committed state or + touch upstream rows directly. Committing in `poll()` leaves nothing to + roll back on a Nack, defeating the point of the handshake. +- [ ] `on_batch_result(SourceBatchResult::Ack)` commits the staged work; + `on_batch_result(SourceBatchResult::Nack)` discards it so the batch is + redelivered against unchanged committed state. A source with no staged + work (e.g. a pure generator) may rely on the SDK's default no-op impl. +- [ ] Returning `Err` from `on_batch_result` is deliberate - it stops the SDK + from polling further rather than risk silently advancing past a failed + rollback. Don't swallow a rollback failure into `Ok(())`. +- [ ] Always return `ConnectorState` in every `ProducedMessages` that made + progress; return `state: None` for an empty poll that made none (avoids + an unnecessary write and can't persist state left over from a failed + batch). +- [ ] `poll()` sleeps **first**, then fetches (never sleep after holding a batch). + +### B4. Transient vs permanent errors + +- [ ] Infra/auth/schema-gone failures map to `Error::PermanentHttpError` / + `Error::InitError` / `Error::SchemaMismatch` — not `InvalidRecord`. +- [ ] Retryable network/5xx/SQLSTATE map to transient variants + (`HttpRequestFailed`, `Connection`, `CannotStoreData`). +- [ ] Do **not** classify retryability by substring-matching `err.to_string()`. +- [ ] `max_retries` means **total attempts** (default 3). README must match code. +- [ ] Cap retry budget so a dead backend cannot delay shutdown unboundedly. + +### B5. Idempotency claims must be real + +- [ ] Stable `ProducedMessage.id` / sink dedup key from natural IDs + (table+PK, document `_id`, `stream:topic:partition:message_id`) — + never random UUIDs per emit. +- [ ] If the backend PK / unique index is informational only (e.g. Redshift), + do not advertise idempotency in README. +- [ ] External workflow IDs (Airflow `dag_run_id`, etc.) must be deterministic + across retries. + +### B6. Secrets / license policy for new SDKs + +- [ ] New backend crates pass `scripts/ci/third-party-licenses.sh` (no BUSL / + incompatible licenses pulled into the tree). +- [ ] Prefer workspace deps; avoid vendoring a license-hostile SDK just to wrap HTTP. + +### B7. Tests that must exist + +#### Sources + +- [ ] Four canonical state tests (restore / no-state / invalid-state / + round-trip), plus two ACK/NACK tests (`given_ack_when_batch_is_staged_ + should_commit_candidate_state`, `given_nack_when_batch_is_staged_ + should_keep_committed_state`) if `on_batch_result` is overridden - six + total. Copy `sources/random_source/src/lib.rs::tests`. A source relying + on the SDK's default no-op `on_batch_result` (no staged work) may skip + the ACK/NACK pair. + +#### Any external backend plugin + +- [ ] At least one real-infra integration test under + `core/integration/tests/connectors//` with `#[iggy_harness]` + + `testcontainers-modules` (or `wiremock` for pure HTTP). +- [ ] No false-green mocks that diverge from real backend semantics + (Decimal, COPY, PK enforcement, etc.). + +### B8. Config validation timing + +- [ ] Structural validation + unknown enum rejection in `new()` / + `open()` — not on first `poll()`/`consume()` after sleep. +- [ ] Connectivity check in `open()`; fail with `Error::InitError`. +- [ ] Invalid restored state: start fresh + `warn!`, but do **not** silently + re-emit an entire index without calling that out in README. +- [ ] Config flag combos that no-op should `warn!` or `Err`, not silently ignore. + +## High-frequency convention nits + +These are "cheap" but burn full review rounds when missed. + +### C1. Copy the closest exemplar + +- [ ] File layout, log labels, error mapping, and test structure match the + nearest sibling (`postgres_*`, `http_sink`, `elasticsearch_*`, …). +- [ ] Do not invent new names for existing knobs. + +### C2. Config knob name canon + +| Concept | Canonical field | Notes | +| ------- | --------------- | ----- | +| Request timeout | `timeout` | Not `request_timeout` | +| Retry attempts | `max_retries` | Total attempts, default 3 | +| Base backoff | `retry_delay` | humantime `Option` | +| Backoff ceiling | `retry_max_delay` | Matches SDK's `RetryPolicy::max_delay` | +| Poll cadence (sources) | `poll_interval` | humantime; sleep first | +| Plugin verbosity | `verbose_logging` | Mirror runtime `verbose` | +| Credentials | `connection_string` / `api_key` / … | Always `SecretString` | + +- [ ] Durations: `Option` + `humantime::Duration` in `new()`; fall back + with `warn!`, never panic. Workspace `humantime` — not a pinned + `humantime-serde`. +- [ ] New fields: `Option` + `#[serde(default)]` where needed. +- [ ] Prefer `#[serde(deny_unknown_fields)]` on plugin config so typo’d knobs + fail loud. + +### C3. Crate / path / docs checklist + +- [ ] `[lib] crate-type = ["cdylib", "lib"]`. +- [ ] Example TOML plugin path uses `../../target/release/lib…` like siblings. +- [ ] Row added to `sinks/README.md` or `sources/README.md`. +- [ ] Sample under `runtime/example_config/connectors/`. +- [ ] README defaults **byte-equal** to consts in code (diff them). +- [ ] No links to non-existent docs. + +### C4. Hot path + +- [ ] No `payload.clone().try_to_bytes()` — use `try_to_bytes(&self)`. +- [ ] `Vec::with_capacity(n)` for per-batch buffers. +- [ ] No `std::sync::Mutex` across `.await`; use `tokio::sync::Mutex`. +- [ ] `&self` on `consume` / `poll` (interior mutability only). +- [ ] No `tokio::spawn` inside plugin code. +- [ ] No `unwrap()`/`expect()` on external I/O outside tests. +- [ ] No eager `format!` around tracing args. + +### C5. Containers / fixtures + +- [ ] Testcontainers named `iggy-test-*` via `fixtures::unique_container_name` + (or fixed `iggy-test-` for reuse fixtures). +- [ ] Custom Docker networks cleaned up. + +## Delivery semantics (document honestly) + +Every new connector README must answer in one short paragraph: + +1. **What happens on transient failure?** (retry N times, then Err) +2. **What happens on permanent failure?** (drop/skip vs fail batch) +3. **What is the duplication window?** (at-least-once because state saves + after Iggy send; or at-most-once if upstream ACK precedes send — say so) +4. **What is the dedup key?** (or "none — duplicates possible") + +If the answer is hand-wavy, the PR is not ready. + +## PR / CI hygiene + +- [ ] Conventional commit: `feat(connectors): …` / `fix(connectors): …`. +- [ ] PR template filled (motivation, linked issue). +- [ ] `cargo fmt --all` + `cargo sort --no-format --workspace` + + `cargo clippy -p --all-targets -- -D warnings` + + `cargo test -p ` green locally. +- [ ] Minimal `Cargo.lock` delta — no unrelated dependency churn. +- [ ] Do not modify unrelated Java/Python/foreign SDK trees in a connectors PR. +- [ ] Mark ready for review only after the above; stale-bot closes waiting PRs. + +## Pre-flight author checklist + +Paste into the PR description (or run mentally before `/ready`): + +```text +[ ] SecretString on all credentials; no secret logs/state +[ ] consume/poll never returns Ok(()) after a failed batch that should retry +[ ] Transient vs permanent errors mapped (no Display substring matching) +[ ] Stable message / dedup IDs (no random UUID per emit) +[ ] README delivery semantics paragraph present and honest +[ ] README defaults match code consts +[ ] deny_unknown_fields on plugin config +[ ] Canonical knob names (timeout, max_retries, retry_delay, poll_interval) +[ ] Sources: 4 state tests (+2 ACK/NACK if on_batch_result overridden); sleep-first poll; state staged in poll, committed only in on_batch_result +[ ] External backend: real-infra integration test (not a lying mock) +[ ] example_config + sinks/sources README row +[ ] fmt / sort --no-format / clippy -D warnings / unit tests green +[ ] Cargo.lock churn limited to this crate's deps +``` + +## Evidence base + +Recurring comments mined from connector PRs including (non-exhaustive): +SurrealDB sink (#3453), Meilisearch sink/source (#3497/#3498), OpenSearch +source (#3515), Quickwit convention (#3523), MySQL source (#3568), JDBC +source (#3588), Doris retry (#3574), Redshift sink (#3654), Airflow trigger +(#3716), Fluss sink (#3782). Highest-density themes: delivery/offset +semantics, idempotency IDs, transient/permanent mapping, config-name drift, +secrets, README/code drift, false-green tests, CI/lockfile hygiene. + +--- + +Discussion / help: see [AGENTS.md](../../../AGENTS.md#discussion-and-support). diff --git a/.claude/skills/connector-runtime/SKILL.md b/.claude/skills/connector-runtime/SKILL.md index 8fb2695a65..0a45a921f6 100644 --- a/.claude/skills/connector-runtime/SKILL.md +++ b/.claude/skills/connector-runtime/SKILL.md @@ -211,7 +211,7 @@ Fatal errors propagate to `main` and exit. Per-connector / per-message errors ar All families labeled by `connector_key` + `connector_type` (histogram adds `stage`): -- **Counters**: `iggy_connector_messages_{produced,sent,consumed,processed,filtered}_total` and `iggy_connector_errors_total`. These are the *rendered* names; each is registered without the `_total`, which the OpenMetrics encoder appends. +- **Counters**: `iggy_connector_messages_{produced,sent,consumed,processed,filtered,errors}_total`. - `messages_filtered_total` - intentional drops via transform `Ok(None)`. - `errors_total` - unexpected drops (decode/encode/build failure, missing field, ...) + batch-level failures, one per failed run. - `sink_runs_total` - FFI `consume()` calls for sink batches, one per contiguous payload-variant run. Divide by `stage_duration_seconds_count{stage="total"}` for runs per batch. @@ -222,7 +222,7 @@ All families labeled by `connector_key` + `connector_type` (histogram adds `stag When adding a metric: -- Add family to `Metrics` struct + `init`, register with name + help text. Never end a `Counter` family's registered name in `_total`: the encoder appends it and the series renders `_total_total`. Gauges get no suffix, so a gauge name may end in `_total` literally. +- Add family to `Metrics` struct + `init`, register with name + help text. - New label sets define `EncodeLabelSet` struct + label enum (hand-impl `EncodeLabelValue` for snake_case values - the derive emits PascalCase). - Histograms: pass `fn() -> Histogram` to `Family::new_with_constructor`. - Add unit tests under `mod tests` with `given_*_when_*_should_*` BDD names. diff --git a/.claude/skills/connector-sink/SKILL.md b/.claude/skills/connector-sink/SKILL.md index d135682824..ae03a13066 100644 --- a/.claude/skills/connector-sink/SKILL.md +++ b/.claude/skills/connector-sink/SKILL.md @@ -34,7 +34,8 @@ for getting them to the external system reliably and efficiently. ## Quick reference -- Skeleton: [TEMPLATE.md](TEMPLATE.md) (load on demand when authoring). +- Skeleton: [TEMPLATE.md](TEMPLATE.md) (fill-in-the-blank kit — implement only `TODO(ConnectorDeveloper)`). +- PR pre-flight: [connector-pr-review](../connector-pr-review/SKILL.md). - Exemplars: `stdout_sink` (minimal), `postgres_sink` (DB + transient detection), `http_sink` (validation, batch modes, retry middleware), `mongodb_sink` (atomic counters), `elasticsearch_sink` / `iceberg_sink` (backend-specific idioms). ## Hard rules diff --git a/.claude/skills/connector-sink/TEMPLATE.md b/.claude/skills/connector-sink/TEMPLATE.md index d9d675fe1c..798c4f8c64 100644 --- a/.claude/skills/connector-sink/TEMPLATE.md +++ b/.claude/skills/connector-sink/TEMPLATE.md @@ -1,100 +1,200 @@ -# Sink plugin skeleton +# Sink plugin fill-in-the-blank kit -Boilerplate for a new `core/connectors/sinks/_sink/`. Adapt the -`MySink` / `Client` types to the backend driver you're integrating. +Copy this kit into `core/connectors/sinks/_sink/`. The scaffolding +covers config, secrets, retry, error classification, batching, logging, +and unit-test shape. **You only implement the marked `TODO(ConnectorDeveloper)` +sections:** build a client from the connection string, and push one +batch. + +Also read [SKILL.md](SKILL.md) and pre-flight with +[connector-pr-review](../connector-pr-review/SKILL.md) before `/ready`. + +Prefer starting from a compiling crate over copying this prose kit: +`core/connectors/sinks/sink_template/` implements the same shape as real, +tested code you can `cargo build`/`cargo test` immediately, with the same +`TODO(ConnectorDeveloper)` markers. Use this kit instead only when copying +a whole crate is more scaffolding than you need. + +## Files to create + +```text +core/connectors/sinks/_sink/ +├── Cargo.toml +├── README.md +├── config.toml +└── src/lib.rs +``` + +Add a workspace member, a row in `sinks/README.md`, and a sample under +`runtime/example_config/connectors/`. + +--- ## Cargo.toml ```toml -# Apache 2.0 header (copy verbatim from any existing sink Cargo.toml) +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [package] name = "iggy_connector__sink" -version = "0.5.1-edge.1" # match the version most other sinks use +version = "0.5.0" # match the version most other sinks use edition = "2024" license = "Apache-2.0" publish = false -# ...keywords, description, repository, homepage all identical to existing sinks +description = "Apache Iggy sink connector" +repository = "https://github.com/apache/iggy" +homepage = "https://iggy.apache.org" [package.metadata.cargo-machete] -ignored = ["dashmap", "once_cell"] # used by sink_connector! macro +ignored = ["dashmap", "once_cell"] [lib] -crate-type = ["cdylib", "lib"] # cdylib = runtime-loadable; lib = unit tests +crate-type = ["cdylib", "lib"] [dependencies] -async-trait = { workspace = true } -dashmap = { workspace = true } +async-trait = { workspace = true } +dashmap = { workspace = true } +humantime = { workspace = true } +iggy_common = { workspace = true } iggy_connector_sdk = { workspace = true } -once_cell = { workspace = true } -serde = { workspace = true } -tokio = { workspace = true } -tracing = { workspace = true } -# + your client crate (reqwest, sqlx, mongodb, ...) +once_cell = { workspace = true } +secrecy = { workspace = true } +serde = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +# TODO(ConnectorDeveloper): add your client crate as a workspace dependency ``` -Run `cargo sort --no-format --workspace` after edits. +Run `cargo sort --no-format --workspace` after edits. Keep `Cargo.lock` +churn limited to this crate's deps. + +--- + +## config.toml (example) + +```toml +# Plugin path matches sibling sinks (relative to connectors runtime cwd). +path = "../../target/release/libiggy_connector__sink" + +[[sinks]] +key = "" +enabled = true +# path is also set via IGGY_CONNECTORS_SINK__PATH in integration tests + +[sinks..plugin_config] +# Never commit real secrets. Use env overrides in tests/ops. +connection_string = "scheme://user:pass@host:port/db" +batch_size = 100 +max_retries = 3 +retry_delay = "500ms" +verbose_logging = false +``` + +--- ## src/lib.rs -Code reads top to bottom. Public types first, then `impl`, then private -helpers. +Replace `` / `` and implement only the `TODO(ConnectorDeveloper)` blocks. ```rust -/* Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements... */ // full Apache header +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. use async_trait::async_trait; +use humantime::Duration as HumanDuration; use iggy_connector_sdk::{ ConsumedMessage, Error, MessagesMetadata, Sink, TopicMetadata, sink_connector, }; +use secrecy::{ExposeSecret, SecretString}; use serde::{Deserialize, Serialize}; use std::str::FromStr; use std::time::Duration; use tokio::sync::Mutex; use tracing::{debug, error, info, warn}; -sink_connector!(MySink); // generates FFI symbols + version export +sink_connector!(NameSink); + +const CONNECTOR_NAME: &str = "Name sink"; +const DEFAULT_BATCH_SIZE: u32 = 100; +const DEFAULT_MAX_RETRIES: u32 = 3; // total attempts +const DEFAULT_RETRY_DELAY: &str = "500ms"; -const CONNECTOR_NAME: &str = "My sink"; +/// Backend client. Replace with the real driver type. +struct BackendClient { + // TODO(ConnectorDeveloper): fields +} #[derive(Debug, Serialize, Deserialize)] -pub struct MySinkConfig { - pub endpoint: String, +#[serde(deny_unknown_fields)] +pub struct NameSinkConfig { + #[serde(serialize_with = "iggy_common::serde_secret::serialize_secret")] + pub connection_string: SecretString, pub batch_size: Option, pub max_retries: Option, - pub retry_delay: Option, // humantime, e.g. "500ms" - pub verbose_logging: Option, // mirror runtime's `verbose` flag - // Every runtime-tunable field is Option; defaults applied in new() + pub retry_delay: Option, + pub verbose_logging: Option, + // TODO(ConnectorDeveloper): optional non-secret knobs (table, index, batch_mode, ...) } #[derive(Debug)] -pub struct MySink { +pub struct NameSink { id: u32, - config: MySinkConfig, + config: NameSinkConfig, batch_size: usize, max_retries: u32, retry_delay: Duration, verbose: bool, - client: Option, + client: Option, state: Mutex, } -#[derive(Debug)] +#[derive(Debug, Default)] struct State { messages_processed: u64, errors: u64, } -impl MySink { - pub fn new(id: u32, config: MySinkConfig) -> Self { - let batch_size = config.batch_size.unwrap_or(100) as usize; - let max_retries = config.max_retries.unwrap_or(3); +impl NameSink { + pub fn new(id: u32, config: NameSinkConfig) -> Self { + let batch_size = config.batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize; + let max_retries = config.max_retries.unwrap_or(DEFAULT_MAX_RETRIES); let retry_delay = config .retry_delay .as_deref() - .and_then(|raw| humantime::Duration::from_str(raw).ok().map(|d| *d)) + .and_then(|raw| HumanDuration::from_str(raw).ok().map(|d| *d)) .unwrap_or_else(|| { - warn!("Invalid retry_delay for {CONNECTOR_NAME} ID: {id}, defaulting to 500ms"); + warn!( + "Invalid retry_delay for {CONNECTOR_NAME} ID: {id}, defaulting to {DEFAULT_RETRY_DELAY}" + ); Duration::from_millis(500) }); let verbose = config.verbose_logging.unwrap_or(false); @@ -106,24 +206,49 @@ impl MySink { retry_delay, verbose, client: None, - state: Mutex::new(State { messages_processed: 0, errors: 0 }), + state: Mutex::new(State::default()), + } + } + + async fn send_batch_with_retry( + &self, + client: &BackendClient, + topic_metadata: &TopicMetadata, + batch: &[ConsumedMessage], + ) -> Result<(), Error> { + let mut attempt = 0u32; + loop { + attempt += 1; + match push_batch(client, topic_metadata, batch).await { + Ok(()) => return Ok(()), + Err(error) if is_permanent(&error) => return Err(error), + Err(error) if attempt >= self.max_retries => return Err(error), + Err(error) => { + warn!( + "{CONNECTOR_NAME} ID: {} retry {attempt}/{}: {error}", + self.id, self.max_retries + ); + tokio::time::sleep(self.retry_delay.saturating_mul(attempt)).await; + } + } } } } #[async_trait] -impl Sink for MySink { +impl Sink for NameSink { async fn open(&mut self) -> Result<(), Error> { + // Structural validation belongs here / in new(), not in consume(). let client = build_client(&self.config) .await .map_err(|e| Error::InitError(format!("client build failed: {e}")))?; - client.ping() + ping(&client) .await .map_err(|e| Error::InitError(format!("connectivity check failed: {e}")))?; self.client = Some(client); info!( - "Opened {CONNECTOR_NAME} connector ID: {}, endpoint: {}", - self.id, self.config.endpoint + "Opened {CONNECTOR_NAME} connector ID: {}, endpoint: ", + self.id ); Ok(()) } @@ -140,28 +265,44 @@ impl Sink for MySink { if self.verbose { info!( - "{CONNECTOR_NAME} ID: {} consuming {} messages from stream: {}, topic: {}, offset: {}, current_offset: {}", - self.id, messages.len(), topic_metadata.stream, topic_metadata.topic, - messages_metadata.partition_id, messages_metadata.current_offset + "{CONNECTOR_NAME} ID: {} consuming {} messages from stream: {}, topic: {}, partition_id: {}, current_offset: {}", + self.id, + messages.len(), + topic_metadata.stream, + topic_metadata.topic, + messages_metadata.partition_id, + messages_metadata.current_offset ); } else { debug!( "{CONNECTOR_NAME} ID: {} consuming {} messages", - self.id, messages.len() + self.id, + messages.len() ); } + // Never swallow a failed batch as Ok(()) — offsets may still advance. let mut last_err: Option = None; for batch in messages.chunks(self.batch_size) { - match self.send_batch(client, batch).await { - Ok(()) => { /* counter */ } + match self + .send_batch_with_retry(client, topic_metadata, batch) + .await + { + Ok(()) => { + let mut state = self.state.lock().await; + state.messages_processed += batch.len() as u64; + } Err(Error::PermanentHttpError(message)) => { error!( "{CONNECTOR_NAME} ID: {} dropping batch (permanent): {message}", self.id ); + let mut state = self.state.lock().await; + state.errors += 1; } Err(error) => { + let mut state = self.state.lock().await; + state.errors += 1; last_err = Some(error); } } @@ -173,9 +314,8 @@ impl Sink for MySink { } async fn close(&mut self) -> Result<(), Error> { - // sqlx pools have `.close().await`; reqwest/mongodb/elasticsearch just drop. if let Some(client) = self.client.take() { - let _ = client; // or `client.close().await;` for sqlx + close_client(client).await; } let state = self.state.lock().await; info!( @@ -186,5 +326,98 @@ impl Sink for MySink { } } -async fn build_client(config: &MySinkConfig) -> Result { /* ... */ } +// ─── Backend surface: implement these ─────────────────────────────────────── + +/// TODO(ConnectorDeveloper): parse `config.connection_string.expose_secret()` and build the client. +async fn build_client(config: &NameSinkConfig) -> Result { + let _secret = config.connection_string.expose_secret(); + Err("TODO(ConnectorDeveloper): build_client".into()) +} + +/// TODO(ConnectorDeveloper): cheap connectivity probe used from open(). +async fn ping(_client: &BackendClient) -> Result<(), String> { + Ok(()) +} + +/// TODO(ConnectorDeveloper): push one batch. Prefer a stable dedup key from +/// `stream:topic:partition:message_id` (or backend natural key). +/// Use `message.payload.try_to_bytes()` — do not clone Payload::Json. +async fn push_batch( + _client: &BackendClient, + _topic_metadata: &TopicMetadata, + _batch: &[ConsumedMessage], +) -> Result<(), Error> { + Err(Error::InitError("TODO(ConnectorDeveloper): push_batch".into())) +} + +/// Map driver errors. Never classify via `err.to_string()` substrings. +fn is_permanent(error: &Error) -> bool { + matches!( + error, + Error::PermanentHttpError(_) | Error::SchemaMismatch(_) | Error::InvalidRecordValue(_) + ) +} + +async fn close_client(_client: BackendClient) { + // sqlx: pool.close().await; most HTTP clients: drop +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_config() -> NameSinkConfig { + NameSinkConfig { + connection_string: SecretString::from("scheme://localhost/db"), + batch_size: Some(10), + max_retries: Some(2), + retry_delay: Some("10ms".into()), + verbose_logging: Some(false), + } + } + + #[test] + fn given_defaults_should_apply_consts() { + let sink = NameSink::new( + 1, + NameSinkConfig { + connection_string: SecretString::from("scheme://localhost/db"), + batch_size: None, + max_retries: None, + retry_delay: None, + verbose_logging: None, + }, + ); + assert_eq!(sink.batch_size, DEFAULT_BATCH_SIZE as usize); + assert_eq!(sink.max_retries, DEFAULT_MAX_RETRIES); + } + + #[test] + fn given_invalid_retry_delay_should_fall_back_to_default() { + let mut config = test_config(); + config.retry_delay = Some("not-a-duration".into()); + let sink = NameSink::new(1, config); + assert_eq!(sink.retry_delay, Duration::from_millis(500)); + } +} ``` + +--- + +## README.md (required paragraphs) + +Your README must include a **Delivery semantics** section answering: + +1. Transient failure behavior (retry N times, then `Err`) +2. Permanent failure behavior (drop/skip vs fail) +3. Duplication window (usually at-least-once) +4. Dedup key (or "none — duplicates possible") + +Diff README defaults against the `DEFAULT_*` consts before opening the PR. + +--- + +## Before `/ready` + +Run the pre-flight checklist in +[connector-pr-review](../connector-pr-review/SKILL.md#pre-flight-author-checklist). diff --git a/.claude/skills/connector-source/SKILL.md b/.claude/skills/connector-source/SKILL.md index 23613c5b05..5ed88cef9a 100644 --- a/.claude/skills/connector-source/SKILL.md +++ b/.claude/skills/connector-source/SKILL.md @@ -9,8 +9,13 @@ A **source** is a Rust `cdylib` that implements `iggy_connector_sdk::Source` and exposes FFI symbols via the `source_connector!` macro. The runtime calls `poll()` in a loop, applies transforms, encodes via the configured `Schema`, sends to -Apache Iggy, and persists the returned `ConnectorState` after every -successful send. +Apache Iggy, and persists the state `poll()` returned - but only after +the send succeeds. Only one batch is ever in flight: the runtime does +not call `poll()` again until it has reported `Ack` or `Nack` for the +current one via `on_batch_result()` (source batch acknowledgment, see PR #3855 +for the full contract). See +[State persistence](#state-persistence-stage-in-poll-commit-in-on_batch_result) +below. > **Universal connector rules** (SecretString, benchmark, verbose flag, drop accounting, filter contract, exemplar patterns) live in > [connectors-overview](../connectors-overview/SKILL.md). This skill @@ -27,14 +32,15 @@ successful send. ## STOP and ask the user before -- Changing the SDK trait surface (`Source::open` / `poll` / `on_batch_result` / `close`) - that's an SDK change. +- Changing the SDK trait surface (`Source::open` / `poll` / `on_batch_result` / `on_batch_result` / `close`) - that's an SDK change, and `poll`/`on_batch_result` are also an FFI change (`iggy_source_handle_v2`, `iggy_source_batch_result` - breaks every pre-built plugin `.so`). - Adding a long-running side task in the plugin - the runtime owns lifecycle, and orphans survive `close()`. Sanctioned only where the source is itself a server the runtime cannot drive, as in `http_source`'s listener, and then only with an explicit shutdown in the last `close()` that awaits its tasks before returning. - Persisting unbounded state - `State` is rewritten every batch. - Adding a source that requires authoritative offsets external to Apache Iggy without coordinating retention. ## Quick reference -- Skeleton: [TEMPLATE.md](TEMPLATE.md) (load on demand). +- Skeleton: [TEMPLATE.md](TEMPLATE.md) (fill-in-the-blank kit — implement only `TODO(ConnectorDeveloper)`). +- PR pre-flight: [connector-pr-review](../connector-pr-review/SKILL.md). - Exemplars: `random_source` (minimal + canonical state tests), `postgres_source` (cursor / delete-after-read / processed-column modes, restart-survives-state tests), `elasticsearch_source` (scroll cursor), `influxdb_source` (time-series scan). ## Hard rules @@ -76,29 +82,61 @@ of backoff, without calling `close()`. it, rather than re-readable upstream, has to hold the batch itself: see `http_source`'s staging. - Returning `Err` from it stops the source immediately, so it is not a retry signal. -### State persistence - -- `ConnectorState` is `Vec` via MessagePack (`rmp_serde`). Use - `ConnectorState::serialize(&state, NAME, id)` and - `ConnectorState::deserialize::(NAME, id)`. -- `poll()` must not commit cursors or destructive work. Return messages with candidate state and - keep the corresponding work staged. -- The runtime sends the batch, saves its candidate state to - `{state_path}/source_{key}.state`, then calls `on_batch_result(Ack)`. Commit staged in-memory - state and external delete or mark operations only on ACK. A NACK discards the candidate so the - same data can be polled again. -- A crash between `poll()` returning and state persistence leaves the prior cursor for the next - poll, so downstream must tolerate at-least-once delivery. -- Cursor sources that stage state until `on_batch_result` can attach state to the corresponding - message batch. Control-plane state that is independent of a publish can ride an empty batch - instead. -- Return `state: None` for an empty poll when no watermark changed. If an empty poll advances a - watermark, stage and return the new state through the same ACK handshake. -- The runtime can still NACK an empty batch when state storage is latched or a pending checkpoint - cannot resolve. Never treat hand-off as durable; `on_batch_result` reports the outcome. -- Treat candidate-state serialization failure as a poll error. Do not send messages without the - state needed to resume them safely. +### State persistence: stage in `poll()`, commit in `on_batch_result()` + +Source connectors use a one-in-flight-batch ACK/NACK contract (#3855) +between the plugin and the runtime: + +1. `poll()` returns messages and *candidate* state without committing + cursor changes or destructive operations (deletes, mark-processed). +2. The runtime sends the batch to Apache Iggy and waits for the + producer result. +3. After a successful send, the runtime persists the candidate state + to `{state_path}/source_{key}.state`. +4. The runtime calls `on_batch_result(SourceBatchResult::Ack)`. A send + or state-save failure calls `on_batch_result(SourceBatchResult::Nack)` + instead. +5. `on_batch_result()` commits or discards the plugin's staged work + before the next `poll()` starts. The SDK allows only one batch in + flight - it will not call `poll()` again until `on_batch_result()` + for the current batch has returned. + +Canonical pattern (`sources/random_source/src/lib.rs`): + +```rust +pending_state: Mutex>, // staged, not yet committed + +async fn poll(&self) -> Result { + // ... fetch ... + let candidate_state = State { cursor: next_cursor }; + *self.pending_state.lock().await = Some(candidate_state.clone()); + Ok(ProducedMessages { + schema: Schema::Json, + messages, + state: Some(ConnectorState::serialize(&candidate_state, NAME, self.id)?), + }) +} + +async fn on_batch_result(&self, result: SourceBatchResult) -> Result<(), Error> { + let candidate_state = self.pending_state.lock().await.take(); + if result == SourceBatchResult::Ack + && let Some(candidate_state) = candidate_state + { + *self.state.lock().await = candidate_state; + } + // Nack: drop candidate_state, committed self.state is untouched - + // the same range is polled again. + Ok(()) +} +``` + +- `ConnectorState` is `Vec` via MessagePack (`rmp_serde`). Use `ConnectorState::serialize(&state, NAME, id)` + `ConnectorState::deserialize::(NAME, id)`. Both return `Option` and log on failure (non-fatal). +- **The default `on_batch_result` is a no-op.** Only override it - and only then does staging via `pending_state` matter - if `poll()` advances a cursor or performs destructive work (delete-after-read, mark-processed). A source with no staged work (e.g. a pure generator) can rely on the default. +- Returning `Err` from `on_batch_result` **stops the SDK from polling further** - a failed rollback must not be allowed to silently advance to the next batch. +- **Always return state in every `ProducedMessages`**, including empty polls that made progress. Return `state: None` for an empty poll that made *no* progress - this avoids an unnecessary state write and cannot persist state left over from a failed batch. - Keep `State` small - rewritten every batch. No unbounded vecs. +- NACK handling must discard staged cursor changes and staged delete/mark operations so polling redelivers the batch. The SDK retries NACKed batches with capped exponential backoff and stops the source after repeated consecutive NACKs. +- Crash recovery is at-least-once at every point except after the plugin has processed the ACK (see the SDK README's crash-point table, `core/connectors/sdk/README.md#source-delivery-acknowledgment`, for the full breakdown). The SDK allows one in-flight batch. Five consecutive NACKs stop the source and require a manual restart. Returning `Err` from `on_batch_result` is fatal, so @@ -131,18 +169,20 @@ Match `ProducedMessages.schema` to the bytes in `messages[i].payload`: ### Concurrency - Runtime spawns ONE `poll()` task per source. No concurrent `poll()`. +- Only one batch is ever in flight: the SDK does not call `poll()` again until `on_batch_result()` has returned for the previous batch (up to a 30s result timeout, after which the SDK treats it as a Nack). - Don't spawn your own long-running Tokio tasks: the runtime owns lifecycle. The exception is a source that listens rather than polls, which has to own its listener; `http_source` is the worked example, and it shuts its tasks down in the last `close()` rather than leaving them to outlive the connector. ### Errors -| Scenario | Variant | -| ------------------------------------------- | ------------------------------------------------- | -| Bad config in `new()`/`open()` | `Error::InitError` | -| Cannot reach external system at startup | `Error::InitError` or `Error::Connection` | -| Transient fetch failure (retry-worthy) | `Error::Connection` or `Error::HttpRequestFailed` | -| Permanent fetch failure (auth, schema gone) | `Error::PermanentHttpError` | -| Row failed to serialize | `Error::Serialization(...)` | -| State serialization failed | `Error::Serialization(...)` | +| Scenario | Variant | +| --------------------------------------------------- | ------------------------------------------------- | +| Bad config in `new()`/`open()` | `Error::InitError` | +| Cannot reach external system at startup | `Error::InitError` or `Error::Connection` | +| Transient fetch failure (retry-worthy) | `Error::Connection` or `Error::HttpRequestFailed` | +| Permanent fetch failure (auth, schema gone) | `Error::PermanentHttpError` | +| Row failed to serialize | `Error::Serialization(...)` | +| State serialization failed | log + skip (non-fatal) | +| `on_batch_result()` failed to roll back staged work | `Err` - stops the SDK from polling further | Returning `Err` from `poll()` is only logged by the SDK's FFI bridge (`sdk/src/source.rs::handle_messages`) - the loop continues, the next @@ -172,15 +212,20 @@ Iggy consumer-loop labels use literal API names (`offset=`, `current_offset=`). 1. `async fn poll(&mut self)` - won't compile. Use `&self` + `Mutex`. 2. Holding `state.lock()` across the fetch I/O - blocks `close()`, causes shutdown timeouts. 3. Forgetting to sleep - 100% CPU on idle source. -4. Committing a cursor or deleting source data in `poll()` - stage it and wait for ACK. -5. Unbounded data in `State` - rewritten every batch. keep O(constant). -6. `std::sync::Mutex` - blocks the executor. Use `tokio::sync::Mutex`. -7. Not setting `ProducedMessage.id` when a stable ID exists - leaves a consumer nothing to dedupe a replayed duplicate on. It does not make the write idempotent server-side, because nothing there reads it. -8. Spawning side tasks - the runtime owns the scheduler. The one exception is a source that listens rather than polls and must own its listener (see [Concurrency](#concurrency) and the STOP list); it owes an explicit shutdown in the last `close()` that awaits its tasks. +4. Returning `state: None` for an empty poll that *did* make progress (e.g. advanced a watermark) - only a no-progress empty poll should return `None`. +5. Committing a cursor or destructive work (delete/mark-processed) directly in `poll()` instead of staging it and applying it in `on_batch_result()` on `Ack` - a Nack (send or state-save failure) has nothing to discard, and the batch is redelivered against already-mutated state. +6. Unbounded data in `State` - rewritten every batch. keep O(constant). +7. `std::sync::Mutex` - blocks the executor. Use `tokio::sync::Mutex`. +8. Not setting `ProducedMessage.id` when a stable ID exists - loses idempotency. +9. Spawning side tasks - the runtime owns the scheduler. ## Tests -Mandatory four canonical source state tests (see [connector-testing](../connector-testing/SKILL.md) for the full pattern). Copy from `sources/random_source/src/lib.rs::tests`. Plus config defaults, payload building, schema selection. +Mandatory six canonical source tests (see [connector-testing](../connector-testing/SKILL.md) for the full pattern): the four +state tests (restore / no-state / invalid-state / round-trip) plus `given_ack_when_batch_is_staged_should_commit_candidate_state` +and `given_nack_when_batch_is_staged_should_keep_committed_state`. Copy from `sources/random_source/src/lib.rs::tests`. Plus +config defaults, payload building, schema selection. A source relying on the default no-op `on_batch_result` (no staged work) +may skip the ack/nack pair. Integration tests under `core/integration/tests/connectors//` for any source backed by external infra. Use `#[iggy_harness]` + a `TestFixture` backed by `testcontainers-modules`. Reference: `core/integration/tests/connectors/postgres/postgres_source.rs` (multi-mode tests) + `restart.rs` (state survives restart). Exercise both ACK and NACK paths when the source stages cursors or destructive work. diff --git a/.claude/skills/connector-source/TEMPLATE.md b/.claude/skills/connector-source/TEMPLATE.md index 04b8c6062e..7b3931f6f7 100644 --- a/.claude/skills/connector-source/TEMPLATE.md +++ b/.claude/skills/connector-source/TEMPLATE.md @@ -1,64 +1,134 @@ -# Source plugin skeleton +# Source plugin fill-in-the-blank kit -Boilerplate for a new `core/connectors/sources/_source/`. Adapt -the `MySource` / `Client` / row types to the backend driver. +Copy this kit into `core/connectors/sources/_source/`. The +scaffolding covers config, secrets, sleep-first poll, lock discipline, +staged/committed state serde (ACK/NACK batch acknowledgment, #3855), +retry classification, logging, and the six canonical state tests. +**You only implement the marked `TODO(ConnectorDeveloper)` sections:** build a +client from the connection string, and fetch the next batch (advancing +a cursor). -`Cargo.toml` is identical to a sink's (see -[connector-sink/TEMPLATE.md](../connector-sink/TEMPLATE.md)) - only -the crate name suffix changes (`iggy_connector__source`) and the -upstream client dep. +Also read [SKILL.md](SKILL.md) and pre-flight with +[connector-pr-review](../connector-pr-review/SKILL.md) before `/ready`. + +Prefer starting from a compiling crate over copying this prose kit: +`core/connectors/sources/source_template/` implements the same shape as +real, tested code you can `cargo build`/`cargo test` immediately, with the +same `TODO(ConnectorDeveloper)` markers. Use this kit instead only when +copying a whole crate is more scaffolding than you need. + +## Files to create + +```text +core/connectors/sources/_source/ +├── Cargo.toml +├── README.md +├── config.toml +└── src/lib.rs +``` + +Add a workspace member, a row in `sources/README.md`, and a sample under +`runtime/example_config/connectors/`. + +--- + +## Cargo.toml + +Same shape as the sink kit (`cdylib` + `lib`, workspace deps, Apache +header). Only the package name suffix changes: + +```toml +name = "iggy_connector__source" +# ... identical metadata / machete ignored / crate-type ... +# TODO(ConnectorDeveloper): add your client crate as a workspace dependency +``` + +--- + +## config.toml (example) + +```toml +path = "../../target/release/libiggy_connector__source" + +[[sources]] +key = "" +enabled = true + +[sources..plugin_config] +connection_string = "scheme://user:pass@host:port/db" +poll_interval = "5s" +batch_size = 100 +max_retries = 3 +retry_delay = "500ms" +verbose_logging = false +``` + +Defaults in this file must match `DEFAULT_*` consts in code. + +--- ## src/lib.rs -Code reads top to bottom. Public types first, `impl Source` after, -helpers below. +Replace `` / `` and implement only the `TODO(ConnectorDeveloper)` blocks. ```rust /* Apache 2.0 header */ -use std::str::FromStr; -use std::time::Duration; - use async_trait::async_trait; +use humantime::Duration as HumanDuration; use iggy_connector_sdk::{ ConnectorState, Error, ProducedMessage, ProducedMessages, Schema, Source, source::SourceBatchResult, source_connector, }; +use secrecy::{ExposeSecret, SecretString}; use serde::{Deserialize, Serialize}; use tokio::sync::Mutex; use tokio::time::sleep; use tracing::{debug, error, info, warn}; -source_connector!(MySource); +source_connector!(NameSource); -const CONNECTOR_NAME: &str = "My source"; +const CONNECTOR_NAME: &str = "Name source"; +const DEFAULT_POLL_INTERVAL: &str = "5s"; +const DEFAULT_BATCH_SIZE: u32 = 100; +const DEFAULT_MAX_RETRIES: u32 = 3; // total attempts +const DEFAULT_RETRY_DELAY: &str = "500ms"; + +struct BackendClient { + // TODO(ConnectorDeveloper): fields +} #[derive(Debug, Serialize, Deserialize)] -pub struct MySourceConfig { - pub endpoint: String, - pub poll_interval: Option, // humantime, "10s" default +#[serde(deny_unknown_fields)] +pub struct NameSourceConfig { + #[serde(serialize_with = "iggy_common::serde_secret::serialize_secret")] + pub connection_string: SecretString, + pub poll_interval: Option, pub batch_size: Option, pub max_retries: Option, pub retry_delay: Option, pub verbose_logging: Option, + // TODO(ConnectorDeveloper): optional non-secret knobs (query, table, index, ...) } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] struct State { - cursor: Option, // WAL LSN, scroll id, timestamp, ... - last_offset: u64, + /// Opaque backend cursor (LSN, scroll id, timestamp, PK, ...). Keep O(1). + cursor: Option, messages_produced: u64, } #[derive(Debug)] -pub struct MySource { +pub struct NameSource { id: u32, - config: MySourceConfig, + config: NameSourceConfig, poll_interval: Duration, + batch_size: usize, + max_retries: u32, + retry_delay: Duration, verbose: bool, - client: Option, + client: Option, state: Mutex, - pending: Mutex>, } impl MySource { @@ -67,45 +137,65 @@ impl MySource { let poll_interval = humantime::Duration::from_str(&raw_interval) .map(|d| *d) .unwrap_or_else(|_| { - warn!("Invalid poll_interval for {CONNECTOR_NAME} ID: {id}, defaulting to 10s"); - Duration::from_secs(10) + warn!( + "Invalid poll_interval for {CONNECTOR_NAME} ID: {id}, defaulting to {DEFAULT_POLL_INTERVAL}" + ); + Duration::from_secs(5) + }); + let batch_size = config.batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize; + let max_retries = config.max_retries.unwrap_or(DEFAULT_MAX_RETRIES); + let retry_delay = config + .retry_delay + .as_deref() + .and_then(|raw| HumanDuration::from_str(raw).ok().map(|d| *d)) + .unwrap_or_else(|| { + warn!( + "Invalid retry_delay for {CONNECTOR_NAME} ID: {id}, defaulting to {DEFAULT_RETRY_DELAY}" + ); + Duration::from_millis(500) }); - let verbose = config.verbose_logging.unwrap_or(false); let restored = state .and_then(|s| s.deserialize::(CONNECTOR_NAME, id)) - .inspect(|s| info!( - "Restored state for {CONNECTOR_NAME} ID: {id}, last_offset: {}, cursor: {:?}", - s.last_offset, s.cursor - )); + .inspect(|s| { + info!( + "Restored state for {CONNECTOR_NAME} ID: {id}, cursor: {:?}, messages_produced: {}", + s.cursor, s.messages_produced + ); + }); Self { id, config, poll_interval, + batch_size, + max_retries, + retry_delay, verbose, client: None, state: Mutex::new(restored.unwrap_or(State { cursor: None, - last_offset: 0, messages_produced: 0, })), - pending: Mutex::new(None), } } } #[async_trait] -impl Source for MySource { +impl Source for NameSource { async fn open(&mut self) -> Result<(), Error> { + // Validate query/cursor shape here — not on first poll after sleep. let client = build_client(&self.config) .await .map_err(|e| Error::InitError(format!("client build failed: {e}")))?; + ping(&client) + .await + .map_err(|e| Error::InitError(format!("connectivity check failed: {e}")))?; self.client = Some(client); info!( - "Opened {CONNECTOR_NAME} connector ID: {}, endpoint: {}", - self.id, self.config.endpoint + "Opened {CONNECTOR_NAME} connector ID: {}, endpoint: ", + self.id ); Ok(()) } @@ -113,9 +203,9 @@ impl Source for MySource { async fn poll(&self) -> Result { sleep(self.poll_interval).await; // sleep first - backpressure - let mut candidate = self.state.lock().await.clone(); + let cursor = { self.state.lock().await.cursor.clone() }; // brief read - let fetched = self.fetch_since(candidate.cursor.as_deref()).await?; + let fetched = self.fetch_since(cursor.as_deref()).await?; // no lock held let mut messages = Vec::with_capacity(fetched.len()); let mut next_cursor = None; @@ -136,45 +226,32 @@ impl Source for MySource { if self.verbose { info!( - "{CONNECTOR_NAME} ID: {} produced {} messages, next cursor: {:?}", - self.id, messages.len(), next_cursor + "{CONNECTOR_NAME} ID: {} polled {} messages, next_cursor: {:?}", + self.id, + fetched.messages.len(), + fetched.next_cursor ); } - if messages.is_empty() { - return Ok(ProducedMessages { - schema: Schema::Json, - messages, - state: None, - }); - } - - candidate.messages_produced += messages.len() as u64; - candidate.cursor = next_cursor; - let persisted = ConnectorState::serialize(&candidate, CONNECTOR_NAME, self.id) - .ok_or_else(|| Error::Serialization("failed to serialize source state".into()))?; - *self.pending.lock().await = Some(candidate); + let persisted = { // brief write + let mut state = self.state.lock().await; + state.messages_produced += messages.len() as u64; + if let Some(c) = next_cursor { + state.cursor = Some(c); + } + ConnectorState::serialize(&*state, CONNECTOR_NAME, self.id) + }; Ok(ProducedMessages { schema: Schema::Json, messages, - state: Some(persisted), + state: persisted, }) } - async fn on_batch_result(&self, result: SourceBatchResult) -> Result<(), Error> { - let (SourceBatchResult::Ack, Some(candidate)) = - (result, self.pending.lock().await.take()) - else { - return Ok(()); - }; - *self.state.lock().await = candidate; - Ok(()) - } - async fn close(&mut self) -> Result<(), Error> { if let Some(client) = self.client.take() { - let _ = client; // or `client.close().await;` for sqlx pools + close_client(client).await; } let state = self.state.lock().await; info!( @@ -185,5 +262,187 @@ impl Source for MySource { } } -async fn build_client(config: &MySourceConfig) -> Result { /* ... */ } +// ─── Backend surface: implement these ─────────────────────────────────────── + +/// TODO(ConnectorDeveloper): parse `config.connection_string.expose_secret()` and build the client. +async fn build_client(config: &NameSourceConfig) -> Result { + let _secret = config.connection_string.expose_secret(); + Err("TODO(ConnectorDeveloper): build_client".into()) +} + +/// TODO(ConnectorDeveloper): cheap connectivity probe used from open(). +async fn ping(_client: &BackendClient) -> Result<(), String> { + Ok(()) +} + +/// TODO(ConnectorDeveloper): fetch up to `limit` rows after `cursor`. +/// Set `ProducedMessage.id` from a stable natural key (never random UUID). +/// Set `origin_timestamp` when the backend has event time (nanoseconds). +async fn fetch_batch( + _client: &BackendClient, + _cursor: Option<&str>, + _limit: usize, +) -> Result { + Err(Error::InitError("TODO(ConnectorDeveloper): fetch_batch".into())) +} + +fn is_permanent(error: &Error) -> bool { + matches!( + error, + Error::PermanentHttpError(_) | Error::SchemaMismatch(_) | Error::InvalidConfigValue(_) + ) +} + +async fn close_client(_client: BackendClient) {} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_config() -> NameSourceConfig { + NameSourceConfig { + connection_string: SecretString::from("scheme://localhost/db"), + poll_interval: Some("100ms".into()), + batch_size: Some(10), + max_retries: Some(2), + retry_delay: Some("10ms".into()), + verbose_logging: Some(false), + } + } + + #[test] + fn given_persisted_state_should_restore_cursor() { + let state = State { + cursor: Some("cursor-1".into()), + messages_produced: 7, + }; + let bytes = rmp_serde::to_vec(&state).expect("serialize"); + let source = NameSource::new(1, test_config(), Some(ConnectorState(bytes))); + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let restored = source.state.lock().await; + assert_eq!(restored.cursor.as_deref(), Some("cursor-1")); + assert_eq!(restored.messages_produced, 7); + }); + } + + #[test] + fn given_no_state_should_start_fresh() { + let source = NameSource::new(1, test_config(), None); + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let restored = source.state.lock().await; + assert!(restored.cursor.is_none()); + assert_eq!(restored.messages_produced, 0); + }); + } + + #[test] + fn given_invalid_state_should_start_fresh() { + let invalid = ConnectorState(b"not valid msgpack".to_vec()); + let source = NameSource::new(1, test_config(), Some(invalid)); + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let restored = source.state.lock().await; + assert!(restored.cursor.is_none()); + assert_eq!(restored.messages_produced, 0); + }); + } + + #[test] + fn state_should_be_serializable_and_deserializable() { + let original = State { + cursor: Some("c".into()), + messages_produced: 3, + }; + let bytes = rmp_serde::to_vec(&original).unwrap(); + let restored: State = rmp_serde::from_slice(&bytes).unwrap(); + assert_eq!(original, restored); + } + + #[test] + fn given_defaults_should_apply_consts() { + let source = NameSource::new( + 1, + NameSourceConfig { + connection_string: SecretString::from("scheme://localhost/db"), + poll_interval: None, + batch_size: None, + max_retries: None, + retry_delay: None, + verbose_logging: None, + }, + None, + ); + assert_eq!(source.batch_size, DEFAULT_BATCH_SIZE as usize); + assert_eq!(source.max_retries, DEFAULT_MAX_RETRIES); + assert_eq!(source.poll_interval, Duration::from_secs(5)); + } + + // Stages `pending_state` directly rather than going through poll() - poll()'s + // TODO(ConnectorDeveloper) fetch is unimplemented in this template, but on_batch_result's + // commit/discard contract is independently testable and must stay covered once + // the backend is filled in. + + #[test] + fn given_ack_when_batch_is_staged_should_commit_candidate_state() { + let source = NameSource::new(1, test_config(), None); + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let candidate = State { + cursor: Some("cursor-2".into()), + messages_produced: 5, + }; + *source.pending_state.lock().await = Some(candidate.clone()); + + source + .on_batch_result(SourceBatchResult::Ack) + .await + .expect("ack should be applied"); + + assert_eq!(*source.state.lock().await, candidate); + assert!(source.pending_state.lock().await.is_none()); + }); + } + + #[test] + fn given_nack_when_batch_is_staged_should_keep_committed_state() { + let source = NameSource::new(1, test_config(), None); + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let committed_before = source.state.lock().await.clone(); + *source.pending_state.lock().await = Some(State { + cursor: Some("cursor-2".into()), + messages_produced: 5, + }); + + source + .on_batch_result(SourceBatchResult::Nack) + .await + .expect("nack should be applied"); + + assert_eq!(*source.state.lock().await, committed_before); + assert!(source.pending_state.lock().await.is_none()); + }); + } +} ``` + +--- + +## README.md (required paragraphs) + +Include a **Delivery semantics** section: + +1. Transient fetch failure → retry N times, then `Err` (loop continues; see `connector-source`) +2. Cursor commits only on `on_batch_result(Ack)` - a Nack (send or state-save failure) discards the staged cursor and redelivers the batch +3. Whether destructive work (delete/mark-processed) is staged in `poll()` and applied only in `on_batch_result()` on Ack (standard - no loss window to document), or must happen earlier for some architectural reason (if so, document the loss window explicitly) +4. Dedup key for `ProducedMessage.id` (or "none") + +--- + +## Before `/ready` + +Run the pre-flight checklist in +[connector-pr-review](../connector-pr-review/SKILL.md#pre-flight-author-checklist). +Mandatory: the six canonical state tests above must stay green. diff --git a/.claude/skills/connectors-overview/SKILL.md b/.claude/skills/connectors-overview/SKILL.md index 230ad5354b..a3a3575eeb 100644 --- a/.claude/skills/connectors-overview/SKILL.md +++ b/.claude/skills/connectors-overview/SKILL.md @@ -7,6 +7,8 @@ description: Entry point and index for the Apache Iggy Connectors subsystem (`co Repo-wide rules (Apache headers, fmt/sort/clippy order, idiomatic Rust traits, imports at top, no `cargo install`, LLM-slop tells) live in [AGENTS.md](../../../AGENTS.md). This file owns **connector-specific** rules and routes to the per-area skills. +New sink/source authors: start with [`core/connectors/docs/authoring-sinks-and-sources.md`](../../../core/connectors/docs/authoring-sinks-and-sources.md) for the end-to-end checklist, then copy `sinks/sink_template` or `sources/source_template`. + ## Contents - [STOP and ask the user before](#stop-and-ask-the-user-before) @@ -25,7 +27,7 @@ Repo-wide rules (Apache headers, fmt/sort/clippy order, idiomatic Rust traits, i ## STOP and ask the user before -- Bumping `iggy_connector_sdk` MAJOR version or changing any FFI signature in `sdk/src/{sink,source}.rs` - breaks every pre-built plugin `.so`. +- Bumping `iggy_connector_sdk` MAJOR version or changing any FFI signature in `sdk/src/{sink,source}.rs` - breaks every pre-built plugin `.so`. Source FFI is `iggy_source_handle_v2` (batch ID carried to the runtime callback) + `iggy_source_batch_result` (plugin-exported ACK/NACK, #3855). - Changing the runtime's wire conventions (postcard FFI payload structs, default consumer group naming, plugin path resolution). - Modifying `runtime/src/state.rs` save protocol (atomic rename + fsync ordering) - corruption risk. - Renaming or repurposing a `Schema` variant - decoders/encoders pinned to wire bytes. @@ -42,27 +44,31 @@ Both compile as `cdylib` shared libraries (`.so`/`.dylib`/`.dll`) loaded by the ```text ┌─ optional transforms ─┐ External ──poll──▶ SOURCE ──FFI──▶ RUNTIME ──encode──▶ Apache Iggy stream - system plugin ▲ ▲ - │ │ - state save (msgpack) │ - │ - ┌─ optional transforms ─┐ │ + system plugin ▲ ▲ + │ │ + Ack/Nack (on_batch_result) │ + │ state save (msgpack) + └────────────────────┘ + ┌─ optional transforms ─┐ Apache Iggy stream ──decode──▶ RUNTIME ──FFI──▶ SINK ──write──▶ External plugin system ``` +Source is a request/response FFI, not fire-and-forget: after the runtime sends the batch and saves the state `poll()` staged, it reports `SourceBatchResult::Ack` or `::Nack` back to the plugin via `iggy_source_batch_result` (#3855) before calling `poll()` again. See [connector-source](../connector-source/SKILL.md#state-persistence-stage-in-poll-commit-in-on_batch_result) for the full handshake. + Headers set on the source side ride through transforms (which may modify, drop, or pass them) and arrive at the sink with `BTreeMap` preserved deterministically. ## Which skill to load -| Task | Skill | -| --------------------------------------------------- | --------------------- | -| Write a new sink plugin | `connector-sink` | -| Write a new source plugin | `connector-source` | -| Add schema / decoder / encoder / SDK trait surface | `connector-sdk` | -| Change runtime internals (FFI, manager, state, ...) | `connector-runtime` | -| Add a transform (field-level or format conversion) | `connector-transform` | -| Write unit / integration tests for any of the above | `connector-testing` | +| Task | Skill | +| ---------------------------------------------------- | --------------------- | +| Write a new sink plugin | `connector-sink` | +| Write a new source plugin | `connector-source` | +| Add schema / decoder / encoder / SDK trait surface | `connector-sdk` | +| Change runtime internals (FFI, manager, state, ...) | `connector-runtime` | +| Add a transform (field-level or format conversion) | `connector-transform` | +| Write unit / integration tests for any of the above | `connector-testing` | +| Review a sink/source PR / pre-flight before `/ready` | `connector-pr-review` | ## Stick to conventions @@ -96,24 +102,14 @@ The connectors codebase is intentionally repetitive across plugins. Cross-plugin ### Secrets -Any credential-bearing field (connection strings, API keys, bearer tokens, AWS keys) must be `SecretString` from the `secrecy` crate. Plain `String` for a credential is a review-blocker: `SecretString` redacts on `Debug`, so it is what keeps a credential out of a log line that formats the whole config. - -**`serde_secret::serialize_secret` EXPOSES the secret. It does not redact.** It calls `expose_secret()` and writes the plaintext. `SecretString` deliberately has no `Serialize` impl, and that absence is the protection - so adding `serialize_with` is what *unblocks* the derive and turns a compile-time guarantee into plaintext output. Use it only where the plaintext is the point: a wire payload, a persisted config, an API response that exposes credentials by design. - -So the default for a plugin config struct is **derive `Deserialize`, but not `Serialize`**. `Deserialize` is required: the SDK glue deserializes the config into the plugin's own struct (`sdk/src/{sink,source}.rs` call `serde_json::from_str::` under a `DeserializeOwned` bound). - -What never happens is the return trip. The runtime holds plugin configuration as a `serde_json::Value` - parsed from TOML, posted as JSON to the control API, or injected by env var - and hands that across the FFI, so nothing re-serializes the plugin's struct. Leaving `Serialize` off makes that compiler-enforced instead of convention-enforced (`sources/http_source/src/lib.rs::HttpSourceConfig` does this, and comments the omission so nobody adds it back). - -Pattern: +Any credential-bearing field (connection strings, API keys, bearer tokens, AWS keys) must be `SecretString` from the `secrecy` crate, with the workspace serde wrapper applied so `Debug` and serialization both redact. Runtime exposes plugin configs over the `/stats` HTTP surface via serialization - plain `String` leaks the secret to anyone who can hit the endpoint. Plain `String` for a credential is a review-blocker. Pattern (from `sinks/postgres_sink/src/lib.rs::PostgresSinkConfig`): ```rust use secrecy::{ExposeSecret, SecretString}; -// `Deserialize` only. Nothing re-serializes a plugin config, and leaving -// `Serialize` off is what makes the credential unserializable rather than -// merely un-serialized. -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct MyConfig { + #[serde(serialize_with = "iggy_common::serde_secret::serialize_secret")] pub connection_string: SecretString, } @@ -123,13 +119,7 @@ let pool = PgPoolOptions::new() .await?; ``` -If a config struct genuinely needs `Serialize`, `serde_secret::serialize_redacted` (and `serialize_optional_redacted`) write `[REDACTED]` in place of the value. Reach for `serialize_secret` only when the caller must get the real thing back. The sinks and sources listed below predate that helper and use the exposing one; the annotation is inert today, but it is not the protection it looks like. - -Note that none of this protects the credential from the runtime's own control API, which returns plugin configuration verbatim - see #3802. Plugin-side annotations are inert there because the runtime never routes through them. - -Plugin-side uses of the exposing helpers: `sinks/{postgres,mongodb,elasticsearch,influxdb,s3,surrealdb}_sink`, `sources/{postgres,elasticsearch,influxdb}_source`. - -That list is plugin-side only, not an inventory of every caller in the tree, and the others are not all mistakes: `runtime/src/api/config.rs` puts `serialize_secret` on `HttpConfig::api_key` (inert for the same reason), and several `core/common` wire-payload types (login, create-user, change-password, PAT) use these helpers by design, because there the credential *is* the payload. +In-tree uses: `sinks/{postgres,mongodb,elasticsearch,influxdb,delta}_sink`, `sources/{postgres,elasticsearch,influxdb}_source`. ### Errors @@ -181,7 +171,7 @@ JSON log format, parser unit tests). | Real-infra sink + integration tests | `sinks/postgres_sink/` + `integration/tests/connectors/postgres/postgres_sink.rs` | | Feature-rich sink config (validation, batch modes, retry) | `sinks/http_sink/` | | Atomic counters on hot path | `sinks/mongodb_sink/` | -| Simplest source (4 canonical state tests) | `sources/random_source/` | +| Simplest source (6 canonical state + ACK/NACK tests) | `sources/random_source/` | | Real-infra source | `sources/postgres_source/` + `integration/tests/connectors/postgres/postgres_source.rs` | Read the relevant exemplar end-to-end before writing or modifying a connector. @@ -209,7 +199,7 @@ Each implemented in at least one in-tree plugin or runtime path. | `flume::unbounded()` channel | `runtime/src/source.rs::spawn_source_handler` / `source_forwarding_loop` | MPSC handoff from SDK async task to runtime loop | | `tokio::sync::watch::channel(())` | `sdk/src/{sink,source}.rs`, `runtime/src/sink.rs`, `runtime/src/manager/*` | One-shot shutdown broadcast | | `dashmap::DashMap` | `runtime/src/manager/sink.rs`, `source.rs::SOURCE_SENDERS`, SDK `INSTANCES` | Lock-free concurrent keyed access | -| `secrecy::SecretString` + `iggy_common::serde_secret::serialize_secret` | `sinks/postgres_sink::PostgresSinkConfig::connection_string` | `Debug` redacts; `serialize_secret` EXPOSES | +| `secrecy::SecretString` + `iggy_common::serde_secret::serialize_secret` | `sinks/postgres_sink::PostgresSinkConfig::connection_string` | Auto-redact on Debug/Display + serialization | ## Drop accounting @@ -222,13 +212,21 @@ Each implemented in at least one in-tree plugin or runtime path. - `&mut self` on `Sink::consume` or `Source::poll` impls - won't compile, flag any creative workaround. - `std::sync::Mutex` held across `.await` - swap for `tokio::sync::Mutex`. - Missing `[lib] crate-type = ["cdylib", "lib"]` in plugin `Cargo.toml`. -- Source plugin without the four canonical state tests (see `connector-testing`). +- Source plugin without the four canonical state tests, or without ACK/NACK tests when `on_batch_result` is overridden (see `connector-testing`). +- Source `poll()` committing a cursor or destructive work directly instead of staging it for `on_batch_result` to commit on `Ack` / discard on `Nack`. - New silent message drop without a metric increment. - Wrapping `format!()` around args passed to `error!`/`warn!`/`info!`/`debug!` - eager `format!` allocates even when level filters the line out. Pass args directly: `error!("foo: {x}")` or `error!(error = %x, "foo")`. - Logging a connection string, API key, or token. - Plain `String` for a credential field - use `SecretString`. - `tokio::spawn` inside plugin code - runtime owns lifecycle. - `std::time::SystemTime::now()` in transforms - non-deterministic, breaks tests. +- Returning `Ok(())` from `consume` after a failed batch (offsets can still advance). +- Random UUIDs as message IDs / dedup keys. +- Classifying retryability via `err.to_string()` substring matches. +- README defaults that disagree with code consts. +- Invented config knob names (e.g. `request_timeout` instead of `timeout`) instead of the canon in `connector-pr-review`. + +For the full PR review checklist (blockers, delivery-semantics paragraph, pre-flight paste), load [connector-pr-review](../connector-pr-review/SKILL.md). ## File map diff --git a/Cargo.lock b/Cargo.lock index e197388942..b41235228f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7579,6 +7579,41 @@ dependencies = [ "tracing", ] +[[package]] +name = "iggy_connector_template_sink" +version = "0.1.0" +dependencies = [ + "async-trait", + "dashmap", + "iggy_common", + "iggy_connector_sdk", + "reqwest 0.13.5", + "reqwest-middleware", + "secrecy", + "serde", + "tokio", + "tracing", +] + +[[package]] +name = "iggy_connector_template_source" +version = "0.1.0" +dependencies = [ + "async-trait", + "dashmap", + "humantime", + "iggy_common", + "iggy_connector_sdk", + "reqwest 0.13.5", + "reqwest-middleware", + "rmp-serde", + "secrecy", + "serde", + "serde_json", + "tokio", + "tracing", +] + [[package]] name = "iggy_examples" version = "0.0.6" diff --git a/Cargo.toml b/Cargo.toml index 21cee6bc0b..105abd816a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,6 +47,7 @@ members = [ "core/connectors/sinks/rabbitmq_sink", "core/connectors/sinks/redshift_sink", "core/connectors/sinks/s3_sink", + "core/connectors/sinks/sink_template", "core/connectors/sinks/stdout_sink", "core/connectors/sinks/surrealdb_sink", "core/connectors/sources/elasticsearch_source", @@ -54,6 +55,7 @@ members = [ "core/connectors/sources/influxdb_source", "core/connectors/sources/postgres_source", "core/connectors/sources/random_source", + "core/connectors/sources/source_template", "core/consensus", "core/cpu_allocation", "core/harness_derive", diff --git a/core/connectors/BLOG_POST.md b/core/connectors/BLOG_POST.md new file mode 100644 index 0000000000..02db07fa96 --- /dev/null +++ b/core/connectors/BLOG_POST.md @@ -0,0 +1,104 @@ +# Announcing sink and source connector templates + +*Draft for the Apache Iggy project blog. Replace this header line with +the final publish date and author byline before posting.* + +Apache Iggy's connectors subsystem has grown fast. In the past few +months alone, contributors have shipped or proposed sink and source +connectors for Postgres, MongoDB, Elasticsearch, Iceberg, Delta Lake, +S3, InfluxDB, Doris, ClickHouse, SurrealDB, Meilisearch, OpenSearch, +Redshift, and more — each one a plugin that moves real data between +Apache Iggy and an external system, often in production. That growth +is great news for the project. It also means new contributors keep +re-solving the same non-backend-specific problems from scratch before +their PR can even get to the interesting part: talking to their actual +system. + +## What we found + +Looking back across recent connector PR reviews, the same handful of +issues came up again and again, and none of them had anything to do +with the destination or source system being integrated: + +- **Credentials typed as plain `String`.** Connection strings and API + keys landing in `Debug`/log output because the field wasn't + `secrecy::SecretString`. +- **Cursor commits that don't survive a failed delivery.** Since + [#3855](https://github.com/apache/iggy/pull/3855), sources use a + formal ACK/NACK handshake — `poll()` stages candidate state, and + only `on_batch_result()` commits it — but that shape has to be + learned and wired up correctly every time. +- **Errors that don't distinguish retry-worthy from permanent.** + Network hiccups and "this payload will never be accepted" ending up + in the same catch-all error variant. +- **Config knobs with drifting names.** `retry_max_delay` here, + `max_retry_delay` there, `request_timeout` somewhere else, for the + same concept. +- **Missing canonical tests.** State restore/round-trip and ACK/NACK + commit/discard behavior left untested because nobody had a reference + test suite to copy. + +None of this is specific to any one backend. It's framework plumbing +that every sink and every source needs, and until now every author +either copied the closest existing plugin and stripped it down, or +started from a blank `lib.rs` and rediscovered each of these the hard +way, one review round at a time. + +## The templates + +[`core/connectors/sinks/sink_template`](sinks/sink_template) and +[`core/connectors/sources/source_template`](sources/source_template) +are compiling, tested crates you copy and fill in — not prose +describing a pattern, but the pattern itself, already wired up and +passing `cargo test`: + +- Config parsing with `#[serde(deny_unknown_fields)]`, so a typo'd TOML + key fails loudly instead of silently doing nothing. +- `SecretString` on every credential-shaped field + (`connection_string`, `auth_token`), via + `iggy_common::serde_secret::serialize_secret`. +- A retry-wrapped HTTP client plus a startup connectivity probe with + its own backoff. +- A `CircuitBreaker` that's actually consulted before each call and + updated once per `consume()`/`poll()`, not per chunk. +- Sink: batching by a configurable size, and a `last_err` pattern that + never swallows a failed batch into `Ok(())`. +- Source: the full #3855 ACK/NACK contract — `poll()` stages a + candidate cursor, `on_batch_result()` commits it on `Ack` or + discards it on `Nack`, so a dropped batch gets re-polled instead of + silently lost. +- The canonical test suites: six tests for the sink, eight for the + source (four state tests — restore, no-state, invalid-state, + round-trip — plus the two ACK/NACK tests, plus config validation and + the circuit-breaker short-circuit path). + +What's left is marked `TODO(ConnectorDeveloper)` in each crate's `src/lib.rs`: +one spot for a sink (`push_batch()`), two for a source +(`build_raw_client()` if you're not talking HTTP, and +`fetch_records()`). Everything else — the parts that used to eat a +review round — is already done. + +## Using one + +Copy the crate, rename the package and the directory, add it to the +workspace `members` list, fill in the `TODO(ConnectorDeveloper)` spots, and +update `config.toml` for your system. Each crate's own `README.md` +walks through the exact steps. Both templates already build, `clippy +--all-targets -- -D warnings` clean, and pass their tests as committed +— the only thing that should break when you fill in the TODOs is the +`Err(Error::InitError("not implemented yet"))` stub they start from. + +## Why this matters beyond Apache Iggy's connectors + +The pattern generalizes past this one subsystem: any plugin system +with a real framework contract — secrets, retries, staged +commit/rollback, canonical tests — benefits more from a working, +compiling example than from a checklist alone. A checklist tells you +what to verify; a template gives you the thing already verified, so +your own diff is just the part only you can write. + +--- + +*Feedback and discussion: see the project's +[GitHub Discussions](https://github.com/apache/iggy/discussions) or +[Discord](https://discord.gg/apache-iggy).* diff --git a/core/connectors/README.md b/core/connectors/README.md index b334009278..6fd79b9796 100644 --- a/core/connectors/README.md +++ b/core/connectors/README.md @@ -129,6 +129,10 @@ Please refer to the **[Source documentation](https://github.com/apache/iggy/tree New connector can be built simply by implementing either `Sink` or `Source` trait. Please check the **[sink](https://github.com/apache/iggy/tree/master/core/connectors/sinks)** or **[source](https://github.com/apache/iggy/tree/master/core/connectors/sources)** documentation, as well as the existing examples under `core/connectors/sinks` and `core/connectors/sources`. +Starting from a blank crate re-solves the same framework plumbing (config validation, `SecretString` secrets, retry + circuit breaker, ACK/NACK cursor staging) every time. +Copy `core/connectors/sinks/sink_template` or `core/connectors/sources/source_template` instead - compiling, tested starting points with only the backend-specific calls marked `TODO(ConnectorDeveloper)`. +See [`core/connectors/docs/authoring-sinks-and-sources.md`](docs/authoring-sinks-and-sources.md) for the full authoring guide and the review checklist it's built from. + ## Transformations Field transformations (depending on the supported payload formats) can be applied to the messages either before they are sent to the specified topic (e.g. when produced by the source connectors), or before consumed by the sink connectors. To add a transformation, implement the `Transform` trait in the SDK, add its `TransformType` variant and extend `transforms::from_config`. Each transform may have its own, custom configuration. diff --git a/core/connectors/docs/authoring-sinks-and-sources.md b/core/connectors/docs/authoring-sinks-and-sources.md new file mode 100644 index 0000000000..54b041839d --- /dev/null +++ b/core/connectors/docs/authoring-sinks-and-sources.md @@ -0,0 +1,196 @@ +# Authoring Apache Iggy sink and source connectors + +> Reference guide for `core/connectors/` docs. For the narrative version +> suitable for the Apache Iggy blog, see [`BLOG_POST.md`](../BLOG_POST.md). +> Companion materials: +> +> - Review checklist skill: `.claude/skills/connector-pr-review/SKILL.md` +> - Sink fill-in-the-blank kit: `.claude/skills/connector-sink/TEMPLATE.md` +> - Source fill-in-the-blank kit: `.claude/skills/connector-source/TEMPLATE.md` +> - Testing skill: `.claude/skills/connector-testing/SKILL.md` + +Connector PRs in Apache Iggy often spend multiple review rounds on the +**same classes of issues**: secrets handling, retry/error mapping, +idempotency claims that the backend does not enforce, README defaults +that disagree with code, and tests that look green while missing the +mandatory source-state suite. + +This guide turns those review patterns into a strict authoring path. +Follow it and a first review can focus on the backend-specific parts +(connection + fetch/push) instead of re-teaching the framework. + +## The only code you should invent + +Copy the closest existing plugin by shape: + +| Shape | Start from | +| ----- | ---------- | +| DB write sink | `core/connectors/sinks/postgres_sink` | +| HTTP sink | `core/connectors/sinks/http_sink` | +| Polling DB source | `core/connectors/sources/postgres_source` | +| Minimal source + state tests | `core/connectors/sources/random_source` | +| Minimal sink | `core/connectors/sinks/stdout_sink` | +| Fill-in-the-blank sink | `core/connectors/sinks/sink_template` | +| Fill-in-the-blank source | `core/connectors/sources/source_template` | + +`sink_template` / `source_template` are compiling, tested starting +points: config parsing, `SecretString`, retry loops, circuit breaker, +`last_err` batch handling (sink), ACK/NACK cursor staging (source), +sleep-first poll, lock discipline, and unit-test stubs are already +wired up. Copy the crate, rename the package, and fill in the +`TODO(ConnectorDeveloper)` spots in `src/lib.rs` — see each crate's `README.md`. +The skills' `TEMPLATE.md` files describe the same pattern in prose, for +when copying a whole crate is more than you need. + +**Your implementation surface is intentionally small:** + +1. Build a client from `connection_string` (secret). +2. Probe connectivity in `open()`. +3. **Sink:** push one batch. +4. **Source:** fetch the next batch and compute the next cursor. + +Everything else should match siblings. + +## Hard rules (review blockers) + +### Secrets + +- Credentials are `secrecy::SecretString` with + `iggy_common::serde_secret::serialize_secret`. +- Never log connection strings, API keys, or tokens. +- Never persist URL userinfo / bearers into source state files. + +### Errors and retries + +- `max_retries` means **total attempts** (default **3**). README must match. +- Map infra/auth/schema failures to permanent variants + (`PermanentHttpError`, `InitError`, `SchemaMismatch`). +- Map network/5xx/retryable SQLSTATE to transient variants. +- Do **not** decide retryability by substring-matching `err.to_string()`. +- Cap backoff so a dead backend cannot stall shutdown. + +### Delivery semantics (document in README) + +Every plugin README needs a short paragraph that answers: + +1. What happens on transient failure? +2. What happens on permanent failure? +3. What is the duplication window? +4. What is the dedup key (or “none”)? + +Do not claim “no data loss” or “exactly once” unless the backend unique +key **and** the runtime path actually enforce it. Today’s runtime can +still advance consumer offsets in ways that punish swallowed `Ok(())` +after a failed sink batch — return `Err` for retryable failures. + +### Source-specific + +Since #3855, sources use a formal one-in-flight-batch ACK/NACK +handshake, not just a convention: `poll()` returns *candidate* state; +the runtime sends the batch, saves that state only on success, then +calls `on_batch_result(SourceBatchResult::Ack | Nack)`. Only +`on_batch_result` may commit staged work, and the SDK will not call +`poll()` again until it returns. + +- `async fn poll(&self)` with state behind `tokio::sync::Mutex`. +- **Sleep first**, then fetch. +- Brief lock → drop → I/O → brief lock write. +- **Stage, don't commit, in `poll()`.** Cursor advances and destructive + work (delete-after-read, mark-processed) go into a `pending_state` / + staged field, applied only in `on_batch_result()` on `Ack`; a `Nack` + discards the staged work so the same range is redelivered. +- Always return state for a poll that made progress; return `state: + None` for an empty poll that made none. +- The default `on_batch_result` is a no-op - override it (and stage + accordingly) whenever `poll()` advances a cursor or does destructive + work. Returning `Err` from it stops the SDK from polling further. +- Ship the **six canonical tests**: four state tests (restore / + no-state / invalid-state / round-trip) plus two ACK/NACK tests + (commit-on-Ack, keep-committed-on-Nack) if `on_batch_result` is + overridden. Copy `random_source`. + +### Sink-specific + +- `async fn consume(&self, …)`. +- Process all batches; keep `last_err`; never convert a failed batch into + bare `Ok(())`. +- Prefer stable dedup keys (`stream:topic:partition:message_id` or a + natural backend key). No random UUIDs per emit. +- Use `payload.try_to_bytes()` — do not clone `Payload::Json`. + +### Config canon + +| Concept | Field name | +| ------- | ---------- | +| Timeout | `timeout` | +| Retry attempts | `max_retries` | +| Base backoff | `retry_delay` | +| Backoff ceiling | `retry_max_delay` | +| Source cadence | `poll_interval` | +| Verbosity | `verbose_logging` | + +Use `Option` + humantime strings for durations, apply defaults in +`new()` with `warn!` on parse failure, and prefer +`#[serde(deny_unknown_fields)]` so typo’d knobs fail loud. + +Validate structure and connectivity in `new()` / `open()`, never on the +first hot-path call after sleep. + +## Required artifacts in the PR + +1. Plugin crate with `[lib] crate-type = ["cdylib", "lib"]`. +2. Unit tests (BDD `given_X_when_Y_should_Z` / `given_X_should_Y`). +3. Sources: four canonical state tests, plus two ACK/NACK tests if `on_batch_result` is overridden. +4. External backend: real-infra integration test under + `core/integration/tests/connectors//` (`#[iggy_harness]` + + testcontainers or wiremock). No false-green mocks. +5. Row in `sinks/README.md` or `sources/README.md`. +6. Sample TOML under `runtime/example_config/connectors/` with plugin + path `../../target/release/lib…`. +7. README defaults **byte-equal** to code consts. +8. Honest delivery-semantics paragraph. + +## Verification before `/ready` + +```bash +cargo fmt --all +cargo sort --no-format --workspace +cargo clippy -p iggy_connector__{sink|source} --all-targets -- -D warnings +cargo test -p iggy_connector__{sink|source} +# if applicable: +cargo test -p integration -- connectors:::: +``` + +Keep `Cargo.lock` churn limited to this plugin’s dependencies. Do not +touch unrelated foreign SDKs in the same PR. + +## Pre-flight checklist (paste into the PR) + +```text +[ ] SecretString on all credentials; no secret logs/state +[ ] consume/poll never returns Ok(()) after a failed batch that should retry +[ ] Transient vs permanent errors mapped (no Display substring matching) +[ ] Stable message / dedup IDs (no random UUID per emit) +[ ] README delivery semantics paragraph present and honest +[ ] README defaults match code consts +[ ] deny_unknown_fields on plugin config +[ ] Canonical knob names (timeout, max_retries, retry_delay, poll_interval) +[ ] Sources: 4 state tests (+2 ACK/NACK if on_batch_result overridden); sleep-first poll; state staged in poll, committed only in on_batch_result +[ ] External backend: real-infra integration test (not a lying mock) +[ ] example_config + sinks/sources README row +[ ] fmt / sort --no-format / clippy -D warnings / unit tests green +[ ] Cargo.lock churn limited to this crate's deps +``` + +Agents and reviewers: load `.claude/skills/connector-pr-review/SKILL.md` +and treat unchecked blockers as CHANGES_REQUESTED. + +## Why this exists + +Across recent connector PRs (SurrealDB, Meilisearch, OpenSearch, Quickwit +convention bring-up, MySQL/JDBC, Doris retry, Redshift, Airflow, Fluss, +and others), the expensive review threads were rarely about the novel +backend call. They were about framework contracts: secrets, retries, +idempotency honesty, config names, README drift, and missing state +tests. Codifying those contracts as a skill + templates is how we make +connector contributions predictable without lowering the bar. diff --git a/core/connectors/runtime/example_config/connectors/sink_template.toml b/core/connectors/runtime/example_config/connectors/sink_template.toml new file mode 100644 index 0000000000..ee9a7bd3de --- /dev/null +++ b/core/connectors/runtime/example_config/connectors/sink_template.toml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +type = "sink" +key = "template" +enabled = true +version = 0 +name = "Template sink" +path = "target/release/libiggy_connector_template_sink" +plugin_config_format = "toml" +verbose = false + +[[streams]] +stream = "example_stream" +topics = ["example_topic"] +schema = "json" +batch_length = 100 +poll_interval = "5ms" +consumer_group = "template_sink_connector" + +[plugin_config] +connection_string = "https://api.example.com" +target = "events" +health_check_path = "/health" +batch_size = 100 +timeout = "30s" +max_retries = 3 +retry_delay = "500ms" +retry_max_delay = "5s" +max_open_retries = 10 +open_retry_max_delay = "60s" +circuit_breaker_threshold = 5 +circuit_breaker_cool_down = "30s" +verbose_logging = false diff --git a/core/connectors/runtime/example_config/connectors/source_template.toml b/core/connectors/runtime/example_config/connectors/source_template.toml new file mode 100644 index 0000000000..02dbe0eab8 --- /dev/null +++ b/core/connectors/runtime/example_config/connectors/source_template.toml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +type = "source" +key = "template" +enabled = true +version = 0 +name = "Template source" +path = "target/release/libiggy_connector_template_source" +plugin_config_format = "toml" +verbose = false + +[[streams]] +stream = "example_stream" +topic = "example_topic" +schema = "json" +batch_length = 100 +linger_time = "5ms" + +[plugin_config] +connection_string = "https://api.example.com" +health_check_path = "/health" +batch_size = 100 +poll_interval = "1s" +timeout = "30s" +max_retries = 3 +retry_delay = "500ms" +retry_max_delay = "5s" +max_open_retries = 10 +open_retry_max_delay = "60s" +circuit_breaker_threshold = 5 +circuit_breaker_cool_down = "30s" +verbose_logging = false diff --git a/core/connectors/sinks/README.md b/core/connectors/sinks/README.md index 83f30a5aca..b5be4d4b2e 100644 --- a/core/connectors/sinks/README.md +++ b/core/connectors/sinks/README.md @@ -20,6 +20,7 @@ Sink connectors are responsible for writing data from Iggy streams to external s | **stdout_sink** | Prints messages to standard output (useful for debugging and development) | | **surrealdb_sink** | Writes messages into SurrealDB with deterministic record IDs for idempotent replay | | **rabbitmq_sink** | Publishes messages to RabbitMQ exchanges via AMQP | +| **sink_template** | Fill-in-the-blank starting point for a new sink; framework/security plumbing done, one `TODO(ConnectorDeveloper)` spot left | The sink is represented by the single `Sink` trait, which defines the basic interface for all sink connectors. It provides methods for initializing the sink, writing data to external destination, and closing the sink. diff --git a/core/connectors/sinks/sink_template/Cargo.toml b/core/connectors/sinks/sink_template/Cargo.toml new file mode 100644 index 0000000000..886962dc6d --- /dev/null +++ b/core/connectors/sinks/sink_template/Cargo.toml @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# TEMPLATE — rename the package (and this directory) to +# `iggy_connector__sink` before publishing, and update the +# `[[sinks]]` entry you add to the workspace root Cargo.toml accordingly. + +[package] +name = "iggy_connector_template_sink" +version = "0.1.0" +description = "Template for an Apache Iggy sink connector — copy this crate and fill in the TODO sections." +edition = "2024" +license = "Apache-2.0" +keywords = ["iggy", "messaging", "streaming"] +categories = ["command-line-utilities", "database", "network-programming"] +homepage = "https://iggy.apache.org" +documentation = "https://iggy.apache.org/docs" +repository = "https://github.com/apache/iggy" +publish = false + +[package.metadata.cargo-machete] +# dashmap is used only inside the `sink_connector!` macro expansion, so a +# naive unused-dependency scan won't see the usage — keep it ignored rather +# than removing it, or the plugin will fail to compile. +ignored = ["dashmap"] + +[lib] +crate-type = ["cdylib", "lib"] + +[dependencies] +async-trait = { workspace = true } +dashmap = { workspace = true } +iggy_common = { workspace = true } +iggy_connector_sdk = { workspace = true } +reqwest = { workspace = true } +reqwest-middleware = { workspace = true } +secrecy = { workspace = true } +serde = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } diff --git a/core/connectors/sinks/sink_template/README.md b/core/connectors/sinks/sink_template/README.md new file mode 100644 index 0000000000..86da056d81 --- /dev/null +++ b/core/connectors/sinks/sink_template/README.md @@ -0,0 +1,78 @@ +# Template sink connector + +Starting point for a new Apache Iggy **sink** connector. Everything except +pushing data to your actual destination is already implemented and follows +the project's required resilience/security patterns — see the module-level +doc comment at the top of `src/lib.rs` for the full rationale, and the +`iggy-connector-review` skill / the "Building Connectors That Pass Review" +blog post for the checklist this template is built against. + +## What's already done for you + +- Config parsing with `#[serde(deny_unknown_fields)]` so a typo in a TOML + file fails loudly instead of silently doing nothing. +- Config validation in `open()` (not `new()`, which has no way to return an + error) — including validating `target` (the destination table/index/ + collection name) against an allowlist pattern *before* it can ever reach a + query, path, or URL. +- `connection_string` and the optional `auth_token` field both typed as + `SecretString`, since either can carry credentials. +- A retry-wrapped HTTP client (`iggy_connector_sdk::retry::build_retry_client`) + and a startup connectivity probe with its own backoff + (`check_connectivity_with_retry`). +- A `CircuitBreaker` that's actually consulted before each `consume()` call + and updated once per call based on the outcome — not just constructed and + forgotten, and not reset mid-batch by a partial success. +- Batching: `consume()` chunks the incoming messages by a configurable + `batch_size` instead of sending everything in one unbounded request. +- The `sink_connector!` FFI macro invocation and a `Cargo.toml` with the + right `crate-type`, workspace-pinned dependencies, and license header. +- `verbose_logging: Option` upgrading the per-batch log line from + `debug!` to `info!`, mirroring the runtime's own `verbose` flag. +- Tests for config/identifier validation, the circuit-breaker short-circuit + path, the `verbose_logging` flag, and `consume()`'s batch loop end to end. + +## What you need to fill in + +Search for `TODO(ConnectorDeveloper)` in `src/lib.rs` — there is exactly one +spot that requires code, plus two more that are conditional/documentation: + +**`push_batch()`** (required) — build the request/write that actually sends +one chunk of messages to your destination, using +`self.config.connection_string` (and `self.config.target`, already validated +by the time this runs) via `self.client` (already retry-wrapped). Distinguish +permanent failures (bad schema, a destination that will reject this payload +shape no matter how many times you retry) from transient ones (network +error, 5xx, timeout) by returning `Error::PermanentHttpError` for the +former — `consume()` drops and counts a permanent failure instead of +propagating it, so a single bad message can't take the whole connector down; +any other error stops `consume()` and is returned as-is. + +**`connection_string`'s doc comment** (documentation only) — describe the +exact shape your connector expects instead of the generic example. + +If your destination isn't HTTP, also revisit **`build_raw_client()`**: swap +the `reqwest::Client` for your driver's connection/pool setup (see +`core/connectors/sinks/postgres_sink` or `core/connectors/sinks/s3_sink` for +non-HTTP examples), store it on `TemplateSink` in place of the HTTP-specific +`client` field, and adjust or remove the `check_connectivity_with_retry` call +in `open()` in favor of whatever connectivity check your driver offers. + +## Using it + +1. Copy this directory, rename it and the package in `Cargo.toml` + (`iggy_connector__sink`), and add it to the `members` list in + the workspace root `Cargo.toml`. +2. Fill in the `TODO(ConnectorDeveloper)` section(s). +3. Update `config.toml` with your real `connection_string` and `target`, and + any settings specific to your system; delete `auth_token` if you don't + need it, or add fields of your own the same way (see + `TemplateSinkConfig`). +4. `cargo build --release -p iggy_connector__sink`, point a + runtime connector config file's `path` at the built `.so`/`.dylib`/`.dll`, + and run the connector runtime — see `core/connectors/README.md` in this + repo for the full runtime quick-start. +5. Before opening a PR: `cargo test`, `cargo clippy --all-targets`, + `cargo fmt --check`, and re-read the connector-review checklist once more + with fresh eyes — most review round-trips come from one of the items in + that list, not from the connector-specific logic in `push_batch()`. diff --git a/core/connectors/sinks/sink_template/config.toml b/core/connectors/sinks/sink_template/config.toml new file mode 100644 index 0000000000..86c91de246 --- /dev/null +++ b/core/connectors/sinks/sink_template/config.toml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +type = "sink" +key = "template" +enabled = true +version = 0 +name = "Template sink" +path = "../../target/release/libiggy_connector_template_sink" +plugin_config_format = "toml" +verbose = false + +[[streams]] +stream = "example_stream" +topics = ["example_topic"] +schema = "json" +batch_length = 100 +poll_interval = "5ms" +consumer_group = "template_sink_connector" + +[plugin_config] +connection_string = "https://api.example.com" +target = "events" +health_check_path = "/health" +batch_size = 100 +timeout = "30s" +max_retries = 3 +retry_delay = "500ms" +retry_max_delay = "5s" +max_open_retries = 10 +open_retry_max_delay = "60s" +circuit_breaker_threshold = 5 +circuit_breaker_cool_down = "30s" +verbose_logging = false +# auth_token = "replace-me" # uncomment if your destination needs bearer/API-key auth diff --git a/core/connectors/sinks/sink_template/src/lib.rs b/core/connectors/sinks/sink_template/src/lib.rs new file mode 100644 index 0000000000..0a86bc7338 --- /dev/null +++ b/core/connectors/sinks/sink_template/src/lib.rs @@ -0,0 +1,608 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Template Apache Iggy **sink** connector. +//! +//! Everything in this file is already wired up and follows the patterns the +//! connector-review checklist expects: config validation happens in +//! `open()`, secrets are `SecretString`, the destination identifier is +//! validated before it's ever interpolated into a request, outbound calls +//! go through a retry-wrapped client plus a circuit breaker, and messages +//! are chunked by a configurable batch size instead of shipped as one +//! unbounded request. +//! +//! There is exactly **one** place you need to write code, marked +//! `TODO(ConnectorDeveloper)`: `TemplateSink::push_batch()` — build the +//! request/write that actually pushes one chunk of messages to your +//! destination, using `self.config.connection_string` (and +//! `self.config.target`, if your destination has a table/index/ +//! collection-shaped name). A second `TODO(ConnectorDeveloper)` on the +//! `connection_string` field's doc comment just asks you to describe its +//! expected shape — not code, but worth personalizing too. `grep` for the +//! marker and you'll find both, plus one more on `build_raw_client()` that +//! only applies if your destination isn't HTTP (see below). +//! +//! This template assumes an HTTP-ish destination and uses `reqwest` wrapped +//! by the SDK's retry middleware, because that's what +//! `iggy_connector_sdk::retry` is built for and it covers the common case. +//! If your destination talks something else (a database, a queue, object +//! storage), swap the client type in `build_raw_client()`/`push_batch()` for +//! your driver of choice and lean on its own retry/pooling behavior — keep the +//! surrounding shape (validation in `open()`, circuit breaker, batching, +//! identifier validation) unchanged. See `core/connectors/sinks/postgres_sink` +//! or `core/connectors/sinks/s3_sink` in this repo for non-HTTP examples of +//! that same shape. + +use async_trait::async_trait; +use iggy_connector_sdk::retry::{ + CircuitBreaker, RetryPolicy, build_retry_client, check_connectivity_with_retry, parse_duration, +}; +use iggy_connector_sdk::{ + ConsumedMessage, Error, MessagesMetadata, Sink, TopicMetadata, sink_connector, +}; +use reqwest::Url; +use reqwest_middleware::ClientWithMiddleware; +use secrecy::{ExposeSecret, SecretString}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; +use tokio::sync::Mutex; +use tracing::{debug, error, info, warn}; + +sink_connector!(TemplateSink); + +const CONNECTOR_NAME: &str = "Template sink"; + +const DEFAULT_BATCH_SIZE: usize = 100; +const DEFAULT_TIMEOUT: &str = "30s"; +const DEFAULT_MAX_RETRIES: u32 = 3; +const DEFAULT_RETRY_DELAY: &str = "500ms"; +const DEFAULT_RETRY_MAX_DELAY: &str = "5s"; +const DEFAULT_MAX_OPEN_RETRIES: u32 = 10; +const DEFAULT_OPEN_RETRY_MAX_DELAY: &str = "60s"; +const DEFAULT_CIRCUIT_BREAKER_THRESHOLD: u32 = 5; +const DEFAULT_CIRCUIT_BREAKER_COOL_DOWN: &str = "30s"; + +// ── Configuration ─────────────────────────────────────────────────────────── +// +// Every tunable except `connection_string` and `target` is optional with a +// sane default, and unknown keys are rejected outright so a typo in a TOML +// file fails at load time instead of silently doing nothing. + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemplateSinkConfig { + /// TODO(ConnectorDeveloper): document the exact shape this connector expects, e.g. + /// "" or "postgres://user:pass@host:5432/db". + /// `SecretString` because DSNs commonly embed credentials — never plain + /// `String` for this field, see `PostgresSinkConfig::connection_string` + /// in `sinks/postgres_sink` for the same pattern. + #[serde(serialize_with = "iggy_common::serde_secret::serialize_secret")] + pub connection_string: SecretString, + + /// The destination table/index/collection/bucket name. Kept as its own + /// field (rather than folded into `connection_string`) specifically so + /// it can be validated in `open()` before ever being interpolated into + /// a query, path, or URL — see `validate_identifier` below. Delete this + /// field if your destination has no such dynamic identifier. + pub target: String, + + /// Example of a secret-shaped setting. `SecretString` keeps it out of + /// `Debug`/log output; delete this field if `connection_string` already + /// carries all required auth. Read it with `.expose_secret()` (from the + /// `secrecy::ExposeSecret` trait) at the one place you actually need the + /// plaintext — e.g. when building an auth header in `build_raw_client()`. + #[serde( + default, + serialize_with = "iggy_common::serde_secret::serialize_optional_secret" + )] + pub auth_token: Option, + + /// Optional path (e.g. "/health") probed with retry during `open()` + /// before the connector is considered ready. Leave unset if your target + /// has no health endpoint — the probe is skipped, not failed, in that case. + pub health_check_path: Option, + + pub batch_size: Option, + pub timeout: Option, + pub max_retries: Option, + pub retry_delay: Option, + pub retry_max_delay: Option, + pub max_open_retries: Option, + pub open_retry_max_delay: Option, + pub circuit_breaker_threshold: Option, + pub circuit_breaker_cool_down: Option, + + /// Upgrades the per-batch log line from `debug!` to `info!`. Mirrors the + /// runtime's own `verbose` flag; keep the field name `verbose_logging` + /// (see `postgres_sink::PostgresSinkConfig::verbose_logging`). + pub verbose_logging: Option, +} + +/// Rejects anything that isn't a plain alphanumeric/underscore identifier. +/// Adjust the allowed character set to whatever your destination's naming +/// rules actually are, but always validate *something* before a +/// config-or-message-derived name is interpolated into a query, path, or +/// URL — see `doris_sink::validate_identifier` / `surrealdb_sink::validate_identifier` +/// in this repo for the same pattern applied to a real destination. +fn validate_identifier(field: &str, value: &str) -> Result<(), Error> { + if value.is_empty() || !value.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + return Err(Error::InvalidConfigValue(format!( + "{field} must be non-empty and contain only ASCII alphanumeric characters and \ + underscores, got: {value:?}" + ))); + } + Ok(()) +} + +// ── Internal state ────────────────────────────────────────────────────────── + +#[derive(Debug, Default)] +struct State { + invocations_count: u64, + messages_written: u64, + messages_failed: u64, +} + +#[derive(Debug)] +pub struct TemplateSink { + id: u32, + config: TemplateSinkConfig, + client: Option, + circuit_breaker: Arc, + batch_size_limit: usize, + retry_delay: Duration, + verbose: bool, + state: Mutex, + records_written_total: AtomicU64, +} + +impl TemplateSink { + pub fn new(id: u32, config: TemplateSinkConfig) -> Self { + let retry_delay = parse_duration(config.retry_delay.as_deref(), DEFAULT_RETRY_DELAY); + let circuit_breaker = Arc::new(CircuitBreaker::new( + config + .circuit_breaker_threshold + .unwrap_or(DEFAULT_CIRCUIT_BREAKER_THRESHOLD), + parse_duration( + config.circuit_breaker_cool_down.as_deref(), + DEFAULT_CIRCUIT_BREAKER_COOL_DOWN, + ), + )); + let batch_size_limit = config.batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1); + let verbose = config.verbose_logging.unwrap_or(false); + + Self { + id, + config, + client: None, + circuit_breaker, + batch_size_limit, + retry_delay, + verbose, + state: Mutex::new(State::default()), + records_written_total: AtomicU64::new(0), + } + } + + /// TODO(ConnectorDeveloper): build your actual client/connection here using + /// `self.config.connection_string` (and `self.config.auth_token`, if + /// your destination needs it). This template builds a plain + /// `reqwest::Client` to hand to `build_retry_client` — if you're not + /// talking HTTP, replace this with e.g. a database connection pool or + /// your driver's equivalent, store it on `self` (add a field, since + /// `client` here is HTTP-specific), and skip the + /// `check_connectivity_with_retry` call below in favor of whatever your + /// driver offers (a ping, a test query). + fn build_raw_client(&self) -> Result { + let timeout = parse_duration(self.config.timeout.as_deref(), DEFAULT_TIMEOUT); + reqwest::Client::builder() + .timeout(timeout) + .build() + .map_err(|e| Error::Connection(format!("failed to build HTTP client: {e}"))) + } + + /// TODO(ConnectorDeveloper): push one chunk of already-batched messages to your + /// destination using `self.config.connection_string` and + /// `self.config.target`, via `self.client` (already retry-wrapped). + /// Distinguish permanent failures (bad schema, destination rejects the + /// payload shape — will not succeed on retry) from transient ones + /// (network error, 5xx, timeout — should retry) by returning + /// `Error::PermanentHttpError` for the former; see the doc comment on + /// that variant in `iggy_connector_sdk::Error` for why the distinction + /// matters. `consume()` drops and counts a `PermanentHttpError` batch but + /// keeps processing the rest; any other error stops `consume()` and is + /// returned, which the runtime treats as fatal for the whole connector — + /// so a permanent classification is what keeps one bad message from + /// taking every future batch down with it. + async fn push_batch( + &self, + client: &ClientWithMiddleware, + batch: &[ConsumedMessage], + ) -> Result<(), Error> { + let _ = (client, batch); // remove once implemented + Err(Error::InitError( + "TemplateSink::push_batch is not implemented yet — see the TODO(ConnectorDeveloper) comment in \ + template_sink/src/lib.rs" + .to_string(), + )) + } +} + +// ── Sink trait ──────────────────────────────────────────────────────────────── + +#[async_trait] +impl Sink for TemplateSink { + async fn open(&mut self) -> Result<(), Error> { + // Structural validation happens here, not in `new()`, because only + // `open()` can return an error — `new()` is a plain factory function + // with nowhere to send a "this config is invalid" result. + if self + .config + .connection_string + .expose_secret() + .trim() + .is_empty() + { + return Err(Error::InvalidConfigValue( + "connection_string must not be empty".to_string(), + )); + } + validate_identifier("target", &self.config.target)?; + + info!( + "Opening {CONNECTOR_NAME} connector with ID: {}, target: {}, batch_size: {}", + self.id, self.config.target, self.batch_size_limit + ); + + let raw_client = self.build_raw_client()?; + + if let Some(health_path) = &self.config.health_check_path { + let base = Url::parse(self.config.connection_string.expose_secret()).map_err(|e| { + Error::InvalidConfigValue(format!("connection_string is not a valid URL: {e}")) + })?; + let health_url = base.join(health_path).map_err(|e| { + Error::InvalidConfigValue(format!("invalid health_check_path: {e}")) + })?; + check_connectivity_with_retry( + &raw_client, + health_url, + CONNECTOR_NAME, + self.id, + RetryPolicy { + max_attempts: self + .config + .max_open_retries + .unwrap_or(DEFAULT_MAX_OPEN_RETRIES), + base_delay: self.retry_delay, + max_delay: parse_duration( + self.config.open_retry_max_delay.as_deref(), + DEFAULT_OPEN_RETRY_MAX_DELAY, + ), + }, + ) + .await?; + } else { + warn!( + "{CONNECTOR_NAME} connector with ID: {} has no health_check_path configured — \ + skipping the startup connectivity probe. Consider adding one.", + self.id + ); + } + + self.client = Some(build_retry_client( + raw_client, + self.config + .max_retries + .unwrap_or(DEFAULT_MAX_RETRIES) + .max(1), + self.retry_delay, + parse_duration( + self.config.retry_max_delay.as_deref(), + DEFAULT_RETRY_MAX_DELAY, + ), + CONNECTOR_NAME, + )); + + info!( + "{CONNECTOR_NAME} connector with ID: {} opened successfully", + self.id + ); + Ok(()) + } + + async fn consume( + &self, + topic_metadata: &TopicMetadata, + messages_metadata: MessagesMetadata, + messages: Vec, + ) -> Result<(), Error> { + let mut state = self.state.lock().await; + state.invocations_count += 1; + let invocation = state.invocations_count; + drop(state); + + if self.verbose { + info!( + "{CONNECTOR_NAME} with ID: {} received: {} messages, schema: {}, stream: {}, \ + topic: {}, partition: {}, offset: {}, invocation: {}", + self.id, + messages.len(), + messages_metadata.schema, + topic_metadata.stream, + topic_metadata.topic, + messages_metadata.partition_id, + messages_metadata.current_offset, + invocation + ); + } else { + debug!( + "{CONNECTOR_NAME} with ID: {} received: {} messages, schema: {}, stream: {}, \ + topic: {}, partition: {}, offset: {}, invocation: {}", + self.id, + messages.len(), + messages_metadata.schema, + topic_metadata.stream, + topic_metadata.topic, + messages_metadata.partition_id, + messages_metadata.current_offset, + invocation + ); + } + + if self.circuit_breaker.is_open().await { + warn!( + "{CONNECTOR_NAME} connector with ID: {} — circuit breaker OPEN, refusing {} messages", + self.id, + messages.len() + ); + return Err(Error::CannotStoreData( + "Circuit breaker is open".to_string(), + )); + } + + let client = self.client.as_ref().ok_or_else(|| { + Error::Connection("client not initialized -- was open() called?".into()) + })?; + + // Track the *last* transient error, not the first — an earlier chunk's + // failure may already be stale by the time later chunks run, and the + // circuit breaker below should react to the most recent signal. A + // `PermanentHttpError` batch (bad schema, will never succeed on retry) + // is dropped and counted here rather than kept as `last_err`: letting + // it propagate would return `Err` from `consume()`, and the runtime + // treats any `Err` here as fatal for the whole connector (see + // `runtime/src/sink.rs::consume_messages`), not just a retry of that + // one batch — so a single unprocessable message would take down every + // future batch instead of just being skipped. + let mut last_err: Option = None; + let mut written = 0u64; + let mut failed = 0u64; + + for batch in messages.chunks(self.batch_size_limit) { + match self.push_batch(client, batch).await { + Ok(()) => written += batch.len() as u64, + Err(Error::PermanentHttpError(message)) => { + failed += batch.len() as u64; + error!( + "{CONNECTOR_NAME} connector with ID: {} dropping a batch of {} \ + (permanent): {message}", + self.id, + batch.len() + ); + } + Err(err) => { + failed += batch.len() as u64; + error!( + "{CONNECTOR_NAME} connector with ID: {} failed a batch of {}: {err}", + self.id, + batch.len() + ); + last_err = Some(err); + } + } + } + + // Record the circuit breaker outcome once per `consume()` call, not + // once per chunk — recording success partway through would reset the + // failure counter mid-consume and prevent the breaker from opening on + // a batch with sustained, mixed-success chunks. A transient failure + // always trips it. A batch where every failure was permanent records + // neither success nor failure — nothing demonstrated the destination + // is reachable, but a schema/data problem isn't a connectivity signal + // either. A batch with at least one real write (or no messages at + // all) counts as success. + match &last_err { + Some(_) => self.circuit_breaker.record_failure().await, + None if written > 0 || messages.is_empty() => self.circuit_breaker.record_success(), + None => {} + } + + let mut state = self.state.lock().await; + state.messages_written += written; + state.messages_failed += failed; + drop(state); + self.records_written_total + .fetch_add(written, Ordering::Relaxed); + + match last_err { + None => Ok(()), + Some(err) => Err(err), + } + } + + async fn close(&mut self) -> Result<(), Error> { + let state = self.state.lock().await; + info!( + "{CONNECTOR_NAME} connector with ID: {} closing. Stats: {} invocations, {} messages written, \ + {} messages failed", + self.id, state.invocations_count, state.messages_written, state.messages_failed + ); + drop(state); + self.client = None; + info!("{CONNECTOR_NAME} connector with ID: {} is closed.", self.id); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_config() -> TemplateSinkConfig { + TemplateSinkConfig { + connection_string: SecretString::from("https://api.example.com"), + target: "events".to_string(), + auth_token: None, + health_check_path: None, + batch_size: Some(10), + timeout: Some("5s".to_string()), + max_retries: Some(2), + retry_delay: Some("10ms".to_string()), + retry_max_delay: Some("100ms".to_string()), + max_open_retries: Some(2), + open_retry_max_delay: Some("100ms".to_string()), + circuit_breaker_threshold: Some(3), + circuit_breaker_cool_down: Some("50ms".to_string()), + verbose_logging: None, + } + } + + #[test] + fn given_verbose_logging_enabled_should_set_verbose_flag() { + let mut config = test_config(); + config.verbose_logging = Some(true); + let sink = TemplateSink::new(1, config); + assert!(sink.verbose); + } + + #[test] + fn given_verbose_logging_disabled_should_not_set_verbose_flag() { + let sink = TemplateSink::new(1, test_config()); + assert!(!sink.verbose); + } + + #[tokio::test] + async fn open_rejects_empty_connection_string() { + let mut config = test_config(); + config.connection_string = SecretString::from(" "); + let mut sink = TemplateSink::new(1, config); + assert!(matches!( + sink.open().await, + Err(Error::InvalidConfigValue(_)) + )); + } + + #[tokio::test] + async fn open_rejects_invalid_target_identifier() { + let mut config = test_config(); + config.target = "events; DROP TABLE users;--".to_string(); + let mut sink = TemplateSink::new(1, config); + assert!(matches!( + sink.open().await, + Err(Error::InvalidConfigValue(_)) + )); + } + + #[test] + fn validate_identifier_accepts_plain_names() { + assert!(validate_identifier("target", "events_v2").is_ok()); + } + + #[test] + fn validate_identifier_rejects_empty() { + assert!(validate_identifier("target", "").is_err()); + } + + #[test] + fn validate_identifier_rejects_path_and_query_characters() { + for bad in [ + "../etc/passwd", + "events?x=1", + "events/../secrets", + "events;drop", + ] { + assert!( + validate_identifier("target", bad).is_err(), + "expected {bad:?} to be rejected" + ); + } + } + + #[tokio::test] + async fn consume_short_circuits_when_breaker_is_open() { + let sink = TemplateSink::new(1, test_config()); + sink.circuit_breaker.record_failure().await; + sink.circuit_breaker.record_failure().await; + sink.circuit_breaker.record_failure().await; + assert!(sink.circuit_breaker.is_open().await); + + let (topic_metadata, messages_metadata) = topic_and_messages_metadata(); + + let result = sink + .consume(&topic_metadata, messages_metadata, Vec::new()) + .await; + assert!(matches!(result, Err(Error::CannotStoreData(_)))); + } + + fn consumed_message() -> ConsumedMessage { + ConsumedMessage { + id: 1, + offset: 0, + checksum: 0, + timestamp: 0, + origin_timestamp: 0, + headers: None, + payload: iggy_connector_sdk::Payload::Raw(b"payload".to_vec()), + } + } + + fn topic_and_messages_metadata() -> (TopicMetadata, MessagesMetadata) { + ( + TopicMetadata { + stream: "s".to_string(), + topic: "t".to_string(), + }, + MessagesMetadata { + partition_id: 1, + current_offset: 0, + schema: iggy_connector_sdk::Schema::Json, + }, + ) + } + + // push_batch() is an unimplemented TODO(ConnectorDeveloper) stub that + // always returns Error::InitError — not a PermanentHttpError — so a + // non-empty consume() call exercises the "last_err" (retryable) branch + // of the chunking loop end to end, including the failed-count and + // circuit-breaker bookkeeping. + #[tokio::test] + async fn consume_with_unimplemented_push_batch_returns_err_and_counts_failure() { + let mut sink = TemplateSink::new(1, test_config()); + sink.open().await.expect("open should succeed"); + let (topic_metadata, messages_metadata) = topic_and_messages_metadata(); + + let result = sink + .consume(&topic_metadata, messages_metadata, vec![consumed_message()]) + .await; + + assert!(matches!(result, Err(Error::InitError(_)))); + assert_eq!(sink.state.lock().await.messages_failed, 1); + assert_eq!(sink.state.lock().await.messages_written, 0); + } +} diff --git a/core/connectors/sources/README.md b/core/connectors/sources/README.md index e63f8cb17f..60f2c6d3bc 100644 --- a/core/connectors/sources/README.md +++ b/core/connectors/sources/README.md @@ -13,6 +13,7 @@ Source connectors are responsible for ingesting data from external sources into | **influxdb_source** | Polls InfluxDB with cursor-based timestamp tracking; supports V2 (Flux, annotated CSV) and V3 (SQL, JSONL) | | **postgres_source** | Reads rows from PostgreSQL tables with multiple strategies: delete after read, mark as processed, or timestamp tracking | | **random_source** | Generates random test messages (useful for testing and development) | +| **source_template** | Fill-in-the-blank starting point for a new source; framework/security plumbing done, two `TODO(ConnectorDeveloper)` spots left | The source is represented by the single `Source` trait, which defines the basic interface for all source connectors. It provides methods for initializing the source, reading data from it, and closing the source. diff --git a/core/connectors/sources/source_template/Cargo.toml b/core/connectors/sources/source_template/Cargo.toml new file mode 100644 index 0000000000..dd29dbacd7 --- /dev/null +++ b/core/connectors/sources/source_template/Cargo.toml @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# TEMPLATE — rename the package (and this directory) to +# `iggy_connector__source` before publishing, and update the +# `[[sources]]` entry you add to the workspace root Cargo.toml accordingly. + +[package] +name = "iggy_connector_template_source" +version = "0.1.0" +description = "Template for an Apache Iggy source connector — copy this crate and fill in the TODO sections." +edition = "2024" +license = "Apache-2.0" +keywords = ["iggy", "messaging", "streaming"] +categories = ["command-line-utilities", "database", "network-programming"] +homepage = "https://iggy.apache.org" +documentation = "https://iggy.apache.org/docs" +repository = "https://github.com/apache/iggy" +publish = false + +[package.metadata.cargo-machete] +# dashmap is used only inside the `source_connector!` macro expansion, so a +# naive unused-dependency scan won't see the usage — keep it ignored rather +# than removing it, or the plugin will fail to compile. +ignored = ["dashmap"] + +[lib] +crate-type = ["cdylib", "lib"] + +[dependencies] +async-trait = { workspace = true } +dashmap = { workspace = true } +humantime = { workspace = true } +iggy_common = { workspace = true } +iggy_connector_sdk = { workspace = true } +reqwest = { workspace = true } +reqwest-middleware = { workspace = true } +rmp-serde = { workspace = true } +secrecy = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } diff --git a/core/connectors/sources/source_template/README.md b/core/connectors/sources/source_template/README.md new file mode 100644 index 0000000000..0ae77ac2cc --- /dev/null +++ b/core/connectors/sources/source_template/README.md @@ -0,0 +1,70 @@ +# Template source connector + +Starting point for a new Apache Iggy **source** connector. Everything except +talking to your actual external system is already implemented and follows +the project's required resilience/security patterns — see the module-level +doc comment at the top of `src/lib.rs` for the full rationale, and the +`iggy-connector-review` skill / the "Building Connectors That Pass Review" +blog post for the checklist this template is built against. + +## What's already done for you + +- Config parsing with `#[serde(deny_unknown_fields)]` so a typo in a TOML + file fails loudly instead of silently doing nothing. +- Config validation in `open()` (not `new()`, which has no way to return an + error). +- `connection_string` and the optional `auth_token` field both typed as + `SecretString`, since either can carry credentials. +- A retry-wrapped HTTP client (`iggy_connector_sdk::retry::build_retry_client`) + and a startup connectivity probe with its own backoff + (`check_connectivity_with_retry`). +- A `CircuitBreaker` that's actually consulted before polling and updated + after every attempt — not just constructed and forgotten. +- Cursor staging: `poll()` never commits its progress directly; it stages a + candidate and `on_batch_result()` commits it only on `Ack`, discarding it + on `Nack` so a failed delivery gets re-polled instead of silently lost. +- The `source_connector!` FFI macro invocation and a `Cargo.toml` with the + right `crate-type`, workspace-pinned dependencies, and license header. +- `verbose_logging: Option` upgrading the per-poll log line from + `debug!` to `info!`, mirroring the runtime's own `verbose` flag. +- The six canonical state/Ack-Nack tests, plus config validation and the + `verbose_logging` flag. + +## What you need to fill in + +Search for `TODO(ConnectorDeveloper)` in `src/lib.rs` — there are exactly +three spots: one required, one conditional, one documentation-only. + +1. **`fetch_records()`** (required) — fetch up to `self.batch_size` new + records from your system, ordered after `cursor` (`None` = start from the + beginning, or from "now" — whichever fits your source). Map each result + to a `FetchedRecord { cursor_value, payload }`, using something + monotonically increasing as `cursor_value` (a timestamp, an ID, a page + token) — that's what lets the cursor-staging logic advance correctly. +2. **`build_raw_client()`** (only if your source isn't HTTP) — replace the + `reqwest::Client` construction with your driver's connection/pool setup + (see `core/connectors/sources/postgres_source` for a real non-HTTP + example), store it on `TemplateSource` (you'll need to add a field — + `client: Option` here is HTTP-specific), and adjust + or remove the `check_connectivity_with_retry` call in `open()` in favor + of whatever connectivity check your driver offers. +3. **`connection_string`'s doc comment** (documentation only) — describe + the exact shape your connector expects instead of the generic example. + +## Using it + +1. Copy this directory, rename it and the package in `Cargo.toml` + (`iggy_connector__source`), and add it to the `members` list in + the workspace root `Cargo.toml`. +2. Fill in `fetch_records()` (and `build_raw_client()` if not HTTP). +3. Update `config.toml` with your real `connection_string` and any + settings specific to your system; delete `auth_token` if you don't need + it, or add fields of your own the same way (see `TemplateSourceConfig`). +4. `cargo build --release -p iggy_connector__source`, point a + runtime connector config file's `path` at the built `.so`/`.dylib`/`.dll`, + and run the connector runtime — see `core/connectors/README.md` in this + repo for the full runtime quick-start. +5. Before opening a PR: `cargo test`, `cargo clippy --all-targets`, + `cargo fmt --check`, and re-read the connector-review checklist once more + with fresh eyes — most review round-trips come from one of the items in + that list, not from the connector-specific logic in `fetch_records()`. diff --git a/core/connectors/sources/source_template/config.toml b/core/connectors/sources/source_template/config.toml new file mode 100644 index 0000000000..721abb0464 --- /dev/null +++ b/core/connectors/sources/source_template/config.toml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +type = "source" +key = "template" +enabled = true +version = 0 +name = "Template source" +path = "../../target/release/libiggy_connector_template_source" +plugin_config_format = "toml" +verbose = false + +[[streams]] +stream = "example_stream" +topic = "example_topic" +schema = "json" +batch_length = 100 +linger_time = "5ms" + +[plugin_config] +connection_string = "https://api.example.com" +health_check_path = "/health" +batch_size = 100 +poll_interval = "1s" +timeout = "30s" +max_retries = 3 +retry_delay = "500ms" +retry_max_delay = "5s" +max_open_retries = 10 +open_retry_max_delay = "60s" +circuit_breaker_threshold = 5 +circuit_breaker_cool_down = "30s" +verbose_logging = false +# auth_token = "replace-me" # uncomment if your source needs bearer/API-key auth diff --git a/core/connectors/sources/source_template/src/lib.rs b/core/connectors/sources/source_template/src/lib.rs new file mode 100644 index 0000000000..430e7e13ac --- /dev/null +++ b/core/connectors/sources/source_template/src/lib.rs @@ -0,0 +1,639 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Template Apache Iggy **source** connector. +//! +//! Everything in this file is already wired up and follows the patterns the +//! connector-review checklist expects: config validation happens in +//! `open()`, secrets are `SecretString`, outbound calls go through a +//! retry-wrapped client plus a circuit breaker, and the read cursor is +//! staged in `poll()` and only committed in `on_batch_result()` on an ACK so +//! a dropped/nacked batch can be re-polled instead of silently lost. +//! +//! There is exactly **one** place you need to write code, marked +//! `TODO(ConnectorDeveloper)`: `TemplateSource::fetch_records()` — fetch up +//! to `batch_size` new records from your external system, starting after +//! `cursor`. `TemplateSource::build_raw_client()` carries a second +//! `TODO(ConnectorDeveloper)` too, but only applies if your source isn't +//! HTTP (see below); a third, on the `connection_string` field's doc +//! comment, just asks you to describe its expected shape — not code. +//! `grep` for the marker and you'll find all three. +//! +//! This template assumes an HTTP-ish source and uses `reqwest` wrapped by +//! the SDK's retry middleware, because that's what `iggy_connector_sdk::retry` +//! is built for and it covers the common case. If your source talks to +//! something else (a database, a queue, a filesystem), swap the client type +//! in `build_raw_client()`/`fetch_records()` for your driver of choice and lean on +//! its own retry/pooling behavior — keep the surrounding shape (validation +//! in `open()`, circuit breaker, cursor staging, batching) unchanged. See +//! `core/connectors/sources/postgres_source` in this repo for a real +//! non-HTTP example of that same shape. + +use async_trait::async_trait; +use iggy_connector_sdk::retry::{ + CircuitBreaker, RetryPolicy, build_retry_client, check_connectivity_with_retry, parse_duration, +}; +use iggy_connector_sdk::{ + ConnectorState, Error, ProducedMessage, ProducedMessages, Schema, Source, + source::SourceBatchResult, source_connector, +}; +use reqwest::Url; +use reqwest_middleware::ClientWithMiddleware; +use secrecy::{ExposeSecret, SecretString}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; +use tokio::sync::Mutex; +use tracing::{debug, error, info, warn}; + +source_connector!(TemplateSource); + +const CONNECTOR_NAME: &str = "Template source"; + +const DEFAULT_POLL_INTERVAL: &str = "1s"; +const DEFAULT_BATCH_SIZE: u32 = 100; +const DEFAULT_TIMEOUT: &str = "30s"; +const DEFAULT_MAX_RETRIES: u32 = 3; +const DEFAULT_RETRY_DELAY: &str = "500ms"; +const DEFAULT_RETRY_MAX_DELAY: &str = "5s"; +const DEFAULT_MAX_OPEN_RETRIES: u32 = 10; +const DEFAULT_OPEN_RETRY_MAX_DELAY: &str = "60s"; +const DEFAULT_CIRCUIT_BREAKER_THRESHOLD: u32 = 5; +const DEFAULT_CIRCUIT_BREAKER_COOL_DOWN: &str = "30s"; + +// ── Configuration ─────────────────────────────────────────────────────────── +// +// Every tunable except `connection_string` is optional with a sane default, +// and unknown keys are rejected outright so a typo in a TOML file fails at +// load time instead of silently doing nothing. + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemplateSourceConfig { + /// TODO(ConnectorDeveloper): document the exact shape this connector expects, e.g. + /// "" or "postgres://user:pass@host:5432/db". + /// `SecretString` because DSNs commonly embed credentials — never plain + /// `String` for this field, see `PostgresSinkConfig::connection_string` + /// in `sinks/postgres_sink` for the same pattern. + #[serde(serialize_with = "iggy_common::serde_secret::serialize_secret")] + pub connection_string: SecretString, + + /// Example of a secret-shaped setting. `SecretString` keeps it out of + /// `Debug`/log output; delete this field if `connection_string` already + /// carries all required auth. Read it with `.expose_secret()` (from the + /// `secrecy::ExposeSecret` trait) at the one place you actually need the + /// plaintext — e.g. when building an auth header in `build_raw_client()`. + #[serde( + default, + serialize_with = "iggy_common::serde_secret::serialize_optional_secret" + )] + pub auth_token: Option, + + /// Optional path (e.g. "/health") probed with retry during `open()` + /// before the connector is considered ready. Leave unset if your target + /// has no health endpoint — the probe is skipped, not failed, in that case. + pub health_check_path: Option, + + pub batch_size: Option, + pub poll_interval: Option, + pub timeout: Option, + pub max_retries: Option, + pub retry_delay: Option, + pub retry_max_delay: Option, + pub max_open_retries: Option, + pub open_retry_max_delay: Option, + pub circuit_breaker_threshold: Option, + pub circuit_breaker_cool_down: Option, + + /// Upgrades the per-poll log line from `debug!` to `info!`. Mirrors the + /// runtime's own `verbose` flag; keep the field name `verbose_logging` + /// (see `postgres_source::PostgresSourceConfig::verbose_logging`). + pub verbose_logging: Option, +} + +// ── Internal state ────────────────────────────────────────────────────────── + +/// Read cursor. Kept as a plain `Option` so it fits whatever ordering +/// field your source uses (a timestamp, an auto-increment ID, an opaque +/// pagination token, ...) — stringify it however makes sense for your data. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +struct State { + cursor: Option, +} + +/// One record as fetched from the external system, before it's turned into +/// an Iggy message. Replace the `payload` type with whatever your +/// `fetch_records()` actually produces. +struct FetchedRecord { + /// The value `cursor` should advance to once this record's batch is + /// acknowledged — typically this record's timestamp/ID/token. + cursor_value: String, + payload: serde_json::Value, +} + +#[derive(Debug)] +pub struct TemplateSource { + id: u32, + config: TemplateSourceConfig, + client: Option, + circuit_breaker: Arc, + batch_size: u32, + poll_interval: Duration, + retry_delay: Duration, + verbose: bool, + state: Mutex, + pending_state: Mutex>, + records_produced: AtomicU64, +} + +impl TemplateSource { + pub fn new(id: u32, config: TemplateSourceConfig, state: Option) -> Self { + let poll_interval = *humantime::Duration::from_str_lossy( + config + .poll_interval + .as_deref() + .unwrap_or(DEFAULT_POLL_INTERVAL), + ); + let retry_delay = parse_duration(config.retry_delay.as_deref(), DEFAULT_RETRY_DELAY); + let circuit_breaker = Arc::new(CircuitBreaker::new( + config + .circuit_breaker_threshold + .unwrap_or(DEFAULT_CIRCUIT_BREAKER_THRESHOLD), + parse_duration( + config.circuit_breaker_cool_down.as_deref(), + DEFAULT_CIRCUIT_BREAKER_COOL_DOWN, + ), + )); + let batch_size = config.batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1); + let verbose = config.verbose_logging.unwrap_or(false); + + let restored_state = state + .and_then(|s| s.deserialize::(CONNECTOR_NAME, id)) + .inspect(|s| { + info!( + "Restored state for {CONNECTOR_NAME} connector with ID: {id}. Cursor: {:?}", + s.cursor + ); + }); + + Self { + id, + config, + client: None, + circuit_breaker, + batch_size, + poll_interval, + retry_delay, + verbose, + state: Mutex::new(restored_state.unwrap_or_default()), + pending_state: Mutex::new(None), + records_produced: AtomicU64::new(0), + } + } + + fn serialize_state(&self, state: &State) -> Option { + ConnectorState::serialize(state, CONNECTOR_NAME, self.id) + } + + /// TODO(ConnectorDeveloper): build your actual client/connection here using + /// `self.config.connection_string` (and `self.config.auth_token`, if + /// your system needs it). This template builds a plain `reqwest::Client` + /// to hand to `build_retry_client` — if you're not talking HTTP, replace + /// this with e.g. a `sqlx::PgPool::connect(...)` or your driver's + /// equivalent, store it on `self` (add a field, since `client` here is + /// HTTP-specific), and skip the `check_connectivity_with_retry` call + /// below in favor of whatever your driver offers (a ping, a test query). + fn build_raw_client(&self) -> Result { + let timeout = parse_duration(self.config.timeout.as_deref(), DEFAULT_TIMEOUT); + reqwest::Client::builder() + .timeout(timeout) + .build() + .map_err(|e| Error::Connection(format!("failed to build HTTP client: {e}"))) + } + + /// TODO(ConnectorDeveloper): fetch up to `self.batch_size` new records from your + /// external system, ordered after `cursor` (`None` means "from the + /// beginning" or "from now" — whichever is right for your source). + /// Use `self.config.connection_string` as the base address and + /// `self.client` (already retry-wrapped) to make the request. Map each + /// result row/document/event to a `FetchedRecord`, using something that + /// monotonically increases (a timestamp, an ID, a page token) as + /// `cursor_value` so the state-staging logic in `poll()` below can + /// advance the cursor correctly. + async fn fetch_records( + &self, + client: &ClientWithMiddleware, + cursor: Option<&str>, + ) -> Result, Error> { + let _ = (client, cursor); // remove once implemented + Err(Error::InitError( + "TemplateSource::fetch_records is not implemented yet — see the TODO(ConnectorDeveloper) comment \ + in template_source/src/lib.rs" + .to_string(), + )) + } +} + +// ── Source trait ──────────────────────────────────────────────────────────── + +#[async_trait] +impl Source for TemplateSource { + async fn open(&mut self) -> Result<(), Error> { + // Structural validation happens here, not in `new()`, because only + // `open()` can return an error — `new()` is a plain factory function + // with nowhere to send a "this config is invalid" result. + if self + .config + .connection_string + .expose_secret() + .trim() + .is_empty() + { + return Err(Error::InvalidConfigValue( + "connection_string must not be empty".to_string(), + )); + } + + info!( + "Opening {CONNECTOR_NAME} connector with ID: {}, batch_size: {}, poll_interval: {:?}", + self.id, self.batch_size, self.poll_interval + ); + + let raw_client = self.build_raw_client()?; + + if let Some(health_path) = &self.config.health_check_path { + let base = Url::parse(self.config.connection_string.expose_secret()).map_err(|e| { + Error::InvalidConfigValue(format!("connection_string is not a valid URL: {e}")) + })?; + let health_url = base.join(health_path).map_err(|e| { + Error::InvalidConfigValue(format!("invalid health_check_path: {e}")) + })?; + check_connectivity_with_retry( + &raw_client, + health_url, + CONNECTOR_NAME, + self.id, + RetryPolicy { + max_attempts: self + .config + .max_open_retries + .unwrap_or(DEFAULT_MAX_OPEN_RETRIES), + base_delay: self.retry_delay, + max_delay: parse_duration( + self.config.open_retry_max_delay.as_deref(), + DEFAULT_OPEN_RETRY_MAX_DELAY, + ), + }, + ) + .await?; + } else { + warn!( + "{CONNECTOR_NAME} connector with ID: {} has no health_check_path configured — \ + skipping the startup connectivity probe. Consider adding one.", + self.id + ); + } + + self.client = Some(build_retry_client( + raw_client, + self.config + .max_retries + .unwrap_or(DEFAULT_MAX_RETRIES) + .max(1), + self.retry_delay, + parse_duration( + self.config.retry_max_delay.as_deref(), + DEFAULT_RETRY_MAX_DELAY, + ), + CONNECTOR_NAME, + )); + + info!( + "{CONNECTOR_NAME} connector with ID: {} opened successfully", + self.id + ); + Ok(()) + } + + async fn poll(&self) -> Result { + // If the breaker is open, sleep for the normal poll interval and + // return an empty (not an error) result. Returning `Err` here would + // make the runtime retry `poll()` again immediately with no delay — + // see `handle_messages` in the SDK's source container — so an empty + // ACK-free result is how a source waits out a known-bad window + // without busy-looping or counting against the NACK budget. + if self.circuit_breaker.is_open().await { + warn!( + "{CONNECTOR_NAME} connector with ID: {} — circuit breaker OPEN, skipping poll", + self.id + ); + tokio::time::sleep(self.poll_interval).await; + return Ok(ProducedMessages { + schema: Schema::Json, + messages: Vec::new(), + state: None, + }); + } + tokio::time::sleep(self.poll_interval).await; + + let client = self.client.as_ref().ok_or_else(|| { + Error::Connection("client not initialized -- was open() called?".into()) + })?; + let cursor = self.state.lock().await.cursor.clone(); + + let records = match self.fetch_records(client, cursor.as_deref()).await { + Ok(records) => { + self.circuit_breaker.record_success(); + records + } + Err(err) => { + if !matches!(err, Error::PermanentHttpError(_)) { + self.circuit_breaker.record_failure().await; + } + return Err(err); + } + }; + + if records.is_empty() { + return Ok(ProducedMessages { + schema: Schema::Json, + messages: Vec::new(), + state: None, + }); + } + + let mut messages = Vec::with_capacity(records.len()); + let mut new_cursor = cursor; + for record in records { + let Ok(payload) = serde_json::to_vec(&record.payload) else { + error!( + "Failed to serialize a record fetched by {CONNECTOR_NAME} connector with ID: {}", + self.id + ); + continue; + }; + // Only advance the candidate cursor once the record actually made it + // into `messages` - advancing on a dropped (unserializable) record + // would stage a cursor past data that was never produced, and a + // later Ack would commit past it permanently. + new_cursor = Some(record.cursor_value); + messages.push(ProducedMessage { + id: None, + headers: None, + checksum: None, + timestamp: None, + origin_timestamp: None, + payload, + }); + } + + if messages.is_empty() { + // Every fetched record failed to serialize - no progress was + // made, so there's nothing to stage and no reason to write + // state, same as the `records.is_empty()` case above. + return Ok(ProducedMessages { + schema: Schema::Json, + messages: Vec::new(), + state: None, + }); + } + + let candidate_state = State { cursor: new_cursor }; + let persisted_state = self.serialize_state(&candidate_state).ok_or_else(|| { + Error::Serialization(format!( + "failed to serialize state for {CONNECTOR_NAME} connector with ID: {}", + self.id + )) + })?; + *self.pending_state.lock().await = Some(candidate_state); + + self.records_produced + .fetch_add(messages.len() as u64, Ordering::Relaxed); + + if self.verbose { + info!( + "{CONNECTOR_NAME} connector with ID: {} produced {} messages", + self.id, + messages.len() + ); + } else { + debug!( + "{CONNECTOR_NAME} connector with ID: {} produced {} messages", + self.id, + messages.len() + ); + } + + Ok(ProducedMessages { + schema: Schema::Json, + messages, + state: Some(persisted_state), + }) + } + + /// The staged cursor from `poll()` is only committed here, and only on + /// an ACK. A NACK (delivery failed, batch timed out, runtime is + /// shutting down) discards the candidate so the same range is re-polled + /// next time — the cursor never moves past data that wasn't confirmed + /// delivered. If `fetch_records()` ever needs to perform a destructive + /// read against the source (delete-after-read, mark-as-processed), + /// stage that side effect the same way and only apply it here on `Ack`. + async fn on_batch_result(&self, result: SourceBatchResult) -> Result<(), Error> { + let candidate_state = self.pending_state.lock().await.take(); + if result == SourceBatchResult::Ack + && let Some(candidate_state) = candidate_state + { + *self.state.lock().await = candidate_state; + } + Ok(()) + } + + async fn close(&mut self) -> Result<(), Error> { + let state = self.state.lock().await; + info!( + "{CONNECTOR_NAME} connector with ID: {} closed. Cursor: {:?}, total records produced: {}", + self.id, + state.cursor, + self.records_produced.load(Ordering::Relaxed) + ); + drop(state); + self.client = None; + Ok(()) + } +} + +// Small local shim so `new()` doesn't need to pull in `humantime::Duration`'s +// `FromStr` (which returns `Result`) just to apply a default — mirrors the +// fallback-with-warning behavior of `iggy_connector_sdk::retry::parse_duration` +// for the one duration field (`poll_interval`) that isn't itself optional in +// spirit (there's always a poll interval, just maybe the default one). +trait DurationExt { + fn from_str_lossy(s: &str) -> humantime::Duration; +} +impl DurationExt for humantime::Duration { + fn from_str_lossy(s: &str) -> humantime::Duration { + use std::str::FromStr; + humantime::Duration::from_str(s).unwrap_or_else(|_| { + warn!("Invalid poll_interval {s:?}, falling back to {DEFAULT_POLL_INTERVAL}"); + humantime::Duration::from_str(DEFAULT_POLL_INTERVAL) + .expect("DEFAULT_POLL_INTERVAL must itself be a valid duration literal") + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_config() -> TemplateSourceConfig { + TemplateSourceConfig { + connection_string: SecretString::from("https://api.example.com"), + auth_token: None, + health_check_path: None, + batch_size: Some(50), + poll_interval: Some("50ms".to_string()), + timeout: Some("5s".to_string()), + max_retries: Some(2), + retry_delay: Some("10ms".to_string()), + retry_max_delay: Some("100ms".to_string()), + max_open_retries: Some(2), + open_retry_max_delay: Some("100ms".to_string()), + circuit_breaker_threshold: Some(3), + circuit_breaker_cool_down: Some("50ms".to_string()), + verbose_logging: None, + } + } + + #[test] + fn given_verbose_logging_enabled_should_set_verbose_flag() { + let mut config = test_config(); + config.verbose_logging = Some(true); + let source = TemplateSource::new(1, config, None); + assert!(source.verbose); + } + + #[test] + fn given_verbose_logging_disabled_should_not_set_verbose_flag() { + let source = TemplateSource::new(1, test_config(), None); + assert!(!source.verbose); + } + + #[tokio::test] + async fn open_rejects_empty_connection_string() { + let mut config = test_config(); + config.connection_string = SecretString::from(" "); + let mut source = TemplateSource::new(1, config, None); + let result = source.open().await; + assert!(matches!(result, Err(Error::InvalidConfigValue(_)))); + } + + #[tokio::test] + async fn given_no_state_should_start_with_no_cursor() { + let source = TemplateSource::new(1, test_config(), None); + assert_eq!(source.state.lock().await.cursor, None); + } + + #[tokio::test] + async fn given_persisted_state_should_restore_cursor() { + let state = State { + cursor: Some("2024-01-01T00:00:00Z".to_string()), + }; + let serialized = rmp_serde::to_vec(&state).expect("failed to serialize state"); + let source = TemplateSource::new(1, test_config(), Some(ConnectorState(serialized))); + assert_eq!( + source.state.lock().await.cursor, + Some("2024-01-01T00:00:00Z".to_string()) + ); + } + + #[tokio::test] + async fn given_invalid_persisted_state_should_start_fresh() { + let source = TemplateSource::new( + 1, + test_config(), + Some(ConnectorState(b"not valid msgpack".to_vec())), + ); + assert_eq!(source.state.lock().await.cursor, None); + } + + #[test] + fn state_should_be_serializable_and_deserializable() { + let original = State { + cursor: Some("2024-01-01T00:00:00Z".to_string()), + }; + + let serialized = rmp_serde::to_vec(&original).expect("failed to serialize state"); + let deserialized: State = + rmp_serde::from_slice(&serialized).expect("failed to deserialize state"); + + assert_eq!(original.cursor, deserialized.cursor); + } + + #[tokio::test] + async fn given_ack_should_commit_staged_cursor() { + let source = TemplateSource::new(1, test_config(), None); + *source.pending_state.lock().await = Some(State { + cursor: Some("next-cursor".to_string()), + }); + + source + .on_batch_result(SourceBatchResult::Ack) + .await + .expect("ACK should be applied"); + + assert_eq!( + source.state.lock().await.cursor, + Some("next-cursor".to_string()) + ); + assert!(source.pending_state.lock().await.is_none()); + } + + #[tokio::test] + async fn given_nack_should_discard_staged_cursor() { + let source = TemplateSource::new(1, test_config(), None); + *source.pending_state.lock().await = Some(State { + cursor: Some("next-cursor".to_string()), + }); + + source + .on_batch_result(SourceBatchResult::Nack) + .await + .expect("NACK should be applied"); + + // The committed cursor is unchanged (still None) — the candidate is + // simply discarded so the same range is polled again. + assert_eq!(source.state.lock().await.cursor, None); + assert!(source.pending_state.lock().await.is_none()); + } + + #[tokio::test(start_paused = true)] + async fn poll_returns_empty_without_error_when_circuit_is_open() { + let source = TemplateSource::new(1, test_config(), None); + source.circuit_breaker.record_failure().await; + source.circuit_breaker.record_failure().await; + source.circuit_breaker.record_failure().await; + assert!(source.circuit_breaker.is_open().await); + + let result = source + .poll() + .await + .expect("open breaker should not error poll()"); + assert!(result.messages.is_empty()); + assert!(result.state.is_none()); + } +} From 0c338c5a7126ebf5d7c8c25a85adf97754084a53 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 10 Oct 2026 20:51:25 -0400 Subject: [PATCH 2/5] Fixed couple of trivial issues --- .claude/skills/connector-pr-review/SKILL.md | 130 +++++++++++++++++- .claude/skills/connector-testing/SKILL.md | 19 ++- core/connectors/sinks/sink_template/README.md | 25 ++++ .../connectors/sinks/sink_template/src/lib.rs | 72 ++++++++-- .../sources/source_template/README.md | 27 ++++ .../sources/source_template/src/lib.rs | 40 +++++- 6 files changed, 289 insertions(+), 24 deletions(-) diff --git a/.claude/skills/connector-pr-review/SKILL.md b/.claude/skills/connector-pr-review/SKILL.md index 4725fcd6ee..0ae07f3cd1 100644 --- a/.claude/skills/connector-pr-review/SKILL.md +++ b/.claude/skills/connector-pr-review/SKILL.md @@ -136,6 +136,75 @@ that state on success, then calls `on_batch_result(Ack | Nack)`. Only re-emit an entire index without calling that out in README. - [ ] Config flag combos that no-op should `warn!` or `Err`, not silently ignore. +### B9. Zero delivered is a failure, even if the call "succeeded" + +A bulk/batch API call returning 200 does not mean every row in it landed. +Recurring, independently-discovered bug across four different sinks: + +- [ ] If your destination's write API can partially or fully reject a batch + while the call itself succeeds (HTTP 200 with per-item errors, a bulk + index response, a stream-load response), count accepted items and + return `Err` when the accepted count is **zero** for a non-empty input — + never `Ok(())`. BigQuery (#4394): "when every row of a non-empty batch + is rejected, `consume` still returns `Ok`." Elasticsearch (#4154): bulk + index knows the rejected count but still returns `Ok`. Doris (#3215): + "if every message in the batch is non-JSON... `consume()` returns + `Ok(())`" with the batch silently gone. S3 (#3103): a formatter failure + produced an empty placeholder that was still counted as uploaded. +- [ ] A per-record encode/serialize failure must not silently substitute an + empty/null/default value that then gets counted as delivered — skip + and count it as lost (visibly), or fail the whole chunk. + +### B10. A wrapping timeout does not cancel in-flight work + +`tokio::time::timeout(dur, fut)` dropping `fut` on expiry does **not** mean +the operation didn't happen — for a blocking-executor call, an FFI call, or +anything already past the "no going back" point in the wire protocol, the +work keeps running after the timeout fires and can still complete +server-side. Confirmed independently on two unrelated backends: + +- [ ] Treat an expired wrapping timeout as **unknown outcome**, not + **didn't happen** — do not assume it's safe to retry the identical + operation without a dedup/idempotency key. RabbitMQ (#3973): "Wrapping + `basic_publish` in `timeout()` does not make it abortable... usually + already delivered." Postgres (#3957): "`timeout` drops the client + future, it does not abort the statement — a DELETE... can still commit + server-side after this logs a failure." +- [ ] Prefer a protocol/transport-level timeout over (or in addition to) a + wrapping `tokio::time::timeout`: `reqwest::ClientBuilder::timeout`, + a connection-string timeout parameter (`?connection_timeout=` for AMQP), + or a driver's own `statement_timeout`. A bare `TcpStream::connect` run + on a blocking executor is not cancelled by an outer async timeout and + leaks a pool thread until the kernel's own SYN timeout (RabbitMQ #3973). +- [ ] This is why `build_raw_client()` in the sink/source templates sets + `.timeout()` on the `reqwest::Client` itself rather than wrapping calls + — copy that pattern, don't wrap instead of it. + +### B11. Ack-path work must fit inside the SDK's fixed batch-result window + +If `on_batch_result` (or any post-ack cleanup) does real I/O — not just an +in-memory state swap — that I/O runs inside the SDK's ~30s batch-result +timeout, and a NACK from that timeout is often **fatal** (stops the source +for good after repeated consecutive NACKs), not a retry signal. + +- [ ] Any retry/timeout budget inside `on_batch_result` must be sized well + under the SDK's window, and **multiplied by how many operations run + inside it** — not assumed to be one shared budget. Postgres (#3957, + critical): "10s is per operation, not per batch... N tables burn + N\*10s inside the SDK's 30s window." +- [ ] User-configurable `max_retries`/`retry_delay` can produce a wait time + that exceeds a hardcoded internal deadline — validate the combination + in `open()` and reject configs that cannot fit, rather than letting + the ack path blow its budget at runtime. Same PR: "`max_retries=3` + with `retry_delay=\"2s\"`, both straight from the readme, needs 12s + and fails startup." +- [ ] A failed destructive/cleanup op inside `on_batch_result` must not be + silently logged and ignored while the batch is still ACKed — count + consecutive failures and return `Err` past a threshold so the SDK + stops the source, rather than resending the same broken operation + forever. Same PR: "a failed slot advance only logs. the batch is + already ACKed, so nothing stops it." + ## High-frequency convention nits These are "cheap" but burn full review rounds when missed. @@ -164,25 +233,66 @@ These are "cheap" but burn full review rounds when missed. - [ ] New fields: `Option` + `#[serde(default)]` where needed. - [ ] Prefer `#[serde(deny_unknown_fields)]` on plugin config so typo’d knobs fail loud. +- [ ] Reject degenerate numeric values (`0`, empty string) for a field where + zero makes a check always-true or always-false, rather than letting it + silently misbehave — `batch_size`/`max_messages_per_file` zero + triggered S3 (#3103) to rotate on every single message. If a sibling + connector already clamps a field like this (`.max(1)`), clamp + consistently rather than leaving one sibling to reject what another + silently accepts. +- [ ] Never derive a value that must stay identical across a retry of the + *same* logical batch (a destination key, filename, or dedup id) from + wall-clock time (`Utc::now()`) — derive it from the batch's own fixed + fields (an offset, a message id, `origin_timestamp`). S3 (#3103): a + `{timestamp}` path placeholder rendered at upload time broke the + README's own idempotent-retry claim. ### C3. Crate / path / docs checklist - [ ] `[lib] crate-type = ["cdylib", "lib"]`. +- [ ] `publish = false` in `[package]` — every in-tree plugin needs it, and + it's easy to forget on a new crate (missed on S3 #3103 and Doris + #3215, both caught in review). - [ ] Example TOML plugin path uses `../../target/release/lib…` like siblings. - [ ] Row added to `sinks/README.md` or `sources/README.md`. - [ ] Sample under `runtime/example_config/connectors/`. - [ ] README defaults **byte-equal** to consts in code (diff them). - [ ] No links to non-existent docs. +- [ ] Helper modules stay crate-private (no `pub mod`) unless genuinely part + of the plugin's public API — S3 (#3103) exposed `buffer`/`client`/etc. + as public on a cdylib plugin where every sibling keeps them private. ### C4. Hot path - [ ] No `payload.clone().try_to_bytes()` — use `try_to_bytes(&self)`. - [ ] `Vec::with_capacity(n)` for per-batch buffers. - [ ] No `std::sync::Mutex` across `.await`; use `tokio::sync::Mutex`. +- [ ] Even a `tokio::sync::Mutex` must not be **held across** a slow network + call or a retry sleep — lock, clone/extract what you need, drop the + guard, *then* do I/O. This isn't just a deadlock rule: holding the + lock across the call serializes every concurrent caller sharing that + plugin instance onto one upload/write at a time, killing the + parallelism the runtime's per-partition tasks were meant to give you. + S3 (#3103): one `buffers.lock().await` held through `put_object()` + *and* the retry sleep capped the whole sink's throughput to one + upload in flight regardless of partition count. - [ ] `&self` on `consume` / `poll` (interior mutability only). - [ ] No `tokio::spawn` inside plugin code. - [ ] No `unwrap()`/`expect()` on external I/O outside tests. - [ ] No eager `format!` around tracing args. +- [ ] Never `let _ = fallible_call()` on a side effect that matters (a + cleanup op, a quoted identifier, a staged delete) — log or propagate + the error. Postgres (#3957): `let _` on a staged-operation error made + an always-failing delete invisible. +- [ ] If retrying raw HTTP status codes (not just SDK `Error` variants), + classify by family — 5xx, 408, 429 retry; other 4xx fail fast. S3 + (#3103) retried `AccessDenied`/`NoSuchBucket`/`InvalidBucketName` + three times each, burning the whole retry budget on failures no retry + could fix. +- [ ] Never log a substring, prefix, or fragment of a credential "for + diagnostics" — it still identifies the credential type/shape for zero + diagnostic value. S3 (#3103) logged the first 3 characters of an AWS + access key (`AKIA`/`ASIA`/`AROA` prefix). ### C5. Containers / fixtures @@ -238,10 +348,22 @@ Paste into the PR description (or run mentally before `/ready`): Recurring comments mined from connector PRs including (non-exhaustive): SurrealDB sink (#3453), Meilisearch sink/source (#3497/#3498), OpenSearch source (#3515), Quickwit convention (#3523), MySQL source (#3568), JDBC -source (#3588), Doris retry (#3574), Redshift sink (#3654), Airflow trigger -(#3716), Fluss sink (#3782). Highest-density themes: delivery/offset -semantics, idempotency IDs, transient/permanent mapping, config-name drift, -secrets, README/code drift, false-green tests, CI/lockfile hygiene. +source (#3588), Doris retry (#3574) and sink review (#3215), Redshift sink +(#3654), Airflow trigger (#3716), Fluss sink (#3782), S3 sink (#3103), +Google BigQuery sink (#4394), Delta sink production hardening (#4219), +RabbitMQ sink (#3973), Elasticsearch sink batch semantics (#4154), +Postgres source ACK/NACK + CDC hardening (#3957), the shared +`retry_async`/`RetryPolicy` SDK helper review (#4104). Highest-density +themes: delivery/offset semantics, idempotency IDs, transient/permanent +mapping, config-name drift, secrets, README/code drift, false-green tests, +CI/lockfile hygiene, zero-delivered-still-reports-Ok, a wrapping timeout +that doesn't cancel in-flight work, ack-path work blowing its fixed budget, +and holding a lock across slow I/O. + +Re-derive or extend this list with `gh pr list --repo apache/iggy --search +"connectors in:title" --state all --limit 100` plus `gh api +repos/apache/iggy/pulls//comments --paginate` for PRs under +`core/connectors/`. --- diff --git a/.claude/skills/connector-testing/SKILL.md b/.claude/skills/connector-testing/SKILL.md index 9e6669e057..fcbbbab75a 100644 --- a/.claude/skills/connector-testing/SKILL.md +++ b/.claude/skills/connector-testing/SKILL.md @@ -100,18 +100,23 @@ fn test_config() -> RandomSourceConfig { ### Async testing pattern -The trait methods are `async`. For unit tests, build a runtime locally - don't add `#[tokio::test]` in `src/lib.rs` (it pulls test macros into prod build): +The trait methods are `async`. Prefer `#[tokio::test]` directly on the test fn - `#[cfg(test)]` +gates the whole module out of a plain `cargo build`, and resolver v2 (default since edition 2021, +which this workspace uses) keeps `tokio`'s `test-util`/`macros` dev-dependency features out of that +build too, so there's no prod-build leak to avoid. This is also the dominant pattern in-tree +(`postgres_source::tests` uses `#[tokio::test]` throughout, no manual runtime): ```rust -#[test] -fn given_X_should_Y() { - let runtime = tokio::runtime::Runtime::new().unwrap(); - runtime.block_on(async { - // await calls here - }); +#[tokio::test] +async fn given_x_should_y() { + // await calls directly, no manual Runtime::new() } ``` +A manual `tokio::runtime::Runtime::new().unwrap().block_on(...)` inside a plain `#[test]` still +works and you'll see it in some older plugin tests (`random_source`), but don't ask for it in +review - it's equivalent, not a correctness requirement. + ### Mandatory: four canonical source state tests Every source must have these. Copy from `sources/random_source/src/lib.rs::tests`: diff --git a/core/connectors/sinks/sink_template/README.md b/core/connectors/sinks/sink_template/README.md index 86da056d81..5924b3422f 100644 --- a/core/connectors/sinks/sink_template/README.md +++ b/core/connectors/sinks/sink_template/README.md @@ -7,6 +7,31 @@ doc comment at the top of `src/lib.rs` for the full rationale, and the `iggy-connector-review` skill / the "Building Connectors That Pass Review" blog post for the checklist this template is built against. +## Delivery Semantics + +The framework around `push_batch()` gives you retry (`max_retries` total attempts, +`retry_delay`/`retry_max_delay` backoff) and a circuit breaker that opens after +`circuit_breaker_threshold` consecutive failures. Everything below that line +depends on what you write in `push_batch()` — fill in the three blanks with your +real behavior before shipping: + +1. **Transient failure**: retries up to `max_retries` total attempts, then + `consume()` returns the error. The runtime's handling of that error is fatal + to the whole connector — see `connector-pr-review` B2. +2. **Permanent failure** (bad schema, destination rejects the payload shape): + return `Error::PermanentHttpError` from `push_batch()` — `consume()` drops + and counts that one batch instead of taking the whole connector down. +3. **Duplication window**: at-least-once. A batch can be retried after a + partial write (e.g. your destination accepted some rows then the connection + dropped before the response), so a retry can re-send already-written rows. +4. **Dedup key**: none by default — `push_batch()` doesn't build one yet. + Before shipping, derive a stable key from `message.id` plus + `topic_metadata`/`messages_metadata` (see `mongodb_sink::build_composite_document_id` + for `stream:topic:partition:message_id`) and make your destination's write + idempotent on it (upsert / `ON CONFLICT` / unique index) — otherwise the + at-least-once retry above will create duplicates. See `connector-pr-review` + B5. + ## What's already done for you - Config parsing with `#[serde(deny_unknown_fields)]` so a typo in a TOML diff --git a/core/connectors/sinks/sink_template/src/lib.rs b/core/connectors/sinks/sink_template/src/lib.rs index 0a86bc7338..80a8c97aba 100644 --- a/core/connectors/sinks/sink_template/src/lib.rs +++ b/core/connectors/sinks/sink_template/src/lib.rs @@ -230,12 +230,36 @@ impl TemplateSink { /// returned, which the runtime treats as fatal for the whole connector — /// so a permanent classification is what keeps one bad message from /// taking every future batch down with it. + /// + /// Delivery is at-least-once: a retried batch can re-send rows your + /// destination already wrote from an earlier attempt that failed after + /// partially succeeding. Build a stable key with `dedup_key()` below for + /// every message you write, and make the write idempotent on it (upsert / + /// `ON CONFLICT` / unique index) — see `connector-pr-review` B5. + /// + /// If your destination's write call can succeed at the transport level + /// while rejecting some or all rows individually (a bulk/batch API + /// returning 200 with per-item errors, a stream-load response, a bulk + /// index response), count accepted rows yourself and return `Err` when + /// that count is zero for a non-empty `batch` — never `Ok(())`. Four + /// unrelated sinks shipped exactly this bug in review (BigQuery, + /// Elasticsearch, Doris, S3): the call "succeeded," `consume()` reported + /// success, and the batch was gone with no signal anywhere. See + /// `connector-pr-review` B9. async fn push_batch( &self, client: &ClientWithMiddleware, + topic_metadata: &TopicMetadata, + messages_metadata: &MessagesMetadata, batch: &[ConsumedMessage], ) -> Result<(), Error> { - let _ = (client, batch); // remove once implemented + let _ = client; // remove once implemented + for message in batch { + // TODO(ConnectorDeveloper): use `key` as the identity your destination's + // write is idempotent on (upsert key, `ON CONFLICT` target, document _id). + let key = dedup_key(topic_metadata, messages_metadata, message); + let _ = key; + } Err(Error::InitError( "TemplateSink::push_batch is not implemented yet — see the TODO(ConnectorDeveloper) comment in \ template_sink/src/lib.rs" @@ -244,6 +268,23 @@ impl TemplateSink { } } +/// Stable per-message dedup key: `stream:topic:partition:message_id`. Mirrors +/// `mongodb_sink::build_composite_document_id` — copy this, not a random UUID, +/// for whatever identity column/key your destination's upsert needs. Offset +/// is deliberately excluded: it only identifies a message within one +/// partition's log, not across the retries and redeliveries this key has to +/// survive. +fn dedup_key( + topic_metadata: &TopicMetadata, + messages_metadata: &MessagesMetadata, + message: &ConsumedMessage, +) -> String { + format!( + "{}:{}:{}:{}", + topic_metadata.stream, topic_metadata.topic, messages_metadata.partition_id, message.id + ) +} + // ── Sink trait ──────────────────────────────────────────────────────────────── #[async_trait] @@ -395,7 +436,10 @@ impl Sink for TemplateSink { let mut failed = 0u64; for batch in messages.chunks(self.batch_size_limit) { - match self.push_batch(client, batch).await { + match self + .push_batch(client, topic_metadata, &messages_metadata, batch) + .await + { Ok(()) => written += batch.len() as u64, Err(Error::PermanentHttpError(message)) => { failed += batch.len() as u64; @@ -498,7 +542,7 @@ mod tests { } #[tokio::test] - async fn open_rejects_empty_connection_string() { + async fn given_empty_connection_string_should_reject_open() { let mut config = test_config(); config.connection_string = SecretString::from(" "); let mut sink = TemplateSink::new(1, config); @@ -509,7 +553,7 @@ mod tests { } #[tokio::test] - async fn open_rejects_invalid_target_identifier() { + async fn given_invalid_target_identifier_should_reject_open() { let mut config = test_config(); config.target = "events; DROP TABLE users;--".to_string(); let mut sink = TemplateSink::new(1, config); @@ -520,17 +564,17 @@ mod tests { } #[test] - fn validate_identifier_accepts_plain_names() { + fn validate_identifier_should_accept_plain_names() { assert!(validate_identifier("target", "events_v2").is_ok()); } #[test] - fn validate_identifier_rejects_empty() { + fn validate_identifier_should_reject_empty() { assert!(validate_identifier("target", "").is_err()); } #[test] - fn validate_identifier_rejects_path_and_query_characters() { + fn validate_identifier_should_reject_path_and_query_characters() { for bad in [ "../etc/passwd", "events?x=1", @@ -544,8 +588,18 @@ mod tests { } } + #[test] + fn dedup_key_should_join_stream_topic_partition_and_message_id() { + let (topic_metadata, messages_metadata) = topic_and_messages_metadata(); + let message = consumed_message(); + assert_eq!( + dedup_key(&topic_metadata, &messages_metadata, &message), + "s:t:1:1" + ); + } + #[tokio::test] - async fn consume_short_circuits_when_breaker_is_open() { + async fn given_open_circuit_breaker_should_short_circuit_consume() { let sink = TemplateSink::new(1, test_config()); sink.circuit_breaker.record_failure().await; sink.circuit_breaker.record_failure().await; @@ -592,7 +646,7 @@ mod tests { // of the chunking loop end to end, including the failed-count and // circuit-breaker bookkeeping. #[tokio::test] - async fn consume_with_unimplemented_push_batch_returns_err_and_counts_failure() { + async fn given_unimplemented_push_batch_should_return_err_and_count_failure() { let mut sink = TemplateSink::new(1, test_config()); sink.open().await.expect("open should succeed"); let (topic_metadata, messages_metadata) = topic_and_messages_metadata(); diff --git a/core/connectors/sources/source_template/README.md b/core/connectors/sources/source_template/README.md index 0ae77ac2cc..234f10d21a 100644 --- a/core/connectors/sources/source_template/README.md +++ b/core/connectors/sources/source_template/README.md @@ -7,6 +7,33 @@ doc comment at the top of `src/lib.rs` for the full rationale, and the `iggy-connector-review` skill / the "Building Connectors That Pass Review" blog post for the checklist this template is built against. +## Delivery Semantics + +The framework around `fetch_records()` gives you retry (`max_retries` total +attempts), a circuit breaker, and the ACK/NACK cursor handshake described +below. Everything past "what fetches" depends on what you write in +`fetch_records()` — fill in the blanks with your real behavior before +shipping: + +1. **Transient failure**: retries up to `max_retries` total attempts, then + `poll()` returns the error; the circuit breaker counts it. The runtime + retries `poll()` again next interval. +2. **Permanent failure** (auth rejected, schema gone): return + `Error::PermanentHttpError` from `fetch_records()` — `poll()` does not + trip the circuit breaker on it, since a bad config/auth state isn't a + connectivity signal the breaker should react to. +3. **Duplication window**: at-least-once. `poll()` stages the cursor as + *candidate* state; the runtime persists it only after the batch is sent, + then reports `Ack`/`Nack`. A NACK (or a crash between send and the + persisted-state write) redelivers the same range on the next poll. +4. **Dedup key**: set `FetchedRecord::natural_id` from a true per-record + identity (row PK, document `_id`, event UUID) whenever your source has + one — it becomes `ProducedMessage.id`, which a downstream sink can key an + idempotent upsert on (see `mongodb_sink`) so the redelivery above doesn't + produce a duplicate at the destination. Leave it `None` only when records + genuinely have no natural identity (a pure generator). See + `connector-pr-review` B5. + ## What's already done for you - Config parsing with `#[serde(deny_unknown_fields)]` so a typo in a TOML diff --git a/core/connectors/sources/source_template/src/lib.rs b/core/connectors/sources/source_template/src/lib.rs index 430e7e13ac..7dafc097f9 100644 --- a/core/connectors/sources/source_template/src/lib.rs +++ b/core/connectors/sources/source_template/src/lib.rs @@ -143,6 +143,19 @@ struct FetchedRecord { /// The value `cursor` should advance to once this record's batch is /// acknowledged — typically this record's timestamp/ID/token. cursor_value: String, + /// TODO(ConnectorDeveloper): set this from a true per-record identity — + /// a numeric row PK, a hash of a document `_id`, an event UUID the + /// upstream system itself assigned — so `ProducedMessage.id` below is + /// stable across a NACK-triggered redelivery of this same record. A + /// *fresh* random id per emit defeats a downstream sink's dedup-by-id + /// (see `connector-pr-review` B5): a redelivered row would arrive with a + /// different id than its first delivery, so a sink keying its upsert on + /// `message.id` would store it twice instead of overwriting. Leave this + /// `None` only when records genuinely have no natural per-record + /// identity (a pure generator, e.g. `random_source`) — `cursor_value` + /// alone usually isn't it either, since a timestamp-based cursor is + /// commonly shared by more than one record. + natural_id: Option, payload: serde_json::Value, } @@ -234,7 +247,8 @@ impl TemplateSource { /// result row/document/event to a `FetchedRecord`, using something that /// monotonically increases (a timestamp, an ID, a page token) as /// `cursor_value` so the state-staging logic in `poll()` below can - /// advance the cursor correctly. + /// advance the cursor correctly, and set `natural_id` from a true + /// per-record identity if one exists (see the field's own doc comment). async fn fetch_records( &self, client: &ClientWithMiddleware, @@ -391,9 +405,10 @@ impl Source for TemplateSource { // into `messages` - advancing on a dropped (unserializable) record // would stage a cursor past data that was never produced, and a // later Ack would commit past it permanently. + let natural_id = record.natural_id; new_cursor = Some(record.cursor_value); messages.push(ProducedMessage { - id: None, + id: natural_id, headers: None, checksum: None, timestamp: None, @@ -453,6 +468,23 @@ impl Source for TemplateSource { /// delivered. If `fetch_records()` ever needs to perform a destructive /// read against the source (delete-after-read, mark-as-processed), /// stage that side effect the same way and only apply it here on `Ack`. + /// + /// If you add real I/O here (an external ack call, a delete, a mark- + /// processed update), two things from a real connector's review history + /// apply directly: + /// + /// 1. This runs inside the SDK's fixed ~30s batch-result window, and a + /// size-your-own-timeout mistake here is fatal, not just slow — a + /// NACK from that window stops the source for good after repeated + /// consecutive NACKs. If you stage more than one operation, budget + /// for *all* of them inside the window, not one op's worth, and + /// reject `max_retries`/`retry_delay` combinations in `open()` that + /// could blow it. See `connector-pr-review` B11. + /// 2. A failed op must not be silently logged while the batch is still + /// ACKed — the cursor has already committed past it by the time you + /// find out. Count consecutive failures and return `Err` past a + /// threshold so the SDK stops the source, instead of resending the + /// same broken operation forever. async fn on_batch_result(&self, result: SourceBatchResult) -> Result<(), Error> { let candidate_state = self.pending_state.lock().await.take(); if result == SourceBatchResult::Ack @@ -534,7 +566,7 @@ mod tests { } #[tokio::test] - async fn open_rejects_empty_connection_string() { + async fn given_empty_connection_string_should_reject_open() { let mut config = test_config(); config.connection_string = SecretString::from(" "); let mut source = TemplateSource::new(1, config, None); @@ -622,7 +654,7 @@ mod tests { } #[tokio::test(start_paused = true)] - async fn poll_returns_empty_without_error_when_circuit_is_open() { + async fn given_open_circuit_breaker_should_return_empty_without_error() { let source = TemplateSource::new(1, test_config(), None); source.circuit_breaker.record_failure().await; source.circuit_breaker.record_failure().await; From 28b1f9b6a422721cc1ee8a38ce31d6e2f128eb96 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Thu, 22 Oct 2026 15:32:51 -0400 Subject: [PATCH 3/5] feat(connectors): add sink/source templates, authoring guide, blog post Squashed for a clean rebase onto current master. --- .claude/skills/connector-pr-review/SKILL.md | 370 ++++++++++ .claude/skills/connector-runtime/SKILL.md | 4 +- .claude/skills/connector-sink/SKILL.md | 3 +- .claude/skills/connector-sink/TEMPLATE.md | 335 +++++++-- .claude/skills/connector-source/SKILL.md | 125 ++-- .claude/skills/connector-source/TEMPLATE.md | 391 ++++++++-- .claude/skills/connector-testing/SKILL.md | 19 +- .claude/skills/connectors-overview/SKILL.md | 72 +- Cargo.lock | 35 + Cargo.toml | 2 + core/connectors/BLOG_POST.md | 104 +++ core/connectors/README.md | 4 + .../docs/authoring-sinks-and-sources.md | 196 +++++ .../connectors/sink_template.toml | 48 ++ .../connectors/source_template.toml | 47 ++ core/connectors/sinks/README.md | 1 + .../connectors/sinks/sink_template/Cargo.toml | 57 ++ core/connectors/sinks/sink_template/README.md | 103 +++ .../sinks/sink_template/config.toml | 49 ++ .../connectors/sinks/sink_template/src/lib.rs | 662 +++++++++++++++++ core/connectors/sources/README.md | 1 + .../sources/source_template/Cargo.toml | 60 ++ .../sources/source_template/README.md | 97 +++ .../sources/source_template/config.toml | 48 ++ .../sources/source_template/src/lib.rs | 671 ++++++++++++++++++ 25 files changed, 3300 insertions(+), 204 deletions(-) create mode 100644 .claude/skills/connector-pr-review/SKILL.md create mode 100644 core/connectors/BLOG_POST.md create mode 100644 core/connectors/docs/authoring-sinks-and-sources.md create mode 100644 core/connectors/runtime/example_config/connectors/sink_template.toml create mode 100644 core/connectors/runtime/example_config/connectors/source_template.toml create mode 100644 core/connectors/sinks/sink_template/Cargo.toml create mode 100644 core/connectors/sinks/sink_template/README.md create mode 100644 core/connectors/sinks/sink_template/config.toml create mode 100644 core/connectors/sinks/sink_template/src/lib.rs create mode 100644 core/connectors/sources/source_template/Cargo.toml create mode 100644 core/connectors/sources/source_template/README.md create mode 100644 core/connectors/sources/source_template/config.toml create mode 100644 core/connectors/sources/source_template/src/lib.rs diff --git a/.claude/skills/connector-pr-review/SKILL.md b/.claude/skills/connector-pr-review/SKILL.md new file mode 100644 index 0000000000..0ae07f3cd1 --- /dev/null +++ b/.claude/skills/connector-pr-review/SKILL.md @@ -0,0 +1,370 @@ +--- +name: connector-pr-review +description: Review checklist for Apache Iggy connector sink/source PRs. Load when reviewing a connectors plugin PR, when authoring a new sink/source and wanting to pre-flight against common review blockers, or when diagnosing why a connectors PR is stuck in review. Encodes recurring review patterns mined from real apache/iggy connector PRs. NOT for runtime/SDK internals (use connector-runtime / connector-sdk). +--- + +# Connector PR review checklist + +> Universal rules live in [connectors-overview](../connectors-overview/SKILL.md). +> Authoring skills: [connector-sink](../connector-sink/SKILL.md), +> [connector-source](../connector-source/SKILL.md), +> [connector-testing](../connector-testing/SKILL.md). +> Fill-in-the-blank kits: those skills' `TEMPLATE.md` files. + +Use this skill to **catch the issues that repeatedly burn review cycles** +before asking for a human re-review. Cite symbols/paths, not stale line numbers. + +## Contents + +- [How to use](#how-to-use) +- [Blockers (must fix before merge)](#blockers-must-fix-before-merge) +- [High-frequency convention nits](#high-frequency-convention-nits) +- [Delivery semantics (document honestly)](#delivery-semantics-document-honestly) +- [PR / CI hygiene](#pr--ci-hygiene) +- [Pre-flight author checklist](#pre-flight-author-checklist) +- [Evidence base](#evidence-base) + +## How to use + +1. Load this skill for any PR under `core/connectors/sinks/` or `core/connectors/sources/`. +2. Walk **Blockers** first. Any hit is CHANGES_REQUESTED. +3. Then **Convention nits** and **Delivery semantics**. +4. End with **PR / CI hygiene** (cheap passes that still delay first review). +5. Prefer "copy the closest exemplar" over inventing new knobs. + +## Blockers (must fix before merge) + +### B1. Secrets + +- [ ] Every credential field is `secrecy::SecretString` with + `#[serde(serialize_with = "iggy_common::serde_secret::serialize_secret")]`. +- [ ] No connection string / API key / token in `info!`/`debug!`/`error!` / + `format!` into SQL / file-state metadata. +- [ ] Source state must not persist URL userinfo or bearer tokens. + +Plain `String` for a credential is a review-blocker. Pattern: +`PostgresSinkConfig::connection_string` in `sinks/postgres_sink`. + +### B2. Swallowing `Err` while offsets advance (sinks) + +- [ ] `consume()` must **not** catch a batch failure and return `Ok(())`. +- [ ] Prefer `last_err` pattern: process all batches, return the last transient + error (see `connector-sink` Hard rules / TEMPLATE). +- [ ] README must not claim "no data loss" / strong idempotency unless the + backend + runtime path actually enforce it. + +Today the runtime can commit consumer offsets even when plugin errors are +poorly surfaced (#2927 / #2928 class issues). Authors must be honest about +loss windows instead of overselling. + +### B3. Source cursor / side-effects staged for `on_batch_result`, not committed in `poll()` + +Since #3855, this is an SDK-enforced contract, not just a convention: +`poll()` returns *candidate* state; the runtime sends the batch, saves +that state on success, then calls `on_batch_result(Ack | Nack)`. Only +`on_batch_result` may commit. + +- [ ] `poll()` stages cursor changes and destructive work (delete-after-read / + mark-processed / ACK upstream) - it does not mutate committed state or + touch upstream rows directly. Committing in `poll()` leaves nothing to + roll back on a Nack, defeating the point of the handshake. +- [ ] `on_batch_result(SourceBatchResult::Ack)` commits the staged work; + `on_batch_result(SourceBatchResult::Nack)` discards it so the batch is + redelivered against unchanged committed state. A source with no staged + work (e.g. a pure generator) may rely on the SDK's default no-op impl. +- [ ] Returning `Err` from `on_batch_result` is deliberate - it stops the SDK + from polling further rather than risk silently advancing past a failed + rollback. Don't swallow a rollback failure into `Ok(())`. +- [ ] Always return `ConnectorState` in every `ProducedMessages` that made + progress; return `state: None` for an empty poll that made none (avoids + an unnecessary write and can't persist state left over from a failed + batch). +- [ ] `poll()` sleeps **first**, then fetches (never sleep after holding a batch). + +### B4. Transient vs permanent errors + +- [ ] Infra/auth/schema-gone failures map to `Error::PermanentHttpError` / + `Error::InitError` / `Error::SchemaMismatch` — not `InvalidRecord`. +- [ ] Retryable network/5xx/SQLSTATE map to transient variants + (`HttpRequestFailed`, `Connection`, `CannotStoreData`). +- [ ] Do **not** classify retryability by substring-matching `err.to_string()`. +- [ ] `max_retries` means **total attempts** (default 3). README must match code. +- [ ] Cap retry budget so a dead backend cannot delay shutdown unboundedly. + +### B5. Idempotency claims must be real + +- [ ] Stable `ProducedMessage.id` / sink dedup key from natural IDs + (table+PK, document `_id`, `stream:topic:partition:message_id`) — + never random UUIDs per emit. +- [ ] If the backend PK / unique index is informational only (e.g. Redshift), + do not advertise idempotency in README. +- [ ] External workflow IDs (Airflow `dag_run_id`, etc.) must be deterministic + across retries. + +### B6. Secrets / license policy for new SDKs + +- [ ] New backend crates pass `scripts/ci/third-party-licenses.sh` (no BUSL / + incompatible licenses pulled into the tree). +- [ ] Prefer workspace deps; avoid vendoring a license-hostile SDK just to wrap HTTP. + +### B7. Tests that must exist + +#### Sources + +- [ ] Four canonical state tests (restore / no-state / invalid-state / + round-trip), plus two ACK/NACK tests (`given_ack_when_batch_is_staged_ + should_commit_candidate_state`, `given_nack_when_batch_is_staged_ + should_keep_committed_state`) if `on_batch_result` is overridden - six + total. Copy `sources/random_source/src/lib.rs::tests`. A source relying + on the SDK's default no-op `on_batch_result` (no staged work) may skip + the ACK/NACK pair. + +#### Any external backend plugin + +- [ ] At least one real-infra integration test under + `core/integration/tests/connectors//` with `#[iggy_harness]` + + `testcontainers-modules` (or `wiremock` for pure HTTP). +- [ ] No false-green mocks that diverge from real backend semantics + (Decimal, COPY, PK enforcement, etc.). + +### B8. Config validation timing + +- [ ] Structural validation + unknown enum rejection in `new()` / + `open()` — not on first `poll()`/`consume()` after sleep. +- [ ] Connectivity check in `open()`; fail with `Error::InitError`. +- [ ] Invalid restored state: start fresh + `warn!`, but do **not** silently + re-emit an entire index without calling that out in README. +- [ ] Config flag combos that no-op should `warn!` or `Err`, not silently ignore. + +### B9. Zero delivered is a failure, even if the call "succeeded" + +A bulk/batch API call returning 200 does not mean every row in it landed. +Recurring, independently-discovered bug across four different sinks: + +- [ ] If your destination's write API can partially or fully reject a batch + while the call itself succeeds (HTTP 200 with per-item errors, a bulk + index response, a stream-load response), count accepted items and + return `Err` when the accepted count is **zero** for a non-empty input — + never `Ok(())`. BigQuery (#4394): "when every row of a non-empty batch + is rejected, `consume` still returns `Ok`." Elasticsearch (#4154): bulk + index knows the rejected count but still returns `Ok`. Doris (#3215): + "if every message in the batch is non-JSON... `consume()` returns + `Ok(())`" with the batch silently gone. S3 (#3103): a formatter failure + produced an empty placeholder that was still counted as uploaded. +- [ ] A per-record encode/serialize failure must not silently substitute an + empty/null/default value that then gets counted as delivered — skip + and count it as lost (visibly), or fail the whole chunk. + +### B10. A wrapping timeout does not cancel in-flight work + +`tokio::time::timeout(dur, fut)` dropping `fut` on expiry does **not** mean +the operation didn't happen — for a blocking-executor call, an FFI call, or +anything already past the "no going back" point in the wire protocol, the +work keeps running after the timeout fires and can still complete +server-side. Confirmed independently on two unrelated backends: + +- [ ] Treat an expired wrapping timeout as **unknown outcome**, not + **didn't happen** — do not assume it's safe to retry the identical + operation without a dedup/idempotency key. RabbitMQ (#3973): "Wrapping + `basic_publish` in `timeout()` does not make it abortable... usually + already delivered." Postgres (#3957): "`timeout` drops the client + future, it does not abort the statement — a DELETE... can still commit + server-side after this logs a failure." +- [ ] Prefer a protocol/transport-level timeout over (or in addition to) a + wrapping `tokio::time::timeout`: `reqwest::ClientBuilder::timeout`, + a connection-string timeout parameter (`?connection_timeout=` for AMQP), + or a driver's own `statement_timeout`. A bare `TcpStream::connect` run + on a blocking executor is not cancelled by an outer async timeout and + leaks a pool thread until the kernel's own SYN timeout (RabbitMQ #3973). +- [ ] This is why `build_raw_client()` in the sink/source templates sets + `.timeout()` on the `reqwest::Client` itself rather than wrapping calls + — copy that pattern, don't wrap instead of it. + +### B11. Ack-path work must fit inside the SDK's fixed batch-result window + +If `on_batch_result` (or any post-ack cleanup) does real I/O — not just an +in-memory state swap — that I/O runs inside the SDK's ~30s batch-result +timeout, and a NACK from that timeout is often **fatal** (stops the source +for good after repeated consecutive NACKs), not a retry signal. + +- [ ] Any retry/timeout budget inside `on_batch_result` must be sized well + under the SDK's window, and **multiplied by how many operations run + inside it** — not assumed to be one shared budget. Postgres (#3957, + critical): "10s is per operation, not per batch... N tables burn + N\*10s inside the SDK's 30s window." +- [ ] User-configurable `max_retries`/`retry_delay` can produce a wait time + that exceeds a hardcoded internal deadline — validate the combination + in `open()` and reject configs that cannot fit, rather than letting + the ack path blow its budget at runtime. Same PR: "`max_retries=3` + with `retry_delay=\"2s\"`, both straight from the readme, needs 12s + and fails startup." +- [ ] A failed destructive/cleanup op inside `on_batch_result` must not be + silently logged and ignored while the batch is still ACKed — count + consecutive failures and return `Err` past a threshold so the SDK + stops the source, rather than resending the same broken operation + forever. Same PR: "a failed slot advance only logs. the batch is + already ACKed, so nothing stops it." + +## High-frequency convention nits + +These are "cheap" but burn full review rounds when missed. + +### C1. Copy the closest exemplar + +- [ ] File layout, log labels, error mapping, and test structure match the + nearest sibling (`postgres_*`, `http_sink`, `elasticsearch_*`, …). +- [ ] Do not invent new names for existing knobs. + +### C2. Config knob name canon + +| Concept | Canonical field | Notes | +| ------- | --------------- | ----- | +| Request timeout | `timeout` | Not `request_timeout` | +| Retry attempts | `max_retries` | Total attempts, default 3 | +| Base backoff | `retry_delay` | humantime `Option` | +| Backoff ceiling | `retry_max_delay` | Matches SDK's `RetryPolicy::max_delay` | +| Poll cadence (sources) | `poll_interval` | humantime; sleep first | +| Plugin verbosity | `verbose_logging` | Mirror runtime `verbose` | +| Credentials | `connection_string` / `api_key` / … | Always `SecretString` | + +- [ ] Durations: `Option` + `humantime::Duration` in `new()`; fall back + with `warn!`, never panic. Workspace `humantime` — not a pinned + `humantime-serde`. +- [ ] New fields: `Option` + `#[serde(default)]` where needed. +- [ ] Prefer `#[serde(deny_unknown_fields)]` on plugin config so typo’d knobs + fail loud. +- [ ] Reject degenerate numeric values (`0`, empty string) for a field where + zero makes a check always-true or always-false, rather than letting it + silently misbehave — `batch_size`/`max_messages_per_file` zero + triggered S3 (#3103) to rotate on every single message. If a sibling + connector already clamps a field like this (`.max(1)`), clamp + consistently rather than leaving one sibling to reject what another + silently accepts. +- [ ] Never derive a value that must stay identical across a retry of the + *same* logical batch (a destination key, filename, or dedup id) from + wall-clock time (`Utc::now()`) — derive it from the batch's own fixed + fields (an offset, a message id, `origin_timestamp`). S3 (#3103): a + `{timestamp}` path placeholder rendered at upload time broke the + README's own idempotent-retry claim. + +### C3. Crate / path / docs checklist + +- [ ] `[lib] crate-type = ["cdylib", "lib"]`. +- [ ] `publish = false` in `[package]` — every in-tree plugin needs it, and + it's easy to forget on a new crate (missed on S3 #3103 and Doris + #3215, both caught in review). +- [ ] Example TOML plugin path uses `../../target/release/lib…` like siblings. +- [ ] Row added to `sinks/README.md` or `sources/README.md`. +- [ ] Sample under `runtime/example_config/connectors/`. +- [ ] README defaults **byte-equal** to consts in code (diff them). +- [ ] No links to non-existent docs. +- [ ] Helper modules stay crate-private (no `pub mod`) unless genuinely part + of the plugin's public API — S3 (#3103) exposed `buffer`/`client`/etc. + as public on a cdylib plugin where every sibling keeps them private. + +### C4. Hot path + +- [ ] No `payload.clone().try_to_bytes()` — use `try_to_bytes(&self)`. +- [ ] `Vec::with_capacity(n)` for per-batch buffers. +- [ ] No `std::sync::Mutex` across `.await`; use `tokio::sync::Mutex`. +- [ ] Even a `tokio::sync::Mutex` must not be **held across** a slow network + call or a retry sleep — lock, clone/extract what you need, drop the + guard, *then* do I/O. This isn't just a deadlock rule: holding the + lock across the call serializes every concurrent caller sharing that + plugin instance onto one upload/write at a time, killing the + parallelism the runtime's per-partition tasks were meant to give you. + S3 (#3103): one `buffers.lock().await` held through `put_object()` + *and* the retry sleep capped the whole sink's throughput to one + upload in flight regardless of partition count. +- [ ] `&self` on `consume` / `poll` (interior mutability only). +- [ ] No `tokio::spawn` inside plugin code. +- [ ] No `unwrap()`/`expect()` on external I/O outside tests. +- [ ] No eager `format!` around tracing args. +- [ ] Never `let _ = fallible_call()` on a side effect that matters (a + cleanup op, a quoted identifier, a staged delete) — log or propagate + the error. Postgres (#3957): `let _` on a staged-operation error made + an always-failing delete invisible. +- [ ] If retrying raw HTTP status codes (not just SDK `Error` variants), + classify by family — 5xx, 408, 429 retry; other 4xx fail fast. S3 + (#3103) retried `AccessDenied`/`NoSuchBucket`/`InvalidBucketName` + three times each, burning the whole retry budget on failures no retry + could fix. +- [ ] Never log a substring, prefix, or fragment of a credential "for + diagnostics" — it still identifies the credential type/shape for zero + diagnostic value. S3 (#3103) logged the first 3 characters of an AWS + access key (`AKIA`/`ASIA`/`AROA` prefix). + +### C5. Containers / fixtures + +- [ ] Testcontainers named `iggy-test-*` via `fixtures::unique_container_name` + (or fixed `iggy-test-` for reuse fixtures). +- [ ] Custom Docker networks cleaned up. + +## Delivery semantics (document honestly) + +Every new connector README must answer in one short paragraph: + +1. **What happens on transient failure?** (retry N times, then Err) +2. **What happens on permanent failure?** (drop/skip vs fail batch) +3. **What is the duplication window?** (at-least-once because state saves + after Iggy send; or at-most-once if upstream ACK precedes send — say so) +4. **What is the dedup key?** (or "none — duplicates possible") + +If the answer is hand-wavy, the PR is not ready. + +## PR / CI hygiene + +- [ ] Conventional commit: `feat(connectors): …` / `fix(connectors): …`. +- [ ] PR template filled (motivation, linked issue). +- [ ] `cargo fmt --all` + `cargo sort --no-format --workspace` + + `cargo clippy -p --all-targets -- -D warnings` + + `cargo test -p ` green locally. +- [ ] Minimal `Cargo.lock` delta — no unrelated dependency churn. +- [ ] Do not modify unrelated Java/Python/foreign SDK trees in a connectors PR. +- [ ] Mark ready for review only after the above; stale-bot closes waiting PRs. + +## Pre-flight author checklist + +Paste into the PR description (or run mentally before `/ready`): + +```text +[ ] SecretString on all credentials; no secret logs/state +[ ] consume/poll never returns Ok(()) after a failed batch that should retry +[ ] Transient vs permanent errors mapped (no Display substring matching) +[ ] Stable message / dedup IDs (no random UUID per emit) +[ ] README delivery semantics paragraph present and honest +[ ] README defaults match code consts +[ ] deny_unknown_fields on plugin config +[ ] Canonical knob names (timeout, max_retries, retry_delay, poll_interval) +[ ] Sources: 4 state tests (+2 ACK/NACK if on_batch_result overridden); sleep-first poll; state staged in poll, committed only in on_batch_result +[ ] External backend: real-infra integration test (not a lying mock) +[ ] example_config + sinks/sources README row +[ ] fmt / sort --no-format / clippy -D warnings / unit tests green +[ ] Cargo.lock churn limited to this crate's deps +``` + +## Evidence base + +Recurring comments mined from connector PRs including (non-exhaustive): +SurrealDB sink (#3453), Meilisearch sink/source (#3497/#3498), OpenSearch +source (#3515), Quickwit convention (#3523), MySQL source (#3568), JDBC +source (#3588), Doris retry (#3574) and sink review (#3215), Redshift sink +(#3654), Airflow trigger (#3716), Fluss sink (#3782), S3 sink (#3103), +Google BigQuery sink (#4394), Delta sink production hardening (#4219), +RabbitMQ sink (#3973), Elasticsearch sink batch semantics (#4154), +Postgres source ACK/NACK + CDC hardening (#3957), the shared +`retry_async`/`RetryPolicy` SDK helper review (#4104). Highest-density +themes: delivery/offset semantics, idempotency IDs, transient/permanent +mapping, config-name drift, secrets, README/code drift, false-green tests, +CI/lockfile hygiene, zero-delivered-still-reports-Ok, a wrapping timeout +that doesn't cancel in-flight work, ack-path work blowing its fixed budget, +and holding a lock across slow I/O. + +Re-derive or extend this list with `gh pr list --repo apache/iggy --search +"connectors in:title" --state all --limit 100` plus `gh api +repos/apache/iggy/pulls//comments --paginate` for PRs under +`core/connectors/`. + +--- + +Discussion / help: see [AGENTS.md](../../../AGENTS.md#discussion-and-support). diff --git a/.claude/skills/connector-runtime/SKILL.md b/.claude/skills/connector-runtime/SKILL.md index b764c31fda..7faa564c6a 100644 --- a/.claude/skills/connector-runtime/SKILL.md +++ b/.claude/skills/connector-runtime/SKILL.md @@ -213,7 +213,7 @@ Fatal errors propagate to `main` and exit. Per-connector / per-message errors ar All families labeled by `connector_key` + `connector_type` (histogram adds `stage`): -- **Counters**: `iggy_connector_messages_{produced,sent,consumed,processed,filtered}_total` and `iggy_connector_errors_total`. These are the *rendered* names; each is registered without the `_total`, which the OpenMetrics encoder appends. +- **Counters**: `iggy_connector_messages_{produced,sent,consumed,processed,filtered,errors}_total`. - `messages_filtered_total` - intentional drops via transform `Ok(None)`. - `errors_total` - unexpected drops (decode/encode/build failure, missing field, ...) + batch-level failures, one per failed run. - `sink_runs_total` - FFI `consume()` calls for sink batches, one per contiguous payload-variant run. Divide by `stage_duration_seconds_count{stage="total"}` for runs per batch. @@ -224,7 +224,7 @@ All families labeled by `connector_key` + `connector_type` (histogram adds `stag When adding a metric: -- Add family to `Metrics` struct + `init`, register with name + help text. Never end a `Counter` family's registered name in `_total`: the encoder appends it and the series renders `_total_total`. Gauges get no suffix, so a gauge name may end in `_total` literally. +- Add family to `Metrics` struct + `init`, register with name + help text. - New label sets define `EncodeLabelSet` struct + label enum (hand-impl `EncodeLabelValue` for snake_case values - the derive emits PascalCase). - Histograms: pass `fn() -> Histogram` to `Family::new_with_constructor`. - Add unit tests under `mod tests` with `given_*_when_*_should_*` BDD names. diff --git a/.claude/skills/connector-sink/SKILL.md b/.claude/skills/connector-sink/SKILL.md index d135682824..ae03a13066 100644 --- a/.claude/skills/connector-sink/SKILL.md +++ b/.claude/skills/connector-sink/SKILL.md @@ -34,7 +34,8 @@ for getting them to the external system reliably and efficiently. ## Quick reference -- Skeleton: [TEMPLATE.md](TEMPLATE.md) (load on demand when authoring). +- Skeleton: [TEMPLATE.md](TEMPLATE.md) (fill-in-the-blank kit — implement only `TODO(ConnectorDeveloper)`). +- PR pre-flight: [connector-pr-review](../connector-pr-review/SKILL.md). - Exemplars: `stdout_sink` (minimal), `postgres_sink` (DB + transient detection), `http_sink` (validation, batch modes, retry middleware), `mongodb_sink` (atomic counters), `elasticsearch_sink` / `iceberg_sink` (backend-specific idioms). ## Hard rules diff --git a/.claude/skills/connector-sink/TEMPLATE.md b/.claude/skills/connector-sink/TEMPLATE.md index d9d675fe1c..798c4f8c64 100644 --- a/.claude/skills/connector-sink/TEMPLATE.md +++ b/.claude/skills/connector-sink/TEMPLATE.md @@ -1,100 +1,200 @@ -# Sink plugin skeleton +# Sink plugin fill-in-the-blank kit -Boilerplate for a new `core/connectors/sinks/_sink/`. Adapt the -`MySink` / `Client` types to the backend driver you're integrating. +Copy this kit into `core/connectors/sinks/_sink/`. The scaffolding +covers config, secrets, retry, error classification, batching, logging, +and unit-test shape. **You only implement the marked `TODO(ConnectorDeveloper)` +sections:** build a client from the connection string, and push one +batch. + +Also read [SKILL.md](SKILL.md) and pre-flight with +[connector-pr-review](../connector-pr-review/SKILL.md) before `/ready`. + +Prefer starting from a compiling crate over copying this prose kit: +`core/connectors/sinks/sink_template/` implements the same shape as real, +tested code you can `cargo build`/`cargo test` immediately, with the same +`TODO(ConnectorDeveloper)` markers. Use this kit instead only when copying +a whole crate is more scaffolding than you need. + +## Files to create + +```text +core/connectors/sinks/_sink/ +├── Cargo.toml +├── README.md +├── config.toml +└── src/lib.rs +``` + +Add a workspace member, a row in `sinks/README.md`, and a sample under +`runtime/example_config/connectors/`. + +--- ## Cargo.toml ```toml -# Apache 2.0 header (copy verbatim from any existing sink Cargo.toml) +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [package] name = "iggy_connector__sink" -version = "0.5.1-edge.1" # match the version most other sinks use +version = "0.5.0" # match the version most other sinks use edition = "2024" license = "Apache-2.0" publish = false -# ...keywords, description, repository, homepage all identical to existing sinks +description = "Apache Iggy sink connector" +repository = "https://github.com/apache/iggy" +homepage = "https://iggy.apache.org" [package.metadata.cargo-machete] -ignored = ["dashmap", "once_cell"] # used by sink_connector! macro +ignored = ["dashmap", "once_cell"] [lib] -crate-type = ["cdylib", "lib"] # cdylib = runtime-loadable; lib = unit tests +crate-type = ["cdylib", "lib"] [dependencies] -async-trait = { workspace = true } -dashmap = { workspace = true } +async-trait = { workspace = true } +dashmap = { workspace = true } +humantime = { workspace = true } +iggy_common = { workspace = true } iggy_connector_sdk = { workspace = true } -once_cell = { workspace = true } -serde = { workspace = true } -tokio = { workspace = true } -tracing = { workspace = true } -# + your client crate (reqwest, sqlx, mongodb, ...) +once_cell = { workspace = true } +secrecy = { workspace = true } +serde = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +# TODO(ConnectorDeveloper): add your client crate as a workspace dependency ``` -Run `cargo sort --no-format --workspace` after edits. +Run `cargo sort --no-format --workspace` after edits. Keep `Cargo.lock` +churn limited to this crate's deps. + +--- + +## config.toml (example) + +```toml +# Plugin path matches sibling sinks (relative to connectors runtime cwd). +path = "../../target/release/libiggy_connector__sink" + +[[sinks]] +key = "" +enabled = true +# path is also set via IGGY_CONNECTORS_SINK__PATH in integration tests + +[sinks..plugin_config] +# Never commit real secrets. Use env overrides in tests/ops. +connection_string = "scheme://user:pass@host:port/db" +batch_size = 100 +max_retries = 3 +retry_delay = "500ms" +verbose_logging = false +``` + +--- ## src/lib.rs -Code reads top to bottom. Public types first, then `impl`, then private -helpers. +Replace `` / `` and implement only the `TODO(ConnectorDeveloper)` blocks. ```rust -/* Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements... */ // full Apache header +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. use async_trait::async_trait; +use humantime::Duration as HumanDuration; use iggy_connector_sdk::{ ConsumedMessage, Error, MessagesMetadata, Sink, TopicMetadata, sink_connector, }; +use secrecy::{ExposeSecret, SecretString}; use serde::{Deserialize, Serialize}; use std::str::FromStr; use std::time::Duration; use tokio::sync::Mutex; use tracing::{debug, error, info, warn}; -sink_connector!(MySink); // generates FFI symbols + version export +sink_connector!(NameSink); + +const CONNECTOR_NAME: &str = "Name sink"; +const DEFAULT_BATCH_SIZE: u32 = 100; +const DEFAULT_MAX_RETRIES: u32 = 3; // total attempts +const DEFAULT_RETRY_DELAY: &str = "500ms"; -const CONNECTOR_NAME: &str = "My sink"; +/// Backend client. Replace with the real driver type. +struct BackendClient { + // TODO(ConnectorDeveloper): fields +} #[derive(Debug, Serialize, Deserialize)] -pub struct MySinkConfig { - pub endpoint: String, +#[serde(deny_unknown_fields)] +pub struct NameSinkConfig { + #[serde(serialize_with = "iggy_common::serde_secret::serialize_secret")] + pub connection_string: SecretString, pub batch_size: Option, pub max_retries: Option, - pub retry_delay: Option, // humantime, e.g. "500ms" - pub verbose_logging: Option, // mirror runtime's `verbose` flag - // Every runtime-tunable field is Option; defaults applied in new() + pub retry_delay: Option, + pub verbose_logging: Option, + // TODO(ConnectorDeveloper): optional non-secret knobs (table, index, batch_mode, ...) } #[derive(Debug)] -pub struct MySink { +pub struct NameSink { id: u32, - config: MySinkConfig, + config: NameSinkConfig, batch_size: usize, max_retries: u32, retry_delay: Duration, verbose: bool, - client: Option, + client: Option, state: Mutex, } -#[derive(Debug)] +#[derive(Debug, Default)] struct State { messages_processed: u64, errors: u64, } -impl MySink { - pub fn new(id: u32, config: MySinkConfig) -> Self { - let batch_size = config.batch_size.unwrap_or(100) as usize; - let max_retries = config.max_retries.unwrap_or(3); +impl NameSink { + pub fn new(id: u32, config: NameSinkConfig) -> Self { + let batch_size = config.batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize; + let max_retries = config.max_retries.unwrap_or(DEFAULT_MAX_RETRIES); let retry_delay = config .retry_delay .as_deref() - .and_then(|raw| humantime::Duration::from_str(raw).ok().map(|d| *d)) + .and_then(|raw| HumanDuration::from_str(raw).ok().map(|d| *d)) .unwrap_or_else(|| { - warn!("Invalid retry_delay for {CONNECTOR_NAME} ID: {id}, defaulting to 500ms"); + warn!( + "Invalid retry_delay for {CONNECTOR_NAME} ID: {id}, defaulting to {DEFAULT_RETRY_DELAY}" + ); Duration::from_millis(500) }); let verbose = config.verbose_logging.unwrap_or(false); @@ -106,24 +206,49 @@ impl MySink { retry_delay, verbose, client: None, - state: Mutex::new(State { messages_processed: 0, errors: 0 }), + state: Mutex::new(State::default()), + } + } + + async fn send_batch_with_retry( + &self, + client: &BackendClient, + topic_metadata: &TopicMetadata, + batch: &[ConsumedMessage], + ) -> Result<(), Error> { + let mut attempt = 0u32; + loop { + attempt += 1; + match push_batch(client, topic_metadata, batch).await { + Ok(()) => return Ok(()), + Err(error) if is_permanent(&error) => return Err(error), + Err(error) if attempt >= self.max_retries => return Err(error), + Err(error) => { + warn!( + "{CONNECTOR_NAME} ID: {} retry {attempt}/{}: {error}", + self.id, self.max_retries + ); + tokio::time::sleep(self.retry_delay.saturating_mul(attempt)).await; + } + } } } } #[async_trait] -impl Sink for MySink { +impl Sink for NameSink { async fn open(&mut self) -> Result<(), Error> { + // Structural validation belongs here / in new(), not in consume(). let client = build_client(&self.config) .await .map_err(|e| Error::InitError(format!("client build failed: {e}")))?; - client.ping() + ping(&client) .await .map_err(|e| Error::InitError(format!("connectivity check failed: {e}")))?; self.client = Some(client); info!( - "Opened {CONNECTOR_NAME} connector ID: {}, endpoint: {}", - self.id, self.config.endpoint + "Opened {CONNECTOR_NAME} connector ID: {}, endpoint: ", + self.id ); Ok(()) } @@ -140,28 +265,44 @@ impl Sink for MySink { if self.verbose { info!( - "{CONNECTOR_NAME} ID: {} consuming {} messages from stream: {}, topic: {}, offset: {}, current_offset: {}", - self.id, messages.len(), topic_metadata.stream, topic_metadata.topic, - messages_metadata.partition_id, messages_metadata.current_offset + "{CONNECTOR_NAME} ID: {} consuming {} messages from stream: {}, topic: {}, partition_id: {}, current_offset: {}", + self.id, + messages.len(), + topic_metadata.stream, + topic_metadata.topic, + messages_metadata.partition_id, + messages_metadata.current_offset ); } else { debug!( "{CONNECTOR_NAME} ID: {} consuming {} messages", - self.id, messages.len() + self.id, + messages.len() ); } + // Never swallow a failed batch as Ok(()) — offsets may still advance. let mut last_err: Option = None; for batch in messages.chunks(self.batch_size) { - match self.send_batch(client, batch).await { - Ok(()) => { /* counter */ } + match self + .send_batch_with_retry(client, topic_metadata, batch) + .await + { + Ok(()) => { + let mut state = self.state.lock().await; + state.messages_processed += batch.len() as u64; + } Err(Error::PermanentHttpError(message)) => { error!( "{CONNECTOR_NAME} ID: {} dropping batch (permanent): {message}", self.id ); + let mut state = self.state.lock().await; + state.errors += 1; } Err(error) => { + let mut state = self.state.lock().await; + state.errors += 1; last_err = Some(error); } } @@ -173,9 +314,8 @@ impl Sink for MySink { } async fn close(&mut self) -> Result<(), Error> { - // sqlx pools have `.close().await`; reqwest/mongodb/elasticsearch just drop. if let Some(client) = self.client.take() { - let _ = client; // or `client.close().await;` for sqlx + close_client(client).await; } let state = self.state.lock().await; info!( @@ -186,5 +326,98 @@ impl Sink for MySink { } } -async fn build_client(config: &MySinkConfig) -> Result { /* ... */ } +// ─── Backend surface: implement these ─────────────────────────────────────── + +/// TODO(ConnectorDeveloper): parse `config.connection_string.expose_secret()` and build the client. +async fn build_client(config: &NameSinkConfig) -> Result { + let _secret = config.connection_string.expose_secret(); + Err("TODO(ConnectorDeveloper): build_client".into()) +} + +/// TODO(ConnectorDeveloper): cheap connectivity probe used from open(). +async fn ping(_client: &BackendClient) -> Result<(), String> { + Ok(()) +} + +/// TODO(ConnectorDeveloper): push one batch. Prefer a stable dedup key from +/// `stream:topic:partition:message_id` (or backend natural key). +/// Use `message.payload.try_to_bytes()` — do not clone Payload::Json. +async fn push_batch( + _client: &BackendClient, + _topic_metadata: &TopicMetadata, + _batch: &[ConsumedMessage], +) -> Result<(), Error> { + Err(Error::InitError("TODO(ConnectorDeveloper): push_batch".into())) +} + +/// Map driver errors. Never classify via `err.to_string()` substrings. +fn is_permanent(error: &Error) -> bool { + matches!( + error, + Error::PermanentHttpError(_) | Error::SchemaMismatch(_) | Error::InvalidRecordValue(_) + ) +} + +async fn close_client(_client: BackendClient) { + // sqlx: pool.close().await; most HTTP clients: drop +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_config() -> NameSinkConfig { + NameSinkConfig { + connection_string: SecretString::from("scheme://localhost/db"), + batch_size: Some(10), + max_retries: Some(2), + retry_delay: Some("10ms".into()), + verbose_logging: Some(false), + } + } + + #[test] + fn given_defaults_should_apply_consts() { + let sink = NameSink::new( + 1, + NameSinkConfig { + connection_string: SecretString::from("scheme://localhost/db"), + batch_size: None, + max_retries: None, + retry_delay: None, + verbose_logging: None, + }, + ); + assert_eq!(sink.batch_size, DEFAULT_BATCH_SIZE as usize); + assert_eq!(sink.max_retries, DEFAULT_MAX_RETRIES); + } + + #[test] + fn given_invalid_retry_delay_should_fall_back_to_default() { + let mut config = test_config(); + config.retry_delay = Some("not-a-duration".into()); + let sink = NameSink::new(1, config); + assert_eq!(sink.retry_delay, Duration::from_millis(500)); + } +} ``` + +--- + +## README.md (required paragraphs) + +Your README must include a **Delivery semantics** section answering: + +1. Transient failure behavior (retry N times, then `Err`) +2. Permanent failure behavior (drop/skip vs fail) +3. Duplication window (usually at-least-once) +4. Dedup key (or "none — duplicates possible") + +Diff README defaults against the `DEFAULT_*` consts before opening the PR. + +--- + +## Before `/ready` + +Run the pre-flight checklist in +[connector-pr-review](../connector-pr-review/SKILL.md#pre-flight-author-checklist). diff --git a/.claude/skills/connector-source/SKILL.md b/.claude/skills/connector-source/SKILL.md index 9c9eb1ac64..126098b9c4 100644 --- a/.claude/skills/connector-source/SKILL.md +++ b/.claude/skills/connector-source/SKILL.md @@ -9,8 +9,13 @@ A **source** is a Rust `cdylib` that implements `iggy_connector_sdk::Source` and exposes FFI symbols via the `source_connector!` macro. The runtime calls `poll()` in a loop, applies transforms, encodes via the configured `Schema`, sends to -Apache Iggy, and persists the returned `ConnectorState` after every -successful send. +Apache Iggy, and persists the state `poll()` returned - but only after +the send succeeds. Only one batch is ever in flight: the runtime does +not call `poll()` again until it has reported `Ack` or `Nack` for the +current one via `on_batch_result()` (source batch acknowledgment, see PR #3855 +for the full contract). See +[State persistence](#state-persistence-stage-in-poll-commit-in-on_batch_result) +below. > **Universal connector rules** (SecretString, benchmark, verbose flag, drop accounting, filter contract, exemplar patterns) live in > [connectors-overview](../connectors-overview/SKILL.md). This skill @@ -27,14 +32,15 @@ successful send. ## STOP and ask the user before -- Changing the SDK trait surface (`Source::open` / `poll` / `on_batch_result` / `close`) - that's an SDK change. +- Changing the SDK trait surface (`Source::open` / `poll` / `on_batch_result` / `on_batch_result` / `close`) - that's an SDK change, and `poll`/`on_batch_result` are also an FFI change (`iggy_source_handle_v2`, `iggy_source_batch_result` - breaks every pre-built plugin `.so`). - Adding a long-running side task in the plugin - the runtime owns lifecycle, and orphans survive `close()`. Sanctioned only where the source is itself a server the runtime cannot drive, as in `http_source`'s listener, and then only with an explicit shutdown in the last `close()` that awaits its tasks before returning. - Persisting unbounded state - `State` is rewritten every batch. - Adding a source that requires authoritative offsets external to Apache Iggy without coordinating retention. ## Quick reference -- Skeleton: [TEMPLATE.md](TEMPLATE.md) (load on demand). +- Skeleton: [TEMPLATE.md](TEMPLATE.md) (fill-in-the-blank kit — implement only `TODO(ConnectorDeveloper)`). +- PR pre-flight: [connector-pr-review](../connector-pr-review/SKILL.md). - Exemplars: `random_source` (minimal + canonical state tests), `postgres_source` (cursor / delete-after-read / processed-column modes, restart-survives-state tests), `elasticsearch_source` (scroll cursor), `influxdb_source` (time-series scan). ## Hard rules @@ -78,29 +84,61 @@ callback, so rebuild older source plugins. it, rather than re-readable upstream, has to hold the batch itself: see `http_source`'s staging. - Returning `Err` from it stops the source immediately, so it is not a retry signal. -### State persistence - -- `ConnectorState` is `Vec` via MessagePack (`rmp_serde`). Use - `ConnectorState::serialize(&state, NAME, id)` and - `ConnectorState::deserialize::(NAME, id)`. -- `poll()` must not commit cursors or destructive work. Return messages with candidate state and - keep the corresponding work staged. -- The runtime sends the batch, saves its candidate state to - `{state_path}/source_{key}.state`, then calls `on_batch_result(Ack)`. Commit staged in-memory - state and external delete or mark operations only on ACK. A NACK discards the candidate so the - same data can be polled again. -- A crash between `poll()` returning and state persistence leaves the prior cursor for the next - poll, so downstream must tolerate at-least-once delivery. -- Cursor sources that stage state until `on_batch_result` can attach state to the corresponding - message batch. Control-plane state that is independent of a publish can ride an empty batch - instead. -- Return `state: None` for an empty poll when no watermark changed. If an empty poll advances a - watermark, stage and return the new state through the same ACK handshake. -- The runtime can still NACK an empty batch when state storage is latched or a pending checkpoint - cannot resolve. Never treat hand-off as durable; `on_batch_result` reports the outcome. -- Treat candidate-state serialization failure as a poll error. Do not send messages without the - state needed to resume them safely. +### State persistence: stage in `poll()`, commit in `on_batch_result()` + +Source connectors use a one-in-flight-batch ACK/NACK contract (#3855) +between the plugin and the runtime: + +1. `poll()` returns messages and *candidate* state without committing + cursor changes or destructive operations (deletes, mark-processed). +2. The runtime sends the batch to Apache Iggy and waits for the + producer result. +3. After a successful send, the runtime persists the candidate state + to `{state_path}/source_{key}.state`. +4. The runtime calls `on_batch_result(SourceBatchResult::Ack)`. A send + or state-save failure calls `on_batch_result(SourceBatchResult::Nack)` + instead. +5. `on_batch_result()` commits or discards the plugin's staged work + before the next `poll()` starts. The SDK allows only one batch in + flight - it will not call `poll()` again until `on_batch_result()` + for the current batch has returned. + +Canonical pattern (`sources/random_source/src/lib.rs`): + +```rust +pending_state: Mutex>, // staged, not yet committed + +async fn poll(&self) -> Result { + // ... fetch ... + let candidate_state = State { cursor: next_cursor }; + *self.pending_state.lock().await = Some(candidate_state.clone()); + Ok(ProducedMessages { + schema: Schema::Json, + messages, + state: Some(ConnectorState::serialize(&candidate_state, NAME, self.id)?), + }) +} + +async fn on_batch_result(&self, result: SourceBatchResult) -> Result<(), Error> { + let candidate_state = self.pending_state.lock().await.take(); + if result == SourceBatchResult::Ack + && let Some(candidate_state) = candidate_state + { + *self.state.lock().await = candidate_state; + } + // Nack: drop candidate_state, committed self.state is untouched - + // the same range is polled again. + Ok(()) +} +``` + +- `ConnectorState` is `Vec` via MessagePack (`rmp_serde`). Use `ConnectorState::serialize(&state, NAME, id)` + `ConnectorState::deserialize::(NAME, id)`. Both return `Option` and log on failure (non-fatal). +- **The default `on_batch_result` is a no-op.** Only override it - and only then does staging via `pending_state` matter - if `poll()` advances a cursor or performs destructive work (delete-after-read, mark-processed). A source with no staged work (e.g. a pure generator) can rely on the default. +- Returning `Err` from `on_batch_result` **stops the SDK from polling further** - a failed rollback must not be allowed to silently advance to the next batch. +- **Always return state in every `ProducedMessages`**, including empty polls that made progress. Return `state: None` for an empty poll that made *no* progress - this avoids an unnecessary state write and cannot persist state left over from a failed batch. - Keep `State` small - rewritten every batch. No unbounded vecs. +- NACK handling must discard staged cursor changes and staged delete/mark operations so polling redelivers the batch. The SDK retries NACKed batches with capped exponential backoff and stops the source after repeated consecutive NACKs. +- Crash recovery is at-least-once at every point except after the plugin has processed the ACK (see the SDK README's crash-point table, `core/connectors/sdk/README.md#source-delivery-acknowledgment`, for the full breakdown). The SDK allows one in-flight batch. Five consecutive NACKs stop a source using the default policy and require a manual restart. Returning `Err` from @@ -134,18 +172,20 @@ Match `ProducedMessages.schema` to the bytes in `messages[i].payload`: ### Concurrency - Runtime spawns ONE `poll()` task per source. No concurrent `poll()`. +- Only one batch is ever in flight: the SDK does not call `poll()` again until `on_batch_result()` has returned for the previous batch (up to a 30s result timeout, after which the SDK treats it as a Nack). - Don't spawn your own long-running Tokio tasks: the runtime owns lifecycle. The exception is a source that listens rather than polls, which has to own its listener; `http_source` is the worked example, and it shuts its tasks down in the last `close()` rather than leaving them to outlive the connector. ### Errors -| Scenario | Variant | -| ------------------------------------------- | ------------------------------------------------- | -| Bad config in `new()`/`open()` | `Error::InitError` | -| Cannot reach external system at startup | `Error::InitError` or `Error::Connection` | -| Transient fetch failure (retry-worthy) | `Error::Connection` or `Error::HttpRequestFailed` | -| Permanent fetch failure (auth, schema gone) | `Error::PermanentHttpError` | -| Row failed to serialize | `Error::Serialization(...)` | -| State serialization failed | `Error::Serialization(...)` | +| Scenario | Variant | +| --------------------------------------------------- | ------------------------------------------------- | +| Bad config in `new()`/`open()` | `Error::InitError` | +| Cannot reach external system at startup | `Error::InitError` or `Error::Connection` | +| Transient fetch failure (retry-worthy) | `Error::Connection` or `Error::HttpRequestFailed` | +| Permanent fetch failure (auth, schema gone) | `Error::PermanentHttpError` | +| Row failed to serialize | `Error::Serialization(...)` | +| State serialization failed | log + skip (non-fatal) | +| `on_batch_result()` failed to roll back staged work | `Err` - stops the SDK from polling further | Returning `Err` from `poll()` is only logged by the SDK's FFI bridge (`sdk/src/source.rs::handle_messages`) - the loop continues, the next @@ -175,15 +215,20 @@ Iggy consumer-loop labels use literal API names (`offset=`, `current_offset=`). 1. `async fn poll(&mut self)` - won't compile. Use `&self` + `Mutex`. 2. Holding `state.lock()` across the fetch I/O - blocks `close()`, causes shutdown timeouts. 3. Forgetting to sleep - 100% CPU on idle source. -4. Committing a cursor or deleting source data in `poll()` - stage it and wait for ACK. -5. Unbounded data in `State` - rewritten every batch. keep O(constant). -6. `std::sync::Mutex` - blocks the executor. Use `tokio::sync::Mutex`. -7. Not setting `ProducedMessage.id` when a stable ID exists - leaves a consumer nothing to dedupe a replayed duplicate on. It does not make the write idempotent server-side, because nothing there reads it. -8. Spawning side tasks - the runtime owns the scheduler. The one exception is a source that listens rather than polls and must own its listener (see [Concurrency](#concurrency) and the STOP list); it owes an explicit shutdown in the last `close()` that awaits its tasks. +4. Returning `state: None` for an empty poll that *did* make progress (e.g. advanced a watermark) - only a no-progress empty poll should return `None`. +5. Committing a cursor or destructive work (delete/mark-processed) directly in `poll()` instead of staging it and applying it in `on_batch_result()` on `Ack` - a Nack (send or state-save failure) has nothing to discard, and the batch is redelivered against already-mutated state. +6. Unbounded data in `State` - rewritten every batch. keep O(constant). +7. `std::sync::Mutex` - blocks the executor. Use `tokio::sync::Mutex`. +8. Not setting `ProducedMessage.id` when a stable ID exists - loses idempotency. +9. Spawning side tasks - the runtime owns the scheduler. ## Tests -Mandatory four canonical source state tests (see [connector-testing](../connector-testing/SKILL.md) for the full pattern). Copy from `sources/random_source/src/lib.rs::tests`. Plus config defaults, payload building, schema selection. +Mandatory six canonical source tests (see [connector-testing](../connector-testing/SKILL.md) for the full pattern): the four +state tests (restore / no-state / invalid-state / round-trip) plus `given_ack_when_batch_is_staged_should_commit_candidate_state` +and `given_nack_when_batch_is_staged_should_keep_committed_state`. Copy from `sources/random_source/src/lib.rs::tests`. Plus +config defaults, payload building, schema selection. A source relying on the default no-op `on_batch_result` (no staged work) +may skip the ack/nack pair. Integration tests under `core/integration/tests/connectors//` for any source backed by external infra. Use `#[iggy_harness]` + a `TestFixture` backed by `testcontainers-modules`. Reference: `core/integration/tests/connectors/postgres/postgres_source.rs` (multi-mode tests) + `restart.rs` (state survives restart). Exercise both ACK and NACK paths when the source stages cursors or destructive work. diff --git a/.claude/skills/connector-source/TEMPLATE.md b/.claude/skills/connector-source/TEMPLATE.md index 04b8c6062e..7b3931f6f7 100644 --- a/.claude/skills/connector-source/TEMPLATE.md +++ b/.claude/skills/connector-source/TEMPLATE.md @@ -1,64 +1,134 @@ -# Source plugin skeleton +# Source plugin fill-in-the-blank kit -Boilerplate for a new `core/connectors/sources/_source/`. Adapt -the `MySource` / `Client` / row types to the backend driver. +Copy this kit into `core/connectors/sources/_source/`. The +scaffolding covers config, secrets, sleep-first poll, lock discipline, +staged/committed state serde (ACK/NACK batch acknowledgment, #3855), +retry classification, logging, and the six canonical state tests. +**You only implement the marked `TODO(ConnectorDeveloper)` sections:** build a +client from the connection string, and fetch the next batch (advancing +a cursor). -`Cargo.toml` is identical to a sink's (see -[connector-sink/TEMPLATE.md](../connector-sink/TEMPLATE.md)) - only -the crate name suffix changes (`iggy_connector__source`) and the -upstream client dep. +Also read [SKILL.md](SKILL.md) and pre-flight with +[connector-pr-review](../connector-pr-review/SKILL.md) before `/ready`. + +Prefer starting from a compiling crate over copying this prose kit: +`core/connectors/sources/source_template/` implements the same shape as +real, tested code you can `cargo build`/`cargo test` immediately, with the +same `TODO(ConnectorDeveloper)` markers. Use this kit instead only when +copying a whole crate is more scaffolding than you need. + +## Files to create + +```text +core/connectors/sources/_source/ +├── Cargo.toml +├── README.md +├── config.toml +└── src/lib.rs +``` + +Add a workspace member, a row in `sources/README.md`, and a sample under +`runtime/example_config/connectors/`. + +--- + +## Cargo.toml + +Same shape as the sink kit (`cdylib` + `lib`, workspace deps, Apache +header). Only the package name suffix changes: + +```toml +name = "iggy_connector__source" +# ... identical metadata / machete ignored / crate-type ... +# TODO(ConnectorDeveloper): add your client crate as a workspace dependency +``` + +--- + +## config.toml (example) + +```toml +path = "../../target/release/libiggy_connector__source" + +[[sources]] +key = "" +enabled = true + +[sources..plugin_config] +connection_string = "scheme://user:pass@host:port/db" +poll_interval = "5s" +batch_size = 100 +max_retries = 3 +retry_delay = "500ms" +verbose_logging = false +``` + +Defaults in this file must match `DEFAULT_*` consts in code. + +--- ## src/lib.rs -Code reads top to bottom. Public types first, `impl Source` after, -helpers below. +Replace `` / `` and implement only the `TODO(ConnectorDeveloper)` blocks. ```rust /* Apache 2.0 header */ -use std::str::FromStr; -use std::time::Duration; - use async_trait::async_trait; +use humantime::Duration as HumanDuration; use iggy_connector_sdk::{ ConnectorState, Error, ProducedMessage, ProducedMessages, Schema, Source, source::SourceBatchResult, source_connector, }; +use secrecy::{ExposeSecret, SecretString}; use serde::{Deserialize, Serialize}; use tokio::sync::Mutex; use tokio::time::sleep; use tracing::{debug, error, info, warn}; -source_connector!(MySource); +source_connector!(NameSource); -const CONNECTOR_NAME: &str = "My source"; +const CONNECTOR_NAME: &str = "Name source"; +const DEFAULT_POLL_INTERVAL: &str = "5s"; +const DEFAULT_BATCH_SIZE: u32 = 100; +const DEFAULT_MAX_RETRIES: u32 = 3; // total attempts +const DEFAULT_RETRY_DELAY: &str = "500ms"; + +struct BackendClient { + // TODO(ConnectorDeveloper): fields +} #[derive(Debug, Serialize, Deserialize)] -pub struct MySourceConfig { - pub endpoint: String, - pub poll_interval: Option, // humantime, "10s" default +#[serde(deny_unknown_fields)] +pub struct NameSourceConfig { + #[serde(serialize_with = "iggy_common::serde_secret::serialize_secret")] + pub connection_string: SecretString, + pub poll_interval: Option, pub batch_size: Option, pub max_retries: Option, pub retry_delay: Option, pub verbose_logging: Option, + // TODO(ConnectorDeveloper): optional non-secret knobs (query, table, index, ...) } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] struct State { - cursor: Option, // WAL LSN, scroll id, timestamp, ... - last_offset: u64, + /// Opaque backend cursor (LSN, scroll id, timestamp, PK, ...). Keep O(1). + cursor: Option, messages_produced: u64, } #[derive(Debug)] -pub struct MySource { +pub struct NameSource { id: u32, - config: MySourceConfig, + config: NameSourceConfig, poll_interval: Duration, + batch_size: usize, + max_retries: u32, + retry_delay: Duration, verbose: bool, - client: Option, + client: Option, state: Mutex, - pending: Mutex>, } impl MySource { @@ -67,45 +137,65 @@ impl MySource { let poll_interval = humantime::Duration::from_str(&raw_interval) .map(|d| *d) .unwrap_or_else(|_| { - warn!("Invalid poll_interval for {CONNECTOR_NAME} ID: {id}, defaulting to 10s"); - Duration::from_secs(10) + warn!( + "Invalid poll_interval for {CONNECTOR_NAME} ID: {id}, defaulting to {DEFAULT_POLL_INTERVAL}" + ); + Duration::from_secs(5) + }); + let batch_size = config.batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize; + let max_retries = config.max_retries.unwrap_or(DEFAULT_MAX_RETRIES); + let retry_delay = config + .retry_delay + .as_deref() + .and_then(|raw| HumanDuration::from_str(raw).ok().map(|d| *d)) + .unwrap_or_else(|| { + warn!( + "Invalid retry_delay for {CONNECTOR_NAME} ID: {id}, defaulting to {DEFAULT_RETRY_DELAY}" + ); + Duration::from_millis(500) }); - let verbose = config.verbose_logging.unwrap_or(false); let restored = state .and_then(|s| s.deserialize::(CONNECTOR_NAME, id)) - .inspect(|s| info!( - "Restored state for {CONNECTOR_NAME} ID: {id}, last_offset: {}, cursor: {:?}", - s.last_offset, s.cursor - )); + .inspect(|s| { + info!( + "Restored state for {CONNECTOR_NAME} ID: {id}, cursor: {:?}, messages_produced: {}", + s.cursor, s.messages_produced + ); + }); Self { id, config, poll_interval, + batch_size, + max_retries, + retry_delay, verbose, client: None, state: Mutex::new(restored.unwrap_or(State { cursor: None, - last_offset: 0, messages_produced: 0, })), - pending: Mutex::new(None), } } } #[async_trait] -impl Source for MySource { +impl Source for NameSource { async fn open(&mut self) -> Result<(), Error> { + // Validate query/cursor shape here — not on first poll after sleep. let client = build_client(&self.config) .await .map_err(|e| Error::InitError(format!("client build failed: {e}")))?; + ping(&client) + .await + .map_err(|e| Error::InitError(format!("connectivity check failed: {e}")))?; self.client = Some(client); info!( - "Opened {CONNECTOR_NAME} connector ID: {}, endpoint: {}", - self.id, self.config.endpoint + "Opened {CONNECTOR_NAME} connector ID: {}, endpoint: ", + self.id ); Ok(()) } @@ -113,9 +203,9 @@ impl Source for MySource { async fn poll(&self) -> Result { sleep(self.poll_interval).await; // sleep first - backpressure - let mut candidate = self.state.lock().await.clone(); + let cursor = { self.state.lock().await.cursor.clone() }; // brief read - let fetched = self.fetch_since(candidate.cursor.as_deref()).await?; + let fetched = self.fetch_since(cursor.as_deref()).await?; // no lock held let mut messages = Vec::with_capacity(fetched.len()); let mut next_cursor = None; @@ -136,45 +226,32 @@ impl Source for MySource { if self.verbose { info!( - "{CONNECTOR_NAME} ID: {} produced {} messages, next cursor: {:?}", - self.id, messages.len(), next_cursor + "{CONNECTOR_NAME} ID: {} polled {} messages, next_cursor: {:?}", + self.id, + fetched.messages.len(), + fetched.next_cursor ); } - if messages.is_empty() { - return Ok(ProducedMessages { - schema: Schema::Json, - messages, - state: None, - }); - } - - candidate.messages_produced += messages.len() as u64; - candidate.cursor = next_cursor; - let persisted = ConnectorState::serialize(&candidate, CONNECTOR_NAME, self.id) - .ok_or_else(|| Error::Serialization("failed to serialize source state".into()))?; - *self.pending.lock().await = Some(candidate); + let persisted = { // brief write + let mut state = self.state.lock().await; + state.messages_produced += messages.len() as u64; + if let Some(c) = next_cursor { + state.cursor = Some(c); + } + ConnectorState::serialize(&*state, CONNECTOR_NAME, self.id) + }; Ok(ProducedMessages { schema: Schema::Json, messages, - state: Some(persisted), + state: persisted, }) } - async fn on_batch_result(&self, result: SourceBatchResult) -> Result<(), Error> { - let (SourceBatchResult::Ack, Some(candidate)) = - (result, self.pending.lock().await.take()) - else { - return Ok(()); - }; - *self.state.lock().await = candidate; - Ok(()) - } - async fn close(&mut self) -> Result<(), Error> { if let Some(client) = self.client.take() { - let _ = client; // or `client.close().await;` for sqlx pools + close_client(client).await; } let state = self.state.lock().await; info!( @@ -185,5 +262,187 @@ impl Source for MySource { } } -async fn build_client(config: &MySourceConfig) -> Result { /* ... */ } +// ─── Backend surface: implement these ─────────────────────────────────────── + +/// TODO(ConnectorDeveloper): parse `config.connection_string.expose_secret()` and build the client. +async fn build_client(config: &NameSourceConfig) -> Result { + let _secret = config.connection_string.expose_secret(); + Err("TODO(ConnectorDeveloper): build_client".into()) +} + +/// TODO(ConnectorDeveloper): cheap connectivity probe used from open(). +async fn ping(_client: &BackendClient) -> Result<(), String> { + Ok(()) +} + +/// TODO(ConnectorDeveloper): fetch up to `limit` rows after `cursor`. +/// Set `ProducedMessage.id` from a stable natural key (never random UUID). +/// Set `origin_timestamp` when the backend has event time (nanoseconds). +async fn fetch_batch( + _client: &BackendClient, + _cursor: Option<&str>, + _limit: usize, +) -> Result { + Err(Error::InitError("TODO(ConnectorDeveloper): fetch_batch".into())) +} + +fn is_permanent(error: &Error) -> bool { + matches!( + error, + Error::PermanentHttpError(_) | Error::SchemaMismatch(_) | Error::InvalidConfigValue(_) + ) +} + +async fn close_client(_client: BackendClient) {} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_config() -> NameSourceConfig { + NameSourceConfig { + connection_string: SecretString::from("scheme://localhost/db"), + poll_interval: Some("100ms".into()), + batch_size: Some(10), + max_retries: Some(2), + retry_delay: Some("10ms".into()), + verbose_logging: Some(false), + } + } + + #[test] + fn given_persisted_state_should_restore_cursor() { + let state = State { + cursor: Some("cursor-1".into()), + messages_produced: 7, + }; + let bytes = rmp_serde::to_vec(&state).expect("serialize"); + let source = NameSource::new(1, test_config(), Some(ConnectorState(bytes))); + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let restored = source.state.lock().await; + assert_eq!(restored.cursor.as_deref(), Some("cursor-1")); + assert_eq!(restored.messages_produced, 7); + }); + } + + #[test] + fn given_no_state_should_start_fresh() { + let source = NameSource::new(1, test_config(), None); + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let restored = source.state.lock().await; + assert!(restored.cursor.is_none()); + assert_eq!(restored.messages_produced, 0); + }); + } + + #[test] + fn given_invalid_state_should_start_fresh() { + let invalid = ConnectorState(b"not valid msgpack".to_vec()); + let source = NameSource::new(1, test_config(), Some(invalid)); + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let restored = source.state.lock().await; + assert!(restored.cursor.is_none()); + assert_eq!(restored.messages_produced, 0); + }); + } + + #[test] + fn state_should_be_serializable_and_deserializable() { + let original = State { + cursor: Some("c".into()), + messages_produced: 3, + }; + let bytes = rmp_serde::to_vec(&original).unwrap(); + let restored: State = rmp_serde::from_slice(&bytes).unwrap(); + assert_eq!(original, restored); + } + + #[test] + fn given_defaults_should_apply_consts() { + let source = NameSource::new( + 1, + NameSourceConfig { + connection_string: SecretString::from("scheme://localhost/db"), + poll_interval: None, + batch_size: None, + max_retries: None, + retry_delay: None, + verbose_logging: None, + }, + None, + ); + assert_eq!(source.batch_size, DEFAULT_BATCH_SIZE as usize); + assert_eq!(source.max_retries, DEFAULT_MAX_RETRIES); + assert_eq!(source.poll_interval, Duration::from_secs(5)); + } + + // Stages `pending_state` directly rather than going through poll() - poll()'s + // TODO(ConnectorDeveloper) fetch is unimplemented in this template, but on_batch_result's + // commit/discard contract is independently testable and must stay covered once + // the backend is filled in. + + #[test] + fn given_ack_when_batch_is_staged_should_commit_candidate_state() { + let source = NameSource::new(1, test_config(), None); + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let candidate = State { + cursor: Some("cursor-2".into()), + messages_produced: 5, + }; + *source.pending_state.lock().await = Some(candidate.clone()); + + source + .on_batch_result(SourceBatchResult::Ack) + .await + .expect("ack should be applied"); + + assert_eq!(*source.state.lock().await, candidate); + assert!(source.pending_state.lock().await.is_none()); + }); + } + + #[test] + fn given_nack_when_batch_is_staged_should_keep_committed_state() { + let source = NameSource::new(1, test_config(), None); + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let committed_before = source.state.lock().await.clone(); + *source.pending_state.lock().await = Some(State { + cursor: Some("cursor-2".into()), + messages_produced: 5, + }); + + source + .on_batch_result(SourceBatchResult::Nack) + .await + .expect("nack should be applied"); + + assert_eq!(*source.state.lock().await, committed_before); + assert!(source.pending_state.lock().await.is_none()); + }); + } +} ``` + +--- + +## README.md (required paragraphs) + +Include a **Delivery semantics** section: + +1. Transient fetch failure → retry N times, then `Err` (loop continues; see `connector-source`) +2. Cursor commits only on `on_batch_result(Ack)` - a Nack (send or state-save failure) discards the staged cursor and redelivers the batch +3. Whether destructive work (delete/mark-processed) is staged in `poll()` and applied only in `on_batch_result()` on Ack (standard - no loss window to document), or must happen earlier for some architectural reason (if so, document the loss window explicitly) +4. Dedup key for `ProducedMessage.id` (or "none") + +--- + +## Before `/ready` + +Run the pre-flight checklist in +[connector-pr-review](../connector-pr-review/SKILL.md#pre-flight-author-checklist). +Mandatory: the six canonical state tests above must stay green. diff --git a/.claude/skills/connector-testing/SKILL.md b/.claude/skills/connector-testing/SKILL.md index 9e6669e057..fcbbbab75a 100644 --- a/.claude/skills/connector-testing/SKILL.md +++ b/.claude/skills/connector-testing/SKILL.md @@ -100,18 +100,23 @@ fn test_config() -> RandomSourceConfig { ### Async testing pattern -The trait methods are `async`. For unit tests, build a runtime locally - don't add `#[tokio::test]` in `src/lib.rs` (it pulls test macros into prod build): +The trait methods are `async`. Prefer `#[tokio::test]` directly on the test fn - `#[cfg(test)]` +gates the whole module out of a plain `cargo build`, and resolver v2 (default since edition 2021, +which this workspace uses) keeps `tokio`'s `test-util`/`macros` dev-dependency features out of that +build too, so there's no prod-build leak to avoid. This is also the dominant pattern in-tree +(`postgres_source::tests` uses `#[tokio::test]` throughout, no manual runtime): ```rust -#[test] -fn given_X_should_Y() { - let runtime = tokio::runtime::Runtime::new().unwrap(); - runtime.block_on(async { - // await calls here - }); +#[tokio::test] +async fn given_x_should_y() { + // await calls directly, no manual Runtime::new() } ``` +A manual `tokio::runtime::Runtime::new().unwrap().block_on(...)` inside a plain `#[test]` still +works and you'll see it in some older plugin tests (`random_source`), but don't ask for it in +review - it's equivalent, not a correctness requirement. + ### Mandatory: four canonical source state tests Every source must have these. Copy from `sources/random_source/src/lib.rs::tests`: diff --git a/.claude/skills/connectors-overview/SKILL.md b/.claude/skills/connectors-overview/SKILL.md index 230ad5354b..a3a3575eeb 100644 --- a/.claude/skills/connectors-overview/SKILL.md +++ b/.claude/skills/connectors-overview/SKILL.md @@ -7,6 +7,8 @@ description: Entry point and index for the Apache Iggy Connectors subsystem (`co Repo-wide rules (Apache headers, fmt/sort/clippy order, idiomatic Rust traits, imports at top, no `cargo install`, LLM-slop tells) live in [AGENTS.md](../../../AGENTS.md). This file owns **connector-specific** rules and routes to the per-area skills. +New sink/source authors: start with [`core/connectors/docs/authoring-sinks-and-sources.md`](../../../core/connectors/docs/authoring-sinks-and-sources.md) for the end-to-end checklist, then copy `sinks/sink_template` or `sources/source_template`. + ## Contents - [STOP and ask the user before](#stop-and-ask-the-user-before) @@ -25,7 +27,7 @@ Repo-wide rules (Apache headers, fmt/sort/clippy order, idiomatic Rust traits, i ## STOP and ask the user before -- Bumping `iggy_connector_sdk` MAJOR version or changing any FFI signature in `sdk/src/{sink,source}.rs` - breaks every pre-built plugin `.so`. +- Bumping `iggy_connector_sdk` MAJOR version or changing any FFI signature in `sdk/src/{sink,source}.rs` - breaks every pre-built plugin `.so`. Source FFI is `iggy_source_handle_v2` (batch ID carried to the runtime callback) + `iggy_source_batch_result` (plugin-exported ACK/NACK, #3855). - Changing the runtime's wire conventions (postcard FFI payload structs, default consumer group naming, plugin path resolution). - Modifying `runtime/src/state.rs` save protocol (atomic rename + fsync ordering) - corruption risk. - Renaming or repurposing a `Schema` variant - decoders/encoders pinned to wire bytes. @@ -42,27 +44,31 @@ Both compile as `cdylib` shared libraries (`.so`/`.dylib`/`.dll`) loaded by the ```text ┌─ optional transforms ─┐ External ──poll──▶ SOURCE ──FFI──▶ RUNTIME ──encode──▶ Apache Iggy stream - system plugin ▲ ▲ - │ │ - state save (msgpack) │ - │ - ┌─ optional transforms ─┐ │ + system plugin ▲ ▲ + │ │ + Ack/Nack (on_batch_result) │ + │ state save (msgpack) + └────────────────────┘ + ┌─ optional transforms ─┐ Apache Iggy stream ──decode──▶ RUNTIME ──FFI──▶ SINK ──write──▶ External plugin system ``` +Source is a request/response FFI, not fire-and-forget: after the runtime sends the batch and saves the state `poll()` staged, it reports `SourceBatchResult::Ack` or `::Nack` back to the plugin via `iggy_source_batch_result` (#3855) before calling `poll()` again. See [connector-source](../connector-source/SKILL.md#state-persistence-stage-in-poll-commit-in-on_batch_result) for the full handshake. + Headers set on the source side ride through transforms (which may modify, drop, or pass them) and arrive at the sink with `BTreeMap` preserved deterministically. ## Which skill to load -| Task | Skill | -| --------------------------------------------------- | --------------------- | -| Write a new sink plugin | `connector-sink` | -| Write a new source plugin | `connector-source` | -| Add schema / decoder / encoder / SDK trait surface | `connector-sdk` | -| Change runtime internals (FFI, manager, state, ...) | `connector-runtime` | -| Add a transform (field-level or format conversion) | `connector-transform` | -| Write unit / integration tests for any of the above | `connector-testing` | +| Task | Skill | +| ---------------------------------------------------- | --------------------- | +| Write a new sink plugin | `connector-sink` | +| Write a new source plugin | `connector-source` | +| Add schema / decoder / encoder / SDK trait surface | `connector-sdk` | +| Change runtime internals (FFI, manager, state, ...) | `connector-runtime` | +| Add a transform (field-level or format conversion) | `connector-transform` | +| Write unit / integration tests for any of the above | `connector-testing` | +| Review a sink/source PR / pre-flight before `/ready` | `connector-pr-review` | ## Stick to conventions @@ -96,24 +102,14 @@ The connectors codebase is intentionally repetitive across plugins. Cross-plugin ### Secrets -Any credential-bearing field (connection strings, API keys, bearer tokens, AWS keys) must be `SecretString` from the `secrecy` crate. Plain `String` for a credential is a review-blocker: `SecretString` redacts on `Debug`, so it is what keeps a credential out of a log line that formats the whole config. - -**`serde_secret::serialize_secret` EXPOSES the secret. It does not redact.** It calls `expose_secret()` and writes the plaintext. `SecretString` deliberately has no `Serialize` impl, and that absence is the protection - so adding `serialize_with` is what *unblocks* the derive and turns a compile-time guarantee into plaintext output. Use it only where the plaintext is the point: a wire payload, a persisted config, an API response that exposes credentials by design. - -So the default for a plugin config struct is **derive `Deserialize`, but not `Serialize`**. `Deserialize` is required: the SDK glue deserializes the config into the plugin's own struct (`sdk/src/{sink,source}.rs` call `serde_json::from_str::` under a `DeserializeOwned` bound). - -What never happens is the return trip. The runtime holds plugin configuration as a `serde_json::Value` - parsed from TOML, posted as JSON to the control API, or injected by env var - and hands that across the FFI, so nothing re-serializes the plugin's struct. Leaving `Serialize` off makes that compiler-enforced instead of convention-enforced (`sources/http_source/src/lib.rs::HttpSourceConfig` does this, and comments the omission so nobody adds it back). - -Pattern: +Any credential-bearing field (connection strings, API keys, bearer tokens, AWS keys) must be `SecretString` from the `secrecy` crate, with the workspace serde wrapper applied so `Debug` and serialization both redact. Runtime exposes plugin configs over the `/stats` HTTP surface via serialization - plain `String` leaks the secret to anyone who can hit the endpoint. Plain `String` for a credential is a review-blocker. Pattern (from `sinks/postgres_sink/src/lib.rs::PostgresSinkConfig`): ```rust use secrecy::{ExposeSecret, SecretString}; -// `Deserialize` only. Nothing re-serializes a plugin config, and leaving -// `Serialize` off is what makes the credential unserializable rather than -// merely un-serialized. -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct MyConfig { + #[serde(serialize_with = "iggy_common::serde_secret::serialize_secret")] pub connection_string: SecretString, } @@ -123,13 +119,7 @@ let pool = PgPoolOptions::new() .await?; ``` -If a config struct genuinely needs `Serialize`, `serde_secret::serialize_redacted` (and `serialize_optional_redacted`) write `[REDACTED]` in place of the value. Reach for `serialize_secret` only when the caller must get the real thing back. The sinks and sources listed below predate that helper and use the exposing one; the annotation is inert today, but it is not the protection it looks like. - -Note that none of this protects the credential from the runtime's own control API, which returns plugin configuration verbatim - see #3802. Plugin-side annotations are inert there because the runtime never routes through them. - -Plugin-side uses of the exposing helpers: `sinks/{postgres,mongodb,elasticsearch,influxdb,s3,surrealdb}_sink`, `sources/{postgres,elasticsearch,influxdb}_source`. - -That list is plugin-side only, not an inventory of every caller in the tree, and the others are not all mistakes: `runtime/src/api/config.rs` puts `serialize_secret` on `HttpConfig::api_key` (inert for the same reason), and several `core/common` wire-payload types (login, create-user, change-password, PAT) use these helpers by design, because there the credential *is* the payload. +In-tree uses: `sinks/{postgres,mongodb,elasticsearch,influxdb,delta}_sink`, `sources/{postgres,elasticsearch,influxdb}_source`. ### Errors @@ -181,7 +171,7 @@ JSON log format, parser unit tests). | Real-infra sink + integration tests | `sinks/postgres_sink/` + `integration/tests/connectors/postgres/postgres_sink.rs` | | Feature-rich sink config (validation, batch modes, retry) | `sinks/http_sink/` | | Atomic counters on hot path | `sinks/mongodb_sink/` | -| Simplest source (4 canonical state tests) | `sources/random_source/` | +| Simplest source (6 canonical state + ACK/NACK tests) | `sources/random_source/` | | Real-infra source | `sources/postgres_source/` + `integration/tests/connectors/postgres/postgres_source.rs` | Read the relevant exemplar end-to-end before writing or modifying a connector. @@ -209,7 +199,7 @@ Each implemented in at least one in-tree plugin or runtime path. | `flume::unbounded()` channel | `runtime/src/source.rs::spawn_source_handler` / `source_forwarding_loop` | MPSC handoff from SDK async task to runtime loop | | `tokio::sync::watch::channel(())` | `sdk/src/{sink,source}.rs`, `runtime/src/sink.rs`, `runtime/src/manager/*` | One-shot shutdown broadcast | | `dashmap::DashMap` | `runtime/src/manager/sink.rs`, `source.rs::SOURCE_SENDERS`, SDK `INSTANCES` | Lock-free concurrent keyed access | -| `secrecy::SecretString` + `iggy_common::serde_secret::serialize_secret` | `sinks/postgres_sink::PostgresSinkConfig::connection_string` | `Debug` redacts; `serialize_secret` EXPOSES | +| `secrecy::SecretString` + `iggy_common::serde_secret::serialize_secret` | `sinks/postgres_sink::PostgresSinkConfig::connection_string` | Auto-redact on Debug/Display + serialization | ## Drop accounting @@ -222,13 +212,21 @@ Each implemented in at least one in-tree plugin or runtime path. - `&mut self` on `Sink::consume` or `Source::poll` impls - won't compile, flag any creative workaround. - `std::sync::Mutex` held across `.await` - swap for `tokio::sync::Mutex`. - Missing `[lib] crate-type = ["cdylib", "lib"]` in plugin `Cargo.toml`. -- Source plugin without the four canonical state tests (see `connector-testing`). +- Source plugin without the four canonical state tests, or without ACK/NACK tests when `on_batch_result` is overridden (see `connector-testing`). +- Source `poll()` committing a cursor or destructive work directly instead of staging it for `on_batch_result` to commit on `Ack` / discard on `Nack`. - New silent message drop without a metric increment. - Wrapping `format!()` around args passed to `error!`/`warn!`/`info!`/`debug!` - eager `format!` allocates even when level filters the line out. Pass args directly: `error!("foo: {x}")` or `error!(error = %x, "foo")`. - Logging a connection string, API key, or token. - Plain `String` for a credential field - use `SecretString`. - `tokio::spawn` inside plugin code - runtime owns lifecycle. - `std::time::SystemTime::now()` in transforms - non-deterministic, breaks tests. +- Returning `Ok(())` from `consume` after a failed batch (offsets can still advance). +- Random UUIDs as message IDs / dedup keys. +- Classifying retryability via `err.to_string()` substring matches. +- README defaults that disagree with code consts. +- Invented config knob names (e.g. `request_timeout` instead of `timeout`) instead of the canon in `connector-pr-review`. + +For the full PR review checklist (blockers, delivery-semantics paragraph, pre-flight paste), load [connector-pr-review](../connector-pr-review/SKILL.md). ## File map diff --git a/Cargo.lock b/Cargo.lock index 7901da9b03..8738cb0ff2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7881,6 +7881,41 @@ dependencies = [ "tracing", ] +[[package]] +name = "iggy_connector_template_sink" +version = "0.1.0" +dependencies = [ + "async-trait", + "dashmap", + "iggy_common", + "iggy_connector_sdk", + "reqwest 0.13.5", + "reqwest-middleware", + "secrecy", + "serde", + "tokio", + "tracing", +] + +[[package]] +name = "iggy_connector_template_source" +version = "0.1.0" +dependencies = [ + "async-trait", + "dashmap", + "humantime", + "iggy_common", + "iggy_connector_sdk", + "reqwest 0.13.5", + "reqwest-middleware", + "rmp-serde", + "secrecy", + "serde", + "serde_json", + "tokio", + "tracing", +] + [[package]] name = "iggy_examples" version = "0.0.6" diff --git a/Cargo.toml b/Cargo.toml index 96a1a62b51..0c6d0d8eac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ members = [ "core/connectors/sinks/rabbitmq_sink", "core/connectors/sinks/redshift_sink", "core/connectors/sinks/s3_sink", + "core/connectors/sinks/sink_template", "core/connectors/sinks/stdout_sink", "core/connectors/sinks/surrealdb_sink", "core/connectors/sources/elasticsearch_source", @@ -56,6 +57,7 @@ members = [ "core/connectors/sources/mqtt_source", "core/connectors/sources/postgres_source", "core/connectors/sources/random_source", + "core/connectors/sources/source_template", "core/consensus", "core/cpu_allocation", "core/harness_derive", diff --git a/core/connectors/BLOG_POST.md b/core/connectors/BLOG_POST.md new file mode 100644 index 0000000000..02db07fa96 --- /dev/null +++ b/core/connectors/BLOG_POST.md @@ -0,0 +1,104 @@ +# Announcing sink and source connector templates + +*Draft for the Apache Iggy project blog. Replace this header line with +the final publish date and author byline before posting.* + +Apache Iggy's connectors subsystem has grown fast. In the past few +months alone, contributors have shipped or proposed sink and source +connectors for Postgres, MongoDB, Elasticsearch, Iceberg, Delta Lake, +S3, InfluxDB, Doris, ClickHouse, SurrealDB, Meilisearch, OpenSearch, +Redshift, and more — each one a plugin that moves real data between +Apache Iggy and an external system, often in production. That growth +is great news for the project. It also means new contributors keep +re-solving the same non-backend-specific problems from scratch before +their PR can even get to the interesting part: talking to their actual +system. + +## What we found + +Looking back across recent connector PR reviews, the same handful of +issues came up again and again, and none of them had anything to do +with the destination or source system being integrated: + +- **Credentials typed as plain `String`.** Connection strings and API + keys landing in `Debug`/log output because the field wasn't + `secrecy::SecretString`. +- **Cursor commits that don't survive a failed delivery.** Since + [#3855](https://github.com/apache/iggy/pull/3855), sources use a + formal ACK/NACK handshake — `poll()` stages candidate state, and + only `on_batch_result()` commits it — but that shape has to be + learned and wired up correctly every time. +- **Errors that don't distinguish retry-worthy from permanent.** + Network hiccups and "this payload will never be accepted" ending up + in the same catch-all error variant. +- **Config knobs with drifting names.** `retry_max_delay` here, + `max_retry_delay` there, `request_timeout` somewhere else, for the + same concept. +- **Missing canonical tests.** State restore/round-trip and ACK/NACK + commit/discard behavior left untested because nobody had a reference + test suite to copy. + +None of this is specific to any one backend. It's framework plumbing +that every sink and every source needs, and until now every author +either copied the closest existing plugin and stripped it down, or +started from a blank `lib.rs` and rediscovered each of these the hard +way, one review round at a time. + +## The templates + +[`core/connectors/sinks/sink_template`](sinks/sink_template) and +[`core/connectors/sources/source_template`](sources/source_template) +are compiling, tested crates you copy and fill in — not prose +describing a pattern, but the pattern itself, already wired up and +passing `cargo test`: + +- Config parsing with `#[serde(deny_unknown_fields)]`, so a typo'd TOML + key fails loudly instead of silently doing nothing. +- `SecretString` on every credential-shaped field + (`connection_string`, `auth_token`), via + `iggy_common::serde_secret::serialize_secret`. +- A retry-wrapped HTTP client plus a startup connectivity probe with + its own backoff. +- A `CircuitBreaker` that's actually consulted before each call and + updated once per `consume()`/`poll()`, not per chunk. +- Sink: batching by a configurable size, and a `last_err` pattern that + never swallows a failed batch into `Ok(())`. +- Source: the full #3855 ACK/NACK contract — `poll()` stages a + candidate cursor, `on_batch_result()` commits it on `Ack` or + discards it on `Nack`, so a dropped batch gets re-polled instead of + silently lost. +- The canonical test suites: six tests for the sink, eight for the + source (four state tests — restore, no-state, invalid-state, + round-trip — plus the two ACK/NACK tests, plus config validation and + the circuit-breaker short-circuit path). + +What's left is marked `TODO(ConnectorDeveloper)` in each crate's `src/lib.rs`: +one spot for a sink (`push_batch()`), two for a source +(`build_raw_client()` if you're not talking HTTP, and +`fetch_records()`). Everything else — the parts that used to eat a +review round — is already done. + +## Using one + +Copy the crate, rename the package and the directory, add it to the +workspace `members` list, fill in the `TODO(ConnectorDeveloper)` spots, and +update `config.toml` for your system. Each crate's own `README.md` +walks through the exact steps. Both templates already build, `clippy +--all-targets -- -D warnings` clean, and pass their tests as committed +— the only thing that should break when you fill in the TODOs is the +`Err(Error::InitError("not implemented yet"))` stub they start from. + +## Why this matters beyond Apache Iggy's connectors + +The pattern generalizes past this one subsystem: any plugin system +with a real framework contract — secrets, retries, staged +commit/rollback, canonical tests — benefits more from a working, +compiling example than from a checklist alone. A checklist tells you +what to verify; a template gives you the thing already verified, so +your own diff is just the part only you can write. + +--- + +*Feedback and discussion: see the project's +[GitHub Discussions](https://github.com/apache/iggy/discussions) or +[Discord](https://discord.gg/apache-iggy).* diff --git a/core/connectors/README.md b/core/connectors/README.md index b334009278..6fd79b9796 100644 --- a/core/connectors/README.md +++ b/core/connectors/README.md @@ -129,6 +129,10 @@ Please refer to the **[Source documentation](https://github.com/apache/iggy/tree New connector can be built simply by implementing either `Sink` or `Source` trait. Please check the **[sink](https://github.com/apache/iggy/tree/master/core/connectors/sinks)** or **[source](https://github.com/apache/iggy/tree/master/core/connectors/sources)** documentation, as well as the existing examples under `core/connectors/sinks` and `core/connectors/sources`. +Starting from a blank crate re-solves the same framework plumbing (config validation, `SecretString` secrets, retry + circuit breaker, ACK/NACK cursor staging) every time. +Copy `core/connectors/sinks/sink_template` or `core/connectors/sources/source_template` instead - compiling, tested starting points with only the backend-specific calls marked `TODO(ConnectorDeveloper)`. +See [`core/connectors/docs/authoring-sinks-and-sources.md`](docs/authoring-sinks-and-sources.md) for the full authoring guide and the review checklist it's built from. + ## Transformations Field transformations (depending on the supported payload formats) can be applied to the messages either before they are sent to the specified topic (e.g. when produced by the source connectors), or before consumed by the sink connectors. To add a transformation, implement the `Transform` trait in the SDK, add its `TransformType` variant and extend `transforms::from_config`. Each transform may have its own, custom configuration. diff --git a/core/connectors/docs/authoring-sinks-and-sources.md b/core/connectors/docs/authoring-sinks-and-sources.md new file mode 100644 index 0000000000..54b041839d --- /dev/null +++ b/core/connectors/docs/authoring-sinks-and-sources.md @@ -0,0 +1,196 @@ +# Authoring Apache Iggy sink and source connectors + +> Reference guide for `core/connectors/` docs. For the narrative version +> suitable for the Apache Iggy blog, see [`BLOG_POST.md`](../BLOG_POST.md). +> Companion materials: +> +> - Review checklist skill: `.claude/skills/connector-pr-review/SKILL.md` +> - Sink fill-in-the-blank kit: `.claude/skills/connector-sink/TEMPLATE.md` +> - Source fill-in-the-blank kit: `.claude/skills/connector-source/TEMPLATE.md` +> - Testing skill: `.claude/skills/connector-testing/SKILL.md` + +Connector PRs in Apache Iggy often spend multiple review rounds on the +**same classes of issues**: secrets handling, retry/error mapping, +idempotency claims that the backend does not enforce, README defaults +that disagree with code, and tests that look green while missing the +mandatory source-state suite. + +This guide turns those review patterns into a strict authoring path. +Follow it and a first review can focus on the backend-specific parts +(connection + fetch/push) instead of re-teaching the framework. + +## The only code you should invent + +Copy the closest existing plugin by shape: + +| Shape | Start from | +| ----- | ---------- | +| DB write sink | `core/connectors/sinks/postgres_sink` | +| HTTP sink | `core/connectors/sinks/http_sink` | +| Polling DB source | `core/connectors/sources/postgres_source` | +| Minimal source + state tests | `core/connectors/sources/random_source` | +| Minimal sink | `core/connectors/sinks/stdout_sink` | +| Fill-in-the-blank sink | `core/connectors/sinks/sink_template` | +| Fill-in-the-blank source | `core/connectors/sources/source_template` | + +`sink_template` / `source_template` are compiling, tested starting +points: config parsing, `SecretString`, retry loops, circuit breaker, +`last_err` batch handling (sink), ACK/NACK cursor staging (source), +sleep-first poll, lock discipline, and unit-test stubs are already +wired up. Copy the crate, rename the package, and fill in the +`TODO(ConnectorDeveloper)` spots in `src/lib.rs` — see each crate's `README.md`. +The skills' `TEMPLATE.md` files describe the same pattern in prose, for +when copying a whole crate is more than you need. + +**Your implementation surface is intentionally small:** + +1. Build a client from `connection_string` (secret). +2. Probe connectivity in `open()`. +3. **Sink:** push one batch. +4. **Source:** fetch the next batch and compute the next cursor. + +Everything else should match siblings. + +## Hard rules (review blockers) + +### Secrets + +- Credentials are `secrecy::SecretString` with + `iggy_common::serde_secret::serialize_secret`. +- Never log connection strings, API keys, or tokens. +- Never persist URL userinfo / bearers into source state files. + +### Errors and retries + +- `max_retries` means **total attempts** (default **3**). README must match. +- Map infra/auth/schema failures to permanent variants + (`PermanentHttpError`, `InitError`, `SchemaMismatch`). +- Map network/5xx/retryable SQLSTATE to transient variants. +- Do **not** decide retryability by substring-matching `err.to_string()`. +- Cap backoff so a dead backend cannot stall shutdown. + +### Delivery semantics (document in README) + +Every plugin README needs a short paragraph that answers: + +1. What happens on transient failure? +2. What happens on permanent failure? +3. What is the duplication window? +4. What is the dedup key (or “none”)? + +Do not claim “no data loss” or “exactly once” unless the backend unique +key **and** the runtime path actually enforce it. Today’s runtime can +still advance consumer offsets in ways that punish swallowed `Ok(())` +after a failed sink batch — return `Err` for retryable failures. + +### Source-specific + +Since #3855, sources use a formal one-in-flight-batch ACK/NACK +handshake, not just a convention: `poll()` returns *candidate* state; +the runtime sends the batch, saves that state only on success, then +calls `on_batch_result(SourceBatchResult::Ack | Nack)`. Only +`on_batch_result` may commit staged work, and the SDK will not call +`poll()` again until it returns. + +- `async fn poll(&self)` with state behind `tokio::sync::Mutex`. +- **Sleep first**, then fetch. +- Brief lock → drop → I/O → brief lock write. +- **Stage, don't commit, in `poll()`.** Cursor advances and destructive + work (delete-after-read, mark-processed) go into a `pending_state` / + staged field, applied only in `on_batch_result()` on `Ack`; a `Nack` + discards the staged work so the same range is redelivered. +- Always return state for a poll that made progress; return `state: + None` for an empty poll that made none. +- The default `on_batch_result` is a no-op - override it (and stage + accordingly) whenever `poll()` advances a cursor or does destructive + work. Returning `Err` from it stops the SDK from polling further. +- Ship the **six canonical tests**: four state tests (restore / + no-state / invalid-state / round-trip) plus two ACK/NACK tests + (commit-on-Ack, keep-committed-on-Nack) if `on_batch_result` is + overridden. Copy `random_source`. + +### Sink-specific + +- `async fn consume(&self, …)`. +- Process all batches; keep `last_err`; never convert a failed batch into + bare `Ok(())`. +- Prefer stable dedup keys (`stream:topic:partition:message_id` or a + natural backend key). No random UUIDs per emit. +- Use `payload.try_to_bytes()` — do not clone `Payload::Json`. + +### Config canon + +| Concept | Field name | +| ------- | ---------- | +| Timeout | `timeout` | +| Retry attempts | `max_retries` | +| Base backoff | `retry_delay` | +| Backoff ceiling | `retry_max_delay` | +| Source cadence | `poll_interval` | +| Verbosity | `verbose_logging` | + +Use `Option` + humantime strings for durations, apply defaults in +`new()` with `warn!` on parse failure, and prefer +`#[serde(deny_unknown_fields)]` so typo’d knobs fail loud. + +Validate structure and connectivity in `new()` / `open()`, never on the +first hot-path call after sleep. + +## Required artifacts in the PR + +1. Plugin crate with `[lib] crate-type = ["cdylib", "lib"]`. +2. Unit tests (BDD `given_X_when_Y_should_Z` / `given_X_should_Y`). +3. Sources: four canonical state tests, plus two ACK/NACK tests if `on_batch_result` is overridden. +4. External backend: real-infra integration test under + `core/integration/tests/connectors//` (`#[iggy_harness]` + + testcontainers or wiremock). No false-green mocks. +5. Row in `sinks/README.md` or `sources/README.md`. +6. Sample TOML under `runtime/example_config/connectors/` with plugin + path `../../target/release/lib…`. +7. README defaults **byte-equal** to code consts. +8. Honest delivery-semantics paragraph. + +## Verification before `/ready` + +```bash +cargo fmt --all +cargo sort --no-format --workspace +cargo clippy -p iggy_connector__{sink|source} --all-targets -- -D warnings +cargo test -p iggy_connector__{sink|source} +# if applicable: +cargo test -p integration -- connectors:::: +``` + +Keep `Cargo.lock` churn limited to this plugin’s dependencies. Do not +touch unrelated foreign SDKs in the same PR. + +## Pre-flight checklist (paste into the PR) + +```text +[ ] SecretString on all credentials; no secret logs/state +[ ] consume/poll never returns Ok(()) after a failed batch that should retry +[ ] Transient vs permanent errors mapped (no Display substring matching) +[ ] Stable message / dedup IDs (no random UUID per emit) +[ ] README delivery semantics paragraph present and honest +[ ] README defaults match code consts +[ ] deny_unknown_fields on plugin config +[ ] Canonical knob names (timeout, max_retries, retry_delay, poll_interval) +[ ] Sources: 4 state tests (+2 ACK/NACK if on_batch_result overridden); sleep-first poll; state staged in poll, committed only in on_batch_result +[ ] External backend: real-infra integration test (not a lying mock) +[ ] example_config + sinks/sources README row +[ ] fmt / sort --no-format / clippy -D warnings / unit tests green +[ ] Cargo.lock churn limited to this crate's deps +``` + +Agents and reviewers: load `.claude/skills/connector-pr-review/SKILL.md` +and treat unchecked blockers as CHANGES_REQUESTED. + +## Why this exists + +Across recent connector PRs (SurrealDB, Meilisearch, OpenSearch, Quickwit +convention bring-up, MySQL/JDBC, Doris retry, Redshift, Airflow, Fluss, +and others), the expensive review threads were rarely about the novel +backend call. They were about framework contracts: secrets, retries, +idempotency honesty, config names, README drift, and missing state +tests. Codifying those contracts as a skill + templates is how we make +connector contributions predictable without lowering the bar. diff --git a/core/connectors/runtime/example_config/connectors/sink_template.toml b/core/connectors/runtime/example_config/connectors/sink_template.toml new file mode 100644 index 0000000000..ee9a7bd3de --- /dev/null +++ b/core/connectors/runtime/example_config/connectors/sink_template.toml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +type = "sink" +key = "template" +enabled = true +version = 0 +name = "Template sink" +path = "target/release/libiggy_connector_template_sink" +plugin_config_format = "toml" +verbose = false + +[[streams]] +stream = "example_stream" +topics = ["example_topic"] +schema = "json" +batch_length = 100 +poll_interval = "5ms" +consumer_group = "template_sink_connector" + +[plugin_config] +connection_string = "https://api.example.com" +target = "events" +health_check_path = "/health" +batch_size = 100 +timeout = "30s" +max_retries = 3 +retry_delay = "500ms" +retry_max_delay = "5s" +max_open_retries = 10 +open_retry_max_delay = "60s" +circuit_breaker_threshold = 5 +circuit_breaker_cool_down = "30s" +verbose_logging = false diff --git a/core/connectors/runtime/example_config/connectors/source_template.toml b/core/connectors/runtime/example_config/connectors/source_template.toml new file mode 100644 index 0000000000..02dbe0eab8 --- /dev/null +++ b/core/connectors/runtime/example_config/connectors/source_template.toml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +type = "source" +key = "template" +enabled = true +version = 0 +name = "Template source" +path = "target/release/libiggy_connector_template_source" +plugin_config_format = "toml" +verbose = false + +[[streams]] +stream = "example_stream" +topic = "example_topic" +schema = "json" +batch_length = 100 +linger_time = "5ms" + +[plugin_config] +connection_string = "https://api.example.com" +health_check_path = "/health" +batch_size = 100 +poll_interval = "1s" +timeout = "30s" +max_retries = 3 +retry_delay = "500ms" +retry_max_delay = "5s" +max_open_retries = 10 +open_retry_max_delay = "60s" +circuit_breaker_threshold = 5 +circuit_breaker_cool_down = "30s" +verbose_logging = false diff --git a/core/connectors/sinks/README.md b/core/connectors/sinks/README.md index 70a28940ac..4f7b08633e 100644 --- a/core/connectors/sinks/README.md +++ b/core/connectors/sinks/README.md @@ -21,6 +21,7 @@ Sink connectors are responsible for writing data from Iggy streams to external s | **stdout_sink** | Prints messages to standard output (useful for debugging and development) | | **surrealdb_sink** | Writes messages into SurrealDB with deterministic record IDs for idempotent replay | | **rabbitmq_sink** | Publishes messages to RabbitMQ exchanges via AMQP | +| **sink_template** | Fill-in-the-blank starting point for a new sink; framework/security plumbing done, one `TODO(ConnectorDeveloper)` spot left | The sink is represented by the single `Sink` trait, which defines the basic interface for all sink connectors. It provides methods for initializing the sink, writing data to external destination, and closing the sink. diff --git a/core/connectors/sinks/sink_template/Cargo.toml b/core/connectors/sinks/sink_template/Cargo.toml new file mode 100644 index 0000000000..886962dc6d --- /dev/null +++ b/core/connectors/sinks/sink_template/Cargo.toml @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# TEMPLATE — rename the package (and this directory) to +# `iggy_connector__sink` before publishing, and update the +# `[[sinks]]` entry you add to the workspace root Cargo.toml accordingly. + +[package] +name = "iggy_connector_template_sink" +version = "0.1.0" +description = "Template for an Apache Iggy sink connector — copy this crate and fill in the TODO sections." +edition = "2024" +license = "Apache-2.0" +keywords = ["iggy", "messaging", "streaming"] +categories = ["command-line-utilities", "database", "network-programming"] +homepage = "https://iggy.apache.org" +documentation = "https://iggy.apache.org/docs" +repository = "https://github.com/apache/iggy" +publish = false + +[package.metadata.cargo-machete] +# dashmap is used only inside the `sink_connector!` macro expansion, so a +# naive unused-dependency scan won't see the usage — keep it ignored rather +# than removing it, or the plugin will fail to compile. +ignored = ["dashmap"] + +[lib] +crate-type = ["cdylib", "lib"] + +[dependencies] +async-trait = { workspace = true } +dashmap = { workspace = true } +iggy_common = { workspace = true } +iggy_connector_sdk = { workspace = true } +reqwest = { workspace = true } +reqwest-middleware = { workspace = true } +secrecy = { workspace = true } +serde = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } diff --git a/core/connectors/sinks/sink_template/README.md b/core/connectors/sinks/sink_template/README.md new file mode 100644 index 0000000000..5924b3422f --- /dev/null +++ b/core/connectors/sinks/sink_template/README.md @@ -0,0 +1,103 @@ +# Template sink connector + +Starting point for a new Apache Iggy **sink** connector. Everything except +pushing data to your actual destination is already implemented and follows +the project's required resilience/security patterns — see the module-level +doc comment at the top of `src/lib.rs` for the full rationale, and the +`iggy-connector-review` skill / the "Building Connectors That Pass Review" +blog post for the checklist this template is built against. + +## Delivery Semantics + +The framework around `push_batch()` gives you retry (`max_retries` total attempts, +`retry_delay`/`retry_max_delay` backoff) and a circuit breaker that opens after +`circuit_breaker_threshold` consecutive failures. Everything below that line +depends on what you write in `push_batch()` — fill in the three blanks with your +real behavior before shipping: + +1. **Transient failure**: retries up to `max_retries` total attempts, then + `consume()` returns the error. The runtime's handling of that error is fatal + to the whole connector — see `connector-pr-review` B2. +2. **Permanent failure** (bad schema, destination rejects the payload shape): + return `Error::PermanentHttpError` from `push_batch()` — `consume()` drops + and counts that one batch instead of taking the whole connector down. +3. **Duplication window**: at-least-once. A batch can be retried after a + partial write (e.g. your destination accepted some rows then the connection + dropped before the response), so a retry can re-send already-written rows. +4. **Dedup key**: none by default — `push_batch()` doesn't build one yet. + Before shipping, derive a stable key from `message.id` plus + `topic_metadata`/`messages_metadata` (see `mongodb_sink::build_composite_document_id` + for `stream:topic:partition:message_id`) and make your destination's write + idempotent on it (upsert / `ON CONFLICT` / unique index) — otherwise the + at-least-once retry above will create duplicates. See `connector-pr-review` + B5. + +## What's already done for you + +- Config parsing with `#[serde(deny_unknown_fields)]` so a typo in a TOML + file fails loudly instead of silently doing nothing. +- Config validation in `open()` (not `new()`, which has no way to return an + error) — including validating `target` (the destination table/index/ + collection name) against an allowlist pattern *before* it can ever reach a + query, path, or URL. +- `connection_string` and the optional `auth_token` field both typed as + `SecretString`, since either can carry credentials. +- A retry-wrapped HTTP client (`iggy_connector_sdk::retry::build_retry_client`) + and a startup connectivity probe with its own backoff + (`check_connectivity_with_retry`). +- A `CircuitBreaker` that's actually consulted before each `consume()` call + and updated once per call based on the outcome — not just constructed and + forgotten, and not reset mid-batch by a partial success. +- Batching: `consume()` chunks the incoming messages by a configurable + `batch_size` instead of sending everything in one unbounded request. +- The `sink_connector!` FFI macro invocation and a `Cargo.toml` with the + right `crate-type`, workspace-pinned dependencies, and license header. +- `verbose_logging: Option` upgrading the per-batch log line from + `debug!` to `info!`, mirroring the runtime's own `verbose` flag. +- Tests for config/identifier validation, the circuit-breaker short-circuit + path, the `verbose_logging` flag, and `consume()`'s batch loop end to end. + +## What you need to fill in + +Search for `TODO(ConnectorDeveloper)` in `src/lib.rs` — there is exactly one +spot that requires code, plus two more that are conditional/documentation: + +**`push_batch()`** (required) — build the request/write that actually sends +one chunk of messages to your destination, using +`self.config.connection_string` (and `self.config.target`, already validated +by the time this runs) via `self.client` (already retry-wrapped). Distinguish +permanent failures (bad schema, a destination that will reject this payload +shape no matter how many times you retry) from transient ones (network +error, 5xx, timeout) by returning `Error::PermanentHttpError` for the +former — `consume()` drops and counts a permanent failure instead of +propagating it, so a single bad message can't take the whole connector down; +any other error stops `consume()` and is returned as-is. + +**`connection_string`'s doc comment** (documentation only) — describe the +exact shape your connector expects instead of the generic example. + +If your destination isn't HTTP, also revisit **`build_raw_client()`**: swap +the `reqwest::Client` for your driver's connection/pool setup (see +`core/connectors/sinks/postgres_sink` or `core/connectors/sinks/s3_sink` for +non-HTTP examples), store it on `TemplateSink` in place of the HTTP-specific +`client` field, and adjust or remove the `check_connectivity_with_retry` call +in `open()` in favor of whatever connectivity check your driver offers. + +## Using it + +1. Copy this directory, rename it and the package in `Cargo.toml` + (`iggy_connector__sink`), and add it to the `members` list in + the workspace root `Cargo.toml`. +2. Fill in the `TODO(ConnectorDeveloper)` section(s). +3. Update `config.toml` with your real `connection_string` and `target`, and + any settings specific to your system; delete `auth_token` if you don't + need it, or add fields of your own the same way (see + `TemplateSinkConfig`). +4. `cargo build --release -p iggy_connector__sink`, point a + runtime connector config file's `path` at the built `.so`/`.dylib`/`.dll`, + and run the connector runtime — see `core/connectors/README.md` in this + repo for the full runtime quick-start. +5. Before opening a PR: `cargo test`, `cargo clippy --all-targets`, + `cargo fmt --check`, and re-read the connector-review checklist once more + with fresh eyes — most review round-trips come from one of the items in + that list, not from the connector-specific logic in `push_batch()`. diff --git a/core/connectors/sinks/sink_template/config.toml b/core/connectors/sinks/sink_template/config.toml new file mode 100644 index 0000000000..86c91de246 --- /dev/null +++ b/core/connectors/sinks/sink_template/config.toml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +type = "sink" +key = "template" +enabled = true +version = 0 +name = "Template sink" +path = "../../target/release/libiggy_connector_template_sink" +plugin_config_format = "toml" +verbose = false + +[[streams]] +stream = "example_stream" +topics = ["example_topic"] +schema = "json" +batch_length = 100 +poll_interval = "5ms" +consumer_group = "template_sink_connector" + +[plugin_config] +connection_string = "https://api.example.com" +target = "events" +health_check_path = "/health" +batch_size = 100 +timeout = "30s" +max_retries = 3 +retry_delay = "500ms" +retry_max_delay = "5s" +max_open_retries = 10 +open_retry_max_delay = "60s" +circuit_breaker_threshold = 5 +circuit_breaker_cool_down = "30s" +verbose_logging = false +# auth_token = "replace-me" # uncomment if your destination needs bearer/API-key auth diff --git a/core/connectors/sinks/sink_template/src/lib.rs b/core/connectors/sinks/sink_template/src/lib.rs new file mode 100644 index 0000000000..80a8c97aba --- /dev/null +++ b/core/connectors/sinks/sink_template/src/lib.rs @@ -0,0 +1,662 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Template Apache Iggy **sink** connector. +//! +//! Everything in this file is already wired up and follows the patterns the +//! connector-review checklist expects: config validation happens in +//! `open()`, secrets are `SecretString`, the destination identifier is +//! validated before it's ever interpolated into a request, outbound calls +//! go through a retry-wrapped client plus a circuit breaker, and messages +//! are chunked by a configurable batch size instead of shipped as one +//! unbounded request. +//! +//! There is exactly **one** place you need to write code, marked +//! `TODO(ConnectorDeveloper)`: `TemplateSink::push_batch()` — build the +//! request/write that actually pushes one chunk of messages to your +//! destination, using `self.config.connection_string` (and +//! `self.config.target`, if your destination has a table/index/ +//! collection-shaped name). A second `TODO(ConnectorDeveloper)` on the +//! `connection_string` field's doc comment just asks you to describe its +//! expected shape — not code, but worth personalizing too. `grep` for the +//! marker and you'll find both, plus one more on `build_raw_client()` that +//! only applies if your destination isn't HTTP (see below). +//! +//! This template assumes an HTTP-ish destination and uses `reqwest` wrapped +//! by the SDK's retry middleware, because that's what +//! `iggy_connector_sdk::retry` is built for and it covers the common case. +//! If your destination talks something else (a database, a queue, object +//! storage), swap the client type in `build_raw_client()`/`push_batch()` for +//! your driver of choice and lean on its own retry/pooling behavior — keep the +//! surrounding shape (validation in `open()`, circuit breaker, batching, +//! identifier validation) unchanged. See `core/connectors/sinks/postgres_sink` +//! or `core/connectors/sinks/s3_sink` in this repo for non-HTTP examples of +//! that same shape. + +use async_trait::async_trait; +use iggy_connector_sdk::retry::{ + CircuitBreaker, RetryPolicy, build_retry_client, check_connectivity_with_retry, parse_duration, +}; +use iggy_connector_sdk::{ + ConsumedMessage, Error, MessagesMetadata, Sink, TopicMetadata, sink_connector, +}; +use reqwest::Url; +use reqwest_middleware::ClientWithMiddleware; +use secrecy::{ExposeSecret, SecretString}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; +use tokio::sync::Mutex; +use tracing::{debug, error, info, warn}; + +sink_connector!(TemplateSink); + +const CONNECTOR_NAME: &str = "Template sink"; + +const DEFAULT_BATCH_SIZE: usize = 100; +const DEFAULT_TIMEOUT: &str = "30s"; +const DEFAULT_MAX_RETRIES: u32 = 3; +const DEFAULT_RETRY_DELAY: &str = "500ms"; +const DEFAULT_RETRY_MAX_DELAY: &str = "5s"; +const DEFAULT_MAX_OPEN_RETRIES: u32 = 10; +const DEFAULT_OPEN_RETRY_MAX_DELAY: &str = "60s"; +const DEFAULT_CIRCUIT_BREAKER_THRESHOLD: u32 = 5; +const DEFAULT_CIRCUIT_BREAKER_COOL_DOWN: &str = "30s"; + +// ── Configuration ─────────────────────────────────────────────────────────── +// +// Every tunable except `connection_string` and `target` is optional with a +// sane default, and unknown keys are rejected outright so a typo in a TOML +// file fails at load time instead of silently doing nothing. + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemplateSinkConfig { + /// TODO(ConnectorDeveloper): document the exact shape this connector expects, e.g. + /// "" or "postgres://user:pass@host:5432/db". + /// `SecretString` because DSNs commonly embed credentials — never plain + /// `String` for this field, see `PostgresSinkConfig::connection_string` + /// in `sinks/postgres_sink` for the same pattern. + #[serde(serialize_with = "iggy_common::serde_secret::serialize_secret")] + pub connection_string: SecretString, + + /// The destination table/index/collection/bucket name. Kept as its own + /// field (rather than folded into `connection_string`) specifically so + /// it can be validated in `open()` before ever being interpolated into + /// a query, path, or URL — see `validate_identifier` below. Delete this + /// field if your destination has no such dynamic identifier. + pub target: String, + + /// Example of a secret-shaped setting. `SecretString` keeps it out of + /// `Debug`/log output; delete this field if `connection_string` already + /// carries all required auth. Read it with `.expose_secret()` (from the + /// `secrecy::ExposeSecret` trait) at the one place you actually need the + /// plaintext — e.g. when building an auth header in `build_raw_client()`. + #[serde( + default, + serialize_with = "iggy_common::serde_secret::serialize_optional_secret" + )] + pub auth_token: Option, + + /// Optional path (e.g. "/health") probed with retry during `open()` + /// before the connector is considered ready. Leave unset if your target + /// has no health endpoint — the probe is skipped, not failed, in that case. + pub health_check_path: Option, + + pub batch_size: Option, + pub timeout: Option, + pub max_retries: Option, + pub retry_delay: Option, + pub retry_max_delay: Option, + pub max_open_retries: Option, + pub open_retry_max_delay: Option, + pub circuit_breaker_threshold: Option, + pub circuit_breaker_cool_down: Option, + + /// Upgrades the per-batch log line from `debug!` to `info!`. Mirrors the + /// runtime's own `verbose` flag; keep the field name `verbose_logging` + /// (see `postgres_sink::PostgresSinkConfig::verbose_logging`). + pub verbose_logging: Option, +} + +/// Rejects anything that isn't a plain alphanumeric/underscore identifier. +/// Adjust the allowed character set to whatever your destination's naming +/// rules actually are, but always validate *something* before a +/// config-or-message-derived name is interpolated into a query, path, or +/// URL — see `doris_sink::validate_identifier` / `surrealdb_sink::validate_identifier` +/// in this repo for the same pattern applied to a real destination. +fn validate_identifier(field: &str, value: &str) -> Result<(), Error> { + if value.is_empty() || !value.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + return Err(Error::InvalidConfigValue(format!( + "{field} must be non-empty and contain only ASCII alphanumeric characters and \ + underscores, got: {value:?}" + ))); + } + Ok(()) +} + +// ── Internal state ────────────────────────────────────────────────────────── + +#[derive(Debug, Default)] +struct State { + invocations_count: u64, + messages_written: u64, + messages_failed: u64, +} + +#[derive(Debug)] +pub struct TemplateSink { + id: u32, + config: TemplateSinkConfig, + client: Option, + circuit_breaker: Arc, + batch_size_limit: usize, + retry_delay: Duration, + verbose: bool, + state: Mutex, + records_written_total: AtomicU64, +} + +impl TemplateSink { + pub fn new(id: u32, config: TemplateSinkConfig) -> Self { + let retry_delay = parse_duration(config.retry_delay.as_deref(), DEFAULT_RETRY_DELAY); + let circuit_breaker = Arc::new(CircuitBreaker::new( + config + .circuit_breaker_threshold + .unwrap_or(DEFAULT_CIRCUIT_BREAKER_THRESHOLD), + parse_duration( + config.circuit_breaker_cool_down.as_deref(), + DEFAULT_CIRCUIT_BREAKER_COOL_DOWN, + ), + )); + let batch_size_limit = config.batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1); + let verbose = config.verbose_logging.unwrap_or(false); + + Self { + id, + config, + client: None, + circuit_breaker, + batch_size_limit, + retry_delay, + verbose, + state: Mutex::new(State::default()), + records_written_total: AtomicU64::new(0), + } + } + + /// TODO(ConnectorDeveloper): build your actual client/connection here using + /// `self.config.connection_string` (and `self.config.auth_token`, if + /// your destination needs it). This template builds a plain + /// `reqwest::Client` to hand to `build_retry_client` — if you're not + /// talking HTTP, replace this with e.g. a database connection pool or + /// your driver's equivalent, store it on `self` (add a field, since + /// `client` here is HTTP-specific), and skip the + /// `check_connectivity_with_retry` call below in favor of whatever your + /// driver offers (a ping, a test query). + fn build_raw_client(&self) -> Result { + let timeout = parse_duration(self.config.timeout.as_deref(), DEFAULT_TIMEOUT); + reqwest::Client::builder() + .timeout(timeout) + .build() + .map_err(|e| Error::Connection(format!("failed to build HTTP client: {e}"))) + } + + /// TODO(ConnectorDeveloper): push one chunk of already-batched messages to your + /// destination using `self.config.connection_string` and + /// `self.config.target`, via `self.client` (already retry-wrapped). + /// Distinguish permanent failures (bad schema, destination rejects the + /// payload shape — will not succeed on retry) from transient ones + /// (network error, 5xx, timeout — should retry) by returning + /// `Error::PermanentHttpError` for the former; see the doc comment on + /// that variant in `iggy_connector_sdk::Error` for why the distinction + /// matters. `consume()` drops and counts a `PermanentHttpError` batch but + /// keeps processing the rest; any other error stops `consume()` and is + /// returned, which the runtime treats as fatal for the whole connector — + /// so a permanent classification is what keeps one bad message from + /// taking every future batch down with it. + /// + /// Delivery is at-least-once: a retried batch can re-send rows your + /// destination already wrote from an earlier attempt that failed after + /// partially succeeding. Build a stable key with `dedup_key()` below for + /// every message you write, and make the write idempotent on it (upsert / + /// `ON CONFLICT` / unique index) — see `connector-pr-review` B5. + /// + /// If your destination's write call can succeed at the transport level + /// while rejecting some or all rows individually (a bulk/batch API + /// returning 200 with per-item errors, a stream-load response, a bulk + /// index response), count accepted rows yourself and return `Err` when + /// that count is zero for a non-empty `batch` — never `Ok(())`. Four + /// unrelated sinks shipped exactly this bug in review (BigQuery, + /// Elasticsearch, Doris, S3): the call "succeeded," `consume()` reported + /// success, and the batch was gone with no signal anywhere. See + /// `connector-pr-review` B9. + async fn push_batch( + &self, + client: &ClientWithMiddleware, + topic_metadata: &TopicMetadata, + messages_metadata: &MessagesMetadata, + batch: &[ConsumedMessage], + ) -> Result<(), Error> { + let _ = client; // remove once implemented + for message in batch { + // TODO(ConnectorDeveloper): use `key` as the identity your destination's + // write is idempotent on (upsert key, `ON CONFLICT` target, document _id). + let key = dedup_key(topic_metadata, messages_metadata, message); + let _ = key; + } + Err(Error::InitError( + "TemplateSink::push_batch is not implemented yet — see the TODO(ConnectorDeveloper) comment in \ + template_sink/src/lib.rs" + .to_string(), + )) + } +} + +/// Stable per-message dedup key: `stream:topic:partition:message_id`. Mirrors +/// `mongodb_sink::build_composite_document_id` — copy this, not a random UUID, +/// for whatever identity column/key your destination's upsert needs. Offset +/// is deliberately excluded: it only identifies a message within one +/// partition's log, not across the retries and redeliveries this key has to +/// survive. +fn dedup_key( + topic_metadata: &TopicMetadata, + messages_metadata: &MessagesMetadata, + message: &ConsumedMessage, +) -> String { + format!( + "{}:{}:{}:{}", + topic_metadata.stream, topic_metadata.topic, messages_metadata.partition_id, message.id + ) +} + +// ── Sink trait ──────────────────────────────────────────────────────────────── + +#[async_trait] +impl Sink for TemplateSink { + async fn open(&mut self) -> Result<(), Error> { + // Structural validation happens here, not in `new()`, because only + // `open()` can return an error — `new()` is a plain factory function + // with nowhere to send a "this config is invalid" result. + if self + .config + .connection_string + .expose_secret() + .trim() + .is_empty() + { + return Err(Error::InvalidConfigValue( + "connection_string must not be empty".to_string(), + )); + } + validate_identifier("target", &self.config.target)?; + + info!( + "Opening {CONNECTOR_NAME} connector with ID: {}, target: {}, batch_size: {}", + self.id, self.config.target, self.batch_size_limit + ); + + let raw_client = self.build_raw_client()?; + + if let Some(health_path) = &self.config.health_check_path { + let base = Url::parse(self.config.connection_string.expose_secret()).map_err(|e| { + Error::InvalidConfigValue(format!("connection_string is not a valid URL: {e}")) + })?; + let health_url = base.join(health_path).map_err(|e| { + Error::InvalidConfigValue(format!("invalid health_check_path: {e}")) + })?; + check_connectivity_with_retry( + &raw_client, + health_url, + CONNECTOR_NAME, + self.id, + RetryPolicy { + max_attempts: self + .config + .max_open_retries + .unwrap_or(DEFAULT_MAX_OPEN_RETRIES), + base_delay: self.retry_delay, + max_delay: parse_duration( + self.config.open_retry_max_delay.as_deref(), + DEFAULT_OPEN_RETRY_MAX_DELAY, + ), + }, + ) + .await?; + } else { + warn!( + "{CONNECTOR_NAME} connector with ID: {} has no health_check_path configured — \ + skipping the startup connectivity probe. Consider adding one.", + self.id + ); + } + + self.client = Some(build_retry_client( + raw_client, + self.config + .max_retries + .unwrap_or(DEFAULT_MAX_RETRIES) + .max(1), + self.retry_delay, + parse_duration( + self.config.retry_max_delay.as_deref(), + DEFAULT_RETRY_MAX_DELAY, + ), + CONNECTOR_NAME, + )); + + info!( + "{CONNECTOR_NAME} connector with ID: {} opened successfully", + self.id + ); + Ok(()) + } + + async fn consume( + &self, + topic_metadata: &TopicMetadata, + messages_metadata: MessagesMetadata, + messages: Vec, + ) -> Result<(), Error> { + let mut state = self.state.lock().await; + state.invocations_count += 1; + let invocation = state.invocations_count; + drop(state); + + if self.verbose { + info!( + "{CONNECTOR_NAME} with ID: {} received: {} messages, schema: {}, stream: {}, \ + topic: {}, partition: {}, offset: {}, invocation: {}", + self.id, + messages.len(), + messages_metadata.schema, + topic_metadata.stream, + topic_metadata.topic, + messages_metadata.partition_id, + messages_metadata.current_offset, + invocation + ); + } else { + debug!( + "{CONNECTOR_NAME} with ID: {} received: {} messages, schema: {}, stream: {}, \ + topic: {}, partition: {}, offset: {}, invocation: {}", + self.id, + messages.len(), + messages_metadata.schema, + topic_metadata.stream, + topic_metadata.topic, + messages_metadata.partition_id, + messages_metadata.current_offset, + invocation + ); + } + + if self.circuit_breaker.is_open().await { + warn!( + "{CONNECTOR_NAME} connector with ID: {} — circuit breaker OPEN, refusing {} messages", + self.id, + messages.len() + ); + return Err(Error::CannotStoreData( + "Circuit breaker is open".to_string(), + )); + } + + let client = self.client.as_ref().ok_or_else(|| { + Error::Connection("client not initialized -- was open() called?".into()) + })?; + + // Track the *last* transient error, not the first — an earlier chunk's + // failure may already be stale by the time later chunks run, and the + // circuit breaker below should react to the most recent signal. A + // `PermanentHttpError` batch (bad schema, will never succeed on retry) + // is dropped and counted here rather than kept as `last_err`: letting + // it propagate would return `Err` from `consume()`, and the runtime + // treats any `Err` here as fatal for the whole connector (see + // `runtime/src/sink.rs::consume_messages`), not just a retry of that + // one batch — so a single unprocessable message would take down every + // future batch instead of just being skipped. + let mut last_err: Option = None; + let mut written = 0u64; + let mut failed = 0u64; + + for batch in messages.chunks(self.batch_size_limit) { + match self + .push_batch(client, topic_metadata, &messages_metadata, batch) + .await + { + Ok(()) => written += batch.len() as u64, + Err(Error::PermanentHttpError(message)) => { + failed += batch.len() as u64; + error!( + "{CONNECTOR_NAME} connector with ID: {} dropping a batch of {} \ + (permanent): {message}", + self.id, + batch.len() + ); + } + Err(err) => { + failed += batch.len() as u64; + error!( + "{CONNECTOR_NAME} connector with ID: {} failed a batch of {}: {err}", + self.id, + batch.len() + ); + last_err = Some(err); + } + } + } + + // Record the circuit breaker outcome once per `consume()` call, not + // once per chunk — recording success partway through would reset the + // failure counter mid-consume and prevent the breaker from opening on + // a batch with sustained, mixed-success chunks. A transient failure + // always trips it. A batch where every failure was permanent records + // neither success nor failure — nothing demonstrated the destination + // is reachable, but a schema/data problem isn't a connectivity signal + // either. A batch with at least one real write (or no messages at + // all) counts as success. + match &last_err { + Some(_) => self.circuit_breaker.record_failure().await, + None if written > 0 || messages.is_empty() => self.circuit_breaker.record_success(), + None => {} + } + + let mut state = self.state.lock().await; + state.messages_written += written; + state.messages_failed += failed; + drop(state); + self.records_written_total + .fetch_add(written, Ordering::Relaxed); + + match last_err { + None => Ok(()), + Some(err) => Err(err), + } + } + + async fn close(&mut self) -> Result<(), Error> { + let state = self.state.lock().await; + info!( + "{CONNECTOR_NAME} connector with ID: {} closing. Stats: {} invocations, {} messages written, \ + {} messages failed", + self.id, state.invocations_count, state.messages_written, state.messages_failed + ); + drop(state); + self.client = None; + info!("{CONNECTOR_NAME} connector with ID: {} is closed.", self.id); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_config() -> TemplateSinkConfig { + TemplateSinkConfig { + connection_string: SecretString::from("https://api.example.com"), + target: "events".to_string(), + auth_token: None, + health_check_path: None, + batch_size: Some(10), + timeout: Some("5s".to_string()), + max_retries: Some(2), + retry_delay: Some("10ms".to_string()), + retry_max_delay: Some("100ms".to_string()), + max_open_retries: Some(2), + open_retry_max_delay: Some("100ms".to_string()), + circuit_breaker_threshold: Some(3), + circuit_breaker_cool_down: Some("50ms".to_string()), + verbose_logging: None, + } + } + + #[test] + fn given_verbose_logging_enabled_should_set_verbose_flag() { + let mut config = test_config(); + config.verbose_logging = Some(true); + let sink = TemplateSink::new(1, config); + assert!(sink.verbose); + } + + #[test] + fn given_verbose_logging_disabled_should_not_set_verbose_flag() { + let sink = TemplateSink::new(1, test_config()); + assert!(!sink.verbose); + } + + #[tokio::test] + async fn given_empty_connection_string_should_reject_open() { + let mut config = test_config(); + config.connection_string = SecretString::from(" "); + let mut sink = TemplateSink::new(1, config); + assert!(matches!( + sink.open().await, + Err(Error::InvalidConfigValue(_)) + )); + } + + #[tokio::test] + async fn given_invalid_target_identifier_should_reject_open() { + let mut config = test_config(); + config.target = "events; DROP TABLE users;--".to_string(); + let mut sink = TemplateSink::new(1, config); + assert!(matches!( + sink.open().await, + Err(Error::InvalidConfigValue(_)) + )); + } + + #[test] + fn validate_identifier_should_accept_plain_names() { + assert!(validate_identifier("target", "events_v2").is_ok()); + } + + #[test] + fn validate_identifier_should_reject_empty() { + assert!(validate_identifier("target", "").is_err()); + } + + #[test] + fn validate_identifier_should_reject_path_and_query_characters() { + for bad in [ + "../etc/passwd", + "events?x=1", + "events/../secrets", + "events;drop", + ] { + assert!( + validate_identifier("target", bad).is_err(), + "expected {bad:?} to be rejected" + ); + } + } + + #[test] + fn dedup_key_should_join_stream_topic_partition_and_message_id() { + let (topic_metadata, messages_metadata) = topic_and_messages_metadata(); + let message = consumed_message(); + assert_eq!( + dedup_key(&topic_metadata, &messages_metadata, &message), + "s:t:1:1" + ); + } + + #[tokio::test] + async fn given_open_circuit_breaker_should_short_circuit_consume() { + let sink = TemplateSink::new(1, test_config()); + sink.circuit_breaker.record_failure().await; + sink.circuit_breaker.record_failure().await; + sink.circuit_breaker.record_failure().await; + assert!(sink.circuit_breaker.is_open().await); + + let (topic_metadata, messages_metadata) = topic_and_messages_metadata(); + + let result = sink + .consume(&topic_metadata, messages_metadata, Vec::new()) + .await; + assert!(matches!(result, Err(Error::CannotStoreData(_)))); + } + + fn consumed_message() -> ConsumedMessage { + ConsumedMessage { + id: 1, + offset: 0, + checksum: 0, + timestamp: 0, + origin_timestamp: 0, + headers: None, + payload: iggy_connector_sdk::Payload::Raw(b"payload".to_vec()), + } + } + + fn topic_and_messages_metadata() -> (TopicMetadata, MessagesMetadata) { + ( + TopicMetadata { + stream: "s".to_string(), + topic: "t".to_string(), + }, + MessagesMetadata { + partition_id: 1, + current_offset: 0, + schema: iggy_connector_sdk::Schema::Json, + }, + ) + } + + // push_batch() is an unimplemented TODO(ConnectorDeveloper) stub that + // always returns Error::InitError — not a PermanentHttpError — so a + // non-empty consume() call exercises the "last_err" (retryable) branch + // of the chunking loop end to end, including the failed-count and + // circuit-breaker bookkeeping. + #[tokio::test] + async fn given_unimplemented_push_batch_should_return_err_and_count_failure() { + let mut sink = TemplateSink::new(1, test_config()); + sink.open().await.expect("open should succeed"); + let (topic_metadata, messages_metadata) = topic_and_messages_metadata(); + + let result = sink + .consume(&topic_metadata, messages_metadata, vec![consumed_message()]) + .await; + + assert!(matches!(result, Err(Error::InitError(_)))); + assert_eq!(sink.state.lock().await.messages_failed, 1); + assert_eq!(sink.state.lock().await.messages_written, 0); + } +} diff --git a/core/connectors/sources/README.md b/core/connectors/sources/README.md index d7e53a34e8..e261ebec88 100644 --- a/core/connectors/sources/README.md +++ b/core/connectors/sources/README.md @@ -14,6 +14,7 @@ Source connectors are responsible for ingesting data from external sources into | **mqtt_source** | Subscribes to MQTT 3.1.1 or MQTT 5 brokers with QoS 0, 1, or 2 | | **postgres_source** | Reads rows from PostgreSQL tables with multiple strategies: delete after read, mark as processed, or timestamp tracking | | **random_source** | Generates random test messages (useful for testing and development) | +| **source_template** | Fill-in-the-blank starting point for a new source; framework/security plumbing done, two `TODO(ConnectorDeveloper)` spots left | The source is represented by the single `Source` trait, which defines the basic interface for all source connectors. It provides methods for initializing the source, reading data from it, and closing the source. diff --git a/core/connectors/sources/source_template/Cargo.toml b/core/connectors/sources/source_template/Cargo.toml new file mode 100644 index 0000000000..dd29dbacd7 --- /dev/null +++ b/core/connectors/sources/source_template/Cargo.toml @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# TEMPLATE — rename the package (and this directory) to +# `iggy_connector__source` before publishing, and update the +# `[[sources]]` entry you add to the workspace root Cargo.toml accordingly. + +[package] +name = "iggy_connector_template_source" +version = "0.1.0" +description = "Template for an Apache Iggy source connector — copy this crate and fill in the TODO sections." +edition = "2024" +license = "Apache-2.0" +keywords = ["iggy", "messaging", "streaming"] +categories = ["command-line-utilities", "database", "network-programming"] +homepage = "https://iggy.apache.org" +documentation = "https://iggy.apache.org/docs" +repository = "https://github.com/apache/iggy" +publish = false + +[package.metadata.cargo-machete] +# dashmap is used only inside the `source_connector!` macro expansion, so a +# naive unused-dependency scan won't see the usage — keep it ignored rather +# than removing it, or the plugin will fail to compile. +ignored = ["dashmap"] + +[lib] +crate-type = ["cdylib", "lib"] + +[dependencies] +async-trait = { workspace = true } +dashmap = { workspace = true } +humantime = { workspace = true } +iggy_common = { workspace = true } +iggy_connector_sdk = { workspace = true } +reqwest = { workspace = true } +reqwest-middleware = { workspace = true } +rmp-serde = { workspace = true } +secrecy = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } diff --git a/core/connectors/sources/source_template/README.md b/core/connectors/sources/source_template/README.md new file mode 100644 index 0000000000..234f10d21a --- /dev/null +++ b/core/connectors/sources/source_template/README.md @@ -0,0 +1,97 @@ +# Template source connector + +Starting point for a new Apache Iggy **source** connector. Everything except +talking to your actual external system is already implemented and follows +the project's required resilience/security patterns — see the module-level +doc comment at the top of `src/lib.rs` for the full rationale, and the +`iggy-connector-review` skill / the "Building Connectors That Pass Review" +blog post for the checklist this template is built against. + +## Delivery Semantics + +The framework around `fetch_records()` gives you retry (`max_retries` total +attempts), a circuit breaker, and the ACK/NACK cursor handshake described +below. Everything past "what fetches" depends on what you write in +`fetch_records()` — fill in the blanks with your real behavior before +shipping: + +1. **Transient failure**: retries up to `max_retries` total attempts, then + `poll()` returns the error; the circuit breaker counts it. The runtime + retries `poll()` again next interval. +2. **Permanent failure** (auth rejected, schema gone): return + `Error::PermanentHttpError` from `fetch_records()` — `poll()` does not + trip the circuit breaker on it, since a bad config/auth state isn't a + connectivity signal the breaker should react to. +3. **Duplication window**: at-least-once. `poll()` stages the cursor as + *candidate* state; the runtime persists it only after the batch is sent, + then reports `Ack`/`Nack`. A NACK (or a crash between send and the + persisted-state write) redelivers the same range on the next poll. +4. **Dedup key**: set `FetchedRecord::natural_id` from a true per-record + identity (row PK, document `_id`, event UUID) whenever your source has + one — it becomes `ProducedMessage.id`, which a downstream sink can key an + idempotent upsert on (see `mongodb_sink`) so the redelivery above doesn't + produce a duplicate at the destination. Leave it `None` only when records + genuinely have no natural identity (a pure generator). See + `connector-pr-review` B5. + +## What's already done for you + +- Config parsing with `#[serde(deny_unknown_fields)]` so a typo in a TOML + file fails loudly instead of silently doing nothing. +- Config validation in `open()` (not `new()`, which has no way to return an + error). +- `connection_string` and the optional `auth_token` field both typed as + `SecretString`, since either can carry credentials. +- A retry-wrapped HTTP client (`iggy_connector_sdk::retry::build_retry_client`) + and a startup connectivity probe with its own backoff + (`check_connectivity_with_retry`). +- A `CircuitBreaker` that's actually consulted before polling and updated + after every attempt — not just constructed and forgotten. +- Cursor staging: `poll()` never commits its progress directly; it stages a + candidate and `on_batch_result()` commits it only on `Ack`, discarding it + on `Nack` so a failed delivery gets re-polled instead of silently lost. +- The `source_connector!` FFI macro invocation and a `Cargo.toml` with the + right `crate-type`, workspace-pinned dependencies, and license header. +- `verbose_logging: Option` upgrading the per-poll log line from + `debug!` to `info!`, mirroring the runtime's own `verbose` flag. +- The six canonical state/Ack-Nack tests, plus config validation and the + `verbose_logging` flag. + +## What you need to fill in + +Search for `TODO(ConnectorDeveloper)` in `src/lib.rs` — there are exactly +three spots: one required, one conditional, one documentation-only. + +1. **`fetch_records()`** (required) — fetch up to `self.batch_size` new + records from your system, ordered after `cursor` (`None` = start from the + beginning, or from "now" — whichever fits your source). Map each result + to a `FetchedRecord { cursor_value, payload }`, using something + monotonically increasing as `cursor_value` (a timestamp, an ID, a page + token) — that's what lets the cursor-staging logic advance correctly. +2. **`build_raw_client()`** (only if your source isn't HTTP) — replace the + `reqwest::Client` construction with your driver's connection/pool setup + (see `core/connectors/sources/postgres_source` for a real non-HTTP + example), store it on `TemplateSource` (you'll need to add a field — + `client: Option` here is HTTP-specific), and adjust + or remove the `check_connectivity_with_retry` call in `open()` in favor + of whatever connectivity check your driver offers. +3. **`connection_string`'s doc comment** (documentation only) — describe + the exact shape your connector expects instead of the generic example. + +## Using it + +1. Copy this directory, rename it and the package in `Cargo.toml` + (`iggy_connector__source`), and add it to the `members` list in + the workspace root `Cargo.toml`. +2. Fill in `fetch_records()` (and `build_raw_client()` if not HTTP). +3. Update `config.toml` with your real `connection_string` and any + settings specific to your system; delete `auth_token` if you don't need + it, or add fields of your own the same way (see `TemplateSourceConfig`). +4. `cargo build --release -p iggy_connector__source`, point a + runtime connector config file's `path` at the built `.so`/`.dylib`/`.dll`, + and run the connector runtime — see `core/connectors/README.md` in this + repo for the full runtime quick-start. +5. Before opening a PR: `cargo test`, `cargo clippy --all-targets`, + `cargo fmt --check`, and re-read the connector-review checklist once more + with fresh eyes — most review round-trips come from one of the items in + that list, not from the connector-specific logic in `fetch_records()`. diff --git a/core/connectors/sources/source_template/config.toml b/core/connectors/sources/source_template/config.toml new file mode 100644 index 0000000000..721abb0464 --- /dev/null +++ b/core/connectors/sources/source_template/config.toml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +type = "source" +key = "template" +enabled = true +version = 0 +name = "Template source" +path = "../../target/release/libiggy_connector_template_source" +plugin_config_format = "toml" +verbose = false + +[[streams]] +stream = "example_stream" +topic = "example_topic" +schema = "json" +batch_length = 100 +linger_time = "5ms" + +[plugin_config] +connection_string = "https://api.example.com" +health_check_path = "/health" +batch_size = 100 +poll_interval = "1s" +timeout = "30s" +max_retries = 3 +retry_delay = "500ms" +retry_max_delay = "5s" +max_open_retries = 10 +open_retry_max_delay = "60s" +circuit_breaker_threshold = 5 +circuit_breaker_cool_down = "30s" +verbose_logging = false +# auth_token = "replace-me" # uncomment if your source needs bearer/API-key auth diff --git a/core/connectors/sources/source_template/src/lib.rs b/core/connectors/sources/source_template/src/lib.rs new file mode 100644 index 0000000000..7dafc097f9 --- /dev/null +++ b/core/connectors/sources/source_template/src/lib.rs @@ -0,0 +1,671 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Template Apache Iggy **source** connector. +//! +//! Everything in this file is already wired up and follows the patterns the +//! connector-review checklist expects: config validation happens in +//! `open()`, secrets are `SecretString`, outbound calls go through a +//! retry-wrapped client plus a circuit breaker, and the read cursor is +//! staged in `poll()` and only committed in `on_batch_result()` on an ACK so +//! a dropped/nacked batch can be re-polled instead of silently lost. +//! +//! There is exactly **one** place you need to write code, marked +//! `TODO(ConnectorDeveloper)`: `TemplateSource::fetch_records()` — fetch up +//! to `batch_size` new records from your external system, starting after +//! `cursor`. `TemplateSource::build_raw_client()` carries a second +//! `TODO(ConnectorDeveloper)` too, but only applies if your source isn't +//! HTTP (see below); a third, on the `connection_string` field's doc +//! comment, just asks you to describe its expected shape — not code. +//! `grep` for the marker and you'll find all three. +//! +//! This template assumes an HTTP-ish source and uses `reqwest` wrapped by +//! the SDK's retry middleware, because that's what `iggy_connector_sdk::retry` +//! is built for and it covers the common case. If your source talks to +//! something else (a database, a queue, a filesystem), swap the client type +//! in `build_raw_client()`/`fetch_records()` for your driver of choice and lean on +//! its own retry/pooling behavior — keep the surrounding shape (validation +//! in `open()`, circuit breaker, cursor staging, batching) unchanged. See +//! `core/connectors/sources/postgres_source` in this repo for a real +//! non-HTTP example of that same shape. + +use async_trait::async_trait; +use iggy_connector_sdk::retry::{ + CircuitBreaker, RetryPolicy, build_retry_client, check_connectivity_with_retry, parse_duration, +}; +use iggy_connector_sdk::{ + ConnectorState, Error, ProducedMessage, ProducedMessages, Schema, Source, + source::SourceBatchResult, source_connector, +}; +use reqwest::Url; +use reqwest_middleware::ClientWithMiddleware; +use secrecy::{ExposeSecret, SecretString}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; +use tokio::sync::Mutex; +use tracing::{debug, error, info, warn}; + +source_connector!(TemplateSource); + +const CONNECTOR_NAME: &str = "Template source"; + +const DEFAULT_POLL_INTERVAL: &str = "1s"; +const DEFAULT_BATCH_SIZE: u32 = 100; +const DEFAULT_TIMEOUT: &str = "30s"; +const DEFAULT_MAX_RETRIES: u32 = 3; +const DEFAULT_RETRY_DELAY: &str = "500ms"; +const DEFAULT_RETRY_MAX_DELAY: &str = "5s"; +const DEFAULT_MAX_OPEN_RETRIES: u32 = 10; +const DEFAULT_OPEN_RETRY_MAX_DELAY: &str = "60s"; +const DEFAULT_CIRCUIT_BREAKER_THRESHOLD: u32 = 5; +const DEFAULT_CIRCUIT_BREAKER_COOL_DOWN: &str = "30s"; + +// ── Configuration ─────────────────────────────────────────────────────────── +// +// Every tunable except `connection_string` is optional with a sane default, +// and unknown keys are rejected outright so a typo in a TOML file fails at +// load time instead of silently doing nothing. + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemplateSourceConfig { + /// TODO(ConnectorDeveloper): document the exact shape this connector expects, e.g. + /// "" or "postgres://user:pass@host:5432/db". + /// `SecretString` because DSNs commonly embed credentials — never plain + /// `String` for this field, see `PostgresSinkConfig::connection_string` + /// in `sinks/postgres_sink` for the same pattern. + #[serde(serialize_with = "iggy_common::serde_secret::serialize_secret")] + pub connection_string: SecretString, + + /// Example of a secret-shaped setting. `SecretString` keeps it out of + /// `Debug`/log output; delete this field if `connection_string` already + /// carries all required auth. Read it with `.expose_secret()` (from the + /// `secrecy::ExposeSecret` trait) at the one place you actually need the + /// plaintext — e.g. when building an auth header in `build_raw_client()`. + #[serde( + default, + serialize_with = "iggy_common::serde_secret::serialize_optional_secret" + )] + pub auth_token: Option, + + /// Optional path (e.g. "/health") probed with retry during `open()` + /// before the connector is considered ready. Leave unset if your target + /// has no health endpoint — the probe is skipped, not failed, in that case. + pub health_check_path: Option, + + pub batch_size: Option, + pub poll_interval: Option, + pub timeout: Option, + pub max_retries: Option, + pub retry_delay: Option, + pub retry_max_delay: Option, + pub max_open_retries: Option, + pub open_retry_max_delay: Option, + pub circuit_breaker_threshold: Option, + pub circuit_breaker_cool_down: Option, + + /// Upgrades the per-poll log line from `debug!` to `info!`. Mirrors the + /// runtime's own `verbose` flag; keep the field name `verbose_logging` + /// (see `postgres_source::PostgresSourceConfig::verbose_logging`). + pub verbose_logging: Option, +} + +// ── Internal state ────────────────────────────────────────────────────────── + +/// Read cursor. Kept as a plain `Option` so it fits whatever ordering +/// field your source uses (a timestamp, an auto-increment ID, an opaque +/// pagination token, ...) — stringify it however makes sense for your data. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +struct State { + cursor: Option, +} + +/// One record as fetched from the external system, before it's turned into +/// an Iggy message. Replace the `payload` type with whatever your +/// `fetch_records()` actually produces. +struct FetchedRecord { + /// The value `cursor` should advance to once this record's batch is + /// acknowledged — typically this record's timestamp/ID/token. + cursor_value: String, + /// TODO(ConnectorDeveloper): set this from a true per-record identity — + /// a numeric row PK, a hash of a document `_id`, an event UUID the + /// upstream system itself assigned — so `ProducedMessage.id` below is + /// stable across a NACK-triggered redelivery of this same record. A + /// *fresh* random id per emit defeats a downstream sink's dedup-by-id + /// (see `connector-pr-review` B5): a redelivered row would arrive with a + /// different id than its first delivery, so a sink keying its upsert on + /// `message.id` would store it twice instead of overwriting. Leave this + /// `None` only when records genuinely have no natural per-record + /// identity (a pure generator, e.g. `random_source`) — `cursor_value` + /// alone usually isn't it either, since a timestamp-based cursor is + /// commonly shared by more than one record. + natural_id: Option, + payload: serde_json::Value, +} + +#[derive(Debug)] +pub struct TemplateSource { + id: u32, + config: TemplateSourceConfig, + client: Option, + circuit_breaker: Arc, + batch_size: u32, + poll_interval: Duration, + retry_delay: Duration, + verbose: bool, + state: Mutex, + pending_state: Mutex>, + records_produced: AtomicU64, +} + +impl TemplateSource { + pub fn new(id: u32, config: TemplateSourceConfig, state: Option) -> Self { + let poll_interval = *humantime::Duration::from_str_lossy( + config + .poll_interval + .as_deref() + .unwrap_or(DEFAULT_POLL_INTERVAL), + ); + let retry_delay = parse_duration(config.retry_delay.as_deref(), DEFAULT_RETRY_DELAY); + let circuit_breaker = Arc::new(CircuitBreaker::new( + config + .circuit_breaker_threshold + .unwrap_or(DEFAULT_CIRCUIT_BREAKER_THRESHOLD), + parse_duration( + config.circuit_breaker_cool_down.as_deref(), + DEFAULT_CIRCUIT_BREAKER_COOL_DOWN, + ), + )); + let batch_size = config.batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1); + let verbose = config.verbose_logging.unwrap_or(false); + + let restored_state = state + .and_then(|s| s.deserialize::(CONNECTOR_NAME, id)) + .inspect(|s| { + info!( + "Restored state for {CONNECTOR_NAME} connector with ID: {id}. Cursor: {:?}", + s.cursor + ); + }); + + Self { + id, + config, + client: None, + circuit_breaker, + batch_size, + poll_interval, + retry_delay, + verbose, + state: Mutex::new(restored_state.unwrap_or_default()), + pending_state: Mutex::new(None), + records_produced: AtomicU64::new(0), + } + } + + fn serialize_state(&self, state: &State) -> Option { + ConnectorState::serialize(state, CONNECTOR_NAME, self.id) + } + + /// TODO(ConnectorDeveloper): build your actual client/connection here using + /// `self.config.connection_string` (and `self.config.auth_token`, if + /// your system needs it). This template builds a plain `reqwest::Client` + /// to hand to `build_retry_client` — if you're not talking HTTP, replace + /// this with e.g. a `sqlx::PgPool::connect(...)` or your driver's + /// equivalent, store it on `self` (add a field, since `client` here is + /// HTTP-specific), and skip the `check_connectivity_with_retry` call + /// below in favor of whatever your driver offers (a ping, a test query). + fn build_raw_client(&self) -> Result { + let timeout = parse_duration(self.config.timeout.as_deref(), DEFAULT_TIMEOUT); + reqwest::Client::builder() + .timeout(timeout) + .build() + .map_err(|e| Error::Connection(format!("failed to build HTTP client: {e}"))) + } + + /// TODO(ConnectorDeveloper): fetch up to `self.batch_size` new records from your + /// external system, ordered after `cursor` (`None` means "from the + /// beginning" or "from now" — whichever is right for your source). + /// Use `self.config.connection_string` as the base address and + /// `self.client` (already retry-wrapped) to make the request. Map each + /// result row/document/event to a `FetchedRecord`, using something that + /// monotonically increases (a timestamp, an ID, a page token) as + /// `cursor_value` so the state-staging logic in `poll()` below can + /// advance the cursor correctly, and set `natural_id` from a true + /// per-record identity if one exists (see the field's own doc comment). + async fn fetch_records( + &self, + client: &ClientWithMiddleware, + cursor: Option<&str>, + ) -> Result, Error> { + let _ = (client, cursor); // remove once implemented + Err(Error::InitError( + "TemplateSource::fetch_records is not implemented yet — see the TODO(ConnectorDeveloper) comment \ + in template_source/src/lib.rs" + .to_string(), + )) + } +} + +// ── Source trait ──────────────────────────────────────────────────────────── + +#[async_trait] +impl Source for TemplateSource { + async fn open(&mut self) -> Result<(), Error> { + // Structural validation happens here, not in `new()`, because only + // `open()` can return an error — `new()` is a plain factory function + // with nowhere to send a "this config is invalid" result. + if self + .config + .connection_string + .expose_secret() + .trim() + .is_empty() + { + return Err(Error::InvalidConfigValue( + "connection_string must not be empty".to_string(), + )); + } + + info!( + "Opening {CONNECTOR_NAME} connector with ID: {}, batch_size: {}, poll_interval: {:?}", + self.id, self.batch_size, self.poll_interval + ); + + let raw_client = self.build_raw_client()?; + + if let Some(health_path) = &self.config.health_check_path { + let base = Url::parse(self.config.connection_string.expose_secret()).map_err(|e| { + Error::InvalidConfigValue(format!("connection_string is not a valid URL: {e}")) + })?; + let health_url = base.join(health_path).map_err(|e| { + Error::InvalidConfigValue(format!("invalid health_check_path: {e}")) + })?; + check_connectivity_with_retry( + &raw_client, + health_url, + CONNECTOR_NAME, + self.id, + RetryPolicy { + max_attempts: self + .config + .max_open_retries + .unwrap_or(DEFAULT_MAX_OPEN_RETRIES), + base_delay: self.retry_delay, + max_delay: parse_duration( + self.config.open_retry_max_delay.as_deref(), + DEFAULT_OPEN_RETRY_MAX_DELAY, + ), + }, + ) + .await?; + } else { + warn!( + "{CONNECTOR_NAME} connector with ID: {} has no health_check_path configured — \ + skipping the startup connectivity probe. Consider adding one.", + self.id + ); + } + + self.client = Some(build_retry_client( + raw_client, + self.config + .max_retries + .unwrap_or(DEFAULT_MAX_RETRIES) + .max(1), + self.retry_delay, + parse_duration( + self.config.retry_max_delay.as_deref(), + DEFAULT_RETRY_MAX_DELAY, + ), + CONNECTOR_NAME, + )); + + info!( + "{CONNECTOR_NAME} connector with ID: {} opened successfully", + self.id + ); + Ok(()) + } + + async fn poll(&self) -> Result { + // If the breaker is open, sleep for the normal poll interval and + // return an empty (not an error) result. Returning `Err` here would + // make the runtime retry `poll()` again immediately with no delay — + // see `handle_messages` in the SDK's source container — so an empty + // ACK-free result is how a source waits out a known-bad window + // without busy-looping or counting against the NACK budget. + if self.circuit_breaker.is_open().await { + warn!( + "{CONNECTOR_NAME} connector with ID: {} — circuit breaker OPEN, skipping poll", + self.id + ); + tokio::time::sleep(self.poll_interval).await; + return Ok(ProducedMessages { + schema: Schema::Json, + messages: Vec::new(), + state: None, + }); + } + tokio::time::sleep(self.poll_interval).await; + + let client = self.client.as_ref().ok_or_else(|| { + Error::Connection("client not initialized -- was open() called?".into()) + })?; + let cursor = self.state.lock().await.cursor.clone(); + + let records = match self.fetch_records(client, cursor.as_deref()).await { + Ok(records) => { + self.circuit_breaker.record_success(); + records + } + Err(err) => { + if !matches!(err, Error::PermanentHttpError(_)) { + self.circuit_breaker.record_failure().await; + } + return Err(err); + } + }; + + if records.is_empty() { + return Ok(ProducedMessages { + schema: Schema::Json, + messages: Vec::new(), + state: None, + }); + } + + let mut messages = Vec::with_capacity(records.len()); + let mut new_cursor = cursor; + for record in records { + let Ok(payload) = serde_json::to_vec(&record.payload) else { + error!( + "Failed to serialize a record fetched by {CONNECTOR_NAME} connector with ID: {}", + self.id + ); + continue; + }; + // Only advance the candidate cursor once the record actually made it + // into `messages` - advancing on a dropped (unserializable) record + // would stage a cursor past data that was never produced, and a + // later Ack would commit past it permanently. + let natural_id = record.natural_id; + new_cursor = Some(record.cursor_value); + messages.push(ProducedMessage { + id: natural_id, + headers: None, + checksum: None, + timestamp: None, + origin_timestamp: None, + payload, + }); + } + + if messages.is_empty() { + // Every fetched record failed to serialize - no progress was + // made, so there's nothing to stage and no reason to write + // state, same as the `records.is_empty()` case above. + return Ok(ProducedMessages { + schema: Schema::Json, + messages: Vec::new(), + state: None, + }); + } + + let candidate_state = State { cursor: new_cursor }; + let persisted_state = self.serialize_state(&candidate_state).ok_or_else(|| { + Error::Serialization(format!( + "failed to serialize state for {CONNECTOR_NAME} connector with ID: {}", + self.id + )) + })?; + *self.pending_state.lock().await = Some(candidate_state); + + self.records_produced + .fetch_add(messages.len() as u64, Ordering::Relaxed); + + if self.verbose { + info!( + "{CONNECTOR_NAME} connector with ID: {} produced {} messages", + self.id, + messages.len() + ); + } else { + debug!( + "{CONNECTOR_NAME} connector with ID: {} produced {} messages", + self.id, + messages.len() + ); + } + + Ok(ProducedMessages { + schema: Schema::Json, + messages, + state: Some(persisted_state), + }) + } + + /// The staged cursor from `poll()` is only committed here, and only on + /// an ACK. A NACK (delivery failed, batch timed out, runtime is + /// shutting down) discards the candidate so the same range is re-polled + /// next time — the cursor never moves past data that wasn't confirmed + /// delivered. If `fetch_records()` ever needs to perform a destructive + /// read against the source (delete-after-read, mark-as-processed), + /// stage that side effect the same way and only apply it here on `Ack`. + /// + /// If you add real I/O here (an external ack call, a delete, a mark- + /// processed update), two things from a real connector's review history + /// apply directly: + /// + /// 1. This runs inside the SDK's fixed ~30s batch-result window, and a + /// size-your-own-timeout mistake here is fatal, not just slow — a + /// NACK from that window stops the source for good after repeated + /// consecutive NACKs. If you stage more than one operation, budget + /// for *all* of them inside the window, not one op's worth, and + /// reject `max_retries`/`retry_delay` combinations in `open()` that + /// could blow it. See `connector-pr-review` B11. + /// 2. A failed op must not be silently logged while the batch is still + /// ACKed — the cursor has already committed past it by the time you + /// find out. Count consecutive failures and return `Err` past a + /// threshold so the SDK stops the source, instead of resending the + /// same broken operation forever. + async fn on_batch_result(&self, result: SourceBatchResult) -> Result<(), Error> { + let candidate_state = self.pending_state.lock().await.take(); + if result == SourceBatchResult::Ack + && let Some(candidate_state) = candidate_state + { + *self.state.lock().await = candidate_state; + } + Ok(()) + } + + async fn close(&mut self) -> Result<(), Error> { + let state = self.state.lock().await; + info!( + "{CONNECTOR_NAME} connector with ID: {} closed. Cursor: {:?}, total records produced: {}", + self.id, + state.cursor, + self.records_produced.load(Ordering::Relaxed) + ); + drop(state); + self.client = None; + Ok(()) + } +} + +// Small local shim so `new()` doesn't need to pull in `humantime::Duration`'s +// `FromStr` (which returns `Result`) just to apply a default — mirrors the +// fallback-with-warning behavior of `iggy_connector_sdk::retry::parse_duration` +// for the one duration field (`poll_interval`) that isn't itself optional in +// spirit (there's always a poll interval, just maybe the default one). +trait DurationExt { + fn from_str_lossy(s: &str) -> humantime::Duration; +} +impl DurationExt for humantime::Duration { + fn from_str_lossy(s: &str) -> humantime::Duration { + use std::str::FromStr; + humantime::Duration::from_str(s).unwrap_or_else(|_| { + warn!("Invalid poll_interval {s:?}, falling back to {DEFAULT_POLL_INTERVAL}"); + humantime::Duration::from_str(DEFAULT_POLL_INTERVAL) + .expect("DEFAULT_POLL_INTERVAL must itself be a valid duration literal") + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_config() -> TemplateSourceConfig { + TemplateSourceConfig { + connection_string: SecretString::from("https://api.example.com"), + auth_token: None, + health_check_path: None, + batch_size: Some(50), + poll_interval: Some("50ms".to_string()), + timeout: Some("5s".to_string()), + max_retries: Some(2), + retry_delay: Some("10ms".to_string()), + retry_max_delay: Some("100ms".to_string()), + max_open_retries: Some(2), + open_retry_max_delay: Some("100ms".to_string()), + circuit_breaker_threshold: Some(3), + circuit_breaker_cool_down: Some("50ms".to_string()), + verbose_logging: None, + } + } + + #[test] + fn given_verbose_logging_enabled_should_set_verbose_flag() { + let mut config = test_config(); + config.verbose_logging = Some(true); + let source = TemplateSource::new(1, config, None); + assert!(source.verbose); + } + + #[test] + fn given_verbose_logging_disabled_should_not_set_verbose_flag() { + let source = TemplateSource::new(1, test_config(), None); + assert!(!source.verbose); + } + + #[tokio::test] + async fn given_empty_connection_string_should_reject_open() { + let mut config = test_config(); + config.connection_string = SecretString::from(" "); + let mut source = TemplateSource::new(1, config, None); + let result = source.open().await; + assert!(matches!(result, Err(Error::InvalidConfigValue(_)))); + } + + #[tokio::test] + async fn given_no_state_should_start_with_no_cursor() { + let source = TemplateSource::new(1, test_config(), None); + assert_eq!(source.state.lock().await.cursor, None); + } + + #[tokio::test] + async fn given_persisted_state_should_restore_cursor() { + let state = State { + cursor: Some("2024-01-01T00:00:00Z".to_string()), + }; + let serialized = rmp_serde::to_vec(&state).expect("failed to serialize state"); + let source = TemplateSource::new(1, test_config(), Some(ConnectorState(serialized))); + assert_eq!( + source.state.lock().await.cursor, + Some("2024-01-01T00:00:00Z".to_string()) + ); + } + + #[tokio::test] + async fn given_invalid_persisted_state_should_start_fresh() { + let source = TemplateSource::new( + 1, + test_config(), + Some(ConnectorState(b"not valid msgpack".to_vec())), + ); + assert_eq!(source.state.lock().await.cursor, None); + } + + #[test] + fn state_should_be_serializable_and_deserializable() { + let original = State { + cursor: Some("2024-01-01T00:00:00Z".to_string()), + }; + + let serialized = rmp_serde::to_vec(&original).expect("failed to serialize state"); + let deserialized: State = + rmp_serde::from_slice(&serialized).expect("failed to deserialize state"); + + assert_eq!(original.cursor, deserialized.cursor); + } + + #[tokio::test] + async fn given_ack_should_commit_staged_cursor() { + let source = TemplateSource::new(1, test_config(), None); + *source.pending_state.lock().await = Some(State { + cursor: Some("next-cursor".to_string()), + }); + + source + .on_batch_result(SourceBatchResult::Ack) + .await + .expect("ACK should be applied"); + + assert_eq!( + source.state.lock().await.cursor, + Some("next-cursor".to_string()) + ); + assert!(source.pending_state.lock().await.is_none()); + } + + #[tokio::test] + async fn given_nack_should_discard_staged_cursor() { + let source = TemplateSource::new(1, test_config(), None); + *source.pending_state.lock().await = Some(State { + cursor: Some("next-cursor".to_string()), + }); + + source + .on_batch_result(SourceBatchResult::Nack) + .await + .expect("NACK should be applied"); + + // The committed cursor is unchanged (still None) — the candidate is + // simply discarded so the same range is polled again. + assert_eq!(source.state.lock().await.cursor, None); + assert!(source.pending_state.lock().await.is_none()); + } + + #[tokio::test(start_paused = true)] + async fn given_open_circuit_breaker_should_return_empty_without_error() { + let source = TemplateSource::new(1, test_config(), None); + source.circuit_breaker.record_failure().await; + source.circuit_breaker.record_failure().await; + source.circuit_breaker.record_failure().await; + assert!(source.circuit_breaker.is_open().await); + + let result = source + .poll() + .await + .expect("open breaker should not error poll()"); + assert!(result.messages.is_empty()); + assert!(result.state.is_none()); + } +} From b51e55e7d6fc6cdd499151251a653ffb75feb979 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 24 Oct 2026 09:21:17 -0400 Subject: [PATCH 4/5] Adding connector related skills --- .claude/skills/connector-pr-review/SKILL.md | 49 ++++++++++++------- .claude/skills/connector-source/SKILL.md | 2 +- .claude/skills/connector-testing/SKILL.md | 7 +-- .claude/skills/connectors-overview/SKILL.md | 2 +- .../sources/source_template/src/lib.rs | 23 +++++---- 5 files changed, 52 insertions(+), 31 deletions(-) diff --git a/.claude/skills/connector-pr-review/SKILL.md b/.claude/skills/connector-pr-review/SKILL.md index 0ae07f3cd1..1a317a6dde 100644 --- a/.claude/skills/connector-pr-review/SKILL.md +++ b/.claude/skills/connector-pr-review/SKILL.md @@ -180,30 +180,40 @@ server-side. Confirmed independently on two unrelated backends: `.timeout()` on the `reqwest::Client` itself rather than wrapping calls — copy that pattern, don't wrap instead of it. -### B11. Ack-path work must fit inside the SDK's fixed batch-result window - -If `on_batch_result` (or any post-ack cleanup) does real I/O — not just an -in-memory state swap — that I/O runs inside the SDK's ~30s batch-result -timeout, and a NACK from that timeout is often **fatal** (stops the source -for good after repeated consecutive NACKs), not a retry signal. - -- [ ] Any retry/timeout budget inside `on_batch_result` must be sized well - under the SDK's window, and **multiplied by how many operations run - inside it** — not assumed to be one shared budget. Postgres (#3957, +### B11. Ack-path work has no SDK deadline, but it blocks every later poll + +`on_batch_result` has no SDK-enforced time limit, and a NACK does **not** +come from a timeout — the SDK (since #4269) only warns past ~30s without a +runtime result and keeps waiting; it never synthesizes a Nack or moves on. +The real risk is different: the SDK allows exactly one batch in flight, so +`poll()` is not called again until `on_batch_result` returns. Real I/O in +there — not just an in-memory state swap — silently stalls the whole source +for as long as it takes, with nothing visibly "failing." + +- [ ] A retry/timeout budget inside `on_batch_result` must account for + **every operation it runs, not one op's worth** — a per-table or + per-item budget multiplies with the item count. Postgres (#3957, critical): "10s is per operation, not per batch... N tables burn - N\*10s inside the SDK's 30s window." -- [ ] User-configurable `max_retries`/`retry_delay` can produce a wait time - that exceeds a hardcoded internal deadline — validate the combination - in `open()` and reject configs that cannot fit, rather than letting - the ack path blow its budget at runtime. Same PR: "`max_retries=3` - with `retry_delay=\"2s\"`, both straight from the readme, needs 12s - and fails startup." + N\*10s" serially inside one `on_batch_result` call, stalling every + later poll by that much. +- [ ] User-configurable `max_retries`/`retry_delay` can combine into a wait + time nobody sized on purpose — validate the combination in `open()` + and reject configs that blow an internal budget, rather than letting + the ack path stall at runtime. Same PR: "`max_retries=3` with + `retry_delay=\"2s\"`, both straight from the readme, needs 12s and + fails startup" once a sane cap was added. - [ ] A failed destructive/cleanup op inside `on_batch_result` must not be silently logged and ignored while the batch is still ACKed — count consecutive failures and return `Err` past a threshold so the SDK stops the source, rather than resending the same broken operation forever. Same PR: "a failed slot advance only logs. the batch is already ACKed, so nothing stops it." +- [ ] A source whose input genuinely cannot be replayed after a restart + (so repeated-Nack-triggered stop would be the wrong failure mode) can + override `Source::batch_policy()` and call + `BatchPolicy::with_max_consecutive_nacks(None)` (SDK 0.5.1-edge.2+) — + prefer that over working around the default 5-consecutive-Nack stop + inside plugin code. ## High-frequency convention nits @@ -261,6 +271,11 @@ These are "cheap" but burn full review rounds when missed. - [ ] Helper modules stay crate-private (no `pub mod`) unless genuinely part of the plugin's public API — S3 (#3103) exposed `buffer`/`client`/etc. as public on a cdylib plugin where every sibling keeps them private. +- [ ] Crate-level doc comment doesn't use `[module]` intra-doc links to a + private module — rustdoc warns on the dead link and the doctest CI job + (`-D warnings`) fails the build. BigQuery (#4394): the crate doc linked + five private modules this way. Use a plain code span (`` `module` ``) + instead when the target isn't `pub`. ### C4. Hot path diff --git a/.claude/skills/connector-source/SKILL.md b/.claude/skills/connector-source/SKILL.md index 126098b9c4..6465548999 100644 --- a/.claude/skills/connector-source/SKILL.md +++ b/.claude/skills/connector-source/SKILL.md @@ -172,7 +172,7 @@ Match `ProducedMessages.schema` to the bytes in `messages[i].payload`: ### Concurrency - Runtime spawns ONE `poll()` task per source. No concurrent `poll()`. -- Only one batch is ever in flight: the SDK does not call `poll()` again until `on_batch_result()` has returned for the previous batch (up to a 30s result timeout, after which the SDK treats it as a Nack). +- Only one batch is ever in flight: the SDK does not call `poll()` again until the runtime has reported `Ack`/`Nack` and `on_batch_result()` has returned for the previous batch. Past 30s without a result the SDK only warns and keeps waiting - it does not synthesize a Nack or move on. - Don't spawn your own long-running Tokio tasks: the runtime owns lifecycle. The exception is a source that listens rather than polls, which has to own its listener; `http_source` is the worked example, and it shuts its tasks down in the last `close()` rather than leaving them to outlive the connector. ### Errors diff --git a/.claude/skills/connector-testing/SKILL.md b/.claude/skills/connector-testing/SKILL.md index fcbbbab75a..c817de39ef 100644 --- a/.claude/skills/connector-testing/SKILL.md +++ b/.claude/skills/connector-testing/SKILL.md @@ -101,9 +101,10 @@ fn test_config() -> RandomSourceConfig { ### Async testing pattern The trait methods are `async`. Prefer `#[tokio::test]` directly on the test fn - `#[cfg(test)]` -gates the whole module out of a plain `cargo build`, and resolver v2 (default since edition 2021, -which this workspace uses) keeps `tokio`'s `test-util`/`macros` dev-dependency features out of that -build too, so there's no prod-build leak to avoid. This is also the dominant pattern in-tree +gates the whole module out of a plain `cargo build`, and this workspace's resolver (`resolver = "3"` +in the root `Cargo.toml`, tied to the edition-2024 crates here) keeps `tokio`'s `test-util`/`macros` +dev-dependency features out of that build too, so there's no prod-build leak to avoid. This is also +the dominant pattern in-tree (`postgres_source::tests` uses `#[tokio::test]` throughout, no manual runtime): ```rust diff --git a/.claude/skills/connectors-overview/SKILL.md b/.claude/skills/connectors-overview/SKILL.md index a3a3575eeb..923e1b7d5e 100644 --- a/.claude/skills/connectors-overview/SKILL.md +++ b/.claude/skills/connectors-overview/SKILL.md @@ -27,7 +27,7 @@ New sink/source authors: start with [`core/connectors/docs/authoring-sinks-and-s ## STOP and ask the user before -- Bumping `iggy_connector_sdk` MAJOR version or changing any FFI signature in `sdk/src/{sink,source}.rs` - breaks every pre-built plugin `.so`. Source FFI is `iggy_source_handle_v2` (batch ID carried to the runtime callback) + `iggy_source_batch_result` (plugin-exported ACK/NACK, #3855). +- Bumping `iggy_connector_sdk` MAJOR version or changing any FFI signature in `sdk/src/{sink,source}.rs` - breaks every pre-built plugin `.so`. Source FFI is `iggy_source_handle_v2` (batch ID carried to the runtime callback) + `iggy_source_batch_result` (plugin-exported ACK/NACK, #3855) + the optional `iggy_source_register_stop_callback` (SDK 0.5.1-edge.2+, #4269) that lets the runtime learn why a source's poll task exited; older plugins simply don't export it. - Changing the runtime's wire conventions (postcard FFI payload structs, default consumer group naming, plugin path resolution). - Modifying `runtime/src/state.rs` save protocol (atomic rename + fsync ordering) - corruption risk. - Renaming or repurposing a `Schema` variant - decoders/encoders pinned to wire bytes. diff --git a/core/connectors/sources/source_template/src/lib.rs b/core/connectors/sources/source_template/src/lib.rs index 7dafc097f9..86070f1548 100644 --- a/core/connectors/sources/source_template/src/lib.rs +++ b/core/connectors/sources/source_template/src/lib.rs @@ -462,29 +462,34 @@ impl Source for TemplateSource { } /// The staged cursor from `poll()` is only committed here, and only on - /// an ACK. A NACK (delivery failed, batch timed out, runtime is + /// an ACK. A NACK (delivery failed, state-save failed, runtime is /// shutting down) discards the candidate so the same range is re-polled /// next time — the cursor never moves past data that wasn't confirmed /// delivered. If `fetch_records()` ever needs to perform a destructive /// read against the source (delete-after-read, mark-as-processed), /// stage that side effect the same way and only apply it here on `Ack`. + /// A missing runtime result does not produce a synthetic NACK — the SDK + /// just keeps waiting (warning past ~30s) rather than guessing. /// /// If you add real I/O here (an external ack call, a delete, a mark- /// processed update), two things from a real connector's review history /// apply directly: /// - /// 1. This runs inside the SDK's fixed ~30s batch-result window, and a - /// size-your-own-timeout mistake here is fatal, not just slow — a - /// NACK from that window stops the source for good after repeated - /// consecutive NACKs. If you stage more than one operation, budget - /// for *all* of them inside the window, not one op's worth, and - /// reject `max_retries`/`retry_delay` combinations in `open()` that - /// could blow it. See `connector-pr-review` B11. + /// 1. `on_batch_result` has no SDK-enforced deadline, but the SDK will + /// not call `poll()` again until it returns — slow I/O here silently + /// stalls the whole source, it doesn't fail loudly. If you stage more + /// than one operation, budget for *all* of them (a per-item retry + /// budget multiplies with the item count), and reject + /// `max_retries`/`retry_delay` combinations in `open()` that would + /// make that stall unbounded. See `connector-pr-review` B11. /// 2. A failed op must not be silently logged while the batch is still /// ACKed — the cursor has already committed past it by the time you /// find out. Count consecutive failures and return `Err` past a /// threshold so the SDK stops the source, instead of resending the - /// same broken operation forever. + /// same broken operation forever. A source whose input genuinely + /// cannot be replayed after restart should instead override + /// `batch_policy()` with `BatchPolicy::with_max_consecutive_nacks(None)` + /// rather than relying on the default 5-consecutive-Nack stop. async fn on_batch_result(&self, result: SourceBatchResult) -> Result<(), Error> { let candidate_state = self.pending_state.lock().await.take(); if result == SourceBatchResult::Ack From e3497595e816991f4b1c97420cee9e4e9a2b21b8 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 24 Oct 2026 10:57:40 -0400 Subject: [PATCH 5/5] Adding connector pr review agent skills --- .agents/skills/connector-pr-review | 1 + 1 file changed, 1 insertion(+) create mode 120000 .agents/skills/connector-pr-review diff --git a/.agents/skills/connector-pr-review b/.agents/skills/connector-pr-review new file mode 120000 index 0000000000..dd9de6d8e8 --- /dev/null +++ b/.agents/skills/connector-pr-review @@ -0,0 +1 @@ +../../.claude/skills/connector-pr-review \ No newline at end of file