diff --git a/Cargo.lock b/Cargo.lock index cefd525776..7d88d27f05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7681,6 +7681,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "iggy_connector_mqtt_sink" +version = "0.5.0" +dependencies = [ + "async-trait", + "base64 0.23.1", + "bytes", + "humantime", + "iggy_common", + "iggy_connector_sdk", + "rumqttc-v4-next", + "rumqttc-v5-next", + "rustls-native-certs", + "rustls-pemfile", + "secrecy", + "serde", + "serde_json", + "simd-json", + "tokio", + "tracing", + "url", + "uuid", +] + [[package]] name = "iggy_connector_mqtt_source" version = "0.5.0" diff --git a/Cargo.toml b/Cargo.toml index f9528ebf60..ac47205acc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ members = [ "core/connectors/sinks/influxdb_sink", "core/connectors/sinks/meilisearch_sink", "core/connectors/sinks/mongodb_sink", + "core/connectors/sinks/mqtt_sink", "core/connectors/sinks/postgres_sink", "core/connectors/sinks/quickwit_sink", "core/connectors/sinks/rabbitmq_sink", diff --git a/core/connectors/runtime/example_config/connectors/mqtt_sink.toml b/core/connectors/runtime/example_config/connectors/mqtt_sink.toml new file mode 100644 index 0000000000..8c10e0a072 --- /dev/null +++ b/core/connectors/runtime/example_config/connectors/mqtt_sink.toml @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +type = "sink" +key = "mqtt" +enabled = true +version = 0 +name = "MQTT sink" +path = "target/release/libiggy_connector_mqtt_sink" +plugin_config_format = "json" + +[[streams]] +stream = "iot" +topics = ["telemetry"] +schema = "raw" +batch_length = 100 +poll_interval = "5ms" +consumer_group = "mqtt_sink_connector" + +[plugin_config] +# Required broker address. Do not add credentials or MQTT options to this URL. +broker_url = "mqtt://127.0.0.1:1883" +protocol = "mqtt5" +clean_start = true +session_expiry_interval = 3600 + +# Select one routing block. Keep one topic_mode line active. +# Static routing publishes every Iggy record to one MQTT topic. +topic = "iggy.messages" +topic_mode = "static" +qos = 1 +retain = false + +# Template routing creates an MQTT topic from the Iggy stream and topic. +# To select it, comment the active static topic and topic_mode lines above. +# topic_mode = "template" +# topic_template = "{stream}/{topic}" + +# Dynamic routing reads mqtt.topic from Iggy headers or an MQTT 5 envelope. +# It uses fallback_topic when the record has no valid MQTT topic. +# To select it, comment the active static topic and topic_mode lines above. +# topic_mode = "dynamic" +# fallback_topic = "{stream}/{topic}" + +# Connection, publication, and retry settings. +keep_alive = "30s" +request_capacity = 32 +publish_timeout = "30s" +shutdown_timeout = "30s" +# Retries after the initial connection attempt. Each publish window receives +# this many retries after its initial uncertain publication attempt. +max_retries = 5 +retry_delay = "1s" +max_retry_delay = "30s" +max_in_flight = 10 +verbose_logging = false + +# Optional broker authentication. +# username = "emqx-user" +# password = "replace-me" + +# Optional TLS configuration for mqtts:// brokers. +# [plugin_config.tls] +# ca_file = "/etc/iggy/certs/emqx-ca.pem" +# client_cert_file = "/etc/iggy/certs/client-cert.pem" +# client_key_file = "/etc/iggy/certs/client-key.pem" diff --git a/core/connectors/sinks/README.md b/core/connectors/sinks/README.md index 70a28940ac..a9c7b200dd 100644 --- a/core/connectors/sinks/README.md +++ b/core/connectors/sinks/README.md @@ -14,6 +14,7 @@ Sink connectors are responsible for writing data from Iggy streams to external s | **iceberg_sink** | Writes data to Apache Iceberg tables via REST catalog with S3/GCS/Azure storage | | **influxdb_sink** | Writes messages to InfluxDB as line-protocol points; supports both V2 (org/bucket, Flux) and V3 (db, SQL) | | **meilisearch_sink** | Indexes messages in Meilisearch for full-text search | +| **mqtt_sink** | Publishes messages to MQTT brokers | | **postgres_sink** | Stores messages in PostgreSQL database tables with configurable schemas | | **quickwit_sink** | Indexes messages in Quickwit search engine for log analytics | | **redshift_sink** | Stores messages in Redshift warehouse tables with configurable schemas vis S3 as staging | diff --git a/core/connectors/sinks/mqtt_sink/Cargo.toml b/core/connectors/sinks/mqtt_sink/Cargo.toml new file mode 100644 index 0000000000..b557673b71 --- /dev/null +++ b/core/connectors/sinks/mqtt_sink/Cargo.toml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "iggy_connector_mqtt_sink" +version = "0.5.0" +description = "Iggy MQTT sink connector." +edition = "2024" +license = "Apache-2.0" +keywords = ["iggy", "mqtt", "messaging", "streaming", "sink"] +categories = ["command-line-utilities", "network-programming"] +homepage = "https://iggy.apache.org" +documentation = "https://iggy.apache.org/docs" +repository = "https://github.com/apache/iggy" +readme = "../../README.md" +publish = false + +[lib] +crate-type = ["cdylib", "lib"] + +[dependencies] +async-trait = { workspace = true } +base64 = { workspace = true } +bytes = { workspace = true } +humantime = { workspace = true } +iggy_common = { workspace = true } +iggy_connector_sdk = { workspace = true } +rumqttc-v4 = { workspace = true } +rumqttc-v5 = { workspace = true } +rustls-native-certs = { workspace = true } +rustls-pemfile = { workspace = true } +secrecy = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +simd-json = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +url = { workspace = true } +uuid = { workspace = true } + +[lints] +workspace = true diff --git a/core/connectors/sinks/mqtt_sink/README.md b/core/connectors/sinks/mqtt_sink/README.md new file mode 100644 index 0000000000..907f04dd6e --- /dev/null +++ b/core/connectors/sinks/mqtt_sink/README.md @@ -0,0 +1,340 @@ +# Apache Iggy MQTT Sink Connector (`iggy_connector_mqtt_sink`) + +The **MQTT Sink Connector** is a dynamically loaded shared library plugin +(`.so` / `.dylib` / `.dll`) for Apache Iggy. It reads Iggy messages and +publishes them to MQTT brokers such as **EMQX** or Mosquitto. + +--- + +## Architecture Overview + +```text +┌──────────────────────────────────────────────────────────────────────────┐ +│ IGGY SERVER │ +│ • Stores messages in streams, topics, and partitions │ +└────────────────────────────────────┬─────────────────────────────────────┘ + │ + │ Native Binary Client Protocol + ▼ +┌──────────────────────────────────────────────────────────────────────────┐ +│ IGGY CONNECTORS RUNTIME PROCESS │ +│ │ +│ ┌────────────────────────────────────────────────────────────────────┐ │ +│ │ MQTT Sink Plugin (libiggy_connector_mqtt_sink.so) │ │ +│ │ • Reads Iggy batches through the C-ABI FFI │ │ +│ │ • Resolves static, template, or dynamic MQTT topics │ │ +│ │ • Tracks QoS 1 and QoS 2 acknowledgements │ │ +│ │ • Reconnects and republishes unconfirmed messages │ │ +│ └─────────────────────────────────┬──────────────────────────────────┘ │ +│ │ │ +│ │ MQTT / TLS Connection │ +│ ▼ │ +│ ┌────────────────────────────────────────────────────────────────────┐ │ +│ │ MQTT BROKER │ │ +│ │ • Client Authentication and Authorization │ │ +│ │ • Topic Routing and QoS Acknowledgements │ │ +│ └────────────────────────────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Features + +* **Multi-Protocol Support**: Supports **MQTT 3.1.1** (`mqtt311`) and **MQTT 5** (`mqtt5`). +* **Quality of Service (QoS)**: Publishes with QoS 0, QoS 1, and QoS 2. +* **Topic Routing**: Supports static topics, `{stream}` and `{topic}` templates, and dynamic MQTT topics from headers or JSON envelopes. +* **Bounded Pipelining**: Limits outstanding publications with `max_in_flight`. +* **Acknowledgement Tracking**: Correlates QoS 1 and QoS 2 acknowledgements by MQTT packet identifier. +* **Reconnect Recovery**: rumqttc retransmits after session resumption. The sink recreates its driver and requeues after a session reset. +* **Idle Connection Keep-Alive**: A managed MQTT task polls the event loop between batches, so it sends and receives keep-alive packets. +* **Malformed Record Reporting**: Logs and skips malformed marked envelopes while it publishes valid records in the batch. +* **MQTT 5 Properties**: Publishes supported envelope properties with the decoded payload. +* **TLS and Mutual TLS**: Supports system roots, custom CA files, client certificates, and client private keys through Rustls. +* **Session Control**: Supports `clean_start` and MQTT 5 `session_expiry_interval`. +* **Bounded Shutdown Cleanup**: Limits lock acquisition and MQTT disconnect cleanup with `shutdown_timeout` after publication work returns. + +--- + +## Configuration + +The connector TOML file has two parts. The runtime section identifies the +sink and selects the Iggy records that it reads. The `[plugin_config]` section +defines the MQTT connection and publication behavior. + +The default sample uses static routing. It publishes every selected Iggy record +to one MQTT topic. Optional authentication, TLS, template routing, and dynamic +routing remain commented. Uncomment the fields that you need and replace the +active `topic_mode` value when you select another routing mode. + +```toml +type = "sink" +key = "mqtt_telemetry" +enabled = true +version = 1 +name = "MQTT Telemetry Sink" +path = "target/release/libiggy_connector_mqtt_sink" +plugin_config_format = "json" + +[[streams]] +stream = "telemetry" +topics = ["events"] +schema = "raw" +batch_length = 100 +poll_interval = "5ms" +consumer_group = "mqtt_telemetry_sink" + +[plugin_config] +broker_url = "mqtt://127.0.0.1:1883" +protocol = "mqtt5" +clean_start = true +session_expiry_interval = 3600 + +# Select one routing block. Keep one topic_mode line active. +# Static routing publishes every selected Iggy record to one MQTT topic. +topic_mode = "static" +topic = "iggy.messages" +qos = 1 +retain = false +max_packet_size = 1048576 + +# Template routing creates an MQTT topic from the Iggy stream and topic. +# To select it, comment the active static topic and topic_mode lines above. +# topic_mode = "template" +# topic_template = "{stream}/{topic}" + +# Dynamic routing reads mqtt.topic from Iggy headers or an MQTT 5 envelope. +# To select it, comment the active static topic and topic_mode lines above. +# topic_mode = "dynamic" +# fallback_topic = "{stream}/{topic}" + +# Optional broker authentication. +# username = "emqx-user" +# password = "replace-me" + +# Optional TLS configuration for mqtts:// brokers. +# [plugin_config.tls] +# ca_file = "/etc/iggy/certs/emqx-ca.pem" +# client_cert_file = "/etc/iggy/certs/client-cert.pem" +# client_key_file = "/etc/iggy/certs/client-key.pem" +``` + +`core/connectors/sinks/mqtt_sink/config.toml` uses this layout. The runtime +example uses the same plugin configuration with its own Iggy route. + +## Runtime Configuration Reference + +| Field | Description | +| :--- | :--- | +| `type` | Set this value to `"sink"`. | +| `key` | Names this connector instance. Each loaded connector needs a unique key. | +| `enabled` | Starts the connector when the runtime starts. | +| `version` | Selects the stored configuration version. | +| `name` | Provides a readable name for logs and the control API. | +| `path` | Gives the path to the MQTT sink shared library. | +| `plugin_config_format` | Set this value to `"json"`. The runtime sends the plugin section as JSON. | +| `streams[].stream` | Selects an Iggy stream to read. | +| `streams[].topics` | Selects one or more Iggy topics in that stream. | +| `streams[].schema` | Selects the decoder for the Iggy record payload. Use `"raw"` for unchanged bytes. Use `"json"` for MQTT 5 envelope routing. | +| `streams[].batch_length` | Limits the number of Iggy records in one runtime batch. | +| `streams[].poll_interval` | Sets the wait time before the next Iggy poll. | +| `streams[].consumer_group` | Stores the Iggy read position for this sink route. Use a unique name for each sink instance. | + +## MQTT Routing Modes + +Static mode publishes every Iggy record from one sink route to the value of +`topic`. This is the default mode. `topic_template` and `fallback_topic` do +not select the output topic in this mode. + +```toml +topic_mode = "static" +topic = "manual/static" +``` + +Template mode creates an MQTT topic from the Iggy stream and topic names. The +template accepts only `{stream}` and `{topic}`. The `topic` field does not +select the output topic in this mode. + +```toml +topic_mode = "template" +topic_template = "iggy/{stream}/{topic}" +``` + +Dynamic mode reads `mqtt.topic` from raw-message headers. With MQTT 5, it can +also read the topic from a versioned JSON envelope. `fallback_topic` selects +the output topic when the record has no valid MQTT topic. + +```toml +topic_mode = "dynamic" +fallback_topic = "iggy/fallback" +``` + +Dynamic mode accepts this MQTT 5 envelope. The connector decodes +`payload_base64` before it publishes the MQTT payload. + +MQTT 5 envelope routing has three required rules: + +1. Set `topic_mode = "dynamic"`. +2. Set `protocol = "mqtt5"`. +3. Set `schema = "json"` for every sink stream that carries envelopes. + +If an MQTT source produces the input, set `include_metadata = true` in that +source and use `schema = "json"` for its destination stream. With +`schema = "raw"`, the sink treats the envelope as ordinary bytes and publishes +the JSON envelope instead of its decoded MQTT payload. + +Static and template routing treat the Iggy payload as opaque. They do not +forward Iggy headers to MQTT, including `mqtt.topic`. + +For `message_expiry_interval`, the connector subtracts the time that the +record spent in Iggy. It skips the record when the remaining lifetime is zero. +The connector ignores `topic_alias` because an alias belongs to the MQTT +connection that created it. + +```json +{ + "version": 1, + "topic": "devices/site-a/telemetry", + "payload_base64": "eyJ0ZW1wZXJhdHVyZSI6MjJ9", + "properties": { + "response_topic": "devices/site-a/replies", + "content_type": "application/json", + "user_properties": [{"key": "site", "value": "a"}] + } +} +``` + +## Plugin Configuration Reference + +| Field | Used when | Default | Description | +| :--- | :--- | :--- | :--- | +| `broker_url` | Always | None | Sets the MQTT broker URL. Use `mqtt://`, `mqtts://`, or `ssl://`. Do not add credentials or MQTT options to this URL. | +| `protocol` | All modes | `"mqtt5"` | Selects `"mqtt5"` or `"mqtt311"`. | +| `client_id` | All modes | Generated for clean sessions | Identifies the MQTT client. For persistent sessions, set a stable, non-empty value. For clean sessions, the connector generates a unique 23-character alphanumeric value when it is omitted. | +| `username` | Authenticated brokers | None | Sets the broker user name. Set `password` with this field. | +| `password` | Authenticated brokers | None | Sets the broker password. The connector does not write it to logs. | +| `clean_start` | All modes | `true` | Starts a clean MQTT 5 session or MQTT 3.1.1 clean session. A session reset makes the sink requeue unconfirmed publications. | +| `session_expiry_interval` | All modes | `3600` | Sets the MQTT 5 session expiry time in seconds. | +| `topic_mode` | All modes | `"static"` | Selects `"static"`, `"template"`, or `"dynamic"`. | +| `topic` | Static mode | `"iggy.messages"` | Sets the MQTT output topic for static routing. It must not begin with `$` or contain `+` or `#`. | +| `topic_template` | Template mode | `"{stream}/{topic}"` | Sets the template for template routing. It accepts only `{stream}` and `{topic}`. The rendered topic must not begin with `$` or contain `+` or `#`. | +| `fallback_topic` | Dynamic mode | `"{stream}/{topic}"` | Sets the MQTT output topic when dynamic routing has no valid topic. The rendered topic must not begin with `$` or contain `+` or `#`. | +| `qos` | All modes | `1` | Sets MQTT publication QoS. Use `0`, `1`, or `2`. | +| `retain` | All modes | `false` | Sets the MQTT retained-message flag. | +| `keep_alive` | All modes | `"30s"` | Sets the MQTT ping interval. MQTT 3.1.1 accepts `"0s"` or at least `"1s"`. MQTT 5 needs at least `"5s"`. The maximum is `"65535s"`. | +| `request_capacity` | All modes | `32` | Limits queued MQTT client requests. Set a value greater than zero. | +| `publish_timeout` | All modes | `"30s"` | Limits the wait for broker connection and publication acknowledgments. The maximum value is `"1h"`. | +| `shutdown_timeout` | All modes | `"30s"` | Limits how long `close()` waits for active `consume()` work and MQTT disconnect cleanup. It does not cancel active publication or reconnect work. | +| `max_retries` | All modes | `5` | Sets the number of retries after the initial connection attempt. Each publish window also gets this many retries after its initial uncertain publication attempt. | +| `retry_delay` | All modes | `"1s"` | Sets the first retry delay. | +| `max_retry_delay` | All modes | `"30s"` | Limits the retry delay. | +| `max_in_flight` | All modes | `10` | Limits publications that wait for acknowledgment. It must be greater than zero and must not exceed `request_capacity`. | +| `max_packet_size` | All modes | `1048576` | Limits an encoded MQTT PUBLISH packet in bytes. Oversized records are logged and skipped. MQTT 5 also uses a lower limit sent by the broker. | +| `verbose_logging` | All modes | `false` | Enables extra plugin log records. | +| `tls.ca_file` | TLS brokers | None | Sets a PEM file that contains custom CA certificates. The system trust store is used when it is omitted. | +| `tls.client_cert_file` | Mutual TLS | None | Sets the PEM client certificate for mutual TLS. Set `tls.client_key_file` with this field. | +| `tls.client_key_file` | Mutual TLS | None | Sets the PEM client key for mutual TLS. Set `tls.client_cert_file` with this field. | + +Use `broker_url` only for the broker address. Set `client_id`, `keep_alive`, +`clean_start`, and `max_in_flight` as separate fields. The connector rejects +the URL options `client_id`, `keep_alive_secs`, `clean_session`, and +`inflight_num`, `max_incoming_packet_size_bytes`, and +`max_outgoing_packet_size_bytes`. + +## TLS and Multiple Routes + +The connector validates the broker certificate and host name. It cannot disable +certificate validation. Keep the complete `[plugin_config.tls]` table in the +connector TOML file because environment overrides cannot update nested fields. + +Use one connector instance for each MQTT route. Give each instance a unique +connector key, client ID, consumer group, and output topic. The runtime loads +each connector TOML file from its configured `config_dir`. + +--- + +## Environment Variable Overrides + +Top-level properties inside `[plugin_config]` can be overridden without +editing the connector file: + +```bash +export IGGY_CONNECTORS_SINK_MQTT_PLUGIN_CONFIG_BROKER_URL="mqtt://127.0.0.1:1883" +export IGGY_CONNECTORS_SINK_MQTT_PLUGIN_CONFIG_USERNAME="emqx-user" +export IGGY_CONNECTORS_SINK_MQTT_PLUGIN_CONFIG_PASSWORD="secret-password" +export IGGY_CONNECTORS_SINK_MQTT_PLUGIN_CONFIG_QOS=1 +export IGGY_CONNECTORS_SINK_MQTT_PLUGIN_CONFIG_MAX_IN_FLIGHT=3 +``` + +Nested fields such as `tls.ca_file` cannot be changed through flat environment +variables. Keep the complete `[plugin_config.tls]` table in the connector TOML. + +--- + +## Publication and Acknowledgement Mechanism + +The sink provides QoS-aware broker delivery for MQTT publications. For QoS 1 +and QoS 2, a publication is confirmed only after the expected MQTT +acknowledgement arrives. + +```text + Iggy Server MQTT Sink Plugin MQTT Broker + │ │ │ + │ 1. Read batch │ │ + ├──────────────────────►│ │ + │ │ 2. MQTT PUBLISH │ + │ ├────────────────────────►│ + │ │ │ + │ │ 3. PUBACK or PUBCOMP │ + │ │◄────────────────────────┤ + │ │ │ + │ 4. Return success │ │ + │◄──────────────────────┤ │ +``` + +For QoS 0, the sink confirms the publication after it queues the request. For +QoS 1, it waits for `PUBACK`. For QoS 2, it waits for `PUBCOMP`. When an +outcome is uncertain, the sink cleans and reconnects. If the broker resumes +the session, rumqttc retransmits unconfirmed requests. If the broker resets +the session, the sink recreates its driver and requeues unconfirmed +publications. Both paths can duplicate uncertain publications while retries +remain. + +Iggy offset durability depends on the connectors runtime. The current runtime +commits offsets when it polls messages, before MQTT publication completes. +The sink therefore provides at-most-once delivery from Iggy after a terminal +failure. If retries end or the broker returns a non-retryable rejection, Iggy +does not deliver the batch to this sink again. QoS 1 and QoS 2 acknowledge +broker delivery, but they do not provide end-to-end at-least-once delivery. + +--- + +## Failure Modes + +| Failure Scenario | Current Behavior | Result | +| :--- | :--- | :--- | +| Invalid configuration | Validation fails before connection | The connector reports an invalid configuration error. | +| MQTT 5 broker capability mismatch | Connection initialization rejects a configured QoS or retained publication mode that the broker does not support. | The connector does not publish records or retry the mismatch. | +| QoS 0 publication | The sink confirms after it queues the local request. | A connection failure before broker delivery can lose the record. | +| Malformed record envelope | The connector logs and skips the malformed record. | The skipped record is lost from this sink. Valid records in the batch are published. | +| Oversized MQTT record | The connector logs and skips a record that exceeds `max_packet_size` or the broker MQTT 5 limit. | The skipped record is lost from this sink. Valid records in the batch are published. | +| Expired MQTT 5 envelope | The connector logs and skips the expired record. | The skipped record is lost from this sink. Other valid records in the batch are published. | +| Invalid dynamic topic | The connector logs and skips the record when the resolved topic begins with `$` or contains `+` or `#`. | The skipped record is lost from this sink. Other valid records in the batch are published. | +| Raw MQTT 5 envelope | The sink treats the JSON envelope as ordinary bytes. | The broker receives the envelope JSON, not the original MQTT payload. | +| Transient connection failure | The event loop reconnects. rumqttc retransmits after session resumption. The sink recreates its driver and requeues after a session reset. | Uncertain publications can duplicate while retries remain. | +| Publication timeout or exhausted retries | The connector cleans and reconnects. It retransmits after session resumption or requeues after a session reset, up to `max_retries`. | After retries end, unconfirmed records are lost from this sink because Iggy already committed the offset. | +| MQTT 5 non-retryable broker rejection | The connector returns the rejection without retrying. | Unconfirmed records are lost from this sink because Iggy already committed the offset. | +| Authentication failure | Connection initialization fails | The sink does not publish records. | +| Shutdown timeout | The sink reports unfinished publications | It returns an error after the timeout. | + +--- + +## Operational Notes + +1. **Duplicate tolerance and order**: QoS 1 and QoS 2 retries can publish duplicates. When a publication is uncertain, the sink replays it and later publications in the same window. This preserves order within one Iggy partition and MQTT topic. It does not preserve global order across partitions or topics. +2. **Credential separation**: Put `username` and `password` in separate fields. The connector rejects credentials embedded in `broker_url`. +3. **Client identity**: For `clean_start = false`, set a stable, unique `client_id` so the broker can resume the MQTT session. For `clean_start = true`, omit `client_id` to generate a unique client ID for this sink instance. +4. **Idle connections**: The MQTT task continues polling between batches. It sends keep-alive pings and reads their responses without waiting for another Iggy batch. +5. **Invalid records**: The connector logs and skips malformed or oversized records. It does not provide a dead-letter queue. +6. **Broker limits**: MQTT 5 uses the lower of `max_packet_size` and the maximum packet size sent by the broker. The sink removes envelope topic aliases because they are connection-local. +7. **Broker rejection classification**: The SDK has no broker-rejection error. The plugin uses `CannotStoreData` with an internal non-retryable flag. The runtime records the error and continues. diff --git a/core/connectors/sinks/mqtt_sink/config.toml b/core/connectors/sinks/mqtt_sink/config.toml new file mode 100644 index 0000000000..253cc769b0 --- /dev/null +++ b/core/connectors/sinks/mqtt_sink/config.toml @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +type = "sink" +key = "mqtt" +enabled = true +version = 0 +name = "MQTT sink" +path = "target/release/libiggy_connector_mqtt_sink" +plugin_config_format = "json" + +[[streams]] +stream = "telemetry" +topics = ["events"] +schema = "raw" +batch_length = 100 +poll_interval = "5ms" +consumer_group = "mqtt_sink_connector" + +[plugin_config] +# Required broker address. Do not add credentials or MQTT options to this URL. +broker_url = "mqtt://127.0.0.1:1883" +protocol = "mqtt5" +clean_start = true +session_expiry_interval = 3600 + +# Select one routing block. Keep one topic_mode line active. +# Static routing publishes every Iggy record to one MQTT topic. +topic = "iggy.messages" +topic_mode = "static" + +# Template routing creates an MQTT topic from the Iggy stream and topic. +# To select it, comment the active static topic and topic_mode lines above. +# topic_mode = "template" +# topic_template = "{stream}/{topic}" + +# Dynamic routing reads mqtt.topic from Iggy headers or an MQTT 5 envelope. +# It uses fallback_topic when the record has no valid MQTT topic. +# To select it, comment the active static topic and topic_mode lines above. +# topic_mode = "dynamic" +# fallback_topic = "{stream}/{topic}" + +qos = 1 +retain = false + +# Connection, publication, and retry settings. +keep_alive = "30s" +request_capacity = 32 +publish_timeout = "30s" +shutdown_timeout = "30s" +max_retries = 5 +retry_delay = "1s" +max_retry_delay = "30s" +max_in_flight = 10 +verbose_logging = false + +# Optional broker authentication. Supply both values or omit both values. +# username = "emqx-user" +# password = "replace-me" + +# Optional TLS configuration for mqtts:// brokers. The client certificate and +# key must be supplied together. +# [plugin_config.tls] +# ca_file = "/etc/iggy/certs/emqx-ca.pem" +# client_cert_file = "/etc/iggy/certs/client-cert.pem" +# client_key_file = "/etc/iggy/certs/client-key.pem" diff --git a/core/connectors/sinks/mqtt_sink/src/driver.rs b/core/connectors/sinks/mqtt_sink/src/driver.rs new file mode 100644 index 0000000000..c132c280fe --- /dev/null +++ b/core/connectors/sinks/mqtt_sink/src/driver.rs @@ -0,0 +1,1837 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use rumqttc_v4::tokio_rustls::rustls; +use rumqttc_v4::tokio_rustls::rustls::{ + ClientConfig, RootCertStore, + pki_types::{CertificateDer, PrivateKeyDer}, +}; +use rumqttc_v4::{ + AsyncClient as Mqtt311Client, ConnectionError as Mqtt311ConnectionError, Event as Mqtt311Event, + EventLoop as Mqtt311EventLoop, Incoming as Mqtt311Incoming, MqttOptions as Mqtt311Options, + Outgoing as Mqtt311Outgoing, PublishOptions as Mqtt311PublishOptions, + TlsConfiguration as Mqtt311TlsConfiguration, Transport as Mqtt311Transport, +}; +use rumqttc_v5::{ + AsyncClient as Mqtt5Client, ConnectionError as Mqtt5ConnectionError, Event as Mqtt5Event, + EventLoop as Mqtt5EventLoop, Incoming as Mqtt5Incoming, MqttOptions as Mqtt5Options, + Outgoing as Mqtt5Outgoing, PublishOptions as Mqtt5PublishOptions, + StateError as Mqtt5StateError, + mqttbytes::v5::{ + ConnectReturnCode as Mqtt5ConnectReturnCode, DisconnectReasonCode, PubAckReason, + PubCompReason, PubRecReason, + }, +}; +use rustls_native_certs::load_native_certs; +use rustls_pemfile::{certs, private_key}; +use secrecy::ExposeSecret; +use serde::Deserialize; +use std::{ + collections::{HashMap, VecDeque}, + fs, + io::{BufReader, Cursor}, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::{Duration, Instant}, +}; +use tokio::{ + sync::{mpsc, oneshot}, + task::JoinHandle, + time::timeout, +}; + +use crate::{MqttSinkConfig, routing::PreparedPublication}; + +mod rumqttc { + pub use rumqttc_v4::*; + pub use rumqttc_v5 as v5; +} + +/// MQTT protocol selection is kept in the plugin because rumqttc exposes +/// separate client/event-loop types for MQTT 3.1.1 and MQTT 5. +#[derive(Debug, Clone, Copy, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum MqttProtocol { + #[serde(rename = "mqtt311")] + Mqtt311, + #[default] + #[serde(rename = "mqtt5")] + Mqtt5, +} + +pub(crate) struct MqttDriver { + client: MqttClient, + events: mpsc::Receiver, + commands: mpsc::Sender, + task: Option>, + max_packet_size: Arc, +} + +pub(crate) struct PublishWindowError { + pub(crate) error: iggy_connector_sdk::Error, + pub(crate) retryable: bool, + pub(crate) requires_clean: bool, + pub(crate) requires_requeue: bool, +} + +pub(crate) enum PublishWindowProgress { + Complete, + SessionPresent(bool), +} + +enum DriverEvent { + ConnectionError(iggy_connector_sdk::Error), + SessionPresent(bool), + OutgoingPublish(u16), + PubAck(u16), + PubComp(u16), + Mqtt5PubAck { + packet_id: u16, + reason: PubAckReason, + }, + Mqtt5PubRec { + packet_id: u16, + reason: PubRecReason, + }, + Mqtt5PubComp { + packet_id: u16, + reason: PubCompReason, + }, + Other, +} + +#[derive(Debug)] +enum PublishWindowFailure { + BrokerRejected(String), + TransientBrokerRejected(String), + Protocol(String), +} + +impl PublishWindowFailure { + fn error(self) -> iggy_connector_sdk::Error { + match self { + Self::BrokerRejected(message) => iggy_connector_sdk::Error::CannotStoreData(message), + Self::TransientBrokerRejected(message) => { + iggy_connector_sdk::Error::Connection(message) + } + Self::Protocol(message) => iggy_connector_sdk::Error::Connection(message), + } + } + + fn retryable(&self) -> bool { + matches!(self, Self::TransientBrokerRejected(_)) + } +} + +pub(crate) struct PublishWindowState { + qos: rumqttc::QoS, + submitted_indexes: VecDeque, + packet_to_index: HashMap, + confirmed: Vec, + confirmed_count: usize, +} + +impl PublishWindowState { + fn new(length: usize, qos: rumqttc::QoS) -> Self { + Self { + qos, + submitted_indexes: VecDeque::with_capacity(length), + packet_to_index: HashMap::with_capacity(length), + confirmed: vec![false; length], + confirmed_count: 0, + } + } + + fn is_complete(&self) -> bool { + self.confirmed_count == self.confirmed.len() + } + + fn submitted(&mut self, index: usize) { + self.submitted_indexes.push_back(index); + } + + fn confirm(&mut self, index: usize) { + if !self.confirmed[index] { + self.confirmed[index] = true; + self.confirmed_count += 1; + } + } + + fn apply(&mut self, event: DriverEvent) -> Result<(), PublishWindowFailure> { + match event { + DriverEvent::OutgoingPublish(packet_id) => { + if self.packet_to_index.contains_key(&packet_id) { + return Ok(()); + } + let Some(index) = self.submitted_indexes.pop_front() else { + return Err(PublishWindowFailure::Protocol( + "received an unexpected MQTT publish packet".to_string(), + )); + }; + if self.qos == rumqttc::QoS::AtMostOnce { + self.confirm(index); + } else { + self.packet_to_index.insert(packet_id, index); + } + } + DriverEvent::PubAck(packet_id) if self.qos == rumqttc::QoS::AtLeastOnce => { + if let Some(index) = self.packet_to_index.remove(&packet_id) { + self.confirm(index); + } + } + DriverEvent::PubComp(packet_id) if self.qos == rumqttc::QoS::ExactlyOnce => { + if let Some(index) = self.packet_to_index.remove(&packet_id) { + self.confirm(index); + } + } + DriverEvent::Mqtt5PubAck { packet_id, reason } + if self.qos == rumqttc::QoS::AtLeastOnce => + { + if let Some(index) = self.packet_to_index.remove(&packet_id) { + if matches!( + reason, + PubAckReason::Success | PubAckReason::NoMatchingSubscribers + ) { + self.confirm(index); + } else { + return Err(mqtt5_puback_rejection(packet_id, reason)); + } + } + } + DriverEvent::Mqtt5PubRec { packet_id, reason } + if self.qos == rumqttc::QoS::ExactlyOnce => + { + if !matches!( + reason, + PubRecReason::Success | PubRecReason::NoMatchingSubscribers + ) && self.packet_to_index.remove(&packet_id).is_some() + { + return Err(mqtt5_pubrec_rejection(packet_id, reason)); + } + } + DriverEvent::Mqtt5PubComp { packet_id, reason } + if self.qos == rumqttc::QoS::ExactlyOnce => + { + if let Some(index) = self.packet_to_index.remove(&packet_id) { + if reason == PubCompReason::Success { + self.confirm(index); + } else { + return Err(PublishWindowFailure::BrokerRejected(format!( + "MQTT 5 PUBCOMP rejected packet ID {packet_id}: {reason:?}" + ))); + } + } + } + DriverEvent::ConnectionError(_) + | DriverEvent::SessionPresent(_) + | DriverEvent::PubAck(_) + | DriverEvent::PubComp(_) + | DriverEvent::Mqtt5PubAck { .. } + | DriverEvent::Mqtt5PubRec { .. } + | DriverEvent::Mqtt5PubComp { .. } + | DriverEvent::Other => {} + } + Ok(()) + } + + pub(crate) fn confirmed_indexes(&self) -> Vec { + confirmed_indexes(&self.confirmed) + } +} + +enum MqttConnection { + Mqtt311 { + client: Mqtt311Client, + event_loop: Box, + }, + Mqtt5 { + client: Mqtt5Client, + event_loop: Box, + }, +} + +#[derive(Clone)] +enum MqttClient { + Mqtt311(Mqtt311Client), + Mqtt5(Mqtt5Client), +} + +enum DriverCommand { + Clean, + DiscardQueuedEvents { + result_sender: oneshot::Sender<()>, + }, + Shutdown { + disconnect_timeout: Duration, + result_sender: oneshot::Sender>, + }, +} + +struct MqttEventLoopTask { + connection: MqttConnection, + id: u32, + qos: u8, + retain: bool, + configured_max_packet_size: usize, + max_packet_size: Arc, + event_sender: mpsc::Sender, + commands: mpsc::Receiver, +} + +impl std::fmt::Debug for MqttConnection { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("MqttConnection") + .field("protocol", &self.protocol()) + .finish_non_exhaustive() + } +} + +impl MqttConnection { + fn protocol(&self) -> MqttProtocol { + match self { + Self::Mqtt311 { .. } => MqttProtocol::Mqtt311, + Self::Mqtt5 { .. } => MqttProtocol::Mqtt5, + } + } + + fn client(&self) -> MqttClient { + match self { + Self::Mqtt311 { client, .. } => MqttClient::Mqtt311(client.clone()), + Self::Mqtt5 { client, .. } => MqttClient::Mqtt5(client.clone()), + } + } + + fn clean(&mut self) { + match self { + Self::Mqtt311 { event_loop, .. } => event_loop.clean(), + Self::Mqtt5 { event_loop, .. } => event_loop.clean(), + } + } + + async fn disconnect( + &mut self, + disconnect_timeout: Duration, + ) -> Result<(), iggy_connector_sdk::Error> { + match self { + Self::Mqtt311 { client, event_loop } => { + client + .disconnect() + .await + .map_err(|error| iggy_connector_sdk::Error::Connection(error.to_string()))?; + wait_for_mqtt311_disconnect(event_loop, disconnect_timeout).await?; + } + Self::Mqtt5 { client, event_loop } => { + client + .disconnect() + .await + .map_err(|error| iggy_connector_sdk::Error::Connection(error.to_string()))?; + wait_for_mqtt5_disconnect(event_loop, disconnect_timeout).await?; + } + } + Ok(()) + } +} + +impl std::fmt::Debug for MqttDriver { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("MqttDriver") + .field("protocol", &self.client.protocol()) + .finish() + } +} + +impl MqttClient { + fn protocol(&self) -> MqttProtocol { + match self { + Self::Mqtt311(_) => MqttProtocol::Mqtt311, + Self::Mqtt5(_) => MqttProtocol::Mqtt5, + } + } +} + +impl Drop for MqttDriver { + fn drop(&mut self) { + if let Some(task) = self.task.take() { + task.abort(); + } + } +} + +impl MqttDriver { + pub(crate) async fn connect( + id: u32, + config: &MqttSinkConfig, + client_id: String, + keep_alive: Duration, + connect_timeout: Duration, + request_capacity: usize, + max_in_flight: usize, + ) -> Result { + install_rustls_provider(); + if request_capacity == 0 { + return Err(iggy_connector_sdk::Error::InvalidConfigValue( + "request_capacity must be greater than zero".to_string(), + )); + } + + let max_in_flight = u16::try_from(max_in_flight).map_err(|_| { + iggy_connector_sdk::Error::InvalidConfigValue( + "max_in_flight must fit into an MQTT packet identifier window".to_string(), + ) + })?; + let keep_alive = u16::try_from(keep_alive.as_secs()).map_err(|_| { + iggy_connector_sdk::Error::InvalidConfigValue( + "keep_alive must be at most 65535s".to_string(), + ) + })?; + let max_packet_size = config + .max_packet_size + .unwrap_or(crate::DEFAULT_MAX_PACKET_SIZE); + let configured_max_packet_size = max_packet_size; + + let (connection, broker_max_packet_size) = match config.protocol { + MqttProtocol::Mqtt311 => { + let options_url = broker_url_with_client_id(&config.broker_url, &client_id)?; + let mut options = Mqtt311Options::parse_url(options_url).map_err(|error| { + iggy_connector_sdk::Error::InvalidConfigValue(format!("broker_url: {error}")) + })?; + options + .set_client_id(client_id) + .set_keep_alive(keep_alive) + .set_request_channel_capacity(request_capacity) + .set_inflight(max_in_flight) + .set_max_packet_size(max_packet_size, max_packet_size) + .set_clean_session(config.clean_start); + if let (Some(username), Some(password)) = + (config.username.as_deref(), config.password.as_ref()) + { + options.set_credentials(username, password.expose_secret().to_string()); + } + if let Some(transport) = tls_transport_v4(config)? { + options.set_transport(transport); + } + let (client, mut event_loop) = Mqtt311Client::builder(options) + .try_build() + .map_err(|error| { + iggy_connector_sdk::Error::InvalidConfigValue(error.to_string()) + })?; + wait_for_connack(&mut event_loop, id, connect_timeout).await?; + ( + MqttConnection::Mqtt311 { + client, + event_loop: Box::new(event_loop), + }, + None, + ) + } + MqttProtocol::Mqtt5 => { + let options_url = broker_url_with_client_id(&config.broker_url, &client_id)?; + let mut options = Mqtt5Options::parse_url(options_url).map_err(|error| { + iggy_connector_sdk::Error::InvalidConfigValue(format!("broker_url: {error}")) + })?; + options + .set_client_id(client_id) + .set_keep_alive(keep_alive) + .set_request_channel_capacity(request_capacity) + .set_outgoing_inflight_upper_limit(max_in_flight) + .set_max_packet_size(Some(u32::try_from(max_packet_size).map_err(|_| { + iggy_connector_sdk::Error::InvalidConfigValue( + "max_packet_size must fit into an MQTT 5 packet size".to_string(), + ) + })?)) + .set_clean_start(config.clean_start) + .set_session_expiry_interval(config.session_expiry_interval); + if let (Some(username), Some(password)) = + (config.username.as_deref(), config.password.as_ref()) + { + options.set_credentials(username, password.expose_secret().to_string()); + } + if let Some(transport) = tls_transport_v5(config)? { + options.set_transport(transport); + } + let (client, mut event_loop) = + Mqtt5Client::builder(options).try_build().map_err(|error| { + iggy_connector_sdk::Error::InvalidConfigValue(error.to_string()) + })?; + let broker_max_packet_size = wait_for_connack_v5( + &mut event_loop, + id, + connect_timeout, + config.qos, + config.retain, + ) + .await?; + ( + MqttConnection::Mqtt5 { + client, + event_loop: Box::new(event_loop), + }, + broker_max_packet_size, + ) + } + }; + + let max_packet_size = match config.protocol { + MqttProtocol::Mqtt311 => max_packet_size, + MqttProtocol::Mqtt5 => broker_max_packet_size + .map(|size| max_packet_size.min(size as usize)) + .unwrap_or(max_packet_size), + }; + let client = connection.client(); + let max_packet_size = Arc::new(AtomicUsize::new(max_packet_size)); + let (event_sender, events) = mpsc::channel(request_capacity); + let (commands, command_receiver) = mpsc::channel(2); + let task = tokio::spawn( + MqttEventLoopTask { + connection, + id, + qos: config.qos, + retain: config.retain, + configured_max_packet_size, + max_packet_size: Arc::clone(&max_packet_size), + event_sender, + commands: command_receiver, + } + .run(), + ); + Ok(Self { + client, + events, + commands, + task: Some(task), + max_packet_size, + }) + } + + pub(crate) fn max_packet_size(&self) -> usize { + self.max_packet_size.load(Ordering::Relaxed) + } + + pub(crate) fn new_publish_window( + &self, + length: usize, + qos: rumqttc::QoS, + ) -> PublishWindowState { + PublishWindowState::new(length, qos) + } + + pub(crate) async fn enqueue_window( + &mut self, + publications: &[&PreparedPublication], + indexes: &[usize], + qos: rumqttc::QoS, + retain: bool, + state: &mut PublishWindowState, + ) -> Result<(), iggy_connector_sdk::Error> { + for (&publication, &index) in publications.iter().zip(indexes) { + self.enqueue_publication(publication, qos, retain).await?; + // AsyncClient queues requests in order but does not return packet + // identifiers. The matching Outgoing::Publish event uses this order. + state.submitted(index); + } + Ok(()) + } + + pub(crate) async fn drive_window( + &mut self, + state: &mut PublishWindowState, + deadline: Instant, + ) -> Result { + while !state.is_complete() { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(PublishWindowError { + error: iggy_connector_sdk::Error::Connection( + "timed out waiting for MQTT publication acknowledgements".to_string(), + ), + retryable: true, + requires_clean: true, + requires_requeue: false, + }); + } + let event = match timeout(remaining, self.events.recv()).await { + Ok(Some(event)) => event, + Ok(None) => { + return Err(PublishWindowError { + error: iggy_connector_sdk::Error::Connection( + "MQTT event loop stopped unexpectedly".to_string(), + ), + retryable: true, + requires_clean: false, + requires_requeue: false, + }); + } + Err(_) => { + return Err(PublishWindowError { + error: iggy_connector_sdk::Error::Connection( + "timed out waiting for MQTT publication acknowledgements".to_string(), + ), + retryable: true, + requires_clean: true, + requires_requeue: false, + }); + } + }; + + let event = match event { + DriverEvent::ConnectionError(error) => { + return Err(PublishWindowError { + retryable: matches!(error, iggy_connector_sdk::Error::Connection(_)), + error, + requires_clean: false, + requires_requeue: false, + }); + } + DriverEvent::SessionPresent(session_present) => { + return Ok(PublishWindowProgress::SessionPresent(session_present)); + } + event => event, + }; + + if let Err(error) = state.apply(event) { + let retryable = error.retryable(); + return Err(PublishWindowError { + error: error.error(), + retryable, + requires_clean: false, + requires_requeue: retryable, + }); + } + } + + Ok(PublishWindowProgress::Complete) + } + + pub(crate) async fn clean(&mut self) -> Result<(), iggy_connector_sdk::Error> { + self.commands.send(DriverCommand::Clean).await.map_err(|_| { + iggy_connector_sdk::Error::Connection( + "MQTT event loop stopped before it could reconnect".to_string(), + ) + }) + } + + pub(crate) async fn discard_queued_events(&mut self) -> Result<(), iggy_connector_sdk::Error> { + discard_queued_events(&mut self.events); + let (result_sender, result_receiver) = oneshot::channel(); + self.commands + .send(DriverCommand::DiscardQueuedEvents { result_sender }) + .await + .map_err(|_| { + iggy_connector_sdk::Error::Connection( + "MQTT event loop stopped before it could discard queued events".to_string(), + ) + })?; + result_receiver.await.map_err(|_| { + iggy_connector_sdk::Error::Connection( + "MQTT event loop stopped before it could discard queued events".to_string(), + ) + })?; + discard_queued_events(&mut self.events); + Ok(()) + } + + async fn enqueue_publication( + &mut self, + publication: &PreparedPublication, + qos: rumqttc::QoS, + retain: bool, + ) -> Result<(), iggy_connector_sdk::Error> { + match &self.client { + MqttClient::Mqtt311(client) => client + .publish( + publication.topic.as_str(), + publication.payload.clone(), + Mqtt311PublishOptions::new(qos).retain(retain), + ) + .await + .map_err(|error| iggy_connector_sdk::Error::Connection(error.to_string())), + MqttClient::Mqtt5(client) => { + let mut options = Mqtt5PublishOptions::new(mqtt5_qos(qos)).retain(retain); + if let Some(properties) = publication.properties.clone() { + options = options.properties(properties); + } + client + .publish( + publication.topic.as_str(), + publication.payload.clone(), + options, + ) + .await + .map_err(|error| iggy_connector_sdk::Error::Connection(error.to_string())) + } + } + } + + pub(crate) async fn disconnect( + &mut self, + disconnect_timeout: Duration, + ) -> Result<(), iggy_connector_sdk::Error> { + let (result_sender, result_receiver) = oneshot::channel(); + let _ = self + .commands + .send(DriverCommand::Shutdown { + disconnect_timeout, + result_sender, + }) + .await; + let result = timeout(disconnect_timeout, result_receiver) + .await + .map_err(|_| { + iggy_connector_sdk::Error::Connection( + "timed out shutting down MQTT event loop".to_string(), + ) + })? + .map_err(|_| { + iggy_connector_sdk::Error::Connection( + "MQTT event loop stopped before disconnect completed".to_string(), + ) + })?; + if let Some(mut task) = self.task.take() + && timeout(disconnect_timeout, &mut task).await.is_err() + { + task.abort(); + let _ = task.await; + } + result + } +} + +impl MqttEventLoopTask { + async fn run(mut self) { + let mut pending_event = None; + loop { + tokio::select! { + biased; + command = self.commands.recv() => match command { + Some(DriverCommand::Clean) => self.connection.clean(), + Some(DriverCommand::DiscardQueuedEvents { result_sender }) => { + pending_event = None; + let _ = result_sender.send(()); + } + Some(DriverCommand::Shutdown { + disconnect_timeout, + result_sender, + }) => { + let _ = result_sender.send(self.connection.disconnect(disconnect_timeout).await); + return; + } + None => return, + }, + permit = self.event_sender.reserve(), if pending_event.is_some() => match permit { + Ok(permit) => { + if let Some(event) = pending_event.take() { + permit.send(event); + } + } + Err(_) => return, + }, + result = poll_connection_event( + &mut self.connection, + self.id, + self.qos, + self.retain, + self.configured_max_packet_size, + &self.max_packet_size, + ), if pending_event.is_none() => match result { + Ok(DriverEvent::Other) => {} + Ok(event) => { + pending_event = Some(event); + } + Err(error) => { + self.connection.clean(); + pending_event = Some(DriverEvent::ConnectionError(error)); + } + }, + } + } + } +} + +fn discard_queued_events(events: &mut mpsc::Receiver) { + while events.try_recv().is_ok() {} +} + +async fn poll_connection_event( + connection: &mut MqttConnection, + id: u32, + qos: u8, + retain: bool, + configured_max_packet_size: usize, + max_packet_size: &AtomicUsize, +) -> Result { + match connection { + MqttConnection::Mqtt311 { event_loop, .. } => match event_loop + .poll() + .await + .map_err(|error| iggy_connector_sdk::Error::Connection(error.to_string()))? + { + Mqtt311Event::Outgoing(Mqtt311Outgoing::Publish(packet_id)) => { + Ok(DriverEvent::OutgoingPublish(packet_id)) + } + Mqtt311Event::Incoming(Mqtt311Incoming::ConnAck(connack)) => { + Ok(DriverEvent::SessionPresent(connack.session_present)) + } + Mqtt311Event::Incoming(Mqtt311Incoming::PubAck(ack)) => { + Ok(DriverEvent::PubAck(ack.pkid)) + } + Mqtt311Event::Incoming(Mqtt311Incoming::PubComp(comp)) => { + Ok(DriverEvent::PubComp(comp.pkid)) + } + Mqtt311Event::Incoming(_) | Mqtt311Event::Outgoing(_) => Ok(DriverEvent::Other), + }, + MqttConnection::Mqtt5 { event_loop, .. } => match event_loop + .poll() + .await + .map_err(|error| mqtt5_connection_error(error, id))? + { + Mqtt5Event::Outgoing(Mqtt5Outgoing::Publish(packet_id)) => { + Ok(DriverEvent::OutgoingPublish(packet_id)) + } + Mqtt5Event::Incoming(Mqtt5Incoming::ConnAck(connack)) => { + validate_mqtt5_connack_capabilities(connack.properties.as_ref(), qos, retain, id)?; + let packet_size = connack + .properties + .as_ref() + .and_then(|properties| properties.max_packet_size) + .map(|size| configured_max_packet_size.min(size as usize)) + .unwrap_or(configured_max_packet_size); + max_packet_size.store(packet_size, Ordering::Relaxed); + Ok(DriverEvent::SessionPresent(connack.session_present)) + } + Mqtt5Event::Incoming(Mqtt5Incoming::PubAck(ack)) => Ok(DriverEvent::Mqtt5PubAck { + packet_id: ack.pkid, + reason: ack.reason, + }), + Mqtt5Event::Incoming(Mqtt5Incoming::PubRec(rec)) => Ok(DriverEvent::Mqtt5PubRec { + packet_id: rec.pkid, + reason: rec.reason, + }), + Mqtt5Event::Incoming(Mqtt5Incoming::PubComp(comp)) => Ok(DriverEvent::Mqtt5PubComp { + packet_id: comp.pkid, + reason: comp.reason, + }), + Mqtt5Event::Auth(_) | Mqtt5Event::Incoming(_) | Mqtt5Event::Outgoing(_) => { + Ok(DriverEvent::Other) + } + }, + } +} + +pub(crate) fn publication_packet_size( + publication: &PreparedPublication, + protocol: MqttProtocol, + qos: u8, + retain: bool, +) -> usize { + match protocol { + MqttProtocol::Mqtt311 => { + let mut publish = rumqttc::mqttbytes::v4::Publish::from_bytes( + publication.topic.as_str(), + mqtt311_qos(qos), + publication.payload.clone(), + ); + publish.retain = retain; + if qos > 0 { + publish.pkid = 1; + } + publish.size() + } + MqttProtocol::Mqtt5 => { + let mut publish = rumqttc::v5::mqttbytes::v5::Publish::new( + publication.topic.as_str(), + mqtt5_qos(mqtt311_qos(qos)), + publication.payload.clone(), + publication.properties.clone(), + ); + publish.retain = retain; + if qos > 0 { + publish.pkid = 1; + } + publish.size() + } + } +} + +fn mqtt311_qos(qos: u8) -> rumqttc::QoS { + match qos { + 0 => rumqttc::QoS::AtMostOnce, + 1 => rumqttc::QoS::AtLeastOnce, + 2 => rumqttc::QoS::ExactlyOnce, + _ => unreachable!("MQTT QoS is validated before publication preparation"), + } +} + +pub(crate) fn validate_tls_config( + config: &MqttSinkConfig, +) -> Result<(), iggy_connector_sdk::Error> { + let Some(tls) = &config.tls else { + return Ok(()); + }; + let broker_url = url::Url::parse(&config.broker_url).map_err(|error| { + iggy_connector_sdk::Error::InvalidConfigValue(format!("broker_url: {error}")) + })?; + if !matches!(broker_url.scheme(), "mqtts" | "ssl") { + return Err(iggy_connector_sdk::Error::InvalidConfigValue( + "tls requires an mqtts:// or ssl:// broker_url".to_string(), + )); + } + if tls.client_cert_file.is_some() != tls.client_key_file.is_some() { + return Err(iggy_connector_sdk::Error::InvalidConfigValue( + "tls client_cert_file and client_key_file must be configured together".to_string(), + )); + } + Ok(()) +} + +fn tls_config( + config: &MqttSinkConfig, +) -> Result>, iggy_connector_sdk::Error> { + let broker_url = url::Url::parse(&config.broker_url).map_err(|error| { + iggy_connector_sdk::Error::InvalidConfigValue(format!("broker_url: {error}")) + })?; + if !matches!(broker_url.scheme(), "mqtts" | "ssl") { + return Ok(None); + } + let tls = config.tls.as_ref(); + let ca = tls + .and_then(|tls| tls.ca_file.as_deref()) + .map(|path| read_certificate_file(path, "tls.ca_file")) + .transpose()?; + let client_auth = match tls.map(|tls| (&tls.client_cert_file, &tls.client_key_file)) { + None => None, + Some((Some(certificate_path), Some(key_path))) => Some(( + read_certificate_file(certificate_path, "tls.client_cert_file")?, + read_private_key_file(key_path, "tls.client_key_file")?, + )), + Some((None, None)) => None, + _ => unreachable!("TLS configuration was validated before loading files"), + }; + + Ok(Some(Arc::new(build_tls_config(ca, client_auth)?))) +} + +fn tls_transport_v4( + config: &MqttSinkConfig, +) -> Result, iggy_connector_sdk::Error> { + tls_config(config).map(|config| { + config.map(|config| { + Mqtt311Transport::tls_with_config(Mqtt311TlsConfiguration::Rustls(config)) + }) + }) +} + +fn tls_transport_v5( + config: &MqttSinkConfig, +) -> Result, iggy_connector_sdk::Error> { + tls_config(config).map(|config| { + config.map(|config| { + rumqttc_v5::Transport::tls_with_config(rumqttc_v5::TlsConfiguration::Rustls(config)) + }) + }) +} + +fn broker_url_with_client_id( + broker_url: &str, + client_id: &str, +) -> Result { + let mut url = url::Url::parse(broker_url).map_err(|error| { + iggy_connector_sdk::Error::InvalidConfigValue(format!("broker_url is invalid: {error}")) + })?; + if matches!(url.scheme(), "mqtts" | "ssl") { + let port = url.port().unwrap_or(8883); + url.set_scheme("mqtt").map_err(|_| { + iggy_connector_sdk::Error::InvalidConfigValue( + "broker_url TLS scheme could not be normalized".to_string(), + ) + })?; + url.set_port(Some(port)).map_err(|_| { + iggy_connector_sdk::Error::InvalidConfigValue( + "broker_url TLS port could not be preserved".to_string(), + ) + })?; + } + let query_pairs = url + .query_pairs() + .filter(|(key, _)| key != "client_id") + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect::>(); + let mut query = url.query_pairs_mut(); + query.clear(); + for (key, value) in &query_pairs { + query.append_pair(key, value); + } + query.append_pair("client_id", client_id); + drop(query); + + Ok(url.into()) +} + +fn install_rustls_provider() { + let _ = rustls::crypto::ring::default_provider().install_default(); +} + +fn read_file(path: &str, field: &str) -> Result, iggy_connector_sdk::Error> { + fs::read(path).map_err(|error| { + iggy_connector_sdk::Error::InvalidConfigValue(format!("{field} could not be read: {error}")) + }) +} + +fn read_certificate_file( + path: &str, + field: &str, +) -> Result>, iggy_connector_sdk::Error> { + let contents = read_file(path, field)?; + let mut reader = BufReader::new(Cursor::new(&contents)); + let certificates = certs(&mut reader) + .collect::, _>>() + .map_err(|error| { + iggy_connector_sdk::Error::InvalidConfigValue(format!("{field} is invalid: {error}")) + })?; + if certificates.is_empty() { + return Err(iggy_connector_sdk::Error::InvalidConfigValue(format!( + "{field} does not contain a PEM certificate" + ))); + } + Ok(certificates) +} + +fn read_private_key_file( + path: &str, + field: &str, +) -> Result, iggy_connector_sdk::Error> { + let contents = read_file(path, field)?; + let mut reader = BufReader::new(Cursor::new(&contents)); + private_key(&mut reader) + .map_err(|error| { + iggy_connector_sdk::Error::InvalidConfigValue(format!("{field} is invalid: {error}")) + })? + .ok_or_else(|| { + iggy_connector_sdk::Error::InvalidConfigValue(format!( + "{field} does not contain a PEM private key" + )) + }) +} + +fn system_root_cert_store() -> Result { + let mut root_cert_store = RootCertStore::empty(); + let native_certificates = load_native_certs(); + for certificate in native_certificates.certs { + root_cert_store.add(certificate).map_err(|error| { + iggy_connector_sdk::Error::InvalidConfigValue(format!( + "system TLS certificate is invalid: {error}" + )) + })?; + } + if root_cert_store.is_empty() { + return Err(iggy_connector_sdk::Error::InvalidConfigValue( + "no system TLS root certificates were found".to_string(), + )); + } + Ok(root_cert_store) +} + +fn build_tls_config( + ca: Option>>, + client_auth: Option<(Vec>, PrivateKeyDer<'static>)>, +) -> Result { + let root_cert_store = match ca { + Some(ca) => { + let mut root_cert_store = RootCertStore::empty(); + for certificate in ca { + root_cert_store.add(certificate).map_err(|error| { + iggy_connector_sdk::Error::InvalidConfigValue(format!( + "tls.ca_file contains an invalid certificate: {error}" + )) + })?; + } + if root_cert_store.is_empty() { + return Err(iggy_connector_sdk::Error::InvalidConfigValue( + "tls.ca_file does not contain a usable certificate".to_string(), + )); + } + root_cert_store + } + None => system_root_cert_store()?, + }; + let client_config = ClientConfig::builder().with_root_certificates(root_cert_store); + let Some((client_certificates, client_key)) = client_auth else { + return Ok(client_config.with_no_client_auth()); + }; + client_config + .with_client_auth_cert(client_certificates, client_key) + .map_err(|error| { + iggy_connector_sdk::Error::InvalidConfigValue(format!( + "TLS client certificate and key do not match: {error}" + )) + }) +} + +fn confirmed_indexes(confirmed: &[bool]) -> Vec { + confirmed + .iter() + .enumerate() + .filter_map(|(index, confirmed)| confirmed.then_some(index)) + .collect() +} + +fn mqtt5_qos(qos: rumqttc::QoS) -> rumqttc::v5::mqttbytes::QoS { + match qos { + rumqttc::QoS::AtMostOnce => rumqttc::v5::mqttbytes::QoS::AtMostOnce, + rumqttc::QoS::AtLeastOnce => rumqttc::v5::mqttbytes::QoS::AtLeastOnce, + rumqttc::QoS::ExactlyOnce => rumqttc::v5::mqttbytes::QoS::ExactlyOnce, + } +} + +fn mqtt5_puback_rejection(packet_id: u16, reason: PubAckReason) -> PublishWindowFailure { + let message = format!("MQTT 5 PUBACK rejected packet ID {packet_id}: {reason:?}"); + match reason { + PubAckReason::UnspecifiedError + | PubAckReason::ImplementationSpecificError + | PubAckReason::QuotaExceeded => PublishWindowFailure::TransientBrokerRejected(message), + _ => PublishWindowFailure::BrokerRejected(message), + } +} + +fn mqtt5_pubrec_rejection(packet_id: u16, reason: PubRecReason) -> PublishWindowFailure { + let message = format!("MQTT 5 PUBREC rejected packet ID {packet_id}: {reason:?}"); + match reason { + PubRecReason::UnspecifiedError + | PubRecReason::ImplementationSpecificError + | PubRecReason::QuotaExceeded => PublishWindowFailure::TransientBrokerRejected(message), + _ => PublishWindowFailure::BrokerRejected(message), + } +} + +async fn wait_for_mqtt311_disconnect( + event_loop: &mut Mqtt311EventLoop, + disconnect_timeout: Duration, +) -> Result<(), iggy_connector_sdk::Error> { + timeout(disconnect_timeout, async { + loop { + match event_loop.poll().await { + Ok(Mqtt311Event::Outgoing(Mqtt311Outgoing::Disconnect)) => return Ok(()), + Ok(_) => {} + Err(error) => { + return Err(iggy_connector_sdk::Error::Connection(error.to_string())); + } + } + } + }) + .await + .map_err(|_| { + iggy_connector_sdk::Error::Connection( + "timed out waiting for MQTT 3.1.1 DISCONNECT to flush".to_string(), + ) + })? +} + +async fn wait_for_mqtt5_disconnect( + event_loop: &mut Mqtt5EventLoop, + disconnect_timeout: Duration, +) -> Result<(), iggy_connector_sdk::Error> { + timeout(disconnect_timeout, async { + loop { + match event_loop.poll().await { + Ok(Mqtt5Event::Outgoing(Mqtt5Outgoing::Disconnect)) => return Ok(()), + Ok(_) => {} + Err(error) => { + return Err(iggy_connector_sdk::Error::Connection(error.to_string())); + } + } + } + }) + .await + .map_err(|_| { + iggy_connector_sdk::Error::Connection( + "timed out waiting for MQTT 5 DISCONNECT to flush".to_string(), + ) + })? +} + +async fn wait_for_connack( + event_loop: &mut Mqtt311EventLoop, + id: u32, + connect_timeout: Duration, +) -> Result<(), iggy_connector_sdk::Error> { + // Ignore unrelated packets while establishing the session; the first + // CONNACK is the broker's definitive connection result. + timeout(connect_timeout, async { + loop { + match event_loop.poll().await { + Ok(Mqtt311Event::Incoming(Mqtt311Incoming::ConnAck(connack))) => { + return mqtt311_connack_result(connack.code, id); + } + Ok(_) => {} + Err(error) => { + return Err(mqtt311_connection_error(error, id)); + } + } + } + }) + .await + .map_err(|_| { + iggy_connector_sdk::Error::Connection(format!( + "timed out waiting for MQTT 3.1.1 CONNACK for connector ID {id}" + )) + })? +} + +async fn wait_for_connack_v5( + event_loop: &mut Mqtt5EventLoop, + id: u32, + connect_timeout: Duration, + qos: u8, + retain: bool, +) -> Result, iggy_connector_sdk::Error> { + timeout(connect_timeout, async { + loop { + match event_loop.poll().await { + Ok(Mqtt5Event::Incoming(Mqtt5Incoming::ConnAck(connack))) => { + mqtt5_connack_result(connack.code, id)?; + validate_mqtt5_connack_capabilities( + connack.properties.as_ref(), + qos, + retain, + id, + )?; + return Ok(connack + .properties + .as_ref() + .and_then(|properties| properties.max_packet_size)); + } + Ok(_) => {} + Err(error) => { + return Err(mqtt5_connection_error(error, id)); + } + } + } + }) + .await + .map_err(|_| { + iggy_connector_sdk::Error::Connection(format!( + "timed out waiting for MQTT 5 CONNACK for connector ID {id}" + )) + })? +} + +fn mqtt311_connection_error(error: Mqtt311ConnectionError, id: u32) -> iggy_connector_sdk::Error { + match error { + Mqtt311ConnectionError::ConnectionRefused(code) => mqtt311_connack_error(code, id), + Mqtt311ConnectionError::Tls(error) => mqtt_tls_connection_error(error, id), + error => iggy_connector_sdk::Error::Connection(format!( + "MQTT 3.1.1 connection failed for connector ID {id}: {error}" + )), + } +} + +fn mqtt311_connack_result( + code: rumqttc::mqttbytes::v4::ConnectReturnCode, + id: u32, +) -> Result<(), iggy_connector_sdk::Error> { + if code == rumqttc::mqttbytes::v4::ConnectReturnCode::Success { + Ok(()) + } else { + Err(mqtt311_connack_error(code, id)) + } +} + +fn mqtt311_connack_error( + code: rumqttc::mqttbytes::v4::ConnectReturnCode, + id: u32, +) -> iggy_connector_sdk::Error { + use rumqttc::mqttbytes::v4::ConnectReturnCode; + + match code { + ConnectReturnCode::BadUserNamePassword | ConnectReturnCode::NotAuthorized => { + iggy_connector_sdk::Error::InvalidConfigValue(format!( + "MQTT 3.1.1 broker rejected authentication for connector ID {id}: {code:?}" + )) + } + ConnectReturnCode::RefusedProtocolVersion | ConnectReturnCode::BadClientId => { + iggy_connector_sdk::Error::InvalidConfigValue(format!( + "MQTT 3.1.1 broker rejected connection configuration for connector ID {id}: {code:?}" + )) + } + ConnectReturnCode::ServiceUnavailable => iggy_connector_sdk::Error::Connection(format!( + "MQTT 3.1.1 broker is unavailable for connector ID {id}" + )), + ConnectReturnCode::Success => { + unreachable!("success is handled before mapping connection failures") + } + } +} + +fn mqtt5_connection_error(error: Mqtt5ConnectionError, id: u32) -> iggy_connector_sdk::Error { + match error { + Mqtt5ConnectionError::ConnectionRefused(code) => mqtt5_connack_error(code, id), + Mqtt5ConnectionError::Tls(error) => mqtt_tls_connection_error(error, id), + Mqtt5ConnectionError::MqttState(Mqtt5StateError::ServerDisconnect { + reason_code: + DisconnectReasonCode::RetainNotSupported | DisconnectReasonCode::QoSNotSupported, + reason_string, + }) => iggy_connector_sdk::Error::InvalidConfigValue(format!( + "MQTT 5 broker rejected the configured publication policy for connector ID {id}: {reason_string:?}" + )), + error => iggy_connector_sdk::Error::Connection(format!( + "MQTT 5 connection failed for connector ID {id}: {error}" + )), + } +} + +fn mqtt_tls_connection_error(error: rumqttc::TlsError, id: u32) -> iggy_connector_sdk::Error { + if let rumqttc::TlsError::Io(io_error) = &error + && io_error + .get_ref() + .and_then(|source| source.downcast_ref::()) + .is_none() + { + return iggy_connector_sdk::Error::Connection(format!( + "MQTT TLS connection failed for connector ID {id}: {error}" + )); + } + + iggy_connector_sdk::Error::InvalidConfigValue(format!( + "MQTT TLS validation failed for connector ID {id}: {error}" + )) +} + +fn mqtt5_connack_result( + code: Mqtt5ConnectReturnCode, + id: u32, +) -> Result<(), iggy_connector_sdk::Error> { + if code == Mqtt5ConnectReturnCode::Success { + Ok(()) + } else { + Err(mqtt5_connack_error(code, id)) + } +} + +fn validate_mqtt5_connack_capabilities( + properties: Option<&rumqttc::v5::mqttbytes::v5::ConnAckProperties>, + qos: u8, + retain: bool, + id: u32, +) -> Result<(), iggy_connector_sdk::Error> { + validate_mqtt5_broker_capabilities( + properties.and_then(|properties| properties.max_qos), + properties.and_then(|properties| properties.retain_available), + qos, + retain, + id, + ) +} + +fn validate_mqtt5_broker_capabilities( + max_qos: Option, + retain_available: Option, + qos: u8, + retain: bool, + id: u32, +) -> Result<(), iggy_connector_sdk::Error> { + if let Some(max_qos) = max_qos + && qos > max_qos + { + return Err(iggy_connector_sdk::Error::InvalidConfigValue(format!( + "MQTT 5 broker maximum QoS {max_qos} is lower than configured qos {qos} for connector ID {id}" + ))); + } + if retain && retain_available == Some(0) { + return Err(iggy_connector_sdk::Error::InvalidConfigValue(format!( + "MQTT 5 broker does not support retained publications for connector ID {id}" + ))); + } + Ok(()) +} + +fn mqtt5_connack_error(code: Mqtt5ConnectReturnCode, id: u32) -> iggy_connector_sdk::Error { + use Mqtt5ConnectReturnCode::*; + + match code { + BadUserNamePassword | NotAuthorized | Banned | BadAuthenticationMethod => { + iggy_connector_sdk::Error::InvalidConfigValue(format!( + "MQTT 5 broker rejected authentication for connector ID {id}: {code:?}" + )) + } + RefusedProtocolVersion | UnsupportedProtocolVersion | ClientIdentifierNotValid => { + iggy_connector_sdk::Error::InvalidConfigValue(format!( + "MQTT 5 broker rejected connection configuration for connector ID {id}: {code:?}" + )) + } + ServerUnavailable + | ServerBusy + | UseAnotherServer + | ServerMoved + | ConnectionRateExceeded + | QuotaExceeded => iggy_connector_sdk::Error::Connection(format!( + "MQTT 5 broker temporarily rejected connection for connector ID {id}: {code:?}" + )), + Success => unreachable!("success is handled before mapping connection failures"), + _ => iggy_connector_sdk::Error::InitError(format!( + "MQTT 5 broker rejected connection for connector ID {id}: {code:?}" + )), + } +} + +#[cfg(test)] +mod tests { + use std::fs; + + use rumqttc::v5::mqttbytes::v5::{ + ConnectReturnCode, DisconnectReasonCode, PubAckReason, PubCompReason, PubRecReason, + }; + + use super::{ + DriverEvent, Mqtt311Options, PublishWindowFailure, PublishWindowState, + broker_url_with_client_id, mqtt5_connack_error, mqtt5_connection_error, + mqtt311_connection_error, read_certificate_file, read_private_key_file, rumqttc, + validate_mqtt5_broker_capabilities, + }; + + #[test] + fn given_invalid_ca_pem_when_validating_should_return_configuration_error() { + let path = + std::env::temp_dir().join(format!("iggy-mqtt-sink-invalid-ca-{}", std::process::id())); + fs::write(&path, b"not a certificate").expect("invalid CA file must write"); + let error = read_certificate_file(path.to_str().unwrap(), "tls.ca_file") + .expect_err("invalid PEM must be rejected"); + fs::remove_file(path).expect("invalid CA file must remove"); + + assert!(matches!( + error, + iggy_connector_sdk::Error::InvalidConfigValue(message) + if message.contains("tls.ca_file") + )); + } + + #[test] + fn given_invalid_private_key_pem_when_validating_should_return_configuration_error() { + let path = + std::env::temp_dir().join(format!("iggy-mqtt-sink-invalid-key-{}", std::process::id())); + fs::write(&path, b"not a private key").expect("invalid key file must write"); + let error = read_private_key_file(path.to_str().unwrap(), "tls.client_key_file") + .expect_err("invalid PEM must be rejected"); + fs::remove_file(path).expect("invalid key file must remove"); + + assert!(matches!( + error, + iggy_connector_sdk::Error::InvalidConfigValue(message) + if message.contains("tls.client_key_file") + )); + } + + #[test] + fn given_permanent_tls_failure_when_classifying_should_return_configuration_error() { + let error = mqtt311_connection_error( + rumqttc::ConnectionError::Tls(rumqttc::TlsError::NoValidCertInChain), + 1, + ); + + assert!(matches!( + error, + iggy_connector_sdk::Error::InvalidConfigValue(message) + if message.contains("TLS validation") + )); + } + + #[test] + fn given_tls_io_failure_when_classifying_should_return_connection_error() { + let error = mqtt5_connection_error( + rumqttc::v5::ConnectionError::Tls(rumqttc::TlsError::Io(std::io::Error::other( + "connection reset", + ))), + 1, + ); + + assert!(matches!( + error, + iggy_connector_sdk::Error::Connection(message) + if message.contains("MQTT TLS connection failed") + )); + } + + #[test] + fn given_wrapped_tls_certificate_failure_when_classifying_should_return_configuration_error() { + let certificate_error = rumqttc::tokio_rustls::rustls::Error::InvalidCertificate( + rumqttc::tokio_rustls::rustls::CertificateError::UnknownIssuer, + ); + let error = mqtt5_connection_error( + rumqttc::v5::ConnectionError::Tls(rumqttc::TlsError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidData, + certificate_error, + ))), + 1, + ); + + assert!(matches!( + error, + iggy_connector_sdk::Error::InvalidConfigValue(message) + if message.contains("TLS validation") + )); + } + + #[test] + fn given_out_of_order_qos_one_acknowledgements_when_applied_should_confirm_matching_publications() + { + let mut state = PublishWindowState::new(3, rumqttc::QoS::AtLeastOnce); + + state.submitted(0); + state.submitted(1); + state.submitted(2); + state.apply(DriverEvent::OutgoingPublish(41)).unwrap(); + state.apply(DriverEvent::OutgoingPublish(42)).unwrap(); + state.apply(DriverEvent::OutgoingPublish(43)).unwrap(); + state.apply(DriverEvent::PubAck(42)).unwrap(); + state.apply(DriverEvent::PubAck(41)).unwrap(); + state.apply(DriverEvent::PubAck(43)).unwrap(); + + assert_eq!(state.confirmed_indexes(), vec![0, 1, 2]); + } + + #[test] + fn given_out_of_order_qos_two_acknowledgements_when_applied_should_confirm_matching_publications() + { + let mut state = PublishWindowState::new(2, rumqttc::QoS::ExactlyOnce); + + state.submitted(0); + state.submitted(1); + state.apply(DriverEvent::OutgoingPublish(51)).unwrap(); + state.apply(DriverEvent::OutgoingPublish(52)).unwrap(); + state.apply(DriverEvent::PubComp(52)).unwrap(); + + assert_eq!(state.confirmed_indexes(), vec![1]); + assert!(!state.is_complete()); + + state.apply(DriverEvent::PubComp(51)).unwrap(); + assert!(state.is_complete()); + } + + #[test] + fn given_qos_zero_when_outgoing_publish_is_applied_should_confirm_without_broker_acknowledgement() + { + let mut state = PublishWindowState::new(1, rumqttc::QoS::AtMostOnce); + + state.submitted(0); + state.apply(DriverEvent::OutgoingPublish(0)).unwrap(); + + assert!(state.is_complete()); + } + + #[test] + fn given_outgoing_publish_beyond_window_when_applied_should_return_protocol_error() { + let mut state = PublishWindowState::new(1, rumqttc::QoS::AtLeastOnce); + + state.submitted(0); + state.apply(DriverEvent::OutgoingPublish(41)).unwrap(); + let error = state.apply(DriverEvent::OutgoingPublish(42)).unwrap_err(); + + assert!(matches!( + error, + PublishWindowFailure::Protocol(message) + if message.contains("unexpected MQTT publish packet") + )); + } + + #[test] + fn given_stale_acknowledgement_when_new_window_reuses_packet_id_should_not_confirm_publication() + { + let mut state = PublishWindowState::new(1, rumqttc::QoS::AtLeastOnce); + + state.apply(DriverEvent::PubAck(71)).unwrap(); + state.submitted(0); + state.apply(DriverEvent::OutgoingPublish(71)).unwrap(); + + assert!(!state.is_complete()); + } + + #[test] + fn given_duplicate_confirmation_should_count_each_publication_once() { + let mut state = PublishWindowState::new(2, rumqttc::QoS::AtLeastOnce); + + state.confirm(0); + state.confirm(0); + + assert!(!state.is_complete()); + + state.confirm(1); + + assert!(state.is_complete()); + } + + #[test] + fn given_session_reset_when_rumqttc_replays_publish_should_keep_its_packet_mapping() { + let mut state = PublishWindowState::new(1, rumqttc::QoS::ExactlyOnce); + state.submitted(0); + state.apply(DriverEvent::OutgoingPublish(81)).unwrap(); + + state.apply(DriverEvent::OutgoingPublish(81)).unwrap(); + state.apply(DriverEvent::PubComp(81)).unwrap(); + + assert!(state.is_complete()); + } + + #[test] + fn given_mqtt_five_no_matching_subscribers_puback_when_applied_should_confirm_publication() { + let mut state = PublishWindowState::new(1, rumqttc::QoS::AtLeastOnce); + state.submitted(0); + state.apply(DriverEvent::OutgoingPublish(61)).unwrap(); + + state + .apply(DriverEvent::Mqtt5PubAck { + packet_id: 61, + reason: PubAckReason::NoMatchingSubscribers, + }) + .unwrap(); + + assert!(state.is_complete()); + } + + #[test] + fn given_mqtt_five_rejected_puback_when_applied_should_return_error_without_confirmation() { + let mut state = PublishWindowState::new(1, rumqttc::QoS::AtLeastOnce); + state.submitted(0); + state.apply(DriverEvent::OutgoingPublish(62)).unwrap(); + + let error = state + .apply(DriverEvent::Mqtt5PubAck { + packet_id: 62, + reason: PubAckReason::NotAuthorized, + }) + .unwrap_err(); + + assert!(matches!( + &error, + PublishWindowFailure::BrokerRejected(message) + if message.contains("PUBACK rejected") + )); + assert!(matches!( + error.error(), + iggy_connector_sdk::Error::CannotStoreData(message) + if message.contains("PUBACK rejected") + )); + assert!(!state.is_complete()); + } + + #[test] + fn given_mqtt_five_quota_exceeded_puback_when_applied_should_request_retry() { + let mut state = PublishWindowState::new(1, rumqttc::QoS::AtLeastOnce); + state.submitted(0); + state.apply(DriverEvent::OutgoingPublish(64)).unwrap(); + + let error = state + .apply(DriverEvent::Mqtt5PubAck { + packet_id: 64, + reason: PubAckReason::QuotaExceeded, + }) + .unwrap_err(); + + assert!(error.retryable()); + assert!(matches!( + error.error(), + iggy_connector_sdk::Error::Connection(message) + if message.contains("QuotaExceeded") + )); + } + + #[test] + fn given_mqtt_five_rejected_pubrec_when_applied_should_return_error_without_confirmation() { + let mut state = PublishWindowState::new(1, rumqttc::QoS::ExactlyOnce); + state.submitted(0); + state.apply(DriverEvent::OutgoingPublish(63)).unwrap(); + + let error = state + .apply(DriverEvent::Mqtt5PubRec { + packet_id: 63, + reason: PubRecReason::NotAuthorized, + }) + .unwrap_err(); + + assert!(matches!( + &error, + PublishWindowFailure::BrokerRejected(message) + if message.contains("PUBREC rejected") + )); + assert!(matches!( + error.error(), + iggy_connector_sdk::Error::CannotStoreData(message) + if message.contains("PUBREC rejected") + )); + assert!(!state.is_complete()); + } + + #[test] + fn given_mqtt_five_quota_exceeded_pubrec_when_applied_should_request_retry() { + let mut state = PublishWindowState::new(1, rumqttc::QoS::ExactlyOnce); + state.submitted(0); + state.apply(DriverEvent::OutgoingPublish(65)).unwrap(); + + let error = state + .apply(DriverEvent::Mqtt5PubRec { + packet_id: 65, + reason: PubRecReason::QuotaExceeded, + }) + .unwrap_err(); + + assert!(error.retryable()); + assert!(matches!( + error.error(), + iggy_connector_sdk::Error::Connection(message) + if message.contains("QuotaExceeded") + )); + } + + #[test] + fn given_mqtt_five_rejected_pubcomp_when_applied_should_return_error_without_confirmation() { + let mut state = PublishWindowState::new(1, rumqttc::QoS::ExactlyOnce); + state.submitted(0); + state.apply(DriverEvent::OutgoingPublish(63)).unwrap(); + + let error = state + .apply(DriverEvent::Mqtt5PubComp { + packet_id: 63, + reason: PubCompReason::PacketIdentifierNotFound, + }) + .unwrap_err(); + + assert!(matches!( + error, + PublishWindowFailure::BrokerRejected(message) + if message.contains("PUBCOMP rejected") + )); + assert!(!state.is_complete()); + } + + #[test] + fn given_mqtt_five_authentication_rejection_should_return_configuration_error() { + let error = mqtt5_connack_error(ConnectReturnCode::BadUserNamePassword, 1); + + assert!(matches!( + error, + iggy_connector_sdk::Error::InvalidConfigValue(message) + if message.contains("authentication") + )); + } + + #[test] + fn given_mqtt_five_busy_broker_should_return_connection_error() { + let error = mqtt5_connack_error(ConnectReturnCode::ServerBusy, 1); + + assert!(matches!( + error, + iggy_connector_sdk::Error::Connection(message) + if message.contains("temporarily rejected") + )); + } + + #[test] + fn given_mqtt_five_broker_qos_limit_below_sink_qos_should_return_configuration_error() { + let error = validate_mqtt5_broker_capabilities(Some(1), None, 2, false, 1) + .expect_err("broker QoS limit must reject a higher configured QoS"); + + assert!(matches!( + error, + iggy_connector_sdk::Error::InvalidConfigValue(message) + if message.contains("maximum QoS 1") + )); + } + + #[test] + fn given_mqtt_five_broker_without_retain_support_should_return_configuration_error() { + let error = validate_mqtt5_broker_capabilities(None, Some(0), 1, true, 1) + .expect_err("broker retain policy must reject retained publications"); + + assert!(matches!( + error, + iggy_connector_sdk::Error::InvalidConfigValue(message) + if message.contains("does not support retained") + )); + } + + #[test] + fn given_mqtt_five_policy_disconnect_should_return_configuration_error() { + let qos_error = mqtt5_connection_error( + rumqttc::v5::ConnectionError::MqttState(rumqttc::v5::StateError::ServerDisconnect { + reason_code: DisconnectReasonCode::QoSNotSupported, + reason_string: None, + }), + 1, + ); + let retain_error = mqtt5_connection_error( + rumqttc::v5::ConnectionError::MqttState(rumqttc::v5::StateError::ServerDisconnect { + reason_code: DisconnectReasonCode::RetainNotSupported, + reason_string: None, + }), + 1, + ); + + assert!(matches!( + qos_error, + iggy_connector_sdk::Error::InvalidConfigValue(_) + )); + assert!(matches!( + retain_error, + iggy_connector_sdk::Error::InvalidConfigValue(_) + )); + } + + #[test] + fn given_broker_url_without_client_id_when_building_driver_options_should_add_client_id() { + let broker_url = broker_url_with_client_id("mqtt://localhost:1883", "sink-client") + .expect("broker URL must accept a client identifier"); + + let options = + Mqtt311Options::parse_url(broker_url).expect("driver options must parse broker URL"); + + assert_eq!(options.client_id(), "sink-client"); + } + + #[test] + fn given_mqtts_broker_url_when_building_driver_options_should_preserve_tls_port() { + let broker_url = broker_url_with_client_id("mqtts://broker.example", "sink-client") + .expect("broker URL must accept a client identifier"); + + assert_eq!( + url::Url::parse(&broker_url) + .expect("broker URL must parse") + .port(), + Some(8883) + ); + } + + #[test] + fn given_idle_reconnect_events_when_starting_window_should_discard_them() { + let (sender, mut events) = tokio::sync::mpsc::channel(2); + sender + .try_send(DriverEvent::ConnectionError( + iggy_connector_sdk::Error::Connection("idle reconnect".to_string()), + )) + .expect("idle connection error must queue"); + sender + .try_send(DriverEvent::SessionPresent(false)) + .expect("idle session event must queue"); + + super::discard_queued_events(&mut events); + + assert!(matches!( + events.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) + )); + } +} diff --git a/core/connectors/sinks/mqtt_sink/src/lib.rs b/core/connectors/sinks/mqtt_sink/src/lib.rs new file mode 100644 index 0000000000..113df76cab --- /dev/null +++ b/core/connectors/sinks/mqtt_sink/src/lib.rs @@ -0,0 +1,1547 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! MQTT sink plugin: converts Iggy batches into MQTT publications. +//! +//! The runtime owns polling and offset commits. This crate owns the broker +//! connection, publication acknowledgement, retry policy, and topic mapping. + +mod driver; +mod routing; + +use async_trait::async_trait; +use humantime::Duration as HumanDuration; +use iggy_connector_sdk::retry::retry_backoff; +use iggy_connector_sdk::{ + ConsumedMessage, Error, MessagesMetadata, Sink, TopicMetadata, sink_connector, +}; +use rumqttc_v4::QoS as MqttQos; +use secrecy::SecretString; +use serde::Deserialize; +use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, +}; +use std::{ + str::FromStr, + time::{Duration, Instant}, +}; +use tokio::sync::Mutex; +use tokio::time::timeout; +use tracing::{debug, error, info, warn}; +use url::Url; +use uuid::Uuid; + +use driver::{MqttDriver, MqttProtocol, PublishWindowProgress, validate_tls_config}; +use routing::{ + DEFAULT_TOPIC_TEMPLATE, PreparedPublication, PublicationOptions, TopicMode, + prepare_batch_topic, prepare_publication_with_batch_topic, validate_topic, + validate_topic_template, +}; + +sink_connector!(MqttSink); + +const CONNECTOR_NAME: &str = "MQTT sink"; +const DEFAULT_KEEP_ALIVE: &str = "30s"; +const DEFAULT_PUBLISH_TIMEOUT: &str = "30s"; +const DEFAULT_SHUTDOWN_TIMEOUT: &str = "30s"; +const DEFAULT_RETRY_DELAY: &str = "1s"; +const DEFAULT_MAX_RETRY_DELAY: &str = "30s"; +const DEFAULT_TOPIC: &str = "iggy.messages"; +const DEFAULT_MAX_RETRIES: u32 = 5; +const DEFAULT_REQUEST_CAPACITY: usize = 32; +const DEFAULT_MAX_IN_FLIGHT: usize = 10; +const DEFAULT_SESSION_EXPIRY_INTERVAL: u32 = 3600; +const DEFAULT_MAX_PACKET_SIZE: usize = 1024 * 1024; +const MAX_PUBLISH_TIMEOUT: Duration = Duration::from_secs(60 * 60); +const MAX_KEEP_ALIVE: Duration = Duration::from_secs(u16::MAX as u64); +const GENERATED_CLIENT_ID_PREFIX: &str = "iggymqttsink"; +const GENERATED_CLIENT_ID_LENGTH: usize = 23; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MqttSinkConfig { + pub broker_url: String, + #[serde(default)] + pub protocol: MqttProtocol, + pub client_id: Option, + pub username: Option, + // Credentials are kept secret so logging a deserialized config cannot + // accidentally expose the broker password. + pub password: Option, + #[serde(default)] + pub tls: Option, + #[serde(default = "default_clean_start")] + pub clean_start: bool, + #[serde(default = "default_session_expiry_interval")] + pub session_expiry_interval: Option, + #[serde(default = "default_topic")] + pub topic: Option, + #[serde(default)] + pub topic_mode: TopicMode, + pub topic_template: Option, + pub fallback_topic: Option, + #[serde(default = "default_qos")] + pub qos: u8, + #[serde(default)] + pub retain: bool, + pub keep_alive: Option, + pub request_capacity: Option, + pub publish_timeout: Option, + pub shutdown_timeout: Option, + pub max_retries: Option, + pub retry_delay: Option, + pub max_retry_delay: Option, + pub max_in_flight: Option, + pub max_packet_size: Option, + pub verbose_logging: Option, +} + +#[derive(Debug, Deserialize)] +pub struct MqttTlsConfig { + pub ca_file: Option, + pub client_cert_file: Option, + pub client_key_file: Option, +} + +fn default_session_expiry_interval() -> Option { + Some(DEFAULT_SESSION_EXPIRY_INTERVAL) +} + +fn default_topic() -> Option { + Some(DEFAULT_TOPIC.to_string()) +} + +fn default_qos() -> u8 { + 1 +} + +fn default_clean_start() -> bool { + true +} + +fn validate_broker_url(config: &MqttSinkConfig) -> Result<(), Error> { + let url = Url::parse(&config.broker_url) + .map_err(|error| Error::InvalidConfigValue(format!("broker_url: {error}")))?; + + if !url.username().is_empty() || url.password().is_some() { + return Err(Error::InvalidConfigValue( + "broker_url must not contain credentials; configure username and password separately" + .to_string(), + )); + } + + for (option, field) in [ + ("keep_alive_secs", "keep_alive"), + ("inflight_num", "max_in_flight"), + ("clean_session", "clean_start"), + ("clean_start", "clean_start"), + ("request_channel_capacity_num", "request_capacity"), + ("client_id", "client_id"), + ("max_incoming_packet_size_bytes", "max_packet_size"), + ("max_outgoing_packet_size_bytes", "max_packet_size"), + ] { + if url.query_pairs().any(|(key, _)| key == option) { + return Err(Error::InvalidConfigValue(format!( + "broker_url must not contain {option}; configure {field} separately" + ))); + } + } + + Ok(()) +} + +#[derive(Debug)] +pub struct MqttSink { + id: u32, + config: MqttSinkConfig, + client_id: String, + keep_alive: Duration, + publish_timeout: Duration, + shutdown_timeout: Duration, + retry_delay: Duration, + max_retry_delay: Duration, + max_retries: u32, + max_in_flight: usize, + max_packet_size: usize, + verbose: bool, + driver: Mutex>, + publish_lock: Arc>, + messages_published: AtomicU64, + publish_errors: AtomicU64, + in_flight_publications: AtomicU64, +} + +impl MqttSink { + pub fn new(id: u32, config: MqttSinkConfig) -> Self { + let max_retries = config.max_retries.unwrap_or(DEFAULT_MAX_RETRIES); + let max_in_flight = config.max_in_flight.unwrap_or(DEFAULT_MAX_IN_FLIGHT); + let max_packet_size = config.max_packet_size.unwrap_or(DEFAULT_MAX_PACKET_SIZE); + let client_id = config.client_id.clone().unwrap_or_else(generated_client_id); + + Self { + id, + verbose: config.verbose_logging.unwrap_or(false), + config, + client_id, + keep_alive: Duration::from_secs(30), + publish_timeout: Duration::from_secs(30), + shutdown_timeout: Duration::from_secs(30), + retry_delay: Duration::from_secs(1), + max_retry_delay: Duration::from_secs(30), + max_retries, + max_in_flight, + max_packet_size, + driver: Mutex::new(None), + publish_lock: Arc::new(Mutex::new(())), + messages_published: AtomicU64::new(0), + publish_errors: AtomicU64::new(0), + in_flight_publications: AtomicU64::new(0), + } + } + + fn qos(&self) -> Result { + match self.config.qos { + 0 => Ok(MqttQos::AtMostOnce), + 1 => Ok(MqttQos::AtLeastOnce), + 2 => Ok(MqttQos::ExactlyOnce), + qos => Err(Error::InvalidConfigValue(format!( + "qos {qos} is unsupported; expected 0, 1, or 2" + ))), + } + } + + fn validate_config( + &self, + ) -> Result<(Duration, Duration, Duration, Duration, Duration, usize), Error> { + if self.config.broker_url.trim().is_empty() { + return Err(Error::InvalidConfigValue( + "broker_url must not be empty".to_string(), + )); + } + if self.config.username.is_some() != self.config.password.is_some() { + return Err(Error::InvalidConfigValue( + "username and password must be configured together".to_string(), + )); + } + validate_broker_url(&self.config)?; + validate_tls_config(&self.config)?; + let keep_alive = parse_duration( + self.config.keep_alive.as_deref(), + DEFAULT_KEEP_ALIVE, + "keep_alive", + )?; + validate_keep_alive(self.config.protocol, keep_alive, "keep_alive")?; + let publish_timeout = parse_duration( + self.config.publish_timeout.as_deref(), + DEFAULT_PUBLISH_TIMEOUT, + "publish_timeout", + )?; + validate_max_duration(publish_timeout, MAX_PUBLISH_TIMEOUT, "publish_timeout")?; + let shutdown_timeout = parse_duration( + self.config.shutdown_timeout.as_deref(), + DEFAULT_SHUTDOWN_TIMEOUT, + "shutdown_timeout", + )?; + let retry_delay = parse_duration( + self.config.retry_delay.as_deref(), + DEFAULT_RETRY_DELAY, + "retry_delay", + )?; + let max_retry_delay = parse_duration( + self.config.max_retry_delay.as_deref(), + DEFAULT_MAX_RETRY_DELAY, + "max_retry_delay", + )?; + self.qos()?; + if let Some(topic) = self.config.topic.as_deref() { + validate_topic(topic, "topic")?; + } + let topic_template = self + .config + .topic_template + .as_deref() + .unwrap_or(DEFAULT_TOPIC_TEMPLATE); + validate_topic_template(topic_template, "topic_template")?; + let fallback_topic = self + .config + .fallback_topic + .as_deref() + .unwrap_or(topic_template); + validate_topic_template(fallback_topic, "fallback_topic")?; + if self.config.request_capacity == Some(0) { + return Err(Error::InvalidConfigValue( + "request_capacity must be greater than zero".to_string(), + )); + } + if self.max_packet_size == 0 { + return Err(Error::InvalidConfigValue( + "max_packet_size must be greater than zero".to_string(), + )); + } + let request_capacity = self + .config + .request_capacity + .unwrap_or(DEFAULT_REQUEST_CAPACITY); + let max_in_flight = self.config.max_in_flight.unwrap_or(DEFAULT_MAX_IN_FLIGHT); + if max_in_flight == 0 { + return Err(Error::InvalidConfigValue( + "max_in_flight must be greater than zero".to_string(), + )); + } + if max_in_flight > request_capacity { + return Err(Error::InvalidConfigValue( + "max_in_flight must not exceed request_capacity".to_string(), + )); + } + if !self.config.clean_start + && !self + .config + .client_id + .as_deref() + .is_some_and(|client_id| !client_id.trim().is_empty()) + { + return Err(Error::InvalidConfigValue( + "client_id must be present and non-empty when clean_start is false".to_string(), + )); + } + Ok(( + keep_alive, + publish_timeout, + shutdown_timeout, + retry_delay, + max_retry_delay.max(retry_delay), + max_in_flight, + )) + } + + fn client_id(&self) -> &str { + &self.client_id + } + + async fn connect_driver(&self) -> Result { + MqttDriver::connect( + self.id, + &self.config, + self.client_id().to_owned(), + self.keep_alive, + self.publish_timeout, + self.config + .request_capacity + .unwrap_or(DEFAULT_REQUEST_CAPACITY), + self.max_in_flight, + ) + .await + } + + fn finish_unconfirmed_publications(&self, confirmed: &[bool]) { + let unfinished = confirmed.iter().filter(|confirmed| !**confirmed).count() as u64; + self.in_flight_publications + .fetch_sub(unfinished, Ordering::Relaxed); + self.publish_errors.fetch_add(unfinished, Ordering::Relaxed); + } + + fn mark_window_confirmed( + confirmed: &mut [bool], + window_indices: &[usize], + confirmed_indexes: &[usize], + ) -> usize { + let mut newly_confirmed = 0; + for &local_index in confirmed_indexes { + let index = window_indices[local_index]; + if !confirmed[index] { + confirmed[index] = true; + newly_confirmed += 1; + } + } + newly_confirmed + } + + fn publish_window_deadline(&self) -> Result { + Instant::now() + .checked_add(self.publish_timeout) + .ok_or_else(|| Error::InvalidConfigValue("publish_timeout is too large".to_string())) + } + + fn prepare_publications( + &self, + topic_metadata: &TopicMetadata, + messages: Vec, + ) -> Result, Error> { + let batch_topic = prepare_batch_topic( + topic_metadata, + self.config.topic_mode, + self.config.topic.as_deref(), + self.config.topic_template.as_deref(), + ) + .map_err(|error| { + warn!( + "Rejecting MQTT batch for connector with ID {} because batch topic is invalid, stream: {}, topic: {}: {error}", + self.id, topic_metadata.stream, topic_metadata.topic + ); + self.publish_errors.fetch_add(1, Ordering::Relaxed); + error + })?; + let mut publications = Vec::with_capacity(messages.len()); + for message in messages { + let offset = message.offset; + match prepare_publication_with_batch_topic( + message, + topic_metadata, + self.config.topic_mode, + batch_topic.as_ref(), + self.config.topic_template.as_deref(), + self.config.fallback_topic.as_deref(), + PublicationOptions { + protocol: self.config.protocol, + qos: self.config.qos, + retain: self.config.retain, + }, + ) { + Ok(publication) => { + if publication.packet_size > self.max_packet_size { + warn!( + "Skipping oversized MQTT record for connector with ID {}: stream: {}, topic: {}, offset: {}, packet_size: {}, max_packet_size: {}", + self.id, + topic_metadata.stream, + topic_metadata.topic, + offset, + publication.packet_size, + self.max_packet_size + ); + self.publish_errors.fetch_add(1, Ordering::Relaxed); + } else { + publications.push(publication); + } + } + Err(error) => { + warn!( + "Skipping invalid MQTT record for connector with ID {}: stream: {}, topic: {}, offset: {}: {error}", + self.id, topic_metadata.stream, topic_metadata.topic, offset + ); + self.publish_errors.fetch_add(1, Ordering::Relaxed); + } + } + } + Ok(publications) + } + + async fn publish_messages( + &self, + topic_metadata: &TopicMetadata, + messages: Vec, + ) -> Result { + let qos = self.qos()?; + let publications = self.prepare_publications(topic_metadata, messages)?; + if publications.is_empty() { + return Ok(0); + } + // A single event loop cannot safely be driven by concurrent consumers. + // Serializing MQTT delivery keeps acknowledgement events associated + // with the publication that caused them. + let _publish_guard = self.publish_lock.lock().await; + let mut confirmed = vec![false; publications.len()]; + let mut skipped = 0_u64; + self.in_flight_publications + .fetch_add(publications.len() as u64, Ordering::Relaxed); + let mut driver = self.driver.lock().await.take(); + + let mut connection_retries = 0; + // Driver recreation restarts window enumeration, so retain the current window's state. + let mut requeued_window_retries = None; + 'publish: loop { + let mut active_driver = match driver.take() { + Some(driver) => driver, + None => match self.connect_driver().await { + Ok(reconnected) => reconnected, + Err(Error::Connection(error)) if connection_retries < self.max_retries => { + warn!( + "Retrying MQTT connection for connector with ID {} (retry {}/{}): {error}", + self.id, + connection_retries + 1, + self.max_retries + ); + let delay = retry_backoff( + self.retry_delay, + connection_retries + 1, + self.max_retry_delay, + ); + tokio::time::sleep(delay).await; + connection_retries += 1; + continue; + } + Err(error) => { + self.finish_unconfirmed_publications(&confirmed); + return Err(error); + } + }, + }; + connection_retries = 0; + if let Err(error) = active_driver.discard_queued_events().await { + self.finish_unconfirmed_publications(&confirmed); + return Err(error); + } + let max_packet_size = active_driver.max_packet_size(); + if max_packet_size < self.max_packet_size { + for (index, publication) in publications.iter().enumerate() { + if confirmed[index] || publication.packet_size <= max_packet_size { + continue; + } + warn!( + "Skipping oversized MQTT record for connector with ID {}: stream: {}, topic: {}, offset: {}, packet_size: {}, max_packet_size: {}", + self.id, + topic_metadata.stream, + topic_metadata.topic, + publication.offset, + publication.packet_size, + max_packet_size + ); + confirmed[index] = true; + skipped += 1; + self.in_flight_publications.fetch_sub(1, Ordering::Relaxed); + self.publish_errors.fetch_add(1, Ordering::Relaxed); + } + } + let pending_indices = unconfirmed_publication_indexes(&confirmed); + for (window_number, window_indices) in + pending_indices.chunks(self.max_in_flight).enumerate() + { + let mut window_retries = if window_number == 0 { + requeued_window_retries.take().unwrap_or(0) + } else { + 0 + }; + let window: Vec<_> = window_indices + .iter() + .map(|&index| &publications[index]) + .collect(); + let mut window_state = active_driver.new_publish_window(window.len(), qos); + let initial_indexes: Vec<_> = (0..window.len()).collect(); + if let Err(error) = active_driver + .enqueue_window( + &window, + &initial_indexes, + qos, + self.config.retain, + &mut window_state, + ) + .await + { + self.finish_unconfirmed_publications(&confirmed); + return Err(error); + } + loop { + let window_deadline = match self.publish_window_deadline() { + Ok(deadline) => deadline, + Err(error) => { + self.finish_unconfirmed_publications(&confirmed); + return Err(error); + } + }; + match active_driver + .drive_window(&mut window_state, window_deadline) + .await + { + Ok(PublishWindowProgress::Complete) => { + for &index in window_indices { + if !confirmed[index] { + confirmed[index] = true; + self.in_flight_publications.fetch_sub(1, Ordering::Relaxed); + } + } + break; + } + // rumqttc retains unacknowledged requests only when the broker resumes + // the previous session. A fresh session clears its pending queue. + Ok(PublishWindowProgress::SessionPresent(true)) => { + if window_retries >= self.max_retries { + self.finish_unconfirmed_publications(&confirmed); + return Err(Error::Connection( + "MQTT reconnect retry limit reached before unconfirmed publications were acknowledged".to_string(), + )); + } + window_retries += 1; + debug!( + "MQTT reconnect for connector with ID {} resumed the previous session; keeping sink packet mappings for rumqttc retransmission (retry {}/{})", + self.id, window_retries, self.max_retries, + ); + } + Ok(PublishWindowProgress::SessionPresent(false)) => { + let confirmed_indexes = window_state.confirmed_indexes(); + let newly_confirmed = Self::mark_window_confirmed( + &mut confirmed, + window_indices, + &confirmed_indexes, + ); + self.in_flight_publications + .fetch_sub(newly_confirmed as u64, Ordering::Relaxed); + let unconfirmed = window.len() - confirmed_indexes.len(); + if window_retries >= self.max_retries { + self.finish_unconfirmed_publications(&confirmed); + return Err(Error::Connection( + "MQTT session was not resumed before unconfirmed publications could be requeued".to_string(), + )); + } + warn!( + "MQTT reconnect for connector with ID {} did not resume the previous session; recreating the driver and requeuing {unconfirmed} unconfirmed publications (retry {}/{})", + self.id, + window_retries + 1, + self.max_retries, + ); + let delay = retry_backoff( + self.retry_delay, + window_retries + 1, + self.max_retry_delay, + ); + tokio::time::sleep(delay).await; + window_retries += 1; + requeued_window_retries = Some(window_retries); + // Leaving this scope drops the driver and its stale event loop. + continue 'publish; + } + Err(window_error) => { + let confirmed_indexes = window_state.confirmed_indexes(); + let newly_confirmed = Self::mark_window_confirmed( + &mut confirmed, + window_indices, + &confirmed_indexes, + ); + self.in_flight_publications + .fetch_sub(newly_confirmed as u64, Ordering::Relaxed); + if !window_error.retryable || window_retries >= self.max_retries { + self.finish_unconfirmed_publications(&confirmed); + return Err(window_error.error); + } + if window_error.requires_requeue { + // Recreate the driver before requeuing because the failed window + // leaves packet mappings and acknowledgement events uncertain. A + // fresh driver prevents stale MQTT events from confirming a retried + // publication. + warn!( + "Retrying MQTT publication for connector with ID {} with a new driver (retry {}/{}): {}", + self.id, + window_retries + 1, + self.max_retries, + window_error.error + ); + let delay = retry_backoff( + self.retry_delay, + window_retries + 1, + self.max_retry_delay, + ); + tokio::time::sleep(delay).await; + window_retries += 1; + requeued_window_retries = Some(window_retries); + continue 'publish; + } + if window_error.requires_clean + && let Err(error) = active_driver.clean().await + { + self.finish_unconfirmed_publications(&confirmed); + return Err(error); + } + warn!( + "Retrying MQTT publication for connector with ID {} (retry {}/{}): {}", + self.id, + window_retries + 1, + self.max_retries, + window_error.error + ); + let delay = retry_backoff( + self.retry_delay, + window_retries + 1, + self.max_retry_delay, + ); + tokio::time::sleep(delay).await; + window_retries += 1; + } + } + } + } + *self.driver.lock().await = Some(active_driver); + return Ok(confirmed.iter().filter(|confirmed| **confirmed).count() as u64 - skipped); + } + } +} + +#[async_trait] +impl Sink for MqttSink { + async fn open(&mut self) -> Result<(), Error> { + let ( + keep_alive, + publish_timeout, + shutdown_timeout, + retry_delay, + max_retry_delay, + max_in_flight, + ) = match self.validate_config() { + Ok(values) => values, + Err(error) => { + error!( + "Failed to validate MQTT sink connector with ID {}: {error}", + self.id + ); + return Err(error); + } + }; + self.keep_alive = keep_alive; + self.publish_timeout = publish_timeout; + self.shutdown_timeout = shutdown_timeout; + self.retry_delay = retry_delay; + self.max_retry_delay = max_retry_delay; + self.max_in_flight = max_in_flight; + let qos = match self.qos() { + Ok(qos) => qos, + Err(error) => { + error!( + "Failed to validate MQTT sink connector with ID {}: {error}", + self.id + ); + return Err(error); + } + }; + // Connection establishment waits for CONNACK, so a successful open + // means the broker accepted the session rather than only queueing a + // local rumqttc request. + let mut retries = 0; + let driver = loop { + match self.connect_driver().await { + Ok(driver) => break driver, + Err(Error::Connection(error)) if retries < self.max_retries => { + warn!( + "Retrying MQTT connection for connector with ID {} (retry {}/{}): {error}", + self.id, + retries + 1, + self.max_retries + ); + let delay = retry_backoff(self.retry_delay, retries + 1, self.max_retry_delay); + tokio::time::sleep(delay).await; + retries += 1; + } + Err(error) => { + error!( + "Failed to open MQTT sink connector with ID {}: {error}", + self.id + ); + return Err(error); + } + } + }; + *self.driver.get_mut() = Some(driver); + info!( + "Opened {CONNECTOR_NAME} connector with ID {}, protocol: {:?}, topic: {:?}, mode: {:?}, qos: {:?}, retain: {}", + self.id, + self.config.protocol, + self.config.topic, + self.config.topic_mode, + qos, + self.config.retain + ); + Ok(()) + } + + async fn consume( + &self, + topic_metadata: &TopicMetadata, + _messages_metadata: MessagesMetadata, + messages: Vec, + ) -> Result<(), Error> { + let published = self.publish_messages(topic_metadata, messages).await?; + self.messages_published + .fetch_add(published, Ordering::Relaxed); + if self.verbose { + info!( + "Published {published} messages to MQTT for connector with ID {}", + self.id + ); + } else { + debug!( + "Published {published} messages to MQTT for connector with ID {}", + self.id + ); + } + Ok(()) + } + + async fn close(&mut self) -> Result<(), Error> { + let shutdown_started = Instant::now(); + let _publish_guard = timeout(self.shutdown_timeout, self.publish_lock.lock()) + .await + .map_err(|_| { + let unfinished = self.in_flight_publications.load(Ordering::Relaxed); + error!( + "MQTT shutdown timed out for connector with ID {} with {unfinished} unfinished publications", + self.id + ); + Error::Connection(format!( + "MQTT shutdown timed out with {unfinished} unfinished publications" + )) + })?; + let remaining = self + .shutdown_timeout + .saturating_sub(shutdown_started.elapsed()); + let driver = self.driver.lock().await.take(); + if let Some(mut driver) = driver { + if remaining.is_zero() { + error!( + "MQTT shutdown timed out for connector with ID {} before disconnecting", + self.id + ); + return Err(Error::Connection( + "MQTT shutdown timed out before disconnecting".to_string(), + )); + } + timeout(remaining, driver.disconnect(remaining)) + .await + .map_err(|_| { + error!( + "MQTT shutdown timed out for connector with ID {} while disconnecting", + self.id + ); + Error::Connection("MQTT shutdown timed out while disconnecting".to_string()) + })??; + } + info!( + "Closed {CONNECTOR_NAME} connector with ID {} processed {} messages with {} errors and {} unfinished publications", + self.id, + self.messages_published.load(Ordering::Relaxed), + self.publish_errors.load(Ordering::Relaxed), + self.in_flight_publications.load(Ordering::Relaxed) + ); + Ok(()) + } +} + +fn generated_client_id() -> String { + let suffix = Uuid::new_v4().simple().to_string(); + let suffix_length = GENERATED_CLIENT_ID_LENGTH - GENERATED_CLIENT_ID_PREFIX.len(); + format!("{GENERATED_CLIENT_ID_PREFIX}{}", &suffix[..suffix_length]) +} + +fn unconfirmed_publication_indexes(confirmed: &[bool]) -> Vec { + confirmed + .iter() + .enumerate() + .filter_map(|(index, is_confirmed)| (!is_confirmed).then_some(index)) + .collect() +} + +fn parse_duration(value: Option<&str>, default: &str, field: &str) -> Result { + let value = value.unwrap_or(default); + HumanDuration::from_str(value) + .map(|duration| *duration) + .map_err(|error| Error::InvalidConfigValue(format!("{field}: {error}"))) +} + +fn validate_max_duration(duration: Duration, maximum: Duration, field: &str) -> Result<(), Error> { + if duration > maximum { + return Err(Error::InvalidConfigValue(format!( + "{field} must not exceed {}s", + maximum.as_secs() + ))); + } + Ok(()) +} + +fn validate_keep_alive( + protocol: MqttProtocol, + keep_alive: Duration, + field: &str, +) -> Result<(), Error> { + if keep_alive > MAX_KEEP_ALIVE { + return Err(Error::InvalidConfigValue(format!( + "{field} must not exceed {}s", + MAX_KEEP_ALIVE.as_secs() + ))); + } + let minimum_keep_alive = match protocol { + MqttProtocol::Mqtt311 => Duration::from_secs(1), + MqttProtocol::Mqtt5 => Duration::from_secs(5), + }; + let invalid = match protocol { + MqttProtocol::Mqtt311 => !keep_alive.is_zero() && keep_alive < minimum_keep_alive, + MqttProtocol::Mqtt5 => keep_alive < minimum_keep_alive, + }; + if invalid { + return Err(Error::InvalidConfigValue(format!( + "{field} must be at least {:?} for {:?}", + minimum_keep_alive, protocol + ))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use iggy_connector_sdk::Payload; + + fn test_config() -> MqttSinkConfig { + MqttSinkConfig { + broker_url: "mqtt://localhost:1883".to_string(), + protocol: MqttProtocol::Mqtt5, + client_id: Some("test-client".to_string()), + username: None, + password: None, + tls: None, + clean_start: false, + session_expiry_interval: Some(DEFAULT_SESSION_EXPIRY_INTERVAL), + topic: Some("test/topic".to_string()), + topic_mode: TopicMode::Static, + topic_template: None, + fallback_topic: None, + qos: 1, + retain: false, + keep_alive: Some("30s".to_string()), + request_capacity: Some(32), + publish_timeout: Some("5s".to_string()), + shutdown_timeout: Some("5s".to_string()), + max_retries: Some(5), + retry_delay: Some("1s".to_string()), + max_retry_delay: Some("30s".to_string()), + max_in_flight: Some(1), + max_packet_size: None, + verbose_logging: Some(false), + } + } + + fn test_messages() -> Vec { + vec![ + ConsumedMessage { + id: 1, + offset: 10, + checksum: 1, + timestamp: 1, + origin_timestamp: 1, + headers: None, + payload: Payload::Raw(b"first".to_vec()), + }, + ConsumedMessage { + id: 2, + offset: 11, + checksum: 1, + timestamp: 1, + origin_timestamp: 1, + headers: None, + payload: Payload::Raw(b"second".to_vec()), + }, + ] + } + + #[test] + fn given_no_max_in_flight_when_new_should_use_pipelining_default() { + let mut config = test_config(); + config.max_in_flight = None; + let sink = MqttSink::new(1, config); + assert_eq!(sink.config.qos, 1); + assert_eq!(sink.max_in_flight, DEFAULT_MAX_IN_FLIGHT); + } + + #[test] + fn given_no_max_retries_should_use_default() { + let mut config = test_config(); + config.max_retries = None; + let sink = MqttSink::new(1, config); + + assert_eq!(sink.max_retries, DEFAULT_MAX_RETRIES); + } + + #[test] + fn given_zero_max_retries_should_not_retry_after_initial_attempt() { + let mut config = test_config(); + config.max_retries = Some(0); + let sink = MqttSink::new(1, config); + + assert_eq!(sink.max_retries, 0); + } + + #[test] + fn given_keep_alive_above_protocol_minimum_should_accept_configuration() { + let sink = MqttSink::new(1, test_config()); + + assert!(sink.validate_config().is_ok()); + } + + #[test] + fn given_mqtt311_zero_keep_alive_should_accept_configuration() { + let mut config = test_config(); + config.protocol = MqttProtocol::Mqtt311; + config.keep_alive = Some("0s".to_string()); + let sink = MqttSink::new(1, config); + + assert!(sink.validate_config().is_ok()); + } + + #[test] + fn given_mqtt5_zero_keep_alive_should_reject_configuration() { + let mut config = test_config(); + config.keep_alive = Some("0s".to_string()); + let sink = MqttSink::new(1, config); + + assert!(matches!( + sink.validate_config(), + Err(Error::InvalidConfigValue(message)) if message.contains("keep_alive") + )); + } + + #[test] + fn given_keep_alive_below_protocol_minimum_should_reject_configuration() { + let mut config = test_config(); + config.protocol = MqttProtocol::Mqtt311; + config.keep_alive = Some("500ms".to_string()); + let sink = MqttSink::new(1, config); + + assert!(matches!( + sink.validate_config(), + Err(Error::InvalidConfigValue(message)) if message.contains("keep_alive") + )); + } + + #[test] + fn given_keep_alive_above_rumqttc_limit_should_reject_configuration() { + let mut config = test_config(); + config.keep_alive = Some("65536s".to_string()); + let sink = MqttSink::new(1, config); + + assert!(matches!( + sink.validate_config(), + Err(Error::InvalidConfigValue(message)) if message == "keep_alive must not exceed 65535s" + )); + } + + #[test] + fn given_unknown_sink_config_field_should_reject_configuration() { + let error = serde_json::from_str::( + r#"{"broker_url":"mqtt://localhost:1883","clean_session":true}"#, + ) + .expect_err("unknown fields must reject the configuration"); + + assert!(error.to_string().contains("unknown field `clean_session`")); + } + + #[test] + fn given_publish_timeout_above_one_hour_when_validating_should_return_configuration_error() { + let mut config = test_config(); + config.publish_timeout = Some("1h1s".to_string()); + let sink = MqttSink::new(1, config); + + assert!(matches!( + sink.validate_config(), + Err(Error::InvalidConfigValue(message)) + if message == "publish_timeout must not exceed 3600s" + )); + } + + #[test] + fn given_unparsable_keep_alive_should_reject_configuration() { + let mut config = test_config(); + config.keep_alive = Some("not a duration".to_string()); + let sink = MqttSink::new(1, config); + + assert!(matches!( + sink.validate_config(), + Err(Error::InvalidConfigValue(message)) if message.contains("keep_alive") + )); + } + + #[test] + fn given_supported_qos_when_qos_should_map_to_rumqttc() { + for qos in [0, 1, 2] { + let mut config = test_config(); + config.qos = qos; + let sink = MqttSink::new(1, config); + assert!(sink.qos().is_ok()); + } + } + + #[test] + fn given_unsupported_qos_when_qos_should_return_configuration_error() { + let mut config = test_config(); + config.qos = 3; + let sink = MqttSink::new(1, config); + assert!(matches!(sink.qos(), Err(Error::InvalidConfigValue(_)))); + } + + #[test] + fn given_max_in_flight_above_request_capacity_when_validating_should_return_configuration_error() + { + let mut config = test_config(); + config.request_capacity = Some(2); + config.max_in_flight = Some(3); + let sink = MqttSink::new(1, config); + + assert!(matches!( + sink.validate_config(), + Err(Error::InvalidConfigValue(_)) + )); + } + + #[test] + fn given_zero_max_in_flight_when_validating_should_return_configuration_error() { + let mut config = test_config(); + config.max_in_flight = Some(0); + let sink = MqttSink::new(1, config); + + assert_eq!(sink.max_in_flight, 0); + assert!(matches!( + sink.validate_config(), + Err(Error::InvalidConfigValue(message)) if message.contains("max_in_flight") + )); + } + + #[test] + fn given_unknown_topic_template_placeholder_when_validating_should_return_configuration_error() + { + let mut config = test_config(); + config.topic_template = Some("{stream}/{partition}".to_string()); + let sink = MqttSink::new(1, config); + + assert!(matches!( + sink.validate_config(), + Err(Error::InvalidConfigValue(message)) if message.contains("unsupported placeholder") + )); + } + + #[test] + fn given_reserved_static_topic_when_validating_should_return_configuration_error() { + let mut config = test_config(); + config.topic = Some("$SYS/broker/uptime".to_string()); + let sink = MqttSink::new(1, config); + + assert!(matches!( + sink.validate_config(), + Err(Error::InvalidConfigValue(message)) if message.contains("must not start") + )); + } + + #[test] + fn given_reserved_fallback_topic_when_validating_should_return_configuration_error() { + let mut config = test_config(); + config.fallback_topic = Some("$SYS/{stream}/{topic}".to_string()); + let sink = MqttSink::new(1, config); + + assert!(matches!( + sink.validate_config(), + Err(Error::InvalidConfigValue(message)) if message.contains("must not start") + )); + } + + #[test] + fn given_broker_url_with_credentials_when_validating_should_return_configuration_error() { + let mut config = test_config(); + config.broker_url = "mqtt://user:password@localhost:1883".to_string(); + let sink = MqttSink::new(1, config); + + assert!(matches!( + sink.validate_config(), + Err(Error::InvalidConfigValue(message)) if message.contains("must not contain credentials") + )); + } + + #[test] + fn given_configured_credentials_when_validating_should_accept_broker_url_without_credentials() { + let mut config = test_config(); + config.username = Some("user".to_string()); + config.password = Some(SecretString::from("password")); + let sink = MqttSink::new(1, config); + + assert!(sink.validate_config().is_ok()); + } + + #[test] + fn given_broker_url_transport_option_when_validating_should_return_configuration_error() { + for (option, field) in [ + ("keep_alive_secs", "keep_alive"), + ("inflight_num", "max_in_flight"), + ("clean_session", "clean_start"), + ("clean_start", "clean_start"), + ("request_channel_capacity_num", "request_capacity"), + ("client_id", "client_id"), + ("max_incoming_packet_size_bytes", "max_packet_size"), + ("max_outgoing_packet_size_bytes", "max_packet_size"), + ] { + let mut config = test_config(); + config.broker_url = format!("mqtt://localhost:1883?{option}=value"); + let sink = MqttSink::new(1, config); + + assert!(matches!( + sink.validate_config(), + Err(Error::InvalidConfigValue(message)) + if message == format!( + "broker_url must not contain {option}; configure {field} separately" + ) + )); + } + } + + #[test] + fn given_client_certificate_without_private_key_when_validating_should_return_configuration_error() + { + let mut config = test_config(); + config.broker_url = "mqtts://localhost:8883".to_string(); + config.tls = Some(MqttTlsConfig { + ca_file: Some("ca.pem".to_string()), + client_cert_file: Some("client.pem".to_string()), + client_key_file: None, + }); + let sink = MqttSink::new(1, config); + + assert!(matches!( + sink.validate_config(), + Err(Error::InvalidConfigValue(message)) if message.contains("together") + )); + } + + #[test] + fn given_mutual_tls_without_ca_file_when_validating_should_accept_configuration() { + let mut config = test_config(); + config.broker_url = "mqtts://localhost:8883".to_string(); + config.tls = Some(MqttTlsConfig { + ca_file: None, + client_cert_file: Some("client.pem".to_string()), + client_key_file: Some("client.key".to_string()), + }); + let sink = MqttSink::new(1, config); + + assert!(sink.validate_config().is_ok()); + } + + #[test] + fn given_tls_file_with_unencrypted_broker_url_when_validating_should_return_configuration_error() + { + let mut config = test_config(); + config.tls = Some(MqttTlsConfig { + ca_file: Some("ca.pem".to_string()), + client_cert_file: None, + client_key_file: None, + }); + let sink = MqttSink::new(1, config); + + assert!(matches!( + sink.validate_config(), + Err(Error::InvalidConfigValue(message)) if message.contains("mqtts:// or ssl://") + )); + } + + #[test] + fn given_clean_start_without_client_id_when_constructing_should_generate_short_client_id() { + let mut config = test_config(); + config.client_id = None; + config.clean_start = true; + let sink = MqttSink::new(42, config); + let mut other_config = test_config(); + other_config.client_id = None; + other_config.clean_start = true; + let other_sink = MqttSink::new(42, other_config); + + assert_eq!(sink.client_id().len(), GENERATED_CLIENT_ID_LENGTH); + assert!( + sink.client_id() + .chars() + .all(|character| character.is_ascii_alphanumeric()) + ); + assert!(sink.client_id().starts_with(GENERATED_CLIENT_ID_PREFIX)); + assert_ne!(sink.client_id(), other_sink.client_id()); + } + + #[test] + fn given_blank_client_id_when_clean_start_is_false_should_return_configuration_error() { + let mut config = test_config(); + config.client_id = Some(" ".to_string()); + let sink = MqttSink::new(1, config); + + assert!(matches!( + sink.validate_config(), + Err(Error::InvalidConfigValue(message)) + if message == "client_id must be present and non-empty when clean_start is false" + )); + } + + #[test] + fn given_unconfirmed_publications_when_finishing_should_clear_and_report_them() { + let sink = MqttSink::new(1, test_config()); + sink.in_flight_publications.store(1, Ordering::Relaxed); + + sink.finish_unconfirmed_publications(&[true, false]); + + assert_eq!(sink.in_flight_publications.load(Ordering::Relaxed), 0); + assert_eq!(sink.publish_errors.load(Ordering::Relaxed), 1); + } + + #[test] + fn given_session_reset_with_partially_confirmed_window_should_requeue_only_unconfirmed() { + let mut confirmed = vec![false; 3]; + let window_indices = [0, 1, 2]; + + let newly_confirmed = + MqttSink::mark_window_confirmed(&mut confirmed, &window_indices, &[1]); + + assert_eq!(newly_confirmed, 1); + assert_eq!(unconfirmed_publication_indexes(&confirmed), vec![0, 2]); + } + + #[test] + fn given_unreachable_broker_when_reconnecting_should_stop_at_max_retries() { + let mut config = test_config(); + config.broker_url = "mqtt://127.0.0.1:1".to_string(); + config.max_retries = Some(2); + let mut sink = MqttSink::new(1, config); + sink.publish_timeout = Duration::from_millis(100); + sink.retry_delay = Duration::from_millis(1); + sink.max_retry_delay = Duration::from_millis(1); + let metadata = TopicMetadata { + stream: "devices".to_string(), + topic: "telemetry".to_string(), + }; + let runtime = tokio::runtime::Runtime::new().expect("runtime must start"); + + let error = runtime.block_on(async { + timeout( + Duration::from_secs(1), + sink.publish_messages(&metadata, test_messages()), + ) + .await + .expect("reconnect attempts must finish") + .expect_err("unreachable broker must return an error") + }); + + assert!(matches!(error, Error::Connection(_))); + assert_eq!(sink.in_flight_publications.load(Ordering::Relaxed), 0); + assert_eq!(sink.publish_errors.load(Ordering::Relaxed), 2); + } + + #[test] + fn given_static_mode_when_preparing_batch_should_share_topic_storage() { + let sink = MqttSink::new(1, test_config()); + let metadata = TopicMetadata { + stream: "devices".to_string(), + topic: "telemetry".to_string(), + }; + let publications = sink + .prepare_publications(&metadata, test_messages()) + .unwrap(); + + assert!(Arc::ptr_eq( + publications[0] + .topic + .as_shared() + .expect("static topics must be shared"), + publications[1] + .topic + .as_shared() + .expect("static topics must be shared"), + )); + assert_eq!( + publications[0].packet_size, + driver::publication_packet_size( + &publications[0], + sink.config.protocol, + sink.config.qos, + sink.config.retain, + ) + ); + } + + #[test] + fn given_template_mode_when_preparing_batch_should_share_topic_storage() { + let mut config = test_config(); + config.topic_mode = TopicMode::Template; + config.topic_template = Some("{stream}/output/{topic}".to_string()); + let sink = MqttSink::new(1, config); + let metadata = TopicMetadata { + stream: "devices".to_string(), + topic: "telemetry".to_string(), + }; + + let publications = sink + .prepare_publications(&metadata, test_messages()) + .unwrap(); + + assert!(Arc::ptr_eq( + publications[0] + .topic + .as_shared() + .expect("template topics must be shared"), + publications[1] + .topic + .as_shared() + .expect("template topics must be shared"), + )); + assert_eq!(publications[0].topic.as_str(), "devices/output/telemetry"); + } + + #[test] + fn given_malformed_dynamic_record_when_preparing_should_skip_record_and_keep_valid_publications() + { + let mut config = test_config(); + config.topic_mode = TopicMode::Dynamic; + config.fallback_topic = Some("fallback/{stream}/{topic}".to_string()); + let sink = MqttSink::new(1, config); + let metadata = TopicMetadata { + stream: "devices".to_string(), + topic: "telemetry".to_string(), + }; + let malformed = br#"{ + "version": 1, + "payload_base64": "not base64" + }"#; + let mut malformed = malformed.to_vec(); + let malformed = simd_json::to_owned_value(&mut malformed).expect("JSON must parse"); + let messages = vec![ + ConsumedMessage { + id: 1, + offset: 10, + checksum: 1, + timestamp: 1, + origin_timestamp: 1, + headers: None, + payload: Payload::Raw(b"valid".to_vec()), + }, + ConsumedMessage { + id: 2, + offset: 11, + checksum: 1, + timestamp: 1, + origin_timestamp: 1, + headers: None, + payload: Payload::Json(malformed), + }, + ConsumedMessage { + id: 3, + offset: 12, + checksum: 1, + timestamp: 1, + origin_timestamp: 1, + headers: None, + payload: Payload::Raw(b"also-valid".to_vec()), + }, + ]; + + let publications = sink + .prepare_publications(&metadata, messages) + .expect("malformed record must not reject valid records"); + + assert_eq!(publications.len(), 2); + assert_eq!(publications[0].payload.as_ref(), b"valid"); + assert_eq!(publications[1].payload.as_ref(), b"also-valid"); + assert_eq!(sink.publish_errors.load(Ordering::Relaxed), 1); + } + + #[test] + fn given_oversized_record_when_preparing_should_skip_record_and_keep_valid_publications() { + let mut config = test_config(); + config.max_packet_size = Some(32); + let sink = MqttSink::new(1, config); + let metadata = TopicMetadata { + stream: "devices".to_string(), + topic: "telemetry".to_string(), + }; + let messages = vec![ + ConsumedMessage { + id: 1, + offset: 10, + checksum: 1, + timestamp: 1, + origin_timestamp: 1, + headers: None, + payload: Payload::Raw(b"valid".to_vec()), + }, + ConsumedMessage { + id: 2, + offset: 11, + checksum: 1, + timestamp: 1, + origin_timestamp: 1, + headers: None, + payload: Payload::Raw(vec![b'x'; 64]), + }, + ]; + + let publications = sink + .prepare_publications(&metadata, messages) + .expect("oversized record must not reject valid records"); + + assert_eq!(publications.len(), 1); + assert_eq!(publications[0].payload.as_ref(), b"valid"); + assert_eq!(sink.publish_errors.load(Ordering::Relaxed), 1); + } + + #[test] + fn given_invalid_batch_topic_when_preparing_should_count_publish_error() { + let mut config = test_config(); + config.topic = Some("invalid\0topic".to_string()); + let sink = MqttSink::new(1, config); + let metadata = TopicMetadata { + stream: "devices".to_string(), + topic: "telemetry".to_string(), + }; + + let error = sink + .prepare_publications(&metadata, test_messages()) + .expect_err("invalid batch topic must reject the batch"); + + assert!(matches!(error, Error::InvalidRecordValue(_))); + assert_eq!(sink.publish_errors.load(Ordering::Relaxed), 1); + } + + #[test] + fn given_no_active_driver_when_closed_should_release_sink() { + let mut sink = MqttSink::new(1, test_config()); + let runtime = tokio::runtime::Runtime::new().expect("runtime must start"); + + runtime + .block_on(sink.close()) + .expect("sink without a driver must close"); + } + + #[test] + fn given_publications_in_flight_when_shutdown_times_out_should_report_them() { + let mut sink = MqttSink::new(1, test_config()); + sink.shutdown_timeout = Duration::from_millis(10); + sink.in_flight_publications.store(2, Ordering::Relaxed); + let publish_lock = Arc::clone(&sink.publish_lock); + let runtime = tokio::runtime::Runtime::new().expect("runtime must start"); + + runtime.block_on(async { + let _publish_guard = publish_lock.lock().await; + let result = timeout(Duration::from_secs(1), sink.close()) + .await + .expect("shutdown must respect its timeout"); + + assert!( + matches!(result, Err(Error::Connection(message)) if message.contains("2 unfinished publications")), + "shutdown must report the number of publications that did not drain" + ); + }); + } + + #[test] + fn given_minimal_json_when_deserialized_should_require_broker_url() { + let error = serde_json::from_str::("{}").unwrap_err(); + assert!(error.to_string().contains("broker_url")); + let config: MqttSinkConfig = serde_json::from_str( + r#"{"broker_url":"mqtt://localhost:1883","client_id":"test-client"}"#, + ) + .unwrap(); + let sink = MqttSink::new(1, config); + assert_eq!(sink.config.topic.as_deref(), Some(DEFAULT_TOPIC)); + assert!(sink.config.clean_start); + assert_eq!(sink.config.qos, 1); + assert_eq!(sink.max_in_flight, DEFAULT_MAX_IN_FLIGHT); + assert_eq!(sink.shutdown_timeout, Duration::from_secs(30)); + } +} diff --git a/core/connectors/sinks/mqtt_sink/src/routing.rs b/core/connectors/sinks/mqtt_sink/src/routing.rs new file mode 100644 index 0000000000..b53ba966d5 --- /dev/null +++ b/core/connectors/sinks/mqtt_sink/src/routing.rs @@ -0,0 +1,992 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use base64::{Engine, engine::general_purpose::STANDARD}; +use bytes::Bytes; +use iggy_common::{HeaderKey, IggyTimestamp}; +use iggy_connector_sdk::{ConsumedMessage, Error, Payload, TopicMetadata}; +use rumqttc_v5::mqttbytes::v5::PublishProperties; +use serde::Deserialize; +use std::sync::Arc; + +use crate::{MqttProtocol, driver}; + +/// Header used by the MQTT source to preserve the original broker topic. +const MQTT_TOPIC_HEADER: &str = "mqtt.topic"; +pub const DEFAULT_TOPIC_TEMPLATE: &str = "{stream}/{topic}"; +const MAX_MQTT_STRING_BYTES: usize = u16::MAX as usize; +const MAX_MQTT_PROPERTIES_BYTES: usize = 268_435_455; +const MICROS_PER_SECOND: u64 = 1_000_000; + +#[derive(Debug, Clone, Copy, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum TopicMode { + #[default] + Static, + Template, + Dynamic, +} + +#[derive(Debug)] +pub struct PreparedPublication { + pub offset: u64, + pub topic: PublicationTopic, + pub payload: Bytes, + pub packet_size: usize, + pub properties: Option, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct PublicationOptions { + pub(crate) protocol: MqttProtocol, + pub(crate) qos: u8, + pub(crate) retain: bool, +} + +#[derive(Debug)] +pub enum PublicationTopic { + Shared(Arc), + Owned(String), +} + +impl PublicationTopic { + pub fn as_str(&self) -> &str { + match self { + Self::Shared(topic) => topic, + Self::Owned(topic) => topic, + } + } + + #[cfg(test)] + pub fn as_shared(&self) -> Option<&Arc> { + match self { + Self::Shared(topic) => Some(topic), + Self::Owned(_) => None, + } + } +} + +#[derive(Debug, Deserialize)] +struct MqttEnvelope { + version: u8, + topic: Option, + properties: Option, + payload_base64: String, +} + +#[derive(Debug, Deserialize)] +struct MqttEnvelopeProperties { + payload_format_indicator: Option, + message_expiry_interval: Option, + response_topic: Option, + correlation_data_base64: Option, + #[serde(default)] + user_properties: Vec, + #[serde(default)] + subscription_identifiers: Vec, + content_type: Option, +} + +#[derive(Debug, Deserialize)] +struct MqttUserProperty { + key: String, + value: String, +} + +pub fn validate_topic(topic: &str, field: &str) -> Result<(), Error> { + validate_publish_topic_value(topic, field).map_err(Error::InvalidConfigValue) +} + +pub fn validate_topic_template(template: &str, field: &str) -> Result<(), Error> { + if template.trim().is_empty() { + return Err(Error::InvalidConfigValue(format!( + "{field} must not be empty" + ))); + } + if template.starts_with('$') { + return Err(Error::InvalidConfigValue(format!( + "{field} must not start with '$'" + ))); + } + validate_mqtt_string(template, field).map_err(Error::InvalidConfigValue)?; + if template + .as_bytes() + .iter() + .any(|character| matches!(*character, b'+' | b'#')) + { + return Err(Error::InvalidConfigValue(format!( + "{field} must not contain MQTT wildcards" + ))); + } + + let mut remaining = template; + while let Some(start) = remaining.find('{') { + if remaining[..start].contains('}') { + return Err(Error::InvalidConfigValue(format!( + "{field} contains an unmatched closing brace" + ))); + } + let placeholder = &remaining[start + 1..]; + let Some(end) = placeholder.find('}') else { + return Err(Error::InvalidConfigValue(format!( + "{field} contains an unterminated placeholder" + ))); + }; + match &placeholder[..end] { + "stream" | "topic" => {} + value => { + return Err(Error::InvalidConfigValue(format!( + "{field} contains unsupported placeholder '{{{value}}}'" + ))); + } + } + remaining = &placeholder[end + 1..]; + } + if remaining.contains('}') { + return Err(Error::InvalidConfigValue(format!( + "{field} contains an unmatched closing brace" + ))); + } + Ok(()) +} + +fn validate_publish_topic_value(topic: &str, field: &str) -> Result<(), String> { + validate_topic_value(topic, field)?; + if topic.starts_with('$') { + return Err(format!("{field} must not start with '$'")); + } + Ok(()) +} + +fn validate_topic_value(topic: &str, field: &str) -> Result<(), String> { + if topic.trim().is_empty() { + return Err(format!("{field} must not be empty")); + } + validate_mqtt_string(topic, field)?; + if topic + .as_bytes() + .iter() + .any(|character| matches!(*character, b'+' | b'#')) + { + return Err(format!("{field} must not contain MQTT wildcards")); + } + Ok(()) +} + +fn validate_mqtt_string(value: &str, field: &str) -> Result<(), String> { + if value.len() > MAX_MQTT_STRING_BYTES { + return Err(format!( + "{field} exceeds the MQTT UTF-8 string limit of {MAX_MQTT_STRING_BYTES} bytes" + )); + } + if value.contains('\0') { + return Err(format!("{field} must not contain a null character")); + } + if value.chars().any(is_mqtt_noncharacter) { + return Err(format!("{field} must not contain Unicode noncharacters")); + } + Ok(()) +} + +fn is_mqtt_noncharacter(character: char) -> bool { + matches!(character as u32, 0xfdd0..=0xfdef) + || matches!(character as u32, value if value & 0xffff == 0xfffe || value & 0xffff == 0xffff) +} + +pub fn prepare_batch_topic( + metadata: &TopicMetadata, + mode: TopicMode, + configured_topic: Option<&str>, + topic_template: Option<&str>, +) -> Result>, Error> { + let template = topic_template.unwrap_or(DEFAULT_TOPIC_TEMPLATE); + let topic = match mode { + TopicMode::Static => configured_topic + .filter(|topic| !topic.trim().is_empty()) + .map(str::to_owned) + .unwrap_or_else(|| render_template(template, metadata)), + TopicMode::Template => render_template(template, metadata), + TopicMode::Dynamic => return Ok(None), + }; + validate_topic(&topic, "resolved topic") + .map_err(|error| Error::InvalidRecordValue(error.to_string()))?; + Ok(Some(Arc::new(topic))) +} + +#[cfg(test)] +pub fn prepare_publication( + message: ConsumedMessage, + metadata: &TopicMetadata, + mode: TopicMode, + configured_topic: Option<&str>, + topic_template: Option<&str>, + fallback_topic: Option<&str>, + protocol: MqttProtocol, +) -> Result { + let batch_topic = prepare_batch_topic(metadata, mode, configured_topic, topic_template)?; + prepare_publication_with_batch_topic( + message, + metadata, + mode, + batch_topic.as_ref(), + topic_template, + fallback_topic, + PublicationOptions { + protocol, + qos: 1, + retain: false, + }, + ) +} + +pub fn prepare_publication_with_batch_topic( + message: ConsumedMessage, + metadata: &TopicMetadata, + mode: TopicMode, + batch_topic: Option<&Arc>, + topic_template: Option<&str>, + fallback_topic: Option<&str>, + options: PublicationOptions, +) -> Result { + // Routing is explicit rather than a precedence chain: static and template + // modes treat the Iggy message as opaque, while dynamic mode opts into + // source-topic/header and MQTT 5 envelope interpretation. + let template = topic_template.unwrap_or(DEFAULT_TOPIC_TEMPLATE); + let fallback = fallback_topic.unwrap_or(template); + let offset = message.offset; + match mode { + TopicMode::Static | TopicMode::Template => { + let topic = batch_topic.ok_or_else(|| { + Error::InvalidConfigValue("missing static or template batch topic".to_string()) + })?; + let mut publication = PreparedPublication { + offset, + topic: PublicationTopic::Shared(Arc::clone(topic)), + payload: message.payload.try_into_vec()?.into(), + packet_size: 0, + properties: None, + }; + publication.packet_size = driver::publication_packet_size( + &publication, + options.protocol, + options.qos, + options.retain, + ); + Ok(publication) + } + TopicMode::Dynamic => { + let (topic, payload, properties) = + prepare_dynamic(message, metadata, fallback, options.protocol)?; + validate_topic(&topic, "resolved topic") + .map_err(|error| Error::InvalidRecordValue(error.to_string()))?; + let mut publication = PreparedPublication { + offset, + topic: PublicationTopic::Owned(topic), + payload, + packet_size: 0, + properties, + }; + publication.packet_size = driver::publication_packet_size( + &publication, + options.protocol, + options.qos, + options.retain, + ); + Ok(publication) + } + } +} + +fn prepare_dynamic( + message: ConsumedMessage, + metadata: &TopicMetadata, + fallback: &str, + protocol: MqttProtocol, +) -> Result<(String, Bytes, Option), Error> { + // Raw messages can carry only the source topic header. JSON messages may + // additionally contain the versioned MQTT 5 envelope used by mqtt_source. + let ConsumedMessage { + headers, + timestamp, + payload, + .. + } = message; + match payload { + payload @ Payload::Raw(_) => { + let topic = headers + .as_ref() + .and_then(|headers| { + HeaderKey::try_from(MQTT_TOPIC_HEADER) + .ok() + .and_then(|key| headers.get(&key)) + }) + .map(|value| value.to_string_value()) + .filter(|topic| !topic.trim().is_empty()) + .unwrap_or_else(|| render_template(fallback, metadata)); + Ok((topic, payload.try_into_vec()?.into(), None)) + } + Payload::Json(value) + if matches!( + &value, + simd_json::OwnedValue::Object(object) + if object.contains_key("version") && object.contains_key("payload_base64") + ) => + { + let json_value = iggy_connector_sdk::convert::owned_value_into_serde_json(value); + let envelope = serde_json::from_value::(json_value).map_err(|error| { + Error::InvalidRecordValue(format!("invalid MQTT envelope: {error}")) + })?; + // A recognized envelope must not be silently downgraded to MQTT + // 3.1.1, because that would discard its publish properties. + if protocol == MqttProtocol::Mqtt311 { + return Err(Error::InvalidRecordValue( + "MQTT envelopes require protocol = mqtt5".to_string(), + )); + } + if envelope.version != 1 { + return Err(Error::InvalidRecordValue(format!( + "unsupported MQTT envelope version: {}", + envelope.version + ))); + } + let topic = envelope + .topic + .filter(|topic| !topic.trim().is_empty()) + .unwrap_or_else(|| render_template(fallback, metadata)); + let payload = + Bytes::from(STANDARD.decode(envelope.payload_base64).map_err(|error| { + Error::InvalidRecordValue(format!( + "invalid MQTT envelope payload_base64: {error}" + )) + })?); + let properties = envelope + .properties + .map(|properties| prepare_publish_properties(properties, timestamp)) + .transpose()?; + Ok((topic, payload, properties)) + } + payload => Ok(( + render_template(fallback, metadata), + payload.try_into_vec()?.into(), + None, + )), + } +} + +fn prepare_publish_properties( + properties: MqttEnvelopeProperties, + message_timestamp: u64, +) -> Result { + if let Some(value) = properties.payload_format_indicator + && !matches!(value, 0 | 1) + { + return Err(Error::InvalidRecordValue( + "MQTT envelope payload_format_indicator must be 0 or 1".to_string(), + )); + } + if !properties.subscription_identifiers.is_empty() { + return Err(Error::InvalidRecordValue( + "MQTT envelope subscription_identifiers are not valid for outbound publications" + .to_string(), + )); + } + if let Some(response_topic) = properties.response_topic.as_deref() { + validate_topic_value(response_topic, "MQTT envelope response_topic") + .map_err(Error::InvalidRecordValue)?; + } + if let Some(content_type) = properties.content_type.as_deref() { + validate_mqtt_string(content_type, "MQTT envelope content_type") + .map_err(Error::InvalidRecordValue)?; + } + for property in &properties.user_properties { + validate_mqtt_string(&property.key, "MQTT envelope user property key") + .map_err(Error::InvalidRecordValue)?; + validate_mqtt_string(&property.value, "MQTT envelope user property value") + .map_err(Error::InvalidRecordValue)?; + } + + let correlation_data = properties + .correlation_data_base64 + .as_deref() + .map(|value| { + STANDARD.decode(value).map_err(|error| { + Error::InvalidRecordValue(format!( + "invalid MQTT envelope correlation_data_base64: {error}" + )) + }) + }) + .transpose()?; + if correlation_data + .as_ref() + .is_some_and(|value| value.len() > MAX_MQTT_STRING_BYTES) + { + return Err(Error::InvalidRecordValue(format!( + "MQTT envelope correlation_data exceeds the MQTT binary data limit of {MAX_MQTT_STRING_BYTES} bytes" + ))); + } + let property_length = mqtt_properties_length(&properties, correlation_data.as_deref()); + if property_length > MAX_MQTT_PROPERTIES_BYTES { + return Err(Error::InvalidRecordValue(format!( + "MQTT envelope properties exceed the MQTT limit of {MAX_MQTT_PROPERTIES_BYTES} bytes" + ))); + } + + Ok(PublishProperties { + payload_format_indicator: properties.payload_format_indicator, + message_expiry_interval: match properties.message_expiry_interval { + Some(message_expiry_interval) => remaining_message_expiry_interval( + Some(message_expiry_interval), + message_timestamp, + IggyTimestamp::now().as_micros(), + )?, + None => None, + }, + // Topic aliases are scoped to the MQTT connection that created them. + // The sink creates a different connection, so it must publish the + // complete topic instead of forwarding the old alias. + topic_alias: None, + response_topic: properties.response_topic, + correlation_data: correlation_data.map(Into::into), + user_properties: properties + .user_properties + .into_iter() + .map(|property| (property.key, property.value)) + .collect(), + subscription_identifiers: properties.subscription_identifiers, + content_type: properties.content_type, + }) +} + +fn remaining_message_expiry_interval( + message_expiry_interval: Option, + message_timestamp: u64, + now_timestamp: u64, +) -> Result, Error> { + let Some(message_expiry_interval) = message_expiry_interval else { + return Ok(None); + }; + + let elapsed_micros = now_timestamp.saturating_sub(message_timestamp); + let elapsed_seconds = elapsed_micros / MICROS_PER_SECOND + + u64::from(!elapsed_micros.is_multiple_of(MICROS_PER_SECOND)); + let remaining = u64::from(message_expiry_interval).saturating_sub(elapsed_seconds); + if remaining == 0 { + return Err(Error::InvalidRecordValue( + "MQTT message expired before sink publication".to_string(), + )); + } + + Ok(Some(remaining as u32)) +} + +fn mqtt_properties_length( + properties: &MqttEnvelopeProperties, + correlation_data: Option<&[u8]>, +) -> usize { + let mut length = 0; + if properties.payload_format_indicator.is_some() { + length += 2; + } + if properties.message_expiry_interval.is_some() { + length += 5; + } + if let Some(response_topic) = &properties.response_topic { + length += 3 + response_topic.len(); + } + if let Some(correlation_data) = correlation_data { + length += 3 + correlation_data.len(); + } + for property in &properties.user_properties { + length += 5 + property.key.len() + property.value.len(); + } + if let Some(content_type) = &properties.content_type { + length += 3 + content_type.len(); + } + length +} + +fn render_template(template: &str, metadata: &TopicMetadata) -> String { + template + .replace("{stream}", &metadata.stream) + .replace("{topic}", &metadata.topic) +} + +#[cfg(test)] +mod tests { + use super::*; + use iggy_common::{HeaderKey, HeaderValue}; + use std::collections::BTreeMap; + + fn metadata() -> TopicMetadata { + TopicMetadata { + stream: "devices".into(), + topic: "telemetry".into(), + } + } + + fn message(payload: Payload) -> ConsumedMessage { + ConsumedMessage { + id: 1, + offset: 1, + checksum: 1, + timestamp: 1, + origin_timestamp: 1, + headers: None, + payload, + } + } + + #[test] + fn template_mode_renders_stream_and_topic() { + let publication = prepare_publication( + message(Payload::Raw(vec![1, 2])), + &metadata(), + TopicMode::Template, + None, + Some("iggy/{stream}/{topic}"), + None, + MqttProtocol::Mqtt5, + ) + .unwrap(); + assert_eq!(publication.topic.as_str(), "iggy/devices/telemetry"); + } + + #[test] + fn static_raw_payload_should_move_storage_into_publication() { + let payload = vec![1, 2, 3]; + let payload_ptr = payload.as_ptr(); + + let publication = prepare_publication( + message(Payload::Raw(payload)), + &metadata(), + TopicMode::Static, + Some("devices/output"), + None, + None, + MqttProtocol::Mqtt5, + ) + .unwrap(); + + assert_eq!(publication.payload.as_ptr(), payload_ptr); + assert_eq!(publication.payload.as_ref(), [1, 2, 3]); + } + + #[test] + fn dynamic_raw_uses_header() { + let key = HeaderKey::try_from(MQTT_TOPIC_HEADER).unwrap(); + let mut headers = BTreeMap::new(); + headers.insert(key, HeaderValue::try_from("devices/living-room").unwrap()); + let mut message = message(Payload::Raw(vec![1])); + message.headers = Some(headers); + let publication = prepare_publication( + message, + &metadata(), + TopicMode::Dynamic, + None, + None, + Some("fallback/{stream}/{topic}"), + MqttProtocol::Mqtt5, + ) + .unwrap(); + assert_eq!(publication.topic.as_str(), "devices/living-room"); + } + + #[test] + fn given_reserved_dynamic_header_when_preparing_should_return_record_error() { + let key = HeaderKey::try_from(MQTT_TOPIC_HEADER).unwrap(); + let mut headers = BTreeMap::new(); + headers.insert(key, HeaderValue::try_from("$SYS/broker/uptime").unwrap()); + let mut message = message(Payload::Raw(vec![1])); + message.headers = Some(headers); + + let error = prepare_publication( + message, + &metadata(), + TopicMode::Dynamic, + None, + None, + Some("fallback/{stream}/{topic}"), + MqttProtocol::Mqtt5, + ) + .unwrap_err(); + + assert!( + matches!(error, Error::InvalidRecordValue(reason) if reason.contains("must not start")) + ); + } + + #[test] + fn given_wildcard_in_template_value_when_preparing_batch_should_return_record_error() { + let metadata = TopicMetadata { + stream: "sensors+".into(), + topic: "telemetry".into(), + }; + + let error = prepare_batch_topic( + &metadata, + TopicMode::Template, + None, + Some("{stream}/{topic}"), + ) + .unwrap_err(); + + assert!(matches!(error, Error::InvalidRecordValue(reason) if reason.contains("wildcards"))); + } + + #[test] + fn dynamic_json_envelope_decodes_payload_and_properties() { + let payload = serde_json::json!({ + "version": 1, + "topic": "devices/test/telemetry", + "properties": { + "topic_alias": 7, + "content_type": "application/json", + "user_properties": [{"key": "device-type", "value": "sensor"}] + }, + "payload_base64": "AQID" + }); + let mut bytes = serde_json::to_vec(&payload).unwrap(); + let json = simd_json::to_owned_value(&mut bytes).unwrap(); + let publication = prepare_publication( + message(Payload::Json(json)), + &metadata(), + TopicMode::Dynamic, + None, + None, + None, + MqttProtocol::Mqtt5, + ) + .unwrap(); + assert_eq!(publication.topic.as_str(), "devices/test/telemetry"); + assert_eq!(publication.payload.as_ref(), [1, 2, 3]); + assert_eq!( + publication.properties.unwrap().user_properties[0], + ("device-type".into(), "sensor".into()) + ); + } + + #[test] + fn given_topic_alias_when_preparing_envelope_should_strip_alias() { + let payload = serde_json::json!({ + "version": 1, + "payload_base64": "AQID", + "properties": {"topic_alias": 7} + }); + let mut bytes = serde_json::to_vec(&payload).unwrap(); + let json = simd_json::to_owned_value(&mut bytes).unwrap(); + + let publication = prepare_publication( + message(Payload::Json(json)), + &metadata(), + TopicMode::Dynamic, + None, + None, + None, + MqttProtocol::Mqtt5, + ) + .unwrap(); + + assert_eq!(publication.properties.unwrap().topic_alias, None); + } + + #[test] + fn given_unexpired_message_when_calculating_expiry_should_return_remaining_interval() { + let now = 100 * MICROS_PER_SECOND; + let timestamp = 60 * MICROS_PER_SECOND; + + let remaining = remaining_message_expiry_interval(Some(60), timestamp, now).unwrap(); + + assert_eq!(remaining, Some(20)); + } + + #[test] + fn given_expired_message_when_calculating_expiry_should_return_record_error() { + let now = 100 * MICROS_PER_SECOND; + let timestamp = 39 * MICROS_PER_SECOND; + + let error = remaining_message_expiry_interval(Some(60), timestamp, now).unwrap_err(); + + assert!(matches!(error, Error::InvalidRecordValue(reason) if reason.contains("expired"))); + } + + #[test] + fn given_expired_envelope_when_preparing_should_return_record_error() { + let payload = serde_json::json!({ + "version": 1, + "payload_base64": "AQID", + "properties": {"message_expiry_interval": 1} + }); + let mut bytes = serde_json::to_vec(&payload).unwrap(); + let json = simd_json::to_owned_value(&mut bytes).unwrap(); + let mut message = message(Payload::Json(json)); + message.timestamp = 1; + + let error = prepare_publication( + message, + &metadata(), + TopicMode::Dynamic, + None, + None, + None, + MqttProtocol::Mqtt5, + ) + .unwrap_err(); + + assert!(matches!(error, Error::InvalidRecordValue(reason) if reason.contains("expired"))); + } + + #[test] + fn dynamic_json_envelope_should_move_topic_storage_into_publication() { + let payload = serde_json::json!({ + "version": 1, + "topic": "devices/test/telemetry", + "payload_base64": "AQID" + }); + let mut bytes = serde_json::to_vec(&payload).unwrap(); + let json = simd_json::to_owned_value(&mut bytes).unwrap(); + let topic_ptr = match &json { + simd_json::OwnedValue::Object(object) => match object.get("topic") { + Some(simd_json::OwnedValue::String(topic)) => topic.as_ptr(), + _ => panic!("envelope topic must be a string"), + }, + _ => panic!("envelope must be an object"), + }; + + let publication = prepare_publication( + message(Payload::Json(json)), + &metadata(), + TopicMode::Dynamic, + None, + None, + None, + MqttProtocol::Mqtt5, + ) + .unwrap(); + + assert!(matches!(&publication.topic, PublicationTopic::Owned(_))); + assert_eq!(publication.topic.as_str().as_ptr(), topic_ptr); + } + + #[test] + fn dynamic_json_envelope_with_invalid_payload_should_return_record_error() { + let payload = serde_json::json!({ + "version": 1, + "payload_base64": "not base64" + }); + let mut bytes = serde_json::to_vec(&payload).unwrap(); + let json = simd_json::to_owned_value(&mut bytes).unwrap(); + + let error = prepare_publication( + message(Payload::Json(json)), + &metadata(), + TopicMode::Dynamic, + None, + None, + None, + MqttProtocol::Mqtt5, + ) + .unwrap_err(); + + assert!(matches!(error, Error::InvalidRecordValue(_))); + } + + #[test] + fn dynamic_json_envelope_with_invalid_field_type_should_return_record_error() { + let payload = serde_json::json!({ + "version": "one", + "payload_base64": "AQID" + }); + let mut bytes = serde_json::to_vec(&payload).unwrap(); + let json = simd_json::to_owned_value(&mut bytes).unwrap(); + + let error = prepare_publication( + message(Payload::Json(json)), + &metadata(), + TopicMode::Dynamic, + None, + None, + None, + MqttProtocol::Mqtt5, + ) + .unwrap_err(); + + assert!( + matches!(error, Error::InvalidRecordValue(reason) if reason.contains("invalid MQTT envelope")) + ); + } + + #[test] + fn dynamic_json_without_envelope_markers_should_use_fallback_topic() { + let payload = serde_json::json!({"topic": "ordinary-json", "value": 1}); + let mut bytes = serde_json::to_vec(&payload).unwrap(); + let json = simd_json::to_owned_value(&mut bytes).unwrap(); + + let publication = prepare_publication( + message(Payload::Json(json)), + &metadata(), + TopicMode::Dynamic, + None, + None, + Some("fallback/{stream}/{topic}"), + MqttProtocol::Mqtt5, + ) + .unwrap(); + + assert_eq!(publication.topic.as_str(), "fallback/devices/telemetry"); + assert!(publication.properties.is_none()); + } + + #[test] + fn dynamic_json_with_only_version_should_use_fallback_topic() { + let payload = serde_json::json!({"version": 1, "value": "ordinary-json"}); + let payload_bytes = serde_json::to_vec(&payload).unwrap(); + let mut bytes = payload_bytes.clone(); + let json = simd_json::to_owned_value(&mut bytes).unwrap(); + + let publication = prepare_publication( + message(Payload::Json(json)), + &metadata(), + TopicMode::Dynamic, + None, + None, + Some("fallback/{stream}/{topic}"), + MqttProtocol::Mqtt5, + ) + .unwrap(); + + assert_eq!(publication.topic.as_str(), "fallback/devices/telemetry"); + assert_eq!(publication.payload.as_ref(), payload_bytes); + assert!(publication.properties.is_none()); + } + + #[test] + fn dynamic_json_with_only_payload_base64_should_use_fallback_topic() { + let payload = serde_json::json!({"payload_base64": "AQID", "value": "ordinary-json"}); + let payload_bytes = serde_json::to_vec(&payload).unwrap(); + let mut bytes = payload_bytes.clone(); + let json = simd_json::to_owned_value(&mut bytes).unwrap(); + + let publication = prepare_publication( + message(Payload::Json(json)), + &metadata(), + TopicMode::Dynamic, + None, + None, + Some("fallback/{stream}/{topic}"), + MqttProtocol::Mqtt5, + ) + .unwrap(); + + assert_eq!(publication.topic.as_str(), "fallback/devices/telemetry"); + assert_eq!(publication.payload.as_ref(), payload_bytes); + assert!(publication.properties.is_none()); + } + + #[test] + fn topic_with_null_character_should_return_configuration_error() { + let error = validate_topic("devices\0telemetry", "topic").unwrap_err(); + + assert!(matches!(error, Error::InvalidConfigValue(message) if message.contains("null"))); + } + + #[test] + fn topic_starting_with_dollar_should_return_configuration_error() { + let error = validate_topic("$SYS/broker/uptime", "topic").unwrap_err(); + + assert!( + matches!(error, Error::InvalidConfigValue(message) if message.contains("must not start")) + ); + } + + #[test] + fn topic_template_starting_with_dollar_should_return_configuration_error() { + let error = validate_topic_template("$SYS/{stream}", "topic_template").unwrap_err(); + + assert!( + matches!(error, Error::InvalidConfigValue(message) if message.contains("must not start")) + ); + } + + #[test] + fn topic_exceeding_mqtt_string_limit_should_return_configuration_error() { + let topic = "a".repeat(MAX_MQTT_STRING_BYTES + 1); + let error = validate_topic(&topic, "topic").unwrap_err(); + + assert!(matches!(error, Error::InvalidConfigValue(message) if message.contains("limit"))); + } + + #[test] + fn dynamic_json_envelope_with_invalid_properties_should_return_record_error() { + for properties in [ + serde_json::json!({"payload_format_indicator": 2}), + serde_json::json!({"response_topic": "devices/#"}), + serde_json::json!({"subscription_identifiers": [1]}), + ] { + let payload = serde_json::json!({ + "version": 1, + "payload_base64": "AQID", + "properties": properties, + }); + let mut bytes = serde_json::to_vec(&payload).unwrap(); + let json = simd_json::to_owned_value(&mut bytes).unwrap(); + + let error = prepare_publication( + message(Payload::Json(json)), + &metadata(), + TopicMode::Dynamic, + None, + None, + None, + MqttProtocol::Mqtt5, + ) + .unwrap_err(); + + assert!(matches!(error, Error::InvalidRecordValue(_))); + } + } + + #[test] + fn dynamic_json_envelope_with_system_response_topic_should_prepare_publication() { + let payload = serde_json::json!({ + "version": 1, + "payload_base64": "AQID", + "properties": {"response_topic": "$reply/device-42"}, + }); + let mut bytes = serde_json::to_vec(&payload).unwrap(); + let json = simd_json::to_owned_value(&mut bytes).unwrap(); + + let publication = prepare_publication( + message(Payload::Json(json)), + &metadata(), + TopicMode::Dynamic, + None, + None, + None, + MqttProtocol::Mqtt5, + ) + .unwrap(); + + assert_eq!( + publication + .properties + .expect("response topic must produce MQTT 5 properties") + .response_topic + .as_deref(), + Some("$reply/device-42") + ); + } +} diff --git a/core/integration/tests/connectors/fixtures/mod.rs b/core/integration/tests/connectors/fixtures/mod.rs index 3912d0a61b..66f0a92b9c 100644 --- a/core/integration/tests/connectors/fixtures/mod.rs +++ b/core/integration/tests/connectors/fixtures/mod.rs @@ -87,7 +87,11 @@ pub use mongodb::{ pub use mqtt::{ Mqtt5InvalidCredentialsFixture, Mqtt5MetadataQos1Fixture, Mqtt5PendingBatchFixture, Mqtt5Qos0Fixture, Mqtt5Qos1Fixture, Mqtt5Qos2Fixture, Mqtt311InvalidCredentialsFixture, - Mqtt311Qos0Fixture, Mqtt311Qos1Fixture, Mqtt311Qos2Fixture, + Mqtt311Qos0Fixture, Mqtt311Qos1Fixture, Mqtt311Qos2Fixture, Mqtt311SinkQosOneFixture, + Mqtt311SinkQosTwoFixture, Mqtt311SinkQosZeroFixture, MqttSinkDynamicTopicFixture, + MqttSinkEnvelopeFixture, MqttSinkFixture, MqttSinkIdleFixture, + MqttSinkInvalidCredentialsFixture, MqttSinkPipelinedFixture, MqttSinkQosTwoFixture, + MqttSinkQosZeroFixture, MqttSinkRawSchemaFixture, MqttSinkTemplateTopicFixture, }; pub use postgres::{ POSTGRES_LARGE_BATCH_SIZE, PostgresOps, PostgresSinkByteaFixture, PostgresSinkFixture, diff --git a/core/integration/tests/connectors/fixtures/mqtt/container.rs b/core/integration/tests/connectors/fixtures/mqtt/container.rs index ee496e64f0..77e09f043e 100644 --- a/core/integration/tests/connectors/fixtures/mqtt/container.rs +++ b/core/integration/tests/connectors/fixtures/mqtt/container.rs @@ -32,6 +32,7 @@ const EMQX_MQTT_PORT: u16 = 1883; const EMQX_DASHBOARD_PORT: u16 = 18083; const EMQX_START_ATTEMPTS: usize = 240; const EMQX_START_INTERVAL: Duration = Duration::from_millis(250); +const EMQX_RESPONSE_BODY_LIMIT: usize = 1024; pub(super) const MQTT_USERNAME: &str = "iggy-mqtt-user"; pub(super) const MQTT_PASSWORD: &str = "iggy-mqtt-password"; pub(super) const INVALID_MQTT_PASSWORD: &str = "invalid-mqtt-password"; @@ -69,7 +70,7 @@ impl MqttBrokerContainer { pub(super) async fn start() -> Result { // EMQX is configured through its environment so the fixture exercises // the same password-based authentication used by the source connector. - let container = GenericImage::new(EMQX_IMAGE, EMQX_TAG) + let image = GenericImage::new(EMQX_IMAGE, EMQX_TAG) .with_wait_for(WaitFor::Nothing) .with_exposed_port(EMQX_MQTT_PORT.tcp()) .with_exposed_port(EMQX_DASHBOARD_PORT.tcp()) @@ -81,7 +82,8 @@ impl MqttBrokerContainer { .with_env_var("EMQX_AUTHENTICATION__1__MECHANISM", "password_based") .with_env_var("EMQX_AUTHENTICATION__1__BACKEND", "built_in_database") .with_env_var("EMQX_AUTHENTICATION__1__USER_ID_TYPE", "username") - .with_container_name(fixtures::unique_container_name("emqx")) + .with_container_name(fixtures::unique_container_name("emqx")); + let container = image .start() .await .map_err(|error| TestBinaryError::FixtureSetup { @@ -170,6 +172,71 @@ impl MqttBrokerContainer { message: format!("Timed out waiting for MQTT client {client_id} after disconnection"), })? } + + pub(super) async fn deny_publish( + &self, + username: &str, + topic: &str, + ) -> Result<(), TestBinaryError> { + let token = wait_for_dashboard_token(&self.dashboard_url).await?; + let client = reqwest::Client::new(); + let source_url = format!("{}/api/v5/authorization/sources", self.dashboard_url); + let source_response = client + .post(source_url) + .bearer_auth(&token) + .json(&json!({ + "enable": true, + "max_rules": 100, + "type": "built_in_database" + })) + .send() + .await + .map_err(|error| TestBinaryError::FixtureSetup { + fixture_type: "MqttBrokerContainer".to_string(), + message: format!("Failed to configure EMQX authorization: {error}"), + })?; + if !source_response.status().is_success() && source_response.status().as_u16() != 409 { + return Err(TestBinaryError::FixtureSetup { + fixture_type: "MqttBrokerContainer".to_string(), + message: format!( + "Failed to enable EMQX built-in authorization: HTTP {}", + source_response.status() + ), + }); + } + + let rules_url = format!( + "{}/api/v5/authorization/sources/built_in_database/rules/users", + self.dashboard_url + ); + let response = client + .post(rules_url) + .bearer_auth(token) + .json(&json!([{ + "username": username, + "rules": [{ + "action": "publish", + "permission": "deny", + "topic": topic + }] + }])) + .send() + .await + .map_err(|error| TestBinaryError::FixtureSetup { + fixture_type: "MqttBrokerContainer".to_string(), + message: format!("Failed to configure EMQX publish rejection: {error}"), + })?; + if !response.status().is_success() { + return Err(TestBinaryError::FixtureSetup { + fixture_type: "MqttBrokerContainer".to_string(), + message: format!( + "Failed to configure EMQX publish rejection: HTTP {}", + response.status() + ), + }); + } + Ok(()) + } } async fn wait_for_dashboard_token(dashboard_url: &str) -> Result { @@ -206,38 +273,86 @@ async fn wait_for_dashboard_token(dashboard_url: &str) -> Result Result<(), TestBinaryError> { let client = reqwest::Client::new(); - let users_url = + let user_url = format!("{dashboard_url}/api/v5/authentication/password_based%3Abuilt_in_database/users"); - let mut last_failure = String::from("no attempt was made"); - for _ in 0..EMQX_START_ATTEMPTS { - match client - .post(&users_url) - .bearer_auth(token) - .json(&json!({ - "user_id": MQTT_USERNAME, - "password": MQTT_PASSWORD, - })) - .send() - .await - { - Ok(response) - if response.status().is_success() || response.status() == StatusCode::CONFLICT => - { - return Ok(()); + + timeout( + EMQX_START_INTERVAL * (EMQX_START_ATTEMPTS as u32 + 1), + async { + let mut last_failure = "EMQX did not return a response".to_string(); + for _ in 0..EMQX_START_ATTEMPTS { + match client + .post(&user_url) + .bearer_auth(token) + .json(&json!({ + "user_id": MQTT_USERNAME, + "password": MQTT_PASSWORD, + })) + .send() + .await + { + Ok(response) + if response.status().is_success() + || response.status() == StatusCode::CONFLICT => + { + return Ok(()); + } + Ok(response) => { + let status = response.status(); + let body = response.text().await.unwrap_or_else(|error| { + format!("failed to read EMQX response body: {error}") + }); + let message = format!( + "EMQX did not accept the MQTT test user: HTTP {status}: {}", + truncate_response_body(&body) + ); + + let authenticator_is_initializing = + status == StatusCode::BAD_REQUEST && body.contains("not_initialized"); + if status.is_client_error() && !authenticator_is_initializing { + return Err(TestBinaryError::FixtureSetup { + fixture_type: "MqttBrokerContainer".to_string(), + message, + }); + } + + last_failure = message; + } + Err(error) => { + last_failure = format!("EMQX did not accept the MQTT test user: {error}"); + } + } + + sleep(EMQX_START_INTERVAL).await; } - Ok(response) => last_failure = format!("HTTP {}", response.status()), - Err(error) => last_failure = error.to_string(), - } - sleep(EMQX_START_INTERVAL).await; - } - Err(TestBinaryError::FixtureSetup { + + Err(TestBinaryError::FixtureSetup { + fixture_type: "MqttBrokerContainer".to_string(), + message: last_failure, + }) + }, + ) + .await + .map_err(|_| TestBinaryError::FixtureSetup { fixture_type: "MqttBrokerContainer".to_string(), - message: format!("Failed to create EMQX MQTT test user: {last_failure}"), - }) + message: format!("Timed out creating the MQTT test user at {dashboard_url}"), + })??; + + Ok(()) +} + +fn truncate_response_body(body: &str) -> String { + let mut characters = body.chars(); + let truncated: String = characters.by_ref().take(EMQX_RESPONSE_BODY_LIMIT).collect(); + if characters.next().is_some() { + format!("{truncated}...") + } else { + truncated + } } async fn wait_for_mqtt( diff --git a/core/integration/tests/connectors/fixtures/mqtt/mod.rs b/core/integration/tests/connectors/fixtures/mqtt/mod.rs index 770fc204d3..1bd2c4483c 100644 --- a/core/integration/tests/connectors/fixtures/mqtt/mod.rs +++ b/core/integration/tests/connectors/fixtures/mqtt/mod.rs @@ -22,6 +22,7 @@ use std::collections::HashMap; mod container; mod publisher; +mod sink; use container::{ DEFAULT_IGGY_TOPIC, DEFAULT_TEST_STREAM, ENV_SOURCE_BATCH_SIZE, ENV_SOURCE_BATCH_TIMEOUT, @@ -31,6 +32,13 @@ use container::{ MqttBrokerContainer, }; +pub use sink::{ + Mqtt311SinkQosOneFixture, Mqtt311SinkQosTwoFixture, Mqtt311SinkQosZeroFixture, + MqttSinkDynamicTopicFixture, MqttSinkEnvelopeFixture, MqttSinkFixture, MqttSinkIdleFixture, + MqttSinkInvalidCredentialsFixture, MqttSinkPipelinedFixture, MqttSinkQosTwoFixture, + MqttSinkQosZeroFixture, MqttSinkRawSchemaFixture, MqttSinkTemplateTopicFixture, +}; + const SOURCE_CLIENT_ID: &str = "iggy-mqtt-integration-source"; struct MqttFixture { diff --git a/core/integration/tests/connectors/fixtures/mqtt/sink.rs b/core/integration/tests/connectors/fixtures/mqtt/sink.rs new file mode 100644 index 0000000000..8e1ec275b1 --- /dev/null +++ b/core/integration/tests/connectors/fixtures/mqtt/sink.rs @@ -0,0 +1,869 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::container::{ + DEFAULT_IGGY_TOPIC, DEFAULT_TEST_STREAM, INVALID_MQTT_PASSWORD, MQTT_PASSWORD, MQTT_USERNAME, + MqttBrokerContainer, +}; +use async_trait::async_trait; +use iggy_connector_sdk::Schema; +use integration::harness::{TestBinaryError, TestFixture}; +use rumqttc_v4::{ + AsyncClient as Mqtt311Client, Broker as Mqtt311Broker, Event as Mqtt311Event, + EventLoop as Mqtt311EventLoop, Incoming as Mqtt311Incoming, MqttOptions as Mqtt311Options, + mqttbytes::QoS as Mqtt311Qos, +}; +use rumqttc_v5::{ + AsyncClient as Mqtt5Client, Broker as Mqtt5Broker, Event as Mqtt5Event, + EventLoop as Mqtt5EventLoop, Incoming as Mqtt5Incoming, MqttOptions as Mqtt5Options, + mqttbytes::{QoS as Mqtt5Qos, v5::PublishProperties}, +}; +use std::{ + collections::{HashMap, HashSet}, + time::Duration, +}; +use tokio::{sync::Mutex, time::timeout}; +use url::Url; + +const DEFAULT_SINK_TOPIC: &str = "iggy.messages"; +const SINK_CLIENT_ID: &str = "iggy-mqtt-integration-sink"; +const SUBSCRIBER_CLIENT_ID: &str = "iggy-mqtt-integration-sink-subscriber"; +const RECEIVE_TIMEOUT: Duration = Duration::from_secs(10); +const MAXIMUM_TEST_PACKET_SIZE: usize = 1024 * 1024; + +const ENV_SINK_BROKER_URL: &str = "IGGY_CONNECTORS_SINK_MQTT_PLUGIN_CONFIG_BROKER_URL"; +const ENV_SINK_CLIENT_ID: &str = "IGGY_CONNECTORS_SINK_MQTT_PLUGIN_CONFIG_CLIENT_ID"; +const ENV_SINK_KEEP_ALIVE: &str = "IGGY_CONNECTORS_SINK_MQTT_PLUGIN_CONFIG_KEEP_ALIVE"; +const ENV_SINK_MAX_IN_FLIGHT: &str = "IGGY_CONNECTORS_SINK_MQTT_PLUGIN_CONFIG_MAX_IN_FLIGHT"; +const ENV_SINK_MAX_RETRIES: &str = "IGGY_CONNECTORS_SINK_MQTT_PLUGIN_CONFIG_MAX_RETRIES"; +const ENV_SINK_PASSWORD: &str = "IGGY_CONNECTORS_SINK_MQTT_PLUGIN_CONFIG_PASSWORD"; +const ENV_SINK_PROTOCOL: &str = "IGGY_CONNECTORS_SINK_MQTT_PLUGIN_CONFIG_PROTOCOL"; +const ENV_SINK_QOS: &str = "IGGY_CONNECTORS_SINK_MQTT_PLUGIN_CONFIG_QOS"; +const ENV_SINK_TOPIC: &str = "IGGY_CONNECTORS_SINK_MQTT_PLUGIN_CONFIG_TOPIC"; +const ENV_SINK_TOPIC_MODE: &str = "IGGY_CONNECTORS_SINK_MQTT_PLUGIN_CONFIG_TOPIC_MODE"; +const ENV_SINK_TOPIC_TEMPLATE: &str = "IGGY_CONNECTORS_SINK_MQTT_PLUGIN_CONFIG_TOPIC_TEMPLATE"; +const ENV_SINK_FALLBACK_TOPIC: &str = "IGGY_CONNECTORS_SINK_MQTT_PLUGIN_CONFIG_FALLBACK_TOPIC"; +const ENV_SINK_USERNAME: &str = "IGGY_CONNECTORS_SINK_MQTT_PLUGIN_CONFIG_USERNAME"; +const ENV_SINK_PATH: &str = "IGGY_CONNECTORS_SINK_MQTT_PATH"; +const ENV_SINK_STREAMS_0_CONSUMER_GROUP: &str = + "IGGY_CONNECTORS_SINK_MQTT_STREAMS_0_CONSUMER_GROUP"; +const ENV_SINK_STREAMS_0_SCHEMA: &str = "IGGY_CONNECTORS_SINK_MQTT_STREAMS_0_SCHEMA"; +const ENV_SINK_STREAMS_0_STREAM: &str = "IGGY_CONNECTORS_SINK_MQTT_STREAMS_0_STREAM"; +const ENV_SINK_STREAMS_0_TOPICS: &str = "IGGY_CONNECTORS_SINK_MQTT_STREAMS_0_TOPICS"; + +pub struct MqttSinkFixture { + broker: MqttBrokerContainer, + subscriber: Mutex, + protocol: MqttProtocol, + qos: u8, + schema: Schema, + max_in_flight: usize, + keep_alive: Duration, + max_retries: usize, + password: String, + topic: Option, + topic_mode: MqttTopicMode, + topic_template: Option, + fallback_topic: Option, +} + +#[derive(Clone, Copy)] +enum MqttProtocol { + Mqtt311, + Mqtt5, +} + +#[derive(Clone, Copy)] +enum MqttTopicMode { + Static, + Template, + Dynamic, +} + +struct MqttRoutingConfig { + topic_mode: MqttTopicMode, + topic: Option, + topic_template: Option, + fallback_topic: Option, + subscription_filters: Vec, +} + +impl MqttTopicMode { + fn as_str(self) -> &'static str { + match self { + Self::Static => "static", + Self::Template => "template", + Self::Dynamic => "dynamic", + } + } +} + +pub struct ReceivedMqttPublication { + pub topic: String, + pub payload: Vec, + pub qos: u8, + pub retain: bool, + pub properties: Option, +} + +impl MqttSinkFixture { + async fn start( + protocol: MqttProtocol, + qos: u8, + schema: Schema, + max_in_flight: usize, + ) -> Result { + Self::start_with_routing( + protocol, + qos, + schema, + max_in_flight, + MqttRoutingConfig { + topic_mode: MqttTopicMode::Static, + topic: Some(DEFAULT_SINK_TOPIC.to_string()), + topic_template: None, + fallback_topic: None, + subscription_filters: vec![DEFAULT_SINK_TOPIC.to_string()], + }, + ) + .await + } + + async fn start_with_routing( + protocol: MqttProtocol, + qos: u8, + schema: Schema, + max_in_flight: usize, + routing: MqttRoutingConfig, + ) -> Result { + let MqttRoutingConfig { + topic_mode, + topic, + topic_template, + fallback_topic, + subscription_filters, + } = routing; + let broker = MqttBrokerContainer::start().await?; + let subscriber = + MqttSubscriber::connect(&broker.broker_url, protocol, &subscription_filters).await?; + Ok(Self { + broker, + subscriber: Mutex::new(subscriber), + protocol, + qos, + schema, + max_in_flight, + keep_alive: Duration::from_secs(30), + max_retries: 5, + password: MQTT_PASSWORD.to_string(), + topic, + topic_mode, + topic_template, + fallback_topic, + }) + } + + async fn start_with_keep_alive( + protocol: MqttProtocol, + qos: u8, + schema: Schema, + keep_alive: Duration, + max_retries: usize, + ) -> Result { + let mut fixture = Self::start(protocol, qos, schema, 1).await?; + fixture.keep_alive = keep_alive; + fixture.max_retries = max_retries; + Ok(fixture) + } + + pub async fn receive_messages(&self, count: usize) -> Result>, TestBinaryError> { + Ok(self + .receive_publications(count) + .await? + .into_iter() + .map(|publication| publication.payload) + .collect()) + } + + pub async fn receive_publications( + &self, + count: usize, + ) -> Result, TestBinaryError> { + self.subscriber + .lock() + .await + .receive_publications(count) + .await + } + + pub async fn receive_until_payloads( + &self, + expected_payloads: &[Vec], + ) -> Result<(), TestBinaryError> { + self.subscriber + .lock() + .await + .receive_until_payloads(expected_payloads) + .await + } + + pub async fn disconnect_sink(&self) -> Result<(), TestBinaryError> { + self.broker.disconnect_client(SINK_CLIENT_ID).await + } + + pub async fn wait_for_sink_reconnect(&self) -> Result<(), TestBinaryError> { + self.broker.wait_for_client(SINK_CLIENT_ID).await + } + + pub async fn deny_sink_publications(&self) -> Result<(), TestBinaryError> { + self.broker + .deny_publish(MQTT_USERNAME, DEFAULT_SINK_TOPIC) + .await + } +} + +#[async_trait] +impl TestFixture for MqttSinkFixture { + async fn setup() -> Result { + Self::start(MqttProtocol::Mqtt5, 1, Schema::Json, 1).await + } + + fn connectors_runtime_envs(&self) -> HashMap { + HashMap::from([ + ( + ENV_SINK_BROKER_URL.to_string(), + self.broker.broker_url.clone(), + ), + (ENV_SINK_CLIENT_ID.to_string(), SINK_CLIENT_ID.to_string()), + ( + ENV_SINK_KEEP_ALIVE.to_string(), + format!("{}s", self.keep_alive.as_secs()), + ), + (ENV_SINK_PASSWORD.to_string(), self.password.clone()), + ( + ENV_SINK_PROTOCOL.to_string(), + match self.protocol { + MqttProtocol::Mqtt311 => "mqtt311", + MqttProtocol::Mqtt5 => "mqtt5", + } + .to_string(), + ), + (ENV_SINK_QOS.to_string(), self.qos.to_string()), + ( + ENV_SINK_TOPIC_MODE.to_string(), + self.topic_mode.as_str().to_string(), + ), + ( + ENV_SINK_MAX_IN_FLIGHT.to_string(), + self.max_in_flight.to_string(), + ), + ( + ENV_SINK_MAX_RETRIES.to_string(), + self.max_retries.to_string(), + ), + (ENV_SINK_USERNAME.to_string(), MQTT_USERNAME.to_string()), + ( + ENV_SINK_STREAMS_0_STREAM.to_string(), + DEFAULT_TEST_STREAM.to_string(), + ), + ( + ENV_SINK_STREAMS_0_TOPICS.to_string(), + format!("[{DEFAULT_IGGY_TOPIC}]"), + ), + ( + ENV_SINK_STREAMS_0_SCHEMA.to_string(), + self.schema.to_string(), + ), + ( + ENV_SINK_STREAMS_0_CONSUMER_GROUP.to_string(), + "mqtt_sink_test_cg".to_string(), + ), + ( + ENV_SINK_PATH.to_string(), + "../../target/debug/libiggy_connector_mqtt_sink".to_string(), + ), + ]) + .into_iter() + .chain( + self.topic + .as_ref() + .map(|topic| (ENV_SINK_TOPIC.to_string(), topic.clone())), + ) + .chain( + self.topic_template + .as_ref() + .map(|template| (ENV_SINK_TOPIC_TEMPLATE.to_string(), template.clone())), + ) + .chain( + self.fallback_topic + .as_ref() + .map(|topic| (ENV_SINK_FALLBACK_TOPIC.to_string(), topic.clone())), + ) + .collect() + } +} + +pub struct MqttSinkIdleFixture { + inner: MqttSinkFixture, +} + +impl std::ops::Deref for MqttSinkIdleFixture { + type Target = MqttSinkFixture; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +#[async_trait] +impl TestFixture for MqttSinkIdleFixture { + async fn setup() -> Result { + Ok(Self { + inner: MqttSinkFixture::start_with_keep_alive( + MqttProtocol::Mqtt5, + 1, + Schema::Json, + Duration::from_secs(5), + 0, + ) + .await?, + }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + self.inner.connectors_runtime_envs() + } +} + +pub struct MqttSinkQosTwoFixture { + inner: MqttSinkFixture, +} + +impl std::ops::Deref for MqttSinkQosTwoFixture { + type Target = MqttSinkFixture; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl std::ops::DerefMut for MqttSinkQosTwoFixture { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +#[async_trait] +impl TestFixture for MqttSinkQosTwoFixture { + async fn setup() -> Result { + Ok(Self { + inner: MqttSinkFixture::start(MqttProtocol::Mqtt5, 2, Schema::Json, 3).await?, + }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + self.inner.connectors_runtime_envs() + } +} + +pub struct MqttSinkQosZeroFixture { + inner: MqttSinkFixture, +} + +impl std::ops::Deref for MqttSinkQosZeroFixture { + type Target = MqttSinkFixture; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl std::ops::DerefMut for MqttSinkQosZeroFixture { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +#[async_trait] +impl TestFixture for MqttSinkQosZeroFixture { + async fn setup() -> Result { + Ok(Self { + inner: MqttSinkFixture::start(MqttProtocol::Mqtt5, 0, Schema::Json, 1).await?, + }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + self.inner.connectors_runtime_envs() + } +} + +pub struct MqttSinkRawSchemaFixture { + inner: MqttSinkFixture, +} + +impl std::ops::Deref for MqttSinkRawSchemaFixture { + type Target = MqttSinkFixture; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl std::ops::DerefMut for MqttSinkRawSchemaFixture { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +#[async_trait] +impl TestFixture for MqttSinkRawSchemaFixture { + async fn setup() -> Result { + Ok(Self { + inner: MqttSinkFixture::start(MqttProtocol::Mqtt5, 1, Schema::Raw, 1).await?, + }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + self.inner.connectors_runtime_envs() + } +} + +macro_rules! mqtt_sink_fixture { + ($name:ident, $protocol:expr, $qos:expr, $max_in_flight:expr) => { + pub struct $name { + inner: MqttSinkFixture, + } + + impl std::ops::Deref for $name { + type Target = MqttSinkFixture; + + fn deref(&self) -> &Self::Target { + &self.inner + } + } + + impl std::ops::DerefMut for $name { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } + } + + #[async_trait] + impl TestFixture for $name { + async fn setup() -> Result { + Ok(Self { + inner: MqttSinkFixture::start($protocol, $qos, Schema::Json, $max_in_flight) + .await?, + }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + self.inner.connectors_runtime_envs() + } + } + }; +} + +mqtt_sink_fixture!(Mqtt311SinkQosZeroFixture, MqttProtocol::Mqtt311, 0, 1); +mqtt_sink_fixture!(Mqtt311SinkQosOneFixture, MqttProtocol::Mqtt311, 1, 1); +mqtt_sink_fixture!(Mqtt311SinkQosTwoFixture, MqttProtocol::Mqtt311, 2, 1); +mqtt_sink_fixture!(MqttSinkPipelinedFixture, MqttProtocol::Mqtt5, 1, 3); + +pub struct MqttSinkInvalidCredentialsFixture { + inner: MqttSinkFixture, +} + +impl std::ops::Deref for MqttSinkInvalidCredentialsFixture { + type Target = MqttSinkFixture; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +#[async_trait] +impl TestFixture for MqttSinkInvalidCredentialsFixture { + async fn setup() -> Result { + let mut inner = MqttSinkFixture::start(MqttProtocol::Mqtt5, 1, Schema::Json, 1).await?; + inner.password = INVALID_MQTT_PASSWORD.to_string(); + Ok(Self { inner }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + self.inner.connectors_runtime_envs() + } +} + +pub struct MqttSinkTemplateTopicFixture { + inner: MqttSinkFixture, +} + +impl std::ops::Deref for MqttSinkTemplateTopicFixture { + type Target = MqttSinkFixture; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +#[async_trait] +impl TestFixture for MqttSinkTemplateTopicFixture { + async fn setup() -> Result { + Ok(Self { + inner: MqttSinkFixture::start_with_routing( + MqttProtocol::Mqtt5, + 1, + Schema::Json, + 1, + MqttRoutingConfig { + topic_mode: MqttTopicMode::Template, + topic: None, + topic_template: Some("integration/{stream}/{topic}".to_string()), + fallback_topic: None, + subscription_filters: vec!["integration/#".to_string()], + }, + ) + .await?, + }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + self.inner.connectors_runtime_envs() + } +} + +pub struct MqttSinkDynamicTopicFixture { + inner: MqttSinkFixture, +} + +impl std::ops::Deref for MqttSinkDynamicTopicFixture { + type Target = MqttSinkFixture; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +#[async_trait] +impl TestFixture for MqttSinkDynamicTopicFixture { + async fn setup() -> Result { + Ok(Self { + inner: MqttSinkFixture::start_with_routing( + MqttProtocol::Mqtt5, + 1, + Schema::Raw, + 1, + MqttRoutingConfig { + topic_mode: MqttTopicMode::Dynamic, + topic: None, + topic_template: None, + fallback_topic: Some("fallback/{stream}/{topic}".to_string()), + subscription_filters: vec![ + "devices/living-room".to_string(), + "fallback/test_stream/test_topic".to_string(), + ], + }, + ) + .await?, + }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + self.inner.connectors_runtime_envs() + } +} + +pub struct MqttSinkEnvelopeFixture { + inner: MqttSinkFixture, +} + +impl std::ops::Deref for MqttSinkEnvelopeFixture { + type Target = MqttSinkFixture; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +#[async_trait] +impl TestFixture for MqttSinkEnvelopeFixture { + async fn setup() -> Result { + Ok(Self { + inner: MqttSinkFixture::start_with_routing( + MqttProtocol::Mqtt5, + 1, + Schema::Json, + 1, + MqttRoutingConfig { + topic_mode: MqttTopicMode::Dynamic, + topic: None, + topic_template: None, + fallback_topic: Some("fallback/{stream}/{topic}".to_string()), + subscription_filters: vec![ + "devices/roof/telemetry".to_string(), + "devices/valid/one".to_string(), + "devices/valid/two".to_string(), + ], + }, + ) + .await?, + }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + self.inner.connectors_runtime_envs() + } +} + +enum MqttSubscriber { + Mqtt311 { + _client: Mqtt311Client, + event_loop: Box, + }, + Mqtt5 { + _client: Mqtt5Client, + event_loop: Box, + }, +} + +impl MqttSubscriber { + async fn connect( + broker_url: &str, + protocol: MqttProtocol, + subscription_filters: &[String], + ) -> Result { + match protocol { + MqttProtocol::Mqtt311 => Self::connect_mqtt311(broker_url, subscription_filters).await, + MqttProtocol::Mqtt5 => Self::connect_mqtt5(broker_url, subscription_filters).await, + } + } + + async fn receive_publications( + &mut self, + count: usize, + ) -> Result, TestBinaryError> { + timeout(RECEIVE_TIMEOUT, async { + let mut messages = Vec::with_capacity(count); + while messages.len() < count { + match self { + Self::Mqtt311 { event_loop, .. } => match event_loop.poll().await { + Ok(Mqtt311Event::Incoming(Mqtt311Incoming::Publish(publish))) => { + messages.push(ReceivedMqttPublication { + topic: String::from_utf8_lossy(&publish.topic).into_owned(), + payload: publish.payload.to_vec(), + qos: publish.qos as u8, + retain: publish.retain, + properties: None, + }); + } + Ok(_) => {} + Err(error) => { + return Err(TestBinaryError::FixtureSetup { + fixture_type: "MqttSubscriber".to_string(), + message: format!("Failed to receive MQTT publication: {error}"), + }); + } + }, + Self::Mqtt5 { event_loop, .. } => match event_loop.poll().await { + Ok(Mqtt5Event::Incoming(Mqtt5Incoming::Publish(publish))) => { + messages.push(ReceivedMqttPublication { + topic: String::from_utf8_lossy(&publish.topic).into_owned(), + payload: publish.payload.to_vec(), + qos: publish.qos as u8, + retain: publish.retain, + properties: publish.properties, + }); + } + Ok(_) => {} + Err(error) => { + return Err(TestBinaryError::FixtureSetup { + fixture_type: "MqttSubscriber".to_string(), + message: format!("Failed to receive MQTT publication: {error}"), + }); + } + }, + } + } + Ok(messages) + }) + .await + .map_err(|_| TestBinaryError::FixtureSetup { + fixture_type: "MqttSubscriber".to_string(), + message: format!("Timed out waiting for {count} MQTT publications"), + })? + } + + async fn receive_until_payloads( + &mut self, + expected_payloads: &[Vec], + ) -> Result<(), TestBinaryError> { + let mut remaining = expected_payloads.iter().cloned().collect::>(); + + timeout(RECEIVE_TIMEOUT, async { + while !remaining.is_empty() { + let publication = self + .receive_publications(1) + .await? + .pop() + .expect("one requested MQTT publication must be returned"); + remaining.remove(&publication.payload); + } + Ok(()) + }) + .await + .map_err(|_| TestBinaryError::FixtureSetup { + fixture_type: "MqttSubscriber".to_string(), + message: "Timed out waiting for all expected MQTT publications".to_string(), + })? + } + + async fn connect_mqtt311( + broker_url: &str, + subscription_filters: &[String], + ) -> Result { + let url = Url::parse(broker_url).map_err(|error| TestBinaryError::FixtureSetup { + fixture_type: "MqttSubscriber".to_string(), + message: format!("Failed to parse MQTT broker URL: {error}"), + })?; + let host = url + .host_str() + .ok_or_else(|| TestBinaryError::FixtureSetup { + fixture_type: "MqttSubscriber".to_string(), + message: "MQTT broker URL has no host".to_string(), + })?; + let port = url + .port_or_known_default() + .ok_or_else(|| TestBinaryError::FixtureSetup { + fixture_type: "MqttSubscriber".to_string(), + message: "MQTT broker URL has no port".to_string(), + })?; + let mut options = Mqtt311Options::new(SUBSCRIBER_CLIENT_ID, Mqtt311Broker::tcp(host, port)); + options.set_credentials(MQTT_USERNAME, MQTT_PASSWORD); + options.set_max_packet_size(MAXIMUM_TEST_PACKET_SIZE, MAXIMUM_TEST_PACKET_SIZE); + let (client, mut event_loop) = Mqtt311Client::builder(options) + .capacity(16) + .try_build() + .map_err(|error| TestBinaryError::FixtureSetup { + fixture_type: "MqttSubscriber".to_string(), + message: format!("Failed to build MQTT client: {error}"), + })?; + for subscription_filter in subscription_filters { + client + .subscribe(subscription_filter, Mqtt311Qos::AtLeastOnce) + .await + .map_err(|error| TestBinaryError::FixtureSetup { + fixture_type: "MqttSubscriber".to_string(), + message: format!("Failed to subscribe to MQTT topic: {error}"), + })?; + } + + timeout(RECEIVE_TIMEOUT, async { + let mut acknowledgements = 0; + while acknowledgements < subscription_filters.len() { + match event_loop.poll().await { + Ok(Mqtt311Event::Incoming(Mqtt311Incoming::SubAck(_))) => acknowledgements += 1, + Ok(_) => {} + Err(error) => { + return Err(TestBinaryError::FixtureSetup { + fixture_type: "MqttSubscriber".to_string(), + message: format!("Failed to establish MQTT subscription: {error}"), + }); + } + } + } + Ok(()) + }) + .await + .map_err(|_| TestBinaryError::FixtureSetup { + fixture_type: "MqttSubscriber".to_string(), + message: "Timed out waiting for MQTT subscription acknowledgement".to_string(), + })??; + + Ok(Self::Mqtt311 { + _client: client, + event_loop: Box::new(event_loop), + }) + } + + async fn connect_mqtt5( + broker_url: &str, + subscription_filters: &[String], + ) -> Result { + let url = Url::parse(broker_url).map_err(|error| TestBinaryError::FixtureSetup { + fixture_type: "MqttSubscriber".to_string(), + message: format!("Failed to parse MQTT broker URL: {error}"), + })?; + let host = url + .host_str() + .ok_or_else(|| TestBinaryError::FixtureSetup { + fixture_type: "MqttSubscriber".to_string(), + message: "MQTT broker URL has no host".to_string(), + })?; + let port = url + .port_or_known_default() + .ok_or_else(|| TestBinaryError::FixtureSetup { + fixture_type: "MqttSubscriber".to_string(), + message: "MQTT broker URL has no port".to_string(), + })?; + let mut options = Mqtt5Options::new(SUBSCRIBER_CLIENT_ID, Mqtt5Broker::tcp(host, port)); + options.set_credentials(MQTT_USERNAME, MQTT_PASSWORD); + options.set_max_packet_size(Some(MAXIMUM_TEST_PACKET_SIZE as u32)); + let (client, mut event_loop) = Mqtt5Client::builder(options) + .capacity(16) + .try_build() + .map_err(|error| TestBinaryError::FixtureSetup { + fixture_type: "MqttSubscriber".to_string(), + message: format!("Failed to build MQTT client: {error}"), + })?; + for subscription_filter in subscription_filters { + client + .subscribe(subscription_filter, Mqtt5Qos::AtLeastOnce) + .await + .map_err(|error| TestBinaryError::FixtureSetup { + fixture_type: "MqttSubscriber".to_string(), + message: format!("Failed to subscribe to MQTT topic: {error}"), + })?; + } + + timeout(RECEIVE_TIMEOUT, async { + let mut acknowledgements = 0; + while acknowledgements < subscription_filters.len() { + match event_loop.poll().await { + Ok(Mqtt5Event::Incoming(Mqtt5Incoming::SubAck(_))) => acknowledgements += 1, + Ok(_) => {} + Err(error) => { + return Err(TestBinaryError::FixtureSetup { + fixture_type: "MqttSubscriber".to_string(), + message: format!("Failed to establish MQTT subscription: {error}"), + }); + } + } + } + Ok(()) + }) + .await + .map_err(|_| TestBinaryError::FixtureSetup { + fixture_type: "MqttSubscriber".to_string(), + message: "Timed out waiting for MQTT subscription acknowledgement".to_string(), + })??; + + Ok(Self::Mqtt5 { + _client: client, + event_loop: Box::new(event_loop), + }) + } +} diff --git a/core/integration/tests/connectors/mqtt/mod.rs b/core/integration/tests/connectors/mqtt/mod.rs index 77beb03cf0..2d00debfa8 100644 --- a/core/integration/tests/connectors/mqtt/mod.rs +++ b/core/integration/tests/connectors/mqtt/mod.rs @@ -15,4 +15,5 @@ // specific language governing permissions and limitations // under the License. +mod mqtt_sink; mod mqtt_source; diff --git a/core/integration/tests/connectors/mqtt/mqtt_sink.rs b/core/integration/tests/connectors/mqtt/mqtt_sink.rs new file mode 100644 index 0000000000..e6d89786b2 --- /dev/null +++ b/core/integration/tests/connectors/mqtt/mqtt_sink.rs @@ -0,0 +1,854 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::connectors::fixtures::{ + Mqtt311SinkQosOneFixture, Mqtt311SinkQosTwoFixture, Mqtt311SinkQosZeroFixture, + MqttSinkDynamicTopicFixture, MqttSinkEnvelopeFixture, MqttSinkFixture, MqttSinkIdleFixture, + MqttSinkInvalidCredentialsFixture, MqttSinkPipelinedFixture, MqttSinkQosTwoFixture, + MqttSinkQosZeroFixture, MqttSinkRawSchemaFixture, MqttSinkTemplateTopicFixture, +}; +use base64::{Engine, engine::general_purpose::STANDARD}; +use bytes::Bytes; +use iggy::prelude::{IggyMessage, Partitioning}; +use iggy_common::{HeaderKey, HeaderValue, Identifier, MessageClient}; +use iggy_connector_sdk::api::{ConnectorRuntimeStats, ConnectorStatus}; +use integration::harness::{TestHarness, seeds}; +use integration::iggy_harness; +use reqwest::Client; +use serde_json::Value; +use std::{collections::BTreeMap, time::Duration}; +use tokio::time::{sleep, timeout}; + +const SINK_STATUS_TIMEOUT: Duration = Duration::from_secs(15); +const MQTT_SINK_KEY: &str = "mqtt"; + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/mqtt/sink.toml")), + seed = seeds::connector_stream +)] +async fn mqtt5_invalid_credentials_should_fail_sink_initialization( + harness: &TestHarness, + fixture: MqttSinkInvalidCredentialsFixture, +) { + let sink = wait_for_sink_status(harness, "error").await; + assert_eq!( + sink.get("last_error") + .and_then(Value::as_object) + .and_then(|error| error.get("message")) + .and_then(Value::as_str), + Some("Invalid configuration: Plugin initialization failed (ID: 1)") + ); + + send_json_messages(harness, vec![serde_json::json!({"message": "blocked"})]).await; + assert!( + timeout(Duration::from_secs(1), fixture.receive_messages(1)) + .await + .is_err(), + "an unauthenticated sink must not publish records" + ); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/mqtt/sink.toml")), + seed = seeds::connector_stream +)] +async fn mqtt5_broker_rejection_should_not_publish_to_the_broker( + harness: &TestHarness, + fixture: MqttSinkFixture, +) { + let errors_before_rejection = sink_errors(harness).await; + fixture + .deny_sink_publications() + .await + .expect("EMQX must deny the sink publication topic"); + send_json_messages(harness, vec![serde_json::json!({"rejected": true})]).await; + + assert!( + timeout(Duration::from_secs(1), fixture.receive_messages(1)) + .await + .is_err(), + "a broker-rejected publication must not reach MQTT subscribers" + ); + wait_for_sink_error_after(harness, errors_before_rejection).await; +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/mqtt/sink.toml")), + seed = seeds::connector_stream +)] +async fn mqtt5_qos1_messages_are_published_to_mqtt( + harness: &TestHarness, + fixture: MqttSinkFixture, +) { + let client = harness + .root_client() + .await + .expect("root client must connect"); + let stream_id: Identifier = seeds::names::STREAM + .try_into() + .expect("stream ID must parse"); + let topic_id: Identifier = seeds::names::TOPIC.try_into().expect("topic ID must parse"); + let payloads = [ + serde_json::json!({"name": "Alice"}), + serde_json::json!({"name": "Bob"}), + serde_json::json!({"name": "Carol"}), + ]; + let mut messages = payloads + .iter() + .enumerate() + .map(|(index, payload)| { + IggyMessage::builder() + .id((index + 1) as u128) + .payload(Bytes::from( + serde_json::to_vec(payload).expect("test payload must serialize"), + )) + .build() + .expect("test message must build") + }) + .collect::>(); + + client + .send_messages( + &stream_id, + &topic_id, + &Partitioning::partition_id(0), + &mut messages, + ) + .await + .expect("messages must send to Iggy"); + + let received = fixture + .receive_messages(payloads.len()) + .await + .expect("messages must reach MQTT"); + let received = received + .iter() + .map(|payload| serde_json::from_slice::(payload)) + .collect::, _>>() + .expect("MQTT payloads must be valid JSON"); + + assert_eq!(received, payloads); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/mqtt/sink.toml")), + seed = seeds::connector_stream +)] +async fn mqtt5_sink_keeps_idle_connection_alive( + harness: &TestHarness, + fixture: MqttSinkIdleFixture, +) { + wait_for_sink_status(harness, "running").await; + sleep(Duration::from_secs(8)).await; + + let payload = serde_json::json!({"after_idle": true}); + send_json_messages(harness, vec![payload.clone()]).await; + + let received = fixture + .receive_messages(1) + .await + .expect("an idle MQTT connection must publish without a retry"); + let received: Value = + serde_json::from_slice(&received[0]).expect("MQTT payload must be valid JSON"); + + assert_eq!(received, payload); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/mqtt/sink.toml")), + seed = seeds::connector_stream +)] +async fn raw_schema_messages_including_payload_larger_than_ten_kib_preserve_payload_bytes( + harness: &TestHarness, + fixture: MqttSinkRawSchemaFixture, +) { + let client = harness + .root_client() + .await + .expect("root client must connect"); + let stream_id: Identifier = seeds::names::STREAM + .try_into() + .expect("stream ID must parse"); + let topic_id: Identifier = seeds::names::TOPIC.try_into().expect("topic ID must parse"); + let payloads = vec![ + b"plain text message".to_vec(), + vec![0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD], + vec![0xDE, 0xAD, 0xBE, 0xEF], + vec![0xA5; 16 * 1024], + ]; + let mut messages = payloads + .iter() + .enumerate() + .map(|(index, payload)| { + IggyMessage::builder() + .id((index + 1) as u128) + .payload(Bytes::copy_from_slice(payload)) + .build() + .expect("test message must build") + }) + .collect::>(); + + client + .send_messages( + &stream_id, + &topic_id, + &Partitioning::partition_id(0), + &mut messages, + ) + .await + .expect("messages must send to Iggy"); + + let received = fixture + .receive_messages(payloads.len()) + .await + .expect("messages must reach MQTT"); + + assert_eq!(received, payloads); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/mqtt/sink.toml")), + seed = seeds::connector_stream +)] +async fn mqtt5_qos2_pipelined_messages_are_published_to_mqtt( + harness: &TestHarness, + fixture: MqttSinkQosTwoFixture, +) { + let client = harness + .root_client() + .await + .expect("root client must connect"); + let stream_id: Identifier = seeds::names::STREAM + .try_into() + .expect("stream ID must parse"); + let topic_id: Identifier = seeds::names::TOPIC.try_into().expect("topic ID must parse"); + let payloads = (0..10) + .map(|sequence| serde_json::json!({"sequence": sequence})) + .collect::>(); + let mut messages = payloads + .iter() + .enumerate() + .map(|(index, payload)| { + IggyMessage::builder() + .id((index + 1) as u128) + .payload(Bytes::from( + serde_json::to_vec(payload).expect("test payload must serialize"), + )) + .build() + .expect("test message must build") + }) + .collect::>(); + + client + .send_messages( + &stream_id, + &topic_id, + &Partitioning::partition_id(0), + &mut messages, + ) + .await + .expect("messages must send to Iggy"); + + let received = fixture + .receive_messages(payloads.len()) + .await + .expect("messages must reach MQTT"); + let received = received + .iter() + .map(|payload| serde_json::from_slice::(payload)) + .collect::, _>>() + .expect("MQTT payloads must be valid JSON"); + + assert_eq!(received, payloads); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/mqtt/sink.toml")), + seed = seeds::connector_stream +)] +async fn mqtt5_qos0_messages_are_published_to_mqtt( + harness: &TestHarness, + fixture: MqttSinkQosZeroFixture, +) { + let client = harness + .root_client() + .await + .expect("root client must connect"); + let stream_id: Identifier = seeds::names::STREAM + .try_into() + .expect("stream ID must parse"); + let topic_id: Identifier = seeds::names::TOPIC.try_into().expect("topic ID must parse"); + let payload = serde_json::json!({"name": "Alice"}); + let mut messages = vec![ + IggyMessage::builder() + .id(1) + .payload(Bytes::from( + serde_json::to_vec(&payload).expect("test payload must serialize"), + )) + .build() + .expect("test message must build"), + ]; + + client + .send_messages( + &stream_id, + &topic_id, + &Partitioning::partition_id(0), + &mut messages, + ) + .await + .expect("message must send to Iggy"); + + let received = fixture + .receive_messages(1) + .await + .expect("message must reach MQTT"); + let received: serde_json::Value = + serde_json::from_slice(&received[0]).expect("MQTT payload must be valid JSON"); + + assert_eq!(received, payload); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/mqtt/sink.toml")), + seed = seeds::connector_stream +)] +async fn mqtt311_qos0_messages_are_published_to_mqtt( + harness: &TestHarness, + fixture: Mqtt311SinkQosZeroFixture, +) { + publish_json_message_and_assert_delivery(harness, &fixture, "mqtt311-qos0").await; +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/mqtt/sink.toml")), + seed = seeds::connector_stream +)] +async fn mqtt311_qos1_messages_are_published_to_mqtt( + harness: &TestHarness, + fixture: Mqtt311SinkQosOneFixture, +) { + publish_json_message_and_assert_delivery(harness, &fixture, "mqtt311-qos1").await; +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/mqtt/sink.toml")), + seed = seeds::connector_stream +)] +async fn mqtt311_qos2_messages_are_published_to_mqtt( + harness: &TestHarness, + fixture: Mqtt311SinkQosTwoFixture, +) { + publish_json_message_and_assert_delivery(harness, &fixture, "mqtt311-qos2").await; +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/mqtt/sink.toml")), + seed = seeds::connector_stream +)] +async fn mqtt5_pipelined_messages_are_published_to_mqtt( + harness: &TestHarness, + fixture: MqttSinkPipelinedFixture, +) { + let client = harness + .root_client() + .await + .expect("root client must connect"); + let stream_id: Identifier = seeds::names::STREAM + .try_into() + .expect("stream ID must parse"); + let topic_id: Identifier = seeds::names::TOPIC.try_into().expect("topic ID must parse"); + let payloads = (0..10) + .map(|index| serde_json::json!({"sequence": index})) + .collect::>(); + let mut messages = payloads + .iter() + .enumerate() + .map(|(index, payload)| { + IggyMessage::builder() + .id((index + 1) as u128) + .payload(Bytes::from( + serde_json::to_vec(payload).expect("test payload must serialize"), + )) + .build() + .expect("test message must build") + }) + .collect::>(); + + client + .send_messages( + &stream_id, + &topic_id, + &Partitioning::partition_id(0), + &mut messages, + ) + .await + .expect("messages must send to Iggy"); + + let received = fixture + .receive_messages(payloads.len()) + .await + .expect("messages must reach MQTT"); + let received = received + .iter() + .map(|payload| serde_json::from_slice::(payload)) + .collect::, _>>() + .expect("MQTT payloads must be valid JSON"); + + assert_eq!(received, payloads); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/mqtt/sink.toml")), + seed = seeds::connector_stream +)] +async fn mqtt5_sink_restart_publishes_pipelined_messages( + harness: &TestHarness, + fixture: MqttSinkPipelinedFixture, +) { + let payloads = (0..10) + .map(|sequence| serde_json::json!({"sequence": sequence})) + .collect::>(); + send_json_messages(harness, payloads.clone()).await; + + let runtime = harness + .connectors_runtime() + .expect("connectors runtime must be configured"); + let response = Client::new() + .post(format!("{}/sinks/mqtt/restart", runtime.http_url())) + .send() + .await + .expect("sink restart request must complete"); + assert_eq!(response.status(), reqwest::StatusCode::NO_CONTENT); + wait_for_sink_status(harness, "running").await; + + let received = fixture + .receive_publications(payloads.len()) + .await + .expect("all publications must survive sink restart"); + let received = received + .iter() + .map(|publication| { + serde_json::from_slice::(&publication.payload) + .expect("MQTT payload must be valid JSON") + }) + .collect::>(); + for payload in payloads { + assert!( + received.contains(&payload), + "publication was lost: {payload}" + ); + } +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/mqtt/sink.toml")), + seed = seeds::connector_stream +)] +async fn mqtt5_sink_publishes_new_message_after_reconnect( + harness: &TestHarness, + fixture: MqttSinkPipelinedFixture, +) { + fixture + .wait_for_sink_reconnect() + .await + .expect("sink must connect before the test starts"); + + let payloads = (0..10) + .map(|sequence| serde_json::json!({"sequence": sequence})) + .collect::>(); + send_json_messages(harness, payloads.clone()).await; + + let first_publication = fixture + .receive_publications(1) + .await + .expect("a publication must reach MQTT before disconnect") + .pop() + .expect("one requested MQTT publication must be returned"); + fixture + .disconnect_sink() + .await + .expect("EMQX must disconnect the sink client"); + fixture + .wait_for_sink_reconnect() + .await + .expect("the sink must reconnect after the blocked publications"); + + let recovery_payload = serde_json::json!({"sequence": "after-reconnect"}); + let recovery_message = IggyMessage::builder() + .id(11) + .payload(Bytes::from( + serde_json::to_vec(&recovery_payload).expect("recovery payload must serialize"), + )) + .build() + .expect("recovery message must build"); + send_messages(harness, vec![recovery_message]).await; + + let original_payloads = payloads + .iter() + .map(|payload| serde_json::to_vec(payload).expect("test payload must serialize")) + .collect::>(); + assert!( + original_payloads + .iter() + .any(|payload| payload == &first_publication.payload), + "the publication received before disconnect must be part of the original batch" + ); + fixture + .receive_until_payloads(&[ + serde_json::to_vec(&recovery_payload).expect("recovery payload must serialize") + ]) + .await + .expect("a publication sent after reconnect must reach MQTT"); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/mqtt/sink.toml")), + seed = seeds::connector_stream +)] +async fn mqtt_sink_static_topic_routes_publications_to_configured_topic( + harness: &TestHarness, + fixture: MqttSinkFixture, +) { + let payload = serde_json::json!({"route": "static"}); + send_json_messages(harness, vec![payload.clone()]).await; + + let received = fixture + .receive_publications(1) + .await + .expect("publication must reach MQTT"); + + assert_eq!(received[0].topic, "iggy.messages"); + assert_eq!( + serde_json::from_slice::(&received[0].payload) + .expect("MQTT payload must be valid JSON"), + payload + ); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/mqtt/sink.toml")), + seed = seeds::connector_stream +)] +async fn mqtt_sink_template_topic_renders_stream_and_topic( + harness: &TestHarness, + fixture: MqttSinkTemplateTopicFixture, +) { + let payload = serde_json::json!({"route": "template"}); + send_json_messages(harness, vec![payload.clone()]).await; + + let received = fixture + .receive_publications(1) + .await + .expect("publication must reach MQTT"); + + assert_eq!(received[0].topic, "integration/test_stream/test_topic"); + assert_eq!( + serde_json::from_slice::(&received[0].payload) + .expect("MQTT payload must be valid JSON"), + payload + ); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/mqtt/sink.toml")), + seed = seeds::connector_stream +)] +async fn mqtt_sink_dynamic_topic_uses_header_and_fallback( + harness: &TestHarness, + fixture: MqttSinkDynamicTopicFixture, +) { + let topic_header = HeaderKey::try_from("mqtt.topic").expect("MQTT topic header key must parse"); + let topic_value = + HeaderValue::try_from("devices/living-room").expect("MQTT topic header value must parse"); + let messages = vec![ + IggyMessage::builder() + .id(1) + .payload(Bytes::from_static(b"header-routed")) + .user_headers(BTreeMap::from([(topic_header, topic_value)])) + .build() + .expect("test message must build"), + IggyMessage::builder() + .id(2) + .payload(Bytes::from_static(b"fallback-routed")) + .build() + .expect("test message must build"), + ]; + send_messages(harness, messages).await; + + let received = fixture + .receive_publications(2) + .await + .expect("publications must reach MQTT"); + + assert_eq!(received[0].topic, "devices/living-room"); + assert_eq!(received[0].payload, b"header-routed"); + assert_eq!(received[1].topic, "fallback/test_stream/test_topic"); + assert_eq!(received[1].payload, b"fallback-routed"); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/mqtt/sink.toml")), + seed = seeds::connector_stream +)] +async fn mqtt_sink_mqtt5_envelope_preserves_supported_properties( + harness: &TestHarness, + fixture: MqttSinkEnvelopeFixture, +) { + let payload = b"envelope payload"; + let envelope = serde_json::json!({ + "version": 1, + "topic": "devices/roof/telemetry", + "payload_base64": STANDARD.encode(payload), + "properties": { + "payload_format_indicator": 1, + "message_expiry_interval": 30, + "response_topic": "devices/roof/replies", + "correlation_data_base64": STANDARD.encode(b"correlation-id"), + "user_properties": [{"key": "source", "value": "integration-test"}], + "content_type": "application/json" + } + }); + send_json_messages(harness, vec![envelope]).await; + + let received = fixture + .receive_publications(1) + .await + .expect("publication must reach MQTT"); + let properties = received[0] + .properties + .as_ref() + .expect("MQTT 5 publication must carry properties"); + + assert_eq!(received[0].topic, "devices/roof/telemetry"); + assert_eq!(received[0].payload, payload); + assert_eq!(received[0].qos, 1); + assert!(!received[0].retain); + assert_eq!(properties.payload_format_indicator, Some(1)); + assert!( + properties + .message_expiry_interval + .is_some_and(|expiry| (1..=30).contains(&expiry)) + ); + assert_eq!( + properties.response_topic.as_deref(), + Some("devices/roof/replies") + ); + assert_eq!( + properties + .correlation_data + .as_ref() + .map(|value| value.as_ref()), + Some(b"correlation-id".as_slice()) + ); + assert_eq!( + properties.user_properties, + vec![("source".to_string(), "integration-test".to_string())] + ); + assert_eq!(properties.content_type.as_deref(), Some("application/json")); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/mqtt/sink.toml")), + seed = seeds::connector_stream +)] +async fn mqtt_sink_skips_malformed_envelope_and_publishes_valid_records( + harness: &TestHarness, + fixture: MqttSinkEnvelopeFixture, +) { + let messages = vec![ + serde_json::json!({ + "version": 1, + "topic": "devices/valid/one", + "payload_base64": STANDARD.encode(b"first") + }), + serde_json::json!({ + "version": "bad", + "payload_base64": STANDARD.encode(b"discarded") + }), + serde_json::json!({ + "version": 1, + "topic": "devices/valid/two", + "payload_base64": STANDARD.encode(b"second") + }), + ]; + send_json_messages(harness, messages).await; + + let received = fixture + .receive_publications(2) + .await + .expect("valid records must reach MQTT"); + + assert_eq!(received[0].topic, "devices/valid/one"); + assert_eq!(received[0].payload, b"first"); + assert_eq!(received[1].topic, "devices/valid/two"); + assert_eq!(received[1].payload, b"second"); +} + +async fn send_json_messages(harness: &TestHarness, payloads: Vec) { + let messages = payloads + .into_iter() + .enumerate() + .map(|(index, payload)| { + IggyMessage::builder() + .id((index + 1) as u128) + .payload(Bytes::from( + serde_json::to_vec(&payload).expect("test payload must serialize"), + )) + .build() + .expect("test message must build") + }) + .collect(); + send_messages(harness, messages).await; +} + +async fn send_messages(harness: &TestHarness, mut messages: Vec) { + let client = harness + .root_client() + .await + .expect("root client must connect"); + let stream_id: Identifier = seeds::names::STREAM + .try_into() + .expect("stream ID must parse"); + let topic_id: Identifier = seeds::names::TOPIC.try_into().expect("topic ID must parse"); + + client + .send_messages( + &stream_id, + &topic_id, + &Partitioning::partition_id(0), + &mut messages, + ) + .await + .expect("messages must send to Iggy"); +} + +async fn publish_json_message_and_assert_delivery( + harness: &TestHarness, + fixture: &MqttSinkFixture, + protocol: &str, +) { + let client = harness + .root_client() + .await + .expect("root client must connect"); + let stream_id: Identifier = seeds::names::STREAM + .try_into() + .expect("stream ID must parse"); + let topic_id: Identifier = seeds::names::TOPIC.try_into().expect("topic ID must parse"); + let payload = serde_json::json!({"protocol": protocol}); + let mut messages = vec![ + IggyMessage::builder() + .id(1) + .payload(Bytes::from( + serde_json::to_vec(&payload).expect("test payload must serialize"), + )) + .build() + .expect("test message must build"), + ]; + + client + .send_messages( + &stream_id, + &topic_id, + &Partitioning::partition_id(0), + &mut messages, + ) + .await + .expect("message must send to Iggy"); + + let received = fixture + .receive_messages(1) + .await + .expect("message must reach MQTT"); + let received: serde_json::Value = + serde_json::from_slice(&received[0]).expect("MQTT payload must be valid JSON"); + + assert_eq!(received, payload); +} + +async fn wait_for_sink_status(harness: &TestHarness, expected_status: &str) -> Value { + let runtime = harness + .connectors_runtime() + .expect("connectors runtime must be configured"); + let http = Client::new(); + let api_url = runtime.http_url(); + + timeout(SINK_STATUS_TIMEOUT, async { + loop { + if let Ok(response) = http.get(format!("{api_url}/sinks/mqtt")).send().await + && let Ok(sink) = response.json::().await + && sink.get("status").and_then(Value::as_str) == Some(expected_status) + { + return sink; + } + sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("MQTT sink connector must reach the expected status") +} + +async fn sink_errors(harness: &TestHarness) -> u64 { + let runtime = harness + .connectors_runtime() + .expect("connectors runtime must be configured"); + let stats = Client::new() + .get(format!("{}/stats", runtime.http_url())) + .send() + .await + .expect("runtime statistics request must complete") + .json::() + .await + .expect("runtime statistics must be valid JSON"); + + stats + .connectors + .iter() + .find(|connector| connector.key == MQTT_SINK_KEY) + .expect("MQTT sink statistics must be present") + .errors +} + +async fn wait_for_sink_error_after(harness: &TestHarness, previous_errors: u64) { + let runtime = harness + .connectors_runtime() + .expect("connectors runtime must be configured"); + let http = Client::new(); + + timeout(SINK_STATUS_TIMEOUT, async { + loop { + if let Ok(response) = http + .get(format!("{}/stats", runtime.http_url())) + .send() + .await + && let Ok(stats) = response.json::().await + && let Some(sink) = stats + .connectors + .iter() + .find(|connector| connector.key == MQTT_SINK_KEY) + && sink.status == ConnectorStatus::Running + && sink.errors > previous_errors + { + return; + } + sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("sink failure must increase the MQTT sink error counter"); +} diff --git a/core/integration/tests/connectors/mqtt/sink.toml b/core/integration/tests/connectors/mqtt/sink.toml new file mode 100644 index 0000000000..b356bf5de9 --- /dev/null +++ b/core/integration/tests/connectors/mqtt/sink.toml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[connectors] +config_type = "local" +config_dir = "../connectors/sinks/mqtt_sink"