diff --git a/docs/observability.md b/docs/observability.md index 1fddc27..1caf30b 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -434,7 +434,7 @@ as loaded and healthy and could not fire for any input ([#63](https://github.com `prometheus/tests/*.test.yaml` holds `promtool test rules` unit tests, which feed a rule synthetic series and assert it fires — paired with a case asserting it stays quiet, because a test that only ever expects silence would have passed -against the broken rule too. Coverage is fifty-seven rules of 78 so far — the five +against the broken rule too. Coverage is fifty-eight rules of 78 so far — the five in `blackbox.rules.yaml`, both in `dns.rules.yaml`, `ContainerHighMemory`, `ContainerNearMemoryLimit`, `ContainerRestartLoop`, `ContainerCpuThrottled` and `PrometheusSizeRetentionActive`, `Watchdog`, the three iLO rules from @@ -449,7 +449,7 @@ SMART rules from [#351](https://github.com/Gerrrt/HomeLab/issues/351), `PatchStateStopped` from [#360](https://github.com/Gerrrt/HomeLab/issues/360), and `SystemUpdateAvailable` from [#378](https://github.com/Gerrrt/HomeLab/issues/378), and the two guest rules from [#257](https://github.com/Gerrrt/HomeLab/issues/257). -The other 21 are still validated for syntax only, which is exactly the +The other 20 are still validated for syntax only, which is exactly the standing #63 had. Both numbers are checked by `scripts/check_docs.py` — the sentence they replaced claimed six and named two, and had been wrong for weeks. diff --git a/stacks/observability/prometheus/rules/stack.rules.yaml b/stacks/observability/prometheus/rules/stack.rules.yaml index 3b9994c..6e5c70a 100644 --- a/stacks/observability/prometheus/rules/stack.rules.yaml +++ b/stacks/observability/prometheus/rules/stack.rules.yaml @@ -72,6 +72,11 @@ groups: PROMETHEUS_RETENTION, so history is shorter than 30 days. - alert: AlertmanagerNotificationsFailing + # The `reason` label is the first thing to read, not the integration. + # On 2026-09-08 this fired with reason="rateLimited": every receiver + # was reachable and answering, and ntfy.sh was refusing with 429 + # because the day's free message budget was spent (#407). "The + # webhook is unreachable" is what a reader assumes, and it was wrong. expr: increase(alertmanager_notifications_failed_total[15m]) > 0 for: 5m labels: @@ -79,10 +84,15 @@ groups: severity: critical category: correctness annotations: - summary: "Alertmanager cannot deliver notifications via {{ $labels.integration }}" + summary: "Alertmanager cannot deliver via {{ $labels.integration }}: {{ $labels.reason }}" description: >- Alerts are firing but not reaching anyone — this makes every other - rule in this repository silently useless. + rule in this repository silently useless. Read the reason before + the URL: rateLimited means the destination answered and refused + (ntfy.sh's daily quota, #407), clientError or serverError means it + answered with an error, and contextDeadlineExceeded means it did + not answer at all. This alert travels the same path it reports on, + so if it reached you, at least one receiver still works. - alert: LokiIngestionStalled expr: | diff --git a/stacks/observability/prometheus/tests/stack.test.yaml b/stacks/observability/prometheus/tests/stack.test.yaml index 064ac27..e20a7aa 100644 --- a/stacks/observability/prometheus/tests/stack.test.yaml +++ b/stacks/observability/prometheus/tests/stack.test.yaml @@ -13,10 +13,11 @@ # alert for a real input; the quiet case proves it does not produce one for the # input production actually supplies. # -# What these do NOT prove: four rules in stack.rules.yaml have no unit test. -# PrometheusConfigReloadFailed, PrometheusRuleEvaluationFailures, -# AlertmanagerNotificationsFailing and LokiIngestionStalled are validated for -# syntax only, which is exactly the standing #63 had. +# What these do NOT prove: three rules in stack.rules.yaml have no unit test. +# PrometheusConfigReloadFailed, PrometheusRuleEvaluationFailures and +# LokiIngestionStalled are validated for syntax only, which is exactly the +# standing #63 had. AlertmanagerNotificationsFailing gained its test the day +# after it fired for real with a reason nobody had expected (#407). # # RemoteWriteJobStale is tested below and needs to be, because the obvious way # to write it cannot fire for any input. `time() - timestamp(up) > 300` reads @@ -55,6 +56,55 @@ tests: Retention is now bounded by PROMETHEUS_RETENTION_SIZE rather than PROMETHEUS_RETENTION, so history is shorter than 30 days. + # --- AlertmanagerNotificationsFailing: fires, and names the reason --------- + # The fixture is the 2026-09-08 incident: the webhook integration reachable + # and refusing with 429 for the day's quota. The point of the case is the + # rendered summary — `reason` must come through, because it is the label + # that told the two failure modes apart on the night and nothing else did. + - interval: 1m + input_series: + - series: 'alertmanager_notifications_failed_total{instance="alertmanager:9093",job="alertmanager",integration="webhook",reason="rateLimited"}' + values: "0+0x10 1+1x20" + alert_rule_test: + - eval_time: 20m + alertname: AlertmanagerNotificationsFailing + exp_alerts: + - exp_labels: + alertname: AlertmanagerNotificationsFailing + instance: alertmanager:9093 + job: alertmanager + integration: webhook + reason: rateLimited + component: stack + severity: critical + category: correctness + exp_annotations: + summary: "Alertmanager cannot deliver via webhook: rateLimited" + description: >- + Alerts are firing but not reaching anyone — this makes every other + rule in this repository silently useless. Read the reason before + the URL: rateLimited means the destination answered and refused + (ntfy.sh's daily quota, #407), clientError or serverError means it + answered with an error, and contextDeadlineExceeded means it did + not answer at all. This alert travels the same path it reports on, + so if it reached you, at least one receiver still works. + + # --- AlertmanagerNotificationsFailing: quiet on a flat counter ------------ + # Alertmanager exports this counter for every integration it knows, at zero, + # forever. A rule written as `> 0` on the counter itself rather than on its + # increase would be fine here and wrong the first time any receiver ever + # failed once, months ago; this case is what pins the increase() form. + - interval: 1m + input_series: + - series: 'alertmanager_notifications_failed_total{instance="alertmanager:9093",job="alertmanager",integration="webhook",reason="rateLimited"}' + values: "3x40" + - series: 'alertmanager_notifications_failed_total{instance="alertmanager:9093",job="alertmanager",integration="email",reason="other"}' + values: "0x40" + alert_rule_test: + - eval_time: 30m + alertname: AlertmanagerNotificationsFailing + exp_alerts: [] + # --- PrometheusSizeRetentionActive: quiet while the ceiling is not bound -- # This is the production state and the one the rule must not alert on: the # counter exists and reads 0 forever. Without this case the rule would pass