feat(metrics): one endpoint for the whole container (#135)

CHANGELOG.md

@@ -5,6 +5,29 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Added
+
+- **One metrics endpoint for the whole container ([#135]).** The main
+  `/metrics` response can now tell the complete story in a single scrape:
+  - The embedded runtime PHP-FPM autotuner's `fpm_tune_*` series always
+    appear on the main endpoint while the tuner runs — no second listener
+    required. `fpm_tune.metrics_addr` stays as an optional extra listener
+    for standalone-tool parity.
+  - `global.metrics_federate` declares local exporters whose exposition is
+    appended to every scrape (per-source timeout and TTL cache, 8 MiB body
+    cap). Each source contributes `cbox_init_federate_up{name}`; one that is
+    down degrades to 0 there instead of failing the scrape. URLs are
+    validated to loopback only — federation merges exporters inside the
+    container, it is not a proxy.
+  - With federation enabled the endpoint serves the plain-text exposition
+    format unconditionally, since federated bodies are appended verbatim.
+  See `configs/examples/metrics-federate.yaml` and
+  [Prometheus Metrics](docs/observability/metrics.md).
+
+[#135]: https://github.com/cboxdk/init/issues/135
+
 ## [3.1.2] - 2026-09-04
 
 ### Fixed

cmd/cbox-init/fpmtune.go

@@ -10,6 +10,7 @@ import (
 	"github.com/cboxdk/fpm-tune/state"
 
 	"github.com/cboxdk/init/internal/config"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // The p95 hybrid is the intended default sizing basis: size on the 95th
@@ -34,10 +35,13 @@ const (
 // are up, is fine even if php-fpm is still coming up. Its config is loaded per
 // round, so a pool's boot-time pm.max_children (set by the calculator before
 // php-fpm started) is the seed it refines, not something it fights.
-func startFPMTune(ctx context.Context, cfg *config.Config, log *slog.Logger) (func(), error) {
+// It also returns the tuner's Prometheus registry so the caller can merge the
+// fpm_tune_* series onto the main metrics endpoint — one scrape, one story —
+// regardless of whether the optional metrics_addr listener is configured.
+func startFPMTune(ctx context.Context, cfg *config.Config, log *slog.Logger) (func(), *prometheus.Registry, error) {
 	ft := cfg.Global.FPMTune
 	if ft == nil || !ft.Enabled {
-		return nil, nil
+		return nil, nil, nil
 	}
 
 	sc := serve.Config{
@@ -62,7 +66,7 @@ func startFPMTune(ctx context.Context, cfg *config.Config, log *slog.Logger) (fu
 
 	loop, err := serve.New(sc, log)
 	if err != nil {
-		return nil, fmt.Errorf("fpm-tune: %w", err)
+		return nil, nil, fmt.Errorf("fpm-tune: %w", err)
 	}
 
 	// The loop gets its own context so shutdown can stop it independently: the
@@ -90,7 +94,7 @@ func startFPMTune(ctx context.Context, cfg *config.Config, log *slog.Logger) (fu
 		<-done // Run's deferred Close() releases the state lock and saves baselines.
 	}
 
-	return stop, nil
+	return stop, loop.Metrics().Registry, nil
 }
 
 // resolveFPMWorkload maps the configured workload name to a class, warning on an

cmd/cbox-init/fpmtune_test.go

@@ -20,7 +20,7 @@ func discardLogger() *slog.Logger {
 func TestStartFPMTuneDisabled(t *testing.T) {
 	for _, ft := range []*config.FPMTuneConfig{nil, {Enabled: false}} {
 		cfg := &config.Config{Global: config.GlobalConfig{FPMTune: ft}}
-		stop, err := startFPMTune(context.Background(), cfg, discardLogger())
+		stop, _, err := startFPMTune(context.Background(), cfg, discardLogger())
 		if err != nil {
 			t.Fatalf("disabled autotuner returned an error: %v", err)
 		}
@@ -49,7 +49,7 @@ func TestStartFPMTuneStartsAndStops(t *testing.T) {
 		},
 	}
 
-	stop, err := startFPMTune(context.Background(), cfg, discardLogger())
+	stop, _, err := startFPMTune(context.Background(), cfg, discardLogger())
 	if err != nil {
 		t.Fatalf("startFPMTune: %v", err)
 	}
@@ -70,7 +70,7 @@ func TestStartFPMTuneStartsAndStops(t *testing.T) {
 	}
 
 	// The state lock is released, so a second loop on the same state path starts.
-	stop2, err := startFPMTune(context.Background(), cfg, discardLogger())
+	stop2, _, err := startFPMTune(context.Background(), cfg, discardLogger())
 	if err != nil {
 		t.Fatalf("second start after stop failed (lock not released?): %v", err)
 	}

cmd/cbox-init/serve.go

@@ -392,10 +392,17 @@ func runServe(cmd *cobra.Command, args []string) {
 	// startup step that can os.Exit, so its stop is never skipped. Non-critical:
 	// if it cannot start (for example a second copy already holds the state lock),
 	// php-fpm keeps its boot-time size and the container runs on.
-	stopFPMTune, err := startFPMTune(ctx, cfg, log)
+	stopFPMTune, fpmTuneRegistry, err := startFPMTune(ctx, cfg, log)
 	if err != nil {
 		slog.Warn("Runtime PHP-FPM autotuner not started", "error", err)
 	}
+	if fpmTuneRegistry != nil && metricsServer != nil {
+		// One scrape, one story: the tuner's fpm_tune_* series ride on the
+		// main metrics endpoint; the separate metrics_addr listener stays
+		// optional for standalone-tool parity.
+		metricsServer.AddGatherer(fpmTuneRegistry)
+		slog.Info("fpm-tune metrics merged onto the main metrics endpoint")
+	}
 
 	// Main event loop - handles shutdown signals and config reloads
 	var shutdownReason string
@@ -684,6 +691,14 @@ func startMetricsServer(ctx context.Context, cfg *config.Config, log *slog.Logge
 
 	server := metrics.NewServer(metricsPort, metricsPath, cfg.Global.MetricsACL, cfg.Global.MetricsTLS, log)
 	server.SetBindHost(cfg.Global.MetricsHost)
+	if len(cfg.Global.MetricsFederate) > 0 {
+		sources := make([]metrics.FederateSource, len(cfg.Global.MetricsFederate))
+		for i, fs := range cfg.Global.MetricsFederate {
+			sources[i] = metrics.FederateSource{Name: fs.Name, URL: fs.URL, Timeout: fs.Timeout, CacheTTL: fs.CacheTTL}
+		}
+		server.SetFederator(metrics.NewFederator(sources, log))
+		slog.Info("Metrics federation enabled", "sources", len(sources))
+	}
 	if err := server.Start(ctx); err != nil {
 		slog.Warn("Failed to start metrics server (continuing without metrics)", "error", err)
 		return nil

configs/examples/metrics-federate.yaml

@@ -0,0 +1,54 @@
+# One scrape, one story.
+#
+# A PHP container easily grows three metrics endpoints: cbox-init's own
+# (supervision), the embedded fpm-tune's (capacity), and an application
+# exporter's (php-fpm status, queues). Federation folds the local exporters
+# into cbox-init's main /metrics response, and since 3.2.0 the embedded
+# fpm-tune series are always there natively - so Prometheus scrapes ONE port
+# and sees the whole story:
+#
+#   cbox_init_*   - is everything running?
+#   fpm_tune_*    - does the workload fit this container? (vertical)
+#   phpfpm_*      - is this container saturated? (horizontal - listen_queue)
+#
+# A source that is down contributes cbox_init_federate_up{name} 0 instead of
+# failing the scrape. URLs must be loopback: federation is for exporters
+# inside the container, not a proxy.
+
+version: "1.0"
+
+global:
+  shutdown_timeout: 30
+  log_format: json
+
+  metrics_enabled: true
+  metrics_port: 9090
+
+  metrics_federate:
+    - name: fpm-exporter
+      url: http://127.0.0.1:9114/metrics
+      # timeout: 2s     # per-fetch timeout (default)
+      # cache_ttl: 5s   # serve the cached body this long between fetches (default)
+
+  # The runtime autotuner's fpm_tune_* series ride on the main endpoint
+  # automatically; metrics_addr remains optional for standalone-tool parity.
+  fpm_tune:
+    enabled: true
+    mode: apply
+    interval: 30s
+
+processes:
+  php-fpm:
+    enabled: true
+    command: ["php-fpm", "-F", "-R"]
+    restart: always
+
+  fpm-exporter:
+    enabled: true
+    command: ["fpm-exporter", "serve"]
+    restart: always
+    depends_on:
+      - php-fpm
+    health_check:
+      type: tcp
+      address: "127.0.0.1:9114"

docs/observability/metrics.md

@@ -19,6 +19,49 @@ global:
   metrics_path: /metrics
 ```
 
+## One Endpoint: Federation and Embedded Engines
+
+Since 3.2.0 the main `/metrics` response can carry the whole container's
+story, so Prometheus scrapes one port:
+
+- **Embedded engines merge natively.** The runtime PHP-FPM autotuner's
+  `fpm_tune_*` series always appear on the main endpoint when `fpm_tune` is
+  enabled. `fpm_tune.metrics_addr` remains optional, for parity with the
+  standalone tool.
+- **Local exporters federate.** Declare them under
+  `global.metrics_federate` and their exposition is appended to every
+  scrape, with a short cache so heavy scraping does not multiply load:
+
+```yaml
+global:
+  metrics_federate:
+    - name: fpm-exporter
+      url: http://127.0.0.1:9114/metrics
+      timeout: 2s     # per-fetch (default)
+      cache_ttl: 5s   # cached between fetches (default)
+```
+
+Each source contributes a health gauge; a source that is down degrades
+instead of failing the scrape:
+
+```text
+cbox_init_federate_up{name="fpm-exporter"} 1
+```
+
+Rules and caveats:
+
+- URLs must point at **loopback** (`127.0.0.1`, `::1`, `localhost`) —
+  federation merges exporters inside the container; it is not a proxy, and
+  config validation rejects anything else.
+- Bodies over 8 MiB and non-200 responses count as down.
+- Metric names must not collide across sources — federation appends
+  expositions verbatim and does not rewrite names.
+- With federation enabled the endpoint always serves the plain-text
+  exposition format (no content negotiation), because federated bodies are
+  appended as-is.
+
+A complete example lives in `configs/examples/metrics-federate.yaml`.
+
 ## Available Metrics
 
 ### Process Lifecycle Metrics

go.mod

@@ -16,6 +16,8 @@ require (
 	github.com/charmbracelet/lipgloss v1.1.0
 	github.com/fsnotify/fsnotify v1.10.1
 	github.com/prometheus/client_golang v1.24.1
+	github.com/prometheus/client_model v0.6.2
+	github.com/prometheus/common v0.70.1
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/shirou/gopsutil/v4 v4.26.7
 	github.com/spf13/cobra v1.10.2
@@ -62,8 +64,6 @@ require (
 	github.com/muesli/termenv v0.16.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
-	github.com/prometheus/common v0.70.1 // indirect
 	github.com/prometheus/procfs v0.21.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/shirou/gopsutil/v3 v3.24.5 // indirect

internal/config/federate_test.go

@@ -0,0 +1,65 @@
+package config
+
+import (
+	"strings"
+	"testing"
+)
+
+func federateConfig(sources ...FederateSourceConfig) *Config {
+	cfg := &Config{
+		Version: "1.0",
+		Processes: map[string]*Process{
+			"dummy": {Command: []string{"sleep", "1"}},
+		},
+	}
+	cfg.Global.MetricsFederate = sources
+	cfg.SetDefaults()
+	return cfg
+}
+
+func validationErrors(cfg *Config) []string {
+	res, _ := cfg.ValidateComprehensive()
+	var msgs []string
+	for _, iss := range res.Errors {
+		msgs = append(msgs, iss.Field+": "+iss.Message)
+	}
+	return msgs
+}
+
+func TestFederateValidationAcceptsLoopback(t *testing.T) {
+	for _, url := range []string{
+		"http://127.0.0.1:9114/metrics",
+		"http://localhost:9114/metrics",
+		"http://[::1]:9114/metrics",
+	} {
+		cfg := federateConfig(FederateSourceConfig{Name: "ok", URL: url})
+		if errs := validationErrors(cfg); len(errs) != 0 {
+			t.Fatalf("loopback URL %q rejected: %v", url, errs)
+		}
+	}
+}
+
+func TestFederateValidationRejectsNonLoopback(t *testing.T) {
+	cfg := federateConfig(FederateSourceConfig{Name: "bad", URL: "http://10.0.0.5:9114/metrics"})
+	errs := validationErrors(cfg)
+	if len(errs) == 0 {
+		t.Fatal("non-loopback URL must be rejected - federation is not a proxy")
+	}
+	if !strings.Contains(strings.Join(errs, " "), "Non-loopback") {
+		t.Fatalf("unexpected errors: %v", errs)
+	}
+}
+
+func TestFederateValidationRejectsDuplicateNamesAndMissingFields(t *testing.T) {
+	cfg := federateConfig(
+		FederateSourceConfig{Name: "dup", URL: "http://127.0.0.1:1/metrics"},
+		FederateSourceConfig{Name: "dup", URL: "http://127.0.0.1:2/metrics"},
+		FederateSourceConfig{URL: "ftp://127.0.0.1/metrics"},
+	)
+	errs := strings.Join(validationErrors(cfg), " | ")
+	for _, want := range []string{"Duplicate", "needs a name", "Unsupported scheme"} {
+		if !strings.Contains(errs, want) {
+			t.Fatalf("missing %q in: %s", want, errs)
+		}
+	}
+}