From f5212ffb0b64b6430161b9de8a99464ba052f99d Mon Sep 17 00:00:00 2001 From: thatguywob Date: Tue, 27 Oct 2026 01:16:02 +0000 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=90=9B=20fix:=20align=20self-managed?= =?UTF-8?q?=20startup=20probe=20budgets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give the self-managed manifest and manual template the hosted ten-minute startup budget and guard their timing values against drift. Hive-Run: hivecommons/hive#14425 Hive-Plan: align-self-managed-startup-budget Hive-Spec: hivecommons/hive#14425#startup-probe-parity Signed-off-by: thatguywob --- ...fixed-14425-self-managed-startup-budget.md | 1 + src/deploy/k8s/deployment.yaml | 5 ++- src/docs/manual-provisioning.md | 4 +- .../hub/saas_provision_startup_probe_test.go | 37 +++++++++++++++++++ 4 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 changelog.d/fixed-14425-self-managed-startup-budget.md diff --git a/changelog.d/fixed-14425-self-managed-startup-budget.md b/changelog.d/fixed-14425-self-managed-startup-budget.md new file mode 100644 index 0000000000..3926517b9e --- /dev/null +++ b/changelog.d/fixed-14425-self-managed-startup-budget.md @@ -0,0 +1 @@ +- Self-managed Kubernetes deployments and the manual provisioning template now allow the same ten-minute startup budget as hosted spokes, avoiding premature restarts during slow first boots. diff --git a/src/deploy/k8s/deployment.yaml b/src/deploy/k8s/deployment.yaml index 3dcf454c98..dc6ccb3630 100644 --- a/src/deploy/k8s/deployment.yaml +++ b/src/deploy/k8s/deployment.yaml @@ -195,14 +195,15 @@ spec: # startupProbe holds the liveness clock off until the process has # finished booting, so a slow start can never race the liveness # timer into a restart loop. failureThreshold x periodSeconds - # (30 x 5s = 150s) is the boot budget. + # plus initialDelaySeconds (10s + 118 x 5s = 600s) matches + # spoke.HostedStartupBudget; keep in sync with its probe constants. startupProbe: httpGet: path: /api/health port: dashboard initialDelaySeconds: 10 periodSeconds: 5 - failureThreshold: 30 + failureThreshold: 118 livenessProbe: # /api/livez checks that the HTTP server responds AND that the # heartbeat loop is still *attempting* sends — i.e. it only fails diff --git a/src/docs/manual-provisioning.md b/src/docs/manual-provisioning.md index 3fc0a6604e..fa9983944c 100644 --- a/src/docs/manual-provisioning.md +++ b/src/docs/manual-provisioning.md @@ -1406,11 +1406,13 @@ spec: - { name: HIVE_HUB_URL, value: "https://hive.hivecommons.dev" } # startupProbe keeps the liveness clock from starting until boot # finishes, so a slow start can't race liveness into a restart loop. + # Match spoke.HostedStartupBudget: 10s + 118 x 5s = 600s; keep + # these timings in sync with its hosted startup probe constants. startupProbe: httpGet: { path: /api/health, port: 3002 } initialDelaySeconds: 10 periodSeconds: 5 - failureThreshold: 30 + failureThreshold: 118 livenessProbe: # /api/livez (NOT /api/health) — it also fails when the heartbeat # loop stops *attempting* sends, so a wedged/dead heartbeat diff --git a/src/pkg/hub/saas_provision_startup_probe_test.go b/src/pkg/hub/saas_provision_startup_probe_test.go index 02561ceacf..1495e17eea 100644 --- a/src/pkg/hub/saas_provision_startup_probe_test.go +++ b/src/pkg/hub/saas_provision_startup_probe_test.go @@ -1,9 +1,12 @@ package hub import ( + "os" "strings" "testing" "time" + + "gopkg.in/yaml.v3" ) // #9692: the hosted spoke's startup probe allowed about 160s, and the first @@ -106,3 +109,37 @@ func TestHostedStartupProbeTemplateHasNoLiteralThreshold(t *testing.T) { t.Errorf("startupProbe block does not render failureThreshold from StartupProbeFailureThreshold:\n%s", block) } } + +// Static operator templates must give first boots the same migration budget as +// hosted provisioning, before a self-upgrade can backfill their probe settings. +func TestSelfManagedStartupProbesMatchHostedConstants(t *testing.T) { + for _, path := range []string{"../../deploy/k8s/deployment.yaml", "../../docs/manual-provisioning.md"} { + t.Run(path, func(t *testing.T) { + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + _, block, ok := strings.Cut(string(data), "startupProbe:") + if !ok { + t.Fatal("missing startupProbe") + } + block, _, ok = strings.Cut(block, "livenessProbe:") + if !ok { + t.Fatal("missing livenessProbe after startupProbe") + } + var probe map[string]any + if err := yaml.Unmarshal([]byte(block), &probe); err != nil { + t.Fatal(err) + } + for field, want := range map[string]int{ + "initialDelaySeconds": hostedStartupProbeInitialDelaySeconds, + "periodSeconds": hostedStartupProbePeriodSeconds, + "failureThreshold": hostedStartupProbeFailureThreshold, + } { + if got := probeInt(t, probe, field); got != want { + t.Errorf("startupProbe.%s = %d, want %d", field, got, want) + } + } + }) + } +} From 215c571f3817a5c6eadde21eb2f27e0e3405dfe3 Mon Sep 17 00:00:00 2001 From: thatguywob Date: Tue, 27 Oct 2026 01:26:01 +0000 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=90=9B=20fix:=20preserve=20rootless?= =?UTF-8?q?=20image=20pull=20diagnostics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: thatguywob --- src/deploy/probe_podman_rootless_netadmin.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/deploy/probe_podman_rootless_netadmin.sh b/src/deploy/probe_podman_rootless_netadmin.sh index ee62c6ccc4..1e916da702 100755 --- a/src/deploy/probe_podman_rootless_netadmin.sh +++ b/src/deploy/probe_podman_rootless_netadmin.sh @@ -144,7 +144,9 @@ printf 'podman=%s rootless=%s cgroups=%s netbackend=%s rootlessnet=%s runtime=%s "$rootlessnet" "$(pinfo .Host.OCIRuntime.Name)" printf 'store=%s\nimage=%s\n\n' "${CONTAINERS_STORAGE_CONF:-}" "$IMAGE" -podman pull -q "$IMAGE" >/dev/null 2>&1 || fail_prereq "cannot pull ${IMAGE}" +# Keep registry and storage errors visible: exit 78 alone cannot distinguish +# an unavailable image from a transient registry failure or a full runner disk. +podman pull -q "$IMAGE" || fail_prereq "cannot pull ${IMAGE}" failures=0 note_fail() { printf ' RESULT: UNEXPECTED — %s\n' "$1"; failures=$((failures + 1)); } From be87431784c2c676676af48006e042e367947a16 Mon Sep 17 00:00:00 2001 From: thatguywob Date: Tue, 27 Oct 2026 01:27:38 +0000 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=90=9B=20fix:=20keep=20rootless=20pro?= =?UTF-8?q?be=20image=20layers=20on=20disk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: thatguywob --- src/deploy/probe_podman_rootless_netadmin.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/deploy/probe_podman_rootless_netadmin.sh b/src/deploy/probe_podman_rootless_netadmin.sh index 1e916da702..910736eec0 100755 --- a/src/deploy/probe_podman_rootless_netadmin.sh +++ b/src/deploy/probe_podman_rootless_netadmin.sh @@ -91,7 +91,9 @@ trap cleanup EXIT if [[ "$SHARED_STORE" != "true" ]]; then if [[ -z "$STORE" ]]; then - STORE="$(mktemp -d -t hive-rootless-probe-store-XXXXXX)" + # Keep multi-GB image layers off quota-limited /tmp (or tmpfs), as the + # rootful probe does. The private store is still removed by cleanup. + STORE="$(mktemp -d -p /var/tmp hive-rootless-probe-store-XXXXXX)" OWN_STORE="true" fi mkdir -p "${STORE}/graph" "${STORE}/run" "${STORE}/tmp" From 022bd1fa2faa83940276ee990c21961153c59e6c Mon Sep 17 00:00:00 2001 From: thatguywob Date: Tue, 27 Oct 2026 02:00:09 +0000 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=93=96=20docs:=20describe=20rootless?= =?UTF-8?q?=20probe=20CI=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: thatguywob --- changelog.d/fixed-rootless-probe-storage.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/fixed-rootless-probe-storage.md diff --git a/changelog.d/fixed-rootless-probe-storage.md b/changelog.d/fixed-rootless-probe-storage.md new file mode 100644 index 0000000000..723c614b26 --- /dev/null +++ b/changelog.d/fixed-rootless-probe-storage.md @@ -0,0 +1 @@ +- The rootless Podman qualification probe now keeps its temporary image store under `/var/tmp` to avoid quota-limited `/tmp` storage and preserves image-pull diagnostics when a prerequisite fails.