diff --git a/changelog.d/fixed-14425-self-managed-startup-budget.md b/changelog.d/fixed-14425-self-managed-startup-budget.md new file mode 100644 index 0000000000..3926517b9e --- /dev/null +++ b/changelog.d/fixed-14425-self-managed-startup-budget.md @@ -0,0 +1 @@ +- Self-managed Kubernetes deployments and the manual provisioning template now allow the same ten-minute startup budget as hosted spokes, avoiding premature restarts during slow first boots. diff --git a/changelog.d/fixed-rootless-probe-storage.md b/changelog.d/fixed-rootless-probe-storage.md new file mode 100644 index 0000000000..723c614b26 --- /dev/null +++ b/changelog.d/fixed-rootless-probe-storage.md @@ -0,0 +1 @@ +- The rootless Podman qualification probe now keeps its temporary image store under `/var/tmp` to avoid quota-limited `/tmp` storage and preserves image-pull diagnostics when a prerequisite fails. diff --git a/src/deploy/k8s/deployment.yaml b/src/deploy/k8s/deployment.yaml index 3dcf454c98..dc6ccb3630 100644 --- a/src/deploy/k8s/deployment.yaml +++ b/src/deploy/k8s/deployment.yaml @@ -195,14 +195,15 @@ spec: # startupProbe holds the liveness clock off until the process has # finished booting, so a slow start can never race the liveness # timer into a restart loop. failureThreshold x periodSeconds - # (30 x 5s = 150s) is the boot budget. + # plus initialDelaySeconds (10s + 118 x 5s = 600s) matches + # spoke.HostedStartupBudget; keep in sync with its probe constants. startupProbe: httpGet: path: /api/health port: dashboard initialDelaySeconds: 10 periodSeconds: 5 - failureThreshold: 30 + failureThreshold: 118 livenessProbe: # /api/livez checks that the HTTP server responds AND that the # heartbeat loop is still *attempting* sends — i.e. it only fails diff --git a/src/deploy/probe_podman_rootless_netadmin.sh b/src/deploy/probe_podman_rootless_netadmin.sh index ee62c6ccc4..910736eec0 100755 --- a/src/deploy/probe_podman_rootless_netadmin.sh +++ b/src/deploy/probe_podman_rootless_netadmin.sh @@ -91,7 +91,9 @@ trap cleanup EXIT if [[ "$SHARED_STORE" != "true" ]]; then if [[ -z "$STORE" ]]; then - STORE="$(mktemp -d -t hive-rootless-probe-store-XXXXXX)" + # Keep multi-GB image layers off quota-limited /tmp (or tmpfs), as the + # rootful probe does. The private store is still removed by cleanup. + STORE="$(mktemp -d -p /var/tmp hive-rootless-probe-store-XXXXXX)" OWN_STORE="true" fi mkdir -p "${STORE}/graph" "${STORE}/run" "${STORE}/tmp" @@ -144,7 +146,9 @@ printf 'podman=%s rootless=%s cgroups=%s netbackend=%s rootlessnet=%s runtime=%s "$rootlessnet" "$(pinfo .Host.OCIRuntime.Name)" printf 'store=%s\nimage=%s\n\n' "${CONTAINERS_STORAGE_CONF:-}" "$IMAGE" -podman pull -q "$IMAGE" >/dev/null 2>&1 || fail_prereq "cannot pull ${IMAGE}" +# Keep registry and storage errors visible: exit 78 alone cannot distinguish +# an unavailable image from a transient registry failure or a full runner disk. +podman pull -q "$IMAGE" || fail_prereq "cannot pull ${IMAGE}" failures=0 note_fail() { printf ' RESULT: UNEXPECTED — %s\n' "$1"; failures=$((failures + 1)); } diff --git a/src/docs/manual-provisioning.md b/src/docs/manual-provisioning.md index 3fc0a6604e..fa9983944c 100644 --- a/src/docs/manual-provisioning.md +++ b/src/docs/manual-provisioning.md @@ -1406,11 +1406,13 @@ spec: - { name: HIVE_HUB_URL, value: "https://hive.hivecommons.dev" } # startupProbe keeps the liveness clock from starting until boot # finishes, so a slow start can't race liveness into a restart loop. + # Match spoke.HostedStartupBudget: 10s + 118 x 5s = 600s; keep + # these timings in sync with its hosted startup probe constants. startupProbe: httpGet: { path: /api/health, port: 3002 } initialDelaySeconds: 10 periodSeconds: 5 - failureThreshold: 30 + failureThreshold: 118 livenessProbe: # /api/livez (NOT /api/health) — it also fails when the heartbeat # loop stops *attempting* sends, so a wedged/dead heartbeat diff --git a/src/pkg/hub/saas_provision_startup_probe_test.go b/src/pkg/hub/saas_provision_startup_probe_test.go index 02561ceacf..1495e17eea 100644 --- a/src/pkg/hub/saas_provision_startup_probe_test.go +++ b/src/pkg/hub/saas_provision_startup_probe_test.go @@ -1,9 +1,12 @@ package hub import ( + "os" "strings" "testing" "time" + + "gopkg.in/yaml.v3" ) // #9692: the hosted spoke's startup probe allowed about 160s, and the first @@ -106,3 +109,37 @@ func TestHostedStartupProbeTemplateHasNoLiteralThreshold(t *testing.T) { t.Errorf("startupProbe block does not render failureThreshold from StartupProbeFailureThreshold:\n%s", block) } } + +// Static operator templates must give first boots the same migration budget as +// hosted provisioning, before a self-upgrade can backfill their probe settings. +func TestSelfManagedStartupProbesMatchHostedConstants(t *testing.T) { + for _, path := range []string{"../../deploy/k8s/deployment.yaml", "../../docs/manual-provisioning.md"} { + t.Run(path, func(t *testing.T) { + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + _, block, ok := strings.Cut(string(data), "startupProbe:") + if !ok { + t.Fatal("missing startupProbe") + } + block, _, ok = strings.Cut(block, "livenessProbe:") + if !ok { + t.Fatal("missing livenessProbe after startupProbe") + } + var probe map[string]any + if err := yaml.Unmarshal([]byte(block), &probe); err != nil { + t.Fatal(err) + } + for field, want := range map[string]int{ + "initialDelaySeconds": hostedStartupProbeInitialDelaySeconds, + "periodSeconds": hostedStartupProbePeriodSeconds, + "failureThreshold": hostedStartupProbeFailureThreshold, + } { + if got := probeInt(t, probe, field); got != want { + t.Errorf("startupProbe.%s = %d, want %d", field, got, want) + } + } + }) + } +}