🐛 fix: align self-managed startup probe budgets

changelog.d/fixed-14425-self-managed-startup-budget.md

@@ -0,0 +1 @@
+- Self-managed Kubernetes deployments and the manual provisioning template now allow the same ten-minute startup budget as hosted spokes, avoiding premature restarts during slow first boots.

changelog.d/fixed-rootless-probe-storage.md

@@ -0,0 +1 @@
+- The rootless Podman qualification probe now keeps its temporary image store under `/var/tmp` to avoid quota-limited `/tmp` storage and preserves image-pull diagnostics when a prerequisite fails.

src/deploy/k8s/deployment.yaml

@@ -195,14 +195,15 @@ spec:
           # startupProbe holds the liveness clock off until the process has
           # finished booting, so a slow start can never race the liveness
           # timer into a restart loop. failureThreshold x periodSeconds
-          # (30 x 5s = 150s) is the boot budget.
+          # plus initialDelaySeconds (10s + 118 x 5s = 600s) matches
+          # spoke.HostedStartupBudget; keep in sync with its probe constants.
           startupProbe:
             httpGet:
               path: /api/health
               port: dashboard
             initialDelaySeconds: 10
             periodSeconds: 5
-            failureThreshold: 30
+            failureThreshold: 118
           livenessProbe:
             # /api/livez checks that the HTTP server responds AND that the
             # heartbeat loop is still *attempting* sends — i.e. it only fails

src/deploy/probe_podman_rootless_netadmin.sh

@@ -91,7 +91,9 @@ trap cleanup EXIT
 
 if [[ "$SHARED_STORE" != "true" ]]; then
   if [[ -z "$STORE" ]]; then
-    STORE="$(mktemp -d -t hive-rootless-probe-store-XXXXXX)"
+    # Keep multi-GB image layers off quota-limited /tmp (or tmpfs), as the
+    # rootful probe does. The private store is still removed by cleanup.
+    STORE="$(mktemp -d -p /var/tmp hive-rootless-probe-store-XXXXXX)"
     OWN_STORE="true"
   fi
   mkdir -p "${STORE}/graph" "${STORE}/run" "${STORE}/tmp"
@@ -144,7 +146,9 @@ printf 'podman=%s rootless=%s cgroups=%s netbackend=%s rootlessnet=%s runtime=%s
   "$rootlessnet" "$(pinfo .Host.OCIRuntime.Name)"
 printf 'store=%s\nimage=%s\n\n' "${CONTAINERS_STORAGE_CONF:-<caller default>}" "$IMAGE"
 
-podman pull -q "$IMAGE" >/dev/null 2>&1 || fail_prereq "cannot pull ${IMAGE}"
+# Keep registry and storage errors visible: exit 78 alone cannot distinguish
+# an unavailable image from a transient registry failure or a full runner disk.
+podman pull -q "$IMAGE" || fail_prereq "cannot pull ${IMAGE}"
 
 failures=0
 note_fail() { printf '  RESULT: UNEXPECTED — %s\n' "$1"; failures=$((failures + 1)); }

src/docs/manual-provisioning.md

@@ -1406,11 +1406,13 @@ spec:
         - { name: HIVE_HUB_URL,    value: "https://hive.hivecommons.dev" }
         # startupProbe keeps the liveness clock from starting until boot
         # finishes, so a slow start can't race liveness into a restart loop.
+        # Match spoke.HostedStartupBudget: 10s + 118 x 5s = 600s; keep
+        # these timings in sync with its hosted startup probe constants.
         startupProbe:
           httpGet: { path: /api/health, port: 3002 }
           initialDelaySeconds: 10
           periodSeconds: 5
-          failureThreshold: 30
+          failureThreshold: 118
         livenessProbe:
           # /api/livez (NOT /api/health) — it also fails when the heartbeat
           # loop stops *attempting* sends, so a wedged/dead heartbeat

src/pkg/hub/saas_provision_startup_probe_test.go

@@ -1,9 +1,12 @@
 package hub
 
 import (
+	"os"
 	"strings"
 	"testing"
 	"time"
+
+	"gopkg.in/yaml.v3"
 )
 
 // #9692: the hosted spoke's startup probe allowed about 160s, and the first
@@ -106,3 +109,37 @@ func TestHostedStartupProbeTemplateHasNoLiteralThreshold(t *testing.T) {
 		t.Errorf("startupProbe block does not render failureThreshold from StartupProbeFailureThreshold:\n%s", block)
 	}
 }
+
+// Static operator templates must give first boots the same migration budget as
+// hosted provisioning, before a self-upgrade can backfill their probe settings.
+func TestSelfManagedStartupProbesMatchHostedConstants(t *testing.T) {
+	for _, path := range []string{"../../deploy/k8s/deployment.yaml", "../../docs/manual-provisioning.md"} {
+		t.Run(path, func(t *testing.T) {
+			data, err := os.ReadFile(path)
+			if err != nil {
+				t.Fatal(err)
+			}
+			_, block, ok := strings.Cut(string(data), "startupProbe:")
+			if !ok {
+				t.Fatal("missing startupProbe")
+			}
+			block, _, ok = strings.Cut(block, "livenessProbe:")
+			if !ok {
+				t.Fatal("missing livenessProbe after startupProbe")
+			}
+			var probe map[string]any
+			if err := yaml.Unmarshal([]byte(block), &probe); err != nil {
+				t.Fatal(err)
+			}
+			for field, want := range map[string]int{
+				"initialDelaySeconds": hostedStartupProbeInitialDelaySeconds,
+				"periodSeconds":       hostedStartupProbePeriodSeconds,
+				"failureThreshold":    hostedStartupProbeFailureThreshold,
+			} {
+				if got := probeInt(t, probe, field); got != want {
+					t.Errorf("startupProbe.%s = %d, want %d", field, got, want)
+				}
+			}
+		})
+	}
+}