fix: harden Trino registry-only routes and fixture readiness

docs/trino-cells.md

@@ -107,6 +107,11 @@ or live observer polls. A stopped green does not make blue's tenants unhealthy.
 When green starts, update the registry and restart the control plane. Every
 running backend must reconcile successfully before a tenant is reported ready;
 one successful coordinator cannot conceal another's missing catalog or failure.
+This state confirms control-plane and coordinator catalog reconciliation, not
+tenant-password volume convergence on every worker. A worker can briefly lack
+a newly added tenant's password file even after the coordinator reports ready.
+Initial testing must check worker file availability before the first write;
+do not blindly retry writes when their commit outcome is unknown.
 The console observes the configured routing-active backend. Usage collection
 polls each running backend independently under the existing leader lease.
 

tests/mw-dev/README.md

@@ -542,6 +542,18 @@ queries the same data through green's independently hydrated catalog. Direct
 coordinator URLs are fixture-only; this does not test Gateway routing or a
 maintenance move of an existing warehouse.
 
+Before each tenant's first write, the fixture checks the expected password file
+is readable on every ready worker, including all three legacy workers. It
+repeats that check for green before querying. These bounded checks use only
+`test -r`; they never read password contents or retry writes. The fixture's
+primary and secondary namespace Roles permit the required pod reads and exec
+under the existing CI grant. No production
+or cluster-wide RBAC changes are required. Control-plane `ready` alone does
+not prove every worker's projected Secret volume has converged.
+The worker checks use a process timeout rather than kubectl's request-timeout
+flag. The pinned client otherwise loses its implicit in-cluster configuration
+and attempts to connect to localhost. No token or kubeconfig is copied.
+
 After these checks, the lane restarts its control plane in registry-only mode,
 verifies both registered backends still query the warehouse, rejects legacy
 ownership and implicit cell selection, and verifies the legacy bundle endpoint

tests/mw-dev/e2e/trino-multicell.sh

@@ -8,6 +8,22 @@ LEGACY_TRINO="$TRINO"
 BLUE_TRINO="https://duckgres-trino-blue.$CELL_NS.svc:8443"
 GREEN_TRINO="https://duckgres-trino-green.$CELL_NS.svc:8443"
 
+wait_cell_registry() {
+  expected_cells="$1"
+  attempt=0
+  while [ "$attempt" -lt 36 ]; do
+    result="$(api --max-time 5 "$API/api/v1/trino/cells" 2>/dev/null || true)"
+    if printf %s "$result" | jq -e --argjson expected "$expected_cells" \
+      '.cells | map(.id) | sort == $expected' >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 5
+    attempt=$((attempt + 1))
+  done
+  fail "cell registry did not converge after control-plane restart"
+}
+
+
 log "multicell initial placement with green stopped"
 api "$API/api/v1/trino/cells" | jq -e '.cells | map(.id) | sort == ["cell-test","legacy"]' >/dev/null \
   || fail "both cells must be registered"
@@ -67,6 +83,7 @@ code="$(curl --connect-timeout 5 --max-time 30 -sS -o /tmp/trino-cell-selection-
 
 TRINO="$BLUE_TRINO"
 wait_cell_auth
+wait_worker_tenant_file blue "$ORG_C"
 trino_query "$DB_C" "$pw_c" "CREATE SCHEMA $CAT_C.cell_test" >/dev/null
 trino_query "$DB_C" "$pw_c" "CREATE TABLE $CAT_C.cell_test.values_test (value BIGINT)" >/dev/null
 trino_query "$DB_C" "$pw_c" "INSERT INTO $CAT_C.cell_test.values_test VALUES (7),(11)" >/dev/null
@@ -114,6 +131,7 @@ done
 wait_cell_ready
 TRINO="$GREEN_TRINO"
 wait_cell_auth
+wait_worker_tenant_file green "$ORG_C"
 result="$(trino_query "$DB_C" "$pw_c" "SELECT COUNT(*), SUM(value) FROM $CAT_C.cell_test.values_test")"
 [ "$result" = '[[2,18]]' ] || fail "green failed to hydrate its independent catalog from the same DuckLake warehouse"
 must_fail "$DB_A" "$pw_a" 'SELECT 1' '401|Unauthorized|Authentication|credentials'
@@ -130,8 +148,7 @@ legacy_owner="$(api "$API/api/v1/orgs/$ORG_A" | jq -r .trino.trino_cell_id)"
 "$KUBECTL" -n "$NS" patch deployment duckgres-control-plane --type=strategic -p \
   '{"spec":{"template":{"spec":{"containers":[{"name":"controlplane","env":[{"name":"DUCKGRES_TRINO_COORDINATOR_URL","$patch":"delete"},{"name":"DUCKGRES_TRINO_REGISTRY_ONLY","value":"true"}]}]}}}}' >/dev/null
 "$KUBECTL" -n "$NS" rollout status deployment/duckgres-control-plane --timeout=180s >/dev/null
-api "$API/api/v1/trino/cells" | jq -e '.cells | map(.id) == ["cell-test"]' >/dev/null \
-  || fail "registry-only startup invented a legacy cell"
+wait_cell_registry '["cell-test"]'
 wait_cell_ready
 for endpoint in "$BLUE_TRINO" "$GREEN_TRINO"; do
   TRINO="$endpoint"
@@ -162,6 +179,7 @@ log "restore legacy fixture configuration"
 patch="$(printf %s "$legacy_env" | jq -c '{spec:{template:{spec:{containers:[{name:"controlplane",env:[.,{name:"DUCKGRES_TRINO_REGISTRY_ONLY","$patch":"delete"}]}]}}}}')"
 "$KUBECTL" -n "$NS" patch deployment duckgres-control-plane --type=strategic -p "$patch" >/dev/null
 "$KUBECTL" -n "$NS" rollout status deployment/duckgres-control-plane --timeout=180s >/dev/null
+wait_cell_registry '["cell-test","legacy"]'
 TRINO="$LEGACY_TRINO"
 [ "$(trino_query "$DB_A" "$pw_a" 'SELECT 1')" = '[[1]]' ] || fail "legacy query failed after registry-only fixture restore"
 log "PASS: registry-only startup + explicit selection + no legacy dependency + restored fixture"

tests/mw-dev/e2e/trino.sh

@@ -32,6 +32,7 @@ TRINO_AUTH_ROTATION_RETRY_SECONDS=5
 fail() { echo "FAIL: $*" >&2; exit 1; }
 log() { echo ">>> $*" >&2; }
 apk add --no-cache curl jq >/dev/null 2>&1
+command -v timeout >/dev/null || fail "timeout is required for bounded worker checks"
 [ -s "$CA" ] || fail "per-run Trino CA is not mounted"
 CP_IP="$(getent hosts "$PGHOST" | awk '{print $1}' | head -1)"
 [ -n "$CP_IP" ] || fail "could not resolve $PGHOST"
@@ -43,6 +44,74 @@ chmod +x "$KUBECTL"
 
 api() { curl --connect-timeout 5 --max-time 60 -fsS -H "$H" "$@"; }
 
+wait_worker_tenant_file() {
+  worker_color="$1"
+  worker_org="$2"
+  case "$PR" in ''|*[!0-9]*|0*) fail "invalid fixture identity" ;; esac
+  case "$worker_color" in
+    legacy)
+      [ "$NS" = "duckgres-ci-pr-$PR" ] || fail "worker mount check escaped fixture identity"
+      case "$worker_org" in
+        "ci-pr-$PR-trinoa"|"ci-pr-$PR-trinob") ;;
+        *) fail "worker mount check crossed tenant boundary" ;;
+      esac
+      worker_namespace="$NS"
+      worker_app=duckgres-trino
+      ;;
+    blue|green)
+      [ "${CELL_NS:-}" = "duckgres-ci-pr-0$PR" ] && [ "$worker_org" = "ci-pr-$PR-trinoc" ] \
+        || fail "worker mount check escaped fixture identity"
+      worker_namespace="$CELL_NS"
+      worker_app="duckgres-trino-$worker_color"
+      ;;
+    *) fail "invalid worker color" ;;
+  esac
+  attempt=0
+  last_worker_stage=""
+  while [ "$attempt" -lt 36 ]; do
+    worker_stage=deployment-readiness
+    worker_code=0
+    deployment="$(timeout 5 "$KUBECTL" -n "$worker_namespace" get deployment "$worker_app-worker" -o json 2>/dev/null)" \
+      || { worker_code=$?; worker_stage=deployment-read; }
+    replicas="$(printf %s "$deployment" | jq -er 'select(.metadata.generation == .status.observedGeneration and .spec.replicas > 0 and .status.readyReplicas == .spec.replicas and .status.updatedReplicas == .spec.replicas) | .spec.replicas' 2>/dev/null || true)"
+    if [ "$worker_code" = 0 ] && [ -n "$replicas" ]; then
+      worker_stage=pod-readiness
+      snapshot="$(timeout 5 "$KUBECTL" -n "$worker_namespace" get pods -l "app=$worker_app,component=worker" -o json 2>/dev/null)" \
+        || { worker_code=$?; worker_stage=pod-read; }
+      workers="$(printf %s "$snapshot" | jq -er --arg app "$worker_app" --argjson replicas "$replicas" \
+        'select($replicas > 0 and (.items | length) == $replicas and all(.items[]; .metadata.deletionTimestamp == null and .metadata.labels.app == $app and .metadata.labels.component == "worker" and (.metadata.name | startswith($app + "-worker-")) and .status.phase == "Running" and any(.status.conditions[]?; .type == "Ready" and .status == "True"))) | .items[].metadata.name' 2>/dev/null || true)"
+      if [ "$worker_code" = 0 ] && [ -n "$workers" ]; then
+        mounted=1
+        worker_stage=worker-file
+        for worker in $workers; do
+          if worker_result="$(timeout 5 "$KUBECTL" -n "$worker_namespace" exec "$worker" -c trino-worker \
+            -- test -r "/etc/trino/tenant-secrets/$worker_org" 2>&1)"; then
+            :
+          else
+            worker_code=$?
+            mounted=0
+            case "$worker_result" in
+              *'command terminated with exit code 1'*) worker_stage=worker-file ;;
+              *Forbidden*|*forbidden*) worker_stage=worker-exec-permission ;;
+              *'deadline exceeded'*|*'timed out'*) worker_stage=worker-exec-timeout ;;
+              *'executable file not found'*) worker_stage=worker-exec-program ;;
+              *) worker_stage=worker-exec ;;
+            esac
+          fi
+        done
+        [ "$mounted" = 1 ] && return 0
+      fi
+    fi
+    if [ "$last_worker_stage" != "$worker_stage:$worker_code" ]; then
+      log "Worker mount readiness: $worker_stage (exit $worker_code)"
+      last_worker_stage="$worker_stage:$worker_code"
+    fi
+    sleep 5
+    attempt=$((attempt + 1))
+  done
+  fail "worker tenant password file did not converge in the isolated cell"
+}
+
 # A fresh managed warehouse has an empty metadata database. DuckLake's Trino
 # connector consumes an existing DuckLake catalog; it does not create the
 # metadata tables itself. Initialize them once through Duckgres's normal
@@ -141,6 +210,7 @@ pw_a="$(provision "$ORG_A" "$DB_A" "$TEAM_A" | jq -r .password)"
 wait_warehouse "$ORG_A"
 bootstrap_ducklake "$ORG_A" "$pw_a"
 wait_trino "$ORG_A" "$DB_A" "$CAT_A"
+wait_worker_tenant_file legacy "$ORG_A"
 
 log "TLS/password auth, discovery, and DDL/DML"
 [ "$(scalar "$DB_A" "$pw_a" 'SELECT 1')" = 1 ] || fail "Trino SELECT 1 failed"
@@ -188,6 +258,7 @@ pw_b="$(provision "$ORG_B" "$DB_B" "$TEAM_B" | jq -r .password)"
 wait_warehouse "$ORG_B"
 bootstrap_ducklake "$ORG_B" "$pw_b"
 wait_trino "$ORG_B" "$DB_B" "$CAT_B"
+wait_worker_tenant_file legacy "$ORG_B"
 [ "$("$KUBECTL" -n "$NS" get pod -l 'app=duckgres-trino,component=coordinator' -o jsonpath='{.items[0].metadata.uid}')" = "$coord_uid_before" ] \
   || fail "adding tenant B restarted the Trino coordinator"
 [ "$(scalar "$DB_B" "$pw_b" 'SELECT 1')" = 1 ] || fail "hot-added tenant cannot authenticate"

tests/mw-dev/manifests.tmpl.yaml

@@ -159,6 +159,10 @@ rules:
   - apiGroups: [""]
     resources: ["pods/log"]
     verbs: ["get"]
+  # The fixture checks password-file availability without reading its contents.
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    verbs: ["get", "create"]
   # The Trino lane deletes coordinator/worker pods and uses rollout status to
   # prove their owning Deployments restore readiness. rollout status lists
   # before watching, so keep this read-only and namespace-scoped; every

tests/mw-dev/trino-multicell.tmpl.yaml

@@ -39,6 +39,13 @@ metadata:
   name: trino-cell-projection
   namespace: ${TRINO_CELL_NAMESPACE}
 rules:
+  # The fixture checks worker file availability without reading file contents.
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    verbs: ["create", "get"]
   - apiGroups: [""]
     resources: ["secrets", "configmaps"]
     verbs: ["create"]

tests/mw-dev/trino_multicell_test.go

@@ -56,6 +56,65 @@ func TestTrinoRegistryOnlyHarnessFollowsLegacyCompatibility(t *testing.T) {
 	}
 }
 
+func TestTrinoCellRegistryWaitsForServiceConvergence(t *testing.T) {
+	raw, err := os.ReadFile("e2e/trino-multicell.sh")
+	if err != nil {
+		t.Fatal(err)
+	}
+	text := string(raw)
+	start := strings.Index(text, "wait_cell_registry() {")
+	if start < 0 {
+		t.Fatal("missing bounded registry convergence helper")
+	}
+	end := strings.Index(text[start:], "\n}\n")
+	if end < 0 {
+		t.Fatal("missing registry helper end")
+	}
+	helper := text[start : start+end+3]
+	for _, tc := range []struct {
+		name, responses string
+		wantSuccess     bool
+	}{
+		{"connection refused then ready", "refused\ncell-test\n", true},
+		{"old endpoint then ready", "cell-test,legacy\ncell-test\n", true},
+		{"wrong registry never accepted", "cell-test,legacy\n", false},
+		{"unavailable API times out", "refused\n", false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			responses := filepath.Join(t.TempDir(), "responses")
+			if err := os.WriteFile(responses, []byte(tc.responses), 0o600); err != nil {
+				t.Fatal(err)
+			}
+			script := `set -eu
+api() {
+  [ "$1" = --max-time ] && [ "$2" = 5 ] || exit 90
+  response=$(head -n 1 "$RESPONSES")
+  if [ "$(wc -l < "$RESPONSES")" -gt 1 ]; then
+    tail -n +2 "$RESPONSES" > "$RESPONSES.next"
+    mv "$RESPONSES.next" "$RESPONSES"
+  fi
+  [ "$response" != refused ] || return 7
+  printf %s "$response" | jq -Rc '{cells:(split(",") | map({id:.}))}'
+}
+sleep() { :; }
+fail() { echo "$*" >&2; exit 1; }
+API=http://fixture.invalid
+` + helper + `
+wait_cell_registry '["cell-test"]'
+`
+			cmd := exec.Command("sh", "-c", script)
+			cmd.Env = append(os.Environ(), "RESPONSES="+responses)
+			output, err := cmd.CombinedOutput()
+			if (err == nil) != tc.wantSuccess {
+				t.Fatalf("success=%v, want=%v: %s", err == nil, tc.wantSuccess, output)
+			}
+			if !tc.wantSuccess && !strings.Contains(string(output), "cell registry did not converge") {
+				t.Fatalf("missing actionable timeout: %s", output)
+			}
+		})
+	}
+}
+
 func TestTrinoMulticellRenderedBackendsAreIsolated(t *testing.T) {
 	envsubst, err := exec.LookPath("envsubst")
 	if err != nil {
@@ -89,6 +148,8 @@ func TestTrinoMulticellRenderedBackendsAreIsolated(t *testing.T) {
 	deployments := map[string]map[string]any{}
 	secrets := []map[string]any{}
 	publicManifests := []map[string]any{}
+	workerPermissions := map[string]bool{}
+	primaryWorkerExec := false
 	for {
 		var manifest map[string]any
 		if err := decoder.Decode(&manifest); err == io.EOF {
@@ -110,12 +171,62 @@ func TestTrinoMulticellRenderedBackendsAreIsolated(t *testing.T) {
 		if manifest["kind"] == "Deployment" {
 			deployments[manifestName(manifest)] = manifest
 		}
+		if manifest["kind"] == "Role" && manifestName(manifest) == "duckgres-control-plane" {
+			if manifest["metadata"].(map[string]any)["namespace"] != "duckgres-ci-pr-123" {
+				t.Fatal("legacy worker inspection must remain in the isolated primary namespace")
+			}
+			for _, rawRule := range manifest["rules"].([]any) {
+				rule := rawRule.(map[string]any)
+				resources, err := json.Marshal(rule["resources"])
+				if err != nil {
+					t.Fatal(err)
+				}
+				if string(resources) == `["pods/exec"]` {
+					verbs, err := json.Marshal(rule["verbs"])
+					if err != nil || string(verbs) != `["get","create"]` {
+						t.Fatal("legacy worker exec must use only get/create")
+					}
+					primaryWorkerExec = true
+				}
+			}
+		}
+		if manifest["kind"] == "Role" && manifestName(manifest) == "trino-cell-projection" {
+			if manifest["metadata"].(map[string]any)["namespace"] != "duckgres-ci-pr-0123" {
+				t.Fatal("worker inspection permissions must remain in the isolated secondary namespace")
+			}
+			for _, rawRule := range manifest["rules"].([]any) {
+				rule := rawRule.(map[string]any)
+				for _, resource := range rule["resources"].([]any) {
+					if resource != "pods" && resource != "pods/exec" {
+						continue
+					}
+					verbs, err := json.Marshal(rule["verbs"])
+					if err != nil {
+						t.Fatal(err)
+					}
+					want := `["get","list"]`
+					if resource == "pods/exec" {
+						want = `["create","get"]`
+					}
+					if string(verbs) != want {
+						t.Fatalf("unexpected worker inspection verbs: %s", verbs)
+					}
+					workerPermissions[resource.(string)] = true
+				}
+			}
+		}
 		if manifest["kind"] == "Secret" {
 			secrets = append(secrets, manifest)
 		} else {
 			publicManifests = append(publicManifests, manifest)
 		}
 	}
+	if !workerPermissions["pods"] || !workerPermissions["pods/exec"] {
+		t.Fatal("missing isolated worker mount inspection permissions")
+	}
+	if !primaryWorkerExec {
+		t.Fatal("missing isolated legacy worker inspection permission")
+	}
 	passwordBytes, err := os.ReadFile(filepath.Join(secretDir, "duckgres-ci-config-store-password"))
 	if err != nil {
 		t.Fatal("renderer must generate a per-run config-store credential:", err)