From d515dec37edef9a47c76914c68671960ac8a3da0 Mon Sep 17 00:00:00 2001 From: Benjamin Knofe-Vider Date: Fri, 11 Sep 2026 16:49:42 +0200 Subject: [PATCH 1/5] test: wait for Trino registry after control-plane rollouts --- tests/mw-dev/e2e/trino-multicell.sh | 19 ++++++++- tests/mw-dev/trino_multicell_test.go | 59 ++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/tests/mw-dev/e2e/trino-multicell.sh b/tests/mw-dev/e2e/trino-multicell.sh index 87fc89fa..0d10904f 100644 --- a/tests/mw-dev/e2e/trino-multicell.sh +++ b/tests/mw-dev/e2e/trino-multicell.sh @@ -8,6 +8,21 @@ LEGACY_TRINO="$TRINO" BLUE_TRINO="https://duckgres-trino-blue.$CELL_NS.svc:8443" GREEN_TRINO="https://duckgres-trino-green.$CELL_NS.svc:8443" +wait_cell_registry() { + expected_cells="$1" + attempt=0 + while [ "$attempt" -lt 36 ]; do + result="$(api --max-time 5 "$API/api/v1/trino/cells" 2>/dev/null || true)" + if printf %s "$result" | jq -e --argjson expected "$expected_cells" \ + '.cells | map(.id) | sort == $expected' >/dev/null 2>&1; then + return 0 + fi + sleep 5 + attempt=$((attempt + 1)) + done + fail "cell registry did not converge after control-plane restart" +} + log "multicell initial placement with green stopped" api "$API/api/v1/trino/cells" | jq -e '.cells | map(.id) | sort == ["cell-test","legacy"]' >/dev/null \ || fail "both cells must be registered" @@ -130,8 +145,7 @@ legacy_owner="$(api "$API/api/v1/orgs/$ORG_A" | jq -r .trino.trino_cell_id)" "$KUBECTL" -n "$NS" patch deployment duckgres-control-plane --type=strategic -p \ '{"spec":{"template":{"spec":{"containers":[{"name":"controlplane","env":[{"name":"DUCKGRES_TRINO_COORDINATOR_URL","$patch":"delete"},{"name":"DUCKGRES_TRINO_REGISTRY_ONLY","value":"true"}]}]}}}}' >/dev/null "$KUBECTL" -n "$NS" rollout status deployment/duckgres-control-plane --timeout=180s >/dev/null -api "$API/api/v1/trino/cells" | jq -e '.cells | map(.id) == ["cell-test"]' >/dev/null \ - || fail "registry-only startup invented a legacy cell" +wait_cell_registry '["cell-test"]' wait_cell_ready for endpoint in "$BLUE_TRINO" "$GREEN_TRINO"; do TRINO="$endpoint" @@ -162,6 +176,7 @@ log "restore legacy fixture configuration" patch="$(printf %s "$legacy_env" | jq -c '{spec:{template:{spec:{containers:[{name:"controlplane",env:[.,{name:"DUCKGRES_TRINO_REGISTRY_ONLY","$patch":"delete"}]}]}}}}')" "$KUBECTL" -n "$NS" patch deployment duckgres-control-plane --type=strategic -p "$patch" >/dev/null "$KUBECTL" -n "$NS" rollout status deployment/duckgres-control-plane --timeout=180s >/dev/null +wait_cell_registry '["cell-test","legacy"]' TRINO="$LEGACY_TRINO" [ "$(trino_query "$DB_A" "$pw_a" 'SELECT 1')" = '[[1]]' ] || fail "legacy query failed after registry-only fixture restore" log "PASS: registry-only startup + explicit selection + no legacy dependency + restored fixture" diff --git a/tests/mw-dev/trino_multicell_test.go b/tests/mw-dev/trino_multicell_test.go index 1dc46538..666cb725 100644 --- a/tests/mw-dev/trino_multicell_test.go +++ b/tests/mw-dev/trino_multicell_test.go @@ -56,6 +56,65 @@ func TestTrinoRegistryOnlyHarnessFollowsLegacyCompatibility(t *testing.T) { } } +func TestTrinoCellRegistryWaitsForServiceConvergence(t *testing.T) { + raw, err := os.ReadFile("e2e/trino-multicell.sh") + if err != nil { + t.Fatal(err) + } + text := string(raw) + start := strings.Index(text, "wait_cell_registry() {") + if start < 0 { + t.Fatal("missing bounded registry convergence helper") + } + end := strings.Index(text[start:], "\n}\n") + if end < 0 { + t.Fatal("missing registry helper end") + } + helper := text[start : start+end+3] + for _, tc := range []struct { + name, responses string + wantSuccess bool + }{ + {"connection refused then ready", "refused\ncell-test\n", true}, + {"old endpoint then ready", "cell-test,legacy\ncell-test\n", true}, + {"wrong registry never accepted", "cell-test,legacy\n", false}, + {"unavailable API times out", "refused\n", false}, + } { + t.Run(tc.name, func(t *testing.T) { + responses := filepath.Join(t.TempDir(), "responses") + if err := os.WriteFile(responses, []byte(tc.responses), 0o600); err != nil { + t.Fatal(err) + } + script := `set -eu +api() { + [ "$1" = --max-time ] && [ "$2" = 5 ] || exit 90 + response=$(head -n 1 "$RESPONSES") + if [ "$(wc -l < "$RESPONSES")" -gt 1 ]; then + tail -n +2 "$RESPONSES" > "$RESPONSES.next" + mv "$RESPONSES.next" "$RESPONSES" + fi + [ "$response" != refused ] || return 7 + printf %s "$response" | jq -Rc '{cells:(split(",") | map({id:.}))}' +} +sleep() { :; } +fail() { echo "$*" >&2; exit 1; } +API=http://fixture.invalid +` + helper + ` +wait_cell_registry '["cell-test"]' +` + cmd := exec.Command("sh", "-c", script) + cmd.Env = append(os.Environ(), "RESPONSES="+responses) + output, err := cmd.CombinedOutput() + if (err == nil) != tc.wantSuccess { + t.Fatalf("success=%v, want=%v: %s", err == nil, tc.wantSuccess, output) + } + if !tc.wantSuccess && !strings.Contains(string(output), "cell registry did not converge") { + t.Fatalf("missing actionable timeout: %s", output) + } + }) + } +} + func TestTrinoMulticellRenderedBackendsAreIsolated(t *testing.T) { envsubst, err := exec.LookPath("envsubst") if err != nil { From e2cdc00e4208c5537efd3b3d73b162ccb95a7014 Mon Sep 17 00:00:00 2001 From: Benjamin Knofe-Vider Date: Fri, 11 Sep 2026 17:22:58 +0200 Subject: [PATCH 2/5] test: gate initial Trino writes on worker password mounts --- docs/trino-cells.md | 5 + tests/mw-dev/README.md | 8 ++ tests/mw-dev/e2e/trino-multicell.sh | 30 ++++++ tests/mw-dev/trino-multicell.tmpl.yaml | 7 ++ tests/mw-dev/trino_multicell_test.go | 29 +++++ tests/mw-dev/trino_worker_projection_test.go | 108 +++++++++++++++++++ 6 files changed, 187 insertions(+) create mode 100644 tests/mw-dev/trino_worker_projection_test.go diff --git a/docs/trino-cells.md b/docs/trino-cells.md index 796491f3..8d3444c6 100644 --- a/docs/trino-cells.md +++ b/docs/trino-cells.md @@ -107,6 +107,11 @@ or live observer polls. A stopped green does not make blue's tenants unhealthy. When green starts, update the registry and restart the control plane. Every running backend must reconcile successfully before a tenant is reported ready; one successful coordinator cannot conceal another's missing catalog or failure. +This state confirms control-plane and coordinator catalog reconciliation, not +tenant-password volume convergence on every worker. A worker can briefly lack +a newly added tenant's password file even after the coordinator reports ready. +Initial testing must check worker file availability before the first write; +do not blindly retry writes when their commit outcome is unknown. The console observes the configured routing-active backend. Usage collection polls each running backend independently under the existing leader lease. diff --git a/tests/mw-dev/README.md b/tests/mw-dev/README.md index fb670c4f..0a13a732 100644 --- a/tests/mw-dev/README.md +++ b/tests/mw-dev/README.md @@ -542,6 +542,14 @@ queries the same data through green's independently hydrated catalog. Direct coordinator URLs are fixture-only; this does not test Gateway routing or a maintenance move of an existing warehouse. +Before the first write, the fixture checks the expected tenant password file +is readable on every ready blue worker. It repeats that check for green before +querying. These bounded checks use only `test -r`; they never read password +contents or retry writes. The fixture's secondary namespace Role permits pod +listing and exec for these checks under the existing CI grant. No production +or cluster-wide RBAC changes are required. Control-plane `ready` alone does +not prove every worker's projected Secret volume has converged. + After these checks, the lane restarts its control plane in registry-only mode, verifies both registered backends still query the warehouse, rejects legacy ownership and implicit cell selection, and verifies the legacy bundle endpoint diff --git a/tests/mw-dev/e2e/trino-multicell.sh b/tests/mw-dev/e2e/trino-multicell.sh index 0d10904f..0a27cfd8 100644 --- a/tests/mw-dev/e2e/trino-multicell.sh +++ b/tests/mw-dev/e2e/trino-multicell.sh @@ -23,6 +23,34 @@ wait_cell_registry() { fail "cell registry did not converge after control-plane restart" } +wait_worker_tenant_file() { + worker_color="$1" + case "$worker_color" in blue|green) ;; *) fail "invalid worker color" ;; esac + case "$PR" in ''|*[!0-9]*|0*) fail "invalid fixture identity" ;; esac + [ "$CELL_NS" = "duckgres-ci-pr-0$PR" ] && [ "$ORG_C" = "ci-pr-$PR-trinoc" ] \ + || fail "worker mount check escaped fixture identity" + worker_app="duckgres-trino-$worker_color" + attempt=0 + while [ "$attempt" -lt 36 ]; do + deployment="$("$KUBECTL" --request-timeout=5s -n "$CELL_NS" get deployment "$worker_app-worker" -o json 2>/dev/null || true)" + replicas="$(printf %s "$deployment" | jq -er 'select(.metadata.generation == .status.observedGeneration and .spec.replicas > 0 and .status.readyReplicas == .spec.replicas and .status.updatedReplicas == .spec.replicas) | .spec.replicas' 2>/dev/null || true)" + snapshot="$("$KUBECTL" --request-timeout=5s -n "$CELL_NS" get pods -l "app=$worker_app,component=worker" -o json 2>/dev/null || true)" + workers="$(printf %s "$snapshot" | jq -er --arg app "$worker_app" --argjson replicas "${replicas:-0}" \ + 'select($replicas > 0 and (.items | length) == $replicas and all(.items[]; .metadata.deletionTimestamp == null and .metadata.labels.app == $app and .metadata.labels.component == "worker" and (.metadata.name | startswith($app + "-worker-")) and .status.phase == "Running" and any(.status.conditions[]?; .type == "Ready" and .status == "True"))) | .items[].metadata.name' 2>/dev/null || true)" + if [ -n "$workers" ]; then + mounted=1 + for worker in $workers; do + "$KUBECTL" --request-timeout=5s -n "$CELL_NS" exec "$worker" -c trino-worker \ + -- test -r "/etc/trino/tenant-secrets/$ORG_C" >/dev/null 2>&1 || mounted=0 + done + [ "$mounted" = 1 ] && return 0 + fi + sleep 5 + attempt=$((attempt + 1)) + done + fail "worker tenant password file did not converge in the isolated cell" +} + log "multicell initial placement with green stopped" api "$API/api/v1/trino/cells" | jq -e '.cells | map(.id) | sort == ["cell-test","legacy"]' >/dev/null \ || fail "both cells must be registered" @@ -82,6 +110,7 @@ code="$(curl --connect-timeout 5 --max-time 30 -sS -o /tmp/trino-cell-selection- TRINO="$BLUE_TRINO" wait_cell_auth +wait_worker_tenant_file blue trino_query "$DB_C" "$pw_c" "CREATE SCHEMA $CAT_C.cell_test" >/dev/null trino_query "$DB_C" "$pw_c" "CREATE TABLE $CAT_C.cell_test.values_test (value BIGINT)" >/dev/null trino_query "$DB_C" "$pw_c" "INSERT INTO $CAT_C.cell_test.values_test VALUES (7),(11)" >/dev/null @@ -129,6 +158,7 @@ done wait_cell_ready TRINO="$GREEN_TRINO" wait_cell_auth +wait_worker_tenant_file green result="$(trino_query "$DB_C" "$pw_c" "SELECT COUNT(*), SUM(value) FROM $CAT_C.cell_test.values_test")" [ "$result" = '[[2,18]]' ] || fail "green failed to hydrate its independent catalog from the same DuckLake warehouse" must_fail "$DB_A" "$pw_a" 'SELECT 1' '401|Unauthorized|Authentication|credentials' diff --git a/tests/mw-dev/trino-multicell.tmpl.yaml b/tests/mw-dev/trino-multicell.tmpl.yaml index 6e62ba31..6e4b70ba 100644 --- a/tests/mw-dev/trino-multicell.tmpl.yaml +++ b/tests/mw-dev/trino-multicell.tmpl.yaml @@ -39,6 +39,13 @@ metadata: name: trino-cell-projection namespace: ${TRINO_CELL_NAMESPACE} rules: + # The fixture checks worker file availability without reading file contents. + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create", "get"] - apiGroups: [""] resources: ["secrets", "configmaps"] verbs: ["create"] diff --git a/tests/mw-dev/trino_multicell_test.go b/tests/mw-dev/trino_multicell_test.go index 666cb725..6d43d011 100644 --- a/tests/mw-dev/trino_multicell_test.go +++ b/tests/mw-dev/trino_multicell_test.go @@ -148,6 +148,7 @@ func TestTrinoMulticellRenderedBackendsAreIsolated(t *testing.T) { deployments := map[string]map[string]any{} secrets := []map[string]any{} publicManifests := []map[string]any{} + workerPermissions := map[string]bool{} for { var manifest map[string]any if err := decoder.Decode(&manifest); err == io.EOF { @@ -169,12 +170,40 @@ func TestTrinoMulticellRenderedBackendsAreIsolated(t *testing.T) { if manifest["kind"] == "Deployment" { deployments[manifestName(manifest)] = manifest } + if manifest["kind"] == "Role" && manifestName(manifest) == "trino-cell-projection" { + if manifest["metadata"].(map[string]any)["namespace"] != "duckgres-ci-pr-0123" { + t.Fatal("worker inspection permissions must remain in the isolated secondary namespace") + } + for _, rawRule := range manifest["rules"].([]any) { + rule := rawRule.(map[string]any) + for _, resource := range rule["resources"].([]any) { + if resource != "pods" && resource != "pods/exec" { + continue + } + verbs, err := json.Marshal(rule["verbs"]) + if err != nil { + t.Fatal(err) + } + want := `["get","list"]` + if resource == "pods/exec" { + want = `["create","get"]` + } + if string(verbs) != want { + t.Fatalf("unexpected worker inspection verbs: %s", verbs) + } + workerPermissions[resource.(string)] = true + } + } + } if manifest["kind"] == "Secret" { secrets = append(secrets, manifest) } else { publicManifests = append(publicManifests, manifest) } } + if !workerPermissions["pods"] || !workerPermissions["pods/exec"] { + t.Fatal("missing isolated worker mount inspection permissions") + } passwordBytes, err := os.ReadFile(filepath.Join(secretDir, "duckgres-ci-config-store-password")) if err != nil { t.Fatal("renderer must generate a per-run config-store credential:", err) diff --git a/tests/mw-dev/trino_worker_projection_test.go b/tests/mw-dev/trino_worker_projection_test.go new file mode 100644 index 00000000..cec50932 --- /dev/null +++ b/tests/mw-dev/trino_worker_projection_test.go @@ -0,0 +1,108 @@ +package e2emwdev_test + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestTrinoWorkerProjectionGate(t *testing.T) { + raw, err := os.ReadFile("e2e/trino-multicell.sh") + if err != nil { + t.Fatal(err) + } + text := string(raw) + start := strings.Index(text, "wait_worker_tenant_file() {") + if start < 0 { + t.Fatal("missing worker file convergence gate") + } + end := strings.Index(text[start:], "\n}\n") + if end < 0 { + t.Fatal("unterminated worker helper") + } + helper := text[start : start+end+3] + for _, tc := range []struct { + name string + ok bool + }{ + {"ready", true}, {"one-worker-lags", true}, {"missing-file", false}, + {"no-pods", false}, {"unready", false}, {"terminating", false}, + {"wrong-label", false}, {"wrong-name", false}, {"denied", false}, + {"stale-generation", false}, {"ready-count", false}, {"updated-count", false}, + {"wrong-identity", false}, {"missing-worker", false}, + } { + t.Run(tc.name, func(t *testing.T) { + counter := filepath.Join(t.TempDir(), "checks") + script := `set -eu +PR=123 +ORG_C=ci-pr-123-trinoc +CELL_NS=duckgres-ci-pr-0123 +[ "$MODE" != wrong-identity ] || CELL_NS=another-namespace +KUBECTL=kubectl +sleep() { :; } +fail() { echo "$*" >&2; exit 1; } +kubectl() { + [ "$1" = --request-timeout=5s ] && [ "$2" = -n ] && [ "$3" = "$CELL_NS" ] || exit 90 + shift 3 + if [ "$MODE" = denied ]; then return 1; fi + case "$1 $2" in + 'get deployment') + [ "$3" = duckgres-trino-blue-worker ] || exit 90 + jq -nc --arg mode "$MODE" '{metadata:{generation:1},spec:{replicas:2},status:{observedGeneration:1,readyReplicas:2,updatedReplicas:2}} | + if $mode == "stale-generation" then .status.observedGeneration=0 + elif $mode == "ready-count" then .status.readyReplicas=1 + elif $mode == "updated-count" then .status.updatedReplicas=1 + else . end' + ;; + 'get pods') + [ "$3" = -l ] && [ "$4" = app=duckgres-trino-blue,component=worker ] || exit 90 + jq -nc --arg mode "$MODE" '{items:[range(1;3) | {metadata:{name:("duckgres-trino-blue-worker-"+tostring),labels:{app:"duckgres-trino-blue",component:"worker"}},status:{phase:"Running",conditions:[{type:"Ready",status:"True"}]}}]} | + if $mode == "no-pods" then .items=[] + elif $mode == "missing-worker" then .items=.items[:1] + elif $mode == "unready" then .items[1].status.conditions[0].status="False" + elif $mode == "terminating" then .items[1].metadata.deletionTimestamp="now" + elif $mode == "wrong-label" then .items[1].metadata.labels.app="another-cell" + elif $mode == "wrong-name" then .items[1].metadata.name="another-worker" + else . end' + ;; + exec*) + [ "$3" = -c ] && [ "$4" = trino-worker ] && [ "$5" = -- ] && [ "$6" = test ] && [ "$7" = -r ] && [ "$8" = /etc/trino/tenant-secrets/ci-pr-123-trinoc ] || exit 90 + printf '%s\n' "$2" >> "$COUNTER" + if [ "$MODE" = missing-file ]; then return 1; fi + if [ "$MODE" = one-worker-lags ] && [ "$2" = duckgres-trino-blue-worker-2 ] && [ "$(wc -l < "$COUNTER")" -lt 3 ]; then return 1; fi + ;; + *) exit 90 ;; + esac +} +` + helper + "\nwait_worker_tenant_file blue\n" + cmd := exec.Command("sh", "-c", script) + cmd.Env = append(os.Environ(), "MODE="+tc.name, "COUNTER="+counter) + out, err := cmd.CombinedOutput() + if (err == nil) != tc.ok { + t.Fatalf("success=%v, want=%v: %s", err == nil, tc.ok, out) + } + if tc.ok { + checks, err := os.ReadFile(counter) + if err != nil || !strings.Contains(string(checks), "worker-1") || !strings.Contains(string(checks), "worker-2") { + t.Fatal("every worker must pass the file check") + } + } + }) + } + blueGate := strings.Index(text, "wait_worker_tenant_file blue") + firstWrite := strings.Index(text, `"CREATE SCHEMA $CAT_C.cell_test"`) + if blueGate < 0 || firstWrite < 0 || blueGate > firstWrite { + t.Fatal("worker gate must precede all initial writes") + } + greenPhase := strings.Index(text, `TRINO="$GREEN_TRINO"`) + if greenPhase < 0 { + t.Fatal("missing green phase") + } + greenGate := strings.Index(text[greenPhase:], "wait_worker_tenant_file green") + greenQuery := strings.Index(text[greenPhase:], `result="$(trino_query`) + if greenGate < 0 || greenQuery < 0 || greenGate > greenQuery { + t.Fatal("green must check worker projection before its first query") + } +} From d92d92bbb728052a3324c251e73c9dfa5ac85e5f Mon Sep 17 00:00:00 2001 From: Benjamin Knofe-Vider Date: Fri, 11 Sep 2026 17:55:34 +0200 Subject: [PATCH 3/5] test: diagnose isolated worker mount readiness stages --- tests/mw-dev/e2e/trino-multicell.sh | 47 +++++++++++++++----- tests/mw-dev/trino_worker_projection_test.go | 21 ++++++++- 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/tests/mw-dev/e2e/trino-multicell.sh b/tests/mw-dev/e2e/trino-multicell.sh index 0a27cfd8..a65320d3 100644 --- a/tests/mw-dev/e2e/trino-multicell.sh +++ b/tests/mw-dev/e2e/trino-multicell.sh @@ -31,19 +31,44 @@ wait_worker_tenant_file() { || fail "worker mount check escaped fixture identity" worker_app="duckgres-trino-$worker_color" attempt=0 + last_worker_stage="" while [ "$attempt" -lt 36 ]; do - deployment="$("$KUBECTL" --request-timeout=5s -n "$CELL_NS" get deployment "$worker_app-worker" -o json 2>/dev/null || true)" + worker_stage=deployment-readiness + worker_code=0 + deployment="$("$KUBECTL" --request-timeout=5s -n "$CELL_NS" get deployment "$worker_app-worker" -o json 2>/dev/null)" \ + || { worker_code=$?; worker_stage=deployment-read; } replicas="$(printf %s "$deployment" | jq -er 'select(.metadata.generation == .status.observedGeneration and .spec.replicas > 0 and .status.readyReplicas == .spec.replicas and .status.updatedReplicas == .spec.replicas) | .spec.replicas' 2>/dev/null || true)" - snapshot="$("$KUBECTL" --request-timeout=5s -n "$CELL_NS" get pods -l "app=$worker_app,component=worker" -o json 2>/dev/null || true)" - workers="$(printf %s "$snapshot" | jq -er --arg app "$worker_app" --argjson replicas "${replicas:-0}" \ - 'select($replicas > 0 and (.items | length) == $replicas and all(.items[]; .metadata.deletionTimestamp == null and .metadata.labels.app == $app and .metadata.labels.component == "worker" and (.metadata.name | startswith($app + "-worker-")) and .status.phase == "Running" and any(.status.conditions[]?; .type == "Ready" and .status == "True"))) | .items[].metadata.name' 2>/dev/null || true)" - if [ -n "$workers" ]; then - mounted=1 - for worker in $workers; do - "$KUBECTL" --request-timeout=5s -n "$CELL_NS" exec "$worker" -c trino-worker \ - -- test -r "/etc/trino/tenant-secrets/$ORG_C" >/dev/null 2>&1 || mounted=0 - done - [ "$mounted" = 1 ] && return 0 + if [ "$worker_code" = 0 ] && [ -n "$replicas" ]; then + worker_stage=pod-readiness + snapshot="$("$KUBECTL" --request-timeout=5s -n "$CELL_NS" get pods -l "app=$worker_app,component=worker" -o json 2>/dev/null)" \ + || { worker_code=$?; worker_stage=pod-read; } + workers="$(printf %s "$snapshot" | jq -er --arg app "$worker_app" --argjson replicas "$replicas" \ + 'select($replicas > 0 and (.items | length) == $replicas and all(.items[]; .metadata.deletionTimestamp == null and .metadata.labels.app == $app and .metadata.labels.component == "worker" and (.metadata.name | startswith($app + "-worker-")) and .status.phase == "Running" and any(.status.conditions[]?; .type == "Ready" and .status == "True"))) | .items[].metadata.name' 2>/dev/null || true)" + if [ "$worker_code" = 0 ] && [ -n "$workers" ]; then + mounted=1 + worker_stage=worker-file + for worker in $workers; do + if worker_result="$("$KUBECTL" --request-timeout=5s -n "$CELL_NS" exec "$worker" -c trino-worker \ + -- test -r "/etc/trino/tenant-secrets/$ORG_C" 2>&1)"; then + : + else + worker_code=$? + mounted=0 + case "$worker_result" in + *'command terminated with exit code 1'*) worker_stage=worker-file ;; + *Forbidden*|*forbidden*) worker_stage=worker-exec-permission ;; + *'deadline exceeded'*|*'timed out'*) worker_stage=worker-exec-timeout ;; + *'executable file not found'*) worker_stage=worker-exec-program ;; + *) worker_stage=worker-exec ;; + esac + fi + done + [ "$mounted" = 1 ] && return 0 + fi + fi + if [ "$last_worker_stage" != "$worker_stage:$worker_code" ]; then + log "Worker mount readiness: $worker_stage (exit $worker_code)" + last_worker_stage="$worker_stage:$worker_code" fi sleep 5 attempt=$((attempt + 1)) diff --git a/tests/mw-dev/trino_worker_projection_test.go b/tests/mw-dev/trino_worker_projection_test.go index cec50932..063d8f1b 100644 --- a/tests/mw-dev/trino_worker_projection_test.go +++ b/tests/mw-dev/trino_worker_projection_test.go @@ -32,6 +32,7 @@ func TestTrinoWorkerProjectionGate(t *testing.T) { {"wrong-label", false}, {"wrong-name", false}, {"denied", false}, {"stale-generation", false}, {"ready-count", false}, {"updated-count", false}, {"wrong-identity", false}, {"missing-worker", false}, + {"exec-timeout", false}, {"exec-denied", false}, } { t.Run(tc.name, func(t *testing.T) { counter := filepath.Join(t.TempDir(), "checks") @@ -42,6 +43,7 @@ CELL_NS=duckgres-ci-pr-0123 [ "$MODE" != wrong-identity ] || CELL_NS=another-namespace KUBECTL=kubectl sleep() { :; } +log() { echo "$*"; } fail() { echo "$*" >&2; exit 1; } kubectl() { [ "$1" = --request-timeout=5s ] && [ "$2" = -n ] && [ "$3" = "$CELL_NS" ] || exit 90 @@ -70,7 +72,9 @@ kubectl() { exec*) [ "$3" = -c ] && [ "$4" = trino-worker ] && [ "$5" = -- ] && [ "$6" = test ] && [ "$7" = -r ] && [ "$8" = /etc/trino/tenant-secrets/ci-pr-123-trinoc ] || exit 90 printf '%s\n' "$2" >> "$COUNTER" - if [ "$MODE" = missing-file ]; then return 1; fi + if [ "$MODE" = missing-file ]; then echo 'command terminated with exit code 1' >&2; return 1; fi + if [ "$MODE" = exec-timeout ]; then echo 'context deadline exceeded private-payload' >&2; return 1; fi + if [ "$MODE" = exec-denied ]; then echo 'Forbidden private-payload' >&2; return 1; fi if [ "$MODE" = one-worker-lags ] && [ "$2" = duckgres-trino-blue-worker-2 ] && [ "$(wc -l < "$COUNTER")" -lt 3 ]; then return 1; fi ;; *) exit 90 ;; @@ -83,6 +87,21 @@ kubectl() { if (err == nil) != tc.ok { t.Fatalf("success=%v, want=%v: %s", err == nil, tc.ok, out) } + if tc.name == "missing-file" && !strings.Contains(string(out), "worker-file (exit 1)") { + t.Fatal("missing safe file-check failure diagnostics") + } + if tc.name == "denied" && !strings.Contains(string(out), "deployment-read (exit 1)") { + t.Fatal("missing safe API failure diagnostics") + } + if tc.name == "exec-timeout" && !strings.Contains(string(out), "worker-exec-timeout (exit 1)") { + t.Fatal("missing safe timeout diagnostics") + } + if tc.name == "exec-denied" && !strings.Contains(string(out), "worker-exec-permission (exit 1)") { + t.Fatal("missing safe permission diagnostics") + } + if strings.Contains(string(out), "private-payload") { + t.Fatal("raw exec stderr must never enter diagnostics") + } if tc.ok { checks, err := os.ReadFile(counter) if err != nil || !strings.Contains(string(checks), "worker-1") || !strings.Contains(string(checks), "worker-2") { From 34f067cc6bfa70b00915ee054fd365e20daacded Mon Sep 17 00:00:00 2001 From: Benjamin Knofe-Vider Date: Fri, 11 Sep 2026 18:19:39 +0200 Subject: [PATCH 4/5] test: preserve in-cluster kubectl configuration with process timeouts --- tests/mw-dev/README.md | 3 +++ tests/mw-dev/e2e/trino-multicell.sh | 6 +++--- tests/mw-dev/e2e/trino.sh | 1 + tests/mw-dev/trino_worker_projection_test.go | 8 ++++++-- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/mw-dev/README.md b/tests/mw-dev/README.md index 0a13a732..b19bb3a5 100644 --- a/tests/mw-dev/README.md +++ b/tests/mw-dev/README.md @@ -549,6 +549,9 @@ contents or retry writes. The fixture's secondary namespace Role permits pod listing and exec for these checks under the existing CI grant. No production or cluster-wide RBAC changes are required. Control-plane `ready` alone does not prove every worker's projected Secret volume has converged. +The worker checks use a process timeout rather than kubectl's request-timeout +flag. The pinned client otherwise loses its implicit in-cluster configuration +and attempts to connect to localhost. No token or kubeconfig is copied. After these checks, the lane restarts its control plane in registry-only mode, verifies both registered backends still query the warehouse, rejects legacy diff --git a/tests/mw-dev/e2e/trino-multicell.sh b/tests/mw-dev/e2e/trino-multicell.sh index a65320d3..1edc689b 100644 --- a/tests/mw-dev/e2e/trino-multicell.sh +++ b/tests/mw-dev/e2e/trino-multicell.sh @@ -35,12 +35,12 @@ wait_worker_tenant_file() { while [ "$attempt" -lt 36 ]; do worker_stage=deployment-readiness worker_code=0 - deployment="$("$KUBECTL" --request-timeout=5s -n "$CELL_NS" get deployment "$worker_app-worker" -o json 2>/dev/null)" \ + deployment="$(timeout 5 "$KUBECTL" -n "$CELL_NS" get deployment "$worker_app-worker" -o json 2>/dev/null)" \ || { worker_code=$?; worker_stage=deployment-read; } replicas="$(printf %s "$deployment" | jq -er 'select(.metadata.generation == .status.observedGeneration and .spec.replicas > 0 and .status.readyReplicas == .spec.replicas and .status.updatedReplicas == .spec.replicas) | .spec.replicas' 2>/dev/null || true)" if [ "$worker_code" = 0 ] && [ -n "$replicas" ]; then worker_stage=pod-readiness - snapshot="$("$KUBECTL" --request-timeout=5s -n "$CELL_NS" get pods -l "app=$worker_app,component=worker" -o json 2>/dev/null)" \ + snapshot="$(timeout 5 "$KUBECTL" -n "$CELL_NS" get pods -l "app=$worker_app,component=worker" -o json 2>/dev/null)" \ || { worker_code=$?; worker_stage=pod-read; } workers="$(printf %s "$snapshot" | jq -er --arg app "$worker_app" --argjson replicas "$replicas" \ 'select($replicas > 0 and (.items | length) == $replicas and all(.items[]; .metadata.deletionTimestamp == null and .metadata.labels.app == $app and .metadata.labels.component == "worker" and (.metadata.name | startswith($app + "-worker-")) and .status.phase == "Running" and any(.status.conditions[]?; .type == "Ready" and .status == "True"))) | .items[].metadata.name' 2>/dev/null || true)" @@ -48,7 +48,7 @@ wait_worker_tenant_file() { mounted=1 worker_stage=worker-file for worker in $workers; do - if worker_result="$("$KUBECTL" --request-timeout=5s -n "$CELL_NS" exec "$worker" -c trino-worker \ + if worker_result="$(timeout 5 "$KUBECTL" -n "$CELL_NS" exec "$worker" -c trino-worker \ -- test -r "/etc/trino/tenant-secrets/$ORG_C" 2>&1)"; then : else diff --git a/tests/mw-dev/e2e/trino.sh b/tests/mw-dev/e2e/trino.sh index cbdf44ef..c95020d8 100644 --- a/tests/mw-dev/e2e/trino.sh +++ b/tests/mw-dev/e2e/trino.sh @@ -32,6 +32,7 @@ TRINO_AUTH_ROTATION_RETRY_SECONDS=5 fail() { echo "FAIL: $*" >&2; exit 1; } log() { echo ">>> $*" >&2; } apk add --no-cache curl jq >/dev/null 2>&1 +command -v timeout >/dev/null || fail "timeout is required for bounded worker checks" [ -s "$CA" ] || fail "per-run Trino CA is not mounted" CP_IP="$(getent hosts "$PGHOST" | awk '{print $1}' | head -1)" [ -n "$CP_IP" ] || fail "could not resolve $PGHOST" diff --git a/tests/mw-dev/trino_worker_projection_test.go b/tests/mw-dev/trino_worker_projection_test.go index 063d8f1b..fdd1e67b 100644 --- a/tests/mw-dev/trino_worker_projection_test.go +++ b/tests/mw-dev/trino_worker_projection_test.go @@ -23,6 +23,9 @@ func TestTrinoWorkerProjectionGate(t *testing.T) { t.Fatal("unterminated worker helper") } helper := text[start : start+end+3] + if strings.Contains(helper, "--request-timeout") || strings.Count(helper, `timeout 5 "$KUBECTL"`) != 3 { + t.Fatal("bound kubectl externally to preserve in-cluster credential discovery") + } for _, tc := range []struct { name string ok bool @@ -43,11 +46,12 @@ CELL_NS=duckgres-ci-pr-0123 [ "$MODE" != wrong-identity ] || CELL_NS=another-namespace KUBECTL=kubectl sleep() { :; } +timeout() { [ "$1" = 5 ] || exit 90; shift; "$@"; } log() { echo "$*"; } fail() { echo "$*" >&2; exit 1; } kubectl() { - [ "$1" = --request-timeout=5s ] && [ "$2" = -n ] && [ "$3" = "$CELL_NS" ] || exit 90 - shift 3 + [ "$1" = -n ] && [ "$2" = "$CELL_NS" ] || exit 90 + shift 2 if [ "$MODE" = denied ]; then return 1; fi case "$1 $2" in 'get deployment') From 13fe8098ffa701e6e15bd4ff314c21e181b835f5 Mon Sep 17 00:00:00 2001 From: Benjamin Knofe-Vider Date: Fri, 11 Sep 2026 19:40:25 +0200 Subject: [PATCH 5/5] test: verify legacy worker password mounts before initial writes --- tests/mw-dev/README.md | 11 +-- tests/mw-dev/e2e/trino-multicell.sh | 56 +--------------- tests/mw-dev/e2e/trino.sh | 70 ++++++++++++++++++++ tests/mw-dev/manifests.tmpl.yaml | 4 ++ tests/mw-dev/trino_multicell_test.go | 23 +++++++ tests/mw-dev/trino_worker_projection_test.go | 61 ++++++++++++++--- 6 files changed, 158 insertions(+), 67 deletions(-) diff --git a/tests/mw-dev/README.md b/tests/mw-dev/README.md index b19bb3a5..6155879e 100644 --- a/tests/mw-dev/README.md +++ b/tests/mw-dev/README.md @@ -542,11 +542,12 @@ queries the same data through green's independently hydrated catalog. Direct coordinator URLs are fixture-only; this does not test Gateway routing or a maintenance move of an existing warehouse. -Before the first write, the fixture checks the expected tenant password file -is readable on every ready blue worker. It repeats that check for green before -querying. These bounded checks use only `test -r`; they never read password -contents or retry writes. The fixture's secondary namespace Role permits pod -listing and exec for these checks under the existing CI grant. No production +Before each tenant's first write, the fixture checks the expected password file +is readable on every ready worker, including all three legacy workers. It +repeats that check for green before querying. These bounded checks use only +`test -r`; they never read password contents or retry writes. The fixture's +primary and secondary namespace Roles permit the required pod reads and exec +under the existing CI grant. No production or cluster-wide RBAC changes are required. Control-plane `ready` alone does not prove every worker's projected Secret volume has converged. The worker checks use a process timeout rather than kubectl's request-timeout diff --git a/tests/mw-dev/e2e/trino-multicell.sh b/tests/mw-dev/e2e/trino-multicell.sh index 1edc689b..e0098f21 100644 --- a/tests/mw-dev/e2e/trino-multicell.sh +++ b/tests/mw-dev/e2e/trino-multicell.sh @@ -23,58 +23,6 @@ wait_cell_registry() { fail "cell registry did not converge after control-plane restart" } -wait_worker_tenant_file() { - worker_color="$1" - case "$worker_color" in blue|green) ;; *) fail "invalid worker color" ;; esac - case "$PR" in ''|*[!0-9]*|0*) fail "invalid fixture identity" ;; esac - [ "$CELL_NS" = "duckgres-ci-pr-0$PR" ] && [ "$ORG_C" = "ci-pr-$PR-trinoc" ] \ - || fail "worker mount check escaped fixture identity" - worker_app="duckgres-trino-$worker_color" - attempt=0 - last_worker_stage="" - while [ "$attempt" -lt 36 ]; do - worker_stage=deployment-readiness - worker_code=0 - deployment="$(timeout 5 "$KUBECTL" -n "$CELL_NS" get deployment "$worker_app-worker" -o json 2>/dev/null)" \ - || { worker_code=$?; worker_stage=deployment-read; } - replicas="$(printf %s "$deployment" | jq -er 'select(.metadata.generation == .status.observedGeneration and .spec.replicas > 0 and .status.readyReplicas == .spec.replicas and .status.updatedReplicas == .spec.replicas) | .spec.replicas' 2>/dev/null || true)" - if [ "$worker_code" = 0 ] && [ -n "$replicas" ]; then - worker_stage=pod-readiness - snapshot="$(timeout 5 "$KUBECTL" -n "$CELL_NS" get pods -l "app=$worker_app,component=worker" -o json 2>/dev/null)" \ - || { worker_code=$?; worker_stage=pod-read; } - workers="$(printf %s "$snapshot" | jq -er --arg app "$worker_app" --argjson replicas "$replicas" \ - 'select($replicas > 0 and (.items | length) == $replicas and all(.items[]; .metadata.deletionTimestamp == null and .metadata.labels.app == $app and .metadata.labels.component == "worker" and (.metadata.name | startswith($app + "-worker-")) and .status.phase == "Running" and any(.status.conditions[]?; .type == "Ready" and .status == "True"))) | .items[].metadata.name' 2>/dev/null || true)" - if [ "$worker_code" = 0 ] && [ -n "$workers" ]; then - mounted=1 - worker_stage=worker-file - for worker in $workers; do - if worker_result="$(timeout 5 "$KUBECTL" -n "$CELL_NS" exec "$worker" -c trino-worker \ - -- test -r "/etc/trino/tenant-secrets/$ORG_C" 2>&1)"; then - : - else - worker_code=$? - mounted=0 - case "$worker_result" in - *'command terminated with exit code 1'*) worker_stage=worker-file ;; - *Forbidden*|*forbidden*) worker_stage=worker-exec-permission ;; - *'deadline exceeded'*|*'timed out'*) worker_stage=worker-exec-timeout ;; - *'executable file not found'*) worker_stage=worker-exec-program ;; - *) worker_stage=worker-exec ;; - esac - fi - done - [ "$mounted" = 1 ] && return 0 - fi - fi - if [ "$last_worker_stage" != "$worker_stage:$worker_code" ]; then - log "Worker mount readiness: $worker_stage (exit $worker_code)" - last_worker_stage="$worker_stage:$worker_code" - fi - sleep 5 - attempt=$((attempt + 1)) - done - fail "worker tenant password file did not converge in the isolated cell" -} log "multicell initial placement with green stopped" api "$API/api/v1/trino/cells" | jq -e '.cells | map(.id) | sort == ["cell-test","legacy"]' >/dev/null \ @@ -135,7 +83,7 @@ code="$(curl --connect-timeout 5 --max-time 30 -sS -o /tmp/trino-cell-selection- TRINO="$BLUE_TRINO" wait_cell_auth -wait_worker_tenant_file blue +wait_worker_tenant_file blue "$ORG_C" trino_query "$DB_C" "$pw_c" "CREATE SCHEMA $CAT_C.cell_test" >/dev/null trino_query "$DB_C" "$pw_c" "CREATE TABLE $CAT_C.cell_test.values_test (value BIGINT)" >/dev/null trino_query "$DB_C" "$pw_c" "INSERT INTO $CAT_C.cell_test.values_test VALUES (7),(11)" >/dev/null @@ -183,7 +131,7 @@ done wait_cell_ready TRINO="$GREEN_TRINO" wait_cell_auth -wait_worker_tenant_file green +wait_worker_tenant_file green "$ORG_C" result="$(trino_query "$DB_C" "$pw_c" "SELECT COUNT(*), SUM(value) FROM $CAT_C.cell_test.values_test")" [ "$result" = '[[2,18]]' ] || fail "green failed to hydrate its independent catalog from the same DuckLake warehouse" must_fail "$DB_A" "$pw_a" 'SELECT 1' '401|Unauthorized|Authentication|credentials' diff --git a/tests/mw-dev/e2e/trino.sh b/tests/mw-dev/e2e/trino.sh index c95020d8..75ab806e 100644 --- a/tests/mw-dev/e2e/trino.sh +++ b/tests/mw-dev/e2e/trino.sh @@ -44,6 +44,74 @@ chmod +x "$KUBECTL" api() { curl --connect-timeout 5 --max-time 60 -fsS -H "$H" "$@"; } +wait_worker_tenant_file() { + worker_color="$1" + worker_org="$2" + case "$PR" in ''|*[!0-9]*|0*) fail "invalid fixture identity" ;; esac + case "$worker_color" in + legacy) + [ "$NS" = "duckgres-ci-pr-$PR" ] || fail "worker mount check escaped fixture identity" + case "$worker_org" in + "ci-pr-$PR-trinoa"|"ci-pr-$PR-trinob") ;; + *) fail "worker mount check crossed tenant boundary" ;; + esac + worker_namespace="$NS" + worker_app=duckgres-trino + ;; + blue|green) + [ "${CELL_NS:-}" = "duckgres-ci-pr-0$PR" ] && [ "$worker_org" = "ci-pr-$PR-trinoc" ] \ + || fail "worker mount check escaped fixture identity" + worker_namespace="$CELL_NS" + worker_app="duckgres-trino-$worker_color" + ;; + *) fail "invalid worker color" ;; + esac + attempt=0 + last_worker_stage="" + while [ "$attempt" -lt 36 ]; do + worker_stage=deployment-readiness + worker_code=0 + deployment="$(timeout 5 "$KUBECTL" -n "$worker_namespace" get deployment "$worker_app-worker" -o json 2>/dev/null)" \ + || { worker_code=$?; worker_stage=deployment-read; } + replicas="$(printf %s "$deployment" | jq -er 'select(.metadata.generation == .status.observedGeneration and .spec.replicas > 0 and .status.readyReplicas == .spec.replicas and .status.updatedReplicas == .spec.replicas) | .spec.replicas' 2>/dev/null || true)" + if [ "$worker_code" = 0 ] && [ -n "$replicas" ]; then + worker_stage=pod-readiness + snapshot="$(timeout 5 "$KUBECTL" -n "$worker_namespace" get pods -l "app=$worker_app,component=worker" -o json 2>/dev/null)" \ + || { worker_code=$?; worker_stage=pod-read; } + workers="$(printf %s "$snapshot" | jq -er --arg app "$worker_app" --argjson replicas "$replicas" \ + 'select($replicas > 0 and (.items | length) == $replicas and all(.items[]; .metadata.deletionTimestamp == null and .metadata.labels.app == $app and .metadata.labels.component == "worker" and (.metadata.name | startswith($app + "-worker-")) and .status.phase == "Running" and any(.status.conditions[]?; .type == "Ready" and .status == "True"))) | .items[].metadata.name' 2>/dev/null || true)" + if [ "$worker_code" = 0 ] && [ -n "$workers" ]; then + mounted=1 + worker_stage=worker-file + for worker in $workers; do + if worker_result="$(timeout 5 "$KUBECTL" -n "$worker_namespace" exec "$worker" -c trino-worker \ + -- test -r "/etc/trino/tenant-secrets/$worker_org" 2>&1)"; then + : + else + worker_code=$? + mounted=0 + case "$worker_result" in + *'command terminated with exit code 1'*) worker_stage=worker-file ;; + *Forbidden*|*forbidden*) worker_stage=worker-exec-permission ;; + *'deadline exceeded'*|*'timed out'*) worker_stage=worker-exec-timeout ;; + *'executable file not found'*) worker_stage=worker-exec-program ;; + *) worker_stage=worker-exec ;; + esac + fi + done + [ "$mounted" = 1 ] && return 0 + fi + fi + if [ "$last_worker_stage" != "$worker_stage:$worker_code" ]; then + log "Worker mount readiness: $worker_stage (exit $worker_code)" + last_worker_stage="$worker_stage:$worker_code" + fi + sleep 5 + attempt=$((attempt + 1)) + done + fail "worker tenant password file did not converge in the isolated cell" +} + # A fresh managed warehouse has an empty metadata database. DuckLake's Trino # connector consumes an existing DuckLake catalog; it does not create the # metadata tables itself. Initialize them once through Duckgres's normal @@ -142,6 +210,7 @@ pw_a="$(provision "$ORG_A" "$DB_A" "$TEAM_A" | jq -r .password)" wait_warehouse "$ORG_A" bootstrap_ducklake "$ORG_A" "$pw_a" wait_trino "$ORG_A" "$DB_A" "$CAT_A" +wait_worker_tenant_file legacy "$ORG_A" log "TLS/password auth, discovery, and DDL/DML" [ "$(scalar "$DB_A" "$pw_a" 'SELECT 1')" = 1 ] || fail "Trino SELECT 1 failed" @@ -189,6 +258,7 @@ pw_b="$(provision "$ORG_B" "$DB_B" "$TEAM_B" | jq -r .password)" wait_warehouse "$ORG_B" bootstrap_ducklake "$ORG_B" "$pw_b" wait_trino "$ORG_B" "$DB_B" "$CAT_B" +wait_worker_tenant_file legacy "$ORG_B" [ "$("$KUBECTL" -n "$NS" get pod -l 'app=duckgres-trino,component=coordinator' -o jsonpath='{.items[0].metadata.uid}')" = "$coord_uid_before" ] \ || fail "adding tenant B restarted the Trino coordinator" [ "$(scalar "$DB_B" "$pw_b" 'SELECT 1')" = 1 ] || fail "hot-added tenant cannot authenticate" diff --git a/tests/mw-dev/manifests.tmpl.yaml b/tests/mw-dev/manifests.tmpl.yaml index e8ead92b..e9a1007f 100644 --- a/tests/mw-dev/manifests.tmpl.yaml +++ b/tests/mw-dev/manifests.tmpl.yaml @@ -159,6 +159,10 @@ rules: - apiGroups: [""] resources: ["pods/log"] verbs: ["get"] + # The fixture checks password-file availability without reading its contents. + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["get", "create"] # The Trino lane deletes coordinator/worker pods and uses rollout status to # prove their owning Deployments restore readiness. rollout status lists # before watching, so keep this read-only and namespace-scoped; every diff --git a/tests/mw-dev/trino_multicell_test.go b/tests/mw-dev/trino_multicell_test.go index 6d43d011..71d53c2f 100644 --- a/tests/mw-dev/trino_multicell_test.go +++ b/tests/mw-dev/trino_multicell_test.go @@ -149,6 +149,7 @@ func TestTrinoMulticellRenderedBackendsAreIsolated(t *testing.T) { secrets := []map[string]any{} publicManifests := []map[string]any{} workerPermissions := map[string]bool{} + primaryWorkerExec := false for { var manifest map[string]any if err := decoder.Decode(&manifest); err == io.EOF { @@ -170,6 +171,25 @@ func TestTrinoMulticellRenderedBackendsAreIsolated(t *testing.T) { if manifest["kind"] == "Deployment" { deployments[manifestName(manifest)] = manifest } + if manifest["kind"] == "Role" && manifestName(manifest) == "duckgres-control-plane" { + if manifest["metadata"].(map[string]any)["namespace"] != "duckgres-ci-pr-123" { + t.Fatal("legacy worker inspection must remain in the isolated primary namespace") + } + for _, rawRule := range manifest["rules"].([]any) { + rule := rawRule.(map[string]any) + resources, err := json.Marshal(rule["resources"]) + if err != nil { + t.Fatal(err) + } + if string(resources) == `["pods/exec"]` { + verbs, err := json.Marshal(rule["verbs"]) + if err != nil || string(verbs) != `["get","create"]` { + t.Fatal("legacy worker exec must use only get/create") + } + primaryWorkerExec = true + } + } + } if manifest["kind"] == "Role" && manifestName(manifest) == "trino-cell-projection" { if manifest["metadata"].(map[string]any)["namespace"] != "duckgres-ci-pr-0123" { t.Fatal("worker inspection permissions must remain in the isolated secondary namespace") @@ -204,6 +224,9 @@ func TestTrinoMulticellRenderedBackendsAreIsolated(t *testing.T) { if !workerPermissions["pods"] || !workerPermissions["pods/exec"] { t.Fatal("missing isolated worker mount inspection permissions") } + if !primaryWorkerExec { + t.Fatal("missing isolated legacy worker inspection permission") + } passwordBytes, err := os.ReadFile(filepath.Join(secretDir, "duckgres-ci-config-store-password")) if err != nil { t.Fatal("renderer must generate a per-run config-store credential:", err) diff --git a/tests/mw-dev/trino_worker_projection_test.go b/tests/mw-dev/trino_worker_projection_test.go index fdd1e67b..2f9e9b35 100644 --- a/tests/mw-dev/trino_worker_projection_test.go +++ b/tests/mw-dev/trino_worker_projection_test.go @@ -9,7 +9,7 @@ import ( ) func TestTrinoWorkerProjectionGate(t *testing.T) { - raw, err := os.ReadFile("e2e/trino-multicell.sh") + raw, err := os.ReadFile("e2e/trino.sh") if err != nil { t.Fatal(err) } @@ -36,35 +36,61 @@ func TestTrinoWorkerProjectionGate(t *testing.T) { {"stale-generation", false}, {"ready-count", false}, {"updated-count", false}, {"wrong-identity", false}, {"missing-worker", false}, {"exec-timeout", false}, {"exec-denied", false}, + {"legacy-a", true}, {"legacy-b", true}, {"legacy-lag", true}, + {"legacy-cross", false}, {"legacy-escape", false}, + {"blue-cross-a", false}, {"green-cross-b", false}, {"other-org", false}, } { t.Run(tc.name, func(t *testing.T) { counter := filepath.Join(t.TempDir(), "checks") script := `set -eu PR=123 +NS=duckgres-ci-pr-123 ORG_C=ci-pr-123-trinoc CELL_NS=duckgres-ci-pr-0123 [ "$MODE" != wrong-identity ] || CELL_NS=another-namespace +TEST_COLOR=blue +TEST_ORG=$ORG_C +EXPECTED_NS=$CELL_NS +EXPECTED_APP=duckgres-trino-blue +EXPECTED_COUNT=2 +case "$MODE" in + legacy-*) + TEST_COLOR=legacy + TEST_ORG=ci-pr-123-trinoa + EXPECTED_NS=$NS + EXPECTED_APP=duckgres-trino + EXPECTED_COUNT=3 + case "$MODE" in + legacy-b) TEST_ORG=ci-pr-123-trinob ;; + legacy-cross) TEST_ORG=$ORG_C ;; + legacy-escape) NS=another-namespace ;; + esac + ;; + blue-cross-a) TEST_ORG=ci-pr-123-trinoa ;; + green-cross-b) TEST_COLOR=green; TEST_ORG=ci-pr-123-trinob ;; + other-org) TEST_ORG=another-org ;; +esac KUBECTL=kubectl sleep() { :; } timeout() { [ "$1" = 5 ] || exit 90; shift; "$@"; } log() { echo "$*"; } fail() { echo "$*" >&2; exit 1; } kubectl() { - [ "$1" = -n ] && [ "$2" = "$CELL_NS" ] || exit 90 + [ "$1" = -n ] && [ "$2" = "$EXPECTED_NS" ] || exit 90 shift 2 if [ "$MODE" = denied ]; then return 1; fi case "$1 $2" in 'get deployment') - [ "$3" = duckgres-trino-blue-worker ] || exit 90 - jq -nc --arg mode "$MODE" '{metadata:{generation:1},spec:{replicas:2},status:{observedGeneration:1,readyReplicas:2,updatedReplicas:2}} | + [ "$3" = "$EXPECTED_APP-worker" ] || exit 90 + jq -nc --arg mode "$MODE" --argjson count "$EXPECTED_COUNT" '{metadata:{generation:1},spec:{replicas:$count},status:{observedGeneration:1,readyReplicas:$count,updatedReplicas:$count}} | if $mode == "stale-generation" then .status.observedGeneration=0 elif $mode == "ready-count" then .status.readyReplicas=1 elif $mode == "updated-count" then .status.updatedReplicas=1 else . end' ;; 'get pods') - [ "$3" = -l ] && [ "$4" = app=duckgres-trino-blue,component=worker ] || exit 90 - jq -nc --arg mode "$MODE" '{items:[range(1;3) | {metadata:{name:("duckgres-trino-blue-worker-"+tostring),labels:{app:"duckgres-trino-blue",component:"worker"}},status:{phase:"Running",conditions:[{type:"Ready",status:"True"}]}}]} | + [ "$3" = -l ] && [ "$4" = "app=$EXPECTED_APP,component=worker" ] || exit 90 + jq -nc --arg mode "$MODE" --arg app "$EXPECTED_APP" --argjson count "$EXPECTED_COUNT" '{items:[range(1;$count+1) | {metadata:{name:($app+"-worker-"+tostring),labels:{app:$app,component:"worker"}},status:{phase:"Running",conditions:[{type:"Ready",status:"True"}]}}]} | if $mode == "no-pods" then .items=[] elif $mode == "missing-worker" then .items=.items[:1] elif $mode == "unready" then .items[1].status.conditions[0].status="False" @@ -74,17 +100,18 @@ kubectl() { else . end' ;; exec*) - [ "$3" = -c ] && [ "$4" = trino-worker ] && [ "$5" = -- ] && [ "$6" = test ] && [ "$7" = -r ] && [ "$8" = /etc/trino/tenant-secrets/ci-pr-123-trinoc ] || exit 90 + [ "$3" = -c ] && [ "$4" = trino-worker ] && [ "$5" = -- ] && [ "$6" = test ] && [ "$7" = -r ] && [ "$8" = "/etc/trino/tenant-secrets/$TEST_ORG" ] || exit 90 printf '%s\n' "$2" >> "$COUNTER" if [ "$MODE" = missing-file ]; then echo 'command terminated with exit code 1' >&2; return 1; fi if [ "$MODE" = exec-timeout ]; then echo 'context deadline exceeded private-payload' >&2; return 1; fi if [ "$MODE" = exec-denied ]; then echo 'Forbidden private-payload' >&2; return 1; fi if [ "$MODE" = one-worker-lags ] && [ "$2" = duckgres-trino-blue-worker-2 ] && [ "$(wc -l < "$COUNTER")" -lt 3 ]; then return 1; fi + if [ "$MODE" = legacy-lag ] && [ "$2" = duckgres-trino-worker-3 ] && [ "$(wc -l < "$COUNTER")" -lt 4 ]; then return 1; fi ;; *) exit 90 ;; esac } -` + helper + "\nwait_worker_tenant_file blue\n" +` + helper + "\nwait_worker_tenant_file \"$TEST_COLOR\" \"$TEST_ORG\"\n" cmd := exec.Command("sh", "-c", script) cmd.Env = append(os.Environ(), "MODE="+tc.name, "COUNTER="+counter) out, err := cmd.CombinedOutput() @@ -111,9 +138,27 @@ kubectl() { if err != nil || !strings.Contains(string(checks), "worker-1") || !strings.Contains(string(checks), "worker-2") { t.Fatal("every worker must pass the file check") } + if strings.HasPrefix(tc.name, "legacy-") && !strings.Contains(string(checks), "worker-3") { + t.Fatal("all three legacy workers must pass") + } } }) } + for _, tenant := range []string{"A", "B"} { + gate := strings.Index(text, `wait_worker_tenant_file legacy "$ORG_`+tenant+`"`) + write := strings.Index(text, `"CREATE SCHEMA $CAT_A.$schema"`) + if tenant == "B" { + write = strings.Index(text, `"CREATE TABLE $CAT_B.main.$foreign_table`) + } + if gate < 0 || write < 0 || gate > write { + t.Fatalf("legacy %s gate must precede first write", tenant) + } + } + raw, err = os.ReadFile("e2e/trino-multicell.sh") + if err != nil { + t.Fatal(err) + } + text = string(raw) blueGate := strings.Index(text, "wait_worker_tenant_file blue") firstWrite := strings.Index(text, `"CREATE SCHEMA $CAT_C.cell_test"`) if blueGate < 0 || firstWrite < 0 || blueGate > firstWrite {