diff --git a/docs/trino-cells.md b/docs/trino-cells.md index 796491f3..8d3444c6 100644 --- a/docs/trino-cells.md +++ b/docs/trino-cells.md @@ -107,6 +107,11 @@ or live observer polls. A stopped green does not make blue's tenants unhealthy. When green starts, update the registry and restart the control plane. Every running backend must reconcile successfully before a tenant is reported ready; one successful coordinator cannot conceal another's missing catalog or failure. +This state confirms control-plane and coordinator catalog reconciliation, not +tenant-password volume convergence on every worker. A worker can briefly lack +a newly added tenant's password file even after the coordinator reports ready. +Initial testing must check worker file availability before the first write; +do not blindly retry writes when their commit outcome is unknown. The console observes the configured routing-active backend. Usage collection polls each running backend independently under the existing leader lease. diff --git a/tests/mw-dev/README.md b/tests/mw-dev/README.md index fb670c4f..6155879e 100644 --- a/tests/mw-dev/README.md +++ b/tests/mw-dev/README.md @@ -542,6 +542,18 @@ queries the same data through green's independently hydrated catalog. Direct coordinator URLs are fixture-only; this does not test Gateway routing or a maintenance move of an existing warehouse. +Before each tenant's first write, the fixture checks the expected password file +is readable on every ready worker, including all three legacy workers. It +repeats that check for green before querying. These bounded checks use only +`test -r`; they never read password contents or retry writes. The fixture's +primary and secondary namespace Roles permit the required pod reads and exec +under the existing CI grant. No production +or cluster-wide RBAC changes are required. Control-plane `ready` alone does +not prove every worker's projected Secret volume has converged. +The worker checks use a process timeout rather than kubectl's request-timeout +flag. The pinned client otherwise loses its implicit in-cluster configuration +and attempts to connect to localhost. No token or kubeconfig is copied. + After these checks, the lane restarts its control plane in registry-only mode, verifies both registered backends still query the warehouse, rejects legacy ownership and implicit cell selection, and verifies the legacy bundle endpoint diff --git a/tests/mw-dev/e2e/trino-multicell.sh b/tests/mw-dev/e2e/trino-multicell.sh index 87fc89fa..e0098f21 100644 --- a/tests/mw-dev/e2e/trino-multicell.sh +++ b/tests/mw-dev/e2e/trino-multicell.sh @@ -8,6 +8,22 @@ LEGACY_TRINO="$TRINO" BLUE_TRINO="https://duckgres-trino-blue.$CELL_NS.svc:8443" GREEN_TRINO="https://duckgres-trino-green.$CELL_NS.svc:8443" +wait_cell_registry() { + expected_cells="$1" + attempt=0 + while [ "$attempt" -lt 36 ]; do + result="$(api --max-time 5 "$API/api/v1/trino/cells" 2>/dev/null || true)" + if printf %s "$result" | jq -e --argjson expected "$expected_cells" \ + '.cells | map(.id) | sort == $expected' >/dev/null 2>&1; then + return 0 + fi + sleep 5 + attempt=$((attempt + 1)) + done + fail "cell registry did not converge after control-plane restart" +} + + log "multicell initial placement with green stopped" api "$API/api/v1/trino/cells" | jq -e '.cells | map(.id) | sort == ["cell-test","legacy"]' >/dev/null \ || fail "both cells must be registered" @@ -67,6 +83,7 @@ code="$(curl --connect-timeout 5 --max-time 30 -sS -o /tmp/trino-cell-selection- TRINO="$BLUE_TRINO" wait_cell_auth +wait_worker_tenant_file blue "$ORG_C" trino_query "$DB_C" "$pw_c" "CREATE SCHEMA $CAT_C.cell_test" >/dev/null trino_query "$DB_C" "$pw_c" "CREATE TABLE $CAT_C.cell_test.values_test (value BIGINT)" >/dev/null trino_query "$DB_C" "$pw_c" "INSERT INTO $CAT_C.cell_test.values_test VALUES (7),(11)" >/dev/null @@ -114,6 +131,7 @@ done wait_cell_ready TRINO="$GREEN_TRINO" wait_cell_auth +wait_worker_tenant_file green "$ORG_C" result="$(trino_query "$DB_C" "$pw_c" "SELECT COUNT(*), SUM(value) FROM $CAT_C.cell_test.values_test")" [ "$result" = '[[2,18]]' ] || fail "green failed to hydrate its independent catalog from the same DuckLake warehouse" must_fail "$DB_A" "$pw_a" 'SELECT 1' '401|Unauthorized|Authentication|credentials' @@ -130,8 +148,7 @@ legacy_owner="$(api "$API/api/v1/orgs/$ORG_A" | jq -r .trino.trino_cell_id)" "$KUBECTL" -n "$NS" patch deployment duckgres-control-plane --type=strategic -p \ '{"spec":{"template":{"spec":{"containers":[{"name":"controlplane","env":[{"name":"DUCKGRES_TRINO_COORDINATOR_URL","$patch":"delete"},{"name":"DUCKGRES_TRINO_REGISTRY_ONLY","value":"true"}]}]}}}}' >/dev/null "$KUBECTL" -n "$NS" rollout status deployment/duckgres-control-plane --timeout=180s >/dev/null -api "$API/api/v1/trino/cells" | jq -e '.cells | map(.id) == ["cell-test"]' >/dev/null \ - || fail "registry-only startup invented a legacy cell" +wait_cell_registry '["cell-test"]' wait_cell_ready for endpoint in "$BLUE_TRINO" "$GREEN_TRINO"; do TRINO="$endpoint" @@ -162,6 +179,7 @@ log "restore legacy fixture configuration" patch="$(printf %s "$legacy_env" | jq -c '{spec:{template:{spec:{containers:[{name:"controlplane",env:[.,{name:"DUCKGRES_TRINO_REGISTRY_ONLY","$patch":"delete"}]}]}}}}')" "$KUBECTL" -n "$NS" patch deployment duckgres-control-plane --type=strategic -p "$patch" >/dev/null "$KUBECTL" -n "$NS" rollout status deployment/duckgres-control-plane --timeout=180s >/dev/null +wait_cell_registry '["cell-test","legacy"]' TRINO="$LEGACY_TRINO" [ "$(trino_query "$DB_A" "$pw_a" 'SELECT 1')" = '[[1]]' ] || fail "legacy query failed after registry-only fixture restore" log "PASS: registry-only startup + explicit selection + no legacy dependency + restored fixture" diff --git a/tests/mw-dev/e2e/trino.sh b/tests/mw-dev/e2e/trino.sh index cbdf44ef..75ab806e 100644 --- a/tests/mw-dev/e2e/trino.sh +++ b/tests/mw-dev/e2e/trino.sh @@ -32,6 +32,7 @@ TRINO_AUTH_ROTATION_RETRY_SECONDS=5 fail() { echo "FAIL: $*" >&2; exit 1; } log() { echo ">>> $*" >&2; } apk add --no-cache curl jq >/dev/null 2>&1 +command -v timeout >/dev/null || fail "timeout is required for bounded worker checks" [ -s "$CA" ] || fail "per-run Trino CA is not mounted" CP_IP="$(getent hosts "$PGHOST" | awk '{print $1}' | head -1)" [ -n "$CP_IP" ] || fail "could not resolve $PGHOST" @@ -43,6 +44,74 @@ chmod +x "$KUBECTL" api() { curl --connect-timeout 5 --max-time 60 -fsS -H "$H" "$@"; } +wait_worker_tenant_file() { + worker_color="$1" + worker_org="$2" + case "$PR" in ''|*[!0-9]*|0*) fail "invalid fixture identity" ;; esac + case "$worker_color" in + legacy) + [ "$NS" = "duckgres-ci-pr-$PR" ] || fail "worker mount check escaped fixture identity" + case "$worker_org" in + "ci-pr-$PR-trinoa"|"ci-pr-$PR-trinob") ;; + *) fail "worker mount check crossed tenant boundary" ;; + esac + worker_namespace="$NS" + worker_app=duckgres-trino + ;; + blue|green) + [ "${CELL_NS:-}" = "duckgres-ci-pr-0$PR" ] && [ "$worker_org" = "ci-pr-$PR-trinoc" ] \ + || fail "worker mount check escaped fixture identity" + worker_namespace="$CELL_NS" + worker_app="duckgres-trino-$worker_color" + ;; + *) fail "invalid worker color" ;; + esac + attempt=0 + last_worker_stage="" + while [ "$attempt" -lt 36 ]; do + worker_stage=deployment-readiness + worker_code=0 + deployment="$(timeout 5 "$KUBECTL" -n "$worker_namespace" get deployment "$worker_app-worker" -o json 2>/dev/null)" \ + || { worker_code=$?; worker_stage=deployment-read; } + replicas="$(printf %s "$deployment" | jq -er 'select(.metadata.generation == .status.observedGeneration and .spec.replicas > 0 and .status.readyReplicas == .spec.replicas and .status.updatedReplicas == .spec.replicas) | .spec.replicas' 2>/dev/null || true)" + if [ "$worker_code" = 0 ] && [ -n "$replicas" ]; then + worker_stage=pod-readiness + snapshot="$(timeout 5 "$KUBECTL" -n "$worker_namespace" get pods -l "app=$worker_app,component=worker" -o json 2>/dev/null)" \ + || { worker_code=$?; worker_stage=pod-read; } + workers="$(printf %s "$snapshot" | jq -er --arg app "$worker_app" --argjson replicas "$replicas" \ + 'select($replicas > 0 and (.items | length) == $replicas and all(.items[]; .metadata.deletionTimestamp == null and .metadata.labels.app == $app and .metadata.labels.component == "worker" and (.metadata.name | startswith($app + "-worker-")) and .status.phase == "Running" and any(.status.conditions[]?; .type == "Ready" and .status == "True"))) | .items[].metadata.name' 2>/dev/null || true)" + if [ "$worker_code" = 0 ] && [ -n "$workers" ]; then + mounted=1 + worker_stage=worker-file + for worker in $workers; do + if worker_result="$(timeout 5 "$KUBECTL" -n "$worker_namespace" exec "$worker" -c trino-worker \ + -- test -r "/etc/trino/tenant-secrets/$worker_org" 2>&1)"; then + : + else + worker_code=$? + mounted=0 + case "$worker_result" in + *'command terminated with exit code 1'*) worker_stage=worker-file ;; + *Forbidden*|*forbidden*) worker_stage=worker-exec-permission ;; + *'deadline exceeded'*|*'timed out'*) worker_stage=worker-exec-timeout ;; + *'executable file not found'*) worker_stage=worker-exec-program ;; + *) worker_stage=worker-exec ;; + esac + fi + done + [ "$mounted" = 1 ] && return 0 + fi + fi + if [ "$last_worker_stage" != "$worker_stage:$worker_code" ]; then + log "Worker mount readiness: $worker_stage (exit $worker_code)" + last_worker_stage="$worker_stage:$worker_code" + fi + sleep 5 + attempt=$((attempt + 1)) + done + fail "worker tenant password file did not converge in the isolated cell" +} + # A fresh managed warehouse has an empty metadata database. DuckLake's Trino # connector consumes an existing DuckLake catalog; it does not create the # metadata tables itself. Initialize them once through Duckgres's normal @@ -141,6 +210,7 @@ pw_a="$(provision "$ORG_A" "$DB_A" "$TEAM_A" | jq -r .password)" wait_warehouse "$ORG_A" bootstrap_ducklake "$ORG_A" "$pw_a" wait_trino "$ORG_A" "$DB_A" "$CAT_A" +wait_worker_tenant_file legacy "$ORG_A" log "TLS/password auth, discovery, and DDL/DML" [ "$(scalar "$DB_A" "$pw_a" 'SELECT 1')" = 1 ] || fail "Trino SELECT 1 failed" @@ -188,6 +258,7 @@ pw_b="$(provision "$ORG_B" "$DB_B" "$TEAM_B" | jq -r .password)" wait_warehouse "$ORG_B" bootstrap_ducklake "$ORG_B" "$pw_b" wait_trino "$ORG_B" "$DB_B" "$CAT_B" +wait_worker_tenant_file legacy "$ORG_B" [ "$("$KUBECTL" -n "$NS" get pod -l 'app=duckgres-trino,component=coordinator' -o jsonpath='{.items[0].metadata.uid}')" = "$coord_uid_before" ] \ || fail "adding tenant B restarted the Trino coordinator" [ "$(scalar "$DB_B" "$pw_b" 'SELECT 1')" = 1 ] || fail "hot-added tenant cannot authenticate" diff --git a/tests/mw-dev/manifests.tmpl.yaml b/tests/mw-dev/manifests.tmpl.yaml index e8ead92b..e9a1007f 100644 --- a/tests/mw-dev/manifests.tmpl.yaml +++ b/tests/mw-dev/manifests.tmpl.yaml @@ -159,6 +159,10 @@ rules: - apiGroups: [""] resources: ["pods/log"] verbs: ["get"] + # The fixture checks password-file availability without reading its contents. + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["get", "create"] # The Trino lane deletes coordinator/worker pods and uses rollout status to # prove their owning Deployments restore readiness. rollout status lists # before watching, so keep this read-only and namespace-scoped; every diff --git a/tests/mw-dev/trino-multicell.tmpl.yaml b/tests/mw-dev/trino-multicell.tmpl.yaml index 6e62ba31..6e4b70ba 100644 --- a/tests/mw-dev/trino-multicell.tmpl.yaml +++ b/tests/mw-dev/trino-multicell.tmpl.yaml @@ -39,6 +39,13 @@ metadata: name: trino-cell-projection namespace: ${TRINO_CELL_NAMESPACE} rules: + # The fixture checks worker file availability without reading file contents. + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create", "get"] - apiGroups: [""] resources: ["secrets", "configmaps"] verbs: ["create"] diff --git a/tests/mw-dev/trino_multicell_test.go b/tests/mw-dev/trino_multicell_test.go index 1dc46538..71d53c2f 100644 --- a/tests/mw-dev/trino_multicell_test.go +++ b/tests/mw-dev/trino_multicell_test.go @@ -56,6 +56,65 @@ func TestTrinoRegistryOnlyHarnessFollowsLegacyCompatibility(t *testing.T) { } } +func TestTrinoCellRegistryWaitsForServiceConvergence(t *testing.T) { + raw, err := os.ReadFile("e2e/trino-multicell.sh") + if err != nil { + t.Fatal(err) + } + text := string(raw) + start := strings.Index(text, "wait_cell_registry() {") + if start < 0 { + t.Fatal("missing bounded registry convergence helper") + } + end := strings.Index(text[start:], "\n}\n") + if end < 0 { + t.Fatal("missing registry helper end") + } + helper := text[start : start+end+3] + for _, tc := range []struct { + name, responses string + wantSuccess bool + }{ + {"connection refused then ready", "refused\ncell-test\n", true}, + {"old endpoint then ready", "cell-test,legacy\ncell-test\n", true}, + {"wrong registry never accepted", "cell-test,legacy\n", false}, + {"unavailable API times out", "refused\n", false}, + } { + t.Run(tc.name, func(t *testing.T) { + responses := filepath.Join(t.TempDir(), "responses") + if err := os.WriteFile(responses, []byte(tc.responses), 0o600); err != nil { + t.Fatal(err) + } + script := `set -eu +api() { + [ "$1" = --max-time ] && [ "$2" = 5 ] || exit 90 + response=$(head -n 1 "$RESPONSES") + if [ "$(wc -l < "$RESPONSES")" -gt 1 ]; then + tail -n +2 "$RESPONSES" > "$RESPONSES.next" + mv "$RESPONSES.next" "$RESPONSES" + fi + [ "$response" != refused ] || return 7 + printf %s "$response" | jq -Rc '{cells:(split(",") | map({id:.}))}' +} +sleep() { :; } +fail() { echo "$*" >&2; exit 1; } +API=http://fixture.invalid +` + helper + ` +wait_cell_registry '["cell-test"]' +` + cmd := exec.Command("sh", "-c", script) + cmd.Env = append(os.Environ(), "RESPONSES="+responses) + output, err := cmd.CombinedOutput() + if (err == nil) != tc.wantSuccess { + t.Fatalf("success=%v, want=%v: %s", err == nil, tc.wantSuccess, output) + } + if !tc.wantSuccess && !strings.Contains(string(output), "cell registry did not converge") { + t.Fatalf("missing actionable timeout: %s", output) + } + }) + } +} + func TestTrinoMulticellRenderedBackendsAreIsolated(t *testing.T) { envsubst, err := exec.LookPath("envsubst") if err != nil { @@ -89,6 +148,8 @@ func TestTrinoMulticellRenderedBackendsAreIsolated(t *testing.T) { deployments := map[string]map[string]any{} secrets := []map[string]any{} publicManifests := []map[string]any{} + workerPermissions := map[string]bool{} + primaryWorkerExec := false for { var manifest map[string]any if err := decoder.Decode(&manifest); err == io.EOF { @@ -110,12 +171,62 @@ func TestTrinoMulticellRenderedBackendsAreIsolated(t *testing.T) { if manifest["kind"] == "Deployment" { deployments[manifestName(manifest)] = manifest } + if manifest["kind"] == "Role" && manifestName(manifest) == "duckgres-control-plane" { + if manifest["metadata"].(map[string]any)["namespace"] != "duckgres-ci-pr-123" { + t.Fatal("legacy worker inspection must remain in the isolated primary namespace") + } + for _, rawRule := range manifest["rules"].([]any) { + rule := rawRule.(map[string]any) + resources, err := json.Marshal(rule["resources"]) + if err != nil { + t.Fatal(err) + } + if string(resources) == `["pods/exec"]` { + verbs, err := json.Marshal(rule["verbs"]) + if err != nil || string(verbs) != `["get","create"]` { + t.Fatal("legacy worker exec must use only get/create") + } + primaryWorkerExec = true + } + } + } + if manifest["kind"] == "Role" && manifestName(manifest) == "trino-cell-projection" { + if manifest["metadata"].(map[string]any)["namespace"] != "duckgres-ci-pr-0123" { + t.Fatal("worker inspection permissions must remain in the isolated secondary namespace") + } + for _, rawRule := range manifest["rules"].([]any) { + rule := rawRule.(map[string]any) + for _, resource := range rule["resources"].([]any) { + if resource != "pods" && resource != "pods/exec" { + continue + } + verbs, err := json.Marshal(rule["verbs"]) + if err != nil { + t.Fatal(err) + } + want := `["get","list"]` + if resource == "pods/exec" { + want = `["create","get"]` + } + if string(verbs) != want { + t.Fatalf("unexpected worker inspection verbs: %s", verbs) + } + workerPermissions[resource.(string)] = true + } + } + } if manifest["kind"] == "Secret" { secrets = append(secrets, manifest) } else { publicManifests = append(publicManifests, manifest) } } + if !workerPermissions["pods"] || !workerPermissions["pods/exec"] { + t.Fatal("missing isolated worker mount inspection permissions") + } + if !primaryWorkerExec { + t.Fatal("missing isolated legacy worker inspection permission") + } passwordBytes, err := os.ReadFile(filepath.Join(secretDir, "duckgres-ci-config-store-password")) if err != nil { t.Fatal("renderer must generate a per-run config-store credential:", err) diff --git a/tests/mw-dev/trino_worker_projection_test.go b/tests/mw-dev/trino_worker_projection_test.go new file mode 100644 index 00000000..2f9e9b35 --- /dev/null +++ b/tests/mw-dev/trino_worker_projection_test.go @@ -0,0 +1,176 @@ +package e2emwdev_test + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestTrinoWorkerProjectionGate(t *testing.T) { + raw, err := os.ReadFile("e2e/trino.sh") + if err != nil { + t.Fatal(err) + } + text := string(raw) + start := strings.Index(text, "wait_worker_tenant_file() {") + if start < 0 { + t.Fatal("missing worker file convergence gate") + } + end := strings.Index(text[start:], "\n}\n") + if end < 0 { + t.Fatal("unterminated worker helper") + } + helper := text[start : start+end+3] + if strings.Contains(helper, "--request-timeout") || strings.Count(helper, `timeout 5 "$KUBECTL"`) != 3 { + t.Fatal("bound kubectl externally to preserve in-cluster credential discovery") + } + for _, tc := range []struct { + name string + ok bool + }{ + {"ready", true}, {"one-worker-lags", true}, {"missing-file", false}, + {"no-pods", false}, {"unready", false}, {"terminating", false}, + {"wrong-label", false}, {"wrong-name", false}, {"denied", false}, + {"stale-generation", false}, {"ready-count", false}, {"updated-count", false}, + {"wrong-identity", false}, {"missing-worker", false}, + {"exec-timeout", false}, {"exec-denied", false}, + {"legacy-a", true}, {"legacy-b", true}, {"legacy-lag", true}, + {"legacy-cross", false}, {"legacy-escape", false}, + {"blue-cross-a", false}, {"green-cross-b", false}, {"other-org", false}, + } { + t.Run(tc.name, func(t *testing.T) { + counter := filepath.Join(t.TempDir(), "checks") + script := `set -eu +PR=123 +NS=duckgres-ci-pr-123 +ORG_C=ci-pr-123-trinoc +CELL_NS=duckgres-ci-pr-0123 +[ "$MODE" != wrong-identity ] || CELL_NS=another-namespace +TEST_COLOR=blue +TEST_ORG=$ORG_C +EXPECTED_NS=$CELL_NS +EXPECTED_APP=duckgres-trino-blue +EXPECTED_COUNT=2 +case "$MODE" in + legacy-*) + TEST_COLOR=legacy + TEST_ORG=ci-pr-123-trinoa + EXPECTED_NS=$NS + EXPECTED_APP=duckgres-trino + EXPECTED_COUNT=3 + case "$MODE" in + legacy-b) TEST_ORG=ci-pr-123-trinob ;; + legacy-cross) TEST_ORG=$ORG_C ;; + legacy-escape) NS=another-namespace ;; + esac + ;; + blue-cross-a) TEST_ORG=ci-pr-123-trinoa ;; + green-cross-b) TEST_COLOR=green; TEST_ORG=ci-pr-123-trinob ;; + other-org) TEST_ORG=another-org ;; +esac +KUBECTL=kubectl +sleep() { :; } +timeout() { [ "$1" = 5 ] || exit 90; shift; "$@"; } +log() { echo "$*"; } +fail() { echo "$*" >&2; exit 1; } +kubectl() { + [ "$1" = -n ] && [ "$2" = "$EXPECTED_NS" ] || exit 90 + shift 2 + if [ "$MODE" = denied ]; then return 1; fi + case "$1 $2" in + 'get deployment') + [ "$3" = "$EXPECTED_APP-worker" ] || exit 90 + jq -nc --arg mode "$MODE" --argjson count "$EXPECTED_COUNT" '{metadata:{generation:1},spec:{replicas:$count},status:{observedGeneration:1,readyReplicas:$count,updatedReplicas:$count}} | + if $mode == "stale-generation" then .status.observedGeneration=0 + elif $mode == "ready-count" then .status.readyReplicas=1 + elif $mode == "updated-count" then .status.updatedReplicas=1 + else . end' + ;; + 'get pods') + [ "$3" = -l ] && [ "$4" = "app=$EXPECTED_APP,component=worker" ] || exit 90 + jq -nc --arg mode "$MODE" --arg app "$EXPECTED_APP" --argjson count "$EXPECTED_COUNT" '{items:[range(1;$count+1) | {metadata:{name:($app+"-worker-"+tostring),labels:{app:$app,component:"worker"}},status:{phase:"Running",conditions:[{type:"Ready",status:"True"}]}}]} | + if $mode == "no-pods" then .items=[] + elif $mode == "missing-worker" then .items=.items[:1] + elif $mode == "unready" then .items[1].status.conditions[0].status="False" + elif $mode == "terminating" then .items[1].metadata.deletionTimestamp="now" + elif $mode == "wrong-label" then .items[1].metadata.labels.app="another-cell" + elif $mode == "wrong-name" then .items[1].metadata.name="another-worker" + else . end' + ;; + exec*) + [ "$3" = -c ] && [ "$4" = trino-worker ] && [ "$5" = -- ] && [ "$6" = test ] && [ "$7" = -r ] && [ "$8" = "/etc/trino/tenant-secrets/$TEST_ORG" ] || exit 90 + printf '%s\n' "$2" >> "$COUNTER" + if [ "$MODE" = missing-file ]; then echo 'command terminated with exit code 1' >&2; return 1; fi + if [ "$MODE" = exec-timeout ]; then echo 'context deadline exceeded private-payload' >&2; return 1; fi + if [ "$MODE" = exec-denied ]; then echo 'Forbidden private-payload' >&2; return 1; fi + if [ "$MODE" = one-worker-lags ] && [ "$2" = duckgres-trino-blue-worker-2 ] && [ "$(wc -l < "$COUNTER")" -lt 3 ]; then return 1; fi + if [ "$MODE" = legacy-lag ] && [ "$2" = duckgres-trino-worker-3 ] && [ "$(wc -l < "$COUNTER")" -lt 4 ]; then return 1; fi + ;; + *) exit 90 ;; + esac +} +` + helper + "\nwait_worker_tenant_file \"$TEST_COLOR\" \"$TEST_ORG\"\n" + cmd := exec.Command("sh", "-c", script) + cmd.Env = append(os.Environ(), "MODE="+tc.name, "COUNTER="+counter) + out, err := cmd.CombinedOutput() + if (err == nil) != tc.ok { + t.Fatalf("success=%v, want=%v: %s", err == nil, tc.ok, out) + } + if tc.name == "missing-file" && !strings.Contains(string(out), "worker-file (exit 1)") { + t.Fatal("missing safe file-check failure diagnostics") + } + if tc.name == "denied" && !strings.Contains(string(out), "deployment-read (exit 1)") { + t.Fatal("missing safe API failure diagnostics") + } + if tc.name == "exec-timeout" && !strings.Contains(string(out), "worker-exec-timeout (exit 1)") { + t.Fatal("missing safe timeout diagnostics") + } + if tc.name == "exec-denied" && !strings.Contains(string(out), "worker-exec-permission (exit 1)") { + t.Fatal("missing safe permission diagnostics") + } + if strings.Contains(string(out), "private-payload") { + t.Fatal("raw exec stderr must never enter diagnostics") + } + if tc.ok { + checks, err := os.ReadFile(counter) + if err != nil || !strings.Contains(string(checks), "worker-1") || !strings.Contains(string(checks), "worker-2") { + t.Fatal("every worker must pass the file check") + } + if strings.HasPrefix(tc.name, "legacy-") && !strings.Contains(string(checks), "worker-3") { + t.Fatal("all three legacy workers must pass") + } + } + }) + } + for _, tenant := range []string{"A", "B"} { + gate := strings.Index(text, `wait_worker_tenant_file legacy "$ORG_`+tenant+`"`) + write := strings.Index(text, `"CREATE SCHEMA $CAT_A.$schema"`) + if tenant == "B" { + write = strings.Index(text, `"CREATE TABLE $CAT_B.main.$foreign_table`) + } + if gate < 0 || write < 0 || gate > write { + t.Fatalf("legacy %s gate must precede first write", tenant) + } + } + raw, err = os.ReadFile("e2e/trino-multicell.sh") + if err != nil { + t.Fatal(err) + } + text = string(raw) + blueGate := strings.Index(text, "wait_worker_tenant_file blue") + firstWrite := strings.Index(text, `"CREATE SCHEMA $CAT_C.cell_test"`) + if blueGate < 0 || firstWrite < 0 || blueGate > firstWrite { + t.Fatal("worker gate must precede all initial writes") + } + greenPhase := strings.Index(text, `TRINO="$GREEN_TRINO"`) + if greenPhase < 0 { + t.Fatal("missing green phase") + } + greenGate := strings.Index(text[greenPhase:], "wait_worker_tenant_file green") + greenQuery := strings.Index(text[greenPhase:], `result="$(trino_query`) + if greenGate < 0 || greenQuery < 0 || greenGate > greenQuery { + t.Fatal("green must check worker projection before its first query") + } +}