diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index a80dfaaf..00b5254d 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -9,6 +9,19 @@ name: E2E Tests on: workflow_dispatch: inputs: + suite: + description: "E2E suite to run" + required: false + default: all + type: choice + options: + - all + - historical-rbac-migration + kubernetes_version: + description: "Exact AKS/Flex Node Kubernetes patch version (for example, 1.34.9)" + required: false + default: "1.35.0" + type: string skip_cleanup: description: "Skip cleanup (keep resources for debugging)" required: false @@ -56,7 +69,10 @@ env: AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} GITHUB_RUN_ID: ${{ github.run_id }} - E2E_WORK_DIR: /tmp/aks-flex-node-e2e-${{ github.run_id }} + E2E_NAME_SUFFIX: ${{ github.run_id }}-${{ github.run_attempt }} + E2E_WORK_DIR: /tmp/aks-flex-node-e2e-${{ github.run_id }}-${{ github.run_attempt }} + E2E_SUITE: ${{ inputs.suite || 'all' }} + E2E_KUBERNETES_VERSION: ${{ inputs.kubernetes_version || '1.35.0' }} jobs: e2e: @@ -95,7 +111,7 @@ jobs: E2E_SKIP_CLEANUP: "1" # Cleanup handled in dedicated step below run: | set -euo pipefail - ./hack/e2e/run.sh all + ./hack/e2e/run.sh "${E2E_SUITE}" - name: Collect logs if: always() @@ -105,16 +121,59 @@ jobs: uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: always() with: - name: e2e-logs-${{ github.run_id }} - path: /tmp/aks-flex-node-e2e-${{ github.run_id }}/logs/ + name: e2e-logs-${{ github.run_id }}-${{ github.run_attempt }} + path: /tmp/aks-flex-node-e2e-${{ github.run_id }}-${{ github.run_attempt }}/logs/ retention-days: 7 - name: Cleanup + id: cleanup if: always() env: E2E_SKIP_CLEANUP: ${{ inputs.skip_cleanup && '1' || '0' }} run: ./hack/e2e/run.sh cleanup - - name: Cleanup runner workspace + - name: Prepare cleanup diagnostics + if: always() + run: | + set -euo pipefail + umask 077 + state_file="${E2E_WORK_DIR}/state.json" + artifact_file="${E2E_WORK_DIR}/cleanup-state.json" + [[ -f "${state_file}" ]] || exit 0 + # Keep the resource names needed for investigation without publishing + # secret-sourced account identifiers, addresses, or cluster access data. + jq '{ + lifecycle, + cleanup_complete, + deployment_name, + run_id, + resource_owner, + name_suffix, + cluster_name, + node_resource_group, + msi_vm_name, + token_vm_name, + offline_vm_name, + kubeadm_vm_name, + arc_vm_name, + arc_machine_name, + vnet_name, + nsg_name + } | with_entries(select(.value != null and .value != ""))' \ + "${state_file}" > "${artifact_file}" + chmod 0600 "${artifact_file}" + + - name: Upload cleanup diagnostics if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: e2e-cleanup-state-${{ github.run_id }}-${{ github.run_attempt }} + path: /tmp/aks-flex-node-e2e-${{ github.run_id }}-${{ github.run_attempt }}/cleanup-state.json + if-no-files-found: ignore + retention-days: 7 + + - name: Cleanup runner workspace + # Preserve retry metadata when Azure cleanup fails or is deliberately + # skipped. A later successful run must not delete another attempt's state. + if: always() && steps.cleanup.outcome == 'success' && inputs.skip_cleanup != true run: ./hack/e2e/run.sh runner-cleanup diff --git a/docs/usages/aks-flex-config.md b/docs/usages/aks-flex-config.md index 8448dfbd..f61b70e0 100644 --- a/docs/usages/aks-flex-config.md +++ b/docs/usages/aks-flex-config.md @@ -9,7 +9,7 @@ The helper does not install anything on the target host. It uses Azure CLI and, - Azure CLI authenticated to the subscription that contains the AKS cluster. - `python3` on the workstation. - `kubectl` on the workstation for `setup-node-rbac` and `--bootstrap-token` config generation. -- Permission to run `az aks get-credentials --admin` and create Kubernetes `ClusterRoleBinding` and bootstrap token `Secret` objects. +- Permission to run `az aks get-credentials --admin`, create Kubernetes `ClusterRoleBinding` and bootstrap token `Secret` objects, and remove the obsolete `aks-flex-node-role` binding when present. ## Save The Helper @@ -46,7 +46,53 @@ Run this once per cluster for bootstrap-token joins: --subscription "$SUBSCRIPTION_ID" ``` -This applies the bootstrap-related `ClusterRoleBinding` objects for the `system:bootstrappers:aks-flex-node` group. +This applies only the CSR creation and approval `ClusterRoleBinding` objects for the `system:bootstrappers:aks-flex-node` group. If any binding still grants that group the obsolete `system:node` role, the command stops after applying the safe bindings and explains how to migrate. It does not silently remove the binding because older and development-mode agents may still use their bootstrap token after joining. + +`v0.1.1` introduced a separate daemon client certificate, but the version alone does not prove that certificate was issued successfully. Upgrade every bootstrap-token agent to `v0.1.1` or later (preferably the latest release), and on every host verify that the certificate exists, is unexpired, and the agent remains healthy after a restart: + +```bash +sudo test -s /etc/aks-flex-node/daemon-credentials/daemon-controller-current.pem +sudo openssl x509 \ + -in /etc/aks-flex-node/daemon-credentials/daemon-controller-current.pem \ + -noout -subject -enddate -checkend 0 +sudo systemctl restart aks-flex-node-agent.service +sudo systemctl is-active aks-flex-node-agent.service +``` + +Then explicitly remove the obsolete binding: + +```bash +./aks-flex-config setup-node-rbac \ + --resource-group "$RESOURCE_GROUP" \ + --cluster-name "$CLUSTER_NAME" \ + --subscription "$SUBSCRIPTION_ID" \ + --remove-legacy-node-role-binding +``` + +This migration is idempotent. It automatically deletes only the canonical `aks-flex-node-role` object created by older helpers. If another binding grants the same unsafe edge, or that object has extra subjects, the helper refuses to guess and identifies the objects for manual review. Bootstrap-token config generation refuses to create a token while any such binding exists, rather than either issuing an over-privileged token or unexpectedly breaking an old daemon. + +To verify no binding still grants the bootstrap group `system:node`, run: + +```bash +kubectl get clusterrolebinding -o json | jq -r ' + .items[] + | select(.roleRef.kind == "ClusterRole" and .roleRef.name == "system:node") + | .metadata.name as $binding + | .subjects[]? + | select(.kind == "Group" and .name == "system:bootstrappers:aks-flex-node") + | $binding' +``` + +The expected result is no output. The canonical `aks-flex-node-role` object is +deleted; a safe, repurposed object with that name is preserved. Once certificate +issuance has been verified, both the kubelet and long-running Flex daemon use +issued client certificates, so removing the unsafe binding does not interrupt +joined nodes. New and in-progress joins retain the CSR permissions installed +above. + +Do not roll back a migrated host to an older or development-mode agent that still uses the bootstrap token for ordinary Kubernetes API requests. After this binding is removed, those requests correctly receive `403 Forbidden`. Restore a supported certificate-using agent instead of restoring the broad binding. + +Finally, delete bootstrap-token Secrets that are no longer needed. In particular, tokens made by helpers before `v0.1.1` had no expiration. Removing the broad binding limits them to bootstrap permissions, but does not revoke them; do not delete a token that is still being used by an in-progress join. ## Generate Node Config diff --git a/docs/usages/joining-nodes.md b/docs/usages/joining-nodes.md index 427bde56..15551bc6 100644 --- a/docs/usages/joining-nodes.md +++ b/docs/usages/joining-nodes.md @@ -16,7 +16,7 @@ Bootstrap token mode is the recommended quickstart path. It uses Kubernetes TLS High-level flow: -1. Run [`scripts/aks-flex-config setup-node-rbac`](../../scripts/aks-flex-config) to setup required node bootstrap RBAC permissions. +1. Run [`scripts/aks-flex-config setup-node-rbac`](../../scripts/aks-flex-config) to set up the least-privilege node bootstrap RBAC permissions. Clusters configured by an older helper require the explicit compatibility migration documented in the helper guide before another token can be generated. 2. Run `scripts/aks-flex-config generate-node-config --bootstrap-token` to create a bootstrap token, fetch AKS cluster metadata, and render the host config. 3. Copy the generated config to `/etc/aks-flex-node/config.json` on the target host. 4. Run `aks-flex-node preflight --config /etc/aks-flex-node/config.json` to validate host, cluster, rootfs, and artifact prerequisites without mutating the node. diff --git a/hack/e2e/README.md b/hack/e2e/README.md index 075b96ca..706e1593 100644 --- a/hack/e2e/README.md +++ b/hack/e2e/README.md @@ -12,6 +12,8 @@ The E2E suite provisions a no-CNI AKS cluster, installs Unbounded-Net as the clu | `python3` | Local registry port readiness checks and helper scripts. | | `ssh` / `scp` | VM access and artifact copy. | | `openssl` | Bootstrap token generation. | +| `curl` / `sha256sum` / `tar` | Download and verify pinned historical release artifacts. | +| `flock` | Serialize atomic updates to the per-run cleanup state. | | `docker` | Build and push the controller image into the in-cluster local registry. | | `git` / `make` | Fetch and render Unbounded-Net manifests. | | `go` | Build the agent binary unless `--binary` is supplied. | @@ -65,6 +67,7 @@ The default `all` command runs: | Command | Description | |---------|-------------| | `all` | Full flow: build, infra, join, validate, unjoin, validate absent, rejoin, validate, lifecycle, agent upgrade, repave, logs, cleanup. | +| `historical-rbac-migration` | On a fresh real AKS cluster and token VM, join with the official v0.1.0 helper/binary, upgrade that host to HEAD, migrate legacy bootstrap RBAC, revoke the old token, and validate restarts. It does not run the other join modes. | | `infra` | Deploy AKS, four standard VMs, the Arc VM, Unbounded-Net CNI, the local registry, and the in-cluster controller. | | `join` | Join all Flex Node VMs. | | `join-msi` | Join only the managed-identity node. | @@ -107,7 +110,7 @@ Additional environment variables: | `E2E_SSH_KEY_FILE` | auto-detected | SSH public key used for VM access. | | `E2E_WORK_DIR` | `/tmp/aks-flex-node-e2e` | Working directory for state, configs, and logs. | | `E2E_KUBECONFIG` | `$E2E_WORK_DIR/kubeconfig` | Per-run kubeconfig path. Defaults to an isolated file instead of the runner-global kubeconfig. | -| `E2E_KUBERNETES_VERSION` | `1.35.0` | Kubernetes version used in generated node configs. | +| `E2E_KUBERNETES_VERSION` | `1.35.0` | Exact Kubernetes version used for the AKS control plane, agent pools, and generated node configs. | | `E2E_CONTAINERD_VERSION` | `2.0.4` | Containerd version used in generated node configs. | | `E2E_RUNC_VERSION` | `1.1.12` | Runc version used in generated node configs. | | `E2E_TARGET_AGENT_POOL_NAME` | `aksflexnodes` | Synthetic target agent pool name used by controller-backed test modes. | @@ -135,9 +138,57 @@ Additional environment variables: | `E2E_POD_READY_TIMEOUT` | `120` | Timeout in seconds while waiting for smoke pods. | | `E2E_AGENT_UPGRADE_TIMEOUT` | `300` | Timeout in seconds while waiting for an AgentUpgrade result. | | `E2E_DRIFT_UPGRADE_TIMEOUT` | `900` | Timeout in seconds while waiting for repave. | +| `E2E_CLEANUP_TIMEOUT` | `900` | Shared deadline in seconds for deployment cancellation and Azure resource deletion. | +| `E2E_CLEANUP_POLL_INTERVAL` | `5` | Poll interval in seconds for deployment and cleanup convergence. | | `AZURE_SUBSCRIPTION_ID` | auto-detected | Azure subscription. | | `AZURE_TENANT_ID` | auto-detected | Azure tenant. | +## Historical RBAC Migration Validation + +Run the focused compatibility suite with: + +```bash +./hack/e2e/run.sh historical-rbac-migration +``` + +For a manual GitHub Actions run, select `historical-rbac-migration` in the +`suite` workflow input. Set `kubernetes_version` to an exact AKS version, such +as the latest supported N-1 patch, to exercise the migration against an older +control-plane version. Check regional availability immediately before running +the workflow with `az aks get-versions --location --output table`. +Infrastructure provisioning, the test, log upload, and cleanup stay in the same +job; the suite deliberately does not call the Arc-inclusive parallel join path. + +The scenario downloads and verifies the official v0.1.0 release archive, +extracted binary, helper, and installer. It then uses the pinned helper and +binary to create the original broad bootstrap RBAC, a non-expiring token, the +legacy config shape, and a real Ready node on the token VM. The historical +daemon runs its production no-op path, not the v0.1.0 file-backed E2E machine +client. On that same host it verifies the HEAD helper fails closed without the +explicit migration flag, activates the HEAD binary through `agent-upgrade`, +removes the legacy binding twice to prove idempotency, checks token access +changes from HTTP 200 to 403, deletes the daemon credential store and verifies +the remaining CSR permissions issue a different certificate, revokes the token +and waits for HTTP 401, and restarts both kubelet and the daemon while checking +the Node UID, Lease, readiness, and certificate-backed API access. +v0.1.0 transitively pins the non-GPU rootfs +`ghcr.io/azure/agent-ubuntu2404:v20260427`. + +There are two intentional compatibility boundaries: + +- The test creates a new AKS control plane and reproduces the v0.1.0 + cluster-side state. It validates a historical node/config/RBAC migration, not + an AKS control plane that has itself been retained since v0.1.0. Selecting an + older `kubernetes_version` proves a newly created control plane at that + version; it still does not reproduce age, prior upgrades, or configuration + drift from a long-lived cluster. +- v0.1.0 tokens lack the `kubernetes.azure.com/managedby=aks` label required by + the production managed CSR approver. The suite explicitly adopts its known + token with that label before the HEAD daemon requests a certificate. The + repository E2E approver does not enforce this label, so this test validates + the host/config/RBAC migration but is not independent proof of the production + approver's ownership check. + ## Join Modes The suite validates five join paths. The E2E subscription must have `Microsoft.HybridCompute`, `Microsoft.HybridConnectivity`, and `Microsoft.GuestConfiguration` registered before the run. diff --git a/hack/e2e/cleanup_safety_test.go b/hack/e2e/cleanup_safety_test.go new file mode 100644 index 00000000..16ee77e5 --- /dev/null +++ b/hack/e2e/cleanup_safety_test.go @@ -0,0 +1,376 @@ +package e2e_test + +import ( + _ "embed" + "os" + "path/filepath" + "strings" + "testing" +) + +//go:embed lib/infra.sh +var infraScript string + +func TestCleanupTargetNameValidation(t *testing.T) { + t.Parallel() + + valid := []string{ + "test", + "e2e-test", + "aks-e2e-test", + "MC_aksflex-e2e-test", + "vm-e2e-msi-test", + "vm-e2e-token-test", + "vm-e2e-offline-test", + "vm-e2e-kubeadm-test", + "vm-e2e-arc-test", + "vm-e2e-arc-test-connected", + "vnet-e2e-test", + "nsg-e2e-test", + "test-rg", + "test-location", + } + + tests := []struct { + name string + argument int + value string + wantSuccess bool + wantError string + }{ + {name: "matching targets", argument: -1, wantSuccess: true}, + {name: "legacy default node resource group", argument: 3, value: "MC_test-rg_aks-e2e-test_test-location", wantSuccess: true}, + {name: "missing optional target", argument: 4, value: "", wantSuccess: true}, + {name: "missing suffix", argument: 0, value: "", wantError: "without an E2E name suffix"}, + {name: "deployment", argument: 1, value: "production-deployment", wantError: "unexpected ARM deployment"}, + {name: "cluster", argument: 2, value: "production-cluster", wantError: "unexpected AKS cluster"}, + {name: "node resource group", argument: 3, value: "production-node-rg", wantError: "unexpected AKS node resource group"}, + {name: "MSI VM", argument: 4, value: "production-msi", wantError: "unexpected MSI VM"}, + {name: "token VM", argument: 5, value: "production-token", wantError: "unexpected token VM"}, + {name: "offline VM", argument: 6, value: "production-offline", wantError: "unexpected offline VM"}, + {name: "kubeadm VM", argument: 7, value: "production-kubeadm", wantError: "unexpected kubeadm VM"}, + {name: "Arc VM", argument: 8, value: "production-arc", wantError: "unexpected Arc VM"}, + {name: "Arc machine", argument: 9, value: "production-machine", wantError: "unexpected Arc machine"}, + {name: "virtual network", argument: 10, value: "production-vnet", wantError: "unexpected virtual network"}, + {name: "network security group", argument: 11, value: "production-nsg", wantError: "unexpected network security group"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + args := append([]string(nil), valid...) + if test.argument >= 0 { + args[test.argument] = test.value + } + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +shift 2 +if _validate_cleanup_target_names "$@"; then + printf 'RESULT=success\n' +else + printf 'RESULT=error\n' +fi +` + output, err := runBash(t, script, append([]string{t.TempDir(), cleanupScript}, args...)...) + if err != nil { + t.Fatalf("target validation harness failed: %v\n%s", err, output) + } + if test.wantSuccess { + if !strings.Contains(string(output), "RESULT=success") { + t.Fatalf("matching cleanup targets were rejected:\n%s", output) + } + return + } + if !strings.Contains(string(output), "RESULT=error") || + !strings.Contains(string(output), test.wantError) { + t.Fatalf("unsafe cleanup target was not rejected with %q:\n%s", test.wantError, output) + } + }) + } +} + +func TestCleanupHandlesLegacyDefaultNodeResourceGroup(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + parentGroupAbsent bool + }{ + {name: "live cluster"}, + {name: "orphan after parent deletion", parentGroupAbsent: true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + _, _, callLog, output, err := runCleanup(t, cleanupOptions{ + runTwice: true, + parentGroupAbsent: test.parentGroupAbsent, + legacyDefaultNodeResource: true, + }) + if err != nil { + t.Fatalf("legacy node resource group cleanup failed: %v\n%s", err, output) + } + if strings.Count(string(output), "RESULT=success") != 2 { + t.Fatalf("legacy node resource group cleanup was not idempotent:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + if test.parentGroupAbsent { + if !strings.Contains(string(calls), "group delete --name MC_test-rg_aks-e2e-test_test-location") { + t.Fatalf("cleanup did not delete the proven orphan node resource group:\n%s", calls) + } + return + } + if !strings.Contains(string(calls), "aks wait --resource-group test-rg --name aks-e2e-test") { + t.Fatalf("cleanup did not wait for AKS to delete its legacy node resource group:\n%s", calls) + } + for _, operation := range []string{ + "group exists --name MC_test-rg_aks-e2e-test_test-location", + "group delete --name MC_test-rg_aks-e2e-test_test-location", + "group wait --name MC_test-rg_aks-e2e-test_test-location", + } { + if strings.Contains(string(calls), operation) { + t.Fatalf("live-cluster cleanup directly managed its node resource group with %q:\n%s", operation, calls) + } + } + }) + } +} + +func TestCleanupResourceOwnerCompatibility(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + state string + githubRunID string + want string + wantError string + }{ + { + name: "new state uses attempt-scoped owner", + state: `{"resource_owner":"12345-2","run_id":"12345","name_suffix":"12345-2"}`, + githubRunID: "12345", + want: "12345-2", + }, + { + name: "legacy state falls back to run ID", + state: `{"run_id":"12345"}`, + githubRunID: "different-run", + want: "12345", + }, + { + name: "partial legacy state falls back to environment", + state: `{}`, + githubRunID: "12345", + want: "12345", + }, + { + name: "new state rejects another attempts owner", + state: `{"resource_owner":"12345-1","run_id":"12345","name_suffix":"12345-2"}`, + githubRunID: "12345", + wantError: "does not match E2E name suffix", + }, + { + name: "new partial state rejects unverifiable owner", + state: `{"resource_owner":"12345-2","run_id":"12345"}`, + githubRunID: "12345", + wantError: "does not match E2E name suffix", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + if err := os.WriteFile(filepath.Join(workDir, "state.json"), []byte(test.state), 0o600); err != nil { + t.Fatalf("write state: %v", err) + } + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +GITHUB_RUN_ID="$2" +source "$3" +if owner="$(_cleanup_resource_owner)"; then + printf 'OWNER=%s\n' "${owner}" +else + printf 'OWNER=error\n' +fi +` + output, err := runBash(t, script, workDir, test.githubRunID, cleanupScript) + if err != nil { + t.Fatalf("resolve cleanup owner: %v\n%s", err, output) + } + if test.wantError != "" { + if !strings.Contains(string(output), "OWNER=error\n") || + !strings.Contains(string(output), test.wantError) { + t.Fatalf("unsafe cleanup owner was not rejected with %q: %q", test.wantError, output) + } + return + } + if !strings.Contains(string(output), "OWNER="+test.want+"\n") { + t.Fatalf("cleanup owner = %q, want %q", output, test.want) + } + }) + } +} + +func TestInfrastructureUsesAttemptScopedOwnershipTag(t *testing.T) { + t.Parallel() + + for _, required := range []string{ + `local resource_owner="${E2E_NAME_SUFFIX}"`, + `--arg resource_owner "${resource_owner}"`, + `run_id: $run_id, resource_owner: $resource_owner`, + `--arg run "${resource_owner}"`, + `'{"github-run": $run, "purpose": $purpose}'`, + } { + if !strings.Contains(infraScript, required) { + t.Errorf("infrastructure ownership metadata is missing %q", required) + } + } +} + +func TestLegacyRunTagCleanupIsScopedToNameSuffix(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + inventoryPath := filepath.Join(workDir, "inventory.json") + inventory := `[ + {"type":"Microsoft.Compute/disks","name":"vm-e2e-token-12345-1_OsDisk_1_abcd","id":"/own-disk"}, + {"type":"Microsoft.Network/networkInterfaces","name":"vm-e2e-msi-12345-1-nic","id":"/own-nic"}, + {"type":"Microsoft.Compute/disks","name":"vm-e2e-token-12345-2_OsDisk_1_efgh","id":"/other-attempt-disk"}, + {"type":"Microsoft.Network/virtualNetworks","name":"vnet-e2e-12345-2","id":"/other-attempt-vnet"}, + {"type":"Microsoft.Compute/disks","name":"production-disk","id":"/unrelated"} +]` + if err := os.WriteFile(inventoryPath, []byte(inventory), 0o600); err != nil { + t.Fatalf("write resource inventory: %v", err) + } + + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +INVENTORY_FILE="$2" +source "$3" +_resource_inventory() { cat "${INVENTORY_FILE}"; } +az() { printf 'AZ=%s\n' "$*"; } +printf '%s\n' 'IDS-BEGIN' +_tagged_resource_ids test-rg 12345 test-subscription 12345-1 0 +printf '%s\n' 'IDS-END' +_delete_tagged_resources test-rg 12345 test-subscription 12345-1 0 +` + output, err := runBash(t, script, workDir, inventoryPath, cleanupScript) + if err != nil { + t.Fatalf("legacy tagged cleanup failed: %v\n%s", err, output) + } + text := string(output) + for _, ownID := range []string{"/own-disk", "/own-nic"} { + if !strings.Contains(text, ownID) { + t.Errorf("legacy tagged cleanup omitted owned resource %q:\n%s", ownID, text) + } + } + for _, foreignID := range []string{"/other-attempt-disk", "/other-attempt-vnet", "/unrelated"} { + if strings.Contains(text, foreignID) { + t.Errorf("legacy tagged cleanup selected foreign resource %q:\n%s", foreignID, text) + } + } +} + +func TestLegacyImplicitOSDiskCleanupIsScopedToValidatedVMNames(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + inventoryPath := filepath.Join(workDir, "inventory.json") + inventory := `[ + {"type":"Microsoft.Compute/disks","name":"vm-e2e-token-12345-1_OsDisk_1_abcd","id":"/own-legacy-disk"}, + {"type":"Microsoft.Compute/disks","name":"vm-e2e-token-12345-2_OsDisk_1_efgh","id":"/other-attempt-disk"}, + {"type":"Microsoft.Compute/disks","name":"vm-e2e-token-12345-1-extra_OsDisk_1_ijkl","id":"/prefix-collision-disk"}, + {"type":"Microsoft.Compute/disks","name":"vm-e2e-token-12345-1-osdisk","id":"/deterministic-disk"}, + {"type":"Microsoft.Network/networkInterfaces","name":"vm-e2e-token-12345-1_OsDisk_1_abcd","id":"/wrong-resource-type"}, + {"type":"Microsoft.Compute/disks","name":"production-disk","id":"/unrelated-disk"} +]` + if err := os.WriteFile(inventoryPath, []byte(inventory), 0o600); err != nil { + t.Fatalf("write resource inventory: %v", err) + } + + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +INVENTORY_FILE="$2" +source "$3" +_resource_inventory() { cat "${INVENTORY_FILE}"; } +az() { printf 'AZ=%s\n' "$*"; } +printf '%s\n' 'IDS-BEGIN' +_legacy_implicit_os_disk_ids test-rg test-subscription 0 \ + vm-e2e-msi-12345-1 vm-e2e-token-12345-1 +printf '%s\n' 'IDS-END' +_delete_legacy_implicit_os_disks test-rg test-subscription 0 \ + vm-e2e-msi-12345-1 vm-e2e-token-12345-1 +` + output, err := runBash(t, script, workDir, inventoryPath, cleanupScript) + if err != nil { + t.Fatalf("legacy implicit OS-disk cleanup failed: %v\n%s", err, output) + } + text := string(output) + if !strings.Contains(text, "/own-legacy-disk") || + !strings.Contains(text, "resource delete --ids /own-legacy-disk") { + t.Fatalf("legacy implicit OS disk was not selected and deleted:\n%s", text) + } + for _, foreignID := range []string{ + "/other-attempt-disk", + "/prefix-collision-disk", + "/deterministic-disk", + "/wrong-resource-type", + "/unrelated-disk", + } { + if strings.Contains(text, foreignID) { + t.Errorf("legacy implicit OS-disk cleanup selected foreign resource %q:\n%s", foreignID, text) + } + } +} + +func TestCleanupValidatesNamesBeforeAzureMutation(t *testing.T) { + t.Parallel() + + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + contents, err := os.ReadFile(cleanupScript) + if err != nil { + t.Fatalf("read cleanup script: %v", err) + } + body := string(contents) + cleanupStart := strings.Index(body, "cleanup() {") + if cleanupStart < 0 { + t.Fatal("cleanup function is absent") + } + body = body[cleanupStart:] + validation := strings.Index(body, "if ! _validate_cleanup_target_names") + if validation < 0 { + t.Fatal("cleanup does not validate deterministic target names") + } + for _, mutation := range []string{ + `_delete_node_resource_group "${node_resource_group}"`, + `az rest --method delete`, + `az vm delete`, + `az aks delete`, + `az network vnet delete`, + `_delete_tagged_resources`, + } { + mutationIndex := strings.Index(body, mutation) + if mutationIndex < 0 { + t.Fatalf("cleanup mutation %q is absent", mutation) + } + if validation >= mutationIndex { + t.Errorf("cleanup validates names after mutation %q", mutation) + } + } +} diff --git a/hack/e2e/e2e_scripts_test.go b/hack/e2e/e2e_scripts_test.go new file mode 100644 index 00000000..265dd738 --- /dev/null +++ b/hack/e2e/e2e_scripts_test.go @@ -0,0 +1,1662 @@ +package e2e_test + +import ( + "context" + "embed" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// Embedding the scripts makes Go's test cache invalidate on shell-only changes. +// +//go:embed run.sh lib/common.sh lib/cleanup.sh lib/bootstrap-rbac-migration.sh lib/runner.sh infra/*.bicep infra/modules/*.bicep +var e2eScripts embed.FS + +func TestRunnerCleanupIsScopedToCurrentAttempt(t *testing.T) { + t.Parallel() + + script, err := e2eScripts.ReadFile("lib/runner.sh") + if err != nil { + t.Fatalf("read embedded runner script: %v", err) + } + text := string(script) + for _, required := range []string{ + `local work_dir="${E2E_WORK_DIR:-}"`, + `^/tmp/aks-flex-node-e2e(-[A-Za-z0-9][A-Za-z0-9._-]*)?$`, + `rm -rf -- "${work_dir}"`, + `Refusing to clean unexpected E2E work directory`, + } { + if !strings.Contains(text, required) { + t.Errorf("runner cleanup is missing attempt-scoped guard %q", required) + } + } + if strings.Contains(text, `find /tmp`) || strings.Contains(text, `-name 'aks-flex-node-e2e-*'`) { + t.Error("runner cleanup still removes work directories owned by other attempts") + } +} + +func TestRunnerCleanupRejectsUnsafeWorkDirectories(t *testing.T) { + t.Parallel() + + runnerScript := e2eScriptPath(t, "lib", "runner.sh") + tests := map[string]func(string) string{ + "unexpected parent": func(targetDir string) string { return targetDir }, + "parent traversal": func(targetDir string) string { return "/tmp/.." + targetDir }, + } + for name, workDirPath := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + targetDir := filepath.Join(t.TempDir(), "aks-flex-node-e2e-test") + if err := os.MkdirAll(targetDir, 0o700); err != nil { + t.Fatalf("create protected directory: %v", err) + } + sentinel := filepath.Join(targetDir, "sentinel") + if err := os.WriteFile(sentinel, []byte("keep"), 0o600); err != nil { + t.Fatalf("write protected sentinel: %v", err) + } + workDir := workDirPath(targetDir) + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +sudo() { return 1; } +go() { return 0; } +if cleanup_runner_workspace; then + printf 'RESULT=unexpected-success\n' +else + printf 'RESULT=rejected\n' +fi +` + output, err := runBash(t, script, workDir, runnerScript) + if err != nil { + t.Fatalf("runner cleanup safety harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=rejected") { + t.Fatalf("runner cleanup accepted unsafe work directory %q:\n%s", workDir, output) + } + if _, err := os.Stat(sentinel); err != nil { + t.Fatalf("runner cleanup removed data outside its allowed path: %v", err) + } + }) + } +} + +func TestBicepModuleDeploymentNamesAreUniquePerRun(t *testing.T) { + t.Parallel() + + mainBicep, err := e2eScripts.ReadFile("infra/main.bicep") + if err != nil { + t.Fatalf("read embedded main.bicep: %v", err) + } + for _, moduleName := range []string{"msi", "token", "offline", "kubeadm", "arc"} { + want := "name: 'deploy-vm-" + moduleName + "-${nameSuffix}'" + if !strings.Contains(string(mainBicep), want) { + t.Errorf("VM module %q does not use a per-run deployment name %q", moduleName, want) + } + } +} + +func TestLoadConfigTreatsEmptyOptionalValuesAsImplicit(t *testing.T) { + t.Parallel() + + commonScript := e2eScriptPath(t, "lib", "common.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +E2E_RESOURCE_GROUP=test-rg +E2E_LOCATION=test-location +AZURE_SUBSCRIPTION_ID=test-subscription +AZURE_TENANT_ID=test-tenant +E2E_NAME_SUFFIX=test +E2E_KUBERNETES_VERSION= +E2E_TARGET_AGENT_POOL_NAME= +E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME= +load_config >/dev/null +printf 'RESULT=%s|%s|%s|%s|%s|%s\n' \ + "${_E2E_KUBERNETES_VERSION_EXPLICIT}" "${E2E_KUBERNETES_VERSION}" \ + "${_E2E_TARGET_AGENT_POOL_NAME_EXPLICIT}" "${E2E_TARGET_AGENT_POOL_NAME}" \ + "${_E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME_EXPLICIT}" "${E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME}" +` + output, err := runBash(t, script, t.TempDir(), commonScript) + if err != nil { + t.Fatalf("load config failed: %v\n%s", err, output) + } + const want = "RESULT=0|1.35.0|0|aksflexnodes|0|aksflexnodes\n" + if !strings.Contains(string(output), want) { + t.Fatalf("empty optional values were treated as explicit: got %q, want %q", output, want) + } +} + +func TestFullE2ECleanupFailureIsAggregated(t *testing.T) { + t.Parallel() + + runScript, err := e2eScripts.ReadFile("run.sh") + if err != nil { + t.Fatalf("read embedded run.sh: %v", err) + } + if !strings.Contains(string(runScript), "cleanup || exit_code=1") { + t.Fatal("full E2E does not aggregate cleanup failure into its final result") + } +} + +func TestRestoreKubernetesVersionFromState(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + configuredVersion string + explicit string + persistedVersion string + liveVersion string + liveSystemVersion string + liveFlexVersion string + lookupFails string + omitSubscription bool + wantVersion string + wantError string + }{ + { + name: "restores persisted version when unset", + configuredVersion: "1.35.0", + explicit: "0", + persistedVersion: "1.34.9", + liveVersion: "1.34.9", + wantVersion: "1.34.9", + }, + { + name: "accepts matching explicit version", + configuredVersion: "1.34.9", + explicit: "1", + persistedVersion: "1.34.9", + liveVersion: "1.34.9", + wantVersion: "1.34.9", + }, + { + name: "rejects mismatched explicit version", + configuredVersion: "1.35.0", + explicit: "1", + persistedVersion: "1.34.9", + liveVersion: "1.34.9", + wantError: "existing cluster state records 1.34.9", + }, + { + name: "rejects invalid persisted version", + configuredVersion: "1.35.0", + explicit: "0", + persistedVersion: "1.34", + liveVersion: "1.34", + wantError: "must be an exact x.y.z patch version", + }, + { + name: "recovers version from legacy state", + configuredVersion: "1.35.0", + explicit: "0", + liveVersion: "1.34.9", + wantVersion: "1.34.9", + }, + { + name: "rejects stale persisted version", + configuredVersion: "1.34.9", + explicit: "0", + persistedVersion: "1.34.9", + liveVersion: "1.35.0", + wantError: "live cluster is 1.35.0", + }, + { + name: "fails closed when live lookup fails", + configuredVersion: "1.35.0", + explicit: "0", + persistedVersion: "1.35.0", + lookupFails: "1", + wantError: "Cannot determine the live Kubernetes version", + }, + { + name: "rejects control plane and pool skew", + configuredVersion: "1.34.9", + explicit: "0", + persistedVersion: "1.34.9", + liveVersion: "1.34.9", + liveSystemVersion: "1.34.9", + liveFlexVersion: "1.35.0", + wantError: "AKS version skew", + }, + { + name: "requires persisted subscription", + configuredVersion: "1.35.0", + explicit: "0", + persistedVersion: "1.35.0", + omitSubscription: true, + wantError: "cluster and subscription", + }, + } + + commonScript := e2eScriptPath(t, "lib", "common.sh") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + liveSystemVersion := tt.liveSystemVersion + if liveSystemVersion == "" { + liveSystemVersion = tt.liveVersion + } + liveFlexVersion := tt.liveFlexVersion + if liveFlexVersion == "" { + liveFlexVersion = tt.liveVersion + } + subscription := "test-subscription" + if tt.omitSubscription { + subscription = "" + } + + script := ` +set -euo pipefail +unset AZURE_SUBSCRIPTION_ID +E2E_WORK_DIR="$1" +source "$2" +E2E_KUBERNETES_VERSION="$3" +_E2E_KUBERNETES_VERSION_EXPLICIT="$4" +LIVE_VERSION="$6" +LIVE_SYSTEM_VERSION="$7" +LIVE_FLEX_VERSION="$8" +LOOKUP_FAILS="$9" +SUBSCRIPTION="${10}" +az() { + [[ "${LOOKUP_FAILS}" != "1" ]] || return 1 + if [[ "$1 $2" == "aks show" ]]; then + [[ "$*" == *"--subscription test-subscription"* ]] || { + echo 'aks lookup omitted persisted subscription' >&2 + return 1 + } + jq -n --arg version "${LIVE_VERSION}" \ + '{id: "/subscriptions/test-subscription/resourceGroups/test-rg/providers/Microsoft.ContainerService/managedClusters/test-aks", version: $version}' + elif [[ "$1" == "rest" && "$*" == *'/agentPools/system?'* ]]; then + printf '%s\n' "${LIVE_SYSTEM_VERSION}" + elif [[ "$1" == "rest" ]]; then + printf '%s\n' "${LIVE_FLEX_VERSION}" + else + return 1 + fi +} +state_set resource_group test-rg +state_set cluster_name test-aks +if [[ -n "${SUBSCRIPTION}" ]]; then + state_set subscription_id "${SUBSCRIPTION}" +fi +if [[ -n "$5" ]]; then + state_set kubernetes_version "$5" +fi +restore_kubernetes_version_from_state +printf 'RESULT=%s\n' "${E2E_KUBERNETES_VERSION}" +` + output, err := runBash(t, script, t.TempDir(), commonScript, + tt.configuredVersion, tt.explicit, tt.persistedVersion, tt.liveVersion, + liveSystemVersion, liveFlexVersion, tt.lookupFails, subscription) + if tt.wantError != "" { + if err == nil { + t.Fatalf("expected error containing %q, got success:\n%s", tt.wantError, output) + } + if !strings.Contains(string(output), tt.wantError) { + t.Fatalf("error output %q does not contain %q", output, tt.wantError) + } + return + } + if err != nil { + t.Fatalf("restore script failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT="+tt.wantVersion+"\n") { + t.Fatalf("output %q does not contain restored version %q", output, tt.wantVersion) + } + }) + } +} + +func TestStateRequiresVerifiedCleanupBeforeReplacement(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + stateJSON string + wantAllow bool + }{ + {name: "missing state", wantAllow: true}, + {name: "empty state", stateJSON: `{}`, wantAllow: true}, + {name: "verified cleanup string", stateJSON: `{"deployment_name":"e2e-old","lifecycle":"cleaned","cleanup_complete":"true"}`, wantAllow: true}, + {name: "verified cleanup boolean", stateJSON: `{"deployment_name":"e2e-old","lifecycle":"cleaned","cleanup_complete":true}`, wantAllow: true}, + {name: "same active deployment", stateJSON: `{"deployment_name":"e2e-new","lifecycle":"provisioning"}`}, + {name: "active prior deployment", stateJSON: `{"deployment_name":"e2e-old","lifecycle":"ready"}`}, + {name: "unknown legacy deployment", stateJSON: `{"cluster_name":"old-aks"}`}, + {name: "invalid json", stateJSON: `{not-json`}, + {name: "non-object json", stateJSON: `[]`}, + } + + commonScript := e2eScriptPath(t, "lib", "common.sh") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + workDir := t.TempDir() + statePath := filepath.Join(workDir, "state.json") + if tt.stateJSON != "" { + if err := os.WriteFile(statePath, []byte(tt.stateJSON), 0o600); err != nil { + t.Fatalf("write initial state: %v", err) + } + } + before, _ := os.ReadFile(statePath) + + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +next_state="$(jq -n '{deployment_name: "e2e-new", lifecycle: "provisioning"}')" +if state_begin_deployment "${next_state}"; then + printf 'DECISION=ALLOWED\n' +else + printf 'DECISION=REJECTED\n' +fi +` + output, err := runBash(t, script, workDir, commonScript) + if err != nil { + t.Fatalf("state guard script failed: %v\n%s", err, output) + } + gotAllow := strings.Contains(string(output), "DECISION=ALLOWED\n") + if gotAllow != tt.wantAllow { + t.Fatalf("allow = %t, want %t; output:\n%s", gotAllow, tt.wantAllow, output) + } + after, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read resulting state: %v", readErr) + } + if !tt.wantAllow && string(after) != string(before) { + t.Fatalf("rejected replacement mutated state: before=%q after=%q", before, after) + } + if tt.wantAllow && !strings.Contains(string(after), `"deployment_name": "e2e-new"`) { + t.Fatalf("allowed replacement did not install new state: %s", after) + } + info, statErr := os.Stat(statePath) + if statErr != nil { + t.Fatalf("stat state file: %v", statErr) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("state file mode = %o, want 600", got) + } + }) + } +} + +func TestConcurrentStateWritesArePrivateAndComplete(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + if err := os.WriteFile(filepath.Join(workDir, "state.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatalf("write initial state: %v", err) + } + commonScript := e2eScriptPath(t, "lib", "common.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +for i in $(seq 1 40); do + state_set "key_${i}" "${i}" & +done +wait +jq -e 'length == 40' "${E2E_STATE_FILE}" +` + if output, err := runBash(t, script, workDir, commonScript); err != nil { + t.Fatalf("concurrent state script failed: %v\n%s", err, output) + } + + info, err := os.Stat(filepath.Join(workDir, "state.json")) + if err != nil { + t.Fatalf("stat state file: %v", err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("state file mode = %o, want 600", got) + } +} + +func TestStateWriteFailurePreservesExistingState(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + statePath := filepath.Join(workDir, "state.json") + const invalidState = `{not-json` + if err := os.WriteFile(statePath, []byte(invalidState), 0o600); err != nil { + t.Fatalf("write invalid state: %v", err) + } + commonScript := e2eScriptPath(t, "lib", "common.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +if state_set test value; then + echo 'unexpected success' >&2 + exit 1 +fi +` + if output, err := runBash(t, script, workDir, commonScript); err != nil { + t.Fatalf("failure-path script failed: %v\n%s", err, output) + } + after, err := os.ReadFile(statePath) + if err != nil { + t.Fatalf("read state: %v", err) + } + if string(after) != invalidState { + t.Fatalf("failed state update replaced recoverable state: %q", after) + } +} + +func TestStateDumpRedactsSecrets(t *testing.T) { + t.Parallel() + + commonScript := e2eScriptPath(t, "lib", "common.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +state_set kubeadm_bootstrap_token abcdef.0123456789abcdef +state_set token_vm_ip 192.0.2.1 +state_dump +` + output, err := runBash(t, script, t.TempDir(), commonScript) + if err != nil { + t.Fatalf("state dump script failed: %v\n%s", err, output) + } + if strings.Contains(string(output), "abcdef.0123456789abcdef") { + t.Fatalf("state dump exposed bootstrap token: %s", output) + } + if !strings.Contains(string(output), "192.0.2.1") { + t.Fatalf("state dump unexpectedly redacted non-secret metadata: %s", output) + } +} + +func TestCancelDeploymentWaitsForTerminalOperations(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + initialState string + terminalState string + cancelFails string + listFails string + wantCancel bool + wantError bool + }{ + {name: "running deployment", initialState: "Running", terminalState: "Canceled", wantCancel: true}, + {name: "already succeeded", initialState: "Succeeded", terminalState: "Succeeded"}, + {name: "cancel races natural completion", initialState: "Running", terminalState: "Succeeded", cancelFails: "1", wantCancel: true}, + {name: "query failure", listFails: "1", wantError: true}, + } + + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + workDir := t.TempDir() + callLog := filepath.Join(workDir, "az-calls.log") + marker := filepath.Join(workDir, "deployment-terminal") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +AZ_CALL_LOG="$3" +AZ_MARKER="$4" +INITIAL_STATE="$5" +TERMINAL_STATE="$6" +CANCEL_FAILS="$7" +LIST_FAILS="$8" +E2E_CLEANUP_TIMEOUT=2 +E2E_CLEANUP_POLL_INTERVAL=0.01 +az() { + printf '%s\n' "$*" >> "${AZ_CALL_LOG}" + if [[ "$1 $2 $3" == "deployment group list" ]]; then + [[ "${LIST_FAILS}" != "1" ]] || return 1 + state="${INITIAL_STATE}" + [[ ! -f "${AZ_MARKER}" ]] || state="${TERMINAL_STATE}" + jq -n --arg state "${state}" '[{name:"e2e-test",properties:{provisioningState:$state}}]' + elif [[ "$1 $2 $3" == "deployment group cancel" ]]; then + : > "${AZ_MARKER}" + [[ "${CANCEL_FAILS}" != "1" ]] + elif [[ "$1 $2 $3 $4" == "deployment operation group list" ]]; then + printf '[{"properties":{"provisioningState":"Succeeded"}}]\n' + else + return 1 + fi +} +if _cancel_active_deployment test-rg e2e-test test-subscription; then + printf 'RESULT=success\n' +else + printf 'RESULT=error\n' +fi +` + output, err := runBash(t, script, workDir, cleanupScript, callLog, marker, + tt.initialState, tt.terminalState, tt.cancelFails, tt.listFails) + if err != nil { + t.Fatalf("cancellation script failed: %v\n%s", err, output) + } + gotError := strings.Contains(string(output), "RESULT=error\n") + if gotError != tt.wantError { + t.Fatalf("error = %t, want %t; output:\n%s", gotError, tt.wantError, output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az call log: %v", readErr) + } + gotCancel := strings.Contains(string(calls), "deployment group cancel") + if gotCancel != tt.wantCancel { + t.Fatalf("cancel call = %t, want %t; calls:\n%s", gotCancel, tt.wantCancel, calls) + } + if !tt.wantError { + operationIndex := strings.Index(string(calls), "deployment operation group list") + if operationIndex < 0 { + t.Fatalf("deployment operations were not verified:\n%s", calls) + } + if tt.wantCancel && operationIndex < strings.Index(string(calls), "deployment group cancel") { + t.Fatalf("operations were checked before cancellation:\n%s", calls) + } + } + }) + } +} + +func TestCleanupIsIdempotentAndWaitsForDependencies(t *testing.T) { + t.Parallel() + + workDir, statePath, callLog, output, err := runCleanup(t, cleanupOptions{runTwice: true}) + _ = workDir + if err != nil { + t.Fatalf("cleanup failed: %v\n%s", err, output) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if !strings.Contains(string(state), `"cleanup_complete": "true"`) || + !strings.Contains(string(state), `"lifecycle": "cleaned"`) { + t.Fatalf("cleanup did not record verified completion: %s", state) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + operationIndex := strings.Index(string(calls), "deployment operation group list") + deleteIndex := strings.Index(string(calls), "vm delete") + if operationIndex < 0 || deleteIndex <= operationIndex { + t.Fatalf("resource deletion began before deployment operations were terminal:\n%s", calls) + } + if strings.Count(string(calls), "resource list") < 6 { + t.Fatalf("cleanup did not perform repeated tagged and exact inventories across two runs:\n%s", calls) + } +} + +func TestCleanupResidualPreventsCleanState(t *testing.T) { + t.Parallel() + + _, statePath, _, output, err := runCleanup(t, cleanupOptions{leaveCluster: true}) + if err != nil { + t.Fatalf("cleanup test harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") { + t.Fatalf("cleanup unexpectedly accepted residual resource:\n%s", output) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if strings.Contains(string(state), `"cleanup_complete": "true"`) || + strings.Contains(string(state), `"lifecycle": "cleaned"`) { + t.Fatalf("failed cleanup was marked clean: %s", state) + } +} + +func TestCleanupUsesExactNamesWithoutRunTag(t *testing.T) { + t.Parallel() + + _, _, callLog, output, err := runCleanup(t, cleanupOptions{noRunTags: true}) + if err != nil { + t.Fatalf("cleanup without tags failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=success") { + t.Fatalf("cleanup without tags did not succeed:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + if strings.Contains(string(calls), "--tag github-run=") { + t.Fatalf("cleanup unexpectedly used an absent run tag:\n%s", calls) + } + if !strings.Contains(string(calls), "resource list --resource-group test-rg --subscription test-subscription --output json") { + t.Fatalf("cleanup skipped exact-name verification:\n%s", calls) + } +} + +func TestCleanupUsesPersistedSubscriptionWithoutEnvironment(t *testing.T) { + t.Parallel() + + _, _, callLog, output, err := runCleanup(t, cleanupOptions{unsetSubscriptionEnv: true}) + if err != nil { + t.Fatalf("cleanup without subscription environment failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=success") { + t.Fatalf("cleanup did not use the persisted subscription:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + if !strings.Contains(string(calls), "--subscription test-subscription") { + t.Fatalf("cleanup did not pass the persisted subscription to Azure CLI:\n%s", calls) + } +} + +func TestCleanupHandlesLegacyStateWithoutVMNames(t *testing.T) { + t.Parallel() + + _, statePath, _, output, err := runCleanup(t, cleanupOptions{noRunTags: true, blankVMNames: true}) + if err != nil { + t.Fatalf("legacy partial-state cleanup failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=success") { + t.Fatalf("legacy partial-state cleanup did not succeed:\n%s", output) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if !strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("legacy partial-state cleanup was not marked complete: %s", state) + } +} + +func TestCleanupDeletesDetachedUntaggedLegacyOSDisk(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{ + runTwice: true, + noRunTags: true, + legacyDetachedDisk: true, + }) + if err != nil { + t.Fatalf("legacy detached OS-disk cleanup failed: %v\n%s", err, output) + } + if strings.Count(string(output), "RESULT=success") != 2 { + t.Fatalf("legacy detached OS-disk cleanup was not idempotent:\n%s", output) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if !strings.Contains(string(state), `"cleanup_complete": "true"`) || + !strings.Contains(string(state), `"lifecycle": "cleaned"`) { + t.Fatalf("legacy detached OS-disk cleanup did not record completion: %s", state) + } + + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + callText := string(calls) + if !strings.Contains(callText, "resource delete --ids /own-legacy-disk") { + t.Fatalf("cleanup did not delete the detached untagged legacy OS disk:\n%s", calls) + } + for _, foreignID := range []string{"/other-attempt-disk", "/unrelated-disk"} { + if strings.Contains(callText, "resource delete --ids "+foreignID) { + t.Errorf("cleanup deleted foreign resource %q:\n%s", foreignID, calls) + } + } +} + +func TestCleanupRetriesAndReportsResidualLegacyOSDisk(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{ + noRunTags: true, + legacyDetachedDisk: true, + legacyDiskDeletePersists: true, + }) + if err != nil { + t.Fatalf("cleanup test harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") { + t.Fatalf("cleanup accepted a residual legacy OS disk:\n%s", output) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if strings.Contains(string(state), `"cleanup_complete": "true"`) || + strings.Contains(string(state), `"lifecycle": "cleaned"`) { + t.Fatalf("residual legacy OS disk was marked clean: %s", state) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + if got := strings.Count(string(calls), "resource delete --ids /own-legacy-disk"); got < 4 { + t.Fatalf("legacy OS-disk deletion was attempted %d times, want at least 4:\n%s", got, calls) + } +} + +func TestCleanupDeletesOrphanNodeResourceGroupWhenParentIsAbsent(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{parentGroupAbsent: true}) + if err != nil { + t.Fatalf("orphan cleanup failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=success") { + t.Fatalf("orphan cleanup did not succeed:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + if !strings.Contains(string(calls), "group delete --name MC_aksflex-e2e-test") { + t.Fatalf("orphan node resource group was not deleted:\n%s", calls) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if !strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("orphan cleanup did not record completion: %s", state) + } +} + +func TestCleanupReliesOnAKSDeletionForLiveNodeResourceGroup(t *testing.T) { + t.Parallel() + + _, _, callLog, output, err := runCleanup(t, cleanupOptions{nodeGroupForbidden: true}) + if err != nil { + t.Fatalf("live-cluster cleanup failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=success") { + t.Fatalf("cleanup did not accept confirmed AKS deletion:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + callText := string(calls) + if !strings.Contains(callText, "aks wait --resource-group test-rg --name aks-e2e-test") { + t.Fatalf("cleanup did not wait for confirmed AKS deletion:\n%s", calls) + } + for _, operation := range []string{ + "group exists --name MC_aksflex-e2e-test", + "group delete --name MC_aksflex-e2e-test", + "group wait --name MC_aksflex-e2e-test", + } { + if strings.Contains(callText, operation) { + t.Fatalf("cleanup directly managed the live cluster's node resource group with %q:\n%s", operation, calls) + } + } +} + +func TestCleanupDoesNotTouchNodeResourceGroupWhenAKSRemains(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{ + aksDeleteRemains: true, + nodeGroupForbidden: true, + }) + if err != nil { + t.Fatalf("AKS deletion failure harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") || + !strings.Contains(string(output), "AKS cluster still exists after cleanup timeout") { + t.Fatalf("cleanup accepted a cluster that remained after deletion:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + callText := string(calls) + if !strings.Contains(callText, "aks show --resource-group test-rg --name aks-e2e-test") { + t.Fatalf("cleanup did not verify that the AKS cluster remained:\n%s", calls) + } + for _, operation := range []string{ + "group exists --name MC_aksflex-e2e-test", + "group delete --name MC_aksflex-e2e-test", + "group wait --name MC_aksflex-e2e-test", + } { + if strings.Contains(callText, operation) { + t.Fatalf("cleanup touched a live cluster's node resource group with %q:\n%s", operation, calls) + } + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("failed AKS deletion was marked clean: %s", state) + } +} + +func TestCleanupDoesNotAssumeAKSDeletionWhenWaitIsUnconfirmed(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{ + aksWaitUnconfirmed: true, + nodeGroupForbidden: true, + }) + if err != nil { + t.Fatalf("unconfirmed AKS deletion harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") || + !strings.Contains(string(output), "Failed to confirm AKS cluster deletion") { + t.Fatalf("cleanup accepted an unconfirmed AKS deletion:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + callText := string(calls) + if !strings.Contains(callText, "aks show --resource-group test-rg --name aks-e2e-test") { + t.Fatalf("cleanup did not inspect AKS after the wait command failed:\n%s", calls) + } + for _, operation := range []string{ + "group exists --name MC_aksflex-e2e-test", + "group delete --name MC_aksflex-e2e-test", + "group wait --name MC_aksflex-e2e-test", + } { + if strings.Contains(callText, operation) { + t.Fatalf("cleanup touched the node resource group after an unconfirmed AKS deletion with %q:\n%s", operation, calls) + } + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("unconfirmed AKS deletion was marked clean: %s", state) + } +} + +func TestCleanupPropagatesOrphanNodeResourceGroupDeleteFailure(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{ + parentGroupAbsent: true, + nodeGroupDeleteFails: true, + }) + if err != nil { + t.Fatalf("orphan deletion failure harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") || + !strings.Contains(string(output), "Failed to delete orphaned AKS node resource group") || + !strings.Contains(string(output), "AuthorizationFailed") { + t.Fatalf("cleanup swallowed the orphan node resource group deletion failure:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + callText := string(calls) + if !strings.Contains(callText, "group delete --name MC_aksflex-e2e-test") { + t.Fatalf("cleanup did not attempt orphan node resource group deletion:\n%s", calls) + } + if strings.Contains(callText, "group wait --name MC_aksflex-e2e-test") { + t.Fatalf("cleanup waited after orphan node resource group deletion failed:\n%s", calls) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("failed orphan deletion was marked clean: %s", state) + } +} + +func TestCleanupTreatsOrphanDeleteNotFoundAsIdempotent(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{ + parentGroupAbsent: true, + nodeGroupDeleteNotFound: true, + }) + if err != nil { + t.Fatalf("orphan not-found harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=success") { + t.Fatalf("cleanup did not accept explicit orphan absence:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + callText := string(calls) + if !strings.Contains(callText, "group delete --name MC_aksflex-e2e-test") { + t.Fatalf("cleanup did not encounter the orphan deletion race:\n%s", calls) + } + if strings.Contains(callText, "group wait --name MC_aksflex-e2e-test") { + t.Fatalf("cleanup waited after Azure reported the orphan absent:\n%s", calls) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if !strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("idempotent orphan cleanup was not marked complete: %s", state) + } +} + +func TestCleanupRejectsUnexpectedOrphanNodeResourceGroup(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{ + parentGroupAbsent: true, + unexpectedNodeResource: true, + }) + if err != nil { + t.Fatalf("orphan cleanup test harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") || + !strings.Contains(string(output), "Refusing to delete unexpected AKS node resource group") { + t.Fatalf("cleanup did not reject an unexpected orphan node resource group:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + if strings.Contains(string(calls), "group delete --name production-node-rg") { + t.Fatalf("cleanup attempted to delete an unexpected resource group:\n%s", calls) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("rejected cleanup was marked complete: %s", state) + } +} + +func TestCleanupRejectsUnexpectedLiveNodeResourceGroup(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{unexpectedLiveNodeRG: true}) + if err != nil { + t.Fatalf("live node resource group cleanup test harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") || + !strings.Contains(string(output), "Refusing to delete unexpected AKS node resource group 'production-live-node-rg'") { + t.Fatalf("cleanup did not reject an unexpected live node resource group:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + for _, mutation := range []string{"group delete", "vm delete", "aks delete", "rest --method delete"} { + if strings.Contains(string(calls), mutation) { + t.Fatalf("cleanup attempted mutation %q for an unexpected live node resource group:\n%s", mutation, calls) + } + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("rejected cleanup was marked complete: %s", state) + } +} + +func TestCleanupRejectsUnexpectedArcMachineID(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{unexpectedArcMachineID: true}) + if err != nil { + t.Fatalf("Arc cleanup test harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") || + !strings.Contains(string(output), "Refusing to delete Arc machine with an unexpected persisted resource ID") { + t.Fatalf("cleanup did not reject an unexpected Arc machine ID:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil && !os.IsNotExist(readErr) { + t.Fatalf("read az calls: %v", readErr) + } + if strings.Contains(string(calls), "rest --method delete") { + t.Fatalf("cleanup attempted to delete an unexpected Arc resource:\n%s", calls) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("rejected cleanup was marked complete: %s", state) + } +} + +func TestCleanupRetriesTransientAzureQueries(t *testing.T) { + t.Parallel() + + workDir, _, callLog, output, err := runCleanup(t, cleanupOptions{transientQueryFailures: true}) + if err != nil { + t.Fatalf("transient-query cleanup failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=success") { + t.Fatalf("cleanup did not recover from transient Azure query failures:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + if strings.Count(string(calls), "group exists") < 3 { + t.Errorf("cleanup did not retry a failed resource-group query:\n%s", calls) + } + for _, marker := range []string{"group-query-failed", "resource-query-failed"} { + failures, readErr := os.ReadFile(filepath.Join(workDir, marker)) + if readErr != nil { + t.Fatalf("read %s: %v", marker, readErr) + } + if strings.TrimSpace(string(failures)) != "2" { + t.Errorf("%s recorded %q transient failures, want 2", marker, strings.TrimSpace(string(failures))) + } + } +} + +func TestCleanupFiltersResourceInventoryByTagWithinGroup(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + callLog := filepath.Join(workDir, "az-calls") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +AZ_CALL_LOG="$3" +az() { + printf '%s\n' "$*" >> "${AZ_CALL_LOG}" + printf '%s\n' '[ + {"name":"owned","tags":{"github-run":"test-run"}}, + {"name":"another-run","tags":{"github-run":"other-run"}}, + {"name":"untagged"} + ]' +} +inventory="$(_resource_inventory test-rg test-subscription test-run 0)" +printf 'INVENTORY=%s\n' "${inventory}" +` + output, err := runBash(t, script, workDir, cleanupScript, callLog) + if err != nil { + t.Fatalf("tagged resource inventory harness failed: %v\n%s", err, output) + } + text := string(output) + if !strings.Contains(text, `"name":"owned"`) || + strings.Contains(text, `"name":"another-run"`) || + strings.Contains(text, `"name":"untagged"`) { + t.Fatalf("resource inventory was not filtered to the requested run tag:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + callText := string(calls) + if !strings.Contains(callText, "resource list --resource-group test-rg --subscription test-subscription --output json") { + t.Fatalf("resource inventory was not scoped to the parent resource group:\n%s", calls) + } + if strings.Contains(callText, "--tag") { + t.Fatalf("resource inventory used Azure CLI's incompatible --resource-group/--tag combination:\n%s", calls) + } +} + +func TestCleanupAzureQueryFailureIncludesDiagnostic(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +E2E_CLEANUP_POLL_INTERVAL=0.01 +az() { + printf '%s\n' '(TooManyRequests) ARM throttled the cleanup query' >&2 + return 1 +} +if _resource_inventory test-rg test-subscription "" 0; then + printf 'RESULT=unexpected-success\n' +else + printf 'RESULT=error\n' +fi +` + output, err := runBash(t, script, workDir, cleanupScript) + if err != nil { + t.Fatalf("Azure query diagnostic harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") || + !strings.Contains(string(output), "Last Azure CLI error: (TooManyRequests) ARM throttled the cleanup query") { + t.Fatalf("cleanup omitted the final Azure CLI diagnostic:\n%s", output) + } +} + +func TestCleanupAzureQueriesStopOnAuthorizationFailure(t *testing.T) { + t.Parallel() + + for _, query := range []string{"group", "inventory"} { + t.Run(query, func(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + callLog := filepath.Join(workDir, "az-calls") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +AZ_CALL_LOG="$3" +QUERY="$4" +E2E_CLEANUP_POLL_INTERVAL=0.01 +az() { + printf '%s\n' "$*" >> "${AZ_CALL_LOG}" + printf '%s\n' "ERROR: Operation returned an invalid status 'Forbidden'" >&2 + return 1 +} +deadline=$((SECONDS + 60)) +if [[ "${QUERY}" == "group" ]]; then + _group_exists_with_retry test-rg test-subscription "${deadline}" || true +else + _resource_inventory test-rg test-subscription "" "${deadline}" || true +fi +printf 'CALLS=%s\n' "$(wc -l < "${AZ_CALL_LOG}")" +` + output, err := runBash(t, script, workDir, cleanupScript, callLog, query) + if err != nil { + t.Fatalf("authorization failure harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "after 1 attempts") || + !strings.Contains(string(output), "invalid status 'Forbidden'") || + !strings.Contains(string(output), "CALLS=1") { + t.Fatalf("authorization failure was retried or lost its diagnostic:\n%s", output) + } + }) + } +} + +func TestCleanupAzureQueriesBoundRetriesByAttemptAndDeadline(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + callLog := filepath.Join(workDir, "az-calls") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +AZ_CALL_LOG="$3" +E2E_CLEANUP_POLL_INTERVAL=0.01 +az() { + printf '%s\n' "$*" >> "${AZ_CALL_LOG}" + printf '%s\n' '(TooManyRequests) ARM throttled the cleanup query' >&2 + return 1 +} +sleep() { :; } +deadline=$((SECONDS + 60)) +_group_exists_with_retry test-rg test-subscription "${deadline}" || true +printf 'GROUP_CALLS=%s\n' "$(wc -l < "${AZ_CALL_LOG}")" +: > "${AZ_CALL_LOG}" +_resource_inventory test-rg test-subscription "" "${deadline}" || true +printf 'INVENTORY_CALLS=%s\n' "$(wc -l < "${AZ_CALL_LOG}")" +` + output, err := runBash(t, script, workDir, cleanupScript, callLog) + if err != nil { + t.Fatalf("bounded retry harness failed: %v\n%s", err, output) + } + for _, expected := range []string{"GROUP_CALLS=5", "INVENTORY_CALLS=5"} { + if !strings.Contains(string(output), expected) { + t.Fatalf("Azure query retries were not bounded by attempt count; missing %q:\n%s", expected, output) + } + } +} + +func TestCleanupTreatsResourceGroupNotFoundAsAbsent(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + callLog := filepath.Join(workDir, "az-calls") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +AZ_CALL_LOG="$3" +az() { + printf '%s\n' "$*" >> "${AZ_CALL_LOG}" + printf '%s\n' '(ResourceGroupNotFound) Resource group was not found.' >&2 + return 1 +} +result="$(_group_exists_with_retry test-rg test-subscription 0)" +printf 'RESULT=%s\n' "${result}" +printf 'CALLS=%s\n' "$(wc -l < "${AZ_CALL_LOG}")" +` + output, err := runBash(t, script, workDir, cleanupScript, callLog) + if err != nil { + t.Fatalf("ResourceGroupNotFound harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=false") || + !strings.Contains(string(output), "CALLS=1") { + t.Fatalf("ResourceGroupNotFound was not treated as successful absence:\n%s", output) + } +} + +func TestCleanupAzureQueriesHonorExpiredDeadline(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + callLog := filepath.Join(workDir, "az-calls") + sleepLog := filepath.Join(workDir, "sleep-delay") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +AZ_CALL_LOG="$3" +SLEEP_LOG="$4" +az() { printf '%s\n' "$*" >> "${AZ_CALL_LOG}"; return 1; } +sleep() { printf '%s\n' "$1" > "${SLEEP_LOG}"; } + +SECONDS=10 +if _group_exists_with_retry test-rg test-subscription 5; then + printf 'GROUP=unexpected-success\n' +else + printf 'GROUP=expired\n' +fi +if _resource_inventory test-rg test-subscription "" 5; then + printf 'INVENTORY=unexpected-success\n' +else + printf 'INVENTORY=expired\n' +fi +calls=0 +[[ ! -f "${AZ_CALL_LOG}" ]] || calls="$(wc -l < "${AZ_CALL_LOG}")" +printf 'CALLS=%s\n' "${calls}" + +E2E_CLEANUP_POLL_INTERVAL=30 +SECONDS=10 +_sleep_before_azure_query_retry 11 +printf 'SLEEP=%s\n' "$(<"${SLEEP_LOG}")" +` + output, err := runBash(t, script, workDir, cleanupScript, callLog, sleepLog) + if err != nil { + t.Fatalf("Azure query deadline harness failed: %v\n%s", err, output) + } + for _, expected := range []string{"GROUP=expired", "INVENTORY=expired", "CALLS=0", "SLEEP=1"} { + if !strings.Contains(string(output), expected) { + t.Fatalf("Azure query deadline was not enforced; missing %q:\n%s", expected, output) + } + } +} + +func TestCleanupQueryFailureDoesNotDeleteResources(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{deploymentQueryFails: true}) + if err != nil { + t.Fatalf("cleanup test harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") { + t.Fatalf("cleanup unexpectedly succeeded:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + if strings.Contains(string(calls), "vm delete") || strings.Contains(string(calls), "aks delete") { + t.Fatalf("cleanup deleted resources after deployment query failure:\n%s", calls) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("query failure was marked clean: %s", state) + } +} + +func TestHistoricalCertificateProbeUsesPrivilegedTemporaryFile(t *testing.T) { + t.Parallel() + + script, err := e2eScripts.ReadFile("lib/bootstrap-rbac-migration.sh") + if err != nil { + t.Fatalf("read embedded migration script: %v", err) + } + for _, required := range []string{ + `ca_file="$(sudo mktemp)"`, + `sudo python3 <<'PY' | sudo tee "${ca_file}" >/dev/null`, + `trap 'sudo rm -f "${ca_file}"' EXIT`, + } { + if !strings.Contains(string(script), required) { + t.Fatalf("migration certificate probe is missing %q", required) + } + } +} + +func TestHistoricalMigrationReissuesDaemonCertificateBeforeTokenRevocation(t *testing.T) { + t.Parallel() + + script, err := e2eScripts.ReadFile("lib/bootstrap-rbac-migration.sh") + if err != nil { + t.Fatalf("read embedded migration script: %v", err) + } + text := string(script) + for _, required := range []string{ + `old_fingerprint="$(openssl x509 -in "${credential_path}" -outform DER`, + `rm -rf -- "${credential_dir}"`, + `new_fingerprint="$(openssl x509 -in "${credential_path}" -outform DER`, + `_require_daemon_certificate_access "${vm_ip}" "${server_url}"`, + } { + if !strings.Contains(text, required) { + t.Fatalf("historical migration is missing certificate reissuance check %q", required) + } + } + + migrationIndex := strings.LastIndex(text, ` --remove-legacy-node-role-binding`) + reissueIndex := strings.LastIndex(text, ` _reissue_daemon_certificate_after_migration "${vm_ip}" "${server_url}"`) + revokeIndex := strings.LastIndex(text, ` with_cluster_lock _revoke_historical_bootstrap_token "${config_file}"`) + if migrationIndex < 0 || reissueIndex <= migrationIndex || revokeIndex <= reissueIndex { + t.Fatalf("certificate reissuance must run after RBAC migration and before token revocation") + } +} + +func TestHistoricalTokenRevocationPropagatesDeleteFailure(t *testing.T) { + t.Parallel() + + script, err := e2eScripts.ReadFile("lib/bootstrap-rbac-migration.sh") + if err != nil { + t.Fatalf("read embedded migration script: %v", err) + } + text := string(script) + start := strings.Index(text, "_revoke_historical_bootstrap_token() {") + if start < 0 { + t.Fatal("historical token revocation helper is absent") + } + end := strings.Index(text[start:], "\n}\n") + if end < 0 { + t.Fatal("historical token revocation helper is malformed") + } + body := text[start : start+end] + for _, required := range []string{ + `if ! kubectl delete secret "bootstrap-token-${token_id}" -n kube-system; then`, + `log_error "Failed to revoke the historical bootstrap token"`, + `return 1`, + } { + if !strings.Contains(body, required) { + t.Fatalf("historical token revocation does not fail closed on delete errors; missing %q", required) + } + } +} + +type cleanupOptions struct { + runTwice bool + leaveCluster bool + aksDeleteRemains bool + aksWaitUnconfirmed bool + parentGroupAbsent bool + deploymentQueryFails bool + noRunTags bool + blankVMNames bool + nodeGroupForbidden bool + nodeGroupDeleteFails bool + nodeGroupDeleteNotFound bool + unexpectedNodeResource bool + unexpectedLiveNodeRG bool + legacyDefaultNodeResource bool + legacyDetachedDisk bool + legacyDiskDeletePersists bool + unexpectedArcMachineID bool + transientQueryFailures bool + unsetSubscriptionEnv bool +} + +func runCleanup(t *testing.T, options cleanupOptions) (string, string, string, []byte, error) { + t.Helper() + workDir := t.TempDir() + statePath := filepath.Join(workDir, "state.json") + callLog := filepath.Join(workDir, "az-calls.log") + nodeGroupDeleted := filepath.Join(workDir, "node-group-deleted") + groupQueryFailed := filepath.Join(workDir, "group-query-failed") + resourceQueryFailed := filepath.Join(workDir, "resource-query-failed") + clusterDeleted := filepath.Join(workDir, "cluster-deleted") + state := `{ + "resource_group": "test-rg", + "location": "test-location", + "subscription_id": "test-subscription", + "deployment_name": "e2e-test", + "run_id": "test-run", + "name_suffix": "test", + "lifecycle": "ready", + "cluster_name": "aks-e2e-test", + "node_resource_group": "MC_aksflex-e2e-test", + "msi_vm_name": "vm-e2e-msi-test", + "token_vm_name": "vm-e2e-token-test", + "offline_vm_name": "vm-e2e-offline-test", + "kubeadm_vm_name": "vm-e2e-kubeadm-test", + "arc_vm_name": "vm-e2e-arc-test", + "arc_machine_name": "", + "vnet_name": "vnet-e2e-test", + "nsg_name": "nsg-e2e-test" +}` + if options.noRunTags { + state = strings.Replace(state, `"run_id": "test-run"`, `"run_id": ""`, 1) + } + if options.blankVMNames { + for _, name := range []string{ + "vm-e2e-msi-test", + "vm-e2e-token-test", + "vm-e2e-offline-test", + "vm-e2e-kubeadm-test", + "vm-e2e-arc-test", + } { + state = strings.ReplaceAll(state, name, "") + } + } + if options.unexpectedNodeResource { + state = strings.Replace(state, `"node_resource_group": "MC_aksflex-e2e-test"`, `"node_resource_group": "production-node-rg"`, 1) + } + if options.legacyDefaultNodeResource { + state = strings.Replace(state, " \"node_resource_group\": \"MC_aksflex-e2e-test\",\n", "", 1) + } + if options.unexpectedArcMachineID { + state = strings.Replace(state, `"arc_machine_name": ""`, `"arc_machine_name": "vm-e2e-arc-test-connected", + "arc_machine_id": "/subscriptions/test-subscription/resourceGroups/production-rg/providers/Microsoft.HybridCompute/machines/production-machine"`, 1) + } + if err := os.WriteFile(statePath, []byte(state), 0o600); err != nil { + t.Fatalf("write cleanup state: %v", err) + } + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +AZ_CALL_LOG="$3" +NODE_GROUP_DELETED="$4" +GROUP_QUERY_FAILED="$5" +RESOURCE_QUERY_FAILED="$6" +LEAVE_CLUSTER="$7" +PARENT_GROUP_ABSENT="$8" +DEPLOYMENT_QUERY_FAILS="$9" +RUN_TWICE="${10}" +TRANSIENT_QUERY_FAILURES="${11}" +UNEXPECTED_LIVE_NODE_RG="${12}" +LEGACY_DEFAULT_NODE_RG="${13}" +LEGACY_DETACHED_DISK="${14}" +LEGACY_DISK_DELETE_PERSISTS="${15}" +UNSET_SUBSCRIPTION_ENV="${16}" +AKS_DELETE_REMAINS="${17}" +NODE_GROUP_FORBIDDEN="${18}" +NODE_GROUP_DELETE_FAILS="${19}" +CLUSTER_DELETED="${20}" +AKS_WAIT_UNCONFIRMED="${21}" +NODE_GROUP_DELETE_NOT_FOUND="${22}" +LEGACY_DISK_DELETED="${E2E_WORK_DIR}/legacy-disk-deleted" +# Keep cleanup fixtures independent from the ambient GitHub Actions run. Tests +# that need tag-based cleanup persist an explicit run_id in their state. +unset GITHUB_RUN_ID +if [[ "${UNSET_SUBSCRIPTION_ENV}" == "1" ]]; then + unset AZURE_SUBSCRIPTION_ID +else + AZURE_SUBSCRIPTION_ID=test-subscription +fi +E2E_SKIP_CLEANUP=0 +E2E_CLEANUP_TIMEOUT=5 +E2E_CLEANUP_POLL_INTERVAL=0.01 +az() { + printf '%s\n' "$*" >> "${AZ_CALL_LOG}" + if [[ "$1 $2 $3" == "deployment group list" ]]; then + [[ "${DEPLOYMENT_QUERY_FAILS}" != "1" ]] || return 1 + printf '[{"name":"e2e-test","properties":{"provisioningState":"Succeeded"}}]\n' + elif [[ "$1 $2 $3 $4" == "deployment operation group list" ]]; then + printf '[{"properties":{"provisioningState":"Succeeded"}}]\n' + elif [[ "$1 $2" == "group exists" ]]; then + if [[ "$*" == *"--name MC_aksflex-e2e-test"* || \ + "$*" == *"--name MC_test-rg_aks-e2e-test_test-location"* ]]; then + if [[ "${NODE_GROUP_FORBIDDEN}" == "1" ]]; then + printf '%s\n' "ERROR: Operation returned an invalid status 'Forbidden'" >&2 + return 1 + fi + [[ -f "${NODE_GROUP_DELETED}" ]] && printf 'false\n' || printf 'true\n' + elif [[ "$*" == *"--name test-rg"* ]]; then + query_failures="$(cat "${GROUP_QUERY_FAILED}" 2>/dev/null || printf '0\n')" + if [[ "${TRANSIENT_QUERY_FAILURES}" == "1" && "${query_failures}" -lt 2 ]]; then + printf '%s\n' "$((query_failures + 1))" > "${GROUP_QUERY_FAILED}" + printf '%s\n' '(TooManyRequests) transient resource-group query failure' >&2 + return 1 + fi + [[ "${PARENT_GROUP_ABSENT}" == "1" ]] && printf 'false\n' || printf 'true\n' + else + printf 'false\n' + fi + elif [[ "$1 $2" == "group delete" ]]; then + if [[ "${NODE_GROUP_DELETE_FAILS}" == "1" ]]; then + printf '%s\n' '(AuthorizationFailed) The client cannot delete this resource group.' >&2 + return 1 + fi + if [[ "${NODE_GROUP_DELETE_NOT_FOUND}" == "1" ]]; then + printf '%s\n' '(ResourceGroupNotFound) Resource group was not found.' >&2 + return 1 + fi + : > "${NODE_GROUP_DELETED}" + elif [[ "$1 $2" == "group wait" ]]; then + return 0 + elif [[ "$1 $2" == "vm list" ]]; then + printf '[]\n' + elif [[ "$1 $2" == "aks list" ]]; then + if [[ -f "${CLUSTER_DELETED}" ]]; then + printf '[]\n' + elif [[ "${UNEXPECTED_LIVE_NODE_RG}" == "1" ]]; then + printf '[{"name":"aks-e2e-test","nodeResourceGroup":"production-live-node-rg"}]\n' + elif [[ "${LEGACY_DEFAULT_NODE_RG}" == "1" ]]; then + printf '[{"name":"aks-e2e-test","location":"test-location","nodeResourceGroup":"MC_test-rg_aks-e2e-test_test-location"}]\n' + else + printf '[{"name":"aks-e2e-test","location":"test-location","nodeResourceGroup":"MC_aksflex-e2e-test"}]\n' + fi + elif [[ "$1 $2" == "aks delete" ]]; then + [[ "${AKS_DELETE_REMAINS}" == "1" ]] || : > "${CLUSTER_DELETED}" + elif [[ "$1 $2" == "aks wait" ]]; then + [[ "${AKS_DELETE_REMAINS}" != "1" && "${AKS_WAIT_UNCONFIRMED}" != "1" ]] + elif [[ "$1 $2" == "aks show" ]]; then + if [[ "${AKS_WAIT_UNCONFIRMED}" == "1" ]]; then + printf '%s\n' "ERROR: Operation returned an invalid status 'Forbidden'" >&2 + return 1 + fi + [[ "${AKS_DELETE_REMAINS}" == "1" ]] + elif [[ "$1 $2" == "resource delete" ]]; then + if [[ "$*" == *"--ids /own-legacy-disk"* && \ + "${LEGACY_DISK_DELETE_PERSISTS}" != "1" ]]; then + : > "${LEGACY_DISK_DELETED}" + fi + return 0 + elif [[ "$1 $2" == "resource list" ]]; then + query_failures="$(cat "${RESOURCE_QUERY_FAILED}" 2>/dev/null || printf '0\n')" + if [[ "${TRANSIENT_QUERY_FAILURES}" == "1" && "${query_failures}" -lt 2 ]]; then + printf '%s\n' "$((query_failures + 1))" > "${RESOURCE_QUERY_FAILED}" + printf '%s\n' '(TooManyRequests) transient resource inventory failure' >&2 + return 1 + fi + if [[ "$*" == *"--tag "* ]]; then + [[ "$*" == *"--output json"* ]] && printf '[]\n' || true + elif [[ "$*" == *"--output json"* ]]; then + resource_json='[]' + if [[ "${LEAVE_CLUSTER}" == "1" ]]; then + resource_json="$(jq -c '. + [{"name":"aks-e2e-test","id":"/subscriptions/test/resourceGroups/test-rg/providers/Microsoft.ContainerService/managedClusters/aks-e2e-test"}]' <<<"${resource_json}")" + fi + if [[ "${LEGACY_DETACHED_DISK}" == "1" ]]; then + resource_json="$(jq -c '. + [ + {"type":"Microsoft.Compute/disks","name":"vm-e2e-token-other_OsDisk_1_efgh","id":"/other-attempt-disk"}, + {"type":"Microsoft.Compute/disks","name":"production-disk","id":"/unrelated-disk"} + ]' <<<"${resource_json}")" + if [[ ! -f "${LEGACY_DISK_DELETED}" ]]; then + resource_json="$(jq -c '. + [{"type":"Microsoft.Compute/disks","name":"vm-e2e-token-test_OsDisk_1_abcd","id":"/own-legacy-disk"}]' <<<"${resource_json}")" + fi + fi + printf '%s\n' "${resource_json}" + fi + else + return 0 + fi +} +run_cleanup() { + if cleanup; then + printf 'RESULT=success\n' + else + printf 'RESULT=error\n' + fi +} +run_cleanup +if [[ "${RUN_TWICE}" == "1" ]]; then + run_cleanup +fi +` + output, err := runBash(t, script, workDir, cleanupScript, callLog, nodeGroupDeleted, + groupQueryFailed, resourceQueryFailed, + boolString(options.leaveCluster), boolString(options.parentGroupAbsent), + boolString(options.deploymentQueryFails), boolString(options.runTwice), + boolString(options.transientQueryFailures), boolString(options.unexpectedLiveNodeRG), + boolString(options.legacyDefaultNodeResource), boolString(options.legacyDetachedDisk), + boolString(options.legacyDiskDeletePersists), boolString(options.unsetSubscriptionEnv), + boolString(options.aksDeleteRemains), boolString(options.nodeGroupForbidden), + boolString(options.nodeGroupDeleteFails), clusterDeleted, + boolString(options.aksWaitUnconfirmed), boolString(options.nodeGroupDeleteNotFound)) + return workDir, statePath, callLog, output, err +} + +func boolString(value bool) string { + if value { + return "1" + } + return "0" +} + +func e2eScriptPath(t *testing.T, elements ...string) string { + t.Helper() + root := t.TempDir() + for _, name := range []string{"common.sh", "cleanup.sh", "bootstrap-rbac-migration.sh", "runner.sh"} { + contents, err := e2eScripts.ReadFile(filepath.ToSlash(filepath.Join("lib", name))) + if err != nil { + t.Fatalf("read embedded %s: %v", name, err) + } + path := filepath.Join(root, "lib", name) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("create embedded script directory: %v", err) + } + if err := os.WriteFile(path, contents, 0o700); err != nil { + t.Fatalf("materialize embedded %s: %v", name, err) + } + } + return filepath.Join(append([]string{root}, elements...)...) +} + +func runBash(t *testing.T, script string, args ...string) ([]byte, error) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + commandArgs := append([]string{"-c", script, "e2e-script-test"}, args...) + cmd := exec.CommandContext(ctx, "bash", commandArgs...) + output, err := cmd.CombinedOutput() + if ctx.Err() != nil { + t.Fatalf("shell test timed out: %v\n%s", ctx.Err(), output) + } + return output, err +} diff --git a/hack/e2e/infra/main.bicep b/hack/e2e/infra/main.bicep index af7d9e0c..71a1bc69 100644 --- a/hack/e2e/infra/main.bicep +++ b/hack/e2e/infra/main.bicep @@ -52,6 +52,7 @@ var kubeadmVmName = 'vm-e2e-kubeadm-${nameSuffix}' var arcVmName = 'vm-e2e-arc-${nameSuffix}' var vnetName = 'vnet-e2e-${nameSuffix}' var nsgName = 'nsg-e2e-${nameSuffix}' +var nodeResourceGroupName = 'MC_aksflex-e2e-${nameSuffix}' var subnetAksName = 'snet-aks' var subnetVmName = 'snet-vm' @@ -125,6 +126,7 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-01-01' = { } properties: { dnsPrefix: clusterName + nodeResourceGroup: nodeResourceGroupName kubernetesVersion: kubernetesVersion enableRBAC: true aadProfile: { @@ -167,7 +169,7 @@ resource flexAgentPool 'Microsoft.ContainerService/managedClusters/agentPools@20 // Flex-node VMs (via reusable module) // --------------------------------------------------------------------------- module vmMsi 'modules/vm.bicep' = { - name: 'deploy-vm-msi' + name: 'deploy-vm-msi-${nameSuffix}' params: { location: location vmName: msiVmName @@ -181,7 +183,7 @@ module vmMsi 'modules/vm.bicep' = { } module vmToken 'modules/vm.bicep' = { - name: 'deploy-vm-token' + name: 'deploy-vm-token-${nameSuffix}' params: { location: location vmName: tokenVmName @@ -197,7 +199,7 @@ module vmToken 'modules/vm.bicep' = { } module vmOffline 'modules/vm.bicep' = { - name: 'deploy-vm-offline' + name: 'deploy-vm-offline-${nameSuffix}' params: { location: location vmName: offlineVmName @@ -211,7 +213,7 @@ module vmOffline 'modules/vm.bicep' = { } module vmKubeadm 'modules/vm.bicep' = { - name: 'deploy-vm-kubeadm' + name: 'deploy-vm-kubeadm-${nameSuffix}' params: { location: location vmName: kubeadmVmName @@ -228,7 +230,7 @@ module vmKubeadm 'modules/vm.bicep' = { // an official Arc evaluation host by disabling walinuxagent, blocking Azure // IMDS, and setting MSFT_ARC_TEST before installing the Connected Machine agent. module vmArc 'modules/vm.bicep' = { - name: 'deploy-vm-arc' + name: 'deploy-vm-arc-${nameSuffix}' params: { location: location vmName: arcVmName diff --git a/hack/e2e/infra/modules/vm.bicep b/hack/e2e/infra/modules/vm.bicep index 6c90df49..1a5dde17 100644 --- a/hack/e2e/infra/modules/vm.bicep +++ b/hack/e2e/infra/modules/vm.bicep @@ -120,12 +120,19 @@ resource vm 'Microsoft.Compute/virtualMachines@2024-03-01' = { version: imageVersion } osDisk: { + name: '${vmName}-osdisk' createOption: 'FromImage' + deleteOption: 'Delete' managedDisk: { storageAccountType: 'StandardSSD_LRS' } } } networkProfile: { - networkInterfaces: [ { id: nic.id } ] + networkInterfaces: [ + { + id: nic.id + properties: { deleteOption: 'Delete' } + } + ] } } } diff --git a/hack/e2e/lib/bootstrap-rbac-migration.sh b/hack/e2e/lib/bootstrap-rbac-migration.sh new file mode 100644 index 00000000..6aa4acf1 --- /dev/null +++ b/hack/e2e/lib/bootstrap-rbac-migration.sh @@ -0,0 +1,928 @@ +#!/usr/bin/env bash +# ============================================================================= +# Real-node migration test for the legacy bootstrap-group system:node binding. +# ============================================================================= +set -euo pipefail + +[[ -n "${_E2E_BOOTSTRAP_RBAC_MIGRATION_LOADED:-}" ]] && return 0 +readonly _E2E_BOOTSTRAP_RBAC_MIGRATION_LOADED=1 + +# shellcheck disable=SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/common.sh" + +readonly historicalReleaseTag="v0.1.0" +readonly historicalCommit="65d8d3896371adf2eb13248c3e73f0e83fd418ef" +readonly historicalArchiveName="aks-flex-node-linux-amd64.tar.gz" +readonly historicalBinaryName="aks-flex-node-linux-amd64" +readonly historicalArchiveURL="https://github.com/Azure/AKSFlexNode/releases/download/${historicalReleaseTag}/${historicalArchiveName}" +readonly historicalHelperURL="https://raw.githubusercontent.com/Azure/AKSFlexNode/${historicalCommit}/scripts/aks-flex-config" +readonly historicalInstallerURL="https://raw.githubusercontent.com/Azure/AKSFlexNode/${historicalCommit}/scripts/install.sh" +readonly historicalArchiveSHA256="50922e15999b2fd9c19a298c3c3d9cb5a0a375858258dc42b92699a4427876e6" +readonly historicalBinarySHA256="50b4a62daeb30e635cc8ea5e18c6a204e54510e164eba75fe0322a8a8d56fdc7" +readonly historicalHelperSHA256="8ae38209e1a63f1b3c9d1fb41423644de5de032c3a978cedef100689fe1f19f7" +readonly historicalInstallerSHA256="c2f7cfc92e62c3a9fb96697b3a3ca170bb8c28d11af8f45d0f26f6fce016e9b4" +# v0.1.0 embeds Unbounded v0.1.8, whose non-GPU default resolves to this +# immutable release tag. The old Flex config schema cannot override OCIImage. +readonly historicalRootFS="ghcr.io/azure/agent-ubuntu2404:v20260427" +readonly legacyNodeRoleBinding="aks-flex-node-role" +readonly flexNodeBootstrapGroup="system:bootstrappers:aks-flex-node" +# client-go's rotating FileStore writes the issued certificate and private key +# into this combined PEM. client.crt/client.key are legacy read-only fallbacks. +readonly daemonCredentialPath="/etc/aks-flex-node/daemon-credentials/daemon-controller-current.pem" + +_historical_artifact_dir() { + echo "${E2E_WORK_DIR}/historical-${historicalReleaseTag}" +} + +_verify_sha256() { + local path="$1" + local expected="$2" + local description="$3" + local actual + + actual="$(sha256sum "${path}" | awk '{print $1}')" + if [[ "${actual}" != "${expected}" ]]; then + log_error "${description} SHA-256 mismatch: got ${actual}, want ${expected}" + return 1 + fi +} + +_download_verified_artifact() { + local url="$1" + local path="$2" + local expected="$3" + local description="$4" + local partial="${path}.download" + + if [[ -f "${path}" ]] && _verify_sha256 "${path}" "${expected}" "${description}"; then + log_info "Using cached, verified ${description}: ${path}" + return 0 + fi + + rm -f "${path}" "${partial}" + log_info "Downloading pinned ${description}" + if ! curl --fail --location --proto '=https' --retry 5 --retry-all-errors \ + --silent --show-error --output "${partial}" "${url}"; then + rm -f "${partial}" + return 1 + fi + if ! _verify_sha256 "${partial}" "${expected}" "${description}"; then + rm -f "${partial}" + return 1 + fi + mv "${partial}" "${path}" +} + +_prepare_historical_artifacts() { + require_cmd curl + require_cmd sha256sum + require_cmd tar + + local artifact_dir archive helper installer binary archive_members version_output + artifact_dir="$(_historical_artifact_dir)" + archive="${artifact_dir}/${historicalArchiveName}" + helper="${artifact_dir}/aks-flex-config" + installer="${artifact_dir}/install.sh" + binary="${artifact_dir}/${historicalBinaryName}" + mkdir -p "${artifact_dir}" + + _download_verified_artifact \ + "${historicalArchiveURL}" "${archive}" "${historicalArchiveSHA256}" \ + "${historicalReleaseTag} release archive" + _download_verified_artifact \ + "${historicalHelperURL}" "${helper}" "${historicalHelperSHA256}" \ + "${historicalReleaseTag} aks-flex-config helper" + _download_verified_artifact \ + "${historicalInstallerURL}" "${installer}" "${historicalInstallerSHA256}" \ + "${historicalReleaseTag} installer" + + archive_members="$(tar -tzf "${archive}")" + if [[ "${archive_members}" != "${historicalBinaryName}" ]]; then + log_error "${historicalReleaseTag} archive has unexpected members: ${archive_members}" + return 1 + fi + rm -f "${binary}" + tar --extract --gzip --file "${archive}" --directory "${artifact_dir}" \ + --no-same-owner --no-same-permissions "${historicalBinaryName}" + _verify_sha256 "${binary}" "${historicalBinarySHA256}" \ + "${historicalReleaseTag} extracted binary" + chmod 0755 "${binary}" "${helper}" "${installer}" + + version_output="$("${binary}" version)" + if [[ "${version_output}" != *"Version: ${historicalReleaseTag}"* || \ + "${version_output}" != *"Git Commit: ${historicalCommit:0:7}"* ]]; then + log_error "Pinned historical binary reports unexpected build metadata: ${version_output}" + return 1 + fi + + log_success "Verified official ${historicalReleaseTag} archive, binary, helper, and installer" + log_info "Historical binary's pinned default rootfs: ${historicalRootFS}" +} + +_historical_config_path() { + echo "${E2E_WORK_DIR}/config-token-${historicalReleaseTag}.json" +} + +_head_legacy_config_path() { + echo "${E2E_WORK_DIR}/config-token-${historicalReleaseTag}-head.json" +} + +_historical_token_id() { + local config_file="$1" + jq -er \ + '.azure.bootstrapToken.token | capture("^(?[a-z0-9]{6})\\.[a-z0-9]{16}$").id' \ + "${config_file}" +} + +_generate_historical_config() { + local artifact_dir helper config_file + local vm_name vm_private_ip cluster_name resource_group subscription_id + artifact_dir="$(_historical_artifact_dir)" + helper="${artifact_dir}/aks-flex-config" + config_file="$(_historical_config_path)" + vm_name="$(state_get token_vm_name)" + vm_private_ip="$(state_get token_vm_private_ip)" + cluster_name="$(state_get cluster_name)" + resource_group="$(state_get resource_group)" + subscription_id="$(state_get subscription_id)" + + if [[ -z "${vm_private_ip}" ]] || ! is_valid_ipv4 "${vm_private_ip}"; then + log_error "Invalid token VM private IP in state: '${vm_private_ip}'" + return 1 + fi + if kubectl get clusterrolebinding "${legacyNodeRoleBinding}" >/dev/null 2>&1; then + log_error "Historical scenario requires a fresh cluster without ${legacyNodeRoleBinding}" + return 1 + fi + + log_info "Applying the real ${historicalReleaseTag} cluster-side RBAC" + with_cluster_lock python3 "${helper}" setup-node-rbac \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" + + log_info "Generating a real non-expiring ${historicalReleaseTag} bootstrap token and config" + # The v0.1.0 helper predates the current secure-output implementation. Create + # the destination first so the bootstrap token is never briefly world-readable + # under a permissive runner umask. + install -m 0600 /dev/null "${config_file}" + with_cluster_lock python3 "${helper}" generate-node-config \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" \ + --bootstrap-token \ + --output "${config_file}" + + install -m 0600 /dev/null "${config_file}.tmp" + jq \ + --arg nodeName "${vm_name}" \ + --arg nodeIP "${vm_private_ip}" \ + --arg kubernetesVersion "${E2E_KUBERNETES_VERSION}" \ + --arg containerdVersion "${E2E_CONTAINERD_VERSION}" \ + --arg runcVersion "${E2E_RUNC_VERSION}" \ + '.agent.logLevel = "debug" + | .agent.nodeName = $nodeName + | .node.kubelet.nodeIP = $nodeIP + | .kubernetes.version = $kubernetesVersion + | .containerd.version = $containerdVersion + | .runc.version = $runcVersion' \ + "${config_file}" > "${config_file}.tmp" + mv "${config_file}.tmp" "${config_file}" + chmod 0600 "${config_file}" + + if ! jq -e \ + --arg nodeName "${vm_name}" \ + --arg nodeIP "${vm_private_ip}" \ + '.agent.nodeName == $nodeName + and (.agent | has("e2eMode") | not) + and (.agent | has("machineOperationMode") | not) + and .node.kubelet.nodeIP == $nodeIP + and (.node.kubelet.serverURL | length > 0) + and (.node.kubelet.caCertData | length > 0) + and (.kubernetes.version | length > 0) + and (has("components") | not)' \ + "${config_file}" >/dev/null; then + log_error "Historical helper did not produce the expected legacy config shape" + return 1 + fi +} + +_require_historical_cluster_state() { + local config_file="$1" + local token_id secret + token_id="$(_historical_token_id "${config_file}")" + + if ! kubectl get clusterrolebinding "${legacyNodeRoleBinding}" -o json | jq -e \ + --arg group "${flexNodeBootstrapGroup}" \ + '.roleRef.apiGroup == "rbac.authorization.k8s.io" + and .roleRef.kind == "ClusterRole" + and .roleRef.name == "system:node" + and any(.subjects[]?; + .apiGroup == "rbac.authorization.k8s.io" + and .kind == "Group" + and .name == $group)' >/dev/null; then + log_error "${historicalReleaseTag} helper did not create the expected legacy node-role binding" + return 1 + fi + + secret="$(kubectl -n kube-system get secret "bootstrap-token-${token_id}" -o json)" + if ! jq -e \ + '(.data.expiration // "") == "" + and (.metadata.labels["kubernetes.azure.com/managedby"] // "") == ""' \ + <<<"${secret}" >/dev/null; then + log_error "Historical bootstrap Secret unexpectedly has expiration or AKS ownership metadata" + return 1 + fi + log_success "Verified ${historicalReleaseTag} legacy RBAC and non-expiring, unmanaged token state" +} + +_install_and_start_historical_node() { + local vm_ip="$1" + local artifact_dir config_file + artifact_dir="$(_historical_artifact_dir)" + config_file="$(_historical_config_path)" + + remote_copy "${artifact_dir}/${historicalArchiveName}" "${vm_ip}" "/tmp/${historicalArchiveName}" + remote_copy "${artifact_dir}/aks-flex-config" "${vm_ip}" "/tmp/aks-flex-config-${historicalReleaseTag}" + remote_copy "${artifact_dir}/install.sh" "${vm_ip}" "/tmp/aks-flex-node-install-${historicalReleaseTag}.sh" + remote_copy "${config_file}" "${vm_ip}" "/tmp/config-${historicalReleaseTag}.json" + + remote_exec "${vm_ip}" \ + "HISTORICAL_TAG=${historicalReleaseTag} HISTORICAL_COMMIT=${historicalCommit:0:7} HISTORICAL_ARCHIVE_SHA256=${historicalArchiveSHA256} HISTORICAL_BINARY_SHA256=${historicalBinarySHA256} HISTORICAL_HELPER_SHA256=${historicalHelperSHA256} HISTORICAL_INSTALLER_SHA256=${historicalInstallerSHA256} E2E_NODE_JOIN_TIMEOUT=${E2E_NODE_JOIN_TIMEOUT} DAEMON_CREDENTIAL_PATH=${daemonCredentialPath} bash -s" <<'REMOTE' +set -euo pipefail + +archive=/tmp/aks-flex-node-linux-amd64.tar.gz +helper="/tmp/aks-flex-config-${HISTORICAL_TAG}" +installer="/tmp/aks-flex-node-install-${HISTORICAL_TAG}.sh" +config="/tmp/config-${HISTORICAL_TAG}.json" +extract_dir="/tmp/aks-flex-node-${HISTORICAL_TAG}" +binary="${extract_dir}/aks-flex-node-linux-amd64" + +printf '%s %s\n' "${HISTORICAL_ARCHIVE_SHA256}" "${archive}" | sha256sum --check --strict - +printf '%s %s\n' "${HISTORICAL_HELPER_SHA256}" "${helper}" | sha256sum --check --strict - +printf '%s %s\n' "${HISTORICAL_INSTALLER_SHA256}" "${installer}" | sha256sum --check --strict - + +archive_members="$(tar -tzf "${archive}")" +if [[ "${archive_members}" != "aks-flex-node-linux-amd64" ]]; then + echo "historical archive has unexpected members: ${archive_members}" >&2 + exit 1 +fi +sudo rm -rf "${extract_dir}" +mkdir -p "${extract_dir}" +tar --extract --gzip --file "${archive}" --directory "${extract_dir}" \ + --no-same-owner --no-same-permissions aks-flex-node-linux-amd64 +printf '%s %s\n' "${HISTORICAL_BINARY_SHA256}" "${binary}" | sha256sum --check --strict - +chmod 0755 "${binary}" "${helper}" "${installer}" + +version_output="$("${binary}" version)" +grep -Fq "Version: ${HISTORICAL_TAG}" <<<"${version_output}" +grep -Fq "Git Commit: ${HISTORICAL_COMMIT}" <<<"${version_output}" + +if sudo test -e /usr/local/lib/aks-flex-node/aks-flex-node-current || \ + sudo test -L /usr/local/lib/aks-flex-node/aks-flex-node-current; then + echo "historical test VM already has a managed agent layout" >&2 + exit 1 +fi + +if command -v apt-get >/dev/null 2>&1; then + packages_installed=0 + for attempt in $(seq 1 5); do + if sudo DEBIAN_FRONTEND=noninteractive apt-get \ + -o Acquire::Retries=5 -o Acquire::http::Timeout=30 update && + sudo DEBIAN_FRONTEND=noninteractive apt-get \ + -o Acquire::Retries=5 -o Acquire::http::Timeout=30 \ + install -y --fix-missing ca-certificates curl nftables systemd-container util-linux; then + packages_installed=1 + break + fi + echo "Host package installation failed; retrying (${attempt}/5)..." + sleep 10 + done + if (( packages_installed != 1 )); then + echo "Host package installation failed after retries" >&2 + exit 1 + fi +fi + +sudo AKS_FLEX_NODE_LOCAL_BINARY="${binary}" \ + AKS_FLEX_NODE_VERSION="${HISTORICAL_TAG}" \ + SKIP_AZCLI=true \ + bash "${installer}" --yes +sudo install -m 0600 "${config}" /etc/aks-flex-node/config.json + +if [[ -L /usr/local/bin/aks-flex-node ]]; then + echo "${HISTORICAL_TAG} installer unexpectedly created a managed compatibility symlink" >&2 + exit 1 +fi +printf '%s %s\n' "${HISTORICAL_BINARY_SHA256}" /usr/local/bin/aks-flex-node | sudo sha256sum --check --strict - +if sudo test -e "${DAEMON_CREDENTIAL_PATH}"; then + echo "daemon certificate existed before historical bootstrap" >&2 + exit 1 +fi + +unit=aks-flex-node-historical-bootstrap +sudo systemctl stop "${unit}.service" 2>/dev/null || true +sudo systemctl reset-failed "${unit}.service" 2>/dev/null || true +sudo systemd-run \ + --unit="${unit}" \ + --description="AKS Flex Node ${HISTORICAL_TAG} E2E" \ + --remain-after-exit \ + /usr/local/bin/aks-flex-node bootstrap --config /etc/aks-flex-node/config.json + +deadline=$((SECONDS + E2E_NODE_JOIN_TIMEOUT)) +while ! sudo systemctl is-active --quiet aks-flex-node-agent.service; do + if sudo systemctl is-failed --quiet "${unit}.service"; then + sudo systemctl status "${unit}.service" --no-pager -l >&2 || true + sudo journalctl -u "${unit}.service" -n 100 --no-pager >&2 || true + exit 1 + fi + if (( SECONDS >= deadline )); then + echo "timed out waiting for historical daemon service" >&2 + sudo systemctl status "${unit}.service" --no-pager -l >&2 || true + sudo systemctl status aks-flex-node-agent.service --no-pager -l >&2 || true + sudo journalctl -u "${unit}.service" -n 100 --no-pager >&2 || true + sudo journalctl -u aks-flex-node-agent.service -n 100 --no-pager >&2 || true + exit 1 + fi + sleep 5 +done + +sleep 5 +sudo systemctl is-active --quiet aks-flex-node-agent.service +sudo grep -Fq 'ExecStart=/usr/local/bin/aks-flex-node agent' \ + /etc/systemd/system/aks-flex-node-agent.service +historical_logs="$(sudo journalctl -u aks-flex-node-agent.service --no-pager)" +grep -Fq 'production agent daemon requires AKS RP machine client implementation' \ + <<<"${historical_logs}" +if grep -Fq 'running agent daemon in e2e mode' <<<"${historical_logs}"; then + echo "${HISTORICAL_TAG} daemon unexpectedly started in E2E mode" >&2 + exit 1 +fi +if sudo test -e "${DAEMON_CREDENTIAL_PATH}"; then + echo "${HISTORICAL_TAG} unexpectedly issued a daemon certificate" >&2 + exit 1 +fi +REMOTE +} + +_bootstrap_token_api_probe() { + local config_file="$1" + local probe="$2" + + python3 - "${config_file}" "${probe}" <<'PY' +import base64 +import json +import ssl +import sys +import urllib.error +import urllib.request + +config_path, probe = sys.argv[1:] +with open(config_path, encoding="utf-8") as stream: + config = json.load(stream) + +token = config["azure"]["bootstrapToken"]["token"] +kubelet = config["node"]["kubelet"] +server = kubelet.get("serverURL") +if not server: + cluster_fqdn = kubelet["clusterFQDN"] + server = cluster_fqdn if "://" in cluster_fqdn else "https://" + cluster_fqdn +ca_pem = base64.b64decode(kubelet["caCertData"], validate=True).decode("ascii") +context = ssl.create_default_context(cadata=ca_pem) +headers = {"Authorization": "Bearer " + token} + +if probe == "list-nodes": + request = urllib.request.Request(server.rstrip("/") + "/api/v1/nodes?limit=1", headers=headers) +elif probe == "create-csr": + body = json.dumps( + { + "apiVersion": "authorization.k8s.io/v1", + "kind": "SelfSubjectAccessReview", + "spec": { + "resourceAttributes": { + "group": "certificates.k8s.io", + "resource": "certificatesigningrequests", + "verb": "create", + } + }, + } + ).encode("utf-8") + request = urllib.request.Request( + server.rstrip("/") + "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews", + data=body, + headers={**headers, "Content-Type": "application/json"}, + method="POST", + ) +else: + raise SystemExit("unsupported probe: " + probe) + +try: + with urllib.request.urlopen(request, context=context, timeout=30) as response: + response_body = response.read() + status = response.status +except urllib.error.HTTPError as error: + print("http:" + str(error.code)) + raise SystemExit(0) + +if probe == "create-csr": + review = json.loads(response_body) + print("allowed" if review.get("status", {}).get("allowed") is True else "denied") +else: + print("http:" + str(status)) +PY +} + +_wait_for_bootstrap_token_probe() { + local expected="$1" + local config_file="$2" + local probe="$3" + local elapsed=0 + local actual="" + + while (( elapsed < E2E_NODE_JOIN_TIMEOUT )); do + if actual="$(_bootstrap_token_api_probe "${config_file}" "${probe}")" && \ + [[ "${actual}" == "${expected}" ]]; then + return 0 + fi + sleep 2 + elapsed=$((elapsed + 2)) + done + + log_error "Bootstrap token probe ${probe} returned '${actual}', want '${expected}' after ${E2E_NODE_JOIN_TIMEOUT}s" + return 1 +} + +_kubelet_client_identity() { + local vm_ip="$1" + + remote_exec "${vm_ip}" 'sudo bash -s' <<'REMOTE' +set -euo pipefail +machine="$(python3 - <<'PY' +import json +with open('/etc/aks-flex-node/daemon-state.json', encoding='utf-8') as stream: + print(json.load(stream)['activeMachine']) +PY +)" +systemd-run --machine="${machine}" --quiet --pipe --wait \ + openssl x509 \ + -in /var/lib/kubelet/pki/kubelet-client-current.pem \ + -noout \ + -subject \ + -nameopt RFC2253 +REMOTE +} + +_daemon_client_identity() { + local vm_ip="$1" + remote_exec "${vm_ip}" \ + "sudo openssl x509 -in ${daemonCredentialPath} -noout -subject -nameopt RFC2253" +} + +_require_daemon_certificate_access() { + local vm_ip="$1" + local server_url="$2" + local quoted_server quoted_credential + printf -v quoted_server '%q' "${server_url}" + printf -v quoted_credential '%q' "${daemonCredentialPath}" + + remote_exec "${vm_ip}" \ + "SERVER_URL=${quoted_server} DAEMON_CREDENTIAL_PATH=${quoted_credential} bash -s" <<'REMOTE' +set -euo pipefail +# The credential config is root-only. Keep the derived CA root-only as well and +# write it through a privileged process; redirecting sudo's stdout to a regular +# user's mktemp file fails on hardened hosts. +ca_file="$(sudo mktemp)" +trap 'sudo rm -f "${ca_file}"' EXIT +sudo python3 <<'PY' | sudo tee "${ca_file}" >/dev/null +import base64 +import json +import sys + +with open('/etc/aks-flex-node/config.json', encoding='utf-8') as stream: + ca_data = json.load(stream)['node']['kubelet']['caCertData'] +sys.stdout.buffer.write(base64.b64decode(ca_data, validate=True)) +PY +status="$(sudo curl --silent --show-error \ + --cert "${DAEMON_CREDENTIAL_PATH}" \ + --key "${DAEMON_CREDENTIAL_PATH}" \ + --cacert "${ca_file}" \ + --output /dev/null \ + --write-out '%{http_code}' \ + "${SERVER_URL}/api/v1/nodes?limit=1")" +if [[ "${status}" != "200" ]]; then + echo "daemon client certificate LIST nodes returned HTTP ${status}, want 200" >&2 + exit 1 +fi +REMOTE +} + +_reissue_daemon_certificate_after_migration() { + local vm_ip="$1" + local server_url="$2" + + remote_exec "${vm_ip}" 'sudo bash -s' <<'REMOTE' +set -euo pipefail +readonly credential_dir="/etc/aks-flex-node/daemon-credentials" +readonly credential_path="${credential_dir}/daemon-controller-current.pem" +readonly service_name="aks-flex-node-agent.service" + +if [[ ! -s "${credential_path}" ]]; then + echo "daemon certificate is missing before forced reissuance" >&2 + exit 1 +fi +old_fingerprint="$(openssl x509 -in "${credential_path}" -outform DER | sha256sum | awk '{print $1}')" + +# Stop the process before deleting its test credential store so the restart +# must authenticate with the still-valid bootstrap token and submit a new CSR. +systemctl stop "${service_name}" +rm -rf -- "${credential_dir}" +systemctl start "${service_name}" + +for _ in $(seq 1 60); do + if systemctl is-active --quiet "${service_name}" && [[ -s "${credential_path}" ]]; then + new_fingerprint="$(openssl x509 -in "${credential_path}" -outform DER | sha256sum | awk '{print $1}')" + if [[ -n "${new_fingerprint}" && "${new_fingerprint}" != "${old_fingerprint}" ]]; then + exit 0 + fi + fi + if systemctl is-failed --quiet "${service_name}"; then + break + fi + sleep 2 +done + +echo "daemon did not obtain a different certificate after legacy RBAC removal" >&2 +systemctl status "${service_name}" --no-pager >&2 || true +journalctl -u "${service_name}" -n 100 --no-pager >&2 || true +exit 1 +REMOTE + + _require_daemon_certificate_access "${vm_ip}" "${server_url}" + log_success "Daemon obtained a new certificate through least-privilege CSR RBAC after legacy binding removal" +} + +_require_old_node_survives_guard() { + local vm_name="$1" + local vm_ip="$2" + + remote_exec "${vm_ip}" \ + "DAEMON_CREDENTIAL_PATH=${daemonCredentialPath} bash -s" <<'REMOTE' +set -euo pipefail +sudo systemctl restart aks-flex-node-agent.service +for _ in $(seq 1 30); do + if sudo systemctl is-active --quiet aks-flex-node-agent.service; then + first_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" + sleep 5 + second_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" + if sudo systemctl is-active --quiet aks-flex-node-agent.service && + [[ "${first_pid}" =~ ^[1-9][0-9]*$ ]] && [[ "${second_pid}" == "${first_pid}" ]]; then + break + fi + fi + sleep 2 +done +if ! sudo systemctl is-active --quiet aks-flex-node-agent.service || + [[ "${second_pid:-}" != "${first_pid:-}" ]]; then + sudo systemctl status aks-flex-node-agent.service --no-pager -l >&2 || true + exit 1 +fi +if sudo test -e "${DAEMON_CREDENTIAL_PATH}"; then + echo "historical daemon unexpectedly created a daemon certificate" >&2 + exit 1 +fi +REMOTE + validate_node_joined "${vm_name}" +} + +_prepare_head_legacy_config() { + local source_config="$1" + local target_config="$2" + + install -m 0600 /dev/null "${target_config}.tmp" + jq \ + --arg machineEndpointURL "${E2E_CONTROLLER_SERVICE_PROXY_PATH}" \ + --arg agentPoolName "${E2E_TARGET_AGENT_POOL_NAME}" \ + --arg ociImage "${historicalRootFS}" \ + '.agent.machineClient.mode = "in-cluster" + | .agent.machineClient.endpointUrl = $machineEndpointURL + | .agent.requireMachineRegistration = true + | .agent.machineOperationMode = "disable" + | .azure.targetAgentPoolName = $agentPoolName + | .bootstrap.ociImage = $ociImage' \ + "${source_config}" > "${target_config}.tmp" + mv "${target_config}.tmp" "${target_config}" + + if ! jq -e \ + --arg endpoint "${E2E_CONTROLLER_SERVICE_PROXY_PATH}" \ + --arg ociImage "${historicalRootFS}" \ + '.agent.machineClient.mode == "in-cluster" + and .agent.machineClient.endpointUrl == $endpoint + and .agent.requireMachineRegistration == true + and .agent.machineOperationMode == "disable" + and .bootstrap.ociImage == $ociImage + and (.node.kubelet.serverURL | length > 0) + and (.kubernetes.version | length > 0) + and (has("components") | not)' \ + "${target_config}" >/dev/null; then + log_error "HEAD upgrade config no longer preserves the historical config fields" + return 1 + fi +} + +_upgrade_historical_node_to_head() { + local vm_ip="$1" + local config_file="$2" + local head_sha + head_sha="$(sha256sum "${E2E_BINARY}" | awk '{print $1}')" + + remote_copy "${E2E_BINARY}" "${vm_ip}" /tmp/aks-flex-node-head + remote_copy "${config_file}" "${vm_ip}" /tmp/config-v0.1.0-head.json + + remote_exec "${vm_ip}" \ + "HISTORICAL_BINARY_SHA256=${historicalBinarySHA256} HEAD_BINARY_SHA256=${head_sha} E2E_NODE_JOIN_TIMEOUT=${E2E_NODE_JOIN_TIMEOUT} DAEMON_CREDENTIAL_PATH=${daemonCredentialPath} bash -s" <<'REMOTE' +set -euo pipefail +candidate=/tmp/aks-flex-node-head +current_link=/usr/local/lib/aks-flex-node/aks-flex-node-current +last_good_link=/usr/local/lib/aks-flex-node/aks-flex-node-last-good +service=/etc/systemd/system/aks-flex-node-agent.service + +chmod 0755 "${candidate}" +printf '%s %s\n' "${HEAD_BINARY_SHA256}" "${candidate}" | sha256sum --check --strict - +printf '%s %s\n' "${HISTORICAL_BINARY_SHA256}" /usr/local/bin/aks-flex-node | sudo sha256sum --check --strict - +if sudo test -e "${current_link}" || sudo test -L "${current_link}"; then + echo "managed layout existed before migration preflight" >&2 + exit 1 +fi + +sudo install -m 0600 /tmp/config-v0.1.0-head.json /etc/aks-flex-node/config.json +sudo "${candidate}" agent-upgrade --preflight | sudo tee /tmp/historical-agent-upgrade-preflight.log + +# Preflight must not mutate the direct v0.1.0 installation. +if sudo test -e "${current_link}" || sudo test -L "${current_link}" || \ + sudo test -L /usr/local/bin/aks-flex-node; then + echo "agent-upgrade preflight mutated the legacy binary layout" >&2 + exit 1 +fi +printf '%s %s\n' "${HISTORICAL_BINARY_SHA256}" /usr/local/bin/aks-flex-node | sudo sha256sum --check --strict - +sudo systemctl is-active --quiet aks-flex-node-agent.service + +sudo "${candidate}" agent-upgrade | sudo tee /tmp/historical-agent-upgrade.log + +deadline=$((SECONDS + E2E_NODE_JOIN_TIMEOUT)) +while true; do + if sudo systemctl is-active --quiet aks-flex-node-agent.service && + sudo test -s "${DAEMON_CREDENTIAL_PATH}"; then + first_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" + sleep 5 + second_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" + if sudo systemctl is-active --quiet aks-flex-node-agent.service && + [[ "${first_pid}" =~ ^[1-9][0-9]*$ ]] && [[ "${second_pid}" == "${first_pid}" ]]; then + break + fi + fi + if (( SECONDS >= deadline )); then + echo "HEAD daemon did not become stable with daemon credentials" >&2 + sudo systemctl status aks-flex-node-agent.service --no-pager -l >&2 || true + sudo journalctl -u aks-flex-node-agent.service -n 150 --no-pager >&2 || true + exit 1 + fi + sleep 2 +done + +for link in /usr/local/bin/aks-flex-node "${current_link}" "${last_good_link}"; do + if ! sudo test -L "${link}"; then + echo "managed binary link missing after upgrade: ${link}" >&2 + exit 1 + fi +done +active="$(sudo readlink -f "${current_link}")" +last_good="$(sudo readlink -f "${last_good_link}")" +printf '%s %s\n' "${HEAD_BINARY_SHA256}" "${active}" | sudo sha256sum --check --strict - +printf '%s %s\n' "${HISTORICAL_BINARY_SHA256}" "${last_good}" | sudo sha256sum --check --strict - +if [[ "$(sudo readlink -f /usr/local/bin/aks-flex-node)" != "${active}" ]]; then + echo "compatibility path does not resolve to the active managed binary" >&2 + exit 1 +fi +sudo grep -Fq "ExecStart=${current_link} agent" "${service}" +pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" +if [[ "$(sudo readlink -f "/proc/${pid}/exe")" != "${active}" ]]; then + echo "daemon is not executing the activated HEAD binary" >&2 + exit 1 +fi + +cert_pub="$(sudo openssl x509 -in "${DAEMON_CREDENTIAL_PATH}" -pubkey -noout | sha256sum | awk '{print $1}')" +key_pub="$(sudo openssl pkey -in "${DAEMON_CREDENTIAL_PATH}" -pubout | sha256sum | awk '{print $1}')" +if [[ "${cert_pub}" != "${key_pub}" ]]; then + echo "daemon certificate and private key do not match" >&2 + exit 1 +fi +REMOTE +} + +_revoke_historical_bootstrap_token() { + local config_file="$1" + local token_id + token_id="$(_historical_token_id "${config_file}")" + if ! kubectl delete secret "bootstrap-token-${token_id}" -n kube-system; then + log_error "Failed to revoke the historical bootstrap token" + return 1 + fi + log_info "Revoked the historical bootstrap token after certificate migration" +} + +_restart_daemon_and_require_certificate_access() { + local vm_ip="$1" + local server_url="$2" + + remote_exec "${vm_ip}" 'bash -s' <<'REMOTE' +set -euo pipefail +sudo systemctl restart aks-flex-node-agent.service +for _ in $(seq 1 30); do + if sudo systemctl is-active --quiet aks-flex-node-agent.service; then + first_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" + sleep 5 + second_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" + if sudo systemctl is-active --quiet aks-flex-node-agent.service && + [[ "${first_pid}" =~ ^[1-9][0-9]*$ ]] && [[ "${second_pid}" == "${first_pid}" ]]; then + exit 0 + fi + fi + sleep 2 +done +sudo systemctl status aks-flex-node-agent.service --no-pager -l >&2 || true +sudo journalctl -u aks-flex-node-agent.service -n 100 --no-pager >&2 || true +exit 1 +REMOTE + _require_daemon_certificate_access "${vm_ip}" "${server_url}" +} + +_restart_kubelet_and_require_lease_renewal() { + local vm_name="$1" + local vm_ip="$2" + local node_uid="$3" + local before_renew + local elapsed=0 + + before_renew="$(kubectl get lease "${vm_name}" -n kube-node-lease -o jsonpath='{.spec.renewTime}')" + remote_exec "${vm_ip}" 'sudo bash -s' <<'REMOTE' +set -euo pipefail +machine="$(python3 - <<'PY' +import json +with open('/etc/aks-flex-node/daemon-state.json', encoding='utf-8') as stream: + print(json.load(stream)['activeMachine']) +PY +)" +systemd-run --machine="${machine}" --quiet --pipe --wait systemctl restart kubelet.service +REMOTE + + while (( elapsed < E2E_NODE_JOIN_TIMEOUT )); do + local current_uid renew ready + current_uid="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.uid}' 2>/dev/null || true)" + renew="$(kubectl get lease "${vm_name}" -n kube-node-lease -o jsonpath='{.spec.renewTime}' 2>/dev/null || true)" + ready="$(kubectl get node "${vm_name}" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" + if [[ "${current_uid}" == "${node_uid}" && -n "${renew}" && \ + "${renew}" != "${before_renew}" && "${ready}" == "True" ]]; then + log_success "Historical node renewed its Lease and stayed Ready after token revocation" + return 0 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + + log_error "Historical node did not reauthenticate after token revocation" + kubectl get node "${vm_name}" -o wide 2>&1 || true + kubectl describe node "${vm_name}" 2>&1 || true + return 1 +} + +historical_rbac_migration_e2e() { + log_section "Historical ${historicalReleaseTag} Node and Bootstrap RBAC Migration" + + local config_file head_config vm_name vm_ip cluster_name resource_group subscription_id + local server_url node_uid node_kubelet_version identity guard_output token token_id + config_file="$(_historical_config_path)" + head_config="$(_head_legacy_config_path)" + vm_name="$(state_get token_vm_name)" + vm_ip="$(state_get token_vm_ip)" + cluster_name="$(state_get cluster_name)" + resource_group="$(state_get resource_group)" + subscription_id="$(state_get subscription_id)" + server_url="$(state_get server_url)" + + _prepare_historical_artifacts + _generate_historical_config + _require_historical_cluster_state "${config_file}" + _install_and_start_historical_node "${vm_ip}" + validate_node_joined "${vm_name}" + + node_kubelet_version="$(kubectl get node "${vm_name}" -o jsonpath='{.status.nodeInfo.kubeletVersion}')" + node_kubelet_version="${node_kubelet_version#v}" + if [[ "${node_kubelet_version}" != "${E2E_KUBERNETES_VERSION}" ]]; then + log_error "Historical node kubelet version is ${node_kubelet_version}, expected ${E2E_KUBERNETES_VERSION}" + return 1 + fi + node_uid="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.uid}')" + identity="$(_kubelet_client_identity "${vm_ip}")" + if [[ "${identity}" != *"CN=system:node:${vm_name}"* ]]; then + log_error "${historicalReleaseTag} kubelet has unexpected client identity: ${identity}" + return 1 + fi + _wait_for_bootstrap_token_probe http:200 "${config_file}" list-nodes + log_success "Official ${historicalReleaseTag} node is Ready on Kubernetes ${node_kubelet_version} with legacy RBAC and an issued kubelet certificate" + + # HEAD must converge the safe CSR bindings but preserve the old daemon until + # an operator explicitly confirms the migration. + if guard_output="$(with_cluster_lock "${REPO_ROOT}/scripts/aks-flex-config" setup-node-rbac \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" 2>&1)"; then + log_error "HEAD RBAC setup accepted a legacy binding without explicit migration" + return 1 + fi + if [[ "${guard_output}" != *"--remove-legacy-node-role-binding"* ]]; then + log_error "HEAD compatibility guard did not explain the explicit migration path" + return 1 + fi + if ! kubectl get clusterrolebinding "${legacyNodeRoleBinding}" >/dev/null 2>&1; then + log_error "HEAD compatibility guard removed the legacy binding" + return 1 + fi + _wait_for_bootstrap_token_probe http:200 "${config_file}" list-nodes + _require_old_node_survives_guard "${vm_name}" "${vm_ip}" + log_success "HEAD fail-closed guard preserved the running historical node" + + # Helpers before v0.1.1 did not add the ownership label required by the + # production AKS managed CSR approver. Adopt this known E2E token explicitly + # before the HEAD daemon requests its dedicated certificate. + token="$(jq -er '.azure.bootstrapToken.token' "${config_file}")" + token_id="$(_historical_token_id "${config_file}")" + with_cluster_lock mark_e2e_bootstrap_token_aks_managed "${token}" + unset token + if [[ "$(kubectl -n kube-system get secret "bootstrap-token-${token_id}" \ + -o jsonpath='{.metadata.labels.kubernetes\.azure\.com/managedby}')" != "aks" ]]; then + log_error "Historical token adoption label was not applied" + return 1 + fi + + machine_configmap_upsert "${vm_name}" "${E2E_KUBERNETES_VERSION}" "${E2E_KUBERNETES_VERSION}" + _prepare_head_legacy_config "${config_file}" "${head_config}" + _upgrade_historical_node_to_head "${vm_ip}" "${head_config}" + validate_node_joined "${vm_name}" + + identity="$(_daemon_client_identity "${vm_ip}")" + if [[ "${identity}" != *"CN=system:node:${vm_name}"* || \ + "${identity}" != *"O=aks-flex-node-daemons"* ]]; then + log_error "Upgraded daemon has unexpected issued client identity: ${identity}" + return 1 + fi + _require_daemon_certificate_access "${vm_ip}" "${server_url}" + log_success "Same host upgraded from direct ${historicalReleaseTag} layout to HEAD and obtained daemon credentials" + + # The first run removes the canonical historical object. The second verifies + # that the explicitly requested migration is idempotent. + with_cluster_lock "${REPO_ROOT}/scripts/aks-flex-config" setup-node-rbac \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" \ + --remove-legacy-node-role-binding + with_cluster_lock "${REPO_ROOT}/scripts/aks-flex-config" setup-node-rbac \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" \ + --remove-legacy-node-role-binding + + if kubectl get clusterrolebinding "${legacyNodeRoleBinding}" >/dev/null 2>&1; then + log_error "Legacy binding '${legacyNodeRoleBinding}' still exists after migration" + return 1 + fi + _wait_for_bootstrap_token_probe http:403 "${config_file}" list-nodes + _wait_for_bootstrap_token_probe allowed "${config_file}" create-csr + _reissue_daemon_certificate_after_migration "${vm_ip}" "${server_url}" + + # Revocation proves the subsequent restarts cannot silently fall back to the + # bootstrap credential. + with_cluster_lock _revoke_historical_bootstrap_token "${config_file}" + _wait_for_bootstrap_token_probe http:401 "${config_file}" list-nodes + + _restart_kubelet_and_require_lease_renewal "${vm_name}" "${vm_ip}" "${node_uid}" + _restart_daemon_and_require_certificate_access "${vm_ip}" "${server_url}" + validate_node_joined "${vm_name}" + + identity="$(_kubelet_client_identity "${vm_ip}")" + if [[ "${identity}" != *"CN=system:node:${vm_name}"* ]]; then + log_error "Kubelet lost its node client identity after migration: ${identity}" + return 1 + fi + identity="$(_daemon_client_identity "${vm_ip}")" + if [[ "${identity}" != *"CN=system:node:${vm_name}"* || \ + "${identity}" != *"O=aks-flex-node-daemons"* ]]; then + log_error "Daemon lost its issued client identity after migration: ${identity}" + return 1 + fi + if [[ "$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.uid}')" != "${node_uid}" ]]; then + log_error "Historical node object was replaced during migration" + return 1 + fi + + log_success "Historical ${historicalReleaseTag} node upgraded in place, migrated RBAC twice, revoked its token, and survived kubelet/daemon restarts" +} diff --git a/hack/e2e/lib/cleanup.sh b/hack/e2e/lib/cleanup.sh index 1ebde191..349925bb 100755 --- a/hack/e2e/lib/cleanup.sh +++ b/hack/e2e/lib/cleanup.sh @@ -178,6 +178,535 @@ collect_logs() { # --------------------------------------------------------------------------- # cleanup - Delete Azure resources # --------------------------------------------------------------------------- +_deployment_state() { + local resource_group="$1" deployment_name="$2" subscription_id="$3" + local deployments + + deployments="$(az deployment group list \ + --resource-group "${resource_group}" \ + --subscription "${subscription_id}" \ + --output json 2>/dev/null)" || return 1 + jq -er --arg name "${deployment_name}" ' + if type != "array" then error("deployment list must be an array") + else ([.[] | select(.name == $name)][0].properties.provisioningState // "") + end + ' <<<"${deployments}" +} + +_wait_for_deployment_operations() { + local resource_group="$1" deployment_name="$2" subscription_id="$3" deadline="$4" + local operations + + while (( SECONDS < deadline )); do + if ! operations="$(az deployment operation group list \ + --resource-group "${resource_group}" \ + --name "${deployment_name}" \ + --subscription "${subscription_id}" \ + --output json 2>/dev/null)"; then + log_error "Failed to query operations for ARM deployment '${deployment_name}'" + return 1 + fi + if jq -e ' + type == "array" and all(.[].properties.provisioningState; + . == "Succeeded" or . == "Failed" or . == "Canceled" or + . == "Cancelled" or . == "Skipped") + ' <<<"${operations}" >/dev/null; then + return 0 + fi + sleep "${E2E_CLEANUP_POLL_INTERVAL:-5}" + done + + log_error "ARM deployment '${deployment_name}' still has active operations after the cleanup timeout" + return 1 +} + +_cancel_active_deployment() { + local resource_group="$1" deployment_name="$2" subscription_id="$3" + local deadline="${4:-$((SECONDS + E2E_CLEANUP_TIMEOUT))}" + local deployment_state + + [[ -n "${deployment_name}" ]] || return 0 + if ! deployment_state="$(_deployment_state \ + "${resource_group}" "${deployment_name}" "${subscription_id}")"; then + log_error "Failed to query ARM deployment '${deployment_name}'; refusing to race resource deletion" + return 1 + fi + + case "${deployment_state}" in + ""|Succeeded|Failed|Canceled|Cancelled) + [[ -z "${deployment_state}" ]] || _wait_for_deployment_operations \ + "${resource_group}" "${deployment_name}" "${subscription_id}" "${deadline}" + return + ;; + esac + + log_warn "ARM deployment '${deployment_name}' is ${deployment_state}; canceling it before resource deletion" + az deployment group cancel \ + --resource-group "${resource_group}" \ + --name "${deployment_name}" \ + --subscription "${subscription_id}" \ + --output none 2>/dev/null || true + + while (( SECONDS < deadline )); do + if ! deployment_state="$(_deployment_state \ + "${resource_group}" "${deployment_name}" "${subscription_id}")"; then + log_error "Failed to query ARM deployment '${deployment_name}' after requesting cancellation" + return 1 + fi + case "${deployment_state}" in + ""|Succeeded|Failed|Canceled|Cancelled) + log_info "ARM deployment is terminal: ${deployment_state:-not found}" + [[ -z "${deployment_state}" ]] || _wait_for_deployment_operations \ + "${resource_group}" "${deployment_name}" "${subscription_id}" "${deadline}" + return + ;; + esac + sleep "${E2E_CLEANUP_POLL_INTERVAL:-5}" + done + + log_error "ARM deployment '${deployment_name}' remained ${deployment_state} after ${E2E_CLEANUP_TIMEOUT}s" + return 1 +} + +_remaining_cleanup_timeout() { + local deadline="$1" + local remaining=$((deadline - SECONDS)) + (( remaining > 0 )) || return 1 + printf '%s\n' "${remaining}" +} + +_azure_query_can_attempt() { + local attempt="$1" deadline="${2:-0}" + + (( attempt < 5 && (deadline <= 0 || SECONDS < deadline) )) +} + +_sleep_before_azure_query_retry() { + local deadline="${1:-0}" + local delay="${E2E_CLEANUP_POLL_INTERVAL:-5}" remaining whole_seconds + + if [[ ! "${delay}" =~ ^[0-9]+([.][0-9]+)?$ ]]; then + log_error "Invalid E2E_CLEANUP_POLL_INTERVAL '${delay}'" + return 1 + fi + if (( deadline > 0 )); then + remaining=$((deadline - SECONDS)) + (( remaining > 0 )) || return 1 + whole_seconds="${delay%%.*}" + if (( whole_seconds >= remaining )); then + delay="${remaining}" + fi + fi + sleep "${delay}" +} + +_azure_error_summary() { + local message="$1" + message="${message//$'\r'/ }" + message="${message//$'\n'/ }" + printf '%.500s\n' "${message}" +} + +_azure_error_is_authorization_failure() { + local message="${1,,}" + + [[ "${message}" == *forbidden* || + "${message}" == *unauthorized* || + "${message}" == *authorizationfailed* || + "${message}" == *authorizationpermissionmismatch* || + "${message}" == *authenticationfailed* || + "${message}" == *invalidauthenticationtoken* || + "${message}" == *expiredauthenticationtoken* || + "${message}" == *"does not have authorization"* || + "${message}" == *"permission denied"* || + "${message}" == *"insufficient privileges"* ]] +} + +_azure_error_is_resource_group_not_found() { + [[ "$1" == *ResourceGroupNotFound* ]] +} + +_group_exists_with_retry() { + local group_name="$1" subscription_id="$2" deadline="${3:-0}" + local exists attempt=0 last_error="" error_file + + if ! error_file="$(mktemp "${E2E_WORK_DIR}/azure-group-query.XXXXXX")"; then + log_error "Failed to create temporary Azure query diagnostics" + return 1 + fi + + while _azure_query_can_attempt "${attempt}" "${deadline}"; do + attempt=$((attempt + 1)) + : > "${error_file}" + if exists="$(az group exists \ + --name "${group_name}" \ + --subscription "${subscription_id}" \ + --output tsv 2>"${error_file}")"; then + case "${exists}" in + true|false) + rm -f -- "${error_file}" + printf '%s\n' "${exists}" + return 0 + ;; + *) + last_error="Azure CLI returned an unexpected resource-group existence result: ${exists}" + ;; + esac + else + last_error="$(<"${error_file}")" + # ARM can report the resource as not found while `az group exists` still + # exits nonzero during asynchronous AKS-managed resource-group deletion. + if _azure_error_is_resource_group_not_found "${last_error}"; then + rm -f -- "${error_file}" + printf 'false\n' + return 0 + fi + fi + if _azure_error_is_authorization_failure "${last_error}"; then + break + fi + if ! _azure_query_can_attempt "${attempt}" "${deadline}" || \ + ! _sleep_before_azure_query_retry "${deadline}"; then + break + fi + done + + rm -f -- "${error_file}" + log_error "Failed to determine whether resource group '${group_name}' exists after ${attempt} attempts" >&2 + if [[ -n "${last_error}" ]]; then + log_error "Last Azure CLI error: $(_azure_error_summary "${last_error}")" >&2 + fi + return 1 +} + +_resource_inventory() { + local resource_group="$1" subscription_id="$2" run_id="${3:-}" deadline="${4:-0}" + local resource_json attempt=0 last_error="" error_file + + if ! error_file="$(mktemp "${E2E_WORK_DIR}/azure-resource-query.XXXXXX")"; then + log_error "Failed to create temporary Azure query diagnostics" + return 1 + fi + + while _azure_query_can_attempt "${attempt}" "${deadline}"; do + attempt=$((attempt + 1)) + : > "${error_file}" + if resource_json="$(az resource list \ + --resource-group "${resource_group}" \ + --subscription "${subscription_id}" \ + --output json 2>"${error_file}")"; then + if resource_json="$(jq -ce --arg run_id "${run_id}" ' + if type != "array" then error("resource list must be an array") + elif $run_id == "" then . + else [.[] | select((.tags // {})["github-run"] == $run_id)] + end + ' <<<"${resource_json}")"; then + rm -f -- "${error_file}" + printf '%s\n' "${resource_json}" + return 0 + fi + last_error="Azure CLI returned a malformed resource inventory" + else + last_error="$(<"${error_file}")" + fi + if _azure_error_is_authorization_failure "${last_error}"; then + break + fi + if ! _azure_query_can_attempt "${attempt}" "${deadline}" || \ + ! _sleep_before_azure_query_retry "${deadline}"; then + break + fi + done + + rm -f -- "${error_file}" + log_error "Failed to inventory resources in '${resource_group}' after ${attempt} attempts" >&2 + if [[ -n "${last_error}" ]]; then + log_error "Last Azure CLI error: $(_azure_error_summary "${last_error}")" >&2 + fi + return 1 +} + +_cleanup_resource_owner() { + local name_suffix="${1:-}" owner + if [[ -z "${name_suffix}" ]]; then + name_suffix="$(state_get name_suffix)" + fi + owner="$(state_get resource_owner)" + if [[ -n "${owner}" ]]; then + if [[ -z "${name_suffix}" || "${owner}" != "${name_suffix}" ]]; then + log_error "Refusing tagged cleanup with resource owner '${owner}' that does not match E2E name suffix '${name_suffix}'" + return 1 + fi + printf '%s\n' "${owner}" + return 0 + fi + + # State written before resource_owner was introduced used run_id as the + # github-run tag value. Keep those deployments recoverable. + state_get run_id "${GITHUB_RUN_ID:-}" +} + +_is_expected_legacy_tagged_resource() { + local resource_type="${1,,}" resource_name="$2" name_suffix="$3" + local role vm_prefix + + [[ -n "${resource_name}" && -n "${name_suffix}" ]] || return 1 + case "${resource_type}" in + microsoft.compute/disks) + for role in msi token offline kubeadm arc; do + vm_prefix="vm-e2e-${role}-${name_suffix}" + if [[ "${resource_name}" == "${vm_prefix}-osdisk" || \ + "${resource_name}" == "${vm_prefix}"_OsDisk_* ]]; then + return 0 + fi + done + ;; + microsoft.network/networkinterfaces) + for role in msi token offline kubeadm arc; do + [[ "${resource_name}" == "vm-e2e-${role}-${name_suffix}-nic" ]] && return 0 + done + ;; + microsoft.network/publicipaddresses) + for role in msi token offline kubeadm arc; do + [[ "${resource_name}" == "vm-e2e-${role}-${name_suffix}-pip" ]] && return 0 + done + ;; + microsoft.network/virtualnetworks) + [[ "${resource_name}" == "vnet-e2e-${name_suffix}" ]] && return 0 + ;; + microsoft.network/networksecuritygroups) + [[ "${resource_name}" == "nsg-e2e-${name_suffix}" ]] && return 0 + ;; + esac + return 1 +} + +_legacy_implicit_os_disk_ids_from_inventory() { + local resource_json="$1" + shift + local resource_rows resource_type resource_name id vm_name + + if ! resource_rows="$(jq -r ' + if type != "array" then error("resource list must be an array") + else .[] | [(.type // ""), (.name // ""), (.id // "")] | + if all(.[]; type == "string") then @tsv + else error("resource type, name, and ID must be strings") + end + end + ' <<<"${resource_json}")"; then + log_error "Azure returned an invalid resource inventory while locating legacy OS disks" + return 1 + fi + + while IFS=$'\t' read -r resource_type resource_name id; do + [[ "${resource_type,,}" == "microsoft.compute/disks" && -n "${id}" ]] || continue + for vm_name in "$@"; do + [[ -n "${vm_name}" ]] || continue + if [[ "${resource_name}" == "${vm_name}"_OsDisk_* ]]; then + printf '%s\n' "${id}" + break + fi + done + done <<<"${resource_rows}" +} + +_legacy_implicit_os_disk_ids() { + local resource_group="$1" subscription_id="$2" deadline="$3" resource_json + shift 3 + + (( $# > 0 )) || return 0 + if ! resource_json="$(_resource_inventory "${resource_group}" "${subscription_id}" "" "${deadline}")"; then + return 1 + fi + _legacy_implicit_os_disk_ids_from_inventory "${resource_json}" "$@" +} + +_delete_legacy_implicit_os_disks() { + local resource_group="$1" subscription_id="$2" deadline="$3" disk_ids disk_id + shift 3 + + if ! disk_ids="$(_legacy_implicit_os_disk_ids "${resource_group}" "${subscription_id}" "${deadline}" "$@")"; then + return 1 + fi + while IFS= read -r disk_id; do + [[ -n "${disk_id}" ]] || continue + log_info "Deleting residual legacy OS disk: ${disk_id##*/}" + az resource delete --ids "${disk_id}" --subscription "${subscription_id}" --output none 2>/dev/null || true + done <<<"${disk_ids}" +} + +_validate_expected_cleanup_name() { + local description="$1" actual="$2" expected="$3" + + [[ -n "${actual}" ]] || return 0 + if [[ "${actual}" != "${expected}" ]]; then + log_error "Refusing to delete unexpected ${description} '${actual}'" + log_error "Expected '${expected}' for this E2E deployment" + return 1 + fi +} + +_validate_cleanup_target_names() { + local name_suffix="$1" deployment_name="$2" cluster_name="$3" node_resource_group="$4" + local msi_vm_name="$5" token_vm_name="$6" offline_vm_name="$7" kubeadm_vm_name="$8" + local arc_vm_name="$9" arc_machine_name="${10}" vnet_name="${11}" nsg_name="${12}" + local resource_group="${13}" location="${14}" + + if [[ -z "${name_suffix}" ]]; then + if [[ -n "${deployment_name}${cluster_name}${node_resource_group}${msi_vm_name}${token_vm_name}${offline_vm_name}${kubeadm_vm_name}${arc_vm_name}${arc_machine_name}${vnet_name}${nsg_name}" ]]; then + log_error "Cannot validate persisted cleanup targets without an E2E name suffix" + return 1 + fi + return 0 + fi + + _validate_expected_cleanup_name "ARM deployment" "${deployment_name}" "e2e-${name_suffix}" || return 1 + _validate_expected_cleanup_name "AKS cluster" "${cluster_name}" "aks-e2e-${name_suffix}" || return 1 + _validate_expected_node_resource_group \ + "${node_resource_group}" "${name_suffix}" "${resource_group}" "${cluster_name}" "${location}" || return 1 + _validate_expected_cleanup_name "MSI VM" "${msi_vm_name}" "vm-e2e-msi-${name_suffix}" || return 1 + _validate_expected_cleanup_name "token VM" "${token_vm_name}" "vm-e2e-token-${name_suffix}" || return 1 + _validate_expected_cleanup_name "offline VM" "${offline_vm_name}" "vm-e2e-offline-${name_suffix}" || return 1 + _validate_expected_cleanup_name "kubeadm VM" "${kubeadm_vm_name}" "vm-e2e-kubeadm-${name_suffix}" || return 1 + _validate_expected_cleanup_name "Arc VM" "${arc_vm_name}" "vm-e2e-arc-${name_suffix}" || return 1 + _validate_expected_cleanup_name "Arc machine" "${arc_machine_name}" "vm-e2e-arc-${name_suffix}-connected" || return 1 + _validate_expected_cleanup_name "virtual network" "${vnet_name}" "vnet-e2e-${name_suffix}" || return 1 + _validate_expected_cleanup_name "network security group" "${nsg_name}" "nsg-e2e-${name_suffix}" || return 1 +} + +_validate_expected_node_resource_group() { + local node_resource_group="$1" name_suffix="$2" + local resource_group="${3:-}" cluster_name="${4:-}" location="${5:-}" + + [[ -n "${node_resource_group}" ]] || return 0 + if [[ -z "${name_suffix}" ]]; then + log_error "Cannot validate AKS node resource group '${node_resource_group}' without an E2E name suffix" + return 1 + fi + + local expected_node_resource_group="MC_aksflex-e2e-${name_suffix}" + if [[ "${node_resource_group}" == "${expected_node_resource_group}" ]]; then + return 0 + fi + + # E2E state written before the template assigned a deterministic node + # resource group uses AKS's documented default naming convention. + local legacy_node_resource_group="" + if [[ -n "${resource_group}" && -n "${cluster_name}" && -n "${location}" ]]; then + legacy_node_resource_group="MC_${resource_group}_${cluster_name}_${location}" + if [[ "${node_resource_group}" == "${legacy_node_resource_group}" ]]; then + return 0 + fi + fi + + log_error "Refusing to delete unexpected AKS node resource group '${node_resource_group}'" + if [[ -n "${legacy_node_resource_group}" ]]; then + log_error "Expected '${expected_node_resource_group}' or legacy '${legacy_node_resource_group}' for this E2E deployment" + else + log_error "Expected '${expected_node_resource_group}' for this E2E deployment" + fi + return 1 +} + +_delete_node_resource_group() { + local node_resource_group="$1" subscription_id="$2" deadline="$3" + local delete_error error_file exists wait_timeout + + [[ -n "${node_resource_group}" ]] || return 0 + if ! exists="$(_group_exists_with_retry "${node_resource_group}" "${subscription_id}" "${deadline}")"; then + log_error "Failed to determine whether AKS node resource group '${node_resource_group}' exists" + return 1 + fi + case "${exists}" in + false) + return 0 + ;; + true) + ;; + *) + log_error "Unexpected existence result for AKS node resource group '${node_resource_group}': ${exists}" + return 1 + ;; + esac + + if ! error_file="$(mktemp "${E2E_WORK_DIR}/azure-group-delete.XXXXXX")"; then + log_error "Failed to create temporary Azure deletion diagnostics" + return 1 + fi + if ! az group delete --name "${node_resource_group}" --subscription "${subscription_id}" \ + --yes --no-wait --output none 2>"${error_file}"; then + delete_error="$(<"${error_file}")" + rm -f -- "${error_file}" + if _azure_error_is_resource_group_not_found "${delete_error}"; then + return 0 + fi + log_error "Failed to delete orphaned AKS node resource group '${node_resource_group}'" + if [[ -n "${delete_error}" ]]; then + log_error "Azure CLI error: $(_azure_error_summary "${delete_error}")" + fi + return 1 + fi + rm -f -- "${error_file}" + if ! wait_timeout="$(_remaining_cleanup_timeout "${deadline}")"; then + log_error "Cleanup deadline reached before deleting AKS node resource group '${node_resource_group}'" + return 1 + fi + if ! az group wait --name "${node_resource_group}" --subscription "${subscription_id}" \ + --deleted --interval 10 --timeout "${wait_timeout}" 2>/dev/null; then + log_error "AKS node resource group still exists after cleanup timeout: ${node_resource_group}" + return 1 + fi +} + +_delete_tagged_resources() { + local resource_group="$1" run_id="$2" subscription_id="$3" legacy_name_suffix="${4:-}" deadline="${5:-0}" + local resource_json resource_type resource_name id + local -a resource_types=( + "Microsoft.Compute/disks" + "Microsoft.Network/networkInterfaces" + "Microsoft.Network/publicIPAddresses" + "Microsoft.Network/virtualNetworks" + "Microsoft.Network/networkSecurityGroups" + ) + + [[ -n "${run_id}" ]] || return 0 + if ! resource_json="$(_resource_inventory "${resource_group}" "${subscription_id}" "${run_id}" "${deadline}")"; then + log_error "Failed to list E2E resources tagged github-run=${run_id}" + return 1 + fi + + for resource_type in "${resource_types[@]}"; do + while IFS=$'\t' read -r resource_name id; do + [[ -n "${id}" ]] || continue + if [[ -n "${legacy_name_suffix}" ]] && \ + ! _is_expected_legacy_tagged_resource "${resource_type}" "${resource_name}" "${legacy_name_suffix}"; then + log_warn "Ignoring legacy-tagged ${resource_type} outside this E2E attempt: ${resource_name}" + continue + fi + log_info "Deleting residual ${resource_type}: ${id##*/}" + az resource delete --ids "${id}" --subscription "${subscription_id}" --output none 2>/dev/null || true + done < <(jq -r --arg resource_type "${resource_type}" \ + '.[] | select((.type | ascii_downcase) == ($resource_type | ascii_downcase)) | [.name, .id] | @tsv' \ + <<<"${resource_json}") + done +} + +_tagged_resource_ids() { + local resource_group="$1" run_id="$2" subscription_id="$3" legacy_name_suffix="${4:-}" deadline="${5:-0}" + local resource_json resource_type resource_name id + + [[ -n "${run_id}" ]] || return 0 + resource_json="$(_resource_inventory "${resource_group}" "${subscription_id}" "${run_id}" "${deadline}")" || return 1 + while IFS=$'\t' read -r resource_type resource_name id; do + [[ -n "${id}" ]] || continue + if [[ -n "${legacy_name_suffix}" ]] && \ + ! _is_expected_legacy_tagged_resource "${resource_type}" "${resource_name}" "${legacy_name_suffix}"; then + continue + fi + printf '%s\n' "${id}" + done < <(jq -r '.[] | [.type, .name, .id] | @tsv' <<<"${resource_json}") +} + cleanup() { log_section "Cleaning Up Resources" @@ -189,6 +718,8 @@ cleanup() { fi local resource_group cluster_name msi_vm_name token_vm_name offline_vm_name kubeadm_vm_name arc_vm_name arc_machine_name arc_machine_id + local subscription_id deployment_name resource_owner persisted_resource_owner legacy_tag_name_suffix + local cleanup_failed vnet_name nsg_name name_suffix node_resource_group location cleanup_deadline resource_group="$(state_get resource_group)" cluster_name="$(state_get cluster_name)" msi_vm_name="$(state_get msi_vm_name)" @@ -197,16 +728,213 @@ cleanup() { kubeadm_vm_name="$(state_get kubeadm_vm_name)" arc_vm_name="$(state_get arc_vm_name)" arc_machine_name="$(state_get arc_machine_name)" + subscription_id="$(state_get subscription_id "${AZURE_SUBSCRIPTION_ID:-}")" arc_machine_id="$(state_get arc_machine_id)" - if [[ -z "${arc_machine_id}" && -n "${arc_machine_name}" ]]; then - arc_machine_id="/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${resource_group}/providers/Microsoft.HybridCompute/machines/${arc_machine_name}" - fi - local deployment_name deployment_name="$(state_get deployment_name)" + persisted_resource_owner="$(state_get resource_owner)" + name_suffix="$(state_get name_suffix)" + vnet_name="$(state_get vnet_name)" + nsg_name="$(state_get nsg_name)" + node_resource_group="$(state_get node_resource_group)" + location="$(state_get location)" + if [[ -z "${name_suffix}" && "${cluster_name}" == aks-e2e-* ]]; then + name_suffix="${cluster_name#aks-e2e-}" + fi + if [[ -z "${deployment_name}" && -n "${name_suffix}" ]]; then + deployment_name="e2e-${name_suffix}" + fi + if [[ -z "${vnet_name}" && -n "${name_suffix}" ]]; then + vnet_name="vnet-e2e-${name_suffix}" + fi + if [[ -z "${nsg_name}" && -n "${name_suffix}" ]]; then + nsg_name="nsg-e2e-${name_suffix}" + fi + if [[ -z "${node_resource_group}" && -z "${persisted_resource_owner}" && \ + -n "${resource_group}" && -n "${cluster_name}" && -n "${location}" ]]; then + node_resource_group="MC_${resource_group}_${cluster_name}_${location}" + fi + if ! resource_owner="$(_cleanup_resource_owner "${name_suffix}")"; then + return 1 + fi + legacy_tag_name_suffix="" + if [[ -z "${persisted_resource_owner}" && -n "${resource_owner}" ]]; then + if [[ -z "${name_suffix}" ]]; then + log_error "Cannot safely clean resources with a legacy run tag without an E2E name suffix" + return 1 + fi + legacy_tag_name_suffix="${name_suffix}" + fi + cleanup_failed=0 + cleanup_deadline=$((SECONDS + E2E_CLEANUP_TIMEOUT)) if [[ -z "${resource_group}" ]]; then - log_warn "No resource group in state; nothing to clean up" - return 0 + if [[ ! -f "${E2E_STATE_FILE}" ]] || jq -e 'length == 0' "${E2E_STATE_FILE}" >/dev/null 2>&1; then + log_warn "No resource group in state; nothing to clean up" + return 0 + fi + log_error "State is nonempty but has no resource_group; refusing to declare cleanup complete" + return 1 + fi + + local resource_group_exists + if ! resource_group_exists="$(_group_exists_with_retry \ + "${resource_group}" "${subscription_id}" "${cleanup_deadline}")"; then + log_error "Failed to determine whether resource group '${resource_group}' exists" + return 1 + fi + + if ! _validate_cleanup_target_names \ + "${name_suffix}" "${deployment_name}" "${cluster_name}" "${node_resource_group}" \ + "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" "${kubeadm_vm_name}" \ + "${arc_vm_name}" "${arc_machine_name}" "${vnet_name}" "${nsg_name}" \ + "${resource_group}" "${location}"; then + return 1 + fi + + if [[ -n "${arc_machine_name}" ]]; then + local expected_arc_machine_id="/subscriptions/${subscription_id}/resourceGroups/${resource_group}/providers/Microsoft.HybridCompute/machines/${arc_machine_name}" + if [[ -n "${arc_machine_id}" && "${arc_machine_id,,}" != "${expected_arc_machine_id,,}" ]]; then + log_error "Refusing to delete Arc machine with an unexpected persisted resource ID: ${arc_machine_id}" + return 1 + fi + # The ID is completely determined by other state fields. Reconstructing it + # avoids trusting a redundant deletion target from stale or damaged state. + arc_machine_id="${expected_arc_machine_id}" + elif [[ -n "${arc_machine_id}" ]]; then + log_error "Cannot validate persisted Arc machine resource ID without arc_machine_name" + return 1 + fi + + case "${resource_group_exists}" in + false) + # The parent group's absence proves the cluster is gone, so this is a + # safe recovery path for an exact-name-validated orphan node group. + if ! _validate_expected_node_resource_group \ + "${node_resource_group}" "${name_suffix}" "${resource_group}" "${cluster_name}" "${location}"; then + return 1 + fi + if ! _delete_node_resource_group "${node_resource_group}" "${subscription_id}" "${cleanup_deadline}"; then + return 1 + fi + state_set "lifecycle" "cleaned" || return 1 + state_set "cleanup_complete" "true" || return 1 + log_success "Resource group is absent; cleanup is complete" + return 0 + ;; + true) + ;; + *) + log_error "Unexpected existence result for resource group '${resource_group}': ${resource_group_exists}" + return 1 + ;; + esac + + state_set "cleanup_complete" "false" || return 1 + state_set "lifecycle" "cleaning" || return 1 + + if ! _cancel_active_deployment \ + "${resource_group}" "${deployment_name}" "${subscription_id}" "${cleanup_deadline}"; then + # Deleting while ARM is still provisioning races new resource creation and + # can report false success, so preserve state for a later cleanup attempt. + state_set "cleanup_complete" "false" || true + return 1 + fi + + # Snapshot IDs that are difficult to recover after deleting their parents. + # This supports cleanup of deployments created before deterministic OS-disk + # names and delete options were added to the Bicep module. + local vm_inventory aks_inventory resource_inventory live_node_resource_group live_cluster_location + local vm_name disk_id nic_id nic_output legacy_disk_output + local -a managed_disk_ids=() legacy_implicit_os_disk_ids=() nic_ids=() + if ! vm_inventory="$(az vm list \ + --resource-group "${resource_group}" \ + --subscription "${subscription_id}" \ + --output json 2>/dev/null)"; then + log_error "Failed to inventory E2E VMs before cleanup" + return 1 + fi + if ! aks_inventory="$(az aks list \ + --resource-group "${resource_group}" \ + --subscription "${subscription_id}" \ + --output json 2>/dev/null)"; then + log_error "Failed to inventory E2E AKS clusters before cleanup" + return 1 + fi + if ! jq -e 'type == "array"' <<<"${vm_inventory}" >/dev/null || \ + ! jq -e 'type == "array"' <<<"${aks_inventory}" >/dev/null; then + log_error "Azure returned an invalid VM or AKS cleanup inventory" + return 1 + fi + if ! resource_inventory="$(_resource_inventory \ + "${resource_group}" "${subscription_id}" "" "${cleanup_deadline}")"; then + log_error "Failed to inventory E2E resources before cleanup" + return 1 + fi + if ! legacy_disk_output="$(_legacy_implicit_os_disk_ids_from_inventory \ + "${resource_inventory}" \ + "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" \ + "${kubeadm_vm_name}" "${arc_vm_name}")"; then + return 1 + fi + while IFS= read -r disk_id; do + [[ -z "${disk_id}" ]] || legacy_implicit_os_disk_ids+=("${disk_id}") + done <<<"${legacy_disk_output}" + for vm_name in "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" "${kubeadm_vm_name}" "${arc_vm_name}"; do + [[ -n "${vm_name}" ]] || continue + if ! disk_id="$(jq -r --arg name "${vm_name}" \ + '.[] | select(.name == $name) | .storageProfile.osDisk.managedDisk.id // empty' \ + <<<"${vm_inventory}")"; then + log_error "Failed to inspect OS disk for VM '${vm_name}'" + return 1 + fi + [[ -z "${disk_id}" ]] || managed_disk_ids+=("${disk_id}") + if ! nic_output="$(jq -r --arg name "${vm_name}" \ + '.[] | select(.name == $name) | .networkProfile.networkInterfaces[]?.id // empty' \ + <<<"${vm_inventory}")"; then + log_error "Failed to inspect network interfaces for VM '${vm_name}'" + return 1 + fi + while read -r nic_id; do + [[ -z "${nic_id}" ]] || nic_ids+=("${nic_id}") + done <<<"${nic_output}" + done + if ! live_node_resource_group="$(jq -r --arg name "${cluster_name}" \ + '.[] | select(.name == $name) | .nodeResourceGroup // empty' \ + <<<"${aks_inventory}")"; then + log_error "Failed to inspect the AKS node resource group" + return 1 + fi + if ! live_cluster_location="$(jq -r --arg name "${cluster_name}" \ + '.[] | select(.name == $name) | .location // empty' \ + <<<"${aks_inventory}")"; then + log_error "Failed to inspect the AKS cluster location" + return 1 + fi + if [[ -n "${live_cluster_location}" ]]; then + if [[ -n "${location}" && "${location,,}" != "${live_cluster_location,,}" ]]; then + log_error "Refusing cleanup because live AKS location '${live_cluster_location}' differs from state '${location}'" + return 1 + fi + location="${live_cluster_location}" + state_set "location" "${location}" || return 1 + fi + if [[ -n "${live_node_resource_group}" ]]; then + if ! _validate_expected_node_resource_group \ + "${live_node_resource_group}" "${name_suffix}" "${resource_group}" "${cluster_name}" "${location}"; then + return 1 + fi + if [[ -n "${node_resource_group}" && "${node_resource_group}" != "${live_node_resource_group}" ]]; then + log_warn "Live AKS node resource group differs from state; using the live cluster value '${live_node_resource_group}'" + fi + node_resource_group="${live_node_resource_group}" + state_set "node_resource_group" "${node_resource_group}" || return 1 + elif ! _validate_expected_node_resource_group \ + "${node_resource_group}" "${name_suffix}" "${resource_group}" "${cluster_name}" "${location}"; then + return 1 + fi + if [[ -n "${node_resource_group}" && "${node_resource_group}" == "${resource_group}" ]]; then + log_error "Refusing to delete parent resource group as an AKS node resource group" + return 1 fi # Arc lifecycle is external to Flex Node. Remove the E2E-owned Arc resource @@ -216,50 +944,209 @@ cleanup() { az rest --method delete --url "https://management.azure.com${arc_machine_id}?api-version=2024-07-10" --output none 2>/dev/null || true fi - # Delete VMs first (faster than waiting for full RG delete) - log_info "[2/8] Deleting MSI VM: ${msi_vm_name}..." - az vm delete --resource-group "${resource_group}" --name "${msi_vm_name}" \ - --force-deletion yes --yes --no-wait 2>/dev/null || true - - log_info "[3/8] Deleting Token VM: ${token_vm_name}..." - az vm delete --resource-group "${resource_group}" --name "${token_vm_name}" \ - --force-deletion yes --yes --no-wait 2>/dev/null || true - - log_info "[4/8] Deleting Offline VM: ${offline_vm_name}..." - az vm delete --resource-group "${resource_group}" --name "${offline_vm_name}" \ - --force-deletion yes --yes --no-wait 2>/dev/null || true - - log_info "[5/8] Deleting Kubeadm VM: ${kubeadm_vm_name}..." - az vm delete --resource-group "${resource_group}" --name "${kubeadm_vm_name}" \ - --force-deletion yes --yes --no-wait 2>/dev/null || true - - log_info "[6/8] Deleting Arc VM: ${arc_vm_name}..." - az vm delete --resource-group "${resource_group}" --name "${arc_vm_name}" \ - --force-deletion yes --yes --no-wait 2>/dev/null || true - - # Clean up leftover networking resources tied to our deployment - log_info "[7/8] Cleaning up networking resources..." - local run_id="${GITHUB_RUN_ID:-}" - if [[ -n "${run_id}" ]]; then - local resource_ids - for res_type in networkInterfaces publicIPAddresses networkSecurityGroups disks; do - resource_ids="$(az resource list --resource-group "${resource_group}" \ - --query "[?tags.\"github-run\"=='${run_id}' && contains(type, '${res_type}')].id" \ - -o tsv 2>/dev/null || true)" - while read -r id; do - [[ -n "${id}" ]] || continue - az resource delete --ids "${id}" --no-wait 2>/dev/null || true - done <<<"${resource_ids}" + # Start independent VM and AKS deletes together, then wait for all of them + # before removing their network dependencies. + for vm_name in "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" "${kubeadm_vm_name}" "${arc_vm_name}"; do + [[ -n "${vm_name}" ]] || continue + log_info "Deleting VM: ${vm_name}..." + az vm delete --resource-group "${resource_group}" --name "${vm_name}" \ + --subscription "${subscription_id}" --force-deletion yes --yes --no-wait 2>/dev/null || true + done + + if [[ -n "${cluster_name}" ]]; then + log_info "Deleting AKS cluster: ${cluster_name}..." + az aks delete --resource-group "${resource_group}" --name "${cluster_name}" \ + --subscription "${subscription_id}" --yes --no-wait 2>/dev/null || true + fi + + for vm_name in "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" "${kubeadm_vm_name}" "${arc_vm_name}"; do + [[ -n "${vm_name}" ]] || continue + local wait_timeout + if ! wait_timeout="$(_remaining_cleanup_timeout "${cleanup_deadline}")"; then + log_error "Cleanup deadline reached while waiting for VMs" + cleanup_failed=1 + break + fi + if ! az vm wait --resource-group "${resource_group}" --name "${vm_name}" \ + --subscription "${subscription_id}" --deleted --interval 10 --timeout "${wait_timeout}" 2>/dev/null; then + if az vm show --resource-group "${resource_group}" --name "${vm_name}" \ + --subscription "${subscription_id}" --output none 2>/dev/null; then + log_error "VM still exists after cleanup timeout: ${vm_name}" + cleanup_failed=1 + fi + fi + az disk delete --resource-group "${resource_group}" --name "${vm_name}-osdisk" \ + --subscription "${subscription_id}" --yes --output none 2>/dev/null || true + az network nic delete --resource-group "${resource_group}" --name "${vm_name}-nic" \ + --subscription "${subscription_id}" --output none 2>/dev/null || true + az network public-ip delete --resource-group "${resource_group}" --name "${vm_name}-pip" \ + --subscription "${subscription_id}" --output none 2>/dev/null || true + done + + for disk_id in "${managed_disk_ids[@]}" "${legacy_implicit_os_disk_ids[@]}"; do + az resource delete --ids "${disk_id}" --subscription "${subscription_id}" --output none 2>/dev/null || true + done + for nic_id in "${nic_ids[@]}"; do + az resource delete --ids "${nic_id}" --subscription "${subscription_id}" --output none 2>/dev/null || true + done + + if [[ -n "${cluster_name}" ]]; then + local aks_wait_timeout + if ! aks_wait_timeout="$(_remaining_cleanup_timeout "${cleanup_deadline}")"; then + log_error "Cleanup deadline reached before AKS deletion completed" + cleanup_failed=1 + elif ! az aks wait \ + --resource-group "${resource_group}" --name "${cluster_name}" \ + --subscription "${subscription_id}" --deleted --interval 10 \ + --timeout "${aks_wait_timeout}" 2>/dev/null; then + cleanup_failed=1 + if az aks show --resource-group "${resource_group}" --name "${cluster_name}" \ + --subscription "${subscription_id}" --output none 2>/dev/null; then + log_error "AKS cluster still exists after cleanup timeout: ${cluster_name}" + else + log_error "Failed to confirm AKS cluster deletion after the wait command failed: ${cluster_name}" + fi + fi + fi + + # AKS owns the managed node resource group's lifecycle and deletes it with + # the cluster. Avoid querying or deleting that sibling group directly: the + # least-privilege cleanup identity may only have access to the parent group. + # https://learn.microsoft.com/azure/aks/faq#can-i-restore-my-cluster-after-i-delete-it- + + if [[ -n "${arc_machine_id}" ]]; then + local arc_wait_timeout + if ! arc_wait_timeout="$(_remaining_cleanup_timeout "${cleanup_deadline}")"; then + log_error "Cleanup deadline reached before Arc machine deletion completed" + cleanup_failed=1 + elif ! az resource wait --ids "${arc_machine_id}" --subscription "${subscription_id}" \ + --api-version 2024-07-10 --deleted --interval 10 --timeout "${arc_wait_timeout}" \ + --output none 2>/dev/null; then + log_error "Arc machine still exists after cleanup timeout: ${arc_machine_id}" + cleanup_failed=1 + fi + fi + + [[ -z "${vnet_name}" ]] || az network vnet delete \ + --resource-group "${resource_group}" --name "${vnet_name}" \ + --subscription "${subscription_id}" --output none 2>/dev/null || true + [[ -z "${nsg_name}" ]] || az network nsg delete \ + --resource-group "${resource_group}" --name "${nsg_name}" \ + --subscription "${subscription_id}" --output none 2>/dev/null || true + + log_info "[8/8] Cleaning up tagged network and disk resources..." + local remaining_ids legacy_remaining_ids empty_inventories=0 + for _ in 1 2 3 4; do + if ! _delete_tagged_resources "${resource_group}" "${resource_owner}" "${subscription_id}" "${legacy_tag_name_suffix}" "${cleanup_deadline}"; then + cleanup_failed=1 + break + fi + # Old VM deployments let Azure choose names such as + # _OsDisk_1_. Those disks can be detached, untagged, and absent + # from `az vm list`, so rediscover and retry them from the full RG inventory. + for disk_id in "${legacy_implicit_os_disk_ids[@]}"; do + az resource delete --ids "${disk_id}" --subscription "${subscription_id}" --output none 2>/dev/null || true done + if ! _delete_legacy_implicit_os_disks \ + "${resource_group}" "${subscription_id}" "${cleanup_deadline}" \ + "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" \ + "${kubeadm_vm_name}" "${arc_vm_name}"; then + log_error "Failed to delete legacy implicit OS disks" + cleanup_failed=1 + break + fi + if ! remaining_ids="$(_tagged_resource_ids "${resource_group}" "${resource_owner}" "${subscription_id}" "${legacy_tag_name_suffix}" "${cleanup_deadline}")"; then + log_error "Failed to verify tagged-resource deletion" + cleanup_failed=1 + break + fi + if ! legacy_remaining_ids="$(_legacy_implicit_os_disk_ids \ + "${resource_group}" "${subscription_id}" "${cleanup_deadline}" \ + "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" \ + "${kubeadm_vm_name}" "${arc_vm_name}")"; then + log_error "Failed to verify legacy implicit OS-disk deletion" + cleanup_failed=1 + break + fi + if [[ -z "${remaining_ids}" && -z "${legacy_remaining_ids}" ]]; then + empty_inventories=$((empty_inventories + 1)) + if (( empty_inventories >= 2 )); then + break + fi + else + empty_inventories=0 + fi + sleep "${E2E_CLEANUP_POLL_INTERVAL:-5}" + done + + if (( empty_inventories < 2 )); then + log_error "Residual-resource inventory did not remain empty for two consecutive checks" + cleanup_failed=1 fi - log_info "[8/8] Deleting AKS cluster: ${cluster_name}..." - az aks delete --resource-group "${resource_group}" --name "${cluster_name}" \ - --yes --no-wait 2>/dev/null || true + if ! remaining_ids="$(_tagged_resource_ids "${resource_group}" "${resource_owner}" "${subscription_id}" "${legacy_tag_name_suffix}" "${cleanup_deadline}")"; then + log_error "Failed final tagged-resource verification" + cleanup_failed=1 + elif [[ -n "${remaining_ids}" ]]; then + log_error "Tagged E2E resources remain after cleanup:" + printf '%s\n' "${remaining_ids}" >&2 + cleanup_failed=1 + fi - # If we created the VNet/NSG via Bicep, they'll be cleaned up when no - # resources reference them, or on next deployment. We don't delete the - # resource group itself since it may be shared. + local final_inventory expected_names known_remaining captured_id resource_name + local -a expected_name_args=() + if ! final_inventory="$(_resource_inventory "${resource_group}" "${subscription_id}" "" "${cleanup_deadline}")"; then + log_error "Failed final exact-name resource verification" + cleanup_failed=1 + final_inventory='[]' + fi + for resource_name in "${cluster_name}" "${arc_machine_name}" "${vnet_name}" "${nsg_name}"; do + [[ -z "${resource_name}" ]] || expected_name_args+=("${resource_name}") + done + for vm_name in "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" "${kubeadm_vm_name}" "${arc_vm_name}"; do + [[ -n "${vm_name}" ]] || continue + expected_name_args+=("${vm_name}" "${vm_name}-nic" "${vm_name}-pip" "${vm_name}-osdisk") + done + if ! expected_names="$(jq -cn --args '$ARGS.positional' -- "${expected_name_args[@]}")"; then + log_error "Failed to build exact-name cleanup inventory" + cleanup_failed=1 + expected_names='[]' + fi + if ! known_remaining="$(jq -er --argjson names "${expected_names}" ' + if type != "array" then error("resource list must be an array") + else [.[] | select(.name as $name | $names | index($name)) | .id] | join("\n") + end + ' <<<"${final_inventory}")"; then + log_error "Azure returned invalid final resource inventory" + cleanup_failed=1 + known_remaining="" + fi + if ! legacy_remaining_ids="$(_legacy_implicit_os_disk_ids_from_inventory \ + "${final_inventory}" \ + "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" \ + "${kubeadm_vm_name}" "${arc_vm_name}")"; then + cleanup_failed=1 + legacy_remaining_ids="" + elif [[ -n "${legacy_remaining_ids}" ]]; then + known_remaining+=$'\n'"${legacy_remaining_ids}" + fi + for captured_id in "${managed_disk_ids[@]}" "${legacy_implicit_os_disk_ids[@]}" "${nic_ids[@]}" "${arc_machine_id}"; do + [[ -n "${captured_id}" ]] || continue + if jq -e --arg id "${captured_id}" '.[] | select(.id == $id)' <<<"${final_inventory}" >/dev/null; then + known_remaining+=$'\n'"${captured_id}" + fi + done + if [[ -n "${known_remaining}" ]]; then + log_error "Known E2E resources remain after cleanup:" + printf '%s\n' "${known_remaining}" >&2 + cleanup_failed=1 + fi + + if [[ "${cleanup_failed}" != "0" ]]; then + return 1 + fi - log_success "Cleanup initiated (async deletes in progress)" + state_set "lifecycle" "cleaned" || return 1 + state_set "cleanup_complete" "true" || return 1 + log_success "Cleanup completed and no known E2E resources remain" } diff --git a/hack/e2e/lib/common.sh b/hack/e2e/lib/common.sh index 60b18c35..ecd4098c 100755 --- a/hack/e2e/lib/common.sh +++ b/hack/e2e/lib/common.sh @@ -120,7 +120,7 @@ check_prerequisites() { log_info "Checking prerequisites..." local missing=0 - for cmd in az docker git go jq kubectl make python3 ssh scp openssl; do + for cmd in az docker flock git go jq kubectl make python3 ssh scp openssl; do if ! command -v "${cmd}" &>/dev/null; then log_error "Missing required tool: ${cmd}" missing=1 @@ -193,13 +193,28 @@ load_config() { E2E_SSH_OPTS="${E2E_SSH_OPTS:--o StrictHostKeyChecking=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR}" # Component versions (match the workflow defaults) + if [[ -n "${E2E_KUBERNETES_VERSION:-}" ]]; then + _E2E_KUBERNETES_VERSION_EXPLICIT=1 + else + _E2E_KUBERNETES_VERSION_EXPLICIT=0 + fi E2E_KUBERNETES_VERSION="${E2E_KUBERNETES_VERSION:-1.35.0}" E2E_CONTAINERD_VERSION="${E2E_CONTAINERD_VERSION:-2.0.4}" E2E_RUNC_VERSION="${E2E_RUNC_VERSION:-1.1.12}" + if [[ -n "${E2E_TARGET_AGENT_POOL_NAME:-}" ]]; then + _E2E_TARGET_AGENT_POOL_NAME_EXPLICIT=1 + else + _E2E_TARGET_AGENT_POOL_NAME_EXPLICIT=0 + fi E2E_TARGET_AGENT_POOL_NAME="${E2E_TARGET_AGENT_POOL_NAME:-aksflexnodes}" # listBootstrapData requires an existing ARM agent pool. The in-cluster # controller still uses E2E_TARGET_AGENT_POOL_NAME for its synthetic machine # contract, while the MSI repave scenario uses this real pool for join data. + if [[ -n "${E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME:-}" ]]; then + _E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME_EXPLICIT=1 + else + _E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME_EXPLICIT=0 + fi E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME="${E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME:-${E2E_TARGET_AGENT_POOL_NAME}}" # Kubelet resource reservation overrides applied to the token node config. @@ -219,6 +234,8 @@ load_config() { E2E_NODE_JOIN_TIMEOUT="${E2E_NODE_JOIN_TIMEOUT:-300}" E2E_POD_READY_TIMEOUT="${E2E_POD_READY_TIMEOUT:-120}" E2E_BOOTSTRAP_SETTLE_TIME="${E2E_BOOTSTRAP_SETTLE_TIME:-60}" + E2E_CLEANUP_TIMEOUT="${E2E_CLEANUP_TIMEOUT:-900}" + E2E_CLEANUP_POLL_INTERVAL="${E2E_CLEANUP_POLL_INTERVAL:-5}" log_info "Configuration loaded:" log_info " Resource Group: ${E2E_RESOURCE_GROUP}" @@ -279,17 +296,98 @@ init_work_dir() { log_debug "Work directory: ${E2E_WORK_DIR}" } +# Atomically reserve a work directory and write its complete deployment +# manifest. Any nonempty state must first reach verified cleanup; otherwise a +# new run could erase the only record capable of removing the old resources. +state_begin_deployment() { + local state_json="$1" + + ( + if ! flock -x 9; then + log_error "Failed to lock E2E deployment state" + return 1 + fi + local tmp existing_state requested_deployment existing_deployment + if ! requested_deployment="$(jq -er ' + if type != "object" then error("state must be an object") + else .deployment_name | select(type == "string" and length > 0) + end + ' <<<"${state_json}")"; then + log_error "New E2E deployment state is invalid or has no deployment_name" + return 1 + fi + + if [[ -f "${E2E_STATE_FILE}" ]]; then + if ! existing_state="$(jq -ec ' + if type == "object" then . else error("state must be an object") end + ' "${E2E_STATE_FILE}")"; then + log_error "Existing E2E state is invalid; refusing to overwrite cleanup metadata" + return 1 + fi + if ! jq -e ' + length == 0 or + (.lifecycle == "cleaned" and + (.cleanup_complete == "true" or .cleanup_complete == true)) + ' <<<"${existing_state}" >/dev/null; then + existing_deployment="$(jq -r '.deployment_name // empty' <<<"${existing_state}")" + log_error "E2E state already tracks deployment '${existing_deployment:-unknown legacy deployment}'" + log_error "Run cleanup with this work directory before deploying '${requested_deployment}', or choose a new E2E_WORK_DIR" + return 1 + fi + fi + + if ! tmp="$(mktemp "${E2E_STATE_FILE}.tmp.XXXXXX")"; then + log_error "Failed to create temporary E2E state file" + return 1 + fi + trap 'rm -f "${tmp}"' EXIT + if ! jq -e 'if type == "object" then . else error("state must be an object") end' \ + <<<"${state_json}" > "${tmp}"; then + log_error "Failed to render initial E2E deployment state" + return 1 + fi + if ! chmod 0600 "${tmp}" || ! mv "${tmp}" "${E2E_STATE_FILE}"; then + log_error "Failed to install initial E2E deployment state" + return 1 + fi + trap - EXIT + ) 9> "${E2E_STATE_FILE}.lock" +} + # Write a key=value into the state file (JSON) state_set() { local key="$1" value="$2" - local tmp="${E2E_STATE_FILE}.tmp" - if [[ -f "${E2E_STATE_FILE}" ]]; then - jq --arg k "${key}" --arg v "${value}" '. + {($k): $v}' "${E2E_STATE_FILE}" > "${tmp}" - else - jq -n --arg k "${key}" --arg v "${value}" '{($k): $v}' > "${tmp}" - fi - mv "${tmp}" "${E2E_STATE_FILE}" + ( + if ! flock -x 9; then + log_error "Failed to lock E2E deployment state" + return 1 + fi + local tmp + if ! tmp="$(mktemp "${E2E_STATE_FILE}.tmp.XXXXXX")"; then + log_error "Failed to create temporary E2E state file" + return 1 + fi + trap 'rm -f "${tmp}"' EXIT + + if [[ -f "${E2E_STATE_FILE}" ]]; then + if ! jq --arg k "${key}" --arg v "${value}" '. + {($k): $v}' \ + "${E2E_STATE_FILE}" > "${tmp}"; then + log_error "Failed to update E2E state key '${key}'" + return 1 + fi + else + if ! jq -n --arg k "${key}" --arg v "${value}" '{($k): $v}' > "${tmp}"; then + log_error "Failed to initialize E2E state key '${key}'" + return 1 + fi + fi + if ! chmod 0600 "${tmp}" || ! mv "${tmp}" "${E2E_STATE_FILE}"; then + log_error "Failed to install updated E2E state" + return 1 + fi + trap - EXIT + ) 9> "${E2E_STATE_FILE}.lock" } # Read a value from the state file @@ -305,11 +403,114 @@ state_get() { fi } +require_exact_kubernetes_version() { + if [[ ! "${E2E_KUBERNETES_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + log_error "E2E_KUBERNETES_VERSION must be an exact x.y.z patch version, got '${E2E_KUBERNETES_VERSION}'" + return 1 + fi +} + +restore_kubernetes_version_from_state() { + local persisted_version resource_group cluster_name subscription_id live_version live_cluster cluster_id + local persisted_target_pool persisted_bootstrap_pool system_pool live_system_version live_flex_version + persisted_version="$(state_get kubernetes_version)" + resource_group="$(state_get resource_group)" + cluster_name="$(state_get cluster_name)" + subscription_id="$(state_get subscription_id "${AZURE_SUBSCRIPTION_ID:-}")" + persisted_target_pool="$(state_get target_agent_pool_name "${E2E_TARGET_AGENT_POOL_NAME:-aksflexnodes}")" + persisted_bootstrap_pool="$(state_get bootstrap_data_agent_pool_name "${E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME:-${persisted_target_pool}}")" + system_pool="$(state_get system_pool_name system)" + + if [[ -z "${resource_group}" || -z "${cluster_name}" || -z "${subscription_id}" ]]; then + log_error "Deployment state does not identify an AKS cluster and subscription; run the infra command first" + return 1 + fi + + if [[ "${_E2E_TARGET_AGENT_POOL_NAME_EXPLICIT:-0}" == "1" && \ + "${E2E_TARGET_AGENT_POOL_NAME:-}" != "${persisted_target_pool}" ]]; then + log_error "E2E_TARGET_AGENT_POOL_NAME is ${E2E_TARGET_AGENT_POOL_NAME:-}, but deployment state records ${persisted_target_pool}" + return 1 + fi + if [[ "${_E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME_EXPLICIT:-0}" == "1" && \ + "${E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME:-}" != "${persisted_bootstrap_pool}" ]]; then + log_error "E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME is ${E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME:-}, but deployment state records ${persisted_bootstrap_pool}" + return 1 + fi + E2E_TARGET_AGENT_POOL_NAME="${persisted_target_pool}" + E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME="${persisted_bootstrap_pool}" + + if ! live_cluster="$(az aks show \ + --subscription "${subscription_id}" \ + --resource-group "${resource_group}" \ + --name "${cluster_name}" \ + --query '{id:id, version:currentKubernetesVersion || kubernetesVersion}' \ + --output json 2>/dev/null)" || \ + ! live_version="$(jq -er '.version | select(type == "string" and length > 0)' <<<"${live_cluster}")" || \ + ! cluster_id="$(jq -er '.id | select(type == "string" and length > 0)' <<<"${live_cluster}")"; then + log_error "Cannot determine the live Kubernetes version for ${resource_group}/${cluster_name}" + return 1 + fi + if ! live_system_version="$(az rest \ + --method get \ + --url "https://management.azure.com${cluster_id}/agentPools/${system_pool}?api-version=2026-05-02-preview" \ + --query 'properties.currentOrchestratorVersion || properties.orchestratorVersion' \ + --output tsv 2>/dev/null)" || [[ -z "${live_system_version}" ]]; then + log_error "Cannot determine the live Kubernetes version for system pool ${system_pool}" + return 1 + fi + if ! live_flex_version="$(az rest \ + --method get \ + --url "https://management.azure.com${cluster_id}/agentPools/${persisted_bootstrap_pool}?api-version=2026-05-02-preview" \ + --query 'properties.currentOrchestratorVersion || properties.orchestratorVersion' \ + --output tsv 2>/dev/null)" || [[ -z "${live_flex_version}" ]]; then + log_error "Cannot determine the live Kubernetes version for FlexNodes pool ${persisted_bootstrap_pool}" + return 1 + fi + if [[ "${live_system_version}" != "${live_version}" || "${live_flex_version}" != "${live_version}" ]]; then + log_error "AKS version skew: control plane=${live_version}, system pool=${live_system_version}, FlexNodes pool=${live_flex_version}" + return 1 + fi + if [[ -n "${persisted_version}" && "${persisted_version}" != "${live_version}" ]]; then + log_error "Deployment state records Kubernetes ${persisted_version}, but the live cluster is ${live_version}" + return 1 + fi + if [[ -z "${persisted_version}" ]]; then + persisted_version="${live_version}" + state_set kubernetes_version "${persisted_version}" + log_info "Recovered Kubernetes version from the live cluster: ${persisted_version}" + fi + if [[ "${_E2E_KUBERNETES_VERSION_EXPLICIT}" == "1" && \ + "${E2E_KUBERNETES_VERSION}" != "${persisted_version}" ]]; then + log_error "E2E_KUBERNETES_VERSION is ${E2E_KUBERNETES_VERSION}, but the existing cluster state records ${persisted_version}" + return 1 + fi + + E2E_KUBERNETES_VERSION="${persisted_version}" + state_set system_pool_kubernetes_version "${live_system_version}" + state_set flex_pool_kubernetes_version "${live_flex_version}" + require_exact_kubernetes_version + log_info "Restored Kubernetes version from deployment state: ${E2E_KUBERNETES_VERSION}" +} + # Dump the state file for debugging state_dump() { if [[ -f "${E2E_STATE_FILE}" ]]; then log_info "Current state:" - jq '.' "${E2E_STATE_FILE}" + jq ' + def redact: + if type == "object" then + with_entries( + if (.key | test("(^|_)(bootstrap_)?token$|secret|password|credential"; "i")) then + .value = "" + else + .value |= redact + end + ) + elif type == "array" then map(redact) + else . + end; + redact + ' "${E2E_STATE_FILE}" else log_info "No state file found" fi diff --git a/hack/e2e/lib/infra.sh b/hack/e2e/lib/infra.sh index f23271f6..3f6aad97 100755 --- a/hack/e2e/lib/infra.sh +++ b/hack/e2e/lib/infra.sh @@ -51,12 +51,62 @@ infra_deploy() { local start start=$(timer_start) + require_exact_kubernetes_version + local bicep_file="${E2E_INFRA_DIR}/main.bicep" if [[ ! -f "${bicep_file}" ]]; then log_error "Bicep template not found: ${bicep_file}" return 1 fi + # Persist deterministic identities before Azure starts provisioning so the + # always-run cleanup stage can remove resources after a partial deployment. + local deployment_name="e2e-${E2E_NAME_SUFFIX}" + local run_id="${GITHUB_RUN_ID:-local-${E2E_NAME_SUFFIX}}" + # A workflow rerun keeps github.run_id but increments github.run_attempt. + # E2E_NAME_SUFFIX includes both, so use it as the ownership tag to keep + # cleanup for one attempt from deleting another attempt's resources. + local resource_owner="${E2E_NAME_SUFFIX}" + local initial_state + initial_state="$(jq -n \ + --arg lifecycle "provisioning" \ + --arg resource_group "${E2E_RESOURCE_GROUP}" \ + --arg location "${E2E_LOCATION}" \ + --arg subscription_id "${AZURE_SUBSCRIPTION_ID}" \ + --arg tenant_id "${AZURE_TENANT_ID}" \ + --arg run_id "${run_id}" \ + --arg resource_owner "${resource_owner}" \ + --arg name_suffix "${E2E_NAME_SUFFIX}" \ + --arg kubernetes_version "${E2E_KUBERNETES_VERSION}" \ + --arg target_agent_pool_name "${E2E_TARGET_AGENT_POOL_NAME}" \ + --arg bootstrap_data_agent_pool_name "${E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME}" \ + --arg system_pool_name "system" \ + --arg deployment_name "${deployment_name}" \ + --arg cluster_name "aks-e2e-${E2E_NAME_SUFFIX}" \ + --arg node_resource_group "MC_aksflex-e2e-${E2E_NAME_SUFFIX}" \ + --arg vnet_name "vnet-e2e-${E2E_NAME_SUFFIX}" \ + --arg nsg_name "nsg-e2e-${E2E_NAME_SUFFIX}" \ + --arg msi_vm_name "vm-e2e-msi-${E2E_NAME_SUFFIX}" \ + --arg token_vm_name "vm-e2e-token-${E2E_NAME_SUFFIX}" \ + --arg offline_vm_name "vm-e2e-offline-${E2E_NAME_SUFFIX}" \ + --arg kubeadm_vm_name "vm-e2e-kubeadm-${E2E_NAME_SUFFIX}" \ + --arg arc_vm_name "vm-e2e-arc-${E2E_NAME_SUFFIX}" \ + --arg arc_machine_name "vm-e2e-arc-${E2E_NAME_SUFFIX}-connected" \ + '{schema_version: 1, lifecycle: $lifecycle, resource_group: $resource_group, + location: $location, subscription_id: $subscription_id, tenant_id: $tenant_id, + run_id: $run_id, resource_owner: $resource_owner, + name_suffix: $name_suffix, kubernetes_version: $kubernetes_version, + target_agent_pool_name: $target_agent_pool_name, + bootstrap_data_agent_pool_name: $bootstrap_data_agent_pool_name, + system_pool_name: $system_pool_name, + deployment_name: $deployment_name, cluster_name: $cluster_name, + node_resource_group: $node_resource_group, + vnet_name: $vnet_name, nsg_name: $nsg_name, msi_vm_name: $msi_vm_name, + token_vm_name: $token_vm_name, offline_vm_name: $offline_vm_name, + kubeadm_vm_name: $kubeadm_vm_name, arc_vm_name: $arc_vm_name, + arc_machine_name: $arc_machine_name}')" + state_begin_deployment "${initial_state}" + # Ensure resource group exists if ! az group show --name "${E2E_RESOURCE_GROUP}" --output none 2>/dev/null; then log_info "Creating resource group: ${E2E_RESOURCE_GROUP} in ${E2E_LOCATION}" @@ -75,16 +125,14 @@ infra_deploy() { configure_ssh_identity # Build tags - local run_id="${GITHUB_RUN_ID:-local-$(date +%s)}" local tags_json tags_json=$(jq -n \ - --arg run "${run_id}" \ + --arg run "${resource_owner}" \ --arg purpose "e2e-test" \ '{"github-run": $run, "purpose": $purpose}') # Deploy log_info "Deploying Bicep template (this may take 5-10 minutes)..." - local deployment_name="e2e-${E2E_NAME_SUFFIX}" az deployment group create \ --resource-group "${E2E_RESOURCE_GROUP}" \ --name "${deployment_name}" \ @@ -210,6 +258,7 @@ infra_deploy() { return 1 fi + state_set "lifecycle" "ready" log_success "Infrastructure deployed in $(timer_elapsed "${start}")s" } @@ -219,9 +268,48 @@ infra_deploy() { infra_get_kubeconfig() { local cluster_name cluster_name="$(state_get cluster_name)" + local cluster_id + cluster_id="$(state_get cluster_id)" local resource_group resource_group="$(state_get resource_group)" + local actual_kubernetes_version system_pool_version flex_pool_version + actual_kubernetes_version="$(az aks show \ + --resource-group "${resource_group}" \ + --name "${cluster_name}" \ + --query 'currentKubernetesVersion || kubernetesVersion' \ + --output tsv)" + if [[ "${actual_kubernetes_version}" != "${E2E_KUBERNETES_VERSION}" ]]; then + log_error "AKS control-plane version is ${actual_kubernetes_version}, expected ${E2E_KUBERNETES_VERSION}" + return 1 + fi + state_set "kubernetes_version" "${actual_kubernetes_version}" + log_info "Verified AKS control-plane version: ${actual_kubernetes_version}" + + system_pool_version="$(az rest \ + --method get \ + --url "https://management.azure.com${cluster_id}/agentPools/system?api-version=2026-05-02-preview" \ + --query 'properties.currentOrchestratorVersion || properties.orchestratorVersion' \ + --output tsv)" + if [[ "${system_pool_version}" != "${E2E_KUBERNETES_VERSION}" ]]; then + log_error "AKS system pool version is ${system_pool_version}, expected ${E2E_KUBERNETES_VERSION}" + return 1 + fi + state_set "system_pool_kubernetes_version" "${system_pool_version}" + log_info "Verified AKS system pool version: ${system_pool_version}" + + flex_pool_version="$(az rest \ + --method get \ + --url "https://management.azure.com${cluster_id}/agentPools/${E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME}?api-version=2026-05-02-preview" \ + --query 'properties.currentOrchestratorVersion || properties.orchestratorVersion' \ + --output tsv)" + if [[ "${flex_pool_version}" != "${E2E_KUBERNETES_VERSION}" ]]; then + log_error "AKS FlexNodes pool version is ${flex_pool_version}, expected ${E2E_KUBERNETES_VERSION}" + return 1 + fi + state_set "flex_pool_kubernetes_version" "${flex_pool_version}" + log_info "Verified AKS FlexNodes pool version: ${flex_pool_version}" + log_info "Fetching kubeconfig for ${cluster_name}..." az aks get-credentials \ --resource-group "${resource_group}" \ diff --git a/hack/e2e/lib/node-join-arc.sh b/hack/e2e/lib/node-join-arc.sh index 73e2176d..1dfbed89 100755 --- a/hack/e2e/lib/node-join-arc.sh +++ b/hack/e2e/lib/node-join-arc.sh @@ -148,8 +148,9 @@ _fetch_arc_bootstrap_config() { return 1 fi + install -m 0600 /dev/null "${output}" remote_exec "${vm_ip}" "sudo cat '${remote_output}'" > "${output}" - chmod 0600 "${output}" + install -m 0600 /dev/null "${output}.tmp" jq \ --arg nodeIP "${vm_private_ip}" \ --arg nodeName "${vm_name}" \ @@ -165,7 +166,6 @@ _fetch_arc_bootstrap_config() { | .node.kubelet.nodeIP = $nodeIP' \ "${output}" > "${output}.tmp" mv "${output}.tmp" "${output}" - chmod 0600 "${output}" remote_exec "${vm_ip}" "sudo rm -f '${remote_output}' '${remote_binary}'" } diff --git a/hack/e2e/lib/node-join-kubeadm.sh b/hack/e2e/lib/node-join-kubeadm.sh index ef9addf2..443516f1 100644 --- a/hack/e2e/lib/node-join-kubeadm.sh +++ b/hack/e2e/lib/node-join-kubeadm.sh @@ -36,10 +36,10 @@ _kubeadm_ensure_rbac() { # - ClusterRoleBindings for CSR creation and auto-approval # - Roles/RoleBindings granting bootstrappers read access to kubeadm config # and kubelet config (required by kubeadm join's preflight phase) - # - ClusterRole/ClusterRoleBinding for bootstrappers to GET nodes + # - ClusterRole/ClusterRoleBinding for kubeadm's bootstrap group to GET nodes # - ConfigMaps: cluster-info (kube-public), kubeadm-config and # kubelet-config (kube-system) consumed by kubeadm join - kubectl apply -f - < "${config_file}" </dev/null || true local removed=0 - local dir - while IFS= read -r dir; do - [[ -n "${dir}" && -d "${dir}" ]] || continue - log_info "Removing ${dir}" + local work_dir="${E2E_WORK_DIR:-}" + if [[ ! "${work_dir}" =~ ^/tmp/aks-flex-node-e2e(-[A-Za-z0-9][A-Za-z0-9._-]*)?$ ]]; then + log_error "Refusing to clean unexpected E2E work directory '${work_dir}'" + return 1 + fi + if [[ -d "${work_dir}" ]]; then + log_info "Removing ${work_dir}" if [[ "${can_sudo}" == "1" ]]; then - sudo rm -rf -- "${dir}" || { log_warn "Failed to remove ${dir}"; continue; } + sudo rm -rf -- "${work_dir}" || { log_warn "Failed to remove ${work_dir}"; return 1; } else - rm -rf -- "${dir}" || { log_warn "Failed to remove ${dir}"; continue; } + rm -rf -- "${work_dir}" || { log_warn "Failed to remove ${work_dir}"; return 1; } fi - removed=$((removed + 1)) - done < <(find /tmp -maxdepth 1 -type d \( \ - -name 'aks-flex-node-e2e' -o \ - -name 'aks-flex-node-e2e-*' \ - \) 2>/dev/null || true) + removed=1 + fi - log_info "Removed ${removed} temporary directories" + log_info "Removed ${removed} temporary directory" if command -v go >/dev/null 2>&1; then log_info "Cleaning Go build cache" diff --git a/hack/e2e/run.sh b/hack/e2e/run.sh index 035193b1..b7f57f8f 100755 --- a/hack/e2e/run.sh +++ b/hack/e2e/run.sh @@ -24,6 +24,7 @@ # validate Verify nodes joined + run smoke tests # validate-absent Verify all flex nodes are gone after unjoin # smoke Run smoke tests only (pods on flex nodes) +# historical-rbac-migration Build a real v0.1.0 node, upgrade it, and migrate RBAC # nspawn-lifecycle Validate generated lifecycle hooks and restart reconciliation # agent-upgrade Validate managed binary upgrade, rollback, and retry # upgrade-drift Run controller-machine Kubernetes version drift repave test @@ -106,6 +107,8 @@ source "${SCRIPT_DIR}/lib/node-join.sh" # shellcheck disable=SC1091 source "${SCRIPT_DIR}/lib/validate.sh" # shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib/bootstrap-rbac-migration.sh" +# shellcheck disable=SC1091 source "${SCRIPT_DIR}/lib/nspawn-lifecycle.sh" # shellcheck disable=SC1091 source "${SCRIPT_DIR}/lib/upgrade-drift.sh" @@ -136,7 +139,7 @@ usage() { parse_args() { while [[ $# -gt 0 ]]; do case "$1" in - all|infra|join|join-msi|join-token|join-offline|join-kubeadm|join-arc|unjoin|unjoin-msi|unjoin-token|unjoin-offline|unjoin-kubeadm|unjoin-arc|validate|validate-absent|smoke|nspawn-lifecycle|agent-upgrade|upgrade-drift|logs|cleanup|runner-cleanup|status) + all|historical-rbac-migration|infra|join|join-msi|join-token|join-offline|join-kubeadm|join-arc|unjoin|unjoin-msi|unjoin-token|unjoin-offline|unjoin-kubeadm|unjoin-arc|validate|validate-absent|smoke|nspawn-lifecycle|agent-upgrade|upgrade-drift|logs|cleanup|runner-cleanup|status) COMMAND="$1"; shift ;; -g|--resource-group) export E2E_RESOURCE_GROUP="$2"; shift 2 ;; -l|--location) export E2E_LOCATION="$2"; shift 2 ;; @@ -218,7 +221,7 @@ cmd_all() { collect_logs || true # Cleanup - cleanup || true + cleanup || exit_code=1 echo "" log_section "E2E Test Complete" @@ -235,6 +238,29 @@ cmd_all() { return ${exit_code} } +# --------------------------------------------------------------------------- +# Command: historical-rbac-migration +# --------------------------------------------------------------------------- +cmd_historical_rbac_migration() { + log_section "AKS Flex Node Historical RBAC Migration E2E" + + if [[ "${SKIP_BUILD}" != "1" ]]; then + ensure_binary + elif [[ -z "${E2E_BINARY:-}" || ! -f "${E2E_BINARY:-}" ]]; then + log_error "--skip-build requires --binary or E2E_BINARY set to an existing file" + return 1 + else + log_info "Skipping build, using: ${E2E_BINARY}" + fi + + # This suite intentionally uses only the token VM. It does not call the + # parallel all-node join path, so an unrelated Arc failure cannot mask the + # historical compatibility result. + infra_deploy + ensure_cluster_dependencies + historical_rbac_migration_e2e +} + # --------------------------------------------------------------------------- # Command: status # --------------------------------------------------------------------------- @@ -259,6 +285,20 @@ main() { check_prerequisites load_config init_work_dir + + case "${COMMAND}" in + all|historical-rbac-migration|infra) + require_exact_kubernetes_version + ;; + join|join-msi|join-token|join-offline|join-kubeadm|join-arc|upgrade-drift) + restore_kubernetes_version_from_state + ;; + cleanup|runner-cleanup|logs|status|unjoin|unjoin-msi|unjoin-token|unjoin-offline|unjoin-kubeadm|unjoin-arc|validate|validate-absent|smoke|nspawn-lifecycle|agent-upgrade) + # Recovery and inspection commands must remain usable even when a caller + # supplied a bad or stale version value. + ;; + esac + configure_ssh_identity trap 'gha_end_group' EXIT @@ -267,6 +307,9 @@ main() { all) cmd_all ;; + historical-rbac-migration) + cmd_historical_rbac_migration + ;; infra) if [[ "${SKIP_BUILD}" != "1" ]]; then ensure_binary diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index de52d59b..9748ac96 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -1284,6 +1284,8 @@ func TestLoadConfigUsesRPConfigOverLegacyAliases(t *testing.T) { func TestLoadConfigAdaptsLegacyConfigAliases(t *testing.T) { t.Parallel() + // e2eMode was persisted by pre-v0.2 test/dev deployments. Keep accepting + // the removed field so those configs can be upgraded with the current agent. configJSON := `{ "azure": { "targetAgentPoolName": "pool1", @@ -1294,6 +1296,9 @@ func TestLoadConfigAdaptsLegacyConfigAliases(t *testing.T) { "resourceId": "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/test-rg/providers/Microsoft.ContainerService/managedClusters/test-cluster" } }, + "agent": { + "e2eMode": true + }, "kubernetes": { "version": "1.30.1" }, @@ -2262,6 +2267,9 @@ func TestServicePrincipalClientSecretFile(t *testing.T) { if err := os.WriteFile(insecureFile, []byte("file-secret"), 0o644); err != nil { t.Fatalf("os.WriteFile: %v", err) } + if err := os.Chmod(insecureFile, 0o644); err != nil { + t.Fatalf("os.Chmod: %v", err) + } tests := []struct { name string diff --git a/pkg/daemon/lifecycle.go b/pkg/daemon/lifecycle.go index 6862fd91..da616742 100644 --- a/pkg/daemon/lifecycle.go +++ b/pkg/daemon/lifecycle.go @@ -173,6 +173,11 @@ func writeAgentServiceAssets(binaryPaths agentUpgradePaths, serviceOptions agent if err := utilio.WriteFile(asset.path, asset.content, asset.mode); err != nil { return fmt.Errorf("write %s: %w", asset.path, err) } + // Atomic replacement preserves an existing file's mode and applies the + // process umask to new files, so reconcile the declared service-asset mode. + if err := os.Chmod(asset.path, asset.mode); err != nil { + return fmt.Errorf("set permissions on %s: %w", asset.path, err) + } } return nil } diff --git a/pkg/daemon/lifecycle_test.go b/pkg/daemon/lifecycle_test.go index b4a404fc..a8691f89 100644 --- a/pkg/daemon/lifecycle_test.go +++ b/pkg/daemon/lifecycle_test.go @@ -29,6 +29,9 @@ func TestEnsureAgentUpgradeServiceAssetsMigratesExistingInstallation(t *testing. t.Fatalf("write legacy unit: %v", err) } recoveryPath := filepath.Join(t.TempDir(), "aks-flex-node-recovery.sh") + if err := os.WriteFile(recoveryPath, []byte("stale"), 0o700); err != nil { + t.Fatalf("write legacy recovery script: %v", err) + } reloaded := false if err := ensureAgentUpgradeServiceAssetsAt( t.Context(), diff --git a/scripts/aks-flex-config b/scripts/aks-flex-config index 313d3e57..85dbdb4f 100755 --- a/scripts/aks-flex-config +++ b/scripts/aks-flex-config @@ -4,18 +4,37 @@ from __future__ import annotations import argparse +import http.client import json import os +import re import secrets +import selectors +import signal import shutil +import socket +import stat import subprocess import sys +import tempfile +import time from datetime import datetime, timedelta, timezone from pathlib import Path -from urllib.parse import urlsplit +from urllib.parse import quote, urlsplit RESOURCE_MANAGER_ENDPOINT = "https://management.azure.com" DEFAULT_AGENT_POOL_NAME = "aksflexnodes" +RBAC_API_GROUP = "rbac.authorization.k8s.io" +LEGACY_NODE_ROLE_BINDING = "aks-flex-node-role" +FLEX_NODE_BOOTSTRAP_GROUP = "system:bootstrappers:aks-flex-node" +RBAC_AUTOUPDATE_ANNOTATION = "rbac.authorization.kubernetes.io/autoupdate" +MANAGED_BOOTSTRAP_BINDINGS = ( + ("aks-flex-node-bootstrapper", "system:node-bootstrapper"), + ( + "aks-flex-node-auto-approve-csr", + "system:certificates.k8s.io:certificatesigningrequests:nodeclient", + ), +) def log_info(message: str) -> None: @@ -65,13 +84,428 @@ def setup_node_rbac(args: argparse.Namespace) -> None: require_command("kubectl") load_admin_kubeconfig(args) - log_info("applying bootstrap token RBAC bindings") - run(["kubectl", "apply", "-f", "-"], input_text=RBAC_MANIFEST) + bindings = reconcile_bootstrap_rbac_bindings() + if args.remove_legacy_node_role_binding: + remove_legacy_node_role_binding(bindings) + else: + require_legacy_node_role_binding_absent(bindings) + + +def cluster_role_bindings() -> list[dict[str, object]]: + raw = run(["kubectl", "get", "clusterrolebindings", "-o", "json"], capture=True) + try: + payload = json.loads(raw) + except json.JSONDecodeError as err: + raise SystemExit(f"ERROR: could not parse ClusterRoleBinding inventory: {err}") from err + + if not isinstance(payload, dict): + raise SystemExit("ERROR: ClusterRoleBinding inventory is not a JSON object") + items = payload.get("items") + if not isinstance(items, list): + raise SystemExit("ERROR: ClusterRoleBinding inventory does not contain an items list") + if not all(isinstance(item, dict) for item in items): + raise SystemExit("ERROR: ClusterRoleBinding inventory contains a malformed item") + return items + + +def expected_role_ref(role_name: str) -> dict[str, str]: + return {"apiGroup": RBAC_API_GROUP, "kind": "ClusterRole", "name": role_name} + + +def bootstrap_group_subject() -> dict[str, str]: + return { + "apiGroup": RBAC_API_GROUP, + "kind": "Group", + "name": FLEX_NODE_BOOTSTRAP_GROUP, + } + + +def binding_has_bootstrap_group(name: str, binding: dict[str, object]) -> bool: + subjects = binding.get("subjects", []) + if subjects is None: + subjects = [] + if not isinstance(subjects, list): + raise SystemExit( + f"ERROR: managed ClusterRoleBinding {name!r} subjects is not a list" + ) + return any( + isinstance(subject, dict) + and subject.get("apiGroup") == RBAC_API_GROUP + and subject.get("kind") == "Group" + and subject.get("name") == FLEX_NODE_BOOTSTRAP_GROUP + and not subject.get("namespace") + for subject in subjects + ) + + +def managed_binding_inventory(bindings: list[dict[str, object]]) -> dict[str, dict[str, object]]: + expected_names = {name for name, _ in MANAGED_BOOTSTRAP_BINDINGS} + managed: dict[str, dict[str, object]] = {} + for binding in bindings: + metadata = binding.get("metadata") + name = metadata.get("name") if isinstance(metadata, dict) else None + if name not in expected_names: + continue + if name in managed: + raise SystemExit(f"ERROR: duplicate managed ClusterRoleBinding {name!r} in inventory") + managed[name] = binding + return managed + + +def validate_managed_binding( + name: str, + role_name: str, + binding: dict[str, object], + *, + require_subject: bool, +) -> bool: + role_ref = binding.get("roleRef") + wanted_role_ref = expected_role_ref(role_name) + if role_ref != wanted_role_ref: + raise SystemExit( + f"ERROR: refusing to modify managed ClusterRoleBinding {name!r}: roleRef is " + f"{role_ref!r}, expected {wanted_role_ref!r}. The roleRef is immutable and replacing " + "this object could discard operator-managed subjects or metadata. Review it manually." + ) + + has_subject = binding_has_bootstrap_group(name, binding) + if require_subject and not has_subject: + raise SystemExit( + f"ERROR: managed ClusterRoleBinding {name!r} does not contain the required " + f"bootstrap group {FLEX_NODE_BOOTSTRAP_GROUP!r} after reconciliation" + ) + return has_subject + + +def desired_managed_binding(name: str, role_name: str) -> dict[str, object]: + return { + "apiVersion": f"{RBAC_API_GROUP}/v1", + "kind": "ClusterRoleBinding", + "metadata": {"name": name}, + "roleRef": expected_role_ref(role_name), + "subjects": [bootstrap_group_subject()], + } + + +def reconcile_bootstrap_rbac_bindings() -> list[dict[str, object]]: + before = cluster_role_bindings() + managed = managed_binding_inventory(before) + actions: list[tuple[str, str, dict[str, object]]] = [] + + # Validate every managed name before making any change. A roleRef is + # immutable, so replacing a customized binding would lose operator-managed + # subjects and metadata. + for name, role_name in MANAGED_BOOTSTRAP_BINDINGS: + binding = managed.get(name) + if binding is None: + actions.append(("create", name, desired_managed_binding(name, role_name))) + continue + + has_subject = validate_managed_binding(name, role_name, binding, require_subject=False) + if has_subject: + continue + + metadata = binding.get("metadata") + annotations = metadata.get("annotations", {}) if isinstance(metadata, dict) else {} + if not isinstance(annotations, dict): + raise SystemExit(f"ERROR: managed ClusterRoleBinding {name!r} annotations is not an object") + if str(annotations.get(RBAC_AUTOUPDATE_ANNOTATION, "")).lower() == "false": + raise SystemExit( + f"ERROR: managed ClusterRoleBinding {name!r} disables RBAC autoupdate but is missing " + f"the required bootstrap group {FLEX_NODE_BOOTSTRAP_GROUP!r}. Add the subject " + "manually or remove the autoupdate=false annotation." + ) + resource_version = metadata.get("resourceVersion") if isinstance(metadata, dict) else None + if not isinstance(resource_version, str) or not resource_version: + raise SystemExit( + f"ERROR: refusing to update managed ClusterRoleBinding {name!r} without a " + "resourceVersion concurrency precondition" + ) + + updated = dict(binding) + subjects = binding.get("subjects", []) + if subjects is None: + subjects = [] + updated["apiVersion"] = f"{RBAC_API_GROUP}/v1" + updated["kind"] = "ClusterRoleBinding" + updated["subjects"] = [*subjects, bootstrap_group_subject()] + actions.append(("replace", name, updated)) + + for operation, name, binding in actions: + log_info(f"{operation} managed bootstrap RBAC binding {name}") + run(["kubectl", operation, "-f", "-"], input_text=json.dumps(binding), capture=True) + + after = cluster_role_bindings() + reconciled = managed_binding_inventory(after) + for name, role_name in MANAGED_BOOTSTRAP_BINDINGS: + binding = reconciled.get(name) + if binding is None: + raise SystemExit(f"ERROR: managed ClusterRoleBinding {name!r} is absent after reconciliation") + validate_managed_binding(name, role_name, binding, require_subject=True) + return after + + +def unsafe_node_role_bindings(bindings: list[dict[str, object]] | None = None) -> list[dict[str, object]]: + if bindings is None: + bindings = cluster_role_bindings() + + unsafe = [] + for item in bindings: + if not isinstance(item, dict): + continue + role_ref = item.get("roleRef") + subjects = item.get("subjects") + if not isinstance(role_ref, dict) or not isinstance(subjects, list): + continue + if ( + role_ref.get("apiGroup") != "rbac.authorization.k8s.io" + or role_ref.get("kind") != "ClusterRole" + or role_ref.get("name") != "system:node" + ): + continue + if any( + isinstance(subject, dict) + and subject.get("apiGroup") == "rbac.authorization.k8s.io" + and subject.get("kind") == "Group" + and subject.get("name") == FLEX_NODE_BOOTSTRAP_GROUP + for subject in subjects + ): + unsafe.append(item) + return unsafe + + +def unsafe_binding_names(bindings: list[dict[str, object]]) -> str: + names = [] + for binding in bindings: + metadata = binding.get("metadata") + name = metadata.get("name") if isinstance(metadata, dict) else None + names.append(name if isinstance(name, str) and name else "") + return ", ".join(sorted(names)) + + +def is_canonical_legacy_node_role_binding(binding: dict[str, object]) -> bool: + metadata = binding.get("metadata") + subjects = binding.get("subjects") + if not isinstance(metadata, dict) or metadata.get("name") != LEGACY_NODE_ROLE_BINDING: + return False + return subjects == [ + { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Group", + "name": FLEX_NODE_BOOTSTRAP_GROUP, + } + ] + + +def require_legacy_node_role_binding_absent(bindings: list[dict[str, object]] | None = None) -> None: + bindings = unsafe_node_role_bindings(bindings) + if not bindings: + return + raise SystemExit( + "ERROR: ClusterRoleBinding(s) " + f"{unsafe_binding_names(bindings)} still grant {FLEX_NODE_BOOTSTRAP_GROUP!r} the " + "system:node role. Some older or development agents may still depend on that access. " + "First upgrade them to a release with daemon client certificates (v0.1.1 or later), " + "verify /etc/aks-flex-node/daemon-credentials/daemon-controller-current.pem exists " + "and the agent remains healthy after restart, then rerun setup-node-rbac with " + "--remove-legacy-node-role-binding." + ) + + +class UnixSocketHTTPConnection(http.client.HTTPConnection): + def __init__(self, socket_path: str, *, timeout: float) -> None: + super().__init__("localhost", timeout=timeout) + self.socket_path = socket_path + + def connect(self) -> None: + connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + connection.settimeout(self.timeout) + try: + connection.connect(self.socket_path) + except OSError: + connection.close() + raise + self.sock = connection + + +def delete_cluster_role_binding_with_preconditions(name: str, uid: str, resource_version: str) -> None: + # kubectl delete does not expose UID/resourceVersion preconditions, and its + # --raw mode sends no request body. A proxy on a mode-0600 Unix socket lets + # us submit the Kubernetes DeleteOptions body without reimplementing + # kubeconfig authentication, exposing admin credentials on a localhost TCP + # port, or silently deleting a concurrently replaced object. + with tempfile.TemporaryDirectory(prefix="aks-flex-config-") as proxy_dir: + socket_path = os.path.join(proxy_dir, "kubectl-proxy.sock") + proxy = subprocess.Popen( + [ + "kubectl", + "proxy", + f"--unix-socket={socket_path}", + r"--accept-hosts=^localhost$", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + ) + try: + if proxy.stdout is None or proxy.stderr is None: + raise SystemExit("ERROR: failed to capture kubectl proxy startup output") + + startup_output = {"stdout": bytearray(), "stderr": bytearray()} + startup_pattern = re.compile( + rb"(?:^|\n)Starting to serve on " + re.escape(os.fsencode(socket_path)) + rb"\r?\n" + ) + ready = False + deadline = time.monotonic() + 10 + with selectors.DefaultSelector() as selector: + selector.register(proxy.stdout, selectors.EVENT_READ, "stdout") + selector.register(proxy.stderr, selectors.EVENT_READ, "stderr") + while time.monotonic() < deadline: + events = selector.select(max(0, deadline - time.monotonic())) + if not events: + break + for key, _ in events: + chunk = os.read(key.fd, 4096) + if not chunk: + selector.unregister(key.fileobj) + continue + stream_output = startup_output[key.data] + stream_output.extend(chunk) + if len(stream_output) > 65536: + del stream_output[:-65536] + if key.data == "stdout" and startup_pattern.search(stream_output): + ready = True + break + if ready: + break + if proxy.poll() is not None and not selector.get_map(): + break + + if not ready: + detail = b"\n".join( + output.strip() for output in startup_output.values() if output.strip() + ).decode(errors="replace") + if proxy.poll() is None: + raise SystemExit( + "ERROR: timed out starting kubectl proxy for conditional RBAC deletion" + f"{': ' + detail if detail else ''}" + ) + raise SystemExit(f"ERROR: kubectl proxy failed to start: {detail or 'unknown error'}") + + try: + socket_info = os.stat(socket_path, follow_symlinks=False) + if not stat.S_ISSOCK(socket_info.st_mode): + raise SystemExit("ERROR: kubectl proxy did not create a Unix socket") + os.chmod(socket_path, 0o600) + socket_mode = stat.S_IMODE(os.stat(socket_path, follow_symlinks=False).st_mode) + except OSError as err: + raise SystemExit(f"ERROR: could not secure kubectl proxy Unix socket: {err}") from err + if socket_mode != 0o600: + raise SystemExit( + f"ERROR: kubectl proxy Unix socket mode is {socket_mode:o}, expected 600" + ) + + body = json.dumps( + { + "apiVersion": "v1", + "kind": "DeleteOptions", + "preconditions": {"uid": uid, "resourceVersion": resource_version}, + } + ).encode() + resource_path = f"/apis/{RBAC_API_GROUP}/v1/clusterrolebindings/{quote(name, safe='')}" + connection = UnixSocketHTTPConnection(socket_path, timeout=30) + try: + connection.request( + "DELETE", + resource_path, + body=body, + headers={"Content-Type": "application/json"}, + ) + response = connection.getresponse() + detail = response.read().decode(errors="replace").strip() + if response.status == 409: + raise SystemExit( + f"ERROR: refusing to delete ClusterRoleBinding {name!r}: it changed after inspection" + ) + if not 200 <= response.status < 300: + raise SystemExit( + f"ERROR: conditional deletion of ClusterRoleBinding {name!r} returned " + f"HTTP {response.status}: {detail}" + ) + except (OSError, http.client.HTTPException) as err: + raise SystemExit( + f"ERROR: conditional deletion of ClusterRoleBinding {name!r} failed: {err}" + ) from err + finally: + connection.close() + finally: + try: + os.killpg(proxy.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + proxy.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + try: + # The kubectl process may have exited while an exec credential + # plugin kept the process group and pipe descriptors alive. + os.killpg(proxy.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + proxy.wait(timeout=5) + except subprocess.TimeoutExpired: + log_error(f"kubectl proxy process {proxy.pid} did not exit after SIGKILL") + if proxy.stdout is not None: + proxy.stdout.close() + if proxy.stderr is not None: + proxy.stderr.close() + + +def remove_legacy_node_role_binding(bindings: list[dict[str, object]] | None = None) -> None: + bindings = unsafe_node_role_bindings(bindings) + if not bindings: + return + if len(bindings) != 1 or not is_canonical_legacy_node_role_binding(bindings[0]): + raise SystemExit( + "ERROR: refusing automatic removal because the unsafe ClusterRoleBinding set is " + f"not the canonical {LEGACY_NODE_ROLE_BINDING!r} object: " + f"{unsafe_binding_names(bindings)}. Review and remove only the bootstrap-group " + "subjects manually." + ) + + metadata = bindings[0].get("metadata") + uid = metadata.get("uid") if isinstance(metadata, dict) else None + resource_version = metadata.get("resourceVersion") if isinstance(metadata, dict) else None + if not isinstance(uid, str) or not uid or not isinstance(resource_version, str) or not resource_version: + raise SystemExit( + "ERROR: refusing automatic removal because the canonical legacy " + "ClusterRoleBinding has no UID/resourceVersion preconditions." + ) + + # Delete only the object version that was inspected above. If another + # actor replaces or edits it between inventory and deletion, the API server + # rejects the request instead of deleting an unreviewed object. + log_info("removing legacy bootstrap node role binding") + delete_cluster_role_binding_with_preconditions(LEGACY_NODE_ROLE_BINDING, uid, resource_version) + remaining = unsafe_node_role_bindings() + if remaining: + raise SystemExit( + "ERROR: unsafe bootstrap node role binding remains after deletion: " + f"{unsafe_binding_names(remaining)}" + ) def generate_bootstrap_token(args: argparse.Namespace) -> str: require_command("kubectl") + # Do not silently break old agents by deleting their binding while rendering + # a config, and never mint another broadly privileged token. Migration is an + # explicit setup-node-rbac action after existing agents have been upgraded. + require_legacy_node_role_binding_absent() + log_info("creating bootstrap token") + token_id = secrets.token_hex(3) token_secret = secrets.token_hex(8) token = f"{token_id}.{token_secret}" @@ -177,7 +611,6 @@ def render_config(args: argparse.Namespace, mode: str, metadata: dict[str, str]) if mode == "bootstrap-token": load_admin_kubeconfig(args) - log_info("creating bootstrap token") token = generate_bootstrap_token(args) server_url = run( ["kubectl", "config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}"], @@ -258,8 +691,16 @@ def build_parser() -> argparse.ArgumentParser: subparser.add_argument("--cluster-name", required=True) subparser.add_argument("--subscription") - rbac = subparsers.add_parser("setup-node-rbac", help="Apply node bootstrap RBAC bindings.") + rbac = subparsers.add_parser("setup-node-rbac", help="Reconcile node bootstrap RBAC bindings.") add_cluster_args(rbac) + rbac.add_argument( + "--remove-legacy-node-role-binding", + action="store_true", + help=( + "Remove the canonical obsolete aks-flex-node-role binding after existing agents " + "have issued daemon certificates and remain healthy after restart." + ), + ) rbac.set_defaults(func=setup_node_rbac) generate = subparsers.add_parser("generate-node-config", help="Render a Flex Node config.") @@ -282,48 +723,6 @@ def build_parser() -> argparse.ArgumentParser: return parser -RBAC_MANIFEST = """ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: aks-flex-node-bootstrapper -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:node-bootstrapper -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:bootstrappers:aks-flex-node ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: aks-flex-node-auto-approve-csr -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:certificates.k8s.io:certificatesigningrequests:nodeclient -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:bootstrappers:aks-flex-node ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: aks-flex-node-role -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:node -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:bootstrappers:aks-flex-node -""".lstrip() - - def main() -> None: parser = build_parser() args = parser.parse_args() diff --git a/scripts/aks_flex_config_test.go b/scripts/aks_flex_config_test.go new file mode 100644 index 00000000..c0226ee2 --- /dev/null +++ b/scripts/aks_flex_config_test.go @@ -0,0 +1,1729 @@ +package scripts + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/yaml" +) + +const ( + bootstrapGroup = "system:bootstrappers:aks-flex-node" + bootstrapBindingName = "aks-flex-node-bootstrapper" + bootstrapRole = "system:node-bootstrapper" + approvalBindingName = "aks-flex-node-auto-approve-csr" + approvalRole = "system:certificates.k8s.io:certificatesigningrequests:nodeclient" + legacyBindingName = "aks-flex-node-role" + legacyNodeRole = "system:node" + fakeDeleteFailure = 37 + fakeGetFailure = 39 + fakeCommandLogEnv = "AKS_FLEX_CONFIG_TEST_COMMAND_LOG" + fakeManifestEnv = "AKS_FLEX_CONFIG_TEST_MANIFEST" + fakeManagedStateEnv = "AKS_FLEX_CONFIG_TEST_MANAGED_STATE" + fakeDeleteOptsEnv = "AKS_FLEX_CONFIG_TEST_DELETE_OPTIONS" + fakeLegacyStateEnv = "AKS_FLEX_CONFIG_TEST_LEGACY_STATE" + fakeDeleteExitEnv = "AKS_FLEX_CONFIG_TEST_DELETE_EXIT" + fakeDeleteKeepsEnv = "AKS_FLEX_CONFIG_TEST_DELETE_KEEPS_STATE" + fakeConcurrentEnv = "AKS_FLEX_CONFIG_TEST_CONCURRENT_REPLACE" + fakeProxyStartupEnv = "AKS_FLEX_CONFIG_TEST_PROXY_STARTUP" + fakeConcurrentManagedEnv = "AKS_FLEX_CONFIG_TEST_CONCURRENT_MANAGED_REPLACE" + fakeManagedPostconditionEnv = "AKS_FLEX_CONFIG_TEST_MANAGED_POSTCONDITION" + fakeSkipManagedMutationEnv = "AKS_FLEX_CONFIG_TEST_SKIP_MANAGED_MUTATION" + fakeGetExitEnv = "AKS_FLEX_CONFIG_TEST_GET_EXIT" + fakeApplyCountEnv = "AKS_FLEX_CONFIG_TEST_APPLY_COUNT" + fakeApplyFailAtEnv = "AKS_FLEX_CONFIG_TEST_APPLY_FAIL_AT" +) + +type commandCall struct { + name string + args []string +} + +type configScriptHarness struct { + pythonPath string + scriptPath string + fakeBinDir string + commandLogPath string + manifestPath string + managedState string + deleteOptsPath string + configPath string + legacyState string + deleteExitCode int + deleteKeeps bool + concurrentSwap bool + proxyStartup string + poisonHTTPProxy bool + concurrentManaged string + managedPostcondition string + skipManagedMutation bool + getExitCode int + applyCountPath string + applyFailAt int +} + +func TestSetupNodeRBACManifestUsesOnlyBootstrapPermissions(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + output, err := harness.runSetupNodeRBAC(false) + if err != nil { + t.Fatalf("setup-node-rbac failed: %v\n%s", err, output) + } + + bindings := readRBACManifest(t, harness.manifestPath) + expectedRoles := map[string]string{ + "aks-flex-node-bootstrapper": "system:node-bootstrapper", + "aks-flex-node-auto-approve-csr": "system:certificates.k8s.io:certificatesigningrequests:nodeclient", + } + if len(bindings) != len(expectedRoles) { + t.Fatalf("RBAC manifest has %d bindings, want %d: %v", len(bindings), len(expectedRoles), bindingNames(bindings)) + } + + seen := make(map[string]struct{}, len(bindings)) + for _, binding := range bindings { + if binding.APIVersion != "rbac.authorization.k8s.io/v1" || binding.Kind != "ClusterRoleBinding" { + t.Errorf("binding %q has apiVersion/kind %q/%q, want rbac.authorization.k8s.io/v1/ClusterRoleBinding", binding.Name, binding.APIVersion, binding.Kind) + } + if binding.Namespace != "" { + t.Errorf("binding %q unexpectedly has namespace %q", binding.Name, binding.Namespace) + } + if _, duplicate := seen[binding.Name]; duplicate { + t.Errorf("binding %q appears more than once", binding.Name) + } + seen[binding.Name] = struct{}{} + + wantRole, expected := expectedRoles[binding.Name] + if !expected { + t.Errorf("unexpected ClusterRoleBinding %q", binding.Name) + continue + } + if binding.RoleRef.APIGroup != rbacv1.GroupName || binding.RoleRef.Kind != "ClusterRole" || binding.RoleRef.Name != wantRole { + t.Errorf("binding %q roleRef = %#v, want ClusterRole %q in %q", binding.Name, binding.RoleRef, wantRole, rbacv1.GroupName) + } + if len(binding.Subjects) != 1 { + t.Errorf("binding %q has %d subjects, want exactly one", binding.Name, len(binding.Subjects)) + continue + } + subject := binding.Subjects[0] + if subject.APIGroup != rbacv1.GroupName || subject.Kind != "Group" || subject.Name != bootstrapGroup || subject.Namespace != "" { + t.Errorf("binding %q subject = %#v, want bootstrapper group %q", binding.Name, subject, bootstrapGroup) + } + if binding.RoleRef.Name == legacyNodeRole { + t.Errorf("bootstrap group must not be bound to broad legacy role %q", legacyNodeRole) + } + } + + if _, found := seen[legacyBindingName]; found { + t.Errorf("RBAC manifest still contains legacy binding %q", legacyBindingName) + } + + calls := readCommandCalls(t, harness.commandLogPath) + applyIndexes, _ := kubectlOperationIndexes(calls) + if len(applyIndexes) != 2 { + t.Fatalf("RBAC reconciliation count = %d, want 2; calls: %s", len(applyIndexes), formatCalls(calls)) + } + for _, index := range applyIndexes { + assertSafeManagedRBACMutation(t, calls[index]) + } +} + +func TestSetupNodeRBACRejectsManagedRoleRefDriftBeforeMutation(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + bindingName string + }{ + {name: "bootstrapper binding", bindingName: bootstrapBindingName}, + {name: "approval binding", bindingName: approvalBindingName}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + bootstrapper := managedBindingFixture(bootstrapBindingName, bootstrapRole, true) + approver := managedBindingFixture(approvalBindingName, approvalRole, true) + if test.bindingName == bootstrapBindingName { + bootstrapper.RoleRef.Name = "view" + bootstrapper.Subjects = append(bootstrapper.Subjects, rbacv1.Subject{ + APIGroup: rbacv1.GroupName, + Kind: "Group", + Name: "operator-bootstrapper-group", + }) + } else { + approver.RoleRef.Name = "view" + approver.Subjects = append(approver.Subjects, rbacv1.Subject{ + APIGroup: rbacv1.GroupName, + Kind: "Group", + Name: "operator-approver-group", + }) + } + writeManagedState(t, harness.managedState, bootstrapper, approver) + before := readFile(t, harness.managedState) + + output, err := harness.runSetupNodeRBAC(false) + if err == nil { + t.Fatalf("setup-node-rbac replaced a customized roleRef\n%s", output) + } + if !strings.Contains(output, test.bindingName) || + !strings.Contains(output, "roleRef is immutable") || + !strings.Contains(output, "Review it manually") { + t.Fatalf("failure did not identify the safe manual remediation:\n%s", output) + } + if after := readFile(t, harness.managedState); after != before { + t.Fatalf("managed bindings changed despite preflight failure\nbefore: %s\nafter: %s", before, after) + } + + calls := readCommandCalls(t, harness.commandLogPath) + mutations, deletes := kubectlOperationIndexes(calls) + if len(mutations) != 0 || len(deletes) != 0 { + t.Fatalf("roleRef preflight performed a mutation: %s", formatCalls(calls)) + } + }) + } +} + +func TestSetupNodeRBACIdentifiesManagedBindingWithMalformedSubjects(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + bindingName string + }{ + {name: "bootstrapper binding", bindingName: bootstrapBindingName}, + {name: "approval binding", bindingName: approvalBindingName}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + bindings := []map[string]any{ + { + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": map[string]any{"name": bootstrapBindingName, "resourceVersion": "7"}, + "roleRef": map[string]any{ + "apiGroup": rbacv1.GroupName, + "kind": "ClusterRole", + "name": bootstrapRole, + }, + "subjects": []any{map[string]any{ + "apiGroup": rbacv1.GroupName, + "kind": "Group", + "name": bootstrapGroup, + }}, + }, + { + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": map[string]any{"name": approvalBindingName, "resourceVersion": "7"}, + "roleRef": map[string]any{ + "apiGroup": rbacv1.GroupName, + "kind": "ClusterRole", + "name": approvalRole, + }, + "subjects": []any{map[string]any{ + "apiGroup": rbacv1.GroupName, + "kind": "Group", + "name": bootstrapGroup, + }}, + }, + } + malformedIndex := 0 + if test.bindingName == approvalBindingName { + malformedIndex = 1 + } + bindings[malformedIndex]["subjects"] = "not-a-list" + data, err := json.Marshal(bindings) + if err != nil { + t.Fatalf("marshal malformed managed bindings: %v", err) + } + if err := os.WriteFile(harness.managedState, append(data, '\n'), 0o600); err != nil { + t.Fatalf("write malformed managed bindings: %v", err) + } + + output, err := harness.runSetupNodeRBAC(false) + if err == nil { + t.Fatalf("setup-node-rbac accepted malformed subjects\n%s", output) + } + if !strings.Contains(output, test.bindingName) || + !strings.Contains(output, "subjects is not a list") { + t.Fatalf("failure did not identify the malformed binding:\n%s", output) + } + + mutations, deletes := kubectlOperationIndexes(readCommandCalls(t, harness.commandLogPath)) + if len(mutations) != 0 || len(deletes) != 0 { + t.Fatalf("malformed-subject preflight performed a mutation") + } + }) + } +} + +func TestSetupNodeRBACPreservesManagedCustomizations(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + bootstrapper := managedBindingFixture(bootstrapBindingName, bootstrapRole, true) + bootstrapper.Labels = map[string]string{"owner": "operator"} + bootstrapper.Annotations = map[string]string{"example.test/note": "keep"} + bootstrapper.Subjects = append(bootstrapper.Subjects, rbacv1.Subject{ + APIGroup: rbacv1.GroupName, + Kind: "Group", + Name: "operator-group", + }) + approver := managedBindingFixture(approvalBindingName, approvalRole, true) + approver.Annotations = map[string]string{"rbac.authorization.kubernetes.io/autoupdate": "false"} + writeManagedState(t, harness.managedState, bootstrapper, approver) + before := readFile(t, harness.managedState) + + output, err := harness.runSetupNodeRBAC(false) + if err != nil { + t.Fatalf("setup-node-rbac rejected valid customized bindings: %v\n%s", err, output) + } + if after := readFile(t, harness.managedState); after != before { + t.Fatalf("already-correct managed bindings changed\nbefore: %s\nafter: %s", before, after) + } + mutations, deletes := kubectlOperationIndexes(readCommandCalls(t, harness.commandLogPath)) + if len(mutations) != 0 || len(deletes) != 0 { + t.Fatalf("already-correct managed bindings were mutated") + } +} + +func TestSetupNodeRBACAddsSubjectWithOptimisticReplace(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + bootstrapper := managedBindingFixture(bootstrapBindingName, bootstrapRole, false) + bootstrapper.Labels = map[string]string{"owner": "operator"} + bootstrapper.Subjects = []rbacv1.Subject{{ + APIGroup: rbacv1.GroupName, + Kind: "Group", + Name: "operator-group", + }} + approver := managedBindingFixture(approvalBindingName, approvalRole, true) + writeManagedState(t, harness.managedState, bootstrapper, approver) + + output, err := harness.runSetupNodeRBAC(false) + if err != nil { + t.Fatalf("setup-node-rbac failed to add the required subject: %v\n%s", err, output) + } + mutations, deletes := kubectlOperationIndexes(readCommandCalls(t, harness.commandLogPath)) + if len(mutations) != 1 || len(deletes) != 0 { + t.Fatalf("managed reconciliation mutation/delete counts = %d/%d, want 1/0", len(mutations), len(deletes)) + } + if call := readCommandCalls(t, harness.commandLogPath)[mutations[0]]; len(call.args) == 0 || call.args[0] != "replace" { + t.Fatalf("managed binding update = %#v, want optimistic kubectl replace", call) + } + + updated := findManagedBinding(t, readManagedState(t, harness.managedState), bootstrapBindingName) + if updated.Labels["owner"] != "operator" { + t.Fatalf("operator metadata was not preserved: %#v", updated.Labels) + } + if !hasSubject(updated.Subjects, "operator-group") || !hasSubject(updated.Subjects, bootstrapGroup) { + t.Fatalf("operator and required subjects were not both preserved: %#v", updated.Subjects) + } + if updated.ResourceVersion != "8" { + t.Fatalf("resourceVersion = %q, want optimistic replacement of version 7", updated.ResourceVersion) + } +} + +func TestSetupNodeRBACRejectsMissingManagedResourceVersion(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + bindingName string + }{ + {name: "bootstrapper binding", bindingName: bootstrapBindingName}, + {name: "approval binding", bindingName: approvalBindingName}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + bootstrapper := managedBindingFixture(bootstrapBindingName, bootstrapRole, true) + approver := managedBindingFixture(approvalBindingName, approvalRole, true) + if test.bindingName == bootstrapBindingName { + bootstrapper.ResourceVersion = "" + bootstrapper.Subjects = nil + } else { + approver.ResourceVersion = "" + approver.Subjects = nil + } + writeManagedState(t, harness.managedState, bootstrapper, approver) + before := readFile(t, harness.managedState) + + output, err := harness.runSetupNodeRBAC(false) + if err == nil { + t.Fatalf("setup-node-rbac replaced a binding without a resourceVersion\n%s", output) + } + if !strings.Contains(output, test.bindingName) || !strings.Contains(output, "resourceVersion") { + t.Fatalf("failure did not identify the missing concurrency precondition:\n%s", output) + } + if after := readFile(t, harness.managedState); after != before { + t.Fatalf("managed bindings changed despite a missing resourceVersion\nbefore: %s\nafter: %s", before, after) + } + + mutations, deletes := kubectlOperationIndexes(readCommandCalls(t, harness.commandLogPath)) + if len(mutations) != 0 || len(deletes) != 0 { + t.Fatalf("missing resourceVersion preflight performed a mutation") + } + }) + } +} + +func TestSetupNodeRBACRespectsAutoupdateFalse(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + includeBootstrap bool + wantFailure bool + }{ + {name: "missing required subject", wantFailure: true}, + {name: "required subject already present", includeBootstrap: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + bootstrapper := managedBindingFixture(bootstrapBindingName, bootstrapRole, test.includeBootstrap) + bootstrapper.Annotations = map[string]string{ + "rbac.authorization.kubernetes.io/autoupdate": "false", + } + approver := managedBindingFixture(approvalBindingName, approvalRole, true) + writeManagedState(t, harness.managedState, bootstrapper, approver) + + output, err := harness.runSetupNodeRBAC(false) + if test.wantFailure && err == nil { + t.Fatalf("setup-node-rbac changed a protected binding\n%s", output) + } + if !test.wantFailure && err != nil { + t.Fatalf("setup-node-rbac rejected a complete protected binding: %v\n%s", err, output) + } + if test.wantFailure && (!strings.Contains(output, "autoupdate") || !strings.Contains(output, "manually")) { + t.Fatalf("failure did not explain protected-binding remediation:\n%s", output) + } + mutations, deletes := kubectlOperationIndexes(readCommandCalls(t, harness.commandLogPath)) + if len(mutations) != 0 || len(deletes) != 0 { + t.Fatalf("protected binding was mutated") + } + }) + } +} + +func TestSetupNodeRBACPreservesConcurrentManagedReplacement(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + bootstrapper := managedBindingFixture(bootstrapBindingName, bootstrapRole, false) + approver := managedBindingFixture(approvalBindingName, approvalRole, true) + writeManagedState(t, harness.managedState, bootstrapper, approver) + harness.concurrentManaged = bootstrapBindingName + + output, err := harness.runSetupNodeRBAC(false) + if err == nil { + t.Fatalf("setup-node-rbac overwrote a concurrently replaced binding\n%s", output) + } + replacement := findManagedBinding(t, readManagedState(t, harness.managedState), bootstrapBindingName) + if replacement.UID != types.UID("operator-replacement") || + replacement.RoleRef.Name != "view" || + !hasSubject(replacement.Subjects, "operator-group") { + t.Fatalf("concurrent operator replacement was not preserved: %#v", replacement) + } +} + +func TestSetupNodeRBACPreservesConcurrentManagedCreate(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + harness.concurrentManaged = bootstrapBindingName + + output, err := harness.runSetupNodeRBAC(false) + if err == nil { + t.Fatalf("setup-node-rbac overwrote a concurrently created binding\n%s", output) + } + replacement := findManagedBinding(t, readManagedState(t, harness.managedState), bootstrapBindingName) + if replacement.UID != types.UID("operator-replacement") || + replacement.RoleRef.Name != "view" || + !hasSubject(replacement.Subjects, "operator-group") { + t.Fatalf("concurrent operator creation was not preserved: %#v", replacement) + } + + mutations, deletes := kubectlOperationIndexes(readCommandCalls(t, harness.commandLogPath)) + if len(mutations) != 1 || len(deletes) != 0 { + t.Fatalf("concurrent create mutation/delete attempts = %d/%d, want 1/0", len(mutations), len(deletes)) + } +} + +func TestSetupNodeRBACChecksManagedPostconditions(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + configure func(*configScriptHarness) + wantSubstring string + }{ + { + name: "binding absent", + configure: func(harness *configScriptHarness) { + harness.skipManagedMutation = true + }, + wantSubstring: "absent after reconciliation", + }, + { + name: "wrong roleRef", + configure: func(harness *configScriptHarness) { + harness.managedPostcondition = bootstrapBindingName + ":wrong-role-ref" + }, + wantSubstring: "roleRef is immutable", + }, + { + name: "required subject missing", + configure: func(harness *configScriptHarness) { + harness.managedPostcondition = bootstrapBindingName + ":missing-subject" + }, + wantSubstring: "does not contain the required bootstrap group", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + test.configure(harness) + output, err := harness.runSetupNodeRBAC(false) + if err == nil { + t.Fatalf("setup-node-rbac succeeded despite a missing postcondition\n%s", output) + } + if !strings.Contains(output, test.wantSubstring) { + t.Fatalf("postcondition failure was not actionable:\n%s", output) + } + }) + } +} + +func TestSetupNodeRBACPreservesLegacyBindingWithoutExplicitMigration(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, 0) + output, err := harness.runSetupNodeRBAC(false) + if err == nil { + t.Fatalf("setup-node-rbac succeeded without explicit migration while legacy binding exists\n%s", output) + } + if !strings.Contains(output, "--remove-legacy-node-role-binding") || + !strings.Contains(output, "v0.1.1") || + !strings.Contains(output, "/etc/aks-flex-node/daemon-credentials/daemon-controller-current.pem") { + t.Fatalf("setup-node-rbac did not explain the compatible migration path:\n%s", output) + } + + state, readErr := os.ReadFile(harness.legacyState) + if readErr != nil { + t.Fatalf("read fake legacy state: %v", readErr) + } + if got := strings.TrimSpace(string(state)); got != "present" { + t.Fatalf("legacy binding state = %q, want present until migration is acknowledged", got) + } + + calls := readCommandCalls(t, harness.commandLogPath) + applyIndexes, deleteIndexes := kubectlOperationIndexes(calls) + if len(applyIndexes) != 2 || len(deleteIndexes) != 0 { + t.Fatalf("kubectl mutation/delete counts = %d/%d, want 2/0; calls: %s", len(applyIndexes), len(deleteIndexes), formatCalls(calls)) + } + if getIndexes := kubectlIndexes(calls, "get"); len(getIndexes) != 2 || getIndexes[0] >= applyIndexes[0] || applyIndexes[1] >= getIndexes[1] { + t.Fatalf("safe RBAC must be preflighted and verified before checking legacy migration: %s", formatCalls(calls)) + } + for _, index := range applyIndexes { + assertSafeManagedRBACMutation(t, calls[index]) + } +} + +func TestSetupNodeRBACMigratesLegacyBindingIdempotently(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, 0) + for run := 1; run <= 2; run++ { + output, err := harness.runSetupNodeRBAC(true) + if err != nil { + t.Fatalf("setup-node-rbac run %d failed: %v\n%s", run, err, output) + } + } + + state, err := os.ReadFile(harness.legacyState) + if err != nil { + t.Fatalf("read fake legacy state: %v", err) + } + if got := strings.TrimSpace(string(state)); got != "absent" { + t.Fatalf("legacy binding state = %q, want absent", got) + } + + calls := readCommandCalls(t, harness.commandLogPath) + applyIndexes, deleteIndexes := kubectlOperationIndexes(calls) + if len(applyIndexes) != 2 || len(deleteIndexes) != 1 { + t.Fatalf("kubectl apply/delete counts = %d/%d, want 2/1; calls: %s", len(applyIndexes), len(deleteIndexes), formatCalls(calls)) + } + proxyIndexes := kubectlIndexes(calls, "proxy") + if len(proxyIndexes) != 1 { + t.Fatalf("conditional deletion started %d kubectl proxies, want 1: %s", len(proxyIndexes), formatCalls(calls)) + } + unixSocketArgs := 0 + for _, arg := range calls[proxyIndexes[0]].args { + if strings.HasPrefix(arg, "--unix-socket=") { + unixSocketArgs++ + } + if strings.HasPrefix(arg, "--port=") || strings.HasPrefix(arg, "--address=") { + t.Errorf("conditional deletion exposed a TCP listener: %q", arg) + } + } + if unixSocketArgs != 1 { + t.Fatalf("conditional deletion must use one private Unix socket: %s", formatCalls(calls)) + } + if applyIndexes[0] >= deleteIndexes[0] { + t.Errorf("legacy binding was deleted before safe RBAC was applied: %s", formatCalls(calls)) + } + assertLegacyDeleteCall(t, calls[deleteIndexes[0]]) + assertLegacyDeletePreconditions(t, harness.deleteOptsPath, "legacy-uid", "7") + if getIndexes := kubectlIndexes(calls, "get"); len(getIndexes) != 5 || getIndexes[1] >= deleteIndexes[0] || deleteIndexes[0] >= getIndexes[2] { + t.Fatalf("migration must inventory before and verify after deletion, then stay idempotent: %s", formatCalls(calls)) + } +} + +func TestSetupNodeRBACRejectsConcurrentLegacyBindingReplacement(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, 0) + harness.concurrentSwap = true + output, err := harness.runSetupNodeRBAC(true) + if err == nil { + t.Fatalf("setup-node-rbac deleted a concurrently replaced binding\n%s", output) + } + + state, readErr := os.ReadFile(harness.legacyState) + if readErr != nil { + t.Fatalf("read fake legacy state: %v", readErr) + } + if got := strings.TrimSpace(string(state)); got != "customized" { + t.Fatalf("legacy binding state = %q, want concurrently replaced object preserved", got) + } + assertLegacyDeletePreconditions(t, harness.deleteOptsPath, "legacy-uid", "7") +} + +func TestSetupNodeRBACFailsWhenLegacyBindingDeleteFails(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, fakeDeleteFailure) + output, err := harness.runSetupNodeRBAC(true) + if err == nil { + t.Fatalf("setup-node-rbac succeeded when legacy binding deletion failed\n%s", output) + } + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + t.Fatalf("setup-node-rbac error = %T %v, want *exec.ExitError", err, err) + } + if got := exitErr.ExitCode(); got == 0 { + t.Fatalf("setup-node-rbac exit code = %d, want nonzero\n%s", got, output) + } + if !strings.Contains(output, "HTTP 500") { + t.Fatalf("setup-node-rbac did not report the API deletion failure:\n%s", output) + } + + state, readErr := os.ReadFile(harness.legacyState) + if readErr != nil { + t.Fatalf("read fake legacy state: %v", readErr) + } + if got := strings.TrimSpace(string(state)); got != "present" { + t.Fatalf("legacy binding state = %q after failed deletion, want present", got) + } + + calls := readCommandCalls(t, harness.commandLogPath) + applyIndexes, deleteIndexes := kubectlOperationIndexes(calls) + if len(applyIndexes) != 2 || len(deleteIndexes) != 1 { + t.Fatalf("kubectl mutation/delete counts = %d/%d, want 2/1; calls: %s", len(applyIndexes), len(deleteIndexes), formatCalls(calls)) + } + if applyIndexes[0] >= deleteIndexes[0] { + t.Errorf("legacy delete failure occurred before safe RBAC was applied; calls: %s", formatCalls(calls)) + } + assertLegacyDeleteCall(t, calls[deleteIndexes[0]]) +} + +func TestSetupNodeRBACFailsWhenLegacyBindingRemainsAfterDelete(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, 0) + harness.deleteKeeps = true + output, err := harness.runSetupNodeRBAC(true) + if err == nil { + t.Fatalf("setup-node-rbac succeeded while unsafe binding remained\n%s", output) + } + if !strings.Contains(output, "remains after deletion") { + t.Fatalf("failure did not report the failed postcondition:\n%s", output) + } + + calls := readCommandCalls(t, harness.commandLogPath) + _, deleteIndexes := kubectlOperationIndexes(calls) + if len(deleteIndexes) != 1 || len(kubectlIndexes(calls, "get")) != 3 { + t.Fatalf("migration did not inventory, delete, and verify: %s", formatCalls(calls)) + } +} + +func TestSetupNodeRBACReportsKubectlProxyStartupFailure(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, 0) + harness.proxyStartup = "exit-before-ready" + output, err := harness.runSetupNodeRBAC(true) + if err == nil { + t.Fatalf("setup-node-rbac succeeded when kubectl proxy exited before readiness\n%s", output) + } + if !strings.Contains(output, "injected proxy startup failure") { + t.Fatalf("setup-node-rbac did not surface kubectl proxy's startup error:\n%s", output) + } + if got := strings.TrimSpace(readFile(t, harness.legacyState)); got != "present" { + t.Fatalf("legacy binding state = %q after proxy startup failure, want present", got) + } +} + +func TestSetupNodeRBACHandlesFragmentedKubectlProxyReadiness(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, 0) + harness.proxyStartup = "fragmented-ready" + output, err := harness.runSetupNodeRBAC(true) + if err != nil { + t.Fatalf("setup-node-rbac did not handle fragmented kubectl proxy output: %v\n%s", err, output) + } + if got := strings.TrimSpace(readFile(t, harness.legacyState)); got != "absent" { + t.Fatalf("legacy binding state = %q after fragmented readiness output, want absent", got) + } +} + +func TestSetupNodeRBACBypassesEnvironmentProxyForUnixSocketDeletion(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, 0) + harness.poisonHTTPProxy = true + output, err := harness.runSetupNodeRBAC(true) + if err != nil { + t.Fatalf("setup-node-rbac sent its Unix-socket deletion through the environment proxy: %v\n%s", err, output) + } + if got := strings.TrimSpace(readFile(t, harness.legacyState)); got != "absent" { + t.Fatalf("legacy binding state = %q after Unix-socket deletion, want absent", got) + } +} + +func TestSetupNodeRBACRefusesAmbiguousUnsafeBindings(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + state string + }{ + {name: "canonical name with additional subject", state: "customized"}, + {name: "unexpected binding name", state: "renamed"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + if err := os.WriteFile(harness.legacyState, []byte(test.state+"\n"), 0o600); err != nil { + t.Fatalf("write fake legacy state: %v", err) + } + output, err := harness.runSetupNodeRBAC(true) + if err == nil { + t.Fatalf("setup-node-rbac removed an ambiguous binding\n%s", output) + } + if !strings.Contains(output, "refusing automatic removal") || !strings.Contains(output, "manually") { + t.Fatalf("failure did not explain manual remediation:\n%s", output) + } + + calls := readCommandCalls(t, harness.commandLogPath) + _, deleteIndexes := kubectlOperationIndexes(calls) + if len(deleteIndexes) != 0 { + t.Fatalf("ambiguous binding was deleted: %s", formatCalls(calls)) + } + }) + } +} + +func TestSetupNodeRBACPreservesSameNamedNonLegacyBinding(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + if err := os.WriteFile(harness.legacyState, []byte("safe-customized\n"), 0o600); err != nil { + t.Fatalf("write fake legacy state: %v", err) + } + output, err := harness.runSetupNodeRBAC(true) + if err != nil { + t.Fatalf("setup-node-rbac rejected a non-legacy same-named binding: %v\n%s", err, output) + } + + state, readErr := os.ReadFile(harness.legacyState) + if readErr != nil { + t.Fatalf("read fake legacy state: %v", readErr) + } + if got := strings.TrimSpace(string(state)); got != "safe-customized" { + t.Fatalf("same-named non-legacy binding state = %q, want preserved", got) + } + if _, deleteIndexes := kubectlOperationIndexes(readCommandCalls(t, harness.commandLogPath)); len(deleteIndexes) != 0 { + t.Fatal("same-named non-legacy binding was deleted") + } +} + +func TestGenerateBootstrapTokenRequiresCompletedLegacyMigration(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + legacyPresent bool + getExitCode int + wantExitCode int + }{ + {name: "migration already complete"}, + {name: "legacy binding present", legacyPresent: true, wantExitCode: 1}, + {name: "legacy state cannot be read", getExitCode: fakeGetFailure, wantExitCode: fakeGetFailure}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, test.legacyPresent, 0) + harness.getExitCode = test.getExitCode + output, err := harness.runGenerateNodeConfig() + if test.wantExitCode != 0 { + if err == nil { + t.Fatalf("generate-node-config succeeded before legacy migration completed\n%s", output) + } + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != test.wantExitCode { + t.Fatalf("generate-node-config error = %v, want exit code %d\n%s", err, test.wantExitCode, output) + } + } else if err != nil { + t.Fatalf("generate-node-config failed: %v\n%s", err, output) + } + + calls := readCommandCalls(t, harness.commandLogPath) + applyIndexes, deleteIndexes := kubectlOperationIndexes(calls) + getIndexes := kubectlIndexes(calls, "get") + if len(getIndexes) != 1 || len(deleteIndexes) != 0 { + t.Fatalf("kubectl get/delete counts = %d/%d, want 1/0; calls: %s", len(getIndexes), len(deleteIndexes), formatCalls(calls)) + } + if test.wantExitCode != 0 { + if len(applyIndexes) != 0 { + t.Fatalf("token Secret was applied before legacy migration completed; calls: %s", formatCalls(calls)) + } + if _, statErr := os.Stat(harness.manifestPath); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("token manifest exists after cleanup failure: %v", statErr) + } + if test.legacyPresent && !strings.Contains(output, "--remove-legacy-node-role-binding") { + t.Fatalf("failure did not explain explicit migration path:\n%s", output) + } + return + } + + if len(applyIndexes) != 1 || getIndexes[0] >= applyIndexes[0] { + t.Fatalf("legacy-state check must precede the single token apply; calls: %s", formatCalls(calls)) + } + manifest, readErr := os.ReadFile(harness.manifestPath) + if readErr != nil { + t.Fatalf("read token manifest: %v", readErr) + } + if !strings.Contains(string(manifest), "kind: Secret") || !strings.Contains(string(manifest), "type: bootstrap.kubernetes.io/token") { + t.Fatalf("applied manifest is not a bootstrap token Secret:\n%s", manifest) + } + }) + } +} + +func TestKubeadmRBACReconciliation(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + applyFailAt int + wantApplies int + wantFailure bool + }{ + {name: "succeeds", wantApplies: 2}, + {name: "initial RBAC apply fails", applyFailAt: 1, wantApplies: 1, wantFailure: true}, + {name: "ConfigMap apply fails", applyFailAt: 2, wantApplies: 2, wantFailure: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, 0) + harness.applyFailAt = test.applyFailAt + output, err := harness.runKubeadmEnsureRBAC() + if test.wantFailure && err == nil { + t.Fatalf("kubeadm RBAC reconciliation succeeded despite injected failure\n%s", output) + } + if test.wantFailure { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 { + t.Fatalf("kubeadm RBAC reconciliation error = %v, want exit code 1\n%s", err, output) + } + } else if err != nil { + t.Fatalf("kubeadm RBAC reconciliation failed: %v\n%s", err, output) + } + + calls := readCommandCalls(t, harness.commandLogPath) + applyIndexes, deleteIndexes := kubectlOperationIndexes(calls) + if len(applyIndexes) != test.wantApplies || len(deleteIndexes) != 0 { + t.Fatalf( + "kubectl apply/delete counts = %d/%d, want %d/%d; calls: %s", + len(applyIndexes), len(deleteIndexes), test.wantApplies, 0, formatCalls(calls), + ) + } + }) + } +} + +func TestRepositoryDoesNotBindBootstrapGroupsToLegacyNodeRole(t *testing.T) { + t.Parallel() + + workingDir, err := os.Getwd() + if err != nil { + t.Fatalf("get working directory: %v", err) + } + repositoryRoot := filepath.Dir(workingDir) + sourceRoots := []string{"cmd", "hack", "pkg", "scripts"} + bindingPattern := regexp.MustCompile(`(?m)^[ \t]*kind:[ \t]*ClusterRoleBinding[ \t]*$`) + legacyRolePattern := regexp.MustCompile(`(?m)^[ \t]*name:[ \t]*system:node[ \t]*$`) + bootstrapGroupPattern := regexp.MustCompile(`(?m)^[ \t]*name:[ \t]*system:bootstrappers:[^ \t\n]+[ \t]*$`) + documentSeparator := regexp.MustCompile(`(?m)^[ \t]*---[ \t]*$`) + + for _, sourceRoot := range sourceRoots { + root := filepath.Join(repositoryRoot, sourceRoot) + walkErr := filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + return nil + } + extension := filepath.Ext(path) + if entry.Name() != "aks-flex-config" && extension != ".go" && extension != ".py" && extension != ".sh" && extension != ".yaml" && extension != ".yml" { + return nil + } + contents, readErr := os.ReadFile(path) + if readErr != nil { + return fmt.Errorf("read %s: %w", path, readErr) + } + normalized := strings.ReplaceAll(string(contents), "\r\n", "\n") + for documentIndex, document := range documentSeparator.Split(normalized, -1) { + if bindingPattern.MatchString(document) && legacyRolePattern.MatchString(document) && bootstrapGroupPattern.MatchString(document) { + relativePath, relErr := filepath.Rel(repositoryRoot, path) + if relErr != nil { + relativePath = path + } + t.Errorf("%s YAML document %d binds a bootstrap group to the broad legacy %q role", relativePath, documentIndex+1, legacyNodeRole) + } + } + return nil + }) + if walkErr != nil { + t.Fatalf("scan %s for unsafe bootstrap RBAC: %v", sourceRoot, walkErr) + } + } +} + +func newConfigScriptHarness(t *testing.T, legacyPresent bool, deleteExitCode int) *configScriptHarness { + t.Helper() + + pythonPath, err := exec.LookPath("python3") + if err != nil { + t.Fatal("python3 is required to test scripts/aks-flex-config") + } + workingDir, err := os.Getwd() + if err != nil { + t.Fatalf("get working directory: %v", err) + } + scriptPath := filepath.Join(workingDir, "aks-flex-config") + if _, err := os.Stat(scriptPath); err != nil { + t.Fatalf("stat aks-flex-config: %v", err) + } + + tempDir := t.TempDir() + fakeBinDir := filepath.Join(tempDir, "bin") + if err := os.Mkdir(fakeBinDir, 0o700); err != nil { + t.Fatalf("create fake bin directory: %v", err) + } + writeExecutable(t, filepath.Join(fakeBinDir, "az"), fakeAZScript) + writeExecutable(t, filepath.Join(fakeBinDir, "kubectl"), fakeKubectlScript) + + legacyState := filepath.Join(tempDir, "legacy-state") + state := "absent\n" + if legacyPresent { + state = "present\n" + } + if err := os.WriteFile(legacyState, []byte(state), 0o600); err != nil { + t.Fatalf("write fake legacy state: %v", err) + } + managedState := filepath.Join(tempDir, "managed-state.json") + if err := os.WriteFile(managedState, []byte("[]\n"), 0o600); err != nil { + t.Fatalf("write fake managed binding state: %v", err) + } + + return &configScriptHarness{ + pythonPath: pythonPath, + scriptPath: scriptPath, + fakeBinDir: fakeBinDir, + commandLogPath: filepath.Join(tempDir, "commands.log"), + manifestPath: filepath.Join(tempDir, "rbac.yaml"), + managedState: managedState, + deleteOptsPath: filepath.Join(tempDir, "delete-options.json"), + configPath: filepath.Join(tempDir, "config.json"), + legacyState: legacyState, + deleteExitCode: deleteExitCode, + applyCountPath: filepath.Join(tempDir, "apply-count"), + } +} + +func (h *configScriptHarness) runSetupNodeRBAC(removeLegacy bool) (string, error) { + args := []string{ + h.scriptPath, + "setup-node-rbac", + "--resource-group", "test-rg", + "--cluster-name", "test-cluster", + "--subscription", "test-subscription", + } + if removeLegacy { + args = append(args, "--remove-legacy-node-role-binding") + } + cmd := exec.Command(h.pythonPath, args...) + cmd.Env = append(os.Environ(), + "PATH="+h.fakeBinDir+string(os.PathListSeparator)+os.Getenv("PATH"), + "KUBECONFIG="+filepath.Join(filepath.Dir(h.fakeBinDir), "kubeconfig"), + "PYTHONDONTWRITEBYTECODE=1", + fakeCommandLogEnv+"="+h.commandLogPath, + fakeManifestEnv+"="+h.manifestPath, + fakeManagedStateEnv+"="+h.managedState, + fakeDeleteOptsEnv+"="+h.deleteOptsPath, + fakeLegacyStateEnv+"="+h.legacyState, + fmt.Sprintf("%s=%d", fakeDeleteExitEnv, h.deleteExitCode), + fmt.Sprintf("%s=%t", fakeDeleteKeepsEnv, h.deleteKeeps), + fmt.Sprintf("%s=%t", fakeConcurrentEnv, h.concurrentSwap), + fakeProxyStartupEnv+"="+h.proxyStartup, + fakeConcurrentManagedEnv+"="+h.concurrentManaged, + fakeManagedPostconditionEnv+"="+h.managedPostcondition, + fmt.Sprintf("%s=%t", fakeSkipManagedMutationEnv, h.skipManagedMutation), + fmt.Sprintf("%s=%d", fakeGetExitEnv, h.getExitCode), + fakeApplyCountEnv+"="+h.applyCountPath, + fmt.Sprintf("%s=%d", fakeApplyFailAtEnv, h.applyFailAt), + ) + if h.poisonHTTPProxy { + cmd.Env = append(cmd.Env, + "HTTP_PROXY=http://127.0.0.1:1", + "http_proxy=http://127.0.0.1:1", + "HTTPS_PROXY=http://127.0.0.1:1", + "https_proxy=http://127.0.0.1:1", + "ALL_PROXY=http://127.0.0.1:1", + "all_proxy=http://127.0.0.1:1", + "NO_PROXY=", + "no_proxy=", + ) + } + output, err := cmd.CombinedOutput() + return string(output), err +} + +func (h *configScriptHarness) runGenerateNodeConfig() (string, error) { + cmd := exec.Command( + h.pythonPath, + h.scriptPath, + "generate-node-config", + "--resource-group", "test-rg", + "--cluster-name", "test-cluster", + "--subscription", "test-subscription", + "--bootstrap-token", + "--output", h.configPath, + ) + cmd.Env = append(os.Environ(), + "PATH="+h.fakeBinDir+string(os.PathListSeparator)+os.Getenv("PATH"), + "KUBECONFIG="+filepath.Join(filepath.Dir(h.fakeBinDir), "kubeconfig"), + "PYTHONDONTWRITEBYTECODE=1", + fakeCommandLogEnv+"="+h.commandLogPath, + fakeManifestEnv+"="+h.manifestPath, + fakeManagedStateEnv+"="+h.managedState, + fakeDeleteOptsEnv+"="+h.deleteOptsPath, + fakeLegacyStateEnv+"="+h.legacyState, + fmt.Sprintf("%s=%d", fakeDeleteExitEnv, h.deleteExitCode), + fmt.Sprintf("%s=%t", fakeDeleteKeepsEnv, h.deleteKeeps), + fmt.Sprintf("%s=%t", fakeConcurrentEnv, h.concurrentSwap), + fakeConcurrentManagedEnv+"="+h.concurrentManaged, + fmt.Sprintf("%s=%t", fakeSkipManagedMutationEnv, h.skipManagedMutation), + fmt.Sprintf("%s=%d", fakeGetExitEnv, h.getExitCode), + fakeApplyCountEnv+"="+h.applyCountPath, + fmt.Sprintf("%s=%d", fakeApplyFailAtEnv, h.applyFailAt), + ) + output, err := cmd.CombinedOutput() + return string(output), err +} + +func (h *configScriptHarness) runKubeadmEnsureRBAC() (string, error) { + workingDir, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("get working directory: %w", err) + } + kubeadmScript := filepath.Join(filepath.Dir(workingDir), "hack", "e2e", "lib", "node-join-kubeadm.sh") + cmd := exec.Command( + "bash", + "-c", + `source "$1"; with_cluster_lock _kubeadm_ensure_rbac "https://test-cluster.example.test:443" "dGVzdC1jYQ=="`, + "bash", + kubeadmScript, + ) + cmd.Env = append(os.Environ(), + "PATH="+h.fakeBinDir+string(os.PathListSeparator)+os.Getenv("PATH"), + "E2E_WORK_DIR="+filepath.Join(filepath.Dir(h.fakeBinDir), "e2e-work"), + "E2E_KUBERNETES_VERSION=1.35.0", + fakeCommandLogEnv+"="+h.commandLogPath, + fakeManifestEnv+"="+h.manifestPath, + fakeManagedStateEnv+"="+h.managedState, + fakeDeleteOptsEnv+"="+h.deleteOptsPath, + fakeLegacyStateEnv+"="+h.legacyState, + fmt.Sprintf("%s=%d", fakeDeleteExitEnv, h.deleteExitCode), + fmt.Sprintf("%s=%t", fakeDeleteKeepsEnv, h.deleteKeeps), + fmt.Sprintf("%s=%t", fakeConcurrentEnv, h.concurrentSwap), + fakeConcurrentManagedEnv+"="+h.concurrentManaged, + fmt.Sprintf("%s=%t", fakeSkipManagedMutationEnv, h.skipManagedMutation), + fmt.Sprintf("%s=%d", fakeGetExitEnv, h.getExitCode), + fakeApplyCountEnv+"="+h.applyCountPath, + fmt.Sprintf("%s=%d", fakeApplyFailAtEnv, h.applyFailAt), + ) + output, err := cmd.CombinedOutput() + return string(output), err +} + +func managedBindingFixture(name, role string, includeBootstrap bool) rbacv1.ClusterRoleBinding { + subjects := []rbacv1.Subject{} + if includeBootstrap { + subjects = append(subjects, rbacv1.Subject{ + APIGroup: rbacv1.GroupName, + Kind: "Group", + Name: bootstrapGroup, + }) + } + return rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rbacv1.SchemeGroupVersion.String(), + Kind: "ClusterRoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(name + "-uid"), + ResourceVersion: "7", + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: role, + }, + Subjects: subjects, + } +} + +func writeManagedState(t *testing.T, path string, bindings ...rbacv1.ClusterRoleBinding) { + t.Helper() + data, err := json.Marshal(bindings) + if err != nil { + t.Fatalf("marshal managed binding state: %v", err) + } + if err := os.WriteFile(path, append(data, '\n'), 0o600); err != nil { + t.Fatalf("write managed binding state: %v", err) + } +} + +func readManagedState(t *testing.T, path string) []rbacv1.ClusterRoleBinding { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read managed binding state: %v", err) + } + var bindings []rbacv1.ClusterRoleBinding + if err := json.Unmarshal(data, &bindings); err != nil { + t.Fatalf("decode managed binding state: %v", err) + } + return bindings +} + +func findManagedBinding(t *testing.T, bindings []rbacv1.ClusterRoleBinding, name string) rbacv1.ClusterRoleBinding { + t.Helper() + for _, binding := range bindings { + if binding.Name == name { + return binding + } + } + t.Fatalf("managed binding %q not found", name) + return rbacv1.ClusterRoleBinding{} +} + +func hasSubject(subjects []rbacv1.Subject, name string) bool { + for _, subject := range subjects { + if subject.APIGroup == rbacv1.GroupName && subject.Kind == "Group" && subject.Name == name { + return true + } + } + return false +} + +func readFile(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(data) +} + +func readRBACManifest(t *testing.T, path string) []rbacv1.ClusterRoleBinding { + t.Helper() + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read captured RBAC manifest: %v", err) + } + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) + var bindings []rbacv1.ClusterRoleBinding + for { + var binding rbacv1.ClusterRoleBinding + err := decoder.Decode(&binding) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("decode captured RBAC manifest: %v", err) + } + if binding.APIVersion == "" && binding.Kind == "" && binding.Name == "" { + continue + } + bindings = append(bindings, binding) + } + return bindings +} + +func bindingNames(bindings []rbacv1.ClusterRoleBinding) []string { + names := make([]string, 0, len(bindings)) + for _, binding := range bindings { + names = append(names, binding.Name) + } + return names +} + +func readCommandCalls(t *testing.T, path string) []commandCall { + t.Helper() + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read fake command log: %v", err) + } + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + calls := make([]commandCall, 0, len(lines)) + for _, line := range lines { + if line == "" { + continue + } + fields := strings.Split(line, "\t") + calls = append(calls, commandCall{name: fields[0], args: fields[1:]}) + } + return calls +} + +func kubectlOperationIndexes(calls []commandCall) (apply []int, delete []int) { + for i, call := range calls { + if call.name == "http-delete" { + delete = append(delete, i) + continue + } + if call.name != "kubectl" || len(call.args) == 0 { + continue + } + switch call.args[0] { + case "apply", "create", "replace": + apply = append(apply, i) + case "auth": + if len(call.args) > 1 && call.args[1] == "reconcile" { + apply = append(apply, i) + } + case "delete": + delete = append(delete, i) + } + } + return apply, delete +} + +func kubectlIndexes(calls []commandCall, operation string) []int { + var indexes []int + for i, call := range calls { + if call.name == "kubectl" && len(call.args) > 0 && call.args[0] == operation { + indexes = append(indexes, i) + } + } + return indexes +} + +func assertLegacyDeleteCall(t *testing.T, call commandCall) { + t.Helper() + + if call.name != "http-delete" || len(call.args) != 1 { + t.Fatalf("migration call = %#v, want conditional Kubernetes HTTP DELETE", call) + } + wantPath := "/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/" + legacyBindingName + if call.args[0] != wantPath { + t.Errorf("migration delete path = %q, want %q", call.args[0], wantPath) + } +} + +func slicesContain(args []string, want string) bool { + for _, arg := range args { + if arg == want { + return true + } + } + return false +} + +func assertLegacyDeletePreconditions(t *testing.T, path, wantUID, wantResourceVersion string) { + t.Helper() + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read delete options: %v", err) + } + var options struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Preconditions struct { + UID string `json:"uid"` + ResourceVersion string `json:"resourceVersion"` + } `json:"preconditions"` + } + if err := json.Unmarshal(data, &options); err != nil { + t.Fatalf("decode delete options: %v", err) + } + if options.APIVersion != "v1" || options.Kind != "DeleteOptions" || + options.Preconditions.UID != wantUID || options.Preconditions.ResourceVersion != wantResourceVersion { + t.Fatalf("delete options = %#v, want UID %q and resourceVersion %q", options, wantUID, wantResourceVersion) + } +} + +func assertSafeManagedRBACMutation(t *testing.T, call commandCall) { + t.Helper() + if call.name != "kubectl" || len(call.args) != 3 || (call.args[0] != "create" && call.args[0] != "replace") { + t.Fatalf("RBAC mutation = %#v, want kubectl create/replace -f -", call) + } + if !slicesContain(call.args, "-f") || !slicesContain(call.args, "-") || slicesContain(call.args, "--force") { + t.Errorf("RBAC mutation must be non-destructive and read stdin: %q", call.args) + } +} + +func formatCalls(calls []commandCall) string { + formatted := make([]string, 0, len(calls)) + for _, call := range calls { + formatted = append(formatted, strings.Join(append([]string{call.name}, call.args...), " ")) + } + return strings.Join(formatted, "; ") +} + +func writeExecutable(t *testing.T, path, contents string) { + t.Helper() + if err := os.WriteFile(path, []byte(contents), 0o700); err != nil { + t.Fatalf("write fake executable %s: %v", path, err) + } +} + +const fakeAZScript = `#!/bin/sh +set -eu +{ + printf 'az' + for arg in "$@"; do + printf '\t%s' "$arg" + done + printf '\n' +} >> "${AKS_FLEX_CONFIG_TEST_COMMAND_LOG:?}" + +case " $* " in +*" account show "*" --query id "*) printf 'test-subscription\n' ;; +*" account show "*" --query tenantId "*) printf 'test-tenant\n' ;; +*" aks show "*" --query id "*) printf '/subscriptions/test/resourceGroups/test-rg/providers/Microsoft.ContainerService/managedClusters/test-cluster\n' ;; +*" aks show "*" --query location "*) printf 'test-region\n' ;; +*" currentKubernetesVersion "*) printf '1.35.0\n' ;; +*" networkProfile.dnsServiceIp "*) printf '10.0.0.10\n' ;; +esac +` + +const fakeKubectlScript = `#!/bin/sh +set -eu +{ + printf 'kubectl' + for arg in "$@"; do + printf '\t%s' "$arg" + done + printf '\n' +} >> "${AKS_FLEX_CONFIG_TEST_COMMAND_LOG:?}" + +case "${1:-}" in +proxy) + socket_path="" + for arg in "$@"; do + case "$arg" in + --unix-socket=*) socket_path="${arg#--unix-socket=}" ;; + --port=*|--address=*) exit 52 ;; + esac + done + if [ -z "$socket_path" ]; then + exit 50 + fi + if [ "${AKS_FLEX_CONFIG_TEST_PROXY_STARTUP:-}" = "exit-before-ready" ]; then + printf '%s\n' 'injected proxy startup failure' >&2 + exit 51 + fi + exec python3 -u - "$socket_path" "${AKS_FLEX_CONFIG_TEST_COMMAND_LOG:?}" "${AKS_FLEX_CONFIG_TEST_DELETE_OPTIONS:?}" "${AKS_FLEX_CONFIG_TEST_LEGACY_STATE:?}" "${AKS_FLEX_CONFIG_TEST_DELETE_EXIT:-0}" "${AKS_FLEX_CONFIG_TEST_DELETE_KEEPS_STATE:-false}" "${AKS_FLEX_CONFIG_TEST_CONCURRENT_REPLACE:-false}" "${AKS_FLEX_CONFIG_TEST_PROXY_STARTUP:-}" <<'PY' +import http.server +import json +import os +import socketserver +import stat +import sys +import time + +socket_path, log_path, options_path, state_path, delete_exit, keep_state, concurrent, startup_mode = sys.argv[1:] + +class Handler(http.server.BaseHTTPRequestHandler): + def do_DELETE(self): + socket_mode = stat.S_IMODE(os.stat(socket_path, follow_symlinks=False).st_mode) + directory_mode = stat.S_IMODE(os.stat(os.path.dirname(socket_path)).st_mode) + if socket_mode != 0o600 or directory_mode != 0o700: + self.respond(500, {"message": f"insecure socket modes {socket_mode:o}/{directory_mode:o}"}) + return + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) + with open(log_path, "a", encoding="utf-8") as stream: + stream.write("http-delete\t" + self.path + "\n") + with open(options_path, "wb") as stream: + stream.write(body) + + if int(delete_exit) != 0: + self.respond(500, {"message": "injected delete failure"}) + return + if concurrent == "true": + with open(state_path, "w", encoding="utf-8") as stream: + stream.write("customized\n") + self.respond(409, {"message": "object changed"}) + return + if self.path != "/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/aks-flex-node-role": + self.respond(404, {"message": "unexpected resource"}) + return + + try: + options = json.loads(body) + except json.JSONDecodeError: + self.respond(400, {"message": "missing DeleteOptions"}) + return + preconditions = options.get("preconditions", {}) + with open(state_path, encoding="utf-8") as stream: + state = stream.read().strip() + if state != "present": + self.respond(404, {"message": "not found"}) + return + if preconditions != {"uid": "legacy-uid", "resourceVersion": "7"}: + self.respond(409, {"message": "precondition failed"}) + return + if keep_state != "true": + with open(state_path, "w", encoding="utf-8") as stream: + stream.write("absent\n") + self.respond(200, {"status": "Success"}) + + def respond(self, status, payload): + encoded = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + def log_message(self, *_): + pass + +class UnixHTTPServer(socketserver.ThreadingMixIn, socketserver.UnixStreamServer): + daemon_threads = True + +os.umask(0o077) +server = UnixHTTPServer(socket_path, Handler) +sys.stderr.write("fake proxy diagnostic before readiness\n") +sys.stderr.flush() +readiness = f"Starting to serve on {socket_path}\n" +if startup_mode == "fragmented-ready": + readiness_prefix = "Starting to " + for fragment in ( + "fake stdout warning before readiness\n" + readiness_prefix, + readiness[len(readiness_prefix):-1], + "\n", + ): + sys.stdout.write(fragment) + sys.stdout.flush() + time.sleep(0.01) +else: + # Keep both lines in one write to catch implementations that mix an OS + # selector with a buffered text reader and strand the readiness line. + sys.stdout.write("fake stdout warning before readiness\n" + readiness) + sys.stdout.flush() +server.serve_forever() +PY + ;; +create|replace) + apply_count=0 + if [ -f "${AKS_FLEX_CONFIG_TEST_APPLY_COUNT:?}" ]; then + apply_count=$(cat "$AKS_FLEX_CONFIG_TEST_APPLY_COUNT") + fi + apply_count=$((apply_count + 1)) + printf '%s\n' "$apply_count" > "$AKS_FLEX_CONFIG_TEST_APPLY_COUNT" + if [ "${AKS_FLEX_CONFIG_TEST_APPLY_FAIL_AT:-0}" -eq "$apply_count" ]; then + exit 38 + fi + + input="${AKS_FLEX_CONFIG_TEST_MANIFEST:?}.input.$$" + cat > "$input" + if [ -s "${AKS_FLEX_CONFIG_TEST_MANIFEST}" ]; then + printf '%s\n' '---' >> "${AKS_FLEX_CONFIG_TEST_MANIFEST}" + fi + cat "$input" >> "${AKS_FLEX_CONFIG_TEST_MANIFEST}" + printf '\n' >> "${AKS_FLEX_CONFIG_TEST_MANIFEST}" + + status=0 + python3 - "$1" "$input" "${AKS_FLEX_CONFIG_TEST_MANAGED_STATE:?}" "${AKS_FLEX_CONFIG_TEST_CONCURRENT_MANAGED_REPLACE:-}" "${AKS_FLEX_CONFIG_TEST_SKIP_MANAGED_MUTATION:-false}" "${AKS_FLEX_CONFIG_TEST_MANAGED_POSTCONDITION:-}" <<'PY' || status=$? +import json +import sys + +operation, input_path, state_path, concurrent_name, skip_mutation, postcondition = sys.argv[1:] +with open(input_path, encoding="utf-8") as stream: + incoming = json.load(stream) +with open(state_path, encoding="utf-8") as stream: + items = json.load(stream) + +name = incoming.get("metadata", {}).get("name") +matches = [index for index, item in enumerate(items) if item.get("metadata", {}).get("name") == name] +if operation == "create": + if matches: + raise SystemExit(48) + if concurrent_name == name: + replacement = { + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": { + "name": name, + "resourceVersion": "concurrent", + "uid": "operator-replacement", + "labels": {"owner": "operator"}, + }, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "view", + }, + "subjects": [ + { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Group", + "name": "operator-group", + } + ], + } + items.append(replacement) + with open(state_path, "w", encoding="utf-8") as stream: + json.dump(items, stream) + raise SystemExit(48) + incoming.setdefault("metadata", {})["resourceVersion"] = "1" + incoming["metadata"]["uid"] = f"{name}-uid" + if skip_mutation != "true": + items.append(incoming) +elif operation == "replace": + if len(matches) != 1: + raise SystemExit(49) + index = matches[0] + current = items[index] + if concurrent_name == name: + replacement = { + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": { + "name": name, + "resourceVersion": "concurrent", + "uid": "operator-replacement", + "labels": {"owner": "operator"}, + }, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "view", + }, + "subjects": [ + { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Group", + "name": "operator-group", + } + ], + } + items[index] = replacement + with open(state_path, "w", encoding="utf-8") as stream: + json.dump(items, stream) + raise SystemExit(41) + if incoming.get("metadata", {}).get("resourceVersion") != current.get("metadata", {}).get("resourceVersion"): + raise SystemExit(41) + incoming["metadata"]["resourceVersion"] = str(int(current["metadata"]["resourceVersion"]) + 1) + if skip_mutation != "true": + items[index] = incoming +else: + raise SystemExit(47) + +if skip_mutation != "true" and postcondition: + postcondition_name, separator, mutation = postcondition.partition(":") + if not separator: + raise SystemExit(50) + if postcondition_name == name: + target = next(item for item in items if item.get("metadata", {}).get("name") == name) + if mutation == "wrong-role-ref": + target["roleRef"]["name"] = "view" + elif mutation == "missing-subject": + target["subjects"] = [ + subject for subject in target.get("subjects", []) + if subject.get("name") != "system:bootstrappers:aks-flex-node" + ] + else: + raise SystemExit(51) + +with open(state_path, "w", encoding="utf-8") as stream: + json.dump(items, stream) +PY + rm -f "$input" + exit "$status" + ;; +apply) + apply_count=0 + if [ -f "${AKS_FLEX_CONFIG_TEST_APPLY_COUNT:?}" ]; then + apply_count=$(cat "$AKS_FLEX_CONFIG_TEST_APPLY_COUNT") + fi + apply_count=$((apply_count + 1)) + printf '%s\n' "$apply_count" > "$AKS_FLEX_CONFIG_TEST_APPLY_COUNT" + if [ "${AKS_FLEX_CONFIG_TEST_APPLY_FAIL_AT:-0}" -eq "$apply_count" ]; then + exit 38 + fi + cat > "${AKS_FLEX_CONFIG_TEST_MANIFEST:?}" + ;; +get) + get_exit="${AKS_FLEX_CONFIG_TEST_GET_EXIT:-0}" + if [ "$get_exit" -ne 0 ]; then + exit "$get_exit" + fi + + case " $* " in + *" clusterrolebindings "*) ;; + *) exit 46 ;; + esac + + python3 - "${AKS_FLEX_CONFIG_TEST_MANAGED_STATE:?}" "${AKS_FLEX_CONFIG_TEST_LEGACY_STATE:?}" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as stream: + items = json.load(stream) +with open(sys.argv[2], encoding="utf-8") as stream: + state = stream.read().strip() + +subject = { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Group", + "name": "system:bootstrappers:aks-flex-node", +} +if state == "present": + items.append({ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": {"name": "aks-flex-node-role", "uid": "legacy-uid", "resourceVersion": "7"}, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "system:node", + }, + "subjects": [subject], + }) +elif state == "customized": + items.append({ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": {"name": "aks-flex-node-role", "uid": "replacement-uid", "resourceVersion": "8"}, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "system:node", + }, + "subjects": [subject, { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Group", + "name": "another-group", + }], + }) +elif state == "renamed": + items.append({ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": {"name": "custom-bootstrap-node-role", "uid": "renamed-uid", "resourceVersion": "9"}, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "system:node", + }, + "subjects": [subject], + }) +elif state == "safe-customized": + items.append({ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": {"name": "aks-flex-node-role"}, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "view", + }, + "subjects": [{ + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Group", + "name": "readers", + }], + }) + +print(json.dumps({"items": items})) +PY + ;; +config) + case " $* " in + *"certificate-authority-data"*) printf 'dGVzdC1jYQ==\n' ;; + *"cluster.server"*) printf 'https://test-cluster.example.test:443\n' ;; + *) exit 45 ;; + esac + ;; +*) + exit 42 + ;; +esac +`