From 08327fe0e807c928bb83dbaa5502753ac92c4785 Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Fri, 21 Aug 2026 22:50:39 +0000 Subject: [PATCH 01/18] Restrict bootstrap token RBAC --- docs/usages/aks-flex-config.md | 14 +- docs/usages/joining-nodes.md | 2 +- hack/e2e/lib/node-join-kubeadm.sh | 35 +- scripts/aks-flex-config | 38 +- scripts/aks_flex_config_test.go | 655 ++++++++++++++++++++++++++++++ 5 files changed, 708 insertions(+), 36 deletions(-) create mode 100644 scripts/aks_flex_config_test.go diff --git a/docs/usages/aks-flex-config.md b/docs/usages/aks-flex-config.md index 8448dfbd..151ac3b8 100644 --- a/docs/usages/aks-flex-config.md +++ b/docs/usages/aks-flex-config.md @@ -9,7 +9,7 @@ The helper does not install anything on the target host. It uses Azure CLI and, - Azure CLI authenticated to the subscription that contains the AKS cluster. - `python3` on the workstation. - `kubectl` on the workstation for `setup-node-rbac` and `--bootstrap-token` config generation. -- Permission to run `az aks get-credentials --admin` and create Kubernetes `ClusterRoleBinding` and bootstrap token `Secret` objects. +- Permission to run `az aks get-credentials --admin`, create Kubernetes `ClusterRoleBinding` and bootstrap token `Secret` objects, and remove the obsolete `aks-flex-node-role` binding when present. ## Save The Helper @@ -46,7 +46,17 @@ Run this once per cluster for bootstrap-token joins: --subscription "$SUBSCRIPTION_ID" ``` -This applies the bootstrap-related `ClusterRoleBinding` objects for the `system:bootstrappers:aks-flex-node` group. +This applies only the CSR creation and approval `ClusterRoleBinding` objects for the `system:bootstrappers:aks-flex-node` group. It also removes the obsolete `aks-flex-node-role` binding created by older versions of the helper; that binding granted bootstrap tokens the broad legacy `system:node` role. The command is safe to rerun, and existing clusters should run it once after updating the helper. + +Bootstrap-token config generation performs the same legacy-binding cleanup before creating a token. If cleanup fails, token generation stops rather than issuing a token into a cluster that may still grant it broad node permissions. + +To remove only the obsolete binding from an existing cluster, run: + +```bash +kubectl delete clusterrolebinding aks-flex-node-role --ignore-not-found=true +``` + +Removing this binding does not interrupt joined nodes: they authenticate with their issued certificates rather than the bootstrap group. New and in-progress joins retain the CSR permissions installed above. ## Generate Node Config diff --git a/docs/usages/joining-nodes.md b/docs/usages/joining-nodes.md index 427bde56..8d2be1a4 100644 --- a/docs/usages/joining-nodes.md +++ b/docs/usages/joining-nodes.md @@ -16,7 +16,7 @@ Bootstrap token mode is the recommended quickstart path. It uses Kubernetes TLS High-level flow: -1. Run [`scripts/aks-flex-config setup-node-rbac`](../../scripts/aks-flex-config) to setup required node bootstrap RBAC permissions. +1. Run [`scripts/aks-flex-config setup-node-rbac`](../../scripts/aks-flex-config) to set up the least-privilege node bootstrap RBAC permissions. Rerunning the command also removes the broad legacy `aks-flex-node-role` binding from clusters configured by older helper versions. 2. Run `scripts/aks-flex-config generate-node-config --bootstrap-token` to create a bootstrap token, fetch AKS cluster metadata, and render the host config. 3. Copy the generated config to `/etc/aks-flex-node/config.json` on the target host. 4. Run `aks-flex-node preflight --config /etc/aks-flex-node/config.json` to validate host, cluster, rootfs, and artifact prerequisites without mutating the node. diff --git a/hack/e2e/lib/node-join-kubeadm.sh b/hack/e2e/lib/node-join-kubeadm.sh index ef9addf2..edc75282 100644 --- a/hack/e2e/lib/node-join-kubeadm.sh +++ b/hack/e2e/lib/node-join-kubeadm.sh @@ -39,7 +39,7 @@ _kubeadm_ensure_rbac() { # - ClusterRole/ClusterRoleBinding for bootstrappers to GET nodes # - ConfigMaps: cluster-info (kube-public), kubeadm-config and # kubelet-config (kube-system) consumed by kubeadm join - kubectl apply -f - < None: @@ -67,11 +68,32 @@ def setup_node_rbac(args: argparse.Namespace) -> None: log_info("applying bootstrap token RBAC bindings") run(["kubectl", "apply", "-f", "-"], input_text=RBAC_MANIFEST) + remove_legacy_node_role_binding() + + +def remove_legacy_node_role_binding() -> None: + # Older helpers granted bootstrap credentials the broad legacy node role. + # Remove it after the least-privilege CSR bindings are safely in place. + log_info("removing legacy bootstrap node role binding") + run( + [ + "kubectl", + "delete", + "clusterrolebinding", + LEGACY_NODE_ROLE_BINDING, + "--ignore-not-found=true", + ] + ) def generate_bootstrap_token(args: argparse.Namespace) -> str: require_command("kubectl") + # Existing clusters may skip setup-node-rbac after updating this helper. + # Never mint another token while the legacy broad binding may still exist. + remove_legacy_node_role_binding() + log_info("creating bootstrap token") + token_id = secrets.token_hex(3) token_secret = secrets.token_hex(8) token = f"{token_id}.{token_secret}" @@ -177,7 +199,6 @@ def render_config(args: argparse.Namespace, mode: str, metadata: dict[str, str]) if mode == "bootstrap-token": load_admin_kubeconfig(args) - log_info("creating bootstrap token") token = generate_bootstrap_token(args) server_url = run( ["kubectl", "config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}"], @@ -258,7 +279,7 @@ def build_parser() -> argparse.ArgumentParser: subparser.add_argument("--cluster-name", required=True) subparser.add_argument("--subscription") - rbac = subparsers.add_parser("setup-node-rbac", help="Apply node bootstrap RBAC bindings.") + rbac = subparsers.add_parser("setup-node-rbac", help="Reconcile node bootstrap RBAC bindings.") add_cluster_args(rbac) rbac.set_defaults(func=setup_node_rbac) @@ -305,19 +326,6 @@ roleRef: kind: ClusterRole name: system:certificates.k8s.io:certificatesigningrequests:nodeclient subjects: -- apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:bootstrappers:aks-flex-node ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: aks-flex-node-role -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:node -subjects: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:bootstrappers:aks-flex-node diff --git a/scripts/aks_flex_config_test.go b/scripts/aks_flex_config_test.go new file mode 100644 index 00000000..ed007095 --- /dev/null +++ b/scripts/aks_flex_config_test.go @@ -0,0 +1,655 @@ +package scripts + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/util/yaml" +) + +const ( + bootstrapGroup = "system:bootstrappers:aks-flex-node" + legacyBindingName = "aks-flex-node-role" + legacyNodeRole = "system:node" + fakeDeleteFailure = 37 + fakeCommandLogEnv = "AKS_FLEX_CONFIG_TEST_COMMAND_LOG" + fakeManifestEnv = "AKS_FLEX_CONFIG_TEST_MANIFEST" + fakeLegacyStateEnv = "AKS_FLEX_CONFIG_TEST_LEGACY_STATE" + fakeDeleteExitEnv = "AKS_FLEX_CONFIG_TEST_DELETE_EXIT" + fakeApplyCountEnv = "AKS_FLEX_CONFIG_TEST_APPLY_COUNT" + fakeApplyFailAtEnv = "AKS_FLEX_CONFIG_TEST_APPLY_FAIL_AT" +) + +type commandCall struct { + name string + args []string +} + +type configScriptHarness struct { + pythonPath string + scriptPath string + fakeBinDir string + commandLogPath string + manifestPath string + configPath string + legacyState string + deleteExitCode int + applyCountPath string + applyFailAt int +} + +func TestSetupNodeRBACManifestUsesOnlyBootstrapPermissions(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + output, err := harness.runSetupNodeRBAC() + if err != nil { + t.Fatalf("setup-node-rbac failed: %v\n%s", err, output) + } + + bindings := readRBACManifest(t, harness.manifestPath) + expectedRoles := map[string]string{ + "aks-flex-node-bootstrapper": "system:node-bootstrapper", + "aks-flex-node-auto-approve-csr": "system:certificates.k8s.io:certificatesigningrequests:nodeclient", + } + if len(bindings) != len(expectedRoles) { + t.Fatalf("RBAC manifest has %d bindings, want %d: %v", len(bindings), len(expectedRoles), bindingNames(bindings)) + } + + seen := make(map[string]struct{}, len(bindings)) + for _, binding := range bindings { + if binding.APIVersion != "rbac.authorization.k8s.io/v1" || binding.Kind != "ClusterRoleBinding" { + t.Errorf("binding %q has apiVersion/kind %q/%q, want rbac.authorization.k8s.io/v1/ClusterRoleBinding", binding.Name, binding.APIVersion, binding.Kind) + } + if binding.Namespace != "" { + t.Errorf("binding %q unexpectedly has namespace %q", binding.Name, binding.Namespace) + } + if _, duplicate := seen[binding.Name]; duplicate { + t.Errorf("binding %q appears more than once", binding.Name) + } + seen[binding.Name] = struct{}{} + + wantRole, expected := expectedRoles[binding.Name] + if !expected { + t.Errorf("unexpected ClusterRoleBinding %q", binding.Name) + continue + } + if binding.RoleRef.APIGroup != rbacv1.GroupName || binding.RoleRef.Kind != "ClusterRole" || binding.RoleRef.Name != wantRole { + t.Errorf("binding %q roleRef = %#v, want ClusterRole %q in %q", binding.Name, binding.RoleRef, wantRole, rbacv1.GroupName) + } + if len(binding.Subjects) != 1 { + t.Errorf("binding %q has %d subjects, want exactly one", binding.Name, len(binding.Subjects)) + continue + } + subject := binding.Subjects[0] + if subject.APIGroup != rbacv1.GroupName || subject.Kind != "Group" || subject.Name != bootstrapGroup || subject.Namespace != "" { + t.Errorf("binding %q subject = %#v, want bootstrapper group %q", binding.Name, subject, bootstrapGroup) + } + if binding.RoleRef.Name == legacyNodeRole { + t.Errorf("bootstrap group must not be bound to broad legacy role %q", legacyNodeRole) + } + } + + if _, found := seen[legacyBindingName]; found { + t.Errorf("RBAC manifest still contains legacy binding %q", legacyBindingName) + } +} + +func TestSetupNodeRBACMigratesLegacyBindingIdempotently(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, 0) + for run := 1; run <= 2; run++ { + output, err := harness.runSetupNodeRBAC() + if err != nil { + t.Fatalf("setup-node-rbac run %d failed: %v\n%s", run, err, output) + } + } + + state, err := os.ReadFile(harness.legacyState) + if err != nil { + t.Fatalf("read fake legacy state: %v", err) + } + if got := strings.TrimSpace(string(state)); got != "absent" { + t.Fatalf("legacy binding state = %q, want absent", got) + } + + calls := readCommandCalls(t, harness.commandLogPath) + applyIndexes, deleteIndexes := kubectlOperationIndexes(calls) + if len(applyIndexes) != 2 || len(deleteIndexes) != 2 { + t.Fatalf("kubectl apply/delete counts = %d/%d, want 2/2; calls: %s", len(applyIndexes), len(deleteIndexes), formatCalls(calls)) + } + for i := range applyIndexes { + if applyIndexes[i] >= deleteIndexes[i] { + t.Errorf("run %d deletes legacy binding before applying safe RBAC; calls: %s", i+1, formatCalls(calls)) + } + assertLegacyDeleteCall(t, calls[deleteIndexes[i]]) + } +} + +func TestSetupNodeRBACFailsWhenLegacyBindingDeleteFails(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, fakeDeleteFailure) + output, err := harness.runSetupNodeRBAC() + if err == nil { + t.Fatalf("setup-node-rbac succeeded when legacy binding deletion failed\n%s", output) + } + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + t.Fatalf("setup-node-rbac error = %T %v, want *exec.ExitError", err, err) + } + if got := exitErr.ExitCode(); got != fakeDeleteFailure { + t.Fatalf("setup-node-rbac exit code = %d, want %d\n%s", got, fakeDeleteFailure, output) + } + + state, readErr := os.ReadFile(harness.legacyState) + if readErr != nil { + t.Fatalf("read fake legacy state: %v", readErr) + } + if got := strings.TrimSpace(string(state)); got != "present" { + t.Fatalf("legacy binding state = %q after failed deletion, want present", got) + } + + calls := readCommandCalls(t, harness.commandLogPath) + applyIndexes, deleteIndexes := kubectlOperationIndexes(calls) + if len(applyIndexes) != 1 || len(deleteIndexes) != 1 { + t.Fatalf("kubectl apply/delete counts = %d/%d, want 1/1; calls: %s", len(applyIndexes), len(deleteIndexes), formatCalls(calls)) + } + if applyIndexes[0] >= deleteIndexes[0] { + t.Errorf("legacy delete failure occurred before safe RBAC was applied; calls: %s", formatCalls(calls)) + } + assertLegacyDeleteCall(t, calls[deleteIndexes[0]]) +} + +func TestGenerateBootstrapTokenCleansLegacyBindingBeforeMintingToken(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + deleteExitCode int + wantFailure bool + }{ + {name: "cleanup succeeds", deleteExitCode: 0}, + {name: "cleanup fails closed", deleteExitCode: fakeDeleteFailure, wantFailure: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, test.deleteExitCode) + output, err := harness.runGenerateNodeConfig() + if test.wantFailure { + if err == nil { + t.Fatalf("generate-node-config succeeded when legacy cleanup failed\n%s", output) + } + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != fakeDeleteFailure { + t.Fatalf("generate-node-config error = %v, want exit code %d\n%s", err, fakeDeleteFailure, output) + } + } else if err != nil { + t.Fatalf("generate-node-config failed: %v\n%s", err, output) + } + + calls := readCommandCalls(t, harness.commandLogPath) + applyIndexes, deleteIndexes := kubectlOperationIndexes(calls) + if len(deleteIndexes) != 1 { + t.Fatalf("kubectl delete count = %d, want 1; calls: %s", len(deleteIndexes), formatCalls(calls)) + } + assertLegacyDeleteCall(t, calls[deleteIndexes[0]]) + if test.wantFailure { + if len(applyIndexes) != 0 { + t.Fatalf("token Secret was applied after cleanup failure; calls: %s", formatCalls(calls)) + } + if _, statErr := os.Stat(harness.manifestPath); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("token manifest exists after cleanup failure: %v", statErr) + } + return + } + + if len(applyIndexes) != 1 || deleteIndexes[0] >= applyIndexes[0] { + t.Fatalf("cleanup must precede the single token apply; calls: %s", formatCalls(calls)) + } + manifest, readErr := os.ReadFile(harness.manifestPath) + if readErr != nil { + t.Fatalf("read token manifest: %v", readErr) + } + if !strings.Contains(string(manifest), "kind: Secret") || !strings.Contains(string(manifest), "type: bootstrap.kubernetes.io/token") { + t.Fatalf("applied manifest is not a bootstrap token Secret:\n%s", manifest) + } + }) + } +} + +func TestKubeadmRBACReconciliation(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + applyFailAt int + deleteExitCode int + wantApplies int + wantDeletes int + wantFailure bool + }{ + {name: "succeeds", wantApplies: 2, wantDeletes: 1}, + {name: "initial RBAC apply fails", applyFailAt: 1, wantApplies: 1, wantFailure: true}, + {name: "legacy cleanup fails", deleteExitCode: fakeDeleteFailure, wantApplies: 1, wantDeletes: 1, wantFailure: true}, + {name: "ConfigMap apply fails", applyFailAt: 2, wantApplies: 2, wantDeletes: 1, wantFailure: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, test.deleteExitCode) + harness.applyFailAt = test.applyFailAt + output, err := harness.runKubeadmEnsureRBAC() + if test.wantFailure && err == nil { + t.Fatalf("kubeadm RBAC reconciliation succeeded despite injected failure\n%s", output) + } + if test.wantFailure { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 { + t.Fatalf("kubeadm RBAC reconciliation error = %v, want exit code 1\n%s", err, output) + } + } else if err != nil { + t.Fatalf("kubeadm RBAC reconciliation failed: %v\n%s", err, output) + } + + calls := readCommandCalls(t, harness.commandLogPath) + applyIndexes, deleteIndexes := kubectlOperationIndexes(calls) + if len(applyIndexes) != test.wantApplies || len(deleteIndexes) != test.wantDeletes { + t.Fatalf( + "kubectl apply/delete counts = %d/%d, want %d/%d; calls: %s", + len(applyIndexes), len(deleteIndexes), test.wantApplies, test.wantDeletes, formatCalls(calls), + ) + } + if len(deleteIndexes) == 1 { + if applyIndexes[0] >= deleteIndexes[0] { + t.Errorf("legacy cleanup ran before safe RBAC apply; calls: %s", formatCalls(calls)) + } + assertLegacyDeleteCall(t, calls[deleteIndexes[0]]) + } + }) + } +} + +func TestRepositoryDoesNotBindBootstrapGroupsToLegacyNodeRole(t *testing.T) { + t.Parallel() + + workingDir, err := os.Getwd() + if err != nil { + t.Fatalf("get working directory: %v", err) + } + repositoryRoot := filepath.Dir(workingDir) + sourceRoots := []string{"cmd", "hack", "pkg", "scripts"} + bindingPattern := regexp.MustCompile(`(?m)^[ \t]*kind:[ \t]*ClusterRoleBinding[ \t]*$`) + legacyRolePattern := regexp.MustCompile(`(?m)^[ \t]*name:[ \t]*system:node[ \t]*$`) + bootstrapGroupPattern := regexp.MustCompile(`(?m)^[ \t]*name:[ \t]*system:bootstrappers:[^ \t\n]+[ \t]*$`) + documentSeparator := regexp.MustCompile(`(?m)^[ \t]*---[ \t]*$`) + + for _, sourceRoot := range sourceRoots { + root := filepath.Join(repositoryRoot, sourceRoot) + walkErr := filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + return nil + } + extension := filepath.Ext(path) + if entry.Name() != "aks-flex-config" && extension != ".go" && extension != ".py" && extension != ".sh" && extension != ".yaml" && extension != ".yml" { + return nil + } + contents, readErr := os.ReadFile(path) + if readErr != nil { + return fmt.Errorf("read %s: %w", path, readErr) + } + normalized := strings.ReplaceAll(string(contents), "\r\n", "\n") + for documentIndex, document := range documentSeparator.Split(normalized, -1) { + if bindingPattern.MatchString(document) && legacyRolePattern.MatchString(document) && bootstrapGroupPattern.MatchString(document) { + relativePath, relErr := filepath.Rel(repositoryRoot, path) + if relErr != nil { + relativePath = path + } + t.Errorf("%s YAML document %d binds a bootstrap group to the broad legacy %q role", relativePath, documentIndex+1, legacyNodeRole) + } + } + return nil + }) + if walkErr != nil { + t.Fatalf("scan %s for unsafe bootstrap RBAC: %v", sourceRoot, walkErr) + } + } +} + +func newConfigScriptHarness(t *testing.T, legacyPresent bool, deleteExitCode int) *configScriptHarness { + t.Helper() + + pythonPath, err := exec.LookPath("python3") + if err != nil { + t.Fatal("python3 is required to test scripts/aks-flex-config") + } + workingDir, err := os.Getwd() + if err != nil { + t.Fatalf("get working directory: %v", err) + } + scriptPath := filepath.Join(workingDir, "aks-flex-config") + if _, err := os.Stat(scriptPath); err != nil { + t.Fatalf("stat aks-flex-config: %v", err) + } + + tempDir := t.TempDir() + fakeBinDir := filepath.Join(tempDir, "bin") + if err := os.Mkdir(fakeBinDir, 0o700); err != nil { + t.Fatalf("create fake bin directory: %v", err) + } + writeExecutable(t, filepath.Join(fakeBinDir, "az"), fakeAZScript) + writeExecutable(t, filepath.Join(fakeBinDir, "kubectl"), fakeKubectlScript) + + legacyState := filepath.Join(tempDir, "legacy-state") + state := "absent\n" + if legacyPresent { + state = "present\n" + } + if err := os.WriteFile(legacyState, []byte(state), 0o600); err != nil { + t.Fatalf("write fake legacy state: %v", err) + } + + return &configScriptHarness{ + pythonPath: pythonPath, + scriptPath: scriptPath, + fakeBinDir: fakeBinDir, + commandLogPath: filepath.Join(tempDir, "commands.log"), + manifestPath: filepath.Join(tempDir, "rbac.yaml"), + configPath: filepath.Join(tempDir, "config.json"), + legacyState: legacyState, + deleteExitCode: deleteExitCode, + applyCountPath: filepath.Join(tempDir, "apply-count"), + } +} + +func (h *configScriptHarness) runSetupNodeRBAC() (string, error) { + cmd := exec.Command( + h.pythonPath, + h.scriptPath, + "setup-node-rbac", + "--resource-group", "test-rg", + "--cluster-name", "test-cluster", + "--subscription", "test-subscription", + ) + cmd.Env = append(os.Environ(), + "PATH="+h.fakeBinDir+string(os.PathListSeparator)+os.Getenv("PATH"), + "KUBECONFIG="+filepath.Join(filepath.Dir(h.fakeBinDir), "kubeconfig"), + "PYTHONDONTWRITEBYTECODE=1", + fakeCommandLogEnv+"="+h.commandLogPath, + fakeManifestEnv+"="+h.manifestPath, + fakeLegacyStateEnv+"="+h.legacyState, + fmt.Sprintf("%s=%d", fakeDeleteExitEnv, h.deleteExitCode), + fakeApplyCountEnv+"="+h.applyCountPath, + fmt.Sprintf("%s=%d", fakeApplyFailAtEnv, h.applyFailAt), + ) + output, err := cmd.CombinedOutput() + return string(output), err +} + +func (h *configScriptHarness) runGenerateNodeConfig() (string, error) { + cmd := exec.Command( + h.pythonPath, + h.scriptPath, + "generate-node-config", + "--resource-group", "test-rg", + "--cluster-name", "test-cluster", + "--subscription", "test-subscription", + "--bootstrap-token", + "--output", h.configPath, + ) + cmd.Env = append(os.Environ(), + "PATH="+h.fakeBinDir+string(os.PathListSeparator)+os.Getenv("PATH"), + "KUBECONFIG="+filepath.Join(filepath.Dir(h.fakeBinDir), "kubeconfig"), + "PYTHONDONTWRITEBYTECODE=1", + fakeCommandLogEnv+"="+h.commandLogPath, + fakeManifestEnv+"="+h.manifestPath, + fakeLegacyStateEnv+"="+h.legacyState, + fmt.Sprintf("%s=%d", fakeDeleteExitEnv, h.deleteExitCode), + fakeApplyCountEnv+"="+h.applyCountPath, + fmt.Sprintf("%s=%d", fakeApplyFailAtEnv, h.applyFailAt), + ) + output, err := cmd.CombinedOutput() + return string(output), err +} + +func (h *configScriptHarness) runKubeadmEnsureRBAC() (string, error) { + workingDir, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("get working directory: %w", err) + } + kubeadmScript := filepath.Join(filepath.Dir(workingDir), "hack", "e2e", "lib", "node-join-kubeadm.sh") + cmd := exec.Command( + "bash", + "-c", + `source "$1"; with_cluster_lock _kubeadm_ensure_rbac "https://test-cluster.example.test:443" "dGVzdC1jYQ=="`, + "bash", + kubeadmScript, + ) + cmd.Env = append(os.Environ(), + "PATH="+h.fakeBinDir+string(os.PathListSeparator)+os.Getenv("PATH"), + "E2E_WORK_DIR="+filepath.Join(filepath.Dir(h.fakeBinDir), "e2e-work"), + "E2E_KUBERNETES_VERSION=1.35.0", + fakeCommandLogEnv+"="+h.commandLogPath, + fakeManifestEnv+"="+h.manifestPath, + fakeLegacyStateEnv+"="+h.legacyState, + fmt.Sprintf("%s=%d", fakeDeleteExitEnv, h.deleteExitCode), + fakeApplyCountEnv+"="+h.applyCountPath, + fmt.Sprintf("%s=%d", fakeApplyFailAtEnv, h.applyFailAt), + ) + output, err := cmd.CombinedOutput() + return string(output), err +} + +func readRBACManifest(t *testing.T, path string) []rbacv1.ClusterRoleBinding { + t.Helper() + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read captured RBAC manifest: %v", err) + } + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) + var bindings []rbacv1.ClusterRoleBinding + for { + var binding rbacv1.ClusterRoleBinding + err := decoder.Decode(&binding) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("decode captured RBAC manifest: %v", err) + } + if binding.APIVersion == "" && binding.Kind == "" && binding.Name == "" { + continue + } + bindings = append(bindings, binding) + } + return bindings +} + +func bindingNames(bindings []rbacv1.ClusterRoleBinding) []string { + names := make([]string, 0, len(bindings)) + for _, binding := range bindings { + names = append(names, binding.Name) + } + return names +} + +func readCommandCalls(t *testing.T, path string) []commandCall { + t.Helper() + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read fake command log: %v", err) + } + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + calls := make([]commandCall, 0, len(lines)) + for _, line := range lines { + if line == "" { + continue + } + fields := strings.Split(line, "\t") + calls = append(calls, commandCall{name: fields[0], args: fields[1:]}) + } + return calls +} + +func kubectlOperationIndexes(calls []commandCall) (apply []int, delete []int) { + for i, call := range calls { + if call.name != "kubectl" || len(call.args) == 0 { + continue + } + switch call.args[0] { + case "apply": + apply = append(apply, i) + case "delete": + delete = append(delete, i) + } + } + return apply, delete +} + +func assertLegacyDeleteCall(t *testing.T, call commandCall) { + t.Helper() + + if call.name != "kubectl" || len(call.args) < 2 || call.args[0] != "delete" { + t.Fatalf("migration call = %#v, want kubectl delete", call) + } + resourceMatches := false + for i, arg := range call.args[1:] { + if arg == "clusterrolebinding/"+legacyBindingName { + resourceMatches = true + break + } + if (arg == "clusterrolebinding" || arg == "clusterrolebindings") && i+2 < len(call.args) && call.args[i+2] == legacyBindingName { + resourceMatches = true + break + } + } + if !resourceMatches { + t.Errorf("migration delete args = %q, want ClusterRoleBinding %q", call.args, legacyBindingName) + } + if !containsIgnoreNotFound(call.args) { + t.Errorf("migration delete args = %q, want --ignore-not-found for idempotency", call.args) + } +} + +func containsIgnoreNotFound(args []string) bool { + for _, arg := range args { + if arg == "--ignore-not-found" || arg == "--ignore-not-found=true" { + return true + } + } + return false +} + +func formatCalls(calls []commandCall) string { + formatted := make([]string, 0, len(calls)) + for _, call := range calls { + formatted = append(formatted, strings.Join(append([]string{call.name}, call.args...), " ")) + } + return strings.Join(formatted, "; ") +} + +func writeExecutable(t *testing.T, path, contents string) { + t.Helper() + if err := os.WriteFile(path, []byte(contents), 0o700); err != nil { + t.Fatalf("write fake executable %s: %v", path, err) + } +} + +const fakeAZScript = `#!/bin/sh +set -eu +{ + printf 'az' + for arg in "$@"; do + printf '\t%s' "$arg" + done + printf '\n' +} >> "${AKS_FLEX_CONFIG_TEST_COMMAND_LOG:?}" + +case " $* " in +*" account show "*" --query id "*) printf 'test-subscription\n' ;; +*" account show "*" --query tenantId "*) printf 'test-tenant\n' ;; +*" aks show "*" --query id "*) printf '/subscriptions/test/resourceGroups/test-rg/providers/Microsoft.ContainerService/managedClusters/test-cluster\n' ;; +*" aks show "*" --query location "*) printf 'test-region\n' ;; +*" currentKubernetesVersion "*) printf '1.35.0\n' ;; +*" networkProfile.dnsServiceIp "*) printf '10.0.0.10\n' ;; +esac +` + +const fakeKubectlScript = `#!/bin/sh +set -eu +{ + printf 'kubectl' + for arg in "$@"; do + printf '\t%s' "$arg" + done + printf '\n' +} >> "${AKS_FLEX_CONFIG_TEST_COMMAND_LOG:?}" + +case "${1:-}" in +apply) + apply_count=0 + if [ -f "${AKS_FLEX_CONFIG_TEST_APPLY_COUNT:?}" ]; then + apply_count=$(cat "$AKS_FLEX_CONFIG_TEST_APPLY_COUNT") + fi + apply_count=$((apply_count + 1)) + printf '%s\n' "$apply_count" > "$AKS_FLEX_CONFIG_TEST_APPLY_COUNT" + if [ "${AKS_FLEX_CONFIG_TEST_APPLY_FAIL_AT:-0}" -eq "$apply_count" ]; then + exit 38 + fi + cat > "${AKS_FLEX_CONFIG_TEST_MANIFEST:?}" + ;; +config) + case " $* " in + *"certificate-authority-data"*) printf 'dGVzdC1jYQ==\n' ;; + *"cluster.server"*) printf 'https://test-cluster.example.test:443\n' ;; + *) exit 45 ;; + esac + ;; +delete) + delete_exit="${AKS_FLEX_CONFIG_TEST_DELETE_EXIT:-0}" + if [ "$delete_exit" -ne 0 ]; then + exit "$delete_exit" + fi + + case " $* " in + *" aks-flex-node-role "*|*" clusterrolebinding/aks-flex-node-role "*) ;; + *) exit 43 ;; + esac + + state=$(cat "${AKS_FLEX_CONFIG_TEST_LEGACY_STATE:?}") + if [ "$state" = "present" ]; then + printf 'absent\n' > "$AKS_FLEX_CONFIG_TEST_LEGACY_STATE" + exit 0 + fi + + for arg in "$@"; do + case "$arg" in + --ignore-not-found|--ignore-not-found=true) exit 0 ;; + esac + done + exit 44 + ;; +*) + exit 42 + ;; +esac +` From 650da9a1d3b5b42eab9356aa588262292b451e8f Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 22 Aug 2026 00:27:26 +0000 Subject: [PATCH 02/18] Harden bootstrap RBAC migration --- .github/workflows/e2e-tests.yml | 11 +- docs/usages/aks-flex-config.md | 35 +- docs/usages/joining-nodes.md | 2 +- hack/e2e/README.md | 41 + hack/e2e/lib/bootstrap-rbac-migration.sh | 860 +++++++++++++++++ hack/e2e/lib/node-join-kubeadm.sh | 12 +- hack/e2e/run.sh | 31 +- pkg/config/config_test.go | 5 + scripts/aks-flex-config | 401 +++++++- scripts/aks_flex_config_test.go | 1119 +++++++++++++++++++--- 10 files changed, 2333 insertions(+), 184 deletions(-) create mode 100644 hack/e2e/lib/bootstrap-rbac-migration.sh diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index a80dfaaf..58fb91c0 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -9,6 +9,14 @@ name: E2E Tests on: workflow_dispatch: inputs: + suite: + description: "E2E suite to run" + required: false + default: all + type: choice + options: + - all + - historical-rbac-migration skip_cleanup: description: "Skip cleanup (keep resources for debugging)" required: false @@ -57,6 +65,7 @@ env: AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} GITHUB_RUN_ID: ${{ github.run_id }} E2E_WORK_DIR: /tmp/aks-flex-node-e2e-${{ github.run_id }} + E2E_SUITE: ${{ inputs.suite || 'all' }} jobs: e2e: @@ -95,7 +104,7 @@ jobs: E2E_SKIP_CLEANUP: "1" # Cleanup handled in dedicated step below run: | set -euo pipefail - ./hack/e2e/run.sh all + ./hack/e2e/run.sh "${E2E_SUITE}" - name: Collect logs if: always() diff --git a/docs/usages/aks-flex-config.md b/docs/usages/aks-flex-config.md index 151ac3b8..aeb61ded 100644 --- a/docs/usages/aks-flex-config.md +++ b/docs/usages/aks-flex-config.md @@ -46,17 +46,42 @@ Run this once per cluster for bootstrap-token joins: --subscription "$SUBSCRIPTION_ID" ``` -This applies only the CSR creation and approval `ClusterRoleBinding` objects for the `system:bootstrappers:aks-flex-node` group. It also removes the obsolete `aks-flex-node-role` binding created by older versions of the helper; that binding granted bootstrap tokens the broad legacy `system:node` role. The command is safe to rerun, and existing clusters should run it once after updating the helper. +This applies only the CSR creation and approval `ClusterRoleBinding` objects for the `system:bootstrappers:aks-flex-node` group. If any binding still grants that group the obsolete `system:node` role, the command stops after applying the safe bindings and explains how to migrate. It does not silently remove the binding because older and development-mode agents may still use their bootstrap token after joining. -Bootstrap-token config generation performs the same legacy-binding cleanup before creating a token. If cleanup fails, token generation stops rather than issuing a token into a cluster that may still grant it broad node permissions. +`v0.1.1` introduced a separate daemon client certificate, but the version alone does not prove that certificate was issued successfully. Upgrade every bootstrap-token agent to `v0.1.1` or later (preferably the latest release), and on every host verify that the certificate exists, is unexpired, and the agent remains healthy after a restart: -To remove only the obsolete binding from an existing cluster, run: +```bash +sudo test -s /etc/aks-flex-node/daemon-credentials/client.crt +sudo openssl x509 \ + -in /etc/aks-flex-node/daemon-credentials/client.crt \ + -noout -subject -enddate -checkend 0 +sudo systemctl restart aks-flex-node-agent.service +sudo systemctl is-active aks-flex-node-agent.service +``` + +Then explicitly remove the obsolete binding: + +```bash +./aks-flex-config setup-node-rbac \ + --resource-group "$RESOURCE_GROUP" \ + --cluster-name "$CLUSTER_NAME" \ + --subscription "$SUBSCRIPTION_ID" \ + --remove-legacy-node-role-binding +``` + +This migration is idempotent. It automatically deletes only the canonical `aks-flex-node-role` object created by older helpers. If another binding grants the same unsafe edge, or that object has extra subjects, the helper refuses to guess and identifies the objects for manual review. Bootstrap-token config generation refuses to create a token while any such binding exists, rather than either issuing an over-privileged token or unexpectedly breaking an old daemon. + +To verify the obsolete binding is gone, run: ```bash -kubectl delete clusterrolebinding aks-flex-node-role --ignore-not-found=true +kubectl get clusterrolebinding aks-flex-node-role ``` -Removing this binding does not interrupt joined nodes: they authenticate with their issued certificates rather than the bootstrap group. New and in-progress joins retain the CSR permissions installed above. +The expected result is `NotFound`. Once certificate issuance has been verified, both the kubelet and long-running Flex daemon use issued client certificates, so removing this binding does not interrupt joined nodes. New and in-progress joins retain the CSR permissions installed above. + +Do not roll back a migrated host to an older or development-mode agent that still uses the bootstrap token for ordinary Kubernetes API requests. After this binding is removed, those requests correctly receive `403 Forbidden`. Restore a supported certificate-using agent instead of restoring the broad binding. + +Finally, delete bootstrap-token Secrets that are no longer needed. In particular, tokens made by helpers before `v0.1.1` had no expiration. Removing the broad binding limits them to bootstrap permissions, but does not revoke them; do not delete a token that is still being used by an in-progress join. ## Generate Node Config diff --git a/docs/usages/joining-nodes.md b/docs/usages/joining-nodes.md index 8d2be1a4..15551bc6 100644 --- a/docs/usages/joining-nodes.md +++ b/docs/usages/joining-nodes.md @@ -16,7 +16,7 @@ Bootstrap token mode is the recommended quickstart path. It uses Kubernetes TLS High-level flow: -1. Run [`scripts/aks-flex-config setup-node-rbac`](../../scripts/aks-flex-config) to set up the least-privilege node bootstrap RBAC permissions. Rerunning the command also removes the broad legacy `aks-flex-node-role` binding from clusters configured by older helper versions. +1. Run [`scripts/aks-flex-config setup-node-rbac`](../../scripts/aks-flex-config) to set up the least-privilege node bootstrap RBAC permissions. Clusters configured by an older helper require the explicit compatibility migration documented in the helper guide before another token can be generated. 2. Run `scripts/aks-flex-config generate-node-config --bootstrap-token` to create a bootstrap token, fetch AKS cluster metadata, and render the host config. 3. Copy the generated config to `/etc/aks-flex-node/config.json` on the target host. 4. Run `aks-flex-node preflight --config /etc/aks-flex-node/config.json` to validate host, cluster, rootfs, and artifact prerequisites without mutating the node. diff --git a/hack/e2e/README.md b/hack/e2e/README.md index 075b96ca..11c86d1c 100644 --- a/hack/e2e/README.md +++ b/hack/e2e/README.md @@ -12,6 +12,7 @@ The E2E suite provisions a no-CNI AKS cluster, installs Unbounded-Net as the clu | `python3` | Local registry port readiness checks and helper scripts. | | `ssh` / `scp` | VM access and artifact copy. | | `openssl` | Bootstrap token generation. | +| `curl` / `sha256sum` / `tar` | Download and verify pinned historical release artifacts. | | `docker` | Build and push the controller image into the in-cluster local registry. | | `git` / `make` | Fetch and render Unbounded-Net manifests. | | `go` | Build the agent binary unless `--binary` is supplied. | @@ -65,6 +66,7 @@ The default `all` command runs: | Command | Description | |---------|-------------| | `all` | Full flow: build, infra, join, validate, unjoin, validate absent, rejoin, validate, lifecycle, agent upgrade, repave, logs, cleanup. | +| `historical-rbac-migration` | On a fresh real AKS cluster and token VM, join with the official v0.1.0 helper/binary, upgrade that host to HEAD, migrate legacy bootstrap RBAC, revoke the old token, and validate restarts. It does not run the other join modes. | | `infra` | Deploy AKS, four standard VMs, the Arc VM, Unbounded-Net CNI, the local registry, and the in-cluster controller. | | `join` | Join all Flex Node VMs. | | `join-msi` | Join only the managed-identity node. | @@ -138,6 +140,45 @@ Additional environment variables: | `AZURE_SUBSCRIPTION_ID` | auto-detected | Azure subscription. | | `AZURE_TENANT_ID` | auto-detected | Azure tenant. | +## Historical RBAC Migration Validation + +Run the focused compatibility suite with: + +```bash +./hack/e2e/run.sh historical-rbac-migration +``` + +For a manual GitHub Actions run, select `historical-rbac-migration` in the +`suite` workflow input. Infrastructure provisioning, the test, log upload, and +cleanup stay in the same job; the suite deliberately does not call the +Arc-inclusive parallel join path. + +The scenario downloads and verifies the official v0.1.0 release archive, +extracted binary, helper, and installer. It then uses the pinned helper and +binary to create the original broad bootstrap RBAC, a non-expiring token, the +legacy config shape, and a real Ready node on the token VM. The historical +daemon runs its production no-op path, not the v0.1.0 file-backed E2E machine +client. On that same host it verifies the HEAD helper fails closed without the +explicit migration flag, activates the HEAD binary through `agent-upgrade`, +removes the legacy binding twice to prove idempotency, checks token access +changes from HTTP 200 to 403 while CSR creation remains authorized, revokes the +token and waits for HTTP 401, and restarts both kubelet and the daemon while +checking the Node UID, Lease, readiness, and certificate-backed API access. +v0.1.0 transitively pins the non-GPU rootfs +`ghcr.io/azure/agent-ubuntu2404:v20260427`. + +There are two intentional compatibility boundaries: + +- The test creates a new AKS control plane and reproduces the v0.1.0 + cluster-side state. It validates a historical node/config/RBAC migration, not + an AKS control plane that has itself been retained since v0.1.0. +- v0.1.0 tokens lack the `kubernetes.azure.com/managedby=aks` label required by + the production managed CSR approver. The suite explicitly adopts its known + token with that label before the HEAD daemon requests a certificate. The + repository E2E approver does not enforce this label, so this test validates + the host/config/RBAC migration but is not independent proof of the production + approver's ownership check. + ## Join Modes The suite validates five join paths. The E2E subscription must have `Microsoft.HybridCompute`, `Microsoft.HybridConnectivity`, and `Microsoft.GuestConfiguration` registered before the run. diff --git a/hack/e2e/lib/bootstrap-rbac-migration.sh b/hack/e2e/lib/bootstrap-rbac-migration.sh new file mode 100644 index 00000000..12cfee7d --- /dev/null +++ b/hack/e2e/lib/bootstrap-rbac-migration.sh @@ -0,0 +1,860 @@ +#!/usr/bin/env bash +# ============================================================================= +# Real-node migration test for the legacy bootstrap-group system:node binding. +# ============================================================================= +set -euo pipefail + +[[ -n "${_E2E_BOOTSTRAP_RBAC_MIGRATION_LOADED:-}" ]] && return 0 +readonly _E2E_BOOTSTRAP_RBAC_MIGRATION_LOADED=1 + +# shellcheck disable=SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/common.sh" + +readonly historicalReleaseTag="v0.1.0" +readonly historicalCommit="65d8d3896371adf2eb13248c3e73f0e83fd418ef" +readonly historicalArchiveName="aks-flex-node-linux-amd64.tar.gz" +readonly historicalBinaryName="aks-flex-node-linux-amd64" +readonly historicalArchiveURL="https://github.com/Azure/AKSFlexNode/releases/download/${historicalReleaseTag}/${historicalArchiveName}" +readonly historicalHelperURL="https://raw.githubusercontent.com/Azure/AKSFlexNode/${historicalCommit}/scripts/aks-flex-config" +readonly historicalInstallerURL="https://raw.githubusercontent.com/Azure/AKSFlexNode/${historicalCommit}/scripts/install.sh" +readonly historicalArchiveSHA256="50922e15999b2fd9c19a298c3c3d9cb5a0a375858258dc42b92699a4427876e6" +readonly historicalBinarySHA256="50b4a62daeb30e635cc8ea5e18c6a204e54510e164eba75fe0322a8a8d56fdc7" +readonly historicalHelperSHA256="8ae38209e1a63f1b3c9d1fb41423644de5de032c3a978cedef100689fe1f19f7" +readonly historicalInstallerSHA256="c2f7cfc92e62c3a9fb96697b3a3ca170bb8c28d11af8f45d0f26f6fce016e9b4" +# v0.1.0 embeds Unbounded v0.1.8, whose non-GPU default resolves to this +# immutable release tag. The old Flex config schema cannot override OCIImage. +readonly historicalRootFS="ghcr.io/azure/agent-ubuntu2404:v20260427" +readonly legacyNodeRoleBinding="aks-flex-node-role" +readonly flexNodeBootstrapGroup="system:bootstrappers:aks-flex-node" + +_historical_artifact_dir() { + echo "${E2E_WORK_DIR}/historical-${historicalReleaseTag}" +} + +_verify_sha256() { + local path="$1" + local expected="$2" + local description="$3" + local actual + + actual="$(sha256sum "${path}" | awk '{print $1}')" + if [[ "${actual}" != "${expected}" ]]; then + log_error "${description} SHA-256 mismatch: got ${actual}, want ${expected}" + return 1 + fi +} + +_download_verified_artifact() { + local url="$1" + local path="$2" + local expected="$3" + local description="$4" + local partial="${path}.download" + + if [[ -f "${path}" ]] && _verify_sha256 "${path}" "${expected}" "${description}"; then + log_info "Using cached, verified ${description}: ${path}" + return 0 + fi + + rm -f "${path}" "${partial}" + log_info "Downloading pinned ${description}" + if ! curl --fail --location --proto '=https' --retry 5 --retry-all-errors \ + --silent --show-error --output "${partial}" "${url}"; then + rm -f "${partial}" + return 1 + fi + if ! _verify_sha256 "${partial}" "${expected}" "${description}"; then + rm -f "${partial}" + return 1 + fi + mv "${partial}" "${path}" +} + +_prepare_historical_artifacts() { + require_cmd curl + require_cmd sha256sum + require_cmd tar + + local artifact_dir archive helper installer binary archive_members version_output + artifact_dir="$(_historical_artifact_dir)" + archive="${artifact_dir}/${historicalArchiveName}" + helper="${artifact_dir}/aks-flex-config" + installer="${artifact_dir}/install.sh" + binary="${artifact_dir}/${historicalBinaryName}" + mkdir -p "${artifact_dir}" + + _download_verified_artifact \ + "${historicalArchiveURL}" "${archive}" "${historicalArchiveSHA256}" \ + "${historicalReleaseTag} release archive" + _download_verified_artifact \ + "${historicalHelperURL}" "${helper}" "${historicalHelperSHA256}" \ + "${historicalReleaseTag} aks-flex-config helper" + _download_verified_artifact \ + "${historicalInstallerURL}" "${installer}" "${historicalInstallerSHA256}" \ + "${historicalReleaseTag} installer" + + archive_members="$(tar -tzf "${archive}")" + if [[ "${archive_members}" != "${historicalBinaryName}" ]]; then + log_error "${historicalReleaseTag} archive has unexpected members: ${archive_members}" + return 1 + fi + rm -f "${binary}" + tar --extract --gzip --file "${archive}" --directory "${artifact_dir}" \ + --no-same-owner --no-same-permissions "${historicalBinaryName}" + _verify_sha256 "${binary}" "${historicalBinarySHA256}" \ + "${historicalReleaseTag} extracted binary" + chmod 0755 "${binary}" "${helper}" "${installer}" + + version_output="$("${binary}" version)" + if [[ "${version_output}" != *"Version: ${historicalReleaseTag}"* || \ + "${version_output}" != *"Git Commit: ${historicalCommit:0:7}"* ]]; then + log_error "Pinned historical binary reports unexpected build metadata: ${version_output}" + return 1 + fi + + log_success "Verified official ${historicalReleaseTag} archive, binary, helper, and installer" + log_info "Historical binary's pinned default rootfs: ${historicalRootFS}" +} + +_historical_config_path() { + echo "${E2E_WORK_DIR}/config-token-${historicalReleaseTag}.json" +} + +_head_legacy_config_path() { + echo "${E2E_WORK_DIR}/config-token-${historicalReleaseTag}-head.json" +} + +_historical_token_id() { + local config_file="$1" + jq -er \ + '.azure.bootstrapToken.token | capture("^(?[a-z0-9]{6})\\.[a-z0-9]{16}$").id' \ + "${config_file}" +} + +_generate_historical_config() { + local artifact_dir helper config_file + local vm_name vm_private_ip cluster_name resource_group subscription_id + artifact_dir="$(_historical_artifact_dir)" + helper="${artifact_dir}/aks-flex-config" + config_file="$(_historical_config_path)" + vm_name="$(state_get token_vm_name)" + vm_private_ip="$(state_get token_vm_private_ip)" + cluster_name="$(state_get cluster_name)" + resource_group="$(state_get resource_group)" + subscription_id="$(state_get subscription_id)" + + if [[ -z "${vm_private_ip}" ]] || ! is_valid_ipv4 "${vm_private_ip}"; then + log_error "Invalid token VM private IP in state: '${vm_private_ip}'" + return 1 + fi + if kubectl get clusterrolebinding "${legacyNodeRoleBinding}" >/dev/null 2>&1; then + log_error "Historical scenario requires a fresh cluster without ${legacyNodeRoleBinding}" + return 1 + fi + + log_info "Applying the real ${historicalReleaseTag} cluster-side RBAC" + with_cluster_lock python3 "${helper}" setup-node-rbac \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" + + log_info "Generating a real non-expiring ${historicalReleaseTag} bootstrap token and config" + with_cluster_lock python3 "${helper}" generate-node-config \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" \ + --bootstrap-token \ + --output "${config_file}" + + jq \ + --arg nodeName "${vm_name}" \ + --arg nodeIP "${vm_private_ip}" \ + --arg kubernetesVersion "${E2E_KUBERNETES_VERSION}" \ + --arg containerdVersion "${E2E_CONTAINERD_VERSION}" \ + --arg runcVersion "${E2E_RUNC_VERSION}" \ + '.agent.logLevel = "debug" + | .agent.nodeName = $nodeName + | .node.kubelet.nodeIP = $nodeIP + | .kubernetes.version = $kubernetesVersion + | .containerd.version = $containerdVersion + | .runc.version = $runcVersion' \ + "${config_file}" > "${config_file}.tmp" + mv "${config_file}.tmp" "${config_file}" + chmod 0600 "${config_file}" + + if ! jq -e \ + --arg nodeName "${vm_name}" \ + --arg nodeIP "${vm_private_ip}" \ + '.agent.nodeName == $nodeName + and (.agent | has("e2eMode") | not) + and (.agent | has("machineOperationMode") | not) + and .node.kubelet.nodeIP == $nodeIP + and (.node.kubelet.serverURL | length > 0) + and (.node.kubelet.caCertData | length > 0) + and (.kubernetes.version | length > 0) + and (has("components") | not)' \ + "${config_file}" >/dev/null; then + log_error "Historical helper did not produce the expected legacy config shape" + return 1 + fi +} + +_require_historical_cluster_state() { + local config_file="$1" + local token_id secret + token_id="$(_historical_token_id "${config_file}")" + + if ! kubectl get clusterrolebinding "${legacyNodeRoleBinding}" -o json | jq -e \ + --arg group "${flexNodeBootstrapGroup}" \ + '.roleRef.apiGroup == "rbac.authorization.k8s.io" + and .roleRef.kind == "ClusterRole" + and .roleRef.name == "system:node" + and any(.subjects[]?; + .apiGroup == "rbac.authorization.k8s.io" + and .kind == "Group" + and .name == $group)' >/dev/null; then + log_error "${historicalReleaseTag} helper did not create the expected legacy node-role binding" + return 1 + fi + + secret="$(kubectl -n kube-system get secret "bootstrap-token-${token_id}" -o json)" + if ! jq -e \ + '(.data.expiration // "") == "" + and (.metadata.labels["kubernetes.azure.com/managedby"] // "") == ""' \ + <<<"${secret}" >/dev/null; then + log_error "Historical bootstrap Secret unexpectedly has expiration or AKS ownership metadata" + return 1 + fi + log_success "Verified ${historicalReleaseTag} legacy RBAC and non-expiring, unmanaged token state" +} + +_install_and_start_historical_node() { + local vm_ip="$1" + local artifact_dir config_file + artifact_dir="$(_historical_artifact_dir)" + config_file="$(_historical_config_path)" + + remote_copy "${artifact_dir}/${historicalArchiveName}" "${vm_ip}" "/tmp/${historicalArchiveName}" + remote_copy "${artifact_dir}/aks-flex-config" "${vm_ip}" "/tmp/aks-flex-config-${historicalReleaseTag}" + remote_copy "${artifact_dir}/install.sh" "${vm_ip}" "/tmp/aks-flex-node-install-${historicalReleaseTag}.sh" + remote_copy "${config_file}" "${vm_ip}" "/tmp/config-${historicalReleaseTag}.json" + + remote_exec "${vm_ip}" \ + "HISTORICAL_TAG=${historicalReleaseTag} HISTORICAL_COMMIT=${historicalCommit:0:7} HISTORICAL_ARCHIVE_SHA256=${historicalArchiveSHA256} HISTORICAL_BINARY_SHA256=${historicalBinarySHA256} HISTORICAL_HELPER_SHA256=${historicalHelperSHA256} HISTORICAL_INSTALLER_SHA256=${historicalInstallerSHA256} E2E_NODE_JOIN_TIMEOUT=${E2E_NODE_JOIN_TIMEOUT} bash -s" <<'REMOTE' +set -euo pipefail + +archive=/tmp/aks-flex-node-linux-amd64.tar.gz +helper="/tmp/aks-flex-config-${HISTORICAL_TAG}" +installer="/tmp/aks-flex-node-install-${HISTORICAL_TAG}.sh" +config="/tmp/config-${HISTORICAL_TAG}.json" +extract_dir="/tmp/aks-flex-node-${HISTORICAL_TAG}" +binary="${extract_dir}/aks-flex-node-linux-amd64" + +printf '%s %s\n' "${HISTORICAL_ARCHIVE_SHA256}" "${archive}" | sha256sum --check --strict - +printf '%s %s\n' "${HISTORICAL_HELPER_SHA256}" "${helper}" | sha256sum --check --strict - +printf '%s %s\n' "${HISTORICAL_INSTALLER_SHA256}" "${installer}" | sha256sum --check --strict - + +archive_members="$(tar -tzf "${archive}")" +if [[ "${archive_members}" != "aks-flex-node-linux-amd64" ]]; then + echo "historical archive has unexpected members: ${archive_members}" >&2 + exit 1 +fi +sudo rm -rf "${extract_dir}" +mkdir -p "${extract_dir}" +tar --extract --gzip --file "${archive}" --directory "${extract_dir}" \ + --no-same-owner --no-same-permissions aks-flex-node-linux-amd64 +printf '%s %s\n' "${HISTORICAL_BINARY_SHA256}" "${binary}" | sha256sum --check --strict - +chmod 0755 "${binary}" "${helper}" "${installer}" + +version_output="$("${binary}" version)" +grep -Fq "Version: ${HISTORICAL_TAG}" <<<"${version_output}" +grep -Fq "Git Commit: ${HISTORICAL_COMMIT}" <<<"${version_output}" + +if [[ -e /usr/local/lib/aks-flex-node/aks-flex-node-current || \ + -L /usr/local/lib/aks-flex-node/aks-flex-node-current ]]; then + echo "historical test VM already has a managed agent layout" >&2 + exit 1 +fi + +if command -v apt-get >/dev/null 2>&1; then + packages_installed=0 + for attempt in $(seq 1 5); do + if sudo DEBIAN_FRONTEND=noninteractive apt-get \ + -o Acquire::Retries=5 -o Acquire::http::Timeout=30 update && + sudo DEBIAN_FRONTEND=noninteractive apt-get \ + -o Acquire::Retries=5 -o Acquire::http::Timeout=30 \ + install -y --fix-missing ca-certificates curl nftables systemd-container util-linux; then + packages_installed=1 + break + fi + echo "Host package installation failed; retrying (${attempt}/5)..." + sleep 10 + done + if (( packages_installed != 1 )); then + echo "Host package installation failed after retries" >&2 + exit 1 + fi +fi + +sudo AKS_FLEX_NODE_LOCAL_BINARY="${binary}" \ + AKS_FLEX_NODE_VERSION="${HISTORICAL_TAG}" \ + SKIP_AZCLI=true \ + bash "${installer}" --yes +sudo install -m 0600 "${config}" /etc/aks-flex-node/config.json + +if [[ -L /usr/local/bin/aks-flex-node ]]; then + echo "${HISTORICAL_TAG} installer unexpectedly created a managed compatibility symlink" >&2 + exit 1 +fi +printf '%s %s\n' "${HISTORICAL_BINARY_SHA256}" /usr/local/bin/aks-flex-node | sudo sha256sum --check --strict - +if [[ -e /etc/aks-flex-node/daemon-credentials/client.crt ]]; then + echo "daemon certificate existed before historical bootstrap" >&2 + exit 1 +fi + +unit=aks-flex-node-historical-bootstrap +sudo systemctl stop "${unit}.service" 2>/dev/null || true +sudo systemctl reset-failed "${unit}.service" 2>/dev/null || true +sudo systemd-run \ + --unit="${unit}" \ + --description="AKS Flex Node ${HISTORICAL_TAG} E2E" \ + --remain-after-exit \ + /usr/local/bin/aks-flex-node bootstrap --config /etc/aks-flex-node/config.json + +deadline=$((SECONDS + E2E_NODE_JOIN_TIMEOUT)) +while ! sudo systemctl is-active --quiet aks-flex-node-agent.service; do + if sudo systemctl is-failed --quiet "${unit}.service"; then + sudo systemctl status "${unit}.service" --no-pager -l >&2 || true + sudo journalctl -u "${unit}.service" -n 100 --no-pager >&2 || true + exit 1 + fi + if (( SECONDS >= deadline )); then + echo "timed out waiting for historical daemon service" >&2 + sudo systemctl status "${unit}.service" --no-pager -l >&2 || true + sudo systemctl status aks-flex-node-agent.service --no-pager -l >&2 || true + sudo journalctl -u "${unit}.service" -n 100 --no-pager >&2 || true + sudo journalctl -u aks-flex-node-agent.service -n 100 --no-pager >&2 || true + exit 1 + fi + sleep 5 +done + +sleep 5 +sudo systemctl is-active --quiet aks-flex-node-agent.service +sudo grep -Fq 'ExecStart=/usr/local/bin/aks-flex-node agent' \ + /etc/systemd/system/aks-flex-node-agent.service +historical_logs="$(sudo journalctl -u aks-flex-node-agent.service --no-pager)" +grep -Fq 'production agent daemon requires AKS RP machine client implementation' \ + <<<"${historical_logs}" +if grep -Fq 'running agent daemon in e2e mode' <<<"${historical_logs}"; then + echo "${HISTORICAL_TAG} daemon unexpectedly started in E2E mode" >&2 + exit 1 +fi +if [[ -e /etc/aks-flex-node/daemon-credentials/client.crt ]]; then + echo "${HISTORICAL_TAG} unexpectedly issued a separate daemon certificate" >&2 + exit 1 +fi +REMOTE +} + +_bootstrap_token_api_probe() { + local config_file="$1" + local probe="$2" + + python3 - "${config_file}" "${probe}" <<'PY' +import base64 +import json +import ssl +import sys +import urllib.error +import urllib.request + +config_path, probe = sys.argv[1:] +with open(config_path, encoding="utf-8") as stream: + config = json.load(stream) + +token = config["azure"]["bootstrapToken"]["token"] +kubelet = config["node"]["kubelet"] +server = kubelet.get("serverURL") +if not server: + cluster_fqdn = kubelet["clusterFQDN"] + server = cluster_fqdn if "://" in cluster_fqdn else "https://" + cluster_fqdn +ca_pem = base64.b64decode(kubelet["caCertData"], validate=True).decode("ascii") +context = ssl.create_default_context(cadata=ca_pem) +headers = {"Authorization": "Bearer " + token} + +if probe == "list-nodes": + request = urllib.request.Request(server.rstrip("/") + "/api/v1/nodes?limit=1", headers=headers) +elif probe == "create-csr": + body = json.dumps( + { + "apiVersion": "authorization.k8s.io/v1", + "kind": "SelfSubjectAccessReview", + "spec": { + "resourceAttributes": { + "group": "certificates.k8s.io", + "resource": "certificatesigningrequests", + "verb": "create", + } + }, + } + ).encode("utf-8") + request = urllib.request.Request( + server.rstrip("/") + "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews", + data=body, + headers={**headers, "Content-Type": "application/json"}, + method="POST", + ) +else: + raise SystemExit("unsupported probe: " + probe) + +try: + with urllib.request.urlopen(request, context=context, timeout=30) as response: + response_body = response.read() + status = response.status +except urllib.error.HTTPError as error: + print("http:" + str(error.code)) + raise SystemExit(0) + +if probe == "create-csr": + review = json.loads(response_body) + print("allowed" if review.get("status", {}).get("allowed") is True else "denied") +else: + print("http:" + str(status)) +PY +} + +_wait_for_bootstrap_token_probe() { + local expected="$1" + local config_file="$2" + local probe="$3" + local elapsed=0 + local actual="" + + while (( elapsed < E2E_NODE_JOIN_TIMEOUT )); do + if actual="$(_bootstrap_token_api_probe "${config_file}" "${probe}")" && \ + [[ "${actual}" == "${expected}" ]]; then + return 0 + fi + sleep 2 + elapsed=$((elapsed + 2)) + done + + log_error "Bootstrap token probe ${probe} returned '${actual}', want '${expected}' after ${E2E_NODE_JOIN_TIMEOUT}s" + return 1 +} + +_kubelet_client_identity() { + local vm_ip="$1" + + remote_exec "${vm_ip}" 'sudo bash -s' <<'REMOTE' +set -euo pipefail +machine="$(python3 - <<'PY' +import json +with open('/etc/aks-flex-node/daemon-state.json', encoding='utf-8') as stream: + print(json.load(stream)['activeMachine']) +PY +)" +systemd-run --machine="${machine}" --quiet --pipe --wait \ + openssl x509 \ + -in /var/lib/kubelet/pki/kubelet-client-current.pem \ + -noout \ + -subject \ + -nameopt RFC2253 +REMOTE +} + +_daemon_client_identity() { + local vm_ip="$1" + remote_exec "${vm_ip}" \ + 'sudo openssl x509 -in /etc/aks-flex-node/daemon-credentials/client.crt -noout -subject -nameopt RFC2253' +} + +_require_daemon_certificate_access() { + local vm_ip="$1" + local server_url="$2" + local quoted_server + printf -v quoted_server '%q' "${server_url}" + + remote_exec "${vm_ip}" "SERVER_URL=${quoted_server} bash -s" <<'REMOTE' +set -euo pipefail +ca_file="$(mktemp)" +trap 'rm -f "${ca_file}"' EXIT +sudo python3 - "${ca_file}" <<'PY' +import base64 +import json +import sys + +with open('/etc/aks-flex-node/config.json', encoding='utf-8') as stream: + ca_data = json.load(stream)['node']['kubelet']['caCertData'] +with open(sys.argv[1], 'wb') as stream: + stream.write(base64.b64decode(ca_data, validate=True)) +PY +status="$(sudo curl --silent --show-error \ + --cert /etc/aks-flex-node/daemon-credentials/client.crt \ + --key /etc/aks-flex-node/daemon-credentials/client.key \ + --cacert "${ca_file}" \ + --output /dev/null \ + --write-out '%{http_code}' \ + "${SERVER_URL}/api/v1/nodes?limit=1")" +if [[ "${status}" != "200" ]]; then + echo "daemon client certificate LIST nodes returned HTTP ${status}, want 200" >&2 + exit 1 +fi +REMOTE +} + +_require_old_node_survives_guard() { + local vm_name="$1" + local vm_ip="$2" + + remote_exec "${vm_ip}" 'bash -s' <<'REMOTE' +set -euo pipefail +sudo systemctl restart aks-flex-node-agent.service +for _ in $(seq 1 30); do + if sudo systemctl is-active --quiet aks-flex-node-agent.service; then + first_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" + sleep 5 + second_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" + if sudo systemctl is-active --quiet aks-flex-node-agent.service && + [[ "${first_pid}" =~ ^[1-9][0-9]*$ ]] && [[ "${second_pid}" == "${first_pid}" ]]; then + break + fi + fi + sleep 2 +done +if ! sudo systemctl is-active --quiet aks-flex-node-agent.service || + [[ "${second_pid:-}" != "${first_pid:-}" ]]; then + sudo systemctl status aks-flex-node-agent.service --no-pager -l >&2 || true + exit 1 +fi +if [[ -e /etc/aks-flex-node/daemon-credentials/client.crt ]]; then + echo "historical daemon unexpectedly created a daemon certificate" >&2 + exit 1 +fi +REMOTE + validate_node_joined "${vm_name}" +} + +_prepare_head_legacy_config() { + local source_config="$1" + local target_config="$2" + + jq \ + --arg machineEndpointURL "${E2E_CONTROLLER_SERVICE_PROXY_PATH}" \ + --arg agentPoolName "${E2E_TARGET_AGENT_POOL_NAME}" \ + --arg ociImage "${historicalRootFS}" \ + '.agent.machineClient.mode = "in-cluster" + | .agent.machineClient.endpointUrl = $machineEndpointURL + | .agent.requireMachineRegistration = true + | .agent.machineOperationMode = "disable" + | .azure.targetAgentPoolName = $agentPoolName + | .bootstrap.ociImage = $ociImage' \ + "${source_config}" > "${target_config}" + chmod 0600 "${target_config}" + + if ! jq -e \ + --arg endpoint "${E2E_CONTROLLER_SERVICE_PROXY_PATH}" \ + --arg ociImage "${historicalRootFS}" \ + '.agent.machineClient.mode == "in-cluster" + and .agent.machineClient.endpointUrl == $endpoint + and .agent.requireMachineRegistration == true + and .agent.machineOperationMode == "disable" + and .bootstrap.ociImage == $ociImage + and (.node.kubelet.serverURL | length > 0) + and (.kubernetes.version | length > 0) + and (has("components") | not)' \ + "${target_config}" >/dev/null; then + log_error "HEAD upgrade config no longer preserves the historical config fields" + return 1 + fi +} + +_upgrade_historical_node_to_head() { + local vm_ip="$1" + local config_file="$2" + local head_sha + head_sha="$(sha256sum "${E2E_BINARY}" | awk '{print $1}')" + + remote_copy "${E2E_BINARY}" "${vm_ip}" /tmp/aks-flex-node-head + remote_copy "${config_file}" "${vm_ip}" /tmp/config-v0.1.0-head.json + + remote_exec "${vm_ip}" \ + "HISTORICAL_BINARY_SHA256=${historicalBinarySHA256} HEAD_BINARY_SHA256=${head_sha} E2E_NODE_JOIN_TIMEOUT=${E2E_NODE_JOIN_TIMEOUT} bash -s" <<'REMOTE' +set -euo pipefail +candidate=/tmp/aks-flex-node-head +current_link=/usr/local/lib/aks-flex-node/aks-flex-node-current +last_good_link=/usr/local/lib/aks-flex-node/aks-flex-node-last-good +service=/etc/systemd/system/aks-flex-node-agent.service + +chmod 0755 "${candidate}" +printf '%s %s\n' "${HEAD_BINARY_SHA256}" "${candidate}" | sha256sum --check --strict - +printf '%s %s\n' "${HISTORICAL_BINARY_SHA256}" /usr/local/bin/aks-flex-node | sudo sha256sum --check --strict - +if [[ -e "${current_link}" || -L "${current_link}" ]]; then + echo "managed layout existed before migration preflight" >&2 + exit 1 +fi + +sudo install -m 0600 /tmp/config-v0.1.0-head.json /etc/aks-flex-node/config.json +sudo "${candidate}" agent-upgrade --preflight | sudo tee /tmp/historical-agent-upgrade-preflight.log + +# Preflight must not mutate the direct v0.1.0 installation. +if [[ -e "${current_link}" || -L "${current_link}" || -L /usr/local/bin/aks-flex-node ]]; then + echo "agent-upgrade preflight mutated the legacy binary layout" >&2 + exit 1 +fi +printf '%s %s\n' "${HISTORICAL_BINARY_SHA256}" /usr/local/bin/aks-flex-node | sudo sha256sum --check --strict - +sudo systemctl is-active --quiet aks-flex-node-agent.service + +sudo "${candidate}" agent-upgrade | sudo tee /tmp/historical-agent-upgrade.log + +deadline=$((SECONDS + E2E_NODE_JOIN_TIMEOUT)) +while true; do + if sudo systemctl is-active --quiet aks-flex-node-agent.service && + [[ -s /etc/aks-flex-node/daemon-credentials/client.crt ]] && + [[ -s /etc/aks-flex-node/daemon-credentials/client.key ]]; then + first_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" + sleep 5 + second_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" + if sudo systemctl is-active --quiet aks-flex-node-agent.service && + [[ "${first_pid}" =~ ^[1-9][0-9]*$ ]] && [[ "${second_pid}" == "${first_pid}" ]]; then + break + fi + fi + if (( SECONDS >= deadline )); then + echo "HEAD daemon did not become stable with daemon credentials" >&2 + sudo systemctl status aks-flex-node-agent.service --no-pager -l >&2 || true + sudo journalctl -u aks-flex-node-agent.service -n 150 --no-pager >&2 || true + exit 1 + fi + sleep 2 +done + +for link in /usr/local/bin/aks-flex-node "${current_link}" "${last_good_link}"; do + if [[ ! -L "${link}" ]]; then + echo "managed binary link missing after upgrade: ${link}" >&2 + exit 1 + fi +done +active="$(sudo readlink -f "${current_link}")" +last_good="$(sudo readlink -f "${last_good_link}")" +printf '%s %s\n' "${HEAD_BINARY_SHA256}" "${active}" | sudo sha256sum --check --strict - +printf '%s %s\n' "${HISTORICAL_BINARY_SHA256}" "${last_good}" | sudo sha256sum --check --strict - +if [[ "$(sudo readlink -f /usr/local/bin/aks-flex-node)" != "${active}" ]]; then + echo "compatibility path does not resolve to the active managed binary" >&2 + exit 1 +fi +sudo grep -Fq "ExecStart=${current_link} agent" "${service}" +pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" +if [[ "$(sudo readlink -f "/proc/${pid}/exe")" != "${active}" ]]; then + echo "daemon is not executing the activated HEAD binary" >&2 + exit 1 +fi + +cert_pub="$(sudo openssl x509 -in /etc/aks-flex-node/daemon-credentials/client.crt -pubkey -noout | sha256sum | awk '{print $1}')" +key_pub="$(sudo openssl pkey -in /etc/aks-flex-node/daemon-credentials/client.key -pubout | sha256sum | awk '{print $1}')" +if [[ "${cert_pub}" != "${key_pub}" ]]; then + echo "daemon certificate and private key do not match" >&2 + exit 1 +fi +REMOTE +} + +_revoke_historical_bootstrap_token() { + local config_file="$1" + local token_id + token_id="$(_historical_token_id "${config_file}")" + kubectl delete secret "bootstrap-token-${token_id}" -n kube-system + log_info "Revoked the historical bootstrap token after certificate migration" +} + +_restart_daemon_and_require_certificate_access() { + local vm_ip="$1" + local server_url="$2" + + remote_exec "${vm_ip}" 'bash -s' <<'REMOTE' +set -euo pipefail +sudo systemctl restart aks-flex-node-agent.service +for _ in $(seq 1 30); do + if sudo systemctl is-active --quiet aks-flex-node-agent.service; then + first_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" + sleep 5 + second_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" + if sudo systemctl is-active --quiet aks-flex-node-agent.service && + [[ "${first_pid}" =~ ^[1-9][0-9]*$ ]] && [[ "${second_pid}" == "${first_pid}" ]]; then + exit 0 + fi + fi + sleep 2 +done +sudo systemctl status aks-flex-node-agent.service --no-pager -l >&2 || true +sudo journalctl -u aks-flex-node-agent.service -n 100 --no-pager >&2 || true +exit 1 +REMOTE + _require_daemon_certificate_access "${vm_ip}" "${server_url}" +} + +_restart_kubelet_and_require_lease_renewal() { + local vm_name="$1" + local vm_ip="$2" + local node_uid="$3" + local before_renew + local elapsed=0 + + before_renew="$(kubectl get lease "${vm_name}" -n kube-node-lease -o jsonpath='{.spec.renewTime}')" + remote_exec "${vm_ip}" 'sudo bash -s' <<'REMOTE' +set -euo pipefail +machine="$(python3 - <<'PY' +import json +with open('/etc/aks-flex-node/daemon-state.json', encoding='utf-8') as stream: + print(json.load(stream)['activeMachine']) +PY +)" +systemd-run --machine="${machine}" --quiet --pipe --wait systemctl restart kubelet.service +REMOTE + + while (( elapsed < E2E_NODE_JOIN_TIMEOUT )); do + local current_uid renew ready + current_uid="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.uid}' 2>/dev/null || true)" + renew="$(kubectl get lease "${vm_name}" -n kube-node-lease -o jsonpath='{.spec.renewTime}' 2>/dev/null || true)" + ready="$(kubectl get node "${vm_name}" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" + if [[ "${current_uid}" == "${node_uid}" && -n "${renew}" && \ + "${renew}" != "${before_renew}" && "${ready}" == "True" ]]; then + log_success "Historical node renewed its Lease and stayed Ready after token revocation" + return 0 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + + log_error "Historical node did not reauthenticate after token revocation" + kubectl get node "${vm_name}" -o wide 2>&1 || true + kubectl describe node "${vm_name}" 2>&1 || true + return 1 +} + +historical_rbac_migration_e2e() { + log_section "Historical ${historicalReleaseTag} Node and Bootstrap RBAC Migration" + + local config_file head_config vm_name vm_ip cluster_name resource_group subscription_id + local server_url node_uid identity guard_output token token_id + config_file="$(_historical_config_path)" + head_config="$(_head_legacy_config_path)" + vm_name="$(state_get token_vm_name)" + vm_ip="$(state_get token_vm_ip)" + cluster_name="$(state_get cluster_name)" + resource_group="$(state_get resource_group)" + subscription_id="$(state_get subscription_id)" + server_url="$(state_get server_url)" + + _prepare_historical_artifacts + _generate_historical_config + _require_historical_cluster_state "${config_file}" + _install_and_start_historical_node "${vm_ip}" + validate_node_joined "${vm_name}" + + node_uid="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.uid}')" + identity="$(_kubelet_client_identity "${vm_ip}")" + if [[ "${identity}" != *"CN=system:node:${vm_name}"* ]]; then + log_error "${historicalReleaseTag} kubelet has unexpected client identity: ${identity}" + return 1 + fi + _wait_for_bootstrap_token_probe http:200 "${config_file}" list-nodes + log_success "Official ${historicalReleaseTag} node is Ready with legacy RBAC and an issued kubelet certificate" + + # HEAD must converge the safe CSR bindings but preserve the old daemon until + # an operator explicitly confirms the migration. + if guard_output="$(with_cluster_lock "${REPO_ROOT}/scripts/aks-flex-config" setup-node-rbac \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" 2>&1)"; then + log_error "HEAD RBAC setup accepted a legacy binding without explicit migration" + return 1 + fi + if [[ "${guard_output}" != *"--remove-legacy-node-role-binding"* ]]; then + log_error "HEAD compatibility guard did not explain the explicit migration path" + return 1 + fi + if ! kubectl get clusterrolebinding "${legacyNodeRoleBinding}" >/dev/null 2>&1; then + log_error "HEAD compatibility guard removed the legacy binding" + return 1 + fi + _wait_for_bootstrap_token_probe http:200 "${config_file}" list-nodes + _require_old_node_survives_guard "${vm_name}" "${vm_ip}" + log_success "HEAD fail-closed guard preserved the running historical node" + + # Helpers before v0.1.1 did not add the ownership label required by the + # production AKS managed CSR approver. Adopt this known E2E token explicitly + # before the HEAD daemon requests its dedicated certificate. + token="$(jq -er '.azure.bootstrapToken.token' "${config_file}")" + token_id="$(_historical_token_id "${config_file}")" + with_cluster_lock mark_e2e_bootstrap_token_aks_managed "${token}" + unset token + if [[ "$(kubectl -n kube-system get secret "bootstrap-token-${token_id}" \ + -o jsonpath='{.metadata.labels.kubernetes\.azure\.com/managedby}')" != "aks" ]]; then + log_error "Historical token adoption label was not applied" + return 1 + fi + + machine_configmap_upsert "${vm_name}" "${E2E_KUBERNETES_VERSION}" "${E2E_KUBERNETES_VERSION}" + _prepare_head_legacy_config "${config_file}" "${head_config}" + _upgrade_historical_node_to_head "${vm_ip}" "${head_config}" + validate_node_joined "${vm_name}" + + identity="$(_daemon_client_identity "${vm_ip}")" + if [[ "${identity}" != *"CN=system:node:${vm_name}"* || \ + "${identity}" != *"O=aks-flex-node-daemons"* ]]; then + log_error "Upgraded daemon has unexpected issued client identity: ${identity}" + return 1 + fi + _require_daemon_certificate_access "${vm_ip}" "${server_url}" + log_success "Same host upgraded from direct ${historicalReleaseTag} layout to HEAD and obtained daemon credentials" + + # The first run removes the canonical historical object. The second verifies + # that the explicitly requested migration is idempotent. + with_cluster_lock "${REPO_ROOT}/scripts/aks-flex-config" setup-node-rbac \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" \ + --remove-legacy-node-role-binding + with_cluster_lock "${REPO_ROOT}/scripts/aks-flex-config" setup-node-rbac \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" \ + --remove-legacy-node-role-binding + + if kubectl get clusterrolebinding "${legacyNodeRoleBinding}" >/dev/null 2>&1; then + log_error "Legacy binding '${legacyNodeRoleBinding}' still exists after migration" + return 1 + fi + _wait_for_bootstrap_token_probe http:403 "${config_file}" list-nodes + _wait_for_bootstrap_token_probe allowed "${config_file}" create-csr + _require_daemon_certificate_access "${vm_ip}" "${server_url}" + + # Revocation proves the subsequent restarts cannot silently fall back to the + # bootstrap credential. + with_cluster_lock _revoke_historical_bootstrap_token "${config_file}" + _wait_for_bootstrap_token_probe http:401 "${config_file}" list-nodes + + _restart_kubelet_and_require_lease_renewal "${vm_name}" "${vm_ip}" "${node_uid}" + _restart_daemon_and_require_certificate_access "${vm_ip}" "${server_url}" + validate_node_joined "${vm_name}" + + identity="$(_kubelet_client_identity "${vm_ip}")" + if [[ "${identity}" != *"CN=system:node:${vm_name}"* ]]; then + log_error "Kubelet lost its node client identity after migration: ${identity}" + return 1 + fi + identity="$(_daemon_client_identity "${vm_ip}")" + if [[ "${identity}" != *"CN=system:node:${vm_name}"* || \ + "${identity}" != *"O=aks-flex-node-daemons"* ]]; then + log_error "Daemon lost its issued client identity after migration: ${identity}" + return 1 + fi + if [[ "$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.uid}')" != "${node_uid}" ]]; then + log_error "Historical node object was replaced during migration" + return 1 + fi + + log_success "Historical ${historicalReleaseTag} node upgraded in place, migrated RBAC twice, revoked its token, and survived kubelet/daemon restarts" +} diff --git a/hack/e2e/lib/node-join-kubeadm.sh b/hack/e2e/lib/node-join-kubeadm.sh index edc75282..255c6ef8 100644 --- a/hack/e2e/lib/node-join-kubeadm.sh +++ b/hack/e2e/lib/node-join-kubeadm.sh @@ -36,7 +36,7 @@ _kubeadm_ensure_rbac() { # - ClusterRoleBindings for CSR creation and auto-approval # - Roles/RoleBindings granting bootstrappers read access to kubeadm config # and kubelet config (required by kubeadm join's preflight phase) - # - ClusterRole/ClusterRoleBinding for bootstrappers to GET nodes + # - ClusterRole/ClusterRoleBinding for kubeadm's bootstrap group to GET nodes # - ConfigMaps: cluster-info (kube-public), kubeadm-config and # kubelet-config (kube-system) consumed by kubeadm join if ! kubectl apply -f - < or E2E_BINARY set to an existing file" + return 1 + else + log_info "Skipping build, using: ${E2E_BINARY}" + fi + + # This suite intentionally uses only the token VM. It does not call the + # parallel all-node join path, so an unrelated Arc failure cannot mask the + # historical compatibility result. + infra_deploy + ensure_cluster_dependencies + historical_rbac_migration_e2e +} + # --------------------------------------------------------------------------- # Command: status # --------------------------------------------------------------------------- @@ -267,6 +293,9 @@ main() { all) cmd_all ;; + historical-rbac-migration) + cmd_historical_rbac_migration + ;; infra) if [[ "${SKIP_BUILD}" != "1" ]]; then ensure_binary diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index de52d59b..b36e6bfe 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -1284,6 +1284,8 @@ func TestLoadConfigUsesRPConfigOverLegacyAliases(t *testing.T) { func TestLoadConfigAdaptsLegacyConfigAliases(t *testing.T) { t.Parallel() + // e2eMode was persisted by pre-v0.2 test/dev deployments. Keep accepting + // the removed field so those configs can be upgraded with the current agent. configJSON := `{ "azure": { "targetAgentPoolName": "pool1", @@ -1294,6 +1296,9 @@ func TestLoadConfigAdaptsLegacyConfigAliases(t *testing.T) { "resourceId": "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/test-rg/providers/Microsoft.ContainerService/managedClusters/test-cluster" } }, + "agent": { + "e2eMode": true + }, "kubernetes": { "version": "1.30.1" }, diff --git a/scripts/aks-flex-config b/scripts/aks-flex-config index f4a3c776..1d0b88d8 100755 --- a/scripts/aks-flex-config +++ b/scripts/aks-flex-config @@ -8,15 +8,29 @@ import json import os import secrets import shutil +import socket import subprocess import sys +import time from datetime import datetime, timedelta, timezone from pathlib import Path -from urllib.parse import urlsplit +from urllib import error as urlerror +from urllib import request as urlrequest +from urllib.parse import quote, urlsplit RESOURCE_MANAGER_ENDPOINT = "https://management.azure.com" DEFAULT_AGENT_POOL_NAME = "aksflexnodes" +RBAC_API_GROUP = "rbac.authorization.k8s.io" LEGACY_NODE_ROLE_BINDING = "aks-flex-node-role" +FLEX_NODE_BOOTSTRAP_GROUP = "system:bootstrappers:aks-flex-node" +RBAC_AUTOUPDATE_ANNOTATION = "rbac.authorization.kubernetes.io/autoupdate" +MANAGED_BOOTSTRAP_BINDINGS = ( + ("aks-flex-node-bootstrapper", "system:node-bootstrapper"), + ( + "aks-flex-node-auto-approve-csr", + "system:certificates.k8s.io:certificatesigningrequests:nodeclient", + ), +) def log_info(message: str) -> None: @@ -66,32 +80,348 @@ def setup_node_rbac(args: argparse.Namespace) -> None: require_command("kubectl") load_admin_kubeconfig(args) - log_info("applying bootstrap token RBAC bindings") - run(["kubectl", "apply", "-f", "-"], input_text=RBAC_MANIFEST) - remove_legacy_node_role_binding() + bindings = reconcile_bootstrap_rbac_bindings() + if args.remove_legacy_node_role_binding: + remove_legacy_node_role_binding(bindings) + else: + require_legacy_node_role_binding_absent(bindings) -def remove_legacy_node_role_binding() -> None: - # Older helpers granted bootstrap credentials the broad legacy node role. - # Remove it after the least-privilege CSR bindings are safely in place. - log_info("removing legacy bootstrap node role binding") - run( +def cluster_role_bindings() -> list[dict[str, object]]: + raw = run(["kubectl", "get", "clusterrolebindings", "-o", "json"], capture=True) + try: + payload = json.loads(raw) + except json.JSONDecodeError as err: + raise SystemExit(f"ERROR: could not parse ClusterRoleBinding inventory: {err}") from err + + if not isinstance(payload, dict): + raise SystemExit("ERROR: ClusterRoleBinding inventory is not a JSON object") + items = payload.get("items") + if not isinstance(items, list): + raise SystemExit("ERROR: ClusterRoleBinding inventory does not contain an items list") + if not all(isinstance(item, dict) for item in items): + raise SystemExit("ERROR: ClusterRoleBinding inventory contains a malformed item") + return items + + +def expected_role_ref(role_name: str) -> dict[str, str]: + return {"apiGroup": RBAC_API_GROUP, "kind": "ClusterRole", "name": role_name} + + +def bootstrap_group_subject() -> dict[str, str]: + return { + "apiGroup": RBAC_API_GROUP, + "kind": "Group", + "name": FLEX_NODE_BOOTSTRAP_GROUP, + } + + +def binding_has_bootstrap_group(binding: dict[str, object]) -> bool: + subjects = binding.get("subjects", []) + if subjects is None: + subjects = [] + if not isinstance(subjects, list): + raise SystemExit("ERROR: managed ClusterRoleBinding subjects is not a list") + return any( + isinstance(subject, dict) + and subject.get("apiGroup") == RBAC_API_GROUP + and subject.get("kind") == "Group" + and subject.get("name") == FLEX_NODE_BOOTSTRAP_GROUP + and not subject.get("namespace") + for subject in subjects + ) + + +def managed_binding_inventory(bindings: list[dict[str, object]]) -> dict[str, dict[str, object]]: + expected_names = {name for name, _ in MANAGED_BOOTSTRAP_BINDINGS} + managed: dict[str, dict[str, object]] = {} + for binding in bindings: + metadata = binding.get("metadata") + name = metadata.get("name") if isinstance(metadata, dict) else None + if name not in expected_names: + continue + if name in managed: + raise SystemExit(f"ERROR: duplicate managed ClusterRoleBinding {name!r} in inventory") + managed[name] = binding + return managed + + +def validate_managed_binding( + name: str, + role_name: str, + binding: dict[str, object], + *, + require_subject: bool, +) -> bool: + role_ref = binding.get("roleRef") + wanted_role_ref = expected_role_ref(role_name) + if role_ref != wanted_role_ref: + raise SystemExit( + f"ERROR: refusing to modify managed ClusterRoleBinding {name!r}: roleRef is " + f"{role_ref!r}, expected {wanted_role_ref!r}. The roleRef is immutable and replacing " + "this object could discard operator-managed subjects or metadata. Review it manually." + ) + + has_subject = binding_has_bootstrap_group(binding) + if require_subject and not has_subject: + raise SystemExit( + f"ERROR: managed ClusterRoleBinding {name!r} does not contain the required " + f"bootstrap group {FLEX_NODE_BOOTSTRAP_GROUP!r} after reconciliation" + ) + return has_subject + + +def desired_managed_binding(name: str, role_name: str) -> dict[str, object]: + return { + "apiVersion": f"{RBAC_API_GROUP}/v1", + "kind": "ClusterRoleBinding", + "metadata": {"name": name}, + "roleRef": expected_role_ref(role_name), + "subjects": [bootstrap_group_subject()], + } + + +def reconcile_bootstrap_rbac_bindings() -> list[dict[str, object]]: + before = cluster_role_bindings() + managed = managed_binding_inventory(before) + actions: list[tuple[str, str, dict[str, object]]] = [] + + # Validate every managed name before making any change. A roleRef is + # immutable, so replacing a customized binding would lose operator-managed + # subjects and metadata. + for name, role_name in MANAGED_BOOTSTRAP_BINDINGS: + binding = managed.get(name) + if binding is None: + actions.append(("create", name, desired_managed_binding(name, role_name))) + continue + + has_subject = validate_managed_binding(name, role_name, binding, require_subject=False) + if has_subject: + continue + + metadata = binding.get("metadata") + annotations = metadata.get("annotations", {}) if isinstance(metadata, dict) else {} + if not isinstance(annotations, dict): + raise SystemExit(f"ERROR: managed ClusterRoleBinding {name!r} annotations is not an object") + if str(annotations.get(RBAC_AUTOUPDATE_ANNOTATION, "")).lower() == "false": + raise SystemExit( + f"ERROR: managed ClusterRoleBinding {name!r} disables RBAC autoupdate but is missing " + f"the required bootstrap group {FLEX_NODE_BOOTSTRAP_GROUP!r}. Add the subject " + "manually or remove the autoupdate=false annotation." + ) + resource_version = metadata.get("resourceVersion") if isinstance(metadata, dict) else None + if not isinstance(resource_version, str) or not resource_version: + raise SystemExit( + f"ERROR: refusing to update managed ClusterRoleBinding {name!r} without a " + "resourceVersion concurrency precondition" + ) + + updated = dict(binding) + subjects = binding.get("subjects", []) + if subjects is None: + subjects = [] + updated["apiVersion"] = f"{RBAC_API_GROUP}/v1" + updated["kind"] = "ClusterRoleBinding" + updated["subjects"] = [*subjects, bootstrap_group_subject()] + actions.append(("replace", name, updated)) + + for operation, name, binding in actions: + log_info(f"{operation} managed bootstrap RBAC binding {name}") + run(["kubectl", operation, "-f", "-"], input_text=json.dumps(binding), capture=True) + + after = cluster_role_bindings() + reconciled = managed_binding_inventory(after) + for name, role_name in MANAGED_BOOTSTRAP_BINDINGS: + binding = reconciled.get(name) + if binding is None: + raise SystemExit(f"ERROR: managed ClusterRoleBinding {name!r} is absent after reconciliation") + validate_managed_binding(name, role_name, binding, require_subject=True) + return after + + +def unsafe_node_role_bindings(bindings: list[dict[str, object]] | None = None) -> list[dict[str, object]]: + if bindings is None: + bindings = cluster_role_bindings() + + unsafe = [] + for item in bindings: + if not isinstance(item, dict): + continue + role_ref = item.get("roleRef") + subjects = item.get("subjects") + if not isinstance(role_ref, dict) or not isinstance(subjects, list): + continue + if ( + role_ref.get("apiGroup") != "rbac.authorization.k8s.io" + or role_ref.get("kind") != "ClusterRole" + or role_ref.get("name") != "system:node" + ): + continue + if any( + isinstance(subject, dict) + and subject.get("apiGroup") == "rbac.authorization.k8s.io" + and subject.get("kind") == "Group" + and subject.get("name") == FLEX_NODE_BOOTSTRAP_GROUP + for subject in subjects + ): + unsafe.append(item) + return unsafe + + +def unsafe_binding_names(bindings: list[dict[str, object]]) -> str: + names = [] + for binding in bindings: + metadata = binding.get("metadata") + name = metadata.get("name") if isinstance(metadata, dict) else None + names.append(name if isinstance(name, str) and name else "") + return ", ".join(sorted(names)) + + +def is_canonical_legacy_node_role_binding(binding: dict[str, object]) -> bool: + metadata = binding.get("metadata") + subjects = binding.get("subjects") + if not isinstance(metadata, dict) or metadata.get("name") != LEGACY_NODE_ROLE_BINDING: + return False + return subjects == [ + { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Group", + "name": FLEX_NODE_BOOTSTRAP_GROUP, + } + ] + + +def require_legacy_node_role_binding_absent(bindings: list[dict[str, object]] | None = None) -> None: + bindings = unsafe_node_role_bindings(bindings) + if not bindings: + return + raise SystemExit( + "ERROR: ClusterRoleBinding(s) " + f"{unsafe_binding_names(bindings)} still grant {FLEX_NODE_BOOTSTRAP_GROUP!r} the " + "system:node role. Some older or development agents may still depend on that access. " + "First upgrade them to a release with daemon client certificates (v0.1.1 or later), " + "verify /etc/aks-flex-node/daemon-credentials/client.crt exists and the agent remains " + "healthy after restart, then rerun setup-node-rbac with " + "--remove-legacy-node-role-binding." + ) + + +def delete_cluster_role_binding_with_preconditions(name: str, uid: str, resource_version: str) -> None: + # kubectl delete does not expose UID/resourceVersion preconditions, and its + # --raw mode sends no request body. A short-lived localhost-only proxy lets + # us submit the Kubernetes DeleteOptions body without reimplementing + # kubeconfig authentication or silently deleting a concurrently replaced + # object. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as reservation: + reservation.bind(("127.0.0.1", 0)) + port = reservation.getsockname()[1] + + proxy = subprocess.Popen( [ "kubectl", - "delete", - "clusterrolebinding", - LEGACY_NODE_ROLE_BINDING, - "--ignore-not-found=true", - ] + "proxy", + f"--port={port}", + "--address=127.0.0.1", + r"--accept-hosts=^127\.0\.0\.1$", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, ) + try: + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + if proxy.poll() is not None: + stdout, stderr = proxy.communicate() + detail = (stderr or stdout).strip() + raise SystemExit(f"ERROR: kubectl proxy failed to start: {detail or 'unknown error'}") + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.2): + break + except OSError: + time.sleep(0.1) + else: + raise SystemExit("ERROR: timed out starting kubectl proxy for conditional RBAC deletion") + + body = json.dumps( + { + "apiVersion": "v1", + "kind": "DeleteOptions", + "preconditions": {"uid": uid, "resourceVersion": resource_version}, + } + ).encode() + resource_path = f"/apis/{RBAC_API_GROUP}/v1/clusterrolebindings/{quote(name, safe='')}" + request = urlrequest.Request( + f"http://127.0.0.1:{port}{resource_path}", + data=body, + headers={"Content-Type": "application/json"}, + method="DELETE", + ) + try: + with urlrequest.urlopen(request, timeout=30) as response: + response.read() + except urlerror.HTTPError as err: + detail = err.read().decode(errors="replace").strip() + if err.code == 409: + raise SystemExit( + f"ERROR: refusing to delete ClusterRoleBinding {name!r}: it changed after inspection" + ) from err + raise SystemExit( + f"ERROR: conditional deletion of ClusterRoleBinding {name!r} returned " + f"HTTP {err.code}: {detail}" + ) from err + except urlerror.URLError as err: + raise SystemExit(f"ERROR: conditional deletion of ClusterRoleBinding {name!r} failed: {err}") from err + finally: + proxy.terminate() + try: + proxy.communicate(timeout=5) + except subprocess.TimeoutExpired: + proxy.kill() + proxy.communicate() + + +def remove_legacy_node_role_binding(bindings: list[dict[str, object]] | None = None) -> None: + bindings = unsafe_node_role_bindings(bindings) + if not bindings: + return + if len(bindings) != 1 or not is_canonical_legacy_node_role_binding(bindings[0]): + raise SystemExit( + "ERROR: refusing automatic removal because the unsafe ClusterRoleBinding set is " + f"not the canonical {LEGACY_NODE_ROLE_BINDING!r} object: " + f"{unsafe_binding_names(bindings)}. Review and remove only the bootstrap-group " + "subjects manually." + ) + + metadata = bindings[0].get("metadata") + uid = metadata.get("uid") if isinstance(metadata, dict) else None + resource_version = metadata.get("resourceVersion") if isinstance(metadata, dict) else None + if not isinstance(uid, str) or not uid or not isinstance(resource_version, str) or not resource_version: + raise SystemExit( + "ERROR: refusing automatic removal because the canonical legacy " + "ClusterRoleBinding has no UID/resourceVersion preconditions." + ) + + # Delete only the object version that was inspected above. If another + # actor replaces or edits it between inventory and deletion, the API server + # rejects the request instead of deleting an unreviewed object. + log_info("removing legacy bootstrap node role binding") + delete_cluster_role_binding_with_preconditions(LEGACY_NODE_ROLE_BINDING, uid, resource_version) + remaining = unsafe_node_role_bindings() + if remaining: + raise SystemExit( + "ERROR: unsafe bootstrap node role binding remains after deletion: " + f"{unsafe_binding_names(remaining)}" + ) def generate_bootstrap_token(args: argparse.Namespace) -> str: require_command("kubectl") - # Existing clusters may skip setup-node-rbac after updating this helper. - # Never mint another token while the legacy broad binding may still exist. - remove_legacy_node_role_binding() + # Do not silently break old agents by deleting their binding while rendering + # a config, and never mint another broadly privileged token. Migration is an + # explicit setup-node-rbac action after existing agents have been upgraded. + require_legacy_node_role_binding_absent() log_info("creating bootstrap token") token_id = secrets.token_hex(3) @@ -281,6 +611,14 @@ def build_parser() -> argparse.ArgumentParser: rbac = subparsers.add_parser("setup-node-rbac", help="Reconcile node bootstrap RBAC bindings.") add_cluster_args(rbac) + rbac.add_argument( + "--remove-legacy-node-role-binding", + action="store_true", + help=( + "Remove the canonical obsolete aks-flex-node-role binding after existing agents " + "have issued daemon certificates and remain healthy after restart." + ), + ) rbac.set_defaults(func=setup_node_rbac) generate = subparsers.add_parser("generate-node-config", help="Render a Flex Node config.") @@ -303,35 +641,6 @@ def build_parser() -> argparse.ArgumentParser: return parser -RBAC_MANIFEST = """ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: aks-flex-node-bootstrapper -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:node-bootstrapper -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:bootstrappers:aks-flex-node ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: aks-flex-node-auto-approve-csr -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:certificates.k8s.io:certificatesigningrequests:nodeclient -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:bootstrappers:aks-flex-node -""".lstrip() - - def main() -> None: parser = build_parser() args = parser.parse_args() diff --git a/scripts/aks_flex_config_test.go b/scripts/aks_flex_config_test.go index ed007095..f4e4e16f 100644 --- a/scripts/aks_flex_config_test.go +++ b/scripts/aks_flex_config_test.go @@ -2,6 +2,7 @@ package scripts import ( "bytes" + "encoding/json" "errors" "fmt" "io" @@ -14,20 +15,35 @@ import ( "testing" rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/yaml" ) const ( - bootstrapGroup = "system:bootstrappers:aks-flex-node" - legacyBindingName = "aks-flex-node-role" - legacyNodeRole = "system:node" - fakeDeleteFailure = 37 - fakeCommandLogEnv = "AKS_FLEX_CONFIG_TEST_COMMAND_LOG" - fakeManifestEnv = "AKS_FLEX_CONFIG_TEST_MANIFEST" - fakeLegacyStateEnv = "AKS_FLEX_CONFIG_TEST_LEGACY_STATE" - fakeDeleteExitEnv = "AKS_FLEX_CONFIG_TEST_DELETE_EXIT" - fakeApplyCountEnv = "AKS_FLEX_CONFIG_TEST_APPLY_COUNT" - fakeApplyFailAtEnv = "AKS_FLEX_CONFIG_TEST_APPLY_FAIL_AT" + bootstrapGroup = "system:bootstrappers:aks-flex-node" + bootstrapBindingName = "aks-flex-node-bootstrapper" + bootstrapRole = "system:node-bootstrapper" + approvalBindingName = "aks-flex-node-auto-approve-csr" + approvalRole = "system:certificates.k8s.io:certificatesigningrequests:nodeclient" + legacyBindingName = "aks-flex-node-role" + legacyNodeRole = "system:node" + fakeDeleteFailure = 37 + fakeGetFailure = 39 + fakeCommandLogEnv = "AKS_FLEX_CONFIG_TEST_COMMAND_LOG" + fakeManifestEnv = "AKS_FLEX_CONFIG_TEST_MANIFEST" + fakeManagedStateEnv = "AKS_FLEX_CONFIG_TEST_MANAGED_STATE" + fakeDeleteOptsEnv = "AKS_FLEX_CONFIG_TEST_DELETE_OPTIONS" + fakeLegacyStateEnv = "AKS_FLEX_CONFIG_TEST_LEGACY_STATE" + fakeDeleteExitEnv = "AKS_FLEX_CONFIG_TEST_DELETE_EXIT" + fakeDeleteKeepsEnv = "AKS_FLEX_CONFIG_TEST_DELETE_KEEPS_STATE" + fakeConcurrentEnv = "AKS_FLEX_CONFIG_TEST_CONCURRENT_REPLACE" + fakeConcurrentManagedEnv = "AKS_FLEX_CONFIG_TEST_CONCURRENT_MANAGED_REPLACE" + fakeManagedPostconditionEnv = "AKS_FLEX_CONFIG_TEST_MANAGED_POSTCONDITION" + fakeSkipManagedMutationEnv = "AKS_FLEX_CONFIG_TEST_SKIP_MANAGED_MUTATION" + fakeGetExitEnv = "AKS_FLEX_CONFIG_TEST_GET_EXIT" + fakeApplyCountEnv = "AKS_FLEX_CONFIG_TEST_APPLY_COUNT" + fakeApplyFailAtEnv = "AKS_FLEX_CONFIG_TEST_APPLY_FAIL_AT" ) type commandCall struct { @@ -36,23 +52,31 @@ type commandCall struct { } type configScriptHarness struct { - pythonPath string - scriptPath string - fakeBinDir string - commandLogPath string - manifestPath string - configPath string - legacyState string - deleteExitCode int - applyCountPath string - applyFailAt int + pythonPath string + scriptPath string + fakeBinDir string + commandLogPath string + manifestPath string + managedState string + deleteOptsPath string + configPath string + legacyState string + deleteExitCode int + deleteKeeps bool + concurrentSwap bool + concurrentManaged string + managedPostcondition string + skipManagedMutation bool + getExitCode int + applyCountPath string + applyFailAt int } func TestSetupNodeRBACManifestUsesOnlyBootstrapPermissions(t *testing.T) { t.Parallel() harness := newConfigScriptHarness(t, false, 0) - output, err := harness.runSetupNodeRBAC() + output, err := harness.runSetupNodeRBAC(false) if err != nil { t.Fatalf("setup-node-rbac failed: %v\n%s", err, output) } @@ -103,6 +127,351 @@ func TestSetupNodeRBACManifestUsesOnlyBootstrapPermissions(t *testing.T) { if _, found := seen[legacyBindingName]; found { t.Errorf("RBAC manifest still contains legacy binding %q", legacyBindingName) } + + calls := readCommandCalls(t, harness.commandLogPath) + applyIndexes, _ := kubectlOperationIndexes(calls) + if len(applyIndexes) != 2 { + t.Fatalf("RBAC reconciliation count = %d, want 2; calls: %s", len(applyIndexes), formatCalls(calls)) + } + for _, index := range applyIndexes { + assertSafeManagedRBACMutation(t, calls[index]) + } +} + +func TestSetupNodeRBACRejectsManagedRoleRefDriftBeforeMutation(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + bindingName string + }{ + {name: "bootstrapper binding", bindingName: bootstrapBindingName}, + {name: "approval binding", bindingName: approvalBindingName}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + bootstrapper := managedBindingFixture(bootstrapBindingName, bootstrapRole, true) + approver := managedBindingFixture(approvalBindingName, approvalRole, true) + if test.bindingName == bootstrapBindingName { + bootstrapper.RoleRef.Name = "view" + bootstrapper.Subjects = append(bootstrapper.Subjects, rbacv1.Subject{ + APIGroup: rbacv1.GroupName, + Kind: "Group", + Name: "operator-bootstrapper-group", + }) + } else { + approver.RoleRef.Name = "view" + approver.Subjects = append(approver.Subjects, rbacv1.Subject{ + APIGroup: rbacv1.GroupName, + Kind: "Group", + Name: "operator-approver-group", + }) + } + writeManagedState(t, harness.managedState, bootstrapper, approver) + before := readFile(t, harness.managedState) + + output, err := harness.runSetupNodeRBAC(false) + if err == nil { + t.Fatalf("setup-node-rbac replaced a customized roleRef\n%s", output) + } + if !strings.Contains(output, test.bindingName) || + !strings.Contains(output, "roleRef is immutable") || + !strings.Contains(output, "Review it manually") { + t.Fatalf("failure did not identify the safe manual remediation:\n%s", output) + } + if after := readFile(t, harness.managedState); after != before { + t.Fatalf("managed bindings changed despite preflight failure\nbefore: %s\nafter: %s", before, after) + } + + calls := readCommandCalls(t, harness.commandLogPath) + mutations, deletes := kubectlOperationIndexes(calls) + if len(mutations) != 0 || len(deletes) != 0 { + t.Fatalf("roleRef preflight performed a mutation: %s", formatCalls(calls)) + } + }) + } +} + +func TestSetupNodeRBACPreservesManagedCustomizations(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + bootstrapper := managedBindingFixture(bootstrapBindingName, bootstrapRole, true) + bootstrapper.Labels = map[string]string{"owner": "operator"} + bootstrapper.Annotations = map[string]string{"example.test/note": "keep"} + bootstrapper.Subjects = append(bootstrapper.Subjects, rbacv1.Subject{ + APIGroup: rbacv1.GroupName, + Kind: "Group", + Name: "operator-group", + }) + approver := managedBindingFixture(approvalBindingName, approvalRole, true) + approver.Annotations = map[string]string{"rbac.authorization.kubernetes.io/autoupdate": "false"} + writeManagedState(t, harness.managedState, bootstrapper, approver) + before := readFile(t, harness.managedState) + + output, err := harness.runSetupNodeRBAC(false) + if err != nil { + t.Fatalf("setup-node-rbac rejected valid customized bindings: %v\n%s", err, output) + } + if after := readFile(t, harness.managedState); after != before { + t.Fatalf("already-correct managed bindings changed\nbefore: %s\nafter: %s", before, after) + } + mutations, deletes := kubectlOperationIndexes(readCommandCalls(t, harness.commandLogPath)) + if len(mutations) != 0 || len(deletes) != 0 { + t.Fatalf("already-correct managed bindings were mutated") + } +} + +func TestSetupNodeRBACAddsSubjectWithOptimisticReplace(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + bootstrapper := managedBindingFixture(bootstrapBindingName, bootstrapRole, false) + bootstrapper.Labels = map[string]string{"owner": "operator"} + bootstrapper.Subjects = []rbacv1.Subject{{ + APIGroup: rbacv1.GroupName, + Kind: "Group", + Name: "operator-group", + }} + approver := managedBindingFixture(approvalBindingName, approvalRole, true) + writeManagedState(t, harness.managedState, bootstrapper, approver) + + output, err := harness.runSetupNodeRBAC(false) + if err != nil { + t.Fatalf("setup-node-rbac failed to add the required subject: %v\n%s", err, output) + } + mutations, deletes := kubectlOperationIndexes(readCommandCalls(t, harness.commandLogPath)) + if len(mutations) != 1 || len(deletes) != 0 { + t.Fatalf("managed reconciliation mutation/delete counts = %d/%d, want 1/0", len(mutations), len(deletes)) + } + if call := readCommandCalls(t, harness.commandLogPath)[mutations[0]]; len(call.args) == 0 || call.args[0] != "replace" { + t.Fatalf("managed binding update = %#v, want optimistic kubectl replace", call) + } + + updated := findManagedBinding(t, readManagedState(t, harness.managedState), bootstrapBindingName) + if updated.Labels["owner"] != "operator" { + t.Fatalf("operator metadata was not preserved: %#v", updated.Labels) + } + if !hasSubject(updated.Subjects, "operator-group") || !hasSubject(updated.Subjects, bootstrapGroup) { + t.Fatalf("operator and required subjects were not both preserved: %#v", updated.Subjects) + } + if updated.ResourceVersion != "8" { + t.Fatalf("resourceVersion = %q, want optimistic replacement of version 7", updated.ResourceVersion) + } +} + +func TestSetupNodeRBACRejectsMissingManagedResourceVersion(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + bindingName string + }{ + {name: "bootstrapper binding", bindingName: bootstrapBindingName}, + {name: "approval binding", bindingName: approvalBindingName}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + bootstrapper := managedBindingFixture(bootstrapBindingName, bootstrapRole, true) + approver := managedBindingFixture(approvalBindingName, approvalRole, true) + if test.bindingName == bootstrapBindingName { + bootstrapper.ResourceVersion = "" + bootstrapper.Subjects = nil + } else { + approver.ResourceVersion = "" + approver.Subjects = nil + } + writeManagedState(t, harness.managedState, bootstrapper, approver) + before := readFile(t, harness.managedState) + + output, err := harness.runSetupNodeRBAC(false) + if err == nil { + t.Fatalf("setup-node-rbac replaced a binding without a resourceVersion\n%s", output) + } + if !strings.Contains(output, test.bindingName) || !strings.Contains(output, "resourceVersion") { + t.Fatalf("failure did not identify the missing concurrency precondition:\n%s", output) + } + if after := readFile(t, harness.managedState); after != before { + t.Fatalf("managed bindings changed despite a missing resourceVersion\nbefore: %s\nafter: %s", before, after) + } + + mutations, deletes := kubectlOperationIndexes(readCommandCalls(t, harness.commandLogPath)) + if len(mutations) != 0 || len(deletes) != 0 { + t.Fatalf("missing resourceVersion preflight performed a mutation") + } + }) + } +} + +func TestSetupNodeRBACRespectsAutoupdateFalse(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + includeBootstrap bool + wantFailure bool + }{ + {name: "missing required subject", wantFailure: true}, + {name: "required subject already present", includeBootstrap: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + bootstrapper := managedBindingFixture(bootstrapBindingName, bootstrapRole, test.includeBootstrap) + bootstrapper.Annotations = map[string]string{ + "rbac.authorization.kubernetes.io/autoupdate": "false", + } + approver := managedBindingFixture(approvalBindingName, approvalRole, true) + writeManagedState(t, harness.managedState, bootstrapper, approver) + + output, err := harness.runSetupNodeRBAC(false) + if test.wantFailure && err == nil { + t.Fatalf("setup-node-rbac changed a protected binding\n%s", output) + } + if !test.wantFailure && err != nil { + t.Fatalf("setup-node-rbac rejected a complete protected binding: %v\n%s", err, output) + } + if test.wantFailure && (!strings.Contains(output, "autoupdate") || !strings.Contains(output, "manually")) { + t.Fatalf("failure did not explain protected-binding remediation:\n%s", output) + } + mutations, deletes := kubectlOperationIndexes(readCommandCalls(t, harness.commandLogPath)) + if len(mutations) != 0 || len(deletes) != 0 { + t.Fatalf("protected binding was mutated") + } + }) + } +} + +func TestSetupNodeRBACPreservesConcurrentManagedReplacement(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + bootstrapper := managedBindingFixture(bootstrapBindingName, bootstrapRole, false) + approver := managedBindingFixture(approvalBindingName, approvalRole, true) + writeManagedState(t, harness.managedState, bootstrapper, approver) + harness.concurrentManaged = bootstrapBindingName + + output, err := harness.runSetupNodeRBAC(false) + if err == nil { + t.Fatalf("setup-node-rbac overwrote a concurrently replaced binding\n%s", output) + } + replacement := findManagedBinding(t, readManagedState(t, harness.managedState), bootstrapBindingName) + if replacement.UID != types.UID("operator-replacement") || + replacement.RoleRef.Name != "view" || + !hasSubject(replacement.Subjects, "operator-group") { + t.Fatalf("concurrent operator replacement was not preserved: %#v", replacement) + } +} + +func TestSetupNodeRBACPreservesConcurrentManagedCreate(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + harness.concurrentManaged = bootstrapBindingName + + output, err := harness.runSetupNodeRBAC(false) + if err == nil { + t.Fatalf("setup-node-rbac overwrote a concurrently created binding\n%s", output) + } + replacement := findManagedBinding(t, readManagedState(t, harness.managedState), bootstrapBindingName) + if replacement.UID != types.UID("operator-replacement") || + replacement.RoleRef.Name != "view" || + !hasSubject(replacement.Subjects, "operator-group") { + t.Fatalf("concurrent operator creation was not preserved: %#v", replacement) + } + + mutations, deletes := kubectlOperationIndexes(readCommandCalls(t, harness.commandLogPath)) + if len(mutations) != 1 || len(deletes) != 0 { + t.Fatalf("concurrent create mutation/delete attempts = %d/%d, want 1/0", len(mutations), len(deletes)) + } +} + +func TestSetupNodeRBACChecksManagedPostconditions(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + configure func(*configScriptHarness) + wantSubstring string + }{ + { + name: "binding absent", + configure: func(harness *configScriptHarness) { + harness.skipManagedMutation = true + }, + wantSubstring: "absent after reconciliation", + }, + { + name: "wrong roleRef", + configure: func(harness *configScriptHarness) { + harness.managedPostcondition = bootstrapBindingName + ":wrong-role-ref" + }, + wantSubstring: "roleRef is immutable", + }, + { + name: "required subject missing", + configure: func(harness *configScriptHarness) { + harness.managedPostcondition = bootstrapBindingName + ":missing-subject" + }, + wantSubstring: "does not contain the required bootstrap group", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + test.configure(harness) + output, err := harness.runSetupNodeRBAC(false) + if err == nil { + t.Fatalf("setup-node-rbac succeeded despite a missing postcondition\n%s", output) + } + if !strings.Contains(output, test.wantSubstring) { + t.Fatalf("postcondition failure was not actionable:\n%s", output) + } + }) + } +} + +func TestSetupNodeRBACPreservesLegacyBindingWithoutExplicitMigration(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, 0) + output, err := harness.runSetupNodeRBAC(false) + if err == nil { + t.Fatalf("setup-node-rbac succeeded without explicit migration while legacy binding exists\n%s", output) + } + if !strings.Contains(output, "--remove-legacy-node-role-binding") || !strings.Contains(output, "v0.1.1") { + t.Fatalf("setup-node-rbac did not explain the compatible migration path:\n%s", output) + } + + state, readErr := os.ReadFile(harness.legacyState) + if readErr != nil { + t.Fatalf("read fake legacy state: %v", readErr) + } + if got := strings.TrimSpace(string(state)); got != "present" { + t.Fatalf("legacy binding state = %q, want present until migration is acknowledged", got) + } + + calls := readCommandCalls(t, harness.commandLogPath) + applyIndexes, deleteIndexes := kubectlOperationIndexes(calls) + if len(applyIndexes) != 2 || len(deleteIndexes) != 0 { + t.Fatalf("kubectl mutation/delete counts = %d/%d, want 2/0; calls: %s", len(applyIndexes), len(deleteIndexes), formatCalls(calls)) + } + if getIndexes := kubectlIndexes(calls, "get"); len(getIndexes) != 2 || getIndexes[0] >= applyIndexes[0] || applyIndexes[1] >= getIndexes[1] { + t.Fatalf("safe RBAC must be preflighted and verified before checking legacy migration: %s", formatCalls(calls)) + } + for _, index := range applyIndexes { + assertSafeManagedRBACMutation(t, calls[index]) + } } func TestSetupNodeRBACMigratesLegacyBindingIdempotently(t *testing.T) { @@ -110,7 +479,7 @@ func TestSetupNodeRBACMigratesLegacyBindingIdempotently(t *testing.T) { harness := newConfigScriptHarness(t, true, 0) for run := 1; run <= 2; run++ { - output, err := harness.runSetupNodeRBAC() + output, err := harness.runSetupNodeRBAC(true) if err != nil { t.Fatalf("setup-node-rbac run %d failed: %v\n%s", run, err, output) } @@ -126,22 +495,44 @@ func TestSetupNodeRBACMigratesLegacyBindingIdempotently(t *testing.T) { calls := readCommandCalls(t, harness.commandLogPath) applyIndexes, deleteIndexes := kubectlOperationIndexes(calls) - if len(applyIndexes) != 2 || len(deleteIndexes) != 2 { - t.Fatalf("kubectl apply/delete counts = %d/%d, want 2/2; calls: %s", len(applyIndexes), len(deleteIndexes), formatCalls(calls)) + if len(applyIndexes) != 2 || len(deleteIndexes) != 1 { + t.Fatalf("kubectl apply/delete counts = %d/%d, want 2/1; calls: %s", len(applyIndexes), len(deleteIndexes), formatCalls(calls)) } - for i := range applyIndexes { - if applyIndexes[i] >= deleteIndexes[i] { - t.Errorf("run %d deletes legacy binding before applying safe RBAC; calls: %s", i+1, formatCalls(calls)) - } - assertLegacyDeleteCall(t, calls[deleteIndexes[i]]) + if applyIndexes[0] >= deleteIndexes[0] { + t.Errorf("legacy binding was deleted before safe RBAC was applied: %s", formatCalls(calls)) + } + assertLegacyDeleteCall(t, calls[deleteIndexes[0]]) + assertLegacyDeletePreconditions(t, harness.deleteOptsPath, "legacy-uid", "7") + if getIndexes := kubectlIndexes(calls, "get"); len(getIndexes) != 5 || getIndexes[1] >= deleteIndexes[0] || deleteIndexes[0] >= getIndexes[2] { + t.Fatalf("migration must inventory before and verify after deletion, then stay idempotent: %s", formatCalls(calls)) } } +func TestSetupNodeRBACRejectsConcurrentLegacyBindingReplacement(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, 0) + harness.concurrentSwap = true + output, err := harness.runSetupNodeRBAC(true) + if err == nil { + t.Fatalf("setup-node-rbac deleted a concurrently replaced binding\n%s", output) + } + + state, readErr := os.ReadFile(harness.legacyState) + if readErr != nil { + t.Fatalf("read fake legacy state: %v", readErr) + } + if got := strings.TrimSpace(string(state)); got != "customized" { + t.Fatalf("legacy binding state = %q, want concurrently replaced object preserved", got) + } + assertLegacyDeletePreconditions(t, harness.deleteOptsPath, "legacy-uid", "7") +} + func TestSetupNodeRBACFailsWhenLegacyBindingDeleteFails(t *testing.T) { t.Parallel() harness := newConfigScriptHarness(t, true, fakeDeleteFailure) - output, err := harness.runSetupNodeRBAC() + output, err := harness.runSetupNodeRBAC(true) if err == nil { t.Fatalf("setup-node-rbac succeeded when legacy binding deletion failed\n%s", output) } @@ -149,8 +540,11 @@ func TestSetupNodeRBACFailsWhenLegacyBindingDeleteFails(t *testing.T) { if !errors.As(err, &exitErr) { t.Fatalf("setup-node-rbac error = %T %v, want *exec.ExitError", err, err) } - if got := exitErr.ExitCode(); got != fakeDeleteFailure { - t.Fatalf("setup-node-rbac exit code = %d, want %d\n%s", got, fakeDeleteFailure, output) + if got := exitErr.ExitCode(); got == 0 { + t.Fatalf("setup-node-rbac exit code = %d, want nonzero\n%s", got, output) + } + if !strings.Contains(output, "HTTP 500") { + t.Fatalf("setup-node-rbac did not report the API deletion failure:\n%s", output) } state, readErr := os.ReadFile(harness.legacyState) @@ -163,8 +557,8 @@ func TestSetupNodeRBACFailsWhenLegacyBindingDeleteFails(t *testing.T) { calls := readCommandCalls(t, harness.commandLogPath) applyIndexes, deleteIndexes := kubectlOperationIndexes(calls) - if len(applyIndexes) != 1 || len(deleteIndexes) != 1 { - t.Fatalf("kubectl apply/delete counts = %d/%d, want 1/1; calls: %s", len(applyIndexes), len(deleteIndexes), formatCalls(calls)) + if len(applyIndexes) != 2 || len(deleteIndexes) != 1 { + t.Fatalf("kubectl mutation/delete counts = %d/%d, want 2/1; calls: %s", len(applyIndexes), len(deleteIndexes), formatCalls(calls)) } if applyIndexes[0] >= deleteIndexes[0] { t.Errorf("legacy delete failure occurred before safe RBAC was applied; calls: %s", formatCalls(calls)) @@ -172,30 +566,112 @@ func TestSetupNodeRBACFailsWhenLegacyBindingDeleteFails(t *testing.T) { assertLegacyDeleteCall(t, calls[deleteIndexes[0]]) } -func TestGenerateBootstrapTokenCleansLegacyBindingBeforeMintingToken(t *testing.T) { +func TestSetupNodeRBACFailsWhenLegacyBindingRemainsAfterDelete(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, 0) + harness.deleteKeeps = true + output, err := harness.runSetupNodeRBAC(true) + if err == nil { + t.Fatalf("setup-node-rbac succeeded while unsafe binding remained\n%s", output) + } + if !strings.Contains(output, "remains after deletion") { + t.Fatalf("failure did not report the failed postcondition:\n%s", output) + } + + calls := readCommandCalls(t, harness.commandLogPath) + _, deleteIndexes := kubectlOperationIndexes(calls) + if len(deleteIndexes) != 1 || len(kubectlIndexes(calls, "get")) != 3 { + t.Fatalf("migration did not inventory, delete, and verify: %s", formatCalls(calls)) + } +} + +func TestSetupNodeRBACRefusesAmbiguousUnsafeBindings(t *testing.T) { t.Parallel() tests := []struct { - name string - deleteExitCode int - wantFailure bool + name string + state string }{ - {name: "cleanup succeeds", deleteExitCode: 0}, - {name: "cleanup fails closed", deleteExitCode: fakeDeleteFailure, wantFailure: true}, + {name: "canonical name with additional subject", state: "customized"}, + {name: "unexpected binding name", state: "renamed"}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { t.Parallel() - harness := newConfigScriptHarness(t, true, test.deleteExitCode) + harness := newConfigScriptHarness(t, false, 0) + if err := os.WriteFile(harness.legacyState, []byte(test.state+"\n"), 0o600); err != nil { + t.Fatalf("write fake legacy state: %v", err) + } + output, err := harness.runSetupNodeRBAC(true) + if err == nil { + t.Fatalf("setup-node-rbac removed an ambiguous binding\n%s", output) + } + if !strings.Contains(output, "refusing automatic removal") || !strings.Contains(output, "manually") { + t.Fatalf("failure did not explain manual remediation:\n%s", output) + } + + calls := readCommandCalls(t, harness.commandLogPath) + _, deleteIndexes := kubectlOperationIndexes(calls) + if len(deleteIndexes) != 0 { + t.Fatalf("ambiguous binding was deleted: %s", formatCalls(calls)) + } + }) + } +} + +func TestSetupNodeRBACPreservesSameNamedNonLegacyBinding(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + if err := os.WriteFile(harness.legacyState, []byte("safe-customized\n"), 0o600); err != nil { + t.Fatalf("write fake legacy state: %v", err) + } + output, err := harness.runSetupNodeRBAC(true) + if err != nil { + t.Fatalf("setup-node-rbac rejected a non-legacy same-named binding: %v\n%s", err, output) + } + + state, readErr := os.ReadFile(harness.legacyState) + if readErr != nil { + t.Fatalf("read fake legacy state: %v", readErr) + } + if got := strings.TrimSpace(string(state)); got != "safe-customized" { + t.Fatalf("same-named non-legacy binding state = %q, want preserved", got) + } + if _, deleteIndexes := kubectlOperationIndexes(readCommandCalls(t, harness.commandLogPath)); len(deleteIndexes) != 0 { + t.Fatal("same-named non-legacy binding was deleted") + } +} + +func TestGenerateBootstrapTokenRequiresCompletedLegacyMigration(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + legacyPresent bool + getExitCode int + wantExitCode int + }{ + {name: "migration already complete"}, + {name: "legacy binding present", legacyPresent: true, wantExitCode: 1}, + {name: "legacy state cannot be read", getExitCode: fakeGetFailure, wantExitCode: fakeGetFailure}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, test.legacyPresent, 0) + harness.getExitCode = test.getExitCode output, err := harness.runGenerateNodeConfig() - if test.wantFailure { + if test.wantExitCode != 0 { if err == nil { - t.Fatalf("generate-node-config succeeded when legacy cleanup failed\n%s", output) + t.Fatalf("generate-node-config succeeded before legacy migration completed\n%s", output) } var exitErr *exec.ExitError - if !errors.As(err, &exitErr) || exitErr.ExitCode() != fakeDeleteFailure { - t.Fatalf("generate-node-config error = %v, want exit code %d\n%s", err, fakeDeleteFailure, output) + if !errors.As(err, &exitErr) || exitErr.ExitCode() != test.wantExitCode { + t.Fatalf("generate-node-config error = %v, want exit code %d\n%s", err, test.wantExitCode, output) } } else if err != nil { t.Fatalf("generate-node-config failed: %v\n%s", err, output) @@ -203,22 +679,25 @@ func TestGenerateBootstrapTokenCleansLegacyBindingBeforeMintingToken(t *testing. calls := readCommandCalls(t, harness.commandLogPath) applyIndexes, deleteIndexes := kubectlOperationIndexes(calls) - if len(deleteIndexes) != 1 { - t.Fatalf("kubectl delete count = %d, want 1; calls: %s", len(deleteIndexes), formatCalls(calls)) + getIndexes := kubectlIndexes(calls, "get") + if len(getIndexes) != 1 || len(deleteIndexes) != 0 { + t.Fatalf("kubectl get/delete counts = %d/%d, want 1/0; calls: %s", len(getIndexes), len(deleteIndexes), formatCalls(calls)) } - assertLegacyDeleteCall(t, calls[deleteIndexes[0]]) - if test.wantFailure { + if test.wantExitCode != 0 { if len(applyIndexes) != 0 { - t.Fatalf("token Secret was applied after cleanup failure; calls: %s", formatCalls(calls)) + t.Fatalf("token Secret was applied before legacy migration completed; calls: %s", formatCalls(calls)) } if _, statErr := os.Stat(harness.manifestPath); !errors.Is(statErr, os.ErrNotExist) { t.Fatalf("token manifest exists after cleanup failure: %v", statErr) } + if test.legacyPresent && !strings.Contains(output, "--remove-legacy-node-role-binding") { + t.Fatalf("failure did not explain explicit migration path:\n%s", output) + } return } - if len(applyIndexes) != 1 || deleteIndexes[0] >= applyIndexes[0] { - t.Fatalf("cleanup must precede the single token apply; calls: %s", formatCalls(calls)) + if len(applyIndexes) != 1 || getIndexes[0] >= applyIndexes[0] { + t.Fatalf("legacy-state check must precede the single token apply; calls: %s", formatCalls(calls)) } manifest, readErr := os.ReadFile(harness.manifestPath) if readErr != nil { @@ -235,23 +714,20 @@ func TestKubeadmRBACReconciliation(t *testing.T) { t.Parallel() tests := []struct { - name string - applyFailAt int - deleteExitCode int - wantApplies int - wantDeletes int - wantFailure bool + name string + applyFailAt int + wantApplies int + wantFailure bool }{ - {name: "succeeds", wantApplies: 2, wantDeletes: 1}, + {name: "succeeds", wantApplies: 2}, {name: "initial RBAC apply fails", applyFailAt: 1, wantApplies: 1, wantFailure: true}, - {name: "legacy cleanup fails", deleteExitCode: fakeDeleteFailure, wantApplies: 1, wantDeletes: 1, wantFailure: true}, - {name: "ConfigMap apply fails", applyFailAt: 2, wantApplies: 2, wantDeletes: 1, wantFailure: true}, + {name: "ConfigMap apply fails", applyFailAt: 2, wantApplies: 2, wantFailure: true}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { t.Parallel() - harness := newConfigScriptHarness(t, true, test.deleteExitCode) + harness := newConfigScriptHarness(t, true, 0) harness.applyFailAt = test.applyFailAt output, err := harness.runKubeadmEnsureRBAC() if test.wantFailure && err == nil { @@ -268,18 +744,12 @@ func TestKubeadmRBACReconciliation(t *testing.T) { calls := readCommandCalls(t, harness.commandLogPath) applyIndexes, deleteIndexes := kubectlOperationIndexes(calls) - if len(applyIndexes) != test.wantApplies || len(deleteIndexes) != test.wantDeletes { + if len(applyIndexes) != test.wantApplies || len(deleteIndexes) != 0 { t.Fatalf( "kubectl apply/delete counts = %d/%d, want %d/%d; calls: %s", - len(applyIndexes), len(deleteIndexes), test.wantApplies, test.wantDeletes, formatCalls(calls), + len(applyIndexes), len(deleteIndexes), test.wantApplies, 0, formatCalls(calls), ) } - if len(deleteIndexes) == 1 { - if applyIndexes[0] >= deleteIndexes[0] { - t.Errorf("legacy cleanup ran before safe RBAC apply; calls: %s", formatCalls(calls)) - } - assertLegacyDeleteCall(t, calls[deleteIndexes[0]]) - } }) } } @@ -365,6 +835,10 @@ func newConfigScriptHarness(t *testing.T, legacyPresent bool, deleteExitCode int if err := os.WriteFile(legacyState, []byte(state), 0o600); err != nil { t.Fatalf("write fake legacy state: %v", err) } + managedState := filepath.Join(tempDir, "managed-state.json") + if err := os.WriteFile(managedState, []byte("[]\n"), 0o600); err != nil { + t.Fatalf("write fake managed binding state: %v", err) + } return &configScriptHarness{ pythonPath: pythonPath, @@ -372,6 +846,8 @@ func newConfigScriptHarness(t *testing.T, legacyPresent bool, deleteExitCode int fakeBinDir: fakeBinDir, commandLogPath: filepath.Join(tempDir, "commands.log"), manifestPath: filepath.Join(tempDir, "rbac.yaml"), + managedState: managedState, + deleteOptsPath: filepath.Join(tempDir, "delete-options.json"), configPath: filepath.Join(tempDir, "config.json"), legacyState: legacyState, deleteExitCode: deleteExitCode, @@ -379,23 +855,34 @@ func newConfigScriptHarness(t *testing.T, legacyPresent bool, deleteExitCode int } } -func (h *configScriptHarness) runSetupNodeRBAC() (string, error) { - cmd := exec.Command( - h.pythonPath, +func (h *configScriptHarness) runSetupNodeRBAC(removeLegacy bool) (string, error) { + args := []string{ h.scriptPath, "setup-node-rbac", "--resource-group", "test-rg", "--cluster-name", "test-cluster", "--subscription", "test-subscription", - ) + } + if removeLegacy { + args = append(args, "--remove-legacy-node-role-binding") + } + cmd := exec.Command(h.pythonPath, args...) cmd.Env = append(os.Environ(), "PATH="+h.fakeBinDir+string(os.PathListSeparator)+os.Getenv("PATH"), "KUBECONFIG="+filepath.Join(filepath.Dir(h.fakeBinDir), "kubeconfig"), "PYTHONDONTWRITEBYTECODE=1", fakeCommandLogEnv+"="+h.commandLogPath, fakeManifestEnv+"="+h.manifestPath, + fakeManagedStateEnv+"="+h.managedState, + fakeDeleteOptsEnv+"="+h.deleteOptsPath, fakeLegacyStateEnv+"="+h.legacyState, fmt.Sprintf("%s=%d", fakeDeleteExitEnv, h.deleteExitCode), + fmt.Sprintf("%s=%t", fakeDeleteKeepsEnv, h.deleteKeeps), + fmt.Sprintf("%s=%t", fakeConcurrentEnv, h.concurrentSwap), + fakeConcurrentManagedEnv+"="+h.concurrentManaged, + fakeManagedPostconditionEnv+"="+h.managedPostcondition, + fmt.Sprintf("%s=%t", fakeSkipManagedMutationEnv, h.skipManagedMutation), + fmt.Sprintf("%s=%d", fakeGetExitEnv, h.getExitCode), fakeApplyCountEnv+"="+h.applyCountPath, fmt.Sprintf("%s=%d", fakeApplyFailAtEnv, h.applyFailAt), ) @@ -420,8 +907,15 @@ func (h *configScriptHarness) runGenerateNodeConfig() (string, error) { "PYTHONDONTWRITEBYTECODE=1", fakeCommandLogEnv+"="+h.commandLogPath, fakeManifestEnv+"="+h.manifestPath, + fakeManagedStateEnv+"="+h.managedState, + fakeDeleteOptsEnv+"="+h.deleteOptsPath, fakeLegacyStateEnv+"="+h.legacyState, fmt.Sprintf("%s=%d", fakeDeleteExitEnv, h.deleteExitCode), + fmt.Sprintf("%s=%t", fakeDeleteKeepsEnv, h.deleteKeeps), + fmt.Sprintf("%s=%t", fakeConcurrentEnv, h.concurrentSwap), + fakeConcurrentManagedEnv+"="+h.concurrentManaged, + fmt.Sprintf("%s=%t", fakeSkipManagedMutationEnv, h.skipManagedMutation), + fmt.Sprintf("%s=%d", fakeGetExitEnv, h.getExitCode), fakeApplyCountEnv+"="+h.applyCountPath, fmt.Sprintf("%s=%d", fakeApplyFailAtEnv, h.applyFailAt), ) @@ -448,8 +942,15 @@ func (h *configScriptHarness) runKubeadmEnsureRBAC() (string, error) { "E2E_KUBERNETES_VERSION=1.35.0", fakeCommandLogEnv+"="+h.commandLogPath, fakeManifestEnv+"="+h.manifestPath, + fakeManagedStateEnv+"="+h.managedState, + fakeDeleteOptsEnv+"="+h.deleteOptsPath, fakeLegacyStateEnv+"="+h.legacyState, fmt.Sprintf("%s=%d", fakeDeleteExitEnv, h.deleteExitCode), + fmt.Sprintf("%s=%t", fakeDeleteKeepsEnv, h.deleteKeeps), + fmt.Sprintf("%s=%t", fakeConcurrentEnv, h.concurrentSwap), + fakeConcurrentManagedEnv+"="+h.concurrentManaged, + fmt.Sprintf("%s=%t", fakeSkipManagedMutationEnv, h.skipManagedMutation), + fmt.Sprintf("%s=%d", fakeGetExitEnv, h.getExitCode), fakeApplyCountEnv+"="+h.applyCountPath, fmt.Sprintf("%s=%d", fakeApplyFailAtEnv, h.applyFailAt), ) @@ -457,6 +958,87 @@ func (h *configScriptHarness) runKubeadmEnsureRBAC() (string, error) { return string(output), err } +func managedBindingFixture(name, role string, includeBootstrap bool) rbacv1.ClusterRoleBinding { + subjects := []rbacv1.Subject{} + if includeBootstrap { + subjects = append(subjects, rbacv1.Subject{ + APIGroup: rbacv1.GroupName, + Kind: "Group", + Name: bootstrapGroup, + }) + } + return rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rbacv1.SchemeGroupVersion.String(), + Kind: "ClusterRoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(name + "-uid"), + ResourceVersion: "7", + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: role, + }, + Subjects: subjects, + } +} + +func writeManagedState(t *testing.T, path string, bindings ...rbacv1.ClusterRoleBinding) { + t.Helper() + data, err := json.Marshal(bindings) + if err != nil { + t.Fatalf("marshal managed binding state: %v", err) + } + if err := os.WriteFile(path, append(data, '\n'), 0o600); err != nil { + t.Fatalf("write managed binding state: %v", err) + } +} + +func readManagedState(t *testing.T, path string) []rbacv1.ClusterRoleBinding { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read managed binding state: %v", err) + } + var bindings []rbacv1.ClusterRoleBinding + if err := json.Unmarshal(data, &bindings); err != nil { + t.Fatalf("decode managed binding state: %v", err) + } + return bindings +} + +func findManagedBinding(t *testing.T, bindings []rbacv1.ClusterRoleBinding, name string) rbacv1.ClusterRoleBinding { + t.Helper() + for _, binding := range bindings { + if binding.Name == name { + return binding + } + } + t.Fatalf("managed binding %q not found", name) + return rbacv1.ClusterRoleBinding{} +} + +func hasSubject(subjects []rbacv1.Subject, name string) bool { + for _, subject := range subjects { + if subject.APIGroup == rbacv1.GroupName && subject.Kind == "Group" && subject.Name == name { + return true + } + } + return false +} + +func readFile(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(data) +} + func readRBACManifest(t *testing.T, path string) []rbacv1.ClusterRoleBinding { t.Helper() @@ -512,12 +1094,20 @@ func readCommandCalls(t *testing.T, path string) []commandCall { func kubectlOperationIndexes(calls []commandCall) (apply []int, delete []int) { for i, call := range calls { + if call.name == "http-delete" { + delete = append(delete, i) + continue + } if call.name != "kubectl" || len(call.args) == 0 { continue } switch call.args[0] { - case "apply": + case "apply", "create", "replace": apply = append(apply, i) + case "auth": + if len(call.args) > 1 && call.args[1] == "reconcile" { + apply = append(apply, i) + } case "delete": delete = append(delete, i) } @@ -525,40 +1115,71 @@ func kubectlOperationIndexes(calls []commandCall) (apply []int, delete []int) { return apply, delete } +func kubectlIndexes(calls []commandCall, operation string) []int { + var indexes []int + for i, call := range calls { + if call.name == "kubectl" && len(call.args) > 0 && call.args[0] == operation { + indexes = append(indexes, i) + } + } + return indexes +} + func assertLegacyDeleteCall(t *testing.T, call commandCall) { t.Helper() - if call.name != "kubectl" || len(call.args) < 2 || call.args[0] != "delete" { - t.Fatalf("migration call = %#v, want kubectl delete", call) - } - resourceMatches := false - for i, arg := range call.args[1:] { - if arg == "clusterrolebinding/"+legacyBindingName { - resourceMatches = true - break - } - if (arg == "clusterrolebinding" || arg == "clusterrolebindings") && i+2 < len(call.args) && call.args[i+2] == legacyBindingName { - resourceMatches = true - break - } + if call.name != "http-delete" || len(call.args) != 1 { + t.Fatalf("migration call = %#v, want conditional Kubernetes HTTP DELETE", call) } - if !resourceMatches { - t.Errorf("migration delete args = %q, want ClusterRoleBinding %q", call.args, legacyBindingName) - } - if !containsIgnoreNotFound(call.args) { - t.Errorf("migration delete args = %q, want --ignore-not-found for idempotency", call.args) + wantPath := "/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/" + legacyBindingName + if call.args[0] != wantPath { + t.Errorf("migration delete path = %q, want %q", call.args[0], wantPath) } } -func containsIgnoreNotFound(args []string) bool { +func slicesContain(args []string, want string) bool { for _, arg := range args { - if arg == "--ignore-not-found" || arg == "--ignore-not-found=true" { + if arg == want { return true } } return false } +func assertLegacyDeletePreconditions(t *testing.T, path, wantUID, wantResourceVersion string) { + t.Helper() + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read delete options: %v", err) + } + var options struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Preconditions struct { + UID string `json:"uid"` + ResourceVersion string `json:"resourceVersion"` + } `json:"preconditions"` + } + if err := json.Unmarshal(data, &options); err != nil { + t.Fatalf("decode delete options: %v", err) + } + if options.APIVersion != "v1" || options.Kind != "DeleteOptions" || + options.Preconditions.UID != wantUID || options.Preconditions.ResourceVersion != wantResourceVersion { + t.Fatalf("delete options = %#v, want UID %q and resourceVersion %q", options, wantUID, wantResourceVersion) + } +} + +func assertSafeManagedRBACMutation(t *testing.T, call commandCall) { + t.Helper() + if call.name != "kubectl" || len(call.args) != 3 || (call.args[0] != "create" && call.args[0] != "replace") { + t.Fatalf("RBAC mutation = %#v, want kubectl create/replace -f -", call) + } + if !slicesContain(call.args, "-f") || !slicesContain(call.args, "-") || slicesContain(call.args, "--force") { + t.Errorf("RBAC mutation must be non-destructive and read stdin: %q", call.args) + } +} + func formatCalls(calls []commandCall) string { formatted := make([]string, 0, len(calls)) for _, call := range calls { @@ -605,6 +1226,205 @@ set -eu } >> "${AKS_FLEX_CONFIG_TEST_COMMAND_LOG:?}" case "${1:-}" in +proxy) + port="" + for arg in "$@"; do + case "$arg" in + --port=*) port="${arg#--port=}" ;; + esac + done + if [ -z "$port" ]; then + exit 50 + fi + exec python3 -u - "$port" "${AKS_FLEX_CONFIG_TEST_COMMAND_LOG:?}" "${AKS_FLEX_CONFIG_TEST_DELETE_OPTIONS:?}" "${AKS_FLEX_CONFIG_TEST_LEGACY_STATE:?}" "${AKS_FLEX_CONFIG_TEST_DELETE_EXIT:-0}" "${AKS_FLEX_CONFIG_TEST_DELETE_KEEPS_STATE:-false}" "${AKS_FLEX_CONFIG_TEST_CONCURRENT_REPLACE:-false}" <<'PY' +import http.server +import json +import sys + +port, log_path, options_path, state_path, delete_exit, keep_state, concurrent = sys.argv[1:] + +class Handler(http.server.BaseHTTPRequestHandler): + def do_DELETE(self): + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) + with open(log_path, "a", encoding="utf-8") as stream: + stream.write("http-delete\t" + self.path + "\n") + with open(options_path, "wb") as stream: + stream.write(body) + + if int(delete_exit) != 0: + self.respond(500, {"message": "injected delete failure"}) + return + if concurrent == "true": + with open(state_path, "w", encoding="utf-8") as stream: + stream.write("customized\n") + self.respond(409, {"message": "object changed"}) + return + if self.path != "/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/aks-flex-node-role": + self.respond(404, {"message": "unexpected resource"}) + return + + try: + options = json.loads(body) + except json.JSONDecodeError: + self.respond(400, {"message": "missing DeleteOptions"}) + return + preconditions = options.get("preconditions", {}) + with open(state_path, encoding="utf-8") as stream: + state = stream.read().strip() + if state != "present": + self.respond(404, {"message": "not found"}) + return + if preconditions != {"uid": "legacy-uid", "resourceVersion": "7"}: + self.respond(409, {"message": "precondition failed"}) + return + if keep_state != "true": + with open(state_path, "w", encoding="utf-8") as stream: + stream.write("absent\n") + self.respond(200, {"status": "Success"}) + + def respond(self, status, payload): + encoded = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + def log_message(self, *_): + pass + +http.server.ThreadingHTTPServer(("127.0.0.1", int(port)), Handler).serve_forever() +PY + ;; +create|replace) + apply_count=0 + if [ -f "${AKS_FLEX_CONFIG_TEST_APPLY_COUNT:?}" ]; then + apply_count=$(cat "$AKS_FLEX_CONFIG_TEST_APPLY_COUNT") + fi + apply_count=$((apply_count + 1)) + printf '%s\n' "$apply_count" > "$AKS_FLEX_CONFIG_TEST_APPLY_COUNT" + if [ "${AKS_FLEX_CONFIG_TEST_APPLY_FAIL_AT:-0}" -eq "$apply_count" ]; then + exit 38 + fi + + input="${AKS_FLEX_CONFIG_TEST_MANIFEST:?}.input.$$" + cat > "$input" + if [ -s "${AKS_FLEX_CONFIG_TEST_MANIFEST}" ]; then + printf '%s\n' '---' >> "${AKS_FLEX_CONFIG_TEST_MANIFEST}" + fi + cat "$input" >> "${AKS_FLEX_CONFIG_TEST_MANIFEST}" + printf '\n' >> "${AKS_FLEX_CONFIG_TEST_MANIFEST}" + + status=0 + python3 - "$1" "$input" "${AKS_FLEX_CONFIG_TEST_MANAGED_STATE:?}" "${AKS_FLEX_CONFIG_TEST_CONCURRENT_MANAGED_REPLACE:-}" "${AKS_FLEX_CONFIG_TEST_SKIP_MANAGED_MUTATION:-false}" "${AKS_FLEX_CONFIG_TEST_MANAGED_POSTCONDITION:-}" <<'PY' || status=$? +import json +import sys + +operation, input_path, state_path, concurrent_name, skip_mutation, postcondition = sys.argv[1:] +with open(input_path, encoding="utf-8") as stream: + incoming = json.load(stream) +with open(state_path, encoding="utf-8") as stream: + items = json.load(stream) + +name = incoming.get("metadata", {}).get("name") +matches = [index for index, item in enumerate(items) if item.get("metadata", {}).get("name") == name] +if operation == "create": + if matches: + raise SystemExit(48) + if concurrent_name == name: + replacement = { + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": { + "name": name, + "resourceVersion": "concurrent", + "uid": "operator-replacement", + "labels": {"owner": "operator"}, + }, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "view", + }, + "subjects": [ + { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Group", + "name": "operator-group", + } + ], + } + items.append(replacement) + with open(state_path, "w", encoding="utf-8") as stream: + json.dump(items, stream) + raise SystemExit(48) + incoming.setdefault("metadata", {})["resourceVersion"] = "1" + incoming["metadata"]["uid"] = f"{name}-uid" + if skip_mutation != "true": + items.append(incoming) +elif operation == "replace": + if len(matches) != 1: + raise SystemExit(49) + index = matches[0] + current = items[index] + if concurrent_name == name: + replacement = { + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": { + "name": name, + "resourceVersion": "concurrent", + "uid": "operator-replacement", + "labels": {"owner": "operator"}, + }, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "view", + }, + "subjects": [ + { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Group", + "name": "operator-group", + } + ], + } + items[index] = replacement + with open(state_path, "w", encoding="utf-8") as stream: + json.dump(items, stream) + raise SystemExit(41) + if incoming.get("metadata", {}).get("resourceVersion") != current.get("metadata", {}).get("resourceVersion"): + raise SystemExit(41) + incoming["metadata"]["resourceVersion"] = str(int(current["metadata"]["resourceVersion"]) + 1) + if skip_mutation != "true": + items[index] = incoming +else: + raise SystemExit(47) + +if skip_mutation != "true" and postcondition: + postcondition_name, separator, mutation = postcondition.partition(":") + if not separator: + raise SystemExit(50) + if postcondition_name == name: + target = next(item for item in items if item.get("metadata", {}).get("name") == name) + if mutation == "wrong-role-ref": + target["roleRef"]["name"] = "view" + elif mutation == "missing-subject": + target["subjects"] = [ + subject for subject in target.get("subjects", []) + if subject.get("name") != "system:bootstrappers:aks-flex-node" + ] + else: + raise SystemExit(51) + +with open(state_path, "w", encoding="utf-8") as stream: + json.dump(items, stream) +PY + rm -f "$input" + exit "$status" + ;; apply) apply_count=0 if [ -f "${AKS_FLEX_CONFIG_TEST_APPLY_COUNT:?}" ]; then @@ -617,36 +1437,97 @@ apply) fi cat > "${AKS_FLEX_CONFIG_TEST_MANIFEST:?}" ;; -config) - case " $* " in - *"certificate-authority-data"*) printf 'dGVzdC1jYQ==\n' ;; - *"cluster.server"*) printf 'https://test-cluster.example.test:443\n' ;; - *) exit 45 ;; - esac - ;; -delete) - delete_exit="${AKS_FLEX_CONFIG_TEST_DELETE_EXIT:-0}" - if [ "$delete_exit" -ne 0 ]; then - exit "$delete_exit" +get) + get_exit="${AKS_FLEX_CONFIG_TEST_GET_EXIT:-0}" + if [ "$get_exit" -ne 0 ]; then + exit "$get_exit" fi case " $* " in - *" aks-flex-node-role "*|*" clusterrolebinding/aks-flex-node-role "*) ;; - *) exit 43 ;; + *" clusterrolebindings "*) ;; + *) exit 46 ;; esac - state=$(cat "${AKS_FLEX_CONFIG_TEST_LEGACY_STATE:?}") - if [ "$state" = "present" ]; then - printf 'absent\n' > "$AKS_FLEX_CONFIG_TEST_LEGACY_STATE" - exit 0 - fi + python3 - "${AKS_FLEX_CONFIG_TEST_MANAGED_STATE:?}" "${AKS_FLEX_CONFIG_TEST_LEGACY_STATE:?}" <<'PY' +import json +import sys - for arg in "$@"; do - case "$arg" in - --ignore-not-found|--ignore-not-found=true) exit 0 ;; - esac - done - exit 44 +with open(sys.argv[1], encoding="utf-8") as stream: + items = json.load(stream) +with open(sys.argv[2], encoding="utf-8") as stream: + state = stream.read().strip() + +subject = { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Group", + "name": "system:bootstrappers:aks-flex-node", +} +if state == "present": + items.append({ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": {"name": "aks-flex-node-role", "uid": "legacy-uid", "resourceVersion": "7"}, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "system:node", + }, + "subjects": [subject], + }) +elif state == "customized": + items.append({ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": {"name": "aks-flex-node-role", "uid": "replacement-uid", "resourceVersion": "8"}, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "system:node", + }, + "subjects": [subject, { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Group", + "name": "another-group", + }], + }) +elif state == "renamed": + items.append({ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": {"name": "custom-bootstrap-node-role", "uid": "renamed-uid", "resourceVersion": "9"}, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "system:node", + }, + "subjects": [subject], + }) +elif state == "safe-customized": + items.append({ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": {"name": "aks-flex-node-role"}, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "view", + }, + "subjects": [{ + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Group", + "name": "readers", + }], + }) + +print(json.dumps({"items": items})) +PY + ;; +config) + case " $* " in + *"certificate-authority-data"*) printf 'dGVzdC1jYQ==\n' ;; + *"cluster.server"*) printf 'https://test-cluster.example.test:443\n' ;; + *) exit 45 ;; + esac ;; *) exit 42 From 74014fda9456d89fdf58b43d76931d6f1c9ab357 Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 22 Aug 2026 00:58:04 +0000 Subject: [PATCH 03/18] Fix daemon credential migration checks --- docs/usages/aks-flex-config.md | 4 +-- hack/e2e/lib/bootstrap-rbac-migration.sh | 37 ++++++++++++++---------- scripts/aks-flex-config | 4 +-- scripts/aks_flex_config_test.go | 4 ++- 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/docs/usages/aks-flex-config.md b/docs/usages/aks-flex-config.md index aeb61ded..eafbbbf6 100644 --- a/docs/usages/aks-flex-config.md +++ b/docs/usages/aks-flex-config.md @@ -51,9 +51,9 @@ This applies only the CSR creation and approval `ClusterRoleBinding` objects for `v0.1.1` introduced a separate daemon client certificate, but the version alone does not prove that certificate was issued successfully. Upgrade every bootstrap-token agent to `v0.1.1` or later (preferably the latest release), and on every host verify that the certificate exists, is unexpired, and the agent remains healthy after a restart: ```bash -sudo test -s /etc/aks-flex-node/daemon-credentials/client.crt +sudo test -s /etc/aks-flex-node/daemon-credentials/daemon-controller-current.pem sudo openssl x509 \ - -in /etc/aks-flex-node/daemon-credentials/client.crt \ + -in /etc/aks-flex-node/daemon-credentials/daemon-controller-current.pem \ -noout -subject -enddate -checkend 0 sudo systemctl restart aks-flex-node-agent.service sudo systemctl is-active aks-flex-node-agent.service diff --git a/hack/e2e/lib/bootstrap-rbac-migration.sh b/hack/e2e/lib/bootstrap-rbac-migration.sh index 12cfee7d..93447ae0 100644 --- a/hack/e2e/lib/bootstrap-rbac-migration.sh +++ b/hack/e2e/lib/bootstrap-rbac-migration.sh @@ -26,6 +26,9 @@ readonly historicalInstallerSHA256="c2f7cfc92e62c3a9fb96697b3a3ca170bb8c28d11af8 readonly historicalRootFS="ghcr.io/azure/agent-ubuntu2404:v20260427" readonly legacyNodeRoleBinding="aks-flex-node-role" readonly flexNodeBootstrapGroup="system:bootstrappers:aks-flex-node" +# client-go's rotating FileStore writes the issued certificate and private key +# into this combined PEM. client.crt/client.key are legacy read-only fallbacks. +readonly daemonCredentialPath="/etc/aks-flex-node/daemon-credentials/daemon-controller-current.pem" _historical_artifact_dir() { echo "${E2E_WORK_DIR}/historical-${historicalReleaseTag}" @@ -240,7 +243,7 @@ _install_and_start_historical_node() { remote_copy "${config_file}" "${vm_ip}" "/tmp/config-${historicalReleaseTag}.json" remote_exec "${vm_ip}" \ - "HISTORICAL_TAG=${historicalReleaseTag} HISTORICAL_COMMIT=${historicalCommit:0:7} HISTORICAL_ARCHIVE_SHA256=${historicalArchiveSHA256} HISTORICAL_BINARY_SHA256=${historicalBinarySHA256} HISTORICAL_HELPER_SHA256=${historicalHelperSHA256} HISTORICAL_INSTALLER_SHA256=${historicalInstallerSHA256} E2E_NODE_JOIN_TIMEOUT=${E2E_NODE_JOIN_TIMEOUT} bash -s" <<'REMOTE' + "HISTORICAL_TAG=${historicalReleaseTag} HISTORICAL_COMMIT=${historicalCommit:0:7} HISTORICAL_ARCHIVE_SHA256=${historicalArchiveSHA256} HISTORICAL_BINARY_SHA256=${historicalBinarySHA256} HISTORICAL_HELPER_SHA256=${historicalHelperSHA256} HISTORICAL_INSTALLER_SHA256=${historicalInstallerSHA256} E2E_NODE_JOIN_TIMEOUT=${E2E_NODE_JOIN_TIMEOUT} DAEMON_CREDENTIAL_PATH=${daemonCredentialPath} bash -s" <<'REMOTE' set -euo pipefail archive=/tmp/aks-flex-node-linux-amd64.tar.gz @@ -307,7 +310,7 @@ if [[ -L /usr/local/bin/aks-flex-node ]]; then exit 1 fi printf '%s %s\n' "${HISTORICAL_BINARY_SHA256}" /usr/local/bin/aks-flex-node | sudo sha256sum --check --strict - -if [[ -e /etc/aks-flex-node/daemon-credentials/client.crt ]]; then +if sudo test -e "${DAEMON_CREDENTIAL_PATH}"; then echo "daemon certificate existed before historical bootstrap" >&2 exit 1 fi @@ -350,8 +353,8 @@ if grep -Fq 'running agent daemon in e2e mode' <<<"${historical_logs}"; then echo "${HISTORICAL_TAG} daemon unexpectedly started in E2E mode" >&2 exit 1 fi -if [[ -e /etc/aks-flex-node/daemon-credentials/client.crt ]]; then - echo "${HISTORICAL_TAG} unexpectedly issued a separate daemon certificate" >&2 +if sudo test -e "${DAEMON_CREDENTIAL_PATH}"; then + echo "${HISTORICAL_TAG} unexpectedly issued a daemon certificate" >&2 exit 1 fi REMOTE @@ -467,16 +470,18 @@ REMOTE _daemon_client_identity() { local vm_ip="$1" remote_exec "${vm_ip}" \ - 'sudo openssl x509 -in /etc/aks-flex-node/daemon-credentials/client.crt -noout -subject -nameopt RFC2253' + "sudo openssl x509 -in ${daemonCredentialPath} -noout -subject -nameopt RFC2253" } _require_daemon_certificate_access() { local vm_ip="$1" local server_url="$2" - local quoted_server + local quoted_server quoted_credential printf -v quoted_server '%q' "${server_url}" + printf -v quoted_credential '%q' "${daemonCredentialPath}" - remote_exec "${vm_ip}" "SERVER_URL=${quoted_server} bash -s" <<'REMOTE' + remote_exec "${vm_ip}" \ + "SERVER_URL=${quoted_server} DAEMON_CREDENTIAL_PATH=${quoted_credential} bash -s" <<'REMOTE' set -euo pipefail ca_file="$(mktemp)" trap 'rm -f "${ca_file}"' EXIT @@ -491,8 +496,8 @@ with open(sys.argv[1], 'wb') as stream: stream.write(base64.b64decode(ca_data, validate=True)) PY status="$(sudo curl --silent --show-error \ - --cert /etc/aks-flex-node/daemon-credentials/client.crt \ - --key /etc/aks-flex-node/daemon-credentials/client.key \ + --cert "${DAEMON_CREDENTIAL_PATH}" \ + --key "${DAEMON_CREDENTIAL_PATH}" \ --cacert "${ca_file}" \ --output /dev/null \ --write-out '%{http_code}' \ @@ -508,7 +513,8 @@ _require_old_node_survives_guard() { local vm_name="$1" local vm_ip="$2" - remote_exec "${vm_ip}" 'bash -s' <<'REMOTE' + remote_exec "${vm_ip}" \ + "DAEMON_CREDENTIAL_PATH=${daemonCredentialPath} bash -s" <<'REMOTE' set -euo pipefail sudo systemctl restart aks-flex-node-agent.service for _ in $(seq 1 30); do @@ -528,7 +534,7 @@ if ! sudo systemctl is-active --quiet aks-flex-node-agent.service || sudo systemctl status aks-flex-node-agent.service --no-pager -l >&2 || true exit 1 fi -if [[ -e /etc/aks-flex-node/daemon-credentials/client.crt ]]; then +if sudo test -e "${DAEMON_CREDENTIAL_PATH}"; then echo "historical daemon unexpectedly created a daemon certificate" >&2 exit 1 fi @@ -580,7 +586,7 @@ _upgrade_historical_node_to_head() { remote_copy "${config_file}" "${vm_ip}" /tmp/config-v0.1.0-head.json remote_exec "${vm_ip}" \ - "HISTORICAL_BINARY_SHA256=${historicalBinarySHA256} HEAD_BINARY_SHA256=${head_sha} E2E_NODE_JOIN_TIMEOUT=${E2E_NODE_JOIN_TIMEOUT} bash -s" <<'REMOTE' + "HISTORICAL_BINARY_SHA256=${historicalBinarySHA256} HEAD_BINARY_SHA256=${head_sha} E2E_NODE_JOIN_TIMEOUT=${E2E_NODE_JOIN_TIMEOUT} DAEMON_CREDENTIAL_PATH=${daemonCredentialPath} bash -s" <<'REMOTE' set -euo pipefail candidate=/tmp/aks-flex-node-head current_link=/usr/local/lib/aks-flex-node/aks-flex-node-current @@ -611,8 +617,7 @@ sudo "${candidate}" agent-upgrade | sudo tee /tmp/historical-agent-upgrade.log deadline=$((SECONDS + E2E_NODE_JOIN_TIMEOUT)) while true; do if sudo systemctl is-active --quiet aks-flex-node-agent.service && - [[ -s /etc/aks-flex-node/daemon-credentials/client.crt ]] && - [[ -s /etc/aks-flex-node/daemon-credentials/client.key ]]; then + sudo test -s "${DAEMON_CREDENTIAL_PATH}"; then first_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" sleep 5 second_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" @@ -651,8 +656,8 @@ if [[ "$(sudo readlink -f "/proc/${pid}/exe")" != "${active}" ]]; then exit 1 fi -cert_pub="$(sudo openssl x509 -in /etc/aks-flex-node/daemon-credentials/client.crt -pubkey -noout | sha256sum | awk '{print $1}')" -key_pub="$(sudo openssl pkey -in /etc/aks-flex-node/daemon-credentials/client.key -pubout | sha256sum | awk '{print $1}')" +cert_pub="$(sudo openssl x509 -in "${DAEMON_CREDENTIAL_PATH}" -pubkey -noout | sha256sum | awk '{print $1}')" +key_pub="$(sudo openssl pkey -in "${DAEMON_CREDENTIAL_PATH}" -pubout | sha256sum | awk '{print $1}')" if [[ "${cert_pub}" != "${key_pub}" ]]; then echo "daemon certificate and private key do not match" >&2 exit 1 diff --git a/scripts/aks-flex-config b/scripts/aks-flex-config index 1d0b88d8..b5d268d8 100755 --- a/scripts/aks-flex-config +++ b/scripts/aks-flex-config @@ -300,8 +300,8 @@ def require_legacy_node_role_binding_absent(bindings: list[dict[str, object]] | f"{unsafe_binding_names(bindings)} still grant {FLEX_NODE_BOOTSTRAP_GROUP!r} the " "system:node role. Some older or development agents may still depend on that access. " "First upgrade them to a release with daemon client certificates (v0.1.1 or later), " - "verify /etc/aks-flex-node/daemon-credentials/client.crt exists and the agent remains " - "healthy after restart, then rerun setup-node-rbac with " + "verify /etc/aks-flex-node/daemon-credentials/daemon-controller-current.pem exists " + "and the agent remains healthy after restart, then rerun setup-node-rbac with " "--remove-legacy-node-role-binding." ) diff --git a/scripts/aks_flex_config_test.go b/scripts/aks_flex_config_test.go index f4e4e16f..9b33408b 100644 --- a/scripts/aks_flex_config_test.go +++ b/scripts/aks_flex_config_test.go @@ -449,7 +449,9 @@ func TestSetupNodeRBACPreservesLegacyBindingWithoutExplicitMigration(t *testing. if err == nil { t.Fatalf("setup-node-rbac succeeded without explicit migration while legacy binding exists\n%s", output) } - if !strings.Contains(output, "--remove-legacy-node-role-binding") || !strings.Contains(output, "v0.1.1") { + if !strings.Contains(output, "--remove-legacy-node-role-binding") || + !strings.Contains(output, "v0.1.1") || + !strings.Contains(output, "/etc/aks-flex-node/daemon-credentials/daemon-controller-current.pem") { t.Fatalf("setup-node-rbac did not explain the compatible migration path:\n%s", output) } From 0f61985c9db6f0ec7009cb3ef0e1dd4ce1baa17b Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:29:14 +0000 Subject: [PATCH 04/18] Fix kubectl proxy port allocation race Let kubectl bind port zero atomically, parse its reported port without buffered-pipe hangs, and bound process-group cleanup. Adds startup failure and fragmented-output coverage for review comment 3834577528. --- scripts/aks-flex-config | 93 +++++++++++++++++++++++++-------- scripts/aks_flex_config_test.go | 80 ++++++++++++++++++++++++++-- 2 files changed, 148 insertions(+), 25 deletions(-) diff --git a/scripts/aks-flex-config b/scripts/aks-flex-config index b5d268d8..640f1652 100755 --- a/scripts/aks-flex-config +++ b/scripts/aks-flex-config @@ -6,9 +6,11 @@ from __future__ import annotations import argparse import json import os +import re import secrets +import selectors +import signal import shutil -import socket import subprocess import sys import time @@ -312,36 +314,67 @@ def delete_cluster_role_binding_with_preconditions(name: str, uid: str, resource # us submit the Kubernetes DeleteOptions body without reimplementing # kubeconfig authentication or silently deleting a concurrently replaced # object. - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as reservation: - reservation.bind(("127.0.0.1", 0)) - port = reservation.getsockname()[1] - proxy = subprocess.Popen( [ "kubectl", "proxy", - f"--port={port}", + "--port=0", "--address=127.0.0.1", r"--accept-hosts=^127\.0\.0\.1$", ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, + start_new_session=True, ) try: + if proxy.stdout is None or proxy.stderr is None: + raise SystemExit("ERROR: failed to capture kubectl proxy startup output") + + startup_output = {"stdout": bytearray(), "stderr": bytearray()} + startup_pattern = re.compile(rb"(?:^|\n)Starting to serve on 127\.0\.0\.1:(\d+)\r?\n") + port = None deadline = time.monotonic() + 10 - while time.monotonic() < deadline: - if proxy.poll() is not None: - stdout, stderr = proxy.communicate() - detail = (stderr or stdout).strip() - raise SystemExit(f"ERROR: kubectl proxy failed to start: {detail or 'unknown error'}") - try: - with socket.create_connection(("127.0.0.1", port), timeout=0.2): + with selectors.DefaultSelector() as selector: + selector.register(proxy.stdout, selectors.EVENT_READ, "stdout") + selector.register(proxy.stderr, selectors.EVENT_READ, "stderr") + while time.monotonic() < deadline: + events = selector.select(max(0, deadline - time.monotonic())) + if not events: + break + for key, _ in events: + chunk = os.read(key.fd, 4096) + if not chunk: + selector.unregister(key.fileobj) + continue + stream_output = startup_output[key.data] + stream_output.extend(chunk) + if len(stream_output) > 65536: + del stream_output[:-65536] + if key.data == "stdout": + match = startup_pattern.search(stream_output) + if match: + candidate = int(match.group(1)) + if not 1 <= candidate <= 65535: + raise SystemExit( + f"ERROR: kubectl proxy reported invalid port {candidate}" + ) + port = candidate + break + if port is not None: + break + if proxy.poll() is not None and not selector.get_map(): break - except OSError: - time.sleep(0.1) - else: - raise SystemExit("ERROR: timed out starting kubectl proxy for conditional RBAC deletion") + + if port is None: + detail = b"\n".join( + output.strip() for output in startup_output.values() if output.strip() + ).decode(errors="replace") + if proxy.poll() is None: + raise SystemExit( + "ERROR: timed out starting kubectl proxy for conditional RBAC deletion" + f"{': ' + detail if detail else ''}" + ) + raise SystemExit(f"ERROR: kubectl proxy failed to start: {detail or 'unknown error'}") body = json.dumps( { @@ -373,12 +406,28 @@ def delete_cluster_role_binding_with_preconditions(name: str, uid: str, resource except urlerror.URLError as err: raise SystemExit(f"ERROR: conditional deletion of ClusterRoleBinding {name!r} failed: {err}") from err finally: - proxy.terminate() try: - proxy.communicate(timeout=5) + os.killpg(proxy.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + proxy.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + try: + # The kubectl process may have exited while an exec credential + # plugin kept the process group and pipe descriptors alive. + os.killpg(proxy.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + proxy.wait(timeout=5) except subprocess.TimeoutExpired: - proxy.kill() - proxy.communicate() + log_error(f"kubectl proxy process {proxy.pid} did not exit after SIGKILL") + if proxy.stdout is not None: + proxy.stdout.close() + if proxy.stderr is not None: + proxy.stderr.close() def remove_legacy_node_role_binding(bindings: list[dict[str, object]] | None = None) -> None: diff --git a/scripts/aks_flex_config_test.go b/scripts/aks_flex_config_test.go index 9b33408b..1aa0b19f 100644 --- a/scripts/aks_flex_config_test.go +++ b/scripts/aks_flex_config_test.go @@ -38,6 +38,7 @@ const ( fakeDeleteExitEnv = "AKS_FLEX_CONFIG_TEST_DELETE_EXIT" fakeDeleteKeepsEnv = "AKS_FLEX_CONFIG_TEST_DELETE_KEEPS_STATE" fakeConcurrentEnv = "AKS_FLEX_CONFIG_TEST_CONCURRENT_REPLACE" + fakeProxyStartupEnv = "AKS_FLEX_CONFIG_TEST_PROXY_STARTUP" fakeConcurrentManagedEnv = "AKS_FLEX_CONFIG_TEST_CONCURRENT_MANAGED_REPLACE" fakeManagedPostconditionEnv = "AKS_FLEX_CONFIG_TEST_MANAGED_POSTCONDITION" fakeSkipManagedMutationEnv = "AKS_FLEX_CONFIG_TEST_SKIP_MANAGED_MUTATION" @@ -64,6 +65,7 @@ type configScriptHarness struct { deleteExitCode int deleteKeeps bool concurrentSwap bool + proxyStartup string concurrentManaged string managedPostcondition string skipManagedMutation bool @@ -500,6 +502,22 @@ func TestSetupNodeRBACMigratesLegacyBindingIdempotently(t *testing.T) { if len(applyIndexes) != 2 || len(deleteIndexes) != 1 { t.Fatalf("kubectl apply/delete counts = %d/%d, want 2/1; calls: %s", len(applyIndexes), len(deleteIndexes), formatCalls(calls)) } + proxyIndexes := kubectlIndexes(calls, "proxy") + if len(proxyIndexes) != 1 { + t.Fatalf("conditional deletion started %d kubectl proxies, want 1: %s", len(proxyIndexes), formatCalls(calls)) + } + portArgs := 0 + for _, arg := range calls[proxyIndexes[0]].args { + if strings.HasPrefix(arg, "--port=") { + portArgs++ + if arg != "--port=0" { + t.Errorf("conditional deletion selected proxy port before launch: %q", arg) + } + } + } + if portArgs != 1 { + t.Fatalf("conditional deletion must let kubectl atomically allocate its proxy port: %s", formatCalls(calls)) + } if applyIndexes[0] >= deleteIndexes[0] { t.Errorf("legacy binding was deleted before safe RBAC was applied: %s", formatCalls(calls)) } @@ -588,6 +606,37 @@ func TestSetupNodeRBACFailsWhenLegacyBindingRemainsAfterDelete(t *testing.T) { } } +func TestSetupNodeRBACReportsKubectlProxyStartupFailure(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, 0) + harness.proxyStartup = "exit-before-ready" + output, err := harness.runSetupNodeRBAC(true) + if err == nil { + t.Fatalf("setup-node-rbac succeeded when kubectl proxy exited before readiness\n%s", output) + } + if !strings.Contains(output, "injected proxy startup failure") { + t.Fatalf("setup-node-rbac did not surface kubectl proxy's startup error:\n%s", output) + } + if got := strings.TrimSpace(readFile(t, harness.legacyState)); got != "present" { + t.Fatalf("legacy binding state = %q after proxy startup failure, want present", got) + } +} + +func TestSetupNodeRBACHandlesFragmentedKubectlProxyReadiness(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, 0) + harness.proxyStartup = "fragmented-ready" + output, err := harness.runSetupNodeRBAC(true) + if err != nil { + t.Fatalf("setup-node-rbac did not handle fragmented kubectl proxy output: %v\n%s", err, output) + } + if got := strings.TrimSpace(readFile(t, harness.legacyState)); got != "absent" { + t.Fatalf("legacy binding state = %q after fragmented readiness output, want absent", got) + } +} + func TestSetupNodeRBACRefusesAmbiguousUnsafeBindings(t *testing.T) { t.Parallel() @@ -881,6 +930,7 @@ func (h *configScriptHarness) runSetupNodeRBAC(removeLegacy bool) (string, error fmt.Sprintf("%s=%d", fakeDeleteExitEnv, h.deleteExitCode), fmt.Sprintf("%s=%t", fakeDeleteKeepsEnv, h.deleteKeeps), fmt.Sprintf("%s=%t", fakeConcurrentEnv, h.concurrentSwap), + fakeProxyStartupEnv+"="+h.proxyStartup, fakeConcurrentManagedEnv+"="+h.concurrentManaged, fakeManagedPostconditionEnv+"="+h.managedPostcondition, fmt.Sprintf("%s=%t", fakeSkipManagedMutationEnv, h.skipManagedMutation), @@ -1238,12 +1288,17 @@ proxy) if [ -z "$port" ]; then exit 50 fi - exec python3 -u - "$port" "${AKS_FLEX_CONFIG_TEST_COMMAND_LOG:?}" "${AKS_FLEX_CONFIG_TEST_DELETE_OPTIONS:?}" "${AKS_FLEX_CONFIG_TEST_LEGACY_STATE:?}" "${AKS_FLEX_CONFIG_TEST_DELETE_EXIT:-0}" "${AKS_FLEX_CONFIG_TEST_DELETE_KEEPS_STATE:-false}" "${AKS_FLEX_CONFIG_TEST_CONCURRENT_REPLACE:-false}" <<'PY' + if [ "${AKS_FLEX_CONFIG_TEST_PROXY_STARTUP:-}" = "exit-before-ready" ]; then + printf '%s\n' 'injected proxy startup failure' >&2 + exit 51 + fi + exec python3 -u - "$port" "${AKS_FLEX_CONFIG_TEST_COMMAND_LOG:?}" "${AKS_FLEX_CONFIG_TEST_DELETE_OPTIONS:?}" "${AKS_FLEX_CONFIG_TEST_LEGACY_STATE:?}" "${AKS_FLEX_CONFIG_TEST_DELETE_EXIT:-0}" "${AKS_FLEX_CONFIG_TEST_DELETE_KEEPS_STATE:-false}" "${AKS_FLEX_CONFIG_TEST_CONCURRENT_REPLACE:-false}" "${AKS_FLEX_CONFIG_TEST_PROXY_STARTUP:-}" <<'PY' import http.server import json import sys +import time -port, log_path, options_path, state_path, delete_exit, keep_state, concurrent = sys.argv[1:] +port, log_path, options_path, state_path, delete_exit, keep_state, concurrent, startup_mode = sys.argv[1:] class Handler(http.server.BaseHTTPRequestHandler): def do_DELETE(self): @@ -1296,7 +1351,26 @@ class Handler(http.server.BaseHTTPRequestHandler): def log_message(self, *_): pass -http.server.ThreadingHTTPServer(("127.0.0.1", int(port)), Handler).serve_forever() +server = http.server.ThreadingHTTPServer(("127.0.0.1", int(port)), Handler) +sys.stderr.write("fake proxy diagnostic before readiness\n") +sys.stderr.flush() +readiness = f"Starting to serve on 127.0.0.1:{server.server_address[1]}\n" +if startup_mode == "fragmented-ready": + readiness_prefix = "Starting to " + for fragment in ( + "fake stdout warning before readiness\n" + readiness_prefix, + readiness[len(readiness_prefix):-1], + "\n", + ): + sys.stdout.write(fragment) + sys.stdout.flush() + time.sleep(0.01) +else: + # Keep both lines in one write to catch implementations that mix an OS + # selector with a buffered text reader and strand the readiness line. + sys.stdout.write("fake stdout warning before readiness\n" + readiness) + sys.stdout.flush() +server.serve_forever() PY ;; create|replace) From 9f82ab590da8cf336794ec1a601f929e7bef1bd8 Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:29:18 +0000 Subject: [PATCH 05/18] Fix historical upgrade link assertions Use privileged filesystem checks for managed binary links under the root-owned 0750 installation directory. --- hack/e2e/lib/bootstrap-rbac-migration.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/hack/e2e/lib/bootstrap-rbac-migration.sh b/hack/e2e/lib/bootstrap-rbac-migration.sh index 93447ae0..08bbcbab 100644 --- a/hack/e2e/lib/bootstrap-rbac-migration.sh +++ b/hack/e2e/lib/bootstrap-rbac-migration.sh @@ -273,8 +273,8 @@ version_output="$("${binary}" version)" grep -Fq "Version: ${HISTORICAL_TAG}" <<<"${version_output}" grep -Fq "Git Commit: ${HISTORICAL_COMMIT}" <<<"${version_output}" -if [[ -e /usr/local/lib/aks-flex-node/aks-flex-node-current || \ - -L /usr/local/lib/aks-flex-node/aks-flex-node-current ]]; then +if sudo test -e /usr/local/lib/aks-flex-node/aks-flex-node-current || \ + sudo test -L /usr/local/lib/aks-flex-node/aks-flex-node-current; then echo "historical test VM already has a managed agent layout" >&2 exit 1 fi @@ -596,7 +596,7 @@ service=/etc/systemd/system/aks-flex-node-agent.service chmod 0755 "${candidate}" printf '%s %s\n' "${HEAD_BINARY_SHA256}" "${candidate}" | sha256sum --check --strict - printf '%s %s\n' "${HISTORICAL_BINARY_SHA256}" /usr/local/bin/aks-flex-node | sudo sha256sum --check --strict - -if [[ -e "${current_link}" || -L "${current_link}" ]]; then +if sudo test -e "${current_link}" || sudo test -L "${current_link}"; then echo "managed layout existed before migration preflight" >&2 exit 1 fi @@ -605,7 +605,8 @@ sudo install -m 0600 /tmp/config-v0.1.0-head.json /etc/aks-flex-node/config.json sudo "${candidate}" agent-upgrade --preflight | sudo tee /tmp/historical-agent-upgrade-preflight.log # Preflight must not mutate the direct v0.1.0 installation. -if [[ -e "${current_link}" || -L "${current_link}" || -L /usr/local/bin/aks-flex-node ]]; then +if sudo test -e "${current_link}" || sudo test -L "${current_link}" || \ + sudo test -L /usr/local/bin/aks-flex-node; then echo "agent-upgrade preflight mutated the legacy binary layout" >&2 exit 1 fi @@ -636,7 +637,7 @@ while true; do done for link in /usr/local/bin/aks-flex-node "${current_link}" "${last_good_link}"; do - if [[ ! -L "${link}" ]]; then + if ! sudo test -L "${link}"; then echo "managed binary link missing after upgrade: ${link}" >&2 exit 1 fi From 4a3590c81637f6a95893b8834c604bfdb21dc40b Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:30:30 +0000 Subject: [PATCH 06/18] Bypass proxies for local RBAC deletion Keep the conditional Kubernetes delete on the loopback kubectl proxy even when enterprise proxy variables are set or NO_PROXY is empty. --- scripts/aks-flex-config | 6 +++++- scripts/aks_flex_config_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/scripts/aks-flex-config b/scripts/aks-flex-config index 640f1652..2b7b8d7e 100755 --- a/scripts/aks-flex-config +++ b/scripts/aks-flex-config @@ -391,7 +391,11 @@ def delete_cluster_role_binding_with_preconditions(name: str, uid: str, resource method="DELETE", ) try: - with urlrequest.urlopen(request, timeout=30) as response: + # This request is always to the loopback-only kubectl proxy. Ignore + # workstation proxy variables so an empty/misconfigured NO_PROXY + # cannot redirect a privileged Kubernetes API mutation elsewhere. + local_opener = urlrequest.build_opener(urlrequest.ProxyHandler({})) + with local_opener.open(request, timeout=30) as response: response.read() except urlerror.HTTPError as err: detail = err.read().decode(errors="replace").strip() diff --git a/scripts/aks_flex_config_test.go b/scripts/aks_flex_config_test.go index 1aa0b19f..49dca199 100644 --- a/scripts/aks_flex_config_test.go +++ b/scripts/aks_flex_config_test.go @@ -66,6 +66,7 @@ type configScriptHarness struct { deleteKeeps bool concurrentSwap bool proxyStartup string + poisonHTTPProxy bool concurrentManaged string managedPostcondition string skipManagedMutation bool @@ -637,6 +638,20 @@ func TestSetupNodeRBACHandlesFragmentedKubectlProxyReadiness(t *testing.T) { } } +func TestSetupNodeRBACBypassesEnvironmentProxyForLoopbackDeletion(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, true, 0) + harness.poisonHTTPProxy = true + output, err := harness.runSetupNodeRBAC(true) + if err != nil { + t.Fatalf("setup-node-rbac sent its loopback deletion through the environment proxy: %v\n%s", err, output) + } + if got := strings.TrimSpace(readFile(t, harness.legacyState)); got != "absent" { + t.Fatalf("legacy binding state = %q after loopback deletion, want absent", got) + } +} + func TestSetupNodeRBACRefusesAmbiguousUnsafeBindings(t *testing.T) { t.Parallel() @@ -938,6 +953,18 @@ func (h *configScriptHarness) runSetupNodeRBAC(removeLegacy bool) (string, error fakeApplyCountEnv+"="+h.applyCountPath, fmt.Sprintf("%s=%d", fakeApplyFailAtEnv, h.applyFailAt), ) + if h.poisonHTTPProxy { + cmd.Env = append(cmd.Env, + "HTTP_PROXY=http://127.0.0.1:1", + "http_proxy=http://127.0.0.1:1", + "HTTPS_PROXY=http://127.0.0.1:1", + "https_proxy=http://127.0.0.1:1", + "ALL_PROXY=http://127.0.0.1:1", + "all_proxy=http://127.0.0.1:1", + "NO_PROXY=", + "no_proxy=", + ) + } output, err := cmd.CombinedOutput() return string(output), err } From 5a4fa259a6a97bde57b0779d29d3274f9ec3dbe4 Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:52:06 +0000 Subject: [PATCH 07/18] Clarify legacy RBAC verification --- docs/usages/aks-flex-config.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/usages/aks-flex-config.md b/docs/usages/aks-flex-config.md index eafbbbf6..f61b70e0 100644 --- a/docs/usages/aks-flex-config.md +++ b/docs/usages/aks-flex-config.md @@ -71,13 +71,24 @@ Then explicitly remove the obsolete binding: This migration is idempotent. It automatically deletes only the canonical `aks-flex-node-role` object created by older helpers. If another binding grants the same unsafe edge, or that object has extra subjects, the helper refuses to guess and identifies the objects for manual review. Bootstrap-token config generation refuses to create a token while any such binding exists, rather than either issuing an over-privileged token or unexpectedly breaking an old daemon. -To verify the obsolete binding is gone, run: +To verify no binding still grants the bootstrap group `system:node`, run: ```bash -kubectl get clusterrolebinding aks-flex-node-role +kubectl get clusterrolebinding -o json | jq -r ' + .items[] + | select(.roleRef.kind == "ClusterRole" and .roleRef.name == "system:node") + | .metadata.name as $binding + | .subjects[]? + | select(.kind == "Group" and .name == "system:bootstrappers:aks-flex-node") + | $binding' ``` -The expected result is `NotFound`. Once certificate issuance has been verified, both the kubelet and long-running Flex daemon use issued client certificates, so removing this binding does not interrupt joined nodes. New and in-progress joins retain the CSR permissions installed above. +The expected result is no output. The canonical `aks-flex-node-role` object is +deleted; a safe, repurposed object with that name is preserved. Once certificate +issuance has been verified, both the kubelet and long-running Flex daemon use +issued client certificates, so removing the unsafe binding does not interrupt +joined nodes. New and in-progress joins retain the CSR permissions installed +above. Do not roll back a migrated host to an older or development-mode agent that still uses the bootstrap token for ordinary Kubernetes API requests. After this binding is removed, those requests correctly receive `403 Forbidden`. Restore a supported certificate-using agent instead of restoring the broad binding. From 02247bc87b5a9afcf7006032d311e31d2cbac393 Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:52:09 +0000 Subject: [PATCH 08/18] Test historical RBAC migration across AKS versions --- .github/workflows/e2e-tests.yml | 6 ++++ hack/e2e/README.md | 16 ++++++--- hack/e2e/lib/bootstrap-rbac-migration.sh | 15 +++++--- hack/e2e/lib/infra.sh | 44 ++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 10 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 58fb91c0..9be30270 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -17,6 +17,11 @@ on: options: - all - historical-rbac-migration + kubernetes_version: + description: "Exact AKS/Flex Node Kubernetes patch version (for example, 1.34.9)" + required: false + default: "1.35.0" + type: string skip_cleanup: description: "Skip cleanup (keep resources for debugging)" required: false @@ -66,6 +71,7 @@ env: GITHUB_RUN_ID: ${{ github.run_id }} E2E_WORK_DIR: /tmp/aks-flex-node-e2e-${{ github.run_id }} E2E_SUITE: ${{ inputs.suite || 'all' }} + E2E_KUBERNETES_VERSION: ${{ inputs.kubernetes_version || '1.35.0' }} jobs: e2e: diff --git a/hack/e2e/README.md b/hack/e2e/README.md index 11c86d1c..69ea6b9a 100644 --- a/hack/e2e/README.md +++ b/hack/e2e/README.md @@ -109,7 +109,7 @@ Additional environment variables: | `E2E_SSH_KEY_FILE` | auto-detected | SSH public key used for VM access. | | `E2E_WORK_DIR` | `/tmp/aks-flex-node-e2e` | Working directory for state, configs, and logs. | | `E2E_KUBECONFIG` | `$E2E_WORK_DIR/kubeconfig` | Per-run kubeconfig path. Defaults to an isolated file instead of the runner-global kubeconfig. | -| `E2E_KUBERNETES_VERSION` | `1.35.0` | Kubernetes version used in generated node configs. | +| `E2E_KUBERNETES_VERSION` | `1.35.0` | Exact Kubernetes version used for the AKS control plane, agent pools, and generated node configs. | | `E2E_CONTAINERD_VERSION` | `2.0.4` | Containerd version used in generated node configs. | | `E2E_RUNC_VERSION` | `1.1.12` | Runc version used in generated node configs. | | `E2E_TARGET_AGENT_POOL_NAME` | `aksflexnodes` | Synthetic target agent pool name used by controller-backed test modes. | @@ -149,9 +149,12 @@ Run the focused compatibility suite with: ``` For a manual GitHub Actions run, select `historical-rbac-migration` in the -`suite` workflow input. Infrastructure provisioning, the test, log upload, and -cleanup stay in the same job; the suite deliberately does not call the -Arc-inclusive parallel join path. +`suite` workflow input. Set `kubernetes_version` to an exact AKS version, such +as the latest supported N-1 patch, to exercise the migration against an older +control-plane version. Check regional availability immediately before running +the workflow with `az aks get-versions --location --output table`. +Infrastructure provisioning, the test, log upload, and cleanup stay in the same +job; the suite deliberately does not call the Arc-inclusive parallel join path. The scenario downloads and verifies the official v0.1.0 release archive, extracted binary, helper, and installer. It then uses the pinned helper and @@ -171,7 +174,10 @@ There are two intentional compatibility boundaries: - The test creates a new AKS control plane and reproduces the v0.1.0 cluster-side state. It validates a historical node/config/RBAC migration, not - an AKS control plane that has itself been retained since v0.1.0. + an AKS control plane that has itself been retained since v0.1.0. Selecting an + older `kubernetes_version` proves a newly created control plane at that + version; it still does not reproduce age, prior upgrades, or configuration + drift from a long-lived cluster. - v0.1.0 tokens lack the `kubernetes.azure.com/managedby=aks` label required by the production managed CSR approver. The suite explicitly adopts its known token with that label before the HEAD daemon requests a certificate. The diff --git a/hack/e2e/lib/bootstrap-rbac-migration.sh b/hack/e2e/lib/bootstrap-rbac-migration.sh index 08bbcbab..4a56d9e0 100644 --- a/hack/e2e/lib/bootstrap-rbac-migration.sh +++ b/hack/e2e/lib/bootstrap-rbac-migration.sh @@ -485,15 +485,14 @@ _require_daemon_certificate_access() { set -euo pipefail ca_file="$(mktemp)" trap 'rm -f "${ca_file}"' EXIT -sudo python3 - "${ca_file}" <<'PY' +sudo python3 <<'PY' > "${ca_file}" import base64 import json import sys with open('/etc/aks-flex-node/config.json', encoding='utf-8') as stream: ca_data = json.load(stream)['node']['kubelet']['caCertData'] -with open(sys.argv[1], 'wb') as stream: - stream.write(base64.b64decode(ca_data, validate=True)) +sys.stdout.buffer.write(base64.b64decode(ca_data, validate=True)) PY status="$(sudo curl --silent --show-error \ --cert "${DAEMON_CREDENTIAL_PATH}" \ @@ -743,7 +742,7 @@ historical_rbac_migration_e2e() { log_section "Historical ${historicalReleaseTag} Node and Bootstrap RBAC Migration" local config_file head_config vm_name vm_ip cluster_name resource_group subscription_id - local server_url node_uid identity guard_output token token_id + local server_url node_uid node_kubelet_version identity guard_output token token_id config_file="$(_historical_config_path)" head_config="$(_head_legacy_config_path)" vm_name="$(state_get token_vm_name)" @@ -759,6 +758,12 @@ historical_rbac_migration_e2e() { _install_and_start_historical_node "${vm_ip}" validate_node_joined "${vm_name}" + node_kubelet_version="$(kubectl get node "${vm_name}" -o jsonpath='{.status.nodeInfo.kubeletVersion}')" + node_kubelet_version="${node_kubelet_version#v}" + if [[ "${node_kubelet_version}" != "${E2E_KUBERNETES_VERSION}" ]]; then + log_error "Historical node kubelet version is ${node_kubelet_version}, expected ${E2E_KUBERNETES_VERSION}" + return 1 + fi node_uid="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.uid}')" identity="$(_kubelet_client_identity "${vm_ip}")" if [[ "${identity}" != *"CN=system:node:${vm_name}"* ]]; then @@ -766,7 +771,7 @@ historical_rbac_migration_e2e() { return 1 fi _wait_for_bootstrap_token_probe http:200 "${config_file}" list-nodes - log_success "Official ${historicalReleaseTag} node is Ready with legacy RBAC and an issued kubelet certificate" + log_success "Official ${historicalReleaseTag} node is Ready on Kubernetes ${node_kubelet_version} with legacy RBAC and an issued kubelet certificate" # HEAD must converge the safe CSR bindings but preserve the old daemon until # an operator explicitly confirms the migration. diff --git a/hack/e2e/lib/infra.sh b/hack/e2e/lib/infra.sh index f23271f6..fda84c96 100755 --- a/hack/e2e/lib/infra.sh +++ b/hack/e2e/lib/infra.sh @@ -51,6 +51,11 @@ infra_deploy() { local start start=$(timer_start) + if [[ ! "${E2E_KUBERNETES_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + log_error "E2E_KUBERNETES_VERSION must be an exact x.y.z patch version, got '${E2E_KUBERNETES_VERSION}'" + return 1 + fi + local bicep_file="${E2E_INFRA_DIR}/main.bicep" if [[ ! -f "${bicep_file}" ]]; then log_error "Bicep template not found: ${bicep_file}" @@ -219,9 +224,48 @@ infra_deploy() { infra_get_kubeconfig() { local cluster_name cluster_name="$(state_get cluster_name)" + local cluster_id + cluster_id="$(state_get cluster_id)" local resource_group resource_group="$(state_get resource_group)" + local actual_kubernetes_version system_pool_version flex_pool_version + actual_kubernetes_version="$(az aks show \ + --resource-group "${resource_group}" \ + --name "${cluster_name}" \ + --query 'currentKubernetesVersion || kubernetesVersion' \ + --output tsv)" + if [[ "${actual_kubernetes_version}" != "${E2E_KUBERNETES_VERSION}" ]]; then + log_error "AKS control-plane version is ${actual_kubernetes_version}, expected ${E2E_KUBERNETES_VERSION}" + return 1 + fi + state_set "kubernetes_version" "${actual_kubernetes_version}" + log_info "Verified AKS control-plane version: ${actual_kubernetes_version}" + + system_pool_version="$(az rest \ + --method get \ + --url "https://management.azure.com${cluster_id}/agentPools/system?api-version=2026-05-02-preview" \ + --query 'properties.currentOrchestratorVersion || properties.orchestratorVersion' \ + --output tsv)" + if [[ "${system_pool_version}" != "${E2E_KUBERNETES_VERSION}" ]]; then + log_error "AKS system pool version is ${system_pool_version}, expected ${E2E_KUBERNETES_VERSION}" + return 1 + fi + state_set "system_pool_kubernetes_version" "${system_pool_version}" + log_info "Verified AKS system pool version: ${system_pool_version}" + + flex_pool_version="$(az rest \ + --method get \ + --url "https://management.azure.com${cluster_id}/agentPools/${E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME}?api-version=2026-05-02-preview" \ + --query 'properties.currentOrchestratorVersion || properties.orchestratorVersion' \ + --output tsv)" + if [[ "${flex_pool_version}" != "${E2E_KUBERNETES_VERSION}" ]]; then + log_error "AKS FlexNodes pool version is ${flex_pool_version}, expected ${E2E_KUBERNETES_VERSION}" + return 1 + fi + state_set "flex_pool_kubernetes_version" "${flex_pool_version}" + log_info "Verified AKS FlexNodes pool version: ${flex_pool_version}" + log_info "Fetching kubeconfig for ${cluster_name}..." az aks get-credentials \ --resource-group "${resource_group}" \ From a55e905165f8af921d1282b7f2f3aeaf418fe37a Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 22 Aug 2026 02:38:51 +0000 Subject: [PATCH 09/18] Harden historical migration E2E recovery Persist deployment identity before provisioning, make cleanup verified and idempotent, protect token-bearing files, and add regression tests for state and cleanup failure paths exposed by the live migration run. --- hack/e2e/README.md | 3 + hack/e2e/e2e_scripts_test.go | 818 +++++++++++++++++++++++ hack/e2e/infra/main.bicep | 2 + hack/e2e/infra/modules/vm.bicep | 9 +- hack/e2e/lib/bootstrap-rbac-migration.sh | 19 +- hack/e2e/lib/cleanup.sh | 566 ++++++++++++++-- hack/e2e/lib/common.sh | 219 +++++- hack/e2e/lib/infra.sh | 50 +- hack/e2e/lib/node-join-arc.sh | 4 +- hack/e2e/lib/node-join-kubeadm.sh | 2 +- hack/e2e/lib/node-join-offline.sh | 1 + hack/e2e/lib/node-join-token.sh | 1 + hack/e2e/run.sh | 16 +- 13 files changed, 1634 insertions(+), 76 deletions(-) create mode 100644 hack/e2e/e2e_scripts_test.go diff --git a/hack/e2e/README.md b/hack/e2e/README.md index 69ea6b9a..12e06074 100644 --- a/hack/e2e/README.md +++ b/hack/e2e/README.md @@ -13,6 +13,7 @@ The E2E suite provisions a no-CNI AKS cluster, installs Unbounded-Net as the clu | `ssh` / `scp` | VM access and artifact copy. | | `openssl` | Bootstrap token generation. | | `curl` / `sha256sum` / `tar` | Download and verify pinned historical release artifacts. | +| `flock` | Serialize atomic updates to the per-run cleanup state. | | `docker` | Build and push the controller image into the in-cluster local registry. | | `git` / `make` | Fetch and render Unbounded-Net manifests. | | `go` | Build the agent binary unless `--binary` is supplied. | @@ -137,6 +138,8 @@ Additional environment variables: | `E2E_POD_READY_TIMEOUT` | `120` | Timeout in seconds while waiting for smoke pods. | | `E2E_AGENT_UPGRADE_TIMEOUT` | `300` | Timeout in seconds while waiting for an AgentUpgrade result. | | `E2E_DRIFT_UPGRADE_TIMEOUT` | `900` | Timeout in seconds while waiting for repave. | +| `E2E_CLEANUP_TIMEOUT` | `900` | Shared deadline in seconds for deployment cancellation and Azure resource deletion. | +| `E2E_CLEANUP_POLL_INTERVAL` | `5` | Poll interval in seconds for deployment and cleanup convergence. | | `AZURE_SUBSCRIPTION_ID` | auto-detected | Azure subscription. | | `AZURE_TENANT_ID` | auto-detected | Azure tenant. | diff --git a/hack/e2e/e2e_scripts_test.go b/hack/e2e/e2e_scripts_test.go new file mode 100644 index 00000000..ea979d80 --- /dev/null +++ b/hack/e2e/e2e_scripts_test.go @@ -0,0 +1,818 @@ +package e2e_test + +import ( + "context" + "embed" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// Embedding the scripts makes Go's test cache invalidate on shell-only changes. +// +//go:embed lib/common.sh lib/cleanup.sh lib/bootstrap-rbac-migration.sh +var e2eScripts embed.FS + +func TestRestoreKubernetesVersionFromState(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + configuredVersion string + explicit string + persistedVersion string + liveVersion string + liveSystemVersion string + liveFlexVersion string + lookupFails string + omitSubscription bool + wantVersion string + wantError string + }{ + { + name: "restores persisted version when unset", + configuredVersion: "1.35.0", + explicit: "0", + persistedVersion: "1.34.9", + liveVersion: "1.34.9", + wantVersion: "1.34.9", + }, + { + name: "accepts matching explicit version", + configuredVersion: "1.34.9", + explicit: "1", + persistedVersion: "1.34.9", + liveVersion: "1.34.9", + wantVersion: "1.34.9", + }, + { + name: "rejects mismatched explicit version", + configuredVersion: "1.35.0", + explicit: "1", + persistedVersion: "1.34.9", + liveVersion: "1.34.9", + wantError: "existing cluster state records 1.34.9", + }, + { + name: "rejects invalid persisted version", + configuredVersion: "1.35.0", + explicit: "0", + persistedVersion: "1.34", + liveVersion: "1.34", + wantError: "must be an exact x.y.z patch version", + }, + { + name: "recovers version from legacy state", + configuredVersion: "1.35.0", + explicit: "0", + liveVersion: "1.34.9", + wantVersion: "1.34.9", + }, + { + name: "rejects stale persisted version", + configuredVersion: "1.34.9", + explicit: "0", + persistedVersion: "1.34.9", + liveVersion: "1.35.0", + wantError: "live cluster is 1.35.0", + }, + { + name: "fails closed when live lookup fails", + configuredVersion: "1.35.0", + explicit: "0", + persistedVersion: "1.35.0", + lookupFails: "1", + wantError: "Cannot determine the live Kubernetes version", + }, + { + name: "rejects control plane and pool skew", + configuredVersion: "1.34.9", + explicit: "0", + persistedVersion: "1.34.9", + liveVersion: "1.34.9", + liveSystemVersion: "1.34.9", + liveFlexVersion: "1.35.0", + wantError: "AKS version skew", + }, + { + name: "requires persisted subscription", + configuredVersion: "1.35.0", + explicit: "0", + persistedVersion: "1.35.0", + omitSubscription: true, + wantError: "cluster and subscription", + }, + } + + commonScript := e2eScriptPath(t, "lib", "common.sh") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + liveSystemVersion := tt.liveSystemVersion + if liveSystemVersion == "" { + liveSystemVersion = tt.liveVersion + } + liveFlexVersion := tt.liveFlexVersion + if liveFlexVersion == "" { + liveFlexVersion = tt.liveVersion + } + subscription := "test-subscription" + if tt.omitSubscription { + subscription = "" + } + + script := ` +set -euo pipefail +unset AZURE_SUBSCRIPTION_ID +E2E_WORK_DIR="$1" +source "$2" +E2E_KUBERNETES_VERSION="$3" +_E2E_KUBERNETES_VERSION_EXPLICIT="$4" +LIVE_VERSION="$6" +LIVE_SYSTEM_VERSION="$7" +LIVE_FLEX_VERSION="$8" +LOOKUP_FAILS="$9" +SUBSCRIPTION="${10}" +az() { + [[ "${LOOKUP_FAILS}" != "1" ]] || return 1 + if [[ "$1 $2" == "aks show" ]]; then + [[ "$*" == *"--subscription test-subscription"* ]] || { + echo 'aks lookup omitted persisted subscription' >&2 + return 1 + } + jq -n --arg version "${LIVE_VERSION}" \ + '{id: "/subscriptions/test-subscription/resourceGroups/test-rg/providers/Microsoft.ContainerService/managedClusters/test-aks", version: $version}' + elif [[ "$1" == "rest" && "$*" == *'/agentPools/system?'* ]]; then + printf '%s\n' "${LIVE_SYSTEM_VERSION}" + elif [[ "$1" == "rest" ]]; then + printf '%s\n' "${LIVE_FLEX_VERSION}" + else + return 1 + fi +} +state_set resource_group test-rg +state_set cluster_name test-aks +if [[ -n "${SUBSCRIPTION}" ]]; then + state_set subscription_id "${SUBSCRIPTION}" +fi +if [[ -n "$5" ]]; then + state_set kubernetes_version "$5" +fi +restore_kubernetes_version_from_state +printf 'RESULT=%s\n' "${E2E_KUBERNETES_VERSION}" +` + output, err := runBash(t, script, t.TempDir(), commonScript, + tt.configuredVersion, tt.explicit, tt.persistedVersion, tt.liveVersion, + liveSystemVersion, liveFlexVersion, tt.lookupFails, subscription) + if tt.wantError != "" { + if err == nil { + t.Fatalf("expected error containing %q, got success:\n%s", tt.wantError, output) + } + if !strings.Contains(string(output), tt.wantError) { + t.Fatalf("error output %q does not contain %q", output, tt.wantError) + } + return + } + if err != nil { + t.Fatalf("restore script failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT="+tt.wantVersion+"\n") { + t.Fatalf("output %q does not contain restored version %q", output, tt.wantVersion) + } + }) + } +} + +func TestStateRequiresVerifiedCleanupBeforeReplacement(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + stateJSON string + wantAllow bool + }{ + {name: "missing state", wantAllow: true}, + {name: "empty state", stateJSON: `{}`, wantAllow: true}, + {name: "verified cleanup string", stateJSON: `{"deployment_name":"e2e-old","lifecycle":"cleaned","cleanup_complete":"true"}`, wantAllow: true}, + {name: "verified cleanup boolean", stateJSON: `{"deployment_name":"e2e-old","lifecycle":"cleaned","cleanup_complete":true}`, wantAllow: true}, + {name: "same active deployment", stateJSON: `{"deployment_name":"e2e-new","lifecycle":"provisioning"}`}, + {name: "active prior deployment", stateJSON: `{"deployment_name":"e2e-old","lifecycle":"ready"}`}, + {name: "unknown legacy deployment", stateJSON: `{"cluster_name":"old-aks"}`}, + {name: "invalid json", stateJSON: `{not-json`}, + {name: "non-object json", stateJSON: `[]`}, + } + + commonScript := e2eScriptPath(t, "lib", "common.sh") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + workDir := t.TempDir() + statePath := filepath.Join(workDir, "state.json") + if tt.stateJSON != "" { + if err := os.WriteFile(statePath, []byte(tt.stateJSON), 0o600); err != nil { + t.Fatalf("write initial state: %v", err) + } + } + before, _ := os.ReadFile(statePath) + + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +next_state="$(jq -n '{deployment_name: "e2e-new", lifecycle: "provisioning"}')" +if state_begin_deployment "${next_state}"; then + printf 'DECISION=ALLOWED\n' +else + printf 'DECISION=REJECTED\n' +fi +` + output, err := runBash(t, script, workDir, commonScript) + if err != nil { + t.Fatalf("state guard script failed: %v\n%s", err, output) + } + gotAllow := strings.Contains(string(output), "DECISION=ALLOWED\n") + if gotAllow != tt.wantAllow { + t.Fatalf("allow = %t, want %t; output:\n%s", gotAllow, tt.wantAllow, output) + } + after, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read resulting state: %v", readErr) + } + if !tt.wantAllow && string(after) != string(before) { + t.Fatalf("rejected replacement mutated state: before=%q after=%q", before, after) + } + if tt.wantAllow && !strings.Contains(string(after), `"deployment_name": "e2e-new"`) { + t.Fatalf("allowed replacement did not install new state: %s", after) + } + info, statErr := os.Stat(statePath) + if statErr != nil { + t.Fatalf("stat state file: %v", statErr) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("state file mode = %o, want 600", got) + } + }) + } +} + +func TestConcurrentStateWritesArePrivateAndComplete(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + if err := os.WriteFile(filepath.Join(workDir, "state.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatalf("write initial state: %v", err) + } + commonScript := e2eScriptPath(t, "lib", "common.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +for i in $(seq 1 40); do + state_set "key_${i}" "${i}" & +done +wait +jq -e 'length == 40' "${E2E_STATE_FILE}" +` + if output, err := runBash(t, script, workDir, commonScript); err != nil { + t.Fatalf("concurrent state script failed: %v\n%s", err, output) + } + + info, err := os.Stat(filepath.Join(workDir, "state.json")) + if err != nil { + t.Fatalf("stat state file: %v", err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("state file mode = %o, want 600", got) + } +} + +func TestStateWriteFailurePreservesExistingState(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + statePath := filepath.Join(workDir, "state.json") + const invalidState = `{not-json` + if err := os.WriteFile(statePath, []byte(invalidState), 0o600); err != nil { + t.Fatalf("write invalid state: %v", err) + } + commonScript := e2eScriptPath(t, "lib", "common.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +if state_set test value; then + echo 'unexpected success' >&2 + exit 1 +fi +` + if output, err := runBash(t, script, workDir, commonScript); err != nil { + t.Fatalf("failure-path script failed: %v\n%s", err, output) + } + after, err := os.ReadFile(statePath) + if err != nil { + t.Fatalf("read state: %v", err) + } + if string(after) != invalidState { + t.Fatalf("failed state update replaced recoverable state: %q", after) + } +} + +func TestStateDumpRedactsSecrets(t *testing.T) { + t.Parallel() + + commonScript := e2eScriptPath(t, "lib", "common.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +state_set kubeadm_bootstrap_token abcdef.0123456789abcdef +state_set token_vm_ip 192.0.2.1 +state_dump +` + output, err := runBash(t, script, t.TempDir(), commonScript) + if err != nil { + t.Fatalf("state dump script failed: %v\n%s", err, output) + } + if strings.Contains(string(output), "abcdef.0123456789abcdef") { + t.Fatalf("state dump exposed bootstrap token: %s", output) + } + if !strings.Contains(string(output), "192.0.2.1") { + t.Fatalf("state dump unexpectedly redacted non-secret metadata: %s", output) + } +} + +func TestCancelDeploymentWaitsForTerminalOperations(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + initialState string + terminalState string + cancelFails string + listFails string + wantCancel bool + wantError bool + }{ + {name: "running deployment", initialState: "Running", terminalState: "Canceled", wantCancel: true}, + {name: "already succeeded", initialState: "Succeeded", terminalState: "Succeeded"}, + {name: "cancel races natural completion", initialState: "Running", terminalState: "Succeeded", cancelFails: "1", wantCancel: true}, + {name: "query failure", listFails: "1", wantError: true}, + } + + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + workDir := t.TempDir() + callLog := filepath.Join(workDir, "az-calls.log") + marker := filepath.Join(workDir, "deployment-terminal") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +AZ_CALL_LOG="$3" +AZ_MARKER="$4" +INITIAL_STATE="$5" +TERMINAL_STATE="$6" +CANCEL_FAILS="$7" +LIST_FAILS="$8" +E2E_CLEANUP_TIMEOUT=2 +E2E_CLEANUP_POLL_INTERVAL=0.01 +az() { + printf '%s\n' "$*" >> "${AZ_CALL_LOG}" + if [[ "$1 $2 $3" == "deployment group list" ]]; then + [[ "${LIST_FAILS}" != "1" ]] || return 1 + state="${INITIAL_STATE}" + [[ ! -f "${AZ_MARKER}" ]] || state="${TERMINAL_STATE}" + jq -n --arg state "${state}" '[{name:"e2e-test",properties:{provisioningState:$state}}]' + elif [[ "$1 $2 $3" == "deployment group cancel" ]]; then + : > "${AZ_MARKER}" + [[ "${CANCEL_FAILS}" != "1" ]] + elif [[ "$1 $2 $3 $4" == "deployment operation group list" ]]; then + printf '[{"properties":{"provisioningState":"Succeeded"}}]\n' + else + return 1 + fi +} +if _cancel_active_deployment test-rg e2e-test test-subscription; then + printf 'RESULT=success\n' +else + printf 'RESULT=error\n' +fi +` + output, err := runBash(t, script, workDir, cleanupScript, callLog, marker, + tt.initialState, tt.terminalState, tt.cancelFails, tt.listFails) + if err != nil { + t.Fatalf("cancellation script failed: %v\n%s", err, output) + } + gotError := strings.Contains(string(output), "RESULT=error\n") + if gotError != tt.wantError { + t.Fatalf("error = %t, want %t; output:\n%s", gotError, tt.wantError, output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az call log: %v", readErr) + } + gotCancel := strings.Contains(string(calls), "deployment group cancel") + if gotCancel != tt.wantCancel { + t.Fatalf("cancel call = %t, want %t; calls:\n%s", gotCancel, tt.wantCancel, calls) + } + if !tt.wantError { + operationIndex := strings.Index(string(calls), "deployment operation group list") + if operationIndex < 0 { + t.Fatalf("deployment operations were not verified:\n%s", calls) + } + if tt.wantCancel && operationIndex < strings.Index(string(calls), "deployment group cancel") { + t.Fatalf("operations were checked before cancellation:\n%s", calls) + } + } + }) + } +} + +func TestCleanupIsIdempotentAndWaitsForDependencies(t *testing.T) { + t.Parallel() + + workDir, statePath, callLog, output, err := runCleanup(t, cleanupOptions{runTwice: true}) + _ = workDir + if err != nil { + t.Fatalf("cleanup failed: %v\n%s", err, output) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if !strings.Contains(string(state), `"cleanup_complete": "true"`) || + !strings.Contains(string(state), `"lifecycle": "cleaned"`) { + t.Fatalf("cleanup did not record verified completion: %s", state) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + operationIndex := strings.Index(string(calls), "deployment operation group list") + deleteIndex := strings.Index(string(calls), "vm delete") + if operationIndex < 0 || deleteIndex <= operationIndex { + t.Fatalf("resource deletion began before deployment operations were terminal:\n%s", calls) + } + if strings.Count(string(calls), "resource list") < 6 { + t.Fatalf("cleanup did not perform repeated tagged and exact inventories across two runs:\n%s", calls) + } +} + +func TestCleanupResidualPreventsCleanState(t *testing.T) { + t.Parallel() + + _, statePath, _, output, err := runCleanup(t, cleanupOptions{leaveCluster: true}) + if err != nil { + t.Fatalf("cleanup test harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") { + t.Fatalf("cleanup unexpectedly accepted residual resource:\n%s", output) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if strings.Contains(string(state), `"cleanup_complete": "true"`) || + strings.Contains(string(state), `"lifecycle": "cleaned"`) { + t.Fatalf("failed cleanup was marked clean: %s", state) + } +} + +func TestCleanupUsesExactNamesWithoutRunTag(t *testing.T) { + t.Parallel() + + _, _, callLog, output, err := runCleanup(t, cleanupOptions{noRunTags: true}) + if err != nil { + t.Fatalf("cleanup without tags failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=success") { + t.Fatalf("cleanup without tags did not succeed:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + if strings.Contains(string(calls), "--tag github-run=") { + t.Fatalf("cleanup unexpectedly used an absent run tag:\n%s", calls) + } + if !strings.Contains(string(calls), "resource list --resource-group test-rg --subscription test-subscription --output json") { + t.Fatalf("cleanup skipped exact-name verification:\n%s", calls) + } +} + +func TestCleanupHandlesLegacyStateWithoutVMNames(t *testing.T) { + t.Parallel() + + _, statePath, _, output, err := runCleanup(t, cleanupOptions{noRunTags: true, blankVMNames: true}) + if err != nil { + t.Fatalf("legacy partial-state cleanup failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=success") { + t.Fatalf("legacy partial-state cleanup did not succeed:\n%s", output) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if !strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("legacy partial-state cleanup was not marked complete: %s", state) + } +} + +func TestCleanupDeletesOrphanNodeResourceGroupWhenParentIsAbsent(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{parentGroupAbsent: true}) + if err != nil { + t.Fatalf("orphan cleanup failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=success") { + t.Fatalf("orphan cleanup did not succeed:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + if !strings.Contains(string(calls), "group delete --name MC_aksflex-e2e-test") { + t.Fatalf("orphan node resource group was not deleted:\n%s", calls) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if !strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("orphan cleanup did not record completion: %s", state) + } +} + +func TestCleanupRejectsUnexpectedOrphanNodeResourceGroup(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{ + parentGroupAbsent: true, + unexpectedNodeResource: true, + }) + if err != nil { + t.Fatalf("orphan cleanup test harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") || + !strings.Contains(string(output), "Refusing to delete unexpected persisted AKS node resource group") { + t.Fatalf("cleanup did not reject an unexpected orphan node resource group:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + if strings.Contains(string(calls), "group delete --name production-node-rg") { + t.Fatalf("cleanup attempted to delete an unexpected resource group:\n%s", calls) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("rejected cleanup was marked complete: %s", state) + } +} + +func TestCleanupRejectsUnexpectedArcMachineID(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{unexpectedArcMachineID: true}) + if err != nil { + t.Fatalf("Arc cleanup test harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") || + !strings.Contains(string(output), "Refusing to delete Arc machine with an unexpected persisted resource ID") { + t.Fatalf("cleanup did not reject an unexpected Arc machine ID:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil && !os.IsNotExist(readErr) { + t.Fatalf("read az calls: %v", readErr) + } + if strings.Contains(string(calls), "rest --method delete") { + t.Fatalf("cleanup attempted to delete an unexpected Arc resource:\n%s", calls) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("rejected cleanup was marked complete: %s", state) + } +} + +func TestCleanupQueryFailureDoesNotDeleteResources(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{deploymentQueryFails: true}) + if err != nil { + t.Fatalf("cleanup test harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") { + t.Fatalf("cleanup unexpectedly succeeded:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + if strings.Contains(string(calls), "vm delete") || strings.Contains(string(calls), "aks delete") { + t.Fatalf("cleanup deleted resources after deployment query failure:\n%s", calls) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("query failure was marked clean: %s", state) + } +} + +func TestHistoricalCertificateProbeUsesPrivilegedTemporaryFile(t *testing.T) { + t.Parallel() + + script, err := e2eScripts.ReadFile("lib/bootstrap-rbac-migration.sh") + if err != nil { + t.Fatalf("read embedded migration script: %v", err) + } + for _, required := range []string{ + `ca_file="$(sudo mktemp)"`, + `sudo python3 <<'PY' | sudo tee "${ca_file}" >/dev/null`, + `trap 'sudo rm -f "${ca_file}"' EXIT`, + } { + if !strings.Contains(string(script), required) { + t.Fatalf("migration certificate probe is missing %q", required) + } + } +} + +type cleanupOptions struct { + runTwice bool + leaveCluster bool + parentGroupAbsent bool + deploymentQueryFails bool + noRunTags bool + blankVMNames bool + unexpectedNodeResource bool + unexpectedArcMachineID bool +} + +func runCleanup(t *testing.T, options cleanupOptions) (string, string, string, []byte, error) { + t.Helper() + workDir := t.TempDir() + statePath := filepath.Join(workDir, "state.json") + callLog := filepath.Join(workDir, "az-calls.log") + nodeGroupDeleted := filepath.Join(workDir, "node-group-deleted") + state := `{ + "resource_group": "test-rg", + "subscription_id": "test-subscription", + "deployment_name": "e2e-test", + "run_id": "test-run", + "name_suffix": "test", + "lifecycle": "ready", + "cluster_name": "aks-e2e-test", + "node_resource_group": "MC_aksflex-e2e-test", + "msi_vm_name": "vm-e2e-msi-test", + "token_vm_name": "vm-e2e-token-test", + "offline_vm_name": "vm-e2e-offline-test", + "kubeadm_vm_name": "vm-e2e-kubeadm-test", + "arc_vm_name": "vm-e2e-arc-test", + "arc_machine_name": "", + "vnet_name": "vnet-e2e-test", + "nsg_name": "nsg-e2e-test" +}` + if options.noRunTags { + state = strings.Replace(state, `"run_id": "test-run"`, `"run_id": ""`, 1) + } + if options.blankVMNames { + for _, name := range []string{ + "vm-e2e-msi-test", + "vm-e2e-token-test", + "vm-e2e-offline-test", + "vm-e2e-kubeadm-test", + "vm-e2e-arc-test", + } { + state = strings.ReplaceAll(state, name, "") + } + } + if options.unexpectedNodeResource { + state = strings.Replace(state, `"node_resource_group": "MC_aksflex-e2e-test"`, `"node_resource_group": "production-node-rg"`, 1) + } + if options.unexpectedArcMachineID { + state = strings.Replace(state, `"arc_machine_name": ""`, `"arc_machine_name": "vm-e2e-arc-test-connected", + "arc_machine_id": "/subscriptions/test-subscription/resourceGroups/production-rg/providers/Microsoft.HybridCompute/machines/production-machine"`, 1) + } + if err := os.WriteFile(statePath, []byte(state), 0o600); err != nil { + t.Fatalf("write cleanup state: %v", err) + } + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +AZ_CALL_LOG="$3" +NODE_GROUP_DELETED="$4" +LEAVE_CLUSTER="$5" +PARENT_GROUP_ABSENT="$6" +DEPLOYMENT_QUERY_FAILS="$7" +RUN_TWICE="$8" +AZURE_SUBSCRIPTION_ID=test-subscription +E2E_SKIP_CLEANUP=0 +E2E_CLEANUP_TIMEOUT=5 +E2E_CLEANUP_POLL_INTERVAL=0.01 +az() { + printf '%s\n' "$*" >> "${AZ_CALL_LOG}" + if [[ "$1 $2 $3" == "deployment group list" ]]; then + [[ "${DEPLOYMENT_QUERY_FAILS}" != "1" ]] || return 1 + printf '[{"name":"e2e-test","properties":{"provisioningState":"Succeeded"}}]\n' + elif [[ "$1 $2 $3 $4" == "deployment operation group list" ]]; then + printf '[{"properties":{"provisioningState":"Succeeded"}}]\n' + elif [[ "$1 $2" == "group exists" ]]; then + if [[ "$*" == *"--name test-rg"* ]]; then + [[ "${PARENT_GROUP_ABSENT}" == "1" ]] && printf 'false\n' || printf 'true\n' + elif [[ "$*" == *"--name MC_aksflex-e2e-test"* ]]; then + [[ -f "${NODE_GROUP_DELETED}" ]] && printf 'false\n' || printf 'true\n' + else + printf 'false\n' + fi + elif [[ "$1 $2" == "group delete" ]]; then + : > "${NODE_GROUP_DELETED}" + elif [[ "$1 $2" == "group wait" ]]; then + return 0 + elif [[ "$1 $2" == "vm list" || "$1 $2" == "aks list" ]]; then + printf '[]\n' + elif [[ "$1 $2" == "resource list" ]]; then + if [[ "$*" == *"--tag "* ]]; then + [[ "$*" == *"--output json"* ]] && printf '[]\n' || true + elif [[ "$*" == *"--output json"* ]]; then + if [[ "${LEAVE_CLUSTER}" == "1" ]]; then + printf '[{"name":"aks-e2e-test","id":"/subscriptions/test/resourceGroups/test-rg/providers/Microsoft.ContainerService/managedClusters/aks-e2e-test"}]\n' + else + printf '[]\n' + fi + fi + else + return 0 + fi +} +run_cleanup() { + if cleanup; then + printf 'RESULT=success\n' + else + printf 'RESULT=error\n' + fi +} +run_cleanup +if [[ "${RUN_TWICE}" == "1" ]]; then + run_cleanup +fi +` + output, err := runBash(t, script, workDir, cleanupScript, callLog, nodeGroupDeleted, + boolString(options.leaveCluster), boolString(options.parentGroupAbsent), + boolString(options.deploymentQueryFails), boolString(options.runTwice)) + return workDir, statePath, callLog, output, err +} + +func boolString(value bool) string { + if value { + return "1" + } + return "0" +} + +func e2eScriptPath(t *testing.T, elements ...string) string { + t.Helper() + root := t.TempDir() + for _, name := range []string{"common.sh", "cleanup.sh", "bootstrap-rbac-migration.sh"} { + contents, err := e2eScripts.ReadFile(filepath.ToSlash(filepath.Join("lib", name))) + if err != nil { + t.Fatalf("read embedded %s: %v", name, err) + } + path := filepath.Join(root, "lib", name) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("create embedded script directory: %v", err) + } + if err := os.WriteFile(path, contents, 0o700); err != nil { + t.Fatalf("materialize embedded %s: %v", name, err) + } + } + return filepath.Join(append([]string{root}, elements...)...) +} + +func runBash(t *testing.T, script string, args ...string) ([]byte, error) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + commandArgs := append([]string{"-c", script, "e2e-script-test"}, args...) + cmd := exec.CommandContext(ctx, "bash", commandArgs...) + output, err := cmd.CombinedOutput() + if ctx.Err() != nil { + t.Fatalf("shell test timed out: %v\n%s", ctx.Err(), output) + } + return output, err +} diff --git a/hack/e2e/infra/main.bicep b/hack/e2e/infra/main.bicep index af7d9e0c..3f4f8ce1 100644 --- a/hack/e2e/infra/main.bicep +++ b/hack/e2e/infra/main.bicep @@ -52,6 +52,7 @@ var kubeadmVmName = 'vm-e2e-kubeadm-${nameSuffix}' var arcVmName = 'vm-e2e-arc-${nameSuffix}' var vnetName = 'vnet-e2e-${nameSuffix}' var nsgName = 'nsg-e2e-${nameSuffix}' +var nodeResourceGroupName = 'MC_aksflex-e2e-${nameSuffix}' var subnetAksName = 'snet-aks' var subnetVmName = 'snet-vm' @@ -125,6 +126,7 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-01-01' = { } properties: { dnsPrefix: clusterName + nodeResourceGroup: nodeResourceGroupName kubernetesVersion: kubernetesVersion enableRBAC: true aadProfile: { diff --git a/hack/e2e/infra/modules/vm.bicep b/hack/e2e/infra/modules/vm.bicep index 6c90df49..1a5dde17 100644 --- a/hack/e2e/infra/modules/vm.bicep +++ b/hack/e2e/infra/modules/vm.bicep @@ -120,12 +120,19 @@ resource vm 'Microsoft.Compute/virtualMachines@2024-03-01' = { version: imageVersion } osDisk: { + name: '${vmName}-osdisk' createOption: 'FromImage' + deleteOption: 'Delete' managedDisk: { storageAccountType: 'StandardSSD_LRS' } } } networkProfile: { - networkInterfaces: [ { id: nic.id } ] + networkInterfaces: [ + { + id: nic.id + properties: { deleteOption: 'Delete' } + } + ] } } } diff --git a/hack/e2e/lib/bootstrap-rbac-migration.sh b/hack/e2e/lib/bootstrap-rbac-migration.sh index 4a56d9e0..1ffd0d8b 100644 --- a/hack/e2e/lib/bootstrap-rbac-migration.sh +++ b/hack/e2e/lib/bootstrap-rbac-migration.sh @@ -162,6 +162,10 @@ _generate_historical_config() { --subscription "${subscription_id}" log_info "Generating a real non-expiring ${historicalReleaseTag} bootstrap token and config" + # The v0.1.0 helper predates the current secure-output implementation. Create + # the destination first so the bootstrap token is never briefly world-readable + # under a permissive runner umask. + install -m 0600 /dev/null "${config_file}" with_cluster_lock python3 "${helper}" generate-node-config \ --resource-group "${resource_group}" \ --cluster-name "${cluster_name}" \ @@ -169,6 +173,7 @@ _generate_historical_config() { --bootstrap-token \ --output "${config_file}" + install -m 0600 /dev/null "${config_file}.tmp" jq \ --arg nodeName "${vm_name}" \ --arg nodeIP "${vm_private_ip}" \ @@ -483,9 +488,12 @@ _require_daemon_certificate_access() { remote_exec "${vm_ip}" \ "SERVER_URL=${quoted_server} DAEMON_CREDENTIAL_PATH=${quoted_credential} bash -s" <<'REMOTE' set -euo pipefail -ca_file="$(mktemp)" -trap 'rm -f "${ca_file}"' EXIT -sudo python3 <<'PY' > "${ca_file}" +# The credential config is root-only. Keep the derived CA root-only as well and +# write it through a privileged process; redirecting sudo's stdout to a regular +# user's mktemp file fails on hardened hosts. +ca_file="$(sudo mktemp)" +trap 'sudo rm -f "${ca_file}"' EXIT +sudo python3 <<'PY' | sudo tee "${ca_file}" >/dev/null import base64 import json import sys @@ -545,6 +553,7 @@ _prepare_head_legacy_config() { local source_config="$1" local target_config="$2" + install -m 0600 /dev/null "${target_config}.tmp" jq \ --arg machineEndpointURL "${E2E_CONTROLLER_SERVICE_PROXY_PATH}" \ --arg agentPoolName "${E2E_TARGET_AGENT_POOL_NAME}" \ @@ -555,8 +564,8 @@ _prepare_head_legacy_config() { | .agent.machineOperationMode = "disable" | .azure.targetAgentPoolName = $agentPoolName | .bootstrap.ociImage = $ociImage' \ - "${source_config}" > "${target_config}" - chmod 0600 "${target_config}" + "${source_config}" > "${target_config}.tmp" + mv "${target_config}.tmp" "${target_config}" if ! jq -e \ --arg endpoint "${E2E_CONTROLLER_SERVICE_PROXY_PATH}" \ diff --git a/hack/e2e/lib/cleanup.sh b/hack/e2e/lib/cleanup.sh index 1ebde191..f1c749a9 100755 --- a/hack/e2e/lib/cleanup.sh +++ b/hack/e2e/lib/cleanup.sh @@ -178,6 +178,205 @@ collect_logs() { # --------------------------------------------------------------------------- # cleanup - Delete Azure resources # --------------------------------------------------------------------------- +_deployment_state() { + local resource_group="$1" deployment_name="$2" subscription_id="$3" + local deployments + + deployments="$(az deployment group list \ + --resource-group "${resource_group}" \ + --subscription "${subscription_id}" \ + --output json 2>/dev/null)" || return 1 + jq -er --arg name "${deployment_name}" ' + if type != "array" then error("deployment list must be an array") + else ([.[] | select(.name == $name)][0].properties.provisioningState // "") + end + ' <<<"${deployments}" +} + +_wait_for_deployment_operations() { + local resource_group="$1" deployment_name="$2" subscription_id="$3" deadline="$4" + local operations + + while (( SECONDS < deadline )); do + if ! operations="$(az deployment operation group list \ + --resource-group "${resource_group}" \ + --name "${deployment_name}" \ + --subscription "${subscription_id}" \ + --output json 2>/dev/null)"; then + log_error "Failed to query operations for ARM deployment '${deployment_name}'" + return 1 + fi + if jq -e ' + type == "array" and all(.[].properties.provisioningState; + . == "Succeeded" or . == "Failed" or . == "Canceled" or + . == "Cancelled" or . == "Skipped") + ' <<<"${operations}" >/dev/null; then + return 0 + fi + sleep "${E2E_CLEANUP_POLL_INTERVAL:-5}" + done + + log_error "ARM deployment '${deployment_name}' still has active operations after the cleanup timeout" + return 1 +} + +_cancel_active_deployment() { + local resource_group="$1" deployment_name="$2" subscription_id="$3" + local deadline="${4:-$((SECONDS + E2E_CLEANUP_TIMEOUT))}" + local deployment_state + + [[ -n "${deployment_name}" ]] || return 0 + if ! deployment_state="$(_deployment_state \ + "${resource_group}" "${deployment_name}" "${subscription_id}")"; then + log_error "Failed to query ARM deployment '${deployment_name}'; refusing to race resource deletion" + return 1 + fi + + case "${deployment_state}" in + ""|Succeeded|Failed|Canceled|Cancelled) + [[ -z "${deployment_state}" ]] || _wait_for_deployment_operations \ + "${resource_group}" "${deployment_name}" "${subscription_id}" "${deadline}" + return + ;; + esac + + log_warn "ARM deployment '${deployment_name}' is ${deployment_state}; canceling it before resource deletion" + az deployment group cancel \ + --resource-group "${resource_group}" \ + --name "${deployment_name}" \ + --subscription "${subscription_id}" \ + --output none 2>/dev/null || true + + while (( SECONDS < deadline )); do + if ! deployment_state="$(_deployment_state \ + "${resource_group}" "${deployment_name}" "${subscription_id}")"; then + log_error "Failed to query ARM deployment '${deployment_name}' after requesting cancellation" + return 1 + fi + case "${deployment_state}" in + ""|Succeeded|Failed|Canceled|Cancelled) + log_info "ARM deployment is terminal: ${deployment_state:-not found}" + [[ -z "${deployment_state}" ]] || _wait_for_deployment_operations \ + "${resource_group}" "${deployment_name}" "${subscription_id}" "${deadline}" + return + ;; + esac + sleep "${E2E_CLEANUP_POLL_INTERVAL:-5}" + done + + log_error "ARM deployment '${deployment_name}' remained ${deployment_state} after ${E2E_CLEANUP_TIMEOUT}s" + return 1 +} + +_remaining_cleanup_timeout() { + local deadline="$1" + local remaining=$((deadline - SECONDS)) + (( remaining > 0 )) || return 1 + printf '%s\n' "${remaining}" +} + +_validate_persisted_node_resource_group() { + local node_resource_group="$1" name_suffix="$2" + + [[ -n "${node_resource_group}" ]] || return 0 + if [[ -z "${name_suffix}" ]]; then + log_error "Cannot validate persisted AKS node resource group '${node_resource_group}' without an E2E name suffix" + return 1 + fi + + local expected_node_resource_group="MC_aksflex-e2e-${name_suffix}" + if [[ "${node_resource_group}" != "${expected_node_resource_group}" ]]; then + log_error "Refusing to delete unexpected persisted AKS node resource group '${node_resource_group}'" + log_error "Expected '${expected_node_resource_group}' for this E2E deployment" + return 1 + fi +} + +_delete_node_resource_group() { + local node_resource_group="$1" subscription_id="$2" deadline="$3" + local exists wait_timeout + + [[ -n "${node_resource_group}" ]] || return 0 + if ! exists="$(az group exists \ + --name "${node_resource_group}" \ + --subscription "${subscription_id}" \ + --output tsv 2>/dev/null)"; then + log_error "Failed to determine whether AKS node resource group '${node_resource_group}' exists" + return 1 + fi + case "${exists}" in + false) + return 0 + ;; + true) + ;; + *) + log_error "Unexpected existence result for AKS node resource group '${node_resource_group}': ${exists}" + return 1 + ;; + esac + + az group delete --name "${node_resource_group}" --subscription "${subscription_id}" \ + --yes --no-wait --output none 2>/dev/null || true + if ! wait_timeout="$(_remaining_cleanup_timeout "${deadline}")"; then + log_error "Cleanup deadline reached before deleting AKS node resource group '${node_resource_group}'" + return 1 + fi + if ! az group wait --name "${node_resource_group}" --subscription "${subscription_id}" \ + --deleted --interval 10 --timeout "${wait_timeout}" 2>/dev/null; then + log_error "AKS node resource group still exists after cleanup timeout: ${node_resource_group}" + return 1 + fi +} + +_delete_tagged_resources() { + local resource_group="$1" run_id="$2" subscription_id="$3" + local resource_json resource_type id + local -a resource_types=( + "Microsoft.Compute/disks" + "Microsoft.Network/networkInterfaces" + "Microsoft.Network/publicIPAddresses" + "Microsoft.Network/virtualNetworks" + "Microsoft.Network/networkSecurityGroups" + ) + + [[ -n "${run_id}" ]] || return 0 + if ! resource_json="$(az resource list \ + --resource-group "${resource_group}" \ + --subscription "${subscription_id}" \ + --tag "github-run=${run_id}" \ + --output json 2>/dev/null)"; then + log_error "Failed to list E2E resources tagged github-run=${run_id}" + return 1 + fi + if ! jq -e 'type == "array"' <<<"${resource_json}" >/dev/null; then + log_error "Azure returned invalid tagged-resource inventory" + return 1 + fi + + for resource_type in "${resource_types[@]}"; do + while read -r id; do + [[ -n "${id}" ]] || continue + log_info "Deleting residual ${resource_type}: ${id##*/}" + az resource delete --ids "${id}" --subscription "${subscription_id}" --output none 2>/dev/null || true + done < <(jq -r --arg resource_type "${resource_type}" \ + '.[] | select((.type | ascii_downcase) == ($resource_type | ascii_downcase)) | .id' \ + <<<"${resource_json}") + done +} + +_tagged_resource_ids() { + local resource_group="$1" run_id="$2" subscription_id="$3" + + [[ -n "${run_id}" ]] || return 0 + az resource list \ + --resource-group "${resource_group}" \ + --subscription "${subscription_id}" \ + --tag "github-run=${run_id}" \ + --query '[].id' \ + --output tsv 2>/dev/null +} + cleanup() { log_section "Cleaning Up Resources" @@ -189,6 +388,7 @@ cleanup() { fi local resource_group cluster_name msi_vm_name token_vm_name offline_vm_name kubeadm_vm_name arc_vm_name arc_machine_name arc_machine_id + local subscription_id deployment_name run_id cleanup_failed vnet_name nsg_name name_suffix node_resource_group cleanup_deadline resource_group="$(state_get resource_group)" cluster_name="$(state_get cluster_name)" msi_vm_name="$(state_get msi_vm_name)" @@ -197,16 +397,152 @@ cleanup() { kubeadm_vm_name="$(state_get kubeadm_vm_name)" arc_vm_name="$(state_get arc_vm_name)" arc_machine_name="$(state_get arc_machine_name)" + subscription_id="$(state_get subscription_id "${AZURE_SUBSCRIPTION_ID}")" arc_machine_id="$(state_get arc_machine_id)" - if [[ -z "${arc_machine_id}" && -n "${arc_machine_name}" ]]; then - arc_machine_id="/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${resource_group}/providers/Microsoft.HybridCompute/machines/${arc_machine_name}" + if [[ -n "${arc_machine_name}" ]]; then + local expected_arc_machine_id="/subscriptions/${subscription_id}/resourceGroups/${resource_group}/providers/Microsoft.HybridCompute/machines/${arc_machine_name}" + if [[ -n "${arc_machine_id}" && "${arc_machine_id,,}" != "${expected_arc_machine_id,,}" ]]; then + log_error "Refusing to delete Arc machine with an unexpected persisted resource ID: ${arc_machine_id}" + return 1 + fi + # The ID is completely determined by other state fields. Reconstructing it + # avoids trusting a redundant deletion target from stale or damaged state. + arc_machine_id="${expected_arc_machine_id}" + elif [[ -n "${arc_machine_id}" ]]; then + log_error "Cannot validate persisted Arc machine resource ID without arc_machine_name" + return 1 fi - local deployment_name deployment_name="$(state_get deployment_name)" + run_id="$(state_get run_id "${GITHUB_RUN_ID:-}")" + name_suffix="$(state_get name_suffix)" + vnet_name="$(state_get vnet_name)" + nsg_name="$(state_get nsg_name)" + node_resource_group="$(state_get node_resource_group)" + if [[ -z "${name_suffix}" && "${cluster_name}" == aks-e2e-* ]]; then + name_suffix="${cluster_name#aks-e2e-}" + fi + if [[ -z "${deployment_name}" && -n "${name_suffix}" ]]; then + deployment_name="e2e-${name_suffix}" + fi + if [[ -z "${vnet_name}" && -n "${name_suffix}" ]]; then + vnet_name="vnet-e2e-${name_suffix}" + fi + if [[ -z "${nsg_name}" && -n "${name_suffix}" ]]; then + nsg_name="nsg-e2e-${name_suffix}" + fi + cleanup_failed=0 + cleanup_deadline=$((SECONDS + E2E_CLEANUP_TIMEOUT)) if [[ -z "${resource_group}" ]]; then - log_warn "No resource group in state; nothing to clean up" - return 0 + if [[ ! -f "${E2E_STATE_FILE}" ]] || jq -e 'length == 0' "${E2E_STATE_FILE}" >/dev/null 2>&1; then + log_warn "No resource group in state; nothing to clean up" + return 0 + fi + log_error "State is nonempty but has no resource_group; refusing to declare cleanup complete" + return 1 + fi + + local resource_group_exists + if ! resource_group_exists="$(az group exists \ + --name "${resource_group}" \ + --subscription "${subscription_id}" \ + --output tsv 2>/dev/null)"; then + log_error "Failed to determine whether resource group '${resource_group}' exists" + return 1 + fi + case "${resource_group_exists}" in + false) + if ! _validate_persisted_node_resource_group "${node_resource_group}" "${name_suffix}"; then + return 1 + fi + if ! _delete_node_resource_group "${node_resource_group}" "${subscription_id}" "${cleanup_deadline}"; then + return 1 + fi + state_set "lifecycle" "cleaned" || return 1 + state_set "cleanup_complete" "true" || return 1 + log_success "Resource group is absent; cleanup is complete" + return 0 + ;; + true) + ;; + *) + log_error "Unexpected existence result for resource group '${resource_group}': ${resource_group_exists}" + return 1 + ;; + esac + + state_set "cleanup_complete" "false" || return 1 + state_set "lifecycle" "cleaning" || return 1 + + if ! _cancel_active_deployment \ + "${resource_group}" "${deployment_name}" "${subscription_id}" "${cleanup_deadline}"; then + # Deleting while ARM is still provisioning races new resource creation and + # can report false success, so preserve state for a later cleanup attempt. + state_set "cleanup_complete" "false" || true + return 1 + fi + + # Snapshot IDs that are difficult to recover after deleting their parents. + # This supports cleanup of deployments created before deterministic OS-disk + # names and delete options were added to the Bicep module. + local vm_inventory aks_inventory live_node_resource_group vm_name disk_id nic_id nic_output + local -a managed_disk_ids=() nic_ids=() + if ! vm_inventory="$(az vm list \ + --resource-group "${resource_group}" \ + --subscription "${subscription_id}" \ + --output json 2>/dev/null)"; then + log_error "Failed to inventory E2E VMs before cleanup" + return 1 + fi + if ! aks_inventory="$(az aks list \ + --resource-group "${resource_group}" \ + --subscription "${subscription_id}" \ + --output json 2>/dev/null)"; then + log_error "Failed to inventory E2E AKS clusters before cleanup" + return 1 + fi + if ! jq -e 'type == "array"' <<<"${vm_inventory}" >/dev/null || \ + ! jq -e 'type == "array"' <<<"${aks_inventory}" >/dev/null; then + log_error "Azure returned an invalid VM or AKS cleanup inventory" + return 1 + fi + for vm_name in "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" "${kubeadm_vm_name}" "${arc_vm_name}"; do + [[ -n "${vm_name}" ]] || continue + if ! disk_id="$(jq -r --arg name "${vm_name}" \ + '.[] | select(.name == $name) | .storageProfile.osDisk.managedDisk.id // empty' \ + <<<"${vm_inventory}")"; then + log_error "Failed to inspect OS disk for VM '${vm_name}'" + return 1 + fi + [[ -z "${disk_id}" ]] || managed_disk_ids+=("${disk_id}") + if ! nic_output="$(jq -r --arg name "${vm_name}" \ + '.[] | select(.name == $name) | .networkProfile.networkInterfaces[]?.id // empty' \ + <<<"${vm_inventory}")"; then + log_error "Failed to inspect network interfaces for VM '${vm_name}'" + return 1 + fi + while read -r nic_id; do + [[ -z "${nic_id}" ]] || nic_ids+=("${nic_id}") + done <<<"${nic_output}" + done + if ! live_node_resource_group="$(jq -r --arg name "${cluster_name}" \ + '.[] | select(.name == $name) | .nodeResourceGroup // empty' \ + <<<"${aks_inventory}")"; then + log_error "Failed to inspect the AKS node resource group" + return 1 + fi + if [[ -n "${live_node_resource_group}" ]]; then + if [[ -n "${node_resource_group}" && "${node_resource_group}" != "${live_node_resource_group}" ]]; then + log_warn "Live AKS node resource group differs from state; using the live cluster value '${live_node_resource_group}'" + fi + node_resource_group="${live_node_resource_group}" + state_set "node_resource_group" "${node_resource_group}" || return 1 + elif ! _validate_persisted_node_resource_group "${node_resource_group}" "${name_suffix}"; then + return 1 + fi + if [[ -n "${node_resource_group}" && "${node_resource_group}" == "${resource_group}" ]]; then + log_error "Refusing to delete parent resource group as an AKS node resource group" + return 1 fi # Arc lifecycle is external to Flex Node. Remove the E2E-owned Arc resource @@ -216,50 +552,178 @@ cleanup() { az rest --method delete --url "https://management.azure.com${arc_machine_id}?api-version=2024-07-10" --output none 2>/dev/null || true fi - # Delete VMs first (faster than waiting for full RG delete) - log_info "[2/8] Deleting MSI VM: ${msi_vm_name}..." - az vm delete --resource-group "${resource_group}" --name "${msi_vm_name}" \ - --force-deletion yes --yes --no-wait 2>/dev/null || true - - log_info "[3/8] Deleting Token VM: ${token_vm_name}..." - az vm delete --resource-group "${resource_group}" --name "${token_vm_name}" \ - --force-deletion yes --yes --no-wait 2>/dev/null || true - - log_info "[4/8] Deleting Offline VM: ${offline_vm_name}..." - az vm delete --resource-group "${resource_group}" --name "${offline_vm_name}" \ - --force-deletion yes --yes --no-wait 2>/dev/null || true - - log_info "[5/8] Deleting Kubeadm VM: ${kubeadm_vm_name}..." - az vm delete --resource-group "${resource_group}" --name "${kubeadm_vm_name}" \ - --force-deletion yes --yes --no-wait 2>/dev/null || true - - log_info "[6/8] Deleting Arc VM: ${arc_vm_name}..." - az vm delete --resource-group "${resource_group}" --name "${arc_vm_name}" \ - --force-deletion yes --yes --no-wait 2>/dev/null || true - - # Clean up leftover networking resources tied to our deployment - log_info "[7/8] Cleaning up networking resources..." - local run_id="${GITHUB_RUN_ID:-}" - if [[ -n "${run_id}" ]]; then - local resource_ids - for res_type in networkInterfaces publicIPAddresses networkSecurityGroups disks; do - resource_ids="$(az resource list --resource-group "${resource_group}" \ - --query "[?tags.\"github-run\"=='${run_id}' && contains(type, '${res_type}')].id" \ - -o tsv 2>/dev/null || true)" - while read -r id; do - [[ -n "${id}" ]] || continue - az resource delete --ids "${id}" --no-wait 2>/dev/null || true - done <<<"${resource_ids}" - done - fi - - log_info "[8/8] Deleting AKS cluster: ${cluster_name}..." - az aks delete --resource-group "${resource_group}" --name "${cluster_name}" \ - --yes --no-wait 2>/dev/null || true - - # If we created the VNet/NSG via Bicep, they'll be cleaned up when no - # resources reference them, or on next deployment. We don't delete the - # resource group itself since it may be shared. - - log_success "Cleanup initiated (async deletes in progress)" + # Start independent VM and AKS deletes together, then wait for all of them + # before removing their network dependencies. + for vm_name in "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" "${kubeadm_vm_name}" "${arc_vm_name}"; do + [[ -n "${vm_name}" ]] || continue + log_info "Deleting VM: ${vm_name}..." + az vm delete --resource-group "${resource_group}" --name "${vm_name}" \ + --subscription "${subscription_id}" --force-deletion yes --yes --no-wait 2>/dev/null || true + done + + if [[ -n "${cluster_name}" ]]; then + log_info "Deleting AKS cluster: ${cluster_name}..." + az aks delete --resource-group "${resource_group}" --name "${cluster_name}" \ + --subscription "${subscription_id}" --yes --no-wait 2>/dev/null || true + fi + + for vm_name in "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" "${kubeadm_vm_name}" "${arc_vm_name}"; do + [[ -n "${vm_name}" ]] || continue + local wait_timeout + if ! wait_timeout="$(_remaining_cleanup_timeout "${cleanup_deadline}")"; then + log_error "Cleanup deadline reached while waiting for VMs" + cleanup_failed=1 + break + fi + if ! az vm wait --resource-group "${resource_group}" --name "${vm_name}" \ + --subscription "${subscription_id}" --deleted --interval 10 --timeout "${wait_timeout}" 2>/dev/null; then + if az vm show --resource-group "${resource_group}" --name "${vm_name}" \ + --subscription "${subscription_id}" --output none 2>/dev/null; then + log_error "VM still exists after cleanup timeout: ${vm_name}" + cleanup_failed=1 + fi + fi + az disk delete --resource-group "${resource_group}" --name "${vm_name}-osdisk" \ + --subscription "${subscription_id}" --yes --output none 2>/dev/null || true + az network nic delete --resource-group "${resource_group}" --name "${vm_name}-nic" \ + --subscription "${subscription_id}" --output none 2>/dev/null || true + az network public-ip delete --resource-group "${resource_group}" --name "${vm_name}-pip" \ + --subscription "${subscription_id}" --output none 2>/dev/null || true + done + + for disk_id in "${managed_disk_ids[@]}"; do + az resource delete --ids "${disk_id}" --subscription "${subscription_id}" --output none 2>/dev/null || true + done + for nic_id in "${nic_ids[@]}"; do + az resource delete --ids "${nic_id}" --subscription "${subscription_id}" --output none 2>/dev/null || true + done + + if [[ -n "${cluster_name}" ]]; then + local aks_wait_timeout + if ! aks_wait_timeout="$(_remaining_cleanup_timeout "${cleanup_deadline}")"; then + log_error "Cleanup deadline reached before AKS deletion completed" + cleanup_failed=1 + elif ! az aks wait \ + --resource-group "${resource_group}" --name "${cluster_name}" \ + --subscription "${subscription_id}" --deleted --interval 10 \ + --timeout "${aks_wait_timeout}" 2>/dev/null; then + if az aks show --resource-group "${resource_group}" --name "${cluster_name}" \ + --subscription "${subscription_id}" --output none 2>/dev/null; then + log_error "AKS cluster still exists after cleanup timeout: ${cluster_name}" + cleanup_failed=1 + fi + fi + fi + + if ! _delete_node_resource_group "${node_resource_group}" "${subscription_id}" "${cleanup_deadline}"; then + cleanup_failed=1 + fi + + if [[ -n "${arc_machine_id}" ]]; then + local arc_wait_timeout + if ! arc_wait_timeout="$(_remaining_cleanup_timeout "${cleanup_deadline}")"; then + log_error "Cleanup deadline reached before Arc machine deletion completed" + cleanup_failed=1 + elif ! az resource wait --ids "${arc_machine_id}" --subscription "${subscription_id}" \ + --api-version 2024-07-10 --deleted --interval 10 --timeout "${arc_wait_timeout}" \ + --output none 2>/dev/null; then + log_error "Arc machine still exists after cleanup timeout: ${arc_machine_id}" + cleanup_failed=1 + fi + fi + + [[ -z "${vnet_name}" ]] || az network vnet delete \ + --resource-group "${resource_group}" --name "${vnet_name}" \ + --subscription "${subscription_id}" --output none 2>/dev/null || true + [[ -z "${nsg_name}" ]] || az network nsg delete \ + --resource-group "${resource_group}" --name "${nsg_name}" \ + --subscription "${subscription_id}" --output none 2>/dev/null || true + + log_info "[8/8] Cleaning up tagged network and disk resources..." + local remaining_ids empty_inventories=0 + for _ in 1 2 3 4; do + if ! _delete_tagged_resources "${resource_group}" "${run_id}" "${subscription_id}"; then + cleanup_failed=1 + break + fi + if ! remaining_ids="$(_tagged_resource_ids "${resource_group}" "${run_id}" "${subscription_id}")"; then + log_error "Failed to verify tagged-resource deletion" + cleanup_failed=1 + break + fi + if [[ -z "${remaining_ids}" ]]; then + empty_inventories=$((empty_inventories + 1)) + if (( empty_inventories >= 2 )); then + break + fi + else + empty_inventories=0 + fi + sleep "${E2E_CLEANUP_POLL_INTERVAL:-5}" + done + + if [[ -n "${run_id}" && "${empty_inventories}" -lt 2 ]]; then + log_error "Tagged-resource inventory did not remain empty for two consecutive checks" + cleanup_failed=1 + fi + + if ! remaining_ids="$(_tagged_resource_ids "${resource_group}" "${run_id}" "${subscription_id}")"; then + log_error "Failed final tagged-resource verification" + cleanup_failed=1 + elif [[ -n "${remaining_ids}" ]]; then + log_error "Tagged E2E resources remain after cleanup:" + printf '%s\n' "${remaining_ids}" >&2 + cleanup_failed=1 + fi + + local final_inventory expected_names known_remaining captured_id resource_name + local -a expected_name_args=() + if ! final_inventory="$(az resource list \ + --resource-group "${resource_group}" \ + --subscription "${subscription_id}" \ + --output json 2>/dev/null)"; then + log_error "Failed final exact-name resource verification" + cleanup_failed=1 + final_inventory='[]' + fi + for resource_name in "${cluster_name}" "${arc_machine_name}" "${vnet_name}" "${nsg_name}"; do + [[ -z "${resource_name}" ]] || expected_name_args+=("${resource_name}") + done + for vm_name in "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" "${kubeadm_vm_name}" "${arc_vm_name}"; do + [[ -n "${vm_name}" ]] || continue + expected_name_args+=("${vm_name}" "${vm_name}-nic" "${vm_name}-pip" "${vm_name}-osdisk") + done + if ! expected_names="$(jq -cn --args '$ARGS.positional' -- "${expected_name_args[@]}")"; then + log_error "Failed to build exact-name cleanup inventory" + cleanup_failed=1 + expected_names='[]' + fi + if ! known_remaining="$(jq -er --argjson names "${expected_names}" ' + if type != "array" then error("resource list must be an array") + else [.[] | select(.name as $name | $names | index($name)) | .id] | join("\n") + end + ' <<<"${final_inventory}")"; then + log_error "Azure returned invalid final resource inventory" + cleanup_failed=1 + known_remaining="" + fi + for captured_id in "${managed_disk_ids[@]}" "${nic_ids[@]}" "${arc_machine_id}"; do + [[ -n "${captured_id}" ]] || continue + if jq -e --arg id "${captured_id}" '.[] | select(.id == $id)' <<<"${final_inventory}" >/dev/null; then + known_remaining+=$'\n'"${captured_id}" + fi + done + if [[ -n "${known_remaining}" ]]; then + log_error "Known E2E resources remain after cleanup:" + printf '%s\n' "${known_remaining}" >&2 + cleanup_failed=1 + fi + + if [[ "${cleanup_failed}" != "0" ]]; then + return 1 + fi + + state_set "lifecycle" "cleaned" || return 1 + state_set "cleanup_complete" "true" || return 1 + log_success "Cleanup completed and no tagged resources remain" } diff --git a/hack/e2e/lib/common.sh b/hack/e2e/lib/common.sh index 60b18c35..01961ebe 100755 --- a/hack/e2e/lib/common.sh +++ b/hack/e2e/lib/common.sh @@ -120,7 +120,7 @@ check_prerequisites() { log_info "Checking prerequisites..." local missing=0 - for cmd in az docker git go jq kubectl make python3 ssh scp openssl; do + for cmd in az docker flock git go jq kubectl make python3 ssh scp openssl; do if ! command -v "${cmd}" &>/dev/null; then log_error "Missing required tool: ${cmd}" missing=1 @@ -193,13 +193,28 @@ load_config() { E2E_SSH_OPTS="${E2E_SSH_OPTS:--o StrictHostKeyChecking=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR}" # Component versions (match the workflow defaults) + if [[ "${E2E_KUBERNETES_VERSION+x}" == "x" ]]; then + _E2E_KUBERNETES_VERSION_EXPLICIT=1 + else + _E2E_KUBERNETES_VERSION_EXPLICIT=0 + fi E2E_KUBERNETES_VERSION="${E2E_KUBERNETES_VERSION:-1.35.0}" E2E_CONTAINERD_VERSION="${E2E_CONTAINERD_VERSION:-2.0.4}" E2E_RUNC_VERSION="${E2E_RUNC_VERSION:-1.1.12}" + if [[ "${E2E_TARGET_AGENT_POOL_NAME+x}" == "x" ]]; then + _E2E_TARGET_AGENT_POOL_NAME_EXPLICIT=1 + else + _E2E_TARGET_AGENT_POOL_NAME_EXPLICIT=0 + fi E2E_TARGET_AGENT_POOL_NAME="${E2E_TARGET_AGENT_POOL_NAME:-aksflexnodes}" # listBootstrapData requires an existing ARM agent pool. The in-cluster # controller still uses E2E_TARGET_AGENT_POOL_NAME for its synthetic machine # contract, while the MSI repave scenario uses this real pool for join data. + if [[ "${E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME+x}" == "x" ]]; then + _E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME_EXPLICIT=1 + else + _E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME_EXPLICIT=0 + fi E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME="${E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME:-${E2E_TARGET_AGENT_POOL_NAME}}" # Kubelet resource reservation overrides applied to the token node config. @@ -219,6 +234,8 @@ load_config() { E2E_NODE_JOIN_TIMEOUT="${E2E_NODE_JOIN_TIMEOUT:-300}" E2E_POD_READY_TIMEOUT="${E2E_POD_READY_TIMEOUT:-120}" E2E_BOOTSTRAP_SETTLE_TIME="${E2E_BOOTSTRAP_SETTLE_TIME:-60}" + E2E_CLEANUP_TIMEOUT="${E2E_CLEANUP_TIMEOUT:-900}" + E2E_CLEANUP_POLL_INTERVAL="${E2E_CLEANUP_POLL_INTERVAL:-5}" log_info "Configuration loaded:" log_info " Resource Group: ${E2E_RESOURCE_GROUP}" @@ -279,17 +296,98 @@ init_work_dir() { log_debug "Work directory: ${E2E_WORK_DIR}" } +# Atomically reserve a work directory and write its complete deployment +# manifest. Any nonempty state must first reach verified cleanup; otherwise a +# new run could erase the only record capable of removing the old resources. +state_begin_deployment() { + local state_json="$1" + + ( + if ! flock -x 9; then + log_error "Failed to lock E2E deployment state" + return 1 + fi + local tmp existing_state requested_deployment existing_deployment + if ! requested_deployment="$(jq -er ' + if type != "object" then error("state must be an object") + else .deployment_name | select(type == "string" and length > 0) + end + ' <<<"${state_json}")"; then + log_error "New E2E deployment state is invalid or has no deployment_name" + return 1 + fi + + if [[ -f "${E2E_STATE_FILE}" ]]; then + if ! existing_state="$(jq -ec ' + if type == "object" then . else error("state must be an object") end + ' "${E2E_STATE_FILE}")"; then + log_error "Existing E2E state is invalid; refusing to overwrite cleanup metadata" + return 1 + fi + if ! jq -e ' + length == 0 or + (.lifecycle == "cleaned" and + (.cleanup_complete == "true" or .cleanup_complete == true)) + ' <<<"${existing_state}" >/dev/null; then + existing_deployment="$(jq -r '.deployment_name // empty' <<<"${existing_state}")" + log_error "E2E state already tracks deployment '${existing_deployment:-unknown legacy deployment}'" + log_error "Run cleanup with this work directory before deploying '${requested_deployment}', or choose a new E2E_WORK_DIR" + return 1 + fi + fi + + if ! tmp="$(mktemp "${E2E_STATE_FILE}.tmp.XXXXXX")"; then + log_error "Failed to create temporary E2E state file" + return 1 + fi + trap 'rm -f "${tmp}"' EXIT + if ! jq -e 'if type == "object" then . else error("state must be an object") end' \ + <<<"${state_json}" > "${tmp}"; then + log_error "Failed to render initial E2E deployment state" + return 1 + fi + if ! chmod 0600 "${tmp}" || ! mv "${tmp}" "${E2E_STATE_FILE}"; then + log_error "Failed to install initial E2E deployment state" + return 1 + fi + trap - EXIT + ) 9> "${E2E_STATE_FILE}.lock" +} + # Write a key=value into the state file (JSON) state_set() { local key="$1" value="$2" - local tmp="${E2E_STATE_FILE}.tmp" - if [[ -f "${E2E_STATE_FILE}" ]]; then - jq --arg k "${key}" --arg v "${value}" '. + {($k): $v}' "${E2E_STATE_FILE}" > "${tmp}" - else - jq -n --arg k "${key}" --arg v "${value}" '{($k): $v}' > "${tmp}" - fi - mv "${tmp}" "${E2E_STATE_FILE}" + ( + if ! flock -x 9; then + log_error "Failed to lock E2E deployment state" + return 1 + fi + local tmp + if ! tmp="$(mktemp "${E2E_STATE_FILE}.tmp.XXXXXX")"; then + log_error "Failed to create temporary E2E state file" + return 1 + fi + trap 'rm -f "${tmp}"' EXIT + + if [[ -f "${E2E_STATE_FILE}" ]]; then + if ! jq --arg k "${key}" --arg v "${value}" '. + {($k): $v}' \ + "${E2E_STATE_FILE}" > "${tmp}"; then + log_error "Failed to update E2E state key '${key}'" + return 1 + fi + else + if ! jq -n --arg k "${key}" --arg v "${value}" '{($k): $v}' > "${tmp}"; then + log_error "Failed to initialize E2E state key '${key}'" + return 1 + fi + fi + if ! chmod 0600 "${tmp}" || ! mv "${tmp}" "${E2E_STATE_FILE}"; then + log_error "Failed to install updated E2E state" + return 1 + fi + trap - EXIT + ) 9> "${E2E_STATE_FILE}.lock" } # Read a value from the state file @@ -305,11 +403,114 @@ state_get() { fi } +require_exact_kubernetes_version() { + if [[ ! "${E2E_KUBERNETES_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + log_error "E2E_KUBERNETES_VERSION must be an exact x.y.z patch version, got '${E2E_KUBERNETES_VERSION}'" + return 1 + fi +} + +restore_kubernetes_version_from_state() { + local persisted_version resource_group cluster_name subscription_id live_version live_cluster cluster_id + local persisted_target_pool persisted_bootstrap_pool system_pool live_system_version live_flex_version + persisted_version="$(state_get kubernetes_version)" + resource_group="$(state_get resource_group)" + cluster_name="$(state_get cluster_name)" + subscription_id="$(state_get subscription_id "${AZURE_SUBSCRIPTION_ID:-}")" + persisted_target_pool="$(state_get target_agent_pool_name "${E2E_TARGET_AGENT_POOL_NAME:-aksflexnodes}")" + persisted_bootstrap_pool="$(state_get bootstrap_data_agent_pool_name "${E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME:-${persisted_target_pool}}")" + system_pool="$(state_get system_pool_name system)" + + if [[ -z "${resource_group}" || -z "${cluster_name}" || -z "${subscription_id}" ]]; then + log_error "Deployment state does not identify an AKS cluster and subscription; run the infra command first" + return 1 + fi + + if [[ "${_E2E_TARGET_AGENT_POOL_NAME_EXPLICIT:-0}" == "1" && \ + "${E2E_TARGET_AGENT_POOL_NAME:-}" != "${persisted_target_pool}" ]]; then + log_error "E2E_TARGET_AGENT_POOL_NAME is ${E2E_TARGET_AGENT_POOL_NAME:-}, but deployment state records ${persisted_target_pool}" + return 1 + fi + if [[ "${_E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME_EXPLICIT:-0}" == "1" && \ + "${E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME:-}" != "${persisted_bootstrap_pool}" ]]; then + log_error "E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME is ${E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME:-}, but deployment state records ${persisted_bootstrap_pool}" + return 1 + fi + E2E_TARGET_AGENT_POOL_NAME="${persisted_target_pool}" + E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME="${persisted_bootstrap_pool}" + + if ! live_cluster="$(az aks show \ + --subscription "${subscription_id}" \ + --resource-group "${resource_group}" \ + --name "${cluster_name}" \ + --query '{id:id, version:currentKubernetesVersion || kubernetesVersion}' \ + --output json 2>/dev/null)" || \ + ! live_version="$(jq -er '.version | select(type == "string" and length > 0)' <<<"${live_cluster}")" || \ + ! cluster_id="$(jq -er '.id | select(type == "string" and length > 0)' <<<"${live_cluster}")"; then + log_error "Cannot determine the live Kubernetes version for ${resource_group}/${cluster_name}" + return 1 + fi + if ! live_system_version="$(az rest \ + --method get \ + --url "https://management.azure.com${cluster_id}/agentPools/${system_pool}?api-version=2026-05-02-preview" \ + --query 'properties.currentOrchestratorVersion || properties.orchestratorVersion' \ + --output tsv 2>/dev/null)" || [[ -z "${live_system_version}" ]]; then + log_error "Cannot determine the live Kubernetes version for system pool ${system_pool}" + return 1 + fi + if ! live_flex_version="$(az rest \ + --method get \ + --url "https://management.azure.com${cluster_id}/agentPools/${persisted_bootstrap_pool}?api-version=2026-05-02-preview" \ + --query 'properties.currentOrchestratorVersion || properties.orchestratorVersion' \ + --output tsv 2>/dev/null)" || [[ -z "${live_flex_version}" ]]; then + log_error "Cannot determine the live Kubernetes version for FlexNodes pool ${persisted_bootstrap_pool}" + return 1 + fi + if [[ "${live_system_version}" != "${live_version}" || "${live_flex_version}" != "${live_version}" ]]; then + log_error "AKS version skew: control plane=${live_version}, system pool=${live_system_version}, FlexNodes pool=${live_flex_version}" + return 1 + fi + if [[ -n "${persisted_version}" && "${persisted_version}" != "${live_version}" ]]; then + log_error "Deployment state records Kubernetes ${persisted_version}, but the live cluster is ${live_version}" + return 1 + fi + if [[ -z "${persisted_version}" ]]; then + persisted_version="${live_version}" + state_set kubernetes_version "${persisted_version}" + log_info "Recovered Kubernetes version from the live cluster: ${persisted_version}" + fi + if [[ "${_E2E_KUBERNETES_VERSION_EXPLICIT}" == "1" && \ + "${E2E_KUBERNETES_VERSION}" != "${persisted_version}" ]]; then + log_error "E2E_KUBERNETES_VERSION is ${E2E_KUBERNETES_VERSION}, but the existing cluster state records ${persisted_version}" + return 1 + fi + + E2E_KUBERNETES_VERSION="${persisted_version}" + state_set system_pool_kubernetes_version "${live_system_version}" + state_set flex_pool_kubernetes_version "${live_flex_version}" + require_exact_kubernetes_version + log_info "Restored Kubernetes version from deployment state: ${E2E_KUBERNETES_VERSION}" +} + # Dump the state file for debugging state_dump() { if [[ -f "${E2E_STATE_FILE}" ]]; then log_info "Current state:" - jq '.' "${E2E_STATE_FILE}" + jq ' + def redact: + if type == "object" then + with_entries( + if (.key | test("(^|_)(bootstrap_)?token$|secret|password|credential"; "i")) then + .value = "" + else + .value |= redact + end + ) + elif type == "array" then map(redact) + else . + end; + redact + ' "${E2E_STATE_FILE}" else log_info "No state file found" fi diff --git a/hack/e2e/lib/infra.sh b/hack/e2e/lib/infra.sh index fda84c96..ea70dd90 100755 --- a/hack/e2e/lib/infra.sh +++ b/hack/e2e/lib/infra.sh @@ -51,10 +51,7 @@ infra_deploy() { local start start=$(timer_start) - if [[ ! "${E2E_KUBERNETES_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - log_error "E2E_KUBERNETES_VERSION must be an exact x.y.z patch version, got '${E2E_KUBERNETES_VERSION}'" - return 1 - fi + require_exact_kubernetes_version local bicep_file="${E2E_INFRA_DIR}/main.bicep" if [[ ! -f "${bicep_file}" ]]; then @@ -62,6 +59,48 @@ infra_deploy() { return 1 fi + # Persist deterministic identities before Azure starts provisioning so the + # always-run cleanup stage can remove resources after a partial deployment. + local deployment_name="e2e-${E2E_NAME_SUFFIX}" + local run_id="${GITHUB_RUN_ID:-local-${E2E_NAME_SUFFIX}}" + local initial_state + initial_state="$(jq -n \ + --arg lifecycle "provisioning" \ + --arg resource_group "${E2E_RESOURCE_GROUP}" \ + --arg location "${E2E_LOCATION}" \ + --arg subscription_id "${AZURE_SUBSCRIPTION_ID}" \ + --arg tenant_id "${AZURE_TENANT_ID}" \ + --arg run_id "${run_id}" \ + --arg name_suffix "${E2E_NAME_SUFFIX}" \ + --arg kubernetes_version "${E2E_KUBERNETES_VERSION}" \ + --arg target_agent_pool_name "${E2E_TARGET_AGENT_POOL_NAME}" \ + --arg bootstrap_data_agent_pool_name "${E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME}" \ + --arg system_pool_name "system" \ + --arg deployment_name "${deployment_name}" \ + --arg cluster_name "aks-e2e-${E2E_NAME_SUFFIX}" \ + --arg node_resource_group "MC_aksflex-e2e-${E2E_NAME_SUFFIX}" \ + --arg vnet_name "vnet-e2e-${E2E_NAME_SUFFIX}" \ + --arg nsg_name "nsg-e2e-${E2E_NAME_SUFFIX}" \ + --arg msi_vm_name "vm-e2e-msi-${E2E_NAME_SUFFIX}" \ + --arg token_vm_name "vm-e2e-token-${E2E_NAME_SUFFIX}" \ + --arg offline_vm_name "vm-e2e-offline-${E2E_NAME_SUFFIX}" \ + --arg kubeadm_vm_name "vm-e2e-kubeadm-${E2E_NAME_SUFFIX}" \ + --arg arc_vm_name "vm-e2e-arc-${E2E_NAME_SUFFIX}" \ + --arg arc_machine_name "vm-e2e-arc-${E2E_NAME_SUFFIX}-connected" \ + '{schema_version: 1, lifecycle: $lifecycle, resource_group: $resource_group, + location: $location, subscription_id: $subscription_id, tenant_id: $tenant_id, + run_id: $run_id, name_suffix: $name_suffix, kubernetes_version: $kubernetes_version, + target_agent_pool_name: $target_agent_pool_name, + bootstrap_data_agent_pool_name: $bootstrap_data_agent_pool_name, + system_pool_name: $system_pool_name, + deployment_name: $deployment_name, cluster_name: $cluster_name, + node_resource_group: $node_resource_group, + vnet_name: $vnet_name, nsg_name: $nsg_name, msi_vm_name: $msi_vm_name, + token_vm_name: $token_vm_name, offline_vm_name: $offline_vm_name, + kubeadm_vm_name: $kubeadm_vm_name, arc_vm_name: $arc_vm_name, + arc_machine_name: $arc_machine_name}')" + state_begin_deployment "${initial_state}" + # Ensure resource group exists if ! az group show --name "${E2E_RESOURCE_GROUP}" --output none 2>/dev/null; then log_info "Creating resource group: ${E2E_RESOURCE_GROUP} in ${E2E_LOCATION}" @@ -80,7 +119,6 @@ infra_deploy() { configure_ssh_identity # Build tags - local run_id="${GITHUB_RUN_ID:-local-$(date +%s)}" local tags_json tags_json=$(jq -n \ --arg run "${run_id}" \ @@ -89,7 +127,6 @@ infra_deploy() { # Deploy log_info "Deploying Bicep template (this may take 5-10 minutes)..." - local deployment_name="e2e-${E2E_NAME_SUFFIX}" az deployment group create \ --resource-group "${E2E_RESOURCE_GROUP}" \ --name "${deployment_name}" \ @@ -215,6 +252,7 @@ infra_deploy() { return 1 fi + state_set "lifecycle" "ready" log_success "Infrastructure deployed in $(timer_elapsed "${start}")s" } diff --git a/hack/e2e/lib/node-join-arc.sh b/hack/e2e/lib/node-join-arc.sh index 73e2176d..1dfbed89 100755 --- a/hack/e2e/lib/node-join-arc.sh +++ b/hack/e2e/lib/node-join-arc.sh @@ -148,8 +148,9 @@ _fetch_arc_bootstrap_config() { return 1 fi + install -m 0600 /dev/null "${output}" remote_exec "${vm_ip}" "sudo cat '${remote_output}'" > "${output}" - chmod 0600 "${output}" + install -m 0600 /dev/null "${output}.tmp" jq \ --arg nodeIP "${vm_private_ip}" \ --arg nodeName "${vm_name}" \ @@ -165,7 +166,6 @@ _fetch_arc_bootstrap_config() { | .node.kubelet.nodeIP = $nodeIP' \ "${output}" > "${output}.tmp" mv "${output}.tmp" "${output}" - chmod 0600 "${output}" remote_exec "${vm_ip}" "sudo rm -f '${remote_output}' '${remote_binary}'" } diff --git a/hack/e2e/lib/node-join-kubeadm.sh b/hack/e2e/lib/node-join-kubeadm.sh index 255c6ef8..443516f1 100644 --- a/hack/e2e/lib/node-join-kubeadm.sh +++ b/hack/e2e/lib/node-join-kubeadm.sh @@ -296,10 +296,10 @@ node_join_kubeadm() { log_info "Creating bootstrap token..." local bootstrap_token bootstrap_token="$(with_cluster_lock _kubeadm_create_bootstrap_token)" - state_set "kubeadm_bootstrap_token" "${bootstrap_token}" # Step 2: Generate the config file for aks-flex-node agent local config_file="${E2E_WORK_DIR}/config-kubeadm.json" + install -m 0600 /dev/null "${config_file}" cat > "${config_file}" < Date: Sat, 22 Aug 2026 03:02:50 +0000 Subject: [PATCH 10/18] Isolate concurrent E2E deployments --- .github/workflows/e2e-tests.yml | 1 + hack/e2e/e2e_scripts_test.go | 68 +++++++++++++++++++++++--- hack/e2e/infra/main.bicep | 10 ++-- hack/e2e/lib/cleanup.sh | 84 ++++++++++++++++++++++----------- 4 files changed, 125 insertions(+), 38 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 9be30270..dfce7ec6 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -69,6 +69,7 @@ env: AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} GITHUB_RUN_ID: ${{ github.run_id }} + E2E_NAME_SUFFIX: ${{ github.run_id }}-${{ github.run_attempt }} E2E_WORK_DIR: /tmp/aks-flex-node-e2e-${{ github.run_id }} E2E_SUITE: ${{ inputs.suite || 'all' }} E2E_KUBERNETES_VERSION: ${{ inputs.kubernetes_version || '1.35.0' }} diff --git a/hack/e2e/e2e_scripts_test.go b/hack/e2e/e2e_scripts_test.go index ea979d80..ce70a085 100644 --- a/hack/e2e/e2e_scripts_test.go +++ b/hack/e2e/e2e_scripts_test.go @@ -13,9 +13,24 @@ import ( // Embedding the scripts makes Go's test cache invalidate on shell-only changes. // -//go:embed lib/common.sh lib/cleanup.sh lib/bootstrap-rbac-migration.sh +//go:embed lib/common.sh lib/cleanup.sh lib/bootstrap-rbac-migration.sh infra/main.bicep var e2eScripts embed.FS +func TestBicepModuleDeploymentNamesAreUniquePerRun(t *testing.T) { + t.Parallel() + + mainBicep, err := e2eScripts.ReadFile("infra/main.bicep") + if err != nil { + t.Fatalf("read embedded main.bicep: %v", err) + } + for _, moduleName := range []string{"msi", "token", "offline", "kubeadm", "arc"} { + want := "name: 'deploy-vm-" + moduleName + "-${nameSuffix}'" + if !strings.Contains(string(mainBicep), want) { + t.Errorf("VM module %q does not use a per-run deployment name %q", moduleName, want) + } + } +} + func TestRestoreKubernetesVersionFromState(t *testing.T) { t.Parallel() @@ -607,6 +622,28 @@ func TestCleanupRejectsUnexpectedArcMachineID(t *testing.T) { } } +func TestCleanupRetriesTransientAzureQueries(t *testing.T) { + t.Parallel() + + _, _, callLog, output, err := runCleanup(t, cleanupOptions{transientQueryFailures: true}) + if err != nil { + t.Fatalf("transient-query cleanup failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=success") { + t.Fatalf("cleanup did not recover from transient Azure query failures:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + if strings.Count(string(calls), "group exists") < 3 { + t.Errorf("cleanup did not retry a failed resource-group query:\n%s", calls) + } + if strings.Count(string(calls), "resource list") < 4 { + t.Errorf("cleanup did not retry a failed resource inventory:\n%s", calls) + } +} + func TestCleanupQueryFailureDoesNotDeleteResources(t *testing.T) { t.Parallel() @@ -660,6 +697,7 @@ type cleanupOptions struct { blankVMNames bool unexpectedNodeResource bool unexpectedArcMachineID bool + transientQueryFailures bool } func runCleanup(t *testing.T, options cleanupOptions) (string, string, string, []byte, error) { @@ -668,6 +706,8 @@ func runCleanup(t *testing.T, options cleanupOptions) (string, string, string, [ statePath := filepath.Join(workDir, "state.json") callLog := filepath.Join(workDir, "az-calls.log") nodeGroupDeleted := filepath.Join(workDir, "node-group-deleted") + groupQueryFailed := filepath.Join(workDir, "group-query-failed") + resourceQueryFailed := filepath.Join(workDir, "resource-query-failed") state := `{ "resource_group": "test-rg", "subscription_id": "test-subscription", @@ -717,10 +757,16 @@ E2E_WORK_DIR="$1" source "$2" AZ_CALL_LOG="$3" NODE_GROUP_DELETED="$4" -LEAVE_CLUSTER="$5" -PARENT_GROUP_ABSENT="$6" -DEPLOYMENT_QUERY_FAILS="$7" -RUN_TWICE="$8" +GROUP_QUERY_FAILED="$5" +RESOURCE_QUERY_FAILED="$6" +LEAVE_CLUSTER="$7" +PARENT_GROUP_ABSENT="$8" +DEPLOYMENT_QUERY_FAILS="$9" +RUN_TWICE="${10}" +TRANSIENT_QUERY_FAILURES="${11}" +# Keep cleanup fixtures independent from the ambient GitHub Actions run. Tests +# that need tag-based cleanup persist an explicit run_id in their state. +unset GITHUB_RUN_ID AZURE_SUBSCRIPTION_ID=test-subscription E2E_SKIP_CLEANUP=0 E2E_CLEANUP_TIMEOUT=5 @@ -733,6 +779,10 @@ az() { elif [[ "$1 $2 $3 $4" == "deployment operation group list" ]]; then printf '[{"properties":{"provisioningState":"Succeeded"}}]\n' elif [[ "$1 $2" == "group exists" ]]; then + if [[ "${TRANSIENT_QUERY_FAILURES}" == "1" && ! -f "${GROUP_QUERY_FAILED}" ]]; then + : > "${GROUP_QUERY_FAILED}" + return 1 + fi if [[ "$*" == *"--name test-rg"* ]]; then [[ "${PARENT_GROUP_ABSENT}" == "1" ]] && printf 'false\n' || printf 'true\n' elif [[ "$*" == *"--name MC_aksflex-e2e-test"* ]]; then @@ -747,6 +797,10 @@ az() { elif [[ "$1 $2" == "vm list" || "$1 $2" == "aks list" ]]; then printf '[]\n' elif [[ "$1 $2" == "resource list" ]]; then + if [[ "${TRANSIENT_QUERY_FAILURES}" == "1" && ! -f "${RESOURCE_QUERY_FAILED}" ]]; then + : > "${RESOURCE_QUERY_FAILED}" + return 1 + fi if [[ "$*" == *"--tag "* ]]; then [[ "$*" == *"--output json"* ]] && printf '[]\n' || true elif [[ "$*" == *"--output json"* ]]; then @@ -773,8 +827,10 @@ if [[ "${RUN_TWICE}" == "1" ]]; then fi ` output, err := runBash(t, script, workDir, cleanupScript, callLog, nodeGroupDeleted, + groupQueryFailed, resourceQueryFailed, boolString(options.leaveCluster), boolString(options.parentGroupAbsent), - boolString(options.deploymentQueryFails), boolString(options.runTwice)) + boolString(options.deploymentQueryFails), boolString(options.runTwice), + boolString(options.transientQueryFailures)) return workDir, statePath, callLog, output, err } diff --git a/hack/e2e/infra/main.bicep b/hack/e2e/infra/main.bicep index 3f4f8ce1..71a1bc69 100644 --- a/hack/e2e/infra/main.bicep +++ b/hack/e2e/infra/main.bicep @@ -169,7 +169,7 @@ resource flexAgentPool 'Microsoft.ContainerService/managedClusters/agentPools@20 // Flex-node VMs (via reusable module) // --------------------------------------------------------------------------- module vmMsi 'modules/vm.bicep' = { - name: 'deploy-vm-msi' + name: 'deploy-vm-msi-${nameSuffix}' params: { location: location vmName: msiVmName @@ -183,7 +183,7 @@ module vmMsi 'modules/vm.bicep' = { } module vmToken 'modules/vm.bicep' = { - name: 'deploy-vm-token' + name: 'deploy-vm-token-${nameSuffix}' params: { location: location vmName: tokenVmName @@ -199,7 +199,7 @@ module vmToken 'modules/vm.bicep' = { } module vmOffline 'modules/vm.bicep' = { - name: 'deploy-vm-offline' + name: 'deploy-vm-offline-${nameSuffix}' params: { location: location vmName: offlineVmName @@ -213,7 +213,7 @@ module vmOffline 'modules/vm.bicep' = { } module vmKubeadm 'modules/vm.bicep' = { - name: 'deploy-vm-kubeadm' + name: 'deploy-vm-kubeadm-${nameSuffix}' params: { location: location vmName: kubeadmVmName @@ -230,7 +230,7 @@ module vmKubeadm 'modules/vm.bicep' = { // an official Arc evaluation host by disabling walinuxagent, blocking Azure // IMDS, and setting MSFT_ARC_TEST before installing the Connected Machine agent. module vmArc 'modules/vm.bicep' = { - name: 'deploy-vm-arc' + name: 'deploy-vm-arc-${nameSuffix}' params: { location: location vmName: arcVmName diff --git a/hack/e2e/lib/cleanup.sh b/hack/e2e/lib/cleanup.sh index f1c749a9..cb2e7889 100755 --- a/hack/e2e/lib/cleanup.sh +++ b/hack/e2e/lib/cleanup.sh @@ -275,6 +275,56 @@ _remaining_cleanup_timeout() { printf '%s\n' "${remaining}" } +_group_exists_with_retry() { + local group_name="$1" subscription_id="$2" + local exists attempt + + for attempt in 1 2 3 4 5; do + if exists="$(az group exists \ + --name "${group_name}" \ + --subscription "${subscription_id}" \ + --output tsv 2>/dev/null)"; then + case "${exists}" in + true|false) + printf '%s\n' "${exists}" + return 0 + ;; + esac + fi + if (( attempt < 5 )); then + sleep "${E2E_CLEANUP_POLL_INTERVAL:-5}" + fi + done + + log_error "Failed to determine whether resource group '${group_name}' exists after ${attempt} attempts" >&2 + return 1 +} + +_resource_inventory() { + local resource_group="$1" subscription_id="$2" run_id="${3:-}" + local resource_json attempt + local -a tag_args=() + [[ -z "${run_id}" ]] || tag_args=(--tag "github-run=${run_id}") + + for attempt in 1 2 3 4 5; do + if resource_json="$(az resource list \ + --resource-group "${resource_group}" \ + --subscription "${subscription_id}" \ + "${tag_args[@]}" \ + --output json 2>/dev/null)" && \ + jq -e 'type == "array"' <<<"${resource_json}" >/dev/null; then + printf '%s\n' "${resource_json}" + return 0 + fi + if (( attempt < 5 )); then + sleep "${E2E_CLEANUP_POLL_INTERVAL:-5}" + fi + done + + log_error "Failed to inventory resources in '${resource_group}' after ${attempt} attempts" >&2 + return 1 +} + _validate_persisted_node_resource_group() { local node_resource_group="$1" name_suffix="$2" @@ -297,10 +347,7 @@ _delete_node_resource_group() { local exists wait_timeout [[ -n "${node_resource_group}" ]] || return 0 - if ! exists="$(az group exists \ - --name "${node_resource_group}" \ - --subscription "${subscription_id}" \ - --output tsv 2>/dev/null)"; then + if ! exists="$(_group_exists_with_retry "${node_resource_group}" "${subscription_id}")"; then log_error "Failed to determine whether AKS node resource group '${node_resource_group}' exists" return 1 fi @@ -341,18 +388,10 @@ _delete_tagged_resources() { ) [[ -n "${run_id}" ]] || return 0 - if ! resource_json="$(az resource list \ - --resource-group "${resource_group}" \ - --subscription "${subscription_id}" \ - --tag "github-run=${run_id}" \ - --output json 2>/dev/null)"; then + if ! resource_json="$(_resource_inventory "${resource_group}" "${subscription_id}" "${run_id}")"; then log_error "Failed to list E2E resources tagged github-run=${run_id}" return 1 fi - if ! jq -e 'type == "array"' <<<"${resource_json}" >/dev/null; then - log_error "Azure returned invalid tagged-resource inventory" - return 1 - fi for resource_type in "${resource_types[@]}"; do while read -r id; do @@ -367,14 +406,11 @@ _delete_tagged_resources() { _tagged_resource_ids() { local resource_group="$1" run_id="$2" subscription_id="$3" + local resource_json [[ -n "${run_id}" ]] || return 0 - az resource list \ - --resource-group "${resource_group}" \ - --subscription "${subscription_id}" \ - --tag "github-run=${run_id}" \ - --query '[].id' \ - --output tsv 2>/dev/null + resource_json="$(_resource_inventory "${resource_group}" "${subscription_id}" "${run_id}")" || return 1 + jq -r '.[].id' <<<"${resource_json}" } cleanup() { @@ -443,10 +479,7 @@ cleanup() { fi local resource_group_exists - if ! resource_group_exists="$(az group exists \ - --name "${resource_group}" \ - --subscription "${subscription_id}" \ - --output tsv 2>/dev/null)"; then + if ! resource_group_exists="$(_group_exists_with_retry "${resource_group}" "${subscription_id}")"; then log_error "Failed to determine whether resource group '${resource_group}' exists" return 1 fi @@ -678,10 +711,7 @@ cleanup() { local final_inventory expected_names known_remaining captured_id resource_name local -a expected_name_args=() - if ! final_inventory="$(az resource list \ - --resource-group "${resource_group}" \ - --subscription "${subscription_id}" \ - --output json 2>/dev/null)"; then + if ! final_inventory="$(_resource_inventory "${resource_group}" "${subscription_id}")"; then log_error "Failed final exact-name resource verification" cleanup_failed=1 final_inventory='[]' From 64dc2daaef22b35a2578b733f5f3861d93c7e4c4 Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 22 Aug 2026 03:29:04 +0000 Subject: [PATCH 11/18] Address E2E cleanup review feedback --- hack/e2e/e2e_scripts_test.go | 46 +++++++++++++++++++++++++++++++++++- hack/e2e/lib/common.sh | 6 ++--- hack/e2e/run.sh | 2 +- 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/hack/e2e/e2e_scripts_test.go b/hack/e2e/e2e_scripts_test.go index ce70a085..82b15cb0 100644 --- a/hack/e2e/e2e_scripts_test.go +++ b/hack/e2e/e2e_scripts_test.go @@ -13,7 +13,7 @@ import ( // Embedding the scripts makes Go's test cache invalidate on shell-only changes. // -//go:embed lib/common.sh lib/cleanup.sh lib/bootstrap-rbac-migration.sh infra/main.bicep +//go:embed run.sh lib/common.sh lib/cleanup.sh lib/bootstrap-rbac-migration.sh infra/main.bicep var e2eScripts embed.FS func TestBicepModuleDeploymentNamesAreUniquePerRun(t *testing.T) { @@ -31,6 +31,50 @@ func TestBicepModuleDeploymentNamesAreUniquePerRun(t *testing.T) { } } +func TestLoadConfigTreatsEmptyOptionalValuesAsImplicit(t *testing.T) { + t.Parallel() + + commonScript := e2eScriptPath(t, "lib", "common.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +E2E_RESOURCE_GROUP=test-rg +E2E_LOCATION=test-location +AZURE_SUBSCRIPTION_ID=test-subscription +AZURE_TENANT_ID=test-tenant +E2E_NAME_SUFFIX=test +E2E_KUBERNETES_VERSION= +E2E_TARGET_AGENT_POOL_NAME= +E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME= +load_config >/dev/null +printf 'RESULT=%s|%s|%s|%s|%s|%s\n' \ + "${_E2E_KUBERNETES_VERSION_EXPLICIT}" "${E2E_KUBERNETES_VERSION}" \ + "${_E2E_TARGET_AGENT_POOL_NAME_EXPLICIT}" "${E2E_TARGET_AGENT_POOL_NAME}" \ + "${_E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME_EXPLICIT}" "${E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME}" +` + output, err := runBash(t, script, t.TempDir(), commonScript) + if err != nil { + t.Fatalf("load config failed: %v\n%s", err, output) + } + const want = "RESULT=0|1.35.0|0|aksflexnodes|0|aksflexnodes\n" + if !strings.Contains(string(output), want) { + t.Fatalf("empty optional values were treated as explicit: got %q, want %q", output, want) + } +} + +func TestFullE2ECleanupFailureIsAggregated(t *testing.T) { + t.Parallel() + + runScript, err := e2eScripts.ReadFile("run.sh") + if err != nil { + t.Fatalf("read embedded run.sh: %v", err) + } + if !strings.Contains(string(runScript), "cleanup || exit_code=1") { + t.Fatal("full E2E does not aggregate cleanup failure into its final result") + } +} + func TestRestoreKubernetesVersionFromState(t *testing.T) { t.Parallel() diff --git a/hack/e2e/lib/common.sh b/hack/e2e/lib/common.sh index 01961ebe..ecd4098c 100755 --- a/hack/e2e/lib/common.sh +++ b/hack/e2e/lib/common.sh @@ -193,7 +193,7 @@ load_config() { E2E_SSH_OPTS="${E2E_SSH_OPTS:--o StrictHostKeyChecking=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR}" # Component versions (match the workflow defaults) - if [[ "${E2E_KUBERNETES_VERSION+x}" == "x" ]]; then + if [[ -n "${E2E_KUBERNETES_VERSION:-}" ]]; then _E2E_KUBERNETES_VERSION_EXPLICIT=1 else _E2E_KUBERNETES_VERSION_EXPLICIT=0 @@ -201,7 +201,7 @@ load_config() { E2E_KUBERNETES_VERSION="${E2E_KUBERNETES_VERSION:-1.35.0}" E2E_CONTAINERD_VERSION="${E2E_CONTAINERD_VERSION:-2.0.4}" E2E_RUNC_VERSION="${E2E_RUNC_VERSION:-1.1.12}" - if [[ "${E2E_TARGET_AGENT_POOL_NAME+x}" == "x" ]]; then + if [[ -n "${E2E_TARGET_AGENT_POOL_NAME:-}" ]]; then _E2E_TARGET_AGENT_POOL_NAME_EXPLICIT=1 else _E2E_TARGET_AGENT_POOL_NAME_EXPLICIT=0 @@ -210,7 +210,7 @@ load_config() { # listBootstrapData requires an existing ARM agent pool. The in-cluster # controller still uses E2E_TARGET_AGENT_POOL_NAME for its synthetic machine # contract, while the MSI repave scenario uses this real pool for join data. - if [[ "${E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME+x}" == "x" ]]; then + if [[ -n "${E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME:-}" ]]; then _E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME_EXPLICIT=1 else _E2E_BOOTSTRAP_DATA_AGENT_POOL_NAME_EXPLICIT=0 diff --git a/hack/e2e/run.sh b/hack/e2e/run.sh index 96bd9a6c..b7f57f8f 100755 --- a/hack/e2e/run.sh +++ b/hack/e2e/run.sh @@ -221,7 +221,7 @@ cmd_all() { collect_logs || true # Cleanup - cleanup + cleanup || exit_code=1 echo "" log_section "E2E Test Complete" From 5abf5f31496bd4123866f6b1b9af255fde918fe1 Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 22 Aug 2026 03:51:36 +0000 Subject: [PATCH 12/18] Harden conditional legacy RBAC deletion Route the preconditioned Kubernetes DELETE over a private Unix socket so migration exposes no privileged local TCP endpoint while retaining race-safe object preconditions. --- scripts/aks-flex-config | 263 ++++++++++++++++++-------------- scripts/aks_flex_config_test.go | 49 +++--- 2 files changed, 176 insertions(+), 136 deletions(-) diff --git a/scripts/aks-flex-config b/scripts/aks-flex-config index 2b7b8d7e..0c011834 100755 --- a/scripts/aks-flex-config +++ b/scripts/aks-flex-config @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import http.client import json import os import re @@ -11,13 +12,14 @@ import secrets import selectors import signal import shutil +import socket +import stat import subprocess import sys +import tempfile import time from datetime import datetime, timedelta, timezone from pathlib import Path -from urllib import error as urlerror -from urllib import request as urlrequest from urllib.parse import quote, urlsplit RESOURCE_MANAGER_ENDPOINT = "https://management.azure.com" @@ -308,130 +310,155 @@ def require_legacy_node_role_binding_absent(bindings: list[dict[str, object]] | ) +class UnixSocketHTTPConnection(http.client.HTTPConnection): + def __init__(self, socket_path: str, *, timeout: float) -> None: + super().__init__("localhost", timeout=timeout) + self.socket_path = socket_path + + def connect(self) -> None: + connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + connection.settimeout(self.timeout) + try: + connection.connect(self.socket_path) + except OSError: + connection.close() + raise + self.sock = connection + + def delete_cluster_role_binding_with_preconditions(name: str, uid: str, resource_version: str) -> None: # kubectl delete does not expose UID/resourceVersion preconditions, and its - # --raw mode sends no request body. A short-lived localhost-only proxy lets + # --raw mode sends no request body. A proxy on a mode-0600 Unix socket lets # us submit the Kubernetes DeleteOptions body without reimplementing - # kubeconfig authentication or silently deleting a concurrently replaced - # object. - proxy = subprocess.Popen( - [ - "kubectl", - "proxy", - "--port=0", - "--address=127.0.0.1", - r"--accept-hosts=^127\.0\.0\.1$", - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - start_new_session=True, - ) - try: - if proxy.stdout is None or proxy.stderr is None: - raise SystemExit("ERROR: failed to capture kubectl proxy startup output") - - startup_output = {"stdout": bytearray(), "stderr": bytearray()} - startup_pattern = re.compile(rb"(?:^|\n)Starting to serve on 127\.0\.0\.1:(\d+)\r?\n") - port = None - deadline = time.monotonic() + 10 - with selectors.DefaultSelector() as selector: - selector.register(proxy.stdout, selectors.EVENT_READ, "stdout") - selector.register(proxy.stderr, selectors.EVENT_READ, "stderr") - while time.monotonic() < deadline: - events = selector.select(max(0, deadline - time.monotonic())) - if not events: - break - for key, _ in events: - chunk = os.read(key.fd, 4096) - if not chunk: - selector.unregister(key.fileobj) - continue - stream_output = startup_output[key.data] - stream_output.extend(chunk) - if len(stream_output) > 65536: - del stream_output[:-65536] - if key.data == "stdout": - match = startup_pattern.search(stream_output) - if match: - candidate = int(match.group(1)) - if not 1 <= candidate <= 65535: - raise SystemExit( - f"ERROR: kubectl proxy reported invalid port {candidate}" - ) - port = candidate + # kubeconfig authentication, exposing admin credentials on a localhost TCP + # port, or silently deleting a concurrently replaced object. + with tempfile.TemporaryDirectory(prefix="aks-flex-config-") as proxy_dir: + socket_path = os.path.join(proxy_dir, "kubectl-proxy.sock") + proxy = subprocess.Popen( + [ + "kubectl", + "proxy", + f"--unix-socket={socket_path}", + r"--accept-hosts=^localhost$", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + ) + try: + if proxy.stdout is None or proxy.stderr is None: + raise SystemExit("ERROR: failed to capture kubectl proxy startup output") + + startup_output = {"stdout": bytearray(), "stderr": bytearray()} + startup_pattern = re.compile( + rb"(?:^|\n)Starting to serve on " + re.escape(os.fsencode(socket_path)) + rb"\r?\n" + ) + ready = False + deadline = time.monotonic() + 10 + with selectors.DefaultSelector() as selector: + selector.register(proxy.stdout, selectors.EVENT_READ, "stdout") + selector.register(proxy.stderr, selectors.EVENT_READ, "stderr") + while time.monotonic() < deadline: + events = selector.select(max(0, deadline - time.monotonic())) + if not events: + break + for key, _ in events: + chunk = os.read(key.fd, 4096) + if not chunk: + selector.unregister(key.fileobj) + continue + stream_output = startup_output[key.data] + stream_output.extend(chunk) + if len(stream_output) > 65536: + del stream_output[:-65536] + if key.data == "stdout" and startup_pattern.search(stream_output): + ready = True break - if port is not None: - break - if proxy.poll() is not None and not selector.get_map(): - break - - if port is None: - detail = b"\n".join( - output.strip() for output in startup_output.values() if output.strip() - ).decode(errors="replace") - if proxy.poll() is None: + if ready: + break + if proxy.poll() is not None and not selector.get_map(): + break + + if not ready: + detail = b"\n".join( + output.strip() for output in startup_output.values() if output.strip() + ).decode(errors="replace") + if proxy.poll() is None: + raise SystemExit( + "ERROR: timed out starting kubectl proxy for conditional RBAC deletion" + f"{': ' + detail if detail else ''}" + ) + raise SystemExit(f"ERROR: kubectl proxy failed to start: {detail or 'unknown error'}") + + try: + socket_info = os.stat(socket_path, follow_symlinks=False) + if not stat.S_ISSOCK(socket_info.st_mode): + raise SystemExit("ERROR: kubectl proxy did not create a Unix socket") + os.chmod(socket_path, 0o600) + socket_mode = stat.S_IMODE(os.stat(socket_path, follow_symlinks=False).st_mode) + except OSError as err: + raise SystemExit(f"ERROR: could not secure kubectl proxy Unix socket: {err}") from err + if socket_mode != 0o600: raise SystemExit( - "ERROR: timed out starting kubectl proxy for conditional RBAC deletion" - f"{': ' + detail if detail else ''}" + f"ERROR: kubectl proxy Unix socket mode is {socket_mode:o}, expected 600" ) - raise SystemExit(f"ERROR: kubectl proxy failed to start: {detail or 'unknown error'}") - - body = json.dumps( - { - "apiVersion": "v1", - "kind": "DeleteOptions", - "preconditions": {"uid": uid, "resourceVersion": resource_version}, - } - ).encode() - resource_path = f"/apis/{RBAC_API_GROUP}/v1/clusterrolebindings/{quote(name, safe='')}" - request = urlrequest.Request( - f"http://127.0.0.1:{port}{resource_path}", - data=body, - headers={"Content-Type": "application/json"}, - method="DELETE", - ) - try: - # This request is always to the loopback-only kubectl proxy. Ignore - # workstation proxy variables so an empty/misconfigured NO_PROXY - # cannot redirect a privileged Kubernetes API mutation elsewhere. - local_opener = urlrequest.build_opener(urlrequest.ProxyHandler({})) - with local_opener.open(request, timeout=30) as response: - response.read() - except urlerror.HTTPError as err: - detail = err.read().decode(errors="replace").strip() - if err.code == 409: + + body = json.dumps( + { + "apiVersion": "v1", + "kind": "DeleteOptions", + "preconditions": {"uid": uid, "resourceVersion": resource_version}, + } + ).encode() + resource_path = f"/apis/{RBAC_API_GROUP}/v1/clusterrolebindings/{quote(name, safe='')}" + connection = UnixSocketHTTPConnection(socket_path, timeout=30) + try: + connection.request( + "DELETE", + resource_path, + body=body, + headers={"Content-Type": "application/json"}, + ) + response = connection.getresponse() + detail = response.read().decode(errors="replace").strip() + if response.status == 409: + raise SystemExit( + f"ERROR: refusing to delete ClusterRoleBinding {name!r}: it changed after inspection" + ) + if not 200 <= response.status < 300: + raise SystemExit( + f"ERROR: conditional deletion of ClusterRoleBinding {name!r} returned " + f"HTTP {response.status}: {detail}" + ) + except (OSError, http.client.HTTPException) as err: raise SystemExit( - f"ERROR: refusing to delete ClusterRoleBinding {name!r}: it changed after inspection" + f"ERROR: conditional deletion of ClusterRoleBinding {name!r} failed: {err}" ) from err - raise SystemExit( - f"ERROR: conditional deletion of ClusterRoleBinding {name!r} returned " - f"HTTP {err.code}: {detail}" - ) from err - except urlerror.URLError as err: - raise SystemExit(f"ERROR: conditional deletion of ClusterRoleBinding {name!r} failed: {err}") from err - finally: - try: - os.killpg(proxy.pid, signal.SIGTERM) - except ProcessLookupError: - pass - try: - proxy.wait(timeout=5) - except subprocess.TimeoutExpired: - pass - try: - # The kubectl process may have exited while an exec credential - # plugin kept the process group and pipe descriptors alive. - os.killpg(proxy.pid, signal.SIGKILL) - except ProcessLookupError: - pass - try: - proxy.wait(timeout=5) - except subprocess.TimeoutExpired: - log_error(f"kubectl proxy process {proxy.pid} did not exit after SIGKILL") - if proxy.stdout is not None: - proxy.stdout.close() - if proxy.stderr is not None: - proxy.stderr.close() + finally: + connection.close() + finally: + try: + os.killpg(proxy.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + proxy.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + try: + # The kubectl process may have exited while an exec credential + # plugin kept the process group and pipe descriptors alive. + os.killpg(proxy.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + proxy.wait(timeout=5) + except subprocess.TimeoutExpired: + log_error(f"kubectl proxy process {proxy.pid} did not exit after SIGKILL") + if proxy.stdout is not None: + proxy.stdout.close() + if proxy.stderr is not None: + proxy.stderr.close() def remove_legacy_node_role_binding(bindings: list[dict[str, object]] | None = None) -> None: diff --git a/scripts/aks_flex_config_test.go b/scripts/aks_flex_config_test.go index 49dca199..0ba3a334 100644 --- a/scripts/aks_flex_config_test.go +++ b/scripts/aks_flex_config_test.go @@ -507,17 +507,17 @@ func TestSetupNodeRBACMigratesLegacyBindingIdempotently(t *testing.T) { if len(proxyIndexes) != 1 { t.Fatalf("conditional deletion started %d kubectl proxies, want 1: %s", len(proxyIndexes), formatCalls(calls)) } - portArgs := 0 + unixSocketArgs := 0 for _, arg := range calls[proxyIndexes[0]].args { - if strings.HasPrefix(arg, "--port=") { - portArgs++ - if arg != "--port=0" { - t.Errorf("conditional deletion selected proxy port before launch: %q", arg) - } + if strings.HasPrefix(arg, "--unix-socket=") { + unixSocketArgs++ + } + if strings.HasPrefix(arg, "--port=") || strings.HasPrefix(arg, "--address=") { + t.Errorf("conditional deletion exposed a TCP listener: %q", arg) } } - if portArgs != 1 { - t.Fatalf("conditional deletion must let kubectl atomically allocate its proxy port: %s", formatCalls(calls)) + if unixSocketArgs != 1 { + t.Fatalf("conditional deletion must use one private Unix socket: %s", formatCalls(calls)) } if applyIndexes[0] >= deleteIndexes[0] { t.Errorf("legacy binding was deleted before safe RBAC was applied: %s", formatCalls(calls)) @@ -638,17 +638,17 @@ func TestSetupNodeRBACHandlesFragmentedKubectlProxyReadiness(t *testing.T) { } } -func TestSetupNodeRBACBypassesEnvironmentProxyForLoopbackDeletion(t *testing.T) { +func TestSetupNodeRBACBypassesEnvironmentProxyForUnixSocketDeletion(t *testing.T) { t.Parallel() harness := newConfigScriptHarness(t, true, 0) harness.poisonHTTPProxy = true output, err := harness.runSetupNodeRBAC(true) if err != nil { - t.Fatalf("setup-node-rbac sent its loopback deletion through the environment proxy: %v\n%s", err, output) + t.Fatalf("setup-node-rbac sent its Unix-socket deletion through the environment proxy: %v\n%s", err, output) } if got := strings.TrimSpace(readFile(t, harness.legacyState)); got != "absent" { - t.Fatalf("legacy binding state = %q after loopback deletion, want absent", got) + t.Fatalf("legacy binding state = %q after Unix-socket deletion, want absent", got) } } @@ -1306,29 +1306,38 @@ set -eu case "${1:-}" in proxy) - port="" + socket_path="" for arg in "$@"; do case "$arg" in - --port=*) port="${arg#--port=}" ;; + --unix-socket=*) socket_path="${arg#--unix-socket=}" ;; + --port=*|--address=*) exit 52 ;; esac done - if [ -z "$port" ]; then + if [ -z "$socket_path" ]; then exit 50 fi if [ "${AKS_FLEX_CONFIG_TEST_PROXY_STARTUP:-}" = "exit-before-ready" ]; then printf '%s\n' 'injected proxy startup failure' >&2 exit 51 fi - exec python3 -u - "$port" "${AKS_FLEX_CONFIG_TEST_COMMAND_LOG:?}" "${AKS_FLEX_CONFIG_TEST_DELETE_OPTIONS:?}" "${AKS_FLEX_CONFIG_TEST_LEGACY_STATE:?}" "${AKS_FLEX_CONFIG_TEST_DELETE_EXIT:-0}" "${AKS_FLEX_CONFIG_TEST_DELETE_KEEPS_STATE:-false}" "${AKS_FLEX_CONFIG_TEST_CONCURRENT_REPLACE:-false}" "${AKS_FLEX_CONFIG_TEST_PROXY_STARTUP:-}" <<'PY' + exec python3 -u - "$socket_path" "${AKS_FLEX_CONFIG_TEST_COMMAND_LOG:?}" "${AKS_FLEX_CONFIG_TEST_DELETE_OPTIONS:?}" "${AKS_FLEX_CONFIG_TEST_LEGACY_STATE:?}" "${AKS_FLEX_CONFIG_TEST_DELETE_EXIT:-0}" "${AKS_FLEX_CONFIG_TEST_DELETE_KEEPS_STATE:-false}" "${AKS_FLEX_CONFIG_TEST_CONCURRENT_REPLACE:-false}" "${AKS_FLEX_CONFIG_TEST_PROXY_STARTUP:-}" <<'PY' import http.server import json +import os +import socketserver +import stat import sys import time -port, log_path, options_path, state_path, delete_exit, keep_state, concurrent, startup_mode = sys.argv[1:] +socket_path, log_path, options_path, state_path, delete_exit, keep_state, concurrent, startup_mode = sys.argv[1:] class Handler(http.server.BaseHTTPRequestHandler): def do_DELETE(self): + socket_mode = stat.S_IMODE(os.stat(socket_path, follow_symlinks=False).st_mode) + directory_mode = stat.S_IMODE(os.stat(os.path.dirname(socket_path)).st_mode) + if socket_mode != 0o600 or directory_mode != 0o700: + self.respond(500, {"message": f"insecure socket modes {socket_mode:o}/{directory_mode:o}"}) + return length = int(self.headers.get("Content-Length", "0")) body = self.rfile.read(length) with open(log_path, "a", encoding="utf-8") as stream: @@ -1378,10 +1387,14 @@ class Handler(http.server.BaseHTTPRequestHandler): def log_message(self, *_): pass -server = http.server.ThreadingHTTPServer(("127.0.0.1", int(port)), Handler) +class UnixHTTPServer(socketserver.ThreadingMixIn, socketserver.UnixStreamServer): + daemon_threads = True + +os.umask(0o077) +server = UnixHTTPServer(socket_path, Handler) sys.stderr.write("fake proxy diagnostic before readiness\n") sys.stderr.flush() -readiness = f"Starting to serve on 127.0.0.1:{server.server_address[1]}\n" +readiness = f"Starting to serve on {socket_path}\n" if startup_mode == "fragmented-ready": readiness_prefix = "Starting to " for fragment in ( From 0f575716371267c36fa531898fe4d309eef1aead Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 22 Aug 2026 03:51:46 +0000 Subject: [PATCH 13/18] Harden historical compatibility E2E Force post-migration daemon certificate issuance, isolate rerun ownership and artifacts by attempt, and validate every destructive cleanup target before deletion while retaining legacy-state cleanup compatibility. --- .github/workflows/e2e-tests.yml | 6 +- hack/e2e/README.md | 7 +- hack/e2e/cleanup_safety_test.go | 269 +++++++++++++++++++++++ hack/e2e/e2e_scripts_test.go | 72 +++++- hack/e2e/lib/bootstrap-rbac-migration.sh | 47 +++- hack/e2e/lib/cleanup.sh | 190 +++++++++++++--- hack/e2e/lib/infra.sh | 10 +- 7 files changed, 557 insertions(+), 44 deletions(-) create mode 100644 hack/e2e/cleanup_safety_test.go diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index dfce7ec6..dedc6926 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -70,7 +70,7 @@ env: AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} GITHUB_RUN_ID: ${{ github.run_id }} E2E_NAME_SUFFIX: ${{ github.run_id }}-${{ github.run_attempt }} - E2E_WORK_DIR: /tmp/aks-flex-node-e2e-${{ github.run_id }} + E2E_WORK_DIR: /tmp/aks-flex-node-e2e-${{ github.run_id }}-${{ github.run_attempt }} E2E_SUITE: ${{ inputs.suite || 'all' }} E2E_KUBERNETES_VERSION: ${{ inputs.kubernetes_version || '1.35.0' }} @@ -121,8 +121,8 @@ jobs: uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: always() with: - name: e2e-logs-${{ github.run_id }} - path: /tmp/aks-flex-node-e2e-${{ github.run_id }}/logs/ + name: e2e-logs-${{ github.run_id }}-${{ github.run_attempt }} + path: /tmp/aks-flex-node-e2e-${{ github.run_id }}-${{ github.run_attempt }}/logs/ retention-days: 7 - name: Cleanup diff --git a/hack/e2e/README.md b/hack/e2e/README.md index 12e06074..706e1593 100644 --- a/hack/e2e/README.md +++ b/hack/e2e/README.md @@ -167,9 +167,10 @@ daemon runs its production no-op path, not the v0.1.0 file-backed E2E machine client. On that same host it verifies the HEAD helper fails closed without the explicit migration flag, activates the HEAD binary through `agent-upgrade`, removes the legacy binding twice to prove idempotency, checks token access -changes from HTTP 200 to 403 while CSR creation remains authorized, revokes the -token and waits for HTTP 401, and restarts both kubelet and the daemon while -checking the Node UID, Lease, readiness, and certificate-backed API access. +changes from HTTP 200 to 403, deletes the daemon credential store and verifies +the remaining CSR permissions issue a different certificate, revokes the token +and waits for HTTP 401, and restarts both kubelet and the daemon while checking +the Node UID, Lease, readiness, and certificate-backed API access. v0.1.0 transitively pins the non-GPU rootfs `ghcr.io/azure/agent-ubuntu2404:v20260427`. diff --git a/hack/e2e/cleanup_safety_test.go b/hack/e2e/cleanup_safety_test.go new file mode 100644 index 00000000..4eeb7ffa --- /dev/null +++ b/hack/e2e/cleanup_safety_test.go @@ -0,0 +1,269 @@ +package e2e_test + +import ( + _ "embed" + "os" + "path/filepath" + "strings" + "testing" +) + +//go:embed lib/infra.sh +var infraScript string + +func TestCleanupTargetNameValidation(t *testing.T) { + t.Parallel() + + valid := []string{ + "test", + "e2e-test", + "aks-e2e-test", + "MC_aksflex-e2e-test", + "vm-e2e-msi-test", + "vm-e2e-token-test", + "vm-e2e-offline-test", + "vm-e2e-kubeadm-test", + "vm-e2e-arc-test", + "vm-e2e-arc-test-connected", + "vnet-e2e-test", + "nsg-e2e-test", + } + + tests := []struct { + name string + argument int + value string + wantSuccess bool + wantError string + }{ + {name: "matching targets", argument: -1, wantSuccess: true}, + {name: "missing optional target", argument: 4, value: "", wantSuccess: true}, + {name: "missing suffix", argument: 0, value: "", wantError: "without an E2E name suffix"}, + {name: "deployment", argument: 1, value: "production-deployment", wantError: "unexpected ARM deployment"}, + {name: "cluster", argument: 2, value: "production-cluster", wantError: "unexpected AKS cluster"}, + {name: "node resource group", argument: 3, value: "production-node-rg", wantError: "unexpected AKS node resource group"}, + {name: "MSI VM", argument: 4, value: "production-msi", wantError: "unexpected MSI VM"}, + {name: "token VM", argument: 5, value: "production-token", wantError: "unexpected token VM"}, + {name: "offline VM", argument: 6, value: "production-offline", wantError: "unexpected offline VM"}, + {name: "kubeadm VM", argument: 7, value: "production-kubeadm", wantError: "unexpected kubeadm VM"}, + {name: "Arc VM", argument: 8, value: "production-arc", wantError: "unexpected Arc VM"}, + {name: "Arc machine", argument: 9, value: "production-machine", wantError: "unexpected Arc machine"}, + {name: "virtual network", argument: 10, value: "production-vnet", wantError: "unexpected virtual network"}, + {name: "network security group", argument: 11, value: "production-nsg", wantError: "unexpected network security group"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + args := append([]string(nil), valid...) + if test.argument >= 0 { + args[test.argument] = test.value + } + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +shift 2 +if _validate_cleanup_target_names "$@"; then + printf 'RESULT=success\n' +else + printf 'RESULT=error\n' +fi +` + output, err := runBash(t, script, append([]string{t.TempDir(), cleanupScript}, args...)...) + if err != nil { + t.Fatalf("target validation harness failed: %v\n%s", err, output) + } + if test.wantSuccess { + if !strings.Contains(string(output), "RESULT=success") { + t.Fatalf("matching cleanup targets were rejected:\n%s", output) + } + return + } + if !strings.Contains(string(output), "RESULT=error") || + !strings.Contains(string(output), test.wantError) { + t.Fatalf("unsafe cleanup target was not rejected with %q:\n%s", test.wantError, output) + } + }) + } +} + +func TestCleanupResourceOwnerCompatibility(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + state string + githubRunID string + want string + wantError string + }{ + { + name: "new state uses attempt-scoped owner", + state: `{"resource_owner":"12345-2","run_id":"12345","name_suffix":"12345-2"}`, + githubRunID: "12345", + want: "12345-2", + }, + { + name: "legacy state falls back to run ID", + state: `{"run_id":"12345"}`, + githubRunID: "different-run", + want: "12345", + }, + { + name: "partial legacy state falls back to environment", + state: `{}`, + githubRunID: "12345", + want: "12345", + }, + { + name: "new state rejects another attempts owner", + state: `{"resource_owner":"12345-1","run_id":"12345","name_suffix":"12345-2"}`, + githubRunID: "12345", + wantError: "does not match E2E name suffix", + }, + { + name: "new partial state rejects unverifiable owner", + state: `{"resource_owner":"12345-2","run_id":"12345"}`, + githubRunID: "12345", + wantError: "does not match E2E name suffix", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + if err := os.WriteFile(filepath.Join(workDir, "state.json"), []byte(test.state), 0o600); err != nil { + t.Fatalf("write state: %v", err) + } + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +GITHUB_RUN_ID="$2" +source "$3" +if owner="$(_cleanup_resource_owner)"; then + printf 'OWNER=%s\n' "${owner}" +else + printf 'OWNER=error\n' +fi +` + output, err := runBash(t, script, workDir, test.githubRunID, cleanupScript) + if err != nil { + t.Fatalf("resolve cleanup owner: %v\n%s", err, output) + } + if test.wantError != "" { + if !strings.Contains(string(output), "OWNER=error\n") || + !strings.Contains(string(output), test.wantError) { + t.Fatalf("unsafe cleanup owner was not rejected with %q: %q", test.wantError, output) + } + return + } + if !strings.Contains(string(output), "OWNER="+test.want+"\n") { + t.Fatalf("cleanup owner = %q, want %q", output, test.want) + } + }) + } +} + +func TestInfrastructureUsesAttemptScopedOwnershipTag(t *testing.T) { + t.Parallel() + + for _, required := range []string{ + `local resource_owner="${E2E_NAME_SUFFIX}"`, + `--arg resource_owner "${resource_owner}"`, + `run_id: $run_id, resource_owner: $resource_owner`, + `--arg run "${resource_owner}"`, + `'{"github-run": $run, "purpose": $purpose}'`, + } { + if !strings.Contains(infraScript, required) { + t.Errorf("infrastructure ownership metadata is missing %q", required) + } + } +} + +func TestLegacyRunTagCleanupIsScopedToNameSuffix(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + inventoryPath := filepath.Join(workDir, "inventory.json") + inventory := `[ + {"type":"Microsoft.Compute/disks","name":"vm-e2e-token-12345-1_OsDisk_1_abcd","id":"/own-disk"}, + {"type":"Microsoft.Network/networkInterfaces","name":"vm-e2e-msi-12345-1-nic","id":"/own-nic"}, + {"type":"Microsoft.Compute/disks","name":"vm-e2e-token-12345-2_OsDisk_1_efgh","id":"/other-attempt-disk"}, + {"type":"Microsoft.Network/virtualNetworks","name":"vnet-e2e-12345-2","id":"/other-attempt-vnet"}, + {"type":"Microsoft.Compute/disks","name":"production-disk","id":"/unrelated"} +]` + if err := os.WriteFile(inventoryPath, []byte(inventory), 0o600); err != nil { + t.Fatalf("write resource inventory: %v", err) + } + + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +INVENTORY_FILE="$2" +source "$3" +_resource_inventory() { cat "${INVENTORY_FILE}"; } +az() { printf 'AZ=%s\n' "$*"; } +printf '%s\n' 'IDS-BEGIN' +_tagged_resource_ids test-rg 12345 test-subscription 12345-1 +printf '%s\n' 'IDS-END' +_delete_tagged_resources test-rg 12345 test-subscription 12345-1 +` + output, err := runBash(t, script, workDir, inventoryPath, cleanupScript) + if err != nil { + t.Fatalf("legacy tagged cleanup failed: %v\n%s", err, output) + } + text := string(output) + for _, ownID := range []string{"/own-disk", "/own-nic"} { + if !strings.Contains(text, ownID) { + t.Errorf("legacy tagged cleanup omitted owned resource %q:\n%s", ownID, text) + } + } + for _, foreignID := range []string{"/other-attempt-disk", "/other-attempt-vnet", "/unrelated"} { + if strings.Contains(text, foreignID) { + t.Errorf("legacy tagged cleanup selected foreign resource %q:\n%s", foreignID, text) + } + } +} + +func TestCleanupValidatesNamesBeforeAzureMutation(t *testing.T) { + t.Parallel() + + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + contents, err := os.ReadFile(cleanupScript) + if err != nil { + t.Fatalf("read cleanup script: %v", err) + } + body := string(contents) + cleanupStart := strings.Index(body, "cleanup() {") + if cleanupStart < 0 { + t.Fatal("cleanup function is absent") + } + body = body[cleanupStart:] + validation := strings.Index(body, "if ! _validate_cleanup_target_names") + if validation < 0 { + t.Fatal("cleanup does not validate deterministic target names") + } + for _, mutation := range []string{ + `_delete_node_resource_group "${node_resource_group}"`, + `az rest --method delete`, + `az vm delete`, + `az aks delete`, + `az network vnet delete`, + `_delete_tagged_resources`, + } { + mutationIndex := strings.Index(body, mutation) + if mutationIndex < 0 { + t.Fatalf("cleanup mutation %q is absent", mutation) + } + if validation >= mutationIndex { + t.Errorf("cleanup validates names after mutation %q", mutation) + } + } +} diff --git a/hack/e2e/e2e_scripts_test.go b/hack/e2e/e2e_scripts_test.go index 82b15cb0..10e5f03b 100644 --- a/hack/e2e/e2e_scripts_test.go +++ b/hack/e2e/e2e_scripts_test.go @@ -13,7 +13,7 @@ import ( // Embedding the scripts makes Go's test cache invalidate on shell-only changes. // -//go:embed run.sh lib/common.sh lib/cleanup.sh lib/bootstrap-rbac-migration.sh infra/main.bicep +//go:embed run.sh lib/common.sh lib/cleanup.sh lib/bootstrap-rbac-migration.sh infra/*.bicep infra/modules/*.bicep var e2eScripts embed.FS func TestBicepModuleDeploymentNamesAreUniquePerRun(t *testing.T) { @@ -620,7 +620,7 @@ func TestCleanupRejectsUnexpectedOrphanNodeResourceGroup(t *testing.T) { t.Fatalf("orphan cleanup test harness failed: %v\n%s", err, output) } if !strings.Contains(string(output), "RESULT=error") || - !strings.Contains(string(output), "Refusing to delete unexpected persisted AKS node resource group") { + !strings.Contains(string(output), "Refusing to delete unexpected AKS node resource group") { t.Fatalf("cleanup did not reject an unexpected orphan node resource group:\n%s", output) } calls, readErr := os.ReadFile(callLog) @@ -639,6 +639,35 @@ func TestCleanupRejectsUnexpectedOrphanNodeResourceGroup(t *testing.T) { } } +func TestCleanupRejectsUnexpectedLiveNodeResourceGroup(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{unexpectedLiveNodeRG: true}) + if err != nil { + t.Fatalf("live node resource group cleanup test harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") || + !strings.Contains(string(output), "Refusing to delete unexpected AKS node resource group 'production-live-node-rg'") { + t.Fatalf("cleanup did not reject an unexpected live node resource group:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + for _, mutation := range []string{"group delete", "vm delete", "aks delete", "rest --method delete"} { + if strings.Contains(string(calls), mutation) { + t.Fatalf("cleanup attempted mutation %q for an unexpected live node resource group:\n%s", mutation, calls) + } + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("rejected cleanup was marked complete: %s", state) + } +} + func TestCleanupRejectsUnexpectedArcMachineID(t *testing.T) { t.Parallel() @@ -732,6 +761,33 @@ func TestHistoricalCertificateProbeUsesPrivilegedTemporaryFile(t *testing.T) { } } +func TestHistoricalMigrationReissuesDaemonCertificateBeforeTokenRevocation(t *testing.T) { + t.Parallel() + + script, err := e2eScripts.ReadFile("lib/bootstrap-rbac-migration.sh") + if err != nil { + t.Fatalf("read embedded migration script: %v", err) + } + text := string(script) + for _, required := range []string{ + `old_fingerprint="$(openssl x509 -in "${credential_path}" -outform DER`, + `rm -rf -- "${credential_dir}"`, + `new_fingerprint="$(openssl x509 -in "${credential_path}" -outform DER`, + `_require_daemon_certificate_access "${vm_ip}" "${server_url}"`, + } { + if !strings.Contains(text, required) { + t.Fatalf("historical migration is missing certificate reissuance check %q", required) + } + } + + migrationIndex := strings.LastIndex(text, ` --remove-legacy-node-role-binding`) + reissueIndex := strings.LastIndex(text, ` _reissue_daemon_certificate_after_migration "${vm_ip}" "${server_url}"`) + revokeIndex := strings.LastIndex(text, ` with_cluster_lock _revoke_historical_bootstrap_token "${config_file}"`) + if migrationIndex < 0 || reissueIndex <= migrationIndex || revokeIndex <= reissueIndex { + t.Fatalf("certificate reissuance must run after RBAC migration and before token revocation") + } +} + type cleanupOptions struct { runTwice bool leaveCluster bool @@ -740,6 +796,7 @@ type cleanupOptions struct { noRunTags bool blankVMNames bool unexpectedNodeResource bool + unexpectedLiveNodeRG bool unexpectedArcMachineID bool transientQueryFailures bool } @@ -808,6 +865,7 @@ PARENT_GROUP_ABSENT="$8" DEPLOYMENT_QUERY_FAILS="$9" RUN_TWICE="${10}" TRANSIENT_QUERY_FAILURES="${11}" +UNEXPECTED_LIVE_NODE_RG="${12}" # Keep cleanup fixtures independent from the ambient GitHub Actions run. Tests # that need tag-based cleanup persist an explicit run_id in their state. unset GITHUB_RUN_ID @@ -838,8 +896,14 @@ az() { : > "${NODE_GROUP_DELETED}" elif [[ "$1 $2" == "group wait" ]]; then return 0 - elif [[ "$1 $2" == "vm list" || "$1 $2" == "aks list" ]]; then + elif [[ "$1 $2" == "vm list" ]]; then printf '[]\n' + elif [[ "$1 $2" == "aks list" ]]; then + if [[ "${UNEXPECTED_LIVE_NODE_RG}" == "1" ]]; then + printf '[{"name":"aks-e2e-test","nodeResourceGroup":"production-live-node-rg"}]\n' + else + printf '[]\n' + fi elif [[ "$1 $2" == "resource list" ]]; then if [[ "${TRANSIENT_QUERY_FAILURES}" == "1" && ! -f "${RESOURCE_QUERY_FAILED}" ]]; then : > "${RESOURCE_QUERY_FAILED}" @@ -874,7 +938,7 @@ fi groupQueryFailed, resourceQueryFailed, boolString(options.leaveCluster), boolString(options.parentGroupAbsent), boolString(options.deploymentQueryFails), boolString(options.runTwice), - boolString(options.transientQueryFailures)) + boolString(options.transientQueryFailures), boolString(options.unexpectedLiveNodeRG)) return workDir, statePath, callLog, output, err } diff --git a/hack/e2e/lib/bootstrap-rbac-migration.sh b/hack/e2e/lib/bootstrap-rbac-migration.sh index 1ffd0d8b..386de6f3 100644 --- a/hack/e2e/lib/bootstrap-rbac-migration.sh +++ b/hack/e2e/lib/bootstrap-rbac-migration.sh @@ -516,6 +516,51 @@ fi REMOTE } +_reissue_daemon_certificate_after_migration() { + local vm_ip="$1" + local server_url="$2" + + remote_exec "${vm_ip}" 'sudo bash -s' <<'REMOTE' +set -euo pipefail +readonly credential_dir="/etc/aks-flex-node/daemon-credentials" +readonly credential_path="${credential_dir}/daemon-controller-current.pem" +readonly service_name="aks-flex-node-agent.service" + +if [[ ! -s "${credential_path}" ]]; then + echo "daemon certificate is missing before forced reissuance" >&2 + exit 1 +fi +old_fingerprint="$(openssl x509 -in "${credential_path}" -outform DER | sha256sum | awk '{print $1}')" + +# Stop the process before deleting its test credential store so the restart +# must authenticate with the still-valid bootstrap token and submit a new CSR. +systemctl stop "${service_name}" +rm -rf -- "${credential_dir}" +systemctl start "${service_name}" + +for _ in $(seq 1 60); do + if systemctl is-active --quiet "${service_name}" && [[ -s "${credential_path}" ]]; then + new_fingerprint="$(openssl x509 -in "${credential_path}" -outform DER | sha256sum | awk '{print $1}')" + if [[ -n "${new_fingerprint}" && "${new_fingerprint}" != "${old_fingerprint}" ]]; then + exit 0 + fi + fi + if systemctl is-failed --quiet "${service_name}"; then + break + fi + sleep 2 +done + +echo "daemon did not obtain a different certificate after legacy RBAC removal" >&2 +systemctl status "${service_name}" --no-pager >&2 || true +journalctl -u "${service_name}" -n 100 --no-pager >&2 || true +exit 1 +REMOTE + + _require_daemon_certificate_access "${vm_ip}" "${server_url}" + log_success "Daemon obtained a new certificate through least-privilege CSR RBAC after legacy binding removal" +} + _require_old_node_survives_guard() { local vm_name="$1" local vm_ip="$2" @@ -849,7 +894,7 @@ historical_rbac_migration_e2e() { fi _wait_for_bootstrap_token_probe http:403 "${config_file}" list-nodes _wait_for_bootstrap_token_probe allowed "${config_file}" create-csr - _require_daemon_certificate_access "${vm_ip}" "${server_url}" + _reissue_daemon_certificate_after_migration "${vm_ip}" "${server_url}" # Revocation proves the subsequent restarts cannot silently fall back to the # bootstrap credential. diff --git a/hack/e2e/lib/cleanup.sh b/hack/e2e/lib/cleanup.sh index cb2e7889..1c083f31 100755 --- a/hack/e2e/lib/cleanup.sh +++ b/hack/e2e/lib/cleanup.sh @@ -325,18 +325,110 @@ _resource_inventory() { return 1 } -_validate_persisted_node_resource_group() { +_cleanup_resource_owner() { + local name_suffix="${1:-}" owner + if [[ -z "${name_suffix}" ]]; then + name_suffix="$(state_get name_suffix)" + fi + owner="$(state_get resource_owner)" + if [[ -n "${owner}" ]]; then + if [[ -z "${name_suffix}" || "${owner}" != "${name_suffix}" ]]; then + log_error "Refusing tagged cleanup with resource owner '${owner}' that does not match E2E name suffix '${name_suffix}'" + return 1 + fi + printf '%s\n' "${owner}" + return 0 + fi + + # State written before resource_owner was introduced used run_id as the + # github-run tag value. Keep those deployments recoverable. + state_get run_id "${GITHUB_RUN_ID:-}" +} + +_is_expected_legacy_tagged_resource() { + local resource_type="${1,,}" resource_name="$2" name_suffix="$3" + local role vm_prefix + + [[ -n "${resource_name}" && -n "${name_suffix}" ]] || return 1 + case "${resource_type}" in + microsoft.compute/disks) + for role in msi token offline kubeadm arc; do + vm_prefix="vm-e2e-${role}-${name_suffix}" + if [[ "${resource_name}" == "${vm_prefix}-osdisk" || \ + "${resource_name}" == "${vm_prefix}"_OsDisk_* ]]; then + return 0 + fi + done + ;; + microsoft.network/networkinterfaces) + for role in msi token offline kubeadm arc; do + [[ "${resource_name}" == "vm-e2e-${role}-${name_suffix}-nic" ]] && return 0 + done + ;; + microsoft.network/publicipaddresses) + for role in msi token offline kubeadm arc; do + [[ "${resource_name}" == "vm-e2e-${role}-${name_suffix}-pip" ]] && return 0 + done + ;; + microsoft.network/virtualnetworks) + [[ "${resource_name}" == "vnet-e2e-${name_suffix}" ]] && return 0 + ;; + microsoft.network/networksecuritygroups) + [[ "${resource_name}" == "nsg-e2e-${name_suffix}" ]] && return 0 + ;; + esac + return 1 +} + +_validate_expected_cleanup_name() { + local description="$1" actual="$2" expected="$3" + + [[ -n "${actual}" ]] || return 0 + if [[ "${actual}" != "${expected}" ]]; then + log_error "Refusing to delete unexpected ${description} '${actual}'" + log_error "Expected '${expected}' for this E2E deployment" + return 1 + fi +} + +_validate_cleanup_target_names() { + local name_suffix="$1" deployment_name="$2" cluster_name="$3" node_resource_group="$4" + local msi_vm_name="$5" token_vm_name="$6" offline_vm_name="$7" kubeadm_vm_name="$8" + local arc_vm_name="$9" arc_machine_name="${10}" vnet_name="${11}" nsg_name="${12}" + + if [[ -z "${name_suffix}" ]]; then + if [[ -n "${deployment_name}${cluster_name}${node_resource_group}${msi_vm_name}${token_vm_name}${offline_vm_name}${kubeadm_vm_name}${arc_vm_name}${arc_machine_name}${vnet_name}${nsg_name}" ]]; then + log_error "Cannot validate persisted cleanup targets without an E2E name suffix" + return 1 + fi + return 0 + fi + + _validate_expected_cleanup_name "ARM deployment" "${deployment_name}" "e2e-${name_suffix}" || return 1 + _validate_expected_cleanup_name "AKS cluster" "${cluster_name}" "aks-e2e-${name_suffix}" || return 1 + _validate_expected_node_resource_group "${node_resource_group}" "${name_suffix}" || return 1 + _validate_expected_cleanup_name "MSI VM" "${msi_vm_name}" "vm-e2e-msi-${name_suffix}" || return 1 + _validate_expected_cleanup_name "token VM" "${token_vm_name}" "vm-e2e-token-${name_suffix}" || return 1 + _validate_expected_cleanup_name "offline VM" "${offline_vm_name}" "vm-e2e-offline-${name_suffix}" || return 1 + _validate_expected_cleanup_name "kubeadm VM" "${kubeadm_vm_name}" "vm-e2e-kubeadm-${name_suffix}" || return 1 + _validate_expected_cleanup_name "Arc VM" "${arc_vm_name}" "vm-e2e-arc-${name_suffix}" || return 1 + _validate_expected_cleanup_name "Arc machine" "${arc_machine_name}" "vm-e2e-arc-${name_suffix}-connected" || return 1 + _validate_expected_cleanup_name "virtual network" "${vnet_name}" "vnet-e2e-${name_suffix}" || return 1 + _validate_expected_cleanup_name "network security group" "${nsg_name}" "nsg-e2e-${name_suffix}" || return 1 +} + +_validate_expected_node_resource_group() { local node_resource_group="$1" name_suffix="$2" [[ -n "${node_resource_group}" ]] || return 0 if [[ -z "${name_suffix}" ]]; then - log_error "Cannot validate persisted AKS node resource group '${node_resource_group}' without an E2E name suffix" + log_error "Cannot validate AKS node resource group '${node_resource_group}' without an E2E name suffix" return 1 fi local expected_node_resource_group="MC_aksflex-e2e-${name_suffix}" if [[ "${node_resource_group}" != "${expected_node_resource_group}" ]]; then - log_error "Refusing to delete unexpected persisted AKS node resource group '${node_resource_group}'" + log_error "Refusing to delete unexpected AKS node resource group '${node_resource_group}'" log_error "Expected '${expected_node_resource_group}' for this E2E deployment" return 1 fi @@ -377,8 +469,8 @@ _delete_node_resource_group() { } _delete_tagged_resources() { - local resource_group="$1" run_id="$2" subscription_id="$3" - local resource_json resource_type id + local resource_group="$1" run_id="$2" subscription_id="$3" legacy_name_suffix="${4:-}" + local resource_json resource_type resource_name id local -a resource_types=( "Microsoft.Compute/disks" "Microsoft.Network/networkInterfaces" @@ -394,23 +486,35 @@ _delete_tagged_resources() { fi for resource_type in "${resource_types[@]}"; do - while read -r id; do + while IFS=$'\t' read -r resource_name id; do [[ -n "${id}" ]] || continue + if [[ -n "${legacy_name_suffix}" ]] && \ + ! _is_expected_legacy_tagged_resource "${resource_type}" "${resource_name}" "${legacy_name_suffix}"; then + log_warn "Ignoring legacy-tagged ${resource_type} outside this E2E attempt: ${resource_name}" + continue + fi log_info "Deleting residual ${resource_type}: ${id##*/}" az resource delete --ids "${id}" --subscription "${subscription_id}" --output none 2>/dev/null || true done < <(jq -r --arg resource_type "${resource_type}" \ - '.[] | select((.type | ascii_downcase) == ($resource_type | ascii_downcase)) | .id' \ + '.[] | select((.type | ascii_downcase) == ($resource_type | ascii_downcase)) | [.name, .id] | @tsv' \ <<<"${resource_json}") done } _tagged_resource_ids() { - local resource_group="$1" run_id="$2" subscription_id="$3" - local resource_json + local resource_group="$1" run_id="$2" subscription_id="$3" legacy_name_suffix="${4:-}" + local resource_json resource_type resource_name id [[ -n "${run_id}" ]] || return 0 resource_json="$(_resource_inventory "${resource_group}" "${subscription_id}" "${run_id}")" || return 1 - jq -r '.[].id' <<<"${resource_json}" + while IFS=$'\t' read -r resource_type resource_name id; do + [[ -n "${id}" ]] || continue + if [[ -n "${legacy_name_suffix}" ]] && \ + ! _is_expected_legacy_tagged_resource "${resource_type}" "${resource_name}" "${legacy_name_suffix}"; then + continue + fi + printf '%s\n' "${id}" + done < <(jq -r '.[] | [.type, .name, .id] | @tsv' <<<"${resource_json}") } cleanup() { @@ -424,7 +528,8 @@ cleanup() { fi local resource_group cluster_name msi_vm_name token_vm_name offline_vm_name kubeadm_vm_name arc_vm_name arc_machine_name arc_machine_id - local subscription_id deployment_name run_id cleanup_failed vnet_name nsg_name name_suffix node_resource_group cleanup_deadline + local subscription_id deployment_name resource_owner persisted_resource_owner legacy_tag_name_suffix + local cleanup_failed vnet_name nsg_name name_suffix node_resource_group cleanup_deadline resource_group="$(state_get resource_group)" cluster_name="$(state_get cluster_name)" msi_vm_name="$(state_get msi_vm_name)" @@ -435,21 +540,8 @@ cleanup() { arc_machine_name="$(state_get arc_machine_name)" subscription_id="$(state_get subscription_id "${AZURE_SUBSCRIPTION_ID}")" arc_machine_id="$(state_get arc_machine_id)" - if [[ -n "${arc_machine_name}" ]]; then - local expected_arc_machine_id="/subscriptions/${subscription_id}/resourceGroups/${resource_group}/providers/Microsoft.HybridCompute/machines/${arc_machine_name}" - if [[ -n "${arc_machine_id}" && "${arc_machine_id,,}" != "${expected_arc_machine_id,,}" ]]; then - log_error "Refusing to delete Arc machine with an unexpected persisted resource ID: ${arc_machine_id}" - return 1 - fi - # The ID is completely determined by other state fields. Reconstructing it - # avoids trusting a redundant deletion target from stale or damaged state. - arc_machine_id="${expected_arc_machine_id}" - elif [[ -n "${arc_machine_id}" ]]; then - log_error "Cannot validate persisted Arc machine resource ID without arc_machine_name" - return 1 - fi deployment_name="$(state_get deployment_name)" - run_id="$(state_get run_id "${GITHUB_RUN_ID:-}")" + persisted_resource_owner="$(state_get resource_owner)" name_suffix="$(state_get name_suffix)" vnet_name="$(state_get vnet_name)" nsg_name="$(state_get nsg_name)" @@ -466,6 +558,17 @@ cleanup() { if [[ -z "${nsg_name}" && -n "${name_suffix}" ]]; then nsg_name="nsg-e2e-${name_suffix}" fi + if ! resource_owner="$(_cleanup_resource_owner "${name_suffix}")"; then + return 1 + fi + legacy_tag_name_suffix="" + if [[ -z "${persisted_resource_owner}" && -n "${resource_owner}" ]]; then + if [[ -z "${name_suffix}" ]]; then + log_error "Cannot safely clean resources with a legacy run tag without an E2E name suffix" + return 1 + fi + legacy_tag_name_suffix="${name_suffix}" + fi cleanup_failed=0 cleanup_deadline=$((SECONDS + E2E_CLEANUP_TIMEOUT)) @@ -483,9 +586,31 @@ cleanup() { log_error "Failed to determine whether resource group '${resource_group}' exists" return 1 fi + + if ! _validate_cleanup_target_names \ + "${name_suffix}" "${deployment_name}" "${cluster_name}" "${node_resource_group}" \ + "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" "${kubeadm_vm_name}" \ + "${arc_vm_name}" "${arc_machine_name}" "${vnet_name}" "${nsg_name}"; then + return 1 + fi + + if [[ -n "${arc_machine_name}" ]]; then + local expected_arc_machine_id="/subscriptions/${subscription_id}/resourceGroups/${resource_group}/providers/Microsoft.HybridCompute/machines/${arc_machine_name}" + if [[ -n "${arc_machine_id}" && "${arc_machine_id,,}" != "${expected_arc_machine_id,,}" ]]; then + log_error "Refusing to delete Arc machine with an unexpected persisted resource ID: ${arc_machine_id}" + return 1 + fi + # The ID is completely determined by other state fields. Reconstructing it + # avoids trusting a redundant deletion target from stale or damaged state. + arc_machine_id="${expected_arc_machine_id}" + elif [[ -n "${arc_machine_id}" ]]; then + log_error "Cannot validate persisted Arc machine resource ID without arc_machine_name" + return 1 + fi + case "${resource_group_exists}" in false) - if ! _validate_persisted_node_resource_group "${node_resource_group}" "${name_suffix}"; then + if ! _validate_expected_node_resource_group "${node_resource_group}" "${name_suffix}"; then return 1 fi if ! _delete_node_resource_group "${node_resource_group}" "${subscription_id}" "${cleanup_deadline}"; then @@ -565,12 +690,15 @@ cleanup() { return 1 fi if [[ -n "${live_node_resource_group}" ]]; then + if ! _validate_expected_node_resource_group "${live_node_resource_group}" "${name_suffix}"; then + return 1 + fi if [[ -n "${node_resource_group}" && "${node_resource_group}" != "${live_node_resource_group}" ]]; then log_warn "Live AKS node resource group differs from state; using the live cluster value '${live_node_resource_group}'" fi node_resource_group="${live_node_resource_group}" state_set "node_resource_group" "${node_resource_group}" || return 1 - elif ! _validate_persisted_node_resource_group "${node_resource_group}" "${name_suffix}"; then + elif ! _validate_expected_node_resource_group "${node_resource_group}" "${name_suffix}"; then return 1 fi if [[ -n "${node_resource_group}" && "${node_resource_group}" == "${resource_group}" ]]; then @@ -675,11 +803,11 @@ cleanup() { log_info "[8/8] Cleaning up tagged network and disk resources..." local remaining_ids empty_inventories=0 for _ in 1 2 3 4; do - if ! _delete_tagged_resources "${resource_group}" "${run_id}" "${subscription_id}"; then + if ! _delete_tagged_resources "${resource_group}" "${resource_owner}" "${subscription_id}" "${legacy_tag_name_suffix}"; then cleanup_failed=1 break fi - if ! remaining_ids="$(_tagged_resource_ids "${resource_group}" "${run_id}" "${subscription_id}")"; then + if ! remaining_ids="$(_tagged_resource_ids "${resource_group}" "${resource_owner}" "${subscription_id}" "${legacy_tag_name_suffix}")"; then log_error "Failed to verify tagged-resource deletion" cleanup_failed=1 break @@ -695,12 +823,12 @@ cleanup() { sleep "${E2E_CLEANUP_POLL_INTERVAL:-5}" done - if [[ -n "${run_id}" && "${empty_inventories}" -lt 2 ]]; then + if [[ -n "${resource_owner}" && "${empty_inventories}" -lt 2 ]]; then log_error "Tagged-resource inventory did not remain empty for two consecutive checks" cleanup_failed=1 fi - if ! remaining_ids="$(_tagged_resource_ids "${resource_group}" "${run_id}" "${subscription_id}")"; then + if ! remaining_ids="$(_tagged_resource_ids "${resource_group}" "${resource_owner}" "${subscription_id}" "${legacy_tag_name_suffix}")"; then log_error "Failed final tagged-resource verification" cleanup_failed=1 elif [[ -n "${remaining_ids}" ]]; then diff --git a/hack/e2e/lib/infra.sh b/hack/e2e/lib/infra.sh index ea70dd90..3f6aad97 100755 --- a/hack/e2e/lib/infra.sh +++ b/hack/e2e/lib/infra.sh @@ -63,6 +63,10 @@ infra_deploy() { # always-run cleanup stage can remove resources after a partial deployment. local deployment_name="e2e-${E2E_NAME_SUFFIX}" local run_id="${GITHUB_RUN_ID:-local-${E2E_NAME_SUFFIX}}" + # A workflow rerun keeps github.run_id but increments github.run_attempt. + # E2E_NAME_SUFFIX includes both, so use it as the ownership tag to keep + # cleanup for one attempt from deleting another attempt's resources. + local resource_owner="${E2E_NAME_SUFFIX}" local initial_state initial_state="$(jq -n \ --arg lifecycle "provisioning" \ @@ -71,6 +75,7 @@ infra_deploy() { --arg subscription_id "${AZURE_SUBSCRIPTION_ID}" \ --arg tenant_id "${AZURE_TENANT_ID}" \ --arg run_id "${run_id}" \ + --arg resource_owner "${resource_owner}" \ --arg name_suffix "${E2E_NAME_SUFFIX}" \ --arg kubernetes_version "${E2E_KUBERNETES_VERSION}" \ --arg target_agent_pool_name "${E2E_TARGET_AGENT_POOL_NAME}" \ @@ -89,7 +94,8 @@ infra_deploy() { --arg arc_machine_name "vm-e2e-arc-${E2E_NAME_SUFFIX}-connected" \ '{schema_version: 1, lifecycle: $lifecycle, resource_group: $resource_group, location: $location, subscription_id: $subscription_id, tenant_id: $tenant_id, - run_id: $run_id, name_suffix: $name_suffix, kubernetes_version: $kubernetes_version, + run_id: $run_id, resource_owner: $resource_owner, + name_suffix: $name_suffix, kubernetes_version: $kubernetes_version, target_agent_pool_name: $target_agent_pool_name, bootstrap_data_agent_pool_name: $bootstrap_data_agent_pool_name, system_pool_name: $system_pool_name, @@ -121,7 +127,7 @@ infra_deploy() { # Build tags local tags_json tags_json=$(jq -n \ - --arg run "${run_id}" \ + --arg run "${resource_owner}" \ --arg purpose "e2e-test" \ '{"github-run": $run, "purpose": $purpose}') From 0d1fc4c03329cbdfa218184760688dd0f54381a7 Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 22 Aug 2026 04:10:22 +0000 Subject: [PATCH 14/18] Harden cleanup for historical E2E resources Recover AKS default node resource groups and untagged implicit OS disks from pre-hardening state without crossing attempt boundaries. --- hack/e2e/cleanup_safety_test.go | 92 +++++++++++++++++ hack/e2e/e2e_scripts_test.go | 134 +++++++++++++++++++++--- hack/e2e/lib/cleanup.sh | 177 ++++++++++++++++++++++++++++---- 3 files changed, 368 insertions(+), 35 deletions(-) diff --git a/hack/e2e/cleanup_safety_test.go b/hack/e2e/cleanup_safety_test.go index 4eeb7ffa..b27fee5f 100644 --- a/hack/e2e/cleanup_safety_test.go +++ b/hack/e2e/cleanup_safety_test.go @@ -27,6 +27,8 @@ func TestCleanupTargetNameValidation(t *testing.T) { "vm-e2e-arc-test-connected", "vnet-e2e-test", "nsg-e2e-test", + "test-rg", + "test-location", } tests := []struct { @@ -37,6 +39,7 @@ func TestCleanupTargetNameValidation(t *testing.T) { wantError string }{ {name: "matching targets", argument: -1, wantSuccess: true}, + {name: "legacy default node resource group", argument: 3, value: "MC_test-rg_aks-e2e-test_test-location", wantSuccess: true}, {name: "missing optional target", argument: 4, value: "", wantSuccess: true}, {name: "missing suffix", argument: 0, value: "", wantError: "without an E2E name suffix"}, {name: "deployment", argument: 1, value: "production-deployment", wantError: "unexpected ARM deployment"}, @@ -90,6 +93,41 @@ fi } } +func TestCleanupHandlesLegacyDefaultNodeResourceGroup(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + parentGroupAbsent bool + }{ + {name: "live cluster"}, + {name: "orphan after parent deletion", parentGroupAbsent: true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + _, _, callLog, output, err := runCleanup(t, cleanupOptions{ + runTwice: true, + parentGroupAbsent: test.parentGroupAbsent, + legacyDefaultNodeResource: true, + }) + if err != nil { + t.Fatalf("legacy node resource group cleanup failed: %v\n%s", err, output) + } + if strings.Count(string(output), "RESULT=success") != 2 { + t.Fatalf("legacy node resource group cleanup was not idempotent:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + if !strings.Contains(string(calls), "group delete --name MC_test-rg_aks-e2e-test_test-location") { + t.Fatalf("cleanup did not delete the derived legacy node resource group:\n%s", calls) + } + }) + } +} + func TestCleanupResourceOwnerCompatibility(t *testing.T) { t.Parallel() @@ -232,6 +270,60 @@ _delete_tagged_resources test-rg 12345 test-subscription 12345-1 } } +func TestLegacyImplicitOSDiskCleanupIsScopedToValidatedVMNames(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + inventoryPath := filepath.Join(workDir, "inventory.json") + inventory := `[ + {"type":"Microsoft.Compute/disks","name":"vm-e2e-token-12345-1_OsDisk_1_abcd","id":"/own-legacy-disk"}, + {"type":"Microsoft.Compute/disks","name":"vm-e2e-token-12345-2_OsDisk_1_efgh","id":"/other-attempt-disk"}, + {"type":"Microsoft.Compute/disks","name":"vm-e2e-token-12345-1-extra_OsDisk_1_ijkl","id":"/prefix-collision-disk"}, + {"type":"Microsoft.Compute/disks","name":"vm-e2e-token-12345-1-osdisk","id":"/deterministic-disk"}, + {"type":"Microsoft.Network/networkInterfaces","name":"vm-e2e-token-12345-1_OsDisk_1_abcd","id":"/wrong-resource-type"}, + {"type":"Microsoft.Compute/disks","name":"production-disk","id":"/unrelated-disk"} +]` + if err := os.WriteFile(inventoryPath, []byte(inventory), 0o600); err != nil { + t.Fatalf("write resource inventory: %v", err) + } + + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +INVENTORY_FILE="$2" +source "$3" +_resource_inventory() { cat "${INVENTORY_FILE}"; } +az() { printf 'AZ=%s\n' "$*"; } +printf '%s\n' 'IDS-BEGIN' +_legacy_implicit_os_disk_ids test-rg test-subscription \ + vm-e2e-msi-12345-1 vm-e2e-token-12345-1 +printf '%s\n' 'IDS-END' +_delete_legacy_implicit_os_disks test-rg test-subscription \ + vm-e2e-msi-12345-1 vm-e2e-token-12345-1 +` + output, err := runBash(t, script, workDir, inventoryPath, cleanupScript) + if err != nil { + t.Fatalf("legacy implicit OS-disk cleanup failed: %v\n%s", err, output) + } + text := string(output) + if !strings.Contains(text, "/own-legacy-disk") || + !strings.Contains(text, "resource delete --ids /own-legacy-disk") { + t.Fatalf("legacy implicit OS disk was not selected and deleted:\n%s", text) + } + for _, foreignID := range []string{ + "/other-attempt-disk", + "/prefix-collision-disk", + "/deterministic-disk", + "/wrong-resource-type", + "/unrelated-disk", + } { + if strings.Contains(text, foreignID) { + t.Errorf("legacy implicit OS-disk cleanup selected foreign resource %q:\n%s", foreignID, text) + } + } +} + func TestCleanupValidatesNamesBeforeAzureMutation(t *testing.T) { t.Parallel() diff --git a/hack/e2e/e2e_scripts_test.go b/hack/e2e/e2e_scripts_test.go index 10e5f03b..c0e0e02f 100644 --- a/hack/e2e/e2e_scripts_test.go +++ b/hack/e2e/e2e_scripts_test.go @@ -583,6 +583,75 @@ func TestCleanupHandlesLegacyStateWithoutVMNames(t *testing.T) { } } +func TestCleanupDeletesDetachedUntaggedLegacyOSDisk(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{ + runTwice: true, + noRunTags: true, + legacyDetachedDisk: true, + }) + if err != nil { + t.Fatalf("legacy detached OS-disk cleanup failed: %v\n%s", err, output) + } + if strings.Count(string(output), "RESULT=success") != 2 { + t.Fatalf("legacy detached OS-disk cleanup was not idempotent:\n%s", output) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if !strings.Contains(string(state), `"cleanup_complete": "true"`) || + !strings.Contains(string(state), `"lifecycle": "cleaned"`) { + t.Fatalf("legacy detached OS-disk cleanup did not record completion: %s", state) + } + + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + callText := string(calls) + if !strings.Contains(callText, "resource delete --ids /own-legacy-disk") { + t.Fatalf("cleanup did not delete the detached untagged legacy OS disk:\n%s", calls) + } + for _, foreignID := range []string{"/other-attempt-disk", "/unrelated-disk"} { + if strings.Contains(callText, "resource delete --ids "+foreignID) { + t.Errorf("cleanup deleted foreign resource %q:\n%s", foreignID, calls) + } + } +} + +func TestCleanupRetriesAndReportsResidualLegacyOSDisk(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{ + noRunTags: true, + legacyDetachedDisk: true, + legacyDiskDeletePersists: true, + }) + if err != nil { + t.Fatalf("cleanup test harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") { + t.Fatalf("cleanup accepted a residual legacy OS disk:\n%s", output) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if strings.Contains(string(state), `"cleanup_complete": "true"`) || + strings.Contains(string(state), `"lifecycle": "cleaned"`) { + t.Fatalf("residual legacy OS disk was marked clean: %s", state) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + if got := strings.Count(string(calls), "resource delete --ids /own-legacy-disk"); got < 4 { + t.Fatalf("legacy OS-disk deletion was attempted %d times, want at least 4:\n%s", got, calls) + } +} + func TestCleanupDeletesOrphanNodeResourceGroupWhenParentIsAbsent(t *testing.T) { t.Parallel() @@ -789,16 +858,19 @@ func TestHistoricalMigrationReissuesDaemonCertificateBeforeTokenRevocation(t *te } type cleanupOptions struct { - runTwice bool - leaveCluster bool - parentGroupAbsent bool - deploymentQueryFails bool - noRunTags bool - blankVMNames bool - unexpectedNodeResource bool - unexpectedLiveNodeRG bool - unexpectedArcMachineID bool - transientQueryFailures bool + runTwice bool + leaveCluster bool + parentGroupAbsent bool + deploymentQueryFails bool + noRunTags bool + blankVMNames bool + unexpectedNodeResource bool + unexpectedLiveNodeRG bool + legacyDefaultNodeResource bool + legacyDetachedDisk bool + legacyDiskDeletePersists bool + unexpectedArcMachineID bool + transientQueryFailures bool } func runCleanup(t *testing.T, options cleanupOptions) (string, string, string, []byte, error) { @@ -811,6 +883,7 @@ func runCleanup(t *testing.T, options cleanupOptions) (string, string, string, [ resourceQueryFailed := filepath.Join(workDir, "resource-query-failed") state := `{ "resource_group": "test-rg", + "location": "test-location", "subscription_id": "test-subscription", "deployment_name": "e2e-test", "run_id": "test-run", @@ -844,6 +917,9 @@ func runCleanup(t *testing.T, options cleanupOptions) (string, string, string, [ if options.unexpectedNodeResource { state = strings.Replace(state, `"node_resource_group": "MC_aksflex-e2e-test"`, `"node_resource_group": "production-node-rg"`, 1) } + if options.legacyDefaultNodeResource { + state = strings.Replace(state, " \"node_resource_group\": \"MC_aksflex-e2e-test\",\n", "", 1) + } if options.unexpectedArcMachineID { state = strings.Replace(state, `"arc_machine_name": ""`, `"arc_machine_name": "vm-e2e-arc-test-connected", "arc_machine_id": "/subscriptions/test-subscription/resourceGroups/production-rg/providers/Microsoft.HybridCompute/machines/production-machine"`, 1) @@ -866,6 +942,10 @@ DEPLOYMENT_QUERY_FAILS="$9" RUN_TWICE="${10}" TRANSIENT_QUERY_FAILURES="${11}" UNEXPECTED_LIVE_NODE_RG="${12}" +LEGACY_DEFAULT_NODE_RG="${13}" +LEGACY_DETACHED_DISK="${14}" +LEGACY_DISK_DELETE_PERSISTS="${15}" +LEGACY_DISK_DELETED="${E2E_WORK_DIR}/legacy-disk-deleted" # Keep cleanup fixtures independent from the ambient GitHub Actions run. Tests # that need tag-based cleanup persist an explicit run_id in their state. unset GITHUB_RUN_ID @@ -885,10 +965,11 @@ az() { : > "${GROUP_QUERY_FAILED}" return 1 fi - if [[ "$*" == *"--name test-rg"* ]]; then - [[ "${PARENT_GROUP_ABSENT}" == "1" ]] && printf 'false\n' || printf 'true\n' - elif [[ "$*" == *"--name MC_aksflex-e2e-test"* ]]; then + if [[ "$*" == *"--name MC_aksflex-e2e-test"* || \ + "$*" == *"--name MC_test-rg_aks-e2e-test_test-location"* ]]; then [[ -f "${NODE_GROUP_DELETED}" ]] && printf 'false\n' || printf 'true\n' + elif [[ "$*" == *"--name test-rg"* ]]; then + [[ "${PARENT_GROUP_ABSENT}" == "1" ]] && printf 'false\n' || printf 'true\n' else printf 'false\n' fi @@ -901,9 +982,17 @@ az() { elif [[ "$1 $2" == "aks list" ]]; then if [[ "${UNEXPECTED_LIVE_NODE_RG}" == "1" ]]; then printf '[{"name":"aks-e2e-test","nodeResourceGroup":"production-live-node-rg"}]\n' + elif [[ "${LEGACY_DEFAULT_NODE_RG}" == "1" ]]; then + printf '[{"name":"aks-e2e-test","location":"test-location","nodeResourceGroup":"MC_test-rg_aks-e2e-test_test-location"}]\n' else printf '[]\n' fi + elif [[ "$1 $2" == "resource delete" ]]; then + if [[ "$*" == *"--ids /own-legacy-disk"* && \ + "${LEGACY_DISK_DELETE_PERSISTS}" != "1" ]]; then + : > "${LEGACY_DISK_DELETED}" + fi + return 0 elif [[ "$1 $2" == "resource list" ]]; then if [[ "${TRANSIENT_QUERY_FAILURES}" == "1" && ! -f "${RESOURCE_QUERY_FAILED}" ]]; then : > "${RESOURCE_QUERY_FAILED}" @@ -912,11 +1001,20 @@ az() { if [[ "$*" == *"--tag "* ]]; then [[ "$*" == *"--output json"* ]] && printf '[]\n' || true elif [[ "$*" == *"--output json"* ]]; then + resource_json='[]' if [[ "${LEAVE_CLUSTER}" == "1" ]]; then - printf '[{"name":"aks-e2e-test","id":"/subscriptions/test/resourceGroups/test-rg/providers/Microsoft.ContainerService/managedClusters/aks-e2e-test"}]\n' - else - printf '[]\n' + resource_json="$(jq -c '. + [{"name":"aks-e2e-test","id":"/subscriptions/test/resourceGroups/test-rg/providers/Microsoft.ContainerService/managedClusters/aks-e2e-test"}]' <<<"${resource_json}")" + fi + if [[ "${LEGACY_DETACHED_DISK}" == "1" ]]; then + resource_json="$(jq -c '. + [ + {"type":"Microsoft.Compute/disks","name":"vm-e2e-token-other_OsDisk_1_efgh","id":"/other-attempt-disk"}, + {"type":"Microsoft.Compute/disks","name":"production-disk","id":"/unrelated-disk"} + ]' <<<"${resource_json}")" + if [[ ! -f "${LEGACY_DISK_DELETED}" ]]; then + resource_json="$(jq -c '. + [{"type":"Microsoft.Compute/disks","name":"vm-e2e-token-test_OsDisk_1_abcd","id":"/own-legacy-disk"}]' <<<"${resource_json}")" + fi fi + printf '%s\n' "${resource_json}" fi else return 0 @@ -938,7 +1036,9 @@ fi groupQueryFailed, resourceQueryFailed, boolString(options.leaveCluster), boolString(options.parentGroupAbsent), boolString(options.deploymentQueryFails), boolString(options.runTwice), - boolString(options.transientQueryFailures), boolString(options.unexpectedLiveNodeRG)) + boolString(options.transientQueryFailures), boolString(options.unexpectedLiveNodeRG), + boolString(options.legacyDefaultNodeResource), boolString(options.legacyDetachedDisk), + boolString(options.legacyDiskDeletePersists)) return workDir, statePath, callLog, output, err } diff --git a/hack/e2e/lib/cleanup.sh b/hack/e2e/lib/cleanup.sh index 1c083f31..9041e649 100755 --- a/hack/e2e/lib/cleanup.sh +++ b/hack/e2e/lib/cleanup.sh @@ -380,6 +380,60 @@ _is_expected_legacy_tagged_resource() { return 1 } +_legacy_implicit_os_disk_ids_from_inventory() { + local resource_json="$1" + shift + local resource_rows resource_type resource_name id vm_name + + if ! resource_rows="$(jq -r ' + if type != "array" then error("resource list must be an array") + else .[] | [(.type // ""), (.name // ""), (.id // "")] | + if all(.[]; type == "string") then @tsv + else error("resource type, name, and ID must be strings") + end + end + ' <<<"${resource_json}")"; then + log_error "Azure returned an invalid resource inventory while locating legacy OS disks" + return 1 + fi + + while IFS=$'\t' read -r resource_type resource_name id; do + [[ "${resource_type,,}" == "microsoft.compute/disks" && -n "${id}" ]] || continue + for vm_name in "$@"; do + [[ -n "${vm_name}" ]] || continue + if [[ "${resource_name}" == "${vm_name}"_OsDisk_* ]]; then + printf '%s\n' "${id}" + break + fi + done + done <<<"${resource_rows}" +} + +_legacy_implicit_os_disk_ids() { + local resource_group="$1" subscription_id="$2" resource_json + shift 2 + + (( $# > 0 )) || return 0 + if ! resource_json="$(_resource_inventory "${resource_group}" "${subscription_id}")"; then + return 1 + fi + _legacy_implicit_os_disk_ids_from_inventory "${resource_json}" "$@" +} + +_delete_legacy_implicit_os_disks() { + local resource_group="$1" subscription_id="$2" disk_ids disk_id + shift 2 + + if ! disk_ids="$(_legacy_implicit_os_disk_ids "${resource_group}" "${subscription_id}" "$@")"; then + return 1 + fi + while IFS= read -r disk_id; do + [[ -n "${disk_id}" ]] || continue + log_info "Deleting residual legacy OS disk: ${disk_id##*/}" + az resource delete --ids "${disk_id}" --subscription "${subscription_id}" --output none 2>/dev/null || true + done <<<"${disk_ids}" +} + _validate_expected_cleanup_name() { local description="$1" actual="$2" expected="$3" @@ -395,6 +449,7 @@ _validate_cleanup_target_names() { local name_suffix="$1" deployment_name="$2" cluster_name="$3" node_resource_group="$4" local msi_vm_name="$5" token_vm_name="$6" offline_vm_name="$7" kubeadm_vm_name="$8" local arc_vm_name="$9" arc_machine_name="${10}" vnet_name="${11}" nsg_name="${12}" + local resource_group="${13}" location="${14}" if [[ -z "${name_suffix}" ]]; then if [[ -n "${deployment_name}${cluster_name}${node_resource_group}${msi_vm_name}${token_vm_name}${offline_vm_name}${kubeadm_vm_name}${arc_vm_name}${arc_machine_name}${vnet_name}${nsg_name}" ]]; then @@ -406,7 +461,8 @@ _validate_cleanup_target_names() { _validate_expected_cleanup_name "ARM deployment" "${deployment_name}" "e2e-${name_suffix}" || return 1 _validate_expected_cleanup_name "AKS cluster" "${cluster_name}" "aks-e2e-${name_suffix}" || return 1 - _validate_expected_node_resource_group "${node_resource_group}" "${name_suffix}" || return 1 + _validate_expected_node_resource_group \ + "${node_resource_group}" "${name_suffix}" "${resource_group}" "${cluster_name}" "${location}" || return 1 _validate_expected_cleanup_name "MSI VM" "${msi_vm_name}" "vm-e2e-msi-${name_suffix}" || return 1 _validate_expected_cleanup_name "token VM" "${token_vm_name}" "vm-e2e-token-${name_suffix}" || return 1 _validate_expected_cleanup_name "offline VM" "${offline_vm_name}" "vm-e2e-offline-${name_suffix}" || return 1 @@ -419,6 +475,7 @@ _validate_cleanup_target_names() { _validate_expected_node_resource_group() { local node_resource_group="$1" name_suffix="$2" + local resource_group="${3:-}" cluster_name="${4:-}" location="${5:-}" [[ -n "${node_resource_group}" ]] || return 0 if [[ -z "${name_suffix}" ]]; then @@ -427,11 +484,27 @@ _validate_expected_node_resource_group() { fi local expected_node_resource_group="MC_aksflex-e2e-${name_suffix}" - if [[ "${node_resource_group}" != "${expected_node_resource_group}" ]]; then - log_error "Refusing to delete unexpected AKS node resource group '${node_resource_group}'" + if [[ "${node_resource_group}" == "${expected_node_resource_group}" ]]; then + return 0 + fi + + # E2E state written before the template assigned a deterministic node + # resource group uses AKS's documented default naming convention. + local legacy_node_resource_group="" + if [[ -n "${resource_group}" && -n "${cluster_name}" && -n "${location}" ]]; then + legacy_node_resource_group="MC_${resource_group}_${cluster_name}_${location}" + if [[ "${node_resource_group}" == "${legacy_node_resource_group}" ]]; then + return 0 + fi + fi + + log_error "Refusing to delete unexpected AKS node resource group '${node_resource_group}'" + if [[ -n "${legacy_node_resource_group}" ]]; then + log_error "Expected '${expected_node_resource_group}' or legacy '${legacy_node_resource_group}' for this E2E deployment" + else log_error "Expected '${expected_node_resource_group}' for this E2E deployment" - return 1 fi + return 1 } _delete_node_resource_group() { @@ -529,7 +602,7 @@ cleanup() { local resource_group cluster_name msi_vm_name token_vm_name offline_vm_name kubeadm_vm_name arc_vm_name arc_machine_name arc_machine_id local subscription_id deployment_name resource_owner persisted_resource_owner legacy_tag_name_suffix - local cleanup_failed vnet_name nsg_name name_suffix node_resource_group cleanup_deadline + local cleanup_failed vnet_name nsg_name name_suffix node_resource_group location cleanup_deadline resource_group="$(state_get resource_group)" cluster_name="$(state_get cluster_name)" msi_vm_name="$(state_get msi_vm_name)" @@ -546,6 +619,7 @@ cleanup() { vnet_name="$(state_get vnet_name)" nsg_name="$(state_get nsg_name)" node_resource_group="$(state_get node_resource_group)" + location="$(state_get location)" if [[ -z "${name_suffix}" && "${cluster_name}" == aks-e2e-* ]]; then name_suffix="${cluster_name#aks-e2e-}" fi @@ -558,6 +632,10 @@ cleanup() { if [[ -z "${nsg_name}" && -n "${name_suffix}" ]]; then nsg_name="nsg-e2e-${name_suffix}" fi + if [[ -z "${node_resource_group}" && -z "${persisted_resource_owner}" && \ + -n "${resource_group}" && -n "${cluster_name}" && -n "${location}" ]]; then + node_resource_group="MC_${resource_group}_${cluster_name}_${location}" + fi if ! resource_owner="$(_cleanup_resource_owner "${name_suffix}")"; then return 1 fi @@ -590,7 +668,8 @@ cleanup() { if ! _validate_cleanup_target_names \ "${name_suffix}" "${deployment_name}" "${cluster_name}" "${node_resource_group}" \ "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" "${kubeadm_vm_name}" \ - "${arc_vm_name}" "${arc_machine_name}" "${vnet_name}" "${nsg_name}"; then + "${arc_vm_name}" "${arc_machine_name}" "${vnet_name}" "${nsg_name}" \ + "${resource_group}" "${location}"; then return 1 fi @@ -610,7 +689,8 @@ cleanup() { case "${resource_group_exists}" in false) - if ! _validate_expected_node_resource_group "${node_resource_group}" "${name_suffix}"; then + if ! _validate_expected_node_resource_group \ + "${node_resource_group}" "${name_suffix}" "${resource_group}" "${cluster_name}" "${location}"; then return 1 fi if ! _delete_node_resource_group "${node_resource_group}" "${subscription_id}" "${cleanup_deadline}"; then @@ -643,8 +723,9 @@ cleanup() { # Snapshot IDs that are difficult to recover after deleting their parents. # This supports cleanup of deployments created before deterministic OS-disk # names and delete options were added to the Bicep module. - local vm_inventory aks_inventory live_node_resource_group vm_name disk_id nic_id nic_output - local -a managed_disk_ids=() nic_ids=() + local vm_inventory aks_inventory resource_inventory live_node_resource_group live_cluster_location + local vm_name disk_id nic_id nic_output legacy_disk_output + local -a managed_disk_ids=() legacy_implicit_os_disk_ids=() nic_ids=() if ! vm_inventory="$(az vm list \ --resource-group "${resource_group}" \ --subscription "${subscription_id}" \ @@ -664,6 +745,19 @@ cleanup() { log_error "Azure returned an invalid VM or AKS cleanup inventory" return 1 fi + if ! resource_inventory="$(_resource_inventory "${resource_group}" "${subscription_id}")"; then + log_error "Failed to inventory E2E resources before cleanup" + return 1 + fi + if ! legacy_disk_output="$(_legacy_implicit_os_disk_ids_from_inventory \ + "${resource_inventory}" \ + "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" \ + "${kubeadm_vm_name}" "${arc_vm_name}")"; then + return 1 + fi + while IFS= read -r disk_id; do + [[ -z "${disk_id}" ]] || legacy_implicit_os_disk_ids+=("${disk_id}") + done <<<"${legacy_disk_output}" for vm_name in "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" "${kubeadm_vm_name}" "${arc_vm_name}"; do [[ -n "${vm_name}" ]] || continue if ! disk_id="$(jq -r --arg name "${vm_name}" \ @@ -689,8 +783,23 @@ cleanup() { log_error "Failed to inspect the AKS node resource group" return 1 fi + if ! live_cluster_location="$(jq -r --arg name "${cluster_name}" \ + '.[] | select(.name == $name) | .location // empty' \ + <<<"${aks_inventory}")"; then + log_error "Failed to inspect the AKS cluster location" + return 1 + fi + if [[ -n "${live_cluster_location}" ]]; then + if [[ -n "${location}" && "${location,,}" != "${live_cluster_location,,}" ]]; then + log_error "Refusing cleanup because live AKS location '${live_cluster_location}' differs from state '${location}'" + return 1 + fi + location="${live_cluster_location}" + state_set "location" "${location}" || return 1 + fi if [[ -n "${live_node_resource_group}" ]]; then - if ! _validate_expected_node_resource_group "${live_node_resource_group}" "${name_suffix}"; then + if ! _validate_expected_node_resource_group \ + "${live_node_resource_group}" "${name_suffix}" "${resource_group}" "${cluster_name}" "${location}"; then return 1 fi if [[ -n "${node_resource_group}" && "${node_resource_group}" != "${live_node_resource_group}" ]]; then @@ -698,7 +807,8 @@ cleanup() { fi node_resource_group="${live_node_resource_group}" state_set "node_resource_group" "${node_resource_group}" || return 1 - elif ! _validate_expected_node_resource_group "${node_resource_group}" "${name_suffix}"; then + elif ! _validate_expected_node_resource_group \ + "${node_resource_group}" "${name_suffix}" "${resource_group}" "${cluster_name}" "${location}"; then return 1 fi if [[ -n "${node_resource_group}" && "${node_resource_group}" == "${resource_group}" ]]; then @@ -752,7 +862,7 @@ cleanup() { --subscription "${subscription_id}" --output none 2>/dev/null || true done - for disk_id in "${managed_disk_ids[@]}"; do + for disk_id in "${managed_disk_ids[@]}" "${legacy_implicit_os_disk_ids[@]}"; do az resource delete --ids "${disk_id}" --subscription "${subscription_id}" --output none 2>/dev/null || true done for nic_id in "${nic_ids[@]}"; do @@ -801,18 +911,40 @@ cleanup() { --subscription "${subscription_id}" --output none 2>/dev/null || true log_info "[8/8] Cleaning up tagged network and disk resources..." - local remaining_ids empty_inventories=0 + local remaining_ids legacy_remaining_ids empty_inventories=0 for _ in 1 2 3 4; do if ! _delete_tagged_resources "${resource_group}" "${resource_owner}" "${subscription_id}" "${legacy_tag_name_suffix}"; then cleanup_failed=1 break fi + # Old VM deployments let Azure choose names such as + # _OsDisk_1_. Those disks can be detached, untagged, and absent + # from `az vm list`, so rediscover and retry them from the full RG inventory. + for disk_id in "${legacy_implicit_os_disk_ids[@]}"; do + az resource delete --ids "${disk_id}" --subscription "${subscription_id}" --output none 2>/dev/null || true + done + if ! _delete_legacy_implicit_os_disks \ + "${resource_group}" "${subscription_id}" \ + "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" \ + "${kubeadm_vm_name}" "${arc_vm_name}"; then + log_error "Failed to delete legacy implicit OS disks" + cleanup_failed=1 + break + fi if ! remaining_ids="$(_tagged_resource_ids "${resource_group}" "${resource_owner}" "${subscription_id}" "${legacy_tag_name_suffix}")"; then log_error "Failed to verify tagged-resource deletion" cleanup_failed=1 break fi - if [[ -z "${remaining_ids}" ]]; then + if ! legacy_remaining_ids="$(_legacy_implicit_os_disk_ids \ + "${resource_group}" "${subscription_id}" \ + "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" \ + "${kubeadm_vm_name}" "${arc_vm_name}")"; then + log_error "Failed to verify legacy implicit OS-disk deletion" + cleanup_failed=1 + break + fi + if [[ -z "${remaining_ids}" && -z "${legacy_remaining_ids}" ]]; then empty_inventories=$((empty_inventories + 1)) if (( empty_inventories >= 2 )); then break @@ -823,8 +955,8 @@ cleanup() { sleep "${E2E_CLEANUP_POLL_INTERVAL:-5}" done - if [[ -n "${resource_owner}" && "${empty_inventories}" -lt 2 ]]; then - log_error "Tagged-resource inventory did not remain empty for two consecutive checks" + if (( empty_inventories < 2 )); then + log_error "Residual-resource inventory did not remain empty for two consecutive checks" cleanup_failed=1 fi @@ -865,7 +997,16 @@ cleanup() { cleanup_failed=1 known_remaining="" fi - for captured_id in "${managed_disk_ids[@]}" "${nic_ids[@]}" "${arc_machine_id}"; do + if ! legacy_remaining_ids="$(_legacy_implicit_os_disk_ids_from_inventory \ + "${final_inventory}" \ + "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" \ + "${kubeadm_vm_name}" "${arc_vm_name}")"; then + cleanup_failed=1 + legacy_remaining_ids="" + elif [[ -n "${legacy_remaining_ids}" ]]; then + known_remaining+=$'\n'"${legacy_remaining_ids}" + fi + for captured_id in "${managed_disk_ids[@]}" "${legacy_implicit_os_disk_ids[@]}" "${nic_ids[@]}" "${arc_machine_id}"; do [[ -n "${captured_id}" ]] || continue if jq -e --arg id "${captured_id}" '.[] | select(.id == $id)' <<<"${final_inventory}" >/dev/null; then known_remaining+=$'\n'"${captured_id}" @@ -883,5 +1024,5 @@ cleanup() { state_set "lifecycle" "cleaned" || return 1 state_set "cleanup_complete" "true" || return 1 - log_success "Cleanup completed and no tagged resources remain" + log_success "Cleanup completed and no known E2E resources remain" } From cfffa4cc1b6dc5cbbebb079d52cfb4f8a12658bf Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Sat, 22 Aug 2026 04:17:18 +0000 Subject: [PATCH 15/18] Fix cleanup with persisted subscription state Allow historical cleanup to use its saved subscription when the environment variable is absent, with a regression test for nounset execution. --- hack/e2e/e2e_scripts_test.go | 29 +++++++++++++++++++++++++++-- hack/e2e/lib/cleanup.sh | 2 +- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/hack/e2e/e2e_scripts_test.go b/hack/e2e/e2e_scripts_test.go index c0e0e02f..8a6b2014 100644 --- a/hack/e2e/e2e_scripts_test.go +++ b/hack/e2e/e2e_scripts_test.go @@ -564,6 +564,25 @@ func TestCleanupUsesExactNamesWithoutRunTag(t *testing.T) { } } +func TestCleanupUsesPersistedSubscriptionWithoutEnvironment(t *testing.T) { + t.Parallel() + + _, _, callLog, output, err := runCleanup(t, cleanupOptions{unsetSubscriptionEnv: true}) + if err != nil { + t.Fatalf("cleanup without subscription environment failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=success") { + t.Fatalf("cleanup did not use the persisted subscription:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + if !strings.Contains(string(calls), "--subscription test-subscription") { + t.Fatalf("cleanup did not pass the persisted subscription to Azure CLI:\n%s", calls) + } +} + func TestCleanupHandlesLegacyStateWithoutVMNames(t *testing.T) { t.Parallel() @@ -871,6 +890,7 @@ type cleanupOptions struct { legacyDiskDeletePersists bool unexpectedArcMachineID bool transientQueryFailures bool + unsetSubscriptionEnv bool } func runCleanup(t *testing.T, options cleanupOptions) (string, string, string, []byte, error) { @@ -945,11 +965,16 @@ UNEXPECTED_LIVE_NODE_RG="${12}" LEGACY_DEFAULT_NODE_RG="${13}" LEGACY_DETACHED_DISK="${14}" LEGACY_DISK_DELETE_PERSISTS="${15}" +UNSET_SUBSCRIPTION_ENV="${16}" LEGACY_DISK_DELETED="${E2E_WORK_DIR}/legacy-disk-deleted" # Keep cleanup fixtures independent from the ambient GitHub Actions run. Tests # that need tag-based cleanup persist an explicit run_id in their state. unset GITHUB_RUN_ID -AZURE_SUBSCRIPTION_ID=test-subscription +if [[ "${UNSET_SUBSCRIPTION_ENV}" == "1" ]]; then + unset AZURE_SUBSCRIPTION_ID +else + AZURE_SUBSCRIPTION_ID=test-subscription +fi E2E_SKIP_CLEANUP=0 E2E_CLEANUP_TIMEOUT=5 E2E_CLEANUP_POLL_INTERVAL=0.01 @@ -1038,7 +1063,7 @@ fi boolString(options.deploymentQueryFails), boolString(options.runTwice), boolString(options.transientQueryFailures), boolString(options.unexpectedLiveNodeRG), boolString(options.legacyDefaultNodeResource), boolString(options.legacyDetachedDisk), - boolString(options.legacyDiskDeletePersists)) + boolString(options.legacyDiskDeletePersists), boolString(options.unsetSubscriptionEnv)) return workDir, statePath, callLog, output, err } diff --git a/hack/e2e/lib/cleanup.sh b/hack/e2e/lib/cleanup.sh index 9041e649..b7585b06 100755 --- a/hack/e2e/lib/cleanup.sh +++ b/hack/e2e/lib/cleanup.sh @@ -611,7 +611,7 @@ cleanup() { kubeadm_vm_name="$(state_get kubeadm_vm_name)" arc_vm_name="$(state_get arc_vm_name)" arc_machine_name="$(state_get arc_machine_name)" - subscription_id="$(state_get subscription_id "${AZURE_SUBSCRIPTION_ID}")" + subscription_id="$(state_get subscription_id "${AZURE_SUBSCRIPTION_ID:-}")" arc_machine_id="$(state_get arc_machine_id)" deployment_name="$(state_get deployment_name)" persisted_resource_owner="$(state_get resource_owner)" From d8f2a79387247b5efecc32bd7b0805e6f4879f39 Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:05:30 +0000 Subject: [PATCH 16/18] Harden E2E cleanup and diagnostics --- .github/workflows/e2e-tests.yml | 45 ++++- hack/e2e/cleanup_safety_test.go | 8 +- hack/e2e/e2e_scripts_test.go | 230 ++++++++++++++++++++++- hack/e2e/lib/bootstrap-rbac-migration.sh | 5 +- hack/e2e/lib/cleanup.sh | 148 +++++++++++---- hack/e2e/lib/runner.sh | 24 +-- pkg/config/config_test.go | 3 + pkg/daemon/lifecycle.go | 5 + pkg/daemon/lifecycle_test.go | 3 + scripts/aks-flex-config | 8 +- scripts/aks_flex_config_test.go | 77 ++++++++ 11 files changed, 491 insertions(+), 65 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index dedc6926..00b5254d 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -126,11 +126,54 @@ jobs: retention-days: 7 - name: Cleanup + id: cleanup if: always() env: E2E_SKIP_CLEANUP: ${{ inputs.skip_cleanup && '1' || '0' }} run: ./hack/e2e/run.sh cleanup - - name: Cleanup runner workspace + - name: Prepare cleanup diagnostics + if: always() + run: | + set -euo pipefail + umask 077 + state_file="${E2E_WORK_DIR}/state.json" + artifact_file="${E2E_WORK_DIR}/cleanup-state.json" + [[ -f "${state_file}" ]] || exit 0 + # Keep the resource names needed for investigation without publishing + # secret-sourced account identifiers, addresses, or cluster access data. + jq '{ + lifecycle, + cleanup_complete, + deployment_name, + run_id, + resource_owner, + name_suffix, + cluster_name, + node_resource_group, + msi_vm_name, + token_vm_name, + offline_vm_name, + kubeadm_vm_name, + arc_vm_name, + arc_machine_name, + vnet_name, + nsg_name + } | with_entries(select(.value != null and .value != ""))' \ + "${state_file}" > "${artifact_file}" + chmod 0600 "${artifact_file}" + + - name: Upload cleanup diagnostics if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: e2e-cleanup-state-${{ github.run_id }}-${{ github.run_attempt }} + path: /tmp/aks-flex-node-e2e-${{ github.run_id }}-${{ github.run_attempt }}/cleanup-state.json + if-no-files-found: ignore + retention-days: 7 + + - name: Cleanup runner workspace + # Preserve retry metadata when Azure cleanup fails or is deliberately + # skipped. A later successful run must not delete another attempt's state. + if: always() && steps.cleanup.outcome == 'success' && inputs.skip_cleanup != true run: ./hack/e2e/run.sh runner-cleanup diff --git a/hack/e2e/cleanup_safety_test.go b/hack/e2e/cleanup_safety_test.go index b27fee5f..26d454fd 100644 --- a/hack/e2e/cleanup_safety_test.go +++ b/hack/e2e/cleanup_safety_test.go @@ -249,9 +249,9 @@ source "$3" _resource_inventory() { cat "${INVENTORY_FILE}"; } az() { printf 'AZ=%s\n' "$*"; } printf '%s\n' 'IDS-BEGIN' -_tagged_resource_ids test-rg 12345 test-subscription 12345-1 +_tagged_resource_ids test-rg 12345 test-subscription 12345-1 0 printf '%s\n' 'IDS-END' -_delete_tagged_resources test-rg 12345 test-subscription 12345-1 +_delete_tagged_resources test-rg 12345 test-subscription 12345-1 0 ` output, err := runBash(t, script, workDir, inventoryPath, cleanupScript) if err != nil { @@ -296,10 +296,10 @@ source "$3" _resource_inventory() { cat "${INVENTORY_FILE}"; } az() { printf 'AZ=%s\n' "$*"; } printf '%s\n' 'IDS-BEGIN' -_legacy_implicit_os_disk_ids test-rg test-subscription \ +_legacy_implicit_os_disk_ids test-rg test-subscription 0 \ vm-e2e-msi-12345-1 vm-e2e-token-12345-1 printf '%s\n' 'IDS-END' -_delete_legacy_implicit_os_disks test-rg test-subscription \ +_delete_legacy_implicit_os_disks test-rg test-subscription 0 \ vm-e2e-msi-12345-1 vm-e2e-token-12345-1 ` output, err := runBash(t, script, workDir, inventoryPath, cleanupScript) diff --git a/hack/e2e/e2e_scripts_test.go b/hack/e2e/e2e_scripts_test.go index 8a6b2014..5017a276 100644 --- a/hack/e2e/e2e_scripts_test.go +++ b/hack/e2e/e2e_scripts_test.go @@ -13,9 +13,79 @@ import ( // Embedding the scripts makes Go's test cache invalidate on shell-only changes. // -//go:embed run.sh lib/common.sh lib/cleanup.sh lib/bootstrap-rbac-migration.sh infra/*.bicep infra/modules/*.bicep +//go:embed run.sh lib/common.sh lib/cleanup.sh lib/bootstrap-rbac-migration.sh lib/runner.sh infra/*.bicep infra/modules/*.bicep var e2eScripts embed.FS +func TestRunnerCleanupIsScopedToCurrentAttempt(t *testing.T) { + t.Parallel() + + script, err := e2eScripts.ReadFile("lib/runner.sh") + if err != nil { + t.Fatalf("read embedded runner script: %v", err) + } + text := string(script) + for _, required := range []string{ + `local work_dir="${E2E_WORK_DIR:-}"`, + `^/tmp/aks-flex-node-e2e(-[A-Za-z0-9][A-Za-z0-9._-]*)?$`, + `rm -rf -- "${work_dir}"`, + `Refusing to clean unexpected E2E work directory`, + } { + if !strings.Contains(text, required) { + t.Errorf("runner cleanup is missing attempt-scoped guard %q", required) + } + } + if strings.Contains(text, `find /tmp`) || strings.Contains(text, `-name 'aks-flex-node-e2e-*'`) { + t.Error("runner cleanup still removes work directories owned by other attempts") + } +} + +func TestRunnerCleanupRejectsUnsafeWorkDirectories(t *testing.T) { + t.Parallel() + + runnerScript := e2eScriptPath(t, "lib", "runner.sh") + tests := map[string]func(string) string{ + "unexpected parent": func(targetDir string) string { return targetDir }, + "parent traversal": func(targetDir string) string { return "/tmp/.." + targetDir }, + } + for name, workDirPath := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + targetDir := filepath.Join(t.TempDir(), "aks-flex-node-e2e-test") + if err := os.MkdirAll(targetDir, 0o700); err != nil { + t.Fatalf("create protected directory: %v", err) + } + sentinel := filepath.Join(targetDir, "sentinel") + if err := os.WriteFile(sentinel, []byte("keep"), 0o600); err != nil { + t.Fatalf("write protected sentinel: %v", err) + } + workDir := workDirPath(targetDir) + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +sudo() { return 1; } +go() { return 0; } +if cleanup_runner_workspace; then + printf 'RESULT=unexpected-success\n' +else + printf 'RESULT=rejected\n' +fi +` + output, err := runBash(t, script, workDir, runnerScript) + if err != nil { + t.Fatalf("runner cleanup safety harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=rejected") { + t.Fatalf("runner cleanup accepted unsafe work directory %q:\n%s", workDir, output) + } + if _, err := os.Stat(sentinel); err != nil { + t.Fatalf("runner cleanup removed data outside its allowed path: %v", err) + } + }) + } +} + func TestBicepModuleDeploymentNamesAreUniquePerRun(t *testing.T) { t.Parallel() @@ -797,14 +867,121 @@ func TestCleanupRetriesTransientAzureQueries(t *testing.T) { if readErr != nil { t.Fatalf("read az calls: %v", readErr) } - if strings.Count(string(calls), "group exists") < 3 { + if strings.Count(string(calls), "group exists") < 8 { t.Errorf("cleanup did not retry a failed resource-group query:\n%s", calls) } - if strings.Count(string(calls), "resource list") < 4 { + if strings.Count(string(calls), "resource list") < 8 { t.Errorf("cleanup did not retry a failed resource inventory:\n%s", calls) } } +func TestCleanupAzureQueryFailureIncludesDiagnostic(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +E2E_CLEANUP_POLL_INTERVAL=0.01 +az() { + printf '%s\n' '(TooManyRequests) ARM throttled the cleanup query' >&2 + return 1 +} +if _resource_inventory test-rg test-subscription "" 0; then + printf 'RESULT=unexpected-success\n' +else + printf 'RESULT=error\n' +fi +` + output, err := runBash(t, script, workDir, cleanupScript) + if err != nil { + t.Fatalf("Azure query diagnostic harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") || + !strings.Contains(string(output), "Last Azure CLI error: (TooManyRequests) ARM throttled the cleanup query") { + t.Fatalf("cleanup omitted the final Azure CLI diagnostic:\n%s", output) + } +} + +func TestCleanupTreatsResourceGroupNotFoundAsAbsent(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + callLog := filepath.Join(workDir, "az-calls") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +AZ_CALL_LOG="$3" +az() { + printf '%s\n' "$*" >> "${AZ_CALL_LOG}" + printf '%s\n' '(ResourceGroupNotFound) Resource group was not found.' >&2 + return 1 +} +result="$(_group_exists_with_retry test-rg test-subscription 0)" +printf 'RESULT=%s\n' "${result}" +printf 'CALLS=%s\n' "$(wc -l < "${AZ_CALL_LOG}")" +` + output, err := runBash(t, script, workDir, cleanupScript, callLog) + if err != nil { + t.Fatalf("ResourceGroupNotFound harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=false") || + !strings.Contains(string(output), "CALLS=1") { + t.Fatalf("ResourceGroupNotFound was not treated as successful absence:\n%s", output) + } +} + +func TestCleanupAzureQueriesHonorExpiredDeadline(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + callLog := filepath.Join(workDir, "az-calls") + sleepLog := filepath.Join(workDir, "sleep-delay") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +AZ_CALL_LOG="$3" +SLEEP_LOG="$4" +az() { printf '%s\n' "$*" >> "${AZ_CALL_LOG}"; return 1; } +sleep() { printf '%s\n' "$1" > "${SLEEP_LOG}"; } + +SECONDS=10 +if _group_exists_with_retry test-rg test-subscription 5; then + printf 'GROUP=unexpected-success\n' +else + printf 'GROUP=expired\n' +fi +if _resource_inventory test-rg test-subscription "" 5; then + printf 'INVENTORY=unexpected-success\n' +else + printf 'INVENTORY=expired\n' +fi +calls=0 +[[ ! -f "${AZ_CALL_LOG}" ]] || calls="$(wc -l < "${AZ_CALL_LOG}")" +printf 'CALLS=%s\n' "${calls}" + +E2E_CLEANUP_POLL_INTERVAL=30 +SECONDS=10 +_sleep_before_azure_query_retry 11 +printf 'SLEEP=%s\n' "$(<"${SLEEP_LOG}")" +` + output, err := runBash(t, script, workDir, cleanupScript, callLog, sleepLog) + if err != nil { + t.Fatalf("Azure query deadline harness failed: %v\n%s", err, output) + } + for _, expected := range []string{"GROUP=expired", "INVENTORY=expired", "CALLS=0", "SLEEP=1"} { + if !strings.Contains(string(output), expected) { + t.Fatalf("Azure query deadline was not enforced; missing %q:\n%s", expected, output) + } + } +} + func TestCleanupQueryFailureDoesNotDeleteResources(t *testing.T) { t.Parallel() @@ -876,6 +1053,34 @@ func TestHistoricalMigrationReissuesDaemonCertificateBeforeTokenRevocation(t *te } } +func TestHistoricalTokenRevocationPropagatesDeleteFailure(t *testing.T) { + t.Parallel() + + script, err := e2eScripts.ReadFile("lib/bootstrap-rbac-migration.sh") + if err != nil { + t.Fatalf("read embedded migration script: %v", err) + } + text := string(script) + start := strings.Index(text, "_revoke_historical_bootstrap_token() {") + if start < 0 { + t.Fatal("historical token revocation helper is absent") + } + end := strings.Index(text[start:], "\n}\n") + if end < 0 { + t.Fatal("historical token revocation helper is malformed") + } + body := text[start : start+end] + for _, required := range []string{ + `if ! kubectl delete secret "bootstrap-token-${token_id}" -n kube-system; then`, + `log_error "Failed to revoke the historical bootstrap token"`, + `return 1`, + } { + if !strings.Contains(body, required) { + t.Fatalf("historical token revocation does not fail closed on delete errors; missing %q", required) + } + } +} + type cleanupOptions struct { runTwice bool leaveCluster bool @@ -986,12 +1191,14 @@ az() { elif [[ "$1 $2 $3 $4" == "deployment operation group list" ]]; then printf '[{"properties":{"provisioningState":"Succeeded"}}]\n' elif [[ "$1 $2" == "group exists" ]]; then - if [[ "${TRANSIENT_QUERY_FAILURES}" == "1" && ! -f "${GROUP_QUERY_FAILED}" ]]; then - : > "${GROUP_QUERY_FAILED}" - return 1 - fi if [[ "$*" == *"--name MC_aksflex-e2e-test"* || \ "$*" == *"--name MC_test-rg_aks-e2e-test_test-location"* ]]; then + query_failures="$(cat "${GROUP_QUERY_FAILED}" 2>/dev/null || printf '0\n')" + if [[ "${TRANSIENT_QUERY_FAILURES}" == "1" && "${query_failures}" -lt 8 ]]; then + printf '%s\n' "$((query_failures + 1))" > "${GROUP_QUERY_FAILED}" + printf '%s\n' '(TooManyRequests) transient resource-group query failure' >&2 + return 1 + fi [[ -f "${NODE_GROUP_DELETED}" ]] && printf 'false\n' || printf 'true\n' elif [[ "$*" == *"--name test-rg"* ]]; then [[ "${PARENT_GROUP_ABSENT}" == "1" ]] && printf 'false\n' || printf 'true\n' @@ -1019,8 +1226,11 @@ az() { fi return 0 elif [[ "$1 $2" == "resource list" ]]; then - if [[ "${TRANSIENT_QUERY_FAILURES}" == "1" && ! -f "${RESOURCE_QUERY_FAILED}" ]]; then - : > "${RESOURCE_QUERY_FAILED}" + query_failures="$(cat "${RESOURCE_QUERY_FAILED}" 2>/dev/null || printf '0\n')" + if [[ "${TRANSIENT_QUERY_FAILURES}" == "1" && -f "${NODE_GROUP_DELETED}" && \ + "${query_failures}" -lt 8 ]]; then + printf '%s\n' "$((query_failures + 1))" > "${RESOURCE_QUERY_FAILED}" + printf '%s\n' '(TooManyRequests) transient resource inventory failure' >&2 return 1 fi if [[ "$*" == *"--tag "* ]]; then @@ -1077,7 +1287,7 @@ func boolString(value bool) string { func e2eScriptPath(t *testing.T, elements ...string) string { t.Helper() root := t.TempDir() - for _, name := range []string{"common.sh", "cleanup.sh", "bootstrap-rbac-migration.sh"} { + for _, name := range []string{"common.sh", "cleanup.sh", "bootstrap-rbac-migration.sh", "runner.sh"} { contents, err := e2eScripts.ReadFile(filepath.ToSlash(filepath.Join("lib", name))) if err != nil { t.Fatalf("read embedded %s: %v", name, err) diff --git a/hack/e2e/lib/bootstrap-rbac-migration.sh b/hack/e2e/lib/bootstrap-rbac-migration.sh index 386de6f3..6aa4acf1 100644 --- a/hack/e2e/lib/bootstrap-rbac-migration.sh +++ b/hack/e2e/lib/bootstrap-rbac-migration.sh @@ -723,7 +723,10 @@ _revoke_historical_bootstrap_token() { local config_file="$1" local token_id token_id="$(_historical_token_id "${config_file}")" - kubectl delete secret "bootstrap-token-${token_id}" -n kube-system + if ! kubectl delete secret "bootstrap-token-${token_id}" -n kube-system; then + log_error "Failed to revoke the historical bootstrap token" + return 1 + fi log_info "Revoked the historical bootstrap token after certificate migration" } diff --git a/hack/e2e/lib/cleanup.sh b/hack/e2e/lib/cleanup.sh index b7585b06..309f4a52 100755 --- a/hack/e2e/lib/cleanup.sh +++ b/hack/e2e/lib/cleanup.sh @@ -275,53 +275,131 @@ _remaining_cleanup_timeout() { printf '%s\n' "${remaining}" } +_azure_query_can_attempt() { + local attempt="$1" deadline="${2:-0}" + + if (( deadline > 0 )); then + (( SECONDS < deadline )) + else + (( attempt < 5 )) + fi +} + +_sleep_before_azure_query_retry() { + local deadline="${1:-0}" + local delay="${E2E_CLEANUP_POLL_INTERVAL:-5}" remaining whole_seconds + + if [[ ! "${delay}" =~ ^[0-9]+([.][0-9]+)?$ ]]; then + log_error "Invalid E2E_CLEANUP_POLL_INTERVAL '${delay}'" + return 1 + fi + if (( deadline > 0 )); then + remaining=$((deadline - SECONDS)) + (( remaining > 0 )) || return 1 + whole_seconds="${delay%%.*}" + if (( whole_seconds >= remaining )); then + delay="${remaining}" + fi + fi + sleep "${delay}" +} + +_azure_error_summary() { + local message="$1" + message="${message//$'\r'/ }" + message="${message//$'\n'/ }" + printf '%.500s\n' "${message}" +} + _group_exists_with_retry() { - local group_name="$1" subscription_id="$2" - local exists attempt + local group_name="$1" subscription_id="$2" deadline="${3:-0}" + local exists attempt=0 last_error="" error_file + + if ! error_file="$(mktemp "${E2E_WORK_DIR}/azure-group-query.XXXXXX")"; then + log_error "Failed to create temporary Azure query diagnostics" + return 1 + fi - for attempt in 1 2 3 4 5; do + while _azure_query_can_attempt "${attempt}" "${deadline}"; do + attempt=$((attempt + 1)) + : > "${error_file}" if exists="$(az group exists \ --name "${group_name}" \ --subscription "${subscription_id}" \ - --output tsv 2>/dev/null)"; then + --output tsv 2>"${error_file}")"; then case "${exists}" in true|false) + rm -f -- "${error_file}" printf '%s\n' "${exists}" return 0 ;; + *) + last_error="Azure CLI returned an unexpected resource-group existence result: ${exists}" + ;; esac + else + last_error="$(<"${error_file}")" + # ARM can report the resource as not found while `az group exists` still + # exits nonzero during asynchronous AKS-managed resource-group deletion. + if [[ "${last_error}" == *ResourceGroupNotFound* ]]; then + rm -f -- "${error_file}" + printf 'false\n' + return 0 + fi fi - if (( attempt < 5 )); then - sleep "${E2E_CLEANUP_POLL_INTERVAL:-5}" + if ! _azure_query_can_attempt "${attempt}" "${deadline}" || \ + ! _sleep_before_azure_query_retry "${deadline}"; then + break fi done + rm -f -- "${error_file}" log_error "Failed to determine whether resource group '${group_name}' exists after ${attempt} attempts" >&2 + if [[ -n "${last_error}" ]]; then + log_error "Last Azure CLI error: $(_azure_error_summary "${last_error}")" >&2 + fi return 1 } _resource_inventory() { - local resource_group="$1" subscription_id="$2" run_id="${3:-}" - local resource_json attempt + local resource_group="$1" subscription_id="$2" run_id="${3:-}" deadline="${4:-0}" + local resource_json attempt=0 last_error="" error_file local -a tag_args=() [[ -z "${run_id}" ]] || tag_args=(--tag "github-run=${run_id}") - for attempt in 1 2 3 4 5; do + if ! error_file="$(mktemp "${E2E_WORK_DIR}/azure-resource-query.XXXXXX")"; then + log_error "Failed to create temporary Azure query diagnostics" + return 1 + fi + + while _azure_query_can_attempt "${attempt}" "${deadline}"; do + attempt=$((attempt + 1)) + : > "${error_file}" if resource_json="$(az resource list \ --resource-group "${resource_group}" \ --subscription "${subscription_id}" \ "${tag_args[@]}" \ - --output json 2>/dev/null)" && \ - jq -e 'type == "array"' <<<"${resource_json}" >/dev/null; then - printf '%s\n' "${resource_json}" - return 0 + --output json 2>"${error_file}")"; then + if jq -e 'type == "array"' <<<"${resource_json}" >/dev/null; then + rm -f -- "${error_file}" + printf '%s\n' "${resource_json}" + return 0 + fi + last_error="Azure CLI returned a malformed resource inventory" + else + last_error="$(<"${error_file}")" fi - if (( attempt < 5 )); then - sleep "${E2E_CLEANUP_POLL_INTERVAL:-5}" + if ! _azure_query_can_attempt "${attempt}" "${deadline}" || \ + ! _sleep_before_azure_query_retry "${deadline}"; then + break fi done + rm -f -- "${error_file}" log_error "Failed to inventory resources in '${resource_group}' after ${attempt} attempts" >&2 + if [[ -n "${last_error}" ]]; then + log_error "Last Azure CLI error: $(_azure_error_summary "${last_error}")" >&2 + fi return 1 } @@ -410,21 +488,21 @@ _legacy_implicit_os_disk_ids_from_inventory() { } _legacy_implicit_os_disk_ids() { - local resource_group="$1" subscription_id="$2" resource_json - shift 2 + local resource_group="$1" subscription_id="$2" deadline="$3" resource_json + shift 3 (( $# > 0 )) || return 0 - if ! resource_json="$(_resource_inventory "${resource_group}" "${subscription_id}")"; then + if ! resource_json="$(_resource_inventory "${resource_group}" "${subscription_id}" "" "${deadline}")"; then return 1 fi _legacy_implicit_os_disk_ids_from_inventory "${resource_json}" "$@" } _delete_legacy_implicit_os_disks() { - local resource_group="$1" subscription_id="$2" disk_ids disk_id - shift 2 + local resource_group="$1" subscription_id="$2" deadline="$3" disk_ids disk_id + shift 3 - if ! disk_ids="$(_legacy_implicit_os_disk_ids "${resource_group}" "${subscription_id}" "$@")"; then + if ! disk_ids="$(_legacy_implicit_os_disk_ids "${resource_group}" "${subscription_id}" "${deadline}" "$@")"; then return 1 fi while IFS= read -r disk_id; do @@ -512,7 +590,7 @@ _delete_node_resource_group() { local exists wait_timeout [[ -n "${node_resource_group}" ]] || return 0 - if ! exists="$(_group_exists_with_retry "${node_resource_group}" "${subscription_id}")"; then + if ! exists="$(_group_exists_with_retry "${node_resource_group}" "${subscription_id}" "${deadline}")"; then log_error "Failed to determine whether AKS node resource group '${node_resource_group}' exists" return 1 fi @@ -542,7 +620,7 @@ _delete_node_resource_group() { } _delete_tagged_resources() { - local resource_group="$1" run_id="$2" subscription_id="$3" legacy_name_suffix="${4:-}" + local resource_group="$1" run_id="$2" subscription_id="$3" legacy_name_suffix="${4:-}" deadline="${5:-0}" local resource_json resource_type resource_name id local -a resource_types=( "Microsoft.Compute/disks" @@ -553,7 +631,7 @@ _delete_tagged_resources() { ) [[ -n "${run_id}" ]] || return 0 - if ! resource_json="$(_resource_inventory "${resource_group}" "${subscription_id}" "${run_id}")"; then + if ! resource_json="$(_resource_inventory "${resource_group}" "${subscription_id}" "${run_id}" "${deadline}")"; then log_error "Failed to list E2E resources tagged github-run=${run_id}" return 1 fi @@ -575,11 +653,11 @@ _delete_tagged_resources() { } _tagged_resource_ids() { - local resource_group="$1" run_id="$2" subscription_id="$3" legacy_name_suffix="${4:-}" + local resource_group="$1" run_id="$2" subscription_id="$3" legacy_name_suffix="${4:-}" deadline="${5:-0}" local resource_json resource_type resource_name id [[ -n "${run_id}" ]] || return 0 - resource_json="$(_resource_inventory "${resource_group}" "${subscription_id}" "${run_id}")" || return 1 + resource_json="$(_resource_inventory "${resource_group}" "${subscription_id}" "${run_id}" "${deadline}")" || return 1 while IFS=$'\t' read -r resource_type resource_name id; do [[ -n "${id}" ]] || continue if [[ -n "${legacy_name_suffix}" ]] && \ @@ -660,7 +738,8 @@ cleanup() { fi local resource_group_exists - if ! resource_group_exists="$(_group_exists_with_retry "${resource_group}" "${subscription_id}")"; then + if ! resource_group_exists="$(_group_exists_with_retry \ + "${resource_group}" "${subscription_id}" "${cleanup_deadline}")"; then log_error "Failed to determine whether resource group '${resource_group}' exists" return 1 fi @@ -745,7 +824,8 @@ cleanup() { log_error "Azure returned an invalid VM or AKS cleanup inventory" return 1 fi - if ! resource_inventory="$(_resource_inventory "${resource_group}" "${subscription_id}")"; then + if ! resource_inventory="$(_resource_inventory \ + "${resource_group}" "${subscription_id}" "" "${cleanup_deadline}")"; then log_error "Failed to inventory E2E resources before cleanup" return 1 fi @@ -913,7 +993,7 @@ cleanup() { log_info "[8/8] Cleaning up tagged network and disk resources..." local remaining_ids legacy_remaining_ids empty_inventories=0 for _ in 1 2 3 4; do - if ! _delete_tagged_resources "${resource_group}" "${resource_owner}" "${subscription_id}" "${legacy_tag_name_suffix}"; then + if ! _delete_tagged_resources "${resource_group}" "${resource_owner}" "${subscription_id}" "${legacy_tag_name_suffix}" "${cleanup_deadline}"; then cleanup_failed=1 break fi @@ -924,20 +1004,20 @@ cleanup() { az resource delete --ids "${disk_id}" --subscription "${subscription_id}" --output none 2>/dev/null || true done if ! _delete_legacy_implicit_os_disks \ - "${resource_group}" "${subscription_id}" \ + "${resource_group}" "${subscription_id}" "${cleanup_deadline}" \ "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" \ "${kubeadm_vm_name}" "${arc_vm_name}"; then log_error "Failed to delete legacy implicit OS disks" cleanup_failed=1 break fi - if ! remaining_ids="$(_tagged_resource_ids "${resource_group}" "${resource_owner}" "${subscription_id}" "${legacy_tag_name_suffix}")"; then + if ! remaining_ids="$(_tagged_resource_ids "${resource_group}" "${resource_owner}" "${subscription_id}" "${legacy_tag_name_suffix}" "${cleanup_deadline}")"; then log_error "Failed to verify tagged-resource deletion" cleanup_failed=1 break fi if ! legacy_remaining_ids="$(_legacy_implicit_os_disk_ids \ - "${resource_group}" "${subscription_id}" \ + "${resource_group}" "${subscription_id}" "${cleanup_deadline}" \ "${msi_vm_name}" "${token_vm_name}" "${offline_vm_name}" \ "${kubeadm_vm_name}" "${arc_vm_name}")"; then log_error "Failed to verify legacy implicit OS-disk deletion" @@ -960,7 +1040,7 @@ cleanup() { cleanup_failed=1 fi - if ! remaining_ids="$(_tagged_resource_ids "${resource_group}" "${resource_owner}" "${subscription_id}" "${legacy_tag_name_suffix}")"; then + if ! remaining_ids="$(_tagged_resource_ids "${resource_group}" "${resource_owner}" "${subscription_id}" "${legacy_tag_name_suffix}" "${cleanup_deadline}")"; then log_error "Failed final tagged-resource verification" cleanup_failed=1 elif [[ -n "${remaining_ids}" ]]; then @@ -971,7 +1051,7 @@ cleanup() { local final_inventory expected_names known_remaining captured_id resource_name local -a expected_name_args=() - if ! final_inventory="$(_resource_inventory "${resource_group}" "${subscription_id}")"; then + if ! final_inventory="$(_resource_inventory "${resource_group}" "${subscription_id}" "" "${cleanup_deadline}")"; then log_error "Failed final exact-name resource verification" cleanup_failed=1 final_inventory='[]' diff --git a/hack/e2e/lib/runner.sh b/hack/e2e/lib/runner.sh index 7bc1f111..fc407848 100644 --- a/hack/e2e/lib/runner.sh +++ b/hack/e2e/lib/runner.sh @@ -32,22 +32,22 @@ cleanup_runner_workspace() { df -h / 2>/dev/null || true local removed=0 - local dir - while IFS= read -r dir; do - [[ -n "${dir}" && -d "${dir}" ]] || continue - log_info "Removing ${dir}" + local work_dir="${E2E_WORK_DIR:-}" + if [[ ! "${work_dir}" =~ ^/tmp/aks-flex-node-e2e(-[A-Za-z0-9][A-Za-z0-9._-]*)?$ ]]; then + log_error "Refusing to clean unexpected E2E work directory '${work_dir}'" + return 1 + fi + if [[ -d "${work_dir}" ]]; then + log_info "Removing ${work_dir}" if [[ "${can_sudo}" == "1" ]]; then - sudo rm -rf -- "${dir}" || { log_warn "Failed to remove ${dir}"; continue; } + sudo rm -rf -- "${work_dir}" || { log_warn "Failed to remove ${work_dir}"; return 1; } else - rm -rf -- "${dir}" || { log_warn "Failed to remove ${dir}"; continue; } + rm -rf -- "${work_dir}" || { log_warn "Failed to remove ${work_dir}"; return 1; } fi - removed=$((removed + 1)) - done < <(find /tmp -maxdepth 1 -type d \( \ - -name 'aks-flex-node-e2e' -o \ - -name 'aks-flex-node-e2e-*' \ - \) 2>/dev/null || true) + removed=1 + fi - log_info "Removed ${removed} temporary directories" + log_info "Removed ${removed} temporary directory" if command -v go >/dev/null 2>&1; then log_info "Cleaning Go build cache" diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index b36e6bfe..9748ac96 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -2267,6 +2267,9 @@ func TestServicePrincipalClientSecretFile(t *testing.T) { if err := os.WriteFile(insecureFile, []byte("file-secret"), 0o644); err != nil { t.Fatalf("os.WriteFile: %v", err) } + if err := os.Chmod(insecureFile, 0o644); err != nil { + t.Fatalf("os.Chmod: %v", err) + } tests := []struct { name string diff --git a/pkg/daemon/lifecycle.go b/pkg/daemon/lifecycle.go index 6862fd91..da616742 100644 --- a/pkg/daemon/lifecycle.go +++ b/pkg/daemon/lifecycle.go @@ -173,6 +173,11 @@ func writeAgentServiceAssets(binaryPaths agentUpgradePaths, serviceOptions agent if err := utilio.WriteFile(asset.path, asset.content, asset.mode); err != nil { return fmt.Errorf("write %s: %w", asset.path, err) } + // Atomic replacement preserves an existing file's mode and applies the + // process umask to new files, so reconcile the declared service-asset mode. + if err := os.Chmod(asset.path, asset.mode); err != nil { + return fmt.Errorf("set permissions on %s: %w", asset.path, err) + } } return nil } diff --git a/pkg/daemon/lifecycle_test.go b/pkg/daemon/lifecycle_test.go index b4a404fc..a8691f89 100644 --- a/pkg/daemon/lifecycle_test.go +++ b/pkg/daemon/lifecycle_test.go @@ -29,6 +29,9 @@ func TestEnsureAgentUpgradeServiceAssetsMigratesExistingInstallation(t *testing. t.Fatalf("write legacy unit: %v", err) } recoveryPath := filepath.Join(t.TempDir(), "aks-flex-node-recovery.sh") + if err := os.WriteFile(recoveryPath, []byte("stale"), 0o700); err != nil { + t.Fatalf("write legacy recovery script: %v", err) + } reloaded := false if err := ensureAgentUpgradeServiceAssetsAt( t.Context(), diff --git a/scripts/aks-flex-config b/scripts/aks-flex-config index 0c011834..85dbdb4f 100755 --- a/scripts/aks-flex-config +++ b/scripts/aks-flex-config @@ -120,12 +120,14 @@ def bootstrap_group_subject() -> dict[str, str]: } -def binding_has_bootstrap_group(binding: dict[str, object]) -> bool: +def binding_has_bootstrap_group(name: str, binding: dict[str, object]) -> bool: subjects = binding.get("subjects", []) if subjects is None: subjects = [] if not isinstance(subjects, list): - raise SystemExit("ERROR: managed ClusterRoleBinding subjects is not a list") + raise SystemExit( + f"ERROR: managed ClusterRoleBinding {name!r} subjects is not a list" + ) return any( isinstance(subject, dict) and subject.get("apiGroup") == RBAC_API_GROUP @@ -166,7 +168,7 @@ def validate_managed_binding( "this object could discard operator-managed subjects or metadata. Review it manually." ) - has_subject = binding_has_bootstrap_group(binding) + has_subject = binding_has_bootstrap_group(name, binding) if require_subject and not has_subject: raise SystemExit( f"ERROR: managed ClusterRoleBinding {name!r} does not contain the required " diff --git a/scripts/aks_flex_config_test.go b/scripts/aks_flex_config_test.go index 0ba3a334..c0226ee2 100644 --- a/scripts/aks_flex_config_test.go +++ b/scripts/aks_flex_config_test.go @@ -198,6 +198,83 @@ func TestSetupNodeRBACRejectsManagedRoleRefDriftBeforeMutation(t *testing.T) { } } +func TestSetupNodeRBACIdentifiesManagedBindingWithMalformedSubjects(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + bindingName string + }{ + {name: "bootstrapper binding", bindingName: bootstrapBindingName}, + {name: "approval binding", bindingName: approvalBindingName}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + harness := newConfigScriptHarness(t, false, 0) + bindings := []map[string]any{ + { + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": map[string]any{"name": bootstrapBindingName, "resourceVersion": "7"}, + "roleRef": map[string]any{ + "apiGroup": rbacv1.GroupName, + "kind": "ClusterRole", + "name": bootstrapRole, + }, + "subjects": []any{map[string]any{ + "apiGroup": rbacv1.GroupName, + "kind": "Group", + "name": bootstrapGroup, + }}, + }, + { + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": map[string]any{"name": approvalBindingName, "resourceVersion": "7"}, + "roleRef": map[string]any{ + "apiGroup": rbacv1.GroupName, + "kind": "ClusterRole", + "name": approvalRole, + }, + "subjects": []any{map[string]any{ + "apiGroup": rbacv1.GroupName, + "kind": "Group", + "name": bootstrapGroup, + }}, + }, + } + malformedIndex := 0 + if test.bindingName == approvalBindingName { + malformedIndex = 1 + } + bindings[malformedIndex]["subjects"] = "not-a-list" + data, err := json.Marshal(bindings) + if err != nil { + t.Fatalf("marshal malformed managed bindings: %v", err) + } + if err := os.WriteFile(harness.managedState, append(data, '\n'), 0o600); err != nil { + t.Fatalf("write malformed managed bindings: %v", err) + } + + output, err := harness.runSetupNodeRBAC(false) + if err == nil { + t.Fatalf("setup-node-rbac accepted malformed subjects\n%s", output) + } + if !strings.Contains(output, test.bindingName) || + !strings.Contains(output, "subjects is not a list") { + t.Fatalf("failure did not identify the malformed binding:\n%s", output) + } + + mutations, deletes := kubectlOperationIndexes(readCommandCalls(t, harness.commandLogPath)) + if len(mutations) != 0 || len(deletes) != 0 { + t.Fatalf("malformed-subject preflight performed a mutation") + } + }) + } +} + func TestSetupNodeRBACPreservesManagedCustomizations(t *testing.T) { t.Parallel() From b3f6b23e81265646def52596be5466e21e288413 Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Mon, 24 Aug 2026 22:08:21 +0000 Subject: [PATCH 17/18] Fix AKS-managed node group cleanup --- hack/e2e/cleanup_safety_test.go | 19 +- hack/e2e/e2e_scripts_test.go | 324 ++++++++++++++++++++++++++++++-- hack/e2e/lib/cleanup.sh | 68 +++++-- 3 files changed, 384 insertions(+), 27 deletions(-) diff --git a/hack/e2e/cleanup_safety_test.go b/hack/e2e/cleanup_safety_test.go index 26d454fd..16ee77e5 100644 --- a/hack/e2e/cleanup_safety_test.go +++ b/hack/e2e/cleanup_safety_test.go @@ -121,8 +121,23 @@ func TestCleanupHandlesLegacyDefaultNodeResourceGroup(t *testing.T) { if readErr != nil { t.Fatalf("read az calls: %v", readErr) } - if !strings.Contains(string(calls), "group delete --name MC_test-rg_aks-e2e-test_test-location") { - t.Fatalf("cleanup did not delete the derived legacy node resource group:\n%s", calls) + if test.parentGroupAbsent { + if !strings.Contains(string(calls), "group delete --name MC_test-rg_aks-e2e-test_test-location") { + t.Fatalf("cleanup did not delete the proven orphan node resource group:\n%s", calls) + } + return + } + if !strings.Contains(string(calls), "aks wait --resource-group test-rg --name aks-e2e-test") { + t.Fatalf("cleanup did not wait for AKS to delete its legacy node resource group:\n%s", calls) + } + for _, operation := range []string{ + "group exists --name MC_test-rg_aks-e2e-test_test-location", + "group delete --name MC_test-rg_aks-e2e-test_test-location", + "group wait --name MC_test-rg_aks-e2e-test_test-location", + } { + if strings.Contains(string(calls), operation) { + t.Fatalf("live-cluster cleanup directly managed its node resource group with %q:\n%s", operation, calls) + } } }) } diff --git a/hack/e2e/e2e_scripts_test.go b/hack/e2e/e2e_scripts_test.go index 5017a276..d9507dd6 100644 --- a/hack/e2e/e2e_scripts_test.go +++ b/hack/e2e/e2e_scripts_test.go @@ -767,6 +767,183 @@ func TestCleanupDeletesOrphanNodeResourceGroupWhenParentIsAbsent(t *testing.T) { } } +func TestCleanupReliesOnAKSDeletionForLiveNodeResourceGroup(t *testing.T) { + t.Parallel() + + _, _, callLog, output, err := runCleanup(t, cleanupOptions{nodeGroupForbidden: true}) + if err != nil { + t.Fatalf("live-cluster cleanup failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=success") { + t.Fatalf("cleanup did not accept confirmed AKS deletion:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + callText := string(calls) + if !strings.Contains(callText, "aks wait --resource-group test-rg --name aks-e2e-test") { + t.Fatalf("cleanup did not wait for confirmed AKS deletion:\n%s", calls) + } + for _, operation := range []string{ + "group exists --name MC_aksflex-e2e-test", + "group delete --name MC_aksflex-e2e-test", + "group wait --name MC_aksflex-e2e-test", + } { + if strings.Contains(callText, operation) { + t.Fatalf("cleanup directly managed the live cluster's node resource group with %q:\n%s", operation, calls) + } + } +} + +func TestCleanupDoesNotTouchNodeResourceGroupWhenAKSRemains(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{ + aksDeleteRemains: true, + nodeGroupForbidden: true, + }) + if err != nil { + t.Fatalf("AKS deletion failure harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") || + !strings.Contains(string(output), "AKS cluster still exists after cleanup timeout") { + t.Fatalf("cleanup accepted a cluster that remained after deletion:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + callText := string(calls) + if !strings.Contains(callText, "aks show --resource-group test-rg --name aks-e2e-test") { + t.Fatalf("cleanup did not verify that the AKS cluster remained:\n%s", calls) + } + for _, operation := range []string{ + "group exists --name MC_aksflex-e2e-test", + "group delete --name MC_aksflex-e2e-test", + "group wait --name MC_aksflex-e2e-test", + } { + if strings.Contains(callText, operation) { + t.Fatalf("cleanup touched a live cluster's node resource group with %q:\n%s", operation, calls) + } + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("failed AKS deletion was marked clean: %s", state) + } +} + +func TestCleanupDoesNotAssumeAKSDeletionWhenWaitIsUnconfirmed(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{ + aksWaitUnconfirmed: true, + nodeGroupForbidden: true, + }) + if err != nil { + t.Fatalf("unconfirmed AKS deletion harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") || + !strings.Contains(string(output), "Failed to confirm AKS cluster deletion") { + t.Fatalf("cleanup accepted an unconfirmed AKS deletion:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + callText := string(calls) + if !strings.Contains(callText, "aks show --resource-group test-rg --name aks-e2e-test") { + t.Fatalf("cleanup did not inspect AKS after the wait command failed:\n%s", calls) + } + for _, operation := range []string{ + "group exists --name MC_aksflex-e2e-test", + "group delete --name MC_aksflex-e2e-test", + "group wait --name MC_aksflex-e2e-test", + } { + if strings.Contains(callText, operation) { + t.Fatalf("cleanup touched the node resource group after an unconfirmed AKS deletion with %q:\n%s", operation, calls) + } + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("unconfirmed AKS deletion was marked clean: %s", state) + } +} + +func TestCleanupPropagatesOrphanNodeResourceGroupDeleteFailure(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{ + parentGroupAbsent: true, + nodeGroupDeleteFails: true, + }) + if err != nil { + t.Fatalf("orphan deletion failure harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=error") || + !strings.Contains(string(output), "Failed to delete orphaned AKS node resource group") || + !strings.Contains(string(output), "AuthorizationFailed") { + t.Fatalf("cleanup swallowed the orphan node resource group deletion failure:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + callText := string(calls) + if !strings.Contains(callText, "group delete --name MC_aksflex-e2e-test") { + t.Fatalf("cleanup did not attempt orphan node resource group deletion:\n%s", calls) + } + if strings.Contains(callText, "group wait --name MC_aksflex-e2e-test") { + t.Fatalf("cleanup waited after orphan node resource group deletion failed:\n%s", calls) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("failed orphan deletion was marked clean: %s", state) + } +} + +func TestCleanupTreatsOrphanDeleteNotFoundAsIdempotent(t *testing.T) { + t.Parallel() + + _, statePath, callLog, output, err := runCleanup(t, cleanupOptions{ + parentGroupAbsent: true, + nodeGroupDeleteNotFound: true, + }) + if err != nil { + t.Fatalf("orphan not-found harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "RESULT=success") { + t.Fatalf("cleanup did not accept explicit orphan absence:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + callText := string(calls) + if !strings.Contains(callText, "group delete --name MC_aksflex-e2e-test") { + t.Fatalf("cleanup did not encounter the orphan deletion race:\n%s", calls) + } + if strings.Contains(callText, "group wait --name MC_aksflex-e2e-test") { + t.Fatalf("cleanup waited after Azure reported the orphan absent:\n%s", calls) + } + state, readErr := os.ReadFile(statePath) + if readErr != nil { + t.Fatalf("read state: %v", readErr) + } + if !strings.Contains(string(state), `"cleanup_complete": "true"`) { + t.Fatalf("idempotent orphan cleanup was not marked complete: %s", state) + } +} + func TestCleanupRejectsUnexpectedOrphanNodeResourceGroup(t *testing.T) { t.Parallel() @@ -856,7 +1033,7 @@ func TestCleanupRejectsUnexpectedArcMachineID(t *testing.T) { func TestCleanupRetriesTransientAzureQueries(t *testing.T) { t.Parallel() - _, _, callLog, output, err := runCleanup(t, cleanupOptions{transientQueryFailures: true}) + workDir, _, callLog, output, err := runCleanup(t, cleanupOptions{transientQueryFailures: true}) if err != nil { t.Fatalf("transient-query cleanup failed: %v\n%s", err, output) } @@ -867,11 +1044,17 @@ func TestCleanupRetriesTransientAzureQueries(t *testing.T) { if readErr != nil { t.Fatalf("read az calls: %v", readErr) } - if strings.Count(string(calls), "group exists") < 8 { + if strings.Count(string(calls), "group exists") < 3 { t.Errorf("cleanup did not retry a failed resource-group query:\n%s", calls) } - if strings.Count(string(calls), "resource list") < 8 { - t.Errorf("cleanup did not retry a failed resource inventory:\n%s", calls) + for _, marker := range []string{"group-query-failed", "resource-query-failed"} { + failures, readErr := os.ReadFile(filepath.Join(workDir, marker)) + if readErr != nil { + t.Fatalf("read %s: %v", marker, readErr) + } + if strings.TrimSpace(string(failures)) != "2" { + t.Errorf("%s recorded %q transient failures, want 2", marker, strings.TrimSpace(string(failures))) + } } } @@ -905,6 +1088,85 @@ fi } } +func TestCleanupAzureQueriesStopOnAuthorizationFailure(t *testing.T) { + t.Parallel() + + for _, query := range []string{"group", "inventory"} { + t.Run(query, func(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + callLog := filepath.Join(workDir, "az-calls") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +AZ_CALL_LOG="$3" +QUERY="$4" +E2E_CLEANUP_POLL_INTERVAL=0.01 +az() { + printf '%s\n' "$*" >> "${AZ_CALL_LOG}" + printf '%s\n' "ERROR: Operation returned an invalid status 'Forbidden'" >&2 + return 1 +} +deadline=$((SECONDS + 60)) +if [[ "${QUERY}" == "group" ]]; then + _group_exists_with_retry test-rg test-subscription "${deadline}" || true +else + _resource_inventory test-rg test-subscription "" "${deadline}" || true +fi +printf 'CALLS=%s\n' "$(wc -l < "${AZ_CALL_LOG}")" +` + output, err := runBash(t, script, workDir, cleanupScript, callLog, query) + if err != nil { + t.Fatalf("authorization failure harness failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "after 1 attempts") || + !strings.Contains(string(output), "invalid status 'Forbidden'") || + !strings.Contains(string(output), "CALLS=1") { + t.Fatalf("authorization failure was retried or lost its diagnostic:\n%s", output) + } + }) + } +} + +func TestCleanupAzureQueriesBoundRetriesByAttemptAndDeadline(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + callLog := filepath.Join(workDir, "az-calls") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +AZ_CALL_LOG="$3" +E2E_CLEANUP_POLL_INTERVAL=0.01 +az() { + printf '%s\n' "$*" >> "${AZ_CALL_LOG}" + printf '%s\n' '(TooManyRequests) ARM throttled the cleanup query' >&2 + return 1 +} +sleep() { :; } +deadline=$((SECONDS + 60)) +_group_exists_with_retry test-rg test-subscription "${deadline}" || true +printf 'GROUP_CALLS=%s\n' "$(wc -l < "${AZ_CALL_LOG}")" +: > "${AZ_CALL_LOG}" +_resource_inventory test-rg test-subscription "" "${deadline}" || true +printf 'INVENTORY_CALLS=%s\n' "$(wc -l < "${AZ_CALL_LOG}")" +` + output, err := runBash(t, script, workDir, cleanupScript, callLog) + if err != nil { + t.Fatalf("bounded retry harness failed: %v\n%s", err, output) + } + for _, expected := range []string{"GROUP_CALLS=5", "INVENTORY_CALLS=5"} { + if !strings.Contains(string(output), expected) { + t.Fatalf("Azure query retries were not bounded by attempt count; missing %q:\n%s", expected, output) + } + } +} + func TestCleanupTreatsResourceGroupNotFoundAsAbsent(t *testing.T) { t.Parallel() @@ -1084,10 +1346,15 @@ func TestHistoricalTokenRevocationPropagatesDeleteFailure(t *testing.T) { type cleanupOptions struct { runTwice bool leaveCluster bool + aksDeleteRemains bool + aksWaitUnconfirmed bool parentGroupAbsent bool deploymentQueryFails bool noRunTags bool blankVMNames bool + nodeGroupForbidden bool + nodeGroupDeleteFails bool + nodeGroupDeleteNotFound bool unexpectedNodeResource bool unexpectedLiveNodeRG bool legacyDefaultNodeResource bool @@ -1106,6 +1373,7 @@ func runCleanup(t *testing.T, options cleanupOptions) (string, string, string, [ nodeGroupDeleted := filepath.Join(workDir, "node-group-deleted") groupQueryFailed := filepath.Join(workDir, "group-query-failed") resourceQueryFailed := filepath.Join(workDir, "resource-query-failed") + clusterDeleted := filepath.Join(workDir, "cluster-deleted") state := `{ "resource_group": "test-rg", "location": "test-location", @@ -1171,6 +1439,12 @@ LEGACY_DEFAULT_NODE_RG="${13}" LEGACY_DETACHED_DISK="${14}" LEGACY_DISK_DELETE_PERSISTS="${15}" UNSET_SUBSCRIPTION_ENV="${16}" +AKS_DELETE_REMAINS="${17}" +NODE_GROUP_FORBIDDEN="${18}" +NODE_GROUP_DELETE_FAILS="${19}" +CLUSTER_DELETED="${20}" +AKS_WAIT_UNCONFIRMED="${21}" +NODE_GROUP_DELETE_NOT_FOUND="${22}" LEGACY_DISK_DELETED="${E2E_WORK_DIR}/legacy-disk-deleted" # Keep cleanup fixtures independent from the ambient GitHub Actions run. Tests # that need tag-based cleanup persist an explicit run_id in their state. @@ -1193,32 +1467,56 @@ az() { elif [[ "$1 $2" == "group exists" ]]; then if [[ "$*" == *"--name MC_aksflex-e2e-test"* || \ "$*" == *"--name MC_test-rg_aks-e2e-test_test-location"* ]]; then + if [[ "${NODE_GROUP_FORBIDDEN}" == "1" ]]; then + printf '%s\n' "ERROR: Operation returned an invalid status 'Forbidden'" >&2 + return 1 + fi + [[ -f "${NODE_GROUP_DELETED}" ]] && printf 'false\n' || printf 'true\n' + elif [[ "$*" == *"--name test-rg"* ]]; then query_failures="$(cat "${GROUP_QUERY_FAILED}" 2>/dev/null || printf '0\n')" - if [[ "${TRANSIENT_QUERY_FAILURES}" == "1" && "${query_failures}" -lt 8 ]]; then + if [[ "${TRANSIENT_QUERY_FAILURES}" == "1" && "${query_failures}" -lt 2 ]]; then printf '%s\n' "$((query_failures + 1))" > "${GROUP_QUERY_FAILED}" printf '%s\n' '(TooManyRequests) transient resource-group query failure' >&2 return 1 fi - [[ -f "${NODE_GROUP_DELETED}" ]] && printf 'false\n' || printf 'true\n' - elif [[ "$*" == *"--name test-rg"* ]]; then [[ "${PARENT_GROUP_ABSENT}" == "1" ]] && printf 'false\n' || printf 'true\n' else printf 'false\n' fi elif [[ "$1 $2" == "group delete" ]]; then + if [[ "${NODE_GROUP_DELETE_FAILS}" == "1" ]]; then + printf '%s\n' '(AuthorizationFailed) The client cannot delete this resource group.' >&2 + return 1 + fi + if [[ "${NODE_GROUP_DELETE_NOT_FOUND}" == "1" ]]; then + printf '%s\n' '(ResourceGroupNotFound) Resource group was not found.' >&2 + return 1 + fi : > "${NODE_GROUP_DELETED}" elif [[ "$1 $2" == "group wait" ]]; then return 0 elif [[ "$1 $2" == "vm list" ]]; then printf '[]\n' elif [[ "$1 $2" == "aks list" ]]; then - if [[ "${UNEXPECTED_LIVE_NODE_RG}" == "1" ]]; then + if [[ -f "${CLUSTER_DELETED}" ]]; then + printf '[]\n' + elif [[ "${UNEXPECTED_LIVE_NODE_RG}" == "1" ]]; then printf '[{"name":"aks-e2e-test","nodeResourceGroup":"production-live-node-rg"}]\n' elif [[ "${LEGACY_DEFAULT_NODE_RG}" == "1" ]]; then printf '[{"name":"aks-e2e-test","location":"test-location","nodeResourceGroup":"MC_test-rg_aks-e2e-test_test-location"}]\n' else - printf '[]\n' + printf '[{"name":"aks-e2e-test","location":"test-location","nodeResourceGroup":"MC_aksflex-e2e-test"}]\n' + fi + elif [[ "$1 $2" == "aks delete" ]]; then + [[ "${AKS_DELETE_REMAINS}" == "1" ]] || : > "${CLUSTER_DELETED}" + elif [[ "$1 $2" == "aks wait" ]]; then + [[ "${AKS_DELETE_REMAINS}" != "1" && "${AKS_WAIT_UNCONFIRMED}" != "1" ]] + elif [[ "$1 $2" == "aks show" ]]; then + if [[ "${AKS_WAIT_UNCONFIRMED}" == "1" ]]; then + printf '%s\n' "ERROR: Operation returned an invalid status 'Forbidden'" >&2 + return 1 fi + [[ "${AKS_DELETE_REMAINS}" == "1" ]] elif [[ "$1 $2" == "resource delete" ]]; then if [[ "$*" == *"--ids /own-legacy-disk"* && \ "${LEGACY_DISK_DELETE_PERSISTS}" != "1" ]]; then @@ -1227,8 +1525,7 @@ az() { return 0 elif [[ "$1 $2" == "resource list" ]]; then query_failures="$(cat "${RESOURCE_QUERY_FAILED}" 2>/dev/null || printf '0\n')" - if [[ "${TRANSIENT_QUERY_FAILURES}" == "1" && -f "${NODE_GROUP_DELETED}" && \ - "${query_failures}" -lt 8 ]]; then + if [[ "${TRANSIENT_QUERY_FAILURES}" == "1" && "${query_failures}" -lt 2 ]]; then printf '%s\n' "$((query_failures + 1))" > "${RESOURCE_QUERY_FAILED}" printf '%s\n' '(TooManyRequests) transient resource inventory failure' >&2 return 1 @@ -1273,7 +1570,10 @@ fi boolString(options.deploymentQueryFails), boolString(options.runTwice), boolString(options.transientQueryFailures), boolString(options.unexpectedLiveNodeRG), boolString(options.legacyDefaultNodeResource), boolString(options.legacyDetachedDisk), - boolString(options.legacyDiskDeletePersists), boolString(options.unsetSubscriptionEnv)) + boolString(options.legacyDiskDeletePersists), boolString(options.unsetSubscriptionEnv), + boolString(options.aksDeleteRemains), boolString(options.nodeGroupForbidden), + boolString(options.nodeGroupDeleteFails), clusterDeleted, + boolString(options.aksWaitUnconfirmed), boolString(options.nodeGroupDeleteNotFound)) return workDir, statePath, callLog, output, err } diff --git a/hack/e2e/lib/cleanup.sh b/hack/e2e/lib/cleanup.sh index 309f4a52..6d5262a5 100755 --- a/hack/e2e/lib/cleanup.sh +++ b/hack/e2e/lib/cleanup.sh @@ -278,11 +278,7 @@ _remaining_cleanup_timeout() { _azure_query_can_attempt() { local attempt="$1" deadline="${2:-0}" - if (( deadline > 0 )); then - (( SECONDS < deadline )) - else - (( attempt < 5 )) - fi + (( attempt < 5 && (deadline <= 0 || SECONDS < deadline) )) } _sleep_before_azure_query_retry() { @@ -311,6 +307,25 @@ _azure_error_summary() { printf '%.500s\n' "${message}" } +_azure_error_is_authorization_failure() { + local message="${1,,}" + + [[ "${message}" == *forbidden* || + "${message}" == *unauthorized* || + "${message}" == *authorizationfailed* || + "${message}" == *authorizationpermissionmismatch* || + "${message}" == *authenticationfailed* || + "${message}" == *invalidauthenticationtoken* || + "${message}" == *expiredauthenticationtoken* || + "${message}" == *"does not have authorization"* || + "${message}" == *"permission denied"* || + "${message}" == *"insufficient privileges"* ]] +} + +_azure_error_is_resource_group_not_found() { + [[ "$1" == *ResourceGroupNotFound* ]] +} + _group_exists_with_retry() { local group_name="$1" subscription_id="$2" deadline="${3:-0}" local exists attempt=0 last_error="" error_file @@ -341,12 +356,15 @@ _group_exists_with_retry() { last_error="$(<"${error_file}")" # ARM can report the resource as not found while `az group exists` still # exits nonzero during asynchronous AKS-managed resource-group deletion. - if [[ "${last_error}" == *ResourceGroupNotFound* ]]; then + if _azure_error_is_resource_group_not_found "${last_error}"; then rm -f -- "${error_file}" printf 'false\n' return 0 fi fi + if _azure_error_is_authorization_failure "${last_error}"; then + break + fi if ! _azure_query_can_attempt "${attempt}" "${deadline}" || \ ! _sleep_before_azure_query_retry "${deadline}"; then break @@ -389,6 +407,9 @@ _resource_inventory() { else last_error="$(<"${error_file}")" fi + if _azure_error_is_authorization_failure "${last_error}"; then + break + fi if ! _azure_query_can_attempt "${attempt}" "${deadline}" || \ ! _sleep_before_azure_query_retry "${deadline}"; then break @@ -587,7 +608,7 @@ _validate_expected_node_resource_group() { _delete_node_resource_group() { local node_resource_group="$1" subscription_id="$2" deadline="$3" - local exists wait_timeout + local delete_error error_file exists wait_timeout [[ -n "${node_resource_group}" ]] || return 0 if ! exists="$(_group_exists_with_retry "${node_resource_group}" "${subscription_id}" "${deadline}")"; then @@ -606,8 +627,24 @@ _delete_node_resource_group() { ;; esac - az group delete --name "${node_resource_group}" --subscription "${subscription_id}" \ - --yes --no-wait --output none 2>/dev/null || true + if ! error_file="$(mktemp "${E2E_WORK_DIR}/azure-group-delete.XXXXXX")"; then + log_error "Failed to create temporary Azure deletion diagnostics" + return 1 + fi + if ! az group delete --name "${node_resource_group}" --subscription "${subscription_id}" \ + --yes --no-wait --output none 2>"${error_file}"; then + delete_error="$(<"${error_file}")" + rm -f -- "${error_file}" + if _azure_error_is_resource_group_not_found "${delete_error}"; then + return 0 + fi + log_error "Failed to delete orphaned AKS node resource group '${node_resource_group}'" + if [[ -n "${delete_error}" ]]; then + log_error "Azure CLI error: $(_azure_error_summary "${delete_error}")" + fi + return 1 + fi + rm -f -- "${error_file}" if ! wait_timeout="$(_remaining_cleanup_timeout "${deadline}")"; then log_error "Cleanup deadline reached before deleting AKS node resource group '${node_resource_group}'" return 1 @@ -768,6 +805,8 @@ cleanup() { case "${resource_group_exists}" in false) + # The parent group's absence proves the cluster is gone, so this is a + # safe recovery path for an exact-name-validated orphan node group. if ! _validate_expected_node_resource_group \ "${node_resource_group}" "${name_suffix}" "${resource_group}" "${cluster_name}" "${location}"; then return 1 @@ -958,17 +997,20 @@ cleanup() { --resource-group "${resource_group}" --name "${cluster_name}" \ --subscription "${subscription_id}" --deleted --interval 10 \ --timeout "${aks_wait_timeout}" 2>/dev/null; then + cleanup_failed=1 if az aks show --resource-group "${resource_group}" --name "${cluster_name}" \ --subscription "${subscription_id}" --output none 2>/dev/null; then log_error "AKS cluster still exists after cleanup timeout: ${cluster_name}" - cleanup_failed=1 + else + log_error "Failed to confirm AKS cluster deletion after the wait command failed: ${cluster_name}" fi fi fi - if ! _delete_node_resource_group "${node_resource_group}" "${subscription_id}" "${cleanup_deadline}"; then - cleanup_failed=1 - fi + # AKS owns the managed node resource group's lifecycle and deletes it with + # the cluster. Avoid querying or deleting that sibling group directly: the + # least-privilege cleanup identity may only have access to the parent group. + # https://learn.microsoft.com/azure/aks/faq#can-i-restore-my-cluster-after-i-delete-it- if [[ -n "${arc_machine_id}" ]]; then local arc_wait_timeout From d677542668480b5155442cceef2aa33ed96f6079 Mon Sep 17 00:00:00 2001 From: Wen Huang <50309350+wenhug@users.noreply.github.com> Date: Mon, 24 Aug 2026 22:49:37 +0000 Subject: [PATCH 18/18] Fix scoped E2E resource inventory --- hack/e2e/e2e_scripts_test.go | 45 ++++++++++++++++++++++++++++++++++++ hack/e2e/lib/cleanup.sh | 10 ++++---- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/hack/e2e/e2e_scripts_test.go b/hack/e2e/e2e_scripts_test.go index d9507dd6..265dd738 100644 --- a/hack/e2e/e2e_scripts_test.go +++ b/hack/e2e/e2e_scripts_test.go @@ -1058,6 +1058,51 @@ func TestCleanupRetriesTransientAzureQueries(t *testing.T) { } } +func TestCleanupFiltersResourceInventoryByTagWithinGroup(t *testing.T) { + t.Parallel() + + workDir := t.TempDir() + cleanupScript := e2eScriptPath(t, "lib", "cleanup.sh") + callLog := filepath.Join(workDir, "az-calls") + script := ` +set -euo pipefail +E2E_WORK_DIR="$1" +source "$2" +AZ_CALL_LOG="$3" +az() { + printf '%s\n' "$*" >> "${AZ_CALL_LOG}" + printf '%s\n' '[ + {"name":"owned","tags":{"github-run":"test-run"}}, + {"name":"another-run","tags":{"github-run":"other-run"}}, + {"name":"untagged"} + ]' +} +inventory="$(_resource_inventory test-rg test-subscription test-run 0)" +printf 'INVENTORY=%s\n' "${inventory}" +` + output, err := runBash(t, script, workDir, cleanupScript, callLog) + if err != nil { + t.Fatalf("tagged resource inventory harness failed: %v\n%s", err, output) + } + text := string(output) + if !strings.Contains(text, `"name":"owned"`) || + strings.Contains(text, `"name":"another-run"`) || + strings.Contains(text, `"name":"untagged"`) { + t.Fatalf("resource inventory was not filtered to the requested run tag:\n%s", output) + } + calls, readErr := os.ReadFile(callLog) + if readErr != nil { + t.Fatalf("read az calls: %v", readErr) + } + callText := string(calls) + if !strings.Contains(callText, "resource list --resource-group test-rg --subscription test-subscription --output json") { + t.Fatalf("resource inventory was not scoped to the parent resource group:\n%s", calls) + } + if strings.Contains(callText, "--tag") { + t.Fatalf("resource inventory used Azure CLI's incompatible --resource-group/--tag combination:\n%s", calls) + } +} + func TestCleanupAzureQueryFailureIncludesDiagnostic(t *testing.T) { t.Parallel() diff --git a/hack/e2e/lib/cleanup.sh b/hack/e2e/lib/cleanup.sh index 6d5262a5..349925bb 100755 --- a/hack/e2e/lib/cleanup.sh +++ b/hack/e2e/lib/cleanup.sh @@ -382,8 +382,6 @@ _group_exists_with_retry() { _resource_inventory() { local resource_group="$1" subscription_id="$2" run_id="${3:-}" deadline="${4:-0}" local resource_json attempt=0 last_error="" error_file - local -a tag_args=() - [[ -z "${run_id}" ]] || tag_args=(--tag "github-run=${run_id}") if ! error_file="$(mktemp "${E2E_WORK_DIR}/azure-resource-query.XXXXXX")"; then log_error "Failed to create temporary Azure query diagnostics" @@ -396,9 +394,13 @@ _resource_inventory() { if resource_json="$(az resource list \ --resource-group "${resource_group}" \ --subscription "${subscription_id}" \ - "${tag_args[@]}" \ --output json 2>"${error_file}")"; then - if jq -e 'type == "array"' <<<"${resource_json}" >/dev/null; then + if resource_json="$(jq -ce --arg run_id "${run_id}" ' + if type != "array" then error("resource list must be an array") + elif $run_id == "" then . + else [.[] | select((.tags // {})["github-run"] == $run_id)] + end + ' <<<"${resource_json}")"; then rm -f -- "${error_file}" printf '%s\n' "${resource_json}" return 0