Restrict Flex Node bootstrap token RBAC

.github/workflows/e2e-tests.yml

@@ -9,6 +9,19 @@ name: E2E Tests
 on:
   workflow_dispatch:
     inputs:
+      suite:
+        description: "E2E suite to run"
+        required: false
+        default: all
+        type: choice
+        options:
+          - all
+          - historical-rbac-migration
+      kubernetes_version:
+        description: "Exact AKS/Flex Node Kubernetes patch version (for example, 1.34.9)"
+        required: false
+        default: "1.35.0"
+        type: string
       skip_cleanup:
         description: "Skip cleanup (keep resources for debugging)"
         required: false
@@ -56,7 +69,10 @@ env:
   AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
   AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
   GITHUB_RUN_ID: ${{ github.run_id }}
-  E2E_WORK_DIR: /tmp/aks-flex-node-e2e-${{ github.run_id }}
+  E2E_NAME_SUFFIX: ${{ github.run_id }}-${{ github.run_attempt }}
+  E2E_WORK_DIR: /tmp/aks-flex-node-e2e-${{ github.run_id }}-${{ github.run_attempt }}
+  E2E_SUITE: ${{ inputs.suite || 'all' }}
+  E2E_KUBERNETES_VERSION: ${{ inputs.kubernetes_version || '1.35.0' }}
 
 jobs:
   e2e:
@@ -95,7 +111,7 @@ jobs:
           E2E_SKIP_CLEANUP: "1"  # Cleanup handled in dedicated step below
         run: |
           set -euo pipefail
-          ./hack/e2e/run.sh all
+          ./hack/e2e/run.sh "${E2E_SUITE}"
 
       - name: Collect logs
         if: always()
@@ -105,16 +121,59 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         if: always()
         with:
-          name: e2e-logs-${{ github.run_id }}
-          path: /tmp/aks-flex-node-e2e-${{ github.run_id }}/logs/
+          name: e2e-logs-${{ github.run_id }}-${{ github.run_attempt }}
+          path: /tmp/aks-flex-node-e2e-${{ github.run_id }}-${{ github.run_attempt }}/logs/
           retention-days: 7
 
       - name: Cleanup
+        id: cleanup
         if: always()
         env:
           E2E_SKIP_CLEANUP: ${{ inputs.skip_cleanup && '1' || '0' }}
         run: ./hack/e2e/run.sh cleanup
 
-      - name: Cleanup runner workspace
+      - name: Prepare cleanup diagnostics
+        if: always()
+        run: |
+          set -euo pipefail
+          umask 077
+          state_file="${E2E_WORK_DIR}/state.json"
+          artifact_file="${E2E_WORK_DIR}/cleanup-state.json"
+          [[ -f "${state_file}" ]] || exit 0
+          # Keep the resource names needed for investigation without publishing
+          # secret-sourced account identifiers, addresses, or cluster access data.
+          jq '{
+            lifecycle,
+            cleanup_complete,
+            deployment_name,
+            run_id,
+            resource_owner,
+            name_suffix,
+            cluster_name,
+            node_resource_group,
+            msi_vm_name,
+            token_vm_name,
+            offline_vm_name,
+            kubeadm_vm_name,
+            arc_vm_name,
+            arc_machine_name,
+            vnet_name,
+            nsg_name
+          } | with_entries(select(.value != null and .value != ""))' \
+            "${state_file}" > "${artifact_file}"
+          chmod 0600 "${artifact_file}"
+
+      - name: Upload cleanup diagnostics
         if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: e2e-cleanup-state-${{ github.run_id }}-${{ github.run_attempt }}
+          path: /tmp/aks-flex-node-e2e-${{ github.run_id }}-${{ github.run_attempt }}/cleanup-state.json
+          if-no-files-found: ignore
+          retention-days: 7
+
+      - name: Cleanup runner workspace
+        # Preserve retry metadata when Azure cleanup fails or is deliberately
+        # skipped. A later successful run must not delete another attempt's state.
+        if: always() && steps.cleanup.outcome == 'success' && inputs.skip_cleanup != true
         run: ./hack/e2e/run.sh runner-cleanup

docs/usages/aks-flex-config.md

@@ -9,7 +9,7 @@ The helper does not install anything on the target host. It uses Azure CLI and,
 - Azure CLI authenticated to the subscription that contains the AKS cluster.
 - `python3` on the workstation.
 - `kubectl` on the workstation for `setup-node-rbac` and `--bootstrap-token` config generation.
-- Permission to run `az aks get-credentials --admin` and create Kubernetes `ClusterRoleBinding` and bootstrap token `Secret` objects.
+- Permission to run `az aks get-credentials --admin`, create Kubernetes `ClusterRoleBinding` and bootstrap token `Secret` objects, and remove the obsolete `aks-flex-node-role` binding when present.
 
 ## Save The Helper
 
@@ -46,7 +46,53 @@ Run this once per cluster for bootstrap-token joins:
   --subscription "$SUBSCRIPTION_ID"
 ```
 
-This applies the bootstrap-related `ClusterRoleBinding` objects for the `system:bootstrappers:aks-flex-node` group.
+This applies only the CSR creation and approval `ClusterRoleBinding` objects for the `system:bootstrappers:aks-flex-node` group. If any binding still grants that group the obsolete `system:node` role, the command stops after applying the safe bindings and explains how to migrate. It does not silently remove the binding because older and development-mode agents may still use their bootstrap token after joining.
+
+`v0.1.1` introduced a separate daemon client certificate, but the version alone does not prove that certificate was issued successfully. Upgrade every bootstrap-token agent to `v0.1.1` or later (preferably the latest release), and on every host verify that the certificate exists, is unexpired, and the agent remains healthy after a restart:
+
+```bash
+sudo test -s /etc/aks-flex-node/daemon-credentials/daemon-controller-current.pem
+sudo openssl x509 \
+  -in /etc/aks-flex-node/daemon-credentials/daemon-controller-current.pem \
+  -noout -subject -enddate -checkend 0
+sudo systemctl restart aks-flex-node-agent.service
+sudo systemctl is-active aks-flex-node-agent.service
+```
+
+Then explicitly remove the obsolete binding:
+
+```bash
+./aks-flex-config setup-node-rbac \
+  --resource-group "$RESOURCE_GROUP" \
+  --cluster-name "$CLUSTER_NAME" \
+  --subscription "$SUBSCRIPTION_ID" \
+  --remove-legacy-node-role-binding
+```
+
+This migration is idempotent. It automatically deletes only the canonical `aks-flex-node-role` object created by older helpers. If another binding grants the same unsafe edge, or that object has extra subjects, the helper refuses to guess and identifies the objects for manual review. Bootstrap-token config generation refuses to create a token while any such binding exists, rather than either issuing an over-privileged token or unexpectedly breaking an old daemon.
+
+To verify no binding still grants the bootstrap group `system:node`, run:
+
+```bash
+kubectl get clusterrolebinding -o json | jq -r '
+  .items[]
+  | select(.roleRef.kind == "ClusterRole" and .roleRef.name == "system:node")
+  | .metadata.name as $binding
+  | .subjects[]?
+  | select(.kind == "Group" and .name == "system:bootstrappers:aks-flex-node")
+  | $binding'
+```
+
+The expected result is no output. The canonical `aks-flex-node-role` object is
+deleted; a safe, repurposed object with that name is preserved. Once certificate
+issuance has been verified, both the kubelet and long-running Flex daemon use
+issued client certificates, so removing the unsafe binding does not interrupt
+joined nodes. New and in-progress joins retain the CSR permissions installed
+above.
+
+Do not roll back a migrated host to an older or development-mode agent that still uses the bootstrap token for ordinary Kubernetes API requests. After this binding is removed, those requests correctly receive `403 Forbidden`. Restore a supported certificate-using agent instead of restoring the broad binding.
+
+Finally, delete bootstrap-token Secrets that are no longer needed. In particular, tokens made by helpers before `v0.1.1` had no expiration. Removing the broad binding limits them to bootstrap permissions, but does not revoke them; do not delete a token that is still being used by an in-progress join.
 
 ## Generate Node Config
 

docs/usages/joining-nodes.md

@@ -16,7 +16,7 @@ Bootstrap token mode is the recommended quickstart path. It uses Kubernetes TLS
 
 High-level flow:
 
-1. Run [`scripts/aks-flex-config setup-node-rbac`](../../scripts/aks-flex-config) to setup required node bootstrap RBAC permissions.
+1. Run [`scripts/aks-flex-config setup-node-rbac`](../../scripts/aks-flex-config) to set up the least-privilege node bootstrap RBAC permissions. Clusters configured by an older helper require the explicit compatibility migration documented in the helper guide before another token can be generated.
 2. Run `scripts/aks-flex-config generate-node-config --bootstrap-token` to create a bootstrap token, fetch AKS cluster metadata, and render the host config.
 3. Copy the generated config to `/etc/aks-flex-node/config.json` on the target host.
 4. Run `aks-flex-node preflight --config /etc/aks-flex-node/config.json` to validate host, cluster, rootfs, and artifact prerequisites without mutating the node.

hack/e2e/README.md

@@ -12,6 +12,8 @@ The E2E suite provisions a no-CNI AKS cluster, installs Unbounded-Net as the clu
 | `python3` | Local registry port readiness checks and helper scripts. |
 | `ssh` / `scp` | VM access and artifact copy. |
 | `openssl` | Bootstrap token generation. |
+| `curl` / `sha256sum` / `tar` | Download and verify pinned historical release artifacts. |
+| `flock` | Serialize atomic updates to the per-run cleanup state. |
 | `docker` | Build and push the controller image into the in-cluster local registry. |
 | `git` / `make` | Fetch and render Unbounded-Net manifests. |
 | `go` | Build the agent binary unless `--binary` is supplied. |
@@ -65,6 +67,7 @@ The default `all` command runs:
 | Command | Description |
 |---------|-------------|
 | `all` | Full flow: build, infra, join, validate, unjoin, validate absent, rejoin, validate, lifecycle, agent upgrade, repave, logs, cleanup. |
+| `historical-rbac-migration` | On a fresh real AKS cluster and token VM, join with the official v0.1.0 helper/binary, upgrade that host to HEAD, migrate legacy bootstrap RBAC, revoke the old token, and validate restarts. It does not run the other join modes. |
 | `infra` | Deploy AKS, four standard VMs, the Arc VM, Unbounded-Net CNI, the local registry, and the in-cluster controller. |
 | `join` | Join all Flex Node VMs. |
 | `join-msi` | Join only the managed-identity node. |
@@ -107,7 +110,7 @@ Additional environment variables:
 | `E2E_SSH_KEY_FILE` | auto-detected | SSH public key used for VM access. |
 | `E2E_WORK_DIR` | `/tmp/aks-flex-node-e2e` | Working directory for state, configs, and logs. |
 | `E2E_KUBECONFIG` | `$E2E_WORK_DIR/kubeconfig` | Per-run kubeconfig path. Defaults to an isolated file instead of the runner-global kubeconfig. |
-| `E2E_KUBERNETES_VERSION` | `1.35.0` | Kubernetes version used in generated node configs. |
+| `E2E_KUBERNETES_VERSION` | `1.35.0` | Exact Kubernetes version used for the AKS control plane, agent pools, and generated node configs. |
 | `E2E_CONTAINERD_VERSION` | `2.0.4` | Containerd version used in generated node configs. |
 | `E2E_RUNC_VERSION` | `1.1.12` | Runc version used in generated node configs. |
 | `E2E_TARGET_AGENT_POOL_NAME` | `aksflexnodes` | Synthetic target agent pool name used by controller-backed test modes. |
@@ -135,9 +138,57 @@ Additional environment variables:
 | `E2E_POD_READY_TIMEOUT` | `120` | Timeout in seconds while waiting for smoke pods. |
 | `E2E_AGENT_UPGRADE_TIMEOUT` | `300` | Timeout in seconds while waiting for an AgentUpgrade result. |
 | `E2E_DRIFT_UPGRADE_TIMEOUT` | `900` | Timeout in seconds while waiting for repave. |
+| `E2E_CLEANUP_TIMEOUT` | `900` | Shared deadline in seconds for deployment cancellation and Azure resource deletion. |
+| `E2E_CLEANUP_POLL_INTERVAL` | `5` | Poll interval in seconds for deployment and cleanup convergence. |
 | `AZURE_SUBSCRIPTION_ID` | auto-detected | Azure subscription. |
 | `AZURE_TENANT_ID` | auto-detected | Azure tenant. |
 
+## Historical RBAC Migration Validation
+
+Run the focused compatibility suite with:
+
+```bash
+./hack/e2e/run.sh historical-rbac-migration
+```
+
+For a manual GitHub Actions run, select `historical-rbac-migration` in the
+`suite` workflow input. Set `kubernetes_version` to an exact AKS version, such
+as the latest supported N-1 patch, to exercise the migration against an older
+control-plane version. Check regional availability immediately before running
+the workflow with `az aks get-versions --location <region> --output table`.
+Infrastructure provisioning, the test, log upload, and cleanup stay in the same
+job; the suite deliberately does not call the Arc-inclusive parallel join path.
+
+The scenario downloads and verifies the official v0.1.0 release archive,
+extracted binary, helper, and installer. It then uses the pinned helper and
+binary to create the original broad bootstrap RBAC, a non-expiring token, the
+legacy config shape, and a real Ready node on the token VM. The historical
+daemon runs its production no-op path, not the v0.1.0 file-backed E2E machine
+client. On that same host it verifies the HEAD helper fails closed without the
+explicit migration flag, activates the HEAD binary through `agent-upgrade`,
+removes the legacy binding twice to prove idempotency, checks token access
+changes from HTTP 200 to 403, deletes the daemon credential store and verifies
+the remaining CSR permissions issue a different certificate, revokes the token
+and waits for HTTP 401, and restarts both kubelet and the daemon while checking
+the Node UID, Lease, readiness, and certificate-backed API access.
+v0.1.0 transitively pins the non-GPU rootfs
+`ghcr.io/azure/agent-ubuntu2404:v20260427`.
+
+There are two intentional compatibility boundaries:
+
+- The test creates a new AKS control plane and reproduces the v0.1.0
+  cluster-side state. It validates a historical node/config/RBAC migration, not
+  an AKS control plane that has itself been retained since v0.1.0. Selecting an
+  older `kubernetes_version` proves a newly created control plane at that
+  version; it still does not reproduce age, prior upgrades, or configuration
+  drift from a long-lived cluster.
+- v0.1.0 tokens lack the `kubernetes.azure.com/managedby=aks` label required by
+  the production managed CSR approver. The suite explicitly adopts its known
+  token with that label before the HEAD daemon requests a certificate. The
+  repository E2E approver does not enforce this label, so this test validates
+  the host/config/RBAC migration but is not independent proof of the production
+  approver's ownership check.
+
 ## Join Modes
 
 The suite validates five join paths. The E2E subscription must have `Microsoft.HybridCompute`, `Microsoft.HybridConnectivity`, and `Microsoft.GuestConfiguration` registered before the run.