[3/3] Test historical bootstrap RBAC migration

.github/workflows/e2e-tests.yml

@@ -9,6 +9,14 @@ name: E2E Tests
 on:
   workflow_dispatch:
     inputs:
+      suite:
+        description: "E2E suite to run"
+        required: false
+        default: all
+        type: choice
+        options:
+          - all
+          - historical-rbac-migration
       kubernetes_version:
         description: "Exact AKS/Flex Node Kubernetes patch version (for example, 1.34.9)"
         required: false
@@ -63,6 +71,7 @@ env:
   GITHUB_RUN_ID: ${{ github.run_id }}
   E2E_NAME_SUFFIX: ${{ github.run_id }}-${{ github.run_attempt }}
   E2E_WORK_DIR: /tmp/aks-flex-node-e2e-${{ github.run_id }}-${{ github.run_attempt }}
+  E2E_SUITE: ${{ inputs.suite || 'all' }}
   E2E_KUBERNETES_VERSION: ${{ inputs.kubernetes_version || '1.35.0' }}
 
 jobs:
@@ -102,7 +111,7 @@ jobs:
           E2E_SKIP_CLEANUP: "1"  # Cleanup handled in dedicated step below
         run: |
           set -euo pipefail
-          ./hack/e2e/run.sh all
+          ./hack/e2e/run.sh "${E2E_SUITE}"
 
       - name: Collect logs
         if: always()

hack/e2e/README.md

@@ -12,6 +12,7 @@ The E2E suite provisions a no-CNI AKS cluster, installs Unbounded-Net as the clu
 | `python3` | Local registry port readiness checks and helper scripts. |
 | `ssh` / `scp` | VM access and artifact copy. |
 | `openssl` | Bootstrap token generation. |
+| `curl` / `sha256sum` / `tar` | Download and verify pinned historical release artifacts. |
 | `flock` | Serialize atomic updates to the per-run cleanup state. |
 | `docker` | Build and push the controller image into the in-cluster local registry. |
 | `git` / `make` | Fetch and render Unbounded-Net manifests. |
@@ -66,6 +67,7 @@ The default `all` command runs:
 | Command | Description |
 |---------|-------------|
 | `all` | Full flow: build, infra, join, validate, unjoin, validate absent, rejoin, validate, lifecycle, agent upgrade, repave, logs, cleanup. |
+| `historical-rbac-migration` | On a fresh real AKS cluster and token VM, join with the official v0.1.0 helper/binary, upgrade that host to HEAD, migrate legacy bootstrap RBAC, revoke the old token, and validate restarts. It does not run the other join modes. |
 | `infra` | Deploy AKS, four standard VMs, the Arc VM, Unbounded-Net CNI, the local registry, and the in-cluster controller. |
 | `join` | Join all Flex Node VMs. |
 | `join-msi` | Join only the managed-identity node. |
@@ -141,6 +143,52 @@ Additional environment variables:
 | `AZURE_SUBSCRIPTION_ID` | auto-detected | Azure subscription. |
 | `AZURE_TENANT_ID` | auto-detected | Azure tenant. |
 
+## Historical RBAC Migration Validation
+
+Run the focused compatibility suite with:
+
+```bash
+./hack/e2e/run.sh historical-rbac-migration
+```
+
+For a manual GitHub Actions run, select `historical-rbac-migration` in the
+`suite` workflow input. Set `kubernetes_version` to an exact AKS version, such
+as the latest supported N-1 patch, to exercise the migration against an older
+control-plane version. Check regional availability immediately before running
+the workflow with `az aks get-versions --location <region> --output table`.
+Infrastructure provisioning, the test, log upload, and cleanup stay in the same
+job; the suite deliberately does not call the Arc-inclusive parallel join path.
+
+The scenario downloads and verifies the official v0.1.0 release archive,
+extracted binary, helper, and installer. It then uses the pinned helper and
+binary to create the original broad bootstrap RBAC, a non-expiring token, the
+legacy config shape, and a real Ready node on the token VM. The historical
+daemon runs its production no-op path, not the v0.1.0 file-backed E2E machine
+client. On that same host it verifies the HEAD helper fails closed without the
+explicit migration flag, activates the HEAD binary through `agent-upgrade`,
+removes the legacy binding twice to prove idempotency, checks token access
+changes from HTTP 200 to 403, deletes the daemon credential store and verifies
+the remaining CSR permissions issue a different certificate, revokes the token
+and waits for HTTP 401, and restarts both kubelet and the daemon while checking
+the Node UID, Lease, readiness, and certificate-backed API access.
+v0.1.0 transitively pins the non-GPU rootfs
+`ghcr.io/azure/agent-ubuntu2404:v20260427`.
+
+There are two intentional compatibility boundaries:
+
+- The test creates a new AKS control plane and reproduces the v0.1.0
+  cluster-side state. It validates a historical node/config/RBAC migration, not
+  an AKS control plane that has itself been retained since v0.1.0. Selecting an
+  older `kubernetes_version` proves a newly created control plane at that
+  version; it still does not reproduce age, prior upgrades, or configuration
+  drift from a long-lived cluster.
+- v0.1.0 tokens lack the `kubernetes.azure.com/managedby=aks` label required by
+  the production managed CSR approver. The suite explicitly adopts its known
+  token with that label before the HEAD daemon requests a certificate. The
+  repository E2E approver does not enforce this label, so this test validates
+  the host/config/RBAC migration but is not independent proof of the production
+  approver's ownership check.
+
 ## Join Modes
 
 The suite validates five join paths. The E2E subscription must have `Microsoft.HybridCompute`, `Microsoft.HybridConnectivity`, and `Microsoft.GuestConfiguration` registered before the run.

hack/e2e/e2e_scripts_test.go

@@ -13,7 +13,7 @@ import (
 
 // Embedding the scripts makes Go's test cache invalidate on shell-only changes.
 //
-//go:embed run.sh lib/common.sh lib/cleanup.sh lib/controller.sh lib/node-join-arc.sh lib/runner.sh infra/*.bicep infra/modules/*.bicep
+//go:embed run.sh lib/bootstrap-rbac-migration.sh lib/common.sh lib/cleanup.sh lib/controller.sh lib/node-join-arc.sh lib/runner.sh infra/*.bicep infra/modules/*.bicep
 var e2eScripts embed.FS
 
 func TestRunnerCleanupIsScopedToCurrentAttempt(t *testing.T) {
@@ -1581,6 +1581,81 @@ func TestCleanupQueryFailureDoesNotDeleteResources(t *testing.T) {
 	}
 }
 
+func TestHistoricalCertificateProbeUsesPrivilegedTemporaryFile(t *testing.T) {
+	t.Parallel()
+
+	script, err := e2eScripts.ReadFile("lib/bootstrap-rbac-migration.sh")
+	if err != nil {
+		t.Fatalf("read embedded migration script: %v", err)
+	}
+	for _, required := range []string{
+		`ca_file="$(sudo mktemp)"`,
+		`sudo python3 <<'PY' | sudo tee "${ca_file}" >/dev/null`,
+		`trap 'sudo rm -f "${ca_file}"' EXIT`,
+		`configured = json.load(stream).get('agent', {}).get('nodeName', '')`,
+		`"${SERVER_URL}/api/v1/nodes/${node_name}"`,
+	} {
+		if !strings.Contains(string(script), required) {
+			t.Fatalf("migration certificate probe is missing %q", required)
+		}
+	}
+}
+
+func TestHistoricalMigrationReissuesDaemonCertificateBeforeTokenRevocation(t *testing.T) {
+	t.Parallel()
+
+	script, err := e2eScripts.ReadFile("lib/bootstrap-rbac-migration.sh")
+	if err != nil {
+		t.Fatalf("read embedded migration script: %v", err)
+	}
+	text := string(script)
+	for _, required := range []string{
+		`old_fingerprint="$(openssl x509 -in "${credential_path}" -outform DER`,
+		`rm -rf -- "${credential_dir}"`,
+		`new_fingerprint="$(openssl x509 -in "${credential_path}" -outform DER`,
+		`_require_daemon_certificate_access "${vm_ip}" "${server_url}"`,
+	} {
+		if !strings.Contains(text, required) {
+			t.Fatalf("historical migration is missing certificate reissuance check %q", required)
+		}
+	}
+
+	migrationIndex := strings.LastIndex(text, `    --remove-legacy-node-role-binding`)
+	reissueIndex := strings.LastIndex(text, `  _reissue_daemon_certificate_after_migration "${vm_ip}" "${server_url}"`)
+	revokeIndex := strings.LastIndex(text, `  with_cluster_lock _revoke_historical_bootstrap_token "${config_file}"`)
+	if migrationIndex < 0 || reissueIndex <= migrationIndex || revokeIndex <= reissueIndex {
+		t.Fatalf("certificate reissuance must run after RBAC migration and before token revocation")
+	}
+}
+
+func TestHistoricalTokenRevocationPropagatesDeleteFailure(t *testing.T) {
+	t.Parallel()
+
+	script, err := e2eScripts.ReadFile("lib/bootstrap-rbac-migration.sh")
+	if err != nil {
+		t.Fatalf("read embedded migration script: %v", err)
+	}
+	text := string(script)
+	start := strings.Index(text, "_revoke_historical_bootstrap_token() {")
+	if start < 0 {
+		t.Fatal("historical token revocation helper is absent")
+	}
+	end := strings.Index(text[start:], "\n}\n")
+	if end < 0 {
+		t.Fatal("historical token revocation helper is malformed")
+	}
+	body := text[start : start+end]
+	for _, required := range []string{
+		`if ! kubectl delete secret "bootstrap-token-${token_id}" -n kube-system; then`,
+		`log_error "Failed to revoke the historical bootstrap token"`,
+		`return 1`,
+	} {
+		if !strings.Contains(body, required) {
+			t.Fatalf("historical token revocation does not fail closed on delete errors; missing %q", required)
+		}
+	}
+}
+
 type cleanupOptions struct {
 	runTwice                  bool
 	leaveCluster              bool
@@ -1825,7 +1900,7 @@ func boolString(value bool) string {
 func e2eScriptPath(t *testing.T, elements ...string) string {
 	t.Helper()
 	root := t.TempDir()
-	for _, name := range []string{"common.sh", "cleanup.sh", "controller.sh", "node-join-arc.sh", "runner.sh"} {
+	for _, name := range []string{"bootstrap-rbac-migration.sh", "common.sh", "cleanup.sh", "controller.sh", "node-join-arc.sh", "runner.sh"} {
 		contents, err := e2eScripts.ReadFile(filepath.ToSlash(filepath.Join("lib", name)))
 		if err != nil {
 			t.Fatalf("read embedded %s: %v", name, err)