diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 2ed751fc..00b5254d 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -9,6 +9,14 @@ name: E2E Tests on: workflow_dispatch: inputs: + suite: + description: "E2E suite to run" + required: false + default: all + type: choice + options: + - all + - historical-rbac-migration kubernetes_version: description: "Exact AKS/Flex Node Kubernetes patch version (for example, 1.34.9)" required: false @@ -63,6 +71,7 @@ env: GITHUB_RUN_ID: ${{ github.run_id }} E2E_NAME_SUFFIX: ${{ github.run_id }}-${{ github.run_attempt }} E2E_WORK_DIR: /tmp/aks-flex-node-e2e-${{ github.run_id }}-${{ github.run_attempt }} + E2E_SUITE: ${{ inputs.suite || 'all' }} E2E_KUBERNETES_VERSION: ${{ inputs.kubernetes_version || '1.35.0' }} jobs: @@ -102,7 +111,7 @@ jobs: E2E_SKIP_CLEANUP: "1" # Cleanup handled in dedicated step below run: | set -euo pipefail - ./hack/e2e/run.sh all + ./hack/e2e/run.sh "${E2E_SUITE}" - name: Collect logs if: always() diff --git a/hack/e2e/README.md b/hack/e2e/README.md index 7daee1f8..706e1593 100644 --- a/hack/e2e/README.md +++ b/hack/e2e/README.md @@ -12,6 +12,7 @@ The E2E suite provisions a no-CNI AKS cluster, installs Unbounded-Net as the clu | `python3` | Local registry port readiness checks and helper scripts. | | `ssh` / `scp` | VM access and artifact copy. | | `openssl` | Bootstrap token generation. | +| `curl` / `sha256sum` / `tar` | Download and verify pinned historical release artifacts. | | `flock` | Serialize atomic updates to the per-run cleanup state. | | `docker` | Build and push the controller image into the in-cluster local registry. | | `git` / `make` | Fetch and render Unbounded-Net manifests. | @@ -66,6 +67,7 @@ The default `all` command runs: | Command | Description | |---------|-------------| | `all` | Full flow: build, infra, join, validate, unjoin, validate absent, rejoin, validate, lifecycle, agent upgrade, repave, logs, cleanup. | +| `historical-rbac-migration` | On a fresh real AKS cluster and token VM, join with the official v0.1.0 helper/binary, upgrade that host to HEAD, migrate legacy bootstrap RBAC, revoke the old token, and validate restarts. It does not run the other join modes. | | `infra` | Deploy AKS, four standard VMs, the Arc VM, Unbounded-Net CNI, the local registry, and the in-cluster controller. | | `join` | Join all Flex Node VMs. | | `join-msi` | Join only the managed-identity node. | @@ -141,6 +143,52 @@ Additional environment variables: | `AZURE_SUBSCRIPTION_ID` | auto-detected | Azure subscription. | | `AZURE_TENANT_ID` | auto-detected | Azure tenant. | +## Historical RBAC Migration Validation + +Run the focused compatibility suite with: + +```bash +./hack/e2e/run.sh historical-rbac-migration +``` + +For a manual GitHub Actions run, select `historical-rbac-migration` in the +`suite` workflow input. Set `kubernetes_version` to an exact AKS version, such +as the latest supported N-1 patch, to exercise the migration against an older +control-plane version. Check regional availability immediately before running +the workflow with `az aks get-versions --location --output table`. +Infrastructure provisioning, the test, log upload, and cleanup stay in the same +job; the suite deliberately does not call the Arc-inclusive parallel join path. + +The scenario downloads and verifies the official v0.1.0 release archive, +extracted binary, helper, and installer. It then uses the pinned helper and +binary to create the original broad bootstrap RBAC, a non-expiring token, the +legacy config shape, and a real Ready node on the token VM. The historical +daemon runs its production no-op path, not the v0.1.0 file-backed E2E machine +client. On that same host it verifies the HEAD helper fails closed without the +explicit migration flag, activates the HEAD binary through `agent-upgrade`, +removes the legacy binding twice to prove idempotency, checks token access +changes from HTTP 200 to 403, deletes the daemon credential store and verifies +the remaining CSR permissions issue a different certificate, revokes the token +and waits for HTTP 401, and restarts both kubelet and the daemon while checking +the Node UID, Lease, readiness, and certificate-backed API access. +v0.1.0 transitively pins the non-GPU rootfs +`ghcr.io/azure/agent-ubuntu2404:v20260427`. + +There are two intentional compatibility boundaries: + +- The test creates a new AKS control plane and reproduces the v0.1.0 + cluster-side state. It validates a historical node/config/RBAC migration, not + an AKS control plane that has itself been retained since v0.1.0. Selecting an + older `kubernetes_version` proves a newly created control plane at that + version; it still does not reproduce age, prior upgrades, or configuration + drift from a long-lived cluster. +- v0.1.0 tokens lack the `kubernetes.azure.com/managedby=aks` label required by + the production managed CSR approver. The suite explicitly adopts its known + token with that label before the HEAD daemon requests a certificate. The + repository E2E approver does not enforce this label, so this test validates + the host/config/RBAC migration but is not independent proof of the production + approver's ownership check. + ## Join Modes The suite validates five join paths. The E2E subscription must have `Microsoft.HybridCompute`, `Microsoft.HybridConnectivity`, and `Microsoft.GuestConfiguration` registered before the run. diff --git a/hack/e2e/e2e_scripts_test.go b/hack/e2e/e2e_scripts_test.go index 03f65f4c..e31ea28a 100644 --- a/hack/e2e/e2e_scripts_test.go +++ b/hack/e2e/e2e_scripts_test.go @@ -13,7 +13,7 @@ import ( // Embedding the scripts makes Go's test cache invalidate on shell-only changes. // -//go:embed run.sh lib/common.sh lib/cleanup.sh lib/controller.sh lib/node-join-arc.sh lib/runner.sh infra/*.bicep infra/modules/*.bicep +//go:embed run.sh lib/bootstrap-rbac-migration.sh lib/common.sh lib/cleanup.sh lib/controller.sh lib/node-join-arc.sh lib/runner.sh infra/*.bicep infra/modules/*.bicep var e2eScripts embed.FS func TestRunnerCleanupIsScopedToCurrentAttempt(t *testing.T) { @@ -1581,6 +1581,81 @@ func TestCleanupQueryFailureDoesNotDeleteResources(t *testing.T) { } } +func TestHistoricalCertificateProbeUsesPrivilegedTemporaryFile(t *testing.T) { + t.Parallel() + + script, err := e2eScripts.ReadFile("lib/bootstrap-rbac-migration.sh") + if err != nil { + t.Fatalf("read embedded migration script: %v", err) + } + for _, required := range []string{ + `ca_file="$(sudo mktemp)"`, + `sudo python3 <<'PY' | sudo tee "${ca_file}" >/dev/null`, + `trap 'sudo rm -f "${ca_file}"' EXIT`, + `configured = json.load(stream).get('agent', {}).get('nodeName', '')`, + `"${SERVER_URL}/api/v1/nodes/${node_name}"`, + } { + if !strings.Contains(string(script), required) { + t.Fatalf("migration certificate probe is missing %q", required) + } + } +} + +func TestHistoricalMigrationReissuesDaemonCertificateBeforeTokenRevocation(t *testing.T) { + t.Parallel() + + script, err := e2eScripts.ReadFile("lib/bootstrap-rbac-migration.sh") + if err != nil { + t.Fatalf("read embedded migration script: %v", err) + } + text := string(script) + for _, required := range []string{ + `old_fingerprint="$(openssl x509 -in "${credential_path}" -outform DER`, + `rm -rf -- "${credential_dir}"`, + `new_fingerprint="$(openssl x509 -in "${credential_path}" -outform DER`, + `_require_daemon_certificate_access "${vm_ip}" "${server_url}"`, + } { + if !strings.Contains(text, required) { + t.Fatalf("historical migration is missing certificate reissuance check %q", required) + } + } + + migrationIndex := strings.LastIndex(text, ` --remove-legacy-node-role-binding`) + reissueIndex := strings.LastIndex(text, ` _reissue_daemon_certificate_after_migration "${vm_ip}" "${server_url}"`) + revokeIndex := strings.LastIndex(text, ` with_cluster_lock _revoke_historical_bootstrap_token "${config_file}"`) + if migrationIndex < 0 || reissueIndex <= migrationIndex || revokeIndex <= reissueIndex { + t.Fatalf("certificate reissuance must run after RBAC migration and before token revocation") + } +} + +func TestHistoricalTokenRevocationPropagatesDeleteFailure(t *testing.T) { + t.Parallel() + + script, err := e2eScripts.ReadFile("lib/bootstrap-rbac-migration.sh") + if err != nil { + t.Fatalf("read embedded migration script: %v", err) + } + text := string(script) + start := strings.Index(text, "_revoke_historical_bootstrap_token() {") + if start < 0 { + t.Fatal("historical token revocation helper is absent") + } + end := strings.Index(text[start:], "\n}\n") + if end < 0 { + t.Fatal("historical token revocation helper is malformed") + } + body := text[start : start+end] + for _, required := range []string{ + `if ! kubectl delete secret "bootstrap-token-${token_id}" -n kube-system; then`, + `log_error "Failed to revoke the historical bootstrap token"`, + `return 1`, + } { + if !strings.Contains(body, required) { + t.Fatalf("historical token revocation does not fail closed on delete errors; missing %q", required) + } + } +} + type cleanupOptions struct { runTwice bool leaveCluster bool @@ -1825,7 +1900,7 @@ func boolString(value bool) string { func e2eScriptPath(t *testing.T, elements ...string) string { t.Helper() root := t.TempDir() - for _, name := range []string{"common.sh", "cleanup.sh", "controller.sh", "node-join-arc.sh", "runner.sh"} { + for _, name := range []string{"bootstrap-rbac-migration.sh", "common.sh", "cleanup.sh", "controller.sh", "node-join-arc.sh", "runner.sh"} { contents, err := e2eScripts.ReadFile(filepath.ToSlash(filepath.Join("lib", name))) if err != nil { t.Fatalf("read embedded %s: %v", name, err) diff --git a/hack/e2e/lib/bootstrap-rbac-migration.sh b/hack/e2e/lib/bootstrap-rbac-migration.sh new file mode 100644 index 00000000..c8d1934f --- /dev/null +++ b/hack/e2e/lib/bootstrap-rbac-migration.sh @@ -0,0 +1,944 @@ +#!/usr/bin/env bash +# ============================================================================= +# Real-node migration test for the legacy bootstrap-group system:node binding. +# ============================================================================= +set -euo pipefail + +[[ -n "${_E2E_BOOTSTRAP_RBAC_MIGRATION_LOADED:-}" ]] && return 0 +readonly _E2E_BOOTSTRAP_RBAC_MIGRATION_LOADED=1 + +# shellcheck disable=SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/common.sh" + +readonly historicalReleaseTag="v0.1.0" +readonly historicalCommit="65d8d3896371adf2eb13248c3e73f0e83fd418ef" +readonly historicalArchiveName="aks-flex-node-linux-amd64.tar.gz" +readonly historicalBinaryName="aks-flex-node-linux-amd64" +readonly historicalArchiveURL="https://github.com/Azure/AKSFlexNode/releases/download/${historicalReleaseTag}/${historicalArchiveName}" +readonly historicalHelperURL="https://raw.githubusercontent.com/Azure/AKSFlexNode/${historicalCommit}/scripts/aks-flex-config" +readonly historicalInstallerURL="https://raw.githubusercontent.com/Azure/AKSFlexNode/${historicalCommit}/scripts/install.sh" +readonly historicalArchiveSHA256="50922e15999b2fd9c19a298c3c3d9cb5a0a375858258dc42b92699a4427876e6" +readonly historicalBinarySHA256="50b4a62daeb30e635cc8ea5e18c6a204e54510e164eba75fe0322a8a8d56fdc7" +readonly historicalHelperSHA256="8ae38209e1a63f1b3c9d1fb41423644de5de032c3a978cedef100689fe1f19f7" +readonly historicalInstallerSHA256="c2f7cfc92e62c3a9fb96697b3a3ca170bb8c28d11af8f45d0f26f6fce016e9b4" +# v0.1.0 embeds Unbounded v0.1.8, whose non-GPU default resolves to this +# immutable release tag. The old Flex config schema cannot override OCIImage. +readonly historicalRootFS="ghcr.io/azure/agent-ubuntu2404:v20260427" +readonly legacyNodeRoleBinding="aks-flex-node-role" +readonly flexNodeBootstrapGroup="system:bootstrappers:aks-flex-node" +# client-go's rotating FileStore writes the issued certificate and private key +# into this combined PEM. client.crt/client.key are legacy read-only fallbacks. +readonly daemonCredentialPath="/etc/aks-flex-node/daemon-credentials/daemon-controller-current.pem" + +_historical_artifact_dir() { + echo "${E2E_WORK_DIR}/historical-${historicalReleaseTag}" +} + +_verify_sha256() { + local path="$1" + local expected="$2" + local description="$3" + local actual + + actual="$(sha256sum "${path}" | awk '{print $1}')" + if [[ "${actual}" != "${expected}" ]]; then + log_error "${description} SHA-256 mismatch: got ${actual}, want ${expected}" + return 1 + fi +} + +_download_verified_artifact() { + local url="$1" + local path="$2" + local expected="$3" + local description="$4" + local partial="${path}.download" + + if [[ -f "${path}" ]] && _verify_sha256 "${path}" "${expected}" "${description}"; then + log_info "Using cached, verified ${description}: ${path}" + return 0 + fi + + rm -f "${path}" "${partial}" + log_info "Downloading pinned ${description}" + if ! curl --fail --location --proto '=https' --retry 5 --retry-all-errors \ + --silent --show-error --output "${partial}" "${url}"; then + rm -f "${partial}" + return 1 + fi + if ! _verify_sha256 "${partial}" "${expected}" "${description}"; then + rm -f "${partial}" + return 1 + fi + mv "${partial}" "${path}" +} + +_prepare_historical_artifacts() { + require_cmd curl + require_cmd sha256sum + require_cmd tar + + local artifact_dir archive helper installer binary archive_members version_output + artifact_dir="$(_historical_artifact_dir)" + archive="${artifact_dir}/${historicalArchiveName}" + helper="${artifact_dir}/aks-flex-config" + installer="${artifact_dir}/install.sh" + binary="${artifact_dir}/${historicalBinaryName}" + mkdir -p "${artifact_dir}" + + _download_verified_artifact \ + "${historicalArchiveURL}" "${archive}" "${historicalArchiveSHA256}" \ + "${historicalReleaseTag} release archive" + _download_verified_artifact \ + "${historicalHelperURL}" "${helper}" "${historicalHelperSHA256}" \ + "${historicalReleaseTag} aks-flex-config helper" + _download_verified_artifact \ + "${historicalInstallerURL}" "${installer}" "${historicalInstallerSHA256}" \ + "${historicalReleaseTag} installer" + + archive_members="$(tar -tzf "${archive}")" + if [[ "${archive_members}" != "${historicalBinaryName}" ]]; then + log_error "${historicalReleaseTag} archive has unexpected members: ${archive_members}" + return 1 + fi + rm -f "${binary}" + tar --extract --gzip --file "${archive}" --directory "${artifact_dir}" \ + --no-same-owner --no-same-permissions "${historicalBinaryName}" + _verify_sha256 "${binary}" "${historicalBinarySHA256}" \ + "${historicalReleaseTag} extracted binary" + chmod 0755 "${binary}" "${helper}" "${installer}" + + version_output="$("${binary}" version)" + if [[ "${version_output}" != *"Version: ${historicalReleaseTag}"* || \ + "${version_output}" != *"Git Commit: ${historicalCommit:0:7}"* ]]; then + log_error "Pinned historical binary reports unexpected build metadata: ${version_output}" + return 1 + fi + + log_success "Verified official ${historicalReleaseTag} archive, binary, helper, and installer" + log_info "Historical binary's pinned default rootfs: ${historicalRootFS}" +} + +_historical_config_path() { + echo "${E2E_WORK_DIR}/config-token-${historicalReleaseTag}.json" +} + +_head_legacy_config_path() { + echo "${E2E_WORK_DIR}/config-token-${historicalReleaseTag}-head.json" +} + +_historical_token_id() { + local config_file="$1" + jq -er \ + '.azure.bootstrapToken.token | capture("^(?[a-z0-9]{6})\\.[a-z0-9]{16}$").id' \ + "${config_file}" +} + +_generate_historical_config() { + local artifact_dir helper config_file + local vm_name vm_private_ip cluster_name resource_group subscription_id + artifact_dir="$(_historical_artifact_dir)" + helper="${artifact_dir}/aks-flex-config" + config_file="$(_historical_config_path)" + vm_name="$(state_get token_vm_name)" + vm_private_ip="$(state_get token_vm_private_ip)" + cluster_name="$(state_get cluster_name)" + resource_group="$(state_get resource_group)" + subscription_id="$(state_get subscription_id)" + + if [[ -z "${vm_private_ip}" ]] || ! is_valid_ipv4 "${vm_private_ip}"; then + log_error "Invalid token VM private IP in state: '${vm_private_ip}'" + return 1 + fi + if kubectl get clusterrolebinding "${legacyNodeRoleBinding}" >/dev/null 2>&1; then + log_error "Historical scenario requires a fresh cluster without ${legacyNodeRoleBinding}" + return 1 + fi + + log_info "Applying the real ${historicalReleaseTag} cluster-side RBAC" + with_cluster_lock python3 "${helper}" setup-node-rbac \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" + + log_info "Generating a real non-expiring ${historicalReleaseTag} bootstrap token and config" + # The v0.1.0 helper predates the current secure-output implementation. Create + # the destination first so the bootstrap token is never briefly world-readable + # under a permissive runner umask. + install -m 0600 /dev/null "${config_file}" + with_cluster_lock python3 "${helper}" generate-node-config \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" \ + --bootstrap-token \ + --output "${config_file}" + + install -m 0600 /dev/null "${config_file}.tmp" + jq \ + --arg nodeName "${vm_name}" \ + --arg nodeIP "${vm_private_ip}" \ + --arg kubernetesVersion "${E2E_KUBERNETES_VERSION}" \ + --arg containerdVersion "${E2E_CONTAINERD_VERSION}" \ + --arg runcVersion "${E2E_RUNC_VERSION}" \ + '.agent.logLevel = "debug" + | .agent.nodeName = $nodeName + | .node.kubelet.nodeIP = $nodeIP + | .kubernetes.version = $kubernetesVersion + | .containerd.version = $containerdVersion + | .runc.version = $runcVersion' \ + "${config_file}" > "${config_file}.tmp" + mv "${config_file}.tmp" "${config_file}" + chmod 0600 "${config_file}" + + if ! jq -e \ + --arg nodeName "${vm_name}" \ + --arg nodeIP "${vm_private_ip}" \ + '.agent.nodeName == $nodeName + and (.agent | has("e2eMode") | not) + and (.agent | has("machineOperationMode") | not) + and .node.kubelet.nodeIP == $nodeIP + and (.node.kubelet.serverURL | length > 0) + and (.node.kubelet.caCertData | length > 0) + and (.kubernetes.version | length > 0) + and (has("components") | not)' \ + "${config_file}" >/dev/null; then + log_error "Historical helper did not produce the expected legacy config shape" + return 1 + fi +} + +_require_historical_cluster_state() { + local config_file="$1" + local token_id secret + token_id="$(_historical_token_id "${config_file}")" + + if ! kubectl get clusterrolebinding "${legacyNodeRoleBinding}" -o json | jq -e \ + --arg group "${flexNodeBootstrapGroup}" \ + '.roleRef.apiGroup == "rbac.authorization.k8s.io" + and .roleRef.kind == "ClusterRole" + and .roleRef.name == "system:node" + and any(.subjects[]?; + .apiGroup == "rbac.authorization.k8s.io" + and .kind == "Group" + and .name == $group)' >/dev/null; then + log_error "${historicalReleaseTag} helper did not create the expected legacy node-role binding" + return 1 + fi + + secret="$(kubectl -n kube-system get secret "bootstrap-token-${token_id}" -o json)" + if ! jq -e \ + '(.data.expiration // "") == "" + and (.metadata.labels["kubernetes.azure.com/managedby"] // "") == ""' \ + <<<"${secret}" >/dev/null; then + log_error "Historical bootstrap Secret unexpectedly has expiration or AKS ownership metadata" + return 1 + fi + log_success "Verified ${historicalReleaseTag} legacy RBAC and non-expiring, unmanaged token state" +} + +_install_and_start_historical_node() { + local vm_ip="$1" + local artifact_dir config_file + artifact_dir="$(_historical_artifact_dir)" + config_file="$(_historical_config_path)" + + remote_copy "${artifact_dir}/${historicalArchiveName}" "${vm_ip}" "/tmp/${historicalArchiveName}" + remote_copy "${artifact_dir}/aks-flex-config" "${vm_ip}" "/tmp/aks-flex-config-${historicalReleaseTag}" + remote_copy "${artifact_dir}/install.sh" "${vm_ip}" "/tmp/aks-flex-node-install-${historicalReleaseTag}.sh" + remote_copy "${config_file}" "${vm_ip}" "/tmp/config-${historicalReleaseTag}.json" + + remote_exec "${vm_ip}" \ + "HISTORICAL_TAG=${historicalReleaseTag} HISTORICAL_COMMIT=${historicalCommit:0:7} HISTORICAL_ARCHIVE_SHA256=${historicalArchiveSHA256} HISTORICAL_BINARY_SHA256=${historicalBinarySHA256} HISTORICAL_HELPER_SHA256=${historicalHelperSHA256} HISTORICAL_INSTALLER_SHA256=${historicalInstallerSHA256} E2E_NODE_JOIN_TIMEOUT=${E2E_NODE_JOIN_TIMEOUT} DAEMON_CREDENTIAL_PATH=${daemonCredentialPath} bash -s" <<'REMOTE' +set -euo pipefail + +archive=/tmp/aks-flex-node-linux-amd64.tar.gz +helper="/tmp/aks-flex-config-${HISTORICAL_TAG}" +installer="/tmp/aks-flex-node-install-${HISTORICAL_TAG}.sh" +config="/tmp/config-${HISTORICAL_TAG}.json" +extract_dir="/tmp/aks-flex-node-${HISTORICAL_TAG}" +binary="${extract_dir}/aks-flex-node-linux-amd64" + +printf '%s %s\n' "${HISTORICAL_ARCHIVE_SHA256}" "${archive}" | sha256sum --check --strict - +printf '%s %s\n' "${HISTORICAL_HELPER_SHA256}" "${helper}" | sha256sum --check --strict - +printf '%s %s\n' "${HISTORICAL_INSTALLER_SHA256}" "${installer}" | sha256sum --check --strict - + +archive_members="$(tar -tzf "${archive}")" +if [[ "${archive_members}" != "aks-flex-node-linux-amd64" ]]; then + echo "historical archive has unexpected members: ${archive_members}" >&2 + exit 1 +fi +sudo rm -rf "${extract_dir}" +mkdir -p "${extract_dir}" +tar --extract --gzip --file "${archive}" --directory "${extract_dir}" \ + --no-same-owner --no-same-permissions aks-flex-node-linux-amd64 +printf '%s %s\n' "${HISTORICAL_BINARY_SHA256}" "${binary}" | sha256sum --check --strict - +chmod 0755 "${binary}" "${helper}" "${installer}" + +version_output="$("${binary}" version)" +grep -Fq "Version: ${HISTORICAL_TAG}" <<<"${version_output}" +grep -Fq "Git Commit: ${HISTORICAL_COMMIT}" <<<"${version_output}" + +if sudo test -e /usr/local/lib/aks-flex-node/aks-flex-node-current || \ + sudo test -L /usr/local/lib/aks-flex-node/aks-flex-node-current; then + echo "historical test VM already has a managed agent layout" >&2 + exit 1 +fi + +if command -v apt-get >/dev/null 2>&1; then + packages_installed=0 + for attempt in $(seq 1 5); do + if sudo DEBIAN_FRONTEND=noninteractive apt-get \ + -o Acquire::Retries=5 -o Acquire::http::Timeout=30 update && + sudo DEBIAN_FRONTEND=noninteractive apt-get \ + -o Acquire::Retries=5 -o Acquire::http::Timeout=30 \ + install -y --fix-missing ca-certificates curl nftables systemd-container util-linux; then + packages_installed=1 + break + fi + echo "Host package installation failed; retrying (${attempt}/5)..." + sleep 10 + done + if (( packages_installed != 1 )); then + echo "Host package installation failed after retries" >&2 + exit 1 + fi +fi + +sudo AKS_FLEX_NODE_LOCAL_BINARY="${binary}" \ + AKS_FLEX_NODE_VERSION="${HISTORICAL_TAG}" \ + SKIP_AZCLI=true \ + bash "${installer}" --yes +sudo install -m 0600 "${config}" /etc/aks-flex-node/config.json + +if [[ -L /usr/local/bin/aks-flex-node ]]; then + echo "${HISTORICAL_TAG} installer unexpectedly created a managed compatibility symlink" >&2 + exit 1 +fi +printf '%s %s\n' "${HISTORICAL_BINARY_SHA256}" /usr/local/bin/aks-flex-node | sudo sha256sum --check --strict - +if sudo test -e "${DAEMON_CREDENTIAL_PATH}"; then + echo "daemon certificate existed before historical bootstrap" >&2 + exit 1 +fi + +unit=aks-flex-node-historical-bootstrap +sudo systemctl stop "${unit}.service" 2>/dev/null || true +sudo systemctl reset-failed "${unit}.service" 2>/dev/null || true +sudo systemd-run \ + --unit="${unit}" \ + --description="AKS Flex Node ${HISTORICAL_TAG} E2E" \ + --remain-after-exit \ + /usr/local/bin/aks-flex-node bootstrap --config /etc/aks-flex-node/config.json + +deadline=$((SECONDS + E2E_NODE_JOIN_TIMEOUT)) +while ! sudo systemctl is-active --quiet aks-flex-node-agent.service; do + if sudo systemctl is-failed --quiet "${unit}.service"; then + sudo systemctl status "${unit}.service" --no-pager -l >&2 || true + sudo journalctl -u "${unit}.service" -n 100 --no-pager >&2 || true + exit 1 + fi + if (( SECONDS >= deadline )); then + echo "timed out waiting for historical daemon service" >&2 + sudo systemctl status "${unit}.service" --no-pager -l >&2 || true + sudo systemctl status aks-flex-node-agent.service --no-pager -l >&2 || true + sudo journalctl -u "${unit}.service" -n 100 --no-pager >&2 || true + sudo journalctl -u aks-flex-node-agent.service -n 100 --no-pager >&2 || true + exit 1 + fi + sleep 5 +done + +sleep 5 +sudo systemctl is-active --quiet aks-flex-node-agent.service +sudo grep -Fq 'ExecStart=/usr/local/bin/aks-flex-node agent' \ + /etc/systemd/system/aks-flex-node-agent.service +historical_logs="$(sudo journalctl -u aks-flex-node-agent.service --no-pager)" +grep -Fq 'production agent daemon requires AKS RP machine client implementation' \ + <<<"${historical_logs}" +if grep -Fq 'running agent daemon in e2e mode' <<<"${historical_logs}"; then + echo "${HISTORICAL_TAG} daemon unexpectedly started in E2E mode" >&2 + exit 1 +fi +if sudo test -e "${DAEMON_CREDENTIAL_PATH}"; then + echo "${HISTORICAL_TAG} unexpectedly issued a daemon certificate" >&2 + exit 1 +fi +REMOTE +} + +_bootstrap_token_api_probe() { + local config_file="$1" + local probe="$2" + + python3 - "${config_file}" "${probe}" <<'PY' +import base64 +import json +import ssl +import sys +import urllib.error +import urllib.request + +config_path, probe = sys.argv[1:] +with open(config_path, encoding="utf-8") as stream: + config = json.load(stream) + +token = config["azure"]["bootstrapToken"]["token"] +kubelet = config["node"]["kubelet"] +server = kubelet.get("serverURL") +if not server: + cluster_fqdn = kubelet["clusterFQDN"] + server = cluster_fqdn if "://" in cluster_fqdn else "https://" + cluster_fqdn +ca_pem = base64.b64decode(kubelet["caCertData"], validate=True).decode("ascii") +context = ssl.create_default_context(cadata=ca_pem) +headers = {"Authorization": "Bearer " + token} + +if probe == "list-nodes": + request = urllib.request.Request(server.rstrip("/") + "/api/v1/nodes?limit=1", headers=headers) +elif probe == "create-csr": + body = json.dumps( + { + "apiVersion": "authorization.k8s.io/v1", + "kind": "SelfSubjectAccessReview", + "spec": { + "resourceAttributes": { + "group": "certificates.k8s.io", + "resource": "certificatesigningrequests", + "verb": "create", + } + }, + } + ).encode("utf-8") + request = urllib.request.Request( + server.rstrip("/") + "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews", + data=body, + headers={**headers, "Content-Type": "application/json"}, + method="POST", + ) +else: + raise SystemExit("unsupported probe: " + probe) + +try: + with urllib.request.urlopen(request, context=context, timeout=30) as response: + response_body = response.read() + status = response.status +except urllib.error.HTTPError as error: + print("http:" + str(error.code)) + raise SystemExit(0) + +if probe == "create-csr": + review = json.loads(response_body) + print("allowed" if review.get("status", {}).get("allowed") is True else "denied") +else: + print("http:" + str(status)) +PY +} + +_wait_for_bootstrap_token_probe() { + local expected="$1" + local config_file="$2" + local probe="$3" + local elapsed=0 + local actual="" + + while (( elapsed < E2E_NODE_JOIN_TIMEOUT )); do + if actual="$(_bootstrap_token_api_probe "${config_file}" "${probe}")" && \ + [[ "${actual}" == "${expected}" ]]; then + return 0 + fi + sleep 2 + elapsed=$((elapsed + 2)) + done + + log_error "Bootstrap token probe ${probe} returned '${actual}', want '${expected}' after ${E2E_NODE_JOIN_TIMEOUT}s" + return 1 +} + +_kubelet_client_identity() { + local vm_ip="$1" + + remote_exec "${vm_ip}" 'sudo bash -s' <<'REMOTE' +set -euo pipefail +machine="$(python3 - <<'PY' +import json +with open('/etc/aks-flex-node/daemon-state.json', encoding='utf-8') as stream: + print(json.load(stream)['activeMachine']) +PY +)" +systemd-run --machine="${machine}" --quiet --pipe --wait \ + openssl x509 \ + -in /var/lib/kubelet/pki/kubelet-client-current.pem \ + -noout \ + -subject \ + -nameopt RFC2253 +REMOTE +} + +_daemon_client_identity() { + local vm_ip="$1" + remote_exec "${vm_ip}" \ + "sudo openssl x509 -in ${daemonCredentialPath} -noout -subject -nameopt RFC2253" +} + +_require_daemon_certificate_access() { + local vm_ip="$1" + local server_url="$2" + local quoted_server quoted_credential + printf -v quoted_server '%q' "${server_url}" + printf -v quoted_credential '%q' "${daemonCredentialPath}" + + remote_exec "${vm_ip}" \ + "SERVER_URL=${quoted_server} DAEMON_CREDENTIAL_PATH=${quoted_credential} bash -s" <<'REMOTE' +set -euo pipefail +# The credential config is root-only. Keep the derived CA root-only as well and +# write it through a privileged process; redirecting sudo's stdout to a regular +# user's mktemp file fails on hardened hosts. +ca_file="$(sudo mktemp)" +trap 'sudo rm -f "${ca_file}"' EXIT +sudo python3 <<'PY' | sudo tee "${ca_file}" >/dev/null +import base64 +import json +import sys + +with open('/etc/aks-flex-node/config.json', encoding='utf-8') as stream: + ca_data = json.load(stream)['node']['kubelet']['caCertData'] +sys.stdout.buffer.write(base64.b64decode(ca_data, validate=True)) +PY +# Match the agent's resolved identity and avoid depending on feature-specific +# cluster-wide Node list permission for this baseline certificate check. +node_name="$(sudo python3 <<'PY' +import json +import socket + +with open('/etc/aks-flex-node/config.json', encoding='utf-8') as stream: + configured = json.load(stream).get('agent', {}).get('nodeName', '') +node_name = configured.strip() if isinstance(configured, str) else '' +print(node_name or socket.gethostname().strip().lower()) +PY +)" +if [[ -z "${node_name}" ]]; then + echo "could not resolve the daemon Node name" >&2 + exit 1 +fi +status="$(sudo curl --silent --show-error \ + --cert "${DAEMON_CREDENTIAL_PATH}" \ + --key "${DAEMON_CREDENTIAL_PATH}" \ + --cacert "${ca_file}" \ + --output /dev/null \ + --write-out '%{http_code}' \ + "${SERVER_URL}/api/v1/nodes/${node_name}")" +if [[ "${status}" != "200" ]]; then + echo "daemon client certificate GET Node ${node_name} returned HTTP ${status}, want 200" >&2 + exit 1 +fi +REMOTE +} + +_reissue_daemon_certificate_after_migration() { + local vm_ip="$1" + local server_url="$2" + + remote_exec "${vm_ip}" 'sudo bash -s' <<'REMOTE' +set -euo pipefail +readonly credential_dir="/etc/aks-flex-node/daemon-credentials" +readonly credential_path="${credential_dir}/daemon-controller-current.pem" +readonly service_name="aks-flex-node-agent.service" + +if [[ ! -s "${credential_path}" ]]; then + echo "daemon certificate is missing before forced reissuance" >&2 + exit 1 +fi +old_fingerprint="$(openssl x509 -in "${credential_path}" -outform DER | sha256sum | awk '{print $1}')" + +# Stop the process before deleting its test credential store so the restart +# must authenticate with the still-valid bootstrap token and submit a new CSR. +systemctl stop "${service_name}" +rm -rf -- "${credential_dir}" +systemctl start "${service_name}" + +for _ in $(seq 1 60); do + if systemctl is-active --quiet "${service_name}" && [[ -s "${credential_path}" ]]; then + new_fingerprint="$(openssl x509 -in "${credential_path}" -outform DER | sha256sum | awk '{print $1}')" + if [[ -n "${new_fingerprint}" && "${new_fingerprint}" != "${old_fingerprint}" ]]; then + exit 0 + fi + fi + if systemctl is-failed --quiet "${service_name}"; then + break + fi + sleep 2 +done + +echo "daemon did not obtain a different certificate after legacy RBAC removal" >&2 +systemctl status "${service_name}" --no-pager >&2 || true +journalctl -u "${service_name}" -n 100 --no-pager >&2 || true +exit 1 +REMOTE + + _require_daemon_certificate_access "${vm_ip}" "${server_url}" + log_success "Daemon obtained a new certificate through least-privilege CSR RBAC after legacy binding removal" +} + +_require_old_node_survives_guard() { + local vm_name="$1" + local vm_ip="$2" + + remote_exec "${vm_ip}" \ + "DAEMON_CREDENTIAL_PATH=${daemonCredentialPath} bash -s" <<'REMOTE' +set -euo pipefail +sudo systemctl restart aks-flex-node-agent.service +for _ in $(seq 1 30); do + if sudo systemctl is-active --quiet aks-flex-node-agent.service; then + first_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" + sleep 5 + second_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" + if sudo systemctl is-active --quiet aks-flex-node-agent.service && + [[ "${first_pid}" =~ ^[1-9][0-9]*$ ]] && [[ "${second_pid}" == "${first_pid}" ]]; then + break + fi + fi + sleep 2 +done +if ! sudo systemctl is-active --quiet aks-flex-node-agent.service || + [[ "${second_pid:-}" != "${first_pid:-}" ]]; then + sudo systemctl status aks-flex-node-agent.service --no-pager -l >&2 || true + exit 1 +fi +if sudo test -e "${DAEMON_CREDENTIAL_PATH}"; then + echo "historical daemon unexpectedly created a daemon certificate" >&2 + exit 1 +fi +REMOTE + validate_node_joined "${vm_name}" +} + +_prepare_head_legacy_config() { + local source_config="$1" + local target_config="$2" + + install -m 0600 /dev/null "${target_config}.tmp" + jq \ + --arg machineEndpointURL "${E2E_CONTROLLER_SERVICE_PROXY_PATH}" \ + --arg agentPoolName "${E2E_TARGET_AGENT_POOL_NAME}" \ + --arg ociImage "${historicalRootFS}" \ + '.agent.machineClient.mode = "in-cluster" + | .agent.machineClient.endpointUrl = $machineEndpointURL + | .agent.requireMachineRegistration = true + | .agent.machineOperationMode = "disable" + | .azure.targetAgentPoolName = $agentPoolName + | .bootstrap.ociImage = $ociImage' \ + "${source_config}" > "${target_config}.tmp" + mv "${target_config}.tmp" "${target_config}" + + if ! jq -e \ + --arg endpoint "${E2E_CONTROLLER_SERVICE_PROXY_PATH}" \ + --arg ociImage "${historicalRootFS}" \ + '.agent.machineClient.mode == "in-cluster" + and .agent.machineClient.endpointUrl == $endpoint + and .agent.requireMachineRegistration == true + and .agent.machineOperationMode == "disable" + and .bootstrap.ociImage == $ociImage + and (.node.kubelet.serverURL | length > 0) + and (.kubernetes.version | length > 0) + and (has("components") | not)' \ + "${target_config}" >/dev/null; then + log_error "HEAD upgrade config no longer preserves the historical config fields" + return 1 + fi +} + +_upgrade_historical_node_to_head() { + local vm_ip="$1" + local config_file="$2" + local head_sha + head_sha="$(sha256sum "${E2E_BINARY}" | awk '{print $1}')" + + remote_copy "${E2E_BINARY}" "${vm_ip}" /tmp/aks-flex-node-head + remote_copy "${config_file}" "${vm_ip}" /tmp/config-v0.1.0-head.json + + remote_exec "${vm_ip}" \ + "HISTORICAL_BINARY_SHA256=${historicalBinarySHA256} HEAD_BINARY_SHA256=${head_sha} E2E_NODE_JOIN_TIMEOUT=${E2E_NODE_JOIN_TIMEOUT} DAEMON_CREDENTIAL_PATH=${daemonCredentialPath} bash -s" <<'REMOTE' +set -euo pipefail +candidate=/tmp/aks-flex-node-head +current_link=/usr/local/lib/aks-flex-node/aks-flex-node-current +last_good_link=/usr/local/lib/aks-flex-node/aks-flex-node-last-good +service=/etc/systemd/system/aks-flex-node-agent.service + +chmod 0755 "${candidate}" +printf '%s %s\n' "${HEAD_BINARY_SHA256}" "${candidate}" | sha256sum --check --strict - +printf '%s %s\n' "${HISTORICAL_BINARY_SHA256}" /usr/local/bin/aks-flex-node | sudo sha256sum --check --strict - +if sudo test -e "${current_link}" || sudo test -L "${current_link}"; then + echo "managed layout existed before migration preflight" >&2 + exit 1 +fi + +sudo install -m 0600 /tmp/config-v0.1.0-head.json /etc/aks-flex-node/config.json +sudo "${candidate}" agent-upgrade --preflight | sudo tee /tmp/historical-agent-upgrade-preflight.log + +# Preflight must not mutate the direct v0.1.0 installation. +if sudo test -e "${current_link}" || sudo test -L "${current_link}" || \ + sudo test -L /usr/local/bin/aks-flex-node; then + echo "agent-upgrade preflight mutated the legacy binary layout" >&2 + exit 1 +fi +printf '%s %s\n' "${HISTORICAL_BINARY_SHA256}" /usr/local/bin/aks-flex-node | sudo sha256sum --check --strict - +sudo systemctl is-active --quiet aks-flex-node-agent.service + +sudo "${candidate}" agent-upgrade | sudo tee /tmp/historical-agent-upgrade.log + +deadline=$((SECONDS + E2E_NODE_JOIN_TIMEOUT)) +while true; do + if sudo systemctl is-active --quiet aks-flex-node-agent.service && + sudo test -s "${DAEMON_CREDENTIAL_PATH}"; then + first_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" + sleep 5 + second_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" + if sudo systemctl is-active --quiet aks-flex-node-agent.service && + [[ "${first_pid}" =~ ^[1-9][0-9]*$ ]] && [[ "${second_pid}" == "${first_pid}" ]]; then + break + fi + fi + if (( SECONDS >= deadline )); then + echo "HEAD daemon did not become stable with daemon credentials" >&2 + sudo systemctl status aks-flex-node-agent.service --no-pager -l >&2 || true + sudo journalctl -u aks-flex-node-agent.service -n 150 --no-pager >&2 || true + exit 1 + fi + sleep 2 +done + +for link in /usr/local/bin/aks-flex-node "${current_link}" "${last_good_link}"; do + if ! sudo test -L "${link}"; then + echo "managed binary link missing after upgrade: ${link}" >&2 + exit 1 + fi +done +active="$(sudo readlink -f "${current_link}")" +last_good="$(sudo readlink -f "${last_good_link}")" +printf '%s %s\n' "${HEAD_BINARY_SHA256}" "${active}" | sudo sha256sum --check --strict - +printf '%s %s\n' "${HISTORICAL_BINARY_SHA256}" "${last_good}" | sudo sha256sum --check --strict - +if [[ "$(sudo readlink -f /usr/local/bin/aks-flex-node)" != "${active}" ]]; then + echo "compatibility path does not resolve to the active managed binary" >&2 + exit 1 +fi +sudo grep -Fq "ExecStart=${current_link} agent" "${service}" +pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" +if [[ "$(sudo readlink -f "/proc/${pid}/exe")" != "${active}" ]]; then + echo "daemon is not executing the activated HEAD binary" >&2 + exit 1 +fi + +cert_pub="$(sudo openssl x509 -in "${DAEMON_CREDENTIAL_PATH}" -pubkey -noout | sha256sum | awk '{print $1}')" +key_pub="$(sudo openssl pkey -in "${DAEMON_CREDENTIAL_PATH}" -pubout | sha256sum | awk '{print $1}')" +if [[ "${cert_pub}" != "${key_pub}" ]]; then + echo "daemon certificate and private key do not match" >&2 + exit 1 +fi +REMOTE +} + +_revoke_historical_bootstrap_token() { + local config_file="$1" + local token_id + token_id="$(_historical_token_id "${config_file}")" + if ! kubectl delete secret "bootstrap-token-${token_id}" -n kube-system; then + log_error "Failed to revoke the historical bootstrap token" + return 1 + fi + log_info "Revoked the historical bootstrap token after certificate migration" +} + +_restart_daemon_and_require_certificate_access() { + local vm_ip="$1" + local server_url="$2" + + remote_exec "${vm_ip}" 'bash -s' <<'REMOTE' +set -euo pipefail +sudo systemctl restart aks-flex-node-agent.service +for _ in $(seq 1 30); do + if sudo systemctl is-active --quiet aks-flex-node-agent.service; then + first_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" + sleep 5 + second_pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" + if sudo systemctl is-active --quiet aks-flex-node-agent.service && + [[ "${first_pid}" =~ ^[1-9][0-9]*$ ]] && [[ "${second_pid}" == "${first_pid}" ]]; then + exit 0 + fi + fi + sleep 2 +done +sudo systemctl status aks-flex-node-agent.service --no-pager -l >&2 || true +sudo journalctl -u aks-flex-node-agent.service -n 100 --no-pager >&2 || true +exit 1 +REMOTE + _require_daemon_certificate_access "${vm_ip}" "${server_url}" +} + +_restart_kubelet_and_require_lease_renewal() { + local vm_name="$1" + local vm_ip="$2" + local node_uid="$3" + local before_renew + local elapsed=0 + + before_renew="$(kubectl get lease "${vm_name}" -n kube-node-lease -o jsonpath='{.spec.renewTime}')" + remote_exec "${vm_ip}" 'sudo bash -s' <<'REMOTE' +set -euo pipefail +machine="$(python3 - <<'PY' +import json +with open('/etc/aks-flex-node/daemon-state.json', encoding='utf-8') as stream: + print(json.load(stream)['activeMachine']) +PY +)" +systemd-run --machine="${machine}" --quiet --pipe --wait systemctl restart kubelet.service +REMOTE + + while (( elapsed < E2E_NODE_JOIN_TIMEOUT )); do + local current_uid renew ready + current_uid="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.uid}' 2>/dev/null || true)" + renew="$(kubectl get lease "${vm_name}" -n kube-node-lease -o jsonpath='{.spec.renewTime}' 2>/dev/null || true)" + ready="$(kubectl get node "${vm_name}" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" + if [[ "${current_uid}" == "${node_uid}" && -n "${renew}" && \ + "${renew}" != "${before_renew}" && "${ready}" == "True" ]]; then + log_success "Historical node renewed its Lease and stayed Ready after token revocation" + return 0 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + + log_error "Historical node did not reauthenticate after token revocation" + kubectl get node "${vm_name}" -o wide 2>&1 || true + kubectl describe node "${vm_name}" 2>&1 || true + return 1 +} + +historical_rbac_migration_e2e() { + log_section "Historical ${historicalReleaseTag} Node and Bootstrap RBAC Migration" + + local config_file head_config vm_name vm_ip cluster_name resource_group subscription_id + local server_url node_uid node_kubelet_version identity guard_output token token_id + config_file="$(_historical_config_path)" + head_config="$(_head_legacy_config_path)" + vm_name="$(state_get token_vm_name)" + vm_ip="$(state_get token_vm_ip)" + cluster_name="$(state_get cluster_name)" + resource_group="$(state_get resource_group)" + subscription_id="$(state_get subscription_id)" + server_url="$(state_get server_url)" + + _prepare_historical_artifacts + _generate_historical_config + _require_historical_cluster_state "${config_file}" + _install_and_start_historical_node "${vm_ip}" + validate_node_joined "${vm_name}" + + node_kubelet_version="$(kubectl get node "${vm_name}" -o jsonpath='{.status.nodeInfo.kubeletVersion}')" + node_kubelet_version="${node_kubelet_version#v}" + if [[ "${node_kubelet_version}" != "${E2E_KUBERNETES_VERSION}" ]]; then + log_error "Historical node kubelet version is ${node_kubelet_version}, expected ${E2E_KUBERNETES_VERSION}" + return 1 + fi + node_uid="$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.uid}')" + identity="$(_kubelet_client_identity "${vm_ip}")" + if [[ "${identity}" != *"CN=system:node:${vm_name}"* ]]; then + log_error "${historicalReleaseTag} kubelet has unexpected client identity: ${identity}" + return 1 + fi + _wait_for_bootstrap_token_probe http:200 "${config_file}" list-nodes + log_success "Official ${historicalReleaseTag} node is Ready on Kubernetes ${node_kubelet_version} with legacy RBAC and an issued kubelet certificate" + + # HEAD must converge the safe CSR bindings but preserve the old daemon until + # an operator explicitly confirms the migration. + if guard_output="$(with_cluster_lock "${REPO_ROOT}/scripts/aks-flex-config" setup-node-rbac \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" 2>&1)"; then + log_error "HEAD RBAC setup accepted a legacy binding without explicit migration" + return 1 + fi + if [[ "${guard_output}" != *"--remove-legacy-node-role-binding"* ]]; then + log_error "HEAD compatibility guard did not explain the explicit migration path" + return 1 + fi + if ! kubectl get clusterrolebinding "${legacyNodeRoleBinding}" >/dev/null 2>&1; then + log_error "HEAD compatibility guard removed the legacy binding" + return 1 + fi + _wait_for_bootstrap_token_probe http:200 "${config_file}" list-nodes + _require_old_node_survives_guard "${vm_name}" "${vm_ip}" + log_success "HEAD fail-closed guard preserved the running historical node" + + # Helpers before v0.1.1 did not add the ownership label required by the + # production AKS managed CSR approver. Adopt this known E2E token explicitly + # before the HEAD daemon requests its dedicated certificate. + token="$(jq -er '.azure.bootstrapToken.token' "${config_file}")" + token_id="$(_historical_token_id "${config_file}")" + with_cluster_lock mark_e2e_bootstrap_token_aks_managed "${token}" + unset token + if [[ "$(kubectl -n kube-system get secret "bootstrap-token-${token_id}" \ + -o jsonpath='{.metadata.labels.kubernetes\.azure\.com/managedby}')" != "aks" ]]; then + log_error "Historical token adoption label was not applied" + return 1 + fi + + machine_configmap_upsert "${vm_name}" "${E2E_KUBERNETES_VERSION}" "${E2E_KUBERNETES_VERSION}" + _prepare_head_legacy_config "${config_file}" "${head_config}" + _upgrade_historical_node_to_head "${vm_ip}" "${head_config}" + validate_node_joined "${vm_name}" + + identity="$(_daemon_client_identity "${vm_ip}")" + if [[ "${identity}" != *"CN=system:node:${vm_name}"* || \ + "${identity}" != *"O=aks-flex-node-daemons"* ]]; then + log_error "Upgraded daemon has unexpected issued client identity: ${identity}" + return 1 + fi + _require_daemon_certificate_access "${vm_ip}" "${server_url}" + log_success "Same host upgraded from direct ${historicalReleaseTag} layout to HEAD and obtained daemon credentials" + + # The first run removes the canonical historical object. The second verifies + # that the explicitly requested migration is idempotent. + with_cluster_lock "${REPO_ROOT}/scripts/aks-flex-config" setup-node-rbac \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" \ + --remove-legacy-node-role-binding + with_cluster_lock "${REPO_ROOT}/scripts/aks-flex-config" setup-node-rbac \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" \ + --remove-legacy-node-role-binding + + if kubectl get clusterrolebinding "${legacyNodeRoleBinding}" >/dev/null 2>&1; then + log_error "Legacy binding '${legacyNodeRoleBinding}' still exists after migration" + return 1 + fi + _wait_for_bootstrap_token_probe http:403 "${config_file}" list-nodes + _wait_for_bootstrap_token_probe allowed "${config_file}" create-csr + _reissue_daemon_certificate_after_migration "${vm_ip}" "${server_url}" + + # Revocation proves the subsequent restarts cannot silently fall back to the + # bootstrap credential. + with_cluster_lock _revoke_historical_bootstrap_token "${config_file}" + _wait_for_bootstrap_token_probe http:401 "${config_file}" list-nodes + + _restart_kubelet_and_require_lease_renewal "${vm_name}" "${vm_ip}" "${node_uid}" + _restart_daemon_and_require_certificate_access "${vm_ip}" "${server_url}" + validate_node_joined "${vm_name}" + + identity="$(_kubelet_client_identity "${vm_ip}")" + if [[ "${identity}" != *"CN=system:node:${vm_name}"* ]]; then + log_error "Kubelet lost its node client identity after migration: ${identity}" + return 1 + fi + identity="$(_daemon_client_identity "${vm_ip}")" + if [[ "${identity}" != *"CN=system:node:${vm_name}"* || \ + "${identity}" != *"O=aks-flex-node-daemons"* ]]; then + log_error "Daemon lost its issued client identity after migration: ${identity}" + return 1 + fi + if [[ "$(kubectl get node "${vm_name}" -o jsonpath='{.metadata.uid}')" != "${node_uid}" ]]; then + log_error "Historical node object was replaced during migration" + return 1 + fi + + log_success "Historical ${historicalReleaseTag} node upgraded in place, migrated RBAC twice, revoked its token, and survived kubelet/daemon restarts" +} diff --git a/hack/e2e/run.sh b/hack/e2e/run.sh index 066e00a4..b7f57f8f 100755 --- a/hack/e2e/run.sh +++ b/hack/e2e/run.sh @@ -24,6 +24,7 @@ # validate Verify nodes joined + run smoke tests # validate-absent Verify all flex nodes are gone after unjoin # smoke Run smoke tests only (pods on flex nodes) +# historical-rbac-migration Build a real v0.1.0 node, upgrade it, and migrate RBAC # nspawn-lifecycle Validate generated lifecycle hooks and restart reconciliation # agent-upgrade Validate managed binary upgrade, rollback, and retry # upgrade-drift Run controller-machine Kubernetes version drift repave test @@ -106,6 +107,8 @@ source "${SCRIPT_DIR}/lib/node-join.sh" # shellcheck disable=SC1091 source "${SCRIPT_DIR}/lib/validate.sh" # shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib/bootstrap-rbac-migration.sh" +# shellcheck disable=SC1091 source "${SCRIPT_DIR}/lib/nspawn-lifecycle.sh" # shellcheck disable=SC1091 source "${SCRIPT_DIR}/lib/upgrade-drift.sh" @@ -136,7 +139,7 @@ usage() { parse_args() { while [[ $# -gt 0 ]]; do case "$1" in - all|infra|join|join-msi|join-token|join-offline|join-kubeadm|join-arc|unjoin|unjoin-msi|unjoin-token|unjoin-offline|unjoin-kubeadm|unjoin-arc|validate|validate-absent|smoke|nspawn-lifecycle|agent-upgrade|upgrade-drift|logs|cleanup|runner-cleanup|status) + all|historical-rbac-migration|infra|join|join-msi|join-token|join-offline|join-kubeadm|join-arc|unjoin|unjoin-msi|unjoin-token|unjoin-offline|unjoin-kubeadm|unjoin-arc|validate|validate-absent|smoke|nspawn-lifecycle|agent-upgrade|upgrade-drift|logs|cleanup|runner-cleanup|status) COMMAND="$1"; shift ;; -g|--resource-group) export E2E_RESOURCE_GROUP="$2"; shift 2 ;; -l|--location) export E2E_LOCATION="$2"; shift 2 ;; @@ -235,6 +238,29 @@ cmd_all() { return ${exit_code} } +# --------------------------------------------------------------------------- +# Command: historical-rbac-migration +# --------------------------------------------------------------------------- +cmd_historical_rbac_migration() { + log_section "AKS Flex Node Historical RBAC Migration E2E" + + if [[ "${SKIP_BUILD}" != "1" ]]; then + ensure_binary + elif [[ -z "${E2E_BINARY:-}" || ! -f "${E2E_BINARY:-}" ]]; then + log_error "--skip-build requires --binary or E2E_BINARY set to an existing file" + return 1 + else + log_info "Skipping build, using: ${E2E_BINARY}" + fi + + # This suite intentionally uses only the token VM. It does not call the + # parallel all-node join path, so an unrelated Arc failure cannot mask the + # historical compatibility result. + infra_deploy + ensure_cluster_dependencies + historical_rbac_migration_e2e +} + # --------------------------------------------------------------------------- # Command: status # --------------------------------------------------------------------------- @@ -261,7 +287,7 @@ main() { init_work_dir case "${COMMAND}" in - all|infra) + all|historical-rbac-migration|infra) require_exact_kubernetes_version ;; join|join-msi|join-token|join-offline|join-kubeadm|join-arc|upgrade-drift) @@ -281,6 +307,9 @@ main() { all) cmd_all ;; + historical-rbac-migration) + cmd_historical_rbac_migration + ;; infra) if [[ "${SKIP_BUILD}" != "1" ]]; then ensure_binary diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index de52d59b..9748ac96 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -1284,6 +1284,8 @@ func TestLoadConfigUsesRPConfigOverLegacyAliases(t *testing.T) { func TestLoadConfigAdaptsLegacyConfigAliases(t *testing.T) { t.Parallel() + // e2eMode was persisted by pre-v0.2 test/dev deployments. Keep accepting + // the removed field so those configs can be upgraded with the current agent. configJSON := `{ "azure": { "targetAgentPoolName": "pool1", @@ -1294,6 +1296,9 @@ func TestLoadConfigAdaptsLegacyConfigAliases(t *testing.T) { "resourceId": "/subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/test-rg/providers/Microsoft.ContainerService/managedClusters/test-cluster" } }, + "agent": { + "e2eMode": true + }, "kubernetes": { "version": "1.30.1" }, @@ -2262,6 +2267,9 @@ func TestServicePrincipalClientSecretFile(t *testing.T) { if err := os.WriteFile(insecureFile, []byte("file-secret"), 0o644); err != nil { t.Fatalf("os.WriteFile: %v", err) } + if err := os.Chmod(insecureFile, 0o644); err != nil { + t.Fatalf("os.Chmod: %v", err) + } tests := []struct { name string diff --git a/pkg/daemon/lifecycle.go b/pkg/daemon/lifecycle.go index 6862fd91..da616742 100644 --- a/pkg/daemon/lifecycle.go +++ b/pkg/daemon/lifecycle.go @@ -173,6 +173,11 @@ func writeAgentServiceAssets(binaryPaths agentUpgradePaths, serviceOptions agent if err := utilio.WriteFile(asset.path, asset.content, asset.mode); err != nil { return fmt.Errorf("write %s: %w", asset.path, err) } + // Atomic replacement preserves an existing file's mode and applies the + // process umask to new files, so reconcile the declared service-asset mode. + if err := os.Chmod(asset.path, asset.mode); err != nil { + return fmt.Errorf("set permissions on %s: %w", asset.path, err) + } } return nil } diff --git a/pkg/daemon/lifecycle_test.go b/pkg/daemon/lifecycle_test.go index b4a404fc..a8691f89 100644 --- a/pkg/daemon/lifecycle_test.go +++ b/pkg/daemon/lifecycle_test.go @@ -29,6 +29,9 @@ func TestEnsureAgentUpgradeServiceAssetsMigratesExistingInstallation(t *testing. t.Fatalf("write legacy unit: %v", err) } recoveryPath := filepath.Join(t.TempDir(), "aks-flex-node-recovery.sh") + if err := os.WriteFile(recoveryPath, []byte("stale"), 0o700); err != nil { + t.Fatalf("write legacy recovery script: %v", err) + } reloaded := false if err := ensureAgentUpgradeServiceAssetsAt( t.Context(),