From cc1cfb091247c855aec97f2ae60d3941cf08c052 Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Mon, 31 Aug 2026 16:36:25 -0700
Subject: [PATCH 1/2] feat(ci): run the e2e tests from the runner instead of
 inside the node

Holodeck's kubernetes.remoteAccess hands the GitHub Actions runner a
usable kubeconfig, so helm, kubectl, the case scripts and log collection
all run there. The repository rsync, the scp of the values override, the
in-VM tooling installs and the pull.sh log retrieval are gone.

Two operations mutate the host and stay on the node: the modprobe of
i2c_core and ipmi_msghandler, and the operator container kill used by the
restart test. Both go through a new tests/scripts/node-exec.sh, which
streams the self-contained node-operations.sh over SSH stdin. With
NODE_SSH_HOST unset it runs the operation locally, which keeps the
developer path documented in tests/README.md working; CI asserts the
variable is set so a node operation can never land on the shared runner.

Moving off the node changes what a kubectl call costs and how it fails,
which exposed several long-standing assumptions in the shell suite:

  - The polling loops bounded themselves by counting sleeps rather than
    elapsed time. On the node an iteration cost about 5s so the two
    agreed; across the network they do not, and a nominal 45 minute wait
    could outlive the 90 minute job. Eighteen loops now measure elapsed
    time against a deadline.

  - A failed query was being read as a satisfied condition. kubectl piped
    into wc -l or jq reports 0 when the API is unreachable, which read as
    "deleted" or "all owned", and a for loop over a failed query iterated
    zero times and reported success without checking anything. Queries
    now separate "the query failed" from "the condition is not met" and
    retry instead of passing.

  - kubectl applies no per-request timeout by default, so a hung request
    could run past a deadline that is only tested between commands.
    Polling and diagnostic calls are bounded now, with a larger budget
    for log and object dumps so the artifacts are not truncated.

  - The artifact upload moved from failure() to always(), since a job
    stopped by timeout-minutes is cancelled rather than failed and would
    otherwise upload nothing.

helm, kubectl and jq are installed on the runner at pinned versions, with
jq checksummed against its release manifest. The SSH key is written under
RUNNER_TEMP rather than the workspace and is removed along with the
kubeconfig in an always() step.

tests/local.sh, ci-run-e2e.sh, push.sh, pull.sh, sync.sh, remote.sh and
prerequisites.sh are kept because they are the documented developer path.
CI simply stops calling them.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 .github/workflows/e2e-tests.yaml              | 176 ++++++++++---
 tests/README.md                               |  17 ++
 tests/holodeck.yaml                           |   1 +
 tests/scripts/.definitions.sh                 |   2 +
 tests/scripts/checks.sh                       | 241 +++++++++++-------
 .../migrate-clusterpolicy-to-nvidiadriver.sh  | 115 +++++----
 tests/scripts/node-exec.sh                    |  58 +++++
 tests/scripts/node-operations.sh              |  93 +++++++
 tests/scripts/update-clusterpolicy.sh         |  16 +-
 tests/scripts/update-nvidiadriver.sh          | 136 +++++-----
 10 files changed, 613 insertions(+), 242 deletions(-)
 create mode 100755 tests/scripts/node-exec.sh
 create mode 100755 tests/scripts/node-operations.sh

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 0e452648a8..87975dd350 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -58,6 +58,12 @@ on:
 permissions:
   contents: read
 
+env:
+  HELM_VERSION: v3.21.4
+  KUBECTL_VERSION: v1.35.4
+  JQ_VERSION: "1.8.2"
+  JQ_SHA256: "b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f"
+
 jobs:
   variables:
     uses: ./.github/workflows/variables.yaml
@@ -82,6 +88,9 @@ jobs:
     permissions:
       contents: read
       id-token: write
+    env:
+      KUBECONFIG: ${{ github.workspace }}/kubeconfig
+      LOG_DIR: ${{ github.workspace }}/logs
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         name: Check out code
@@ -95,11 +104,29 @@ jobs:
           path: ${{ github.workspace }}
       - name: Set up Helm
         uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
+        with:
+          version: ${{ env.HELM_VERSION }}
       - name: Verify the published Helm chart is available
         env:
           HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }}
           HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }}
         run: helm show chart "${HELM_CHART}" --version "${HELM_CHART_VERSION}"
+      - name: Install kubectl
+        uses: azure/setup-kubectl@c0c8b32d33a5244f1e5947304550403b63930415 # v4
+        with:
+          version: ${{ env.KUBECTL_VERSION }}
+      - name: Install jq and verify runner tooling
+        run: |
+          set -euo pipefail
+          mkdir -p "${RUNNER_TEMP}/bin"
+          curl -fsSL -o "${RUNNER_TEMP}/bin/jq" \
+            "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
+          echo "${JQ_SHA256}  ${RUNNER_TEMP}/bin/jq" | sha256sum -c -
+          chmod +x "${RUNNER_TEMP}/bin/jq"
+          echo "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH"
+          helm version
+          kubectl version --client
+          "${RUNNER_TEMP}/bin/jq" --version
       - name: Set up Holodeck
         uses: NVIDIA/holodeck@e0f3932cf284d92421a55536839fa821a14116aa # v0.3.7
         with:
@@ -112,24 +139,40 @@ jobs:
         uses: mikefarah/yq@c14f446382944492701b16c1ddb48bb9dbe683e3 # v4
         with:
           cmd: yq '.status.properties[] | select(.name == "public-dns-name") | .value' /github/workspace/.cache/holodeck.yaml
-      - name: Set test environment
+      - name: Configure node SSH access
         env:
+          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
           PUBLIC_DNS_NAME: ${{ steps.get_public_dns_name.outputs.result }}
         run: |
-          echo "instance_hostname=ubuntu@${PUBLIC_DNS_NAME}" >> $GITHUB_ENV
-          echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
-      - name: Write SSH key
-        env:
-          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
+          set -euo pipefail
+          test -n "${PUBLIC_DNS_NAME}"
+          SSH_DIR="${RUNNER_TEMP}/holodeck-ssh"
+          mkdir -p "${SSH_DIR}"
+          chmod 700 "${SSH_DIR}"
+          install -m 600 /dev/null "${SSH_DIR}/id_rsa"
+          printf '%s\n' "${AWS_SSH_KEY}" > "${SSH_DIR}/id_rsa"
+          install -m 600 /dev/null "${SSH_DIR}/known_hosts"
+          {
+            echo "NODE_SSH_HOST=ubuntu@${PUBLIC_DNS_NAME}"
+            echo "NODE_SSH_KEY=${SSH_DIR}/id_rsa"
+            echo "NODE_SSH_KNOWN_HOSTS=${SSH_DIR}/known_hosts"
+          } >> "$GITHUB_ENV"
+      - name: Verify cluster access
         run: |
-          echo "${AWS_SSH_KEY}" > ${private_key} && chmod 400 ${private_key}
-      - name: Copy values override file to remote
+          set -euo pipefail
+          test -r "${KUBECONFIG}"
+          kubectl cluster-info
+          kubectl get nodes -o wide
+      - name: Select values override file
         if: ${{ inputs.use_values_override }}
         run: |
-          scp -i ${private_key} -o StrictHostKeyChecking=no \
-            ${{ github.workspace }}/values-overrides.yaml \
-            ${instance_hostname}:/tmp/values-overrides.yaml
-          echo "VALUES_FILE=/tmp/values-overrides.yaml" >> $GITHUB_ENV
+          set -euo pipefail
+          echo "VALUES_FILE=${GITHUB_WORKSPACE}/values-overrides.yaml" >> "$GITHUB_ENV"
+      - name: Load kernel modules on the node
+        run: |
+          set -euo pipefail
+          test -n "${NODE_SSH_HOST}"
+          ./tests/scripts/node-exec.sh load-modules
       - name: Run e2e tests
         env:
           OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }}
@@ -137,20 +180,32 @@ jobs:
           HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }}
           HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }}
           GPU_PRODUCT_NAME: "Tesla-T4"
-          SKIP_LAUNCH: "true"
           CONTAINER_RUNTIME: "containerd"
-          TEST_CASE: "./tests/cases/defaults.sh"
         run: |
-          ./tests/ci-run-e2e.sh ${OPERATOR_IMAGE} ${OPERATOR_VERSION} ${GPU_PRODUCT_NAME} ${TEST_CASE} || rc=$?
-          ./tests/scripts/pull.sh /tmp/logs logs
-          exit $rc
+          set -euo pipefail
+          test -n "${NODE_SSH_HOST}"
+          mkdir -p "${LOG_DIR}"
+          ./tests/cases/defaults.sh
+      - name: Collect cluster diagnostics
+        if: always()
+        run: |
+          mkdir -p "${LOG_DIR}"
+          kubectl get nodes -o wide --request-timeout=15s > "${LOG_DIR}/nodes.txt" 2>&1 || true
+          kubectl get pods -A -o wide --request-timeout=15s > "${LOG_DIR}/pods.txt" 2>&1 || true
+          kubectl get events -A --sort-by=.lastTimestamp --request-timeout=15s > "${LOG_DIR}/events.txt" 2>&1 || true
+          timeout 30s helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
       - name: Archive test logs
-        if: ${{ failure() }}
+        if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
           name: containerd-e2e-test-logs
           path: ./logs/
           retention-days: 15
+      - name: Remove credentials from the runner
+        if: always()
+        run: |
+          rm -rf "${RUNNER_TEMP}/holodeck-ssh"
+          rm -f "${KUBECONFIG}"
 
   e2e-tests-nvidiadriver:
     needs: [variables, publish-helm-oci-chart]
@@ -159,6 +214,9 @@ jobs:
     permissions:
       contents: read
       id-token: write
+    env:
+      KUBECONFIG: ${{ github.workspace }}/kubeconfig
+      LOG_DIR: ${{ github.workspace }}/logs
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         name: Check out code
@@ -172,11 +230,29 @@ jobs:
           path: ${{ github.workspace }}
       - name: Set up Helm
         uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
+        with:
+          version: ${{ env.HELM_VERSION }}
       - name: Verify the published Helm chart is available
         env:
           HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }}
           HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }}
         run: helm show chart "${HELM_CHART}" --version "${HELM_CHART_VERSION}"
+      - name: Install kubectl
+        uses: azure/setup-kubectl@c0c8b32d33a5244f1e5947304550403b63930415 # v4
+        with:
+          version: ${{ env.KUBECTL_VERSION }}
+      - name: Install jq and verify runner tooling
+        run: |
+          set -euo pipefail
+          mkdir -p "${RUNNER_TEMP}/bin"
+          curl -fsSL -o "${RUNNER_TEMP}/bin/jq" \
+            "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
+          echo "${JQ_SHA256}  ${RUNNER_TEMP}/bin/jq" | sha256sum -c -
+          chmod +x "${RUNNER_TEMP}/bin/jq"
+          echo "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH"
+          helm version
+          kubectl version --client
+          "${RUNNER_TEMP}/bin/jq" --version
       - name: Set up Holodeck
         uses: NVIDIA/holodeck@e0f3932cf284d92421a55536839fa821a14116aa # v0.3.7
         with:
@@ -189,24 +265,40 @@ jobs:
         uses: mikefarah/yq@c14f446382944492701b16c1ddb48bb9dbe683e3 # v4
         with:
           cmd: yq '.status.properties[] | select(.name == "public-dns-name") | .value' /github/workspace/.cache/holodeck.yaml
-      - name: Set test environment
+      - name: Configure node SSH access
         env:
+          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
           PUBLIC_DNS_NAME: ${{ steps.get_public_dns_name.outputs.result }}
         run: |
-          echo "instance_hostname=ubuntu@${PUBLIC_DNS_NAME}" >> $GITHUB_ENV
-          echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
-      - name: Write SSH key
-        env:
-          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
+          set -euo pipefail
+          test -n "${PUBLIC_DNS_NAME}"
+          SSH_DIR="${RUNNER_TEMP}/holodeck-ssh"
+          mkdir -p "${SSH_DIR}"
+          chmod 700 "${SSH_DIR}"
+          install -m 600 /dev/null "${SSH_DIR}/id_rsa"
+          printf '%s\n' "${AWS_SSH_KEY}" > "${SSH_DIR}/id_rsa"
+          install -m 600 /dev/null "${SSH_DIR}/known_hosts"
+          {
+            echo "NODE_SSH_HOST=ubuntu@${PUBLIC_DNS_NAME}"
+            echo "NODE_SSH_KEY=${SSH_DIR}/id_rsa"
+            echo "NODE_SSH_KNOWN_HOSTS=${SSH_DIR}/known_hosts"
+          } >> "$GITHUB_ENV"
+      - name: Verify cluster access
         run: |
-          echo "${AWS_SSH_KEY}" > ${private_key} && chmod 400 ${private_key}
-      - name: Copy values override file to remote
+          set -euo pipefail
+          test -r "${KUBECONFIG}"
+          kubectl cluster-info
+          kubectl get nodes -o wide
+      - name: Select values override file
         if: ${{ inputs.use_values_override }}
         run: |
-          scp -i ${private_key} -o StrictHostKeyChecking=no \
-            ${{ github.workspace }}/values-overrides.yaml \
-            ${instance_hostname}:/tmp/values-overrides.yaml
-          echo "VALUES_FILE=/tmp/values-overrides.yaml" >> $GITHUB_ENV
+          set -euo pipefail
+          echo "VALUES_FILE=${GITHUB_WORKSPACE}/values-overrides.yaml" >> "$GITHUB_ENV"
+      - name: Load kernel modules on the node
+        run: |
+          set -euo pipefail
+          test -n "${NODE_SSH_HOST}"
+          ./tests/scripts/node-exec.sh load-modules
       - name: Run e2e tests
         env:
           OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }}
@@ -214,17 +306,29 @@ jobs:
           HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }}
           HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }}
           GPU_PRODUCT_NAME: "Tesla-T4"
-          SKIP_LAUNCH: "true"
           CONTAINER_RUNTIME: "containerd"
-          TEST_CASE: "./tests/cases/nvidia-driver.sh"
         run: |
-          ./tests/ci-run-e2e.sh ${OPERATOR_IMAGE} ${OPERATOR_VERSION} ${GPU_PRODUCT_NAME} ${TEST_CASE} || rc=$?
-          ./tests/scripts/pull.sh /tmp/logs logs
-          exit $rc
+          set -euo pipefail
+          test -n "${NODE_SSH_HOST}"
+          mkdir -p "${LOG_DIR}"
+          ./tests/cases/nvidia-driver.sh
+      - name: Collect cluster diagnostics
+        if: always()
+        run: |
+          mkdir -p "${LOG_DIR}"
+          kubectl get nodes -o wide --request-timeout=15s > "${LOG_DIR}/nodes.txt" 2>&1 || true
+          kubectl get pods -A -o wide --request-timeout=15s > "${LOG_DIR}/pods.txt" 2>&1 || true
+          kubectl get events -A --sort-by=.lastTimestamp --request-timeout=15s > "${LOG_DIR}/events.txt" 2>&1 || true
+          timeout 30s helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
       - name: Archive test logs
-        if: ${{ failure() }}
+        if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
           name: nvidiadriver-e2e-test-logs
           path: ./logs/
           retention-days: 15
+      - name: Remove credentials from the runner
+        if: always()
+        run: |
+          rm -rf "${RUNNER_TEMP}/holodeck-ssh"
+          rm -f "${KUBECONFIG}"
diff --git a/tests/README.md b/tests/README.md
index 9b14de2462..3a4096ce5c 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -1,5 +1,22 @@
 # GPU operator test utilities
 
+## Testing in CI
+CI no longer uses `local.sh` or `ci-run-e2e.sh`, and no longer syncs the project
+folder to the test instance. Those remain the developer path described below.
+
+Instead, the e2e workflow provisions a Holodeck environment with
+`kubernetes.remoteAccess` enabled, which gives the GitHub Actions runner a
+kubeconfig for the cluster. The case scripts (`cases/defaults.sh`,
+`cases/nvidia-driver.sh`) then run on the runner itself, and everything they do
+-- helm, kubectl, log collection -- goes over that kubeconfig.
+
+Only two operations still need a shell on the instance, and both go through
+`scripts/node-exec.sh`, which dispatches `scripts/node-operations.sh` over SSH:
+loading the `i2c_core` and `ipmi_msghandler` kernel modules, and killing the
+gpu-operator container for the operator restart test. With `NODE_SSH_HOST`
+unset, `node-exec.sh` runs the operation locally, so the developer path below is
+unaffected.
+
 ## Testing locally
 The `local.sh` script allows for triggering basic end-to-end testing of the GPU
 operator from a local machine.
diff --git a/tests/holodeck.yaml b/tests/holodeck.yaml
index 3050eaf680..880aa417a6 100644
--- a/tests/holodeck.yaml
+++ b/tests/holodeck.yaml
@@ -24,3 +24,4 @@ spec:
     version: v1.35.4
     crictlVersion: v1.35.0
     calicoVersion: v3.31.5
+    remoteAccess: true
diff --git a/tests/scripts/.definitions.sh b/tests/scripts/.definitions.sh
index 737a019c6e..3babf0ff24 100644
--- a/tests/scripts/.definitions.sh
+++ b/tests/scripts/.definitions.sh
@@ -14,6 +14,8 @@ TERRAFORM="terraform -chdir=${TERRAFORM_DIR}"
 
 # Set default values if not defined
 : ${HELM:="helm"}
+: "${KUBECTL_REQUEST_TIMEOUT:="15s"}"
+: "${KUBECTL_LOG_TIMEOUT:="120s"}"
 : ${LOG_DIR:="/tmp/logs"}
 : ${PROJECT:="$(basename "${PROJECT_DIR}")"}
 : ${TEST_NAMESPACE:="test-operator"}
diff --git a/tests/scripts/checks.sh b/tests/scripts/checks.sh
index 5056d48b2f..d701044132 100755
--- a/tests/scripts/checks.sh
+++ b/tests/scripts/checks.sh
@@ -2,17 +2,16 @@
 
 check_pod_ready() {
 	local pod_label=$1
-	local current_time=0
+	local deadline=$((SECONDS + 60 * 45))
 	while :; do
 		echo "Checking $pod_label pod"
-		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE}
+		kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 		echo "Checking $pod_label pod readiness"
-		is_pod_ready=$(kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
+		is_pod_ready=$(kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
 
 		if [ "${is_pod_ready}" = "True" ]; then
-			# Check if the pod is not in terminating state
-			is_pod_terminating=$(kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} -o jsonpath='{.items[0].metadata.deletionGracePeriodSeconds}' 2>/dev/null || echo "terminated")
+			is_pod_terminating=$(kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" -o jsonpath='{.items[0].metadata.deletionGracePeriodSeconds}' 2>/dev/null || echo "terminated")
 			if [ "${is_pod_terminating}" != "" ]; then
 				echo "pod $pod_label is in terminating state..."
 			else
@@ -21,152 +20,160 @@ check_pod_ready() {
 			fi
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
 			exit 1;
 		fi
 
-		# Echo useful information on stdout
-		kubectl get pods -n ${TEST_NAMESPACE}
+		kubectl get pods -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5
 	done
 }
 
 check_pod_deleted() {
 	local pod_label=$1
-	local current_time=0
+	local deadline=$((SECONDS + 60 * 45))
+	local pod_list
 	while :; do
 		echo "Checking $pod_label pod"
-		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE}
+		kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 		echo "Checking if $pod_label pod has been deleted"
-		# note: $(kubectl get pods <options> -o jsonpath='.items' | jq length) does not work for older kubectl clients
-		num_pods=$(kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} -o json | jq '.items' | jq length)
-
-		if [ "${num_pods}" = 0 ]; then
-			echo "Pod $pod_label has been deleted"
-			break;
+		if pod_list=$(kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" -o name --ignore-not-found --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+			if [ -z "${pod_list}" ]; then
+				echo "Pod $pod_label has been deleted"
+				break;
+			else
+				echo "Pod $pod_label has not been deleted"
+			fi
 		else
-			echo "Pod $pod_label has not been deleted"
+			api_unreachable "checking whether pod $pod_label has been deleted"
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
 			exit 1;
 		fi
 
-		# Echo useful information on stdout
-		kubectl get pods -n ${TEST_NAMESPACE}
+		kubectl get pods -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5
 	done
 }
 
 check_no_restarts() {
 	local pod_label=$1
-	restartCount=$(kubectl get pod -lapp=$pod_label -n ${TEST_NAMESPACE} -o jsonpath='{.items[*].status.containerStatuses[0].restartCount}')
-	if [ $restartCount -gt 1 ]; then
-		echo "$pod_label restarted multiple times: $restartCount"
-		kubectl logs -p -lapp=$pod_label --all-containers -n ${TEST_NAMESPACE}
+	local restart_count
+	restart_count=$(kubectl get pod -lapp="${pod_label}" -n "${TEST_NAMESPACE}" -o jsonpath='{.items[*].status.containerStatuses[0].restartCount}' --request-timeout="${KUBECTL_REQUEST_TIMEOUT}")
+	if ! [[ "${restart_count}" =~ ^[0-9]+$ ]]; then
+		echo "expected one restart count for ${pod_label}, got '${restart_count}'"
+		exit 1
+	fi
+	if [ "${restart_count}" -gt 1 ]; then
+		echo "$pod_label restarted multiple times: ${restart_count}"
+		kubectl logs -p -lapp="${pod_label}" --all-containers -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
 		exit 1
 	fi
 	echo "Repeated restarts not observed for pod $pod_label"
 	return 0
 }
 
-# This function kills the operator and waits for the operator to be back in a running state
-# Timeout is 100 seconds
 test_restart_operator() {
 	local ns=${1}
 	local runtime=${2}
 
-	if [[ x"${runtime}" == x"containerd" ]]; then
-		# The operator is the only container that has the string '"gpu-operator"'
-		# TODO: This requires permissions on containerd.sock
-		sudo crictl rm --force "$(sudo crictl ps --name gpu-operator | awk '{if(NR>1)print $1}')"
-	else
-		# The operator is the only container that has the string '"gpu-operator"'
-	    docker kill "$(docker ps --format '{{.ID}} {{.Command}}' | grep "gpu-operator" | cut -f 1 -d ' ')"
-	fi
+	local checks_script_dir
+	checks_script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+	"${checks_script_dir}"/node-exec.sh restart-operator-container "${runtime}"
 
-	for i in $(seq 1 10); do
-		# Sleep a reasonable amount of time for k8s to update the container status to crashing
+	for _ in $(seq 1 10); do
 		sleep 10
 
-		state=$(kubectl get pods -n "${ns}" -l "app.kubernetes.io/component=gpu-operator" \
-			-o jsonpath='{.items[0].status.phase}')
+		local operator_phase
+		operator_phase=$(kubectl get pods -n "${ns}" -l "app.kubernetes.io/component=gpu-operator" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" \
+			-o jsonpath='{.items[0].status.phase}' || true)
 
-		echo "Checking state of the GPU Operator, it is: '$state'"
-		if [ "$state" = "Running" ]; then
+		echo "Checking state of the GPU Operator, it is: '${operator_phase}'"
+		if [ "${operator_phase}" = "Running" ]; then
 			return 0
 		fi
 	done
 
 	echo "Timeout reached, the GPU Operator is still not ready. See below for logs:"
-	kubectl logs -n gpu-operator "$(kubectl get pods -n "${ns}" -o json | jq -r '.items[0].metadata.name')"
+	kubectl logs -n "${ns}" --request-timeout="${KUBECTL_LOG_TIMEOUT}" "$(kubectl get pods -n "${ns}" -o json --request-timeout="${KUBECTL_LOG_TIMEOUT}" | jq -r '.items[0].metadata.name')"
 	exit 1
 }
 
+list_all_pods() {
+	kubectl get pods --all-namespaces -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name --no-headers --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
+}
+
+collect_pod_logs() {
+	local log_dir=$1
+	local namespaced_pods=$2
+	local namespace pod_name
+
+	while read -r namespace pod_name; do
+		[[ -n "${pod_name}" ]] || continue
+		echo "Generating logs for pod: ${pod_name} ns: ${namespace}"
+		local artifact="${log_dir}/${namespace}_${pod_name}"
+		echo "------------------------------------------------" >> "${artifact}.describe"
+		kubectl -n "${namespace}" describe pods "${pod_name}" --request-timeout="${KUBECTL_LOG_TIMEOUT}" >> "${artifact}.describe" || true
+		kubectl -n "${namespace}" logs "${pod_name}" --all-containers=true --request-timeout="${KUBECTL_LOG_TIMEOUT}" > "${artifact}.logs" || true
+	done <<< "${namespaced_pods}"
+}
+
 check_gpu_pod_ready() {
 	local log_dir=$1
-	local current_time=0
+	local deadline=$((SECONDS + 60 * 45))
+	local next_collection=0
 
-	# Ensure the log directory exists
-	mkdir -p ${log_dir}
+	mkdir -p "${log_dir}"
 
 	while :; do
-		pods="$(kubectl get --all-namespaces pods -o json | jq '.items[] | {name: .metadata.name, ns: .metadata.namespace}' | jq -s -c .)"
-		status=$(kubectl get pods gpu-operator-test -o json | jq -r .status.phase)
-		if [ "${status}" = "Succeeded" ]; then
+		local pod_phase
+		pod_phase=$(kubectl get pods gpu-operator-test --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" -o jsonpath='{.status.phase}' || true)
+		if [ "${pod_phase}" = "Succeeded" ]; then
 			echo "GPU pod terminated successfully"
-			rc=0
+			collect_pod_logs "${log_dir}" "$(list_all_pods)"
 			break;
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
+			collect_pod_logs "${log_dir}" "$(list_all_pods)"
 			exit 1
 		fi
 
-		# Echo useful information on stdout
-		kubectl get pods --all-namespaces
-
-		for pod in $(echo "$pods" | jq -r .[].name); do
-			ns=$(echo "$pods" | jq -r ".[] | select(.name == \"$pod\") | .ns")
-			echo "Generating logs for pod: ${pod} ns: ${ns}"
-			echo "------------------------------------------------" >> "${log_dir}/${pod}.describe"
-			kubectl -n "${ns}" describe pods "${pod}" >> "${log_dir}/${pod}.describe"
-			kubectl -n "${ns}" logs "${pod}" --all-containers=true > "${log_dir}/${pod}.logs" || true
-		done
-
 		echo "Generating cluster logs"
 		echo "------------------------------------------------" >> "${log_dir}/cluster.logs"
-		kubectl get --all-namespaces pods >> "${log_dir}/cluster.logs"
+		kubectl get pods --all-namespaces --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" | tee -a "${log_dir}/cluster.logs" || true
+
+		if (( SECONDS >= next_collection )); then
+			collect_pod_logs "${log_dir}" "$(list_all_pods)"
+			next_collection=$((SECONDS + 30))
+		fi
 
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5;
 	done
 }
 
 # TODO: deduplicate the logic found in this file by moving the duplicate to a common method and parameterizing the labels to select on
 check_nvidia_driver_pods_ready() {
-	local current_time=0
+	local deadline=$((SECONDS + 60 * 45))
 	while :; do
 		echo "Checking nvidia driver pod"
-		kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE}
+		kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 		echo "Checking nvidia driver pod readiness"
-		is_pod_ready=$(kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
+		is_pod_ready=$(kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
 
 		if [ "${is_pod_ready}" = "True" ]; then
-			# Check if the pod is not in terminating state
-			is_pod_terminating=$(kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -ojsonpath='{.items[0].metadata.deletionGracePeriodSeconds}' 2>/dev/null || echo "terminated")
+			is_pod_terminating=$(kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" -ojsonpath='{.items[0].metadata.deletionGracePeriodSeconds}' 2>/dev/null || echo "terminated")
 			if [ "${is_pod_terminating}" != "" ]; then
 				echo "nvidia driver pod is in terminating state..."
 			else
@@ -175,52 +182,65 @@ check_nvidia_driver_pods_ready() {
 			fi
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
 			exit 1;
 		fi
 
-		# Echo useful information on stdout
-		kubectl get pods -n ${TEST_NAMESPACE}
+		kubectl get pods -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5
 	done
 }
 
 check_no_driver_pod_restarts() {
-	restartCount=$(kubectl get pod -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o jsonpath='{.items[*].status.containerStatuses[0].restartCount}')
-	if [ $restartCount -gt 1 ]; then
-		echo "nvidia driver pod restarted multiple times: $restartCount"
-		kubectl logs -p -l "app.kubernetes.io/component=nvidia-driver" --all-containers -n ${TEST_NAMESPACE}
+	local restart_count
+	restart_count=$(kubectl get pod -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" -o jsonpath='{.items[*].status.containerStatuses[0].restartCount}' --request-timeout="${KUBECTL_REQUEST_TIMEOUT}")
+	if ! [[ "${restart_count}" =~ ^[0-9]+$ ]]; then
+		echo "expected one restart count for the nvidia driver pod, got '${restart_count}'"
+		exit 1
+	fi
+	if [ "${restart_count}" -gt 1 ]; then
+		echo "nvidia driver pod restarted multiple times: ${restart_count}"
+		kubectl logs -p -l "app.kubernetes.io/component=nvidia-driver" --all-containers -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
 		exit 1
 	fi
 	echo "Repeated restarts not observed for the nvidia driver pod"
 	return 0
 }
 
+api_unreachable() {
+	echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') WARNING: cluster API unreachable while ${1}; treating as not ready and retrying"
+}
+
+kubectl_count() {
+	local resource_lines
+	resource_lines=$(kubectl "$@" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}") || return 1
+	echo "${resource_lines}" | grep -c . || true
+}
+
 print_driver_upgrade_debug() {
 	echo "current state of driver upgrade"
-	kubectl get node -l nvidia.com/gpu.present \
-		-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers
+	kubectl get node -l nvidia.com/gpu.present --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" \
+		-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers || true
 
 	echo ""
 	echo "driver pods"
-	kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o wide || true
+	kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 	echo ""
 	echo "gpu operator operands"
-	kubectl get pods -n ${TEST_NAMESPACE} -o wide || true
+	kubectl get pods -n "${TEST_NAMESPACE}" -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 	echo ""
 	echo "driver daemonsets"
-	kubectl get daemonsets -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o wide || true
+	kubectl get daemonsets -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 	echo ""
 	echo "NVIDIADriver status"
 	local nvidiadriver_status
-	if nvidiadriver_status=$(kubectl get nvidiadriver -o json 2>/dev/null); then
+	if nvidiadriver_status=$(kubectl get nvidiadriver -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" 2>/dev/null); then
 		echo "${nvidiadriver_status}" | jq -r '
 			(["NAME", "DEFAULT", "STATE", "REASON", "MESSAGE"] | @tsv),
 			(
@@ -241,39 +261,64 @@ print_driver_upgrade_debug() {
 }
 
 wait_for_driver_upgrade_done() {
-	gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present --no-headers | wc -l)
-	local current_time=0
+	local deadline=$((SECONDS + 60 * 45))
+	local next_debug=0
+	local node_list=""
+	local node_count=""
+	local upgraded_count=0
+	local upgrade_state=""
+	local gpu_node_count=""
+
 	echo "waiting for the gpu driver upgrade to complete"
 	while :; do
-		local upgraded_count=0
-		for node in $(kubectl get nodes -o NAME); do
-			upgrade_state=$(kubectl get $node -ojsonpath='{.metadata.labels.nvidia\.com/gpu-driver-upgrade-state}')
-			if [ "${upgrade_state}" = "upgrade-done" ]; then
-				upgraded_count=$((${upgraded_count} + 1))
+		upgraded_count=0
+
+		if [[ "${gpu_node_count:-0}" -le 0 ]]; then
+			if node_count=$(kubectl_count get node -l nvidia.com/gpu.present --no-headers); then
+				if (( node_count > 0 )); then
+					gpu_node_count="${node_count}"
+				fi
+			else
+				api_unreachable "counting the GPU nodes"
 			fi
-		done
-		if [[ $upgraded_count -eq $gpu_node_count ]]; then
+		fi
+
+		if node_list=$(kubectl get nodes -o NAME --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+			for node in ${node_list}; do
+				if upgrade_state=$(kubectl get "$node" -ojsonpath='{.metadata.labels.nvidia\.com/gpu-driver-upgrade-state}' --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+					if [ "${upgrade_state}" = "upgrade-done" ]; then
+						upgraded_count=$((upgraded_count + 1))
+					fi
+				else
+					api_unreachable "reading the upgrade state of ${node}"
+				fi
+			done
+		else
+			api_unreachable "listing the nodes"
+		fi
+
+		if [[ "${gpu_node_count:-0}" -gt 0 ]] && [[ "${upgraded_count}" -eq "${gpu_node_count}" ]]; then
 			echo "gpu driver upgrade completed successfully"
 			break;
 		else
-			echo "gpu driver still in progress. $upgraded_count/$gpu_node_count node(s) upgraded"
+			echo "gpu driver still in progress. $upgraded_count/${gpu_node_count:-unknown} node(s) upgraded"
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
 			print_driver_upgrade_debug
 			exit 1;
 		fi
 
-		if [[ $((current_time % 30)) -eq 0 ]]; then
+		if (( SECONDS >= next_debug )); then
 			print_driver_upgrade_debug
+			next_debug=$((SECONDS + 30))
 		else
-			kubectl get node -l nvidia.com/gpu.present \
-				-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers
+			kubectl get node -l nvidia.com/gpu.present --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" \
+				-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers || true
 		fi
 		
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5
 	done
 }
diff --git a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
index 4f56d9f198..4f5e3fc2c9 100755
--- a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
+++ b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
@@ -23,153 +23,176 @@ get_helm_release_name() {
 }
 
 wait_for_legacy_driver_daemonset_deleted() {
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for ClusterPolicy-owned driver DaemonSet to be deleted"
     while :; do
-        daemonset_count=$(kubectl get daemonset -l app=nvidia-driver-daemonset -n "${TEST_NAMESPACE}" --no-headers 2>/dev/null | wc -l)
-        if [[ "${daemonset_count}" -eq 0 ]]; then
-            break
+        if daemonsets=$(kubectl get daemonset -l app=nvidia-driver-daemonset -n "${TEST_NAMESPACE}" -o name --ignore-not-found --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+            if [[ -z "${daemonsets}" ]]; then
+                break
+            fi
+        else
+            api_unreachable "checking for the ClusterPolicy-owned driver DaemonSet"
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for legacy driver DaemonSet deletion"
-            kubectl get daemonset -n "${TEST_NAMESPACE}" -o wide
+            kubectl get daemonset -n "${TEST_NAMESPACE}" -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_orphaned_legacy_driver_pod() {
     local pod_name=$1
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for legacy driver pod/${pod_name} to become orphaned"
     while :; do
-        owner_count=$(kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o json | jq '.metadata.ownerReferences | length')
-        if [[ "${owner_count}" -eq 0 ]]; then
-            echo "legacy driver pod/${pod_name} is orphaned"
-            break
+        if pod_json=$(kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+            owner_count=$(echo "${pod_json}" | jq '.metadata.ownerReferences | length')
+            if [[ "${owner_count}" -eq 0 ]]; then
+                echo "legacy driver pod/${pod_name} is orphaned"
+                break
+            fi
+        else
+            api_unreachable "checking whether legacy driver pod/${pod_name} is orphaned"
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for legacy driver pod to become orphaned"
-            kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml
+            kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_default_nvidiadriver() {
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for default NVIDIADriver to be rendered"
     while :; do
-        default_count=$(kubectl get nvidiadriver -o json 2>/dev/null | jq '[.items[] | select(.spec.default == true)] | length')
-        if [[ "${default_count}" -eq 1 ]]; then
-            break
+        if nvidiadrivers=$(kubectl get nvidiadriver -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+            default_count=$(echo "${nvidiadrivers}" | jq '[.items[] | select(.spec.default == true)] | length')
+            if [[ "${default_count}" -eq 1 ]]; then
+                break
+            fi
+        else
+            api_unreachable "counting the default NVIDIADrivers"
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for default NVIDIADriver"
-            kubectl get nvidiadriver || true
+            kubectl get nvidiadriver --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_nvidiadriver_owner_labels() {
     local driver_name=$1
-    local elapsed_time=0
-    local gpu_node_count
+    local deadline=$((SECONDS + 300))
+    local gpu_node_count=""
+    local node_count=""
+    local owned_count
 
-    gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
-    echo "Waiting for ${gpu_node_count} GPU node(s) to be owned by NVIDIADriver/${driver_name}"
+    echo "Waiting for the GPU node(s) to be owned by NVIDIADriver/${driver_name}"
 
     while :; do
-        owned_count=$(kubectl get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers | wc -l)
-        if [[ "${owned_count}" -eq "${gpu_node_count}" ]]; then
-            break
+        if [[ "${gpu_node_count:-0}" -le 0 ]]; then
+            if node_count=$(kubectl_count get node -l nvidia.com/gpu.present=true --no-headers); then
+                if (( node_count > 0 )); then
+                    gpu_node_count="${node_count}"
+                fi
+            else
+                api_unreachable "counting the GPU nodes"
+            fi
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if owned_count=$(kubectl_count get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers); then
+            if [[ "${gpu_node_count:-0}" -gt 0 ]] && [[ "${owned_count}" -eq "${gpu_node_count}" ]]; then
+                break
+            fi
+        else
+            api_unreachable "counting the nodes owned by NVIDIADriver/${driver_name}"
+        fi
+
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver owner labels"
-            kubectl get nodes -l nvidia.com/gpu.present=true -o json |
-                jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv'
+            kubectl get nodes -l nvidia.com/gpu.present=true -o json --request-timeout="${KUBECTL_LOG_TIMEOUT}" |
+                jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv' || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_nvidiadriver_daemonset() {
     local driver_name=$1
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for NVIDIADriver-owned driver DaemonSet"
     while :; do
-        daemonset_count=$(kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" -o json |
+        daemonset_count=$(kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" |
             jq --arg driver_name "${driver_name}" '[.items[] | select(.spec.template.spec.nodeSelector["nvidia.com/gpu-operator.driver.owner"] == $driver_name)] | length')
         if [[ "${daemonset_count}" -gt 0 ]]; then
             break
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver-owned driver DaemonSet"
-            kubectl get daemonset -n "${TEST_NAMESPACE}" -o yaml
+            kubectl get daemonset -n "${TEST_NAMESPACE}" -o yaml --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_legacy_driver_pod_deleted() {
     local pod_name=$1
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for orphaned legacy driver pod/${pod_name} to be deleted by the upgrade flow"
     while :; do
-        if ! kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" >/dev/null 2>&1; then
-            break
+        if legacy_pod=$(kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o name --ignore-not-found --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+            if [[ -z "${legacy_pod}" ]]; then
+                break
+            fi
+        else
+            api_unreachable "checking whether legacy driver pod/${pod_name} is deleted"
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for orphaned legacy driver pod deletion"
             print_driver_upgrade_debug
-            kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml || true
+            kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
             exit 1
         fi
 
         print_driver_upgrade_debug
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 legacy_driver_pod=$(kubectl get pod -l app=nvidia-driver-daemonset -n "${TEST_NAMESPACE}" -o jsonpath='{.items[0].metadata.name}')
 if [[ -z "${legacy_driver_pod}" ]]; then
     echo "legacy ClusterPolicy driver pod not found"
-    kubectl get pods -n "${TEST_NAMESPACE}" -o wide
+    kubectl get pods -n "${TEST_NAMESPACE}" -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
     exit 1
 fi
 
 operator_name=$(get_helm_release_name)
 if [[ -z "${operator_name}" ]]; then
     echo "GPU Operator Helm release not found in namespace ${TEST_NAMESPACE}"
-    ${HELM} list -n "${TEST_NAMESPACE}"
+    ${HELM} list -n "${TEST_NAMESPACE}" || true
     exit 1
 fi
 
diff --git a/tests/scripts/node-exec.sh b/tests/scripts/node-exec.sh
new file mode 100755
index 0000000000..7eda1fa247
--- /dev/null
+++ b/tests/scripts/node-exec.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# Copyright NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+readonly SCRIPT_DIR
+readonly NODE_OPERATIONS="${SCRIPT_DIR}/node-operations.sh"
+
+require_readable_file() {
+	local name="${1}"
+	local value="${2}"
+
+	if [[ -z "${value}" ]]; then
+		echo "Error: ${name} must be set when NODE_SSH_HOST is set" >&2
+		exit 1
+	fi
+	if [[ ! -r "${value}" ]]; then
+		echo "Error: ${name} '${value}' does not exist or is not readable" >&2
+		exit 1
+	fi
+}
+
+if [[ -z "${NODE_SSH_HOST:-}" ]]; then
+	echo "Running '$*' locally"
+	exec bash "${NODE_OPERATIONS}" "$@"
+fi
+
+require_readable_file "NODE_SSH_KEY" "${NODE_SSH_KEY:-}"
+require_readable_file "NODE_SSH_KNOWN_HOSTS" "${NODE_SSH_KNOWN_HOSTS:-}"
+
+REMOTE_ARGS=""
+if (( $# )); then
+	printf -v REMOTE_ARGS ' %q' "$@"
+fi
+
+echo "Running '$*' on ${NODE_SSH_HOST}"
+ssh -i "${NODE_SSH_KEY}" \
+	-o BatchMode=yes \
+	-o IdentitiesOnly=yes \
+	-o ConnectTimeout=30 \
+	-o StrictHostKeyChecking=accept-new \
+	-o UserKnownHostsFile="${NODE_SSH_KNOWN_HOSTS}" \
+	"${NODE_SSH_HOST}" \
+	"bash -s --${REMOTE_ARGS}" < "${NODE_OPERATIONS}"
diff --git a/tests/scripts/node-operations.sh b/tests/scripts/node-operations.sh
new file mode 100755
index 0000000000..a37fa5ad49
--- /dev/null
+++ b/tests/scripts/node-operations.sh
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+
+# Copyright NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+usage() {
+	cat <<'EOF'
+Usage: node-operations.sh <operation> [args...]
+
+Operations:
+  load-modules
+      Load the kernel modules required by the GPU Operator.
+
+  restart-operator-container <runtime>
+      Kill the running gpu-operator container so that kubernetes restarts it.
+      Supported runtimes: containerd, docker.
+EOF
+}
+
+load_modules() {
+	echo "Load kernel modules i2c_core and ipmi_msghandler"
+	sudo modprobe -a i2c_core ipmi_msghandler
+}
+
+restart_operator_container() {
+	local runtime="${1:-}"
+	local container_id=""
+
+	case "${runtime}" in
+	containerd)
+		# The operator is the only container that has the string '"gpu-operator"'
+		# TODO: This requires permissions on containerd.sock
+		container_id="$(sudo crictl ps --name gpu-operator | awk '{if(NR>1)print $1}')" || true
+		if [[ -z "${container_id}" ]]; then
+			echo "Error: no running gpu-operator container found via crictl" >&2
+			return 1
+		fi
+		sudo crictl rm --force "${container_id}"
+		;;
+	docker)
+		# The operator is the only container that has the string '"gpu-operator"'
+		container_id="$(docker ps --format '{{.ID}} {{.Command}}' | grep "gpu-operator" | cut -f 1 -d ' ')" || true
+		if [[ -z "${container_id}" ]]; then
+			echo "Error: no running gpu-operator container found via docker" >&2
+			return 1
+		fi
+		docker kill "${container_id}"
+		;;
+	*)
+		echo "Error: unknown runtime '${runtime}'. Supported runtimes: containerd, docker" >&2
+		return 1
+		;;
+	esac
+}
+
+main() {
+	if [[ $# -lt 1 ]]; then
+		usage >&2
+		exit 2
+	fi
+
+	local operation="${1}"
+	shift
+
+	case "${operation}" in
+		load-modules)
+			load_modules "$@"
+			;;
+		restart-operator-container)
+			restart_operator_container "$@"
+			;;
+		*)
+			echo "Error: unknown operation '${operation}'" >&2
+			usage >&2
+			exit 2
+			;;
+	esac
+}
+
+main "$@"
diff --git a/tests/scripts/update-clusterpolicy.sh b/tests/scripts/update-clusterpolicy.sh
index 7a53901c74..232b33dd38 100755
--- a/tests/scripts/update-clusterpolicy.sh
+++ b/tests/scripts/update-clusterpolicy.sh
@@ -150,7 +150,7 @@ test_gpu_sharing() {
     kubectl wait --for=condition=available --timeout=300s deployment/nvidia-plugin-test -n $TEST_NAMESPACE
     if [ $? -ne 0 ]; then
         echo "cannot run parallel pods with GPU sharing enabled"
-        kubectl get pods -l app=nvidia-plugin-test -n $TEST_NAMESPACE
+        kubectl get pods -l app=nvidia-plugin-test -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
         exit 1
     fi
 
@@ -243,14 +243,22 @@ test_custom_labels_override() {
   for operand in $operands
   do
     echo "checking $operand labels"
-    for pod in $(kubectl get pods -n "$TEST_NAMESPACE" -l app="$operand" --output=jsonpath={.items..metadata.name})
+    if ! operand_pods=$(kubectl get pods -n "$TEST_NAMESPACE" -l app="$operand" --output=jsonpath={.items..metadata.name} --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+      echo "cannot list $operand pods to verify the overridden labels"
+      exit 1
+    fi
+    if [ -z "$operand_pods" ]; then
+      echo "no $operand pods found when verifying the overridden labels"
+      exit 1
+    fi
+    for pod in $operand_pods
     do
-      cp_label_value=$(kubectl get pod -n "$TEST_NAMESPACE" "$pod" --output jsonpath={.metadata.labels.cloudprovider})
+      cp_label_value=$(kubectl get pod -n "$TEST_NAMESPACE" "$pod" --output jsonpath='{.metadata.labels.cloudprovider}' --request-timeout="${KUBECTL_REQUEST_TIMEOUT}")
       if [ "$cp_label_value" != "aws" ]; then
           echo "Custom Label cloudprovider is incorrect when clusterpolicy labels are overridden - $pod"
           exit 1
       fi
-      platform_label_value=$(kubectl get pod -n "$TEST_NAMESPACE" "$pod" --output jsonpath={.metadata.labels.platform})
+      platform_label_value=$(kubectl get pod -n "$TEST_NAMESPACE" "$pod" --output jsonpath='{.metadata.labels.platform}' --request-timeout="${KUBECTL_REQUEST_TIMEOUT}")
       if [ "$platform_label_value" != "kubernetes" ]; then
           echo "Custom Label platform is incorrect when clusterpolicy labels are overridden - $pod"
           exit 1
diff --git a/tests/scripts/update-nvidiadriver.sh b/tests/scripts/update-nvidiadriver.sh
index d104673654..3104e738aa 100755
--- a/tests/scripts/update-nvidiadriver.sh
+++ b/tests/scripts/update-nvidiadriver.sh
@@ -8,7 +8,6 @@ fi
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 source ${SCRIPT_DIR}/.definitions.sh
 
-# Import the check definitions
 source ${SCRIPT_DIR}/checks.sh
 
 NVIDIA_DRIVER_NAME="${NVIDIA_DRIVER_NAME:-e2e-driver}"
@@ -16,12 +15,12 @@ DEFAULT_NVIDIA_DRIVER_NAME="${DEFAULT_NVIDIA_DRIVER_NAME:-e2e-default-driver}"
 DUPLICATE_DEFAULT_NVIDIA_DRIVER_NAME="${DUPLICATE_DEFAULT_NVIDIA_DRIVER_NAME:-e2e-duplicate-default-driver}"
 
 get_default_nvidiadriver_name() {
-    kubectl get nvidiadriver -o json |
+    kubectl get nvidiadriver -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" |
         jq -r '.items[] | select(.spec.default == true) | .metadata.name' | head -n 1
 }
 
 get_default_nvidiadriver_count() {
-    kubectl get nvidiadriver -o json |
+    kubectl get nvidiadriver -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" |
         jq '[.items[] | select(.spec.default == true)] | length'
 }
 
@@ -56,7 +55,7 @@ set_default_driver() {
 
 wait_for_default_nvidiadriver() {
     local expected_name=$1
-    local current_time=0
+    local deadline=$((SECONDS + 120))
 
     echo "Waiting for NVIDIADriver/${expected_name} to be the only default"
     while :; do
@@ -67,14 +66,13 @@ wait_for_default_nvidiadriver() {
             break
         fi
 
-        if [[ "${current_time}" -gt 120 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${expected_name} to be the only default"
-            kubectl get nvidiadriver
+            kubectl get nvidiadriver --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 
@@ -83,7 +81,7 @@ test_arbitrary_name_default_nvidiadriver() {
     current_default=$(get_default_nvidiadriver_name)
     if [[ -z "${current_default}" ]]; then
         echo "default NVIDIADriver not found"
-        kubectl get nvidiadriver
+        kubectl get nvidiadriver --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
         exit 1
     fi
 
@@ -110,7 +108,7 @@ create_nvidiadriver() {
     default_name=$(get_default_nvidiadriver_name)
     if [[ -z "${default_name}" ]]; then
         echo "default NVIDIADriver not found"
-        kubectl get nvidiadriver
+        kubectl get nvidiadriver --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
         exit 1
     fi
 
@@ -120,41 +118,54 @@ create_nvidiadriver() {
 
 wait_for_nvidiadriver_owner() {
     local driver_name=$1
-    local current_time=0
-    local gpu_node_count
+    local deadline=$((SECONDS + 60 * 15))
+    local gpu_node_count=""
+    local node_count=""
+    local owned_count
 
-    gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
-    echo "Waiting for ${gpu_node_count} GPU node(s) to be owned by NVIDIADriver/${driver_name}"
+    echo "Waiting for the GPU node(s) to be owned by NVIDIADriver/${driver_name}"
 
     while :; do
-        owned_count=$(kubectl get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers | wc -l)
-        if [[ "${owned_count}" -eq "${gpu_node_count}" ]]; then
-            echo "All GPU nodes are owned by NVIDIADriver/${driver_name}"
-            break
+        if [[ "${gpu_node_count:-0}" -le 0 ]]; then
+            if node_count=$(kubectl_count get node -l nvidia.com/gpu.present=true --no-headers); then
+                if (( node_count > 0 )); then
+                    gpu_node_count="${node_count}"
+                fi
+            else
+                api_unreachable "counting the GPU nodes"
+            fi
+        fi
+
+        if owned_count=$(kubectl_count get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers); then
+            if [[ "${gpu_node_count:-0}" -gt 0 ]] && [[ "${owned_count}" -eq "${gpu_node_count}" ]]; then
+                echo "All GPU nodes are owned by NVIDIADriver/${driver_name}"
+                break
+            fi
+        else
+            api_unreachable "counting the nodes owned by NVIDIADriver/${driver_name}"
         fi
 
-        if [[ "${current_time}" -gt $((60 * 15)) ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} ownership"
-            kubectl get nodes -l nvidia.com/gpu.present=true -o json |
-                jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv'
+            kubectl get nodes -l nvidia.com/gpu.present=true -o json --request-timeout="${KUBECTL_LOG_TIMEOUT}" |
+                jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv' || true
             exit 1
         fi
 
-        echo "NVIDIADriver/${driver_name} owns ${owned_count}/${gpu_node_count} GPU node(s)"
+        echo "NVIDIADriver/${driver_name} owns ${owned_count:-unknown}/${gpu_node_count:-unknown} GPU node(s)"
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 
 get_nvidiadriver_daemonsets() {
     local driver_name=$1
-    kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o json |
+    kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" |
         jq --arg driver_name "${driver_name}" '.items | map(select(.spec.template.spec.nodeSelector["nvidia.com/gpu-operator.driver.owner"] == $driver_name))'
 }
 
 wait_for_nvidiadriver_daemonsets() {
     local driver_name=$1
-    local current_time=0
+    local deadline=$((SECONDS + 60 * 15))
 
     echo "Waiting for daemonsets owned by NVIDIADriver/${driver_name}"
     while :; do
@@ -164,47 +175,41 @@ wait_for_nvidiadriver_daemonsets() {
             break
         fi
 
-        if [[ "${current_time}" -gt $((60 * 15)) ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for daemonsets owned by NVIDIADriver/${driver_name}"
-            kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o yaml
+            kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o yaml --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 
 test_driver_image_updates() {
-    # Update driver image version
     kubectl patch nvidiadriver/"${NVIDIA_DRIVER_NAME}" --type='json' -p='[{"op": "replace", "path": "/spec/version", "value": '"$TARGET_DRIVER_VERSION"'}]'
     if [ "$?" -ne 0 ]; then
         echo "cannot update driver image with version $TARGET_DRIVER_VERSION for driver-daemonset"
         exit 1
     fi
 
-    # Verify update is applied to Driver Daemonset
-    local current_time=0
+    local deadline=$((SECONDS + 120))
     while :; do
         if get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" | jq -e --arg version "${TARGET_DRIVER_VERSION}" 'length > 0 and all(.[]; .spec.template.spec.containers[0].image | contains($version))' >/dev/null; then
             break
         fi
 
-        if [[ "${current_time}" -gt 120 ]]; then
+        if (( SECONDS > deadline )); then
             echo "Image update failed for driver daemonset to version $TARGET_DRIVER_VERSION"
-            get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}"
+            get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" || true
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
     echo "driver daemonset image updated successfully to version $TARGET_DRIVER_VERSION"
 
-    # Delete driver pod to trigger update due to OnDelete policy
     kubectl delete pod -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE"
 
-    # Wait for the driver upgrade to transition to "upgrade-done" state
     wait_for_driver_upgrade_done
     
     echo "ensuring that the new driver pods with version $TARGET_DRIVER_VERSION come up successfully"
@@ -221,39 +226,46 @@ test_custom_labels_override() {
     exit 1
   fi
 
-  # Wait for the operator to update the pod template with new labels
   echo "Waiting for DaemonSet pod template to be updated with new labels..."
-  local current_time=0
+  local deadline=$((SECONDS + 120))
   while :; do
     if get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" | jq -e 'length > 0 and all(.[]; .spec.template.metadata.labels.cloudprovider == "aws" and .spec.template.metadata.labels.platform == "kubernetes")' >/dev/null; then
       break
     fi
 
-    if [[ "${current_time}" -gt 120 ]]; then
+    if (( SECONDS > deadline )); then
       echo "timeout reached waiting for DaemonSet pod template labels"
-      get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}"
+      get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" || true
       exit 1
     fi
 
     sleep 5
-    current_time=$((${current_time} + 5))
   done
 
   # Delete driver pod to force recreation with updated labels. Existing pods are not automatically restarted due to the DaemonSet's 'OnDelete` updateStrategy.
   echo "Deleting driver pod to trigger recreation with updated labels..."
   kubectl delete pod -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE"
 
-  # Wait for the driver upgrade to transition to "upgrade-done" state
   wait_for_driver_upgrade_done
 
   check_nvidia_driver_pods_ready
 
   echo "checking nvidia-driver-daemonset labels"
-  labeled_pod_count=$(kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/component=nvidia-driver,cloudprovider=aws,platform=kubernetes" --no-headers | wc -l)
-  gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
+  if ! labeled_pod_count=$(kubectl_count get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/component=nvidia-driver,cloudprovider=aws,platform=kubernetes" --no-headers); then
+    echo "cannot count the labelled NVIDIADriver/${NVIDIA_DRIVER_NAME} pods"
+    exit 1
+  fi
+  if ! gpu_node_count=$(kubectl_count get node -l nvidia.com/gpu.present=true --no-headers); then
+    echo "cannot count the GPU nodes"
+    exit 1
+  fi
+  if [[ "${gpu_node_count}" -eq 0 ]]; then
+    echo "no GPU nodes found while verifying NVIDIADriver/${NVIDIA_DRIVER_NAME} labels"
+    exit 1
+  fi
   if [[ "${labeled_pod_count}" -ne "${gpu_node_count}" ]]; then
     echo "Custom labels are missing from one or more NVIDIADriver/${NVIDIA_DRIVER_NAME} pods"
-    kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/component=nvidia-driver" --show-labels
+    kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/component=nvidia-driver" --show-labels --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
     exit 1
   fi
 }
@@ -263,12 +275,22 @@ assert_nvidiadriver_owner_count() {
     local gpu_node_count
     local owned_count
 
-    gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
-    owned_count=$(kubectl get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers | wc -l)
+    if ! gpu_node_count=$(kubectl_count get node -l nvidia.com/gpu.present=true --no-headers); then
+        echo "cannot count the GPU nodes"
+        exit 1
+    fi
+    if ! owned_count=$(kubectl_count get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers); then
+        echo "cannot count the nodes owned by NVIDIADriver/${driver_name}"
+        exit 1
+    fi
+    if [[ "${gpu_node_count}" -eq 0 ]]; then
+        echo "no GPU nodes found while checking NVIDIADriver/${driver_name} ownership"
+        exit 1
+    fi
     if [[ "${owned_count}" -ne "${gpu_node_count}" ]]; then
         echo "Expected ${gpu_node_count} GPU node(s) to remain owned by NVIDIADriver/${driver_name}, found ${owned_count}"
-        kubectl get nodes -l nvidia.com/gpu.present=true -o json |
-            jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv'
+        kubectl get nodes -l nvidia.com/gpu.present=true -o json --request-timeout="${KUBECTL_LOG_TIMEOUT}" |
+            jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv' || true
         exit 1
     fi
 }
@@ -276,35 +298,34 @@ assert_nvidiadriver_owner_count() {
 wait_for_nvidiadriver_condition_message() {
     local driver_name=$1
     local message=$2
-    local current_time=0
+    local deadline=$((SECONDS + 120))
 
     echo "Waiting for NVIDIADriver/${driver_name} status message to contain: ${message}"
     while :; do
-        if kubectl get nvidiadriver/"${driver_name}" -o json | jq -e --arg message "${message}" '
+        if kubectl get nvidiadriver/"${driver_name}" -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" | jq -e --arg message "${message}" '
             (.status.state // "") == "notReady" and
             ([.status.conditions[]?.message // ""] | any(contains($message)))
         ' >/dev/null; then
             break
         fi
 
-        if [[ "${current_time}" -gt 120 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} status message"
-            kubectl get nvidiadriver/"${driver_name}" -o yaml
+            kubectl get nvidiadriver/"${driver_name}" -o yaml --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 
 wait_for_nvidiadriver_ready() {
     local driver_name=$1
-    local current_time=0
+    local deadline=$((SECONDS + 120))
 
     echo "Waiting for NVIDIADriver/${driver_name} to report Ready"
     while :; do
-        if kubectl get nvidiadriver/"${driver_name}" -o json | jq -e '
+        if kubectl get nvidiadriver/"${driver_name}" -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" | jq -e '
             (.status.state // "") == "ready" and
             ([.status.conditions[]? | select(.type == "Ready" and .status == "True")] | length > 0) and
             ([.status.conditions[]? | select(.type == "Error" and .status == "True")] | length == 0)
@@ -312,14 +333,13 @@ wait_for_nvidiadriver_ready() {
             break
         fi
 
-        if [[ "${current_time}" -gt 120 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} to report Ready"
-            kubectl get nvidiadriver/"${driver_name}" -o yaml
+            kubectl get nvidiadriver/"${driver_name}" -o yaml --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 

From c1aa25139696523ff6f5f0c5c1024fe3812f2980 Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Mon, 31 Aug 2026 18:59:27 -0700
Subject: [PATCH 2/2] ci: probe why the cluster became unreachable

A containerd run lost the API server twenty seconds after the driver pod
began reloading host kernel modules, and never got it back: 45 minutes of
retries, and the post-run diagnostics still could not connect.

Two very different causes fit that evidence equally well. Either the node
lost its network during the module reload, or the runner's egress address
changed and no longer matches the security group rule Holodeck opened for
it. The logs cannot separate them because nothing touches the node after
the kernel modules are loaded, so a silent API server is the only symptom
we ever see.

Record the egress address while things still work, then on every run
report it again alongside a direct readyz check, an ssh probe of the node
on port 22, and the instance state from EC2. Whichever of those still
answers tells us which side broke. It runs on always() so a passing run
leaves a baseline to compare a failing one against, and every probe is
bounded and swallowed so it cannot change the result of the job.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 .github/workflows/e2e-tests.yaml | 70 ++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 87975dd350..8d0a237c0f 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -156,6 +156,8 @@ jobs:
             echo "NODE_SSH_HOST=ubuntu@${PUBLIC_DNS_NAME}"
             echo "NODE_SSH_KEY=${SSH_DIR}/id_rsa"
             echo "NODE_SSH_KNOWN_HOSTS=${SSH_DIR}/known_hosts"
+            echo "NODE_PUBLIC_DNS_NAME=${PUBLIC_DNS_NAME}"
+            echo "EGRESS_IP_AT_START=$(curl -fsS --max-time 15 https://api.ipify.org || echo unknown)"
           } >> "$GITHUB_ENV"
       - name: Verify cluster access
         run: |
@@ -194,6 +196,39 @@ jobs:
           kubectl get pods -A -o wide --request-timeout=15s > "${LOG_DIR}/pods.txt" 2>&1 || true
           kubectl get events -A --sort-by=.lastTimestamp --request-timeout=15s > "${LOG_DIR}/events.txt" 2>&1 || true
           timeout 30s helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
+      - name: Diagnose cluster reachability
+        if: always()
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-west-1
+        run: |
+          mkdir -p "${LOG_DIR}"
+          {
+            echo "runner egress IP at start: ${EGRESS_IP_AT_START:-unknown}"
+            echo "runner egress IP now:      $(curl -fsS --max-time 15 https://api.ipify.org || echo unavailable)"
+            echo
+            echo "--- kube API over HTTPS ---"
+            kubectl get --raw='/readyz' --request-timeout=15s || echo "kube API unreachable"
+            echo
+            echo "--- node over SSH (port 22) ---"
+            timeout 45 ssh -i "${NODE_SSH_KEY}" \
+              -o BatchMode=yes -o IdentitiesOnly=yes -o ConnectTimeout=15 \
+              -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="${NODE_SSH_KNOWN_HOSTS}" \
+              "${NODE_SSH_HOST}" 'uptime; ip -br addr; systemctl is-active kubelet containerd' \
+              || echo "ssh probe failed"
+            echo
+            echo "--- EC2 instance state ---"
+            if command -v aws >/dev/null; then
+              aws ec2 describe-instances \
+                --filters "Name=dns-name,Values=${NODE_PUBLIC_DNS_NAME}" \
+                --query 'Reservations[].Instances[].{Id:InstanceId,State:State.Name,Launch:LaunchTime}' \
+                --output json || echo "describe-instances failed"
+            else
+              echo "aws cli not installed on the runner"
+            fi
+          } > "${LOG_DIR}/reachability.txt" 2>&1 || true
+          cat "${LOG_DIR}/reachability.txt"
       - name: Archive test logs
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
@@ -282,6 +317,8 @@ jobs:
             echo "NODE_SSH_HOST=ubuntu@${PUBLIC_DNS_NAME}"
             echo "NODE_SSH_KEY=${SSH_DIR}/id_rsa"
             echo "NODE_SSH_KNOWN_HOSTS=${SSH_DIR}/known_hosts"
+            echo "NODE_PUBLIC_DNS_NAME=${PUBLIC_DNS_NAME}"
+            echo "EGRESS_IP_AT_START=$(curl -fsS --max-time 15 https://api.ipify.org || echo unknown)"
           } >> "$GITHUB_ENV"
       - name: Verify cluster access
         run: |
@@ -320,6 +357,39 @@ jobs:
           kubectl get pods -A -o wide --request-timeout=15s > "${LOG_DIR}/pods.txt" 2>&1 || true
           kubectl get events -A --sort-by=.lastTimestamp --request-timeout=15s > "${LOG_DIR}/events.txt" 2>&1 || true
           timeout 30s helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
+      - name: Diagnose cluster reachability
+        if: always()
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-west-1
+        run: |
+          mkdir -p "${LOG_DIR}"
+          {
+            echo "runner egress IP at start: ${EGRESS_IP_AT_START:-unknown}"
+            echo "runner egress IP now:      $(curl -fsS --max-time 15 https://api.ipify.org || echo unavailable)"
+            echo
+            echo "--- kube API over HTTPS ---"
+            kubectl get --raw='/readyz' --request-timeout=15s || echo "kube API unreachable"
+            echo
+            echo "--- node over SSH (port 22) ---"
+            timeout 45 ssh -i "${NODE_SSH_KEY}" \
+              -o BatchMode=yes -o IdentitiesOnly=yes -o ConnectTimeout=15 \
+              -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="${NODE_SSH_KNOWN_HOSTS}" \
+              "${NODE_SSH_HOST}" 'uptime; ip -br addr; systemctl is-active kubelet containerd' \
+              || echo "ssh probe failed"
+            echo
+            echo "--- EC2 instance state ---"
+            if command -v aws >/dev/null; then
+              aws ec2 describe-instances \
+                --filters "Name=dns-name,Values=${NODE_PUBLIC_DNS_NAME}" \
+                --query 'Reservations[].Instances[].{Id:InstanceId,State:State.Name,Launch:LaunchTime}' \
+                --output json || echo "describe-instances failed"
+            else
+              echo "aws cli not installed on the runner"
+            fi
+          } > "${LOG_DIR}/reachability.txt" 2>&1 || true
+          cat "${LOG_DIR}/reachability.txt"
       - name: Archive test logs
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7