From 980ff16ae5112c521787d60970325d4f384a0717 Mon Sep 17 00:00:00 2001
From: Danny Avila <danny@librechat.ai>
Date: Tue, 4 Aug 2026 08:41:22 -0400
Subject: [PATCH 1/7] fix: recycle sandboxes before clock drift

---
 .github/workflows/ci.yml                      |   3 +
 docker-compose.scalable.yml                   |   6 +-
 docker/Dockerfile.worker-sandbox              |   7 +-
 docker/sandbox-runner-healthcheck.sh          | 106 +++++++++++++-
 .../templates/worker-sandbox-deployment.yaml  |  12 ++
 helm/codeapi/values.yaml                      |   8 ++
 tests/block_root_package_delivery.sh          |  71 +++++++++
 tests/sandbox_runner_healthcheck.sh           | 136 ++++++++++++++++++
 8 files changed, 340 insertions(+), 9 deletions(-)
 create mode 100755 tests/sandbox_runner_healthcheck.sh

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a8318f0..7ec8559 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -31,6 +31,9 @@ jobs:
       - name: Block-root package delivery
         run: tests/block_root_package_delivery.sh
 
+      - name: Sandbox-runner liveness checks
+        run: tests/sandbox_runner_healthcheck.sh
+
       - name: Validate sandbox Dockerfiles
         run: |
           docker buildx build --check -f api/Dockerfile .
diff --git a/docker-compose.scalable.yml b/docker-compose.scalable.yml
index 49c8914..8f93bf3 100644
--- a/docker-compose.scalable.yml
+++ b/docker-compose.scalable.yml
@@ -221,7 +221,11 @@ services:
     deploy:
       replicas: 3  # Scale based on queue depth
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:3113/health"]
+      test:
+        - CMD-SHELL
+        - >-
+          /usr/local/bin/sandbox-runner-healthcheck.sh &&
+          curl -fsS --max-time 5 http://localhost:3113/health >/dev/null
       interval: 10s
       timeout: 5s
       retries: 3
diff --git a/docker/Dockerfile.worker-sandbox b/docker/Dockerfile.worker-sandbox
index cd3edd3..d972906 100644
--- a/docker/Dockerfile.worker-sandbox
+++ b/docker/Dockerfile.worker-sandbox
@@ -235,7 +235,8 @@ COPY --from=launcher-builder /launcher/target/release/sandbox-launcher /usr/loca
 # --- Launcher entrypoint (DNS resolution + socat relay before VM boot) ---
 COPY launcher/entrypoint.sh /usr/local/bin/launcher-entrypoint.sh
 COPY docker/start-direct-sandbox.sh /usr/local/bin/start-direct-sandbox.sh
-RUN chmod +x /usr/local/bin/launcher-entrypoint.sh /usr/local/bin/start-direct-sandbox.sh
+COPY docker/sandbox-runner-healthcheck.sh /usr/local/bin/sandbox-runner-healthcheck.sh
+RUN chmod +x /usr/local/bin/launcher-entrypoint.sh /usr/local/bin/start-direct-sandbox.sh /usr/local/bin/sandbox-runner-healthcheck.sh
 
 # --- Worker ---
 WORKDIR /worker
@@ -253,7 +254,9 @@ WORKDIR /
 EXPOSE 2000 3113
 
 HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \
-  CMD curl -f http://localhost:3113/health || exit 1
+  CMD /usr/local/bin/sandbox-runner-healthcheck.sh \
+    && curl -fsS --max-time 5 http://localhost:3113/health >/dev/null \
+    || exit 1
 
 CMD ["/supervisor.sh"]
 
diff --git a/docker/sandbox-runner-healthcheck.sh b/docker/sandbox-runner-healthcheck.sh
index fdb03de..8c72e09 100755
--- a/docker/sandbox-runner-healthcheck.sh
+++ b/docker/sandbox-runner-healthcheck.sh
@@ -2,16 +2,63 @@
 set -euo pipefail
 
 fd_limit="${SANDBOX_RUNNER_FD_LIVENESS_LIMIT:-40000}"
+clock_skew_limit_seconds="${SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS:-10}"
+clock_skew_jitter_seconds="${SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS:-2}"
 timeout_seconds="${SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS:-5}"
 port="${PORT:-2000}"
 url="${SANDBOX_RUNNER_HEALTHCHECK_URL:-http://127.0.0.1:${port}/api/v2/runtimes}"
 
-case "$fd_limit" in
-    ''|*[!0-9]*)
-        echo "invalid SANDBOX_RUNNER_FD_LIVENESS_LIMIT: $fd_limit" >&2
+validate_non_negative_integer() {
+    local name="$1"
+    local value="$2"
+
+    case "$value" in
+        ''|*[!0-9]*)
+            echo "invalid ${name}: ${value}" >&2
+            exit 2
+            ;;
+    esac
+}
+
+validate_non_negative_integer SANDBOX_RUNNER_FD_LIVENESS_LIMIT "$fd_limit"
+validate_non_negative_integer \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS \
+    "$clock_skew_limit_seconds"
+validate_non_negative_integer \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS \
+    "$clock_skew_jitter_seconds"
+
+if [ "$clock_skew_limit_seconds" -ge 30 ]; then
+    echo "SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS must be less than the 30-second execution-manifest tolerance" >&2
+    exit 2
+fi
+
+if [ "$clock_skew_limit_seconds" -gt 0 ] && ! command -v date >/dev/null 2>&1; then
+    echo "sandbox-runner clock-skew check requires date" >&2
+    exit 2
+fi
+
+effective_clock_skew_limit_seconds="$clock_skew_limit_seconds"
+if [ "$clock_skew_limit_seconds" -gt 0 ] && [ "$clock_skew_jitter_seconds" -gt 0 ]; then
+    if [ "$clock_skew_jitter_seconds" -ge "$clock_skew_limit_seconds" ]; then
+        echo "SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS must be less than SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS" >&2
         exit 2
-        ;;
-esac
+    fi
+    if ! command -v cksum >/dev/null 2>&1; then
+        echo "sandbox-runner clock-skew jitter requires cksum" >&2
+        exit 2
+    fi
+
+    # Pods from one rollout have nearly identical ages and drift rates. Give
+    # each pod a stable threshold within the configured range so they recycle
+    # across different days instead of all restarting together.
+    jitter_key="${HOSTNAME:-sandbox-runner}"
+    jitter_checksum=$(printf '%s' "$jitter_key" | cksum)
+    jitter_checksum="${jitter_checksum%% *}"
+    effective_clock_skew_limit_seconds=$((
+        clock_skew_limit_seconds - (jitter_checksum % (clock_skew_jitter_seconds + 1))
+    ))
+fi
 
 if [ "$fd_limit" -gt 0 ]; then
     fd_count=$(find /proc/1/fd -mindepth 1 -maxdepth 1 2>/dev/null | wc -l | tr -d '[:space:]')
@@ -21,4 +68,51 @@ if [ "$fd_limit" -gt 0 ]; then
     fi
 fi
 
-curl -fsS --max-time "$timeout_seconds" "$url" >/dev/null
+if [ "$clock_skew_limit_seconds" -eq 0 ]; then
+    curl -fsS --max-time "$timeout_seconds" "$url" >/dev/null
+    exit 0
+fi
+
+host_before_seconds=$(date -u +%s)
+response_headers=$(curl -fsS --max-time "$timeout_seconds" --dump-header - --output /dev/null "$url")
+host_after_seconds=$(date -u +%s)
+
+guest_date=$(printf '%s\n' "$response_headers" | awk '
+    tolower($1) == "date:" {
+        sub(/\r$/, "")
+        sub(/^[^:]*:[[:space:]]*/, "")
+        value = $0
+    }
+    END { print value }
+')
+
+if [ -z "$guest_date" ]; then
+    echo "sandbox-runner unhealthy: guest response is missing the HTTP Date header" >&2
+    exit 1
+fi
+
+if ! guest_seconds=$(date -u -d "$guest_date" +%s 2>/dev/null); then
+    echo "sandbox-runner unhealthy: guest returned an invalid HTTP Date header: ${guest_date}" >&2
+    exit 1
+fi
+
+# The response can take time to arrive, so compare guest time with the host
+# interval that enclosed the request instead of with a single sample. This
+# prevents probe latency from looking like backward clock drift.
+if [ "$guest_seconds" -lt "$host_before_seconds" ]; then
+    skew_seconds=$((guest_seconds - host_before_seconds))
+elif [ "$guest_seconds" -gt "$host_after_seconds" ]; then
+    skew_seconds=$((guest_seconds - host_after_seconds))
+else
+    skew_seconds=0
+fi
+
+absolute_skew_seconds="$skew_seconds"
+if [ "$absolute_skew_seconds" -lt 0 ]; then
+    absolute_skew_seconds=$((-absolute_skew_seconds))
+fi
+
+if [ "$absolute_skew_seconds" -ge "$effective_clock_skew_limit_seconds" ]; then
+    echo "sandbox-runner unhealthy: guest clock skew is ${skew_seconds}s, pod limit is ${effective_clock_skew_limit_seconds}s (configured maximum ${clock_skew_limit_seconds}s)" >&2
+    exit 1
+fi
diff --git a/helm/codeapi/templates/worker-sandbox-deployment.yaml b/helm/codeapi/templates/worker-sandbox-deployment.yaml
index 8a82a61..1af386c 100644
--- a/helm/codeapi/templates/worker-sandbox-deployment.yaml
+++ b/helm/codeapi/templates/worker-sandbox-deployment.yaml
@@ -27,6 +27,14 @@ Split runtime deployments:
 {{- if and (not $packagesFromPvc) (not .Values.workerSandbox.kvmEnabled) }}
 {{- fail "workerSandbox.packages.source=image requires workerSandbox.kvmEnabled=true because the baked runner boots from a libkrun block root image" }}
 {{- end }}
+{{- $clockSkewLimit := int .Values.workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds }}
+{{- $clockSkewJitter := int .Values.workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds }}
+{{- if or (lt $clockSkewLimit 0) (ge $clockSkewLimit 30) }}
+{{- fail "workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds must be between 0 and 29 so it stays below the 30-second execution-manifest tolerance" }}
+{{- end }}
+{{- if or (lt $clockSkewJitter 0) (and (gt $clockSkewLimit 0) (ge $clockSkewJitter $clockSkewLimit)) }}
+{{- fail "workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds must be non-negative and less than clockSkewLivenessLimitSeconds when the clock-skew guard is enabled" }}
+{{- end }}
 {{- range .Values.workerSandbox.sandboxExtraEnv }}
 {{- $name := .name | default "" }}
 {{- if and $name (or (regexMatch "(?i)(SECRET|TOKEN|PASSWORD|KEY)" $name) (hasPrefix "REDIS_" $name) (hasPrefix "AWS_" $name) (hasPrefix "S3_" $name) (hasPrefix "MINIO_" $name) (hasPrefix "CODEAPI_" $name) (eq $name "FILE_SERVER_URL") (eq $name "TOOL_CALL_SERVER_URL")) }}
@@ -258,6 +266,10 @@ spec:
               value: {{ .Values.workerSandbox.launcher.filterVsockEnotconn | quote }}
             - name: SANDBOX_RUNNER_FD_LIVENESS_LIMIT
               value: {{ .Values.workerSandbox.sandboxRunner.fdLivenessLimit | quote }}
+            - name: SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS
+              value: {{ .Values.workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds | quote }}
+            - name: SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS
+              value: {{ .Values.workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds | quote }}
             - name: EGRESS_GATEWAY_URL
               value: "http://{{ include "codeapi.fullname" . }}-egress-gateway:{{ .Values.egressGateway.service.port }}"
             - name: SANDBOX_LOG_LEVEL
diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml
index c926e76..05a957b 100644
--- a/helm/codeapi/values.yaml
+++ b/helm/codeapi/values.yaml
@@ -200,6 +200,14 @@ workerSandbox:
     # Restart a sandbox-runner before launcher/VMM host-side fd retention can
     # exhaust the process file descriptor limit. Set 0 to disable.
     fdLivenessLimit: 40000
+    # Restart a sandbox-runner when its microVM guest wall clock differs from
+    # the host by this many seconds. This must remain comfortably below the
+    # 30-second execution-manifest clock tolerance. Set 0 to disable.
+    clockSkewLivenessLimitSeconds: 10
+    # Subtract a stable 0..N-second offset from each pod's clock-skew limit so
+    # replicas created together do not all recycle at once. Must be less than
+    # clockSkewLivenessLimitSeconds.
+    clockSkewLivenessJitterSeconds: 2
     inheritSharedScheduling: true
     nodeSelector: {}
     tolerations: []
diff --git a/tests/block_root_package_delivery.sh b/tests/block_root_package_delivery.sh
index c549533..721a5af 100755
--- a/tests/block_root_package_delivery.sh
+++ b/tests/block_root_package_delivery.sh
@@ -36,6 +36,23 @@ assert_not_contains() {
     fi
 }
 
+assert_env_value() {
+    local file="$1"
+    local name="$2"
+    local expected="$3"
+    local message="$4"
+    if ! awk -v name="$name" -v expected="$expected" '
+        $0 ~ "- name: " name {
+            getline
+            if ($0 ~ "value: \\\"" expected "\\\"") found = 1
+        }
+        END { exit found ? 0 : 1 }
+    ' "$file"; then
+        echo "$message" >&2
+        exit 1
+    fi
+}
+
 assert_compose_mode() {
     local compose_file="$1"
     local service="$2"
@@ -139,6 +156,10 @@ assert_contains \
     "$ROOT/docker/Dockerfile.worker-sandbox" \
     '^FROM worker-sandbox-legacy AS worker-sandbox-false$' \
     "worker-sandbox-false must inherit the direct target"
+assert_contains \
+    "$ROOT/docker/Dockerfile.worker-sandbox" \
+    'sandbox-runner-healthcheck.sh' \
+    "the combined worker image must check guest clock skew"
 
 awk '
     /^FROM / {
@@ -196,6 +217,16 @@ assert_not_contains \
     "$TMP_DIR/helm-image.yaml" \
     'app.kubernetes.io/component: package-init' \
     "default Helm render must not create package-init"
+assert_env_value \
+    "$TMP_DIR/helm-image.yaml" \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS \
+    10 \
+    "default Helm render must keep clock skew below the manifest tolerance"
+assert_env_value \
+    "$TMP_DIR/helm-image.yaml" \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS \
+    2 \
+    "default Helm render must stagger clock-skew recycling across replicas"
 
 helm template codeapi "$TMP_DIR/chart" \
     --set executionManifest.privateKey=test \
@@ -243,6 +274,46 @@ assert_contains \
     'value: "0"' \
     "fdLivenessLimit=0 must reach the healthcheck instead of reverting to 40000"
 
+helm template codeapi "$TMP_DIR/chart" \
+    --set executionManifest.privateKey=test \
+    --set executionManifest.publicKey=test \
+    --set workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds=0 \
+    > "$TMP_DIR/helm-no-clock-skew-liveness.yaml"
+
+assert_env_value \
+    "$TMP_DIR/helm-no-clock-skew-liveness.yaml" \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS \
+    0 \
+    "clockSkewLivenessLimitSeconds=0 must reach the healthcheck instead of reverting to 10"
+
+if helm template codeapi "$TMP_DIR/chart" \
+    --set executionManifest.privateKey=test \
+    --set executionManifest.publicKey=test \
+    --set workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds=30 \
+    > "$TMP_DIR/invalid-clock-skew-limit.yaml" 2> "$TMP_DIR/invalid-clock-skew-limit.log"; then
+    echo "clock-skew liveness limit must stay below manifest tolerance" >&2
+    exit 1
+fi
+
+assert_contains \
+    "$TMP_DIR/invalid-clock-skew-limit.log" \
+    'clockSkewLivenessLimitSeconds must be between 0 and 29' \
+    "clock-skew limit validation failed for an unexpected reason"
+
+if helm template codeapi "$TMP_DIR/chart" \
+    --set executionManifest.privateKey=test \
+    --set executionManifest.publicKey=test \
+    --set workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds=10 \
+    > "$TMP_DIR/invalid-clock-skew-jitter.yaml" 2> "$TMP_DIR/invalid-clock-skew-jitter.log"; then
+    echo "clock-skew liveness jitter must stay below the configured limit" >&2
+    exit 1
+fi
+
+assert_contains \
+    "$TMP_DIR/invalid-clock-skew-jitter.log" \
+    'clockSkewLivenessJitterSeconds must be non-negative and less than clockSkewLivenessLimitSeconds' \
+    "clock-skew jitter validation failed for an unexpected reason"
+
 if helm template codeapi "$TMP_DIR/chart" \
     --set executionManifest.privateKey=test \
     --set executionManifest.publicKey=test \
diff --git a/tests/sandbox_runner_healthcheck.sh b/tests/sandbox_runner_healthcheck.sh
new file mode 100755
index 0000000..5c844ff
--- /dev/null
+++ b/tests/sandbox_runner_healthcheck.sh
@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+TMP_DIR="$(mktemp -d)"
+trap 'rm -rf "$TMP_DIR"' EXIT
+
+mkdir -p "$TMP_DIR/bin"
+
+cat > "$TMP_DIR/bin/curl" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ "${TEST_CURL_FAIL:-false}" == "true" ]]; then
+    exit 22
+fi
+
+printf 'HTTP/1.1 200 OK\r\n'
+if [[ -n "${TEST_GUEST_DATE:-}" ]]; then
+    printf 'Date: %s\r\n' "$TEST_GUEST_DATE"
+fi
+printf '\r\n'
+EOF
+
+cat > "$TMP_DIR/bin/date" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ "$#" -eq 2 && "$1" == "-u" && "$2" == "+%s" ]]; then
+    printf '%s\n' "${TEST_HOST_SECONDS:?}"
+    exit 0
+fi
+
+if [[ "$#" -eq 4 && "$1" == "-u" && "$2" == "-d" && "$4" == "+%s" ]]; then
+    if [[ "${TEST_GUEST_DATE_INVALID:-false}" == "true" ]]; then
+        exit 1
+    fi
+    printf '%s\n' "${TEST_GUEST_SECONDS:?}"
+    exit 0
+fi
+
+printf 'unexpected date arguments: %q\n' "$*" >&2
+exit 2
+EOF
+
+cat > "$TMP_DIR/bin/cksum" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+
+cat >/dev/null
+printf '%s 1\n' "${TEST_CKSUM:-0}"
+EOF
+
+chmod +x "$TMP_DIR/bin/curl" "$TMP_DIR/bin/date" "$TMP_DIR/bin/cksum"
+
+run_check() {
+    env \
+        PATH="$TMP_DIR/bin:$PATH" \
+        SANDBOX_RUNNER_FD_LIVENESS_LIMIT=0 \
+        SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS=0 \
+        TEST_HOST_SECONDS=100 \
+        TEST_GUEST_DATE='Tue, 04 Aug 2026 08:00:00 GMT' \
+        "$@" \
+        bash "$ROOT/docker/sandbox-runner-healthcheck.sh"
+}
+
+run_check \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \
+    TEST_GUEST_SECONDS=91
+
+if run_check \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \
+    TEST_GUEST_SECONDS=90 2> "$TMP_DIR/backward.log"; then
+    echo "healthcheck accepted guest clock at the backward-skew limit" >&2
+    exit 1
+fi
+grep -F 'guest clock skew is -10s, pod limit is 10s (configured maximum 10s)' "$TMP_DIR/backward.log" >/dev/null
+
+if run_check \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \
+    TEST_GUEST_SECONDS=110 2> "$TMP_DIR/forward.log"; then
+    echo "healthcheck accepted guest clock at the forward-skew limit" >&2
+    exit 1
+fi
+grep -F 'guest clock skew is 10s, pod limit is 10s (configured maximum 10s)' "$TMP_DIR/forward.log" >/dev/null
+
+if run_check \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS=2 \
+    TEST_CKSUM=2 \
+    TEST_GUEST_SECONDS=92 2> "$TMP_DIR/jitter.log"; then
+    echo "healthcheck did not apply its deterministic per-pod threshold" >&2
+    exit 1
+fi
+grep -F 'guest clock skew is -8s, pod limit is 8s (configured maximum 10s)' "$TMP_DIR/jitter.log" >/dev/null
+
+if run_check \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \
+    TEST_GUEST_DATE= \
+    TEST_GUEST_SECONDS=100 2> "$TMP_DIR/missing.log"; then
+    echo "healthcheck accepted a response without a Date header" >&2
+    exit 1
+fi
+grep -F 'guest response is missing the HTTP Date header' "$TMP_DIR/missing.log" >/dev/null
+
+if run_check \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \
+    TEST_GUEST_DATE_INVALID=true \
+    TEST_GUEST_SECONDS=100 2> "$TMP_DIR/malformed.log"; then
+    echo "healthcheck accepted a malformed Date header" >&2
+    exit 1
+fi
+grep -F 'guest returned an invalid HTTP Date header' "$TMP_DIR/malformed.log" >/dev/null
+
+run_check \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=0 \
+    TEST_GUEST_DATE= \
+    TEST_GUEST_SECONDS=100
+
+if run_check \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=invalid \
+    TEST_GUEST_SECONDS=100 2> "$TMP_DIR/invalid.log"; then
+    echo "healthcheck accepted an invalid clock-skew limit" >&2
+    exit 1
+fi
+grep -F 'invalid SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS: invalid' "$TMP_DIR/invalid.log" >/dev/null
+
+if run_check \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=30 \
+    TEST_GUEST_SECONDS=100 2> "$TMP_DIR/unsafe-limit.log"; then
+    echo "healthcheck accepted a limit at the manifest tolerance" >&2
+    exit 1
+fi
+grep -F 'must be less than the 30-second execution-manifest tolerance' "$TMP_DIR/unsafe-limit.log" >/dev/null
+
+echo "sandbox-runner healthcheck checks passed"

From ad8b8f365aeaba2e9ba7a5349d3916548dafb22c Mon Sep 17 00:00:00 2001
From: Danny Avila <danny@librechat.ai>
Date: Tue, 4 Aug 2026 08:43:25 -0400
Subject: [PATCH 2/7] test: strengthen clock healthcheck coverage

---
 docker/sandbox-runner-healthcheck.sh |  4 ++--
 tests/sandbox_runner_healthcheck.sh  | 28 ++++++++++++++++++++++++++--
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/docker/sandbox-runner-healthcheck.sh b/docker/sandbox-runner-healthcheck.sh
index 8c72e09..f02c7ec 100755
--- a/docker/sandbox-runner-healthcheck.sh
+++ b/docker/sandbox-runner-healthcheck.sh
@@ -50,8 +50,8 @@ if [ "$clock_skew_limit_seconds" -gt 0 ] && [ "$clock_skew_jitter_seconds" -gt 0
     fi
 
     # Pods from one rollout have nearly identical ages and drift rates. Give
-    # each pod a stable threshold within the configured range so they recycle
-    # across different days instead of all restarting together.
+    # each pod a stable threshold within the configured range to reduce the
+    # chance that same-age replicas restart together.
     jitter_key="${HOSTNAME:-sandbox-runner}"
     jitter_checksum=$(printf '%s' "$jitter_key" | cksum)
     jitter_checksum="${jitter_checksum%% *}"
diff --git a/tests/sandbox_runner_healthcheck.sh b/tests/sandbox_runner_healthcheck.sh
index 5c844ff..00a3530 100755
--- a/tests/sandbox_runner_healthcheck.sh
+++ b/tests/sandbox_runner_healthcheck.sh
@@ -27,7 +27,12 @@ cat > "$TMP_DIR/bin/date" <<'EOF'
 set -euo pipefail
 
 if [[ "$#" -eq 2 && "$1" == "-u" && "$2" == "+%s" ]]; then
-    printf '%s\n' "${TEST_HOST_SECONDS:?}"
+    if [[ -e "${TEST_HOST_SAMPLE_STATE:?}" ]]; then
+        printf '%s\n' "${TEST_HOST_AFTER_SECONDS:?}"
+    else
+        : > "$TEST_HOST_SAMPLE_STATE"
+        printf '%s\n' "${TEST_HOST_BEFORE_SECONDS:?}"
+    fi
     exit 0
 fi
 
@@ -54,11 +59,14 @@ EOF
 chmod +x "$TMP_DIR/bin/curl" "$TMP_DIR/bin/date" "$TMP_DIR/bin/cksum"
 
 run_check() {
+    rm -f "$TMP_DIR/host-sample"
     env \
         PATH="$TMP_DIR/bin:$PATH" \
         SANDBOX_RUNNER_FD_LIVENESS_LIMIT=0 \
         SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS=0 \
-        TEST_HOST_SECONDS=100 \
+        TEST_HOST_BEFORE_SECONDS=100 \
+        TEST_HOST_AFTER_SECONDS=100 \
+        TEST_HOST_SAMPLE_STATE="$TMP_DIR/host-sample" \
         TEST_GUEST_DATE='Tue, 04 Aug 2026 08:00:00 GMT' \
         "$@" \
         bash "$ROOT/docker/sandbox-runner-healthcheck.sh"
@@ -68,6 +76,14 @@ run_check \
     SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \
     TEST_GUEST_SECONDS=91
 
+# A guest timestamp inside the host interval is healthy even when either
+# individual host sample differs by more than the configured limit.
+run_check \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=1 \
+    TEST_HOST_BEFORE_SECONDS=100 \
+    TEST_HOST_AFTER_SECONDS=104 \
+    TEST_GUEST_SECONDS=102
+
 if run_check \
     SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \
     TEST_GUEST_SECONDS=90 2> "$TMP_DIR/backward.log"; then
@@ -112,6 +128,14 @@ if run_check \
 fi
 grep -F 'guest returned an invalid HTTP Date header' "$TMP_DIR/malformed.log" >/dev/null
 
+if run_check \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \
+    TEST_CURL_FAIL=true \
+    TEST_GUEST_SECONDS=100; then
+    echo "healthcheck accepted a failed guest request" >&2
+    exit 1
+fi
+
 run_check \
     SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=0 \
     TEST_GUEST_DATE= \

From 77fd5adbf1d50cf7658d9e7f1ee59081fe8645f2 Mon Sep 17 00:00:00 2001
From: Danny Avila <danny@librechat.ai>
Date: Tue, 4 Aug 2026 08:47:33 -0400
Subject: [PATCH 3/7] fix: address clock healthcheck review findings

---
 docker-compose.scalable.yml                   |  2 +-
 docker/Dockerfile.worker-sandbox              |  2 +-
 .../templates/worker-sandbox-deployment.yaml  | 16 ++++++---
 tests/block_root_package_delivery.sh          | 36 +++++++++++++++++++
 4 files changed, 50 insertions(+), 6 deletions(-)

diff --git a/docker-compose.scalable.yml b/docker-compose.scalable.yml
index 8f93bf3..40a25a0 100644
--- a/docker-compose.scalable.yml
+++ b/docker-compose.scalable.yml
@@ -157,7 +157,7 @@ services:
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:3112/v1/health"]
       interval: 10s
-      timeout: 5s
+      timeout: 12s
       retries: 3
 
   # =============================================================================
diff --git a/docker/Dockerfile.worker-sandbox b/docker/Dockerfile.worker-sandbox
index d972906..bb04897 100644
--- a/docker/Dockerfile.worker-sandbox
+++ b/docker/Dockerfile.worker-sandbox
@@ -253,7 +253,7 @@ RUN chmod +x /supervisor.sh
 WORKDIR /
 EXPOSE 2000 3113
 
-HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \
+HEALTHCHECK --interval=30s --timeout=12s --start-period=30s --retries=3 \
   CMD /usr/local/bin/sandbox-runner-healthcheck.sh \
     && curl -fsS --max-time 5 http://localhost:3113/health >/dev/null \
     || exit 1
diff --git a/helm/codeapi/templates/worker-sandbox-deployment.yaml b/helm/codeapi/templates/worker-sandbox-deployment.yaml
index 1af386c..62c780c 100644
--- a/helm/codeapi/templates/worker-sandbox-deployment.yaml
+++ b/helm/codeapi/templates/worker-sandbox-deployment.yaml
@@ -27,8 +27,16 @@ Split runtime deployments:
 {{- if and (not $packagesFromPvc) (not .Values.workerSandbox.kvmEnabled) }}
 {{- fail "workerSandbox.packages.source=image requires workerSandbox.kvmEnabled=true because the baked runner boots from a libkrun block root image" }}
 {{- end }}
-{{- $clockSkewLimit := int .Values.workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds }}
-{{- $clockSkewJitter := int .Values.workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds }}
+{{- $clockSkewLimitValue := toString .Values.workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds }}
+{{- $clockSkewJitterValue := toString .Values.workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds }}
+{{- if not (regexMatch "^(0|[1-9][0-9]*)$" $clockSkewLimitValue) }}
+{{- fail "workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds must be a non-negative integer" }}
+{{- end }}
+{{- if not (regexMatch "^(0|[1-9][0-9]*)$" $clockSkewJitterValue) }}
+{{- fail "workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds must be a non-negative integer" }}
+{{- end }}
+{{- $clockSkewLimit := int $clockSkewLimitValue }}
+{{- $clockSkewJitter := int $clockSkewJitterValue }}
 {{- if or (lt $clockSkewLimit 0) (ge $clockSkewLimit 30) }}
 {{- fail "workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds must be between 0 and 29 so it stays below the 30-second execution-manifest tolerance" }}
 {{- end }}
@@ -267,9 +275,9 @@ spec:
             - name: SANDBOX_RUNNER_FD_LIVENESS_LIMIT
               value: {{ .Values.workerSandbox.sandboxRunner.fdLivenessLimit | quote }}
             - name: SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS
-              value: {{ .Values.workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds | quote }}
+              value: {{ $clockSkewLimit | quote }}
             - name: SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS
-              value: {{ .Values.workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds | quote }}
+              value: {{ $clockSkewJitter | quote }}
             - name: EGRESS_GATEWAY_URL
               value: "http://{{ include "codeapi.fullname" . }}-egress-gateway:{{ .Values.egressGateway.service.port }}"
             - name: SANDBOX_LOG_LEVEL
diff --git a/tests/block_root_package_delivery.sh b/tests/block_root_package_delivery.sh
index 721a5af..bc09bc3 100755
--- a/tests/block_root_package_delivery.sh
+++ b/tests/block_root_package_delivery.sh
@@ -160,6 +160,14 @@ assert_contains \
     "$ROOT/docker/Dockerfile.worker-sandbox" \
     'sandbox-runner-healthcheck.sh' \
     "the combined worker image must check guest clock skew"
+assert_contains \
+    "$ROOT/docker/Dockerfile.worker-sandbox" \
+    '^HEALTHCHECK --interval=30s --timeout=12s ' \
+    "the combined healthcheck timeout must cover both sequential probes"
+assert_contains \
+    "$ROOT/docker-compose.scalable.yml" \
+    '^[[:space:]]+timeout: 12s$' \
+    "the scalable Compose healthcheck timeout must cover both sequential probes"
 
 awk '
     /^FROM / {
@@ -300,6 +308,20 @@ assert_contains \
     'clockSkewLivenessLimitSeconds must be between 0 and 29' \
     "clock-skew limit validation failed for an unexpected reason"
 
+if helm template codeapi "$TMP_DIR/chart" \
+    --set executionManifest.privateKey=test \
+    --set executionManifest.publicKey=test \
+    --set workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds=29.5 \
+    > "$TMP_DIR/fractional-clock-skew-limit.yaml" 2> "$TMP_DIR/fractional-clock-skew-limit.log"; then
+    echo "clock-skew liveness limit must reject fractional values" >&2
+    exit 1
+fi
+
+assert_contains \
+    "$TMP_DIR/fractional-clock-skew-limit.log" \
+    'clockSkewLivenessLimitSeconds must be a non-negative integer' \
+    "fractional clock-skew limit validation failed for an unexpected reason"
+
 if helm template codeapi "$TMP_DIR/chart" \
     --set executionManifest.privateKey=test \
     --set executionManifest.publicKey=test \
@@ -314,6 +336,20 @@ assert_contains \
     'clockSkewLivenessJitterSeconds must be non-negative and less than clockSkewLivenessLimitSeconds' \
     "clock-skew jitter validation failed for an unexpected reason"
 
+if helm template codeapi "$TMP_DIR/chart" \
+    --set executionManifest.privateKey=test \
+    --set executionManifest.publicKey=test \
+    --set workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds=1.5 \
+    > "$TMP_DIR/fractional-clock-skew-jitter.yaml" 2> "$TMP_DIR/fractional-clock-skew-jitter.log"; then
+    echo "clock-skew liveness jitter must reject fractional values" >&2
+    exit 1
+fi
+
+assert_contains \
+    "$TMP_DIR/fractional-clock-skew-jitter.log" \
+    'clockSkewLivenessJitterSeconds must be a non-negative integer' \
+    "fractional clock-skew jitter validation failed for an unexpected reason"
+
 if helm template codeapi "$TMP_DIR/chart" \
     --set executionManifest.privateKey=test \
     --set executionManifest.publicKey=test \

From e213accc2633ee8d800cd2f93487f4414d5d7857 Mon Sep 17 00:00:00 2001
From: Danny Avila <danny@librechat.ai>
Date: Tue, 4 Aug 2026 08:59:04 -0400
Subject: [PATCH 4/7] fix: address clock-skew review findings

---
 docker-compose.scalable.yml                   |  8 ++----
 docker/Dockerfile.worker-sandbox              |  9 +++----
 docker/sandbox-runner-healthcheck.sh          | 20 +++++++++++++--
 .../templates/worker-sandbox-deployment.yaml  | 15 +++++------
 helm/codeapi/values.yaml                      |  3 ++-
 tests/block_root_package_delivery.sh          | 25 ++++++++-----------
 tests/sandbox_runner_healthcheck.sh           | 24 +++++++++++++++---
 7 files changed, 65 insertions(+), 39 deletions(-)

diff --git a/docker-compose.scalable.yml b/docker-compose.scalable.yml
index 40a25a0..49c8914 100644
--- a/docker-compose.scalable.yml
+++ b/docker-compose.scalable.yml
@@ -157,7 +157,7 @@ services:
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:3112/v1/health"]
       interval: 10s
-      timeout: 12s
+      timeout: 5s
       retries: 3
 
   # =============================================================================
@@ -221,11 +221,7 @@ services:
     deploy:
       replicas: 3  # Scale based on queue depth
     healthcheck:
-      test:
-        - CMD-SHELL
-        - >-
-          /usr/local/bin/sandbox-runner-healthcheck.sh &&
-          curl -fsS --max-time 5 http://localhost:3113/health >/dev/null
+      test: ["CMD", "curl", "-f", "http://localhost:3113/health"]
       interval: 10s
       timeout: 5s
       retries: 3
diff --git a/docker/Dockerfile.worker-sandbox b/docker/Dockerfile.worker-sandbox
index bb04897..cd3edd3 100644
--- a/docker/Dockerfile.worker-sandbox
+++ b/docker/Dockerfile.worker-sandbox
@@ -235,8 +235,7 @@ COPY --from=launcher-builder /launcher/target/release/sandbox-launcher /usr/loca
 # --- Launcher entrypoint (DNS resolution + socat relay before VM boot) ---
 COPY launcher/entrypoint.sh /usr/local/bin/launcher-entrypoint.sh
 COPY docker/start-direct-sandbox.sh /usr/local/bin/start-direct-sandbox.sh
-COPY docker/sandbox-runner-healthcheck.sh /usr/local/bin/sandbox-runner-healthcheck.sh
-RUN chmod +x /usr/local/bin/launcher-entrypoint.sh /usr/local/bin/start-direct-sandbox.sh /usr/local/bin/sandbox-runner-healthcheck.sh
+RUN chmod +x /usr/local/bin/launcher-entrypoint.sh /usr/local/bin/start-direct-sandbox.sh
 
 # --- Worker ---
 WORKDIR /worker
@@ -253,10 +252,8 @@ RUN chmod +x /supervisor.sh
 WORKDIR /
 EXPOSE 2000 3113
 
-HEALTHCHECK --interval=30s --timeout=12s --start-period=30s --retries=3 \
-  CMD /usr/local/bin/sandbox-runner-healthcheck.sh \
-    && curl -fsS --max-time 5 http://localhost:3113/health >/dev/null \
-    || exit 1
+HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \
+  CMD curl -f http://localhost:3113/health || exit 1
 
 CMD ["/supervisor.sh"]
 
diff --git a/docker/sandbox-runner-healthcheck.sh b/docker/sandbox-runner-healthcheck.sh
index f02c7ec..1c887f2 100755
--- a/docker/sandbox-runner-healthcheck.sh
+++ b/docker/sandbox-runner-healthcheck.sh
@@ -5,6 +5,7 @@ fd_limit="${SANDBOX_RUNNER_FD_LIVENESS_LIMIT:-40000}"
 clock_skew_limit_seconds="${SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS:-10}"
 clock_skew_jitter_seconds="${SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS:-2}"
 timeout_seconds="${SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS:-5}"
+manifest_clock_tolerance_seconds=30
 port="${PORT:-2000}"
 url="${SANDBOX_RUNNER_HEALTHCHECK_URL:-http://127.0.0.1:${port}/api/v2/runtimes}"
 
@@ -27,9 +28,18 @@ validate_non_negative_integer \
 validate_non_negative_integer \
     SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS \
     "$clock_skew_jitter_seconds"
+validate_non_negative_integer \
+    SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS \
+    "$timeout_seconds"
+
+if [ "$timeout_seconds" -eq 0 ]; then
+    echo "SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be greater than zero" >&2
+    exit 2
+fi
 
-if [ "$clock_skew_limit_seconds" -ge 30 ]; then
-    echo "SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS must be less than the 30-second execution-manifest tolerance" >&2
+if [ "$clock_skew_limit_seconds" -gt 0 ] && \
+    [ $((clock_skew_limit_seconds + timeout_seconds)) -ge "$manifest_clock_tolerance_seconds" ]; then
+    echo "SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS plus SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be less than the ${manifest_clock_tolerance_seconds}-second execution-manifest tolerance" >&2
     exit 2
 fi
 
@@ -91,6 +101,12 @@ if [ -z "$guest_date" ]; then
     exit 1
 fi
 
+http_date_pattern='^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), [0-9]{2} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2} GMT$'
+if ! [[ "$guest_date" =~ $http_date_pattern ]]; then
+    echo "sandbox-runner unhealthy: guest returned an invalid HTTP Date header: ${guest_date}" >&2
+    exit 1
+fi
+
 if ! guest_seconds=$(date -u -d "$guest_date" +%s 2>/dev/null); then
     echo "sandbox-runner unhealthy: guest returned an invalid HTTP Date header: ${guest_date}" >&2
     exit 1
diff --git a/helm/codeapi/templates/worker-sandbox-deployment.yaml b/helm/codeapi/templates/worker-sandbox-deployment.yaml
index 62c780c..3484a61 100644
--- a/helm/codeapi/templates/worker-sandbox-deployment.yaml
+++ b/helm/codeapi/templates/worker-sandbox-deployment.yaml
@@ -37,8 +37,8 @@ Split runtime deployments:
 {{- end }}
 {{- $clockSkewLimit := int $clockSkewLimitValue }}
 {{- $clockSkewJitter := int $clockSkewJitterValue }}
-{{- if or (lt $clockSkewLimit 0) (ge $clockSkewLimit 30) }}
-{{- fail "workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds must be between 0 and 29 so it stays below the 30-second execution-manifest tolerance" }}
+{{- if or (lt $clockSkewLimit 0) (ge $clockSkewLimit 25) }}
+{{- fail "workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds must be between 0 and 24 so the 5-second healthcheck timeout stays below the 30-second execution-manifest tolerance" }}
 {{- end }}
 {{- if or (lt $clockSkewJitter 0) (and (gt $clockSkewLimit 0) (ge $clockSkewJitter $clockSkewLimit)) }}
 {{- fail "workerSandbox.sandboxRunner.clockSkewLivenessJitterSeconds must be non-negative and less than clockSkewLivenessLimitSeconds when the clock-skew guard is enabled" }}
@@ -382,14 +382,15 @@ spec:
                 - /usr/local/bin/sandbox-runner-healthcheck.sh
             initialDelaySeconds: 60
             periodSeconds: 30
-            timeoutSeconds: 5
+            timeoutSeconds: 7
           readinessProbe:
-            httpGet:
-              path: /api/v2/health
-              port: sandbox
+            exec:
+              command:
+                - /usr/local/bin/sandbox-runner-healthcheck.sh
             initialDelaySeconds: 30
             periodSeconds: 10
-            timeoutSeconds: 5
+            timeoutSeconds: 7
+            failureThreshold: 1
           resources:
             {{- $sandboxResources := deepCopy .Values.workerSandbox.resources }}
             {{- if $useKvmDevicePlugin }}
diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml
index 05a957b..2385814 100644
--- a/helm/codeapi/values.yaml
+++ b/helm/codeapi/values.yaml
@@ -202,7 +202,8 @@ workerSandbox:
     fdLivenessLimit: 40000
     # Restart a sandbox-runner when its microVM guest wall clock differs from
     # the host by this many seconds. This must remain comfortably below the
-    # 30-second execution-manifest clock tolerance. Set 0 to disable.
+    # 30-second execution-manifest clock tolerance; the chart reserves the
+    # five-second probe timeout and accepts at most 24. Set 0 to disable.
     clockSkewLivenessLimitSeconds: 10
     # Subtract a stable 0..N-second offset from each pod's clock-skew limit so
     # replicas created together do not all recycle at once. Must be less than
diff --git a/tests/block_root_package_delivery.sh b/tests/block_root_package_delivery.sh
index bc09bc3..20ec6ea 100755
--- a/tests/block_root_package_delivery.sh
+++ b/tests/block_root_package_delivery.sh
@@ -156,18 +156,6 @@ assert_contains \
     "$ROOT/docker/Dockerfile.worker-sandbox" \
     '^FROM worker-sandbox-legacy AS worker-sandbox-false$' \
     "worker-sandbox-false must inherit the direct target"
-assert_contains \
-    "$ROOT/docker/Dockerfile.worker-sandbox" \
-    'sandbox-runner-healthcheck.sh' \
-    "the combined worker image must check guest clock skew"
-assert_contains \
-    "$ROOT/docker/Dockerfile.worker-sandbox" \
-    '^HEALTHCHECK --interval=30s --timeout=12s ' \
-    "the combined healthcheck timeout must cover both sequential probes"
-assert_contains \
-    "$ROOT/docker-compose.scalable.yml" \
-    '^[[:space:]]+timeout: 12s$' \
-    "the scalable Compose healthcheck timeout must cover both sequential probes"
 
 awk '
     /^FROM / {
@@ -235,6 +223,15 @@ assert_env_value \
     SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS \
     2 \
     "default Helm render must stagger clock-skew recycling across replicas"
+assert_contains \
+    "$TMP_DIR/helm-image.yaml" \
+    '^[[:space:]]+failureThreshold: 1$' \
+    "sandbox-runner readiness must fail immediately when clock skew is detected"
+
+if [ "$(grep -Fc -- '- /usr/local/bin/sandbox-runner-healthcheck.sh' "$TMP_DIR/helm-image.yaml")" -lt 2 ]; then
+    echo "sandbox-runner liveness and readiness must both check guest clock skew" >&2
+    exit 1
+fi
 
 helm template codeapi "$TMP_DIR/chart" \
     --set executionManifest.privateKey=test \
@@ -297,7 +294,7 @@ assert_env_value \
 if helm template codeapi "$TMP_DIR/chart" \
     --set executionManifest.privateKey=test \
     --set executionManifest.publicKey=test \
-    --set workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds=30 \
+    --set workerSandbox.sandboxRunner.clockSkewLivenessLimitSeconds=25 \
     > "$TMP_DIR/invalid-clock-skew-limit.yaml" 2> "$TMP_DIR/invalid-clock-skew-limit.log"; then
     echo "clock-skew liveness limit must stay below manifest tolerance" >&2
     exit 1
@@ -305,7 +302,7 @@ fi
 
 assert_contains \
     "$TMP_DIR/invalid-clock-skew-limit.log" \
-    'clockSkewLivenessLimitSeconds must be between 0 and 29' \
+    'clockSkewLivenessLimitSeconds must be between 0 and 24' \
     "clock-skew limit validation failed for an unexpected reason"
 
 if helm template codeapi "$TMP_DIR/chart" \
diff --git a/tests/sandbox_runner_healthcheck.sh b/tests/sandbox_runner_healthcheck.sh
index 00a3530..6daab24 100755
--- a/tests/sandbox_runner_healthcheck.sh
+++ b/tests/sandbox_runner_healthcheck.sh
@@ -128,6 +128,15 @@ if run_check \
 fi
 grep -F 'guest returned an invalid HTTP Date header' "$TMP_DIR/malformed.log" >/dev/null
 
+if run_check \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \
+    TEST_GUEST_DATE=now \
+    TEST_GUEST_SECONDS=100 2> "$TMP_DIR/non-http-date.log"; then
+    echo "healthcheck accepted a GNU-date-parseable value that is not an HTTP-date" >&2
+    exit 1
+fi
+grep -F 'guest returned an invalid HTTP Date header: now' "$TMP_DIR/non-http-date.log" >/dev/null
+
 if run_check \
     SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \
     TEST_CURL_FAIL=true \
@@ -150,11 +159,20 @@ fi
 grep -F 'invalid SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS: invalid' "$TMP_DIR/invalid.log" >/dev/null
 
 if run_check \
-    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=30 \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=25 \
     TEST_GUEST_SECONDS=100 2> "$TMP_DIR/unsafe-limit.log"; then
-    echo "healthcheck accepted a limit at the manifest tolerance" >&2
+    echo "healthcheck accepted a limit without request-latency headroom" >&2
+    exit 1
+fi
+grep -F 'plus SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be less than the 30-second execution-manifest tolerance' "$TMP_DIR/unsafe-limit.log" >/dev/null
+
+if run_check \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=10 \
+    SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS=0 \
+    TEST_GUEST_SECONDS=100 2> "$TMP_DIR/zero-timeout.log"; then
+    echo "healthcheck accepted a zero request timeout" >&2
     exit 1
 fi
-grep -F 'must be less than the 30-second execution-manifest tolerance' "$TMP_DIR/unsafe-limit.log" >/dev/null
+grep -F 'SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be greater than zero' "$TMP_DIR/zero-timeout.log" >/dev/null
 
 echo "sandbox-runner healthcheck checks passed"

From ef1055b0de78b0a0cc75623a63e8169b04445a48 Mon Sep 17 00:00:00 2001
From: Danny Avila <danny@librechat.ai>
Date: Tue, 4 Aug 2026 09:05:58 -0400
Subject: [PATCH 5/7] fix: opt in to clock recycling from helm

---
 docker/sandbox-runner-healthcheck.sh | 2 +-
 tests/sandbox_runner_healthcheck.sh  | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/docker/sandbox-runner-healthcheck.sh b/docker/sandbox-runner-healthcheck.sh
index 1c887f2..4da2521 100755
--- a/docker/sandbox-runner-healthcheck.sh
+++ b/docker/sandbox-runner-healthcheck.sh
@@ -2,7 +2,7 @@
 set -euo pipefail
 
 fd_limit="${SANDBOX_RUNNER_FD_LIVENESS_LIMIT:-40000}"
-clock_skew_limit_seconds="${SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS:-10}"
+clock_skew_limit_seconds="${SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS:-0}"
 clock_skew_jitter_seconds="${SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS:-2}"
 timeout_seconds="${SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS:-5}"
 manifest_clock_tolerance_seconds=30
diff --git a/tests/sandbox_runner_healthcheck.sh b/tests/sandbox_runner_healthcheck.sh
index 6daab24..52cf8d6 100755
--- a/tests/sandbox_runner_healthcheck.sh
+++ b/tests/sandbox_runner_healthcheck.sh
@@ -150,6 +150,12 @@ run_check \
     TEST_GUEST_DATE= \
     TEST_GUEST_SECONDS=100
 
+# Compose and standalone Docker callers only observe health status, so the
+# clock-skew guard stays disabled unless an orchestrator explicitly opts in.
+run_check \
+    TEST_GUEST_DATE= \
+    TEST_GUEST_SECONDS=100
+
 if run_check \
     SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=invalid \
     TEST_GUEST_SECONDS=100 2> "$TMP_DIR/invalid.log"; then

From 4241b96965f0e142383e17965c5828d7b7d65dad Mon Sep 17 00:00:00 2001
From: Danny Avila <danny@librechat.ai>
Date: Tue, 4 Aug 2026 09:09:42 -0400
Subject: [PATCH 6/7] fix: bound clock healthcheck arithmetic

---
 docker/sandbox-runner-healthcheck.sh | 25 ++++++++++++++++++-------
 tests/sandbox_runner_healthcheck.sh  | 16 ++++++++++++++++
 2 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/docker/sandbox-runner-healthcheck.sh b/docker/sandbox-runner-healthcheck.sh
index 4da2521..ec15997 100755
--- a/docker/sandbox-runner-healthcheck.sh
+++ b/docker/sandbox-runner-healthcheck.sh
@@ -14,8 +14,9 @@ validate_non_negative_integer() {
     local value="$2"
 
     case "$value" in
-        ''|*[!0-9]*)
-            echo "invalid ${name}: ${value}" >&2
+        0|[1-9]|[1-9][0-9]*) ;;
+        *)
+            echo "invalid ${name}: ${value} (expected a canonical non-negative integer)" >&2
             exit 2
             ;;
     esac
@@ -37,10 +38,19 @@ if [ "$timeout_seconds" -eq 0 ]; then
     exit 2
 fi
 
-if [ "$clock_skew_limit_seconds" -gt 0 ] && \
-    [ $((clock_skew_limit_seconds + timeout_seconds)) -ge "$manifest_clock_tolerance_seconds" ]; then
-    echo "SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS plus SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be less than the ${manifest_clock_tolerance_seconds}-second execution-manifest tolerance" >&2
-    exit 2
+if [ "$clock_skew_limit_seconds" -gt 0 ]; then
+    if [ "${#clock_skew_limit_seconds}" -gt 2 ] || \
+        [ "$clock_skew_limit_seconds" -ge "$manifest_clock_tolerance_seconds" ]; then
+        echo "SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS plus SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be less than the ${manifest_clock_tolerance_seconds}-second execution-manifest tolerance" >&2
+        exit 2
+    fi
+
+    remaining_tolerance_seconds=$((manifest_clock_tolerance_seconds - clock_skew_limit_seconds))
+    if [ "${#timeout_seconds}" -gt 2 ] || \
+        [ "$timeout_seconds" -ge "$remaining_tolerance_seconds" ]; then
+        echo "SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS plus SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be less than the ${manifest_clock_tolerance_seconds}-second execution-manifest tolerance" >&2
+        exit 2
+    fi
 fi
 
 if [ "$clock_skew_limit_seconds" -gt 0 ] && ! command -v date >/dev/null 2>&1; then
@@ -50,7 +60,8 @@ fi
 
 effective_clock_skew_limit_seconds="$clock_skew_limit_seconds"
 if [ "$clock_skew_limit_seconds" -gt 0 ] && [ "$clock_skew_jitter_seconds" -gt 0 ]; then
-    if [ "$clock_skew_jitter_seconds" -ge "$clock_skew_limit_seconds" ]; then
+    if [ "${#clock_skew_jitter_seconds}" -gt 2 ] || \
+        [ "$clock_skew_jitter_seconds" -ge "$clock_skew_limit_seconds" ]; then
         echo "SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS must be less than SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS" >&2
         exit 2
     fi
diff --git a/tests/sandbox_runner_healthcheck.sh b/tests/sandbox_runner_healthcheck.sh
index 52cf8d6..6a5d43f 100755
--- a/tests/sandbox_runner_healthcheck.sh
+++ b/tests/sandbox_runner_healthcheck.sh
@@ -164,6 +164,22 @@ if run_check \
 fi
 grep -F 'invalid SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS: invalid' "$TMP_DIR/invalid.log" >/dev/null
 
+if run_check \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=08 \
+    TEST_GUEST_SECONDS=100 2> "$TMP_DIR/leading-zero.log"; then
+    echo "healthcheck accepted a non-canonical clock-skew limit" >&2
+    exit 1
+fi
+grep -F 'expected a canonical non-negative integer' "$TMP_DIR/leading-zero.log" >/dev/null
+
+if run_check \
+    SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=9223372036854775807 \
+    TEST_GUEST_SECONDS=100 2> "$TMP_DIR/overflow-limit.log"; then
+    echo "healthcheck accepted a clock-skew limit that could overflow arithmetic" >&2
+    exit 1
+fi
+grep -F 'plus SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS must be less than the 30-second execution-manifest tolerance' "$TMP_DIR/overflow-limit.log" >/dev/null
+
 if run_check \
     SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=25 \
     TEST_GUEST_SECONDS=100 2> "$TMP_DIR/unsafe-limit.log"; then

From 783573def0df389328627c01ea457093f28fbe8e Mon Sep 17 00:00:00 2001
From: Danny Avila <danny@librechat.ai>
Date: Tue, 4 Aug 2026 09:14:45 -0400
Subject: [PATCH 7/7] fix: preserve sandbox workspace readiness

---
 docker/sandbox-runner-healthcheck.sh | 2 +-
 tests/sandbox_runner_healthcheck.sh  | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/docker/sandbox-runner-healthcheck.sh b/docker/sandbox-runner-healthcheck.sh
index ec15997..c639848 100755
--- a/docker/sandbox-runner-healthcheck.sh
+++ b/docker/sandbox-runner-healthcheck.sh
@@ -7,7 +7,7 @@ clock_skew_jitter_seconds="${SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_JITTER_SECONDS:-
 timeout_seconds="${SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS:-5}"
 manifest_clock_tolerance_seconds=30
 port="${PORT:-2000}"
-url="${SANDBOX_RUNNER_HEALTHCHECK_URL:-http://127.0.0.1:${port}/api/v2/runtimes}"
+url="${SANDBOX_RUNNER_HEALTHCHECK_URL:-http://127.0.0.1:${port}/api/v2/health}"
 
 validate_non_negative_integer() {
     local name="$1"
diff --git a/tests/sandbox_runner_healthcheck.sh b/tests/sandbox_runner_healthcheck.sh
index 6a5d43f..583ff7f 100755
--- a/tests/sandbox_runner_healthcheck.sh
+++ b/tests/sandbox_runner_healthcheck.sh
@@ -15,6 +15,11 @@ if [[ "${TEST_CURL_FAIL:-false}" == "true" ]]; then
     exit 22
 fi
 
+if [[ -n "${TEST_EXPECTED_URL:-}" && "${!#}" != "$TEST_EXPECTED_URL" ]]; then
+    printf 'unexpected healthcheck URL: %s\n' "${!#}" >&2
+    exit 2
+fi
+
 printf 'HTTP/1.1 200 OK\r\n'
 if [[ -n "${TEST_GUEST_DATE:-}" ]]; then
     printf 'Date: %s\r\n' "$TEST_GUEST_DATE"
@@ -68,6 +73,7 @@ run_check() {
         TEST_HOST_AFTER_SECONDS=100 \
         TEST_HOST_SAMPLE_STATE="$TMP_DIR/host-sample" \
         TEST_GUEST_DATE='Tue, 04 Aug 2026 08:00:00 GMT' \
+        TEST_EXPECTED_URL='http://127.0.0.1:2000/api/v2/health' \
         "$@" \
         bash "$ROOT/docker/sandbox-runner-healthcheck.sh"
 }