From bbd5d99d975f007090e7e5f21030c36f479fd73d Mon Sep 17 00:00:00 2001 From: Taylor Addison Date: Thu, 22 Oct 2026 22:22:07 -0400 Subject: [PATCH] cleanup: reclaim what a busy fleet grows (app-named sims, idle AVDs, stale updates) ultra kept dropping under its 100 GB admission floor while cleanup.sh found almost nothing. Since the 10-17 reclaim, files created under ~/Library/Developer (simulators) and ~/.android/avd were 43 of the ~100 GB that are new, and the script could take none of it: - CI simulators: Jerv's ci/sim.py names its devices "ci- jerv ", which the kit-shaped regex never matched; nine of them held 19 GB and had never been erased. The match now allows one app word before the platform. - Android AVDs: one per runner (ci-, Jerv's shots-), 17 of them, 20 GB, removed by nothing. android-emulator-runner recreates the AVD at the start of every job (force-avd-creation defaults to true), so an idle runner's AVD is pure waste. Removed while its runner is idle, no emulator holds it, and it was not written in the last 30 minutes. - _work/_update: a runner self-update's staging copy (0.45 GB) left behind on nine runners. Removed when older than six hours, idle, unused. All three are per-runner steps, so they run while other runners build. Co-Authored-By: Claude Opus 5.5 --- cleanup.sh | 71 +++++++++++++++++++++++++++++++++++++++-- scripts/test-cleanup.sh | 61 ++++++++++++++++++++++++++++++++++- 2 files changed, 129 insertions(+), 3 deletions(-) diff --git a/cleanup.sh b/cleanup.sh index 81c5874..a7ffcdb 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -19,7 +19,8 @@ # - never touch a runner's checkouts or build caches in _work; wiping them # makes every job re-clone and recompile from cold. The one exception is # the Playwright tool cache, and only under disk pressure. -# - never touch a simulator that is not a runner's own ci- device. +# - never touch a simulator or Android AVD that is not a runner's own +# ci- (or shots-) device. set -uo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -291,6 +292,10 @@ say " ${n} directories" # could take. A device is erased only while it is shut down and its runner is # not building; a ci- device whose runner no longer exists is deleted. Other # devices (the person's, or a shared "iPhone 17") are never touched. +# A repo that picks its own device can name it "ci- +# " (Jerv's ci/sim.py: "ci-ultra-jerv-2 jerv iOS 27.0"). Matching only +# the kit's shape missed those: on ultra (2026-10-22) nine of them held 19 GB +# and none had ever been erased. # --------------------------------------------------------------------------- sim_limit="$SIM_MAX_GB"; [ "$PRESSURE" = 1 ] && sim_limit="$SIM_PRESSURE_GB" say "==> CI simulators over ${sim_limit} GB" @@ -323,12 +328,74 @@ except Exception: sys.exit(0) for devs in devices.values(): for d in devs: - m = re.match(r"ci-(\S+) (iOS|tvOS|watchOS|visionOS|xrOS) [0-9.]+$", d.get("name", "")) + m = re.match(r"ci-(\S+) (?:\S+ )?(iOS|tvOS|watchOS|visionOS|xrOS) [0-9.]+$", d.get("name", "")) if m: print("|".join([d["udid"], d.get("state", ""), str(d.get("dataPathSize", 0)), m.group(1)])) ') say " ${n} devices" +# --------------------------------------------------------------------------- +# Each idle runner's own Android emulator. The kit's android-ci names one AVD +# per runner "ci-" (Jerv's screenshots: "shots-"), and +# android-emulator-runner creates it again at the start of every job +# (force-avd-creation defaults to true), so between jobs an AVD is 1-1.5 GB +# that the next job overwrites anyway. Nothing removed them: on ultra +# (2026-10-22) 17 of them held 20 GB. An AVD is left alone while its runner is +# building, while an emulator has it open, or within 30 minutes of its last +# write (a job that is creating it). Other AVDs (night QA's fleet-explore-*, a +# person's) are never touched. +# --------------------------------------------------------------------------- +AVD_HOME="${ANDROID_AVD_HOME:-$HOME/.android/avd}" +AVD_PREFIXES="${FLEET_CLEANUP_AVD_PREFIXES:-ci shots}" +avd_open() { + local dir="$1" pid + pid="$(cat "$dir/hardware-qemu.ini.lock" 2>/dev/null | tr -dc '0-9')" + [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null && return 0 + pgrep -f -- "-avd $2( |\$)" >/dev/null 2>&1 +} +say "==> idle runners' Android emulators" +n=0 +for dir in "$AVD_HOME"/*.avd; do + [ -d "$dir" ] || continue + avd="$(basename "$dir" .avd)" + runner="" + for p in $AVD_PREFIXES; do + case "$avd" in "$p"-?*) runner="${avd#"$p"-}"; break ;; esac + done + [ -n "$runner" ] || continue + building "$runner" && continue + [ -n "$(find "$dir" -maxdepth 1 -mmin -30 2>/dev/null | head -1)" ] && continue + avd_open "$dir" "$avd" && continue + why="" + case "$known" in *" $runner "*) ;; *) why=": no such runner here" ;; esac + say " $(du -sh "$dir" 2>/dev/null | cut -f1) $avd$why" + run rm -rf "$dir" "$AVD_HOME/$avd.ini" + n=$((n + 1)) +done +say " ${n} AVDs" + +# --------------------------------------------------------------------------- +# A runner's leftover self-update. The runner unpacks a new version into +# _work/_update, copies it into bin./externals. and is meant +# to remove the staging copy; nine runners on ultra kept theirs (0.45 GB each, +# 2026-10-22). Only a copy older than six hours on an idle runner that no +# process is running from goes: an update in progress takes minutes. +# --------------------------------------------------------------------------- +say "==> leftover runner self-updates" +n=0 +for d in "$ROOT"/*/; do + d="${d%/}" + up="$d/_work/_update" + [ -f "$d/.runner" ] && [ -d "$up" ] || continue + [ -n "$(find "$up" -maxdepth 0 -mmin +360 2>/dev/null)" ] || continue + building "$(runner_name "$d")" && continue + pgrep -f "$up/" >/dev/null 2>&1 && continue + say " $(du -sh "$up" 2>/dev/null | cut -f1) $(basename "$d")/_work/_update" + run rm -rf "$up" + n=$((n + 1)) +done +say " ${n} directories" + # --------------------------------------------------------------------------- # Under pressure only: each idle runner's Playwright browsers. They are 0.5-1.1 GB # per web runner (13 GB on runner-host) and the next job that needs them diff --git a/scripts/test-cleanup.sh b/scripts/test-cleanup.sh index 6b259f1..c3c6669 100755 --- a/scripts/test-cleanup.sh +++ b/scripts/test-cleanup.sh @@ -75,7 +75,7 @@ EOF } clean() { - HOME="$T/home" FLEET_ROOT="$T" PATH="$T/bin:$PATH" bash "$T/cleanup.sh" "$@" 2>&1 + HOME="$T/home" FLEET_ROOT="$T" PATH="$T/bin:$PATH" ANDROID_AVD_HOME="" bash "$T/cleanup.sh" "$@" 2>&1 } calls() { grep -c "$1" "$T/calls" 2>/dev/null; } gone() { [ -e "$1" ] && echo kept || echo removed; } @@ -222,6 +222,65 @@ clean --apply > "$T/out" ok "no missing-file noise" "$(grep -c -E 'No such file|command not found' "$T/out")" "0" ok "went by df, so under pressure" "$(grep -c 'UNDER PRESSURE' "$T/out")" "1" +echo "== a repo's own ci- devices count as CI devices ==" +setup 12 +echo 120 > "$T/free" +echo 1 > "$T/.cleanup-last-full" +echo "RL-web" > "$T/busy" +cat > "$T/sims.json" <<'EOF' +{"devices": { + "com.apple.CoreSimulator.SimRuntime.iOS-27-0": [ + {"name": "ci-RL-ios jerv iOS 27.0", "udid": "U-JERV", "state": "Shutdown", "dataPathSize": 5368709120}, + {"name": "ci-RL-web jerv iOS 27.0", "udid": "U-JWEB", "state": "Shutdown", "dataPathSize": 5368709120}, + {"name": "iPhone 17 Pro", "udid": "U-PERSON", "state": "Shutdown", "dataPathSize": 12884901888} + ], + "com.apple.CoreSimulator.SimRuntime.watchOS-27-0": [ + {"name": "ci-RL-gone jerv watchOS 27.0", "udid": "U-JGONE", "state": "Shutdown", "dataPathSize": 1048576} + ]}} +EOF +clean --apply > "$T/out" +ok "erased an idle runner's app device over 3 GB" "$(calls 'simctl erase U-JERV')" "1" +ok "left a busy runner's app device" "$(calls 'U-JWEB')" "0" +ok "deleted an app device whose runner is gone" "$(calls 'simctl delete U-JGONE')" "1" +ok "never touched a device outside CI" "$(calls 'U-PERSON')" "0" + +echo "== idle runners' AVDs and leftover self-updates go, nothing else ==" +setup 13 +echo 120 > "$T/free" +echo 1 > "$T/.cleanup-last-full" +echo "RL-web" > "$T/busy" +A="$T/home/.android/avd" +mkdir -p "$A" +avd() { + mkdir -p "$A/$1.avd"; : > "$A/$1.avd/userdata-qemu.img.qcow2"; echo "path=$A/$1.avd" > "$A/$1.ini" + [ "${2:-old}" = old ] && touch -t 202601010000 "$A/$1.avd/userdata-qemu.img.qcow2" "$A/$1.avd" +} +avd ci-RL-ios; avd shots-RL-android; avd ci-RL-web; avd ci-RL-gone; avd fleet-explore-1 +avd ci-RL-android new +avd shots-RL-ios +sleep 300 >/dev/null 2>&1 & +EMU=$! +echo "$EMU" > "$A/shots-RL-ios.avd/hardware-qemu.ini.lock" +touch -t 202601010000 "$A/shots-RL-ios.avd/hardware-qemu.ini.lock" "$A/shots-RL-ios.avd" +for r in ios web android; do mkdir -p "$T/$r/_work/_update/bin"; done +touch -t 202601010000 "$T/ios/_work/_update" "$T/web/_work/_update" +clean > "$T/out" +ok "a dry run keeps every AVD" "$(gone "$A/ci-RL-ios.avd")" "kept" +ok "a dry run keeps the leftover update" "$(gone "$T/ios/_work/_update")" "kept" +clean --apply > "$T/out" +ok "removed an idle runner's AVD" "$(gone "$A/ci-RL-ios.avd")" "removed" +ok "and its .ini" "$(gone "$A/ci-RL-ios.ini")" "removed" +ok "removed an idle runner's screenshot AVD" "$(gone "$A/shots-RL-android.avd")" "removed" +ok "removed the AVD of a runner that no longer exists" "$(gone "$A/ci-RL-gone.avd")" "removed" +ok "kept a busy runner's AVD" "$(gone "$A/ci-RL-web.avd")" "kept" +ok "kept an AVD written in the last 30 minutes" "$(gone "$A/ci-RL-android.avd")" "kept" +ok "kept an AVD an emulator has open" "$(gone "$A/shots-RL-ios.avd")" "kept" +ok "never touched an AVD outside CI" "$(gone "$A/fleet-explore-1.avd")" "kept" +ok "removed an idle runner's old self-update" "$(gone "$T/ios/_work/_update")" "removed" +ok "kept a busy runner's self-update" "$(gone "$T/web/_work/_update")" "kept" +ok "kept a self-update under six hours old" "$(gone "$T/android/_work/_update")" "kept" +{ kill "$EMU"; wait "$EMU"; } 2>/dev/null + echo echo "passed $PASS, failed $FAIL" [ "$FAIL" -eq 0 ]