diff --git a/cleanup.sh b/cleanup.sh index 81c5874..a7ffcdb 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -19,7 +19,8 @@ # - never touch a runner's checkouts or build caches in _work; wiping them # makes every job re-clone and recompile from cold. The one exception is # the Playwright tool cache, and only under disk pressure. -# - never touch a simulator that is not a runner's own ci- device. +# - never touch a simulator or Android AVD that is not a runner's own +# ci- (or shots-) device. set -uo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -291,6 +292,10 @@ say " ${n} directories" # could take. A device is erased only while it is shut down and its runner is # not building; a ci- device whose runner no longer exists is deleted. Other # devices (the person's, or a shared "iPhone 17") are never touched. +# A repo that picks its own device can name it "ci- +# " (Jerv's ci/sim.py: "ci-ultra-jerv-2 jerv iOS 27.0"). Matching only +# the kit's shape missed those: on ultra (2026-10-22) nine of them held 19 GB +# and none had ever been erased. # --------------------------------------------------------------------------- sim_limit="$SIM_MAX_GB"; [ "$PRESSURE" = 1 ] && sim_limit="$SIM_PRESSURE_GB" say "==> CI simulators over ${sim_limit} GB" @@ -323,12 +328,74 @@ except Exception: sys.exit(0) for devs in devices.values(): for d in devs: - m = re.match(r"ci-(\S+) (iOS|tvOS|watchOS|visionOS|xrOS) [0-9.]+$", d.get("name", "")) + m = re.match(r"ci-(\S+) (?:\S+ )?(iOS|tvOS|watchOS|visionOS|xrOS) [0-9.]+$", d.get("name", "")) if m: print("|".join([d["udid"], d.get("state", ""), str(d.get("dataPathSize", 0)), m.group(1)])) ') say " ${n} devices" +# --------------------------------------------------------------------------- +# Each idle runner's own Android emulator. The kit's android-ci names one AVD +# per runner "ci-" (Jerv's screenshots: "shots-"), and +# android-emulator-runner creates it again at the start of every job +# (force-avd-creation defaults to true), so between jobs an AVD is 1-1.5 GB +# that the next job overwrites anyway. Nothing removed them: on ultra +# (2026-10-22) 17 of them held 20 GB. An AVD is left alone while its runner is +# building, while an emulator has it open, or within 30 minutes of its last +# write (a job that is creating it). Other AVDs (night QA's fleet-explore-*, a +# person's) are never touched. +# --------------------------------------------------------------------------- +AVD_HOME="${ANDROID_AVD_HOME:-$HOME/.android/avd}" +AVD_PREFIXES="${FLEET_CLEANUP_AVD_PREFIXES:-ci shots}" +avd_open() { + local dir="$1" pid + pid="$(cat "$dir/hardware-qemu.ini.lock" 2>/dev/null | tr -dc '0-9')" + [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null && return 0 + pgrep -f -- "-avd $2( |\$)" >/dev/null 2>&1 +} +say "==> idle runners' Android emulators" +n=0 +for dir in "$AVD_HOME"/*.avd; do + [ -d "$dir" ] || continue + avd="$(basename "$dir" .avd)" + runner="" + for p in $AVD_PREFIXES; do + case "$avd" in "$p"-?*) runner="${avd#"$p"-}"; break ;; esac + done + [ -n "$runner" ] || continue + building "$runner" && continue + [ -n "$(find "$dir" -maxdepth 1 -mmin -30 2>/dev/null | head -1)" ] && continue + avd_open "$dir" "$avd" && continue + why="" + case "$known" in *" $runner "*) ;; *) why=": no such runner here" ;; esac + say " $(du -sh "$dir" 2>/dev/null | cut -f1) $avd$why" + run rm -rf "$dir" "$AVD_HOME/$avd.ini" + n=$((n + 1)) +done +say " ${n} AVDs" + +# --------------------------------------------------------------------------- +# A runner's leftover self-update. The runner unpacks a new version into +# _work/_update, copies it into bin./externals. and is meant +# to remove the staging copy; nine runners on ultra kept theirs (0.45 GB each, +# 2026-10-22). Only a copy older than six hours on an idle runner that no +# process is running from goes: an update in progress takes minutes. +# --------------------------------------------------------------------------- +say "==> leftover runner self-updates" +n=0 +for d in "$ROOT"/*/; do + d="${d%/}" + up="$d/_work/_update" + [ -f "$d/.runner" ] && [ -d "$up" ] || continue + [ -n "$(find "$up" -maxdepth 0 -mmin +360 2>/dev/null)" ] || continue + building "$(runner_name "$d")" && continue + pgrep -f "$up/" >/dev/null 2>&1 && continue + say " $(du -sh "$up" 2>/dev/null | cut -f1) $(basename "$d")/_work/_update" + run rm -rf "$up" + n=$((n + 1)) +done +say " ${n} directories" + # --------------------------------------------------------------------------- # Under pressure only: each idle runner's Playwright browsers. They are 0.5-1.1 GB # per web runner (13 GB on runner-host) and the next job that needs them diff --git a/scripts/test-cleanup.sh b/scripts/test-cleanup.sh index 6b259f1..c3c6669 100755 --- a/scripts/test-cleanup.sh +++ b/scripts/test-cleanup.sh @@ -75,7 +75,7 @@ EOF } clean() { - HOME="$T/home" FLEET_ROOT="$T" PATH="$T/bin:$PATH" bash "$T/cleanup.sh" "$@" 2>&1 + HOME="$T/home" FLEET_ROOT="$T" PATH="$T/bin:$PATH" ANDROID_AVD_HOME="" bash "$T/cleanup.sh" "$@" 2>&1 } calls() { grep -c "$1" "$T/calls" 2>/dev/null; } gone() { [ -e "$1" ] && echo kept || echo removed; } @@ -222,6 +222,65 @@ clean --apply > "$T/out" ok "no missing-file noise" "$(grep -c -E 'No such file|command not found' "$T/out")" "0" ok "went by df, so under pressure" "$(grep -c 'UNDER PRESSURE' "$T/out")" "1" +echo "== a repo's own ci- devices count as CI devices ==" +setup 12 +echo 120 > "$T/free" +echo 1 > "$T/.cleanup-last-full" +echo "RL-web" > "$T/busy" +cat > "$T/sims.json" <<'EOF' +{"devices": { + "com.apple.CoreSimulator.SimRuntime.iOS-27-0": [ + {"name": "ci-RL-ios jerv iOS 27.0", "udid": "U-JERV", "state": "Shutdown", "dataPathSize": 5368709120}, + {"name": "ci-RL-web jerv iOS 27.0", "udid": "U-JWEB", "state": "Shutdown", "dataPathSize": 5368709120}, + {"name": "iPhone 17 Pro", "udid": "U-PERSON", "state": "Shutdown", "dataPathSize": 12884901888} + ], + "com.apple.CoreSimulator.SimRuntime.watchOS-27-0": [ + {"name": "ci-RL-gone jerv watchOS 27.0", "udid": "U-JGONE", "state": "Shutdown", "dataPathSize": 1048576} + ]}} +EOF +clean --apply > "$T/out" +ok "erased an idle runner's app device over 3 GB" "$(calls 'simctl erase U-JERV')" "1" +ok "left a busy runner's app device" "$(calls 'U-JWEB')" "0" +ok "deleted an app device whose runner is gone" "$(calls 'simctl delete U-JGONE')" "1" +ok "never touched a device outside CI" "$(calls 'U-PERSON')" "0" + +echo "== idle runners' AVDs and leftover self-updates go, nothing else ==" +setup 13 +echo 120 > "$T/free" +echo 1 > "$T/.cleanup-last-full" +echo "RL-web" > "$T/busy" +A="$T/home/.android/avd" +mkdir -p "$A" +avd() { + mkdir -p "$A/$1.avd"; : > "$A/$1.avd/userdata-qemu.img.qcow2"; echo "path=$A/$1.avd" > "$A/$1.ini" + [ "${2:-old}" = old ] && touch -t 202601010000 "$A/$1.avd/userdata-qemu.img.qcow2" "$A/$1.avd" +} +avd ci-RL-ios; avd shots-RL-android; avd ci-RL-web; avd ci-RL-gone; avd fleet-explore-1 +avd ci-RL-android new +avd shots-RL-ios +sleep 300 >/dev/null 2>&1 & +EMU=$! +echo "$EMU" > "$A/shots-RL-ios.avd/hardware-qemu.ini.lock" +touch -t 202601010000 "$A/shots-RL-ios.avd/hardware-qemu.ini.lock" "$A/shots-RL-ios.avd" +for r in ios web android; do mkdir -p "$T/$r/_work/_update/bin"; done +touch -t 202601010000 "$T/ios/_work/_update" "$T/web/_work/_update" +clean > "$T/out" +ok "a dry run keeps every AVD" "$(gone "$A/ci-RL-ios.avd")" "kept" +ok "a dry run keeps the leftover update" "$(gone "$T/ios/_work/_update")" "kept" +clean --apply > "$T/out" +ok "removed an idle runner's AVD" "$(gone "$A/ci-RL-ios.avd")" "removed" +ok "and its .ini" "$(gone "$A/ci-RL-ios.ini")" "removed" +ok "removed an idle runner's screenshot AVD" "$(gone "$A/shots-RL-android.avd")" "removed" +ok "removed the AVD of a runner that no longer exists" "$(gone "$A/ci-RL-gone.avd")" "removed" +ok "kept a busy runner's AVD" "$(gone "$A/ci-RL-web.avd")" "kept" +ok "kept an AVD written in the last 30 minutes" "$(gone "$A/ci-RL-android.avd")" "kept" +ok "kept an AVD an emulator has open" "$(gone "$A/shots-RL-ios.avd")" "kept" +ok "never touched an AVD outside CI" "$(gone "$A/fleet-explore-1.avd")" "kept" +ok "removed an idle runner's old self-update" "$(gone "$T/ios/_work/_update")" "removed" +ok "kept a busy runner's self-update" "$(gone "$T/web/_work/_update")" "kept" +ok "kept a self-update under six hours old" "$(gone "$T/android/_work/_update")" "kept" +{ kill "$EMU"; wait "$EMU"; } 2>/dev/null + echo echo "passed $PASS, failed $FAIL" [ "$FAIL" -eq 0 ]