Skip to content

cleanup: reclaim what a busy fleet grows (app-named sims, idle AVDs, stale updates) #114

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
addisdev merged 1 commit into from
Oct 23, 2026
Merged

cleanup: reclaim what a busy fleet grows (app-named sims, idle AVDs, stale updates) #114

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 69 additions & 2 deletions cleanup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@
# - never touch a runner's checkouts or build caches in _work; wiping them
# makes every job re-clone and recompile from cold. The one exception is
# the Playwright tool cache, and only under disk pressure.
# - never touch a simulator that is not a runner's own ci- device.
# - never touch a simulator or Android AVD that is not a runner's own
# ci- (or shots-) device.
set -uo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
Expand Down Expand Up @@ -291,6 +292,10 @@ say " ${n} directories"
# could take. A device is erased only while it is shut down and its runner is
# not building; a ci- device whose runner no longer exists is deleted. Other
# devices (the person's, or a shared "iPhone 17") are never touched.
# A repo that picks its own device can name it "ci-<runner> <app> <platform>
# <version>" (Jerv's ci/sim.py: "ci-ultra-jerv-2 jerv iOS 27.0"). Matching only
# the kit's shape missed those: on ultra (2026-10-22) nine of them held 19 GB
# and none had ever been erased.
# ---------------------------------------------------------------------------
sim_limit="$SIM_MAX_GB"; [ "$PRESSURE" = 1 ] && sim_limit="$SIM_PRESSURE_GB"
say "==> CI simulators over ${sim_limit} GB"
Expand Down Expand Up @@ -323,12 +328,74 @@ except Exception:
sys.exit(0)
for devs in devices.values():
for d in devs:
m = re.match(r"ci-(\S+) (iOS|tvOS|watchOS|visionOS|xrOS) [0-9.]+$", d.get("name", ""))
m = re.match(r"ci-(\S+) (?:\S+ )?(iOS|tvOS|watchOS|visionOS|xrOS) [0-9.]+$", d.get("name", ""))
if m:
print("|".join([d["udid"], d.get("state", ""), str(d.get("dataPathSize", 0)), m.group(1)]))
')
say " ${n} devices"

# ---------------------------------------------------------------------------
# Each idle runner's own Android emulator. The kit's android-ci names one AVD
# per runner "ci-<runner>" (Jerv's screenshots: "shots-<runner>"), and
# android-emulator-runner creates it again at the start of every job
# (force-avd-creation defaults to true), so between jobs an AVD is 1-1.5 GB
# that the next job overwrites anyway. Nothing removed them: on ultra
# (2026-10-22) 17 of them held 20 GB. An AVD is left alone while its runner is
# building, while an emulator has it open, or within 30 minutes of its last
# write (a job that is creating it). Other AVDs (night QA's fleet-explore-*, a
# person's) are never touched.
# ---------------------------------------------------------------------------
AVD_HOME="${ANDROID_AVD_HOME:-$HOME/.android/avd}"
AVD_PREFIXES="${FLEET_CLEANUP_AVD_PREFIXES:-ci shots}"
avd_open() {
local dir="$1" pid
pid="$(cat "$dir/hardware-qemu.ini.lock" 2>/dev/null | tr -dc '0-9')"
[ -n "$pid" ] && kill -0 "$pid" 2>/dev/null && return 0
pgrep -f -- "-avd $2( |\$)" >/dev/null 2>&1
}
say "==> idle runners' Android emulators"
n=0
for dir in "$AVD_HOME"/*.avd; do
[ -d "$dir" ] || continue
avd="$(basename "$dir" .avd)"
runner=""
for p in $AVD_PREFIXES; do
case "$avd" in "$p"-?*) runner="${avd#"$p"-}"; break ;; esac
done
[ -n "$runner" ] || continue
building "$runner" && continue
[ -n "$(find "$dir" -maxdepth 1 -mmin -30 2>/dev/null | head -1)" ] && continue
avd_open "$dir" "$avd" && continue
why=""
case "$known" in *" $runner "*) ;; *) why=": no such runner here" ;; esac
say " $(du -sh "$dir" 2>/dev/null | cut -f1) $avd$why"
run rm -rf "$dir" "$AVD_HOME/$avd.ini"
n=$((n + 1))
done
say " ${n} AVDs"

# ---------------------------------------------------------------------------
# A runner's leftover self-update. The runner unpacks a new version into
# _work/_update, copies it into bin.<version>/externals.<version> and is meant
# to remove the staging copy; nine runners on ultra kept theirs (0.45 GB each,
# 2026-10-22). Only a copy older than six hours on an idle runner that no
# process is running from goes: an update in progress takes minutes.
# ---------------------------------------------------------------------------
say "==> leftover runner self-updates"
n=0
for d in "$ROOT"/*/; do
d="${d%/}"
up="$d/_work/_update"
[ -f "$d/.runner" ] && [ -d "$up" ] || continue
[ -n "$(find "$up" -maxdepth 0 -mmin +360 2>/dev/null)" ] || continue
building "$(runner_name "$d")" && continue
pgrep -f "$up/" >/dev/null 2>&1 && continue
say " $(du -sh "$up" 2>/dev/null | cut -f1) $(basename "$d")/_work/_update"
run rm -rf "$up"
n=$((n + 1))
done
say " ${n} directories"

# ---------------------------------------------------------------------------
# Under pressure only: each idle runner's Playwright browsers. They are 0.5-1.1 GB
# per web runner (13 GB on runner-host) and the next job that needs them
Expand Down
61 changes: 60 additions & 1 deletion scripts/test-cleanup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ EOF
}

clean() {
HOME="$T/home" FLEET_ROOT="$T" PATH="$T/bin:$PATH" bash "$T/cleanup.sh" "$@" 2>&1
HOME="$T/home" FLEET_ROOT="$T" PATH="$T/bin:$PATH" ANDROID_AVD_HOME="" bash "$T/cleanup.sh" "$@" 2>&1
}
calls() { grep -c "$1" "$T/calls" 2>/dev/null; }
gone() { [ -e "$1" ] && echo kept || echo removed; }
Expand Down Expand Up @@ -222,6 +222,65 @@ clean --apply > "$T/out"
ok "no missing-file noise" "$(grep -c -E 'No such file|command not found' "$T/out")" "0"
ok "went by df, so under pressure" "$(grep -c 'UNDER PRESSURE' "$T/out")" "1"

echo "== a repo's own ci-<runner> <app> devices count as CI devices =="
setup 12
echo 120 > "$T/free"
echo 1 > "$T/.cleanup-last-full"
echo "RL-web" > "$T/busy"
cat > "$T/sims.json" <<'EOF'
{"devices": {
"com.apple.CoreSimulator.SimRuntime.iOS-27-0": [
{"name": "ci-RL-ios jerv iOS 27.0", "udid": "U-JERV", "state": "Shutdown", "dataPathSize": 5368709120},
{"name": "ci-RL-web jerv iOS 27.0", "udid": "U-JWEB", "state": "Shutdown", "dataPathSize": 5368709120},
{"name": "iPhone 17 Pro", "udid": "U-PERSON", "state": "Shutdown", "dataPathSize": 12884901888}
],
"com.apple.CoreSimulator.SimRuntime.watchOS-27-0": [
{"name": "ci-RL-gone jerv watchOS 27.0", "udid": "U-JGONE", "state": "Shutdown", "dataPathSize": 1048576}
]}}
EOF
clean --apply > "$T/out"
ok "erased an idle runner's app device over 3 GB" "$(calls 'simctl erase U-JERV')" "1"
ok "left a busy runner's app device" "$(calls 'U-JWEB')" "0"
ok "deleted an app device whose runner is gone" "$(calls 'simctl delete U-JGONE')" "1"
ok "never touched a device outside CI" "$(calls 'U-PERSON')" "0"

echo "== idle runners' AVDs and leftover self-updates go, nothing else =="
setup 13
echo 120 > "$T/free"
echo 1 > "$T/.cleanup-last-full"
echo "RL-web" > "$T/busy"
A="$T/home/.android/avd"
mkdir -p "$A"
avd() {
mkdir -p "$A/$1.avd"; : > "$A/$1.avd/userdata-qemu.img.qcow2"; echo "path=$A/$1.avd" > "$A/$1.ini"
[ "${2:-old}" = old ] && touch -t 202601010000 "$A/$1.avd/userdata-qemu.img.qcow2" "$A/$1.avd"
}
avd ci-RL-ios; avd shots-RL-android; avd ci-RL-web; avd ci-RL-gone; avd fleet-explore-1
avd ci-RL-android new
avd shots-RL-ios
sleep 300 >/dev/null 2>&1 &
EMU=$!
echo "$EMU" > "$A/shots-RL-ios.avd/hardware-qemu.ini.lock"
touch -t 202601010000 "$A/shots-RL-ios.avd/hardware-qemu.ini.lock" "$A/shots-RL-ios.avd"
for r in ios web android; do mkdir -p "$T/$r/_work/_update/bin"; done
touch -t 202601010000 "$T/ios/_work/_update" "$T/web/_work/_update"
clean > "$T/out"
ok "a dry run keeps every AVD" "$(gone "$A/ci-RL-ios.avd")" "kept"
ok "a dry run keeps the leftover update" "$(gone "$T/ios/_work/_update")" "kept"
clean --apply > "$T/out"
ok "removed an idle runner's AVD" "$(gone "$A/ci-RL-ios.avd")" "removed"
ok "and its .ini" "$(gone "$A/ci-RL-ios.ini")" "removed"
ok "removed an idle runner's screenshot AVD" "$(gone "$A/shots-RL-android.avd")" "removed"
ok "removed the AVD of a runner that no longer exists" "$(gone "$A/ci-RL-gone.avd")" "removed"
ok "kept a busy runner's AVD" "$(gone "$A/ci-RL-web.avd")" "kept"
ok "kept an AVD written in the last 30 minutes" "$(gone "$A/ci-RL-android.avd")" "kept"
ok "kept an AVD an emulator has open" "$(gone "$A/shots-RL-ios.avd")" "kept"
ok "never touched an AVD outside CI" "$(gone "$A/fleet-explore-1.avd")" "kept"
ok "removed an idle runner's old self-update" "$(gone "$T/ios/_work/_update")" "removed"
ok "kept a busy runner's self-update" "$(gone "$T/web/_work/_update")" "kept"
ok "kept a self-update under six hours old" "$(gone "$T/android/_work/_update")" "kept"
{ kill "$EMU"; wait "$EMU"; } 2>/dev/null

echo
echo "passed $PASS, failed $FAIL"
[ "$FAIL" -eq 0 ]
Loading