From 9f9a75f3ed5899286f04c307c7bb7341cd946437 Mon Sep 17 00:00:00 2001 From: Taylor Addison Date: Fri, 23 Oct 2026 13:28:07 -0400 Subject: [PATCH] cleanup: keep runners' snapshot AVDs, stop orphaned CI emulators cleanly The launch kit's android-ci emulator_snapshot boots each runner's emulator from a quick-boot snapshot kept in its own AVD (ci--a--). cleanup.sh deleted every idle runner's ci- AVD after 30 minutes, which would make the next job boot three times instead of once. - Snapshot AVDs of runners that exist are kept until 14 days without a boot (FLEET_CLEANUP_AVD_KEEP_DAYS). Their total is held under FLEET_CLEANUP_AVD_KEEP_GB (30) by dropping the least recently used idle snapshots/ (the AVD stays), also on a quiet --auto tick; below the admission floor every idle snapshot goes before any other step. Plain ci- AVDs (recreated every job) and AVDs of gone runners still go. - hooks/emulator-control.sh: the job-completed hook stops an emulator the job left on its runner's AVD, and every cleanup tick stops one whose parent is launchd and whose runner runs no job (two greenfolio emulators ran 9 h after cancelled jobs on 10-23). adb emu kill, then SIGTERM, SIGKILL last, dropping the snapshot of a save-capable boot that needed SIGKILL. - Tests: scripts/test-cleanup.sh (snapshot keep/stale/gone, floor, budget, orphans), hooks/tests/emulator.sh (owner matching, job-end stop). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 3 + CHANGELOG.md | 22 ++++ cleanup.sh | 257 ++++++++++++++++++++++++++++++-------- docs/configuration.md | 5 + docs/reference/scripts.md | 37 +++++- hooks/emulator-control.sh | 167 +++++++++++++++++++++++++ hooks/job-completed.sh | 6 + hooks/tests/emulator.sh | 105 ++++++++++++++++ scripts/test-cleanup.sh | 138 +++++++++++++++++++- 9 files changed, 685 insertions(+), 55 deletions(-) create mode 100755 hooks/emulator-control.sh create mode 100755 hooks/tests/emulator.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28f47a6..8b8b917 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,6 +126,9 @@ jobs: - name: Simulator cleanup tests run: bash hooks/tests/simulator.sh + - name: Emulator stop tests + run: bash hooks/tests/emulator.sh + - name: Admission control tests run: bash hooks/tests/run.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index c3e6521..a0e9db8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,28 @@ version numbers (`v0.1.0`, `v0.1.1`, etc.). ### Added +- **`cleanup.sh` keeps runners' quick-boot snapshot AVDs.** The launch kit's + android-ci `emulator_snapshot` boots each runner's emulator from a snapshot + saved in its own AVD, `ci--a--` (~6 GB). The + AVD step deleted every idle runner's `ci-` AVD after 30 minutes, so the next + job would have booted three times (initialise, save, test) instead of once. + Now a snapshot AVD is kept while its runner exists and booted it in the last + `FLEET_CLEANUP_AVD_KEEP_DAYS` (14); the snapshot AVDs in all are held under + `FLEET_CLEANUP_AVD_KEEP_GB` (30) by dropping the least recently used idle + snapshots (even on a quiet `--auto` tick), and below the admission floor every + idle snapshot goes first, before any other step. Plain `ci-` AVDs, + recreated every job, still go after 30 idle minutes, and an AVD of a runner + that no longer exists still goes. +- **Orphaned CI emulators are stopped.** android-emulator-runner leaves its + emulator detached, and a cancelled or timed-out job left it running (ultra, + 2026-10-23: two greenfolio emulators for 9 h); shims/ps answers the runner's + own orphan scan, so nothing stopped them. The job-completed hook now stops an + emulator left on its runner's AVD (`hooks/emulator-control.sh`), and every + cleanup tick stops one whose parent is launchd and whose runner runs no job. + Both use `adb emu kill`, then SIGTERM, SIGKILL last (dropping that AVD's + snapshot if the boot could have been saving), so a snapshot is never cut off + mid-save. + - **Admission lanes (`FLEET_ADMIT_LANES`).** One host-wide FIFO made a 5-second job wait behind 12-minute Xcode builds: on ultra load rose about 8 per concurrent job on 30 cores, and 74 % of jobs finish in under 90 s. Each diff --git a/cleanup.sh b/cleanup.sh index 569d483..d1367cc 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -24,6 +24,10 @@ # browsers. # - never touch a simulator or Android AVD that is not a runner's own # ci- (or shots-) device. +# - never throw away a runner's quick-boot snapshot AVD (the kit's +# emulator_snapshot) while its runner exists and used it in the last +# FLEET_CLEANUP_AVD_KEEP_DAYS, except to stay under +# FLEET_CLEANUP_AVD_KEEP_GB or, below the admission floor, first of all. set -uo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -77,6 +81,92 @@ else fi free_gb() { fleet_usable_free_gb; } PLAIN_HARD_GB="${FLEET_ADMIT_MIN_PLAIN_FREE_GB:-15}" +# The admission floor itself: below it every job is held, so the snapshots go +# first (see the AVD step). +FLOOR_GB="${FLEET_ADMIT_MIN_FREE_DISK_GB:-40}" + +# A runner's name as the kit spells it in AVD and simulator names. +runner_name() { + python3 -c "import json,sys;print(json.load(open(sys.argv[1],encoding='utf-8-sig'))['agentName'])" \ + "$1/.runner" 2>/dev/null +} + +AVD_HOME="${ANDROID_AVD_HOME:-$HOME/.android/avd}" +AVD_PREFIXES="${FLEET_CLEANUP_AVD_PREFIXES:-ci shots}" +AVD_KEEP_DAYS="${FLEET_CLEANUP_AVD_KEEP_DAYS:-14}" +case "$AVD_KEEP_DAYS" in '' | *[!0-9]*) AVD_KEEP_DAYS=14 ;; esac +AVD_KEEP_GB="${FLEET_CLEANUP_AVD_KEEP_GB:-30}" +case "$AVD_KEEP_GB" in '' | *[!0-9]*) AVD_KEEP_GB=30 ;; esac +EMU_ORPHAN_MIN_S="${FLEET_CLEANUP_EMU_ORPHAN_MIN_S:-300}" +case "$EMU_ORPHAN_MIN_S" in '' | *[!0-9]*) EMU_ORPHAN_MIN_S=300 ;; esac +# emu_list, emu_avd_owner, emu_stop: the job-completed hook's emulator helpers. +if [ -f "$HERE/hooks/emulator-control.sh" ]; then + # shellcheck source=hooks/emulator-control.sh + . "$HERE/hooks/emulator-control.sh" +fi +EMU_PREFIXES="$AVD_PREFIXES" +EMU_AVD_HOME="$AVD_HOME" + +# The kit's snapshot AVDs ("--a--"), in +# KB. A few du calls on a handful of directories: cheap enough for every tick. +avd_snapshot_kb() { + local t=0 kb d p + for p in $AVD_PREFIXES; do + for d in "$AVD_HOME/$p"-*-a[0-9]*-*.avd; do + [ -d "$d" ] || continue + kb="$(du -sk "$d" 2>/dev/null | cut -f1)" + case "$kb" in '' | *[!0-9]*) kb=0 ;; esac + t=$((t + kb)) + done + done + printf '%s' "$t" +} + +# --------------------------------------------------------------------------- +# Orphaned CI emulators, on every tick (it costs one ps when there are none). +# android-emulator-runner starts the emulator detached, so it belongs to launchd +# even while its job runs, and a job cancelled or timed out mid-step leaves it +# running: on ultra (2026-10-23) two greenfolio emulators ran 9 h that way, +# each holding 4 GB of RAM and its runner's AVD. One goes when its parent is +# launchd, it is older than FLEET_CLEANUP_EMU_ORPHAN_MIN_S, and its runner is +# running no job (no Runner.Worker under the runner's directory) or no longer +# exists. It is stopped the snapshot-safe way (hooks/emulator-control.sh). +# --------------------------------------------------------------------------- +reap_orphan_emulators() { + command -v emu_list >/dev/null 2>&1 || return 0 + local procs map names workers d name pid secs avd port save owner runner where dir + procs="$(emu_list | awk -v min="$EMU_ORPHAN_MIN_S" '$2 == 1 && $3 >= min')" + [ -n "$procs" ] || return 0 + map="" + for d in "$ROOT"/*/; do + [ -f "$d/.runner" ] || continue + name="$(runner_name "$d" | tr -c 'A-Za-z0-9._\n-' '_')" + [ -n "$name" ] && map="$map$name ${d%/} +" + done + names="$(printf '%s' "$map" | awk '{print $1}' | tr '\n' ' ')" + workers="$("${FLEET_PS:-/bin/ps}" -axo command= 2>/dev/null | grep 'Runner\.Worker')" + say "==> orphaned CI emulators (parent launchd, runner running no job)" + while read -r pid _ secs avd port save; do + [ -n "$pid" ] || continue + owner="$(emu_avd_owner "$avd" "$names")" || continue + runner="${owner%% *}" + where="${owner##* }" + if [ "$where" = here ]; then + dir="$(printf '%s' "$map" | awk -v n="$runner" '$1 == n { sub(/^[^ ]+ /, ""); print; exit }')" + if [ -n "$dir" ] && printf '%s\n' "$workers" \ + | awk -v d="$dir/bin" 'index($0, d) == 1 { f = 1 } END { exit !f }'; then + continue + fi + fi + if [ "$APPLY" = "1" ]; then + say " $avd (pid $pid, port $port, up ${secs}s, runner $runner $where): $(emu_stop "$pid" "$port" "$avd" "$save")" + else + say " [dry-run] stop $avd (pid $pid, port $port, up ${secs}s, runner $runner $where)" + fi + done <<< "$procs" +} +reap_orphan_emulators # --auto runs every 15 minutes, so its no-op has to cost two cheap reads and # nothing else: the host it guards is usually saturated when the disk is short. @@ -85,7 +175,13 @@ PLAIN_NOW="$(fleet_plain_free_gb)" PRESSURE=0 [ -n "$FREE_NOW" ] && [ "$FREE_NOW" -lt "$PRESSURE_GB" ] && PRESSURE=1 [ -n "$PLAIN_NOW" ] && [ "$PLAIN_NOW" -lt $((PLAIN_HARD_GB + 10)) ] && PRESSURE=1 -if [ "$AUTO" = "1" ] && [ "$PRESSURE" = "0" ]; then +FLOOR=0 +[ -n "$FREE_NOW" ] && [ "$FREE_NOW" -lt "$FLOOR_GB" ] && FLOOR=1 +[ -n "$PLAIN_NOW" ] && [ "$PLAIN_NOW" -lt "$PLAIN_HARD_GB" ] && FLOOR=1 +AVD_SNAP_KB="$(avd_snapshot_kb)" +AVD_OVER=0 +[ "$AVD_SNAP_KB" -gt $((AVD_KEEP_GB * 1048576)) ] && AVD_OVER=1 +if [ "$AUTO" = "1" ] && [ "$PRESSURE" = "0" ] && [ "$AVD_OVER" = "0" ]; then last="$(cat "$STAMP" 2>/dev/null)" case "$last" in '' | *[!0-9]*) last=0 ;; esac if [ $(( $(date +%s) - last )) -lt "$FULL_EVERY_S" ]; then @@ -101,7 +197,7 @@ if ! mkdir "$LOCK" 2>/dev/null; then fi fi trap 'rm -rf "$LOCK"' EXIT -say "==> $(date '+%F %T') free ${FREE_NOW} GB (${PLAIN_NOW} GB without purgeable), pressure line ${PRESSURE_GB} GB$([ "$PRESSURE" = 1 ] && echo ' — UNDER PRESSURE')" +say "==> $(date '+%F %T') free ${FREE_NOW} GB (${PLAIN_NOW} GB without purgeable), pressure line ${PRESSURE_GB} GB$([ "$PRESSURE" = 1 ] && echo ' — UNDER PRESSURE')$([ "$FLOOR" = 1 ] && echo ', BELOW THE FLOOR')$([ "$AVD_OVER" = 1 ] && echo " — snapshot AVDs over ${AVD_KEEP_GB} GB")" # --------------------------------------------------------------------------- # Jobs in flight. Deleting DerivedData out from under a live xcodebuild produces @@ -111,11 +207,6 @@ say "==> $(date '+%F %T') free ${FREE_NOW} GB (${PLAIN_NOW} GB without purgeabl # only the runners that are building. Refusing everything while any job ran # meant this never ran on a busy day, which is when the disk runs out. # --------------------------------------------------------------------------- -# A runner directory's GitHub name, which is what GitHub and the slots use. -runner_name() { - python3 -c "import json,sys;print(json.load(open(sys.argv[1],encoding='utf-8-sig'))['agentName'])" \ - "$1/.runner" 2>/dev/null -} # The runners on THIS host. GitHub lists every runner of a repository, so with # the same repo mirrored on two hosts (runner-host and ultra, 2026-10-09) the # other host's busy runners came back too, and each host's host-wide steps @@ -211,6 +302,113 @@ host_idle() { before=$(free_gb) say "==> free before: ${before} GB" +# This host's runners, spelled as the kit spells them in device and AVD names. +known=" " +for d in "$ROOT"/*/; do + [ -f "$d/.runner" ] || continue + known="$known$(runner_name "$d" | tr -c 'A-Za-z0-9._\n-' '_') " +done + +# --------------------------------------------------------------------------- +# Runners' own Android emulators (AVDs). First, because below the admission +# floor their snapshots are what goes before anything else. +# +# Plain "ci-" (Jerv's screenshots: "shots-"): android-emulator- +# runner creates it again at the start of every job (force-avd-creation), so +# between jobs it is 1-1.5 GB the next job overwrites anyway; on ultra +# (2026-10-22) 17 of them held 20 GB. One goes once its runner is idle and it is +# 30 minutes untouched. +# +# Snapshot "ci--a--" (the kit's android-ci with +# emulator_snapshot, taylab-launch-kit v1.38.0): the AVD and its quick-boot +# snapshot (~6 GB; the guest's 4 GB of RAM is most of it) are what the runner's +# next job boots from in seconds instead of cold. Deleting one makes that job +# boot three times (initialise, save, test) instead of once, so it is KEPT +# while its runner exists, until FLEET_CLEANUP_AVD_KEEP_DAYS without a boot. +# Two limits on the kept ones, both met by dropping snapshots/ (the AVD stays, +# so the next job only saves a new snapshot), least recently used first, never +# a building runner's, an open one, or one written in the last 30 minutes: +# - FLEET_CLEANUP_AVD_KEEP_GB: the snapshot AVDs in all; +# - below the admission floor (FLEET_ADMIT_MIN_FREE_DISK_GB, usable): every +# idle snapshot, before any other step, since the floor holds every job. +# A snapshot AVD whose runner no longer exists goes like a plain one. Other +# AVDs (night QA's fleet-explore-*, a person's) are never touched. +# --------------------------------------------------------------------------- +avd_open() { + local dir="$1" pid + pid="$(cat "$dir/hardware-qemu.ini.lock" 2>/dev/null | tr -dc '0-9')" + [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null && return 0 + pgrep -f -- "-avd $2( |\$)" >/dev/null 2>&1 +} +avd_last_use() { + find "$1" -maxdepth 1 -exec stat -f %m {} + 2>/dev/null | sort -n | tail -1 +} +du_kb() { + local kb + kb="$(du -sk "$1" 2>/dev/null | cut -f1)" + case "$kb" in '' | *[!0-9]*) kb=0 ;; esac + printf '%s' "$kb" +} +say "==> runners' Android emulators (snapshot AVDs: kept ${AVD_KEEP_DAYS}d unused, ${AVD_KEEP_GB} GB in all)$([ "$FLOOR" = 1 ] && echo ' — below the floor: idle snapshots go first')" +AVD_KEPT="$(mktemp -t cleanup-avd 2>/dev/null || echo "/tmp/.cleanup-avd-$$")" +: > "$AVD_KEPT" +snap_total="$AVD_SNAP_KB" +n=0 +s=0 +for dir in "$AVD_HOME"/*.avd; do + [ -d "$dir" ] || continue + avd="$(basename "$dir" .avd)" + owner="$(emu_avd_owner "$avd" "$known" 2>/dev/null)" || continue + runner="${owner%% *}" + rest="${owner#* }" + kind="${rest%% *}" + where="${rest##* }" + building "$runner" && continue + avd_open "$dir" "$avd" && continue + recent=0 + [ -n "$(find "$dir" -maxdepth 1 -mmin -30 2>/dev/null | head -1)" ] && recent=1 + if [ "$kind" = snapshot ] && [ "$where" = here ]; then + if [ -z "$(find "$dir" -maxdepth 1 -mtime -"$AVD_KEEP_DAYS" -print -quit 2>/dev/null)" ]; then + kb="$(du_kb "$dir")" + say " $(du -sh "$dir" 2>/dev/null | cut -f1) $avd: no boot in ${AVD_KEEP_DAYS}d" + run rm -rf "$dir" "$AVD_HOME/$avd.ini" + snap_total=$((snap_total - kb)) + n=$((n + 1)) + continue + fi + [ "$recent" = 1 ] && continue + [ -d "$dir/snapshots" ] || continue + if [ "$FLOOR" = 1 ]; then + kb="$(du_kb "$dir/snapshots")" + say " $(du -sh "$dir/snapshots" 2>/dev/null | cut -f1) $avd/snapshots: below the floor" + run rm -rf "$dir/snapshots" "$dir/ci-snapshot.key" + snap_total=$((snap_total - kb)) + s=$((s + 1)) + continue + fi + echo "$(avd_last_use "$dir") $(du_kb "$dir/snapshots") $dir" >> "$AVD_KEPT" + continue + fi + [ "$recent" = 1 ] && continue + why="" + [ "$where" = gone ] && why=": no such runner here" + [ "$kind" = snapshot ] && snap_total=$((snap_total - $(du_kb "$dir"))) + say " $(du -sh "$dir" 2>/dev/null | cut -f1) $avd$why" + run rm -rf "$dir" "$AVD_HOME/$avd.ini" + n=$((n + 1)) +done +cap_kb=$((AVD_KEEP_GB * 1048576)) +while read -r _ kb dir; do + [ -n "$dir" ] || continue + [ "$snap_total" -gt "$cap_kb" ] || break + say " $(du -sh "$dir/snapshots" 2>/dev/null | cut -f1) $(basename "$dir" .avd)/snapshots: least recently used over ${AVD_KEEP_GB} GB" + run rm -rf "$dir/snapshots" "$dir/ci-snapshot.key" + snap_total=$((snap_total - kb)) + s=$((s + 1)) +done < <(sort -n "$AVD_KEPT") +rm -f "$AVD_KEPT" +say " ${n} AVDs, ${s} snapshots; snapshot AVDs now $((snap_total / 1048576)) GB" + # --------------------------------------------------------------------------- # DerivedData older than a week. Xcode rebuilds whatever it needs; the only cost # of being wrong here is one slow build. Keyed on mtime, so a project being @@ -306,11 +504,6 @@ say " ${n} directories" # --------------------------------------------------------------------------- sim_limit="$SIM_MAX_GB"; [ "$PRESSURE" = 1 ] && sim_limit="$SIM_PRESSURE_GB" say "==> CI simulators over ${sim_limit} GB" -known=" " -for d in "$ROOT"/*/; do - [ -f "$d/.runner" ] || continue - known="$known$(runner_name "$d" | tr -c 'A-Za-z0-9._\n-' '_') " -done n=0 while IFS='|' read -r udid state bytes runner; do [ -n "$udid" ] || continue @@ -341,46 +534,6 @@ for devs in devices.values(): ') say " ${n} devices" -# --------------------------------------------------------------------------- -# Each idle runner's own Android emulator. The kit's android-ci names one AVD -# per runner "ci-" (Jerv's screenshots: "shots-"), and -# android-emulator-runner creates it again at the start of every job -# (force-avd-creation defaults to true), so between jobs an AVD is 1-1.5 GB -# that the next job overwrites anyway. Nothing removed them: on ultra -# (2026-10-22) 17 of them held 20 GB. An AVD is left alone while its runner is -# building, while an emulator has it open, or within 30 minutes of its last -# write (a job that is creating it). Other AVDs (night QA's fleet-explore-*, a -# person's) are never touched. -# --------------------------------------------------------------------------- -AVD_HOME="${ANDROID_AVD_HOME:-$HOME/.android/avd}" -AVD_PREFIXES="${FLEET_CLEANUP_AVD_PREFIXES:-ci shots}" -avd_open() { - local dir="$1" pid - pid="$(cat "$dir/hardware-qemu.ini.lock" 2>/dev/null | tr -dc '0-9')" - [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null && return 0 - pgrep -f -- "-avd $2( |\$)" >/dev/null 2>&1 -} -say "==> idle runners' Android emulators" -n=0 -for dir in "$AVD_HOME"/*.avd; do - [ -d "$dir" ] || continue - avd="$(basename "$dir" .avd)" - runner="" - for p in $AVD_PREFIXES; do - case "$avd" in "$p"-?*) runner="${avd#"$p"-}"; break ;; esac - done - [ -n "$runner" ] || continue - building "$runner" && continue - [ -n "$(find "$dir" -maxdepth 1 -mmin -30 2>/dev/null | head -1)" ] && continue - avd_open "$dir" "$avd" && continue - why="" - case "$known" in *" $runner "*) ;; *) why=": no such runner here" ;; esac - say " $(du -sh "$dir" 2>/dev/null | cut -f1) $avd$why" - run rm -rf "$dir" "$AVD_HOME/$avd.ini" - n=$((n + 1)) -done -say " ${n} AVDs" - # --------------------------------------------------------------------------- # A runner's leftover self-update. The runner unpacks a new version into # _work/_update, copies it into bin./externals. and is meant diff --git a/docs/configuration.md b/docs/configuration.md index 3228aa8..be9ddc7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -74,6 +74,11 @@ Used by `cleanup.sh` and `cleanupctl.sh`. See | `FLEET_CLEANUP_SIM_MAX_GB` | `3` | Erase an idle runner's `ci-` simulator past this size. | | `FLEET_CLEANUP_SIM_PRESSURE_GB` | `1` | ...and past this under pressure. | | `FLEET_CLEANUP_FULL_EVERY_S` | `86400` | `--auto` runs a full pass at least this often. | +| `FLEET_CLEANUP_AVD_KEEP_DAYS` | `14` | Keep a runner's quick-boot snapshot AVD (`ci--a--`, the kit's `emulator_snapshot`) until it has gone this long without a boot. | +| `FLEET_CLEANUP_AVD_KEEP_GB` | `30` | The snapshot AVDs in all. Over it, even on a quiet `--auto` tick, idle runners' snapshots (`snapshots/` inside the AVD) go, least recently used first; the AVD stays. Below the admission floor every idle snapshot goes, before any other step. | +| `FLEET_CLEANUP_AVD_PREFIXES` | `ci shots` | AVD name prefixes that mark a runner's own emulator. Nothing else is touched. | +| `FLEET_CLEANUP_EMU_ORPHAN_MIN_S` | `300` | Stop a CI emulator whose parent is launchd and whose runner runs no job once it is this old (on every tick). | +| `FLEET_EMULATOR_CLEANUP` | `1` | The job-completed hook stops an emulator the job left running on its runner's AVD (`adb emu kill`, then SIGTERM, SIGKILL last). `0` turns it off. | | `FLEET_CLEANUP_INTERVAL` | `900` | Seconds between `--auto` ticks. Reinstall after changing: `./cleanupctl.sh install`. | | `FLEET_CLEANUP_LOG` | `~/Library/Logs/runner-cleanup.log` | Where the cleanup LaunchAgent writes stdout/stderr. | diff --git a/docs/reference/scripts.md b/docs/reference/scripts.md index 7c5fd76..6890b26 100644 --- a/docs/reference/scripts.md +++ b/docs/reference/scripts.md @@ -27,7 +27,7 @@ without `--repair` — have no `--apply` because there is nothing to guard. | `healthctl.sh` | Install a LaunchAgent that runs `health.sh --repair` on a timer | n/a — subcommands | | `cleanupctl.sh` | Install a LaunchAgent that runs `cleanup.sh --apply --auto` every 15 minutes | n/a — subcommands | | `runs.sh` | What is building across every repo, and on which machine | read-only | -| `cleanup.sh` | Erase idle CI simulators; prune stale DerivedData, dead simulators, old `_diag`, superseded runner versions | **yes** | +| `cleanup.sh` | Erase idle CI simulators and AVDs (keeping runners' snapshot AVDs), stop orphaned CI emulators; prune stale DerivedData, dead simulators, old `_diag`, superseded runner versions | **yes** | | `preflight.sh` | Check a host against what the workflows assume | read-only, installs nothing | | `scripts/deregister.sh` | Remove one named runner completely | **yes** | | `scripts/drain-runner.sh` | Stop a runner gracefully, or resume it | no — every state it writes is reversible | @@ -258,6 +258,9 @@ Source: [`cleanup.sh`](https://github.com/addisdev/actions-runners/blob/main/cle | `FLEET_CLEANUP_SIM_MAX_GB` | 3 | Erase a runner's `ci-` simulator past this size | | `FLEET_CLEANUP_SIM_PRESSURE_GB` | 1 | ...and past this under pressure | | `FLEET_CLEANUP_FULL_EVERY_S` | 86400 | `--auto` runs a full pass at least this often | +| `FLEET_CLEANUP_AVD_KEEP_DAYS` | 14 | Keep a runner's snapshot AVD until this long without a boot | +| `FLEET_CLEANUP_AVD_KEEP_GB` | 30 | Snapshot AVDs in all; over it, least recently used snapshots go | +| `FLEET_CLEANUP_EMU_ORPHAN_MIN_S` | 300 | Age at which an unowned CI emulator is stopped | Host-wide steps, only while no job is building: DerivedData directories older than 7 days, simulators whose runtime is no longer installed, and stale @@ -272,6 +275,23 @@ deleted. Under pressure, idle runners' Playwright browsers in `_work/_tool/ms-playwright` are removed; the next job that needs them downloads them again. +Runners' Android emulators, first of the deleting steps. A plain +`ci-` (or `shots-`) AVD, which android-emulator-runner recreates +every job, goes once its runner is idle and it is 30 minutes untouched. A +snapshot AVD, `ci--a--` (the launch kit's +`emulator_snapshot`), is kept while its runner exists and has booted it in the +last `FLEET_CLEANUP_AVD_KEEP_DAYS`: deleting it would make that runner's next +job boot three times instead of once. Over `FLEET_CLEANUP_AVD_KEEP_GB`, idle +runners' snapshots (`snapshots/`, not the AVD) go least recently used first; +below the admission floor every idle runner's snapshot goes before anything +else. A building runner's AVD, one an emulator has open, and one written in the +last 30 minutes are never touched. + +Every tick, even a quiet `--auto` one: a CI emulator (`-avd ci-…`/`shots-…`) +whose parent is launchd, older than `FLEET_CLEANUP_EMU_ORPHAN_MIN_S`, whose +runner runs no job (no `Runner.Worker` under its directory) or no longer exists +is stopped as `hooks/emulator-control.sh` does it. + Always: runner `_diag` logs older than 14 days and runner versions older than the linked one (a self-update leaves the previous pair behind). It prints free disk before and after. @@ -773,6 +793,21 @@ lease has a detached `Runner.Worker` guardian, so cancellation and abrupt worker failure follow the same cleanup path. Source: [`hooks/simulator-control.sh`](https://github.com/addisdev/actions-runners/blob/main/hooks/simulator-control.sh). +### `hooks/emulator-control.sh` + +Internal helper used by the job-completed hook (`complete`) and sourced by +`cleanup.sh`. It stops an Android emulator the job left running on its +runner's own AVD (`ci-…`, `shots-…`) the way a quick-boot +snapshot survives: the console's `adb emu kill`, then SIGTERM, then SIGKILL; a +save-capable boot that needed SIGKILL has its snapshot removed, since it may +have been mid-save. Night QA's `fleet-explore-*` and any other AVD are never +touched. `FLEET_EMULATOR_CLEANUP=0` turns the hook path off. +Source: [`hooks/emulator-control.sh`](https://github.com/addisdev/actions-runners/blob/main/hooks/emulator-control.sh). + +```bash +hooks/tests/emulator.sh # stand-in emulators, fake adb; no real emulator is touched +``` + ### `hooks/shm-reap.sh` Internal helper used by both job hooks. Removes System V shared memory segments diff --git a/hooks/emulator-control.sh b/hooks/emulator-control.sh new file mode 100755 index 0000000..c78a41e --- /dev/null +++ b/hooks/emulator-control.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# CI Android emulators, stopped so that a quick-boot snapshot survives them. +# +# emulator-control.sh complete # job-completed.sh: stop THIS runner's emulators +# +# and sourced by cleanup.sh, which reaps emulators no job owns any more. +# +# The kit's android-ci (taylab-launch-kit, `emulator_snapshot`) keeps one AVD per +# runner and system image, "ci--a--", with a +# quick-boot snapshot (~4 GB of guest RAM) that the next job on that runner +# boots from. android-emulator-runner stops its emulator with `adb emu kill`, +# which is the clean path. But a job that is cancelled or times out mid-step +# leaves its emulator running (on ultra, 2026-10-23, two greenfolio emulators +# ran 9 h after their jobs were cancelled), and nothing at job end stops it: +# the runner's own orphan scan is answered "not this job's" by shims/ps. A +# SIGKILLed emulator that was saving its snapshot leaves half a snapshot. +# +# So an emulator is stopped the way the emulator expects: the console's +# `emu kill`, then SIGTERM, and SIGKILL only when neither worked. A boot that +# could save (no -no-snapshot, -no-snapshot-save or -read-only) and had to be +# SIGKILLed may have been mid-save, so its AVD's snapshot is thrown away; the +# kit saves a new one on the runner's next job. +# +# Only AVDs named for a runner ("ci-…", "shots-…", FLEET_CLEANUP_AVD_PREFIXES) +# are ever touched. Night QA's fleet-explore-* and a person's own AVDs are not. +# +# Never fails its caller: the job-completed hook must not turn a green job red. + +EMU_PREFIXES="${FLEET_CLEANUP_AVD_PREFIXES:-ci shots}" +EMU_AVD_HOME="${ANDROID_AVD_HOME:-$HOME/.android/avd}" + +emu_adb() { + printf '%s' "${FLEET_ADB:-${ANDROID_HOME:-${ANDROID_SDK_ROOT:-$HOME/Library/Android/sdk}}/platform-tools/adb}" +} + +# One line per emulator process booting a runner's AVD: +# pid ppid age_seconds avd port could_save(1|0) +# The emulator binary is the command's first word (or the second, after an +# interpreter, which is how the tests' stand-in runs). +emu_list() { + "${FLEET_PS:-/bin/ps}" -axo pid=,ppid=,etime=,command= 2>/dev/null | awk -v prefixes="$EMU_PREFIXES" ' + function secs(t, d, a, p, n, i, s) { + d = 0 + if (index(t, "-")) { split(t, a, "-"); d = a[1]; t = a[2] } + n = split(t, p, ":"); s = 0 + for (i = 1; i <= n; i++) s = s * 60 + p[i] + return d * 86400 + s + } + BEGIN { np = split(prefixes, P, " ") } + { + hit = 0 + for (k = 4; k <= 5 && k <= NF; k++) { + exe = $k; sub(/.*\//, "", exe) + if (exe ~ /^(emulator|qemu-system)/) hit = 1 + } + if (!hit) next + avd = ""; port = "-"; save = 1 + for (i = 5; i <= NF; i++) { + if ($i == "-avd" && i < NF) avd = $(i + 1) + else if ($i == "-port" && i < NF) port = $(i + 1) + else if ($i == "-no-snapshot" || $i == "-no-snapshot-save" || $i == "-read-only") save = 0 + } + if (avd == "") next + ok = 0 + for (j = 1; j <= np; j++) + if (index(avd, P[j] "-") == 1 && length(avd) > length(P[j]) + 1) ok = 1 + if (ok) print $1, $2, secs($3), avd, port, save + }' +} + +# Whose AVD is this? Given the runner names on this host (space-separated, as +# the kit spells them: anything but [A-Za-z0-9_.-] made "_"), prints +# +# "-" is the plain AVD android-emulator-runner recreates every +# job; "--a--" is the kit's snapshot AVD. +# The longest matching runner wins, so ultra-jerv-2's AVD is never ultra-jerv's. +emu_avd_owner() { + local avd="$1" known="$2" p rest="" r best="" kind="" cand + for p in $EMU_PREFIXES; do + case "$avd" in "$p"-?*) rest="${avd#"$p"-}"; break ;; esac + done + [ -n "$rest" ] || return 1 + for r in $known; do + if [ "$rest" = "$r" ]; then + cand=plain + else + case "$rest" in "$r"-a[0-9]*-?*) cand=snapshot ;; *) continue ;; esac + fi + if [ "${#r}" -gt "${#best}" ]; then best="$r"; kind="$cand"; fi + done + if [ -n "$best" ]; then + printf '%s %s here\n' "$best" "$kind" + elif [[ "$rest" =~ ^(.+)-a[0-9]+-.+$ ]]; then + printf '%s snapshot gone\n' "${BASH_REMATCH[1]}" + else + printf '%s plain gone\n' "$rest" + fi +} + +emu_wait_gone() { + local pid="$1" n="$2" i=0 + while kill -0 "$pid" 2>/dev/null; do + [ "$i" -ge "$n" ] && return 1 + sleep 1 + i=$((i + 1)) + done + return 0 +} + +# emu_stop : prints how it went. +emu_stop() { + local pid="$1" port="$2" avd="$3" save="$4" adb apid + kill -0 "$pid" 2>/dev/null || { echo "already gone"; return 0; } + adb="$(emu_adb)" + if [ "$port" != - ] && [ -x "$adb" ]; then + # In the background: an adb whose server is wedged must not hold the hook. + "$adb" -s "emulator-$port" emu kill >/dev/null 2>&1 & + apid=$! + # A save-capable boot writes its snapshot (~4 GB) after `emu kill`. + if emu_wait_gone "$pid" "${FLEET_EMU_KILL_WAIT_S:-120}"; then + kill "$apid" 2>/dev/null; wait "$apid" 2>/dev/null + echo "stopped (emu kill)"; return 0 + fi + kill "$apid" 2>/dev/null; wait "$apid" 2>/dev/null + fi + kill -TERM "$pid" 2>/dev/null + if emu_wait_gone "$pid" "${FLEET_EMU_TERM_WAIT_S:-30}"; then + echo "stopped (SIGTERM)"; return 0 + fi + kill -KILL "$pid" 2>/dev/null + emu_wait_gone "$pid" 5 + if [ "$save" = 1 ] && [ -d "$EMU_AVD_HOME/$avd.avd/snapshots" ]; then + rm -rf "$EMU_AVD_HOME/$avd.avd/snapshots/default_boot" "$EMU_AVD_HOME/$avd.avd/ci-snapshot.key" + echo "SIGKILLed; its snapshot may have been mid-save, so it was removed" + return 0 + fi + echo "SIGKILLed" +} + +# The job-completed path: this runner's emulators, whatever their parent. A +# runner runs one job at a time, so an emulator on its AVD is this job's, and +# one still up now would hold the AVD's lock against the next job. +emu_complete() { + [ "${FLEET_EMULATOR_CLEANUP:-1}" = 1 ] || return 0 + local me pid avd port save owner + me="$(printf '%s' "${RUNNER_NAME:-}" | tr -c 'A-Za-z0-9_.-' '_')" + [ -n "$me" ] || return 0 + emu_list | while read -r pid _ _ avd port save; do + owner="$(emu_avd_owner "$avd" "$me")" || continue + case "$owner" in *" here") ;; *) continue ;; esac + echo "emulator-control: $avd (pid $pid, port $port) left running by this job: $(emu_stop "$pid" "$port" "$avd" "$save")" + done +} + +if [ "${BASH_SOURCE[0]}" = "$0" ]; then + trap 'exit 0' EXIT + _here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + _root="${FLEET_ROOT:-$(cd "$_here/.." && pwd)}" + # shellcheck source=/dev/null + [ -f "$_root/fleet.env" ] && . "$_root/fleet.env" + EMU_PREFIXES="${FLEET_CLEANUP_AVD_PREFIXES:-ci shots}" + case "${1:-}" in + complete) emu_complete ;; + list) emu_list ;; + esac +fi diff --git a/hooks/job-completed.sh b/hooks/job-completed.sh index c1a2881..0090b9b 100755 --- a/hooks/job-completed.sh +++ b/hooks/job-completed.sh @@ -27,6 +27,12 @@ ROOT="${FLEET_ROOT:-$(cd "$HERE/.." && pwd)}" # this completion hook. [ -f "$HERE/simulator-control.sh" ] && FLEET_ROOT="$ROOT" bash "$HERE/simulator-control.sh" complete +# Stop an Android emulator this job left running (a cancelled or timed-out +# emulator step), with the console's `emu kill` before any signal, so the kit's +# per-runner quick-boot snapshot is never cut off mid-save and the AVD's lock is +# free for this runner's next job. FLEET_EMULATOR_CLEANUP=0 turns it off. +[ -f "$HERE/emulator-control.sh" ] && FLEET_ROOT="$ROOT" bash "$HERE/emulator-control.sh" complete + # Same sweep as the started hook. Running it here too means a host that goes # quiet after a cancelled Postgres job is not left near the 32-segment cap until # the next job happens to start. diff --git a/hooks/tests/emulator.sh b/hooks/tests/emulator.sh new file mode 100755 index 0000000..224b571 --- /dev/null +++ b/hooks/tests/emulator.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Exercises emulator-control.sh with stand-in emulators and a fake adb. No real +# emulator or AVD is touched: ps is filtered to this test's own processes. +set -uo pipefail + +HOOKS="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ROOT="${TMPDIR:-/tmp}/emulator-hook-test/root" +PASS=0 +FAIL=0 + +ok() { + if [ "$2" = "$3" ]; then + echo " PASS $1 ($2)" + PASS=$((PASS + 1)) + else + echo " FAIL $1 — expected '$3', got '$2'" + FAIL=$((FAIL + 1)) + fi +} + +cleanup() { + pkill -f -- "$ROOT/" 2>/dev/null + rm -rf "$ROOT" +} +trap cleanup EXIT + +setup() { + cleanup + mkdir -p "$ROOT/bin" "$ROOT/home/.android/avd" + : > "$ROOT/calls" + cat > "$ROOT/bin/fakeps" < "$ROOT/bin/adb" <> "$ROOT/calls" +pkill -TERM -f -- "-port \${2#emulator-} -avd" 2>/dev/null +exit 0 +EOF + cat > "$ROOT/bin/qemu-system-aarch64-headless" <<'EOF' +#!/usr/bin/env bash +trap 'exit 0' TERM +while :; do sleep 1; done +EOF + chmod +x "$ROOT/bin/"* +} + +emu() { ( "$ROOT/bin/qemu-system-aarch64-headless" "$@" >/dev/null 2>&1 & ); } +alive() { pgrep -f -- "-avd $1" >/dev/null 2>&1 && echo running || echo stopped; } +complete() { + HOME="$ROOT/home" FLEET_ROOT="$ROOT" ANDROID_AVD_HOME="" FLEET_PS="$ROOT/bin/fakeps" \ + FLEET_ADB="$ROOT/bin/adb" FLEET_EMU_KILL_WAIT_S=3 FLEET_EMU_TERM_WAIT_S=1 "$@" \ + bash "$HOOKS/emulator-control.sh" complete 2>&1 +} + +echo "== whose AVD is it ==" +# shellcheck source=hooks/emulator-control.sh +. "$HOOKS/emulator-control.sh" +K="ultra-jerv ultra-jerv-2 ultra-greenfolio-android ultra-dozehound-firetv" +ok "a plain AVD" "$(emu_avd_owner ci-ultra-jerv "$K")" "ultra-jerv plain here" +ok "the longest runner wins" "$(emu_avd_owner ci-ultra-jerv-2 "$K")" "ultra-jerv-2 plain here" +ok "a phone snapshot AVD" "$(emu_avd_owner ci-ultra-greenfolio-android-a34-google_apis-pixel_6 "$K")" \ + "ultra-greenfolio-android snapshot here" +ok "a TV snapshot AVD" "$(emu_avd_owner ci-ultra-dozehound-firetv-a34-android-tv-tv_1080p "$K")" \ + "ultra-dozehound-firetv snapshot here" +ok "ultra-jerv-2's snapshot AVD is not ultra-jerv's" \ + "$(emu_avd_owner ci-ultra-jerv-2-a34-google_apis-pixel_6 "$K")" "ultra-jerv-2 snapshot here" +ok "a snapshot AVD of a runner that is gone" \ + "$(emu_avd_owner ci-ultra-jerv-9-a34-google_apis-pixel_6 "$K")" "ultra-jerv-9 snapshot gone" +ok "a screenshot AVD" "$(emu_avd_owner shots-ultra-jerv "$K")" "ultra-jerv plain here" +ok "not a runner's AVD" "$(emu_avd_owner fleet-explore-1 "$K" || echo none)" "none" + +echo "== job end: this runner's emulator is stopped with emu kill, nobody else's ==" +setup +emu -port 5554 -avd ci-RL-a-a34-google_apis-pixel_6 -no-snapshot-save +emu -port 5556 -avd ci-RL-a-2-a34-google_apis-pixel_6 -no-snapshot-save +emu -port 5558 -avd fleet-explore-1 +sleep 1 +OUT="$(complete env RUNNER_NAME=RL-a)" +ok "stopped this runner's emulator" "$(alive ci-RL-a-a34)" "stopped" +ok "through the console" "$(grep -c 'adb -s emulator-5554 emu kill' "$ROOT/calls")" "1" +ok "and said so in the job log" "$(printf '%s\n' "$OUT" | grep -c 'ci-RL-a-a34.*stopped (emu kill)')" "1" +ok "left RL-a-2's emulator" "$(alive ci-RL-a-2-a34)" "running" +ok "left night QA's emulator" "$(alive fleet-explore-1)" "running" + +echo "== job end with nothing running is silent ==" +OUT="$(complete env RUNNER_NAME=RL-b)" +ok "printed nothing" "$OUT" "" + +echo "== FLEET_EMULATOR_CLEANUP=0 turns it off ==" +emu -port 5560 -avd ci-RL-c -no-snapshot +sleep 1 +complete env RUNNER_NAME=RL-c FLEET_EMULATOR_CLEANUP=0 >/dev/null +ok "left the emulator running" "$(alive ci-RL-c)" "running" +complete env RUNNER_NAME=RL-c >/dev/null +ok "and stops it when on" "$(alive ci-RL-c)" "stopped" + +echo "== it never fails the hook ==" +complete env RUNNER_NAME= FLEET_PS=/nonexistent >/dev/null +ok "exit 0 without a runner name or ps" "$?" "0" + +echo +echo "passed $PASS, failed $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/scripts/test-cleanup.sh b/scripts/test-cleanup.sh index f78b344..e3382f1 100755 --- a/scripts/test-cleanup.sh +++ b/scripts/test-cleanup.sh @@ -27,7 +27,7 @@ setup() { mkdir -p "$T/bin" "$T/home/Library/Developer/Xcode/DerivedData" mkdir -p "$T/hooks" cp "$HERE/cleanup.sh" "$T/" - cp "$HERE/hooks/free-disk.sh" "$T/hooks/" + cp "$HERE/hooks/free-disk.sh" "$HERE/hooks/emulator-control.sh" "$T/hooks/" for r in web android ios; do mkdir -p "$T/$r/_work/_tool/ms-playwright/chromium" "$T/$r/_diag" printf '{"agentName":"RL-%s","gitHubUrl":"https://github.com/acme/%s"}' "$r" "$r" > "$T/$r/.runner" @@ -70,12 +70,27 @@ EOF cat > "$T/bin/gh" < "$T/bin/fakeps" < "$T/bin/adb" <> "$T/calls" +port="\${2#emulator-}" +pkill -TERM -f -- "-port \$port -avd" 2>/dev/null +exit 0 EOF chmod +x "$T/bin/"* } clean() { - HOME="$T/home" FLEET_ROOT="$T" PATH="$T/bin:$PATH" ANDROID_AVD_HOME="" bash "$T/cleanup.sh" "$@" 2>&1 + HOME="$T/home" FLEET_ROOT="$T" PATH="$T/bin:$PATH" ANDROID_AVD_HOME="" \ + FLEET_PS="$T/bin/fakeps" FLEET_ADB="$T/bin/adb" bash "$T/cleanup.sh" "$@" 2>&1 } calls() { grep -c "$1" "$T/calls" 2>/dev/null; } gone() { [ -e "$1" ] && echo kept || echo removed; } @@ -319,6 +334,125 @@ clean --apply > "$T/out" ok "under pressure, an idle runner's recent cache goes too" "$(gone "$T/android/_work/android/DerivedData-App")" "removed" ok "...but never a building runner's" "$(gone "$T/web/_work/web/DerivedData-App")" "kept" +echo "== snapshot AVDs: kept for a runner here, gone when stale or orphaned ==" +setup 40 +echo 120 > "$T/free" +echo 1 > "$T/.cleanup-last-full" +echo "RL-web" > "$T/busy" +A="$T/home/.android/avd" +mkdir -p "$A" +# snap [with-snapshot] +snap() { + mkdir -p "$A/$1.avd"; : > "$A/$1.avd/userdata-qemu.img.qcow2"; echo "path=$A/$1.avd" > "$A/$1.ini" + [ "${4:-}" = with-snapshot ] && { mkdir -p "$A/$1.avd/snapshots/default_boot"; : > "$A/$1.avd/snapshots/default_boot/ram.bin"; echo key > "$A/$1.avd/ci-snapshot.key"; } + case "$2" in + old) stamp=202601010000 ;; + days) stamp="$(date -v-"$3"d +%Y%m%d%H%M)" ;; + *) return 0 ;; + esac + find "$A/$1.avd" -exec touch -t "$stamp" {} + +} +snap ci-RL-ios-a34-google_apis-pixel_6 days 2 with-snapshot +snap ci-RL-ios-a34-android-tv-tv_1080p days 3 with-snapshot +snap ci-RL-android-a34-google_apis-pixel_6 days 20 with-snapshot +snap ci-RL-web-a34-google_apis-pixel_6 old with-snapshot +snap ci-RL-gone-a34-android-tv-tv_1080p days 2 with-snapshot +snap ci-RL-ios-2-a34-google_apis-pixel_6 days 2 with-snapshot +snap ci-RL-android x x +touch -t 202601010000 "$A/ci-RL-android.avd/userdata-qemu.img.qcow2" "$A/ci-RL-android.avd" +clean --apply > "$T/out" +ok "kept an idle runner's snapshot AVD used this week" "$(gone "$A/ci-RL-ios-a34-google_apis-pixel_6.avd/snapshots/default_boot")" "kept" +ok "kept the same runner's TV snapshot AVD" "$(gone "$A/ci-RL-ios-a34-android-tv-tv_1080p.avd/snapshots")" "kept" +ok "removed a snapshot AVD with no boot in 14 days" "$(gone "$A/ci-RL-android-a34-google_apis-pixel_6.avd")" "removed" +ok "and its .ini" "$(gone "$A/ci-RL-android-a34-google_apis-pixel_6.ini")" "removed" +ok "kept a building runner's snapshot AVD, however old" "$(gone "$A/ci-RL-web-a34-google_apis-pixel_6.avd")" "kept" +ok "removed the snapshot AVD of a runner that no longer exists" "$(gone "$A/ci-RL-gone-a34-android-tv-tv_1080p.avd")" "removed" +ok "RL-ios-2's AVD is not RL-ios's (no such runner: removed)" "$(gone "$A/ci-RL-ios-2-a34-google_apis-pixel_6.avd")" "removed" +ok "still removed an idle runner's plain AVD" "$(gone "$A/ci-RL-android.avd")" "removed" + +echo "== below the admission floor, idle runners' snapshots go first ==" +setup 41 +echo 30 > "$T/free" +date +%s > "$T/.cleanup-last-full" +echo "RL-web" > "$T/busy" +A="$T/home/.android/avd" +mkdir -p "$A" +snap ci-RL-ios-a34-google_apis-pixel_6 days 2 with-snapshot +snap ci-RL-web-a34-google_apis-pixel_6 days 2 with-snapshot +snap ci-RL-android-a34-google_apis-pixel_6 new x with-snapshot +clean --apply --auto > "$T/out" +ok "said it is below the floor" "$(grep -c 'BELOW THE FLOOR' "$T/out")" "1" +ok "dropped an idle runner's snapshot" "$(gone "$A/ci-RL-ios-a34-google_apis-pixel_6.avd/snapshots")" "removed" +ok "and its snapshot key" "$(gone "$A/ci-RL-ios-a34-google_apis-pixel_6.avd/ci-snapshot.key")" "removed" +ok "but kept the AVD itself" "$(gone "$A/ci-RL-ios-a34-google_apis-pixel_6.avd/userdata-qemu.img.qcow2")" "kept" +ok "kept a building runner's snapshot" "$(gone "$A/ci-RL-web-a34-google_apis-pixel_6.avd/snapshots")" "kept" +ok "kept a snapshot written in the last 30 minutes" "$(gone "$A/ci-RL-android-a34-google_apis-pixel_6.avd/snapshots")" "kept" +ok "the snapshots went before any other step" \ + "$(awk '/Android emulators/ {a = NR} /DerivedData older/ {d = NR} END {print (a && d && a < d) ? "first" : "not first"}' "$T/out")" "first" + +echo "== over FLEET_CLEANUP_AVD_KEEP_GB, the least recently used snapshots go, even on a quiet --auto tick ==" +setup 42 +echo 120 > "$T/free" +date +%s > "$T/.cleanup-last-full" +A="$T/home/.android/avd" +mkdir -p "$A" +snap ci-RL-ios-a34-google_apis-pixel_6 x x with-snapshot +snap ci-RL-android-a34-google_apis-pixel_6 x x with-snapshot +dd if=/dev/zero of="$A/ci-RL-ios-a34-google_apis-pixel_6.avd/snapshots/default_boot/ram.bin" bs=1048576 count=700 2>/dev/null +dd if=/dev/zero of="$A/ci-RL-android-a34-google_apis-pixel_6.avd/snapshots/default_boot/ram.bin" bs=1048576 count=700 2>/dev/null +find "$A/ci-RL-ios-a34-google_apis-pixel_6.avd" -exec touch -t "$(date -v-3d +%Y%m%d%H%M)" {} + +find "$A/ci-RL-android-a34-google_apis-pixel_6.avd" -exec touch -t "$(date -v-1d +%Y%m%d%H%M)" {} + +OUT="$(FLEET_CLEANUP_AVD_KEEP_GB=2 clean --apply --auto)" +ok "under the limit, a quiet tick stays quiet" "$OUT" "" +FLEET_CLEANUP_AVD_KEEP_GB=1 clean --apply --auto > "$T/out" +ok "over the limit, a quiet tick runs" "$(grep -c 'snapshot AVDs over 1 GB' "$T/out")" "1" +ok "dropped the least recently used snapshot" "$(gone "$A/ci-RL-ios-a34-google_apis-pixel_6.avd/snapshots")" "removed" +ok "kept its AVD" "$(gone "$A/ci-RL-ios-a34-google_apis-pixel_6.avd")" "kept" +ok "and stopped once under the limit" "$(gone "$A/ci-RL-android-a34-google_apis-pixel_6.avd/snapshots/default_boot/ram.bin")" "kept" +rm -rf "$A" + +echo "== orphaned CI emulators are stopped the snapshot-safe way ==" +setup 43 +echo 120 > "$T/free" +date +%s > "$T/.cleanup-last-full" +A="$T/home/.android/avd" +mkdir -p "$A" "$T/emu" "$T/web/bin" +snap ci-RL-ios-a34-google_apis-pixel_6 days 1 with-snapshot +cat > "$T/emu/qemu-system-aarch64-headless" <<'EOF' +#!/usr/bin/env bash +case " $* " in *" -stubborn "*) trap '' TERM ;; *) trap 'exit 0' TERM ;; esac +while :; do sleep 1; done +EOF +chmod +x "$T/emu/qemu-system-aarch64-headless" +# Detached, so launchd is the parent, as android-emulator-runner leaves it. +emu() { ( "$T/emu/qemu-system-aarch64-headless" "$@" >/dev/null 2>&1 & ); } +emu -port 5554 -avd ci-RL-android-a34-google_apis-pixel_6 -no-snapshot-save +emu -port 5556 -avd ci-RL-web-a34-google_apis-pixel_6 -no-snapshot-save +emu -port 5558 -avd ci-RL-ios-a34-google_apis-pixel_6 -stubborn +emu -port 5560 -avd fleet-explore-1 +emu -port 5562 -avd ci-RL-gone -no-snapshot +# RL-web is running a job: a Runner.Worker under its directory. +ln -s /bin/sleep "$T/web/bin/Runner.Worker" +( "$T/web/bin/Runner.Worker" 300 >/dev/null 2>&1 & ) +sleep 1 +alive() { pgrep -f -- "-avd $1" >/dev/null 2>&1 && echo running || echo stopped; } +FLEET_CLEANUP_EMU_ORPHAN_MIN_S=0 clean --auto > "$T/out" +ok "a dry run stops nothing" "$(alive ci-RL-android-a34)" "running" +ok "a dry run says what it would stop" "$(grep -c '\[dry-run\] stop ci-RL-android-a34' "$T/out")" "1" +FLEET_CLEANUP_EMU_ORPHAN_MIN_S=3600 clean --apply --auto > "$T/out" +ok "a young emulator is not an orphan yet" "$(alive ci-RL-android-a34)" "running" +FLEET_CLEANUP_EMU_ORPHAN_MIN_S=0 FLEET_EMU_KILL_WAIT_S=2 FLEET_EMU_TERM_WAIT_S=1 clean --apply --auto > "$T/out" +ok "stopped an idle runner's emulator" "$(alive ci-RL-android-a34)" "stopped" +ok "with the console's emu kill" "$(grep -c 'adb -s emulator-5554 emu kill' "$T/calls")" "1" +ok "and said so" "$(grep -c 'ci-RL-android-a34.*stopped (emu kill)' "$T/out")" "1" +ok "left the emulator of a runner running a job" "$(alive ci-RL-web-a34)" "running" +ok "never touched night QA's emulator" "$(alive fleet-explore-1)" "running" +ok "stopped the emulator of a runner that no longer exists" "$(alive 'ci-RL-gone ')" "stopped" +ok "SIGKILLed one that ignored emu kill and SIGTERM" "$(alive ci-RL-ios-a34)" "stopped" +ok "and, since it could have been saving, dropped its snapshot" "$(gone "$A/ci-RL-ios-a34-google_apis-pixel_6.avd/snapshots/default_boot")" "removed" +ok "but kept the AVD" "$(gone "$A/ci-RL-ios-a34-google_apis-pixel_6.avd")" "kept" +pkill -f -- "$T/" 2>/dev/null + echo echo "passed $PASS, failed $FAIL" [ "$FAIL" -eq 0 ]