diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28f47a6..8b8b917 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,6 +126,9 @@ jobs: - name: Simulator cleanup tests run: bash hooks/tests/simulator.sh + - name: Emulator stop tests + run: bash hooks/tests/emulator.sh + - name: Admission control tests run: bash hooks/tests/run.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index c3e6521..a0e9db8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,28 @@ version numbers (`v0.1.0`, `v0.1.1`, etc.). ### Added +- **`cleanup.sh` keeps runners' quick-boot snapshot AVDs.** The launch kit's + android-ci `emulator_snapshot` boots each runner's emulator from a snapshot + saved in its own AVD, `ci--a--` (~6 GB). The + AVD step deleted every idle runner's `ci-` AVD after 30 minutes, so the next + job would have booted three times (initialise, save, test) instead of once. + Now a snapshot AVD is kept while its runner exists and booted it in the last + `FLEET_CLEANUP_AVD_KEEP_DAYS` (14); the snapshot AVDs in all are held under + `FLEET_CLEANUP_AVD_KEEP_GB` (30) by dropping the least recently used idle + snapshots (even on a quiet `--auto` tick), and below the admission floor every + idle snapshot goes first, before any other step. Plain `ci-` AVDs, + recreated every job, still go after 30 idle minutes, and an AVD of a runner + that no longer exists still goes. +- **Orphaned CI emulators are stopped.** android-emulator-runner leaves its + emulator detached, and a cancelled or timed-out job left it running (ultra, + 2026-10-23: two greenfolio emulators for 9 h); shims/ps answers the runner's + own orphan scan, so nothing stopped them. The job-completed hook now stops an + emulator left on its runner's AVD (`hooks/emulator-control.sh`), and every + cleanup tick stops one whose parent is launchd and whose runner runs no job. + Both use `adb emu kill`, then SIGTERM, SIGKILL last (dropping that AVD's + snapshot if the boot could have been saving), so a snapshot is never cut off + mid-save. + - **Admission lanes (`FLEET_ADMIT_LANES`).** One host-wide FIFO made a 5-second job wait behind 12-minute Xcode builds: on ultra load rose about 8 per concurrent job on 30 cores, and 74 % of jobs finish in under 90 s. Each diff --git a/cleanup.sh b/cleanup.sh index 569d483..d1367cc 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -24,6 +24,10 @@ # browsers. # - never touch a simulator or Android AVD that is not a runner's own # ci- (or shots-) device. +# - never throw away a runner's quick-boot snapshot AVD (the kit's +# emulator_snapshot) while its runner exists and used it in the last +# FLEET_CLEANUP_AVD_KEEP_DAYS, except to stay under +# FLEET_CLEANUP_AVD_KEEP_GB or, below the admission floor, first of all. set -uo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -77,6 +81,92 @@ else fi free_gb() { fleet_usable_free_gb; } PLAIN_HARD_GB="${FLEET_ADMIT_MIN_PLAIN_FREE_GB:-15}" +# The admission floor itself: below it every job is held, so the snapshots go +# first (see the AVD step). +FLOOR_GB="${FLEET_ADMIT_MIN_FREE_DISK_GB:-40}" + +# A runner's name as the kit spells it in AVD and simulator names. +runner_name() { + python3 -c "import json,sys;print(json.load(open(sys.argv[1],encoding='utf-8-sig'))['agentName'])" \ + "$1/.runner" 2>/dev/null +} + +AVD_HOME="${ANDROID_AVD_HOME:-$HOME/.android/avd}" +AVD_PREFIXES="${FLEET_CLEANUP_AVD_PREFIXES:-ci shots}" +AVD_KEEP_DAYS="${FLEET_CLEANUP_AVD_KEEP_DAYS:-14}" +case "$AVD_KEEP_DAYS" in '' | *[!0-9]*) AVD_KEEP_DAYS=14 ;; esac +AVD_KEEP_GB="${FLEET_CLEANUP_AVD_KEEP_GB:-30}" +case "$AVD_KEEP_GB" in '' | *[!0-9]*) AVD_KEEP_GB=30 ;; esac +EMU_ORPHAN_MIN_S="${FLEET_CLEANUP_EMU_ORPHAN_MIN_S:-300}" +case "$EMU_ORPHAN_MIN_S" in '' | *[!0-9]*) EMU_ORPHAN_MIN_S=300 ;; esac +# emu_list, emu_avd_owner, emu_stop: the job-completed hook's emulator helpers. +if [ -f "$HERE/hooks/emulator-control.sh" ]; then + # shellcheck source=hooks/emulator-control.sh + . "$HERE/hooks/emulator-control.sh" +fi +EMU_PREFIXES="$AVD_PREFIXES" +EMU_AVD_HOME="$AVD_HOME" + +# The kit's snapshot AVDs ("--a--"), in +# KB. A few du calls on a handful of directories: cheap enough for every tick. +avd_snapshot_kb() { + local t=0 kb d p + for p in $AVD_PREFIXES; do + for d in "$AVD_HOME/$p"-*-a[0-9]*-*.avd; do + [ -d "$d" ] || continue + kb="$(du -sk "$d" 2>/dev/null | cut -f1)" + case "$kb" in '' | *[!0-9]*) kb=0 ;; esac + t=$((t + kb)) + done + done + printf '%s' "$t" +} + +# --------------------------------------------------------------------------- +# Orphaned CI emulators, on every tick (it costs one ps when there are none). +# android-emulator-runner starts the emulator detached, so it belongs to launchd +# even while its job runs, and a job cancelled or timed out mid-step leaves it +# running: on ultra (2026-10-23) two greenfolio emulators ran 9 h that way, +# each holding 4 GB of RAM and its runner's AVD. One goes when its parent is +# launchd, it is older than FLEET_CLEANUP_EMU_ORPHAN_MIN_S, and its runner is +# running no job (no Runner.Worker under the runner's directory) or no longer +# exists. It is stopped the snapshot-safe way (hooks/emulator-control.sh). +# --------------------------------------------------------------------------- +reap_orphan_emulators() { + command -v emu_list >/dev/null 2>&1 || return 0 + local procs map names workers d name pid secs avd port save owner runner where dir + procs="$(emu_list | awk -v min="$EMU_ORPHAN_MIN_S" '$2 == 1 && $3 >= min')" + [ -n "$procs" ] || return 0 + map="" + for d in "$ROOT"/*/; do + [ -f "$d/.runner" ] || continue + name="$(runner_name "$d" | tr -c 'A-Za-z0-9._\n-' '_')" + [ -n "$name" ] && map="$map$name ${d%/} +" + done + names="$(printf '%s' "$map" | awk '{print $1}' | tr '\n' ' ')" + workers="$("${FLEET_PS:-/bin/ps}" -axo command= 2>/dev/null | grep 'Runner\.Worker')" + say "==> orphaned CI emulators (parent launchd, runner running no job)" + while read -r pid _ secs avd port save; do + [ -n "$pid" ] || continue + owner="$(emu_avd_owner "$avd" "$names")" || continue + runner="${owner%% *}" + where="${owner##* }" + if [ "$where" = here ]; then + dir="$(printf '%s' "$map" | awk -v n="$runner" '$1 == n { sub(/^[^ ]+ /, ""); print; exit }')" + if [ -n "$dir" ] && printf '%s\n' "$workers" \ + | awk -v d="$dir/bin" 'index($0, d) == 1 { f = 1 } END { exit !f }'; then + continue + fi + fi + if [ "$APPLY" = "1" ]; then + say " $avd (pid $pid, port $port, up ${secs}s, runner $runner $where): $(emu_stop "$pid" "$port" "$avd" "$save")" + else + say " [dry-run] stop $avd (pid $pid, port $port, up ${secs}s, runner $runner $where)" + fi + done <<< "$procs" +} +reap_orphan_emulators # --auto runs every 15 minutes, so its no-op has to cost two cheap reads and # nothing else: the host it guards is usually saturated when the disk is short. @@ -85,7 +175,13 @@ PLAIN_NOW="$(fleet_plain_free_gb)" PRESSURE=0 [ -n "$FREE_NOW" ] && [ "$FREE_NOW" -lt "$PRESSURE_GB" ] && PRESSURE=1 [ -n "$PLAIN_NOW" ] && [ "$PLAIN_NOW" -lt $((PLAIN_HARD_GB + 10)) ] && PRESSURE=1 -if [ "$AUTO" = "1" ] && [ "$PRESSURE" = "0" ]; then +FLOOR=0 +[ -n "$FREE_NOW" ] && [ "$FREE_NOW" -lt "$FLOOR_GB" ] && FLOOR=1 +[ -n "$PLAIN_NOW" ] && [ "$PLAIN_NOW" -lt "$PLAIN_HARD_GB" ] && FLOOR=1 +AVD_SNAP_KB="$(avd_snapshot_kb)" +AVD_OVER=0 +[ "$AVD_SNAP_KB" -gt $((AVD_KEEP_GB * 1048576)) ] && AVD_OVER=1 +if [ "$AUTO" = "1" ] && [ "$PRESSURE" = "0" ] && [ "$AVD_OVER" = "0" ]; then last="$(cat "$STAMP" 2>/dev/null)" case "$last" in '' | *[!0-9]*) last=0 ;; esac if [ $(( $(date +%s) - last )) -lt "$FULL_EVERY_S" ]; then @@ -101,7 +197,7 @@ if ! mkdir "$LOCK" 2>/dev/null; then fi fi trap 'rm -rf "$LOCK"' EXIT -say "==> $(date '+%F %T') free ${FREE_NOW} GB (${PLAIN_NOW} GB without purgeable), pressure line ${PRESSURE_GB} GB$([ "$PRESSURE" = 1 ] && echo ' — UNDER PRESSURE')" +say "==> $(date '+%F %T') free ${FREE_NOW} GB (${PLAIN_NOW} GB without purgeable), pressure line ${PRESSURE_GB} GB$([ "$PRESSURE" = 1 ] && echo ' — UNDER PRESSURE')$([ "$FLOOR" = 1 ] && echo ', BELOW THE FLOOR')$([ "$AVD_OVER" = 1 ] && echo " — snapshot AVDs over ${AVD_KEEP_GB} GB")" # --------------------------------------------------------------------------- # Jobs in flight. Deleting DerivedData out from under a live xcodebuild produces @@ -111,11 +207,6 @@ say "==> $(date '+%F %T') free ${FREE_NOW} GB (${PLAIN_NOW} GB without purgeabl # only the runners that are building. Refusing everything while any job ran # meant this never ran on a busy day, which is when the disk runs out. # --------------------------------------------------------------------------- -# A runner directory's GitHub name, which is what GitHub and the slots use. -runner_name() { - python3 -c "import json,sys;print(json.load(open(sys.argv[1],encoding='utf-8-sig'))['agentName'])" \ - "$1/.runner" 2>/dev/null -} # The runners on THIS host. GitHub lists every runner of a repository, so with # the same repo mirrored on two hosts (runner-host and ultra, 2026-10-09) the # other host's busy runners came back too, and each host's host-wide steps @@ -211,6 +302,113 @@ host_idle() { before=$(free_gb) say "==> free before: ${before} GB" +# This host's runners, spelled as the kit spells them in device and AVD names. +known=" " +for d in "$ROOT"/*/; do + [ -f "$d/.runner" ] || continue + known="$known$(runner_name "$d" | tr -c 'A-Za-z0-9._\n-' '_') " +done + +# --------------------------------------------------------------------------- +# Runners' own Android emulators (AVDs). First, because below the admission +# floor their snapshots are what goes before anything else. +# +# Plain "ci-" (Jerv's screenshots: "shots-"): android-emulator- +# runner creates it again at the start of every job (force-avd-creation), so +# between jobs it is 1-1.5 GB the next job overwrites anyway; on ultra +# (2026-10-22) 17 of them held 20 GB. One goes once its runner is idle and it is +# 30 minutes untouched. +# +# Snapshot "ci--a--" (the kit's android-ci with +# emulator_snapshot, taylab-launch-kit v1.38.0): the AVD and its quick-boot +# snapshot (~6 GB; the guest's 4 GB of RAM is most of it) are what the runner's +# next job boots from in seconds instead of cold. Deleting one makes that job +# boot three times (initialise, save, test) instead of once, so it is KEPT +# while its runner exists, until FLEET_CLEANUP_AVD_KEEP_DAYS without a boot. +# Two limits on the kept ones, both met by dropping snapshots/ (the AVD stays, +# so the next job only saves a new snapshot), least recently used first, never +# a building runner's, an open one, or one written in the last 30 minutes: +# - FLEET_CLEANUP_AVD_KEEP_GB: the snapshot AVDs in all; +# - below the admission floor (FLEET_ADMIT_MIN_FREE_DISK_GB, usable): every +# idle snapshot, before any other step, since the floor holds every job. +# A snapshot AVD whose runner no longer exists goes like a plain one. Other +# AVDs (night QA's fleet-explore-*, a person's) are never touched. +# --------------------------------------------------------------------------- +avd_open() { + local dir="$1" pid + pid="$(cat "$dir/hardware-qemu.ini.lock" 2>/dev/null | tr -dc '0-9')" + [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null && return 0 + pgrep -f -- "-avd $2( |\$)" >/dev/null 2>&1 +} +avd_last_use() { + find "$1" -maxdepth 1 -exec stat -f %m {} + 2>/dev/null | sort -n | tail -1 +} +du_kb() { + local kb + kb="$(du -sk "$1" 2>/dev/null | cut -f1)" + case "$kb" in '' | *[!0-9]*) kb=0 ;; esac + printf '%s' "$kb" +} +say "==> runners' Android emulators (snapshot AVDs: kept ${AVD_KEEP_DAYS}d unused, ${AVD_KEEP_GB} GB in all)$([ "$FLOOR" = 1 ] && echo ' — below the floor: idle snapshots go first')" +AVD_KEPT="$(mktemp -t cleanup-avd 2>/dev/null || echo "/tmp/.cleanup-avd-$$")" +: > "$AVD_KEPT" +snap_total="$AVD_SNAP_KB" +n=0 +s=0 +for dir in "$AVD_HOME"/*.avd; do + [ -d "$dir" ] || continue + avd="$(basename "$dir" .avd)" + owner="$(emu_avd_owner "$avd" "$known" 2>/dev/null)" || continue + runner="${owner%% *}" + rest="${owner#* }" + kind="${rest%% *}" + where="${rest##* }" + building "$runner" && continue + avd_open "$dir" "$avd" && continue + recent=0 + [ -n "$(find "$dir" -maxdepth 1 -mmin -30 2>/dev/null | head -1)" ] && recent=1 + if [ "$kind" = snapshot ] && [ "$where" = here ]; then + if [ -z "$(find "$dir" -maxdepth 1 -mtime -"$AVD_KEEP_DAYS" -print -quit 2>/dev/null)" ]; then + kb="$(du_kb "$dir")" + say " $(du -sh "$dir" 2>/dev/null | cut -f1) $avd: no boot in ${AVD_KEEP_DAYS}d" + run rm -rf "$dir" "$AVD_HOME/$avd.ini" + snap_total=$((snap_total - kb)) + n=$((n + 1)) + continue + fi + [ "$recent" = 1 ] && continue + [ -d "$dir/snapshots" ] || continue + if [ "$FLOOR" = 1 ]; then + kb="$(du_kb "$dir/snapshots")" + say " $(du -sh "$dir/snapshots" 2>/dev/null | cut -f1) $avd/snapshots: below the floor" + run rm -rf "$dir/snapshots" "$dir/ci-snapshot.key" + snap_total=$((snap_total - kb)) + s=$((s + 1)) + continue + fi + echo "$(avd_last_use "$dir") $(du_kb "$dir/snapshots") $dir" >> "$AVD_KEPT" + continue + fi + [ "$recent" = 1 ] && continue + why="" + [ "$where" = gone ] && why=": no such runner here" + [ "$kind" = snapshot ] && snap_total=$((snap_total - $(du_kb "$dir"))) + say " $(du -sh "$dir" 2>/dev/null | cut -f1) $avd$why" + run rm -rf "$dir" "$AVD_HOME/$avd.ini" + n=$((n + 1)) +done +cap_kb=$((AVD_KEEP_GB * 1048576)) +while read -r _ kb dir; do + [ -n "$dir" ] || continue + [ "$snap_total" -gt "$cap_kb" ] || break + say " $(du -sh "$dir/snapshots" 2>/dev/null | cut -f1) $(basename "$dir" .avd)/snapshots: least recently used over ${AVD_KEEP_GB} GB" + run rm -rf "$dir/snapshots" "$dir/ci-snapshot.key" + snap_total=$((snap_total - kb)) + s=$((s + 1)) +done < <(sort -n "$AVD_KEPT") +rm -f "$AVD_KEPT" +say " ${n} AVDs, ${s} snapshots; snapshot AVDs now $((snap_total / 1048576)) GB" + # --------------------------------------------------------------------------- # DerivedData older than a week. Xcode rebuilds whatever it needs; the only cost # of being wrong here is one slow build. Keyed on mtime, so a project being @@ -306,11 +504,6 @@ say " ${n} directories" # --------------------------------------------------------------------------- sim_limit="$SIM_MAX_GB"; [ "$PRESSURE" = 1 ] && sim_limit="$SIM_PRESSURE_GB" say "==> CI simulators over ${sim_limit} GB" -known=" " -for d in "$ROOT"/*/; do - [ -f "$d/.runner" ] || continue - known="$known$(runner_name "$d" | tr -c 'A-Za-z0-9._\n-' '_') " -done n=0 while IFS='|' read -r udid state bytes runner; do [ -n "$udid" ] || continue @@ -341,46 +534,6 @@ for devs in devices.values(): ') say " ${n} devices" -# --------------------------------------------------------------------------- -# Each idle runner's own Android emulator. The kit's android-ci names one AVD -# per runner "ci-" (Jerv's screenshots: "shots-"), and -# android-emulator-runner creates it again at the start of every job -# (force-avd-creation defaults to true), so between jobs an AVD is 1-1.5 GB -# that the next job overwrites anyway. Nothing removed them: on ultra -# (2026-10-22) 17 of them held 20 GB. An AVD is left alone while its runner is -# building, while an emulator has it open, or within 30 minutes of its last -# write (a job that is creating it). Other AVDs (night QA's fleet-explore-*, a -# person's) are never touched. -# --------------------------------------------------------------------------- -AVD_HOME="${ANDROID_AVD_HOME:-$HOME/.android/avd}" -AVD_PREFIXES="${FLEET_CLEANUP_AVD_PREFIXES:-ci shots}" -avd_open() { - local dir="$1" pid - pid="$(cat "$dir/hardware-qemu.ini.lock" 2>/dev/null | tr -dc '0-9')" - [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null && return 0 - pgrep -f -- "-avd $2( |\$)" >/dev/null 2>&1 -} -say "==> idle runners' Android emulators" -n=0 -for dir in "$AVD_HOME"/*.avd; do - [ -d "$dir" ] || continue - avd="$(basename "$dir" .avd)" - runner="" - for p in $AVD_PREFIXES; do - case "$avd" in "$p"-?*) runner="${avd#"$p"-}"; break ;; esac - done - [ -n "$runner" ] || continue - building "$runner" && continue - [ -n "$(find "$dir" -maxdepth 1 -mmin -30 2>/dev/null | head -1)" ] && continue - avd_open "$dir" "$avd" && continue - why="" - case "$known" in *" $runner "*) ;; *) why=": no such runner here" ;; esac - say " $(du -sh "$dir" 2>/dev/null | cut -f1) $avd$why" - run rm -rf "$dir" "$AVD_HOME/$avd.ini" - n=$((n + 1)) -done -say " ${n} AVDs" - # --------------------------------------------------------------------------- # A runner's leftover self-update. The runner unpacks a new version into # _work/_update, copies it into bin./externals. and is meant diff --git a/docs/configuration.md b/docs/configuration.md index 3228aa8..be9ddc7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -74,6 +74,11 @@ Used by `cleanup.sh` and `cleanupctl.sh`. See | `FLEET_CLEANUP_SIM_MAX_GB` | `3` | Erase an idle runner's `ci-` simulator past this size. | | `FLEET_CLEANUP_SIM_PRESSURE_GB` | `1` | ...and past this under pressure. | | `FLEET_CLEANUP_FULL_EVERY_S` | `86400` | `--auto` runs a full pass at least this often. | +| `FLEET_CLEANUP_AVD_KEEP_DAYS` | `14` | Keep a runner's quick-boot snapshot AVD (`ci--a--`, the kit's `emulator_snapshot`) until it has gone this long without a boot. | +| `FLEET_CLEANUP_AVD_KEEP_GB` | `30` | The snapshot AVDs in all. Over it, even on a quiet `--auto` tick, idle runners' snapshots (`snapshots/` inside the AVD) go, least recently used first; the AVD stays. Below the admission floor every idle snapshot goes, before any other step. | +| `FLEET_CLEANUP_AVD_PREFIXES` | `ci shots` | AVD name prefixes that mark a runner's own emulator. Nothing else is touched. | +| `FLEET_CLEANUP_EMU_ORPHAN_MIN_S` | `300` | Stop a CI emulator whose parent is launchd and whose runner runs no job once it is this old (on every tick). | +| `FLEET_EMULATOR_CLEANUP` | `1` | The job-completed hook stops an emulator the job left running on its runner's AVD (`adb emu kill`, then SIGTERM, SIGKILL last). `0` turns it off. | | `FLEET_CLEANUP_INTERVAL` | `900` | Seconds between `--auto` ticks. Reinstall after changing: `./cleanupctl.sh install`. | | `FLEET_CLEANUP_LOG` | `~/Library/Logs/runner-cleanup.log` | Where the cleanup LaunchAgent writes stdout/stderr. | diff --git a/docs/reference/scripts.md b/docs/reference/scripts.md index 7c5fd76..6890b26 100644 --- a/docs/reference/scripts.md +++ b/docs/reference/scripts.md @@ -27,7 +27,7 @@ without `--repair` — have no `--apply` because there is nothing to guard. | `healthctl.sh` | Install a LaunchAgent that runs `health.sh --repair` on a timer | n/a — subcommands | | `cleanupctl.sh` | Install a LaunchAgent that runs `cleanup.sh --apply --auto` every 15 minutes | n/a — subcommands | | `runs.sh` | What is building across every repo, and on which machine | read-only | -| `cleanup.sh` | Erase idle CI simulators; prune stale DerivedData, dead simulators, old `_diag`, superseded runner versions | **yes** | +| `cleanup.sh` | Erase idle CI simulators and AVDs (keeping runners' snapshot AVDs), stop orphaned CI emulators; prune stale DerivedData, dead simulators, old `_diag`, superseded runner versions | **yes** | | `preflight.sh` | Check a host against what the workflows assume | read-only, installs nothing | | `scripts/deregister.sh` | Remove one named runner completely | **yes** | | `scripts/drain-runner.sh` | Stop a runner gracefully, or resume it | no — every state it writes is reversible | @@ -258,6 +258,9 @@ Source: [`cleanup.sh`](https://github.com/addisdev/actions-runners/blob/main/cle | `FLEET_CLEANUP_SIM_MAX_GB` | 3 | Erase a runner's `ci-` simulator past this size | | `FLEET_CLEANUP_SIM_PRESSURE_GB` | 1 | ...and past this under pressure | | `FLEET_CLEANUP_FULL_EVERY_S` | 86400 | `--auto` runs a full pass at least this often | +| `FLEET_CLEANUP_AVD_KEEP_DAYS` | 14 | Keep a runner's snapshot AVD until this long without a boot | +| `FLEET_CLEANUP_AVD_KEEP_GB` | 30 | Snapshot AVDs in all; over it, least recently used snapshots go | +| `FLEET_CLEANUP_EMU_ORPHAN_MIN_S` | 300 | Age at which an unowned CI emulator is stopped | Host-wide steps, only while no job is building: DerivedData directories older than 7 days, simulators whose runtime is no longer installed, and stale @@ -272,6 +275,23 @@ deleted. Under pressure, idle runners' Playwright browsers in `_work/_tool/ms-playwright` are removed; the next job that needs them downloads them again. +Runners' Android emulators, first of the deleting steps. A plain +`ci-` (or `shots-`) AVD, which android-emulator-runner recreates +every job, goes once its runner is idle and it is 30 minutes untouched. A +snapshot AVD, `ci--a--` (the launch kit's +`emulator_snapshot`), is kept while its runner exists and has booted it in the +last `FLEET_CLEANUP_AVD_KEEP_DAYS`: deleting it would make that runner's next +job boot three times instead of once. Over `FLEET_CLEANUP_AVD_KEEP_GB`, idle +runners' snapshots (`snapshots/`, not the AVD) go least recently used first; +below the admission floor every idle runner's snapshot goes before anything +else. A building runner's AVD, one an emulator has open, and one written in the +last 30 minutes are never touched. + +Every tick, even a quiet `--auto` one: a CI emulator (`-avd ci-…`/`shots-…`) +whose parent is launchd, older than `FLEET_CLEANUP_EMU_ORPHAN_MIN_S`, whose +runner runs no job (no `Runner.Worker` under its directory) or no longer exists +is stopped as `hooks/emulator-control.sh` does it. + Always: runner `_diag` logs older than 14 days and runner versions older than the linked one (a self-update leaves the previous pair behind). It prints free disk before and after. @@ -773,6 +793,21 @@ lease has a detached `Runner.Worker` guardian, so cancellation and abrupt worker failure follow the same cleanup path. Source: [`hooks/simulator-control.sh`](https://github.com/addisdev/actions-runners/blob/main/hooks/simulator-control.sh). +### `hooks/emulator-control.sh` + +Internal helper used by the job-completed hook (`complete`) and sourced by +`cleanup.sh`. It stops an Android emulator the job left running on its +runner's own AVD (`ci-…`, `shots-…`) the way a quick-boot +snapshot survives: the console's `adb emu kill`, then SIGTERM, then SIGKILL; a +save-capable boot that needed SIGKILL has its snapshot removed, since it may +have been mid-save. Night QA's `fleet-explore-*` and any other AVD are never +touched. `FLEET_EMULATOR_CLEANUP=0` turns the hook path off. +Source: [`hooks/emulator-control.sh`](https://github.com/addisdev/actions-runners/blob/main/hooks/emulator-control.sh). + +```bash +hooks/tests/emulator.sh # stand-in emulators, fake adb; no real emulator is touched +``` + ### `hooks/shm-reap.sh` Internal helper used by both job hooks. Removes System V shared memory segments diff --git a/hooks/emulator-control.sh b/hooks/emulator-control.sh new file mode 100755 index 0000000..c78a41e --- /dev/null +++ b/hooks/emulator-control.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# CI Android emulators, stopped so that a quick-boot snapshot survives them. +# +# emulator-control.sh complete # job-completed.sh: stop THIS runner's emulators +# +# and sourced by cleanup.sh, which reaps emulators no job owns any more. +# +# The kit's android-ci (taylab-launch-kit, `emulator_snapshot`) keeps one AVD per +# runner and system image, "ci--a--", with a +# quick-boot snapshot (~4 GB of guest RAM) that the next job on that runner +# boots from. android-emulator-runner stops its emulator with `adb emu kill`, +# which is the clean path. But a job that is cancelled or times out mid-step +# leaves its emulator running (on ultra, 2026-10-23, two greenfolio emulators +# ran 9 h after their jobs were cancelled), and nothing at job end stops it: +# the runner's own orphan scan is answered "not this job's" by shims/ps. A +# SIGKILLed emulator that was saving its snapshot leaves half a snapshot. +# +# So an emulator is stopped the way the emulator expects: the console's +# `emu kill`, then SIGTERM, and SIGKILL only when neither worked. A boot that +# could save (no -no-snapshot, -no-snapshot-save or -read-only) and had to be +# SIGKILLed may have been mid-save, so its AVD's snapshot is thrown away; the +# kit saves a new one on the runner's next job. +# +# Only AVDs named for a runner ("ci-…", "shots-…", FLEET_CLEANUP_AVD_PREFIXES) +# are ever touched. Night QA's fleet-explore-* and a person's own AVDs are not. +# +# Never fails its caller: the job-completed hook must not turn a green job red. + +EMU_PREFIXES="${FLEET_CLEANUP_AVD_PREFIXES:-ci shots}" +EMU_AVD_HOME="${ANDROID_AVD_HOME:-$HOME/.android/avd}" + +emu_adb() { + printf '%s' "${FLEET_ADB:-${ANDROID_HOME:-${ANDROID_SDK_ROOT:-$HOME/Library/Android/sdk}}/platform-tools/adb}" +} + +# One line per emulator process booting a runner's AVD: +# pid ppid age_seconds avd port could_save(1|0) +# The emulator binary is the command's first word (or the second, after an +# interpreter, which is how the tests' stand-in runs). +emu_list() { + "${FLEET_PS:-/bin/ps}" -axo pid=,ppid=,etime=,command= 2>/dev/null | awk -v prefixes="$EMU_PREFIXES" ' + function secs(t, d, a, p, n, i, s) { + d = 0 + if (index(t, "-")) { split(t, a, "-"); d = a[1]; t = a[2] } + n = split(t, p, ":"); s = 0 + for (i = 1; i <= n; i++) s = s * 60 + p[i] + return d * 86400 + s + } + BEGIN { np = split(prefixes, P, " ") } + { + hit = 0 + for (k = 4; k <= 5 && k <= NF; k++) { + exe = $k; sub(/.*\//, "", exe) + if (exe ~ /^(emulator|qemu-system)/) hit = 1 + } + if (!hit) next + avd = ""; port = "-"; save = 1 + for (i = 5; i <= NF; i++) { + if ($i == "-avd" && i < NF) avd = $(i + 1) + else if ($i == "-port" && i < NF) port = $(i + 1) + else if ($i == "-no-snapshot" || $i == "-no-snapshot-save" || $i == "-read-only") save = 0 + } + if (avd == "") next + ok = 0 + for (j = 1; j <= np; j++) + if (index(avd, P[j] "-") == 1 && length(avd) > length(P[j]) + 1) ok = 1 + if (ok) print $1, $2, secs($3), avd, port, save + }' +} + +# Whose AVD is this? Given the runner names on this host (space-separated, as +# the kit spells them: anything but [A-Za-z0-9_.-] made "_"), prints +# +# "-" is the plain AVD android-emulator-runner recreates every +# job; "--a--" is the kit's snapshot AVD. +# The longest matching runner wins, so ultra-jerv-2's AVD is never ultra-jerv's. +emu_avd_owner() { + local avd="$1" known="$2" p rest="" r best="" kind="" cand + for p in $EMU_PREFIXES; do + case "$avd" in "$p"-?*) rest="${avd#"$p"-}"; break ;; esac + done + [ -n "$rest" ] || return 1 + for r in $known; do + if [ "$rest" = "$r" ]; then + cand=plain + else + case "$rest" in "$r"-a[0-9]*-?*) cand=snapshot ;; *) continue ;; esac + fi + if [ "${#r}" -gt "${#best}" ]; then best="$r"; kind="$cand"; fi + done + if [ -n "$best" ]; then + printf '%s %s here\n' "$best" "$kind" + elif [[ "$rest" =~ ^(.+)-a[0-9]+-.+$ ]]; then + printf '%s snapshot gone\n' "${BASH_REMATCH[1]}" + else + printf '%s plain gone\n' "$rest" + fi +} + +emu_wait_gone() { + local pid="$1" n="$2" i=0 + while kill -0 "$pid" 2>/dev/null; do + [ "$i" -ge "$n" ] && return 1 + sleep 1 + i=$((i + 1)) + done + return 0 +} + +# emu_stop : prints how it went. +emu_stop() { + local pid="$1" port="$2" avd="$3" save="$4" adb apid + kill -0 "$pid" 2>/dev/null || { echo "already gone"; return 0; } + adb="$(emu_adb)" + if [ "$port" != - ] && [ -x "$adb" ]; then + # In the background: an adb whose server is wedged must not hold the hook. + "$adb" -s "emulator-$port" emu kill >/dev/null 2>&1 & + apid=$! + # A save-capable boot writes its snapshot (~4 GB) after `emu kill`. + if emu_wait_gone "$pid" "${FLEET_EMU_KILL_WAIT_S:-120}"; then + kill "$apid" 2>/dev/null; wait "$apid" 2>/dev/null + echo "stopped (emu kill)"; return 0 + fi + kill "$apid" 2>/dev/null; wait "$apid" 2>/dev/null + fi + kill -TERM "$pid" 2>/dev/null + if emu_wait_gone "$pid" "${FLEET_EMU_TERM_WAIT_S:-30}"; then + echo "stopped (SIGTERM)"; return 0 + fi + kill -KILL "$pid" 2>/dev/null + emu_wait_gone "$pid" 5 + if [ "$save" = 1 ] && [ -d "$EMU_AVD_HOME/$avd.avd/snapshots" ]; then + rm -rf "$EMU_AVD_HOME/$avd.avd/snapshots/default_boot" "$EMU_AVD_HOME/$avd.avd/ci-snapshot.key" + echo "SIGKILLed; its snapshot may have been mid-save, so it was removed" + return 0 + fi + echo "SIGKILLed" +} + +# The job-completed path: this runner's emulators, whatever their parent. A +# runner runs one job at a time, so an emulator on its AVD is this job's, and +# one still up now would hold the AVD's lock against the next job. +emu_complete() { + [ "${FLEET_EMULATOR_CLEANUP:-1}" = 1 ] || return 0 + local me pid avd port save owner + me="$(printf '%s' "${RUNNER_NAME:-}" | tr -c 'A-Za-z0-9_.-' '_')" + [ -n "$me" ] || return 0 + emu_list | while read -r pid _ _ avd port save; do + owner="$(emu_avd_owner "$avd" "$me")" || continue + case "$owner" in *" here") ;; *) continue ;; esac + echo "emulator-control: $avd (pid $pid, port $port) left running by this job: $(emu_stop "$pid" "$port" "$avd" "$save")" + done +} + +if [ "${BASH_SOURCE[0]}" = "$0" ]; then + trap 'exit 0' EXIT + _here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + _root="${FLEET_ROOT:-$(cd "$_here/.." && pwd)}" + # shellcheck source=/dev/null + [ -f "$_root/fleet.env" ] && . "$_root/fleet.env" + EMU_PREFIXES="${FLEET_CLEANUP_AVD_PREFIXES:-ci shots}" + case "${1:-}" in + complete) emu_complete ;; + list) emu_list ;; + esac +fi diff --git a/hooks/job-completed.sh b/hooks/job-completed.sh index c1a2881..0090b9b 100755 --- a/hooks/job-completed.sh +++ b/hooks/job-completed.sh @@ -27,6 +27,12 @@ ROOT="${FLEET_ROOT:-$(cd "$HERE/.." && pwd)}" # this completion hook. [ -f "$HERE/simulator-control.sh" ] && FLEET_ROOT="$ROOT" bash "$HERE/simulator-control.sh" complete +# Stop an Android emulator this job left running (a cancelled or timed-out +# emulator step), with the console's `emu kill` before any signal, so the kit's +# per-runner quick-boot snapshot is never cut off mid-save and the AVD's lock is +# free for this runner's next job. FLEET_EMULATOR_CLEANUP=0 turns it off. +[ -f "$HERE/emulator-control.sh" ] && FLEET_ROOT="$ROOT" bash "$HERE/emulator-control.sh" complete + # Same sweep as the started hook. Running it here too means a host that goes # quiet after a cancelled Postgres job is not left near the 32-segment cap until # the next job happens to start. diff --git a/hooks/tests/emulator.sh b/hooks/tests/emulator.sh new file mode 100755 index 0000000..224b571 --- /dev/null +++ b/hooks/tests/emulator.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Exercises emulator-control.sh with stand-in emulators and a fake adb. No real +# emulator or AVD is touched: ps is filtered to this test's own processes. +set -uo pipefail + +HOOKS="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ROOT="${TMPDIR:-/tmp}/emulator-hook-test/root" +PASS=0 +FAIL=0 + +ok() { + if [ "$2" = "$3" ]; then + echo " PASS $1 ($2)" + PASS=$((PASS + 1)) + else + echo " FAIL $1 — expected '$3', got '$2'" + FAIL=$((FAIL + 1)) + fi +} + +cleanup() { + pkill -f -- "$ROOT/" 2>/dev/null + rm -rf "$ROOT" +} +trap cleanup EXIT + +setup() { + cleanup + mkdir -p "$ROOT/bin" "$ROOT/home/.android/avd" + : > "$ROOT/calls" + cat > "$ROOT/bin/fakeps" < "$ROOT/bin/adb" <> "$ROOT/calls" +pkill -TERM -f -- "-port \${2#emulator-} -avd" 2>/dev/null +exit 0 +EOF + cat > "$ROOT/bin/qemu-system-aarch64-headless" <<'EOF' +#!/usr/bin/env bash +trap 'exit 0' TERM +while :; do sleep 1; done +EOF + chmod +x "$ROOT/bin/"* +} + +emu() { ( "$ROOT/bin/qemu-system-aarch64-headless" "$@" >/dev/null 2>&1 & ); } +alive() { pgrep -f -- "-avd $1" >/dev/null 2>&1 && echo running || echo stopped; } +complete() { + HOME="$ROOT/home" FLEET_ROOT="$ROOT" ANDROID_AVD_HOME="" FLEET_PS="$ROOT/bin/fakeps" \ + FLEET_ADB="$ROOT/bin/adb" FLEET_EMU_KILL_WAIT_S=3 FLEET_EMU_TERM_WAIT_S=1 "$@" \ + bash "$HOOKS/emulator-control.sh" complete 2>&1 +} + +echo "== whose AVD is it ==" +# shellcheck source=hooks/emulator-control.sh +. "$HOOKS/emulator-control.sh" +K="ultra-jerv ultra-jerv-2 ultra-greenfolio-android ultra-dozehound-firetv" +ok "a plain AVD" "$(emu_avd_owner ci-ultra-jerv "$K")" "ultra-jerv plain here" +ok "the longest runner wins" "$(emu_avd_owner ci-ultra-jerv-2 "$K")" "ultra-jerv-2 plain here" +ok "a phone snapshot AVD" "$(emu_avd_owner ci-ultra-greenfolio-android-a34-google_apis-pixel_6 "$K")" \ + "ultra-greenfolio-android snapshot here" +ok "a TV snapshot AVD" "$(emu_avd_owner ci-ultra-dozehound-firetv-a34-android-tv-tv_1080p "$K")" \ + "ultra-dozehound-firetv snapshot here" +ok "ultra-jerv-2's snapshot AVD is not ultra-jerv's" \ + "$(emu_avd_owner ci-ultra-jerv-2-a34-google_apis-pixel_6 "$K")" "ultra-jerv-2 snapshot here" +ok "a snapshot AVD of a runner that is gone" \ + "$(emu_avd_owner ci-ultra-jerv-9-a34-google_apis-pixel_6 "$K")" "ultra-jerv-9 snapshot gone" +ok "a screenshot AVD" "$(emu_avd_owner shots-ultra-jerv "$K")" "ultra-jerv plain here" +ok "not a runner's AVD" "$(emu_avd_owner fleet-explore-1 "$K" || echo none)" "none" + +echo "== job end: this runner's emulator is stopped with emu kill, nobody else's ==" +setup +emu -port 5554 -avd ci-RL-a-a34-google_apis-pixel_6 -no-snapshot-save +emu -port 5556 -avd ci-RL-a-2-a34-google_apis-pixel_6 -no-snapshot-save +emu -port 5558 -avd fleet-explore-1 +sleep 1 +OUT="$(complete env RUNNER_NAME=RL-a)" +ok "stopped this runner's emulator" "$(alive ci-RL-a-a34)" "stopped" +ok "through the console" "$(grep -c 'adb -s emulator-5554 emu kill' "$ROOT/calls")" "1" +ok "and said so in the job log" "$(printf '%s\n' "$OUT" | grep -c 'ci-RL-a-a34.*stopped (emu kill)')" "1" +ok "left RL-a-2's emulator" "$(alive ci-RL-a-2-a34)" "running" +ok "left night QA's emulator" "$(alive fleet-explore-1)" "running" + +echo "== job end with nothing running is silent ==" +OUT="$(complete env RUNNER_NAME=RL-b)" +ok "printed nothing" "$OUT" "" + +echo "== FLEET_EMULATOR_CLEANUP=0 turns it off ==" +emu -port 5560 -avd ci-RL-c -no-snapshot +sleep 1 +complete env RUNNER_NAME=RL-c FLEET_EMULATOR_CLEANUP=0 >/dev/null +ok "left the emulator running" "$(alive ci-RL-c)" "running" +complete env RUNNER_NAME=RL-c >/dev/null +ok "and stops it when on" "$(alive ci-RL-c)" "stopped" + +echo "== it never fails the hook ==" +complete env RUNNER_NAME= FLEET_PS=/nonexistent >/dev/null +ok "exit 0 without a runner name or ps" "$?" "0" + +echo +echo "passed $PASS, failed $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/scripts/test-cleanup.sh b/scripts/test-cleanup.sh index f78b344..e3382f1 100755 --- a/scripts/test-cleanup.sh +++ b/scripts/test-cleanup.sh @@ -27,7 +27,7 @@ setup() { mkdir -p "$T/bin" "$T/home/Library/Developer/Xcode/DerivedData" mkdir -p "$T/hooks" cp "$HERE/cleanup.sh" "$T/" - cp "$HERE/hooks/free-disk.sh" "$T/hooks/" + cp "$HERE/hooks/free-disk.sh" "$HERE/hooks/emulator-control.sh" "$T/hooks/" for r in web android ios; do mkdir -p "$T/$r/_work/_tool/ms-playwright/chromium" "$T/$r/_diag" printf '{"agentName":"RL-%s","gitHubUrl":"https://github.com/acme/%s"}' "$r" "$r" > "$T/$r/.runner" @@ -70,12 +70,27 @@ EOF cat > "$T/bin/gh" < "$T/bin/fakeps" < "$T/bin/adb" <> "$T/calls" +port="\${2#emulator-}" +pkill -TERM -f -- "-port \$port -avd" 2>/dev/null +exit 0 EOF chmod +x "$T/bin/"* } clean() { - HOME="$T/home" FLEET_ROOT="$T" PATH="$T/bin:$PATH" ANDROID_AVD_HOME="" bash "$T/cleanup.sh" "$@" 2>&1 + HOME="$T/home" FLEET_ROOT="$T" PATH="$T/bin:$PATH" ANDROID_AVD_HOME="" \ + FLEET_PS="$T/bin/fakeps" FLEET_ADB="$T/bin/adb" bash "$T/cleanup.sh" "$@" 2>&1 } calls() { grep -c "$1" "$T/calls" 2>/dev/null; } gone() { [ -e "$1" ] && echo kept || echo removed; } @@ -319,6 +334,125 @@ clean --apply > "$T/out" ok "under pressure, an idle runner's recent cache goes too" "$(gone "$T/android/_work/android/DerivedData-App")" "removed" ok "...but never a building runner's" "$(gone "$T/web/_work/web/DerivedData-App")" "kept" +echo "== snapshot AVDs: kept for a runner here, gone when stale or orphaned ==" +setup 40 +echo 120 > "$T/free" +echo 1 > "$T/.cleanup-last-full" +echo "RL-web" > "$T/busy" +A="$T/home/.android/avd" +mkdir -p "$A" +# snap [with-snapshot] +snap() { + mkdir -p "$A/$1.avd"; : > "$A/$1.avd/userdata-qemu.img.qcow2"; echo "path=$A/$1.avd" > "$A/$1.ini" + [ "${4:-}" = with-snapshot ] && { mkdir -p "$A/$1.avd/snapshots/default_boot"; : > "$A/$1.avd/snapshots/default_boot/ram.bin"; echo key > "$A/$1.avd/ci-snapshot.key"; } + case "$2" in + old) stamp=202601010000 ;; + days) stamp="$(date -v-"$3"d +%Y%m%d%H%M)" ;; + *) return 0 ;; + esac + find "$A/$1.avd" -exec touch -t "$stamp" {} + +} +snap ci-RL-ios-a34-google_apis-pixel_6 days 2 with-snapshot +snap ci-RL-ios-a34-android-tv-tv_1080p days 3 with-snapshot +snap ci-RL-android-a34-google_apis-pixel_6 days 20 with-snapshot +snap ci-RL-web-a34-google_apis-pixel_6 old with-snapshot +snap ci-RL-gone-a34-android-tv-tv_1080p days 2 with-snapshot +snap ci-RL-ios-2-a34-google_apis-pixel_6 days 2 with-snapshot +snap ci-RL-android x x +touch -t 202601010000 "$A/ci-RL-android.avd/userdata-qemu.img.qcow2" "$A/ci-RL-android.avd" +clean --apply > "$T/out" +ok "kept an idle runner's snapshot AVD used this week" "$(gone "$A/ci-RL-ios-a34-google_apis-pixel_6.avd/snapshots/default_boot")" "kept" +ok "kept the same runner's TV snapshot AVD" "$(gone "$A/ci-RL-ios-a34-android-tv-tv_1080p.avd/snapshots")" "kept" +ok "removed a snapshot AVD with no boot in 14 days" "$(gone "$A/ci-RL-android-a34-google_apis-pixel_6.avd")" "removed" +ok "and its .ini" "$(gone "$A/ci-RL-android-a34-google_apis-pixel_6.ini")" "removed" +ok "kept a building runner's snapshot AVD, however old" "$(gone "$A/ci-RL-web-a34-google_apis-pixel_6.avd")" "kept" +ok "removed the snapshot AVD of a runner that no longer exists" "$(gone "$A/ci-RL-gone-a34-android-tv-tv_1080p.avd")" "removed" +ok "RL-ios-2's AVD is not RL-ios's (no such runner: removed)" "$(gone "$A/ci-RL-ios-2-a34-google_apis-pixel_6.avd")" "removed" +ok "still removed an idle runner's plain AVD" "$(gone "$A/ci-RL-android.avd")" "removed" + +echo "== below the admission floor, idle runners' snapshots go first ==" +setup 41 +echo 30 > "$T/free" +date +%s > "$T/.cleanup-last-full" +echo "RL-web" > "$T/busy" +A="$T/home/.android/avd" +mkdir -p "$A" +snap ci-RL-ios-a34-google_apis-pixel_6 days 2 with-snapshot +snap ci-RL-web-a34-google_apis-pixel_6 days 2 with-snapshot +snap ci-RL-android-a34-google_apis-pixel_6 new x with-snapshot +clean --apply --auto > "$T/out" +ok "said it is below the floor" "$(grep -c 'BELOW THE FLOOR' "$T/out")" "1" +ok "dropped an idle runner's snapshot" "$(gone "$A/ci-RL-ios-a34-google_apis-pixel_6.avd/snapshots")" "removed" +ok "and its snapshot key" "$(gone "$A/ci-RL-ios-a34-google_apis-pixel_6.avd/ci-snapshot.key")" "removed" +ok "but kept the AVD itself" "$(gone "$A/ci-RL-ios-a34-google_apis-pixel_6.avd/userdata-qemu.img.qcow2")" "kept" +ok "kept a building runner's snapshot" "$(gone "$A/ci-RL-web-a34-google_apis-pixel_6.avd/snapshots")" "kept" +ok "kept a snapshot written in the last 30 minutes" "$(gone "$A/ci-RL-android-a34-google_apis-pixel_6.avd/snapshots")" "kept" +ok "the snapshots went before any other step" \ + "$(awk '/Android emulators/ {a = NR} /DerivedData older/ {d = NR} END {print (a && d && a < d) ? "first" : "not first"}' "$T/out")" "first" + +echo "== over FLEET_CLEANUP_AVD_KEEP_GB, the least recently used snapshots go, even on a quiet --auto tick ==" +setup 42 +echo 120 > "$T/free" +date +%s > "$T/.cleanup-last-full" +A="$T/home/.android/avd" +mkdir -p "$A" +snap ci-RL-ios-a34-google_apis-pixel_6 x x with-snapshot +snap ci-RL-android-a34-google_apis-pixel_6 x x with-snapshot +dd if=/dev/zero of="$A/ci-RL-ios-a34-google_apis-pixel_6.avd/snapshots/default_boot/ram.bin" bs=1048576 count=700 2>/dev/null +dd if=/dev/zero of="$A/ci-RL-android-a34-google_apis-pixel_6.avd/snapshots/default_boot/ram.bin" bs=1048576 count=700 2>/dev/null +find "$A/ci-RL-ios-a34-google_apis-pixel_6.avd" -exec touch -t "$(date -v-3d +%Y%m%d%H%M)" {} + +find "$A/ci-RL-android-a34-google_apis-pixel_6.avd" -exec touch -t "$(date -v-1d +%Y%m%d%H%M)" {} + +OUT="$(FLEET_CLEANUP_AVD_KEEP_GB=2 clean --apply --auto)" +ok "under the limit, a quiet tick stays quiet" "$OUT" "" +FLEET_CLEANUP_AVD_KEEP_GB=1 clean --apply --auto > "$T/out" +ok "over the limit, a quiet tick runs" "$(grep -c 'snapshot AVDs over 1 GB' "$T/out")" "1" +ok "dropped the least recently used snapshot" "$(gone "$A/ci-RL-ios-a34-google_apis-pixel_6.avd/snapshots")" "removed" +ok "kept its AVD" "$(gone "$A/ci-RL-ios-a34-google_apis-pixel_6.avd")" "kept" +ok "and stopped once under the limit" "$(gone "$A/ci-RL-android-a34-google_apis-pixel_6.avd/snapshots/default_boot/ram.bin")" "kept" +rm -rf "$A" + +echo "== orphaned CI emulators are stopped the snapshot-safe way ==" +setup 43 +echo 120 > "$T/free" +date +%s > "$T/.cleanup-last-full" +A="$T/home/.android/avd" +mkdir -p "$A" "$T/emu" "$T/web/bin" +snap ci-RL-ios-a34-google_apis-pixel_6 days 1 with-snapshot +cat > "$T/emu/qemu-system-aarch64-headless" <<'EOF' +#!/usr/bin/env bash +case " $* " in *" -stubborn "*) trap '' TERM ;; *) trap 'exit 0' TERM ;; esac +while :; do sleep 1; done +EOF +chmod +x "$T/emu/qemu-system-aarch64-headless" +# Detached, so launchd is the parent, as android-emulator-runner leaves it. +emu() { ( "$T/emu/qemu-system-aarch64-headless" "$@" >/dev/null 2>&1 & ); } +emu -port 5554 -avd ci-RL-android-a34-google_apis-pixel_6 -no-snapshot-save +emu -port 5556 -avd ci-RL-web-a34-google_apis-pixel_6 -no-snapshot-save +emu -port 5558 -avd ci-RL-ios-a34-google_apis-pixel_6 -stubborn +emu -port 5560 -avd fleet-explore-1 +emu -port 5562 -avd ci-RL-gone -no-snapshot +# RL-web is running a job: a Runner.Worker under its directory. +ln -s /bin/sleep "$T/web/bin/Runner.Worker" +( "$T/web/bin/Runner.Worker" 300 >/dev/null 2>&1 & ) +sleep 1 +alive() { pgrep -f -- "-avd $1" >/dev/null 2>&1 && echo running || echo stopped; } +FLEET_CLEANUP_EMU_ORPHAN_MIN_S=0 clean --auto > "$T/out" +ok "a dry run stops nothing" "$(alive ci-RL-android-a34)" "running" +ok "a dry run says what it would stop" "$(grep -c '\[dry-run\] stop ci-RL-android-a34' "$T/out")" "1" +FLEET_CLEANUP_EMU_ORPHAN_MIN_S=3600 clean --apply --auto > "$T/out" +ok "a young emulator is not an orphan yet" "$(alive ci-RL-android-a34)" "running" +FLEET_CLEANUP_EMU_ORPHAN_MIN_S=0 FLEET_EMU_KILL_WAIT_S=2 FLEET_EMU_TERM_WAIT_S=1 clean --apply --auto > "$T/out" +ok "stopped an idle runner's emulator" "$(alive ci-RL-android-a34)" "stopped" +ok "with the console's emu kill" "$(grep -c 'adb -s emulator-5554 emu kill' "$T/calls")" "1" +ok "and said so" "$(grep -c 'ci-RL-android-a34.*stopped (emu kill)' "$T/out")" "1" +ok "left the emulator of a runner running a job" "$(alive ci-RL-web-a34)" "running" +ok "never touched night QA's emulator" "$(alive fleet-explore-1)" "running" +ok "stopped the emulator of a runner that no longer exists" "$(alive 'ci-RL-gone ')" "stopped" +ok "SIGKILLed one that ignored emu kill and SIGTERM" "$(alive ci-RL-ios-a34)" "stopped" +ok "and, since it could have been saving, dropped its snapshot" "$(gone "$A/ci-RL-ios-a34-google_apis-pixel_6.avd/snapshots/default_boot")" "removed" +ok "but kept the AVD" "$(gone "$A/ci-RL-ios-a34-google_apis-pixel_6.avd")" "kept" +pkill -f -- "$T/" 2>/dev/null + echo echo "passed $PASS, failed $FAIL" [ "$FAIL" -eq 0 ]