From d3b696927dc7bb5c14cd903224f6557fa4b3401c Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 15:58:34 +0200 Subject: [PATCH 01/24] Vagrant: make installation of pinned packages optional The pin lists record the package versions of a VM whose output was verified, so a provision reproduces that machine. They only apply to the box they were captured from: pointing the Vagrantfile at a new Ubuntu series makes them a list of versions that series never carried, and the provisioner aborts before a new list can be captured. Gate the step on VM_APT_PIN, which defaults to 1. `VM_APT_PIN=0 vagrant up` skips it, which is what a base-box bootstrap needs. Co-Authored-By: Claude Opus 5 (1M context) --- Vagrantfile | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 5ad06e6a0..513f04a7a 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -23,9 +23,15 @@ $frontend = <<-SHELL libjpeg-dev \ make - # Install/check packages from list for reproducibility - DEBIAN_FRONTEND=noninteractive apt-get install \ - --allow-downgrades -y $(cat /home/vagrant/vm_apt.txt) + # Install/check packages from list for reproducibility. + # Set VM_APT_PIN=0 to skip this step. That is needed when bootstrapping a + # new Ubuntu base box, whose archive does not carry the pinned versions. + if [ "${VM_APT_PIN:-1}" = "1" ]; then + DEBIAN_FRONTEND=noninteractive apt-get install \ + --allow-downgrades -y $(cat /home/vagrant/vm_apt.txt) + else + echo "VM_APT_PIN=0 -- skipping installation of pinned packages" + fi # Force packages to be set as automatically installed apt-mark auto $(cat /vagrant/vm_apt_list.txt | grep "\\[installed,automatic\\]" | awk -F/ -v ORS=" " 'NR>1 {print $1}') @@ -127,9 +133,15 @@ $epub = <<-SHELL wget \ libc6-dev - # Install/check packages from list for reproducibility - DEBIAN_FRONTEND=noninteractive apt-get install \ - --allow-downgrades -y $(cat /home/vagrant/vm_apt.txt) + # Install/check packages from list for reproducibility. + # Set VM_APT_PIN=0 to skip this step. That is needed when bootstrapping a + # new Ubuntu base box, whose archive does not carry the pinned versions. + if [ "${VM_APT_PIN:-1}" = "1" ]; then + DEBIAN_FRONTEND=noninteractive apt-get install \ + --allow-downgrades -y $(cat /home/vagrant/vm_apt.txt) + else + echo "VM_APT_PIN=0 -- skipping installation of pinned packages" + fi # Force packages to be set as automatically installed apt-mark auto $(cat /vagrant/vm_apt_list.txt | grep "\\[installed,automatic\\]" | awk -F/ -v ORS=" " 'NR>1 {print $1}') @@ -218,6 +230,10 @@ $epub = <<-SHELL SHELL +# Installation of the pinned package versions is enabled by default. +# Set VM_APT_PIN=0 to disable it for a base-box bootstrap. +vm_apt_pin = ENV.fetch("VM_APT_PIN", "1") + Vagrant.configure("2") do |config| config.vm.provider "virtualbox" do |vb| @@ -236,7 +252,8 @@ Vagrant.configure("2") do |config| web.vm.provision "file", source: "./frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/data/toolchain.ini", destination: "/home/vagrant/toolchain.ini" web.vm.provision "file", source: "./frontend/vm_apt_web.txt", destination: "/home/vagrant/vm_apt.txt" - web.vm.provision :shell, inline: $frontend + web.vm.provision :shell, inline: $frontend, + env: { "VM_APT_PIN" => vm_apt_pin } end config.vm.define "epub" do |epub| @@ -248,7 +265,8 @@ Vagrant.configure("2") do |config| epub.vm.provision "file", source: "./frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/data/toolchain.ini", destination: "/home/vagrant/toolchain.ini" epub.vm.provision "file", source: "./frontend/vm_apt_epub.txt", destination: "/home/vagrant/vm_apt.txt" - epub.vm.provision :shell, inline: $epub + epub.vm.provision :shell, inline: $epub, + env: { "VM_APT_PIN" => vm_apt_pin } end end From 00181eb199cc2154665cdcd48317371b6c3ae890 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 15:59:46 +0200 Subject: [PATCH 02/24] Vagrant: move VM package lists into frontend/vm/ Group the two pin lists with the script that regenerates them, rather than leaving them loose in frontend/. Update the provision source paths for both VMs. Co-Authored-By: Claude Opus 5 (1M context) --- Vagrantfile | 4 ++-- frontend/{ => vm}/vm_apt_epub.txt | 0 frontend/{ => vm}/vm_apt_web.txt | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename frontend/{ => vm}/vm_apt_epub.txt (100%) rename frontend/{ => vm}/vm_apt_web.txt (100%) diff --git a/Vagrantfile b/Vagrantfile index 513f04a7a..9b1fb8b87 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -251,7 +251,7 @@ Vagrant.configure("2") do |config| web.vm.synced_folder './content', '/vagrant/content' web.vm.provision "file", source: "./frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/data/toolchain.ini", destination: "/home/vagrant/toolchain.ini" - web.vm.provision "file", source: "./frontend/vm_apt_web.txt", destination: "/home/vagrant/vm_apt.txt" + web.vm.provision "file", source: "./frontend/vm/vm_apt_web.txt", destination: "/home/vagrant/vm_apt.txt" web.vm.provision :shell, inline: $frontend, env: { "VM_APT_PIN" => vm_apt_pin } end @@ -264,7 +264,7 @@ Vagrant.configure("2") do |config| epub.vm.synced_folder './content', '/vagrant/content' epub.vm.provision "file", source: "./frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/data/toolchain.ini", destination: "/home/vagrant/toolchain.ini" - epub.vm.provision "file", source: "./frontend/vm_apt_epub.txt", destination: "/home/vagrant/vm_apt.txt" + epub.vm.provision "file", source: "./frontend/vm/vm_apt_epub.txt", destination: "/home/vagrant/vm_apt.txt" epub.vm.provision :shell, inline: $epub, env: { "VM_APT_PIN" => vm_apt_pin } end diff --git a/frontend/vm_apt_epub.txt b/frontend/vm/vm_apt_epub.txt similarity index 100% rename from frontend/vm_apt_epub.txt rename to frontend/vm/vm_apt_epub.txt diff --git a/frontend/vm_apt_web.txt b/frontend/vm/vm_apt_web.txt similarity index 100% rename from frontend/vm_apt_web.txt rename to frontend/vm/vm_apt_web.txt From b015c39268f97e55f3b3962f249e66a84be7fcee Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 17:49:01 +0200 Subject: [PATCH 03/24] Vagrant: cache GNAT toolchain downloads on the host A full reprovision re-downloaded 3.3 GB of GNAT-FSF tarballs, and the three gnat ones twice, since each VM fetched independently. Fetch them into a host-side folder mounted at /vagrant_cache/gnat instead, so destroying a VM no longer throws them away. Redirect it with LEARN_VM_CACHE_GNAT; the default is the gitignored .toolchains/gnat. vm_toolchain_fetch.sh verifies each tarball against the .sha256 sidecar upstream publishes, on every use rather than only after downloading, and downloads to a per-VM .part file so the two VMs cannot collide. It runs on the host too: `--all` warms the cache before `vagrant up`. This also drops `rm *.tar.gz`, a bare glob in the provisioner's working directory rather than the file just downloaded. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 1 + Vagrantfile | 82 ++++++++++++++---- frontend/vm/vm_toolchain_fetch.sh | 135 ++++++++++++++++++++++++++++++ 3 files changed, 200 insertions(+), 18 deletions(-) create mode 100644 frontend/vm/vm_toolchain_fetch.sh diff --git a/.gitignore b/.gitignore index d49e32744..50ead7d5e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ env .idea .vagrant* +.toolchains/ .vscode .yarn *.bkp diff --git a/Vagrantfile b/Vagrantfile index 9b1fb8b87..9c249e97a 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -50,6 +50,27 @@ $frontend = <<-SHELL echo default_version_gnat: $default_version_gnat echo toolchain_versions_gnat: $toolchain_versions_gnat + # Toolchain download cache: fetch each tarball into the host-side folder + # mounted at /vagrant_cache/gnat, verified against its upstream .sha256. + # The script also runs on the host -- `vm_toolchain_fetch.sh --all` warms + # the cache before `vagrant up`. + export LEARN_VM_CACHE_GNAT=/vagrant_cache/gnat + toolchain_fetch=/vagrant/frontend/vm/vm_toolchain_fetch.sh + + install_toolchain () { + local tool=$1 + local ver=$2 + local tarball + local tmp + + tarball=$(${toolchain_fetch} "${tool}" "${ver}") + # Extract on the VM's own disk, never onto the shared cache folder. + tmp=$(mktemp -d) + tar xzf "${tarball}" -C "${tmp}" + mv "${tmp}"/${tool}-* ${path_ada_toolchain_root}/${tool}/${ver} + rm -rf "${tmp}" + } + # Install FSF GNAT # (Required tool: gnatchop) mkdir -p ${path_ada_toolchain_root} @@ -60,10 +81,7 @@ $frontend = <<-SHELL mkdir ${path_ada_toolchain_root}/gnat for tool_version in ${gnat_version[@]}; do echo Installing GNAT $tool_version - wget -O gnat.tar.gz https://github.com/alire-project/GNAT-FSF-builds/releases/download/gnat-${tool_version}/gnat-x86_64-linux-${tool_version}.tar.gz && \ - tar xzf gnat.tar.gz && \ - mv gnat-* ${path_ada_toolchain_root}/gnat/${tool_version} && \ - rm *.tar.gz + install_toolchain gnat ${tool_version} done ln -sf ${path_ada_toolchain_root}/gnat/${default_version_gnat} ${path_ada_toolchain_default}/gnat @@ -168,6 +186,27 @@ $epub = <<-SHELL echo toolchain_versions_gnatprove $toolchain_versions_gnatprove echo toolchain_versions_gprbuild $toolchain_versions_gprbuild + # Toolchain download cache: fetch each tarball into the host-side folder + # mounted at /vagrant_cache/gnat, verified against its upstream .sha256. + # The script also runs on the host -- `vm_toolchain_fetch.sh --all` warms + # the cache before `vagrant up`. + export LEARN_VM_CACHE_GNAT=/vagrant_cache/gnat + toolchain_fetch=/vagrant/frontend/vm/vm_toolchain_fetch.sh + + install_toolchain () { + local tool=$1 + local ver=$2 + local tarball + local tmp + + tarball=$(${toolchain_fetch} "${tool}" "${ver}") + # Extract on the VM's own disk, never onto the shared cache folder. + tmp=$(mktemp -d) + tar xzf "${tarball}" -C "${tmp}" + mv "${tmp}"/${tool}-* ${path_ada_toolchain_root}/${tool}/${ver} + rm -rf "${tmp}" + } + # Install FSF GNAT mkdir -p ${path_ada_toolchain_root} mkdir -p ${path_ada_toolchain_default} @@ -177,30 +216,21 @@ $epub = <<-SHELL mkdir ${path_ada_toolchain_root}/gnat for tool_version in ${gnat_version[@]}; do echo Installing GNAT $tool_version - wget -O gnat.tar.gz https://github.com/alire-project/GNAT-FSF-builds/releases/download/gnat-${tool_version}/gnat-x86_64-linux-${tool_version}.tar.gz && \ - tar xzf gnat.tar.gz && \ - mv gnat-* ${path_ada_toolchain_root}/gnat/${tool_version} && \ - rm *.tar.gz + install_toolchain gnat ${tool_version} done gnat_prove_version=(${toolchain_versions_gnatprove}) mkdir ${path_ada_toolchain_root}/gnatprove for tool_version in ${gnat_prove_version[@]}; do echo Installing GNATprove $tool_version - wget -O gnatprove.tar.gz https://github.com/alire-project/GNAT-FSF-builds/releases/download/gnatprove-${tool_version}/gnatprove-x86_64-linux-${tool_version}.tar.gz && \ - tar xzf gnatprove.tar.gz && \ - mv gnatprove-* ${path_ada_toolchain_root}/gnatprove/${tool_version} && \ - rm *.tar.gz + install_toolchain gnatprove ${tool_version} done gprbuild_version=(${toolchain_versions_gprbuild}) mkdir ${path_ada_toolchain_root}/gprbuild for tool_version in ${gprbuild_version[@]}; do echo Installing GPRbuild $tool_version - wget -O gprbuild.tar.gz https://github.com/alire-project/GNAT-FSF-builds/releases/download/gprbuild-${tool_version}/gprbuild-x86_64-linux-${tool_version}.tar.gz && \ - tar xzf gprbuild.tar.gz && \ - mv gprbuild-* ${path_ada_toolchain_root}/gprbuild/${tool_version} && \ - rm *.tar.gz + install_toolchain gprbuild ${tool_version} done rm -f ${path_ada_toolchain_default}/* @@ -230,10 +260,22 @@ $epub = <<-SHELL SHELL +require 'fileutils' + # Installation of the pinned package versions is enabled by default. # Set VM_APT_PIN=0 to disable it for a base-box bootstrap. vm_apt_pin = ENV.fetch("VM_APT_PIN", "1") +# Host-side download cache for the GNAT-FSF toolchain tarballs, so that +# destroying a VM does not throw them away. Redirect it with +# LEARN_VM_CACHE_GNAT -- it holds several GB and may belong on another disk. +vm_cache_gnat = ENV.fetch("LEARN_VM_CACHE_GNAT", + File.expand_path(".toolchains/gnat", __dir__)) + +# Vagrant refuses to start if a synced folder's source does not exist, so the +# cache directory has to be created before it is declared below. +FileUtils.mkdir_p(vm_cache_gnat) + Vagrant.configure("2") do |config| config.vm.provider "virtualbox" do |vb| @@ -249,11 +291,13 @@ Vagrant.configure("2") do |config| web.vm.synced_folder './frontend', '/vagrant/frontend' web.vm.synced_folder './content', '/vagrant/content' + web.vm.synced_folder vm_cache_gnat, '/vagrant_cache/gnat' web.vm.provision "file", source: "./frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/data/toolchain.ini", destination: "/home/vagrant/toolchain.ini" web.vm.provision "file", source: "./frontend/vm/vm_apt_web.txt", destination: "/home/vagrant/vm_apt.txt" web.vm.provision :shell, inline: $frontend, - env: { "VM_APT_PIN" => vm_apt_pin } + env: { "VM_APT_PIN" => vm_apt_pin, + "LEARN_VM_NAME" => "web" } end config.vm.define "epub" do |epub| @@ -262,11 +306,13 @@ Vagrant.configure("2") do |config| epub.vm.synced_folder './frontend', '/vagrant/frontend' epub.vm.synced_folder './content', '/vagrant/content' + epub.vm.synced_folder vm_cache_gnat, '/vagrant_cache/gnat' epub.vm.provision "file", source: "./frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/data/toolchain.ini", destination: "/home/vagrant/toolchain.ini" epub.vm.provision "file", source: "./frontend/vm/vm_apt_epub.txt", destination: "/home/vagrant/vm_apt.txt" epub.vm.provision :shell, inline: $epub, - env: { "VM_APT_PIN" => vm_apt_pin } + env: { "VM_APT_PIN" => vm_apt_pin, + "LEARN_VM_NAME" => "epub" } end end diff --git a/frontend/vm/vm_toolchain_fetch.sh b/frontend/vm/vm_toolchain_fetch.sh new file mode 100644 index 000000000..47f09b09e --- /dev/null +++ b/frontend/vm/vm_toolchain_fetch.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# +# Fetch a GNAT-FSF-builds toolchain tarball into the download cache and print +# its path. +# +# The cache exists so that destroying a VM does not throw the toolchains away: +# a full reprovision otherwise re-downloads several GB. Upstream publishes a +# .sha256 sidecar for every asset, and it is verified on every use -- not only +# after downloading -- so a file corrupted later is re-fetched rather than +# failing extraction with a confusing tar error. +# +# Runs on the host as well as inside a VM. On the host it can be used to warm +# the cache before `vagrant up`, so that provisioning downloads nothing. +# +# Usage: +# vm_toolchain_fetch.sh # print the verified cached path +# vm_toolchain_fetch.sh --all # every version in toolchain.ini +# +# Environment: +# LEARN_VM_CACHE_GNAT cache directory +# (default: /.toolchains/gnat; the Vagrantfile +# passes /vagrant_cache/gnat inside the VMs) +# LEARN_VM_NAME suffix for temporary files, so that two VMs sharing +# one cache cannot collide (default: the hostname) + +set -eu +set -o pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo=$(cd "${here}/../.." && pwd) + +toolchain_ini="${repo}/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/data/toolchain.ini" + +cache="${LEARN_VM_CACHE_GNAT:-${repo}/.toolchains/gnat}" +tag="${LEARN_VM_NAME:-$(hostname)}" + +base_url=https://github.com/alire-project/GNAT-FSF-builds/releases/download + +usage () { + sed -n '3,25p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit "${1:-1}" +} + +# Fetch one tarball, verified against its upstream .sha256 sidecar. +# Prints the cached path on stdout; progress goes to stderr so the path can be +# captured with $(...). +fetch () { + local tool=$1 + local ver=$2 + local name="${tool}-x86_64-linux-${ver}.tar.gz" + local url="${base_url}/${tool}-${ver}/${name}" + local tarball="${cache}/${name}" + local sum="${cache}/${name}.sha256" + # Two VMs share one cache and fetch the same gnat tarballs, so their + # temporaries must not collide. + local part="${tarball}.${tag}.part" + local sum_part="${sum}.${tag}.part" + local rc=0 + + mkdir -p "${cache}" + + if [ ! -s "${sum}" ]; then + echo "Fetching checksum for ${name}" >&2 + wget -q -O "${sum_part}" "${url}.sha256" || rc=$? + if [ "${rc}" -ne 0 ] || [ ! -s "${sum_part}" ]; then + rm -f "${sum_part}" + echo "error: could not fetch ${url}.sha256 (exit ${rc})" >&2 + return 1 + fi + mv "${sum_part}" "${sum}" + fi + + if [ -f "${tarball}" ] && verify "${sum}" "${tarball}"; then + echo "Using cached ${name}" >&2 + else + [ -f "${tarball}" ] && echo "Cached ${name} failed verification; re-fetching" >&2 + rm -f "${tarball}" + echo "Downloading ${name}" >&2 + # The redirection creates ${part} before wget runs, so a failure must be + # caught rather than left to `set -e` -- otherwise an empty file survives. + wget -O "${part}" "${url}" || rc=$? + if [ "${rc}" -ne 0 ]; then + rm -f "${part}" + echo "error: could not download ${url} (exit ${rc})" >&2 + return 1 + fi + if ! verify "${sum}" "${part}"; then + rm -f "${part}" + echo "error: ${name} failed checksum verification after download" >&2 + return 1 + fi + mv "${part}" "${tarball}" + fi + + echo "${tarball}" +} + +verify () { + local sum=$1 + local file=$2 + # The sidecar holds a bare hash with no filename, so pair it up here. + echo "$(cat "${sum}") ${file}" | sha256sum -c - > /dev/null 2>&1 +} + +# Read a whitespace-separated list of versions out of toolchain.ini. +versions_of () { + local tool=$1 + if [ ! -f "${toolchain_ini}" ]; then + echo "error: ${toolchain_ini} not found" >&2 + return 1 + fi + sed -n '/^\[toolchains\]/,/^\[/p' "${toolchain_ini}" \ + | sed -n "s/^${tool}[[:space:]]*=[[:space:]]*//p" +} + +fetch_all () { + local tool ver + for tool in gnat gnatprove gprbuild; do + for ver in $(versions_of "${tool}"); do + fetch "${tool}" "${ver}" > /dev/null + done + done + echo "Cache ready: ${cache}" >&2 + du -sh "${cache}" 2>/dev/null >&2 || true +} + +case "${1:-}" in + --all) fetch_all ;; + -h|--help|"") usage 0 ;; + -*) echo "error: unknown option '$1'" >&2; usage 1 ;; + *) + [ $# -eq 2 ] || { echo "error: expected " >&2; usage 1; } + fetch "$1" "$2" + ;; +esac From d09d6784142b2f5ace572d96295c5819183e2d2d Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 17:50:00 +0200 Subject: [PATCH 04/24] Vagrant: cache the pnpm store on the host pnpm install re-downloaded a 530 MB store on every reprovision, in both VMs. Point pnpm's store at a host-side folder mounted at /vagrant_cache/node instead. Redirect it with LEARN_VM_CACHE_NODE; the default is the gitignored .toolchains/node. Putting the store on a shared folder costs nothing: node_modules already lives on one, so pnpm has been copying rather than hardlinking all along (hardlinks are refused on vboxsf, and a file under node_modules/.pnpm reports a link count of 1). Co-Authored-By: Claude Opus 5 (1M context) --- Vagrantfile | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 9c249e97a..54458759b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -103,7 +103,11 @@ $frontend = <<-SHELL echo 'export COREPACK_ENABLE_DOWNLOAD_PROMPT=0' >> /home/vagrant/.bashrc yes | corepack enable - sudo -u vagrant bash -c "export COREPACK_ENABLE_DOWNLOAD_PROMPT=0; cd /vagrant/frontend && pnpm install --frozen-lockfile" + # Keep pnpm's content-addressed store on the host cache, so a destroyed VM + # does not take it with it. Nothing is lost by the store living on a shared + # folder: node_modules is on one too, so pnpm already copies rather than + # hardlinks. + sudo -u vagrant bash -c "export COREPACK_ENABLE_DOWNLOAD_PROMPT=0; cd /vagrant/frontend && pnpm config set store-dir /vagrant_cache/node && pnpm install --frozen-lockfile" SHELL @@ -256,7 +260,11 @@ $epub = <<-SHELL echo 'export COREPACK_ENABLE_DOWNLOAD_PROMPT=0' >> /home/vagrant/.bashrc yes | corepack enable - sudo -u vagrant bash -c "export COREPACK_ENABLE_DOWNLOAD_PROMPT=0; cd /vagrant/frontend && pnpm install --frozen-lockfile" + # Keep pnpm's content-addressed store on the host cache, so a destroyed VM + # does not take it with it. Nothing is lost by the store living on a shared + # folder: node_modules is on one too, so pnpm already copies rather than + # hardlinks. + sudo -u vagrant bash -c "export COREPACK_ENABLE_DOWNLOAD_PROMPT=0; cd /vagrant/frontend && pnpm config set store-dir /vagrant_cache/node && pnpm install --frozen-lockfile" SHELL @@ -272,9 +280,15 @@ vm_apt_pin = ENV.fetch("VM_APT_PIN", "1") vm_cache_gnat = ENV.fetch("LEARN_VM_CACHE_GNAT", File.expand_path(".toolchains/gnat", __dir__)) +# Host-side pnpm store, so that `pnpm install --frozen-lockfile` does not +# re-download half a gigabyte on every reprovision. Redirect it with +# LEARN_VM_CACHE_NODE. +vm_cache_node = ENV.fetch("LEARN_VM_CACHE_NODE", + File.expand_path(".toolchains/node", __dir__)) + # Vagrant refuses to start if a synced folder's source does not exist, so the -# cache directory has to be created before it is declared below. -FileUtils.mkdir_p(vm_cache_gnat) +# cache directories have to be created before they are declared below. +[vm_cache_gnat, vm_cache_node].each { |d| FileUtils.mkdir_p(d) } Vagrant.configure("2") do |config| @@ -292,6 +306,7 @@ Vagrant.configure("2") do |config| web.vm.synced_folder './frontend', '/vagrant/frontend' web.vm.synced_folder './content', '/vagrant/content' web.vm.synced_folder vm_cache_gnat, '/vagrant_cache/gnat' + web.vm.synced_folder vm_cache_node, '/vagrant_cache/node' web.vm.provision "file", source: "./frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/data/toolchain.ini", destination: "/home/vagrant/toolchain.ini" web.vm.provision "file", source: "./frontend/vm/vm_apt_web.txt", destination: "/home/vagrant/vm_apt.txt" @@ -307,6 +322,7 @@ Vagrant.configure("2") do |config| epub.vm.synced_folder './frontend', '/vagrant/frontend' epub.vm.synced_folder './content', '/vagrant/content' epub.vm.synced_folder vm_cache_gnat, '/vagrant_cache/gnat' + epub.vm.synced_folder vm_cache_node, '/vagrant_cache/node' epub.vm.provision "file", source: "./frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/data/toolchain.ini", destination: "/home/vagrant/toolchain.ini" epub.vm.provision "file", source: "./frontend/vm/vm_apt_epub.txt", destination: "/home/vagrant/vm_apt.txt" From ad6df731c911569c4e68f4c42938a8bdbcacff1e Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 17:52:07 +0200 Subject: [PATCH 05/24] Vagrant: cache apt archives on the host Keep downloaded .deb files in a host-side folder shared by both VMs rather than discarding them with the VM. Redirect it with LEARN_VM_CACHE_APT; the default is the gitignored .toolchains/apt. Dir::Cache::Archives is redirected instead of bind-mounting over /var/cache/apt/archives, so apt's lock and partial/ handling stays explicit. APT::Sandbox::User is set to root because the _apt user cannot read a vboxsf share owned by vagrant. Co-Authored-By: Claude Opus 5 (1M context) --- Vagrantfile | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 54458759b..a0236bf89 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,6 +1,17 @@ $frontend = <<-SHELL #!/bin/bash -eux + # Keep downloaded .deb files in a host-side cache, so a reprovision does not + # re-fetch them. Redirected rather than bind-mounted over + # /var/cache/apt/archives, so apt's lock and partial/ handling stays + # explicit. This must precede anything that runs apt, including the + # NodeSource setup script below. + mkdir -p /vagrant_cache/apt/partial + echo 'Dir::Cache::Archives "/vagrant_cache/apt";' > /etc/apt/apt.conf.d/99-learn-cache + # apt drops privileges to the _apt user to download, which cannot read a + # vboxsf share owned by vagrant; without this it warns on every invocation. + echo 'APT::Sandbox::User "root";' >> /etc/apt/apt.conf.d/99-learn-cache + # Enable the NodeSource repository curl -sL https://deb.nodesource.com/setup_24.x | bash - @@ -114,6 +125,17 @@ SHELL $epub = <<-SHELL #!/bin/bash -eux + # Keep downloaded .deb files in a host-side cache, so a reprovision does not + # re-fetch them. Redirected rather than bind-mounted over + # /var/cache/apt/archives, so apt's lock and partial/ handling stays + # explicit. This must precede anything that runs apt, including the + # NodeSource setup script below. + mkdir -p /vagrant_cache/apt/partial + echo 'Dir::Cache::Archives "/vagrant_cache/apt";' > /etc/apt/apt.conf.d/99-learn-cache + # apt drops privileges to the _apt user to download, which cannot read a + # vboxsf share owned by vagrant; without this it warns on every invocation. + echo 'APT::Sandbox::User "root";' >> /etc/apt/apt.conf.d/99-learn-cache + # Enable the NodeSource repository curl -sL https://deb.nodesource.com/setup_22.x | bash - @@ -286,9 +308,14 @@ vm_cache_gnat = ENV.fetch("LEARN_VM_CACHE_GNAT", vm_cache_node = ENV.fetch("LEARN_VM_CACHE_NODE", File.expand_path(".toolchains/node", __dir__)) +# Host-side apt archive, shared by both VMs. Redirect it with +# LEARN_VM_CACHE_APT. +vm_cache_apt = ENV.fetch("LEARN_VM_CACHE_APT", + File.expand_path(".toolchains/apt", __dir__)) + # Vagrant refuses to start if a synced folder's source does not exist, so the # cache directories have to be created before they are declared below. -[vm_cache_gnat, vm_cache_node].each { |d| FileUtils.mkdir_p(d) } +[vm_cache_gnat, vm_cache_node, vm_cache_apt].each { |d| FileUtils.mkdir_p(d) } Vagrant.configure("2") do |config| @@ -307,6 +334,7 @@ Vagrant.configure("2") do |config| web.vm.synced_folder './content', '/vagrant/content' web.vm.synced_folder vm_cache_gnat, '/vagrant_cache/gnat' web.vm.synced_folder vm_cache_node, '/vagrant_cache/node' + web.vm.synced_folder vm_cache_apt, '/vagrant_cache/apt' web.vm.provision "file", source: "./frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/data/toolchain.ini", destination: "/home/vagrant/toolchain.ini" web.vm.provision "file", source: "./frontend/vm/vm_apt_web.txt", destination: "/home/vagrant/vm_apt.txt" @@ -323,6 +351,7 @@ Vagrant.configure("2") do |config| epub.vm.synced_folder './content', '/vagrant/content' epub.vm.synced_folder vm_cache_gnat, '/vagrant_cache/gnat' epub.vm.synced_folder vm_cache_node, '/vagrant_cache/node' + epub.vm.synced_folder vm_cache_apt, '/vagrant_cache/apt' epub.vm.provision "file", source: "./frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/data/toolchain.ini", destination: "/home/vagrant/toolchain.ini" epub.vm.provision "file", source: "./frontend/vm/vm_apt_epub.txt", destination: "/home/vagrant/vm_apt.txt" From a71a5d2db0c43b271c66051f28f2aa60a564db0e Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 17:53:33 +0200 Subject: [PATCH 06/24] Vagrant: add cache reporting and cleanup scripts Dropping a version from toolchain.ini leaves its tarball in the download cache and nothing prunes it, so orphans accumulate unnoticed. Both scripts derive the set of wanted versions from the same toolchain.ini the provisioner reads, so "orphan" cannot drift from "needed". vm_cache_clean.sh defaults to a dry run: the cache directory is configurable and may be anywhere on the host, so deleting has to be asked for explicitly. Co-Authored-By: Claude Opus 5 (1M context) --- frontend/vm/vm_cache_clean.sh | 56 +++++++++++++++++ frontend/vm/vm_cache_report.sh | 110 +++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 frontend/vm/vm_cache_clean.sh create mode 100644 frontend/vm/vm_cache_report.sh diff --git a/frontend/vm/vm_cache_clean.sh b/frontend/vm/vm_cache_clean.sh new file mode 100644 index 000000000..79d54b2f4 --- /dev/null +++ b/frontend/vm/vm_cache_clean.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# Remove orphaned entries from the toolchain download cache -- tarballs whose +# version is no longer listed in toolchain.ini, and .part files left behind by +# an interrupted download. +# +# Defaults to a dry run. The cache directory is user-configurable and may sit +# anywhere on the host, so deleting requires saying so explicitly. +# +# The orphan list comes from vm_cache_report.sh, which derives it from the +# same toolchain.ini the provisioner reads. +# +# Usage: +# vm_cache_clean.sh # dry run -- show what would be removed +# vm_cache_clean.sh --delete # actually remove it +# +# Environment: +# LEARN_VM_CACHE_GNAT cache directory (default: /.toolchains/gnat) + +set -eu +set -o pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +report="${here}/vm_cache_report.sh" + +delete=false +case "${1:-}" in + --delete) delete=true ;; + -n|--dry-run|"") ;; + -h|--help) sed -n '3,18p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "error: unknown option '$1'" >&2; exit 1 ;; +esac + +orphans=$("${report}" --orphans) + +if [ -z "${orphans}" ]; then + echo "No orphaned cache entries." + exit 0 +fi + +count=$(echo "${orphans}" | wc -l) + +if [ "${delete}" = true ]; then + echo "${orphans}" | while read -r f; do + echo "removing ${f}" + rm -f "${f}" + done + echo "Removed ${count} entr$([ "${count}" -eq 1 ] && echo y || echo ies)." +else + echo "Would remove ${count} orphaned entr$([ "${count}" -eq 1 ] && echo y || echo ies):" + echo "${orphans}" | while read -r f; do + printf ' %-8s %s\n' "$(du -h "${f}" 2>/dev/null | cut -f1)" "${f}" + done + echo + echo "This was a dry run. Re-run with --delete to remove them." +fi diff --git a/frontend/vm/vm_cache_report.sh b/frontend/vm/vm_cache_report.sh new file mode 100644 index 000000000..be2bd8056 --- /dev/null +++ b/frontend/vm/vm_cache_report.sh @@ -0,0 +1,110 @@ +#!/bin/bash +# +# Report on the VM download caches: size, contents, and which toolchain +# entries no longer correspond to a version listed in toolchain.ini. +# +# Dropping a version from toolchain.ini leaves its tarball behind and nothing +# prunes it, so orphans accumulate silently. This reports them; +# vm_cache_clean.sh removes them. +# +# The set of wanted versions is read from the same toolchain.ini the +# provisioner reads, so "orphan" cannot drift from "needed". +# +# Usage: +# vm_cache_report.sh # all caches +# vm_cache_report.sh --orphans # just the orphan paths, one per line +# +# Environment: +# LEARN_VM_CACHE_GNAT / _NODE / _APT override the cache locations +# (defaults: /.toolchains/{gnat,node,apt}) + +set -eu +set -o pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo=$(cd "${here}/../.." && pwd) + +toolchain_ini="${repo}/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/data/toolchain.ini" + +cache_gnat="${LEARN_VM_CACHE_GNAT:-${repo}/.toolchains/gnat}" +cache_node="${LEARN_VM_CACHE_NODE:-${repo}/.toolchains/node}" +cache_apt="${LEARN_VM_CACHE_APT:-${repo}/.toolchains/apt}" + +versions_of () { + sed -n '/^\[toolchains\]/,/^\[/p' "${toolchain_ini}" \ + | sed -n "s/^$1[[:space:]]*=[[:space:]]*//p" +} + +# Print the basenames the GNAT cache is supposed to contain. +wanted_names () { + local tool ver + for tool in gnat gnatprove gprbuild; do + for ver in $(versions_of "${tool}"); do + echo "${tool}-x86_64-linux-${ver}.tar.gz" + echo "${tool}-x86_64-linux-${ver}.tar.gz.sha256" + done + done +} + +# Anything in the GNAT cache that is not wanted. Leftover .part files count as +# orphans too: a complete download is always renamed into place, so one that +# survives is debris from an interrupted run. +orphans () { + [ -d "${cache_gnat}" ] || return 0 + local wanted + # LC_ALL=C throughout: comm compares byte-wise, so both inputs must be + # sorted that way too. Locale collation ignores the '-', which puts + # gnat-... and gnatprove-... in an order comm rejects. + wanted=$(wanted_names | LC_ALL=C sort) + find "${cache_gnat}" -maxdepth 1 -type f -printf '%f\n' 2>/dev/null \ + | LC_ALL=C sort \ + | LC_ALL=C comm -23 - <(echo "${wanted}") \ + | while read -r f; do echo "${cache_gnat}/${f}"; done +} + +size_of () { + if [ -d "$1" ]; then du -sh "$1" 2>/dev/null | cut -f1; else echo "-"; fi +} + +if [ "${1:-}" = "--orphans" ]; then + orphans + exit 0 +fi + +if [ ! -f "${toolchain_ini}" ]; then + echo "error: ${toolchain_ini} not found" >&2 + exit 1 +fi + +printf '%-8s %-8s %s\n' "CACHE" "SIZE" "LOCATION" +printf '%-8s %-8s %s\n' "gnat" "$(size_of "${cache_gnat}")" "${cache_gnat}" +printf '%-8s %-8s %s\n' "node" "$(size_of "${cache_node}")" "${cache_node}" +printf '%-8s %-8s %s\n' "apt" "$(size_of "${cache_apt}")" "${cache_apt}" + +echo +echo "Toolchain versions wanted by toolchain.ini:" +for tool in gnat gnatprove gprbuild; do + printf ' %-10s %s\n' "${tool}" "$(versions_of "${tool}" | tr -s ' ')" +done + +echo +echo "Toolchain tarballs present:" +if [ -d "${cache_gnat}" ]; then + find "${cache_gnat}" -maxdepth 1 -type f -name '*.tar.gz' -printf ' %f\n' \ + 2>/dev/null | sort || true +else + echo " (cache directory does not exist yet)" +fi + +echo +orphan_list=$(orphans) +if [ -z "${orphan_list}" ]; then + echo "No orphaned entries." +else + echo "Orphaned entries (not wanted by toolchain.ini):" + echo "${orphan_list}" | while read -r f; do + printf ' %-8s %s\n' "$(du -h "${f}" 2>/dev/null | cut -f1)" "${f##*/}" + done + echo + echo "Remove them with: frontend/vm/vm_cache_clean.sh --delete" +fi From 8c7b56227d583dbc285bc184213179d266007615 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 17:54:47 +0200 Subject: [PATCH 07/24] Vagrant: make the cache scripts executable and invoke them via bash The scripts went in without the executable bit: this repo has core.fileMode disabled, so chmod on the working tree never reached the index. Set the mode explicitly. The provisioner also calls vm_toolchain_fetch.sh via bash rather than directly, since the script reaches the VM over a vboxsf share whose mount options need not expose the executable bit either. Co-Authored-By: Claude Opus 5 (1M context) --- Vagrantfile | 10 ++++++++-- frontend/vm/vm_cache_clean.sh | 0 frontend/vm/vm_cache_report.sh | 0 frontend/vm/vm_toolchain_fetch.sh | 0 4 files changed, 8 insertions(+), 2 deletions(-) mode change 100644 => 100755 frontend/vm/vm_cache_clean.sh mode change 100644 => 100755 frontend/vm/vm_cache_report.sh mode change 100644 => 100755 frontend/vm/vm_toolchain_fetch.sh diff --git a/Vagrantfile b/Vagrantfile index a0236bf89..64872a828 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -74,7 +74,10 @@ $frontend = <<-SHELL local tarball local tmp - tarball=$(${toolchain_fetch} "${tool}" "${ver}") + # Invoked via bash rather than directly: the repo has + # core.fileMode disabled and the script arrives over a vboxsf + # share, so the executable bit cannot be relied on here. + tarball=$(bash ${toolchain_fetch} "${tool}" "${ver}") # Extract on the VM's own disk, never onto the shared cache folder. tmp=$(mktemp -d) tar xzf "${tarball}" -C "${tmp}" @@ -225,7 +228,10 @@ $epub = <<-SHELL local tarball local tmp - tarball=$(${toolchain_fetch} "${tool}" "${ver}") + # Invoked via bash rather than directly: the repo has + # core.fileMode disabled and the script arrives over a vboxsf + # share, so the executable bit cannot be relied on here. + tarball=$(bash ${toolchain_fetch} "${tool}" "${ver}") # Extract on the VM's own disk, never onto the shared cache folder. tmp=$(mktemp -d) tar xzf "${tarball}" -C "${tmp}" diff --git a/frontend/vm/vm_cache_clean.sh b/frontend/vm/vm_cache_clean.sh old mode 100644 new mode 100755 diff --git a/frontend/vm/vm_cache_report.sh b/frontend/vm/vm_cache_report.sh old mode 100644 new mode 100755 diff --git a/frontend/vm/vm_toolchain_fetch.sh b/frontend/vm/vm_toolchain_fetch.sh old mode 100644 new mode 100755 From 35694c4b9b085cdd1872072e2c41888ee5fcd0ad Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 20:18:54 +0200 Subject: [PATCH 08/24] Vagrant: expand relative cache paths consistently The cache locations are read in two places -- the Vagrantfile and the helper scripts in `frontend/vm/` -- and only the defaults were expanded to absolute paths. A relative value in `LEARN_VM_CACHE_GNAT` and friends was therefore taken verbatim on both sides and resolved against each process's own working directory, so the Vagrantfile and the scripts could disagree about where the cache is. Expand the environment value as well, against the repository root in both cases. Absolute values are unaffected. Verified by resolving an absolute override, a relative override and the default from three different working directories: all three now report the same location from each. Co-Authored-By: Claude Opus 5 (1M context) --- Vagrantfile | 15 +++++++++------ frontend/vm/vm_cache_report.sh | 16 +++++++++++++--- frontend/vm/vm_toolchain_fetch.sh | 12 +++++++++++- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 64872a828..e942ff51f 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -305,20 +305,23 @@ vm_apt_pin = ENV.fetch("VM_APT_PIN", "1") # Host-side download cache for the GNAT-FSF toolchain tarballs, so that # destroying a VM does not throw them away. Redirect it with # LEARN_VM_CACHE_GNAT -- it holds several GB and may belong on another disk. -vm_cache_gnat = ENV.fetch("LEARN_VM_CACHE_GNAT", - File.expand_path(".toolchains/gnat", __dir__)) +vm_cache_gnat = File.expand_path( + ENV.fetch("LEARN_VM_CACHE_GNAT", ".toolchains/gnat"), __dir__) # Host-side pnpm store, so that `pnpm install --frozen-lockfile` does not # re-download half a gigabyte on every reprovision. Redirect it with # LEARN_VM_CACHE_NODE. -vm_cache_node = ENV.fetch("LEARN_VM_CACHE_NODE", - File.expand_path(".toolchains/node", __dir__)) +vm_cache_node = File.expand_path( + ENV.fetch("LEARN_VM_CACHE_NODE", ".toolchains/node"), __dir__) # Host-side apt archive, shared by both VMs. Redirect it with # LEARN_VM_CACHE_APT. -vm_cache_apt = ENV.fetch("LEARN_VM_CACHE_APT", - File.expand_path(".toolchains/apt", __dir__)) +vm_cache_apt = File.expand_path( + ENV.fetch("LEARN_VM_CACHE_APT", ".toolchains/apt"), __dir__) +# Expanded against this file's directory, so that a relative override still +# names one place: the helper scripts in frontend/vm/ resolve it the same way. +# # Vagrant refuses to start if a synced folder's source does not exist, so the # cache directories have to be created before they are declared below. [vm_cache_gnat, vm_cache_node, vm_cache_apt].each { |d| FileUtils.mkdir_p(d) } diff --git a/frontend/vm/vm_cache_report.sh b/frontend/vm/vm_cache_report.sh index be2bd8056..d61955c72 100755 --- a/frontend/vm/vm_cache_report.sh +++ b/frontend/vm/vm_cache_report.sh @@ -26,9 +26,19 @@ repo=$(cd "${here}/../.." && pwd) toolchain_ini="${repo}/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/data/toolchain.ini" -cache_gnat="${LEARN_VM_CACHE_GNAT:-${repo}/.toolchains/gnat}" -cache_node="${LEARN_VM_CACHE_NODE:-${repo}/.toolchains/node}" -cache_apt="${LEARN_VM_CACHE_APT:-${repo}/.toolchains/apt}" +# Expand a cache location against the repository root, matching how the +# Vagrantfile expands it against its own directory. An absolute value is used +# as-is; a relative one must not depend on the caller's working directory. +abspath () { + case "$1" in + /*) echo "$1" ;; + *) echo "${repo}/$1" ;; + esac +} + +cache_gnat=$(abspath "${LEARN_VM_CACHE_GNAT:-.toolchains/gnat}") +cache_node=$(abspath "${LEARN_VM_CACHE_NODE:-.toolchains/node}") +cache_apt=$(abspath "${LEARN_VM_CACHE_APT:-.toolchains/apt}") versions_of () { sed -n '/^\[toolchains\]/,/^\[/p' "${toolchain_ini}" \ diff --git a/frontend/vm/vm_toolchain_fetch.sh b/frontend/vm/vm_toolchain_fetch.sh index 47f09b09e..285a354f7 100755 --- a/frontend/vm/vm_toolchain_fetch.sh +++ b/frontend/vm/vm_toolchain_fetch.sh @@ -31,7 +31,17 @@ repo=$(cd "${here}/../.." && pwd) toolchain_ini="${repo}/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/data/toolchain.ini" -cache="${LEARN_VM_CACHE_GNAT:-${repo}/.toolchains/gnat}" +# Expand a cache location against the repository root, matching how the +# Vagrantfile expands it against its own directory. An absolute value is used +# as-is; a relative one must not depend on the caller's working directory. +abspath () { + case "$1" in + /*) echo "$1" ;; + *) echo "${repo}/$1" ;; + esac +} + +cache=$(abspath "${LEARN_VM_CACHE_GNAT:-.toolchains/gnat}") tag="${LEARN_VM_NAME:-$(hostname)}" base_url=https://github.com/alire-project/GNAT-FSF-builds/releases/download From 437a9d1ed0d4dac5ee21e2e4fa936f02e51ac4a0 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 20:19:16 +0200 Subject: [PATCH 09/24] Vagrant: auto-correct a clashing dev-server port Bringing up the web VM while another one already holds host port 8080 aborts with "Vagrant cannot forward the specified ports on this VM", since an explicitly declared `forwarded_port` is fatal on collision by default. That happens whenever a second checkout of this repository brings up its own VMs. Enable `auto_correct` on the rule so Vagrant picks a free host port and warns instead of refusing to boot. `vagrant port web` reports the port actually in use. The guest port is unchanged, so a single-VM setup still gets 8080 and the address documented in the README stays correct. Co-Authored-By: Claude Opus 5 (1M context) --- Vagrantfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index e942ff51f..39bc712d0 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -337,7 +337,8 @@ Vagrant.configure("2") do |config| config.vm.define "web" do |web| web.vm.box = "bento/ubuntu-24.04" web.vm.box_version = "202510.26.0" - web.vm.network "forwarded_port", guest: 8080, host: 8080, host_ip: "127.0.0.1" + web.vm.network "forwarded_port", guest: 8080, host: 8080, + host_ip: "127.0.0.1", auto_correct: true web.vm.synced_folder './frontend', '/vagrant/frontend' web.vm.synced_folder './content', '/vagrant/content' From d0c19697498e11041e3ee5b9fa6c68b1a64e56ae Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 20:19:27 +0200 Subject: [PATCH 10/24] Vagrant: make the dev-server host port configurable `auto_correct` keeps a second web VM from failing to boot, but the port it lands on is then arbitrary and has to be looked up after the fact. Read the host port from `LEARN_WEB_PORT`, defaulting to 8080. A checkout that sets it gets a predictable address; `auto_correct` remains as the fallback when even that port is taken. The default reproduces the previous behaviour exactly, so an unmodified single-VM setup is unaffected. Co-Authored-By: Claude Opus 5 (1M context) --- Vagrantfile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 39bc712d0..8a196cc23 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -302,6 +302,15 @@ require 'fileutils' # Set VM_APT_PIN=0 to disable it for a base-box bootstrap. vm_apt_pin = ENV.fetch("VM_APT_PIN", "1") +# Several checkouts of this repository can run their own web/epub VMs at the +# same time: VirtualBox names each VM after its directory, and the ports below +# can be moved out of one another's way. + +# Host port for the web VM's dev server (guest 8080). Override it to run more +# than one web VM at once. auto_correct still picks a free port if this one is +# taken too -- `vagrant port web` then reports what was chosen. +web_port = Integer(ENV.fetch("LEARN_WEB_PORT", "8080")) + # Host-side download cache for the GNAT-FSF toolchain tarballs, so that # destroying a VM does not throw them away. Redirect it with # LEARN_VM_CACHE_GNAT -- it holds several GB and may belong on another disk. @@ -337,7 +346,7 @@ Vagrant.configure("2") do |config| config.vm.define "web" do |web| web.vm.box = "bento/ubuntu-24.04" web.vm.box_version = "202510.26.0" - web.vm.network "forwarded_port", guest: 8080, host: 8080, + web.vm.network "forwarded_port", guest: 8080, host: web_port, host_ip: "127.0.0.1", auto_correct: true web.vm.synced_folder './frontend', '/vagrant/frontend' From 38e70700cead72aba619ecd6f808c1613056c730 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 20:19:42 +0200 Subject: [PATCH 11/24] Vagrant: make the SSH host ports configurable The SSH host ports were whatever Vagrant chose: 2222 for the machine that booted first and 2200, the base of the auto-correct range, for the second. That ordering is not stable once several checkouts run their own VMs, so the ports shift and any command using a fixed one breaks. Declare both explicitly under the reserved `ssh` forwarding id, which overrides Vagrant's own rule rather than adding a second one, and read them from `LEARN_WEB_SSH_PORT` and `LEARN_EPUB_SSH_PORT`. The defaults reproduce the values a single checkout gets today. `auto_correct` stays enabled, so these remain a preference rather than a guarantee and `vagrant ssh-config` is still authoritative. Co-Authored-By: Claude Opus 5 (1M context) --- Vagrantfile | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Vagrantfile b/Vagrantfile index 8a196cc23..ac7d8728b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -311,6 +311,20 @@ vm_apt_pin = ENV.fetch("VM_APT_PIN", "1") # taken too -- `vagrant port web` then reports what was chosen. web_port = Integer(ENV.fetch("LEARN_WEB_PORT", "8080")) +# SSH host ports. Vagrant forwards SSH under the reserved id "ssh", so +# redeclaring that id overrides its default rather than adding a second rule. +# The defaults reproduce what Vagrant picks unaided for a single checkout: +# 2222 for the first machine, and 2200 -- the base of the auto-correct range +# -- for the second. Pinning them keeps the numbers predictable when several +# checkouts run at once. +# +# auto_correct stays on, so a clash degrades to a warning rather than a +# refusal to boot. These are therefore a preference, not a guarantee: +# `vagrant ssh-config ` remains authoritative, and anything scripted +# should ask rather than assume. +web_ssh_port = Integer(ENV.fetch("LEARN_WEB_SSH_PORT", "2222")) +epub_ssh_port = Integer(ENV.fetch("LEARN_EPUB_SSH_PORT", "2200")) + # Host-side download cache for the GNAT-FSF toolchain tarballs, so that # destroying a VM does not throw them away. Redirect it with # LEARN_VM_CACHE_GNAT -- it holds several GB and may belong on another disk. @@ -348,6 +362,8 @@ Vagrant.configure("2") do |config| web.vm.box_version = "202510.26.0" web.vm.network "forwarded_port", guest: 8080, host: web_port, host_ip: "127.0.0.1", auto_correct: true + web.vm.network "forwarded_port", guest: 22, host: web_ssh_port, + id: "ssh", auto_correct: true web.vm.synced_folder './frontend', '/vagrant/frontend' web.vm.synced_folder './content', '/vagrant/content' @@ -365,6 +381,8 @@ Vagrant.configure("2") do |config| config.vm.define "epub" do |epub| epub.vm.box = "bento/ubuntu-24.04" epub.vm.box_version = "202510.26.0" + epub.vm.network "forwarded_port", guest: 22, host: epub_ssh_port, + id: "ssh", auto_correct: true epub.vm.synced_folder './frontend', '/vagrant/frontend' epub.vm.synced_folder './content', '/vagrant/content' From 78bebcca38b91dae05d6e1832846f13fc59cf928 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 20:20:03 +0200 Subject: [PATCH 12/24] Vagrant: scope cache temporaries to the checkout `LEARN_VM_NAME` distinguishes the temporary files `vm_toolchain_fetch.sh` writes while downloading, so that the web and epub VMs sharing one cache cannot collide. It held the bare machine name, which is unique only within a single checkout: two `web` VMs from different checkouts would both write the same `.part` file and overwrite each other mid-download. Prefix it with the checkout's directory name, giving values such as `ada-learning-material-web`. Verified by running two fetches of the same tarball concurrently under different names into one cache: both completed, each wrote its own temporary, and the resulting tarball passes its checksum. Co-Authored-By: Claude Opus 5 (1M context) --- Vagrantfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index ac7d8728b..c1e28f40b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -349,6 +349,11 @@ vm_cache_apt = File.expand_path( # cache directories have to be created before they are declared below. [vm_cache_gnat, vm_cache_node, vm_cache_apt].each { |d| FileUtils.mkdir_p(d) } +# The download caches are shared between checkouts, so a temporary file there +# has to name the checkout as well as the machine: two "web" VMs would +# otherwise write to the same .part file. +project = File.basename(__dir__) + Vagrant.configure("2") do |config| config.vm.provider "virtualbox" do |vb| @@ -375,7 +380,7 @@ Vagrant.configure("2") do |config| web.vm.provision "file", source: "./frontend/vm/vm_apt_web.txt", destination: "/home/vagrant/vm_apt.txt" web.vm.provision :shell, inline: $frontend, env: { "VM_APT_PIN" => vm_apt_pin, - "LEARN_VM_NAME" => "web" } + "LEARN_VM_NAME" => "#{project}-web" } end config.vm.define "epub" do |epub| @@ -394,7 +399,7 @@ Vagrant.configure("2") do |config| epub.vm.provision "file", source: "./frontend/vm/vm_apt_epub.txt", destination: "/home/vagrant/vm_apt.txt" epub.vm.provision :shell, inline: $epub, env: { "VM_APT_PIN" => vm_apt_pin, - "LEARN_VM_NAME" => "epub" } + "LEARN_VM_NAME" => "#{project}-epub" } end end From eb87e2b0972c648df8f31bc8ade0551dc0de2b41 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 20:52:22 +0200 Subject: [PATCH 13/24] Vagrant: stop caching the pnpm store on the host `pnpm install` fails when its store sits on a shared folder: [ERR_PNPM_EBADF] [importPackage .../acorn] EBADF: bad file descriptor, copyfile '/vagrant_cache/node/v11/files/...' Downloading works -- the packages resolve and land in the store -- but importing them into `node_modules` aborts partway through. It is not a matter of import strategy: `package-import-method=copy` fails the same way, and `copyfile` and `copy_file_range` both succeed in isolation between the same two directories. The failure only appears under pnpm's concurrent import of several hundred packages. Even working, the cache would not pay for itself. The store holds unpacked, content-addressed files, so it is considerably larger than the compressed tarballs actually fetched; copying it in and out of the shared folder would cost more time than the download it replaces. The toolchain and apt caches are unaffected and stay. Co-Authored-By: Claude Opus 5 (1M context) --- Vagrantfile | 22 +++------------------- frontend/vm/vm_cache_report.sh | 6 ++---- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index c1e28f40b..7d2099a11 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -117,11 +117,7 @@ $frontend = <<-SHELL echo 'export COREPACK_ENABLE_DOWNLOAD_PROMPT=0' >> /home/vagrant/.bashrc yes | corepack enable - # Keep pnpm's content-addressed store on the host cache, so a destroyed VM - # does not take it with it. Nothing is lost by the store living on a shared - # folder: node_modules is on one too, so pnpm already copies rather than - # hardlinks. - sudo -u vagrant bash -c "export COREPACK_ENABLE_DOWNLOAD_PROMPT=0; cd /vagrant/frontend && pnpm config set store-dir /vagrant_cache/node && pnpm install --frozen-lockfile" + sudo -u vagrant bash -c "export COREPACK_ENABLE_DOWNLOAD_PROMPT=0; cd /vagrant/frontend && pnpm install --frozen-lockfile" SHELL @@ -288,11 +284,7 @@ $epub = <<-SHELL echo 'export COREPACK_ENABLE_DOWNLOAD_PROMPT=0' >> /home/vagrant/.bashrc yes | corepack enable - # Keep pnpm's content-addressed store on the host cache, so a destroyed VM - # does not take it with it. Nothing is lost by the store living on a shared - # folder: node_modules is on one too, so pnpm already copies rather than - # hardlinks. - sudo -u vagrant bash -c "export COREPACK_ENABLE_DOWNLOAD_PROMPT=0; cd /vagrant/frontend && pnpm config set store-dir /vagrant_cache/node && pnpm install --frozen-lockfile" + sudo -u vagrant bash -c "export COREPACK_ENABLE_DOWNLOAD_PROMPT=0; cd /vagrant/frontend && pnpm install --frozen-lockfile" SHELL @@ -331,12 +323,6 @@ epub_ssh_port = Integer(ENV.fetch("LEARN_EPUB_SSH_PORT", "2200")) vm_cache_gnat = File.expand_path( ENV.fetch("LEARN_VM_CACHE_GNAT", ".toolchains/gnat"), __dir__) -# Host-side pnpm store, so that `pnpm install --frozen-lockfile` does not -# re-download half a gigabyte on every reprovision. Redirect it with -# LEARN_VM_CACHE_NODE. -vm_cache_node = File.expand_path( - ENV.fetch("LEARN_VM_CACHE_NODE", ".toolchains/node"), __dir__) - # Host-side apt archive, shared by both VMs. Redirect it with # LEARN_VM_CACHE_APT. vm_cache_apt = File.expand_path( @@ -347,7 +333,7 @@ vm_cache_apt = File.expand_path( # # Vagrant refuses to start if a synced folder's source does not exist, so the # cache directories have to be created before they are declared below. -[vm_cache_gnat, vm_cache_node, vm_cache_apt].each { |d| FileUtils.mkdir_p(d) } +[vm_cache_gnat, vm_cache_apt].each { |d| FileUtils.mkdir_p(d) } # The download caches are shared between checkouts, so a temporary file there # has to name the checkout as well as the machine: two "web" VMs would @@ -373,7 +359,6 @@ Vagrant.configure("2") do |config| web.vm.synced_folder './frontend', '/vagrant/frontend' web.vm.synced_folder './content', '/vagrant/content' web.vm.synced_folder vm_cache_gnat, '/vagrant_cache/gnat' - web.vm.synced_folder vm_cache_node, '/vagrant_cache/node' web.vm.synced_folder vm_cache_apt, '/vagrant_cache/apt' web.vm.provision "file", source: "./frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/data/toolchain.ini", destination: "/home/vagrant/toolchain.ini" @@ -392,7 +377,6 @@ Vagrant.configure("2") do |config| epub.vm.synced_folder './frontend', '/vagrant/frontend' epub.vm.synced_folder './content', '/vagrant/content' epub.vm.synced_folder vm_cache_gnat, '/vagrant_cache/gnat' - epub.vm.synced_folder vm_cache_node, '/vagrant_cache/node' epub.vm.synced_folder vm_cache_apt, '/vagrant_cache/apt' epub.vm.provision "file", source: "./frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/data/toolchain.ini", destination: "/home/vagrant/toolchain.ini" diff --git a/frontend/vm/vm_cache_report.sh b/frontend/vm/vm_cache_report.sh index d61955c72..91ec80912 100755 --- a/frontend/vm/vm_cache_report.sh +++ b/frontend/vm/vm_cache_report.sh @@ -15,8 +15,8 @@ # vm_cache_report.sh --orphans # just the orphan paths, one per line # # Environment: -# LEARN_VM_CACHE_GNAT / _NODE / _APT override the cache locations -# (defaults: /.toolchains/{gnat,node,apt}) +# LEARN_VM_CACHE_GNAT / _APT override the cache locations +# (defaults: /.toolchains/{gnat,apt}) set -eu set -o pipefail @@ -37,7 +37,6 @@ abspath () { } cache_gnat=$(abspath "${LEARN_VM_CACHE_GNAT:-.toolchains/gnat}") -cache_node=$(abspath "${LEARN_VM_CACHE_NODE:-.toolchains/node}") cache_apt=$(abspath "${LEARN_VM_CACHE_APT:-.toolchains/apt}") versions_of () { @@ -88,7 +87,6 @@ fi printf '%-8s %-8s %s\n' "CACHE" "SIZE" "LOCATION" printf '%-8s %-8s %s\n' "gnat" "$(size_of "${cache_gnat}")" "${cache_gnat}" -printf '%-8s %-8s %s\n' "node" "$(size_of "${cache_node}")" "${cache_node}" printf '%-8s %-8s %s\n' "apt" "$(size_of "${cache_apt}")" "${cache_apt}" echo From 38707dbe73a7f6e782ca8ca0e9881fe3705e4c32 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 21:55:37 +0200 Subject: [PATCH 14/24] Vagrant: add a script to capture a VM's package set The pin lists in `frontend/vm/` record the exact package versions of a VM whose build output has been checked, so that a later `vagrant up` reproduces that machine rather than whatever the archive serves on the day. Nothing recorded how to regenerate them: the command existed only implicitly, in the shape of the committed files. `vm_apt_capture.sh` writes a snapshot of one or both VMs over `vagrant ssh`, using `dpkg-query -W -f='${binary:Package}=${Version}\n'` -- the format that keeps the `:arch` suffix on multi-arch packages and omits it elsewhere, which is what `apt-get install` expects back. It goes through `vagrant ssh` rather than a fixed port so it keeps working when the forwarded ports move. A failed capture leaves the existing list untouched: the download is written to a temporary file whose exit status is checked explicitly, since the redirection creates that file before the command runs. Co-Authored-By: Claude Opus 5 (1M context) --- frontend/vm/vm_apt_capture.sh | 72 +++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100755 frontend/vm/vm_apt_capture.sh diff --git a/frontend/vm/vm_apt_capture.sh b/frontend/vm/vm_apt_capture.sh new file mode 100755 index 000000000..6101722d2 --- /dev/null +++ b/frontend/vm/vm_apt_capture.sh @@ -0,0 +1,72 @@ +#!/bin/bash -eu +# +# Capture the apt package set of a provisioned VM into its pin list. +# +# The pin lists (vm_apt_web.txt, vm_apt_epub.txt) record the exact package +# versions of a VM whose output has been verified, so that a later +# `vagrant up` reproduces that machine rather than whatever the archive +# serves on the day. This script is how a new snapshot is taken. +# +# Run it on the host, from the working copy the VMs were created in. +# Only snapshot a VM that has been provisioned and whose build output has +# been checked: whatever is installed at that moment is frozen into the +# list and demanded of every future provision. +# +# Usage: +# frontend/vm/vm_apt_capture.sh web +# frontend/vm/vm_apt_capture.sh epub +# frontend/vm/vm_apt_capture.sh # both + +set -o pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo=$(cd "${here}/../.." && pwd) + +# The one thing this script exists to record. `${binary:Package}` keeps the +# `:arch` suffix that multi-arch packages carry (bind9-libs:amd64) and omits +# it elsewhere, which is the format `apt-get install` expects back. +dpkg_format='${binary:Package}=${Version}\n' + +capture () { + local vm=$1 + local out="${repo}/frontend/vm/vm_apt_${vm}.txt" + local tmp="${out}.new" + local rc=0 + + echo "Capturing package set of the '${vm}' VM ..." + + # Go through `vagrant ssh` rather than a hardcoded port/key, so the script + # keeps working if the forwarded ports move. + # + # `|| rc=$?` matters: the redirection below creates ${tmp} before the + # command runs, so letting `set -e` abort here would leave an empty file + # behind. Capture the status instead and clean up explicitly. + ( cd "${repo}" \ + && vagrant ssh "${vm}" -c "dpkg-query -W -f='${dpkg_format}'" -- -T ) \ + > "${tmp}" || rc=$? + + if [ "${rc}" -ne 0 ] || [ ! -s "${tmp}" ]; then + rm -f "${tmp}" + echo "error: could not capture the package set of '${vm}'" \ + "(exit ${rc}); '${out}' left unchanged" >&2 + return 1 + fi + + mv "${tmp}" "${out}" + echo " -> frontend/vm/vm_apt_${vm}.txt ($(wc -l < "${out}") packages)" +} + +vms=("$@") +if [ ${#vms[@]} -eq 0 ]; then + vms=(web epub) +fi + +for vm in "${vms[@]}"; do + capture "${vm}" +done + +cat <<'EOF' + +Review the diff before committing it. The snapshot freezes whatever is +installed right now, including anything installed by hand while debugging. +EOF From 1738a9b85c9d5d3f7b5390881016cc2745d49af9 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 22:27:41 +0200 Subject: [PATCH 15/24] Docs: add a how-to for the Vagrant VMs The VMs grew several options that were documented only as comments in the `Vagrantfile`: environment variables for the host ports and the download caches, the switch that skips apt package pinning, and the scripts in `frontend/vm/` that fill the caches and regenerate the pinned lists. None of it was discoverable from the top-level README. `frontend/vm/README.md` is the longer form of that README's "Getting started" section. It is organized by task rather than by file: bringing the VMs up and connecting to them, running the VMs of several checkouts side by side, using the download caches, what the pinned package lists are and how to update them, and the sequence for moving to a new Ubuntu base box. Each documented variable and default was checked against the `Vagrantfile`, and each command against the script it invokes. Co-Authored-By: Claude Opus 5 (1M context) --- frontend/vm/README.md | 265 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 frontend/vm/README.md diff --git a/frontend/vm/README.md b/frontend/vm/README.md new file mode 100644 index 000000000..3e622ea5e --- /dev/null +++ b/frontend/vm/README.md @@ -0,0 +1,265 @@ +# Vagrant VM: How to + +This file provides more details about setting up the Vagrant VMs for this +repository. It covers: + +- what `vagrant up` actually installs, and how to connect to each VM +- how to run the VMs of several checkouts side by side +- how to avoid re-downloading several GB every time a VM is rebuilt +- what the pinned package lists are, and how to update them +- how to bring up a VM on a new Ubuntu base box + +Everything here is optional, though. A simple `vagrant up` should be +sufficient. + +## Bringing up the VMs + +To create both VMs and install everything they need, run: + +``` +$ vagrant up +``` + +This creates two VMs and provisions each of them. Provisioning installs the +apt packages, the Node.js release from NodeSource, the Ada toolchains listed +in `toolchain.ini` (into `/opt/ada`), a Python virtual environment at +`/vagrant/venv`, and the frontend's npm dependencies. + +The two VMs get different toolchains: `web` installs every GNAT version, +since it only needs `gnatchop`, while `epub` also installs GNATprove and +GPRbuild because it builds and runs the course examples. + +Expect the first run to take a while: the toolchains alone are over 2 GB. +The section on caching below removes that cost from every later run. + +To bring up just one of them: + +``` +$ vagrant up web +$ vagrant up epub +``` + +## Connecting + +Once the VMs are up, open a shell on either of them with: + +``` +$ vagrant ssh web +$ vagrant ssh epub +``` + +Inside either VM, the checkout is mounted at `/vagrant`: `frontend/` and +`content/` are the live directories from your host, so edits on either side +are visible immediately. + +The Ada toolchain is on the `PATH` of a login shell. A command run +non-interactively does not read `~/.profile`, so use a login shell when you +need the toolchain: + +``` +$ vagrant ssh epub -c "bash -lc 'gnatchop --version'" +``` + +## Running several checkouts at the same time + +If you have more than one checkout of this repository, each can have its own +pair of VMs. VirtualBox names each VM after the directory it was created in, +so they do not clash — but the host ports do, and `vagrant up` fails with +"Vagrant cannot forward the specified ports on this VM". + +Give the second checkout its own ports: + +``` +$ export LEARN_WEB_PORT=8081 +$ export LEARN_WEB_SSH_PORT=2232 +$ export LEARN_EPUB_SSH_PORT=2230 +$ vagrant up +``` + +| Variable | Default | Effect | +|---|---|---| +| `LEARN_WEB_PORT` | `8080` | Host port for the `web` development server | +| `LEARN_WEB_SSH_PORT` | `2222` | Host port for SSH to `web` | +| `LEARN_EPUB_SSH_PORT` | `2200` | Host port for SSH to `epub` | + +Set them in the shell you use for that checkout, so that every later +`vagrant` command in it agrees. + +These are a preference, not a guarantee: if a port you ask for is also taken, +Vagrant moves it and warns rather than failing. To see what a machine ended +up with: + +``` +$ vagrant port web +$ vagrant ssh-config web +``` + +`vagrant ssh` always connects correctly, so prefer it over a hard-coded port. + +## Avoiding repeated downloads + +Destroying a VM throws away everything provisioning downloaded. To avoid +paying for that twice, the Ada toolchain tarballs and the `.deb` files are +kept on the host instead, in directories mounted into both VMs. + +Nothing needs to be configured for this — the caches fill themselves on the +first `vagrant up` and are reused from then on. The defaults live in +`.toolchains/` inside the checkout and are gitignored: + +| Variable | Default | Holds | +|---|---|---| +| `LEARN_VM_CACHE_GNAT` | `.toolchains/gnat` | Ada toolchain tarballs and their checksums | +| `LEARN_VM_CACHE_APT` | `.toolchains/apt` | Downloaded `.deb` files | + +Because the default is inside the checkout, a second checkout starts with an +empty cache. To share one set of downloads between them, point both variables +somewhere outside. For example: + +``` +$ export LEARN_VM_CACHE_GNAT=~/vm-cache/gnat +$ export LEARN_VM_CACHE_APT=~/vm-cache/apt +``` + +Use absolute paths. A relative one is resolved against the repository root, +not your current directory. + +### Filling the cache in advance + +You can download the toolchains before creating any VM, so that provisioning +fetches nothing. This runs on the host: + +``` +$ frontend/vm/vm_toolchain_fetch.sh --all +``` + +This is useful when you expect to rebuild a VM several times, or want the +download out of the way before going offline. To fetch a single version: + +``` +$ frontend/vm/vm_toolchain_fetch.sh gnat 15.1.0-2 +``` + +Each tarball is checked against the SHA-256 published alongside it upstream, +every time it is used rather than only when downloaded, so a truncated or +damaged file is replaced automatically instead of breaking a later build. + +### Checking and cleaning up + +To see what the caches currently hold, run: + +``` +$ frontend/vm/vm_cache_report.sh +``` + +This shows where each cache is, how large it is, and which toolchain +tarballs are no longer listed in `toolchain.ini` — those accumulate whenever +a version is dropped, and nothing removes them on its own. + +To get rid of those leftovers, run: + +``` +$ frontend/vm/vm_cache_clean.sh # shows what it would remove +$ frontend/vm/vm_cache_clean.sh --delete # removes it +``` + +The first form changes nothing, so it is safe to run to see the list. + +## The pinned package lists + +Both VMs run Ubuntu, so their system software is installed with `apt`, the +package manager Debian and Ubuntu share. Left to itself, `apt` installs +whichever version of a package the Ubuntu archive happens to offer on the day +you ask for it, which means two people running `vagrant up` a month apart get +two different machines. + +To avoid that, this directory keeps a record of the apt packages each VM +should have: `vm_apt_web.txt` and `vm_apt_epub.txt`, one file per VM, listing +one `package=version` per line. Provisioning installs exactly those versions, +so a `vagrant up` reproduces a known machine rather than today's archive. + +That matters because these VMs produce published artifacts. A new texlive or +font package can change the PDFs without anything failing, and the difference +would only show up when someone compared a course against an earlier release. + +Provisioning passes `--allow-downgrades` deliberately: if a package on the VM +is newer than the pinned version, it is moved *back*. The pinned versions are +expected to lag the archive, and that is the point. + +### Updating the packages in a VM + +Upgrading is a deliberate step, not something that happens on its own. Bring +the VMs up, upgrade in place, and check the result before recording it: + +``` +$ vagrant ssh epub +$ sudo apt update && sudo apt full-upgrade +``` + +Then build the content and confirm the output is still correct — at minimum a +successful `make site`, and a generated course PDF compared against the +published one. Only then record the new state, from the host: + +``` +$ frontend/vm/vm_apt_capture.sh epub +$ frontend/vm/vm_apt_capture.sh # or both VMs at once +``` + +This rewrites the lists in place. Read the diff before committing it: the +snapshot freezes whatever is installed at that moment — including anything +you installed by hand while debugging — and every later `vagrant up` will +then demand it. + +## Bringing up a new Ubuntu base box + +Moving the VMs to a newer Ubuntu release has one complication: the pinned +lists name versions that the new release's archive has never carried, so +provisioning aborts before it can get far enough to record new ones. The +first run therefore has to be made without pinning: + +| Variable | Default | Effect | +|---|---|---| +| `VM_APT_PIN` | `1` | Set to `0` to skip installing the pinned versions | + +The full sequence is: + +1. Set the new box and version in the `Vagrantfile`. Both the `web` and the + `epub` block need it: + + ```ruby + web.vm.box = "bento/ubuntu-" + web.vm.box_version = "" + ``` + +2. Rebuild the VMs with pinning switched off: + + ``` + $ vagrant destroy -f + $ VM_APT_PIN=0 vagrant up + ``` + + Expect to iterate here. A new Ubuntu release is where renamed or dropped + packages turn up, and the failure will be in the package list in the + `Vagrantfile` rather than in the pinned lists, which are switched off. + +3. Verify the result, as when updating packages: a successful build on both + VMs, and a generated course PDF compared against the published one. + +4. Record the new package set: + + ``` + $ frontend/vm/vm_apt_capture.sh + ``` + +5. Rebuild once more, this time *with* pinning, to confirm the new lists + install cleanly from scratch: + + ``` + $ vagrant destroy -f + $ vagrant up + ``` + + This step is not optional: a list that cannot be replayed is worse than no + list, because it will only fail for the next person. + +6. Commit the captured lists together with the `box_version` change. On their + own, neither half describes a working configuration. From f82f05e312b5807b14e102e97c1559deaed88c46 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 22:43:35 +0200 Subject: [PATCH 16/24] Vagrant: split the cache scripts by cache `vm_cache_report.sh` and `vm_cache_clean.sh` handled two unrelated caches, and the generic names hid how unequal that handling was: almost everything they did applied to the GNAT toolchain cache, while the apt cache contributed one line of output and could not be cleaned at all. Move the work into one script per cache, `vm_cache_gnat.sh` and `vm_cache_apt.sh`, each taking the same verbs. The two existing scripts become entry points that dispatch to them and keep their options unchanged, so the documented commands behave exactly as before; either cache script can also be invoked directly. `vm_cache_apt.sh` implements only `summary`, which is all the apt cache supports today. Verified against output captured before the change: `vm_cache_report.sh`, `vm_cache_report.sh --orphans` and `vm_cache_clean.sh` are byte-identical over a cache holding required entries, a dropped version and a stale `.part` file. `--delete` removes the same three entries, leaves the six required ones and is idempotent. `--help` and an unknown argument were checked on all four scripts. Co-Authored-By: Claude Opus 5 (1M context) --- frontend/vm/README.md | 9 ++ frontend/vm/vm_cache_apt.sh | 51 +++++++++++ frontend/vm/vm_cache_clean.sh | 48 +++------- frontend/vm/vm_cache_gnat.sh | 161 +++++++++++++++++++++++++++++++++ frontend/vm/vm_cache_report.sh | 119 +++++------------------- 5 files changed, 255 insertions(+), 133 deletions(-) create mode 100755 frontend/vm/vm_cache_apt.sh create mode 100755 frontend/vm/vm_cache_gnat.sh diff --git a/frontend/vm/README.md b/frontend/vm/README.md index 3e622ea5e..8d281bd4f 100644 --- a/frontend/vm/README.md +++ b/frontend/vm/README.md @@ -164,6 +164,15 @@ $ frontend/vm/vm_cache_clean.sh --delete # removes it The first form changes nothing, so it is safe to run to see the list. +Both commands are entry points covering every cache. The work is done by one +script per cache — `vm_cache_gnat.sh` and `vm_cache_apt.sh` — which take the +same verbs and can be run directly when you only care about one of them: + +``` +$ frontend/vm/vm_cache_gnat.sh report +$ frontend/vm/vm_cache_gnat.sh clean --delete +``` + ## The pinned package lists Both VMs run Ubuntu, so their system software is installed with `apt`, the diff --git a/frontend/vm/vm_cache_apt.sh b/frontend/vm/vm_cache_apt.sh new file mode 100755 index 000000000..1fae624c3 --- /dev/null +++ b/frontend/vm/vm_cache_apt.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Operate on the cache of apt package files. +# +# Reports the location and size of the cache. +# +# The cache holds the .deb files apt downloaded during provisioning, so that +# rebuilding a VM does not fetch them again. Both VMs use the same cache. +# +# Invoked by vm_cache_report.sh. May also be invoked directly to operate on +# this cache alone. +# +# Usage: +# vm_cache_apt.sh summary # one line for the caches table +# +# Environment: +# LEARN_VM_CACHE_APT cache directory (default: /.toolchains/apt) + +set -eu +set -o pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo=$(cd "${here}/../.." && pwd) + +# Expand a cache location against the repository root, matching how the +# Vagrantfile expands it against its own directory. An absolute value is used +# as-is; a relative one must not depend on the caller's working directory. +abspath () { + case "$1" in + /*) echo "$1" ;; + *) echo "${repo}/$1" ;; + esac +} + +cache=$(abspath "${LEARN_VM_CACHE_APT:-.toolchains/apt}") + +size_of () { + if [ -d "$1" ]; then du -sh "$1" 2>/dev/null | cut -f1; else echo "-"; fi +} + +do_summary () { + printf '%-8s %-8s %s\n' "apt" "$(size_of "${cache}")" "${cache}" +} + +case "${1:-}" in + summary) do_summary ;; + -h|--help|"") + sed -n '3,17p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) + echo "error: unknown command '$1'" >&2; exit 1 ;; +esac diff --git a/frontend/vm/vm_cache_clean.sh b/frontend/vm/vm_cache_clean.sh index 79d54b2f4..9e6f525a5 100755 --- a/frontend/vm/vm_cache_clean.sh +++ b/frontend/vm/vm_cache_clean.sh @@ -1,14 +1,13 @@ #!/bin/bash # -# Remove orphaned entries from the toolchain download cache -- tarballs whose -# version is no longer listed in toolchain.ini, and .part files left behind by -# an interrupted download. +# Remove entries that are no longer required from the VM download caches. # -# Defaults to a dry run. The cache directory is user-configurable and may sit -# anywhere on the host, so deleting requires saying so explicitly. +# The caches are handled by one script each. Removal is currently implemented +# for the GNAT toolchain cache only; see vm_cache_gnat.sh for the definition +# of an entry that is no longer required. # -# The orphan list comes from vm_cache_report.sh, which derives it from the -# same toolchain.ini the provisioner reads. +# Defaults to a dry run. The cache directories are configurable and may be +# located anywhere on the host, so removal must be requested explicitly. # # Usage: # vm_cache_clean.sh # dry run -- show what would be removed @@ -21,36 +20,11 @@ set -eu set -o pipefail here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -report="${here}/vm_cache_report.sh" +gnat="${here}/vm_cache_gnat.sh" -delete=false case "${1:-}" in - --delete) delete=true ;; - -n|--dry-run|"") ;; - -h|--help) sed -n '3,18p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; - *) echo "error: unknown option '$1'" >&2; exit 1 ;; + --delete) exec "${gnat}" clean --delete ;; + -n|--dry-run|"") exec "${gnat}" clean ;; + -h|--help) sed -n '3,17p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "error: unknown option '$1'" >&2; exit 1 ;; esac - -orphans=$("${report}" --orphans) - -if [ -z "${orphans}" ]; then - echo "No orphaned cache entries." - exit 0 -fi - -count=$(echo "${orphans}" | wc -l) - -if [ "${delete}" = true ]; then - echo "${orphans}" | while read -r f; do - echo "removing ${f}" - rm -f "${f}" - done - echo "Removed ${count} entr$([ "${count}" -eq 1 ] && echo y || echo ies)." -else - echo "Would remove ${count} orphaned entr$([ "${count}" -eq 1 ] && echo y || echo ies):" - echo "${orphans}" | while read -r f; do - printf ' %-8s %s\n' "$(du -h "${f}" 2>/dev/null | cut -f1)" "${f}" - done - echo - echo "This was a dry run. Re-run with --delete to remove them." -fi diff --git a/frontend/vm/vm_cache_gnat.sh b/frontend/vm/vm_cache_gnat.sh new file mode 100755 index 000000000..f4aafe37a --- /dev/null +++ b/frontend/vm/vm_cache_gnat.sh @@ -0,0 +1,161 @@ +#!/bin/bash +# +# Operate on the cache of GNAT toolchain tarballs. +# +# Reports the contents of the cache, identifies entries that are no longer +# required, and removes them. +# +# An entry is no longer required when its version is not listed in +# toolchain.ini, or when it is a .part file left behind by an interrupted +# download. The list of required versions is read from the same toolchain.ini +# the provisioner reads, so the two cannot diverge. +# +# Invoked by vm_cache_report.sh and vm_cache_clean.sh. May also be invoked +# directly to operate on this cache alone. +# +# Usage: +# vm_cache_gnat.sh summary # one line for the caches table +# vm_cache_gnat.sh report # versions, contents and orphans +# vm_cache_gnat.sh orphans # orphan paths, one per line +# vm_cache_gnat.sh clean # dry run -- what would be removed +# vm_cache_gnat.sh clean --delete # remove it +# +# Environment: +# LEARN_VM_CACHE_GNAT cache directory (default: /.toolchains/gnat) + +set -eu +set -o pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo=$(cd "${here}/../.." && pwd) + +toolchain_ini="${repo}/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/data/toolchain.ini" + +# Expand a cache location against the repository root, matching how the +# Vagrantfile expands it against its own directory. An absolute value is used +# as-is; a relative one must not depend on the caller's working directory. +abspath () { + case "$1" in + /*) echo "$1" ;; + *) echo "${repo}/$1" ;; + esac +} + +cache=$(abspath "${LEARN_VM_CACHE_GNAT:-.toolchains/gnat}") + +size_of () { + if [ -d "$1" ]; then du -sh "$1" 2>/dev/null | cut -f1; else echo "-"; fi +} + +versions_of () { + sed -n '/^\[toolchains\]/,/^\[/p' "${toolchain_ini}" \ + | sed -n "s/^$1[[:space:]]*=[[:space:]]*//p" +} + +# The basenames this cache is required to contain. +wanted_names () { + local tool ver + for tool in gnat gnatprove gprbuild; do + for ver in $(versions_of "${tool}"); do + echo "${tool}-x86_64-linux-${ver}.tar.gz" + echo "${tool}-x86_64-linux-${ver}.tar.gz.sha256" + done + done +} + +# Entries present in the cache that are not required. A .part file is also +# treated as such: a completed download is always renamed into place, so one +# that remains is the residue of an interrupted run. +orphans () { + [ -d "${cache}" ] || return 0 + local wanted + # LC_ALL=C throughout: comm compares byte-wise, so both inputs must be + # sorted that way too. Locale collation ignores the '-', which puts + # gnat-... and gnatprove-... in an order comm rejects. + wanted=$(wanted_names | LC_ALL=C sort) + find "${cache}" -maxdepth 1 -type f -printf '%f\n' 2>/dev/null \ + | LC_ALL=C sort \ + | LC_ALL=C comm -23 - <(echo "${wanted}") \ + | while read -r f; do echo "${cache}/${f}"; done +} + +require_ini () { + if [ ! -f "${toolchain_ini}" ]; then + echo "error: ${toolchain_ini} not found" >&2 + exit 1 + fi +} + +do_summary () { + printf '%-8s %-8s %s\n' "gnat" "$(size_of "${cache}")" "${cache}" +} + +do_report () { + require_ini + local tool orphan_list f + + echo "Toolchain versions wanted by toolchain.ini:" + for tool in gnat gnatprove gprbuild; do + printf ' %-10s %s\n' "${tool}" "$(versions_of "${tool}" | tr -s ' ')" + done + + echo + echo "Toolchain tarballs present:" + if [ -d "${cache}" ]; then + find "${cache}" -maxdepth 1 -type f -name '*.tar.gz' -printf ' %f\n' \ + 2>/dev/null | sort || true + else + echo " (cache directory does not exist yet)" + fi + + echo + orphan_list=$(orphans) + if [ -z "${orphan_list}" ]; then + echo "No orphaned entries." + else + echo "Orphaned entries (not wanted by toolchain.ini):" + echo "${orphan_list}" | while read -r f; do + printf ' %-8s %s\n' "$(du -h "${f}" 2>/dev/null | cut -f1)" "${f##*/}" + done + echo + echo "Remove them with: frontend/vm/vm_cache_clean.sh --delete" + fi +} + +do_clean () { + local delete=$1 + local list count f + + list=$(orphans) + if [ -z "${list}" ]; then + echo "No orphaned cache entries." + return 0 + fi + count=$(echo "${list}" | wc -l) + + if [ "${delete}" = true ]; then + echo "${list}" | while read -r f; do + echo "removing ${f}" + rm -f "${f}" + done + echo "Removed ${count} entr$([ "${count}" -eq 1 ] && echo y || echo ies)." + else + echo "Would remove ${count} orphaned entr$([ "${count}" -eq 1 ] && echo y || echo ies):" + echo "${list}" | while read -r f; do + printf ' %-8s %s\n' "$(du -h "${f}" 2>/dev/null | cut -f1)" "${f}" + done + echo + echo "This was a dry run. Re-run with --delete to remove them." + fi +} + +case "${1:-}" in + summary) do_summary ;; + report) do_report ;; + orphans) orphans ;; + clean) do_clean "$([ "${2:-}" = "--delete" ] && echo true || echo false)" ;; + -h|--help|"") + sed -n '3,24p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) + echo "error: unknown command '$1'" >&2; exit 1 ;; +esac diff --git a/frontend/vm/vm_cache_report.sh b/frontend/vm/vm_cache_report.sh index 91ec80912..6aa93701c 100755 --- a/frontend/vm/vm_cache_report.sh +++ b/frontend/vm/vm_cache_report.sh @@ -1,14 +1,12 @@ #!/bin/bash # -# Report on the VM download caches: size, contents, and which toolchain -# entries no longer correspond to a version listed in toolchain.ini. +# Report on the VM download caches. # -# Dropping a version from toolchain.ini leaves its tarball behind and nothing -# prunes it, so orphans accumulate silently. This reports them; -# vm_cache_clean.sh removes them. +# Prints one summary line per cache, followed by the detailed report of each +# cache that provides one. # -# The set of wanted versions is read from the same toolchain.ini the -# provisioner reads, so "orphan" cannot drift from "needed". +# The caches are handled by one script each: vm_cache_gnat.sh and +# vm_cache_apt.sh. Either may also be invoked directly. # # Usage: # vm_cache_report.sh # all caches @@ -22,97 +20,26 @@ set -eu set -o pipefail here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -repo=$(cd "${here}/../.." && pwd) -toolchain_ini="${repo}/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/data/toolchain.ini" - -# Expand a cache location against the repository root, matching how the -# Vagrantfile expands it against its own directory. An absolute value is used -# as-is; a relative one must not depend on the caller's working directory. -abspath () { - case "$1" in - /*) echo "$1" ;; - *) echo "${repo}/$1" ;; - esac -} - -cache_gnat=$(abspath "${LEARN_VM_CACHE_GNAT:-.toolchains/gnat}") -cache_apt=$(abspath "${LEARN_VM_CACHE_APT:-.toolchains/apt}") - -versions_of () { - sed -n '/^\[toolchains\]/,/^\[/p' "${toolchain_ini}" \ - | sed -n "s/^$1[[:space:]]*=[[:space:]]*//p" -} - -# Print the basenames the GNAT cache is supposed to contain. -wanted_names () { - local tool ver - for tool in gnat gnatprove gprbuild; do - for ver in $(versions_of "${tool}"); do - echo "${tool}-x86_64-linux-${ver}.tar.gz" - echo "${tool}-x86_64-linux-${ver}.tar.gz.sha256" - done - done -} - -# Anything in the GNAT cache that is not wanted. Leftover .part files count as -# orphans too: a complete download is always renamed into place, so one that -# survives is debris from an interrupted run. -orphans () { - [ -d "${cache_gnat}" ] || return 0 - local wanted - # LC_ALL=C throughout: comm compares byte-wise, so both inputs must be - # sorted that way too. Locale collation ignores the '-', which puts - # gnat-... and gnatprove-... in an order comm rejects. - wanted=$(wanted_names | LC_ALL=C sort) - find "${cache_gnat}" -maxdepth 1 -type f -printf '%f\n' 2>/dev/null \ - | LC_ALL=C sort \ - | LC_ALL=C comm -23 - <(echo "${wanted}") \ - | while read -r f; do echo "${cache_gnat}/${f}"; done -} - -size_of () { - if [ -d "$1" ]; then du -sh "$1" 2>/dev/null | cut -f1; else echo "-"; fi -} - -if [ "${1:-}" = "--orphans" ]; then - orphans - exit 0 -fi - -if [ ! -f "${toolchain_ini}" ]; then - echo "error: ${toolchain_ini} not found" >&2 - exit 1 -fi +gnat="${here}/vm_cache_gnat.sh" +apt="${here}/vm_cache_apt.sh" + +case "${1:-}" in + --orphans) + "${gnat}" orphans + exit 0 + ;; + -h|--help) + sed -n '3,17p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit 0 + ;; + "") ;; + *) echo "error: unknown option '$1'" >&2; exit 1 ;; +esac printf '%-8s %-8s %s\n' "CACHE" "SIZE" "LOCATION" -printf '%-8s %-8s %s\n' "gnat" "$(size_of "${cache_gnat}")" "${cache_gnat}" -printf '%-8s %-8s %s\n' "apt" "$(size_of "${cache_apt}")" "${cache_apt}" - -echo -echo "Toolchain versions wanted by toolchain.ini:" -for tool in gnat gnatprove gprbuild; do - printf ' %-10s %s\n' "${tool}" "$(versions_of "${tool}" | tr -s ' ')" -done - -echo -echo "Toolchain tarballs present:" -if [ -d "${cache_gnat}" ]; then - find "${cache_gnat}" -maxdepth 1 -type f -name '*.tar.gz' -printf ' %f\n' \ - 2>/dev/null | sort || true -else - echo " (cache directory does not exist yet)" -fi +"${gnat}" summary +"${apt}" summary echo -orphan_list=$(orphans) -if [ -z "${orphan_list}" ]; then - echo "No orphaned entries." -else - echo "Orphaned entries (not wanted by toolchain.ini):" - echo "${orphan_list}" | while read -r f; do - printf ' %-8s %s\n' "$(du -h "${f}" 2>/dev/null | cut -f1)" "${f##*/}" - done - echo - echo "Remove them with: frontend/vm/vm_cache_clean.sh --delete" -fi +"${gnat}" report From 1123fd2233874d08bf8b94d7b55733f7ab2169f8 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 23:02:23 +0200 Subject: [PATCH 17/24] Vagrant: detect unneeded package files in the apt cache The apt cache grew without bound: every package upgrade adds a .deb file and nothing ever removed the superseded one. `vm_cache_apt.sh` could only report the size of the directory. Give it the same verbs as the toolchain cache script: `report`, `orphans` and `clean`. A package file is no longer required when its package and version appear in neither `vm_apt_web.txt` nor `vm_apt_epub.txt`, which are the lists the provisioner installs from, so the criterion cannot drift from what the VMs actually need. Matching accounts for the `%3a` encoding of an epoch separator in a filename and for the architecture qualifier that a multi-arch entry carries in the lists but that the filename may render as `all` or `amd64`. Only regular `.deb` files directly in the directory are considered; apt keeps `lock`, `partial/` and `apt/` there as well. A missing pin list is an error rather than a reason to treat the whole cache as removable, and a report in which most entries look unneeded prints a warning, since that is what a cache filled by a `VM_APT_PIN=0` bootstrap looks like before the lists have been captured. Verified against the cache of a provisioned VM: all 390 package files match the pinned lists, none spuriously reported. On a constructed cache covering the epoch, `all` and `amd64` cases, superseded versions are reported and required ones are not; `clean --delete` removes exactly those and is idempotent; `lock`, `partial/` and `apt/` are never listed; a missing list exits non-zero without deleting anything. Co-Authored-By: Claude Opus 5 (1M context) --- frontend/vm/vm_cache_apt.sh | 143 ++++++++++++++++++++++++++++++++++-- 1 file changed, 138 insertions(+), 5 deletions(-) diff --git a/frontend/vm/vm_cache_apt.sh b/frontend/vm/vm_cache_apt.sh index 1fae624c3..8e1d8886e 100755 --- a/frontend/vm/vm_cache_apt.sh +++ b/frontend/vm/vm_cache_apt.sh @@ -2,16 +2,32 @@ # # Operate on the cache of apt package files. # -# Reports the location and size of the cache. +# Reports the contents of the cache, identifies entries that are no longer +# required, and removes them. # # The cache holds the .deb files apt downloaded during provisioning, so that # rebuilding a VM does not fetch them again. Both VMs use the same cache. # -# Invoked by vm_cache_report.sh. May also be invoked directly to operate on -# this cache alone. +# An entry is no longer required when its package and version appear in +# neither vm_apt_web.txt nor vm_apt_epub.txt. Those are the same lists the +# provisioner installs from, so the two cannot diverge. A .deb file is named +# __.deb, with the epoch separator of a +# version encoded as %3a. +# +# Note that the lists only describe a VM provisioned with package pinning +# enabled. After a bootstrap with VM_APT_PIN=0, or after upgrading a VM in +# place, the cache legitimately holds versions no list mentions yet; capture +# the lists first, otherwise those entries are reported as no longer required. +# +# Invoked by vm_cache_report.sh and vm_cache_clean.sh. May also be invoked +# directly to operate on this cache alone. # # Usage: -# vm_cache_apt.sh summary # one line for the caches table +# vm_cache_apt.sh summary # one line for the caches table +# vm_cache_apt.sh report # contents and entries to remove +# vm_cache_apt.sh orphans # their paths, one per line +# vm_cache_apt.sh clean # dry run -- what would be removed +# vm_cache_apt.sh clean --delete # remove it # # Environment: # LEARN_VM_CACHE_APT cache directory (default: /.toolchains/apt) @@ -22,6 +38,8 @@ set -o pipefail here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) repo=$(cd "${here}/../.." && pwd) +pin_lists=("${here}/vm_apt_web.txt" "${here}/vm_apt_epub.txt") + # Expand a cache location against the repository root, matching how the # Vagrantfile expands it against its own directory. An absolute value is used # as-is; a relative one must not depend on the caller's working directory. @@ -38,14 +56,129 @@ size_of () { if [ -d "$1" ]; then du -sh "$1" 2>/dev/null | cut -f1; else echo "-"; fi } +require_lists () { + local f + for f in "${pin_lists[@]}"; do + if [ ! -f "${f}" ]; then + echo "error: ${f} not found" >&2 + exit 1 + fi + done +} + +# " " for every entry of both pin lists. The architecture +# qualifier a multi-arch entry carries (bind9-libs:amd64=...) is dropped: the +# same package and version may be cached as _amd64.deb or _all.deb. +wanted_pairs () { + cat "${pin_lists[@]}" \ + | sed -e 's/:[a-z0-9][a-z0-9-]*=/=/' -e 's/=/ /' \ + | LC_ALL=C sort -u +} + +# " " for every .deb in the cache. Only regular files +# directly in the directory are considered: apt also keeps `lock`, `partial/` +# and `apt/` there, which are not ours to touch. +present_pairs () { + [ -d "${cache}" ] || return 0 + find "${cache}" -maxdepth 1 -type f -name '*.deb' -printf '%f\n' 2>/dev/null \ + | sed 's/\.deb$//' \ + | awk -F_ '{ gsub(/%3a/, ":", $2); print $1, $2 }' \ + | LC_ALL=C sort -u +} + +# Paths of the .deb files whose package and version are in neither list. +orphans () { + [ -d "${cache}" ] || return 0 + local stale + # LC_ALL=C throughout: comm compares byte-wise, so both inputs must be + # sorted the same way. + stale=$(LC_ALL=C comm -13 <(wanted_pairs) <(present_pairs)) + [ -n "${stale}" ] || return 0 + # Map each stale pair back to the file carrying it. A version in a filename + # has its epoch separator encoded, so encode before matching. + echo "${stale}" | while read -r pkg ver; do + find "${cache}" -maxdepth 1 -type f \ + -name "${pkg}_${ver//:/%3a}_*.deb" -print 2>/dev/null + done +} + do_summary () { printf '%-8s %-8s %s\n' "apt" "$(size_of "${cache}")" "${cache}" } +do_report () { + require_lists + local total stale_list stale_count f + + if [ ! -d "${cache}" ]; then + echo "Package files present: (cache directory does not exist yet)" + return 0 + fi + + total=$(find "${cache}" -maxdepth 1 -type f -name '*.deb' 2>/dev/null | wc -l) + stale_list=$(orphans) + stale_count=$([ -z "${stale_list}" ] && echo 0 || echo "${stale_list}" | wc -l) + + echo "Package files present: ${total}" + echo "Required by the pinned lists: $((total - stale_count))" + + echo + if [ "${stale_count}" -eq 0 ]; then + echo "No entries to remove." + return 0 + fi + + echo "Entries no longer required (in neither pinned list):" + echo "${stale_list}" | while read -r f; do + printf ' %-8s %s\n' "$(du -h "${f}" 2>/dev/null | cut -f1)" "${f##*/}" + done + + if [ "${stale_count}" -gt $((total - stale_count)) ]; then + echo + echo "Most of the cache is reported as no longer required, which suggests" + echo "the pinned lists do not describe the VMs the cache was filled from." + echo "Capture the lists before removing anything." + fi + + echo + echo "Remove them with: frontend/vm/vm_cache_clean.sh --delete" +} + +do_clean () { + require_lists + local delete=$1 + local list count f + + list=$(orphans) + if [ -z "${list}" ]; then + echo "No apt cache entries to remove." + return 0 + fi + count=$(echo "${list}" | wc -l) + + if [ "${delete}" = true ]; then + echo "${list}" | while read -r f; do + echo "removing ${f}" + rm -f "${f}" + done + echo "Removed ${count} package file$([ "${count}" -eq 1 ] || echo s)." + else + echo "Would remove ${count} package file$([ "${count}" -eq 1 ] || echo s):" + echo "${list}" | while read -r f; do + printf ' %-8s %s\n' "$(du -h "${f}" 2>/dev/null | cut -f1)" "${f}" + done + echo + echo "This was a dry run. Re-run with --delete to remove them." + fi +} + case "${1:-}" in summary) do_summary ;; + report) do_report ;; + orphans) orphans ;; + clean) do_clean "$([ "${2:-}" = "--delete" ] && echo true || echo false)" ;; -h|--help|"") - sed -n '3,17p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + sed -n '3,34p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; *) echo "error: unknown command '$1'" >&2; exit 1 ;; esac From ff622e06bb642a56b1b7b84bb0c4df3fa8b2b6fb Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 23:03:02 +0200 Subject: [PATCH 18/24] Vagrant: include the apt cache in report and clean The entry points covered both caches only nominally: the report showed one summary line for the apt cache and no detail, and cleaning skipped it entirely by handing the whole invocation to the toolchain script. Call both cache scripts from both entry points. `vm_cache_report.sh` now prints the apt detail after the toolchain detail and includes apt paths in `--orphans`; `vm_cache_clean.sh` runs the requested verb against each cache in turn instead of replacing itself with one of them, so its dry run and `--delete` apply to both. The documented options are unchanged. Verified on a cache holding an unneeded tarball and an unneeded package file: both entry points report and remove from both caches, and the dry run remains the default. Co-Authored-By: Claude Opus 5 (1M context) --- frontend/vm/vm_cache_clean.sh | 21 ++++++++++++++------- frontend/vm/vm_cache_report.sh | 4 ++++ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/frontend/vm/vm_cache_clean.sh b/frontend/vm/vm_cache_clean.sh index 9e6f525a5..b52e5c95a 100755 --- a/frontend/vm/vm_cache_clean.sh +++ b/frontend/vm/vm_cache_clean.sh @@ -2,9 +2,9 @@ # # Remove entries that are no longer required from the VM download caches. # -# The caches are handled by one script each. Removal is currently implemented -# for the GNAT toolchain cache only; see vm_cache_gnat.sh for the definition -# of an entry that is no longer required. +# The caches are handled by one script each: vm_cache_gnat.sh and +# vm_cache_apt.sh, which define what makes an entry no longer required and +# may also be invoked directly. # # Defaults to a dry run. The cache directories are configurable and may be # located anywhere on the host, so removal must be requested explicitly. @@ -14,17 +14,24 @@ # vm_cache_clean.sh --delete # actually remove it # # Environment: -# LEARN_VM_CACHE_GNAT cache directory (default: /.toolchains/gnat) +# LEARN_VM_CACHE_GNAT / _APT override the cache locations +# (defaults: /.toolchains/{gnat,apt}) set -eu set -o pipefail here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) gnat="${here}/vm_cache_gnat.sh" +apt="${here}/vm_cache_apt.sh" +args=() case "${1:-}" in - --delete) exec "${gnat}" clean --delete ;; - -n|--dry-run|"") exec "${gnat}" clean ;; - -h|--help) sed -n '3,17p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + --delete) args=(clean --delete) ;; + -n|--dry-run|"") args=(clean) ;; + -h|--help) sed -n '3,18p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; *) echo "error: unknown option '$1'" >&2; exit 1 ;; esac + +"${gnat}" "${args[@]}" +echo +"${apt}" "${args[@]}" diff --git a/frontend/vm/vm_cache_report.sh b/frontend/vm/vm_cache_report.sh index 6aa93701c..7d6036a25 100755 --- a/frontend/vm/vm_cache_report.sh +++ b/frontend/vm/vm_cache_report.sh @@ -27,6 +27,7 @@ apt="${here}/vm_cache_apt.sh" case "${1:-}" in --orphans) "${gnat}" orphans + "${apt}" orphans exit 0 ;; -h|--help) @@ -43,3 +44,6 @@ printf '%-8s %-8s %s\n' "CACHE" "SIZE" "LOCATION" echo "${gnat}" report + +echo +"${apt}" report From 400f869ad067030e68c4b73b824377e6d5ad1c54 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 23:03:39 +0200 Subject: [PATCH 19/24] Vagrant: label each cache in the combined report With both caches reporting detail, the two sections ran together: the apt figures followed the toolchain orphan list with no indication that a new cache had started. Print a heading before each detail section. The cache scripts are unchanged, so a section keeps its current form when one of them is run on its own, where the context is already unambiguous. Co-Authored-By: Claude Opus 5 (1M context) --- frontend/vm/vm_cache_report.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frontend/vm/vm_cache_report.sh b/frontend/vm/vm_cache_report.sh index 7d6036a25..e9d5afe6d 100755 --- a/frontend/vm/vm_cache_report.sh +++ b/frontend/vm/vm_cache_report.sh @@ -43,7 +43,11 @@ printf '%-8s %-8s %s\n' "CACHE" "SIZE" "LOCATION" "${apt}" summary echo +echo "GNAT toolchain cache" +echo "--------------------" "${gnat}" report echo +echo "apt package cache" +echo "-----------------" "${apt}" report From 85f54ec4144f7c8d9ee343529c8969daae3dd36b Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 23:03:56 +0200 Subject: [PATCH 20/24] Docs: describe apt cache cleanup The how-to described the cleanup scripts as removing stale toolchain tarballs, which is no longer the whole story now that the apt cache is covered too. Say what each cache accumulates, and add the caveat specific to the apt cache: a package file counts as required only if one of the pinned lists names it, so a cache filled by a `VM_APT_PIN=0` bootstrap, or by a VM upgraded in place, is almost entirely reported as removable until the lists have been regenerated. Deleting in that state is harmless -- provisioning only installs what the lists name -- but it discards files the next `vagrant up` immediately fetches again. Co-Authored-By: Claude Opus 5 (1M context) --- frontend/vm/README.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/frontend/vm/README.md b/frontend/vm/README.md index 8d281bd4f..5e7b3f8c6 100644 --- a/frontend/vm/README.md +++ b/frontend/vm/README.md @@ -151,9 +151,11 @@ To see what the caches currently hold, run: $ frontend/vm/vm_cache_report.sh ``` -This shows where each cache is, how large it is, and which toolchain -tarballs are no longer listed in `toolchain.ini` — those accumulate whenever -a version is dropped, and nothing removes them on its own. +This shows where each cache is, how large it is, and which of its entries +are no longer required. Both caches accumulate those on their own: the +toolchain cache keeps tarballs whose version has been dropped from +`toolchain.ini`, and the apt cache keeps package files superseded by a +later capture of the pinned lists. To get rid of those leftovers, run: @@ -164,6 +166,16 @@ $ frontend/vm/vm_cache_clean.sh --delete # removes it The first form changes nothing, so it is safe to run to see the list. +One caveat for the apt cache: a package file counts as required only if one +of the pinned lists names it. Immediately after a bootstrap with +`VM_APT_PIN=0`, or after upgrading a VM in place, the lists do not yet +describe the VM, so almost the whole cache is reported as removable. The +report prints a warning whenever most of the cache is no longer required. +Nothing breaks if you delete anyway — provisioning only installs what the +lists name, so it never asks for a file that was removed — but the next +`vagrant up` re-downloads what it needs. Therefore, regenerate the lists +first, so that the cache remains useful. + Both commands are entry points covering every cache. The work is done by one script per cache — `vm_cache_gnat.sh` and `vm_cache_apt.sh` — which take the same verbs and can be run directly when you only care about one of them: From cdf7f4e9fc9f7b24d2dfab784cb07d71496b2e89 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 23:26:49 +0200 Subject: [PATCH 21/24] Vagrant: report every cache even if one fails Both entry points run under `set -e`, so a non-zero exit from the first cache script ended the run before the second was reached. A missing `toolchain.ini` therefore suppressed the apt report entirely, and a missing pinned list suppressed the toolchain cleanup -- silently, since the output simply stopped rather than reporting a problem. Run every cache script regardless, and return the worst status. One unusable cache no longer hides the state of the others. Verified by removing `toolchain.ini` and, separately, a pinned list: in each case the other cache is still reported and cleaned, and both invocations exit non-zero. Co-Authored-By: Claude Opus 5 (1M context) --- frontend/vm/vm_cache_clean.sh | 8 ++++++-- frontend/vm/vm_cache_report.sh | 17 ++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/frontend/vm/vm_cache_clean.sh b/frontend/vm/vm_cache_clean.sh index b52e5c95a..cf0107d0d 100755 --- a/frontend/vm/vm_cache_clean.sh +++ b/frontend/vm/vm_cache_clean.sh @@ -32,6 +32,10 @@ case "${1:-}" in *) echo "error: unknown option '$1'" >&2; exit 1 ;; esac -"${gnat}" "${args[@]}" +# Each cache is processed even if another fails, so that one broken cache +# does not leave the rest uncleaned. The worst status is returned. +rc=0 +"${gnat}" "${args[@]}" || rc=$? echo -"${apt}" "${args[@]}" +"${apt}" "${args[@]}" || rc=$? +exit "${rc}" diff --git a/frontend/vm/vm_cache_report.sh b/frontend/vm/vm_cache_report.sh index e9d5afe6d..dc249c4ec 100755 --- a/frontend/vm/vm_cache_report.sh +++ b/frontend/vm/vm_cache_report.sh @@ -26,9 +26,12 @@ apt="${here}/vm_cache_apt.sh" case "${1:-}" in --orphans) - "${gnat}" orphans - "${apt}" orphans - exit 0 + # Each cache is reported even if another fails, so that one broken cache + # does not hide the rest. The worst status is returned. + rc=0 + "${gnat}" orphans || rc=$? + "${apt}" orphans || rc=$? + exit "${rc}" ;; -h|--help) sed -n '3,17p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' @@ -42,12 +45,16 @@ printf '%-8s %-8s %s\n' "CACHE" "SIZE" "LOCATION" "${gnat}" summary "${apt}" summary +rc=0 + echo echo "GNAT toolchain cache" echo "--------------------" -"${gnat}" report +"${gnat}" report || rc=$? echo echo "apt package cache" echo "-----------------" -"${apt}" report +"${apt}" report || rc=$? + +exit "${rc}" From 2e69dc489d9d12a6bf90ab0fecf59b5a29045ef6 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 23:31:32 +0200 Subject: [PATCH 22/24] Vagrant: name the fetch script after the cache it fills Two scripts own the toolchain cache: one downloads into it, the other reports and prunes it. Their names shared nothing, so neither suggested the other existed, let alone that both resolve `LEARN_VM_CACHE_GNAT` and parse `toolchain.ini` to decide what belongs there. Rename `vm_toolchain_fetch.sh` to `vm_cache_gnat_fetch.sh`, so the common prefix names the cache both act on. The provisioner and the script's own documentation follow the new name; nothing else changes. Co-Authored-By: Claude Opus 5 (1M context) --- Vagrantfile | 8 ++++---- .../vm/{vm_toolchain_fetch.sh => vm_cache_gnat_fetch.sh} | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) rename frontend/vm/{vm_toolchain_fetch.sh => vm_cache_gnat_fetch.sh} (95%) diff --git a/Vagrantfile b/Vagrantfile index 7d2099a11..6c42e06f9 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -63,10 +63,10 @@ $frontend = <<-SHELL # Toolchain download cache: fetch each tarball into the host-side folder # mounted at /vagrant_cache/gnat, verified against its upstream .sha256. - # The script also runs on the host -- `vm_toolchain_fetch.sh --all` warms + # The script also runs on the host -- `vm_cache_gnat_fetch.sh --all` warms # the cache before `vagrant up`. export LEARN_VM_CACHE_GNAT=/vagrant_cache/gnat - toolchain_fetch=/vagrant/frontend/vm/vm_toolchain_fetch.sh + toolchain_fetch=/vagrant/frontend/vm/vm_cache_gnat_fetch.sh install_toolchain () { local tool=$1 @@ -213,10 +213,10 @@ $epub = <<-SHELL # Toolchain download cache: fetch each tarball into the host-side folder # mounted at /vagrant_cache/gnat, verified against its upstream .sha256. - # The script also runs on the host -- `vm_toolchain_fetch.sh --all` warms + # The script also runs on the host -- `vm_cache_gnat_fetch.sh --all` warms # the cache before `vagrant up`. export LEARN_VM_CACHE_GNAT=/vagrant_cache/gnat - toolchain_fetch=/vagrant/frontend/vm/vm_toolchain_fetch.sh + toolchain_fetch=/vagrant/frontend/vm/vm_cache_gnat_fetch.sh install_toolchain () { local tool=$1 diff --git a/frontend/vm/vm_toolchain_fetch.sh b/frontend/vm/vm_cache_gnat_fetch.sh similarity index 95% rename from frontend/vm/vm_toolchain_fetch.sh rename to frontend/vm/vm_cache_gnat_fetch.sh index 285a354f7..436dc2cb2 100755 --- a/frontend/vm/vm_toolchain_fetch.sh +++ b/frontend/vm/vm_cache_gnat_fetch.sh @@ -13,8 +13,8 @@ # the cache before `vagrant up`, so that provisioning downloads nothing. # # Usage: -# vm_toolchain_fetch.sh # print the verified cached path -# vm_toolchain_fetch.sh --all # every version in toolchain.ini +# vm_cache_gnat_fetch.sh # print the verified cached path +# vm_cache_gnat_fetch.sh --all # every version in toolchain.ini # # Environment: # LEARN_VM_CACHE_GNAT cache directory @@ -47,7 +47,7 @@ tag="${LEARN_VM_NAME:-$(hostname)}" base_url=https://github.com/alire-project/GNAT-FSF-builds/releases/download usage () { - sed -n '3,25p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + sed -n '3,24p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' exit "${1:-1}" } From 9e4cf8b3055f19f36d3ef91e7f49fe3786e4dd15 Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 23:32:17 +0200 Subject: [PATCH 23/24] Vagrant: add a fetch verb to the toolchain cache script Downloading into the toolchain cache and pruning it are operations on one cache, but callers had to know which of two scripts performed which. Give `vm_cache_gnat.sh` a `fetch` verb that delegates to `vm_cache_gnat_fetch.sh`, making it the only interface to that cache. Downloading stays in its own file, which is large enough to be worth separating, but callers no longer see it: the provisioner runs `vm_cache_gnat.sh fetch `. This also makes the difference between the caches legible. The apt cache has no `fetch`, because its contents depend on what apt resolves during provisioning and cannot be fetched ahead of time. Verified through the verb: a cold fetch downloads, a second reports `Using cached`, standard output carries only the cached path, and both errors and exit statuses pass through unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- Vagrantfile | 12 ++++++------ frontend/vm/vm_cache_gnat.sh | 13 ++++++++++--- frontend/vm/vm_cache_gnat_fetch.sh | 5 ++++- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 6c42e06f9..80b1724cd 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -63,10 +63,10 @@ $frontend = <<-SHELL # Toolchain download cache: fetch each tarball into the host-side folder # mounted at /vagrant_cache/gnat, verified against its upstream .sha256. - # The script also runs on the host -- `vm_cache_gnat_fetch.sh --all` warms + # The script also runs on the host -- `vm_cache_gnat.sh fetch --all` warms # the cache before `vagrant up`. export LEARN_VM_CACHE_GNAT=/vagrant_cache/gnat - toolchain_fetch=/vagrant/frontend/vm/vm_cache_gnat_fetch.sh + gnat_cache=/vagrant/frontend/vm/vm_cache_gnat.sh install_toolchain () { local tool=$1 @@ -77,7 +77,7 @@ $frontend = <<-SHELL # Invoked via bash rather than directly: the repo has # core.fileMode disabled and the script arrives over a vboxsf # share, so the executable bit cannot be relied on here. - tarball=$(bash ${toolchain_fetch} "${tool}" "${ver}") + tarball=$(bash ${gnat_cache} fetch "${tool}" "${ver}") # Extract on the VM's own disk, never onto the shared cache folder. tmp=$(mktemp -d) tar xzf "${tarball}" -C "${tmp}" @@ -213,10 +213,10 @@ $epub = <<-SHELL # Toolchain download cache: fetch each tarball into the host-side folder # mounted at /vagrant_cache/gnat, verified against its upstream .sha256. - # The script also runs on the host -- `vm_cache_gnat_fetch.sh --all` warms + # The script also runs on the host -- `vm_cache_gnat.sh fetch --all` warms # the cache before `vagrant up`. export LEARN_VM_CACHE_GNAT=/vagrant_cache/gnat - toolchain_fetch=/vagrant/frontend/vm/vm_cache_gnat_fetch.sh + gnat_cache=/vagrant/frontend/vm/vm_cache_gnat.sh install_toolchain () { local tool=$1 @@ -227,7 +227,7 @@ $epub = <<-SHELL # Invoked via bash rather than directly: the repo has # core.fileMode disabled and the script arrives over a vboxsf # share, so the executable bit cannot be relied on here. - tarball=$(bash ${toolchain_fetch} "${tool}" "${ver}") + tarball=$(bash ${gnat_cache} fetch "${tool}" "${ver}") # Extract on the VM's own disk, never onto the shared cache folder. tmp=$(mktemp -d) tar xzf "${tarball}" -C "${tmp}" diff --git a/frontend/vm/vm_cache_gnat.sh b/frontend/vm/vm_cache_gnat.sh index f4aafe37a..6e534f62b 100755 --- a/frontend/vm/vm_cache_gnat.sh +++ b/frontend/vm/vm_cache_gnat.sh @@ -2,8 +2,8 @@ # # Operate on the cache of GNAT toolchain tarballs. # -# Reports the contents of the cache, identifies entries that are no longer -# required, and removes them. +# Downloads toolchain tarballs into the cache, reports its contents, +# identifies entries that are no longer required, and removes them. # # An entry is no longer required when its version is not listed in # toolchain.ini, or when it is a .part file left behind by an interrupted @@ -14,6 +14,9 @@ # directly to operate on this cache alone. # # Usage: +# vm_cache_gnat.sh fetch +# # download one toolchain into the cache +# vm_cache_gnat.sh fetch --all # download every version in toolchain.ini # vm_cache_gnat.sh summary # one line for the caches table # vm_cache_gnat.sh report # versions, contents and orphans # vm_cache_gnat.sh orphans # orphan paths, one per line @@ -43,6 +46,9 @@ abspath () { cache=$(abspath "${LEARN_VM_CACHE_GNAT:-.toolchains/gnat}") +# Downloading is large enough to keep in its own file. +fetch_impl="${here}/vm_cache_gnat_fetch.sh" + size_of () { if [ -d "$1" ]; then du -sh "$1" 2>/dev/null | cut -f1; else echo "-"; fi } @@ -150,12 +156,13 @@ do_clean () { } case "${1:-}" in + fetch) shift; exec bash "${fetch_impl}" "$@" ;; summary) do_summary ;; report) do_report ;; orphans) orphans ;; clean) do_clean "$([ "${2:-}" = "--delete" ] && echo true || echo false)" ;; -h|--help|"") - sed -n '3,24p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + sed -n '3,27p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; *) echo "error: unknown command '$1'" >&2; exit 1 ;; esac diff --git a/frontend/vm/vm_cache_gnat_fetch.sh b/frontend/vm/vm_cache_gnat_fetch.sh index 436dc2cb2..b4ba88af0 100755 --- a/frontend/vm/vm_cache_gnat_fetch.sh +++ b/frontend/vm/vm_cache_gnat_fetch.sh @@ -3,6 +3,9 @@ # Fetch a GNAT-FSF-builds toolchain tarball into the download cache and print # its path. # +# This is the implementation of `vm_cache_gnat.sh fetch`, which is the +# documented way to invoke it. +# # The cache exists so that destroying a VM does not throw the toolchains away: # a full reprovision otherwise re-downloads several GB. Upstream publishes a # .sha256 sidecar for every asset, and it is verified on every use -- not only @@ -47,7 +50,7 @@ tag="${LEARN_VM_NAME:-$(hostname)}" base_url=https://github.com/alire-project/GNAT-FSF-builds/releases/download usage () { - sed -n '3,24p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + sed -n '3,27p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' exit "${1:-1}" } From d926788d7e2573a56775cf5ebe5d2fa819b79d2f Mon Sep 17 00:00:00 2001 From: gusthoff Date: Fri, 11 Sep 2026 23:32:34 +0200 Subject: [PATCH 24/24] Docs: correct the cache script descriptions Three statements in the how-to no longer matched the scripts: the pre-warm commands named the download script directly rather than the `fetch` verb that now fronts it, the example of invoking a single cache showed the toolchain script twice, and the two cache scripts were described as taking the same verbs, which stopped being true once only one of them gained `fetch`. Co-Authored-By: Claude Opus 5 (1M context) --- frontend/vm/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/vm/README.md b/frontend/vm/README.md index 5e7b3f8c6..57d68257b 100644 --- a/frontend/vm/README.md +++ b/frontend/vm/README.md @@ -129,14 +129,14 @@ You can download the toolchains before creating any VM, so that provisioning fetches nothing. This runs on the host: ``` -$ frontend/vm/vm_toolchain_fetch.sh --all +$ frontend/vm/vm_cache_gnat.sh fetch --all ``` This is useful when you expect to rebuild a VM several times, or want the download out of the way before going offline. To fetch a single version: ``` -$ frontend/vm/vm_toolchain_fetch.sh gnat 15.1.0-2 +$ frontend/vm/vm_cache_gnat.sh fetch gnat 15.1.0-2 ``` Each tarball is checked against the SHA-256 published alongside it upstream, @@ -177,12 +177,12 @@ lists name, so it never asks for a file that was removed — but the next first, so that the cache remains useful. Both commands are entry points covering every cache. The work is done by one -script per cache — `vm_cache_gnat.sh` and `vm_cache_apt.sh` — which take the -same verbs and can be run directly when you only care about one of them: +script per cache — `vm_cache_gnat.sh` and `vm_cache_apt.sh` — which can be +run directly when you only care about one of them: ``` $ frontend/vm/vm_cache_gnat.sh report -$ frontend/vm/vm_cache_gnat.sh clean --delete +$ frontend/vm/vm_cache_apt.sh clean --delete ``` ## The pinned package lists