Skip to content

ci: cache the cabal store and run the worker tests in a parallel job #70

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
FeathBow merged 3 commits into from
Oct 18, 2026
Merged

ci: cache the cabal store and run the worker tests in a parallel job #70

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c7ebcdd
test(hf): check the default run in both inference modes and compare o…
FeathBow Oct 17, 2026
eda4dec
ci: cache the cabal store and run the worker tests in their own job i…
FeathBow Oct 17, 2026
55796f0
Merge branch 'main' into ci/speed
FeathBow Oct 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 96 additions & 26 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,20 @@ jobs:
echo "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 $RUNNER_TEMP/actionlint.tar.gz" | sha256sum --check
tar -xzf "$RUNNER_TEMP/actionlint.tar.gz" -C "$RUNNER_TEMP" actionlint
"$RUNNER_TEMP/actionlint" -color
- name: Format
shell: bash
run: |
curl --fail --silent --show-error --location --output "$RUNNER_TEMP/fourmolu.zip" https://github.com/fourmolu/fourmolu/releases/download/v0.20.1.0/fourmolu-0.20.1.0-linux-x86_64.zip
echo "bdd97063b445145b39191259c428bca489fa30578b82d9066fc7377db390f1d0 $RUNNER_TEMP/fourmolu.zip" | sha256sum --check
unzip -q "$RUNNER_TEMP/fourmolu.zip" -d "$RUNNER_TEMP"
git ls-files -z '*.hs' | xargs -0 "$RUNNER_TEMP/fourmolu-0.20.1.0-linux-x86_64/fourmolu" --mode check --record-brace-space true
- name: HLint
shell: bash
run: |
curl --fail --silent --show-error --location --output "$RUNNER_TEMP/hlint.tar.gz" https://github.com/ndmitchell/hlint/releases/download/v3.10/hlint-3.10-x86_64-linux.tar.gz
echo "ccabc8802a58154699a3583b8dddc5ea2e6d65753a62c45c0e80088ebb16b42b $RUNNER_TEMP/hlint.tar.gz" | sha256sum --check
tar -xzf "$RUNNER_TEMP/hlint.tar.gz" -C "$RUNNER_TEMP"
"$RUNNER_TEMP/hlint-3.10/hlint" src test app

haskell:
name: Haskell
Expand All @@ -57,21 +71,68 @@ jobs:
with:
persist-credentials: false
- uses: haskell-actions/setup@cd0d9bdd65b20557f41bea4dbe43d0b5fbbfe553 # v2.11.0
id: setup
with:
ghc-version: '9.14.1'
cabal-version: '3.18.1.0'
- name: Build
- name: Resolve the build plan
run: |
cabal update
cabal build all --enable-tests
cabal build all --enable-tests --dry-run
- uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
id: store
with:
path: ${{ steps.setup.outputs.cabal-store }}
key: cabal-store-${{ runner.os }}-ghc-${{ steps.setup.outputs.ghc-version }}-${{ hashFiles('dist-newstyle/cache/plan.json') }}
restore-keys: cabal-store-${{ runner.os }}-ghc-${{ steps.setup.outputs.ghc-version }}-
- name: Build
run: cabal build all --enable-tests
- uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
if: steps.store.outputs.cache-hit != 'true'
with:
path: ${{ steps.setup.outputs.cabal-store }}
key: ${{ steps.store.outputs.cache-primary-key }}
- name: Test
run: cabal test all --test-show-details=direct
- name: Type fixtures
run: cabal exec -- sh test/types.sh ghc
- name: Confidence bounds against decimal arithmetic
run: |
mkdir -p "$RUNNER_TEMP/confidence"
cabal exec -- ghc -XGHC2021 -Wall -Werror -package invar -outputdir "$RUNNER_TEMP/confidence" -o "$RUNNER_TEMP/confidence/reference" test/confidence/Main.hs
python3 test/confidence/check.py "$RUNNER_TEMP/confidence/reference"

workers:
name: Workers
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: haskell-actions/setup@cd0d9bdd65b20557f41bea4dbe43d0b5fbbfe553 # v2.11.0
id: setup
with:
ghc-version: '9.14.1'
cabal-version: '3.18.1.0'
- name: Resolve the build plan
run: |
cabal update
cabal build all --enable-tests --dry-run
- uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ${{ steps.setup.outputs.cabal-store }}
key: cabal-store-${{ runner.os }}-ghc-${{ steps.setup.outputs.ghc-version }}-${{ hashFiles('dist-newstyle/cache/plan.json') }}
restore-keys: cabal-store-${{ runner.os }}-ghc-${{ steps.setup.outputs.ghc-version }}-
- name: Build the core
run: cabal build exe:invar
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: '3.12'
cache: pip
cache-dependency-path: worker/requirements.txt
- name: Install the pinned numerical dependencies
run: python -m pip install --no-cache-dir --requirement worker/requirements.txt
- name: Test the shared worker modules and the hf package with the built core; mlx and vllm packages run on their native hosts
run: python -m pip install --requirement worker/requirements.txt
- name: Test the shared worker modules and the hf package with the built core, in three concurrent groups; mlx and vllm packages run on their native hosts
shell: bash
env:
HF_HUB_OFFLINE: '1'
Expand All @@ -84,25 +145,34 @@ jobs:
core_directory="$(dirname "$core")"
export PATH="$core_directory:$PATH"
python -B -m unittest worker.tests.invocation worker.tests.probepacked worker.tests.probestore
python -B -m unittest discover -s worker/tests/hf -t . -p '*.py'
- name: Format
shell: bash
run: |
curl --fail --silent --show-error --location --output "$RUNNER_TEMP/fourmolu.zip" https://github.com/fourmolu/fourmolu/releases/download/v0.20.1.0/fourmolu-0.20.1.0-linux-x86_64.zip
echo "bdd97063b445145b39191259c428bca489fa30578b82d9066fc7377db390f1d0 $RUNNER_TEMP/fourmolu.zip" | sha256sum --check
unzip -q "$RUNNER_TEMP/fourmolu.zip" -d "$RUNNER_TEMP"
git ls-files -z '*.hs' | xargs -0 "$RUNNER_TEMP/fourmolu-0.20.1.0-linux-x86_64/fourmolu" --mode check --record-brace-space true
- name: HLint
shell: bash
run: |
curl --fail --silent --show-error --location --output "$RUNNER_TEMP/hlint.tar.gz" https://github.com/ndmitchell/hlint/releases/download/v3.10/hlint-3.10-x86_64-linux.tar.gz
echo "ccabc8802a58154699a3583b8dddc5ea2e6d65753a62c45c0e80088ebb16b42b $RUNNER_TEMP/hlint.tar.gz" | sha256sum --check
tar -xzf "$RUNNER_TEMP/hlint.tar.gz" -C "$RUNNER_TEMP"
"$RUNNER_TEMP/hlint-3.10/hlint" src test app
- name: Type fixtures
run: cabal exec -- sh test/types.sh ghc
- name: Confidence bounds against decimal arithmetic
run: |
mkdir -p "$RUNNER_TEMP/confidence"
cabal exec -- ghc -XGHC2021 -Wall -Werror -package invar -outputdir "$RUNNER_TEMP/confidence" -o "$RUNNER_TEMP/confidence/reference" test/confidence/Main.hs
python3 test/confidence/check.py "$RUNNER_TEMP/confidence/reference"
# The two slowest modules run alone; every other module, including new ones, runs in the third group.
slow=(distillation overlap)
rest=()
for path in worker/tests/hf/*.py; do
name="$(basename "$path" .py)"
case " __init__ ${slow[*]} " in
*" $name "*) ;;
*) rest+=("worker.tests.hf.$name") ;;
esac
done
groups=("${slow[@]}" rest)
pids=()
for group in "${slow[@]}"; do
python -B -m unittest "worker.tests.hf.$group" > "$RUNNER_TEMP/$group.log" 2>&1 &
pids+=("$!")
done
python -B -m unittest "${rest[@]}" > "$RUNNER_TEMP/rest.log" 2>&1 &
pids+=("$!")
failed=()
for index in "${!groups[@]}"; do
wait "${pids[index]}" || failed+=("${groups[index]}")
done
for group in "${groups[@]}"; do
echo "::group::hf tests: $group"
cat "$RUNNER_TEMP/$group.log"
echo "::endgroup::"
done
if [ "${#failed[@]}" -gt 0 ]; then
echo "Failed hf test groups: ${failed[*]}"
exit 1
fi
21 changes: 11 additions & 10 deletions worker/tests/hf/overlap.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,20 +168,21 @@ def test_two_optimizer_steps_per_update_are_counted_in_the_history(self):
steps, = [item["state"]["steps"] for item in history["artifacts"]]
self.assertTrue(steps and all(count == 2 for count in steps))

def test_a_default_two_cycle_run_goes_through_the_runtime_scores_the_reference_and_equals_an_explicit_staleness_zero(self):
def test_a_default_two_cycle_run_journals_its_declaration_scores_the_reference_and_repeats_exactly(self):
runs = {}
for mode in ("serial", "resident"):
with self.subTest(mode=mode):
records, default = self.train("default" + mode, 2, **{"inference-mode": mode})
_, explicit = self.train("explicit" + mode, 2, staleness=0, **{"inference-mode": mode})
self.assertEqual(journaled(default)[0]["entry"], "declaration")
records, runs[mode] = self.train("default" + mode, 2, **{"inference-mode": mode})
self.assertEqual(journaled(runs[mode])[0]["entry"], "declaration")
published = [value["policy"] for value in records if value.get("phase") == "published"]
self.assertEqual(len({self.settings["policy"], *published}), 3)
self.assertEqual([value is not None for value in references(default)], [False, False, True, True])
comparison = compared({"run": default, **independent(self.root, default / "generation2", "default" + mode)},
{"run": explicit, **independent(self.root, explicit / "generation2", "explicit" + mode)})
self.assertTrue(comparison["equal"])
self.assertTrue(comparison["schedule_equal"])
self.assertTrue(comparison["execution"]["declared_equal"])
self.assertEqual([value is not None for value in references(runs[mode])], [False, False, True, True])
_, repeated = self.train("repeated", 2, staleness=0)
comparison = compared({"run": runs["serial"], **independent(self.root, runs["serial"] / "generation2", "defaultserial")},
{"run": repeated, **independent(self.root, repeated / "generation2", "repeated")})
self.assertTrue(comparison["equal"])
self.assertTrue(comparison["schedule_equal"])
self.assertTrue(comparison["execution"]["declared_equal"])

def test_staleness_is_a_schedule_difference(self):
_, fresh = self.train("fresh", 1, staleness=0)
Expand Down
Loading