T6: run the Verify suite five times every night on Linux and macOS, and name every flaky test

.github/workflows/cppverify-stability.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+"""Compare the results of several runs of a lit suite (lit -o FILE.json).
+
+Prints each test whose result is not PASS, UNSUPPORTED, or XFAIL in some run,
+with its result in every run, and exits non-zero if there is one. A test whose
+results differ between runs is flaky: its verdict depends on more than the
+program, such as the load or the solver's search order.
+"""
+
+import json
+import sys
+
+GOOD = {"PASS", "UNSUPPORTED", "XFAIL"}
+
+runs = []
+for path in sys.argv[1:]:
+    with open(path) as f:
+        runs.append({t["name"]: t["code"] for t in json.load(f)["tests"]})
+
+names = sorted(set().union(*runs))
+flaky, failing = [], []
+for name in names:
+    codes = [run.get(name, "MISSING") for run in runs]
+    if all(code in GOOD for code in codes):
+        continue
+    (failing if len(set(codes)) == 1 else flaky).append((name, codes))
+
+print(f"{len(names)} tests, {len(runs)} runs: {len(flaky)} flaky, "
+      f"{len(failing)} failing in every run")
+for title, tests in (("Flaky", flaky), ("Failing in every run", failing)):
+    if tests:
+        print(f"\n{title}:")
+        for name, codes in tests:
+            print(f"  {name}: {' '.join(codes)}")
+sys.exit(1 if flaky or failing else 0)

.github/workflows/cppverify-stability.yml

@@ -0,0 +1,109 @@
+name: CppVerify stability
+
+# Every night, the Verify suite runs five times on Linux and on macOS, at the
+# parallelism CI uses. A test whose result differs between runs has a verdict
+# that depends on more than the program, such as the load or the solver's
+# search order; the job fails and names it with its result in each run.
+
+on:
+  schedule:
+    - cron: "17 4 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  CVC5_VERSION: "1.4.2"
+  RUNS: 5
+
+jobs:
+  linux:
+    name: Linux
+    runs-on: ubuntu-24.04
+    timeout-minutes: 355
+    env:
+      CCACHE_DIR: ${{ github.workspace }}/.ccache
+      CCACHE_MAXSIZE: 4G
+      CCACHE_COMPRESS: "1"
+    steps:
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install build tools and cvc5
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ninja-build ccache
+          curl -sSfL -o "$RUNNER_TEMP/cvc5.zip" \
+            "https://github.com/cvc5/cvc5/releases/download/cvc5-$CVC5_VERSION/cvc5-Linux-x86_64-static.zip"
+          unzip -q "$RUNNER_TEMP/cvc5.zip" -d "$RUNNER_TEMP/cvc5"
+          dirname "$(find "$RUNNER_TEMP/cvc5" -name cvc5 -type f)" >> "$GITHUB_PATH"
+
+      - uses: actions/cache/restore@v4
+        with:
+          path: .ccache
+          key: ccache-linux-release-${{ github.sha }}
+          restore-keys: ccache-linux-release-
+
+      - name: Configure
+        run: |
+          cmake -S llvm -B build -G Ninja \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DLLVM_ENABLE_PROJECTS=clang \
+            -DLLVM_TARGETS_TO_BUILD=Native \
+            -DCPPVERIFY_VENDOR_Z3=ON \
+            -DCPPVERIFY_PREFER_SYSTEM_Z3=OFF \
+            -DLLVM_ENABLE_ASSERTIONS=OFF \
+            -DLLVM_CCACHE_BUILD=ON
+
+      - name: Build
+        run: ninja -C build clang cpp-verify clang-format FileCheck not count split-file llvm-config
+
+      - name: Run the suite ${{ env.RUNS }} times
+        run: |
+          build/bin/cpp-verify --version
+          for run in $(seq "$RUNS"); do
+            build/bin/llvm-lit -s --no-progress-bar clang/test/Verify -o "run$run.json" || true
+          done
+          python3 .github/workflows/cppverify-stability.py run*.json
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: stability-linux
+          path: run*.json
+
+  macos:
+    name: macOS
+    runs-on: macos-14
+    timeout-minutes: 355
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install build tools
+        run: brew install ninja
+
+      - name: Build with setup.sh
+        run: |
+          ./setup.sh
+          ninja -C build FileCheck not count split-file llvm-config
+
+      - name: Run the suite ${{ env.RUNS }} times
+        run: |
+          build/bin/cpp-verify --version
+          for run in $(seq "$RUNS"); do
+            build/bin/llvm-lit -s --no-progress-bar clang/test/Verify -o "run$run.json" || true
+          done
+          python3 .github/workflows/cppverify-stability.py run*.json
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: stability-macos
+          path: run*.json