From 967033c652584b73b259d3758929fa9ab6532812 Mon Sep 17 00:00:00 2001 From: omartelo Date: Fri, 16 Oct 2026 12:59:46 -0300 Subject: [PATCH 1/4] ci: temporary loop measuring the native EGL flake --- .github/workflows/native-egl-loop.yml | 80 +++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 .github/workflows/native-egl-loop.yml diff --git a/.github/workflows/native-egl-loop.yml b/.github/workflows/native-egl-loop.yml new file mode 100644 index 00000000..043b4176 --- /dev/null +++ b/.github/workflows/native-egl-loop.yml @@ -0,0 +1,80 @@ +name: native-egl-loop + +# TEMPORARY: measures the eglInitialize 0x3001 flake of the native render +# tests. Removed before the PR is ready. +on: + pull_request: + paths: [native/**, .github/workflows/native-egl-loop.yml] + +permissions: + contents: read + +jobs: + loop: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + variant: [baseline, surfaceless] + defaults: + run: + working-directory: native + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: native/go.mod + cache-dependency-path: native/go.sum + - name: System packages + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + gcc pkg-config libwayland-dev libx11-dev libx11-xcb-dev libxkbcommon-x11-dev \ + libgles2-mesa-dev libegl1-mesa-dev libffi-dev libxcursor-dev libvulkan-dev \ + libgl1-mesa-dri libegl-mesa0 xvfb fontconfig fonts-dejavu-core + - name: Pinned Ghostty commit + id: ghostty + run: echo "commit=$(cat GHOSTTY_COMMIT)" >> "$GITHUB_OUTPUT" + - name: Cache libghostty-vt + id: cache + uses: actions/cache@v4 + with: + path: ~/libghostty-vt + key: libghostty-vt-${{ runner.os }}-${{ steps.ghostty.outputs.commit }} + - name: Set up Zig + if: steps.cache.outputs.cache-hit != 'true' + uses: mlugg/setup-zig@v2 + with: + version: 0.16.0 + - name: Build libghostty-vt + if: steps.cache.outputs.cache-hit != 'true' + working-directory: ${{ runner.temp }} + run: | + git init ghostty + cd ghostty + git fetch --depth 1 https://github.com/ghostty-org/ghostty "${{ steps.ghostty.outputs.commit }}" + git checkout FETCH_HEAD + zig build -Demit-lib-vt -Doptimize=ReleaseFast --prefix ~/libghostty-vt + - name: Point pkg-config at libghostty-vt + run: echo "PKG_CONFIG_PATH=$HOME/libghostty-vt/share/pkgconfig" >> "$GITHUB_ENV" + - name: Warm build + run: go test -count=1 -run XXX ./... + - name: Loop + env: + LIBGL_ALWAYS_SOFTWARE: "1" + EGL_LOG_LEVEL: debug + VARIANT: ${{ matrix.variant }} + run: | + [ "$VARIANT" = surfaceless ] && export EGL_PLATFORM=surfaceless + N=60; runs=0; badruns=0; skips=0 + for i in $(seq $N); do + xvfb-run -a go test -count=1 -v ./... > run.log 2>&1 || { echo "run $i exit $?"; tail -50 run.log; } + s=$(grep -c 'no headless GPU' run.log || true) + runs=$((runs+1)); skips=$((skips+s)); [ "$s" -gt 0 ] && badruns=$((badruns+1)) + if [ "$s" -gt 0 ]; then + echo "=== run $i: $s skip(s)" + grep -B40 'no headless GPU' run.log | grep -v '^--- PASS' | tail -120 + fi + done + echo "RESULT variant=$VARIANT runs=$runs badruns=$badruns skips=$skips" + grep -c -- '--- PASS' run.log | sed 's/^/passes in last run: /' From 4268d0a712a989b2e4b16a11bce49381caa205b0 Mon Sep 17 00:00:00 2001 From: omartelo Date: Fri, 16 Oct 2026 13:07:25 -0300 Subject: [PATCH 2/4] ci: log Xvfb errors and peak X connections in the loop --- .github/workflows/native-egl-loop.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/native-egl-loop.yml b/.github/workflows/native-egl-loop.yml index 043b4176..3bb3240d 100644 --- a/.github/workflows/native-egl-loop.yml +++ b/.github/workflows/native-egl-loop.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - variant: [baseline, surfaceless] + variant: [baseline] defaults: run: working-directory: native @@ -66,9 +66,13 @@ jobs: VARIANT: ${{ matrix.variant }} run: | [ "$VARIANT" = surfaceless ] && export EGL_PLATFORM=surfaceless - N=60; runs=0; badruns=0; skips=0 + N=40; runs=0; badruns=0; skips=0 for i in $(seq $N); do - xvfb-run -a go test -count=1 -v ./... > run.log 2>&1 || { echo "run $i exit $?"; tail -50 run.log; } + ( peak=0; while :; do n=$(ss -xH | grep -c X11-unix || true); [ "$n" -gt "$peak" ] && { peak=$n; echo $peak > peak.txt; }; sleep 0.05; done ) & sampler=$! + : > peak.txt + xvfb-run -a -e xvfb.err go test -count=1 -v ./... > run.log 2>&1 || { echo "run $i exit $?"; tail -50 run.log; } + kill $sampler; echo "run $i peak X sockets: $(cat peak.txt)" + [ -s xvfb.err ] && { echo "--- xvfb.err run $i"; sort xvfb.err | uniq -c | sort -rn | head -20; } s=$(grep -c 'no headless GPU' run.log || true) runs=$((runs+1)); skips=$((skips+s)); [ "$s" -gt 0 ] && badruns=$((badruns+1)) if [ "$s" -gt 0 ]; then From cf8da27852d3dd6450f223e8bb17b861767bfba6 Mon Sep 17 00:00:00 2001 From: omartelo Date: Fri, 16 Oct 2026 13:13:14 -0300 Subject: [PATCH 3/4] ci: compare Xvfb -noreset against baseline in the loop --- .github/workflows/native-egl-loop.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/native-egl-loop.yml b/.github/workflows/native-egl-loop.yml index 3bb3240d..8bcef781 100644 --- a/.github/workflows/native-egl-loop.yml +++ b/.github/workflows/native-egl-loop.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - variant: [baseline] + variant: [baseline, noreset] defaults: run: working-directory: native @@ -65,14 +65,16 @@ jobs: EGL_LOG_LEVEL: debug VARIANT: ${{ matrix.variant }} run: | - [ "$VARIANT" = surfaceless ] && export EGL_PLATFORM=surfaceless - N=40; runs=0; badruns=0; skips=0 + srv="$(sed -n 's/^XVFBARGS="\(.*\)"/\1/p' "$(command -v xvfb-run)")"; echo "xvfb-run default: $srv" + [ "$VARIANT" = noreset ] && srv="-noreset $srv" + N=80 + runs=0; badruns=0; skips=0 for i in $(seq $N); do ( peak=0; while :; do n=$(ss -xH | grep -c X11-unix || true); [ "$n" -gt "$peak" ] && { peak=$n; echo $peak > peak.txt; }; sleep 0.05; done ) & sampler=$! - : > peak.txt - xvfb-run -a -e xvfb.err go test -count=1 -v ./... > run.log 2>&1 || { echo "run $i exit $?"; tail -50 run.log; } + : > peak.txt; : > xvfb.err + xvfb-run -a -s "$srv" -e xvfb.err go test -count=1 -v ./... > run.log 2>&1 || { echo "run $i exit $?"; tail -50 run.log; } kill $sampler; echo "run $i peak X sockets: $(cat peak.txt)" - [ -s xvfb.err ] && { echo "--- xvfb.err run $i"; sort xvfb.err | uniq -c | sort -rn | head -20; } + echo "run $i server generations: $(grep -c 'xkbcomp) reports' xvfb.err || true)" s=$(grep -c 'no headless GPU' run.log || true) runs=$((runs+1)); skips=$((skips+s)); [ "$s" -gt 0 ] && badruns=$((badruns+1)) if [ "$s" -gt 0 ]; then From 0f785c8b6d77a6cce363250b194ba4f7f9fd66b6 Mon Sep 17 00:00:00 2001 From: omartelo Date: Fri, 16 Oct 2026 13:23:14 -0300 Subject: [PATCH 4/4] ci(native): keep Xvfb from resetting between render tests --- .github/workflows/native-egl-loop.yml | 86 --------------------------- .github/workflows/native.yml | 5 +- 2 files changed, 4 insertions(+), 87 deletions(-) delete mode 100644 .github/workflows/native-egl-loop.yml diff --git a/.github/workflows/native-egl-loop.yml b/.github/workflows/native-egl-loop.yml deleted file mode 100644 index 8bcef781..00000000 --- a/.github/workflows/native-egl-loop.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: native-egl-loop - -# TEMPORARY: measures the eglInitialize 0x3001 flake of the native render -# tests. Removed before the PR is ready. -on: - pull_request: - paths: [native/**, .github/workflows/native-egl-loop.yml] - -permissions: - contents: read - -jobs: - loop: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - variant: [baseline, noreset] - defaults: - run: - working-directory: native - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-go@v7 - with: - go-version-file: native/go.mod - cache-dependency-path: native/go.sum - - name: System packages - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ - gcc pkg-config libwayland-dev libx11-dev libx11-xcb-dev libxkbcommon-x11-dev \ - libgles2-mesa-dev libegl1-mesa-dev libffi-dev libxcursor-dev libvulkan-dev \ - libgl1-mesa-dri libegl-mesa0 xvfb fontconfig fonts-dejavu-core - - name: Pinned Ghostty commit - id: ghostty - run: echo "commit=$(cat GHOSTTY_COMMIT)" >> "$GITHUB_OUTPUT" - - name: Cache libghostty-vt - id: cache - uses: actions/cache@v4 - with: - path: ~/libghostty-vt - key: libghostty-vt-${{ runner.os }}-${{ steps.ghostty.outputs.commit }} - - name: Set up Zig - if: steps.cache.outputs.cache-hit != 'true' - uses: mlugg/setup-zig@v2 - with: - version: 0.16.0 - - name: Build libghostty-vt - if: steps.cache.outputs.cache-hit != 'true' - working-directory: ${{ runner.temp }} - run: | - git init ghostty - cd ghostty - git fetch --depth 1 https://github.com/ghostty-org/ghostty "${{ steps.ghostty.outputs.commit }}" - git checkout FETCH_HEAD - zig build -Demit-lib-vt -Doptimize=ReleaseFast --prefix ~/libghostty-vt - - name: Point pkg-config at libghostty-vt - run: echo "PKG_CONFIG_PATH=$HOME/libghostty-vt/share/pkgconfig" >> "$GITHUB_ENV" - - name: Warm build - run: go test -count=1 -run XXX ./... - - name: Loop - env: - LIBGL_ALWAYS_SOFTWARE: "1" - EGL_LOG_LEVEL: debug - VARIANT: ${{ matrix.variant }} - run: | - srv="$(sed -n 's/^XVFBARGS="\(.*\)"/\1/p' "$(command -v xvfb-run)")"; echo "xvfb-run default: $srv" - [ "$VARIANT" = noreset ] && srv="-noreset $srv" - N=80 - runs=0; badruns=0; skips=0 - for i in $(seq $N); do - ( peak=0; while :; do n=$(ss -xH | grep -c X11-unix || true); [ "$n" -gt "$peak" ] && { peak=$n; echo $peak > peak.txt; }; sleep 0.05; done ) & sampler=$! - : > peak.txt; : > xvfb.err - xvfb-run -a -s "$srv" -e xvfb.err go test -count=1 -v ./... > run.log 2>&1 || { echo "run $i exit $?"; tail -50 run.log; } - kill $sampler; echo "run $i peak X sockets: $(cat peak.txt)" - echo "run $i server generations: $(grep -c 'xkbcomp) reports' xvfb.err || true)" - s=$(grep -c 'no headless GPU' run.log || true) - runs=$((runs+1)); skips=$((skips+s)); [ "$s" -gt 0 ] && badruns=$((badruns+1)) - if [ "$s" -gt 0 ]; then - echo "=== run $i: $s skip(s)" - grep -B40 'no headless GPU' run.log | grep -v '^--- PASS' | tail -120 - fi - done - echo "RESULT variant=$VARIANT runs=$runs badruns=$badruns skips=$skips" - grep -c -- '--- PASS' run.log | sed 's/^/passes in last run: /' diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml index 70940f99..df7f4abe 100644 --- a/.github/workflows/native.yml +++ b/.github/workflows/native.yml @@ -84,11 +84,14 @@ jobs: # The render tests skip where no GPU answers, which on a laptop is a # shrug and here would be a gate that checks nothing: a skip fails the job. + # -noreset: each headless window holds the only X connection and drops it + # on Release, so Xvfb regenerates about 60 times a run, and an + # eglInitialize that lands mid-reset fails xcb_connect (1 run in 5 did). - name: go test env: LIBGL_ALWAYS_SOFTWARE: "1" run: | - xvfb-run -a go test -v ./... > test.log 2>&1 || { cat test.log; exit 1; } + xvfb-run -a -s "-noreset -screen 0 1280x1024x24" go test -v ./... > test.log 2>&1 || { cat test.log; exit 1; } grep -E '^(ok|FAIL|---)' test.log if grep -q 'no headless GPU' test.log; then echo "::error::render tests skipped: no headless GPU under Xvfb"