ci: run e2e tests in parallel, move kernel-free checks to the colab p…

.github/workflows/ci.yml

@@ -72,14 +72,16 @@ jobs:
           restore-keys: playwright-${{ runner.os }}-
       - name: Install the browser
         run: python/.venv/bin/playwright install --with-deps chromium
-      # Cold Pyodide/JupyterLite downloads make the first run timing-sensitive,
-      # so failed tests are retried once. The tests share one browser context
-      # (see tests/e2e/conftest.py); --durations shows where the time goes.
+      # Every page with editable cells boots its own Pyodide kernel (a minute
+      # or more on a runner), which is what makes this slow; the tests are
+      # independent, so they run in parallel (-n 4: the runner has 4 cores).
+      # Cold downloads make the first run timing-sensitive, so failed tests are
+      # retried once. --durations shows where the time goes.
       - name: Browser smoke test
         run: |
           (cd tests/example-course/_build/html && python3 -m http.server 8765 >/dev/null 2>&1 &)
           sleep 2
-          NMA_E2E_URL=http://localhost:8765 python/.venv/bin/pytest -v --durations=15 --reruns 1 tests/e2e
+          NMA_E2E_URL=http://localhost:8765 python/.venv/bin/pytest -v --durations=15 --reruns 1 -n 4 tests/e2e
 
   course-check:
     needs: [python, js]

AGENTS.md

@@ -110,9 +110,9 @@ templates/wheels.yml          # optional caller for course-wheels.yml
   `nma.yml` without testing that in a real course.
 - **One `style` slot**: `site.css` = kit CSS (Rubik embedded as data URIs, so
   no relative `url()` depends on where MyST serves the file and the kit's CSS
-  makes no font requests; the built site still requests Roboto from
-  `fonts.gstatic.com` from something that isn't ours, see
-  `tests/e2e/test_branding_browser.py::test_no_google_font_requests`)
+  makes no font requests; the YouTube players embedded by `{nma-video}`
+  fetch Roboto from Google inside their own iframes, which
+  `tests/e2e/test_branding_browser.py` deliberately doesn't count)
   + `:root` overrides from `nma.brand` + the course's
   `nma.style.extra` file.
 - **CSS variables are the API**: `--nma-brand-*` (the five colours a course may
@@ -183,14 +183,20 @@ npm run build:dev && npm test    # dev build: local asset paths
 tests/scripts/build-example.sh
 (cd tests/example-course/_build/html && python3 -m http.server 8765 &)
 python/.venv/bin/playwright install chromium
-NMA_E2E_URL=http://localhost:8765 python/.venv/bin/pytest -q --reruns 1 tests/e2e
+NMA_E2E_URL=http://localhost:8765 python/.venv/bin/pytest -q --reruns 1 -n 4 tests/e2e
 # NMA_E2E_COURSE=<course root> tests another course's build;
 # NMA_E2E_REQUIREJS=<path to require.js> if cdnjs is blocked.
 
 # Golden check for a change: build a course before and after, then
 python3 tests/scripts/compare_builds.py <before course root> <after course root>
 ```
 
+Every page with editable cells boots its own Pyodide kernel (about a minute
+or more in CI, CPU time, not downloads: a shared browser context did not help),
+so the e2e tests run in parallel (`-n 4`) and tests that don't need a kernel
+(the branding checks) use the `runtime: colab` tutorial, which loads in about
+a second. Keep it that way when adding tests.
+
 The e2e suite is timing-sensitive on cold caches (first Pyodide/JupyterLite
 downloads); CI retries failed tests once.
 

python/pyproject.toml

@@ -20,7 +20,7 @@ dependencies = [
 
 [project.optional-dependencies]
 # matplotlib, numpy, ipywidgets: the render tests execute plotting cells.
-test = ["pytest", "pytest-rerunfailures", "playwright", "matplotlib", "numpy", "ipywidgets"]
+test = ["pytest", "pytest-rerunfailures", "pytest-xdist", "playwright", "matplotlib", "numpy", "ipywidgets"]
 
 [project.scripts]
 nma-kit = "nma_kit.cli:main"

tests/e2e/conftest.py

@@ -8,9 +8,6 @@
 loads require.js from cdnjs; in sandboxes where cdnjs is blocked, set this to
 serve it locally. Service workers are then blocked, because Playwright can't
 intercept requests that go through JupyterLite's service worker.
-
-NMA_E2E_FRESH_CONTEXT=1 (optional): a new browser context per test instead of one
-shared by the whole run (slower: every test downloads Pyodide again).
 """
 
 import os
@@ -42,40 +39,23 @@ def _browser():
         browser.close()
 
 
-def _new_context(browser):
+@pytest.fixture
+def page(_browser):
     options = {"ignore_https_errors": True}
     if REQUIREJS:
         options["service_workers"] = "block"
-    context = browser.new_context(**options)
+    context = _browser.new_context(**options)
     if REQUIREJS:
         context.route(
             "**/require.min.js",
             lambda route: route.fulfill(path=REQUIREJS, content_type="application/javascript"),
         )
-    return context
-
-
-@pytest.fixture(scope="session")
-def _shared_context(_browser):
-    """One context for the whole run. Pages in a context share the HTTP cache, so
-    Pyodide and its packages are downloaded once, not once per test (the slow
-    part of this suite). Set NMA_E2E_FRESH_CONTEXT=1 for a new context per test,
-    to rule out state leaking between tests."""
-    context = _new_context(_browser)
-    yield context
-    context.close()
-
-
-@pytest.fixture
-def page(_browser, _shared_context):
-    fresh = os.environ.get("NMA_E2E_FRESH_CONTEXT") == "1"
-    context = _new_context(_browser) if fresh else _shared_context
     page = context.new_page()
-    page.set_default_timeout(120_000)
+    # Pages boot a Pyodide kernel, and several run in parallel on a 4-core CI runner:
+    # 120 s was too close for the slowest tests. A passing test never waits this long.
+    page.set_default_timeout(180_000)
     yield page
-    page.close()
-    if fresh:
-        context.close()
+    context.close()
 
 
 @pytest.fixture

tests/e2e/test_branding_browser.py

@@ -1,4 +1,8 @@
-"""Browser checks for the kit's site branding (_nma/site.css via `nma-kit sync-assets`)."""
+"""Browser checks for the kit's site branding (_nma/site.css via `nma-kit sync-assets`).
+
+They use the `runtime: colab` tutorial: it has no kernel to boot, so a page load
+takes about a second (pages with editable cells take over a minute in CI).
+"""
 
 import pytest
 
@@ -7,16 +11,24 @@
 # Deterministic checks: a failure is not cold-cache flakiness, so don't rerun it.
 pytestmark = pytest.mark.flaky(reruns=0)
 
-PAGE = "/tutorials/w2d1-examples/w2d1-tutorial1/"
+PAGE = "/tutorials/w2d1-examples/w2d1-tutorial3/"
 GOOGLE_FONTS = ("fonts.googleapis.com", "fonts.gstatic.com")
 
 
+def in_main_frame(request):
+    """Third-party iframes (the YouTube players) bring their own fonts: not the site's."""
+    try:
+        return request.frame.parent_frame is None
+    except Exception:  # service worker and worker requests have no frame
+        return True
+
+
 def load(page, base_url):
-    """Open the page; return the font requests it made as (url, referer, resource type)."""
+    """Open the page; return the font requests the site itself made as (url, referer, type)."""
     fonts = []
 
     def record(request):
-        if any(host in request.url for host in GOOGLE_FONTS):
+        if any(host in request.url for host in GOOGLE_FONTS) and in_main_frame(request):
             fonts.append((request.url, request.headers.get("referer", "?"), request.resource_type))
 
     page.on("request", record)
@@ -36,14 +48,13 @@ def test_brand_font_and_colours_apply(page, base_url):
         "getComputedStyle(document.documentElement).getPropertyValue('--nma-brand-primary').trim()")
     assert primary.upper() == "#008CBF"
 
-    # What the kit controls: Rubik is never fetched from Google.
     assert not [f for f in fonts if "rubik" in f[0].lower()], fonts
 
 
-def test_no_google_font_requests(page, base_url):
-    """The kit's stylesheet makes no font requests. Anything reported here comes
-    from something else on the page (the theme, thebe, ...): the Referer is the
-    stylesheet that asked for it."""
+def test_site_makes_no_google_font_requests(page, base_url):
+    """Neither the kit's stylesheet nor the theme asks Google for fonts. (Embedded
+    YouTube players do, from inside their iframes; those are not counted.) The
+    Referer shows which stylesheet asked, if this ever fails."""
     fonts = load(page, base_url)
     assert not fonts, "\n".join(f"{kind}: {url}\n    asked by {referer}" for url, referer, kind in sorted(set(fonts)))
 

tests/e2e/test_cells_browser.py

@@ -28,9 +28,17 @@ def visible_cell_count(md: str) -> int:
     return count
 
 
-def open_page(page, base_url, path):
+def open_page(page, base_url, path, boot=True):
+    """Open a page with editable cells and wait for its panels.
+
+    boot=True also waits for the network to go quiet, which means the Pyodide
+    kernel has finished booting (a minute or more on a CI runner). Tests that
+    never run Python pass boot=False: they don't need it, and a kernel booting
+    in the background only slows the other tests running in parallel.
+    """
     page.goto(base_url + path)
-    page.wait_for_load_state("networkidle")
+    if boot:
+        page.wait_for_load_state("networkidle")
     page.wait_for_selector(".myst-anywidget.nma-cell")
     page.wait_for_timeout(2000)  # let hydration settle
 
@@ -60,11 +68,11 @@ def out_text(panel):
 
 def run(panel):
     panel.locator("button.run").click()
-    expect(status(panel)).to_have_text(DONE, timeout=120_000)
+    expect(status(panel)).to_have_text(DONE, timeout=180_000)
 
 
 def test_one_panel_per_visible_cell(page, base_url):
-    open_page(page, base_url, TUTORIAL1)
+    open_page(page, base_url, TUTORIAL1, boot=False)
     assert panels(page).count() == visible_cell_count(TUTORIAL1_MD)
     for i in range(panels(page).count()):
         panel = panels(page).nth(i)
@@ -171,7 +179,7 @@ def test_restart(page, base_url):
     last = panels(page).last
     run(last)
     last.locator("button.restart").click()
-    expect(status(last)).to_have_text("", timeout=120_000)  # restart takes a few seconds
+    expect(status(last)).to_have_text("", timeout=180_000)  # restart takes a few seconds
     run(last)
     expect(status(last)).to_contain_text("✓")
     assert FALLBACK_NOTE in out_text(panel_with(page, "def integrate_exponential"))