diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 762c212..297038b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,14 +72,16 @@ jobs: restore-keys: playwright-${{ runner.os }}- - name: Install the browser run: python/.venv/bin/playwright install --with-deps chromium - # Cold Pyodide/JupyterLite downloads make the first run timing-sensitive, - # so failed tests are retried once. The tests share one browser context - # (see tests/e2e/conftest.py); --durations shows where the time goes. + # Every page with editable cells boots its own Pyodide kernel (a minute + # or more on a runner), which is what makes this slow; the tests are + # independent, so they run in parallel (-n 4: the runner has 4 cores). + # Cold downloads make the first run timing-sensitive, so failed tests are + # retried once. --durations shows where the time goes. - name: Browser smoke test run: | (cd tests/example-course/_build/html && python3 -m http.server 8765 >/dev/null 2>&1 &) sleep 2 - NMA_E2E_URL=http://localhost:8765 python/.venv/bin/pytest -v --durations=15 --reruns 1 tests/e2e + NMA_E2E_URL=http://localhost:8765 python/.venv/bin/pytest -v --durations=15 --reruns 1 -n 4 tests/e2e course-check: needs: [python, js] diff --git a/AGENTS.md b/AGENTS.md index 14979cc..6cb3204 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -110,9 +110,9 @@ templates/wheels.yml # optional caller for course-wheels.yml `nma.yml` without testing that in a real course. - **One `style` slot**: `site.css` = kit CSS (Rubik embedded as data URIs, so no relative `url()` depends on where MyST serves the file and the kit's CSS - makes no font requests; the built site still requests Roboto from - `fonts.gstatic.com` from something that isn't ours, see - `tests/e2e/test_branding_browser.py::test_no_google_font_requests`) + makes no font requests; the YouTube players embedded by `{nma-video}` + fetch Roboto from Google inside their own iframes, which + `tests/e2e/test_branding_browser.py` deliberately doesn't count) + `:root` overrides from `nma.brand` + the course's `nma.style.extra` file. - **CSS variables are the API**: `--nma-brand-*` (the five colours a course may @@ -183,7 +183,7 @@ npm run build:dev && npm test # dev build: local asset paths tests/scripts/build-example.sh (cd tests/example-course/_build/html && python3 -m http.server 8765 &) python/.venv/bin/playwright install chromium -NMA_E2E_URL=http://localhost:8765 python/.venv/bin/pytest -q --reruns 1 tests/e2e +NMA_E2E_URL=http://localhost:8765 python/.venv/bin/pytest -q --reruns 1 -n 4 tests/e2e # NMA_E2E_COURSE= tests another course's build; # NMA_E2E_REQUIREJS= if cdnjs is blocked. @@ -191,6 +191,12 @@ NMA_E2E_URL=http://localhost:8765 python/.venv/bin/pytest -q --reruns 1 tests/e2 python3 tests/scripts/compare_builds.py ``` +Every page with editable cells boots its own Pyodide kernel (about a minute +or more in CI, CPU time, not downloads: a shared browser context did not help), +so the e2e tests run in parallel (`-n 4`) and tests that don't need a kernel +(the branding checks) use the `runtime: colab` tutorial, which loads in about +a second. Keep it that way when adding tests. + The e2e suite is timing-sensitive on cold caches (first Pyodide/JupyterLite downloads); CI retries failed tests once. diff --git a/python/pyproject.toml b/python/pyproject.toml index 7ed3c2d..1b45212 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ [project.optional-dependencies] # matplotlib, numpy, ipywidgets: the render tests execute plotting cells. -test = ["pytest", "pytest-rerunfailures", "playwright", "matplotlib", "numpy", "ipywidgets"] +test = ["pytest", "pytest-rerunfailures", "pytest-xdist", "playwright", "matplotlib", "numpy", "ipywidgets"] [project.scripts] nma-kit = "nma_kit.cli:main" diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 585a5a4..075b19f 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -8,9 +8,6 @@ loads require.js from cdnjs; in sandboxes where cdnjs is blocked, set this to serve it locally. Service workers are then blocked, because Playwright can't intercept requests that go through JupyterLite's service worker. - -NMA_E2E_FRESH_CONTEXT=1 (optional): a new browser context per test instead of one -shared by the whole run (slower: every test downloads Pyodide again). """ import os @@ -42,40 +39,23 @@ def _browser(): browser.close() -def _new_context(browser): +@pytest.fixture +def page(_browser): options = {"ignore_https_errors": True} if REQUIREJS: options["service_workers"] = "block" - context = browser.new_context(**options) + context = _browser.new_context(**options) if REQUIREJS: context.route( "**/require.min.js", lambda route: route.fulfill(path=REQUIREJS, content_type="application/javascript"), ) - return context - - -@pytest.fixture(scope="session") -def _shared_context(_browser): - """One context for the whole run. Pages in a context share the HTTP cache, so - Pyodide and its packages are downloaded once, not once per test (the slow - part of this suite). Set NMA_E2E_FRESH_CONTEXT=1 for a new context per test, - to rule out state leaking between tests.""" - context = _new_context(_browser) - yield context - context.close() - - -@pytest.fixture -def page(_browser, _shared_context): - fresh = os.environ.get("NMA_E2E_FRESH_CONTEXT") == "1" - context = _new_context(_browser) if fresh else _shared_context page = context.new_page() - page.set_default_timeout(120_000) + # Pages boot a Pyodide kernel, and several run in parallel on a 4-core CI runner: + # 120 s was too close for the slowest tests. A passing test never waits this long. + page.set_default_timeout(180_000) yield page - page.close() - if fresh: - context.close() + context.close() @pytest.fixture diff --git a/tests/e2e/test_branding_browser.py b/tests/e2e/test_branding_browser.py index eb3eaf7..18f3ce1 100644 --- a/tests/e2e/test_branding_browser.py +++ b/tests/e2e/test_branding_browser.py @@ -1,4 +1,8 @@ -"""Browser checks for the kit's site branding (_nma/site.css via `nma-kit sync-assets`).""" +"""Browser checks for the kit's site branding (_nma/site.css via `nma-kit sync-assets`). + +They use the `runtime: colab` tutorial: it has no kernel to boot, so a page load +takes about a second (pages with editable cells take over a minute in CI). +""" import pytest @@ -7,16 +11,24 @@ # Deterministic checks: a failure is not cold-cache flakiness, so don't rerun it. pytestmark = pytest.mark.flaky(reruns=0) -PAGE = "/tutorials/w2d1-examples/w2d1-tutorial1/" +PAGE = "/tutorials/w2d1-examples/w2d1-tutorial3/" GOOGLE_FONTS = ("fonts.googleapis.com", "fonts.gstatic.com") +def in_main_frame(request): + """Third-party iframes (the YouTube players) bring their own fonts: not the site's.""" + try: + return request.frame.parent_frame is None + except Exception: # service worker and worker requests have no frame + return True + + def load(page, base_url): - """Open the page; return the font requests it made as (url, referer, resource type).""" + """Open the page; return the font requests the site itself made as (url, referer, type).""" fonts = [] def record(request): - if any(host in request.url for host in GOOGLE_FONTS): + if any(host in request.url for host in GOOGLE_FONTS) and in_main_frame(request): fonts.append((request.url, request.headers.get("referer", "?"), request.resource_type)) page.on("request", record) @@ -36,14 +48,13 @@ def test_brand_font_and_colours_apply(page, base_url): "getComputedStyle(document.documentElement).getPropertyValue('--nma-brand-primary').trim()") assert primary.upper() == "#008CBF" - # What the kit controls: Rubik is never fetched from Google. assert not [f for f in fonts if "rubik" in f[0].lower()], fonts -def test_no_google_font_requests(page, base_url): - """The kit's stylesheet makes no font requests. Anything reported here comes - from something else on the page (the theme, thebe, ...): the Referer is the - stylesheet that asked for it.""" +def test_site_makes_no_google_font_requests(page, base_url): + """Neither the kit's stylesheet nor the theme asks Google for fonts. (Embedded + YouTube players do, from inside their iframes; those are not counted.) The + Referer shows which stylesheet asked, if this ever fails.""" fonts = load(page, base_url) assert not fonts, "\n".join(f"{kind}: {url}\n asked by {referer}" for url, referer, kind in sorted(set(fonts))) diff --git a/tests/e2e/test_cells_browser.py b/tests/e2e/test_cells_browser.py index ad89a97..b06643f 100644 --- a/tests/e2e/test_cells_browser.py +++ b/tests/e2e/test_cells_browser.py @@ -28,9 +28,17 @@ def visible_cell_count(md: str) -> int: return count -def open_page(page, base_url, path): +def open_page(page, base_url, path, boot=True): + """Open a page with editable cells and wait for its panels. + + boot=True also waits for the network to go quiet, which means the Pyodide + kernel has finished booting (a minute or more on a CI runner). Tests that + never run Python pass boot=False: they don't need it, and a kernel booting + in the background only slows the other tests running in parallel. + """ page.goto(base_url + path) - page.wait_for_load_state("networkidle") + if boot: + page.wait_for_load_state("networkidle") page.wait_for_selector(".myst-anywidget.nma-cell") page.wait_for_timeout(2000) # let hydration settle @@ -60,11 +68,11 @@ def out_text(panel): def run(panel): panel.locator("button.run").click() - expect(status(panel)).to_have_text(DONE, timeout=120_000) + expect(status(panel)).to_have_text(DONE, timeout=180_000) def test_one_panel_per_visible_cell(page, base_url): - open_page(page, base_url, TUTORIAL1) + open_page(page, base_url, TUTORIAL1, boot=False) assert panels(page).count() == visible_cell_count(TUTORIAL1_MD) for i in range(panels(page).count()): panel = panels(page).nth(i) @@ -171,7 +179,7 @@ def test_restart(page, base_url): last = panels(page).last run(last) last.locator("button.restart").click() - expect(status(last)).to_have_text("", timeout=120_000) # restart takes a few seconds + expect(status(last)).to_have_text("", timeout=180_000) # restart takes a few seconds run(last) expect(status(last)).to_contain_text("✓") assert FALLBACK_NOTE in out_text(panel_with(page, "def integrate_exponential"))