From ee047caa8ed18417bd0868ab16e70956ec1078e7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 22:45:53 +0000 Subject: [PATCH] Measure what a sub-interpreter cell actually costs The performance snapshot quoted 0.7 ms spawn and 0.5 MiB RSS against the "sub-interpreter backend". Those are numbers for the thread backend that `backend="subinterpreter"` selects today, not for a CPython sub-interpreter, so they understate the roadmap item that replaces it by more than an order of magnitude. Two of the four rows were also not produced by anything in the repository. Add scripts/cell_cost.py, which measures the primitives that backend would be built on, and quote it. On free-threaded CPython 3.14 a cell costs 10.9 ms and 3.5 MiB bare, 56.8 ms and 13.3 MiB once it has imported a realistic module surface -- against 1.62 ms for a fork(), which is a real boundary. The one regime where a cell is cheap is dispatch onto an already-warm pooled interpreter, at 0.82 ms, which is an argument for a cell pool rather than an interpreter per request. The script also separates two things that get conflated: on the free-threaded build four plain threads scaled 3.3x against four cells' 3.9x, while on a GIL build the same threads scaled 0.96x. Parallelism comes from the build; sub-interpreters buy a private sys.modules and a private set of globals. Lifecycle measurement is gated to CPython 3.14+. Holding ~20 interpreters that imported http.client or email.message and then destroying them aborts the process with `munmap_chunk(): invalid pointer` on 3.13's private `_interpreters`; the same workload is clean on 3.14's public `concurrent.interpreters`. `--allow-unstable-api` measures anyway. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Fs1hTtmF4Hm9h617AG9Gse --- README.md | 72 ++++++-- scripts/cell_cost.py | 379 ++++++++++++++++++++++++++++++++++++++++ tests/test_cell_cost.py | 124 +++++++++++++ 3 files changed, 563 insertions(+), 12 deletions(-) create mode 100755 scripts/cell_cost.py create mode 100644 tests/test_cell_cost.py diff --git a/README.md b/README.md index 43e4dfa..942db59 100644 --- a/README.md +++ b/README.md @@ -292,18 +292,66 @@ boundary statement. ## Performance snapshot -These are indicative figures from one reference machine *(Ryzen 7950X, Linux -6.9, sub-interpreter backend)* — hardware-, kernel-, and build-dependent, and -**not** a benchmark to copy into a comparison. Reproduce them on your own host -with `python scripts/benchmark.py` (add `--backend process` for the process -boundary); the encrypted-throughput and RSS rows are not yet covered by it. - -| Metric | Value | -| ----------------------- | ------- | -| Spawn latency | 0.7 ms | -| Round‑trip (1 kB) | 70 µs | -| Max encrypted msgs/core | 1.9 M/s | -| Baseline RSS | 0.5 MiB | +All figures are hardware-, kernel-, and build-dependent, and **not** benchmarks +to copy into a comparison. Reproduce them on your own host. + +### Shipped backends + +From one reference machine *(Ryzen 7950X, Linux 6.9)*. Reproduce with +`python scripts/benchmark.py` (add `--backend process` for the process +boundary): + +| Metric | Value | +| ----------------- | ------ | +| Spawn latency | 0.7 ms | +| Round-trip (1 kB) | 70 us | + +These are numbers for the **thread** backend -- the runtime that +`backend="subinterpreter"` selects today (see +[Sub-interpreter status](#sub-interpreter-status)). They are not what a real +CPython sub-interpreter costs, and the roadmap item that lands real +sub-interpreters will make them substantially worse; the next section measures +what it will cost. + +### What a real sub-interpreter costs + +`python scripts/cell_cost.py` measures the CPython primitives the planned +sub-interpreter backend would be built on, so the roadmap can be argued from +numbers taken on a real machine rather than from the assumption that a +sub-interpreter is cheap. From one run on free-threaded CPython 3.14 (4-core +container, `--iterations 30`): + +| Import surface in the cell | create+exec (p50) | close (p50) | RSS/cell | +| ----------------------------------------- | ----------------- | ----------- | -------- | +| bare (`x = 1`) | 10.9 ms | 3.9 ms | 3.5 MiB | +| `json, re, dataclasses` | 29.2 ms | 7.8 ms | 7.7 MiB | +| `+ email, http.client, logging, argparse` | 56.8 ms | 13.3 ms | 13.3 MiB | + +| Reference point | p50 | +| --------------------------------------- | ------- | +| dispatch onto an already-warm pool cell | 0.82 ms | +| `fork()` of a warm parent | 1.62 ms | + +Three things follow, and they shape the roadmap: + +* **Creating a cell is not cheap.** A sub-interpreter re-imports every module it + uses with no copy-on-write sharing, so the import surface -- not the + interpreter object -- dominates. A `fork()`, which *is* a real boundary, costs + less than the cheapest possible cell. +* **Cells are only cheap when pooled.** 0.82 ms to dispatch onto a warm + interpreter is the number worth designing around, which means a pool of cells + with pre-warmed import surfaces, not an interpreter per request. +* **Parallelism comes from the build, not from the interpreters.** On the same + 4-core box four cells scaled 3.9x -- but four *plain threads* on that + free-threaded build scaled 3.3x too. On a GIL build (3.13) those threads + scaled 0.96x while cells scaled 3.3x. So sub-interpreters buy parallelism on a + GIL build, and buy a private `sys.modules` and a private set of globals per + tenant on a free-threaded one. + +The encrypted-throughput and baseline-RSS rows previously quoted here are not +produced by either script, and have been dropped rather than left as figures +nobody can reproduce. + --- diff --git a/scripts/cell_cost.py b/scripts/cell_cost.py new file mode 100755 index 0000000..93d66e0 --- /dev/null +++ b/scripts/cell_cost.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 +"""Measure what a sub-interpreter execution cell actually costs on this host. + +``scripts/benchmark.py`` measures the *current* backends through the public +``iso.spawn`` API. This script measures the CPython primitives the planned +sub-interpreter backend would be built on, so the roadmap can be argued from +numbers taken on a real machine instead of from the assumption that a +sub-interpreter is cheap: + +* ``create + exec + close`` for a fresh interpreter, at three import surfaces, + because a sub-interpreter re-imports every module it uses and that import + work -- not the interpreter object -- dominates the cost; +* resident memory held per live interpreter, for the same three surfaces; +* dispatch latency onto an already-warm pooled interpreter, which is the only + regime where a cell is cheap; +* ``fork()`` of a warm parent, as the reference point: it is the boundary the + process backend already gives you, so a cell that costs more than a fork is + paying for isolation it does not provide; +* parallel scaling of cells against plain threads, which separates the two + things free-threading and sub-interpreters actually buy -- parallelism comes + from the free-threaded build, not from the interpreters. + +Usage:: + + python scripts/cell_cost.py + python scripts/cell_cost.py --json + python scripts/cell_cost.py --iterations 100 + +Requires CPython 3.12+ for ``_interpreters``; 3.14+ exposes the same thing as +the public ``concurrent.interpreters``. Run it on a free-threaded build to get +the parallel-scaling rows. + +On CPython 3.13 the lifecycle rows are gated behind ``--allow-unstable-api``: +holding ~20 interpreters that have imported ``http.client`` or ``email.message`` +and then destroying them aborts the process with ``munmap_chunk(): invalid +pointer``. The same workload is clean on 3.14's ``concurrent.interpreters``, +which is why the roadmap targets 3.14+ rather than the private module. +""" + +from __future__ import annotations + +import argparse +import json +import os +import statistics +import sys +import threading +import time +from typing import Callable, Optional + +# --- interpreter API shim ------------------------------------------------- +# +# 3.14 promoted the private ``_interpreters`` module to ``concurrent.interpreters`` +# with an object API. Support both so the same numbers can be taken on 3.12/3.13 +# and on 3.14+, which is the comparison the roadmap decision needs. + +_IMPL: Optional[str] +try: # pragma: no cover - exercised by whichever branch the host provides + import concurrent.interpreters as _interp + + _IMPL = "concurrent.interpreters" +except ImportError: # pragma: no cover - 3.12/3.13 + try: + import _interpreters as _interp # type: ignore[no-redef] + + _IMPL = "_interpreters" + except ImportError: + _interp = None # type: ignore[assignment] + _IMPL = None + + +class Cell: + """One sub-interpreter, with create/exec/close spelled the same on 3.12+.""" + + __slots__ = ("_id", "_obj") + + def __init__(self) -> None: + if _IMPL == "concurrent.interpreters": + self._obj = _interp.create() + self._id = self._obj.id + else: + self._id = _interp.create() + self._obj = None + + def exec(self, source: str) -> None: + if self._obj is not None: + self._obj.exec(source) + else: + _interp.run_string(self._id, source) + + def close(self) -> None: + if self._obj is not None: + self._obj.close() + else: + _interp.destroy(self._id) + + +# --- helpers -------------------------------------------------------------- + +BARE = "x = 1" +SMALL = "import json, re, dataclasses" +TYPICAL = "import json, re, dataclasses, email, http.client, logging, argparse" + +SURFACES: tuple[tuple[str, str], ...] = ( + ("bare", BARE), + ("json/re/dataclasses", SMALL), + ("typical stdlib", TYPICAL), +) + +CPU_WORK = ( + "def _f(n):\n" + " s = 0\n" + " for i in range(n):\n" + " s += i * i\n" + " return s\n" + "_f(4_000_000)\n" +) + + +def rss_kib() -> int: + """Resident set size of this process in KiB, or -1 where unavailable.""" + try: + with open("/proc/self/status", encoding="utf-8") as fh: + for line in fh: + if line.startswith("VmRSS:"): + return int(line.split()[1]) + except OSError: + pass + return -1 + + +def summarize(samples: list[float]) -> dict[str, float]: + ordered = sorted(samples) + idx = min(len(ordered) - 1, int(len(ordered) * 0.95)) + return { + "median": statistics.median(ordered), + "p95": ordered[idx], + "min": ordered[0], + "max": ordered[-1], + } + + +def time_ms(fn: Callable[[], None]) -> float: + start = time.perf_counter() + fn() + return (time.perf_counter() - start) * 1e3 + + +# --- measurements --------------------------------------------------------- + + +def measure_lifecycle(iterations: int) -> list[dict[str, object]]: + """create+exec, close, and held RSS for each import surface.""" + rows: list[dict[str, object]] = [] + for label, body in SURFACES: + cells: list[Cell] = [] + create: list[float] = [] + before = rss_kib() + for _ in range(iterations): + start = time.perf_counter() + cell = Cell() + cell.exec(body) + create.append((time.perf_counter() - start) * 1e3) + cells.append(cell) + held = rss_kib() + close: list[float] = [] + for cell in cells: + close.append(time_ms(cell.close)) + rss_per_cell = (held - before) / iterations if before >= 0 and held >= 0 else -1.0 + rows.append( + { + "surface": label, + "create_exec_ms": summarize(create), + "close_ms": summarize(close), + "rss_kib_per_cell": rss_per_cell, + } + ) + return rows + + +def measure_pool_dispatch(iterations: int, pool_size: int = 16) -> dict[str, float]: + """Latency of running work on an interpreter that is already warm.""" + pool = [Cell() for _ in range(pool_size)] + for cell in pool: + cell.exec(SMALL) + samples: list[float] = [] + for i in range(iterations): + cell = pool[i % pool_size] + samples.append(time_ms(lambda: cell.exec("y = sum(range(100))"))) + for cell in pool: + cell.close() + return summarize(samples) + + +def measure_fork(iterations: int) -> Optional[dict[str, float]]: + """fork()+wait of a warm parent -- the reference boundary cost. + + Measured before any interpreter is created. ``fork()`` copies page tables, + so its cost tracks the size of the parent's address space; forking after + the lifecycle rows have churned through hundreds of interpreters measures + that churn rather than the primitive, and overstates the fork by an order + of magnitude. + """ + if not hasattr(os, "fork"): + return None + exec(SMALL, {}) # warm the parent so the fork is pure copy-on-write + samples: list[float] = [] + for _ in range(iterations): + start = time.perf_counter() + pid = os.fork() + if pid == 0: # pragma: no cover - child never returns + os._exit(0) + os.waitpid(pid, 0) + samples.append((time.perf_counter() - start) * 1e3) + return summarize(samples) + + +def measure_parallel(workers: int) -> dict[str, float]: + """Speedup of N cells and of N plain threads over running N times in a row. + + The two rows together are the point: on a free-threaded build plain threads + already scale, so sub-interpreters are not what buys parallelism. They buy a + separate ``sys.modules`` and a separate set of globals per tenant. + """ + sequential = time_ms(lambda: [exec(CPU_WORK, {}) for _ in range(workers)]) + + def run_threads() -> None: + threads = [ + threading.Thread(target=exec, args=(CPU_WORK, {})) for _ in range(workers) + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + threaded = time_ms(run_threads) + + pool = [Cell() for _ in range(workers)] + for cell in pool: + cell.exec("pass") + + def run_cells() -> None: + threads = [threading.Thread(target=c.exec, args=(CPU_WORK,)) for c in pool] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + run_cells() # warm: first exec in a cell pays for its own code objects + celled = time_ms(run_cells) + for cell in pool: + cell.close() + + return { + "workers": float(workers), + "sequential_ms": sequential, + "threads_ms": threaded, + "threads_speedup": sequential / threaded if threaded else 0.0, + "cells_ms": celled, + "cells_speedup": sequential / celled if celled else 0.0, + } + + +# --- reporting ------------------------------------------------------------ + + +#: The lifecycle rows destroy many interpreters that have imported a realistic +#: module surface. That is exactly the pattern that aborts the process on the +#: private 3.13 API, so it is opt-in below 3.14 rather than a crash by default. +STABLE_INTERPRETERS_API = (3, 14) + + +def lifecycle_is_safe() -> bool: + return sys.version_info[:2] >= STABLE_INTERPRETERS_API + + +def build_report( + iterations: int, workers: int, *, allow_unstable: bool = False +) -> dict[str, object]: + # Order matters: fork() is measured first, on a parent that has not yet + # created any interpreter, so it reports the primitive rather than the + # address-space growth the later rows cause. See measure_fork. + fork = measure_fork(iterations) + run_lifecycle = lifecycle_is_safe() or allow_unstable + return { + "python": sys.version.split()[0], + "free_threaded": not getattr(sys, "_is_gil_enabled", lambda: True)(), + "interpreters_api": _IMPL, + "iterations": iterations, + "fork_ms": fork, + "lifecycle": measure_lifecycle(iterations) if run_lifecycle else None, + "pool_dispatch_ms": measure_pool_dispatch(iterations), + "parallel": measure_parallel(workers), + } + + +def print_report(report: dict[str, object]) -> None: + gil = "free-threaded" if report["free_threaded"] else "GIL enabled" + print( + f"PyIsolate cell cost python={report['python']} ({gil}) " + f"api={report['interpreters_api']} n={report['iterations']}\n" + ) + + lifecycle = report["lifecycle"] + if lifecycle is None: + print( + "lifecycle rows skipped: destroying interpreters that imported a real\n" + "module surface aborts the process on this build's private API.\n" + f"Re-run on CPython {STABLE_INTERPRETERS_API[0]}.{STABLE_INTERPRETERS_API[1]}+, " + "or pass --allow-unstable-api to measure anyway." + ) + else: + print(f"{'import surface':<22}{'create+exec':>14}{'close':>10}{'RSS/cell':>12}") + print(f"{'':<22}{'p50 ms':>14}{'p50 ms':>10}{'KiB':>12}") + for row in lifecycle: # type: ignore[union-attr] + create = row["create_exec_ms"]["median"] # type: ignore[index] + close = row["close_ms"]["median"] # type: ignore[index] + rss = row["rss_kib_per_cell"] # type: ignore[index] + print(f"{row['surface']:<22}{create:>14.2f}{close:>10.2f}{rss:>12.0f}") + + pool = report["pool_dispatch_ms"] + print(f"\n{'dispatch onto warm cell':<22}{pool['median']:>14.3f}{'':>10}{'':>12}") + fork = report["fork_ms"] + if fork is not None: + print(f"{'fork() of warm parent':<22}{fork['median']:>14.3f}") + + par = report["parallel"] + print( + f"\nparallel scaling at {int(par['workers'])} workers: " + f"threads {par['threads_speedup']:.2f}x cells {par['cells_speedup']:.2f}x" + ) + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--iterations", + type=int, + default=50, + help="samples per measurement (default: 50)", + ) + parser.add_argument( + "--workers", + type=int, + default=min(4, os.cpu_count() or 1), + help="concurrent workers for the scaling rows (default: min(4, cpus))", + ) + parser.add_argument("--json", action="store_true", help="emit JSON instead of a table") + parser.add_argument( + "--allow-unstable-api", + action="store_true", + help=( + "run the lifecycle rows on a pre-3.14 build, where destroying " + "interpreters that imported http.client or email.message is known " + "to abort the process" + ), + ) + args = parser.parse_args(argv) + + if _IMPL is None: + print( + "no interpreters API on this build: needs CPython 3.12+ for " + "_interpreters, or 3.14+ for concurrent.interpreters", + file=sys.stderr, + ) + return 2 + + report = build_report( + args.iterations, args.workers, allow_unstable=args.allow_unstable_api + ) + if args.json: + print(json.dumps(report, indent=2)) + else: + print_report(report) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_cell_cost.py b/tests/test_cell_cost.py new file mode 100644 index 0000000..f9e0fcf --- /dev/null +++ b/tests/test_cell_cost.py @@ -0,0 +1,124 @@ +"""Tests for scripts/cell_cost.py. + +The measurements themselves are wall-clock dependent and are not asserted on. +What is worth locking down is the reporting contract the README quotes from, +and the version guard: the lifecycle rows destroy interpreters that have +imported a real module surface, which aborts the process on CPython 3.13's +private ``_interpreters``, so they must stay opt-in below 3.14. +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + + +def _load_cell_cost(): + spec = importlib.util.spec_from_file_location( + "pyisolate_cell_cost", ROOT / "scripts" / "cell_cost.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_summarize_reports_expected_statistics(): + cc = _load_cell_cost() + summary = cc.summarize([10.0, 20.0, 30.0, 40.0]) + assert summary["min"] == 10.0 + assert summary["max"] == 40.0 + assert summary["median"] == 25.0 + assert summary["p95"] >= summary["median"] + + +def test_exposes_expected_entry_points(): + cc = _load_cell_cost() + for name in ( + "measure_lifecycle", + "measure_pool_dispatch", + "measure_fork", + "measure_parallel", + "build_report", + "main", + ): + assert callable(getattr(cc, name)), name + + +def test_lifecycle_gate_tracks_the_stable_api_version(): + """Below 3.14 the lifecycle rows must not run unless explicitly allowed.""" + cc = _load_cell_cost() + assert cc.STABLE_INTERPRETERS_API == (3, 14) + expected = sys.version_info[:2] >= cc.STABLE_INTERPRETERS_API + assert cc.lifecycle_is_safe() is expected + + +def test_build_report_skips_lifecycle_on_an_unstable_api(monkeypatch): + """The guard, not the Python actually running the tests, decides.""" + cc = _load_cell_cost() + monkeypatch.setattr(cc, "lifecycle_is_safe", lambda: False) + monkeypatch.setattr(cc, "measure_fork", lambda n: None) + monkeypatch.setattr(cc, "measure_pool_dispatch", lambda n: {"median": 0.0}) + monkeypatch.setattr(cc, "measure_parallel", lambda w: {"workers": float(w)}) + + def _must_not_run(_n): # pragma: no cover - the point is that it is not called + raise AssertionError("lifecycle rows ran on an unstable interpreters API") + + monkeypatch.setattr(cc, "measure_lifecycle", _must_not_run) + + report = cc.build_report(1, 1) + assert report["lifecycle"] is None + + # ...and that the opt-out is what re-enables them. + monkeypatch.setattr(cc, "measure_lifecycle", lambda n: [{"surface": "bare"}]) + forced = cc.build_report(1, 1, allow_unstable=True) + assert forced["lifecycle"] == [{"surface": "bare"}] + + +def test_report_carries_the_fields_the_readme_quotes(): + cc = _load_cell_cost() + if cc._IMPL is None: + pytest.skip("no interpreters API on this build") + report = cc.build_report(2, 2, allow_unstable=False) + for key in ( + "python", + "free_threaded", + "interpreters_api", + "fork_ms", + "lifecycle", + "pool_dispatch_ms", + "parallel", + ): + assert key in report, key + assert report["parallel"]["workers"] == 2.0 + assert report["pool_dispatch_ms"]["median"] >= 0.0 + + +def test_print_report_handles_a_skipped_lifecycle(capsys): + """The table renderer must not blow up on the guarded path.""" + cc = _load_cell_cost() + cc.print_report( + { + "python": "3.13.0", + "free_threaded": False, + "interpreters_api": "_interpreters", + "iterations": 5, + "fork_ms": {"median": 1.0, "p95": 1.0, "min": 1.0, "max": 1.0}, + "lifecycle": None, + "pool_dispatch_ms": {"median": 0.5, "p95": 0.6, "min": 0.4, "max": 0.7}, + "parallel": { + "workers": 2.0, + "sequential_ms": 2.0, + "threads_ms": 2.0, + "threads_speedup": 1.0, + "cells_ms": 1.0, + "cells_speedup": 2.0, + }, + } + ) + out = capsys.readouterr().out + assert "lifecycle rows skipped" in out + assert "--allow-unstable-api" in out