From c89c7d9842062bae13f58b35f84071d8432994ce Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Fri, 23 Oct 2026 23:37:39 -0400 Subject: [PATCH 1/5] macOS containment: run uncontained sessions and evaluations under a packaged Seatbelt profile (#292) --- CHANGELOG.md | 3 +- README.md | 10 +- docs/install.md | 6 +- docs/reference/environment.md | 5 +- pyproject.toml | 1 + src/outerloop/attempt.py | 3 +- src/outerloop/containment/__init__.py | 104 ++++++++++++++++ src/outerloop/containment/macos.sb | 17 +++ src/outerloop/harness.py | 21 +++- src/outerloop/orchestrator.py | 21 +++- src/outerloop/tick.py | 10 +- tests/conftest.py | 9 ++ tests/test_containment.py | 165 ++++++++++++++++++++++++++ 13 files changed, 355 insertions(+), 20 deletions(-) create mode 100644 src/outerloop/containment/__init__.py create mode 100644 src/outerloop/containment/macos.sb create mode 100644 tests/test_containment.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dc5443d..96ca95cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,9 @@ Versions follow [SemVer](https://semver.org). ## [Unreleased] +- On macOS without an image, sessions, evaluations and panel lenses run under a packaged Seatbelt profile (`sandbox-exec`, #292): they write only the workspace and a scratch directory, read only system and toolchain paths, and connect out only over HTTPS and DNS. Linux and Apptainer runs are unchanged. - `BENCHMARKS.md` shows `—` instead of "provenance unknown" in the main-commit column of a baseline row that nothing has improved on yet, as on a freshly seeded repository. -- The README and the getting-started guide link to the quickstart repository. It runs two CPU benchmarks with the local forge and local compute, and needs no GitHub account or cluster (#563). The docs and the startup warning no longer say an uncontained session runs in a sandbox: it gets a scrubbed environment and its own home, but can read any file the account can. +- The README and the getting-started guide link to the quickstart repository. It runs two CPU benchmarks with the local forge and local compute, and needs no GitHub account or cluster (#563). The docs and the startup warning no longer say an uncontained session runs in a sandbox: on Linux without an image, it gets a scrubbed environment and its own home, but can read any file the account can. - The advisory review covers exactly the head its triggering event named (`REVIEW_HEAD_SHA`, #462). If the PR has moved off that head, nothing is reviewed or posted. Otherwise the diff is read between fixed commits, every envelope records the reviewed `head_sha`, and the review is attached to that commit. When GitHub refuses its inline comments, the fallback is a body-only review on the same commit, never an unpinned comment (#569). The summarizer skips clean when its lenses read different heads. For a fork PR (`REVIEW_CHECKOUT_BASE=1`) the brief says the tree is the base branch and the change is only in the diff. Upgrading: envelopes without `head_sha` (from an older reviewer ref) still post; callers that do not pass REVIEW_HEAD_SHA behave as before. - When check runs cannot be read on a token deployment, the warning now says the token cannot read them (fine-grained tokens have no checks permission) instead of asking for an App permission. diff --git a/README.md b/README.md index fe6ca928..0d49ce42 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,10 @@ outerloop init --forge local --compute local --no-image --target me/quickstart \ outerloop start ``` -Without a container the author's session runs as your user and can read any file -your account can, so give it a spend-capped key. +On macOS, sessions and evaluations run under a Seatbelt profile: they write only +their workspace and scratch directory and cannot read your home. On Linux without a +container they can read any file your account can. Either way, use a spend-capped +API key. For your own repository: three commands and one file. You need a repo with a benchmark command, an API key for the model that will write the code, and a Slurm cluster or one @@ -126,8 +128,8 @@ build. See [the reviewer guide](https://docs.outerloop.science/reviewer/). output are rendered to the agent as fenced data, not instructions, which lowers the risk of injected text being followed without guaranteeing it. A contained session gets no GitHub credential in its environment and - cannot read the operator's key files; an uncontained local session runs - as your user and can read any file you can, key files included. + cannot read the operator's key files. Without a container, a macOS session + runs under Seatbelt; a Linux session can read any file you can, key files included. - **Budgets in code.** Launches, GPU-hours, and runs per week are enforced by the kernel, not left to the agent. - **No model lock-in.** Claude Code, Codex, and hermes-agent are wired diff --git a/docs/install.md b/docs/install.md index 6497846a..83ee4fb1 100644 --- a/docs/install.md +++ b/docs/install.md @@ -10,7 +10,7 @@ Outerloop is self-hosted. You run it, with your keys, on your compute, against y - `uv` on PATH, or in the directory its installer uses. Evaluations and launches run through `uv run`, and `outerloop start` stops with a message when it cannot find it. - A GitHub repository you administer. - An API key for the author's model, and separate keys for the judges. -- For contained runs, Linux with [Apptainer](https://apptainer.org). macOS has no Apptainer, so runs there stay uncontained. +- For contained runs, Linux with [Apptainer](https://apptainer.org). macOS uses Seatbelt through `/usr/bin/sandbox-exec` without a VM. The orchestrator is CPU-only and makes outbound connections only. Anywhere that can reach GitHub and your model provider works. @@ -46,7 +46,7 @@ With `OUTERLOOP_COMPUTE=local`, or with no `sbatch` on PATH, `outerloop start` r Local compute has no lanes, so GPU benchmarks need no GPU partition: evaluations and the author's launches run on the machine's own GPUs, shared first come first served. Launch arrays run their tasks in parallel as GPUs become available, and `OUTERLOOP_LOCAL_GPUS` overrides automatic detection through `nvidia-smi` (`0` disables GPU allocation). The pool also follows the loop's own `CUDA_VISIBLE_DEVICES`: start each deployment with its own list (for example `0,1` and `2,3`) to give it its own GPUs. The author's session itself starts with `CUDA_VISIBLE_DEVICES` empty, so an experiment that needs a GPU goes through the author's `launch` syscall, where it is recorded and metered against `gpu_hours_per_run`. Uncontained, that is a default the session could reset; containment is what keeps the GPU out of the session, which is one more reason to run with the image. -Without a container image, sessions and evaluations run as your account on your machine, and the loop says so once at start. A session gets a scrubbed environment and its own home directory, but it can read any file your account can, keys included, so use a spend-capped key. The verification panel is off in this mode unless you set `OUTERLOOP_PANEL_UNCONTAINED=1`, because an uncontained judge holds a shell next to its own key file. A pull request opened without a panel says so. A Codex or Hermes author needs the image in every mode. For priorities, walltime accounting, or several machines, use Slurm. +Without a container image, sessions and evaluations run as your account on your machine, and the loop says so once at start. On macOS a Seatbelt profile (`/usr/bin/sandbox-exec`) confines them: they write only the workspace and a scratch directory, read only system and toolchain paths, and connect out only over HTTPS and DNS. The profile does not open the keychain, so use an API key rather than a subscription login. On Linux they get a scrubbed environment and their own home but can read any file your account can, keys included. Use a spend-capped key. The verification panel is off in this mode unless you set `OUTERLOOP_PANEL_UNCONTAINED=1`, because an uncontained judge holds a shell next to its own key file. A pull request opened without a panel says so. A Codex or Hermes author needs the image in every mode. For priorities, walltime accounting, or several machines, use Slurm. ### One machine as its own Slurm cluster @@ -208,7 +208,7 @@ GPU jobs request at least 8 cores and 64 GB of host memory per GPU. On nodes wit ## The agent image -On a cluster, evals and launches run inside the Apptainer image at `OUTERLOOP_IMAGE` (default `~/outerloop-images/agent-py312.sif`), in a jail that binds only the checked-out tree. An eval that needs data must fetch it into the tree. The published image lives at [huggingface.co/outerloop-science/agent-image](https://huggingface.co/outerloop-science/agent-image), built from `containers/agent-py312.def`. On Linux with Apptainer installed, `outerloop init` downloads it (about 200 MB), verifies its published checksum and records it. `--image` points at your own, and `--no-image` keeps runs uncontained even when an image is on disk by writing `OUTERLOOP_IMAGE=`. +On a cluster, evals and launches run inside the Apptainer image at `OUTERLOOP_IMAGE` (default `~/outerloop-images/agent-py312.sif`), in a jail that binds only the checked-out tree. An eval that needs data must fetch it into the tree. The published image lives at [huggingface.co/outerloop-science/agent-image](https://huggingface.co/outerloop-science/agent-image), built from `containers/agent-py312.def`. On Linux with Apptainer installed, `outerloop init` downloads it (about 200 MB), verifies its published checksum and records it. `--image` points at your own, and `--no-image` skips Apptainer even when an image is on disk (macOS still uses Seatbelt when available) by writing `OUTERLOOP_IMAGE=`. Only the local loop falls back to running uncontained on its own. On Slurm the tick does not service a target without an image, and its log names the setting below, so a lost image never silently drops containment. Codex and Hermes authors need the image in every mode. diff --git a/docs/reference/environment.md b/docs/reference/environment.md index a88a069b..90e00ac2 100644 --- a/docs/reference/environment.md +++ b/docs/reference/environment.md @@ -206,8 +206,9 @@ PAT rather than a GitHub App. One identity or the other, never both. ### `OUTERLOOP_IMAGE` The Apptainer image sessions and evals run in. An empty value is the -off-switch for local Claude runs: they stay uncontained even when an image -is on disk. On Slurm the tick does not service a target without an image, +off-switch for local Claude runs: they skip Apptainer even when an image +is on disk. Without an image, macOS runs sessions and evaluations under the packaged +Seatbelt profile; Linux runs them uncontained. On Slurm the tick does not service a target without an image, and Codex and Hermes authors need one in every mode. - Default: `~/outerloop-images/agent-py312.sif` diff --git a/pyproject.toml b/pyproject.toml index 0a2fe72a..279e5a1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,7 @@ testpaths = ["tests"] # the tests affected by your edits. addopts = "-ra --strict-markers -n auto" markers = [ + "seatbelt: exercises the macOS containment wrapper (including real enforcement)", "real_locate_harness: exercises the real harness lookup (test_init fixture skips its patch)", "slow: long-running; nightly/manual only", "llm: calls a paid LLM API; manual only, never CI", diff --git a/src/outerloop/attempt.py b/src/outerloop/attempt.py index 77336110..b1f2f0c0 100644 --- a/src/outerloop/attempt.py +++ b/src/outerloop/attempt.py @@ -33,6 +33,7 @@ from outerloop.appauth import add_credential_args from outerloop.brief import BudgetState, distill_lessons from outerloop.compute import LocalCompute +from outerloop.containment import seatbelt_available from outerloop.contract import ( Benchmark, BenchmarkNotFoundError, @@ -3772,7 +3773,7 @@ def _panel_lenses_from_args( prepared.append((kind, backend, model, lens_key)) for kind, backend, model, lens_key in prepared: try: - if not args.image: + if not args.image and not seatbelt_available(): log.warning( "panel lens %s runs uncontained (no image): the judge shares " "this account with the operator's keys", diff --git a/src/outerloop/containment/__init__.py b/src/outerloop/containment/__init__.py new file mode 100644 index 00000000..379f3467 --- /dev/null +++ b/src/outerloop/containment/__init__.py @@ -0,0 +1,104 @@ +"""Seatbelt containment for launches without an Apptainer image.""" + +from __future__ import annotations + +import logging +import os +import shutil +import sys +import tempfile +from collections.abc import Iterable +from importlib.resources import files +from pathlib import Path + +SANDBOX_EXEC = Path("/usr/bin/sandbox-exec") +_warned = False + + +def seatbelt_available() -> bool: + return sys.platform == "darwin" and SANDBOX_EXEC.is_file() + + +def executable_roots(*executables: str) -> list[Path]: + """Include launch prefixes as well as symlink targets (venvs need both).""" + roots = {Path(sys.prefix).resolve(), Path(sys.base_prefix).resolve()} + for executable in (*executables, sys.executable, "uv", "python", "python3", "node"): + located = shutil.which(executable) + if not located: + continue + for path in (Path(located).absolute(), Path(located).resolve()): + directory = path.parent + prefix = directory.parent + needs_prefix = directory.name == "bin" and ( + path.name.startswith("python") + or path.name == "node" + or (prefix / "pyvenv.cfg").is_file() + or (prefix / "package.json").is_file() + ) + roots.add(prefix if needs_prefix else directory) + uv_python = Path( + os.environ.get( + "UV_PYTHON_INSTALL_DIR", + str( + Path(os.environ.get("XDG_DATA_HOME", str(Path.home() / ".local/share"))) + / "uv/python" + ), + ) + ) + if uv_python.is_dir(): + roots.add(uv_python.resolve()) + # a tool installed straight under home (~/bin/python) must not open all of home + home = Path.home().resolve() + return sorted(root for root in roots if root not in (home, *home.parents)) + + +def seatbelt_wrap( + argv: list[str], + *, + workspace: Path, + scratch: Path, + read_only: Iterable[Path], + contained: bool = False, +) -> list[str]: + """Wrap an uncontained macOS launch; callers own the scratch lifetime.""" + global _warned + if contained or sys.platform != "darwin": + return argv + if not seatbelt_available(): + if not _warned: + logging.getLogger(__name__).warning( + "sandbox-exec is missing: sessions and evaluations run uncontained" + ) + _warned = True + return argv + roots = sorted({str(root.resolve()) for root in read_only}) + rules = "\n ".join(f'(subpath (param "READ_ONLY_{i}"))' for i in range(len(roots))) + profile = ( + files(__package__).joinpath("macos.sb").read_text().replace("; READ_ONLY_ROOTS", rules) + ) + # A fresh file avoids following anything left by a previous session. + fd, name = tempfile.mkstemp(prefix="seatbelt-", suffix=".sb", dir=scratch) + with os.fdopen(fd, "w") as stream: + stream.write(profile) + parameters = {"WORKSPACE": str(workspace.resolve()), "SCRATCH": str(scratch.resolve())} + parameters.update({f"READ_ONLY_{i}": root for i, root in enumerate(roots)}) + flags = [flag for key, value in parameters.items() for flag in ("-D", f"{key}={value}")] + return [str(SANDBOX_EXEC), *flags, "-f", name, *argv] + + +def session_wrap( + argv: list[str], + workspace: Path, + scratch: Path, + env: dict[str, str], + read_only: Iterable[Path] = (), +) -> list[str]: + if sys.platform == "darwin": + env["TMPDIR"] = str(scratch.resolve()) + env["CLAUDE_CODE_TMPDIR"] = str(scratch.resolve()) + return seatbelt_wrap( + argv, + workspace=workspace, + scratch=scratch, + read_only=[*executable_roots(argv[0]), *read_only], + ) diff --git a/src/outerloop/containment/macos.sb b/src/outerloop/containment/macos.sb new file mode 100644 index 00000000..1862c4a6 --- /dev/null +++ b/src/outerloop/containment/macos.sb @@ -0,0 +1,17 @@ +(version 1) +(deny default) +(import "system.sb") +(allow process-fork process-exec signal) +(allow sysctl-read mach-lookup ipc-posix-shm ipc-posix-sem system-socket iokit-open) +(allow file-read-metadata) +(allow file-read* + (subpath "/usr") (subpath "/bin") (subpath "/sbin") (subpath "/System") + (subpath "/Library") (subpath "/opt/homebrew") (subpath "/private/etc") + (subpath "/private/var/db") (subpath "/dev") + ; READ_ONLY_ROOTS +) +(allow file-read* file-write* + (subpath (param "WORKSPACE")) (subpath (param "SCRATCH")) + (literal "/dev/null") (literal "/dev/tty") (subpath "/private/var/folders")) +(allow network-outbound (remote tcp "*:443") (remote udp "*:53") + (literal "/private/var/run/mDNSResponder")) diff --git a/src/outerloop/harness.py b/src/outerloop/harness.py index 149207c2..8d90b6a7 100644 --- a/src/outerloop/harness.py +++ b/src/outerloop/harness.py @@ -27,6 +27,7 @@ from pathlib import Path from typing import Any, Protocol +from outerloop.containment import executable_roots, session_wrap from outerloop.endpoint_wait import session_url as endpoint_session_url from outerloop.endpoints import EndpointProfile from outerloop.hermes_install import hermes_ready, hermes_runtime @@ -47,10 +48,8 @@ # What a session's environment contains — nothing else survives from the # parent. HOME is deliberately NOT inherited: it is redirected to a per-run # directory so a session cannot read key files under the real home or poison -# other runs via ~/.claude state. (Residual risk: the filesystem itself is -# not sandboxed — same-user absolute paths remain readable. See the threat -# model; the bot PAT must not live on the account that runs sessions until -# OS-level sandboxing lands.) +# other runs via ~/.claude state. Without an image, macOS uses Seatbelt; +# Linux still permits reads of same-user absolute paths. SESSION_ENV_ALLOWLIST = ("PATH", "TERM", "LANG", "LC_ALL", "TMPDIR") # Fallbacks only — every orchestrated path threads effective_limits in @@ -798,6 +797,8 @@ def run( if self.vertex is not None and self.vertex.adc_file: # Also mounted when an endpoint overrides Vertex authentication. provision_credential_file(Path(self.vertex.adc_file)) + if not self.container_image: + command = session_wrap(command, workspace, session_home, env) process = subprocess.Popen( command, cwd=workspace, @@ -1416,6 +1417,14 @@ def run( env[f"APPTAINERENV_{key_env}"] = self.api_key env["APPTAINERENV_CODEX_HOME"] = env["CODEX_HOME"] provision_credential_file(session_home / ".codex" / "auth.json") + if not self.container_image: + command = session_wrap( + command, + workspace, + session_home, + env, + [Path(__file__).parent, *executable_roots(self.binary)], + ) process = subprocess.Popen( command, cwd=workspace, @@ -1788,6 +1797,10 @@ def run( # cwd is the per-run home, NOT the workspace: --save_sample writes # its trajectory JSON to cwd, and artifacts must never land in the # clone (they would enter the diff). + if not self.container_image: + command = session_wrap( + command, workspace, session_home, env, [repo, Path(__file__).parent] + ) process = subprocess.Popen( command, cwd=session_home, diff --git a/src/outerloop/orchestrator.py b/src/outerloop/orchestrator.py index a8d8abec..7904df7f 100644 --- a/src/outerloop/orchestrator.py +++ b/src/outerloop/orchestrator.py @@ -27,6 +27,7 @@ from secrets import randbits from typing import TYPE_CHECKING, Any, Literal, Protocol +from outerloop.containment import executable_roots, seatbelt_available, seatbelt_wrap from outerloop.image import apptainer_from_env if TYPE_CHECKING: @@ -258,7 +259,12 @@ def evaluate( # bound into the container — never the whole host /tmp. try: cache_dir = Path( - tempfile.mkdtemp(prefix="uv-cache-", dir=os.environ.get("TMPDIR", "/tmp")) + tempfile.mkdtemp( + prefix="uv-cache-", + dir=eval_home + if not self.container_image and seatbelt_available() + else os.environ.get("TMPDIR", "/tmp"), + ) ) except OSError as exc: import shutil @@ -331,7 +337,16 @@ def _run( command, ] else: - argv = ["sh", "-c", command] + import shlex + + argv = seatbelt_wrap( + ["sh", "-c", command], + workspace=workspace, + scratch=eval_home, + read_only=executable_roots(*shlex.split(command)[:1]) + if seatbelt_available() + else [], + ) env = {k: os.environ[k] for k in ("PATH", "LANG", "TMPDIR") if k in os.environ} # uv's cache does heavy small-file IO; on shared filesystems (NFS) # that flakes with stale-handle/copy errors. Keep the cache on @@ -352,6 +367,8 @@ def _run( env["APPTAINERENV_UV_LINK_MODE"] = "copy" env["APPTAINERENV_UV_PROJECT_ENVIRONMENT"] = env["UV_PROJECT_ENVIRONMENT"] env["HOME"] = str(eval_home) + if not self.container_image and seatbelt_available(): + env["TMPDIR"] = str(eval_home) if extra_env: # explicit injections only (the base env is a scrubbed # allowlist): today this carries the benchmark's run seed. diff --git a/src/outerloop/tick.py b/src/outerloop/tick.py index 65983b93..0e2b69ca 100644 --- a/src/outerloop/tick.py +++ b/src/outerloop/tick.py @@ -42,6 +42,7 @@ local_mode, quote_command, ) +from outerloop.containment import seatbelt_available from outerloop.disk import DEFAULT_MIN_FREE_BYTES, check_disk from outerloop.endpoint_wait import EndpointWaitReason from outerloop.forge import ( @@ -3744,12 +3745,15 @@ def _service_spec_from_env( if not _UNCONTAINED_WARNED: _UNCONTAINED_WARNED = True log.warning( - "%s; sessions and evaluations run as this account and can read its " - "files; the panel is %s; a codex or hermes author needs the " - "image (docs/install.md, running without a container)", + "%s; sessions and evaluations %s; the panel is %s; " + "a codex or hermes author needs the image " + "(docs/install.md, running without a container)", "OUTERLOOP_UNCONTAINED=1: no container" if uncontained else f"local mode: no container image at {image}", + "use macOS Seatbelt (workspace, scratch and runtime reads; HTTPS/DNS egress)" + if seatbelt_available() + else "run as this account and can read its files", "on by OUTERLOOP_PANEL_UNCONTAINED=1" if os.environ.get("OUTERLOOP_PANEL_UNCONTAINED") == "1" else "off (OUTERLOOP_PANEL_UNCONTAINED=1 turns it on)", diff --git a/tests/conftest.py b/tests/conftest.py index 9b4eda5e..dbc73ac2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -110,3 +110,12 @@ def codex_host(tmp_path, monkeypatch): host.chmod(0o755) monkeypatch.setattr(CodexHarness, "_code_mode_host", lambda self: host) return host + + +@pytest.fixture(autouse=True) +def _mock_seatbelt_for_unit_tests(request, monkeypatch): + """Fake-process tests inspect bare argv; policy tests exercise the OS wrapper.""" + if request.node.get_closest_marker("seatbelt"): + return + monkeypatch.setattr("outerloop.orchestrator.seatbelt_wrap", lambda argv, **kwargs: argv) + monkeypatch.setattr("outerloop.harness.session_wrap", lambda argv, *args, **kwargs: argv) diff --git a/tests/test_containment.py b/tests/test_containment.py new file mode 100644 index 00000000..dd20f1ec --- /dev/null +++ b/tests/test_containment.py @@ -0,0 +1,165 @@ +"""Packaged policy, launch wiring, and real macOS enforcement.""" + +import subprocess +import sys +import tempfile +from importlib.resources import files +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from outerloop import containment +from outerloop.harness import ClaudeCodeHarness +from outerloop.orchestrator import SubprocessEvaluator + +pytestmark = pytest.mark.seatbelt + + +def test_packaged_profile(): + assert "(deny default)" in files("outerloop.containment").joinpath("macos.sb").read_text() + + +@pytest.mark.parametrize(("platform", "contained"), [("linux", False), ("darwin", True)]) +def test_no_wrap(platform, contained, tmp_path, monkeypatch): + monkeypatch.setattr(sys, "platform", platform) + argv = ["python", "-V"] + assert ( + containment.seatbelt_wrap( + argv, workspace=tmp_path, scratch=tmp_path, read_only=[], contained=contained + ) + is argv + ) + + +def test_wrap_parameters(tmp_path, monkeypatch): + monkeypatch.setattr(sys, "platform", "darwin") + binary = tmp_path / "sandbox-exec" + binary.touch() + monkeypatch.setattr(containment, "SANDBOX_EXEC", binary) + real = tmp_path / 'runtime "quoted"' + real.mkdir() + link = tmp_path / "link" + link.symlink_to(real) + argv = containment.seatbelt_wrap( + ["python", "-V"], workspace=tmp_path, scratch=tmp_path, read_only=[link] + ) + assert argv[0] == str(binary) + assert argv[-2:] == ["python", "-V"] + assert f"READ_ONLY_0={real.resolve()}" in argv + profile = Path(argv[argv.index("-f") + 1]).read_text() + assert '(param "READ_ONLY_0")' in profile + assert str(real) not in profile + + +def test_missing_warns_once(tmp_path, monkeypatch, caplog): + monkeypatch.setattr(sys, "platform", "darwin") + monkeypatch.setattr(containment, "SANDBOX_EXEC", tmp_path / "missing") + monkeypatch.setattr(containment, "_warned", False) + for _ in range(2): + assert containment.seatbelt_wrap( + ["true"], workspace=tmp_path, scratch=tmp_path, read_only=[] + ) == ["true"] + assert caplog.text.count("sandbox-exec is missing") == 1 + + +def test_executable_roots(tmp_path, monkeypatch): + prefix = tmp_path / "python-install" + (prefix / "bin").mkdir(parents=True) + python = prefix / "bin/python" + python.touch(mode=0o755) + venv = tmp_path / "venv" + (venv / "bin").mkdir(parents=True) + (venv / "bin/python").symlink_to(python) + home = tmp_path / "home" + (home / "bin").mkdir(parents=True) + (home / "bin/python").touch(mode=0o755) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("UV_PYTHON_INSTALL_DIR", str(prefix)) + roots = containment.executable_roots(str(venv / "bin/python"), str(home / "bin/python")) + assert home.resolve() not in roots + assert prefix.resolve() in roots + assert venv.resolve() in roots + + +@pytest.mark.parametrize("contained", [False, True]) +def test_eval_wrap(tmp_path, monkeypatch, contained): + wrap = Mock(return_value=["wrapped"]) + monkeypatch.setattr("outerloop.orchestrator.seatbelt_wrap", wrap) + process = Mock(returncode=0) + process.communicate.return_value = ('{"score": 1}', "") + popen = Mock(return_value=process) + monkeypatch.setattr("outerloop.orchestrator.subprocess.Popen", popen) + workspace = tmp_path / "workspace" + workspace.mkdir() + assert ( + SubprocessEvaluator(container_image="image" if contained else "").evaluate( + workspace, "python -m eval", "score" + ) + == 1 + ) + assert wrap.call_count == (0 if contained else 1) + if not contained: + assert popen.call_args.args[0] == ["wrapped"] + assert wrap.call_args.kwargs["workspace"] == workspace + + +@pytest.mark.parametrize("contained", [False, True]) +def test_session_wrap(tmp_path, monkeypatch, contained): + wrap = Mock(return_value=["wrapped"]) + monkeypatch.setattr("outerloop.harness.session_wrap", wrap) + process = Mock(returncode=0) + process.communicate.return_value = ('{"result": "done", "is_error": false}', "") + popen = Mock(return_value=process) + monkeypatch.setattr("outerloop.harness.subprocess.Popen", popen) + workspace = tmp_path / "workspace" + workspace.mkdir() + ClaudeCodeHarness( + api_key="test-key", binary="/usr/bin/true", container_image="image" if contained else "" + ).run("task", workspace) + assert wrap.call_count == (0 if contained else 1) + if not contained: + assert popen.call_args.args[0] == ["wrapped"] + assert wrap.call_args.args[1] == workspace + + +@pytest.mark.skipif(not containment.seatbelt_available(), reason="requires macOS sandbox-exec") +def test_real_seatbelt(): + # /var/folders is allowed by the policy, so the denied fixture must be elsewhere. + with tempfile.TemporaryDirectory(dir="/private/tmp", prefix="seatbelt-test-") as directory: + root = Path(directory) + workspace, scratch, home = (root / name for name in ("workspace", "scratch", "home")) + for path in (workspace, scratch, home): + path.mkdir() + secret = home / "secret" + secret.write_text("private") + code = ( + "from pathlib import Path\n" + f'Path({str(workspace / "written")!r}).write_text("ok")\n' + "try:\n" + f" Path({str(secret)!r}).read_text()\n" + "except PermissionError:\n" + ' print("denied")\n' + "else:\n" + ' raise AssertionError("private read allowed")\n' + ) + argv = containment.seatbelt_wrap( + [sys.executable, "-c", code], + workspace=workspace, + scratch=scratch, + read_only=containment.executable_roots(sys.executable), + ) + result = subprocess.run(argv, capture_output=True, text=True, cwd=workspace) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "denied" + assert (workspace / "written").read_text() == "ok" + + +def test_session_scratch_environment(tmp_path, monkeypatch): + monkeypatch.setattr(sys, "platform", "darwin") + wrap = Mock(return_value=["wrapped"]) + monkeypatch.setattr(containment, "seatbelt_wrap", wrap) + env = {"TMPDIR": "/tmp", "CLAUDE_CODE_TMPDIR": "/tmp"} + assert containment.session_wrap(["claude"], tmp_path, tmp_path, env) == ["wrapped"] + assert env["TMPDIR"] == str(tmp_path.resolve()) + assert env["CLAUDE_CODE_TMPDIR"] == str(tmp_path.resolve()) From 72ad0d49afa62b8dcb7dd99f47b84cc4619900ac Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Fri, 23 Oct 2026 23:49:41 -0400 Subject: [PATCH 2/5] Seatbelt review fixes: only the per-user cache dir under /private/var/folders, profile passed inline, local endpoints and the Vertex ADC file readable, roots filtered after resolving --- CHANGELOG.md | 2 +- README.md | 8 ++-- docs/install.md | 2 +- src/outerloop/containment/__init__.py | 53 +++++++++++++++++++++------ src/outerloop/containment/macos.sb | 5 ++- src/outerloop/harness.py | 5 ++- tests/test_containment.py | 6 +-- 7 files changed, 58 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96ca95cb..af078c3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Versions follow [SemVer](https://semver.org). ## [Unreleased] -- On macOS without an image, sessions, evaluations and panel lenses run under a packaged Seatbelt profile (`sandbox-exec`, #292): they write only the workspace and a scratch directory, read only system and toolchain paths, and connect out only over HTTPS and DNS. Linux and Apptainer runs are unchanged. +- On macOS without an image, sessions, evaluations and panel lenses run under a packaged Seatbelt profile (`sandbox-exec`, #292): they write only the workspace and a scratch directory, read only system and toolchain paths, and connect out only to TCP port 443, DNS and local ports. Linux and Apptainer runs are unchanged. - `BENCHMARKS.md` shows `—` instead of "provenance unknown" in the main-commit column of a baseline row that nothing has improved on yet, as on a freshly seeded repository. - The README and the getting-started guide link to the quickstart repository. It runs two CPU benchmarks with the local forge and local compute, and needs no GitHub account or cluster (#563). The docs and the startup warning no longer say an uncontained session runs in a sandbox: on Linux without an image, it gets a scrubbed environment and its own home, but can read any file the account can. - The advisory review covers exactly the head its triggering event named (`REVIEW_HEAD_SHA`, #462). If the PR has moved off that head, nothing is reviewed or posted. Otherwise the diff is read between fixed commits, every envelope records the reviewed `head_sha`, and the review is attached to that commit. When GitHub refuses its inline comments, the fallback is a body-only review on the same commit, never an unpinned comment (#569). The summarizer skips clean when its lenses read different heads. For a fork PR (`REVIEW_CHECKOUT_BASE=1`) the brief says the tree is the base branch and the change is only in the diff. Upgrading: envelopes without `head_sha` (from an older reviewer ref) still post; callers that do not pass REVIEW_HEAD_SHA behave as before. diff --git a/README.md b/README.md index 0d49ce42..96803978 100644 --- a/README.md +++ b/README.md @@ -41,10 +41,10 @@ outerloop init --forge local --compute local --no-image --target me/quickstart \ outerloop start ``` -On macOS, sessions and evaluations run under a Seatbelt profile: they write only -their workspace and scratch directory and cannot read your home. On Linux without a -container they can read any file your account can. Either way, use a spend-capped -API key. +On macOS, sessions and evaluations run under a Seatbelt profile: they write their +workspace and a scratch directory, read system and toolchain paths, and cannot read +the rest of your home. On Linux without a container they can read any file your +account can. Either way, use a spend-capped API key. For your own repository: three commands and one file. You need a repo with a benchmark command, an API key for the model that will write the code, and a Slurm cluster or one diff --git a/docs/install.md b/docs/install.md index 83ee4fb1..81c1dc3d 100644 --- a/docs/install.md +++ b/docs/install.md @@ -46,7 +46,7 @@ With `OUTERLOOP_COMPUTE=local`, or with no `sbatch` on PATH, `outerloop start` r Local compute has no lanes, so GPU benchmarks need no GPU partition: evaluations and the author's launches run on the machine's own GPUs, shared first come first served. Launch arrays run their tasks in parallel as GPUs become available, and `OUTERLOOP_LOCAL_GPUS` overrides automatic detection through `nvidia-smi` (`0` disables GPU allocation). The pool also follows the loop's own `CUDA_VISIBLE_DEVICES`: start each deployment with its own list (for example `0,1` and `2,3`) to give it its own GPUs. The author's session itself starts with `CUDA_VISIBLE_DEVICES` empty, so an experiment that needs a GPU goes through the author's `launch` syscall, where it is recorded and metered against `gpu_hours_per_run`. Uncontained, that is a default the session could reset; containment is what keeps the GPU out of the session, which is one more reason to run with the image. -Without a container image, sessions and evaluations run as your account on your machine, and the loop says so once at start. On macOS a Seatbelt profile (`/usr/bin/sandbox-exec`) confines them: they write only the workspace and a scratch directory, read only system and toolchain paths, and connect out only over HTTPS and DNS. The profile does not open the keychain, so use an API key rather than a subscription login. On Linux they get a scrubbed environment and their own home but can read any file your account can, keys included. Use a spend-capped key. The verification panel is off in this mode unless you set `OUTERLOOP_PANEL_UNCONTAINED=1`, because an uncontained judge holds a shell next to its own key file. A pull request opened without a panel says so. A Codex or Hermes author needs the image in every mode. For priorities, walltime accounting, or several machines, use Slurm. +Without a container image, sessions and evaluations run as your account on your machine, and the loop says so once at start. On macOS a Seatbelt profile (`/usr/bin/sandbox-exec`) confines them: they write only the workspace and a scratch directory, read only system and toolchain paths, and connect out only to TCP port 443, DNS and local ports (a local model endpoint). If `sandbox-exec` is missing, the loop warns and runs them uncontained. The profile does not open the keychain, so use an API key rather than a subscription login. On Linux they get a scrubbed environment and their own home but can read any file your account can, keys included. Use a spend-capped key. The verification panel is off in this mode unless you set `OUTERLOOP_PANEL_UNCONTAINED=1`, because an uncontained judge holds a shell next to its own key file. A pull request opened without a panel says so. A Codex or Hermes author needs the image in every mode. For priorities, walltime accounting, or several machines, use Slurm. ### One machine as its own Slurm cluster diff --git a/src/outerloop/containment/__init__.py b/src/outerloop/containment/__init__.py index 379f3467..ec13211c 100644 --- a/src/outerloop/containment/__init__.py +++ b/src/outerloop/containment/__init__.py @@ -2,11 +2,12 @@ from __future__ import annotations +import functools import logging import os import shutil +import subprocess import sys -import tempfile from collections.abc import Iterable from importlib.resources import files from pathlib import Path @@ -47,9 +48,30 @@ def executable_roots(*executables: str) -> list[Path]: ) if uv_python.is_dir(): roots.add(uv_python.resolve()) - # a tool installed straight under home (~/bin/python) must not open all of home + return _outside_home(roots) + + +def _outside_home(paths: Iterable[Path]) -> list[Path]: + """A tool installed straight under home (~/bin/python) must not open all of home.""" home = Path.home().resolve() - return sorted(root for root in roots if root not in (home, *home.parents)) + resolved = {path.resolve() for path in paths} + return sorted(path for path in resolved if path not in (home, *home.parents)) + + +@functools.cache +def _user_cache_dir() -> str: + """Per-user cache dir under /private/var/folders (the CLI's keychain access + writes there); the rest of /private/var/folders holds other runs' temp files.""" + try: + out = subprocess.run( + ["/usr/bin/getconf", "DARWIN_USER_CACHE_DIR"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + return str(Path(out).resolve()) if out else "/nonexistent" + except (OSError, subprocess.CalledProcessError): + return "/nonexistent" def seatbelt_wrap( @@ -58,6 +80,7 @@ def seatbelt_wrap( workspace: Path, scratch: Path, read_only: Iterable[Path], + read_files: Iterable[Path] = (), contained: bool = False, ) -> list[str]: """Wrap an uncontained macOS launch; callers own the scratch lifetime.""" @@ -71,19 +94,25 @@ def seatbelt_wrap( ) _warned = True return argv - roots = sorted({str(root.resolve()) for root in read_only}) - rules = "\n ".join(f'(subpath (param "READ_ONLY_{i}"))' for i in range(len(roots))) + roots = [str(root) for root in _outside_home(read_only)] + singles = sorted({str(path.resolve()) for path in read_files}) + rules = "\n ".join( + [f'(subpath (param "READ_ONLY_{i}"))' for i in range(len(roots))] + + [f'(literal (param "READ_FILE_{i}"))' for i in range(len(singles))] + ) profile = ( files(__package__).joinpath("macos.sb").read_text().replace("; READ_ONLY_ROOTS", rules) ) - # A fresh file avoids following anything left by a previous session. - fd, name = tempfile.mkstemp(prefix="seatbelt-", suffix=".sb", dir=scratch) - with os.fdopen(fd, "w") as stream: - stream.write(profile) - parameters = {"WORKSPACE": str(workspace.resolve()), "SCRATCH": str(scratch.resolve())} + parameters = { + "WORKSPACE": str(workspace.resolve()), + "SCRATCH": str(scratch.resolve()), + "USER_CACHE": _user_cache_dir(), + } parameters.update({f"READ_ONLY_{i}": root for i, root in enumerate(roots)}) + parameters.update({f"READ_FILE_{i}": path for i, path in enumerate(singles)}) flags = [flag for key, value in parameters.items() for flag in ("-D", f"{key}={value}")] - return [str(SANDBOX_EXEC), *flags, "-f", name, *argv] + # the profile travels in argv, so no file the sandboxed process can write + return [str(SANDBOX_EXEC), *flags, "-p", profile, *argv] def session_wrap( @@ -92,6 +121,7 @@ def session_wrap( scratch: Path, env: dict[str, str], read_only: Iterable[Path] = (), + read_files: Iterable[Path] = (), ) -> list[str]: if sys.platform == "darwin": env["TMPDIR"] = str(scratch.resolve()) @@ -101,4 +131,5 @@ def session_wrap( workspace=workspace, scratch=scratch, read_only=[*executable_roots(argv[0]), *read_only], + read_files=read_files, ) diff --git a/src/outerloop/containment/macos.sb b/src/outerloop/containment/macos.sb index 1862c4a6..89700041 100644 --- a/src/outerloop/containment/macos.sb +++ b/src/outerloop/containment/macos.sb @@ -12,6 +12,7 @@ ) (allow file-read* file-write* (subpath (param "WORKSPACE")) (subpath (param "SCRATCH")) - (literal "/dev/null") (literal "/dev/tty") (subpath "/private/var/folders")) + (literal "/dev/null") (literal "/dev/tty") (subpath (param "USER_CACHE"))) +; ports, not protocols: TCP 443 anywhere, DNS, and local endpoints (allow network-outbound (remote tcp "*:443") (remote udp "*:53") - (literal "/private/var/run/mDNSResponder")) + (remote tcp "localhost:*") (literal "/private/var/run/mDNSResponder")) diff --git a/src/outerloop/harness.py b/src/outerloop/harness.py index 8d90b6a7..52df6d7b 100644 --- a/src/outerloop/harness.py +++ b/src/outerloop/harness.py @@ -798,7 +798,10 @@ def run( # Also mounted when an endpoint overrides Vertex authentication. provision_credential_file(Path(self.vertex.adc_file)) if not self.container_image: - command = session_wrap(command, workspace, session_home, env) + adc = self.vertex.adc_file if self.vertex is not None else "" + command = session_wrap( + command, workspace, session_home, env, read_files=[Path(adc)] if adc else [] + ) process = subprocess.Popen( command, cwd=workspace, diff --git a/tests/test_containment.py b/tests/test_containment.py index dd20f1ec..53bf9dba 100644 --- a/tests/test_containment.py +++ b/tests/test_containment.py @@ -47,7 +47,7 @@ def test_wrap_parameters(tmp_path, monkeypatch): assert argv[0] == str(binary) assert argv[-2:] == ["python", "-V"] assert f"READ_ONLY_0={real.resolve()}" in argv - profile = Path(argv[argv.index("-f") + 1]).read_text() + profile = argv[argv.index("-p") + 1] assert '(param "READ_ONLY_0")' in profile assert str(real) not in profile @@ -125,8 +125,8 @@ def test_session_wrap(tmp_path, monkeypatch, contained): @pytest.mark.skipif(not containment.seatbelt_available(), reason="requires macOS sandbox-exec") def test_real_seatbelt(): - # /var/folders is allowed by the policy, so the denied fixture must be elsewhere. - with tempfile.TemporaryDirectory(dir="/private/tmp", prefix="seatbelt-test-") as directory: + # the default temp dir sits under /private/var/folders, beside other runs' scratch + with tempfile.TemporaryDirectory(prefix="seatbelt-test-") as directory: root = Path(directory) workspace, scratch, home = (root / name for name in ("workspace", "scratch", "home")) for path in (workspace, scratch, home): From f1cd7893056c2924385e48a898168ddc994c732c Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 24 Oct 2026 00:01:30 -0400 Subject: [PATCH 3/5] macOS install hint: runs without an image use the Seatbelt profile --- src/outerloop/image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/outerloop/image.py b/src/outerloop/image.py index 8325eaec..9525a2a9 100644 --- a/src/outerloop/image.py +++ b/src/outerloop/image.py @@ -80,8 +80,8 @@ def install_hint() -> str: is why init never runs it.""" if sys.platform == "darwin": return ( - "macOS has no Apptainer; runs stay uncontained here (a macOS containment is on the\n" - " roadmap). A Linux machine, or Slurm, runs contained." + "macOS has no Apptainer; runs without an image use the kernel's Seatbelt profile\n" + " (sandbox-exec) instead. A Linux machine, or Slurm, runs in the image." ) dist, _version = _linux_flavor() apparmor_note = ( From b9a05225c56e49dfc246183d575b37ab715837b4 Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 24 Oct 2026 00:16:26 -0400 Subject: [PATCH 4/5] Seatbelt: validation checks keep their uv cache in the scratch home --- src/outerloop/orchestrator.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/outerloop/orchestrator.py b/src/outerloop/orchestrator.py index 7904df7f..6951164c 100644 --- a/src/outerloop/orchestrator.py +++ b/src/outerloop/orchestrator.py @@ -431,7 +431,13 @@ def check(self, workspace: Path, command: str) -> None: ) except OSError as exc: raise EvalError(f"could not create check home: {exc}") from exc - cache_dir = Path(tempfile.mkdtemp(prefix="outerloop-check-cache-")) + cache_dir = Path( + tempfile.mkdtemp( + prefix="outerloop-check-cache-", + # under Seatbelt the scratch home is the only writable place + dir=eval_home if not self.container_image and seatbelt_available() else None, + ) + ) try: self._run(workspace, command, eval_home, cache_dir) finally: From 7fdf773f8a5167dfcf99d31861c2fa6b0baf4d0b Mon Sep 17 00:00:00 2001 From: Mengye Ren Date: Sat, 24 Oct 2026 00:31:23 -0400 Subject: [PATCH 5/5] Seatbelt eval: locate the tool by a plain split so a malformed command fails as an eval error; test the check() cache --- src/outerloop/orchestrator.py | 8 +++----- tests/test_containment.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/outerloop/orchestrator.py b/src/outerloop/orchestrator.py index 6951164c..b05bed87 100644 --- a/src/outerloop/orchestrator.py +++ b/src/outerloop/orchestrator.py @@ -337,15 +337,13 @@ def _run( command, ] else: - import shlex - + # the first word locates the tool; a plain split never raises on a + # malformed command, which sh then reports as a failed eval argv = seatbelt_wrap( ["sh", "-c", command], workspace=workspace, scratch=eval_home, - read_only=executable_roots(*shlex.split(command)[:1]) - if seatbelt_available() - else [], + read_only=executable_roots(*command.split()[:1]) if seatbelt_available() else [], ) env = {k: os.environ[k] for k in ("PATH", "LANG", "TMPDIR") if k in os.environ} # uv's cache does heavy small-file IO; on shared filesystems (NFS) diff --git a/tests/test_containment.py b/tests/test_containment.py index 53bf9dba..f978ac2f 100644 --- a/tests/test_containment.py +++ b/tests/test_containment.py @@ -104,6 +104,21 @@ def test_eval_wrap(tmp_path, monkeypatch, contained): assert wrap.call_args.kwargs["workspace"] == workspace +def test_check_cache_inside_scratch_and_malformed_command(tmp_path, monkeypatch): + wrap = Mock(return_value=["wrapped"]) + monkeypatch.setattr("outerloop.orchestrator.seatbelt_wrap", wrap) + monkeypatch.setattr("outerloop.orchestrator.seatbelt_available", lambda: True) + process = Mock(returncode=0) + process.communicate.return_value = ("", "") + popen = Mock(return_value=process) + monkeypatch.setattr("outerloop.orchestrator.subprocess.Popen", popen) + workspace = tmp_path / "workspace" + workspace.mkdir() + SubprocessEvaluator().check(workspace, "python -c 'unterminated") + scratch = wrap.call_args.kwargs["scratch"] + assert Path(popen.call_args.kwargs["env"]["UV_CACHE_DIR"]).is_relative_to(scratch) + + @pytest.mark.parametrize("contained", [False, True]) def test_session_wrap(tmp_path, monkeypatch, contained): wrap = Mock(return_value=["wrapped"])