diff --git a/contrib/frontierharness/README.md b/contrib/frontierharness/README.md new file mode 100644 index 000000000..759514592 --- /dev/null +++ b/contrib/frontierharness/README.md @@ -0,0 +1,77 @@ +# FrontierHarness Eval adapter + +Runs anton on [FrontierHarness Eval](https://github.com/frontier-harness-eval/eval) v1.0: 21 +Terminal-Bench tasks through Harbor and 9 DeepSWE tasks through Pier, with the model fixed at +Kimi K3. The eval's own scripts provision the runtime, run the tasks and score them; this folder +plugs anton into them as a custom agent. + +| File | Role | +| --- | --- | +| `anton_harbor.py`, `anton_pier.py` | Harbor and Pier agent classes | +| `anton_common.py` | What both share: install, run, token accounting | +| `install.sh` | The eval's `--install-script`: checks both runners can load the adapter | +| `run-task.sh` | The eval's `--cmd`: runs one task through Harbor or Pier | +| `eval-anton-usage.patch` | Teaches the eval's cost script to read anton's per-call usage | + +## Run it with the eval's workflow on Runta + +From a checkout of the eval at `e837a70bd6beb4e72eeeda62dd06e3bd34f6cb63`, with `ANTON` pointing +at an anton checkout and `SHA` the anton commit to evaluate: + +```bash +git apply "$ANTON/contrib/frontierharness/eval-anton-usage.patch" +runta secret set MINDS_API_KEY --prompt +FH=skills/frontierharness-eval/scripts + +bash $FH/provision-golden-checkpoint.sh \ + --runtime fh-build --checkpoint fh-golden-anton-$SHA --keep-runtime \ + --harness anton --repo https://github.com/mindsdb/anton --commit $SHA \ + --provider custom --model mindshub/kimi-k3 \ + --secret-name MINDS_API_KEY --secret-host api.mindshub.ai \ + --cpus 4 --memory 8192 --disk-size-gib 50 \ + --install-script "$ANTON/contrib/frontierharness/install.sh" + +bash $FH/run-trials.sh \ + --checkpoint fh-golden-anton-$SHA --harness anton --run-id anton-$SHA --out runs \ + --provider custom --model mindshub/kimi-k3 \ + --secret-name MINDS_API_KEY --secret-host api.mindshub.ai \ + --cmd '/work/harness/contrib/frontierharness/run-task.sh {suite} {task} {model} {jobs}' + +node $FH/normalize-results.mjs --run runs/anton-$SHA --label "anton $SHA" +node $FH/generate-chart.mjs --run runs/anton-$SHA +node $FH/build-report.mjs --run runs/anton-$SHA +``` + +`--model mindshub/kimi-k3` only labels the run in reports and pricing. The adapter asks the +MindsHub gateway for `kimi-k3`, which it serves with Fireworks Kimi K3, the provider of the +published runs. + +To call Fireworks directly instead, store the key as `FIREWORKS_API_KEY` and provision a new +checkpoint with `--provider fireworks` in place of `--provider custom --model ... --secret-name +... --secret-host ...` (explicit flags override the preset, and the run must use the +checkpoint's provider). Run the trials the same way, with the `--cmd` prefixed by +`ANTON_FH_BASE_URL=https://api.fireworks.ai/inference/v1 ANTON_FH_MODEL=accounts/fireworks/models/kimi-k3`. +That route has not been run yet: MindsHub adjusts some tool schemas for Kimi K3 that raw +Fireworks may reject. + +`run-task.sh` also runs on a laptop with Docker, Harbor 0.22.0 and Pier 0.3.1 installed as uv +tools: set `ANTON_FH_API_KEY` and `DEEP_SWE` (a deep-swe checkout at +`435ee89ec2f2e2289f33b0da4f992f0b7b7266b9`). + +## What the adapter does + +- Installs anton inside each task container from the GitHub tarball of the commit + `run-task.sh` is checked out at, with uv's Python 3.12. +- Sends the task instruction as one non-interactive turn of `python -m anton.cloud_turn`, so + anton never waits for a user. +- Turns off Jev, analytics and auto-updates, so every model call goes to Kimi K3. +- Keeps anton's `.anton/` folder out of git while the agent works and deletes it after, so it + never reaches the graded diff. +- Exits 1 when the turn does not end in `turn_completed`; Harbor retries that twice, as for the + published harnesses, and Pier does not retry. +- Reads token counts from anton's `llm_usage` log lines, including calls made from its + scratchpad, and counts calls served by another model in `non_k3_calls`. + +The adapter has run on local Docker but not on Runta yet. The first Runta run should confirm +that anton's Python trusts Runta's egress CA inside task containers, that the stub key is +swapped for the real one on the gateway host, and that Pier lets the agent reach that host. diff --git a/contrib/frontierharness/anton_common.py b/contrib/frontierharness/anton_common.py new file mode 100644 index 000000000..71d0b14e2 --- /dev/null +++ b/contrib/frontierharness/anton_common.py @@ -0,0 +1,180 @@ +"""Anton adapter logic shared by the Harbor and Pier agent classes; run-task.sh sets its env.""" + +from __future__ import annotations + +import json +import os +import shlex +import uuid +from pathlib import Path + +# The benchmark fixes the model; the MindsHub alias serves Fireworks Kimi K3. +BENCHMARK_MODEL = "kimi-k3" + +EVENTS_FILE = "anton-events.jsonl" +STDERR_FILE = "anton-stderr.log" +USAGE_PREFIX = "llm_usage " + + +def _required_env(name: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + raise RuntimeError(f"{name} is not set; run-task.sh sets it") + return value + + +def anton_ref() -> str: + return _required_env("ANTON_FH_REF") + + +def base_url() -> str: + return _required_env("ANTON_FH_BASE_URL") + + +def install_script(ref: str) -> str: + # The GitHub tarball needs no git in the container, and uv brings its own + # Python, so the task image's Python version does not matter. + source = f"anton-agent @ https://github.com/mindsdb/anton/archive/{ref}.tar.gz" + return ( + "set -euo pipefail; " + 'export PATH="$HOME/.local/bin:$PATH"; ' + "command -v uv >/dev/null 2>&1 || curl -LsSf https://astral.sh/uv/install.sh | sh; " + f"uv tool install --force --python 3.12 {shlex.quote(source)}; " + '"$(uv tool dir)/anton-agent/bin/python" -c "import anton.cloud_turn"' + ) + + +def anton_env(api_key: str | None) -> dict[str, str]: + if not api_key: + raise RuntimeError("ANTON_FH_API_KEY is not set; run-task.sh sets it") + # Fireworks directly names it accounts/fireworks/models/kimi-k3. + model = os.environ.get("ANTON_FH_MODEL", "").strip() or BENCHMARK_MODEL + return { + "ANTON_PLANNING_PROVIDER": "openai-compatible", + "ANTON_CODING_PROVIDER": "openai-compatible", + "ANTON_PLANNING_MODEL": model, + "ANTON_CODING_MODEL": model, + "ANTON_OPENAI_BASE_URL": base_url(), + "ANTON_OPENAI_API_KEY": api_key, + "ANTON_USAGE_LOG": "1", + # Artifacts would otherwise land under the workspace and in the graded diff. + "ANTON_CLOUD_ARTIFACTS_ROOT": "/tmp/anton-artifacts", + # Every model call must use the benchmark model, so no Jev calls. + "ANTON_VERIFIER_JEV": "off", + "ANTON_DECISIONS_MODEL": "off", + "ANTON_BROWSER_JEV": "off", + "ANTON_ANALYTICS_ENABLED": "false", + "ANTON_POSTHOG_KEY": "", + "ANTON_DISABLE_AUTOUPDATES": "true", + } + + +def run_script(instruction: str, logs_dir: str) -> str: + request = json.dumps({ + "protocol_version": 1, + "conversation_id": f"fh-{uuid.uuid4().hex}", + "input": instruction, + # On api.mindshub.ai anton would otherwise ask Jev, a non-benchmark model. + "jev_decisions": False, + }) + events = f"{logs_dir}/{EVENTS_FILE}" + stderr = f"{logs_dir}/{STDERR_FILE}" + # Anton keeps state and scratchpad venvs in /.anton. Git-exclude it + # while the agent works (tasks ask it to commit everything), then remove it + # before the verifier looks at the workspace. + return f"""set -uo pipefail +export PATH="$HOME/.local/bin:$PATH" +ws="$(pwd)" +mkdir -p {shlex.quote(logs_dir)} +printf '%s\\n' {shlex.quote(request)} > /tmp/anton-request.json +excl="$ws/.git/info/exclude" +if [ -d "$ws/.git" ]; then + mkdir -p "$ws/.git/info" + [ -f "$excl" ] && cp "$excl" /tmp/anton-exclude.bak + printf '%s\\n' '.anton/' >> "$excl" +fi +echo "anton workspace: $ws" > {shlex.quote(stderr)} +ANTON_CLOUD_WORKSPACE_PATH="$ws" "$(uv tool dir)/anton-agent/bin/python" -m anton.cloud_turn \\ + < /tmp/anton-request.json > {shlex.quote(events)} 2>> {shlex.quote(stderr)} +rc=$? +rm -rf "$ws/.anton" +if [ -d "$ws/.git" ]; then + if [ -f /tmp/anton-exclude.bak ]; then mv /tmp/anton-exclude.bak "$excl"; else rm -f "$excl"; fi +fi +tail -n 1 {shlex.quote(events)} +# cloud_turn exits 0 even when the turn fails; a failed turn is a harness error, +# so the runner treats it like the baselines' crashes and retries. +if [ "$rc" -eq 0 ] && ! tail -n 1 {shlex.quote(events)} | grep -q '"kind": *"turn_completed"'; then + rc=1 +fi +exit $rc +""" + + +def usage_records(text: str) -> list[dict]: + records = [] + for line in text.splitlines(): + if not line.startswith(USAGE_PREFIX): + continue + try: + record = json.loads(line[len(USAGE_PREFIX):]) + except json.JSONDecodeError: + continue + if isinstance(record, dict): + records.append(record) + return records + + +def usage_summary(logs_dir: Path) -> dict | None: + path = logs_dir / STDERR_FILE + if not path.exists(): + return None + records = usage_records(path.read_text(errors="replace")) + if not records: + return None + + def total(key: str) -> int: + return sum(int(r.get(key) or 0) for r in records) + + fresh, cache_read, cache_write = total("input"), total("cache_read"), total("cache_creation") + served = [str(r.get("served_model") or "unknown") for r in records] + return { + # Anton's `input` is fresh tokens only; the benchmark counts all prompt tokens. + "n_input_tokens": fresh + cache_read + cache_write, + "n_cache_tokens": cache_read, + "n_output_tokens": total("output"), + "calls": len(records), + "cache_write_tokens": cache_write, + "served_models": sorted(set(served)), + # A call with no served model is counted too: the gateway may have fallen back. + "non_k3_calls": sum(BENCHMARK_MODEL not in model for model in served), + "terminal": terminal_kind(logs_dir), + } + + +def fill_context(context, logs_dir: Path) -> None: + """Copy anton's token totals onto a Harbor or Pier AgentContext.""" + summary = usage_summary(logs_dir) + if summary is None: + return + context.n_input_tokens = summary.pop("n_input_tokens") + context.n_cache_tokens = summary.pop("n_cache_tokens") + context.n_output_tokens = summary.pop("n_output_tokens") + if hasattr(context, "n_agent_steps"): + context.n_agent_steps = summary["calls"] + context.metadata = {**(context.metadata or {}), "anton": summary} + + +def terminal_kind(logs_dir: Path) -> str | None: + path = logs_dir / EVENTS_FILE + if not path.exists(): + return None + kind = None + for line in path.read_text(errors="replace").splitlines(): + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(event, dict) and event.get("kind") in ("turn_completed", "turn_failed"): + kind = event["kind"] + return kind diff --git a/contrib/frontierharness/anton_harbor.py b/contrib/frontierharness/anton_harbor.py new file mode 100644 index 000000000..cc6b1a722 --- /dev/null +++ b/contrib/frontierharness/anton_harbor.py @@ -0,0 +1,40 @@ +"""Anton as a Harbor installed agent: `harbor run --agent anton_harbor:AntonHarbor`.""" + +from __future__ import annotations + +from harbor.agents.installed.base import BaseInstalledAgent, with_prompt_template +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext + +import anton_common + + +class AntonHarbor(BaseInstalledAgent): + @staticmethod + def name() -> str: + return "anton" + + def version(self) -> str | None: + return anton_common.anton_ref() + + async def install(self, environment: BaseEnvironment) -> None: + await self.ensure_system_dependencies(environment, ("curl",)) + await self.exec_as_agent( + environment, command=anton_common.install_script(anton_common.anton_ref()) + ) + + @with_prompt_template + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + await self.exec_as_agent( + environment, + command=anton_common.run_script(instruction, str(self.environment_logs_dir)), + env=anton_common.anton_env(self._get_env("ANTON_FH_API_KEY")), + ) + + def populate_context_post_run(self, context: AgentContext) -> None: + anton_common.fill_context(context, self.logs_dir) diff --git a/contrib/frontierharness/anton_pier.py b/contrib/frontierharness/anton_pier.py new file mode 100644 index 000000000..062ff4786 --- /dev/null +++ b/contrib/frontierharness/anton_pier.py @@ -0,0 +1,61 @@ +"""Anton as a Pier installed agent: `pier run --agent-import-path anton_pier:AntonPier`.""" + +from __future__ import annotations + +from pier.agents.installed.base import BaseInstalledAgent, with_prompt_template +from pier.agents.network import allowlist_from_urls +from pier.environments.base import BaseEnvironment +from pier.models.agent.context import AgentContext +from pier.models.agent.install import AgentInstallSpec, InstallStep +from pier.models.agent.network import NetworkAllowlist +from pier.models.trial.paths import EnvironmentPaths + +import anton_common + +_ROOT_DEPS = ( + "if command -v apt-get >/dev/null 2>&1; then apt-get update && apt-get install -y curl ca-certificates;" + " elif command -v apk >/dev/null 2>&1; then apk add --no-cache curl bash ca-certificates;" + " elif command -v dnf >/dev/null 2>&1; then dnf install -y curl;" + " elif command -v yum >/dev/null 2>&1; then yum install -y curl;" + " fi" +) + + +class AntonPier(BaseInstalledAgent): + @staticmethod + def name() -> str: + return "anton" + + def install_spec(self) -> AgentInstallSpec: + ref = anton_common.anton_ref() + return AgentInstallSpec( + agent_name=self.name(), + version=ref, + steps=[ + InstallStep(user="root", env={"DEBIAN_FRONTEND": "noninteractive"}, run=_ROOT_DEPS), + InstallStep(user="agent", run=anton_common.install_script(ref)), + ], + ) + + def network_allowlist(self) -> NetworkAllowlist: + # DeepSWE agents run with no network; the model gateway is the one exception. + return allowlist_from_urls([anton_common.base_url()]) + + def version(self) -> str | None: + return anton_common.anton_ref() + + @with_prompt_template + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + await self.exec_as_agent( + environment, + command=anton_common.run_script(instruction, str(EnvironmentPaths.agent_dir)), + env=self.build_process_env(anton_common.anton_env(self._get_env("ANTON_FH_API_KEY"))), + ) + + def populate_context_post_run(self, context: AgentContext) -> None: + anton_common.fill_context(context, self.logs_dir) diff --git a/contrib/frontierharness/eval-anton-usage.patch b/contrib/frontierharness/eval-anton-usage.patch new file mode 100644 index 000000000..255184e51 --- /dev/null +++ b/contrib/frontierharness/eval-anton-usage.patch @@ -0,0 +1,33 @@ +diff --git a/skills/frontierharness-eval/scripts/usage_details.py b/skills/frontierharness-eval/scripts/usage_details.py +index 94b4dd5..fb94a20 100644 +--- a/skills/frontierharness-eval/scripts/usage_details.py ++++ b/skills/frontierharness-eval/scripts/usage_details.py +@@ -203,6 +203,20 @@ def _hermes_totals(text: str) -> tuple[int, int, int, int] | None: + return fresh + cached, cached, cache_write, output + + ++def _anton(text: str) -> Detail: ++ # ANTON_USAGE_LOG lines on stderr: "llm_usage {json}", input is fresh-only. ++ prefix = "llm_usage " ++ records = "\n".join(line[len(prefix):] for line in text.splitlines() if line.startswith(prefix)) ++ usages = [] ++ for event in _jsonl(records): ++ fresh = int(event.get("input") or 0) ++ cached = int(event.get("cache_read") or 0) ++ cache_write = int(event.get("cache_creation") or 0) ++ if fresh or cached or cache_write or event.get("output"): ++ usages.append((fresh + cached + cache_write, cached)) ++ return _detail(usages) ++ ++ + # harness name -> (glob relative to the trial agent dir, parser) + REGISTRY: dict[str, tuple[str, Any]] = { + "codex": ("sessions/**/*.jsonl", _codex), +@@ -217,6 +231,7 @@ REGISTRY: dict[str, tuple[str, Any]] = { + "dsh-creator": ("dsh-session.jsonl", _dsh), + "kimi-code": (".kimi-code/sessions/*/*/agents/main/wire.jsonl", _kimi_code), + "hermes": ("hermes-calls.jsonl", _hermes), ++ "anton": ("anton-stderr.log", _anton), + } + + diff --git a/contrib/frontierharness/install.sh b/contrib/frontierharness/install.sh new file mode 100755 index 000000000..a42e4b958 --- /dev/null +++ b/contrib/frontierharness/install.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# The eval's --install-script: runs in /work/harness, the anton clone at the pinned commit, +# after Harbor and Pier are installed. Anton itself is installed per task, inside each task +# container, so this only checks that both runners can load the adapter. +set -euo pipefail + +here=/work/harness/contrib/frontierharness +export PATH="$HOME/.local/bin:$PATH" +[ -f "$here/run-task.sh" ] || { echo "install.sh: $here not found; is /work/harness the anton clone?" >&2; exit 1; } + +tools=$(uv tool dir) +for pair in harbor:anton_harbor datacurve-pier:anton_pier; do + tool=${pair%%:*} module=${pair#*:} + PYTHONPATH="$here" "$tools/$tool/bin/python" -c "import $module" \ + || { echo "install.sh: $tool cannot import $module" >&2; exit 1; } +done +echo "anton adapter ready at $here (anton $(git -C /work/harness rev-parse --short HEAD))" diff --git a/contrib/frontierharness/run-task.sh b/contrib/frontierharness/run-task.sh new file mode 100755 index 000000000..f3ab65921 --- /dev/null +++ b/contrib/frontierharness/run-task.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Run one FrontierHarness Eval task with anton through Harbor (Terminal-Bench) or Pier (DeepSWE). +# The eval's run-trials.sh calls it through --cmd; see README.md. +# +# Usage: run-task.sh +set -euo pipefail + +[ $# -eq 4 ] || { echo "usage: run-task.sh " >&2; exit 2; } +suite=$1 task=$2 model=$3 jobs=$4 +here=$(cd "$(dirname "$0")" && pwd) + +export PATH="$HOME/.local/bin:$PATH" +export PYTHONPATH="$here${PYTHONPATH:+:$PYTHONPATH}" +# Task containers install the same anton commit this script was checked out at. +export ANTON_FH_REF=${ANTON_FH_REF:-$(git -C "$here" rev-parse HEAD)} +export ANTON_FH_BASE_URL=${ANTON_FH_BASE_URL:-https://api.mindshub.ai/v1} +# On Runta the key is a stub that the egress proxy swaps for the stored secret. +export ANTON_FH_API_KEY=${ANTON_FH_API_KEY:-${MINDS_API_KEY:-${FIREWORKS_API_KEY:-}}} +DEEP_SWE=${DEEP_SWE:-/work/deep-swe} +# Written by the eval's provisioning on Runta, where egress TLS goes through a proxy CA. +CA_OVERLAY=${CA_OVERLAY:-/work/runta-ca-overlay.yaml} + +case "$suite" in + terminal-bench) + overlay=() + [ -f "$CA_OVERLAY" ] && overlay=(--extra-docker-compose "$CA_OVERLAY") + exec harbor run -d terminal-bench@2.0 -i "$task" -a anton_harbor:AntonHarbor -m "$model" \ + --jobs-dir "$jobs" ${overlay[@]+"${overlay[@]}"} -r 2 -y ;; + datacurve) + exec pier run -p "$DEEP_SWE/tasks/$task" --agent-import-path anton_pier:AntonPier -m "$model" \ + --jobs-dir "$jobs" -y ;; + *) + echo "run-task.sh: unknown suite $suite" >&2; exit 2 ;; +esac