diff --git a/.gitignore b/.gitignore index f530a304..90c1d7ed 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,6 @@ pyrightconfig.json .spec-workflow site/* + +# benchmark artifacts +bench/results/ diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 00000000..5825d6e1 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,64 @@ +# 压测 + +- [使用文档](../docs/mkdocs/zh/bench.md) +- [设计文档](../docs/design/01-bench-design.md) + +当前压测目标：`LlmAgent`。 + +## LlmAgent + +### 直接测试 + +在 `bench/` 目录下执行： + +```bash +python -m trpc_agent_bench.run_bench \ + --targets llmagent.base \ + --benchmarks turn1,turn5,turn10,loop1,loop5,loop10 \ + --iterations 10 \ + --results-dir results/direct +``` + +环境：16 vCPU，Python 3.12，mock 无额外延迟，`--iterations 10`。 + +| benchmark | p50 | p95 | memory | +|---|---|---|---| +| `turn1` | 25.79 ms | 26.35 ms | 341.4 KiB | +| `turn5` | 133.32 ms | 145.81 ms | 462.6 KiB | +| `turn10` | 291.23 ms | 319.68 ms | 664.9 KiB | +| `loop1` | 27.79 ms | 30.72 ms | 357.5 KiB | +| `loop5` | 138.92 ms | 147.69 ms | 452.6 KiB | +| `loop10` | 300.37 ms | 314.15 ms | 572.3 KiB | + +### 压力测试 + +```bash +python -m trpc_agent_bench.run_bench \ + --targets llmagent.base \ + --benchmarks turn1,turn5,turn10,loop1,loop5,loop10 \ + --concurrency 1,8,16,32 \ + --iterations 10 \ + --results-dir results/stress +``` + +环境：16 vCPU，Python 3.12，mock 无额外延迟，`--iterations 10`。 + +| benchmark | c1 p50 | c8 p50 | c16 p50 | c32 p50 | +|---|---|---|---|---| +| `turn1` | 27.52 ms | 196.51 ms | 400.87 ms | 772.52 ms | +| `turn5` | 134.43 ms | 1079.48 ms | 2124.66 ms | 4500.42 ms | +| `turn10` | 294.16 ms | 2317.78 ms | 4738.24 ms | 9519.07 ms | +| `loop1` | 27.61 ms | 208.85 ms | 414.94 ms | 818.57 ms | +| `loop5` | 137.59 ms | 1078.72 ms | 2115.57 ms | 4444.14 ms | +| `loop10` | 301.68 ms | 2288.40 ms | 4575.68 ms | 9370.65 ms | + +直接测试和压力测试是两次独立运行，两张表的 c1 数字会有少量抖动，但 benchmark 行和口径保持一致。 + +### 结论 + +- `turnN` 的耗时随 user turn 数近似线性增长：`turn5` 是 `turn1` 的 5.17 倍，`turn10` 是 11.29 倍；session history 变长后单轮成本略有上升。 +- `loopN` 的耗时随 LLM 调用数近似线性增长：`loop5` 是 `loop1` 的 5.00 倍，`loop10` 是 10.81 倍。`loopN` 的 `N-1` 次 tool round 没有带来明显的额外放大，主要成本还是一次 LLM round trip。 +- 同样的 5 次 / 10 次 LLM 调用下，`loop5` 比 `turn5` 高约 4.2%，`loop10` 比 `turn10` 高约 3.1%：多轮 session 和一轮 Agent 执行的总成本接近，两种形态可以互相作为对照。 +- 内存随会话/调用次数增长：`turn1` 约 341 KiB，`turn10` 约 665 KiB；`loop10` 约 572 KiB。 +- 压力测试下，`turn1` 从 c1 的 27.52 ms 增长到 c32 的 772.52 ms；`turn10` 从 294.16 ms 增长到 9519.07 ms，c32 约为 c1 的 30 倍。耗时随并发数近似线性增长，说明当前瓶颈在单进程 mock server：c8/c16/c32 的请求会在 mock 侧排队，这组数据反映的是「框架 + 单 mock server」的整体吞吐，不是纯框架的并行扩展性。 +- 并发下 turn / loop 的相对关系保持一致：`turn5` 和 `loop5`、`turn10` 和 `loop10` 的耗时都很接近，说明多轮 session 与一轮 Agent 执行的总成本仍然主要由 LLM 调用次数决定。 diff --git a/bench/llmagent/__init__.py b/bench/llmagent/__init__.py new file mode 100644 index 00000000..35d03d6a --- /dev/null +++ b/bench/llmagent/__init__.py @@ -0,0 +1 @@ +"""LlmAgent benchmark target package.""" diff --git a/bench/llmagent/base.py b/bench/llmagent/base.py new file mode 100644 index 00000000..1494021d --- /dev/null +++ b/bench/llmagent/base.py @@ -0,0 +1,148 @@ +"""Benchmark target for trpc-agent-python's LlmAgent.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any +from typing import AsyncIterator + +from trpc_agent_sdk.agents import LlmAgent +from trpc_agent_sdk.models import OpenAIModel +from trpc_agent_sdk.runners import Runner +from trpc_agent_sdk.sessions import BaseSessionService +from trpc_agent_sdk.sessions import InMemorySessionService +from trpc_agent_sdk.types import Content +from trpc_agent_sdk.types import Part + +from trpc_agent_bench.bench import AgentBenchConf +from trpc_agent_bench.bench import AgentHandle +from trpc_agent_bench.bench import AgentTarget +from trpc_agent_bench.bench import ToolsFactory +from trpc_agent_bench.bench import resolve_tools + +INSTRUCTION = "You are a deterministic benchmark agent. Answer briefly." + + +@dataclass +class PreparedLlmAgent: + model: OpenAIModel + session_service: BaseSessionService + + +class LlmAgentHandle(AgentHandle): + + def __init__( + self, + *, + agent: LlmAgent, + session_service: BaseSessionService, + ) -> None: + self.agent = agent + self.session_service = session_service + + async def run_turn( + self, + *, + app_name: str, + user_id: str, + session_id: str, + message: str, + ) -> AsyncIterator[Any]: + await self._ensure_session(app_name, user_id, session_id) + runner = Runner( + app_name=app_name, + agent=self.agent, + session_service=self.session_service, + enable_post_turn_processing=False, + close_session_service_on_close=False, + ) + try: + async for event in runner.run_async( + user_id=user_id, + session_id=session_id, + new_message=Content(parts=[Part.from_text(text=message)]), + ): + yield event + finally: + await runner.close() + + async def _ensure_session( + self, + app_name: str, + user_id: str, + session_id: str, + ) -> None: + session = await self.session_service.get_session( + app_name=app_name, + user_id=user_id, + session_id=session_id, + ) + if session is None: + await self.session_service.create_session( + app_name=app_name, + user_id=user_id, + session_id=session_id, + state={}, + ) + + +class LlmAgentTarget(AgentTarget): + name = "llm_agent" + import_module = "trpc_agent_sdk" + + def create_session_service(self) -> BaseSessionService: + return InMemorySessionService() + + async def prepare(self, *, conf: AgentBenchConf) -> PreparedLlmAgent: + if conf.base_url is None: + raise RuntimeError("AgentBenchConf.base_url is not set; call setup() first") + + session_service = self.create_session_service() + try: + model = OpenAIModel( + model_name="bench-model", + api_key="bench", + base_url=conf.base_url, + ) + except Exception: + await session_service.close() + raise + + return PreparedLlmAgent( + model=model, + session_service=session_service, + ) + + async def instantiate( + self, + prepared: PreparedLlmAgent, + *, + tools_factory: ToolsFactory, + ) -> LlmAgent: + tools = await resolve_tools(tools_factory) + return LlmAgent( + name="bench_llm_agent", + description="LlmAgent benchmark target.", + model=prepared.model, + instruction=INSTRUCTION, + tools=tools, + parallel_tool_calls=True, + ) + + async def handle( + self, + prepared: PreparedLlmAgent, + *, + tools_factory: ToolsFactory, + ) -> AgentHandle: + agent = await self.instantiate(prepared, tools_factory=tools_factory) + return LlmAgentHandle( + agent=agent, + session_service=prepared.session_service, + ) + + async def cleanup(self, prepared: PreparedLlmAgent) -> None: + await prepared.session_service.close() + + +TARGET = LlmAgentTarget() diff --git a/docs/design/01-bench-design.md b/docs/design/01-bench-design.md new file mode 100644 index 00000000..0734568f --- /dev/null +++ b/docs/design/01-bench-design.md @@ -0,0 +1,845 @@ +# trpc-agent-python 压测套件设计 + +> **结论摘要**：压测套件按 Agno `cookbook/performance/` 的思路设计，只测框架自身开销：模型调用用 Mock 替代，agent 的真实 loop 照常执行。套件是一个 `trpc_agent_bench` 通用包，顶层只放入口 `run_bench.py`；包内 `bench/` 下是公开 API 和实现（`_eval.py`、`_model.py`、`_target.py`、`_report.py`），内置 benchmark 放在 `bench/_builtin/`，子进程入口是 `bench/_runner.py`；仓库的 `bench/llmagent/` 放内置 target 适配，用户自定义的 target/benchmark 可以是任意可 import 的 module。 + +## 1. 背景与目标 + +需要量化 agent 编排层在 import、构造、单轮 run、流式、工具循环、会话持久化、内存占用上的框架开销，并支持多个 target 的横向对比。 + +直接用真实 LLM 做对比不可行： + +- 模型时延和排队时间远大于编排层开销，差异会被噪声淹没； +- 同一 prompt 的返回不稳定，模型调用次数、工具调用次数都会变化； +- 需要真实凭证和网络，CI 里不可复现。 + +Agno 的 performance 套件给了我们一个成熟模板：**在框架自己的模型调用处注入 mock，跑真实 loop，只测框架开销**。本方案照这个模板设计，更多请参考[第 4 章](#4-参考agno-的-performance-套件)。 + +### 1.1 目标 + +- **测框架开销**：模型 Mock 固定返回，真实执行 import、构造、消息拼装、tool dispatch、事件流、session 读写。 +- **可复现**：无网络、无凭证；同一脚本每次的调用次数、工具执行次数、最终文本一致。 +- **可对比**：不同 target 用同一套 benchmark、同一种 mock，输出 median/p95 和倍数。 +- **可回归**：同一套脚本可重复跑，`--iterations 10` 做调测；报告可直接看。 +- **可扩展**：新增 benchmark 或 target 只加一个文件。 + +### 1.2 非目标 + +- 不评估回答质量、token 计费准确性； +- 不压测真实模型服务（TTFT/TPOT、GPU、KV cache）； +- 不做 pass@k、flakiness 这类质量指标——我们的 workload 是确定性的； + +## 2. 方案设计 + +### 2.1 设计原则与总体架构 + +**设计原则** + +1. **只测框架开销**：模型、网络、真实 LLM 全部用 Mock 替代；报告里明确每个数字不包含 provider 服务时间。 +2. **驱动真实 loop**：不重实现 agent，只在模型调用处注入 mock；消息拼装、tool dispatch、事件流、session 读写都走真实路径。 +3. **统一 mock 路径**：所有 target 都走本机 HTTP mock，数字包含相同的 HTTP client、SSE 解析和 wire 转换开销。 +4. **确定性 + 正确性断言**：`TextBlock` / `FunctionBlock` 脚本按请求里的 message 位置无状态取响应；每次 run 校验最终文本、工具执行、错误事件；失败直接抛异常。 +5. **进程隔离**：每个 benchmark 文件单独起一个新进程；模型 server 独立进程，不计入被测进程的 CPU/RSS。 +6. **时间/内存分开测**：`tracemalloc` 会拖慢执行，timed pass 不追踪内存。 +7. **统计用 median/p95**：GC 长尾会让 mean 失真；保留原始样本。 +8. **模块依赖规则**：包内 `bench/_eval.py`、`bench/_model.py`、`bench/_target.py`、`bench/_setup.py`、`bench/_builtin/`、`bench/_report.py`、`run_bench.py` 只使用 `trpc_agent_sdk` 的公共 tool/log 类型，不 import agent/runner 实现；只有仓库的 `bench/llmagent/` 知道具体 target 的构造和运行方式。 + +**总体架构** + +```text +┌───────────────── trpc_agent_bench/run_bench.py ─────────────────────────────┐ +│ │ +│ target module paths report.py results/ │ +│ ┌────────────────┐ ┌─────────────────┐ ┌────────┐ │ +│ │ target agent A │ │ JSON + Markdown │ │ *.json │ │ +│ │ target agent B │ │ HTML + 倍数表 │ └────────┘ │ +│ └───────┬────────┘ └────────▲────────┘ │ +│ │ spawn per (target, benchmark) │ │ +│ ▼ │ │ +│ ┌──────────────────────────────┐ │ │ +│ │ fresh subprocess │ │ │ +│ │ benchmark file │ │ │ +│ │ ├─ target adapter │ │ │ +│ │ ├─ model mock │ │ │ +│ │ │ MockOpenAI server │ │ │ +│ │ ├─ PerformanceEval │ │ │ +│ │ │ warmup → runtime │ │ │ +│ │ │ → memory │ │ │ +│ │ └─ result JSON ────────────┼──────────┘ │ +│ └──────────────────────────────┘ │ +│ │ +│ 通用部分：_eval.py + _model.py + _target.py + _builtin/ + _report.py │ +│ 内置适配：仓库的 bench/llmagent/；自定义 target/benchmark：任意可 import │ +│ 的 module │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +一次完整运行的流程： + +```text +1. run_bench.py 读取 --targets 、--benchmarks、--concurrency +2. benchmark 只描述 turn script；session service 由 target 创建和清理 +3. 对每个选中的 benchmark 调 get_bench_profile()；usable=False 时打印 [skip] 并跳过 +4. 如果选中的 benchmark 有 requires_iterations=True，命令行必须提供 --iterations +5. 对 supports_concurrency=True 的 benchmark，按 --concurrency 的每个值展开 +6. 对每个 (target module, benchmark module, concurrency)： + a. spawn 一个 fresh Python 进程 + b. 进程通过 BENCH_TARGET 找到 target module，读取 module.TARGET + c. setup() 启动 MockOpenAIServer，把 base_url 写进 AgentBenchConf + d. target.prepare() 创建 session service / runtime / run_dir + e. PerformanceEval 跑 warmup → runtime → memory + f. teardown() 关 target 资源和 mock server + g. 写 results//[_cN].json +7. 收集 summary.json（machine info + 所有结果） +8. report.py 生成 target 对比表、Markdown/HTML +9. sanity 失败或 benchmark 崩溃 → run_bench.py 退出非 0 +``` + +**包结构与调用链** + +```text +trpc-agent-python/ +├── trpc_agent_bench/ # 通用层：不 import agent/runner 实现 +│ ├── __init__.py # 只导出 run_bench +│ ├── run_bench.py # 唯一入口：子进程编排 +│ ├── bench/ # 公开 API + 性能压测实现 +│ │ ├── __init__.py # 公开导出 +│ │ ├── _config.py # AgentBenchConf / ModelConf +│ │ ├── _eval.py # PerformanceEval、统计、断言、run_benchmarks +│ │ ├── _model.py # TextBlock/FunctionBlock、内部 MockOpenAIServer +│ │ ├── _profile.py # BenchProfile +│ │ ├── _setup.py # setup / teardown / BenchSetup +│ │ ├── _target.py # AgentTarget / AgentHandle、load_target() +│ │ ├── _report.py # JSON / Markdown / HTML +│ │ ├── _runner.py # 单 benchmark 子进程入口 +│ │ └── _builtin/ # 内置 benchmark +│ │ └── _utils/ # session helper、命名 helper +│ └── eval/ # 效果评测占位，当前只有 __init__.py +├── bench/ +│ ├── README.md # 目标、命令、结果与结论 +│ ├── llmagent/ # 内置 target 模块 +│ │ ├── __init__.py +│ │ └── base.py # LlmAgentTarget +│ └── results/ # gitignored +└── tests/ + └── trpc_agent_bench/ + ├── test_run_bench.py + └── bench/ +``` + +包内 `bench/_builtin/` 一文件一项，`bench/_runner.py` 按 module 路径加载并调用 `main()`；仓库的 `bench/llmagent/` 放内置 target module。`bench/` 下的实现模块统一用 `_` 前缀，公开名字只从 `bench/__init__.py` 导出。 + +```text +bench/_builtin/.py + │ load_target("") # 只知道 AgentTarget / AgentHandle + ▼ +.py # 提供 module.TARGET + │ LlmAgent + Runner + ▼ +AgentHandle.run_turn(...) ──► events +``` + +benchmark 文件只通过 `AgentHandle.run_turn()` 发起对话，不直接依赖某个 Agent 实现；target module 把具体 Agent 接成这个接口，并管理 session service 的生命周期。 + +模块依赖规则用一条测试守住： + +| 路径 | 允许 import | 说明 | +|---|---|---| +| `bench/_eval.py` / `bench/_target.py` / `bench/_report.py` / `run_bench.py` | stdlib + `trpc_agent_sdk` 公共 tool 类型 | 不依赖 agent/runner 实现 | +| `bench/_model.py` | 同上 + fastapi/uvicorn | 本地 mock server | +| `bench/_setup.py` | 同上 | setup/teardown 与 mock server 生命周期 | +| `bench/_builtin/` | 以上模块 + stdlib | 不依赖 agent/runner 实现 | +| 仓库 `bench/llmagent/` | `trpc_agent_sdk` 的 agents/models/runners/sessions | 只有这一层会构造和运行具体 Agent | + +扫描整个 `trpc_agent_bench/` 的 import，不应该出现具体 agent/runner 的实现 import；agent 实现只放在 `bench/llmagent/`。这样通用层不会意外耦合到具体 target 的实现。 + +### 2.2 模型 Mock 与 Target 抽象 + +**模型 Mock** + +所有 target 统一走内部 `MockOpenAIServer`：本地起一个 OpenAI 兼容 server，target 通过自己的 OpenAI client 访问。这样 v1/v2 以及后续接入的 agent 都走同一条 HTTP、SSE 和 wire 转换路径，数字可比；同时不访问外网，也不包含真实模型推理。 + +`MockOpenAIServer` 不是公开 API，由 `setup()` 创建和启动，固定监听 `127.0.0.1`，端口随机。启动后把地址写进 `AgentBenchConf.base_url`，target 从 `conf.base_url` 读取。 + +模型返回内容用 `TextBlock` / `FunctionBlock` 脚本： + +```python +@dataclass(frozen=True) +class TextBlock: + text: str = "ok" + chunks: int = 1 + +@dataclass(frozen=True) +class FunctionBlock: + name: str + args: dict[str, Any] +``` + +```python +Reply = TextBlock | list[FunctionBlock] +Turn = list[Reply] +``` + +脚本结构是 `list[Turn]`，每个 `Turn` 是 `list[Reply]`。一个 `Reply` 要么是一个 `TextBlock`，要么是一组 `FunctionBlock`。 + +- `TextBlock` 表示一次文本响应，一个 reply 只有一个 `TextBlock`；`chunks` 控制拆成几个 stream chunk +- `FunctionBlock` 表示模型发起一次 tool call；一个 reply 里的多个 `FunctionBlock` 表示并行调用 +- `FunctionResponseBlock` 不存在；tool response 由 agent 真正执行工具后产生 + +`setup(turns, conf)` 把脚本序列化给 mock server。server 根据请求里的 user/assistant message 数量计算： + +```text +turn = user message 数量 +reply = 当前 turn 里 assistant message 数量 +``` + +然后返回 `turns[turn - 1][reply]`。这个推导是无状态的，并发 session 各自按自己的 messages 取响应，不共享游标。 + +`TextBlock` 转成文本 chunk；`FunctionBlock` 列表转成 OpenAI 的 `tool_calls`。`MockOpenAIServer` 的职责： + +```text +POST /v1/chat/completions stream=true/false；SSE；tool_calls deltas；usage +GET /v1/models client 探测模型/context length 时返回 +GET /health 就绪检查 +``` + +实现要点： + +- 用 FastAPI + uvicorn（仓库已有依赖）；每个 benchmark 在测量前启动，测量结束后停止； +- 独立子进程运行，端口随机；benchmark 进程只通过 `base_url` 连接； +- `ModelConf.request_delay_ms` / `chunk_delay_ms` 控制 mock 响应节奏； +- 脚本由 `setup(turns)` 提供，server 负责序列化成 OpenAI 的 wire format。 + +模型延迟通过 `AgentBenchConf.model_conf` 配置： + +```python +from trpc_agent_bench.bench import AgentBenchConf, ModelConf, TextBlock, setup + +conf = AgentBenchConf( + model_conf=ModelConf(request_delay_ms=50, chunk_delay_ms=10), +) + +box = await setup( + [[TextBlock("hello", chunks=3)]], + conf=conf, +) +``` + +`request_delay_ms` 在流式和非流式 chat 请求上都生效；`chunk_delay_ms` 只在多个 stream chunk 之间生效。 + +**Target 协议与 adapter 骨架** + +```python +# trpc_agent_bench/bench/_target.py +class AgentTarget(ABC): + name: str + import_module: str # target 的 import 路径 + + @abstractmethod + async def prepare(self, *, conf: AgentBenchConf) -> Any: ... + @abstractmethod + async def instantiate(self, prepared: Any, *, tools_factory: ToolsFactory) -> Any: ... + @abstractmethod + async def handle(self, prepared: Any, *, tools_factory: ToolsFactory) -> AgentHandle: ... + @abstractmethod + async def cleanup(self, prepared: Any) -> None: ... + +class AgentHandle(ABC): + @abstractmethod + async def run_turn( + self, + *, + app_name: str, + user_id: str, + session_id: str, + message: str, + ) -> AsyncIterator[Event]: ... +``` + +`AgentTarget` 和 `AgentHandle` 都是 ABC，具体实现必须显式继承并实现各自的抽象方法。 + +`prepare()` 从 `conf.base_url` 读取 mock server 地址，创建 session service、agent runtime 和 target 自己的临时目录；`instantiate()` 才构造 agent，这样 `instantiate_agent.py` 的计时只包含 agent 构造。 + +`AgentBenchConf` 的字段： + +```python +@dataclass +class AgentBenchConf: + app_name: str = "bench_app" + user_id: str = "bench_user" + agent_conf: Any = None + model_conf: ModelConf = field(default_factory=ModelConf) + base_url: str | None = field(default=None, repr=False) +``` + +`agent_conf` 是 target 自己的配置，类型和字段由 target 定义，压测套件不解析。当前 `LlmAgent` target 没有额外配置；后续 target 可以把 instruction、model 参数、目录等放进自己的 conf 类型，通过 `conf.agent_conf` 传入。 + +**session service 归 target** + +target 在 `prepare()` 里创建 session service，在 `cleanup()` 里关闭。benchmark 不创建 session service，也不管理它的生命周期。 + +当前只有 `LlmAgent` target，使用 `InMemorySessionService`： + +```python +# bench/llmagent/base.py +class LlmAgentTarget(AgentTarget): + name = "llm_agent" + import_module = "trpc_agent_sdk" + + def create_session_service(self) -> BaseSessionService: + return InMemorySessionService() +``` + +在 `bench/` 目录下通过 `--targets llmagent.base` 选择该 target；benchmark 仍然是 `turn1` / `turn5` / `turn10` 和 `loop1` / `loop5` / `loop10`。 + +**setup / teardown** + +```text +setup(turns, conf) + ├─ MockOpenAIServer(turns, conf.model_conf).start() + ├─ conf.base_url = base_url + ├─ await target.prepare(conf=conf) + └─ BenchSetup(target, prepared, conf, _server) + +teardown(box) + ├─ await target.cleanup(prepared) + └─ box._server.stop() +``` + +`target.cleanup()` 负责关闭 session service、target runtime 和 target 的 `run_dir`。Runner 在每次 turn 结束时关闭。 + +### 2.3 压测项与断言 + +**压测项清单** + +核心 benchmark： + +| 文件 | 操作 | 脚本 | 断言 | 主要指标 | +|---|---|---|---|---| +| `import_time.py` | 新进程 import 被测 Agent 包 | — | import 成功 | 减掉只启动解释器时间后的 median/p95 | +| `instantiate_agent.py` | 构造不带工具的 agent，不发起对话 | — | 返回对象不为空 | 每次构造耗时 + 内存峰值 | +| `instantiate_agent_with_tools.py` | 构造带 5 个工具的 agent，不发起对话 | — | 返回对象不为空 | 每次构造耗时 + 内存峰值 | +| `turn1.py` | 1 个 user turn，1 次 LLM 调用，文本 1 chunk；target 决定 session service；`--concurrency` 控制并发数 | `TEXT` | 每次采样正常完成 | 会话耗时、内存峰值 | +| `turn5.py` | 5 个 user turn，每个 turn 1 次 LLM 调用，文本 1 chunk；target 决定 session service；支持并发 | `TEXT ×5` | 每次采样正常完成 | 会话耗时、内存峰值 | +| `turn10.py` | 10 个 user turn，每个 turn 1 次 LLM 调用，文本 1 chunk；target 决定 session service；支持并发 | `TEXT ×10` | 每次采样正常完成 | 会话耗时、内存峰值 | +| `loop1.py` | 1 个 user turn，1 次 LLM 调用，最终文本 5 chunks，无 tool round | `TextBlock(chunks=5)` | 正常完成 | 会话耗时、内存峰值 | +| `loop5.py` | 1 个 user turn，5 次 LLM 调用，4 次 tool round，最终文本 5 chunks | `FC ×4, TEXT(chunks=5)` | 每次采样正常完成、tool 成功 | 会话耗时、内存峰值 | +| `loop10.py` | 1 个 user turn，10 次 LLM 调用，9 次 tool round，最终文本 5 chunks | `FC ×9, TEXT(chunks=5)` | 每次采样正常完成、tool 成功 | 会话耗时、内存峰值 | +| `tools.py` | 一轮 Agent 执行（1 次 tool round） | `FunctionBlock -> TextBlock` | 工具执行成功且无 error、最终文本 == "ok" | median/p95、内存峰值 | +| `memory_footprint.py` | 批量创建不带工具 / 带工具的 agent 后一直留在内存里 | — | 构造出的 agent 数量正确 | 每个常驻 agent 占用的内存 | + +`turnN` 按 user turn 数命名：`turn1` / `turn5` / `turn10`，每个 user turn 只调一次 LLM，文本固定 1 chunk，用来测 session 历史增长和多轮调用开销。`loopN` 按一轮 Agent 执行内的 LLM 调用数命名：`loop1` / `loop5` / `loop10`，除最后一次外每次都返回 function call，所以有 `N-1` 次 tool round，最终文本固定 5 chunks，用来测一轮 Agent 执行内 tool round 次数增加时的开销。默认集包含 `turn1` / `turn5` / `turn10`、`loop1` / `loop5` / `loop10` 以及 import/构造/tools/memory 等 12 项。session service 由 target 创建和清理，不由 benchmark 管理。`--concurrency` 只对 `get_bench_profile().supports_concurrency` 为 `True` 的 benchmark 生效；并发大于 1 时结果名追加 `_cN`，HTML 中显示 `concurrency=N`。每个并发 session 使用独立的 `user_id`（`_0`、`_1` 等），避免共享 session 状态。 + +扩展 benchmark： + +| 文件 | 操作 | 想验证 | +|---|---|---| +| `tools_parallel.py` | 一个响应里 M 个 tool call | 并行工具调度 | + +扩展项不再单独分组，内置 benchmark 按固定顺序展示；自定义 benchmark 排在最后，按名称自然排序。 + +**benchmark 钩子** + +每个 benchmark module 可以选择提供一个 `get_bench_profile()`： + +```python +from trpc_agent_bench.bench import BenchProfile + + +def get_bench_profile() -> BenchProfile: + """Describe availability, concurrency, and iteration requirements.""" +``` + +没写时按默认 profile 处理： + +```python +BenchProfile( + usable=True, + supports_concurrency=False, + requires_iterations=True, + skip_reason="", +) +``` + +`run_bench` 在启动子进程前调用 `get_bench_profile()`。`usable=False` 时打印 `[skip] ()` 并跳过该 benchmark，继续跑其他项；`supports_concurrency=True` 时，`--concurrency 1,8` 会为每个值启动独立子进程，并发大于 1 的结果名追加 `_cN`；为 `False` 时只跑一次。`requires_iterations=True` 时，选中的 benchmark 会要求命令行提供 `--iterations`；`import_time` / `memory_footprint` 这类不需要 `--iterations` 的 benchmark 返回 `requires_iterations=False`，因此只选它们时可以省略 `--iterations`。普通 benchmark（`tools` 等）不需要写这个方法。 + +**一个 benchmark 文件长什么样** + +```python +# my_benches/my_benchmark.py +import uuid + +from trpc_agent_bench.bench import ( + PerformanceEval, + TextBlock, + ensure_turn_completed, + get_iterations, + run_benchmarks, + setup, + teardown, +) + + +TURN_SCRIPT = [[TextBlock()]] + + +async def main() -> int: + box = await setup(TURN_SCRIPT) + handle = await box.target.handle(box.prepared, tools_factory=lambda: []) + + async def run_once() -> None: + events = [ + event async for event in handle.run_turn( + app_name=box.conf.app_name, + user_id=box.conf.user_id, + session_id=str(uuid.uuid4()), + message="Say ok.", + ) + ] + ensure_turn_completed(events, expected_text="ok") + + try: + await run_benchmarks( + [ + PerformanceEval( + name="my_benchmark", + func=run_once, + num_iterations=get_iterations(), + warmup_runs=1, + ), + ], + ) + finally: + await teardown(box) + return 0 +``` + +`run_benchmarks` 在调用方的事件循环里跑异步 eval；runner 也支持 `async def main()`。 + +**benchmark 说明（DESCRIPTION）** + +每个 benchmark module 顶层定义一个 `DESCRIPTION` 字符串，写清楚两件事： + +1. 这个 benchmark 在压什么场景； +2. 报告里的 p50/p95（以及 memory 行）分别代表什么。 + +```python +# turn5.py +DESCRIPTION = ( + "5 个 user turn，每个 turn 1 次 LLM 调用，每个 turn 的文本固定 1 chunk。" + "指标：p50/p95 = 整个会话的耗时；memory = 本次采样中 tracemalloc 记录的峰值。" +) +``` + +数据流： + +```text +benchmark module.DESCRIPTION + → runner.load_benchmark() 写入 BENCH_DESCRIPTION + → write_result() 写入 result JSON 的 description 字段 + → report 的 Benchmark 说明 bullet list 展示 +``` + +`PerformanceEval` 也支持逐项 `description=` 覆盖，给同一 module 里多个结果用。自定义 benchmark 没有 `DESCRIPTION` 时，report 显示 `-`，不影响运行。 + +**正确性断言** + +```python +def ensure_turn_completed(events, *, expected_text=None, expect_tool_success=False): + if any(e.is_error() for e in events): + raise RuntimeError("turn returned an error event") + if not any(e.turn_complete for e in events): + raise RuntimeError("turn did not complete") + final = "".join( + part.text + for e in events + if not e.partial and e.content is not None + for part in e.content.parts + if part.text + ) + if expected_text is not None and final != expected_text: + raise RuntimeError(f"unexpected final text: {final!r}") + if expect_tool_success: + calls = [ + part.function_call + for e in events + if e.content is not None + for part in e.content.parts + if part.function_call + ] + results = [ + part.function_response + for e in events + if e.content is not None + for part in e.content.parts + if part.function_response + ] + if not calls or not results: + raise RuntimeError("tool loop did not execute") + if any(isinstance(r.response, dict) and r.response.get("error") for r in results): + raise RuntimeError("tool call failed") +``` + +断言成本是纳秒级，相对几百微秒到毫秒的 run 可以忽略；但能防止“错误路径被当成正常结果”。 + +### 2.4 测量、统计与报告 + +**测量口径** + +`PerformanceEval` 的执行顺序： + +```text +PerformanceEval.run() + ├─ warmup_runs = 10 直接 func()，不计入统计 + ├─ runtime pass N 次，Timer 只包住 func() + ├─ memory baseline 空函数跑 3 次 tracemalloc，取平均 + ├─ memory pass N 次，tracemalloc peak - baseline + └─ PerformanceResult + run_times / memory_usages + avg / min / max / std / median / p95 +``` + +要点： + +- **时间/内存分开跑**：`tracemalloc` 会拖慢执行，timed iteration 不追踪内存； +- **warmup 丢弃**：首次 import、首次建连、首次初始化不算； +- **median/p95 优先**：GC 长尾会让 mean 失真； +- **原始样本保留**：JSON 里存 `run_times` / `memory_usages`，报告可以重算。 + +import time： + +```text +for sample in range(SAMPLES): + t = subprocess.run([python, "-c", "import "]).wall_time +baseline = median(subprocess.run([python, "-c", "pass"]).wall_time) +adjusted = [max(0, t - baseline) for t in samples] +``` + +每次都是 fresh 进程；`-X importtime` 的 top 模块另存，方便定位是哪一段依赖变重了。 + +memory footprint： + +```text +per_agent = (tracemalloc_after - tracemalloc_before) / batch_size +``` + +批量创建 N 个 agent 后一直留在内存里，不释放；测的是每个常驻 agent 占用的内存，不是构造过程中的瞬时峰值。构造峰值由 `instantiate_agent.py` 的 memory pass 覆盖。 + +统计口径： + +| 指标 | 定义 | +|---|---| +| `median_run_time` | 排序后的中位数 | +| `p95_run_time` | inclusive quantile，95th | +| `avg/min/max/std` | 辅助诊断 | +| `median_memory_usage` | 每次 run 的 peak 中位数 | +| `run_times` / `memory_usages` | 原始样本，供报告重算 | + +**进程隔离** + +`run_bench.py` 对每个 `(target, benchmark)` 起一个 fresh Python 进程： + +```text +run_bench.py --targets , + ├─ target-a × import_time.py → fresh process + ├─ target-a × instantiate_agent.py → fresh process + ├─ target-a × turn1.py → fresh process + ├─ target-a × turn5.py → fresh process + ├─ target-a × turn10.py → fresh process + ├─ target-a × loop1.py → fresh process + ├─ target-a × loop5.py → fresh process + ├─ target-a × loop10.py → fresh process + ├─ target-b × import_time.py → fresh process + ├─ ... + └─ summary.json +``` + +原因： + +- 不继承前一个 benchmark 的 import 缓存 / 内存分配器状态； +- 不同 target 的模块级状态、线程池、全局 registry 互不污染； +- 每个进程的 RSS/CPU 可以单独统计。 + +模型 server 独立于 benchmark 进程：server 是另一个子进程，agent 进程只计自己的 CPU/RSS；server 的请求时长单独记录。 + +**结果 JSON** + +```json +{ + "name": "turn5", + "description": "5 个 user turn，每个 turn 1 次 LLM 调用，每个 turn 的文本固定 1 chunk；指标：p50/p95 = 整个会话的耗时", + "display_name": "turn5", + "label": "", + "target": "", + "num_iterations": 10, + "warmup_runs": 1, + "result": { + "run_times": [0.000081, 0.000084], + "memory_usages": [0.000012, 0.000011], + "median_run_time": 0.000081, + "p95_run_time": 0.000084, + "median_memory_usage": 0.000012 + } +} +``` + +**target 对比表** + +多 target 时必须指定 `--baseline`，baseline 可以是 `--targets` 里的任意一个。每个非 baseline target 单独一张对比表，`turn1` / `turn5` / `turn10` 和 `loop1` / `loop5` / `loop10` 分别放在相邻位置： + +```text +# trpc-agent-python bench report +# results: results/ + +## Benchmark 说明 + +- `import_time`：每次新开一个 Python 进程 import 被测 Agent 包，再减去只启动解释器的时间。指标：p50/p95 = 新进程 import 耗时。 +- `instantiate_agent`：只调用 `target.instantiate(..., tools_factory=...)` 构造一个不带工具的 agent，不发起对话。指标：p50/p95 = 单次构造耗时。 +- `turn1`：1 个 user turn，1 次 LLM 调用，文本固定 1 chunk。指标：p50/p95 = 整个会话的耗时。 +- `turn5`：5 个 user turn，每个 turn 1 次 LLM 调用，每个 turn 的文本固定 1 chunk。指标：p50/p95 = 整个会话的耗时。 +- `turn10`：10 个 user turn，每个 turn 1 次 LLM 调用，每个 turn 的文本固定 1 chunk。指标：p50/p95 = 整个会话的耗时。 +- `loop5`：1 个 user turn，5 次 LLM 调用，4 次 tool round，最终文本 5 chunks。指标：p50/p95 = 整个 turn 耗时。 +- `loop10`：1 个 user turn，10 次 LLM 调用，9 次 tool round，最终文本 5 chunks。指标：p50/p95 = 整个 turn 耗时。 +- `memory_per_agent`：批量创建 agent 后一直留在内存里，不释放。指标：memory = 每个常驻 agent 占用的内存。 + +## target A vs target B + +| benchmark | baseline p50 | target B p50 | baseline p95 | target B p95 | baseline memory | target B memory | +|------------------|--------------|--------------|--------------|--------------|-----------------|-----------------| +| import_time | 147 ms | 120 ms -18.4% | 155 ms | 128 ms -17.4% | - | - | +| instantiate_agent| 4.7 us | 3.1 us -34.0% | 5.2 us | 3.5 us -32.7% | 2.0 KiB | 1.7 KiB -15.0% | +| turn1 | 65 us | 41 us -36.9% | 72 us | 46 us -36.1% | 1.2 KiB | 1.0 KiB -16.7% | +| turn5 | 2.9 ms | 1.1 ms -62.1% | 3.0 ms | 1.2 ms -60.0% | 1.3 MiB | 2.1 MiB +61.5% | +| turn10 | 5.8 ms | 2.2 ms -62.1% | 6.0 ms | 2.4 ms -60.0% | 2.3 MiB | 3.1 MiB +34.8% | +| loop1 | 65 us | 41 us -36.9% | 72 us | 46 us -36.1% | 1.2 KiB | 1.0 KiB -16.7% | +| loop5 | 2.9 ms | 1.1 ms -62.1% | 3.0 ms | 1.2 ms -60.0% | 1.3 MiB | 2.1 MiB +61.5% | +| loop10 | 5.8 ms | 2.2 ms -62.1% | 6.0 ms | 2.4 ms -60.0% | 2.3 MiB | 3.1 MiB +34.8% | + +# 数字仅为格式示例，不是实测结果。 +``` + +口径： + +```text +Δ p50 = (target p50 - baseline p50) / baseline p50 * 100% +Δ p95 = (target p95 - baseline p95) / baseline p95 * 100% +Δ memory = (target median_memory_usage - baseline median_memory_usage) + / baseline median_memory_usage * 100% +``` + +当前指标都是时间或内存，负数表示更快/更省，正数表示更慢/更多；百分比直接跟在 target 的数字后面，HTML 里负数绿色、正数红色。HTML 里 benchmark 列和 target 的 p50/p95/memory 列加粗，baseline 列保持正常字重，方便扫读。`memory_per_agent*` 只测内存，p50/p95 显示 `-`，memory 列填值；`import_time` 不测内存，memory 列显示 `-`。只有一个 target 时不需要 baseline，报告退化成 `benchmark | p50 | p95 | memory` 的绝对值表。 + +报告同时输出： + +- `report.md`：Benchmark 说明 bullet list + 分节对比表 + 口径； +- `report.html`：自包含，按 target 分节，带百分比颜色；带并发的结果在 benchmark 名下面显示 `concurrency=N`； +- `summary.json`：机器可读，CI artifact，带 `targets` / `baseline` 和每条 result 的 `description`、`display_name`、`label`。 + +**运行方式** + +```bash +cd bench +python -m trpc_agent_bench.run_bench \ + --targets llmagent.base \ + --benchmarks turn5,loop5 \ + --concurrency 1,8 \ + --iterations 10 +``` + +命令在 `bench/` 目录下执行，当前目录已在 `sys.path` 里。`--targets` 填 target module 路径；`--baseline` 在多 target 时必填，必须是 `--targets` 里的一个；`--benchmarks` 内置项可以用短名，自定义项填 module 路径。用户自定义的 module 只要在 `PYTHONPATH` 上即可。 + +调测用 `--iterations 10`。只有选中的 benchmark 有 `requires_iterations=True` 时才必须传；只跑 `import_time` / `memory_footprint` 时可以省略。结果统一写到 `results/`。完整跑在空闲机器上手动/定时执行。 + +内部通过环境变量把配置传给每个 benchmark 子进程： + +| 变量 | 作用 | +|---|---| +| `BENCH_TARGET` | target module 路径 | +| `BENCH_BENCHMARK` | benchmark module 路径 | +| `BENCH_CONCURRENCY` | 当前子进程的并发数；由 `--concurrency` 展开 | +| `BENCH_RESULT_SUFFIX` | 并发大于 1 时的结果名后缀，如 `_c8` | +| `BENCH_ITERATIONS` | 需要 iterations 的 benchmark 的迭代数；由 `--iterations` 写入 | +| `BENCH_RESULTS_DIR` | 结果输出目录 | +| `BENCH_DESCRIPTION` | runner 从 benchmark module 的 `DESCRIPTION` 写入，供 `write_result` 记录 | +| `BENCH_QUIET` | 不打印每项 summary | +| `GIT_COMMIT` | 写入结果 JSON 的 machine 信息 | +| `PYTHONPATH` | 仓库根目录 + 执行命令时的当前目录 | + +## 3. 压测目标 Agent + +前面的方案设计是通用的：mock、Target 协议、benchmark 和报告都不绑定具体实现。这一章把它落到 `trpc-agent-python` 的 `LlmAgent` 上。内置 target 放在 `bench/llmagent/`。 + +同一套 benchmark 会跑 import、构造、单轮 run、turn 多轮、一轮 Agent 执行内的多次 LLM 调用和 memory。对每个 target，adapter 只需提供三个入口： + +```python +await prepare(conf=...) # 从 conf.base_url 连接 mock，创建 target 资源 +instantiate(prepared, tools_factory=...) # 构造 agent；工具创建算在这里 +handle(prepared, tools_factory=...).run_turn(...) # 跑一轮；benchmark 自己 drain 事件 +``` + +`run_turn` 使用 `LlmAgent + Runner`：每次 turn 先确保 session 存在，再新建 `Runner`，共用 target 自己创建的 session service。`app_name` / `user_id` / `session_id` 由 benchmark 调用 `run_turn` 时显式传入，handle 不保存固定身份。`cleanup()` 负责关闭 session service；Runner 在每次 turn 结束时关闭。 + +```python +# bench/llmagent/base.py +async def prepare(self, *, conf: AgentBenchConf) -> PreparedLlmAgent: + session_service = self.create_session_service() # InMemorySessionService + model = OpenAIModel( + model_name="bench-model", + api_key="bench", + base_url=conf.base_url, # setup() 写入的 mock 地址 + ) + return PreparedLlmAgent(model=model, session_service=session_service) + +async def instantiate(prepared, *, tools_factory): + tools = await resolve_tools(tools_factory) + return LlmAgent( + name="bench_llm_agent", + model=prepared.model, + instruction=INSTRUCTION, + tools=tools, + parallel_tool_calls=True, + ) + +async def run_turn(self, *, app_name, user_id, session_id, message): + await ensure_session(...) + runner = Runner( + app_name=app_name, + agent=self.agent, + session_service=self.session_service, + enable_post_turn_processing=False, # 不触发总结/记忆的额外模型调用 + close_session_service_on_close=False, # session service 由 target 管理 + ) + try: + async for event in runner.run_async( + user_id=user_id, + session_id=session_id, + new_message=Content(parts=[Part.from_text(text=message)]), + ): + yield event + finally: + await runner.close() +``` + +`LlmAgent` 的 `turnN` 脚本只返回 `TextBlock`，不带 `FunctionBlock`；`loop1` / `loop5` / `loop10`、`tools` 和 `tools_parallel` 的脚本才返回 `FunctionBlock`。mock 通过 `conf.base_url` 注入，agent 的真实 loop、tool dispatch、事件流和 session 读写都会执行。 + +## 4. 参考：Agno 的 performance 套件 + +Agno 在 `cookbook/performance/` 下有一套完整的性能基准，核心思路写在 `cookbook/performance/README.md`： + +> measures framework overhead: the time and memory an agent framework itself adds to importing, constructing, and running an agent, isolated from any model provider. All benchmarks replace the model with an in-process mock at the framework's own model boundary. + +### 4.1 它测哪些东西 + +| benchmark 文件 | 操作 | 度量 | +|---|---|---| +| `import_time.py` | fresh 进程 import `agno` / `agno.agent` | 减掉解释器启动的 median/p95 | +| `instantiate_agent.py` | 构造 `Agent` | 每次构造的耗时 + 内存峰值 | +| `instantiate_agent_with_tools.py` | 构造带 5 个工具的 Agent | 同上 | +| `instantiate_team.py` / `instantiate_workflow.py` | 构造 Team / Workflow | 同上 | +| `run_agent.py` | 一次 `run()` / `arun()` | 每次 run 的框架开销 | +| `run_agent_streaming.py` | 一次流式 run，drain 整个事件流 | 流式机制固定开销 | +| `run_agent_with_tools.py` | 两轮 tool loop | tool dispatch + 第二轮模型调用 | +| `run_agent_with_storage.py` | 带 in-memory DB 和 history 的 run | session 持久化开销 | +| `memory_footprint.py` | 批量创建 agent 后一直留在内存里 | 每个常驻 agent 占用的内存 | + +### 4.2 它的 mock 与测量口径 + +`cookbook/performance/_bench.py` 里两个类： + +```python +class MockModel(Model): + """Minimal offline model: returns a canned text response without any network call.""" + + def __init__(self, response_content: str = "ok"): + super().__init__(id="mock-model", name="mock-model", provider="mock") + self._mock_response = ModelResponse( + content=response_content, role="assistant", response_usage=MessageMetrics() + ) + + def invoke(self, *args, **kwargs) -> ModelResponse: + return self._mock_response + + async def ainvoke(self, *args, **kwargs) -> ModelResponse: + return self._mock_response + + def invoke_stream(self, *args, **kwargs) -> Iterator[ModelResponse]: + yield self._mock_response +``` + +```python +class MockToolModel(MockModel): + """Offline model that requests one tool call, then answers once the tool result is present.""" + + def _make_response(self, messages) -> ModelResponse: + has_tool_result = any( + getattr(m, "role", None) == "tool" for m in (messages or []) + ) + if has_tool_result: + return ModelResponse(content="done", role="assistant", ...) + return ModelResponse( + role="assistant", + tool_calls=[{ + "id": "call_1", + "type": "function", + "function": {"name": "add_numbers", "arguments": '{"a": 1, "b": 2}'}, + }], + ... + ) +``` + +两个关键点： + +- mock 继承框架自己的 `Model` 基类，所以真实 loop 照常执行，只有 provider 调用被换掉； +- tool 决策是**无状态**的：看请求的 messages 里有没有 `role="tool"`，不看“第几次调用”，所以并发/重放都稳定。 + +`libs/agno/agno/eval/performance.py` 的 `PerformanceEval`： + +```text +PerformanceEval.run() + ├─ warmup_runs = 10 直接 func()，不计入统计 + ├─ runtime pass N 次，Timer 只包住 func()，记录 run_times + ├─ memory pass N 次，tracemalloc.start() → func() → get_traced_memory() + │ 和 runtime 分开跑，因为 tracemalloc 会拖慢执行 + └─ PerformanceResult + avg / min / max / std / median / p95 + p95 = statistics.quantiles(..., method="inclusive")[94] +``` + +套件运行方式： + +```text +run_all.py + ├─ subprocess: import_time.py → import_time.json + ├─ subprocess: instantiate_agent.py → instantiate_agent.json + ├─ subprocess: run_agent.py → run_agent.json + ├─ ... + └─ summary.json（machine info + 所有 benchmark 结果） +report.py → 自包含 HTML 报告（图表 + 统计 + 跨框架倍数） +baselines/*.json → committed baseline +TEST_LOG.md → 每次跑的环境、结果、异常 +``` + +每个 benchmark 文件都在新进程里运行，避免继承前一个 benchmark 的 import 缓存、内存分配器状态和 GC 状态。调测用 `--iterations 10`，结果写到 `results/`；不传时只有不需要 `--iterations` 的 benchmark 可以运行，不会冒充 baseline。 + +### 4.3 我们采用什么、改什么 + +`comparison/run_all.py` 把同一操作跑在 Agno、LangGraph、PydanticAI、CrewAI 上，每个框架在**自己的模型调用处**注入 mock： + +| 框架 | mock 位置 | +|---|---| +| Agno | `Model` 子类 | +| LangGraph | `GenericFakeChatModel` | +| PydanticAI | `TestModel` | +| CrewAI | `BaseLLM` 子类 | + +`comparison/README.md` 明确写了 fairness notes 和 exclusions（例如 CrewAI 的 tool 协议是版本内部格式，mock 会变成“测 mock”）。报告里的数字是各框架的 **floor**，不是完整 provider 路径成本。 + +| Agno 的做法 | 我们采用 | 我们的调整 | +|---|---|---| +| in-process mock at the model call site | ✅ | 统一改成在本机 HTTP 层注入 mock，v1/v2 都走同一条 client/stream 路径 | +| 按指标分文件（import/instantiate/run/streaming/tools/storage/memory） | ✅ | 文件名和分组直接对齐 | +| `PerformanceEval`：warmup + median/p95 + 时间/内存分开 | ✅ | 复刻同样的统计口径 | +| 每文件 fresh process | ✅ | `run_bench.py` 按 `(target, benchmark)` 起子进程 | +| 每次 run 断言正确性 | ✅ | `ensure_turn_completed` | +| HTML report + 调测 | ✅ | 报告同时输出 Markdown 和 HTML；调测用 `--iterations 10` | +| 跨框架对比 | ✅ | 对比对象是同一套操作下的不同 target | +| pass@k / 质量指标 | ❌ | 我们的 workload 确定性，不需要 | diff --git a/docs/mkdocs/en/bench.md b/docs/mkdocs/en/bench.md new file mode 100644 index 00000000..0e2026a9 --- /dev/null +++ b/docs/mkdocs/en/bench.md @@ -0,0 +1,496 @@ +# Benchmark Suite + +`trpc_agent_bench` measures only the framework's own overhead: import, construction, a single-turn run, multi-turn sessions, Agent execution with tools, and memory usage. Model calls are served by a local OpenAI-compatible mock server, but Agent execution, tool dispatch, event streaming, and session reads/writes all run for real — no live LLM calls are required. + +- [Design document](../../design/01-bench-design.md) +- [Framework benchmark data](../../../bench/README.md) + +## Installation + +To use the PyPI package: + +```bash +pip install "trpc-agent-py[bench]" +``` + +For framework development, use an editable install (commented out by default): + +```bash +# pip install -e ".[bench]" +``` + +## Quick start + +The repository ships with `bench/llmagent`, so you can run it directly from the `bench/` directory: + +```bash +cd bench + +python -m trpc_agent_bench.run_bench \ + --targets llmagent.base \ + --benchmarks turn1,turn5,turn10,loop1,loop5,loop10 \ + --iterations 10 \ + --results-dir results/direct +``` + +If you only installed the PyPI package and don't have `bench/llmagent` from the repository, write your own target module as described in [Developing the agent under test](#developing-the-agent-under-test) below, expose a `TARGET` in it, and run it with `--targets your.module`. + +Use `--concurrency` for stress testing. Each concurrency value starts its own child process: + +```bash +python -m trpc_agent_bench.run_bench \ + --targets llmagent.base \ + --benchmarks turn1,turn5,turn10,loop1,loop5,loop10 \ + --concurrency 1,8,16,32 \ + --iterations 10 \ + --results-dir results/stress +``` + +You don't need `--baseline` with a single target. With multiple targets it is required and must be one of the modules in `--targets`: + +```bash +python -m trpc_agent_bench.run_bench \ + --targets llmagent.base,my_bench.my_target \ + --baseline llmagent.base \ + --benchmarks turn5,loop5 \ + --iterations 10 +``` + +Each run produces the following under the results root: + +```text +results/direct/ +├── llmagent.base/ +│ ├── turn1.json +│ ├── turn5.json +│ ├── turn10.json +│ ├── loop1.json +│ ├── loop5.json +│ └── loop10.json +├── summary.json +├── report.md +└── report.html +``` + +Each JSON file contains `target`, `description`, `display_name`, `label`, `num_iterations`, `warmup_runs`, `result.run_times`, `result.memory_usages`, and `result.median_run_time` / `p95_run_time` / `median_memory_usage`. Reports are available as Markdown and self-contained HTML; result names include a `_cN` suffix when concurrency is used. + +## Command-line options and built-in benchmarks + +### Command-line options + +| Argument | Values | Default | Description | +|---|---|---|---| +| `--targets` | Target module paths, comma-separated | Required | Target modules to benchmark, for example `llmagent.base` | +| `--baseline` | One of the module paths in `--targets` | Required with multiple targets | Comparison baseline | +| `--benchmarks` | Built-in short names or custom benchmark module paths, comma-separated | Built-in default set (12 items) | Run only the listed items | +| `--concurrency` | A positive integer or a comma-separated list such as `1,8,16,32` | `1` | Only applies to benchmarks whose `get_bench_profile().supports_concurrency` is `True` | +| `--iterations` | Positive integer | Required when the selected benchmarks need it | 10 is recommended; benchmarks such as `import_time` / `memory_footprint` that don't need it may omit it | +| `--results-dir` | Directory path | `results/` under the current directory | Result root directory | + +For the full list of options, run: + +```bash +python -m trpc_agent_bench.run_bench --help +``` + +### Built-in benchmarks + +| Scenario | Metric | Description | +|---|---|---| +| `import_time` | Time to import the Agent package under test in a fresh process | Starts a new Python process to import the Agent package, then subtracts the time taken to start the interpreter alone | +| `instantiate_agent` | Construction time, peak memory | Calls only `target.instantiate()` to create an agent without tools; no conversation | +| `instantiate_agent_with_tools` | Construction time, peak memory | Creates an agent with 5 tools; no conversation | +| `turn1` / `turn5` / `turn10` | p50/p95, memory | 1 / 5 / 10 user turns, one LLM call per turn, text in 1 chunk | +| `loop1` / `loop5` / `loop10` | p50/p95, memory | 1 user turn with 1 / 5 / 10 LLM calls; every call except the last returns a function call, and the final text arrives in 5 chunks | +| `tools` | p50/p95, memory | Calls `add_numbers` first, then produces the final text after receiving the tool result | +| `tools_parallel` | p50/p95 | Returns 3 tool calls with different arguments in a single reply | +| `memory_footprint` | Memory footprint per resident agent | Creates agents without / with tools in bulk and keeps them in memory, measuring how much memory each resident agent occupies | + +`turnN` measures the growth of multi-turn conversations within the same session; `loopN` measures the cost of multiple consecutive LLM calls inside one Agent execution (`N-1` tool rounds). Each `--iterations` sample is independent and creates a new session, so it is not the same thing as a multi-turn conversation. + +- For `instantiate_agent*`, memory is the peak recorded by tracemalloc during construction. +- For `memory_footprint`, memory is the total memory of N agents still resident after creation divided by N — not the peak during construction. + +Built-in short names map to `trpc_agent_bench/bench/_builtin/.py`; custom benchmarks use a dotted module path, for example `my_bench.my_benchmark`. + +## Integrating with the benchmark suite + +From the entry point's perspective, the pieces relate as follows: + +```text +run_bench (parent process) + │ parses arguments, pre-checks benchmark profiles + │ starts one child process per (target, benchmark, concurrency) + ▼ +benchmark.main() (child process) + ├─ setup(script) + │ ├─ load_target() -> AgentTarget + │ ├─ start MockOpenAIServer + │ └─ target.prepare(conf) -> prepared + ├─ target.handle(prepared) -> AgentHandle (holds the Agent) + ├─ PerformanceEval + │ └─ calls handle.run_turn() according to script / iterations / concurrency + └─ teardown() + ├─ target.cleanup(prepared) + └─ stop MockOpenAIServer +``` + +`run_bench` launches the benchmark scripts; a benchmark orchestrates the `handle.run_turn()` calls; `AgentTarget` creates the Agent and returns a Handle, and `AgentHandle` runs one conversation turn. + +To integrate an agent under test, developers only need `AgentTarget`/`AgentHandle`; to define a custom benchmark scenario, they only need to write a benchmark. + +### Developing the agent under test + +Expose `TARGET` in any importable module. The example below has two parts: `MyTarget` handles integration, and `MyHandle` runs a single conversation turn. + +```python +# my_bench/my_target.py +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, AsyncIterator + +from trpc_agent_sdk.agents import LlmAgent +from trpc_agent_sdk.models import OpenAIModel +from trpc_agent_sdk.runners import Runner +from trpc_agent_sdk.sessions import BaseSessionService, InMemorySessionService +from trpc_agent_sdk.types import Content, Part + +from trpc_agent_bench.bench import ( + AgentBenchConf, + AgentHandle, + AgentTarget, + ToolsFactory, + resolve_tools, +) + + +@dataclass +class Prepared: + model: OpenAIModel + session_service: BaseSessionService + + +class MyHandle(AgentHandle): + def __init__(self, *, agent: LlmAgent, session_service: BaseSessionService): + self.agent = agent + self.session_service = session_service + + async def run_turn( + self, + *, + app_name: str, + user_id: str, + session_id: str, + message: str, + ) -> AsyncIterator[Any]: + # Run one Agent turn. + runner = Runner( + app_name=app_name, + agent=self.agent, + session_service=self.session_service, + enable_post_turn_processing=False, + close_session_service_on_close=False, + ) + try: + async for event in runner.run_async( + user_id=user_id, + session_id=session_id, + new_message=Content(parts=[Part.from_text(text=message)]), + ): + yield event + finally: + await runner.close() + + +class MyTarget(AgentTarget): + name = "my_agent" + import_module = "my_agent" # used by import_time + + async def prepare(self, *, conf: AgentBenchConf) -> Prepared: + # Called from setup(): create the model and session service. + session_service = InMemorySessionService() + try: + model = OpenAIModel( + model_name="bench-model", + api_key="bench", + base_url=conf.base_url, + ) + except Exception: + await session_service.close() + raise + + return Prepared(model=model, session_service=session_service) + + async def instantiate( + self, + prepared: Prepared, + *, + tools_factory: ToolsFactory, + ) -> LlmAgent: + # Create the Agent; tools are resolved here. + tools = await resolve_tools(tools_factory) + return LlmAgent( + name="my_bench_agent", + model=prepared.model, + instruction="You are a deterministic benchmark agent.", + tools=tools, + ) + + async def handle( + self, + prepared: Prepared, + *, + tools_factory: ToolsFactory, + ) -> AgentHandle: + # Return the handle the benchmark will call. + agent = await self.instantiate(prepared, tools_factory=tools_factory) + return MyHandle(agent=agent, session_service=prepared.session_service) + + async def cleanup(self, prepared: Prepared) -> None: + # Called from teardown(): release target resources. + await prepared.session_service.close() + + +TARGET = MyTarget() +``` + +Benchmark the target agent with: + +```bash +python -m trpc_agent_bench.run_bench \ + --targets my_bench.my_target \ + --benchmarks turn1 \ + --iterations 10 +``` + +`run_bench` adds the current directory to the child process's `PYTHONPATH`, so the target module only needs to be in the current directory. + +### Developing a benchmark + +A benchmark is just a Python module that provides `main()`; returning 0 from `main()` means the benchmark succeeded. Built-in benchmarks live under `trpc_agent_bench/bench/_builtin/` and can be referenced by short name; custom benchmarks use a dotted module path. + +#### 1. Minimal example + +The following single-turn text example can be copied and run as-is. + +```python +# my_bench/my_benchmark.py +from __future__ import annotations + +import asyncio +import sys +import uuid + +from trpc_agent_bench.bench import ( + PerformanceEval, + TextBlock, + drain, + ensure_turn_completed, + get_iterations, + run_benchmarks, + setup, + teardown, +) + +# Written to the result JSON and shown in the report. +DESCRIPTION = "A single text turn; metric: p50/p95 = latency of one turn." + + +async def main() -> int: + # The script decides what the mock model returns. + box = await setup([[TextBlock("ok")]]) + handle = await box.target.handle(box.prepared, tools_factory=lambda: []) + + async def run_once() -> None: + events = await drain( + handle.run_turn( + app_name=box.conf.app_name, + user_id=box.conf.user_id, + session_id=str(uuid.uuid4()), + message="Say ok.", + ), + ) + # Validate the final text; add expect_tool_success=True when tools are involved. + ensure_turn_completed(events, expected_text="ok") + + try: + # num_iterations comes from --iterations; one warmup run first. + await run_benchmarks([ + PerformanceEval( + name="my_benchmark", + func=run_once, + num_iterations=get_iterations(), + warmup_runs=1, + ), + ]) + finally: + # Clean up the target and the MockOpenAIServer. + await teardown(box) + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) +``` + +Run it with: + +```bash +python -m trpc_agent_bench.run_bench \ + --targets my_bench.my_target \ + --benchmarks my_bench.my_benchmark \ + --iterations 10 +``` + +`run_bench` adds the current directory to the child process's `PYTHONPATH`, so the target and benchmark modules only need to be in the current directory. + +#### 2. Simulating multi-turn conversations + +```python +# bench/your_bench.py +... +from trpc_agent_sdk.tools import BaseTool, FunctionTool +from trpc_agent_bench.bench import FunctionBlock, TextBlock + +# One Turn is one user turn; one Reply is one LLM response. +# FunctionBlock triggers tool execution; TextBlock ends the turn. +SCRIPT = [ + [[FunctionBlock("add_numbers", {"a": 1, "b": 2})], TextBlock("ok")], + [[FunctionBlock("add_numbers", {"a": 3, "b": 4})], TextBlock("ok")], +] + + +def add_numbers(a: int, b: int) -> int: + """Add two integers and return the result.""" + return a + b + + +def make_tools() -> list[BaseTool]: + return [FunctionTool(add_numbers)] + + +async def main() -> int: + box = await setup(SCRIPT) + handle = await box.target.handle(box.prepared, tools_factory=make_tools) + + async def run_once() -> None: + session_id = str(uuid.uuid4()) + # Call run_turn once per Turn, sharing the same session. + for _ in SCRIPT: + events = await drain( + handle.run_turn( + app_name=box.conf.app_name, + user_id=box.conf.user_id, + session_id=session_id, + message="Say ok.", + ), + ) + ensure_turn_completed( + events, + expected_text="ok", + expect_tool_success=True, + ) + + ... +``` + +`tools_factory` may also be an async function; `TextBlock(chunks=N)` controls the number of stream chunks, and the text must be long enough. + +#### 3. Simulating LLM latency + +```python +# bench/your_bench.py +... +from trpc_agent_bench.bench import AgentBenchConf, ModelConf, TextBlock + + +async def main() -> int: + conf = AgentBenchConf( + model_conf=ModelConf( + request_delay_ms=50, + chunk_delay_ms=10, + ), + ) + box = await setup([[TextBlock("hello", chunks=3)]], conf=conf) + ... +``` + +- `request_delay_ms`: delay between the start of the request and the first response; applies to both streaming and non-streaming responses. +- `chunk_delay_ms`: delay between chunks starting from the 2nd chunk of a streaming response; has no effect when `chunks=1`. + +`MockOpenAIServer` is not public API; `setup()` starts it internally. + +#### 4. Concurrency benchmarks + +```python +# bench/your_bench.py +... +from trpc_agent_bench.bench import BenchProfile, get_concurrency + + +def get_bench_profile() -> BenchProfile: + return BenchProfile(supports_concurrency=True) + + +async def main() -> int: + ... + current = get_concurrency() + name = f"my_benchmark_c{current}" if current > 1 else "my_benchmark" + label = f"concurrency={current}" if current > 1 else "" + + async def run_session(index: int) -> None: + user_id = f"{box.conf.user_id}_{index}" + session_id = str(uuid.uuid4()) + for _ in SCRIPT: + events = await drain( + handle.run_turn( + app_name=box.conf.app_name, + user_id=user_id, + session_id=session_id, + message="Say ok.", + ), + ) + ensure_turn_completed(events, expected_text="ok", expect_tool_success=True) + + async def run_once() -> None: + await asyncio.gather(*(run_session(i) for i in range(current))) + + await run_benchmarks([ + PerformanceEval( + name=name, + display_name="my_benchmark", + label=label, + func=run_once, + num_iterations=get_iterations(), + warmup_runs=1, + ), + ]) + ... +``` + +With `supports_concurrency=True`, `run_bench` starts a separate child process for each `--concurrency` value. Result names get a `_cN` suffix, and each session uses its own `user_id`. + +#### 5. Disabling a benchmark + +```python +# bench/your_bench.py +import os + +from trpc_agent_bench.bench import BenchProfile + + +def get_bench_profile() -> BenchProfile: + if not os.environ.get("MY_BACKEND_URL"): + return BenchProfile( + usable=False, + skip_reason="MY_BACKEND_URL is not set", + ) + return BenchProfile() + + +async def main() -> int: + ... +``` + +When `usable=False`, `run_bench` prints `[skip] ()` and skips it without affecting other benchmarks. diff --git a/docs/mkdocs/zh/bench.md b/docs/mkdocs/zh/bench.md new file mode 100644 index 00000000..22fd5bcc --- /dev/null +++ b/docs/mkdocs/zh/bench.md @@ -0,0 +1,496 @@ +# 压测套件 + +`trpc_agent_bench` 只测框架自身开销：import、构造、单轮 run、多轮会话、带工具的 Agent 执行和内存占用。模型调用由本机 OpenAI-compatible mock server 替代，但 Agent 执行、tool dispatch、事件流和 session 读写都会真实执行，不需要真实 LLM 调用。 + +- [设计文档](../../design/01-bench-design.md) +- [框架压测数据](../../../bench/README.md) + +## 安装 + +直接使用 PyPI 包： + +```bash +pip install "trpc-agent-py[bench]" +``` + +框架自身开发调测时使用 editable 安装（默认注释掉）： + +```bash +# pip install -e ".[bench]" +``` + +## 快速开始 + +仓库里自带 `bench/llmagent`，在 `bench/` 目录下可以直接跑： + +```bash +cd bench + +python -m trpc_agent_bench.run_bench \ + --targets llmagent.base \ + --benchmarks turn1,turn5,turn10,loop1,loop5,loop10 \ + --iterations 10 \ + --results-dir results/direct +``` + +如果只安装了 PyPI 包、没有仓库里的 `bench/llmagent`，可以按后面的「开发压测 target」写一个自己的 target module，在里面提供 `TARGET`，再用 `--targets your.module` 运行。 + +压力测试用 `--concurrency`。每个并发值会启动独立子进程： + +```bash +python -m trpc_agent_bench.run_bench \ + --targets llmagent.base \ + --benchmarks turn1,turn5,turn10,loop1,loop5,loop10 \ + --concurrency 1,8,16,32 \ + --iterations 10 \ + --results-dir results/stress +``` + +只跑一个 target 时不需要 `--baseline`；多个 target 时必须指定，而且必须是 `--targets` 里的一个： + +```bash +python -m trpc_agent_bench.run_bench \ + --targets llmagent.base,my_bench.my_target \ + --baseline llmagent.base \ + --benchmarks turn5,loop5 \ + --iterations 10 +``` + +每次运行会在结果根目录生成： + +```text +results/direct/ +├── llmagent.base/ +│ ├── turn1.json +│ ├── turn5.json +│ ├── turn10.json +│ ├── loop1.json +│ ├── loop5.json +│ └── loop10.json +├── summary.json +├── report.md +└── report.html +``` + +单个 JSON 里有 `target`、`description`、`display_name`、`label`、`num_iterations`、`warmup_runs`、`result.run_times`、`result.memory_usages`，以及 `result.median_run_time` / `p95_run_time` / `median_memory_usage`。报告支持 Markdown 和自包含 HTML，带并发的结果名带 `_cN` 后缀。 + +## 命令行参数与内置benchmark + +### 命令行参数 + +| 参数 | 可填的值 | 默认 | 说明 | +|---|---|---|---| +| `--targets` | target module 路径，逗号分隔 | 必填 | 要压测的 target module 列表，例如 `llmagent.base` | +| `--baseline` | `--targets` 里的一个 module 路径 | 多 target 时必填 | 对比基准 | +| `--benchmarks` | 内置短名或自定义 benchmark module 路径，逗号分隔 | 内置默认集（12 项） | 只跑指定的几项 | +| `--concurrency` | 正整数或逗号分隔的多个值，如 `1,8,16,32` | `1` | 只对 `get_bench_profile().supports_concurrency` 为 `True` 的 benchmark 生效 | +| `--iterations` | 正整数 | 选中的 benchmark 需要时必填 | 推荐 10；`import_time` / `memory_footprint` 这类不需要 `--iterations` 的 benchmark 可以省略 | +| `--results-dir` | 目录路径 | 当前目录下的 `results/` | 结果根目录 | + +完整参数可以直接看： + +```bash +python -m trpc_agent_bench.run_bench --help +``` + +### 内置 benchmark + +| 压测场景 | 指标 | 说明 | +|---|---|---| +| `import_time` | 新进程 import 被测 Agent 包的耗时 | 每次新开一个 Python 进程 import 被测 Agent 包，再减去只启动解释器的时间 | +| `instantiate_agent` | 构造耗时、内存峰值 | 只调用 `target.instantiate()` 创建一个不带工具的 agent，不发起对话 | +| `instantiate_agent_with_tools` | 构造耗时、内存峰值 | 创建一个带 5 个工具的 agent，不发起对话 | +| `turn1` / `turn5` / `turn10` | p50/p95、内存 | 1 / 5 / 10 个 user turn，每个 turn 1 次 LLM，文本 1 chunk | +| `loop1` / `loop5` / `loop10` | p50/p95、内存 | 1 个 user turn，1 / 5 / 10 次 LLM；除最后一次外都返回 function call，最终文本 5 chunks | +| `tools` | p50/p95、内存 | 先调 `add_numbers`，拿到 tool result 后再出最终文本 | +| `tools_parallel` | p50/p95 | 一次回复里返回 3 个不同参数的 tool call | +| `memory_footprint` | 每个常驻 agent 的内存占用 | 批量创建不带工具 / 带工具的 agent 后一直留在内存里，测每个 agent 常驻占用多少内存 | + +`turnN` 测同一个 session 内多轮会话的增长；`loopN` 测一轮 Agent 执行内连续多次 LLM 调用（`N-1` 次 tool round）的开销。`--iterations` 是独立采样，每次都会新建 session，不等于多轮对话。 + +- `instantiate_agent*` 的 memory 是构造过程中 tracemalloc 记录的峰值。 +- `memory_footprint` 的 memory 是创建 N 个 agent 后一直留在内存里，占用的内存除以 N；不是构造过程中的峰值。 + +内置短名对应 `trpc_agent_bench/bench/_builtin/.py`；自定义 benchmark 用带点的 module 路径，例如 `my_bench.my_benchmark`。 + +## 接入压测套件 + +从压测入口来看，有如下关系图： + +```text +run_bench（父进程） + │ 解析参数、预检 benchmark profile + │ 每个 (target, benchmark, concurrency) 起一个子进程 + ▼ +benchmark.main()（子进程） + ├─ setup(script) + │ ├─ load_target() -> AgentTarget + │ ├─ 启动 MockOpenAIServer + │ └─ target.prepare(conf) -> prepared + ├─ target.handle(prepared) -> AgentHandle（内部持有 Agent） + ├─ PerformanceEval + │ └─ 按 script / iterations / concurrency 调 handle.run_turn() + └─ teardown() + ├─ target.cleanup(prepared) + └─ 停止 MockOpenAIServer +``` + +`run_bench` 负责拉起压测脚本；`benchmark` 负责编排 `handle.run_turn()` 的调用；`AgentTarget` 负责创建 Agent 和返回 Handle，`AgentHandle` 负责跑一轮对话。 + +开发者只需AgentTarget/AgentHandle即可接入被测 Agent，只需开发benchmark即可自定义压测场景。 + +### 开发被测 Agent + +在任意可 import 的 module 里提供 `TARGET`。下面的例子包含两部分：`MyTarget` 负责接入，`MyHandle` 负责跑一轮对话。 + +```python +# my_bench/my_target.py +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, AsyncIterator + +from trpc_agent_sdk.agents import LlmAgent +from trpc_agent_sdk.models import OpenAIModel +from trpc_agent_sdk.runners import Runner +from trpc_agent_sdk.sessions import BaseSessionService, InMemorySessionService +from trpc_agent_sdk.types import Content, Part + +from trpc_agent_bench.bench import ( + AgentBenchConf, + AgentHandle, + AgentTarget, + ToolsFactory, + resolve_tools, +) + + +@dataclass +class Prepared: + model: OpenAIModel + session_service: BaseSessionService + + +class MyHandle(AgentHandle): + def __init__(self, *, agent: LlmAgent, session_service: BaseSessionService): + self.agent = agent + self.session_service = session_service + + async def run_turn( + self, + *, + app_name: str, + user_id: str, + session_id: str, + message: str, + ) -> AsyncIterator[Any]: + # 执行一轮Agent交互 + runner = Runner( + app_name=app_name, + agent=self.agent, + session_service=self.session_service, + enable_post_turn_processing=False, + close_session_service_on_close=False, + ) + try: + async for event in runner.run_async( + user_id=user_id, + session_id=session_id, + new_message=Content(parts=[Part.from_text(text=message)]), + ): + yield event + finally: + await runner.close() + + +class MyTarget(AgentTarget): + name = "my_agent" + import_module = "my_agent" # import_time 用 + + async def prepare(self, *, conf: AgentBenchConf) -> Prepared: + # setup() 时调用：创建 model 和 session service。 + session_service = InMemorySessionService() + try: + model = OpenAIModel( + model_name="bench-model", + api_key="bench", + base_url=conf.base_url, + ) + except Exception: + await session_service.close() + raise + + return Prepared(model=model, session_service=session_service) + + async def instantiate( + self, + prepared: Prepared, + *, + tools_factory: ToolsFactory, + ) -> LlmAgent: + # 创建 Agent；tool 在这里解析。 + tools = await resolve_tools(tools_factory) + return LlmAgent( + name="my_bench_agent", + model=prepared.model, + instruction="You are a deterministic benchmark agent.", + tools=tools, + ) + + async def handle( + self, + prepared: Prepared, + *, + tools_factory: ToolsFactory, + ) -> AgentHandle: + # 返回给 benchmark 调用的 handle。 + agent = await self.instantiate(prepared, tools_factory=tools_factory) + return MyHandle(agent=agent, session_service=prepared.session_service) + + async def cleanup(self, prepared: Prepared) -> None: + # teardown() 时调用：释放 target 资源。 + await prepared.session_service.close() + + +TARGET = MyTarget() +``` + +通过如下命令来压测目标 Agent： + +```bash +python -m trpc_agent_bench.run_bench \ + --targets my_bench.my_target \ + --benchmarks turn1 \ + --iterations 10 +``` + +`run_bench` 会把当前目录加入子进程的 `PYTHONPATH`，所以 target module 在当前目录下即可。 + +### 开发 benchmark + +一个 benchmark 就是一个 Python module，里面提供 `main()`；`main()` 返回 0 表示这项压测成功。内置 benchmark 在 `trpc_agent_bench/bench/_builtin/` 下，可以用短名引用；自定义 benchmark 用带点的 module 路径。 + +#### 1. 最小示例 + +下面是一个单轮文本示例，复制后可以直接运行。 + +```python +# my_bench/my_benchmark.py +from __future__ import annotations + +import asyncio +import sys +import uuid + +from trpc_agent_bench.bench import ( + PerformanceEval, + TextBlock, + drain, + ensure_turn_completed, + get_iterations, + run_benchmarks, + setup, + teardown, +) + +# 会写进结果 JSON，并显示在报告里。 +DESCRIPTION = "单个文本 turn；指标：p50/p95 = 一次 turn 的耗时。" + + +async def main() -> int: + # script 决定 mock 模型返回什么。 + box = await setup([[TextBlock("ok")]]) + handle = await box.target.handle(box.prepared, tools_factory=lambda: []) + + async def run_once() -> None: + events = await drain( + handle.run_turn( + app_name=box.conf.app_name, + user_id=box.conf.user_id, + session_id=str(uuid.uuid4()), + message="Say ok.", + ), + ) + # 校验最终文本；带 tool 时加 expect_tool_success=True。 + ensure_turn_completed(events, expected_text="ok") + + try: + # num_iterations 来自 --iterations；先跑一次 warmup。 + await run_benchmarks([ + PerformanceEval( + name="my_benchmark", + func=run_once, + num_iterations=get_iterations(), + warmup_runs=1, + ), + ]) + finally: + # 清理 target 和 MockOpenAIServer。 + await teardown(box) + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) +``` + +写完后用这条命令运行： + +```bash +python -m trpc_agent_bench.run_bench \ + --targets my_bench.my_target \ + --benchmarks my_bench.my_benchmark \ + --iterations 10 +``` + +`run_bench` 会把当前目录加入子进程的 `PYTHONPATH`，所以 target 和 benchmark module 放在当前目录下即可。 + +#### 2. 模拟多轮对话 + +```python +# bench/your_bench.py +... +from trpc_agent_sdk.tools import BaseTool, FunctionTool +from trpc_agent_bench.bench import FunctionBlock, TextBlock + +# 一个 Turn 是一轮 user turn；一个 Reply 是一次 LLM 响应。 +# FunctionBlock 触发 tool 执行；TextBlock 结束这一轮。 +SCRIPT = [ + [[FunctionBlock("add_numbers", {"a": 1, "b": 2})], TextBlock("ok")], + [[FunctionBlock("add_numbers", {"a": 3, "b": 4})], TextBlock("ok")], +] + + +def add_numbers(a: int, b: int) -> int: + """把两个整数相加并返回结果。""" + return a + b + + +def make_tools() -> list[BaseTool]: + return [FunctionTool(add_numbers)] + + +async def main() -> int: + box = await setup(SCRIPT) + handle = await box.target.handle(box.prepared, tools_factory=make_tools) + + async def run_once() -> None: + session_id = str(uuid.uuid4()) + # 每个 Turn 调一次 run_turn，共用同一个 session。 + for _ in SCRIPT: + events = await drain( + handle.run_turn( + app_name=box.conf.app_name, + user_id=box.conf.user_id, + session_id=session_id, + message="Say ok.", + ), + ) + ensure_turn_completed( + events, + expected_text="ok", + expect_tool_success=True, + ) + + ... +``` + +`tools_factory` 也可以是 async 函数；`TextBlock(chunks=N)` 控制 stream chunk 数，文本要够长。 + +#### 3. 模拟LLM延迟 + +```python +# bench/your_bench.py +... +from trpc_agent_bench.bench import AgentBenchConf, ModelConf, TextBlock + + +async def main() -> int: + conf = AgentBenchConf( + model_conf=ModelConf( + request_delay_ms=50, + chunk_delay_ms=10, + ), + ) + box = await setup([[TextBlock("hello", chunks=3)]], conf=conf) + ... +``` + +- `request_delay_ms`：请求开始到第一个响应之前的等待，流式和非流式都生效。 +- `chunk_delay_ms`：流式响应中第 2 个 chunk 起，每个 chunk 之间等待；`chunks=1` 时不会生效。 + +`MockOpenAIServer` 不是公开 API；`setup()` 会在内部启动它。 + +#### 4. 并发压测 + +```python +# bench/your_bench.py +... +from trpc_agent_bench.bench import BenchProfile, get_concurrency + + +def get_bench_profile() -> BenchProfile: + return BenchProfile(supports_concurrency=True) + + +async def main() -> int: + ... + current = get_concurrency() + name = f"my_benchmark_c{current}" if current > 1 else "my_benchmark" + label = f"concurrency={current}" if current > 1 else "" + + async def run_session(index: int) -> None: + user_id = f"{box.conf.user_id}_{index}" + session_id = str(uuid.uuid4()) + for _ in SCRIPT: + events = await drain( + handle.run_turn( + app_name=box.conf.app_name, + user_id=user_id, + session_id=session_id, + message="Say ok.", + ), + ) + ensure_turn_completed(events, expected_text="ok", expect_tool_success=True) + + async def run_once() -> None: + await asyncio.gather(*(run_session(i) for i in range(current))) + + await run_benchmarks([ + PerformanceEval( + name=name, + display_name="my_benchmark", + label=label, + func=run_once, + num_iterations=get_iterations(), + warmup_runs=1, + ), + ]) + ... +``` + +`supports_concurrency=True` 后，`run_bench` 会为每个 `--concurrency` 值启动独立子进程。结果名带 `_cN` 后缀，每个 session 使用独立的 `user_id`。 + +#### 5. 禁用 benchmark + +```python +# bench/your_bench.py +import os + +from trpc_agent_bench.bench import BenchProfile + + +def get_bench_profile() -> BenchProfile: + if not os.environ.get("MY_BACKEND_URL"): + return BenchProfile( + usable=False, + skip_reason="MY_BACKEND_URL is not set", + ) + return BenchProfile() + + +async def main() -> int: + ... +``` + +`usable=False` 时，`run_bench` 打印 `[skip] ()` 并跳过，不影响其他 benchmark。 diff --git a/mkdocs.yml b/mkdocs.yml index beac2732..336129b9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -67,6 +67,8 @@ nav: - Human in the Loop: en/human_in_the_loop.md - Plan Mode: en/plan.md - Cancellation: en/cancel.md + - Benchmark: + - Framework cost bench: en/bench.md - 中文: - 概览: zh/index.md - Agent: @@ -115,3 +117,5 @@ nav: - Human in the Loop: zh/human_in_the_loop.md - Plan Mode: zh/plan.md - Cancellation: zh/cancel.md + - 性能测试: + - 框架性能压测: zh/bench.md diff --git a/pyproject.toml b/pyproject.toml index f512e178..490e305c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,6 +114,10 @@ eval = [ "tabulate", ] +bench = [ + "fastapi>=0.95.0", +] + optimize = [ "gepa>=0.0.7", "rich>=13.0.0", @@ -179,7 +183,7 @@ trpc_agent_cmd = "trpc_agent_sdk._cli:main" # 使用 include 保证三个包都包含进来 [tool.hatch.build.targets.wheel] -packages = ["trpc_agent_sdk"] +packages = ["trpc_agent_sdk", "trpc_agent_bench"] [project.urls] Homepage = "https://github.com/trpc-group/trpc-agent-python.git" diff --git a/tests/trpc_agent_bench/__init__.py b/tests/trpc_agent_bench/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/trpc_agent_bench/bench/__init__.py b/tests/trpc_agent_bench/bench/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/trpc_agent_bench/bench/test_builtin.py b/tests/trpc_agent_bench/bench/test_builtin.py new file mode 100644 index 00000000..4ee5f4ab --- /dev/null +++ b/tests/trpc_agent_bench/bench/test_builtin.py @@ -0,0 +1,161 @@ +"""Turn and loop benchmark scripts, profiles, and conversation construction.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +import pytest + +from trpc_agent_bench.bench._builtin import import_time +from trpc_agent_bench.bench._builtin import loop1 +from trpc_agent_bench.bench._builtin import loop5 +from trpc_agent_bench.bench._builtin import loop10 +from trpc_agent_bench.bench._builtin import memory_footprint +from trpc_agent_bench.bench._builtin import turn1 +from trpc_agent_bench.bench._builtin import turn5 +from trpc_agent_bench.bench._builtin import turn10 +from trpc_agent_bench.bench._builtin._script_fixtures import make_loop_script +from trpc_agent_bench.bench._builtin._script_fixtures import make_turn_script +from trpc_agent_bench.bench._builtin._utils._session_bench import _expected_text +from trpc_agent_bench.bench._builtin._utils._session_bench import _has_function +from trpc_agent_bench.bench._builtin._utils._session_bench import _make_conversation +from trpc_agent_bench.bench._builtin._utils._session_bench import _session_user_id +from trpc_agent_bench.bench import AgentHandle +from trpc_agent_bench.bench import FunctionBlock +from trpc_agent_bench.bench import TextBlock + + +def _event(): + part = SimpleNamespace( + text="ok", + thought=False, + function_call=None, + function_response=None, + ) + event = SimpleNamespace( + content=SimpleNamespace(parts=[part]), + partial=False, + turn_complete=True, + ) + event.is_error = lambda: False + return event + + +def test_concurrent_sessions_use_distinct_user_ids() -> None: + + class Handle(AgentHandle): + + def __init__(self) -> None: + self.user_ids = [] + self.session_ids = [] + self.app_names = [] + + async def run_turn(self, *, app_name, user_id, session_id, message): + self.user_ids.append(user_id) + self.session_ids.append(session_id) + self.app_names.append(app_name) + yield _event() + + handle = Handle() + func = _make_conversation( + handle=handle, + app_name="bench_app", + base_user_id="bench_user", + script=[[TextBlock()]], + current=3, + ) + asyncio.run(func()) + + assert sorted(handle.user_ids) == ["bench_user_0", "bench_user_1", "bench_user_2"] + assert sorted(handle.app_names) == ["bench_app"] * 3 + assert len(set(handle.session_ids)) == 3 + + +def test_session_user_id_appends_index() -> None: + assert [_session_user_id("bench_user", index) for index in range(3)] == [ + "bench_user_0", + "bench_user_1", + "bench_user_2", + ] + + +def test_turn_counts_are_fixed() -> None: + assert turn1.TURN_COUNT == 1 + assert turn5.TURN_COUNT == 5 + assert turn10.TURN_COUNT == 10 + + +def test_turn_scripts_are_single_chunk_text() -> None: + for module in (turn1, turn5, turn10): + assert len(module.TURN_SCRIPT) == module.TURN_COUNT + for turn in module.TURN_SCRIPT: + assert len(turn) == 1 + assert isinstance(turn[0], TextBlock) + assert turn[0].chunks == 1 + + +def test_loop_scripts_have_n_calls_and_n_minus_one_tool_rounds() -> None: + for module, llm_calls in ((loop1, 1), (loop5, 5), (loop10, 10)): + assert module.LLM_CALLS == llm_calls + assert module.TOOL_ROUNDS == llm_calls - 1 + assert len(module.TURN_SCRIPT) == 1 + + replies = module.TURN_SCRIPT[0] + assert len(replies) == llm_calls + for reply in replies[:-1]: + assert isinstance(reply, list) + assert all(isinstance(block, FunctionBlock) for block in reply) + assert isinstance(replies[-1], TextBlock) + assert replies[-1].chunks == 5 + + +def test_loop_tool_calls_use_distinct_arguments() -> None: + replies = make_loop_script(5)[0] + args = [reply[0].args for reply in replies[:-1]] + + assert args == [ + { + "a": 1, + "b": 2 + }, + { + "a": 2, + "b": 3 + }, + { + "a": 3, + "b": 4 + }, + { + "a": 4, + "b": 5 + }, + ] + + +def test_script_builders_reject_empty_scripts() -> None: + with pytest.raises(ValueError, match="user_turns"): + make_turn_script(0) + with pytest.raises(ValueError, match="llm_calls"): + make_loop_script(0) + + +def test_expected_text_and_tool_detection() -> None: + assert _expected_text([TextBlock("done")]) == "done" + assert _expected_text([[FunctionBlock("add_numbers", {"a": 1, "b": 2})], TextBlock("ok")]) == "ok" + assert _has_function([[FunctionBlock("add_numbers", {"a": 1, "b": 2})], TextBlock()]) is True + assert _has_function([TextBlock()]) is False + + +def test_turn_and_loop_modules_declare_profiles() -> None: + for module in (turn1, turn5, turn10, loop1, loop5, loop10): + profile = module.get_bench_profile() + assert profile.usable is True + assert profile.supports_concurrency is True + assert profile.requires_iterations is True + + +def test_fixed_sample_benchmarks_do_not_require_iterations() -> None: + assert import_time.get_bench_profile().requires_iterations is False + assert memory_footprint.get_bench_profile().requires_iterations is False diff --git a/tests/trpc_agent_bench/bench/test_common.py b/tests/trpc_agent_bench/bench/test_common.py new file mode 100644 index 00000000..46108310 --- /dev/null +++ b/tests/trpc_agent_bench/bench/test_common.py @@ -0,0 +1,33 @@ +"""Naming helpers shared by built-in benchmarks.""" + +from __future__ import annotations + +import pytest + +from trpc_agent_bench.bench._builtin._utils._common import concurrency_label +from trpc_agent_bench.bench._builtin._utils._common import new_session_id +from trpc_agent_bench.bench._builtin._utils._common import result_name +from trpc_agent_bench.bench._builtin._utils._common import result_suffix + + +def test_new_session_id_is_unique() -> None: + values = {new_session_id() for _ in range(5)} + + assert len(values) == 5 + + +def test_result_name_uses_run_bench_suffix(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("BENCH_CONCURRENCY", "1") + monkeypatch.delenv("BENCH_RESULT_SUFFIX", raising=False) + assert result_name("turn5") == "turn5" + assert result_suffix() == "" + assert concurrency_label() == "" + + monkeypatch.setenv("BENCH_CONCURRENCY", "8") + assert result_name("turn5") == "turn5_c8" + assert result_suffix() == "_c8" + assert concurrency_label() == "concurrency=8" + + # run_bench controls the suffix explicitly, including the no-suffix case. + monkeypatch.setenv("BENCH_RESULT_SUFFIX", "") + assert result_name("turn5") == "turn5" diff --git a/tests/trpc_agent_bench/bench/test_config.py b/tests/trpc_agent_bench/bench/test_config.py new file mode 100644 index 00000000..1ea55fdc --- /dev/null +++ b/tests/trpc_agent_bench/bench/test_config.py @@ -0,0 +1,41 @@ +"""Benchmark configuration objects.""" + +from __future__ import annotations + +from types import SimpleNamespace + +from trpc_agent_bench.bench import AgentBenchConf +from trpc_agent_bench.bench import ModelConf + + +def test_agent_bench_conf_defaults() -> None: + conf = AgentBenchConf() + + assert conf.app_name == "bench_app" + assert conf.user_id == "bench_user" + assert conf.agent_conf is None + assert conf.model_conf == ModelConf() + assert conf.base_url is None + assert not hasattr(conf, "session_service") + + +def test_agent_bench_conf_keeps_agent_conf() -> None: + agent_conf = SimpleNamespace(skills_path="/tmp/skills") + conf = AgentBenchConf( + app_name="my_app", + user_id="my_user", + agent_conf=agent_conf, + ) + + assert conf.app_name == "my_app" + assert conf.user_id == "my_user" + assert conf.agent_conf is agent_conf + + +def test_model_conf_is_mutable() -> None: + conf = ModelConf() + conf.request_delay_ms = 10 + conf.chunk_delay_ms = 2 + + assert conf.request_delay_ms == 10 + assert conf.chunk_delay_ms == 2 diff --git a/tests/trpc_agent_bench/bench/test_core.py b/tests/trpc_agent_bench/bench/test_core.py new file mode 100644 index 00000000..8127efdc --- /dev/null +++ b/tests/trpc_agent_bench/bench/test_core.py @@ -0,0 +1,254 @@ +"""Unit tests for stats, assertions, and behaviors.""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from trpc_agent_bench.bench._eval import ensure_turn_completed +from trpc_agent_bench.bench._eval import get_iterations +from trpc_agent_bench.bench._eval import percentile +from trpc_agent_bench.bench._eval import write_result +from trpc_agent_bench.bench._model import FunctionBlock +from trpc_agent_bench.bench._model import TextBlock +from trpc_agent_bench.bench._model import _split_text +from trpc_agent_bench.bench._model import reply_from_script +from trpc_agent_bench.bench._model import request_position +from trpc_agent_bench.bench._model import script_spec +from trpc_agent_bench.bench._model import turns_from_spec +from trpc_agent_bench.bench._setup import get_concurrency + + +def _part(text=None, function_call=None, function_response=None): + return SimpleNamespace( + text=text, + thought=False, + function_call=function_call, + function_response=function_response, + ) + + +def _event(parts, *, partial=False, turn_complete=False, error=False): + event = SimpleNamespace( + content=SimpleNamespace(parts=parts), + partial=partial, + turn_complete=turn_complete, + ) + event.is_error = lambda: error + return event + + +def test_percentile_inclusive() -> None: + assert percentile([], 50) == 0.0 + assert percentile([1.0], 95) == 1.0 + assert percentile([1.0, 2.0, 3.0, 4.0], 50) == 2.5 + assert percentile([1.0, 2.0, 3.0, 4.0], 95) == pytest.approx(3.85) + + +def test_get_iterations_reads_and_validates_env(monkeypatch: pytest.MonkeyPatch, ) -> None: + monkeypatch.delenv("BENCH_ITERATIONS", raising=False) + with pytest.raises(RuntimeError, match="BENCH_ITERATIONS is not set"): + get_iterations() + + monkeypatch.setenv("BENCH_ITERATIONS", "5") + assert get_iterations() == 5 + + monkeypatch.setenv("BENCH_ITERATIONS", "0") + with pytest.raises(ValueError, match="positive"): + get_iterations() + + monkeypatch.setenv("BENCH_ITERATIONS", "many") + with pytest.raises(ValueError, match="integer"): + get_iterations() + + +def test_get_concurrency_defaults_and_validates(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("BENCH_CONCURRENCY", raising=False) + assert get_concurrency() == 1 + + monkeypatch.setenv("BENCH_CONCURRENCY", "8") + assert get_concurrency() == 8 + + monkeypatch.setenv("BENCH_CONCURRENCY", "0") + with pytest.raises(ValueError, match="positive"): + get_concurrency() + + monkeypatch.setenv("BENCH_CONCURRENCY", "many") + with pytest.raises(ValueError, match="integer"): + get_concurrency() + + +def test_turn_completed_accepts_partial_only_text() -> None: + events = [_event([_part(text="ok")], partial=True)] + ensure_turn_completed(events, expected_text="ok") + + +def test_turn_completed_rejects_error() -> None: + events = [_event([_part(text="ok")], partial=True, error=True)] + with pytest.raises(RuntimeError, match="error event"): + ensure_turn_completed(events, expected_text="ok") + + +def test_turn_completed_rejects_missing_text() -> None: + events = [_event([])] + with pytest.raises(RuntimeError, match="did not complete"): + ensure_turn_completed(events) + + +def test_turn_completed_ignores_surrounding_whitespace() -> None: + events = [_event([_part(text="\n\ndone\n")], partial=True)] + ensure_turn_completed(events, expected_text="done") + + +def test_turn_completed_rejects_wrong_text() -> None: + events = [_event([_part(text="nope")], partial=True)] + with pytest.raises(RuntimeError, match="unexpected final text"): + ensure_turn_completed(events, expected_text="ok") + + +def test_tool_success_requires_call_and_response() -> None: + call = SimpleNamespace(name="add_numbers") + response = SimpleNamespace(name="add_numbers", response={"result": 3}) + events = [ + _event([_part(function_call=call)], partial=True), + _event([_part(function_response=response)]), + ] + ensure_turn_completed(events, expect_tool_success=True) + + +def test_text_block_chunks() -> None: + reply = reply_from_script(TextBlock("ok", chunks=3), turn=1, step=0) + assert reply.text == "ok" + assert reply.chunks == 3 + + +def test_function_block_reply_has_tool_call() -> None: + reply = reply_from_script( + [FunctionBlock("add_numbers", { + "a": 1, + "b": 2 + })], + turn=1, + step=0, + ) + assert reply.tool_calls[0]["name"] == "add_numbers" + assert reply.tool_calls[0]["args"] == {"a": 1, "b": 2} + assert reply.finish_reason == "tool_calls" + + +def test_parallel_function_blocks_have_distinct_arguments() -> None: + reply = reply_from_script( + [ + FunctionBlock("add_numbers", { + "a": 1, + "b": 2 + }), + FunctionBlock("add_numbers", { + "a": 2, + "b": 3 + }), + FunctionBlock("add_numbers", { + "a": 3, + "b": 4 + }), + ], + turn=1, + step=0, + ) + assert len(reply.tool_calls) == 3 + arguments = [call["args"] for call in reply.tool_calls] + assert len({(item["a"], item["b"]) for item in arguments}) == 3 + + +def test_split_text_returns_exact_chunk_count() -> None: + assert _split_text("ok", 10) == ["o", "k", "", "", "", "", "", "", "", ""] + assert _split_text("abcdef", 3) == ["ab", "cd", "ef"] + + +def test_script_spec_round_trip() -> None: + turns = [ + [TextBlock("ok", chunks=2)], + [ + [FunctionBlock("add_numbers", { + "a": 1, + "b": 2 + })], + TextBlock("done"), + ], + ] + assert turns_from_spec(script_spec(turns)) == turns + + +def test_request_position_counts_turns_and_replies() -> None: + messages = [ + { + "role": "user", + "content": "first" + }, + { + "role": "assistant", + "content": "ok" + }, + { + "role": "user", + "content": "second" + }, + { + "role": "assistant", + "content": "tool call" + }, + { + "role": "tool", + "content": "result" + }, + ] + assert request_position(messages) == (1, 1) + assert request_position([{"role": "user", "content": "only"}]) == (0, 0) + + +def test_write_result_uses_description_env_and_explicit_override( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("BENCH_QUIET", "1") + monkeypatch.setenv("BENCH_DESCRIPTION", "env description") + + payload = write_result( + name="from_env", + result={"median_run_time": 0.1}, + results_dir=tmp_path, + ) + assert payload["description"] == "env description" + assert json.loads((tmp_path / "from_env.json").read_text(encoding="utf-8"))["description"] == "env description" + + payload = write_result( + name="explicit", + result={"median_run_time": 0.1}, + description="explicit description", + results_dir=tmp_path, + ) + assert payload["description"] == "explicit description" + + +def test_write_result_keeps_display_name_and_label( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("BENCH_QUIET", "1") + + payload = write_result( + name="turn1_redis_c8", + display_name="turn1_redis", + label="concurrency=8", + result={"median_run_time": 0.1}, + results_dir=tmp_path, + ) + + assert payload["display_name"] == "turn1_redis" + assert payload["label"] == "concurrency=8" + stored = json.loads((tmp_path / "turn1_redis_c8.json").read_text(encoding="utf-8")) + assert stored["display_name"] == "turn1_redis" + assert stored["label"] == "concurrency=8" diff --git a/tests/trpc_agent_bench/bench/test_loading.py b/tests/trpc_agent_bench/bench/test_loading.py new file mode 100644 index 00000000..9ef33c3c --- /dev/null +++ b/tests/trpc_agent_bench/bench/test_loading.py @@ -0,0 +1,55 @@ +"""Module-based target loading.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from trpc_agent_bench.bench._target import load_target + + +def _write_module(root: Path, module: str, source: str) -> None: + parts = module.split(".") + for index in range(1, len(parts)): + package = root.joinpath(*parts[:index]) + package.mkdir(parents=True, exist_ok=True) + (package / "__init__.py").write_text("", encoding="utf-8") + path = root.joinpath(*parts).with_suffix(".py") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source, encoding="utf-8") + + +def test_load_target_from_module(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _write_module(tmp_path, "target_pkg_a.my_target", "class Target: pass\nTARGET = Target()\n") + monkeypatch.syspath_prepend(str(tmp_path)) + + target = load_target("target_pkg_a.my_target") + + assert type(target).__name__ == "Target" + + +def test_load_target_uses_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _write_module(tmp_path, "target_pkg_b.env_target", "TARGET = object()\n") + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.setenv("BENCH_TARGET", "target_pkg_b.env_target") + + assert load_target() is not None + + +def test_load_target_requires_target_attribute( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _write_module(tmp_path, "target_pkg_c.no_target", "X = 1\n") + monkeypatch.syspath_prepend(str(tmp_path)) + + with pytest.raises(RuntimeError, match="does not define TARGET"): + load_target("target_pkg_c.no_target") + + +def test_load_target_requires_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("BENCH_TARGET", raising=False) + + with pytest.raises(RuntimeError, match="BENCH_TARGET is not set"): + load_target() diff --git a/tests/trpc_agent_bench/bench/test_mock_openai_server.py b/tests/trpc_agent_bench/bench/test_mock_openai_server.py new file mode 100644 index 00000000..e133f503 --- /dev/null +++ b/tests/trpc_agent_bench/bench/test_mock_openai_server.py @@ -0,0 +1,175 @@ +"""The HTTP mock must speak enough OpenAI wire format for the SDK clients.""" + +from __future__ import annotations + +import time + +import pytest + +from openai import OpenAI + +from trpc_agent_bench.bench import FunctionBlock +from trpc_agent_bench.bench import ModelConf +from trpc_agent_bench.bench import TextBlock +from trpc_agent_bench.bench._model import MockOpenAIServer + + +@pytest.fixture() +def text_server(): + server = MockOpenAIServer([[TextBlock("ok", chunks=2)]]) + server.start() + try: + yield server + finally: + server.stop() + + +@pytest.fixture() +def tool_server(): + server = MockOpenAIServer([ + [ + [FunctionBlock("add_numbers", { + "a": 1, + "b": 2 + })], + TextBlock("ok"), + ], + ]) + server.start() + try: + yield server + finally: + server.stop() + + +def test_non_stream_text(text_server: MockOpenAIServer) -> None: + client = OpenAI(api_key="bench", base_url=text_server.base_url) + response = client.chat.completions.create( + model="bench-model", + messages=[{ + "role": "user", + "content": "hi" + }], + ) + assert response.choices[0].message.content == "ok" + assert response.choices[0].finish_reason == "stop" + + +def test_stream_text(text_server: MockOpenAIServer) -> None: + client = OpenAI(api_key="bench", base_url=text_server.base_url) + stream = client.chat.completions.create( + model="bench-model", + messages=[{ + "role": "user", + "content": "hi" + }], + stream=True, + stream_options={"include_usage": True}, + ) + chunks = [] + usage = None + for chunk in stream: + if chunk.choices: + delta = chunk.choices[0].delta + if delta.content: + chunks.append(delta.content) + elif chunk.usage: + usage = chunk.usage + assert "".join(chunks) == "ok" + assert usage is not None and usage.total_tokens == 17 + + +def test_stream_applies_request_and_chunk_latency() -> None: + server = MockOpenAIServer( + [[TextBlock("abcdef", chunks=3)]], + ModelConf(request_delay_ms=100, chunk_delay_ms=20), + ) + server.start() + try: + client = OpenAI(api_key="bench", base_url=server.base_url) + started = time.perf_counter() + stream = client.chat.completions.create( + model="bench-model", + messages=[{ + "role": "user", + "content": "hi" + }], + stream=True, + ) + first_chunk_at = None + chunks = [] + for chunk in stream: + if first_chunk_at is None: + first_chunk_at = time.perf_counter() - started + if chunk.choices: + content = chunk.choices[0].delta.content + if content: + chunks.append(content) + elapsed = time.perf_counter() - started + finally: + server.stop() + + assert "".join(chunks) == "abcdef" + assert first_chunk_at is not None + assert first_chunk_at >= 0.08 + assert elapsed >= 0.11 + + +def test_stream_tool_call(tool_server: MockOpenAIServer) -> None: + client = OpenAI(api_key="bench", base_url=tool_server.base_url) + stream = client.chat.completions.create( + model="bench-model", + messages=[{ + "role": "user", + "content": "add" + }], + stream=True, + ) + calls = [] + finish_reason = None + for chunk in stream: + if not chunk.choices: + continue + delta = chunk.choices[0].delta + if delta.tool_calls: + calls.extend(delta.tool_calls) + if chunk.choices[0].finish_reason: + finish_reason = chunk.choices[0].finish_reason + assert finish_reason == "tool_calls" + assert len(calls) == 1 + assert calls[0].function.name == "add_numbers" + assert calls[0].function.arguments == '{"a": 1, "b": 2}' + + +def test_non_stream_after_tool_result(tool_server: MockOpenAIServer) -> None: + client = OpenAI(api_key="bench", base_url=tool_server.base_url) + response = client.chat.completions.create( + model="bench-model", + messages=[ + { + "role": "user", + "content": "add" + }, + { + "role": + "assistant", + "content": + None, + "tool_calls": [ + { + "id": "call_1_0_0", + "type": "function", + "function": { + "name": "add_numbers", + "arguments": '{"a": 1, "b": 2}', + }, + }, + ], + }, + { + "role": "tool", + "content": "3" + }, + ], + ) + assert response.choices[0].message.content == "ok" diff --git a/tests/trpc_agent_bench/bench/test_module_imports.py b/tests/trpc_agent_bench/bench/test_module_imports.py new file mode 100644 index 00000000..0849582c --- /dev/null +++ b/tests/trpc_agent_bench/bench/test_module_imports.py @@ -0,0 +1,39 @@ +"""Generic modules must not import agent implementations.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +PACKAGE = Path(__file__).resolve().parents[3] / "trpc_agent_bench" +FORBIDDEN = ( + "trpc_harness", + "trpc_agent_sdk.agents", + "trpc_agent_sdk.models", + "trpc_agent_sdk.runners", + "trpc_agent_sdk.sessions", +) + + +def _generic_files() -> list[Path]: + return sorted(PACKAGE.rglob("*.py")) + + +def _imports(path: Path) -> set[str]: + tree = ast.parse(path.read_text(encoding="utf-8")) + names: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + names.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + names.add(node.module) + return names + + +def test_generic_modules_do_not_import_agent_implementations() -> None: + offenders = [] + for path in _generic_files(): + imports = _imports(path) + if any(name == forbidden or name.startswith(f"{forbidden}.") for name in imports for forbidden in FORBIDDEN): + offenders.append(path.name) + assert not offenders, f"generic modules import agent implementations: {offenders}" diff --git a/tests/trpc_agent_bench/bench/test_public_api.py b/tests/trpc_agent_bench/bench/test_public_api.py new file mode 100644 index 00000000..c06b6247 --- /dev/null +++ b/tests/trpc_agent_bench/bench/test_public_api.py @@ -0,0 +1,182 @@ +"""Public benchmark API and target contracts.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from trpc_agent_sdk.tools import BaseTool +from trpc_agent_sdk.tools import FunctionTool + +import trpc_agent_bench.bench as bench_api +from trpc_agent_bench.bench import AgentBenchConf +from trpc_agent_bench.bench import AgentTarget +from trpc_agent_bench.bench import BenchProfile +from trpc_agent_bench.bench import FunctionBlock +from trpc_agent_bench.bench import ModelConf +from trpc_agent_bench.bench import Reply +from trpc_agent_bench.bench import TextBlock +from trpc_agent_bench.bench import Turn +from trpc_agent_bench.bench import resolve_tools +from trpc_agent_bench.bench import teardown +from trpc_agent_bench.bench import _setup + + +def _add(a: int, b: int) -> int: + """Add two numbers.""" + return a + b + + +def test_public_bench_api_is_minimal() -> None: + assert set(bench_api.__all__) == { + "AgentBenchConf", + "AgentHandle", + "AgentTarget", + "BenchProfile", + "BenchSetup", + "FunctionBlock", + "ModelConf", + "PerformanceEval", + "PerformanceResult", + "Reply", + "TextBlock", + "ToolsFactory", + "Turn", + "drain", + "ensure_turn_completed", + "get_concurrency", + "get_iterations", + "resolve_tools", + "run_benchmarks", + "setup", + "teardown", + } + assert not hasattr(bench_api, "MockOpenAIServer") + assert not hasattr(bench_api, "load_target") + assert not hasattr(bench_api, "script_spec") + assert not hasattr(bench_api, "write_result") + + +def test_core_exports_script_types() -> None: + assert Reply == TextBlock | list[FunctionBlock] + assert Turn == list[Reply] + + +def test_resolve_tools_accepts_sync_and_async_factories() -> None: + + def sync_factory() -> list[BaseTool]: + return [FunctionTool(_add)] + + async def async_factory() -> list[BaseTool]: + return [FunctionTool(_add)] + + assert isinstance(asyncio.run(resolve_tools(sync_factory))[0], BaseTool) + assert isinstance(asyncio.run(resolve_tools(async_factory))[0], BaseTool) + + +def test_agent_target_is_abstract() -> None: + with pytest.raises(TypeError): + AgentTarget() + + +def test_bench_profile_defaults() -> None: + profile = BenchProfile() + + assert profile.usable is True + assert profile.supports_concurrency is False + assert profile.requires_iterations is True + assert profile.skip_reason == "" + + +def test_setup_and_teardown_own_server_and_target_resources(monkeypatch: pytest.MonkeyPatch, ) -> None: + captured = {} + + class FakeServer: + + def __init__(self, turns, conf): + captured["turns"] = turns + captured["model_conf"] = conf + self.base_url = None + self.started = False + self.stopped = False + + def start(self) -> str: + self.started = True + self.base_url = "http://mock.test/v1" + return self.base_url + + def stop(self) -> None: + self.stopped = True + + class FakeTarget(AgentTarget): + name = "fake" + import_module = "fake" + + def __init__(self) -> None: + self.seen_conf = None + self.cleaned = None + + async def prepare(self, *, conf): + self.seen_conf = conf + return "prepared" + + async def instantiate(self, prepared, *, tools_factory): + return None + + async def handle(self, prepared, *, tools_factory): + return None + + async def cleanup(self, prepared): + self.cleaned = prepared + + target = FakeTarget() + monkeypatch.setattr(_setup, "MockOpenAIServer", FakeServer) + monkeypatch.setattr(_setup, "load_target", lambda: target) + + conf = AgentBenchConf(model_conf=ModelConf(request_delay_ms=25, chunk_delay_ms=5), ) + box = asyncio.run(_setup.setup([[TextBlock()]], conf=conf)) + + assert box.target is target + assert box.conf is conf + assert box.prepared == "prepared" + assert conf.base_url == "http://mock.test/v1" + assert isinstance(box._server, FakeServer) + assert box._server.started is True + assert captured["model_conf"] is conf.model_conf + + asyncio.run(teardown(box)) + + assert target.cleaned == "prepared" + assert box._server.stopped is True + + +def test_setup_failure_resets_base_url_and_stops_server(monkeypatch: pytest.MonkeyPatch, ) -> None: + servers = [] + + class FakeServer: + + def __init__(self, turns, conf): + self.stopped = False + servers.append(self) + + def start(self) -> str: + return "http://mock.test/v1" + + def stop(self) -> None: + self.stopped = True + + class BadTarget: + + async def prepare(self, *, conf): + raise RuntimeError("prepare failed") + + monkeypatch.setattr(_setup, "MockOpenAIServer", FakeServer) + monkeypatch.setattr(_setup, "load_target", lambda: BadTarget()) + + conf = AgentBenchConf() + with pytest.raises(RuntimeError, match="prepare failed"): + asyncio.run(_setup.setup([[TextBlock()]], conf=conf)) + + assert conf.base_url is None + assert servers[0].stopped is True diff --git a/tests/trpc_agent_bench/bench/test_report.py b/tests/trpc_agent_bench/bench/test_report.py new file mode 100644 index 00000000..1dc28529 --- /dev/null +++ b/tests/trpc_agent_bench/bench/test_report.py @@ -0,0 +1,348 @@ +"""Report generation: descriptions, baseline deltas, and table shape.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +from trpc_agent_bench.bench._report import build_report + + +def _write_result( + root: Path, + *, + target: str, + name: str, + description: str = "", + median_run_time: float = 0.0, + p95_run_time: float = 0.0, + median_memory_usage: float = 0.0, + display_name: str = "", + label: str = "", +) -> None: + directory = root / target + directory.mkdir(parents=True, exist_ok=True) + payload = { + "name": name, + "display_name": display_name or name, + "label": label, + "description": description, + "target": target, + "result": { + "run_times": [median_run_time] if median_run_time else [], + "memory_usages": [median_memory_usage] if median_memory_usage else [], + "median_run_time": median_run_time, + "p95_run_time": p95_run_time, + "median_memory_usage": median_memory_usage, + }, + } + (directory / f"{name}.json").write_text(json.dumps(payload), encoding="utf-8") + + +def _markdown_cell_counts(report: Path) -> list[int]: + counts = [] + for line in report.read_text(encoding="utf-8").splitlines(): + if not line.startswith("|"): + continue + counts.append(line.strip().strip("|").count("|") + 1) + return counts + + +def _markdown_benchmark_names(markdown: str) -> list[str]: + names = [] + for line in markdown.splitlines(): + if not line.startswith("| "): + continue + cells = [cell.strip() for cell in line.strip().strip("|").split("|")] + if not cells or cells[0] == "benchmark": + continue + names.append(cells[0]) + return names + + +def _html_header_count(document: str) -> int: + thead = re.search(r"(.*?)", document, re.S) + assert thead is not None + return len(re.findall(r" list[int]: + tbody = re.search(r"(.*?)", document, re.S) + assert tbody is not None + rows = re.findall(r"(.*?)", tbody.group(1), re.S) + return [row.count(" None: + _write_result( + tmp_path, + target="hermes_v1", + name="import_time", + description="新进程 import 耗时。", + median_run_time=5.23, + p95_run_time=5.28, + ) + _write_result( + tmp_path, + target="hermes_v2", + name="import_time", + description="新进程 import 耗时。", + median_run_time=4.28, + p95_run_time=5.17, + ) + _write_result( + tmp_path, + target="hermes_v1", + name="run_agent", + description="单轮文本 turn。", + median_run_time=0.1, + p95_run_time=0.12, + ) + _write_result( + tmp_path, + target="hermes_v2", + name="run_agent", + description="单轮文本 turn。", + median_run_time=0.03, + p95_run_time=0.04, + ) + _write_result( + tmp_path, + target="hermes_v1", + name="memory_per_agent", + description="每个常驻 agent 占用的内存。", + median_memory_usage=0.0048, + ) + _write_result( + tmp_path, + target="hermes_v2", + name="memory_per_agent", + description="每个常驻 agent 占用的内存。", + median_memory_usage=0.0085, + ) + + build_report(tmp_path, ["hermes_v1", "hermes_v2"], baseline="hermes_v1") + + counts = _markdown_cell_counts(tmp_path / "report.md") + assert set(counts) == {7}, counts + + markdown = (tmp_path / "report.md").read_text(encoding="utf-8") + assert "| group |" not in markdown + assert "| extra |" not in markdown + assert "| 说明 |" not in markdown + assert "## hermes_v1 vs hermes_v2" in markdown + assert ("| benchmark | baseline p50 | hermes_v2 p50 | " + "baseline p95 | hermes_v2 p95 | " + "baseline memory | hermes_v2 memory |") in markdown + assert "单轮文本 turn。" in markdown + assert "-18.2%" in markdown + assert "-70.0%" in markdown + assert "+77.1%" in markdown + assert _markdown_benchmark_names(markdown) == sorted( + _markdown_benchmark_names(markdown), + key=str.lower, + ) + + memory_row = next(line for line in markdown.splitlines() if line.startswith("| ") and "memory_per_agent" in line) + assert memory_row.endswith("| - | - | - | - | 4.9 KiB | 8.7 KiB +77.1% |") + + document = (tmp_path / "report.html").read_text(encoding="utf-8") + assert "color-scheme: light" in document + assert "background: #ffffff" in document + assert "color: #000000" in document + assert "group" not in document + assert "extra" not in document + assert "class='bar'" not in document + assert "th { background: #f2f2f2; font-weight: normal; }" in document + assert "th.benchmark, th.target { font-weight: 700; }" in document + assert "td.benchmark, td.target { font-weight: 600; }" in document + assert "td.extra" not in document + assert "benchmark" in document + assert "hermes_v2 p50" in document + assert "" in document + assert "" in document + assert document.count("") == 1 + assert _html_header_count(document) == 7 + assert set(_html_row_counts(document)) == {7} + assert "单轮文本 turn。" in document + assert "1.77x" not in document + + +def test_single_target_report_has_description_and_no_baseline(tmp_path: Path) -> None: + _write_result( + tmp_path, + target="hermes_v2", + name="run_agent", + description="单轮文本 turn。", + median_run_time=0.03, + p95_run_time=0.04, + ) + _write_result( + tmp_path, + target="hermes_v2", + name="memory_per_agent", + description="每个常驻 agent 占用的内存。", + median_memory_usage=0.0085, + ) + + build_report(tmp_path, ["hermes_v2"]) + + counts = _markdown_cell_counts(tmp_path / "report.md") + assert set(counts) == {4}, counts + markdown = (tmp_path / "report.md").read_text(encoding="utf-8") + assert "| benchmark | p50 | p95 | memory |" in markdown + assert "baseline" not in markdown + assert "单轮文本 turn。" in markdown + + document = (tmp_path / "report.html").read_text(encoding="utf-8") + assert _html_header_count(document) == 4 + assert set(_html_row_counts(document)) == {4} + assert "单轮文本 turn。" in document + + +def test_baseline_can_be_any_target_and_creates_one_table_per_comparison(tmp_path: Path, ) -> None: + for target, median in (("target_a", 0.1), ("target_b", 0.05), ("target_c", 0.2)): + _write_result( + tmp_path, + target=target, + name="run_agent", + description="单轮文本 turn。", + median_run_time=median, + p95_run_time=median * 1.2, + ) + + build_report(tmp_path, ["target_a", "target_b", "target_c"], baseline="target_b") + + markdown = (tmp_path / "report.md").read_text(encoding="utf-8") + assert "## target_b vs target_a" in markdown + assert "## target_b vs target_c" in markdown + assert "## target_b vs baseline" not in markdown + assert "+100.0%" in markdown + assert "+300.0%" in markdown + + document = (tmp_path / "report.html").read_text(encoding="utf-8") + assert document.count("
") == 2 + + +def test_missing_description_and_missing_target_show_dash(tmp_path: Path) -> None: + _write_result( + tmp_path, + target="hermes_v1", + name="run_agent", + median_run_time=0.1, + p95_run_time=0.12, + ) + _write_result( + tmp_path, + target="hermes_v1", + name="import_time", + median_run_time=5.23, + p95_run_time=5.28, + ) + _write_result( + tmp_path, + target="hermes_v2", + name="run_agent", + median_run_time=0.03, + p95_run_time=0.04, + ) + + build_report(tmp_path, ["hermes_v1", "hermes_v2"], baseline="hermes_v1") + + markdown = (tmp_path / "report.md").read_text(encoding="utf-8") + import_row = next(line for line in markdown.splitlines() if "| import_time |" in line) + assert "| import_time | 5.23 s | - | 5.28 s | - | - | - |" in import_row + run_row = next(line for line in markdown.splitlines() if "| run_agent |" in line) + assert ("| run_agent | 100.00 ms | 30.00 ms -70.0% | " + "120.00 ms | 40.00 ms -66.7% | - | - |") in run_row + + +def test_turn_is_time_and_memory_and_stray_summary_is_ignored(tmp_path: Path) -> None: + _write_result( + tmp_path, + target="hermes_v1", + name="turn10", + median_run_time=0.1, + p95_run_time=0.12, + median_memory_usage=0.5, + ) + _write_result( + tmp_path, + target="hermes_v2", + name="turn10", + median_run_time=0.05, + p95_run_time=0.06, + median_memory_usage=0.7, + ) + stray = tmp_path / "quick" + stray.mkdir() + (stray / "summary.json").write_text(json.dumps({"results": []}), encoding="utf-8") + + build_report(tmp_path, ["hermes_v1", "hermes_v2"], baseline="hermes_v1") + + markdown = (tmp_path / "report.md").read_text(encoding="utf-8") + row = next(line for line in markdown.splitlines() if line.startswith("| ") and "turn10" in line) + assert "ms" in row + assert "KiB" in row + assert "-50.0%" in row + assert "unknown" not in markdown + + +def test_report_renders_concurrency_sublabel(tmp_path: Path) -> None: + _write_result( + tmp_path, + target="hermes_v2", + name="turn1_redis_c8", + display_name="turn1_redis", + label="concurrency=8", + description="单轮 Redis。", + median_run_time=0.03, + p95_run_time=0.04, + ) + + build_report(tmp_path, ["hermes_v2"]) + + markdown = (tmp_path / "report.md").read_text(encoding="utf-8") + assert "| turn1_redis (concurrency=8) |" in markdown + + document = (tmp_path / "report.html").read_text(encoding="utf-8") + assert "
concurrency=8
" in document + + +def test_turn_benchmarks_are_ordered_by_turn_count(tmp_path: Path) -> None: + for name in ("turn10", "turn1", "turn5"): + _write_result( + tmp_path, + target="hermes_v2", + name=name, + median_run_time=0.1, + p95_run_time=0.12, + ) + + build_report(tmp_path, ["hermes_v2"]) + + markdown = (tmp_path / "report.md").read_text(encoding="utf-8") + assert _markdown_benchmark_names(markdown) == ["turn1", "turn5", "turn10"] + + +def test_loop_benchmarks_follow_turns_and_sort_by_call_count(tmp_path: Path) -> None: + for name in ("loop10", "turn5", "loop1", "turn1", "loop5"): + _write_result( + tmp_path, + target="hermes_v2", + name=name, + median_run_time=0.1, + p95_run_time=0.12, + ) + + build_report(tmp_path, ["hermes_v2"]) + + markdown = (tmp_path / "report.md").read_text(encoding="utf-8") + assert _markdown_benchmark_names(markdown) == [ + "turn1", + "turn5", + "loop1", + "loop5", + "loop10", + ] diff --git a/tests/trpc_agent_bench/bench/test_runner.py b/tests/trpc_agent_bench/bench/test_runner.py new file mode 100644 index 00000000..9811ab4d --- /dev/null +++ b/tests/trpc_agent_bench/bench/test_runner.py @@ -0,0 +1,75 @@ +"""Benchmark module loading and the subprocess entry point.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from trpc_agent_bench.bench import _runner + + +def _write_module(root: Path, module: str, source: str) -> None: + parts = module.split(".") + for index in range(1, len(parts)): + package = root.joinpath(*parts[:index]) + package.mkdir(parents=True, exist_ok=True) + (package / "__init__.py").write_text("", encoding="utf-8") + path = root.joinpath(*parts).with_suffix(".py") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source, encoding="utf-8") + + +def test_load_benchmark_returns_main(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _write_module(tmp_path, "bench_pkg_a.my_bench", "def main():\n return 7\n") + monkeypatch.syspath_prepend(str(tmp_path)) + + entry = _runner.load_benchmark("bench_pkg_a.my_bench") + + assert entry() == 7 + + +def test_runner_awaits_async_main(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _write_module( + tmp_path, + "bench_pkg_async.async_bench", + "async def main():\n return 9\n", + ) + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.setenv("BENCH_BENCHMARK", "bench_pkg_async.async_bench") + + assert _runner.main() == 9 + + +def test_load_benchmark_requires_main(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _write_module(tmp_path, "bench_pkg_b.no_main", "X = 1\n") + monkeypatch.syspath_prepend(str(tmp_path)) + + with pytest.raises(RuntimeError, match="does not define main"): + _runner.load_benchmark("bench_pkg_b.no_main") + + +def test_load_benchmark_sets_description( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _write_module( + tmp_path, + "bench_pkg_c.with_description", + 'DESCRIPTION = "bench description"\ndef main():\n return 7\n', + ) + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.delenv("BENCH_DESCRIPTION", raising=False) + + entry = _runner.load_benchmark("bench_pkg_c.with_description") + + assert entry() == 7 + assert os.environ["BENCH_DESCRIPTION"] == "bench description" + + +def test_runner_requires_benchmark_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("BENCH_BENCHMARK", raising=False) + + with pytest.raises(RuntimeError, match="BENCH_BENCHMARK is not set"): + _runner.main() diff --git a/tests/trpc_agent_bench/test_run_bench.py b/tests/trpc_agent_bench/test_run_bench.py new file mode 100644 index 00000000..0079f784 --- /dev/null +++ b/tests/trpc_agent_bench/test_run_bench.py @@ -0,0 +1,378 @@ +"""run_bench command, environment construction, and skip/concurrency behavior.""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +import pytest + +from trpc_agent_bench import run_bench + + +@pytest.fixture(autouse=True) +def _clear_backend_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("BENCH_REDIS_URL", raising=False) + monkeypatch.delenv("BENCH_SQL_URL", raising=False) + + +def _run( + argv: list[str], + monkeypatch: pytest.MonkeyPatch, +) -> list[tuple[list[str], dict[str, object]]]: + calls: list[tuple[list[str], dict[str, object]]] = [] + real_run = run_bench.subprocess.run + + def fake_run(command: list[str], **kwargs: object): + if command[:3] == [sys.executable, "-m", "trpc_agent_bench.bench._runner"]: + calls.append((command, kwargs)) + return run_bench.subprocess.CompletedProcess(command, 0) + return real_run(command, **kwargs) + + monkeypatch.setattr(run_bench.subprocess, "run", fake_run) + monkeypatch.setattr(sys, "argv", ["run_bench.py", *argv]) + assert run_bench.main() == 0 + return calls + + +def _environments(calls: list[tuple[list[str], dict[str, object]]]) -> list[dict[str, object]]: + return [kwargs["env"] for _, kwargs in calls] # type: ignore[list-item] + + +def test_run_bench_builds_module_commands( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = _run( + [ + "--targets", + "target_pkg_d.agent_a,target_pkg_d.agent_b", + "--baseline", + "target_pkg_d.agent_a", + "--benchmarks", + "turn1,turn5", + "--iterations", + "3", + "--results-dir", + str(tmp_path / "results"), + ], + monkeypatch, + ) + + assert len(calls) == 4 + command, kwargs = calls[0] + assert command == [sys.executable, "-m", "trpc_agent_bench.bench._runner"] + environment = kwargs["env"] + assert isinstance(environment, dict) + assert environment["BENCH_TARGET"] == "target_pkg_d.agent_a" + assert environment["BENCH_BENCHMARK"] == "trpc_agent_bench.bench._builtin.turn1" + assert environment["BENCH_ITERATIONS"] == "3" + assert environment["BENCH_CONCURRENCY"] == "1" + assert environment["BENCH_RESULT_SUFFIX"] == "" + assert environment["BENCH_RESULTS_DIR"] == str((tmp_path / "results" / "target_pkg_d.agent_a").resolve()) + pythonpath = environment["PYTHONPATH"] + assert isinstance(pythonpath, str) + assert str(run_bench.BENCH_DIR) in pythonpath.split(os.pathsep) + + builtin_benchmarks = [ + env["BENCH_BENCHMARK"] for env in _environments(calls) + if str(env["BENCH_BENCHMARK"]).startswith("trpc_agent_bench.bench._builtin.") + ] + assert builtin_benchmarks == [ + "trpc_agent_bench.bench._builtin.turn1", + "trpc_agent_bench.bench._builtin.turn5", + "trpc_agent_bench.bench._builtin.turn1", + "trpc_agent_bench.bench._builtin.turn5", + ] + + summary = json.loads((tmp_path / "results" / "summary.json").read_text(encoding="utf-8")) + assert summary["targets"] == ["target_pkg_d.agent_a", "target_pkg_d.agent_b"] + assert summary["baseline"] == "target_pkg_d.agent_a" + + +def test_default_benchmarks_have_no_backend_variants( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = _run( + [ + "--targets", + "target_pkg_d.agent_a", + "--iterations", + "1", + "--results-dir", + str(tmp_path / "results"), + ], + monkeypatch, + ) + + names = [str(env["BENCH_BENCHMARK"]) for env in _environments(calls)] + assert len(names) == 12 + assert all(not name.endswith(("_redis", "_sql")) for name in names) + assert "trpc_agent_bench.bench._builtin.turn1" in names + assert "trpc_agent_bench.bench._builtin.turn10" in names + assert "trpc_agent_bench.bench._builtin.loop1" in names + assert "trpc_agent_bench.bench._builtin.loop10" in names + assert "trpc_agent_bench.bench._builtin.streaming" not in names + + +def test_unusable_benchmark_is_skipped( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + module = tmp_path / "bench_pkg_skip" + module.mkdir() + (module / "__init__.py").write_text("", encoding="utf-8") + (module / "skipped.py").write_text( + "from trpc_agent_bench.bench import BenchProfile\n" + "def get_bench_profile():\n" + " return BenchProfile(usable=False, skip_reason='not available')\n" + "def main():\n" + " return 0\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(tmp_path)) + + calls = _run( + [ + "--targets", + "target_pkg_d.agent_a", + "--benchmarks", + "bench_pkg_skip.skipped", + "--results-dir", + str(tmp_path / "results"), + ], + monkeypatch, + ) + + assert calls == [] + captured = capsys.readouterr() + assert "[skip] bench_pkg_skip.skipped (not available)" in captured.out + + +def test_concurrency_expands_only_supported_benchmarks( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = _run( + [ + "--targets", + "target_pkg_d.agent_a", + "--benchmarks", + "turn1,import_time", + "--concurrency", + "1,8", + "--iterations", + "1", + "--results-dir", + str(tmp_path / "results"), + ], + monkeypatch, + ) + + environments = _environments(calls) + assert len(environments) == 3 + turn1_envs = [env for env in environments if env["BENCH_BENCHMARK"].endswith(".turn1")] + import_envs = [env for env in environments if env["BENCH_BENCHMARK"].endswith(".import_time")] + assert [(env["BENCH_CONCURRENCY"], env["BENCH_RESULT_SUFFIX"]) for env in turn1_envs] == [ + ("1", ""), + ("8", "_c8"), + ] + assert [(env["BENCH_CONCURRENCY"], env["BENCH_RESULT_SUFFIX"]) for env in import_envs] == [ + ("1", ""), + ] + + +def test_multiple_targets_require_baseline( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr( + sys, + "argv", + [ + "run_bench.py", + "--targets", + "target_pkg_d.agent_a,target_pkg_d.agent_b", + "--benchmarks", + "turn1", + "--iterations", + "1", + "--results-dir", + str(tmp_path / "results"), + ], + ) + + with pytest.raises(SystemExit) as exc: + run_bench.main() + + assert exc.value.code == 2 + assert "--baseline is required" in capsys.readouterr().err + + +def test_baseline_must_be_in_targets( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr( + sys, + "argv", + [ + "run_bench.py", + "--targets", + "target_pkg_d.agent_a,target_pkg_d.agent_b", + "--baseline", + "target_pkg_d.other", + "--benchmarks", + "turn1", + "--iterations", + "1", + "--results-dir", + str(tmp_path / "results"), + ], + ) + + with pytest.raises(SystemExit) as exc: + run_bench.main() + + assert exc.value.code == 2 + assert "must be one of --targets" in capsys.readouterr().err + + +def test_invalid_concurrency_is_rejected( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr( + sys, + "argv", + [ + "run_bench.py", + "--targets", + "target_pkg_d.agent_a", + "--benchmarks", + "turn1", + "--concurrency", + "0", + "--iterations", + "1", + "--results-dir", + str(tmp_path / "results"), + ], + ) + + with pytest.raises(SystemExit) as exc: + run_bench.main() + + assert exc.value.code == 2 + assert "--concurrency values must be positive" in capsys.readouterr().err + + +def test_fixed_benchmarks_do_not_require_iterations( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("BENCH_ITERATIONS", raising=False) + + calls = _run( + [ + "--targets", + "target_pkg_d.agent_a", + "--benchmarks", + "import_time,memory_footprint", + "--results-dir", + str(tmp_path / "results"), + ], + monkeypatch, + ) + + assert len(calls) == 2 + assert all("BENCH_ITERATIONS" not in env for env in _environments(calls)) + + +def test_iterations_required_for_selected_benchmarks( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr( + sys, + "argv", + [ + "run_bench.py", + "--targets", + "target_pkg_d.agent_a", + "--benchmarks", + "turn1", + "--results-dir", + str(tmp_path / "results"), + ], + ) + + with pytest.raises(SystemExit) as exc: + run_bench.main() + + assert exc.value.code == 2 + assert "--iterations is required for: turn1" in capsys.readouterr().err + + +def test_missing_profile_defaults_to_usable_without_concurrency( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = tmp_path / "bench_pkg_e" + module.mkdir() + (module / "__init__.py").write_text("", encoding="utf-8") + (module / "no_hooks.py").write_text("def main():\n return 0\n", encoding="utf-8") + monkeypatch.syspath_prepend(str(tmp_path)) + + calls = _run( + [ + "--targets", + "target_pkg_d.agent_a", + "--benchmarks", + "bench_pkg_e.no_hooks", + "--concurrency", + "2", + "--iterations", + "1", + "--results-dir", + str(tmp_path / "results"), + ], + monkeypatch, + ) + + assert len(calls) == 1 + environment = _environments(calls)[0] + assert environment["BENCH_CONCURRENCY"] == "1" + assert environment["BENCH_RESULT_SUFFIX"] == "" + assert environment["BENCH_ITERATIONS"] == "1" + + +def test_preflight_import_sees_target_and_restores_env( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = tmp_path / "bench_pkg_f" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "probe.py").write_text( + "import os\n" + "SEEN = os.environ.get('BENCH_TARGET')\n" + "def main():\n" + " return 0\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.delenv("BENCH_TARGET", raising=False) + + module = run_bench._load_benchmark_module("bench_pkg_f.probe", "target_pkg_d.agent_a") + + assert module.SEEN == "target_pkg_d.agent_a" + assert os.environ.get("BENCH_TARGET") is None diff --git a/trpc_agent_bench/__init__.py b/trpc_agent_bench/__init__.py new file mode 100644 index 00000000..495f7fa8 --- /dev/null +++ b/trpc_agent_bench/__init__.py @@ -0,0 +1,3 @@ +"""Framework-overhead benchmark suite for trpc-agent based agents.""" + +__all__ = ["run_bench"] diff --git a/trpc_agent_bench/bench/__init__.py b/trpc_agent_bench/bench/__init__.py new file mode 100644 index 00000000..2c5e6345 --- /dev/null +++ b/trpc_agent_bench/bench/__init__.py @@ -0,0 +1,47 @@ +"""Public benchmark authoring API.""" + +from ._config import AgentBenchConf +from ._config import ModelConf +from ._eval import PerformanceEval +from ._eval import PerformanceResult +from ._eval import drain +from ._eval import ensure_turn_completed +from ._eval import get_iterations +from ._eval import run_benchmarks +from ._model import FunctionBlock +from ._model import Reply +from ._model import TextBlock +from ._model import Turn +from ._profile import BenchProfile +from ._setup import BenchSetup +from ._setup import get_concurrency +from ._setup import setup +from ._setup import teardown +from ._target import AgentHandle +from ._target import AgentTarget +from ._target import ToolsFactory +from ._target import resolve_tools + +__all__ = [ + "AgentBenchConf", + "AgentHandle", + "AgentTarget", + "BenchProfile", + "BenchSetup", + "FunctionBlock", + "ModelConf", + "PerformanceEval", + "PerformanceResult", + "Reply", + "TextBlock", + "ToolsFactory", + "Turn", + "drain", + "ensure_turn_completed", + "get_concurrency", + "get_iterations", + "resolve_tools", + "run_benchmarks", + "setup", + "teardown", +] diff --git a/trpc_agent_bench/bench/_builtin/__init__.py b/trpc_agent_bench/bench/_builtin/__init__.py new file mode 100644 index 00000000..7aa1aa6d --- /dev/null +++ b/trpc_agent_bench/bench/_builtin/__init__.py @@ -0,0 +1 @@ +"""Built-in benchmark modules.""" diff --git a/trpc_agent_bench/bench/_builtin/_script_fixtures.py b/trpc_agent_bench/bench/_builtin/_script_fixtures.py new file mode 100644 index 00000000..82ffefa7 --- /dev/null +++ b/trpc_agent_bench/bench/_builtin/_script_fixtures.py @@ -0,0 +1,37 @@ +"""Script builders and text fixtures shared by built-in benchmarks.""" + +from __future__ import annotations + +from .. import FunctionBlock +from .. import Reply +from .. import TextBlock +from .. import Turn + +REAL_TEXT = ("The quick brown fox jumps over the lazy dog. " + "Pack my box with five dozen liquor jugs. " + "How vexingly quick daft zebras jump!") + + +def make_turn_script(user_turns: int, *, chunks: int = 1) -> list[Turn]: + """Build a pure-text multi-turn conversation.""" + + if user_turns < 1: + raise ValueError("user_turns must be positive") + return [[TextBlock(REAL_TEXT, chunks=chunks)] for _ in range(user_turns)] + + +def make_loop_script(llm_calls: int, *, chunks: int = 5) -> list[Turn]: + """Build one user turn with ``llm_calls`` model replies. + + Every reply except the last one requests a tool call, so the agent really + executes ``llm_calls - 1`` tool rounds before the final text reply. + """ + + if llm_calls < 1: + raise ValueError("llm_calls must be positive") + replies: list[Reply] = [[FunctionBlock("add_numbers", { + "a": index + 1, + "b": index + 2 + })] for index in range(llm_calls - 1)] + replies.append(TextBlock(REAL_TEXT, chunks=chunks)) + return [replies] diff --git a/trpc_agent_bench/bench/_builtin/_tool_fixtures.py b/trpc_agent_bench/bench/_builtin/_tool_fixtures.py new file mode 100644 index 00000000..7e7e4b0c --- /dev/null +++ b/trpc_agent_bench/bench/_builtin/_tool_fixtures.py @@ -0,0 +1,47 @@ +"""Private tool fixtures shared by construction benchmarks.""" + +from __future__ import annotations + +from typing import List + +from trpc_agent_sdk.tools import BaseTool +from trpc_agent_sdk.tools import FunctionTool + + +def add_numbers(a: int, b: int) -> int: + """Add two integers and return the sum.""" + return a + b + + +def multiply_numbers(a: int, b: int) -> int: + """Multiply two integers and return the product.""" + return a * b + + +def get_weather(city: str) -> str: + """Return a fixed weather string for a city.""" + return f"sunny in {city}" + + +def get_time(city: str) -> str: + """Return a fixed time string for a city.""" + return f"12:00 in {city}" + + +def get_news(topic: str) -> str: + """Return a fixed news string for a topic.""" + return f"no news about {topic}" + + +def make_add_tool() -> List[BaseTool]: + return [FunctionTool(add_numbers)] + + +def make_tools() -> List[BaseTool]: + return [ + FunctionTool(add_numbers), + FunctionTool(multiply_numbers), + FunctionTool(get_weather), + FunctionTool(get_time), + FunctionTool(get_news), + ] diff --git a/trpc_agent_bench/bench/_builtin/_utils/__init__.py b/trpc_agent_bench/bench/_builtin/_utils/__init__.py new file mode 100644 index 00000000..ff4d8731 --- /dev/null +++ b/trpc_agent_bench/bench/_builtin/_utils/__init__.py @@ -0,0 +1 @@ +"""Shared helpers for built-in benchmark modules.""" diff --git a/trpc_agent_bench/bench/_builtin/_utils/_common.py b/trpc_agent_bench/bench/_builtin/_utils/_common.py new file mode 100644 index 00000000..a3d412b3 --- /dev/null +++ b/trpc_agent_bench/bench/_builtin/_utils/_common.py @@ -0,0 +1,32 @@ +"""Internal naming helpers shared by built-in benchmarks.""" + +from __future__ import annotations + +import uuid + +from ... import get_concurrency + + +def new_session_id() -> str: + return str(uuid.uuid4()) + + +def result_suffix() -> str: + """Return the suffix ``run_bench`` assigned to this process.""" + + import os + + suffix = os.environ.get("BENCH_RESULT_SUFFIX") + if suffix is not None: + return suffix + value = get_concurrency() + return f"_c{value}" if value > 1 else "" + + +def result_name(base: str) -> str: + return f"{base}{result_suffix()}" + + +def concurrency_label() -> str: + value = get_concurrency() + return f"concurrency={value}" if value > 1 else "" diff --git a/trpc_agent_bench/bench/_builtin/_utils/_session_bench.py b/trpc_agent_bench/bench/_builtin/_utils/_session_bench.py new file mode 100644 index 00000000..2741d616 --- /dev/null +++ b/trpc_agent_bench/bench/_builtin/_utils/_session_bench.py @@ -0,0 +1,117 @@ +"""Shared turn and loop implementation for session benchmarks.""" + +from __future__ import annotations + +import asyncio + +from ._common import concurrency_label +from ._common import new_session_id +from ._common import result_name +from ... import BenchSetup +from ... import PerformanceEval +from ... import ToolsFactory +from ... import drain +from ... import ensure_turn_completed +from ... import get_concurrency +from ... import get_iterations +from ... import run_benchmarks +from ... import setup +from ... import teardown +from ... import FunctionBlock +from ... import TextBlock +from ... import Turn + + +def _session_user_id(base_user_id: str, index: int) -> str: + return f"{base_user_id}_{index}" + + +def _expected_text(turn: Turn) -> str | None: + for reply in reversed(turn): + if isinstance(reply, TextBlock): + return reply.text + return None + + +def _has_function(turn: Turn) -> bool: + return any(isinstance(reply, list) and any(isinstance(block, FunctionBlock) for block in reply) for reply in turn) + + +def _make_conversation( + *, + handle, + app_name: str, + base_user_id: str, + script: list[Turn], + current: int, +): + + async def one_session(index: int) -> None: + user_id = _session_user_id(base_user_id, index) + session_id = new_session_id() + for turn in script: + events = await drain( + handle.run_turn( + app_name=app_name, + user_id=user_id, + session_id=session_id, + message="Say ok.", + ), ) + ensure_turn_completed( + events, + expected_text=_expected_text(turn), + expect_tool_success=_has_function(turn), + ) + + async def arun_conversation() -> None: + await asyncio.gather(*(one_session(index) for index in range(current))) + + return arun_conversation + + +async def _run_turns( + script: list[Turn], + *, + name: str, + description: str, + tools_factory: ToolsFactory | None, +) -> None: + current = get_concurrency() + box: BenchSetup = await setup(script) + handle = await box.target.handle( + box.prepared, + tools_factory=tools_factory or (lambda: []), + ) + try: + await run_benchmarks([ + PerformanceEval( + name=result_name(name), + display_name=name, + label=concurrency_label(), + func=_make_conversation( + handle=handle, + app_name=box.conf.app_name, + base_user_id=box.conf.user_id, + script=script, + current=current, + ), + num_iterations=get_iterations(), + warmup_runs=1, + description=description, + ), + ], ) + finally: + await teardown(box) + + +def run_turns( + script: list[Turn], + *, + name: str, + description: str, + tools_factory: ToolsFactory | None = None, +) -> int: + if not script: + raise ValueError("script must contain at least one turn") + asyncio.run(_run_turns(script, name=name, description=description, tools_factory=tools_factory)) + return 0 diff --git a/trpc_agent_bench/bench/_builtin/import_time.py b/trpc_agent_bench/bench/_builtin/import_time.py new file mode 100644 index 00000000..f8e28d69 --- /dev/null +++ b/trpc_agent_bench/bench/_builtin/import_time.py @@ -0,0 +1,59 @@ +"""Import time in a fresh interpreter, minus interpreter startup.""" + +from __future__ import annotations + +import statistics +import subprocess +import sys +import time + +from .._eval import percentile +from .._eval import write_result +from .. import BenchProfile +from .._target import load_target + +DESCRIPTION = ("每次新开一个 Python 进程 import 被测 Agent 包，再减去只启动解释器的时间。" + "指标：p50/p95 = 新进程 import 耗时。") +SAMPLES = 15 + + +def _run(command: list[str]) -> float: + started = time.perf_counter() + completed = subprocess.run(command, capture_output=True, text=True, check=False) + elapsed = time.perf_counter() - started + if completed.returncode != 0: + raise RuntimeError(f"command failed ({completed.returncode}): {' '.join(command)}\n{completed.stderr}", ) + return elapsed + + +def get_bench_profile() -> BenchProfile: + return BenchProfile(requires_iterations=False) + + +def main() -> int: + target = load_target() + samples = SAMPLES + imports = [_run([sys.executable, "-c", f"import {target.import_module}"]) for _ in range(samples)] + baselines = [_run([sys.executable, "-c", "pass"]) for _ in range(samples)] + baseline = statistics.median(baselines) + adjusted = [max(0.0, sample - baseline) for sample in imports] + + result = { + "run_times": adjusted, + "memory_usages": [], + "median_run_time": percentile(adjusted, 50), + "p95_run_time": percentile(adjusted, 95), + "avg_run_time": statistics.fmean(adjusted), + "min_run_time": min(adjusted), + "max_run_time": max(adjusted), + "std_run_time": statistics.pstdev(adjusted) if len(adjusted) > 1 else 0.0, + "median_memory_usage": 0.0, + "num_iterations": len(adjusted), + "warmup_runs": 0, + } + write_result(name="import_time", result=result) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/trpc_agent_bench/bench/_builtin/instantiate_agent.py b/trpc_agent_bench/bench/_builtin/instantiate_agent.py new file mode 100644 index 00000000..790ccd64 --- /dev/null +++ b/trpc_agent_bench/bench/_builtin/instantiate_agent.py @@ -0,0 +1,48 @@ +"""Agent construction only; the model mock and runtime are already prepared.""" + +from __future__ import annotations + +import asyncio +import sys + +from .. import teardown +from .. import setup +from .. import PerformanceEval +from .. import get_iterations +from .. import run_benchmarks +from .. import TextBlock + +DESCRIPTION = ("只调用 target.instantiate() 构造一个不带工具的 agent，不执行 turn。" + "指标：p50/p95 = 单次构造耗时；memory = 构造过程中的 tracemalloc 峰值。") +TURN_SCRIPT = [[TextBlock()]] + + +async def _run() -> None: + box = await setup(TURN_SCRIPT) + + async def make_agent(): + agent = await box.target.instantiate(box.prepared, tools_factory=lambda: []) + if agent is None: + raise RuntimeError("target.instantiate returned None") + return agent + + try: + await run_benchmarks([ + PerformanceEval( + name="instantiate_agent", + func=make_agent, + num_iterations=get_iterations(), + warmup_runs=10, + ), + ], ) + finally: + await teardown(box) + + +def main() -> int: + asyncio.run(_run()) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/trpc_agent_bench/bench/_builtin/instantiate_agent_with_tools.py b/trpc_agent_bench/bench/_builtin/instantiate_agent_with_tools.py new file mode 100644 index 00000000..6238a209 --- /dev/null +++ b/trpc_agent_bench/bench/_builtin/instantiate_agent_with_tools.py @@ -0,0 +1,52 @@ +"""Agent construction with a benchmark-owned five-tool factory.""" + +from __future__ import annotations + +import asyncio +import sys + +from .. import teardown +from .. import setup +from .. import PerformanceEval +from .. import get_iterations +from .. import run_benchmarks +from .. import TextBlock +from ._tool_fixtures import make_tools + +DESCRIPTION = ("只调用 target.instantiate() 构造一个带 5 个工具的 agent，不执行 turn。" + "指标：p50/p95 = 单次构造耗时；memory = 构造过程中的 tracemalloc 峰值。") +TURN_SCRIPT = [[TextBlock()]] + + +async def _run() -> None: + box = await setup(TURN_SCRIPT) + + async def make_agent(): + agent = await box.target.instantiate( + box.prepared, + tools_factory=make_tools, + ) + if agent is None: + raise RuntimeError("target.instantiate returned None") + return agent + + try: + await run_benchmarks([ + PerformanceEval( + name="instantiate_agent_with_tools", + func=make_agent, + num_iterations=get_iterations(), + warmup_runs=10, + ), + ], ) + finally: + await teardown(box) + + +def main() -> int: + asyncio.run(_run()) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/trpc_agent_bench/bench/_builtin/loop1.py b/trpc_agent_bench/bench/_builtin/loop1.py new file mode 100644 index 00000000..72ccdc43 --- /dev/null +++ b/trpc_agent_bench/bench/_builtin/loop1.py @@ -0,0 +1,33 @@ +"""One LLM call in a single user turn.""" + +from __future__ import annotations + +import sys + +from ._utils._session_bench import run_turns +from .. import BenchProfile +from ._script_fixtures import make_loop_script +from ._tool_fixtures import make_add_tool + +LLM_CALLS = 1 +TOOL_ROUNDS = LLM_CALLS - 1 +TURN_SCRIPT = make_loop_script(LLM_CALLS) +DESCRIPTION = ("1 个 user turn，1 次 LLM 调用，0 次 tool round，最终文本 5 chunks。" + "指标：p50/p95 = 整次 turn 的耗时；memory = 本次采样中 tracemalloc 记录的峰值。") + + +def get_bench_profile() -> BenchProfile: + return BenchProfile(supports_concurrency=True) + + +def main() -> int: + return run_turns( + TURN_SCRIPT, + name="loop1", + description=DESCRIPTION, + tools_factory=make_add_tool, + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/trpc_agent_bench/bench/_builtin/loop10.py b/trpc_agent_bench/bench/_builtin/loop10.py new file mode 100644 index 00000000..13b3b3be --- /dev/null +++ b/trpc_agent_bench/bench/_builtin/loop10.py @@ -0,0 +1,33 @@ +"""Ten LLM calls in a single user turn.""" + +from __future__ import annotations + +import sys + +from ._utils._session_bench import run_turns +from .. import BenchProfile +from ._script_fixtures import make_loop_script +from ._tool_fixtures import make_add_tool + +LLM_CALLS = 10 +TOOL_ROUNDS = LLM_CALLS - 1 +TURN_SCRIPT = make_loop_script(LLM_CALLS) +DESCRIPTION = ("1 个 user turn，10 次 LLM 调用，9 次 tool round，最终文本 5 chunks。" + "指标：p50/p95 = 整次 turn 的耗时；memory = 本次采样中 tracemalloc 记录的峰值。") + + +def get_bench_profile() -> BenchProfile: + return BenchProfile(supports_concurrency=True) + + +def main() -> int: + return run_turns( + TURN_SCRIPT, + name="loop10", + description=DESCRIPTION, + tools_factory=make_add_tool, + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/trpc_agent_bench/bench/_builtin/loop5.py b/trpc_agent_bench/bench/_builtin/loop5.py new file mode 100644 index 00000000..7abcebf6 --- /dev/null +++ b/trpc_agent_bench/bench/_builtin/loop5.py @@ -0,0 +1,33 @@ +"""Five LLM calls in a single user turn.""" + +from __future__ import annotations + +import sys + +from ._utils._session_bench import run_turns +from .. import BenchProfile +from ._script_fixtures import make_loop_script +from ._tool_fixtures import make_add_tool + +LLM_CALLS = 5 +TOOL_ROUNDS = LLM_CALLS - 1 +TURN_SCRIPT = make_loop_script(LLM_CALLS) +DESCRIPTION = ("1 个 user turn，5 次 LLM 调用，4 次 tool round，最终文本 5 chunks。" + "指标：p50/p95 = 整次 turn 的耗时；memory = 本次采样中 tracemalloc 记录的峰值。") + + +def get_bench_profile() -> BenchProfile: + return BenchProfile(supports_concurrency=True) + + +def main() -> int: + return run_turns( + TURN_SCRIPT, + name="loop5", + description=DESCRIPTION, + tools_factory=make_add_tool, + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/trpc_agent_bench/bench/_builtin/memory_footprint.py b/trpc_agent_bench/bench/_builtin/memory_footprint.py new file mode 100644 index 00000000..fac9a37a --- /dev/null +++ b/trpc_agent_bench/bench/_builtin/memory_footprint.py @@ -0,0 +1,108 @@ +"""Retained memory per resident agent (no tools and the five-tool profile).""" + +from __future__ import annotations + +import asyncio +import gc +import os +import sys +import tracemalloc + +from .. import teardown +from .. import setup +from .._eval import percentile +from .._eval import write_result +from .. import TextBlock +from .. import BenchProfile +from ._tool_fixtures import make_tools + +DESCRIPTION = ("批量构造 agent 并一直持有，不释放。" + "memory = 每个常驻 agent 占用的内存，不是构造过程中的瞬时峰值。") +AGENTS_PER_SAMPLE = max(1, int(os.environ.get("BENCH_MEMORY_AGENTS", "1000"))) +SAMPLES = 5 +TURN_SCRIPT = [[TextBlock()]] + + +async def _per_agent_footprint(factory) -> float: + gc.collect() + tracemalloc.start() + before, _ = tracemalloc.get_traced_memory() + agents = [await factory() for _ in range(AGENTS_PER_SAMPLE)] + after, _ = tracemalloc.get_traced_memory() + tracemalloc.stop() + del agents + gc.collect() + return max(0.0, (after - before) / (1024.0 * 1024.0)) / AGENTS_PER_SAMPLE + + +async def _write_memory_result( + *, + name: str, + description: str, + factory, + samples: int, +) -> None: + usages = [await _per_agent_footprint(factory) for _ in range(samples)] + median = percentile(usages, 50) + result = { + "run_times": [], + "memory_usages": usages, + "median_run_time": 0.0, + "p95_run_time": 0.0, + "avg_run_time": 0.0, + "min_run_time": 0.0, + "max_run_time": 0.0, + "std_run_time": 0.0, + "median_memory_usage": median, + "avg_memory_usage": sum(usages) / len(usages), + "min_memory_usage": min(usages), + "max_memory_usage": max(usages), + "num_iterations": samples, + "warmup_runs": 0, + } + write_result( + name=name, + result=result, + description=description, + ) + + +async def _run() -> None: + box = await setup(TURN_SCRIPT) + samples = SAMPLES + try: + await _write_memory_result( + name="memory_per_agent", + description=(f"批量构造 {AGENTS_PER_SAMPLE} 个不带工具的 agent 并一直持有，不释放。" + "指标：memory = 每个常驻 agent 占用的内存，不是构造过程中的瞬时峰值。"), + factory=lambda: box.target.instantiate( + box.prepared, + tools_factory=lambda: [], + ), + samples=samples, + ) + await _write_memory_result( + name="memory_per_agent_with_tools", + description=(f"批量构造 {AGENTS_PER_SAMPLE} 个带 5 个工具的 agent 并一直持有，不释放。" + "指标：memory = 每个常驻 agent 占用的内存，不是构造过程中的瞬时峰值。"), + factory=lambda: box.target.instantiate( + box.prepared, + tools_factory=make_tools, + ), + samples=samples, + ) + finally: + await teardown(box) + + +def get_bench_profile() -> BenchProfile: + return BenchProfile(requires_iterations=False) + + +def main() -> int: + asyncio.run(_run()) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/trpc_agent_bench/bench/_builtin/tools.py b/trpc_agent_bench/bench/_builtin/tools.py new file mode 100644 index 00000000..1fa9cb27 --- /dev/null +++ b/trpc_agent_bench/bench/_builtin/tools.py @@ -0,0 +1,82 @@ +"""One round of agent execution: one model-requested ``add_numbers`` call, then a final answer.""" + +from __future__ import annotations + +import asyncio +import sys +from typing import List + +from trpc_agent_sdk.tools import BaseTool +from trpc_agent_sdk.tools import FunctionTool + +from .. import teardown +from ._utils._common import new_session_id +from .. import setup +from .. import PerformanceEval +from .. import drain +from .. import ensure_turn_completed +from .. import get_iterations +from .. import run_benchmarks +from .. import FunctionBlock +from .. import TextBlock + +DESCRIPTION = ("一轮 Agent 执行内，模型先请求 add_numbers tool call，执行后再返回最终文本。" + "指标：p50/p95 = 整轮 Agent 执行的耗时；memory = 本次采样中 tracemalloc 记录的峰值。") +TURN_SCRIPT = [ + [ + [FunctionBlock("add_numbers", { + "a": 1, + "b": 2 + })], + TextBlock(), + ], +] + + +def add_numbers(a: int, b: int) -> int: + """Add two integers and return the sum.""" + return a + b + + +def make_tools() -> List[BaseTool]: + return [FunctionTool(add_numbers)] + + +async def _run() -> None: + box = await setup(TURN_SCRIPT) + handle = await box.target.handle(box.prepared, tools_factory=make_tools) + + async def arun_agent() -> None: + events = await drain( + handle.run_turn( + app_name=box.conf.app_name, + user_id=box.conf.user_id, + session_id=new_session_id(), + message="Add 1 and 2.", + ), ) + ensure_turn_completed( + events, + expected_text="ok", + expect_tool_success=True, + ) + + try: + await run_benchmarks([ + PerformanceEval( + name="tools", + func=arun_agent, + num_iterations=get_iterations(), + warmup_runs=10, + ), + ], ) + finally: + await teardown(box) + + +def main() -> int: + asyncio.run(_run()) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/trpc_agent_bench/bench/_builtin/tools_parallel.py b/trpc_agent_bench/bench/_builtin/tools_parallel.py new file mode 100644 index 00000000..7cac37b1 --- /dev/null +++ b/trpc_agent_bench/bench/_builtin/tools_parallel.py @@ -0,0 +1,96 @@ +"""Parallel tool calls: one assistant response requests three tools.""" + +from __future__ import annotations + +import asyncio +import sys +from typing import List + +from trpc_agent_sdk.tools import BaseTool +from trpc_agent_sdk.tools import FunctionTool + +from .. import teardown +from ._utils._common import new_session_id +from .. import setup +from .. import PerformanceEval +from .. import drain +from .. import ensure_turn_completed +from .. import get_iterations +from .. import run_benchmarks +from .. import FunctionBlock +from .. import TextBlock + +DESCRIPTION = ("一次回复里返回 3 个不同参数的 tool call。" + "指标：p50/p95 = 整次 turn 的耗时；memory = 本次采样中 tracemalloc 记录的峰值。") +TURN_SCRIPT = [ + [ + [ + FunctionBlock("add_numbers", { + "a": 1, + "b": 2 + }), + FunctionBlock("add_numbers", { + "a": 2, + "b": 3 + }), + FunctionBlock("add_numbers", { + "a": 3, + "b": 4 + }), + ], + TextBlock(), + ], +] + + +def add_numbers(a: int, b: int) -> int: + """Add two integers and return the sum.""" + return a + b + + +def make_tools() -> List[BaseTool]: + return [FunctionTool(add_numbers)] + + +async def _run() -> None: + box = await setup(TURN_SCRIPT) + handle = await box.target.handle(box.prepared, tools_factory=make_tools) + + async def arun_agent() -> None: + events = await drain( + handle.run_turn( + app_name=box.conf.app_name, + user_id=box.conf.user_id, + session_id=new_session_id(), + message="Add three pairs.", + ), ) + ensure_turn_completed( + events, + expected_text="ok", + expect_tool_success=True, + ) + responses = sum(1 for event in events if event.content is not None for part in event.content.parts + if part.function_response is not None) + if responses < 3: + raise RuntimeError(f"expected 3 tool responses, got {responses}") + + try: + await run_benchmarks([ + PerformanceEval( + name="tools_parallel", + func=arun_agent, + num_iterations=get_iterations(), + warmup_runs=2, + ), + ], ) + finally: + await teardown(box) + + +def main() -> int: + asyncio.run(_run()) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/trpc_agent_bench/bench/_builtin/turn1.py b/trpc_agent_bench/bench/_builtin/turn1.py new file mode 100644 index 00000000..54f80294 --- /dev/null +++ b/trpc_agent_bench/bench/_builtin/turn1.py @@ -0,0 +1,26 @@ +"""One user turn with a single LLM call.""" + +from __future__ import annotations + +import sys + +from ._utils._session_bench import run_turns +from .. import BenchProfile +from ._script_fixtures import make_turn_script + +TURN_COUNT = 1 +TURN_SCRIPT = make_turn_script(TURN_COUNT) +DESCRIPTION = ("1 个 user turn，1 次 LLM 调用，文本固定 1 chunk。" + "指标：p50/p95 = 整个会话的耗时；memory = 本次采样中 tracemalloc 记录的峰值。") + + +def get_bench_profile() -> BenchProfile: + return BenchProfile(supports_concurrency=True) + + +def main() -> int: + return run_turns(TURN_SCRIPT, name="turn1", description=DESCRIPTION) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/trpc_agent_bench/bench/_builtin/turn10.py b/trpc_agent_bench/bench/_builtin/turn10.py new file mode 100644 index 00000000..1dd49378 --- /dev/null +++ b/trpc_agent_bench/bench/_builtin/turn10.py @@ -0,0 +1,26 @@ +"""Ten user turns with one LLM call each.""" + +from __future__ import annotations + +import sys + +from ._utils._session_bench import run_turns +from .. import BenchProfile +from ._script_fixtures import make_turn_script + +TURN_COUNT = 10 +TURN_SCRIPT = make_turn_script(TURN_COUNT) +DESCRIPTION = ("10 个 user turn，每个 turn 1 次 LLM 调用，每个 turn 的文本固定 1 chunk。" + "指标：p50/p95 = 整个会话的耗时；memory = 本次采样中 tracemalloc 记录的峰值。") + + +def get_bench_profile() -> BenchProfile: + return BenchProfile(supports_concurrency=True) + + +def main() -> int: + return run_turns(TURN_SCRIPT, name="turn10", description=DESCRIPTION) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/trpc_agent_bench/bench/_builtin/turn5.py b/trpc_agent_bench/bench/_builtin/turn5.py new file mode 100644 index 00000000..ec20cb53 --- /dev/null +++ b/trpc_agent_bench/bench/_builtin/turn5.py @@ -0,0 +1,26 @@ +"""Five user turns with one LLM call each.""" + +from __future__ import annotations + +import sys + +from ._utils._session_bench import run_turns +from .. import BenchProfile +from ._script_fixtures import make_turn_script + +TURN_COUNT = 5 +TURN_SCRIPT = make_turn_script(TURN_COUNT) +DESCRIPTION = ("5 个 user turn，每个 turn 1 次 LLM 调用，每个 turn 的文本固定 1 chunk。" + "指标：p50/p95 = 整个会话的耗时；memory = 本次采样中 tracemalloc 记录的峰值。") + + +def get_bench_profile() -> BenchProfile: + return BenchProfile(supports_concurrency=True) + + +def main() -> int: + return run_turns(TURN_SCRIPT, name="turn5", description=DESCRIPTION) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/trpc_agent_bench/bench/_config.py b/trpc_agent_bench/bench/_config.py new file mode 100644 index 00000000..da1414af --- /dev/null +++ b/trpc_agent_bench/bench/_config.py @@ -0,0 +1,26 @@ +"""User-facing benchmark configuration objects.""" + +from __future__ import annotations + +from dataclasses import dataclass +from dataclasses import field +from typing import Any + + +@dataclass +class ModelConf: + """Behavior of the internal OpenAI-compatible mock model.""" + + request_delay_ms: float = 0.0 + chunk_delay_ms: float = 0.0 + + +@dataclass +class AgentBenchConf: + """User configuration shared by every benchmark run.""" + + app_name: str = "bench_app" + user_id: str = "bench_user" + agent_conf: Any = None + model_conf: ModelConf = field(default_factory=ModelConf) + base_url: str | None = field(default=None, repr=False) diff --git a/trpc_agent_bench/bench/_eval.py b/trpc_agent_bench/bench/_eval.py new file mode 100644 index 00000000..cdb0d42e --- /dev/null +++ b/trpc_agent_bench/bench/_eval.py @@ -0,0 +1,325 @@ +"""Measurement, statistics, correctness assertions, and result writing.""" + +from __future__ import annotations + +import asyncio +import gc +import inspect +import json +import os +import statistics +import time +import tracemalloc +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from typing import AsyncIterator +from typing import Callable +from typing import Iterable + + +def get_iterations() -> int: + """Return the iteration count selected by ``run_bench --iterations``.""" + + raw = os.environ.get("BENCH_ITERATIONS") + if raw is None: + raise RuntimeError("BENCH_ITERATIONS is not set; run through run_bench or set it explicitly", ) + try: + value = int(raw) + except ValueError as exc: + raise ValueError(f"BENCH_ITERATIONS must be an integer: {raw!r}") from exc + if value <= 0: + raise ValueError("BENCH_ITERATIONS must be positive") + return value + + +def percentile(values: list[float], percent: float) -> float: + """Inclusive linear-interpolation percentile; safe for small samples.""" + if not values: + return 0.0 + ordered = sorted(values) + if len(ordered) == 1: + return ordered[0] + rank = (len(ordered) - 1) * percent / 100.0 + lower = int(rank) + upper = min(lower + 1, len(ordered) - 1) + fraction = rank - lower + return ordered[lower] + (ordered[upper] - ordered[lower]) * fraction + + +@dataclass +class PerformanceResult: + name: str + run_times: list[float] + memory_usages: list[float] + + @property + def median_run_time(self) -> float: + return statistics.median(self.run_times) if self.run_times else 0.0 + + @property + def p95_run_time(self) -> float: + return percentile(self.run_times, 95) + + @property + def median_memory_usage(self) -> float: + return statistics.median(self.memory_usages) if self.memory_usages else 0.0 + + def to_dict(self) -> dict[str, Any]: + run_times = self.run_times + memory = self.memory_usages + return { + "name": self.name, + "run_times": run_times, + "memory_usages": memory, + "median_run_time": self.median_run_time, + "p95_run_time": self.p95_run_time, + "avg_run_time": statistics.fmean(run_times) if run_times else 0.0, + "min_run_time": min(run_times) if run_times else 0.0, + "max_run_time": max(run_times) if run_times else 0.0, + "std_run_time": statistics.pstdev(run_times) if len(run_times) > 1 else 0.0, + "median_memory_usage": self.median_memory_usage, + } + + +class PerformanceEval: + """Warmup, timed pass, then a separate memory pass. + + ``tracemalloc`` slows execution, so the timed pass never starts it. The + function may be sync or async; async functions run in one event loop for + the whole eval, not one loop per iteration. + """ + + def __init__( + self, + *, + name: str, + func: Callable[..., Any], + num_iterations: int, + warmup_runs: int = 5, + measure_memory: bool = True, + description: str = "", + display_name: str = "", + label: str = "", + ) -> None: + self.name = name + self.func = func + self.num_iterations = max(1, num_iterations) + self.warmup_runs = max(0, warmup_runs) + self.measure_memory = measure_memory + self.description = description + self.display_name = display_name or name + self.label = label + + async def run(self) -> PerformanceResult: + if inspect.iscoroutinefunction(self.func): + return await self._run_async() + return self._run_sync() + + def _run_sync(self) -> PerformanceResult: + for _ in range(self.warmup_runs): + self.func() + run_times = [self._timed_sync() for _ in range(self.num_iterations)] + if not self.measure_memory: + return PerformanceResult(self.name, run_times, []) + baseline = self._memory_baseline_sync() + memory = [self._memory_sync(baseline) for _ in range(self.num_iterations)] + return PerformanceResult(self.name, run_times, memory) + + async def _run_async(self) -> PerformanceResult: + for _ in range(self.warmup_runs): + await self.func() + run_times = [await self._timed_async() for _ in range(self.num_iterations)] + if not self.measure_memory: + return PerformanceResult(self.name, run_times, []) + baseline = await self._memory_baseline_async() + memory = [await self._memory_async(baseline) for _ in range(self.num_iterations)] + return PerformanceResult(self.name, run_times, memory) + + def _timed_sync(self) -> float: + started = time.perf_counter() + self.func() + return time.perf_counter() - started + + async def _timed_async(self) -> float: + started = time.perf_counter() + await self.func() + return time.perf_counter() - started + + def _memory_baseline_sync(self) -> float: + samples = [] + for _ in range(3): + gc.collect() + tracemalloc.start() + (lambda: None)() + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + samples.append(peak) + return statistics.fmean(samples) + + async def _memory_baseline_async(self) -> float: + samples = [] + for _ in range(3): + gc.collect() + tracemalloc.start() + await asyncio.sleep(0) + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + samples.append(peak) + return statistics.fmean(samples) + + def _memory_sync(self, baseline: float) -> float: + gc.collect() + tracemalloc.start() + self.func() + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + return max(0.0, peak - baseline) / (1024.0 * 1024.0) + + async def _memory_async(self, baseline: float) -> float: + gc.collect() + tracemalloc.start() + await self.func() + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + return max(0.0, peak - baseline) / (1024.0 * 1024.0) + + +async def drain(iterator: AsyncIterator[Any]) -> list[Any]: + """Consume an async event stream into a list.""" + return [item async for item in iterator] + + +def ensure_turn_completed( + events: Iterable[Any], + *, + expected_text: str | None = None, + expect_tool_success: bool = False, +) -> None: + """Fail loudly when a benchmark turn took an error or silent path.""" + event_list = list(events) + if any(event.is_error() for event in event_list): + raise RuntimeError("turn returned an error event") + + non_partial: list[str] = [] + partial: list[str] = [] + calls = [] + results = [] + has_non_partial_content = False + for event in event_list: + content = getattr(event, "content", None) + if content is None: + continue + for part in getattr(content, "parts", None) or []: + if getattr(part, "thought", False): + continue + text = getattr(part, "text", None) + if text: + if getattr(event, "partial", False): + partial.append(text) + else: + non_partial.append(text) + has_non_partial_content = True + if getattr(part, "function_call", None) is not None: + calls.append(part.function_call) + if getattr(part, "function_response", None) is not None: + results.append(part.function_response) + has_non_partial_content = True + + turn_complete = any( + getattr(event, "turn_complete", False) or ( + callable(getattr(event, "is_final_response", None)) and event.is_final_response()) for event in event_list) + if not turn_complete and not has_non_partial_content and not partial: + raise RuntimeError("turn did not complete") + + final_text = "".join(non_partial) or "".join(partial) + if expected_text is not None and final_text.strip() != expected_text.strip(): + raise RuntimeError(f"unexpected final text: {final_text!r}") + + if expect_tool_success: + if not calls or not results: + raise RuntimeError("tool loop did not execute") + for response in results: + payload = getattr(response, "response", None) + if isinstance(payload, dict) and payload.get("error"): + raise RuntimeError("tool call failed") + + +def _payload( + *, + name: str, + result: dict[str, Any], + description: str = "", + display_name: str = "", + label: str = "", +) -> dict[str, Any]: + return { + "name": name, + "display_name": display_name or name, + "label": label, + "description": description, + "target": os.environ.get("BENCH_TARGET", ""), + "num_iterations": result.get("num_iterations"), + "warmup_runs": result.get("warmup_runs"), + "result": result, + } + + +def write_result( + *, + name: str, + result: dict[str, Any], + description: str = "", + results_dir: str | Path | None = None, + display_name: str = "", + label: str = "", +) -> dict[str, Any]: + description = description or os.environ.get("BENCH_DESCRIPTION", "") + payload = _payload( + name=name, + result=result, + description=description, + display_name=display_name, + label=label, + ) + raw_directory = results_dir or os.environ.get("BENCH_RESULTS_DIR", "") + if raw_directory: + directory = Path(raw_directory) + directory.mkdir(parents=True, exist_ok=True) + (directory / f"{name}.json").write_text( + json.dumps(payload, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + summary = { + "name": name, + "target": payload["target"], + "median_run_time": result.get("median_run_time"), + "p95_run_time": result.get("p95_run_time"), + } + if not os.environ.get("BENCH_QUIET"): + print(json.dumps(summary, ensure_ascii=False)) + return payload + + +async def run_benchmarks( + evals: list[PerformanceEval], + *, + results_dir: str | Path | None = None, +) -> list[dict[str, Any]]: + """Run evals inside the caller's event loop.""" + payloads = [] + for evaluation in evals: + result = (await evaluation.run()).to_dict() + result["name"] = evaluation.name + result["num_iterations"] = evaluation.num_iterations + result["warmup_runs"] = evaluation.warmup_runs + payloads.append( + write_result( + name=evaluation.name, + result=result, + description=evaluation.description, + results_dir=results_dir, + display_name=evaluation.display_name, + label=evaluation.label, + ), ) + return payloads diff --git a/trpc_agent_bench/bench/_model.py b/trpc_agent_bench/bench/_model.py new file mode 100644 index 00000000..41aa86a7 --- /dev/null +++ b/trpc_agent_bench/bench/_model.py @@ -0,0 +1,392 @@ +"""Model mock: an OpenAI-compatible HTTP server driven by scripted turns. + +The server runs in a separate process so its CPU/RSS do not pollute the +benchmark process. Targets reach it through their normal OpenAI client. +""" + +from __future__ import annotations + +import asyncio +import json +import multiprocessing +import time +from dataclasses import asdict +from dataclasses import dataclass +from dataclasses import field +from typing import Any +from typing import AsyncIterator + +from fastapi import FastAPI +from fastapi import HTTPException +from fastapi import Request +from fastapi.responses import JSONResponse +from fastapi.responses import StreamingResponse + +from ._config import ModelConf + +_MOCK_HOST = "127.0.0.1" + + +@dataclass(frozen=True) +class TextBlock: + """One text response block.""" + + text: str = "ok" + chunks: int = 1 + + +@dataclass(frozen=True) +class FunctionBlock: + """One model-requested function call.""" + + name: str + args: dict[str, Any] = field(default_factory=dict) + + +Reply = TextBlock | list[FunctionBlock] +Turn = list[Reply] + + +@dataclass +class ModelReply: + text: str | None = None + tool_calls: list[dict[str, Any]] = field(default_factory=list) + chunks: int = 1 + finish_reason: str = "stop" + error: str | None = None + + +def _reply_spec(reply: Reply) -> dict[str, Any]: + if isinstance(reply, TextBlock): + return {"kind": "text", "text": reply.text, "chunks": reply.chunks} + if not reply: + raise ValueError("a function reply must contain at least one block") + return { + "kind": "functions", + "calls": [{ + "name": block.name, + "args": block.args + } for block in reply], + } + + +def _reply_from_spec(spec: dict[str, Any]) -> Reply: + kind = spec.get("kind") + if kind == "text": + return TextBlock(text=spec.get("text", "ok"), chunks=spec.get("chunks", 1)) + if kind == "functions": + calls = spec.get("calls") or [] + if not calls: + raise ValueError("a function reply must contain at least one call") + return [FunctionBlock(name=call["name"], args=call.get("args") or {}) for call in calls] + raise ValueError(f"unknown reply kind: {kind!r}") + + +def script_spec(turns: list[Turn]) -> dict[str, Any]: + """Serialize a turn script for the mock server process.""" + + return { + "turns": [[_reply_spec(reply) for reply in turn] for turn in turns], + } + + +def turns_from_spec(spec: dict[str, Any]) -> list[Turn]: + return [[_reply_from_spec(reply) for reply in turn] for turn in spec.get("turns", [])] + + +def reply_from_script( + reply: Reply, + *, + turn: int, + step: int, +) -> ModelReply: + """Convert one scripted reply into a model reply.""" + + if isinstance(reply, TextBlock): + return ModelReply( + text=reply.text, + chunks=max(1, reply.chunks), + ) + if not reply: + raise ValueError("a reply must contain at least one block") + if not all(isinstance(block, FunctionBlock) for block in reply): + raise TypeError("a function reply may only contain FunctionBlock entries") + return ModelReply( + tool_calls=[{ + "id": f"call_{turn}_{step}_{index}", + "name": block.name, + "args": block.args, + } for index, block in enumerate(reply)], + finish_reason="tool_calls", + ) + + +def request_position(messages: list[dict[str, Any]]) -> tuple[int, int]: + """Return the zero-based turn and reply index for one model request.""" + + user_indexes = [index for index, message in enumerate(messages) if message.get("role") == "user"] + if not user_indexes: + raise ValueError("model request has no user message") + turn = len(user_indexes) - 1 + last_user = user_indexes[-1] + step = sum(1 for message in messages[last_user + 1:] if message.get("role") == "assistant") + return turn, step + + +def _split_text(text: str, chunks: int) -> list[str]: + """Split text into exactly ``chunks`` parts without losing characters.""" + + chunks = max(1, chunks) + if chunks == 1: + return [text] + base, extra = divmod(len(text), chunks) + parts = [] + start = 0 + for index in range(chunks): + size = base + (1 if index < extra else 0) + parts.append(text[start:start + size]) + start += size + return parts + + +# ───────────────────────────────────────────────────────────────────────────── +# OpenAI-compatible HTTP server +# ───────────────────────────────────────────────────────────────────────────── + + +def _chunk_payload(delta: dict[str, Any], finish_reason: str | None = None) -> dict[str, Any]: + return { + "id": "chatcmpl-bench", + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": "bench-model", + "choices": [{ + "index": 0, + "delta": delta, + "finish_reason": finish_reason + }], + } + + +def _usage_payload() -> dict[str, Any]: + return { + "id": "chatcmpl-bench", + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": "bench-model", + "choices": [], + "usage": { + "prompt_tokens": 16, + "completion_tokens": 1, + "total_tokens": 17 + }, + } + + +def _completion_payload(reply: ModelReply) -> dict[str, Any]: + message: dict[str, Any] = {"role": "assistant", "content": reply.text} + if reply.tool_calls: + message["content"] = None + message["tool_calls"] = [{ + "id": call["id"], + "type": "function", + "function": { + "name": call["name"], + "arguments": json.dumps(call["args"]), + }, + } for call in reply.tool_calls] + return { + "id": "chatcmpl-bench", + "object": "chat.completion", + "created": int(time.time()), + "model": "bench-model", + "choices": [ + { + "index": 0, + "message": message, + "finish_reason": reply.finish_reason, + }, + ], + "usage": { + "prompt_tokens": 16, + "completion_tokens": 1, + "total_tokens": 17 + }, + } + + +def create_app(spec: dict[str, Any], model_conf: dict[str, Any] | None = None): + """Build the FastAPI app used by the mock server and its tests.""" + + turns = turns_from_spec(spec) + conf = ModelConf(**(model_conf or {})) + app = FastAPI(title="trpc-agent-bench-mock") + + @app.get("/health") + async def health() -> dict[str, str]: + return {"status": "ok"} + + @app.get("/v1/models") + async def models() -> dict[str, Any]: + return { + "object": "list", + "data": [{ + "id": "bench-model", + "object": "model", + "created": 0, + "owned_by": "bench" + }], + } + + @app.post("/v1/chat/completions") + async def chat(request: Request): + body = await request.json() + messages = body.get("messages") or [] + turn, step = request_position(messages) + try: + reply = turns[turn][step] + except IndexError as exc: + raise HTTPException( + status_code=500, + detail=f"no scripted reply for turn={turn + 1} step={step}", + ) from exc + if conf.request_delay_ms > 0: + await asyncio.sleep(conf.request_delay_ms / 1000.0) + model_reply = reply_from_script(reply, turn=turn + 1, step=step) + if model_reply.error: + raise HTTPException(status_code=500, detail=model_reply.error) + if body.get("stream"): + include_usage = bool((body.get("stream_options") or {}).get("include_usage")) + return StreamingResponse( + _sse_stream(model_reply, conf, include_usage=include_usage), + media_type="text/event-stream", + ) + return JSONResponse(_completion_payload(model_reply)) + + return app + + +async def _sse_stream( + reply: ModelReply, + conf: ModelConf, + *, + include_usage: bool, +) -> AsyncIterator[str]: + + def emit(payload: dict[str, Any]) -> str: + return f"data: {json.dumps(payload)}\n\n" + + yield emit(_chunk_payload({"role": "assistant", "content": ""})) + + if reply.tool_calls: + for index, call in enumerate(reply.tool_calls): + if index and conf.chunk_delay_ms: + await asyncio.sleep(conf.chunk_delay_ms / 1000.0) + yield emit( + _chunk_payload( + { + "tool_calls": [ + { + "index": index, + "id": call["id"], + "type": "function", + "function": { + "name": call["name"], + "arguments": json.dumps(call["args"]), + }, + }, + ], + }, ), ) + yield emit(_chunk_payload({}, "tool_calls")) + else: + text = reply.text or "" + for index, chunk in enumerate(_split_text(text, reply.chunks)): + if index and conf.chunk_delay_ms: + await asyncio.sleep(conf.chunk_delay_ms / 1000.0) + yield emit(_chunk_payload({"content": chunk})) + yield emit(_chunk_payload({}, "stop")) + + if include_usage: + yield emit(_usage_payload()) + yield "data: [DONE]\n\n" + + +def _serve(spec: dict[str, Any], model_conf: dict[str, Any], connection: Any) -> None: + import uvicorn + + app = create_app(spec, model_conf) + config = uvicorn.Config( + app, + host="127.0.0.1", + port=0, + log_level="error", + access_log=False, + ) + server = uvicorn.Server(config) + + async def run() -> None: + task = asyncio.create_task(server.serve()) + while not server.started: + await asyncio.sleep(0.01) + port = server.servers[0].sockets[0].getsockname()[1] + connection.send(port) + await task + + try: + asyncio.run(run()) + except KeyboardInterrupt: + pass + + +class MockOpenAIServer: + """A separately hosted OpenAI-compatible endpoint.""" + + def __init__( + self, + turns: list[Turn], + conf: ModelConf | None = None, + ) -> None: + self.spec = script_spec(turns) + self.model_conf = conf or ModelConf() + self.base_url: str | None = None + self._process: multiprocessing.Process | None = None + self._connection: Any = None + + def start(self) -> str: + context = multiprocessing.get_context("fork") + parent_connection, child_connection = context.Pipe() + process = context.Process( + target=_serve, + args=(self.spec, asdict(self.model_conf), child_connection), + daemon=True, + ) + process.start() + try: + if not parent_connection.poll(20.0): + raise RuntimeError("mock OpenAI server did not report a port") + port = parent_connection.recv() + except Exception: + process.terminate() + process.join(timeout=5.0) + raise + self._process = process + self._connection = parent_connection + self.base_url = f"http://{_MOCK_HOST}:{port}/v1" + return self.base_url + + def stop(self) -> None: + process = self._process + self._process = None + if process is not None and process.is_alive(): + process.terminate() + process.join(timeout=5.0) + if self._connection is not None: + self._connection.close() + self._connection = None + + def __enter__(self) -> "MockOpenAIServer": + self.start() + return self + + def __exit__(self, exc_type: Any, exc: Any, traceback: Any) -> None: + self.stop() diff --git a/trpc_agent_bench/bench/_profile.py b/trpc_agent_bench/bench/_profile.py new file mode 100644 index 00000000..9b722f55 --- /dev/null +++ b/trpc_agent_bench/bench/_profile.py @@ -0,0 +1,15 @@ +"""Benchmark capability metadata used by ``run_bench`` before execution.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class BenchProfile: + """What a benchmark can do and what it needs from the runner.""" + + usable: bool = True + supports_concurrency: bool = False + requires_iterations: bool = True + skip_reason: str = "" diff --git a/trpc_agent_bench/bench/_report.py b/trpc_agent_bench/bench/_report.py new file mode 100644 index 00000000..a43d3e1c --- /dev/null +++ b/trpc_agent_bench/bench/_report.py @@ -0,0 +1,440 @@ +"""Turn benchmark JSON files into Markdown and self-contained HTML.""" + +from __future__ import annotations + +import html +import json +import re +from collections import defaultdict +from pathlib import Path +from typing import Any + + +def _format_time(seconds: float | None) -> str: + if seconds is None: + return "-" + if seconds <= 0: + return "0" + if seconds < 1e-3: + return f"{seconds * 1e6:.1f} us" + if seconds < 1: + return f"{seconds * 1e3:.2f} ms" + return f"{seconds:.2f} s" + + +def _format_memory(mebibytes: float | None) -> str: + if mebibytes is None: + return "-" + return f"{mebibytes * 1024:.1f} KiB" + + +def _format_delta(delta: float | None) -> str: + if delta is None: + return "-" + return f"{delta:+.1f}%" + + +def _delta(baseline: float | None, target: float | None) -> float | None: + if baseline is None or target is None or baseline == 0: + return None + return (target - baseline) / baseline * 100.0 + + +def _delta_class(delta: float | None) -> str: + if delta is None or delta == 0: + return "" + return "better" if delta < 0 else "worse" + + +def _number(result: dict[str, Any] | None, field: str) -> float | None: + if result is None: + return None + value = result.get(field) + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return float(value) + + +def _is_memory(name: str) -> bool: + return name.startswith("memory") + + +def _p50(result: dict[str, Any] | None, memory: bool) -> float | None: + if memory: + return None + return _number(result, "median_run_time") + + +def _p95(result: dict[str, Any] | None, memory: bool) -> float | None: + if memory: + return None + return _number(result, "p95_run_time") + + +def _memory(result: dict[str, Any] | None) -> float | None: + if result is None: + return None + usages = result.get("memory_usages") + if isinstance(usages, list) and not usages: + return None + value = _number(result, "median_memory_usage") + if value is None or (value == 0 and not isinstance(usages, list)): + return None + return value + + +def _load(results_dir: Path, targets: list[str] | None = None) -> list[dict[str, Any]]: + if targets: + paths = [path for target in targets for path in sorted((results_dir / target).glob("*.json"))] + else: + paths = sorted(results_dir.glob("*/*.json")) + results = [] + for path in paths: + payload = json.loads(path.read_text(encoding="utf-8")) + if "result" not in payload: + continue + results.append(payload) + return results + + +def _rows( + results: list[dict[str, Any]], +) -> tuple[ + list[str], + dict[str, dict[str, Any]], + dict[str, str], + dict[str, str], + dict[str, str], +]: + targets: list[str] = [] + table: dict[str, dict[str, Any]] = defaultdict(dict) + descriptions: dict[str, str] = {} + display_names: dict[str, str] = {} + labels: dict[str, str] = {} + for payload in results: + target = payload.get("target") or "unknown" + if target not in targets: + targets.append(target) + result = payload.get("result") or {} + name = payload.get("name") or result.get("name") or "unknown" + table[name][target] = result + display_name = payload.get("display_name") or name + display_names.setdefault(name, display_name) + label = payload.get("label") or "" + if label: + labels.setdefault(name, label) + description = payload.get("description") or "" + if description and not descriptions.get(name): + descriptions[name] = description + return targets, table, descriptions, display_names, labels + + +def _natural_key(name: str) -> list[Any]: + return [int(part) if part.isdigit() else part.lower() for part in re.split(r"(\d+)", name) if part] + + +_ORDER = ( + ("import_time", 0), + ("instantiate_agent", 1), + ("turn", 2), + ("loop", 3), + ("memory", 4), + ("tools_parallel", 6), + ("tools", 5), +) + + +def _name_key(name: str) -> tuple[int, int, list[Any]]: + for prefix, rank in _ORDER: + if prefix in ("turn", "loop"): + if not name.startswith(prefix): + continue + suffix = name[len(prefix):] + if suffix and suffix[0].isdigit(): + return rank, 0, _natural_key(name) + continue + if name == prefix or name.startswith(f"{prefix}_"): + return rank, 0, _natural_key(name) + return 7, 0, _natural_key(name) + + +def _ordered(table: dict[str, dict[str, Any]], ) -> list[tuple[str, dict[str, Any]]]: + return sorted(table.items(), key=lambda item: _name_key(item[0])) + + +def _md_cell(value: str) -> str: + return value.replace("|", "\\|").replace("\n", " ") + + +def _html_cell(value: str) -> str: + return html.escape(value).replace("\n", " ") + + +def _description(descriptions: dict[str, str], name: str) -> str: + return descriptions.get(name) or "-" + + +def _markdown_name( + name: str, + display_names: dict[str, str], + labels: dict[str, str], +) -> str: + display = _md_cell(display_names.get(name, name)) + label = labels.get(name) + return f"{display} ({_md_cell(label)})" if label else display + + +def _html_name( + name: str, + display_names: dict[str, str], + labels: dict[str, str], +) -> str: + display = _html_cell(display_names.get(name, name)) + label = labels.get(name) + if not label: + return display + return f"{display}
{_html_cell(label)}
" + + +def _format_value(value: float | None, memory: bool) -> str: + return _format_memory(value) if memory else _format_time(value) + + +def _format_p95(value: float | None, memory: bool) -> str: + return "-" if memory else _format_time(value) + + +def _with_delta(value: str, delta: float | None) -> str: + if delta is None: + return value + return f"{value} {_format_delta(delta)}" + + +def _markdown_absolute( + table: dict[str, dict[str, Any]], + target: str, + display_names: dict[str, str], + labels: dict[str, str], +) -> list[str]: + lines = [ + "| benchmark | p50 | p95 | memory |", + "|---|---|---|---|", + ] + for name, by_target in _ordered(table): + memory_only = _is_memory(name) + result = by_target.get(target) + lines.append( + f"| {_markdown_name(name, display_names, labels)} | " + f"{_format_value(_p50(result, memory_only), memory_only)} | " + f"{_format_p95(_p95(result, memory_only), memory_only)} | " + f"{_format_memory(_memory(result))} |", ) + return lines + + +def _markdown_comparison( + table: dict[str, dict[str, Any]], + baseline: str, + target: str, + display_names: dict[str, str], + labels: dict[str, str], +) -> list[str]: + lines = [ + f"| benchmark | baseline p50 | {_md_cell(target)} p50 | " + f"baseline p95 | {_md_cell(target)} p95 | " + f"baseline memory | {_md_cell(target)} memory |", + "|---|---|---|---|---|---|---|", + ] + for name, by_target in _ordered(table): + memory_only = _is_memory(name) + base = by_target.get(baseline) + current = by_target.get(target) + base_p50 = _p50(base, memory_only) + current_p50 = _p50(current, memory_only) + base_p95 = _p95(base, memory_only) + current_p95 = _p95(current, memory_only) + base_memory = _memory(base) + current_memory = _memory(current) + lines.append( + f"| {_markdown_name(name, display_names, labels)} | " + f"{_format_value(base_p50, memory_only)} | " + f"{_with_delta(_format_value(current_p50, memory_only), _delta(base_p50, current_p50))} | " + f"{_format_p95(base_p95, memory_only)} | " + f"{_with_delta(_format_p95(current_p95, memory_only), _delta(base_p95, current_p95))} | " + f"{_format_memory(base_memory)} | " + f"{_with_delta(_format_memory(current_memory), _delta(base_memory, current_memory))} |", ) + return lines + + +def _html_delta(delta: float | None) -> str: + if delta is None: + return "" + css = _delta_class(delta) + return f" {_format_delta(delta)}" + + +def _html_absolute( + table: dict[str, dict[str, Any]], + target: str, + display_names: dict[str, str], + labels: dict[str, str], +) -> str: + rows = [] + for name, by_target in _ordered(table): + memory_only = _is_memory(name) + result = by_target.get(target) + p50 = _p50(result, memory_only) + p95 = _p95(result, memory_only) + rows.append( + f"" + f"" + f"" + f"", ) + return ("
{_html_name(name, display_names, labels)}{_format_value(p50, memory_only)}{_format_p95(p95, memory_only)}{_format_memory(_memory(result))}
" + "" + f"{''.join(rows)}
benchmarkp50p95memory
") + + +def _html_comparison( + table: dict[str, dict[str, Any]], + baseline: str, + target: str, + display_names: dict[str, str], + labels: dict[str, str], +) -> str: + rows = [] + for name, by_target in _ordered(table): + memory_only = _is_memory(name) + base = by_target.get(baseline) + current = by_target.get(target) + base_p50 = _p50(base, memory_only) + current_p50 = _p50(current, memory_only) + base_p95 = _p95(base, memory_only) + current_p95 = _p95(current, memory_only) + base_memory = _memory(base) + current_memory = _memory(current) + rows.append( + f"{_html_name(name, display_names, labels)}" + f"{_format_value(base_p50, memory_only)}" + f"{_format_value(current_p50, memory_only)}" + f"{_html_delta(_delta(base_p50, current_p50))}" + f"{_format_p95(base_p95, memory_only)}" + f"{_format_p95(current_p95, memory_only)}" + f"{_html_delta(_delta(base_p95, current_p95))}" + f"{_format_memory(base_memory)}" + f"{_format_memory(current_memory)}" + f"{_html_delta(_delta(base_memory, current_memory))}", ) + return ("
" + "" + f"" + f"" + f"" + f"{''.join(rows)}
benchmarkbaseline p50{_html_cell(target)} p50baseline p95{_html_cell(target)} p95baseline memory{_html_cell(target)} memory
") + + +def build_report( + results_dir: str | Path, + targets: list[str] | None = None, + baseline: str | None = None, +) -> Path: + directory = Path(results_dir) + results = _load(directory, targets) + discovered_targets, table, descriptions, display_names, labels = _rows(results) + target_names = targets or discovered_targets + if not target_names: + raise RuntimeError(f"no benchmark results under {directory}") + + if len(target_names) > 1: + if baseline is None: + raise ValueError("baseline is required when comparing multiple targets") + if baseline not in target_names: + raise ValueError(f"baseline {baseline!r} is not in targets") + comparison_targets = [target for target in target_names if target != baseline] + else: + if baseline is not None and baseline != target_names[0]: + raise ValueError("baseline must match the only target when targets has one entry") + comparison_targets = [] + + lines = [ + "# trpc-agent bench report", + "", + f"- results: `{directory}`", + f"- targets: {', '.join(target_names)}", + "", + "## Benchmark 说明", + "", + ] + for name, _ in _ordered(table): + lines.append(f"- `{name}`：{_md_cell(_description(descriptions, name))}") + lines.append("") + + if comparison_targets: + for target in comparison_targets: + lines.extend([f"## {baseline} vs {target}", ""]) + lines.extend(_markdown_comparison(table, baseline, target, display_names, labels), ) + lines.append("") + else: + lines.extend([f"## {target_names[0]}", ""]) + lines.extend(_markdown_absolute(table, target_names[0], display_names, labels), ) + lines.append("") + + lines.extend([ + "Notes:", + "- benchmark 按固定顺序排列，`turn1` / `turn5` / `turn10` 和 `loop1` / `loop5` / `loop10` 分别放在一起；同组内数字按数值比较。", + "- Benchmark 说明来自 result JSON 的 `description`；自定义 benchmark 未提供时显示 `-`。", + "- p50/p95 是耗时的中位数/95 分位；memory 是 `median_memory_usage`。", + "- `-` 表示该 benchmark 没有测这个指标：`import_time` 不测内存，" + "`memory_per_agent*` 不测耗时。", + "- 带并发的结果文件名和 `name` 带 `_cN` 后缀；表格里用 `display_name` " + "和 `concurrency=N` 标签展示。", + ], ) + if comparison_targets: + lines.append("- 百分比（跟在 target 数字后面）= (target - baseline) / baseline；" + "负数表示 target 更快/更省，正数表示更慢/更多。", ) + markdown = "\n".join(lines) + "\n" + (directory / "report.md").write_text(markdown, encoding="utf-8") + + description_items = "".join( + f"
  • {_html_cell(name)}：{_html_cell(_description(descriptions, name))}
    • " + for name, _ in _ordered(table)) + sections = [] + if comparison_targets: + for target in comparison_targets: + sections.append( + f"

    {_html_cell(baseline or '')} vs {_html_cell(target)}

    " + f"{_html_comparison(table, baseline or '', target, display_names, labels)}
    ", ) + else: + target = target_names[0] + sections.append( + f"

    {_html_cell(target)}

    " + f"{_html_absolute(table, target, display_names, labels)}
    ", ) + + document = f""" +trpc-agent bench report + +

    trpc-agent bench report

    +

    results: {html.escape(str(directory))}

    +

    Benchmark 说明

    +
      {description_items}
    +{''.join(sections)} + +""" + (directory / "report.html").write_text(document, encoding="utf-8") + return directory diff --git a/trpc_agent_bench/bench/_runner.py b/trpc_agent_bench/bench/_runner.py new file mode 100644 index 00000000..db80c8a9 --- /dev/null +++ b/trpc_agent_bench/bench/_runner.py @@ -0,0 +1,39 @@ +"""Run one benchmark module by its import path.""" + +from __future__ import annotations + +import asyncio +import importlib +import inspect +import os +import sys +from typing import Any +from typing import Callable + + +def load_benchmark(module_path: str) -> Callable[[], Any]: + """Import a benchmark module and return its ``main`` entry point.""" + + try: + module = importlib.import_module(module_path) + except ImportError as exc: + raise RuntimeError(f"failed to import benchmark module {module_path!r}: {exc}") from exc + entry = getattr(module, "main", None) + if entry is None: + raise RuntimeError(f"benchmark module {module_path!r} does not define main()") + os.environ["BENCH_DESCRIPTION"] = str(getattr(module, "DESCRIPTION", "")) + return entry + + +def main() -> int: + module_path = os.environ.get("BENCH_BENCHMARK") + if not module_path: + raise RuntimeError("BENCH_BENCHMARK is not set") + result = load_benchmark(module_path)() + if inspect.isawaitable(result): + result = asyncio.run(result) + return result + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/trpc_agent_bench/bench/_setup.py b/trpc_agent_bench/bench/_setup.py new file mode 100644 index 00000000..d727e964 --- /dev/null +++ b/trpc_agent_bench/bench/_setup.py @@ -0,0 +1,81 @@ +"""Benchmark bootstrap and teardown.""" + +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass +from dataclasses import field +from typing import Any + +from ._config import AgentBenchConf +from ._model import MockOpenAIServer +from ._model import Turn +from ._target import AgentTarget +from ._target import load_target + + +@dataclass +class BenchSetup: + target: AgentTarget + prepared: Any + conf: AgentBenchConf + _server: MockOpenAIServer = field(repr=False) + + +async def setup( + turns: list[Turn], + *, + conf: AgentBenchConf | None = None, +) -> BenchSetup: + for logger_name in ("trpc_agent_sdk", ): + logging.getLogger(logger_name).setLevel(logging.WARNING) + try: + from trpc_agent_sdk.log import DefaultLogger + from trpc_agent_sdk.log import LogLevel + from trpc_agent_sdk.log import set_default_logger + + set_default_logger(DefaultLogger(min_level=LogLevel.WARNING)) + except Exception: + pass + + conf = conf or AgentBenchConf() + target = load_target() + server = MockOpenAIServer(turns, conf.model_conf) + try: + conf.base_url = server.start() + prepared = await target.prepare(conf=conf) + except Exception: + conf.base_url = None + server.stop() + raise + return BenchSetup( + target=target, + prepared=prepared, + conf=conf, + _server=server, + ) + + +async def teardown(box: BenchSetup) -> None: + try: + await box.target.cleanup(box.prepared) + finally: + box._server.stop() + + +def get_concurrency() -> int: + """Return the concurrency for this benchmark process. + + ``run_bench`` sets ``BENCH_CONCURRENCY`` for each subprocess. Direct runs + default to 1. + """ + + raw = os.environ.get("BENCH_CONCURRENCY", "1").strip() + try: + value = int(raw) + except ValueError as exc: + raise ValueError(f"BENCH_CONCURRENCY must be an integer: {raw!r}") from exc + if value <= 0: + raise ValueError("BENCH_CONCURRENCY must be positive") + return value diff --git a/trpc_agent_bench/bench/_target.py b/trpc_agent_bench/bench/_target.py new file mode 100644 index 00000000..f3580895 --- /dev/null +++ b/trpc_agent_bench/bench/_target.py @@ -0,0 +1,113 @@ +"""Target base classes and target loading. + +This module is part of the generic layer: it must not import specific agent +implementations. Targets are loaded from any importable module that exposes +``TARGET``. +""" + +from __future__ import annotations + +import importlib +import inspect +import os +from abc import ABC +from abc import abstractmethod +from typing import Any +from typing import AsyncIterator +from typing import Awaitable +from typing import Callable +from typing import List +from typing import Union + +from trpc_agent_sdk.tools import BaseTool + +from ._config import AgentBenchConf + +ToolsFactory = Callable[ + [], + Union[List[BaseTool], Awaitable[List[BaseTool]]], +] + + +async def resolve_tools(factory: ToolsFactory) -> List[BaseTool]: + """Resolve a sync or async tools factory into a concrete tool list.""" + + tools = factory() + if inspect.isawaitable(tools): + tools = await tools + return list(tools) + + +class AgentTarget(ABC): + """Base contract every concrete target adapter must implement.""" + + name: str + import_module: str + + @abstractmethod + async def prepare(self, *, conf: AgentBenchConf) -> Any: + """Build the target runtime using ``conf.base_url``. + + The target owns its runtime, session service, and any temporary + directories. It must clean them up in :meth:`cleanup`. + """ + + @abstractmethod + async def instantiate( + self, + prepared: Any, + *, + tools_factory: ToolsFactory, + ) -> Any: + """Construct the agent. + + ``tools_factory`` is called as part of construction so benchmark timing + includes tool creation. It may return tools directly or via an awaitable. + """ + + @abstractmethod + async def handle( + self, + prepared: Any, + *, + tools_factory: ToolsFactory, + ) -> "AgentHandle": + """Wrap the constructed agent in the uniform turn interface.""" + + @abstractmethod + async def cleanup(self, prepared: Any) -> None: + """Close session services and target runtime; remove target temp paths.""" + + +class AgentHandle(ABC): + """Base class for the per-turn handle returned by targets.""" + + @abstractmethod + async def run_turn( + self, + *, + app_name: str, + user_id: str, + session_id: str, + message: str, + ) -> AsyncIterator[Any]: + """Run one turn and yield framework events.""" + + +def load_target(name: str | None = None) -> AgentTarget: + """Load a target from a Python module path. + + ``name`` defaults to ``BENCH_TARGET``. The module must expose ``TARGET``. + """ + + target_name = name or os.environ.get("BENCH_TARGET") + if not target_name: + raise RuntimeError("BENCH_TARGET is not set") + try: + module = importlib.import_module(target_name) + except ImportError as exc: + raise RuntimeError(f"failed to import target module {target_name!r}: {exc}") from exc + target = getattr(module, "TARGET", None) + if target is None: + raise RuntimeError(f"target module {target_name!r} does not define TARGET") + return target diff --git a/trpc_agent_bench/eval/__init__.py b/trpc_agent_bench/eval/__init__.py new file mode 100644 index 00000000..f4aecadc --- /dev/null +++ b/trpc_agent_bench/eval/__init__.py @@ -0,0 +1,4 @@ +"""Quality evaluation suite. + +Runs agents against real model providers. Implementation is not included yet. +""" diff --git a/trpc_agent_bench/run_bench.py b/trpc_agent_bench/run_bench.py new file mode 100644 index 00000000..84dbf122 --- /dev/null +++ b/trpc_agent_bench/run_bench.py @@ -0,0 +1,235 @@ +"""Run every (target module, benchmark module) pair in a fresh process.""" + +from __future__ import annotations + +import argparse +import importlib +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Any + +from .bench._profile import BenchProfile + +PACKAGE_DIR = Path(__file__).resolve().parent +BENCH_DIR = PACKAGE_DIR.parent +DEFAULT_BENCHMARKS = ( + "import_time", + "instantiate_agent", + "instantiate_agent_with_tools", + "turn1", + "turn5", + "turn10", + "loop1", + "loop5", + "loop10", + "tools", + "memory_footprint", + "tools_parallel", +) +BUILTIN_BENCHMARK_PREFIX = "trpc_agent_bench.bench._builtin." + + +def _parse_list(value: str, default: tuple[str, ...]) -> list[str]: + items = [item.strip() for item in value.split(",") if item.strip()] + return items or list(default) + + +def _resolve_benchmark(name: str) -> str: + """Built-in short names expand to modules; custom names stay as-is.""" + + return name if "." in name else f"{BUILTIN_BENCHMARK_PREFIX}{name}" + + +def _load_benchmark_module(module_path: str, target: str): + # Custom benchmark modules may import their target at module scope. Preflight + # imports happen in this process, so give them a valid BENCH_TARGET and + # restore the caller's value afterwards. + previous_target = os.environ.get("BENCH_TARGET") + os.environ["BENCH_TARGET"] = target + try: + return importlib.import_module(module_path) + except ImportError as exc: + raise RuntimeError(f"failed to import benchmark module {module_path!r}: {exc}", ) from exc + finally: + if previous_target is None: + os.environ.pop("BENCH_TARGET", None) + else: + os.environ["BENCH_TARGET"] = previous_target + + +def _bench_profile(module: Any) -> BenchProfile: + getter = getattr(module, "get_bench_profile", None) + return getter() if getter is not None else BenchProfile() + + +def _positive_int(raw: str) -> int: + try: + value = int(raw) + except ValueError as exc: + raise argparse.ArgumentTypeError(f"must be an integer: {raw!r}") from exc + if value <= 0: + raise argparse.ArgumentTypeError("must be positive") + return value + + +def _parse_concurrency(raw: str) -> list[int]: + values = [] + for item in raw.split(","): + item = item.strip() + if not item: + continue + try: + value = int(item) + except ValueError as exc: + raise ValueError(f"--concurrency must contain integers: {item!r}") from exc + if value <= 0: + raise ValueError("--concurrency values must be positive") + values.append(value) + if not values: + raise ValueError("--concurrency must contain at least one positive integer") + return sorted(set(values)) + + +def _pythonpath() -> str: + entries = [str(BENCH_DIR), str(Path.cwd())] + existing = os.environ.get("PYTHONPATH") + if existing: + entries.append(existing) + return os.pathsep.join(dict.fromkeys(entries)) + + +def _collect( + results_root: Path, + targets: list[str], + baseline: str | None, +) -> dict[str, Any]: + results: list[dict[str, Any]] = [] + for target in targets: + for path in sorted((results_root / target).glob("*.json")): + results.append(json.loads(path.read_text(encoding="utf-8"))) + return { + "results_dir": str(results_root), + "targets": targets, + "baseline": baseline, + "results": results, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run the trpc-agent benchmark suite.") + parser.add_argument( + "--targets", + required=True, + help="comma-separated target module paths, e.g. llmagent.base", + ) + parser.add_argument( + "--benchmarks", + default="", + help=("comma-separated built-in names or custom benchmark module paths; " + "defaults to the built-in suite"), + ) + parser.add_argument( + "--baseline", + default=None, + help="target module path used as the comparison baseline; required for multiple targets", + ) + parser.add_argument( + "--concurrency", + default="1", + help="comma-separated concurrency values, e.g. 1,8", + ) + parser.add_argument( + "--iterations", + type=_positive_int, + default=None, + help=("iteration count for benchmarks that require it; " + "required when any selected benchmark requires iterations"), + ) + parser.add_argument( + "--results-dir", + default=None, + help="results root, default: ./results", + ) + args = parser.parse_args() + + targets = _parse_list(args.targets, ()) + if not targets: + parser.error("--targets must list at least one target module") + baseline = args.baseline.strip() if args.baseline else None + if len(targets) > 1: + if not baseline: + parser.error("--baseline is required when --targets lists more than one target") + if baseline not in targets: + parser.error(f"--baseline must be one of --targets: {baseline!r}") + elif baseline is not None and baseline != targets[0]: + parser.error("--baseline must match the only target when --targets has one entry") + + try: + concurrency_values = _parse_concurrency(args.concurrency) + except ValueError as exc: + parser.error(str(exc)) + + raw_benchmarks = args.benchmarks.strip() + if raw_benchmarks: + benchmark_names = _parse_list(raw_benchmarks, ()) + else: + benchmark_names = list(DEFAULT_BENCHMARKS) + + benchmarks = [] + for name in benchmark_names: + module_path = _resolve_benchmark(name) + try: + module = _load_benchmark_module(module_path, targets[0]) + except RuntimeError as exc: + parser.error(str(exc)) + profile = _bench_profile(module) + if not profile.usable: + reason = f" ({profile.skip_reason})" if profile.skip_reason else "" + print(f"[skip] {name}{reason}", flush=True) + continue + benchmarks.append((name, module_path, module, profile)) + + needs_iterations = [name for name, _, _, profile in benchmarks if profile.requires_iterations] + if needs_iterations and args.iterations is None: + parser.error("--iterations is required for: " + ", ".join(needs_iterations), ) + + results_root = (Path(args.results_dir) if args.results_dir else Path("results")).resolve() + + for target in targets: + for name, module_path, module, profile in benchmarks: + supports_concurrency = profile.supports_concurrency + run_concurrency = concurrency_values if supports_concurrency else [1] + for current in run_concurrency: + environment = os.environ.copy() + environment["BENCH_TARGET"] = target + environment["BENCH_BENCHMARK"] = module_path + environment["BENCH_RESULTS_DIR"] = str(results_root / target) + environment["BENCH_CONCURRENCY"] = str(current) + result_suffix = f"_c{current}" if supports_concurrency and current > 1 else "" + environment["BENCH_RESULT_SUFFIX"] = result_suffix + environment["PYTHONPATH"] = _pythonpath() + if args.iterations is not None: + environment["BENCH_ITERATIONS"] = str(args.iterations) + command = [sys.executable, "-m", "trpc_agent_bench.bench._runner"] + suffix = f" c={current}" if supports_concurrency and current > 1 else "" + print(f"[bench] {target} / {name}{suffix}", flush=True) + subprocess.run(command, env=environment, check=True) + + summary = _collect(results_root, targets, baseline) + results_root.mkdir(parents=True, exist_ok=True) + (results_root / "summary.json").write_text( + json.dumps(summary, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + + from .bench._report import build_report + + build_report(results_root, targets, baseline) + return 0 + + +if __name__ == "__main__": + sys.exit(main())