feat(bench): route and separate model reasoning conditions

bench/README.md

@@ -57,6 +57,20 @@ node bench/cli.mjs ab --task csv-sum
 - 文件夹名称格式为 `benchmark-YYYY-MM-DD-NN`（按当天日期编号，从 `-00` 起）
 - 跑完自动在组目录写 `DSH-OPS Benchmark Report <UTC>.md`
 
+## 成本 A/B 的条件 / A/B measurement conditions
+
+```console
+node bench/cli.mjs ab --model DeepSeek-V4.1-Flash --reasoning-effort high --seal
+node bench/cli.mjs ab --model model-a --model model-b --reasoning-effort low --reasoning-effort high --seal
+node bench/cli.mjs report --matrix bench/runs/benchmark-YYYY-MM-DD-NN
+```
+
+`--model` 会真实改变两臂路由；`--reasoning-effort` 只接受 `off|low|high|max`。Provider 默认取 LLM overlay 中的字面值，也可用 `--provider` 指定；请先在 overlay/provider 配置中声明要测试的模型。条件补丁作为最后一个 `--patch` 同时传入两臂，不修改原来的 `llm.patch.yml`。每臂保留实际 `dump-config.txt`，`--seal` 将同源 `{provider, model, reasoningEffort}` 写入 `run.json` 的 `meta.condition`。未指定思考等级且 overlay 也没有该值时，记录 `null`（宿主默认值未固定）。
+
+重复的模型与档位形成笛卡尔积，每个条件位于独立的 `condition-NNN` 目录；根目录的 `matrix.json` 指向各组证据。多组报告逐条件展示，**跨条件不可比，不合并 Δ**。不同条件（含已知与未知）交给 compare 会退 `2` 并列出两侧条件。不传这些旗标时仍使用原有 overlay，条件标为 `unknown`；旧 run 缺少该字段仍可读，旧 `meta.model` 标签不作为路由证据。
+
+Model flags set the actual shared route, with the condition patch applied after the provider overlay. Both arms archive their composed configuration. Repeated models and reasoning levels form independent Cartesian groups; `report --matrix` renders one section per condition and never combines deltas. Cross-condition comparisons exit 2, naming both conditions. Without new flags, routing is unchanged and the condition is `unknown`; historical labels do not prove routing. An omitted reasoning level remains `null` unless the overlay pins it. All examples are optional model-driven runs, outside CI.
+
 ## 任务一览
 
 六个任务各自压一种能力，`ops_*` 工具名来自 `publicToolName()`。每个任务的期望值都由夹具生成器现算（`derive*`），判卷在 `bench/lib/verify.mjs`。

bench/compare.mjs

@@ -30,6 +30,8 @@
 import fs from 'node:fs'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
+import { describeCondition } from './lib/condition.mjs'
+import { matrixComparisons, renderMatrix } from './lib/matrix.mjs'
 
 import {
   AGGREGATES,
@@ -63,6 +65,7 @@ const RED = '#f85149'
 export const EXIT = { ok: 0, failure: 1, usage: 2 }
 
 const USAGE = `usage: node bench/compare.mjs <base> <cand> [options]
+       node bench/compare.mjs --matrix <group-directory> [options]
 
   <base> <cand>      each side is one of:
                        <label>   a run label, matched against bench/runs/*-<label>
@@ -117,6 +120,7 @@ export function parseArgs(argv) {
   const options = {
     base: null,
     cand: null,
+    matrix: null,
     folder: null,
     runsDir: RUNS_DIR,
     aggregate: 'median',
@@ -164,6 +168,7 @@ export function parseArgs(argv) {
       } else if (name === '--out') options.out = take()
       else if (name === '--folder') options.folder = assertFolderName(take(), '--folder')
       else if (name === '--runs') options.runsDir = path.resolve(take())
+      else if (name === '--matrix') options.matrix = path.resolve(take())
       else throw new Error(`unknown option "${name}"`)
       continue
     }
@@ -178,6 +183,10 @@ export function parseArgs(argv) {
     throw new Error('--noise-floor-pct must be a non-negative number')
   }
   if (options.help) return options
+  if (options.matrix !== null) {
+    if (positional.length || options.folder !== null || options.noise !== null) throw new Error('--matrix takes its groups from matrix.json; do not pass side labels, --folder or --noise')
+    return options
+  }
   if (positional.length !== 2) throw new Error('expected exactly two sides: <base> <cand>')
   options.base = positional[0]
   options.cand = positional[1]
@@ -521,6 +530,8 @@ export function renderReport(comparison) {
   const lines = [
     REPORT_TITLE,
     '',
+    `Condition: ${describeCondition(comparison.condition)}`,
+    '',
     ...environmentLines(comparison.environment),
     ...taskSection(comparison),
     '',
@@ -616,6 +627,17 @@ export async function main(argv, io = {}) {
     out(USAGE)
     return EXIT.ok
   }
+  if (options.matrix !== null) {
+    try {
+      const comparisons = matrixComparisons(options.matrix, options, { resolveSide, compare: compareSummaries })
+      const payload = options.json ? JSON.stringify({ format: 'dsh-ops-bench-matrix-report-v1', comparableAcrossConditions: false, comparisons }, null, 2) : renderMatrix(comparisons, renderReport)
+      out(payload)
+      const target = options.out ? path.resolve(options.out) : path.join(options.matrix, reportFileName(now()))
+      fs.writeFileSync(target, payload + '\n')
+      err(`compare matrix: wrote ${target}`)
+      return comparisons.some(value => value.gate.fail) ? EXIT.failure : EXIT.ok
+    } catch (error) { err(`compare matrix: ${error.message}`); return EXIT.usage }
+  }
 
   let base
   let cand

bench/lib/compare-core.mjs

@@ -33,6 +33,7 @@
  */
 
 import { CORE_BUCKETS, median, minValue, pairMetrics } from './aggregate.mjs'
+import { assertSameCondition } from './condition.mjs'
 import { compareEnvironment } from './environment.mjs'
 
 export { median, minValue }
@@ -482,6 +483,7 @@ export function classifyExcluded(pairs, taskId = null) {
  * @returns {object} the noise-floor document; `pct: null` when nothing could be paired.
  */
 export function noiseFloor(baseA, baseB, options = {}) {
+  assertSameCondition(baseA, baseB)
   compareEnvironment(baseA, baseB, options)
   const aggregate = AGGREGATES.includes(options.aggregate) ? options.aggregate : 'median'
   const merged = pairMetrics(normalizeResults(baseA), normalizeResults(baseB))
@@ -549,6 +551,8 @@ export function resolveNoiseFloor(options = {}, base = null, cand = null) {
   }
   const noise = options.noise
   if (noise && typeof noise === 'object' && !Array.isArray(noise) && (noise.base || noise.cand)) {
+    assertSameCondition(base, noise.base ?? base)
+    assertSameCondition(base, noise.cand ?? base)
     return noiseFloor(noise.base ?? base, noise.cand ?? base, { aggregate: options.aggregate, allowCrossPlatform: options.allowCrossPlatform })
   }
   return {
@@ -704,6 +708,7 @@ export function formatNumber(value) {
  * @returns {object} the comparison structure, including the gate and the verdict.
  */
 export function compareSummaries(base, cand, options = {}) {
+  const condition = assertSameCondition(base, cand)
   const environment = compareEnvironment(base, cand, options)
   if (options.noise) compareEnvironment(base, options.noise.base, options)
   const aggregate = AGGREGATES.includes(options.aggregate) ? options.aggregate : 'median'
@@ -857,6 +862,7 @@ export function compareSummaries(base, cand, options = {}) {
 
   return {
     format: COMPARE_FORMAT,
+    condition,
     environment,
     base: { label: sideLabel(base, 'base'), profile: sideProfile(base, 'base'), model: baseModel, dshVersion: sideDshVersion(base, 'base') },
     cand: { label: sideLabel(cand, 'cand'), profile: sideProfile(cand, 'cand'), model: candModel, dshVersion: sideDshVersion(cand, 'cand') },

bench/lib/condition.mjs

@@ -0,0 +1,82 @@
+/** Explicit, reproducible model routing; unknown historical conditions stay unknown. */
+import fs from 'node:fs'
+import path from 'node:path'
+import { parse, stringify } from 'yaml'
+
+export const REASONING_EFFORTS = Object.freeze(['off', 'low', 'high', 'max'])
+export class ConditionError extends Error {}
+const literal = value => typeof value === 'string' && value.trim() !== '' && !/[\x00-\x1f]/u.test(value)
+
+export function reasoningEffort(value) {
+  if (!REASONING_EFFORTS.includes(value)) throw new ConditionError(`--reasoning-effort must be one of ${REASONING_EFFORTS.join(', ')}; got ${JSON.stringify(value)}`)
+  return value
+}
+
+export function conditionOf(run) {
+  const value = Object.hasOwn(run?.meta ?? {}, 'condition') ? run.meta.condition : run?.condition ?? null
+  if (value === null) return null
+  if (!literal(value.provider) || !literal(value.model)
+    || value.reasoningEffort !== null && !REASONING_EFFORTS.includes(value.reasoningEffort)) {
+    throw new ConditionError(`Invalid recorded condition: ${JSON.stringify(value)}`)
+  }
+  return { provider: value.provider, model: value.model, reasoningEffort: value.reasoningEffort }
+}
+
+export function describeCondition(value) {
+  return value === null || value === undefined ? 'unknown' : JSON.stringify(conditionOf({ condition: value }))
+}
+
+export function assertSameCondition(base, cand) {
+  const left = conditionOf(base), right = conditionOf(cand)
+  if (JSON.stringify(left) !== JSON.stringify(right)) {
+    throw new ConditionError(`Conditions differ; base=${describeCondition(left)}; ops=${describeCondition(right)}. Cross-condition pairing is refused.`)
+  }
+  return left
+}
+
+/** Refuse to spend tokens when the composed host profile did not apply the requested route. */
+export function assertEffectiveCondition(text, expected) {
+  if (expected === null || expected === undefined) return
+  const rows = parse(text, { customTags: [{ tag: 'tag:yaml.org,2002:js', resolve: value => ({ expression: value }) }] })
+  const entries = Array.isArray(rows) ? rows.filter(row => row?.id === 'agent-default-model' && row.disabled !== true) : []
+  const actual = entries.length === 1 ? entries[0].config : null
+  if (!actual || actual.provider !== expected.provider || actual.model !== expected.model
+    || expected.reasoningEffort !== null && actual.reasoningEffort !== expected.reasoningEffort) {
+    throw new ConditionError(`Composed agent-default-model route does not match requested condition ${describeCondition(expected)}; nothing was run.`)
+  }
+}
+
+/** The overlay's literal route supplies omitted CLI fields; no expression is evaluated. */
+export function planConditions(options, patch, read = file => fs.readFileSync(file, 'utf8')) {
+  if (!options.models.length && !options.reasoningEfforts.length && options.provider === null) return [null]
+  let route = {}
+  if (patch !== null) {
+    const rows = parse(read(patch), { customTags: [{ tag: 'tag:yaml.org,2002:js', resolve: value => ({ expression: value }) }] })
+    if (!Array.isArray(rows)) throw new ConditionError('The LLM overlay must be a list of patch entries')
+    for (const row of rows) if (row?.id === 'agent-default-model') route = row.config ?? {}
+  }
+  const provider = options.provider ?? route.provider
+  const models = options.models.length ? options.models : [route.model]
+  const efforts = options.reasoningEfforts.length ? options.reasoningEfforts : [route.reasoningEffort ?? null]
+  if (!literal(provider)) throw new ConditionError('Explicit conditions need a literal provider in the LLM overlay or --provider <id>')
+  if (models.some(model => !literal(model))) throw new ConditionError('Explicit conditions need --model <id> or a literal model in the LLM overlay')
+  for (const effort of efforts) if (effort !== null) reasoningEffort(effort)
+  return [...new Set(models)].flatMap(model => [...new Set(efforts)].map(reasoningEffort => ({ provider, model, reasoningEffort })))
+}
+
+/** Both arms receive the identical final patch after the original provider overlay. */
+export function writeConditionPatch(dir, condition) {
+  if (condition === null) return null
+  const { provider, model, reasoningEffort } = conditionOf({ condition })
+  fs.mkdirSync(dir, { recursive: true })
+  const target = path.join(dir, 'condition.patch.yml')
+  const text = stringify([{ id: 'agent-default-model', config: {
+    provider, model, ...(reasoningEffort === null ? {} : { reasoningEffort }),
+  } }])
+  fs.writeFileSync(target, text, { flag: 'wx', mode: 0o600 })
+  return target
+}
+
+export function patchArgs(llmPatch, conditionPatch) {
+  return [llmPatch, conditionPatch].filter(Boolean).flatMap(file => ['--patch', file])
+}

bench/lib/matrix.mjs

@@ -0,0 +1,59 @@
+/** Separate condition groups; there is deliberately no cross-condition aggregate. */
+import fs from 'node:fs'
+import path from 'node:path'
+import { describeCondition, assertSameCondition } from './condition.mjs'
+
+export const MATRIX_FORMAT = 'dsh-ops-bench-matrix-v1'
+export const MATRIX_FILE = 'matrix.json'
+
+export async function runMatrix({ conditions, root, argv, options, io, run }) {
+  const out = io.out ?? console.log
+  const manifest = { format: MATRIX_FORMAT, groups: [] }
+  if (!options.show) {
+    fs.mkdirSync(root, { recursive: true })
+    fs.writeFileSync(path.join(root, MATRIX_FILE), JSON.stringify(manifest, null, 2), { flag: 'wx' })
+  }
+  let exit = 0
+  for (const [index, condition] of conditions.entries()) {
+    const directory = `condition-${String(index + 1).padStart(3, '0')}`
+    const group = { directory, condition, base: null, cand: null, exitCode: null }
+    manifest.groups.push(group)
+    out(`[matrix] ${index + 1}/${conditions.length}: ${describeCondition(condition)}`)
+    const code = await run(argv, { ...io, conditionOverride: condition, groupOverride: path.join(root, directory),
+      onResult(result) {
+        for (const side of ['base', 'cand']) if (result.summaryFiles?.[side]) group[side] = path.relative(root, result.summaryFiles[side]).split(path.sep).join('/')
+        io.onResult?.(result)
+      },
+    })
+    group.exitCode = code
+    if (!options.show) fs.writeFileSync(path.join(root, MATRIX_FILE), JSON.stringify(manifest, null, 2) + '\n')
+    if (code !== 0) exit = code === 2 ? 2 : exit || 1
+    if (code === 2 || io.interrupt?.stopped) break
+  }
+  out('[matrix] Cross-condition results are not comparable; no combined delta is computed.')
+  return exit
+}
+
+export function matrixComparisons(root, options, { resolveSide, compare }) {
+  const dir = path.resolve(root), manifest = JSON.parse(fs.readFileSync(path.join(dir, MATRIX_FILE), 'utf8'))
+  if (manifest.format !== MATRIX_FORMAT || !Array.isArray(manifest.groups) || !manifest.groups.length) throw new Error('Not a populated benchmark matrix')
+  const side = file => {
+    if (typeof file !== 'string') throw new Error('Matrix group has no completed side summaries')
+    const target = path.resolve(dir, file), relative = path.relative(dir, target)
+    if (path.isAbsolute(relative) || relative.startsWith(`..${path.sep}`) || relative === '..') throw new Error('Matrix summary escapes its group root')
+    return resolveSide(target, options).doc
+  }
+  return manifest.groups.map(group => {
+    const base = side(group.base), cand = side(group.cand)
+    assertSameCondition(base, { condition: group.condition })
+    assertSameCondition(cand, { condition: group.condition })
+    return compare(base, cand, options)
+  })
+}
+
+export function renderMatrix(comparisons, render) {
+  return ['# Benchmark condition matrix', '', 'Cross-condition results are not comparable. Each section is independent; deltas are never combined.', '',
+    ...comparisons.flatMap((comparison, index) => [`## Condition ${index + 1}: ${describeCondition(comparison.condition)}`, '',
+      render(comparison).replace(/^# [^\n]*\n\n/, '').replace(/^Condition: [^\n]*\n\n/, '').replace(/^## /gm, '### '), '']),
+  ].join('\n')
+}

bench/report/panel.mjs

@@ -375,6 +375,8 @@ function renderConclusion() {
     : '噪声底(A/A)±' + SUMMARY.noise.pct.toFixed(1) + '% → ' + (SUMMARY.readable ? '超过噪声 ⇒ 可读方向' : '落在噪声内 ⇒ 只报区间,不报方向')
 }
 function renderOverview(node) {
+  const condition = obj(RUN.meta).condition
+  node.appendChild(el('p', 'mono', 'Condition: ' + (condition === undefined || condition === null ? 'unknown' : j(condition))))
   const environment = el('div', 'card')
   environment.appendChild(el('h3', null, 'Platform / component configuration'))
   environment.appendChild(el('pre', null, JSON.stringify({ platform: obj(RUN.meta).platform || 'unknown', components: Object.fromEntries(Object.entries(obj(RUN.arms)).map(([key, arm]) => [key, obj(arm).components || null])) }, null, 2)))