feat(bench): configure and compare candidate component switches

bench/README.md

@@ -71,6 +71,25 @@ node bench/cli.mjs report --matrix bench/runs/benchmark-YYYY-MM-DD-NN
 
 Model flags set the actual shared route, with the condition patch applied after the provider overlay. Both arms archive their composed configuration. Repeated models and reasoning levels form independent Cartesian groups; `report --matrix` renders one section per condition and never combines deltas. Cross-condition comparisons exit 2, naming both conditions. Without new flags, routing is unchanged and the condition is `unknown`; historical labels do not prove routing. An omitted reasoning level remains `null` unless the overlay pins it. All examples are optional model-driven runs, outside CI.
 
+### 组件开关 / Component switches
+
+```console
+node bench/cli.mjs setup --switch hide-official=all --switch ptc --switch panel=false --dry-run
+node bench/cli.mjs setup --switch hide-official=read,grep --switch discovery --switch shell
+```
+
+`--switch` 可重复，支持 `hide-official`、`ptc`、`panel`、`discovery`、`shell`；省略值等同 `true`，普通开关也接受 `false`。`hide-official=all` 只选择已有替代实现的 `read,glob,grep`，也可指定不重复的子集或 `false`。只修改 candidate 的 `cordis.patch.yml`；baseline 不写入组件行。每次 setup 只改明确指定的开关，不自动搜索组合。
+
+写入幂等；本地已有相反值需 `--force`，继承的字面配置键会保留。动态表达式、非映射配置、重复行和无法安全编辑的目标形态需手工处理，`--force` 也不会猜测。`--dry-run` 打印请求的行和值，不启动宿主、不修改 profile。
+
+实跑先读合成配置，写入后再核验配置并执行一次有超时的最小 headless turn；这可能调用已配置模型。没有已完成 turn、宿主错误或 `READINESS_BLOCKED` 都会让 setup 失败并指向 [#207](https://github.com/T-Auto/dsh-ops/issues/207)，不会启动 benchmark sweep。这个探测不代替 Desktop 重启后的人工验证。
+
+实际配置写入摘要及封存 `run.json` 的 `arms.cand.switches`，报告头和 panel 显示 candidate 开关。baseline 的空集合与 candidate 的非空集合属于正常 A/B；两份运行的 candidate 集合不同则 compare 退出 `2`，列出差异，不合并 Δ。未求值配置记录 `null`，旧数据缺失开关元数据仍可读取为未知；这些字段是配置事实，不证明组件已经发布到会话。
+
+Repeat `--switch` to set candidate components. Values default to `true`; booleans also accept `false`. `hide-official=all` expands only the implemented `read,glob,grep` replacements, or select a unique subset. Setup edits only the candidate patch, preserves inherited literal configuration, and changes only named switches. Conflicting local values require `--force`; dynamic or unsafe shapes require manual edits even with force. Dry-run prints the requested values without launching the host or writing profiles.
+
+Real setup verifies composed configuration and runs one bounded headless readiness turn, which may call the configured model. Missing successful completion, host errors and `READINESS_BLOCKED` fail setup with the #207 diagnostic before a sweep starts. This does not substitute for Desktop validation. Summaries and sealed `arms.cand.switches` retain effective configuration; report headers and the panel show it. Comparisons reject different candidate switch sets with exit 2, while the baseline's empty set is expected. Unevaluated configuration and historical missing metadata remain unknown, without implying publication readiness. Switch combinations are selected explicitly and are outside the CI gate.
+
 ## 任务一览
 
 六个任务各自压一种能力，`ops_*` 工具名来自 `publicToolName()`。每个任务的期望值都由夹具生成器现算（`derive*`），判卷在 `bench/lib/verify.mjs`。

bench/compare.mjs

@@ -531,6 +531,7 @@ export function renderReport(comparison) {
     REPORT_TITLE,
     '',
     `Condition: ${describeCondition(comparison.condition)}`,
+    `Candidate switches: ${JSON.stringify(comparison.switches ?? null)} (null means unknown)`,
     '',
     ...environmentLines(comparison.environment),
     ...taskSection(comparison),

bench/lib/compare-core.mjs

@@ -35,6 +35,7 @@
 import { CORE_BUCKETS, median, minValue, pairMetrics } from './aggregate.mjs'
 import { assertSameCondition } from './condition.mjs'
 import { compareEnvironment } from './environment.mjs'
+import { assertSameSwitches } from './switches.mjs'
 
 export { median, minValue }
 
@@ -483,6 +484,7 @@ export function classifyExcluded(pairs, taskId = null) {
  * @returns {object} the noise-floor document; `pct: null` when nothing could be paired.
  */
 export function noiseFloor(baseA, baseB, options = {}) {
+  assertSameSwitches(baseA, baseB)
   assertSameCondition(baseA, baseB)
   compareEnvironment(baseA, baseB, options)
   const aggregate = AGGREGATES.includes(options.aggregate) ? options.aggregate : 'median'
@@ -708,6 +710,8 @@ export function formatNumber(value) {
  * @returns {object} the comparison structure, including the gate and the verdict.
  */
 export function compareSummaries(base, cand, options = {}) {
+  const switches = assertSameSwitches(base, cand)
+  if (options.noise) assertSameSwitches(base, options.noise.base)
   const condition = assertSameCondition(base, cand)
   const environment = compareEnvironment(base, cand, options)
   if (options.noise) compareEnvironment(base, options.noise.base, options)
@@ -862,6 +866,7 @@ export function compareSummaries(base, cand, options = {}) {
 
   return {
     format: COMPARE_FORMAT,
+    switches,
     condition,
     environment,
     base: { label: sideLabel(base, 'base'), profile: sideProfile(base, 'base'), model: baseModel, dshVersion: sideDshVersion(base, 'base') },

bench/lib/switches.mjs

@@ -0,0 +1,46 @@
+import { parse } from 'yaml'
+
+export const SWITCH_ROWS = Object.freeze(Object.fromEntries(['hide-official', 'ptc', 'panel', 'discovery', 'shell'].map(name => [name, `dsh-ops-${name}`])))
+export const ACTIVATION_KEYS = Object.freeze({ discovery: 'enabled', panel: 'capture', shell: 'publishBashTool' })
+export const parseRows = text => parse(text, { customTags: [{ tag: 'tag:yaml.org,2002:js', resolve: value => ({ expression: value }) }] })
+
+/** Configuration facts only; readiness is established separately by setup's host probe. */
+export function switchesFromDump(text) {
+  const rows = parseRows(text)
+  if (!Array.isArray(rows)) return null
+  const switches = {}
+  for (const [name, id] of Object.entries(SWITCH_ROWS)) {
+    const matches = rows.filter(row => row?.id === id)
+    if (!matches.length) continue
+    if (matches.length !== 1) throw new Error(`Ambiguous component row ${id}`)
+    const row = matches[0]
+    const knownConfig = row.config === undefined || (row.config !== null && typeof row.config === 'object'
+      && !Array.isArray(row.config) && !Object.hasOwn(row.config, 'expression'))
+    const activation = row.config?.[ACTIVATION_KEYS[name]]
+    const enabled = row.disabled === true || activation === false ? false
+      : knownConfig && (row.disabled === undefined || row.disabled === false) && (activation === undefined || activation === true) ? true : null
+    switches[name] = { enabled }
+    if (name === 'hide-official') switches[name].replaceTools = Array.isArray(row.config?.replaceTools)
+      && row.config.replaceTools.every(value => typeof value === 'string') ? [...row.config.replaceTools].sort() : null
+  }
+  return switches
+}
+
+function canonical(value) {
+  if (Array.isArray(value)) return value.map(canonical)
+  if (value && typeof value === 'object') return Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])]))
+  return value
+}
+
+export function candidateSwitches(run) {
+  return run?.candidateSwitches ?? run?.arms?.cand?.switches ?? null
+}
+
+/** Both summaries in one A/B carry the same candidate set; baseline {} is not compared to it. */
+export function assertSameSwitches(base, cand) {
+  const left = candidateSwitches(base), right = candidateSwitches(cand)
+  if (JSON.stringify(canonical(left)) !== JSON.stringify(canonical(right))) {
+    throw new Error(`Candidate switches differ: base=${JSON.stringify(left)}; cand=${JSON.stringify(right)}; incompatible sweeps cannot be paired`)
+  }
+  return left
+}

bench/report/panel.mjs

@@ -376,6 +376,7 @@ function renderConclusion() {
 }
 function renderOverview(node) {
   const condition = obj(RUN.meta).condition
+  node.appendChild(el('p', 'mono', 'Candidate switches: ' + j(obj(obj(RUN.arms).cand).switches || null)))
   node.appendChild(el('p', 'mono', 'Condition: ' + (condition === undefined || condition === null ? 'unknown' : j(condition))))
   const environment = el('div', 'card')
   environment.appendChild(el('h3', null, 'Platform / component configuration'))

bench/run.mjs

@@ -45,6 +45,7 @@ import { renderReport, reportFileName } from './compare.mjs'
 import { reasoningEffort, planConditions, writeConditionPatch, patchArgs, describeCondition, assertEffectiveCondition } from './lib/condition.mjs'
 import { runMatrix } from './lib/matrix.mjs'
 import { componentFacts } from './lib/environment.mjs'
+import { switchesFromDump } from './lib/switches.mjs'
 
 /** The runner-only flags that take a value; `--task`/`--repeat`/`--folder`/`--base`/`--cand` are parsed by the CLI. */
 const VALUE_FLAGS = ['--sleep-ms', '--out', '--heartbeat-ms', '--stall-ms', '--timeout', '--dsh', '--llm-patch', '--model', '--reasoning-effort', '--provider']
@@ -271,6 +272,7 @@ export function profileFingerprints(arms, context = {}) {
       fs.writeFileSync(path.join(context.dirs[arm.side], 'dump-config.txt'), text, { mode: 0o600 })
     }
     arm.components = componentFacts(text)
+    arm.switches = switchesFromDump(text)
   }
   return hashes
 }

bench/run.schema.json

@@ -76,6 +76,7 @@
   "$defs": {
     "arm": { "description": "One arm of the comparison: what it was called, which profile contents it ran, and how it was launched.", "type": "object", "additionalProperties": false, "required": ["label", "profile", "profileHash"],
       "properties": {
+        "switches": { "description": "Effective component switches from the composed profile; null for unknown legacy records.", "type": ["object", "null"], "additionalProperties": { "type": "object", "required": ["enabled"], "properties": { "enabled": { "type": ["boolean", "null"] }, "replaceTools": { "type": ["array", "null"], "items": { "type": "string" } } }, "additionalProperties": false } },
         "components": { "description": "Component states read from composed configuration, not a publication/readiness receipt. Null for older or injected records.", "type": ["object", "null"], "additionalProperties": { "type": "string", "enum": ["enabled", "disabled", "absent", "unknown"] } },
         "label": { "description": "Arm label, also the profile name it ran, e.g. bench-base.", "type": "string" },
         "profile": { "description": "Profile name whose resolved configuration was measured.", "type": "string" },

bench/run/schedule.mjs

@@ -208,6 +208,8 @@ function buildSummary(arm, rows, ctx) {
     meta: { platform: ctx.spec.meta?.platform ?? `${process.platform}-${process.arch}` },
     profileHash: arm.profileHash ?? null,
     components: arm.components ?? null,
+    switches: arm.switches ?? null,
+    candidateSwitches: ctx.spec.arms?.find(item => item.side === 'cand')?.switches ?? null,
     format: SUMMARY_FORMAT,
     label: arm.label,
     profile: arm.profile,
@@ -242,6 +244,7 @@ function noiseFor(spec, summaries, arms) {
 /** The arm descriptor the seal step (P8a) lifts into `run.json`'s `arms`. */
 function descriptor(arm) {
   return {
+    switches: arm.switches ?? null,
     components: arm.components ?? null,
     label: arm.label,
     profile: arm.profile,

bench/setup.mjs

@@ -29,13 +29,14 @@ import { fileURLToPath } from 'node:url'
 import { REPO_ROOT, resolveDshEntry, spawnPlan } from './lib/env.mjs'
 import { USAGE as PROFILE_USAGE, dshHomeFor, nextStepLines, parseArgs, prepareProfiles, spawnInvoker } from './setup/profile.mjs'
 import { EXTRA_FLAGS, USAGE_EXTRA, closingLines, preflight } from './setup/preflight.mjs'
+import { SWITCH_USAGE, extractSwitches, configureSwitches } from './setup/switches.mjs'
 
 /** Exit codes: 0 prepared, 1 a check or a step failed, 2 usage. `bench/cli.mjs`'s own
  * `EXIT.pending` (3) never applies here — this module exists, so `setup` is implemented. */
 export const EXIT = { ok: 0, failure: 1, usage: 2 }
 
 /** The whole help: `profile.mjs`'s flags plus setup B's own. */
-export const USAGE = `${PROFILE_USAGE}\n${USAGE_EXTRA}`
+export const USAGE = `${PROFILE_USAGE}\n${USAGE_EXTRA}\n${SWITCH_USAGE}`
 
 /**
  * The injectable half of `main`. `bench/cli.mjs` passes `{out, err}`; a suite passes the
@@ -69,8 +70,12 @@ export const USAGE = `${PROFILE_USAGE}\n${USAGE_EXTRA}`
 export async function main(argv, io = {}) {
   const out = io.out ?? ((line) => console.log(line))
   const err = io.err ?? ((line) => console.error(line))
-  let options
-  try { options = parseArgs(argv, { ...EXTRA_FLAGS, platform: io.platform }) } catch (error) {
+  let options, switches
+  try {
+    const extracted = extractSwitches(argv)
+    switches = extracted.switches
+    options = parseArgs(extracted.argv, { ...EXTRA_FLAGS, platform: io.platform })
+  } catch (error) {
     err(`${error.message}\n\n${USAGE}`)
     return EXIT.usage
   }
@@ -122,6 +127,8 @@ export async function main(argv, io = {}) {
     err(`\nsetup is not finished: ${prepared.problems.join('; ')}`)
     return EXIT.failure
   }
+  try { configureSwitches(options, { home, log, invoke, prefix: plan.args, overlay: io.overlay }, switches) }
+  catch (error) { err(`component switches: ${error.message}`); return EXIT.failure }
   out('\n— next —')
   for (const line of closingLines(options)) out(line)
   out(options.dryRun ? '\n[dry-run] everything above is a plan; nothing was changed.' : '\nsetup complete.')