From 0ab4394e310d53cc30a58274b3cebbded27e7ab8 Mon Sep 17 00:00:00 2001 From: Shen Haoming <167516473+Haoming9527@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:05:09 +0800 Subject: [PATCH] feat(bench): configure and compare candidate component switches --- bench/README.md | 19 +++++ bench/compare.mjs | 1 + bench/lib/compare-core.mjs | 5 ++ bench/lib/switches.mjs | 46 +++++++++++ bench/report/panel.mjs | 1 + bench/run.mjs | 2 + bench/run.schema.json | 1 + bench/run/schedule.mjs | 3 + bench/setup.mjs | 13 ++- bench/setup/switches.mjs | 131 ++++++++++++++++++++++++++++++ docs/CHANGELOG.md | 3 + test/bench-compare.test.mjs | 2 +- test/bench-schedule.test.mjs | 2 + test/bench-seal.test.mjs | 6 +- test/bench-switches.test.mjs | 151 +++++++++++++++++++++++++++++++++++ 15 files changed, 380 insertions(+), 6 deletions(-) create mode 100644 bench/lib/switches.mjs create mode 100644 bench/setup/switches.mjs create mode 100644 test/bench-switches.test.mjs diff --git a/bench/README.md b/bench/README.md index bdaaa527..1db50ff4 100644 --- a/bench/README.md +++ b/bench/README.md @@ -71,6 +71,25 @@ node bench/cli.mjs report --matrix bench/runs/benchmark-YYYY-MM-DD-NN Model flags set the actual shared route, with the condition patch applied after the provider overlay. Both arms archive their composed configuration. Repeated models and reasoning levels form independent Cartesian groups; `report --matrix` renders one section per condition and never combines deltas. Cross-condition comparisons exit 2, naming both conditions. Without new flags, routing is unchanged and the condition is `unknown`; historical labels do not prove routing. An omitted reasoning level remains `null` unless the overlay pins it. All examples are optional model-driven runs, outside CI. +### 组件开关 / Component switches + +```console +node bench/cli.mjs setup --switch hide-official=all --switch ptc --switch panel=false --dry-run +node bench/cli.mjs setup --switch hide-official=read,grep --switch discovery --switch shell +``` + +`--switch` 可重复，支持 `hide-official`、`ptc`、`panel`、`discovery`、`shell`；省略值等同 `true`，普通开关也接受 `false`。`hide-official=all` 只选择已有替代实现的 `read,glob,grep`，也可指定不重复的子集或 `false`。只修改 candidate 的 `cordis.patch.yml`；baseline 不写入组件行。每次 setup 只改明确指定的开关，不自动搜索组合。 + +写入幂等；本地已有相反值需 `--force`，继承的字面配置键会保留。动态表达式、非映射配置、重复行和无法安全编辑的目标形态需手工处理，`--force` 也不会猜测。`--dry-run` 打印请求的行和值，不启动宿主、不修改 profile。 + +实跑先读合成配置，写入后再核验配置并执行一次有超时的最小 headless turn；这可能调用已配置模型。没有已完成 turn、宿主错误或 `READINESS_BLOCKED` 都会让 setup 失败并指向 [#207](https://github.com/T-Auto/dsh-ops/issues/207)，不会启动 benchmark sweep。这个探测不代替 Desktop 重启后的人工验证。 + +实际配置写入摘要及封存 `run.json` 的 `arms.cand.switches`，报告头和 panel 显示 candidate 开关。baseline 的空集合与 candidate 的非空集合属于正常 A/B；两份运行的 candidate 集合不同则 compare 退出 `2`，列出差异，不合并 Δ。未求值配置记录 `null`，旧数据缺失开关元数据仍可读取为未知；这些字段是配置事实，不证明组件已经发布到会话。 + +Repeat `--switch` to set candidate components. Values default to `true`; booleans also accept `false`. `hide-official=all` expands only the implemented `read,glob,grep` replacements, or select a unique subset. Setup edits only the candidate patch, preserves inherited literal configuration, and changes only named switches. Conflicting local values require `--force`; dynamic or unsafe shapes require manual edits even with force. Dry-run prints the requested values without launching the host or writing profiles. + +Real setup verifies composed configuration and runs one bounded headless readiness turn, which may call the configured model. Missing successful completion, host errors and `READINESS_BLOCKED` fail setup with the #207 diagnostic before a sweep starts. This does not substitute for Desktop validation. Summaries and sealed `arms.cand.switches` retain effective configuration; report headers and the panel show it. Comparisons reject different candidate switch sets with exit 2, while the baseline's empty set is expected. Unevaluated configuration and historical missing metadata remain unknown, without implying publication readiness. Switch combinations are selected explicitly and are outside the CI gate. + ## 任务一览 六个任务各自压一种能力，`ops_*` 工具名来自 `publicToolName()`。每个任务的期望值都由夹具生成器现算（`derive*`），判卷在 `bench/lib/verify.mjs`。 diff --git a/bench/compare.mjs b/bench/compare.mjs index 85a40616..aac13e4a 100644 --- a/bench/compare.mjs +++ b/bench/compare.mjs @@ -531,6 +531,7 @@ export function renderReport(comparison) { REPORT_TITLE, '', `Condition: ${describeCondition(comparison.condition)}`, + `Candidate switches: ${JSON.stringify(comparison.switches ?? null)} (null means unknown)`, '', ...environmentLines(comparison.environment), ...taskSection(comparison), diff --git a/bench/lib/compare-core.mjs b/bench/lib/compare-core.mjs index c9fe4da7..96a3f947 100644 --- a/bench/lib/compare-core.mjs +++ b/bench/lib/compare-core.mjs @@ -35,6 +35,7 @@ import { CORE_BUCKETS, median, minValue, pairMetrics } from './aggregate.mjs' import { assertSameCondition } from './condition.mjs' import { compareEnvironment } from './environment.mjs' +import { assertSameSwitches } from './switches.mjs' export { median, minValue } @@ -483,6 +484,7 @@ export function classifyExcluded(pairs, taskId = null) { * @returns {object} the noise-floor document; `pct: null` when nothing could be paired. */ export function noiseFloor(baseA, baseB, options = {}) { + assertSameSwitches(baseA, baseB) assertSameCondition(baseA, baseB) compareEnvironment(baseA, baseB, options) const aggregate = AGGREGATES.includes(options.aggregate) ? options.aggregate : 'median' @@ -708,6 +710,8 @@ export function formatNumber(value) { * @returns {object} the comparison structure, including the gate and the verdict. */ export function compareSummaries(base, cand, options = {}) { + const switches = assertSameSwitches(base, cand) + if (options.noise) assertSameSwitches(base, options.noise.base) const condition = assertSameCondition(base, cand) const environment = compareEnvironment(base, cand, options) if (options.noise) compareEnvironment(base, options.noise.base, options) @@ -862,6 +866,7 @@ export function compareSummaries(base, cand, options = {}) { return { format: COMPARE_FORMAT, + switches, condition, environment, base: { label: sideLabel(base, 'base'), profile: sideProfile(base, 'base'), model: baseModel, dshVersion: sideDshVersion(base, 'base') }, diff --git a/bench/lib/switches.mjs b/bench/lib/switches.mjs new file mode 100644 index 00000000..a7e35234 --- /dev/null +++ b/bench/lib/switches.mjs @@ -0,0 +1,46 @@ +import { parse } from 'yaml' + +export const SWITCH_ROWS = Object.freeze(Object.fromEntries(['hide-official', 'ptc', 'panel', 'discovery', 'shell'].map(name => [name, `dsh-ops-${name}`]))) +export const ACTIVATION_KEYS = Object.freeze({ discovery: 'enabled', panel: 'capture', shell: 'publishBashTool' }) +export const parseRows = text => parse(text, { customTags: [{ tag: 'tag:yaml.org,2002:js', resolve: value => ({ expression: value }) }] }) + +/** Configuration facts only; readiness is established separately by setup's host probe. */ +export function switchesFromDump(text) { + const rows = parseRows(text) + if (!Array.isArray(rows)) return null + const switches = {} + for (const [name, id] of Object.entries(SWITCH_ROWS)) { + const matches = rows.filter(row => row?.id === id) + if (!matches.length) continue + if (matches.length !== 1) throw new Error(`Ambiguous component row ${id}`) + const row = matches[0] + const knownConfig = row.config === undefined || (row.config !== null && typeof row.config === 'object' + && !Array.isArray(row.config) && !Object.hasOwn(row.config, 'expression')) + const activation = row.config?.[ACTIVATION_KEYS[name]] + const enabled = row.disabled === true || activation === false ? false + : knownConfig && (row.disabled === undefined || row.disabled === false) && (activation === undefined || activation === true) ? true : null + switches[name] = { enabled } + if (name === 'hide-official') switches[name].replaceTools = Array.isArray(row.config?.replaceTools) + && row.config.replaceTools.every(value => typeof value === 'string') ? [...row.config.replaceTools].sort() : null + } + return switches +} + +function canonical(value) { + if (Array.isArray(value)) return value.map(canonical) + if (value && typeof value === 'object') return Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) + return value +} + +export function candidateSwitches(run) { + return run?.candidateSwitches ?? run?.arms?.cand?.switches ?? null +} + +/** Both summaries in one A/B carry the same candidate set; baseline {} is not compared to it. */ +export function assertSameSwitches(base, cand) { + const left = candidateSwitches(base), right = candidateSwitches(cand) + if (JSON.stringify(canonical(left)) !== JSON.stringify(canonical(right))) { + throw new Error(`Candidate switches differ: base=${JSON.stringify(left)}; cand=${JSON.stringify(right)}; incompatible sweeps cannot be paired`) + } + return left +} diff --git a/bench/report/panel.mjs b/bench/report/panel.mjs index 4be565cc..59dfbc25 100644 --- a/bench/report/panel.mjs +++ b/bench/report/panel.mjs @@ -376,6 +376,7 @@ function renderConclusion() { } function renderOverview(node) { const condition = obj(RUN.meta).condition + node.appendChild(el('p', 'mono', 'Candidate switches: ' + j(obj(obj(RUN.arms).cand).switches || null))) node.appendChild(el('p', 'mono', 'Condition: ' + (condition === undefined || condition === null ? 'unknown' : j(condition)))) const environment = el('div', 'card') environment.appendChild(el('h3', null, 'Platform / component configuration')) diff --git a/bench/run.mjs b/bench/run.mjs index 50b61d38..cae36179 100644 --- a/bench/run.mjs +++ b/bench/run.mjs @@ -45,6 +45,7 @@ import { renderReport, reportFileName } from './compare.mjs' import { reasoningEffort, planConditions, writeConditionPatch, patchArgs, describeCondition, assertEffectiveCondition } from './lib/condition.mjs' import { runMatrix } from './lib/matrix.mjs' import { componentFacts } from './lib/environment.mjs' +import { switchesFromDump } from './lib/switches.mjs' /** The runner-only flags that take a value; `--task`/`--repeat`/`--folder`/`--base`/`--cand` are parsed by the CLI. */ const VALUE_FLAGS = ['--sleep-ms', '--out', '--heartbeat-ms', '--stall-ms', '--timeout', '--dsh', '--llm-patch', '--model', '--reasoning-effort', '--provider'] @@ -271,6 +272,7 @@ export function profileFingerprints(arms, context = {}) { fs.writeFileSync(path.join(context.dirs[arm.side], 'dump-config.txt'), text, { mode: 0o600 }) } arm.components = componentFacts(text) + arm.switches = switchesFromDump(text) } return hashes } diff --git a/bench/run.schema.json b/bench/run.schema.json index ef4ab204..f5f9c01a 100644 --- a/bench/run.schema.json +++ b/bench/run.schema.json @@ -76,6 +76,7 @@ "$defs": { "arm": { "description": "One arm of the comparison: what it was called, which profile contents it ran, and how it was launched.", "type": "object", "additionalProperties": false, "required": ["label", "profile", "profileHash"], "properties": { + "switches": { "description": "Effective component switches from the composed profile; null for unknown legacy records.", "type": ["object", "null"], "additionalProperties": { "type": "object", "required": ["enabled"], "properties": { "enabled": { "type": ["boolean", "null"] }, "replaceTools": { "type": ["array", "null"], "items": { "type": "string" } } }, "additionalProperties": false } }, "components": { "description": "Component states read from composed configuration, not a publication/readiness receipt. Null for older or injected records.", "type": ["object", "null"], "additionalProperties": { "type": "string", "enum": ["enabled", "disabled", "absent", "unknown"] } }, "label": { "description": "Arm label, also the profile name it ran, e.g. bench-base.", "type": "string" }, "profile": { "description": "Profile name whose resolved configuration was measured.", "type": "string" }, diff --git a/bench/run/schedule.mjs b/bench/run/schedule.mjs index 143dd8c8..409d7c55 100644 --- a/bench/run/schedule.mjs +++ b/bench/run/schedule.mjs @@ -208,6 +208,8 @@ function buildSummary(arm, rows, ctx) { meta: { platform: ctx.spec.meta?.platform ?? `${process.platform}-${process.arch}` }, profileHash: arm.profileHash ?? null, components: arm.components ?? null, + switches: arm.switches ?? null, + candidateSwitches: ctx.spec.arms?.find(item => item.side === 'cand')?.switches ?? null, format: SUMMARY_FORMAT, label: arm.label, profile: arm.profile, @@ -242,6 +244,7 @@ function noiseFor(spec, summaries, arms) { /** The arm descriptor the seal step (P8a) lifts into `run.json`'s `arms`. */ function descriptor(arm) { return { + switches: arm.switches ?? null, components: arm.components ?? null, label: arm.label, profile: arm.profile, diff --git a/bench/setup.mjs b/bench/setup.mjs index 705fcf36..824960c0 100644 --- a/bench/setup.mjs +++ b/bench/setup.mjs @@ -29,13 +29,14 @@ import { fileURLToPath } from 'node:url' import { REPO_ROOT, resolveDshEntry, spawnPlan } from './lib/env.mjs' import { USAGE as PROFILE_USAGE, dshHomeFor, nextStepLines, parseArgs, prepareProfiles, spawnInvoker } from './setup/profile.mjs' import { EXTRA_FLAGS, USAGE_EXTRA, closingLines, preflight } from './setup/preflight.mjs' +import { SWITCH_USAGE, extractSwitches, configureSwitches } from './setup/switches.mjs' /** Exit codes: 0 prepared, 1 a check or a step failed, 2 usage. `bench/cli.mjs`'s own * `EXIT.pending` (3) never applies here — this module exists, so `setup` is implemented. */ export const EXIT = { ok: 0, failure: 1, usage: 2 } /** The whole help: `profile.mjs`'s flags plus setup B's own. */ -export const USAGE = `${PROFILE_USAGE}\n${USAGE_EXTRA}` +export const USAGE = `${PROFILE_USAGE}\n${USAGE_EXTRA}\n${SWITCH_USAGE}` /** * The injectable half of `main`. `bench/cli.mjs` passes `{out, err}`; a suite passes the @@ -69,8 +70,12 @@ export const USAGE = `${PROFILE_USAGE}\n${USAGE_EXTRA}` export async function main(argv, io = {}) { const out = io.out ?? ((line) => console.log(line)) const err = io.err ?? ((line) => console.error(line)) - let options - try { options = parseArgs(argv, { ...EXTRA_FLAGS, platform: io.platform }) } catch (error) { + let options, switches + try { + const extracted = extractSwitches(argv) + switches = extracted.switches + options = parseArgs(extracted.argv, { ...EXTRA_FLAGS, platform: io.platform }) + } catch (error) { err(`${error.message}\n\n${USAGE}`) return EXIT.usage } @@ -122,6 +127,8 @@ export async function main(argv, io = {}) { err(`\nsetup is not finished: ${prepared.problems.join('; ')}`) return EXIT.failure } + try { configureSwitches(options, { home, log, invoke, prefix: plan.args, overlay: io.overlay }, switches) } + catch (error) { err(`component switches: ${error.message}`); return EXIT.failure } out('\n— next —') for (const line of closingLines(options)) out(line) out(options.dryRun ? '\n[dry-run] everything above is a plan; nothing was changed.' : '\nsetup complete.') diff --git a/bench/setup/switches.mjs b/bench/setup/switches.mjs new file mode 100644 index 00000000..01fc2622 --- /dev/null +++ b/bench/setup/switches.mjs @@ -0,0 +1,131 @@ +import fs from 'node:fs' +import path from 'node:path' +import { OFFICIAL_REPLACEMENTS } from '../../lib/official-tool-catalog.js' +import { LLM_OVERLAY, planEntryPatch } from './profile.mjs' +import { ACTIVATION_KEYS, SWITCH_ROWS, parseRows, switchesFromDump } from '../lib/switches.mjs' + +export const SWITCH_USAGE = ' --switch [=value] candidate only; repeat hide-official=all|read,glob,grep|false, ptc, panel, discovery, shell (booleans true/false)' + +export function extractSwitches(argv) { + const rest = [], switches = {} + for (let i = 0; i < argv.length; i++) { + if (argv[i] !== '--switch' && !argv[i].startsWith('--switch=')) { rest.push(argv[i]); continue } + const spec = argv[i] === '--switch' ? argv[++i] : argv[i].slice(9) + if (!spec || spec.startsWith('--')) throw new Error('--switch needs a name') + const equal = spec.indexOf('='), name = equal < 0 ? spec : spec.slice(0, equal), value = equal < 0 ? 'true' : spec.slice(equal + 1) + if (!Object.hasOwn(SWITCH_ROWS, name)) throw new Error(`Unknown switch ${name}; available: ${Object.keys(SWITCH_ROWS).join(', ')}`) + if (Object.hasOwn(switches, name)) throw new Error(`Duplicate switch ${name}`) + if (name !== 'hide-official') { + if (!['true', 'false'].includes(value)) throw new Error(`${name} requires true or false`) + switches[name] = { enabled: value === 'true' } + } else { + const names = Object.keys(OFFICIAL_REPLACEMENTS) + const selected = ['all', 'true'].includes(value) ? names : value === 'false' ? [] : value.split(',') + if (selected.some(item => !names.includes(item)) || new Set(selected).size !== selected.length) throw new Error(`hide-official requires all, false or a unique list from ${names.join(', ')}`) + switches[name] = { enabled: value !== 'false', replaceTools: [...selected].sort() } + } + } + return { argv: rest, switches } +} + +function literal(value) { + if (typeof value === 'number' && !Number.isFinite(value)) throw new Error('Non-finite config needs a manual patch') + if (value && typeof value === 'object') { + if (Object.hasOwn(value, 'expression')) throw new Error('A dynamic component config needs a manual patch; refusing to rewrite it') + for (const child of Object.values(value)) literal(child) + } + if (value !== null && !['object', 'string', 'boolean', 'number'].includes(typeof value)) throw new Error('Unsupported config value') + return JSON.stringify(value) +} + +function configMapping(value) { + if (value === undefined) return {} + if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Component config requires a literal mapping; edit it manually') + literal(value) + return value +} +const escapePattern = value => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + +/** Line-local, atomic planning. Unsupported inline/aliased/multiline target shapes are refused. */ +export function planSwitchPatch(text, switches, { force = false, composed = [] } = {}) { + let next = text + for (const [name, wanted] of Object.entries(switches)) { + if (!Object.hasOwn(SWITCH_ROWS, name)) throw new Error(`Unknown switch ${name}`) + const id = SWITCH_ROWS[name], rows = parseRows(next) + if (!Array.isArray(rows)) throw new Error('Candidate patch must be a plain top-level list') + const own = rows.filter(row => row?.id === id) + if (own.length > 1) throw new Error(`Duplicate row ${id}`) + const inherited = composed.filter(row => row?.id === id) + if (inherited.length > 1) throw new Error(`Ambiguous composed row ${id}`) + const row = own[0] + if (row?.disabled !== undefined && row.disabled !== !wanted.enabled && !force) throw new Error(`${id}.disabled conflicts; use --force`) + if (row && row.disabled !== undefined && typeof row.disabled !== 'boolean') throw new Error(`${id}.disabled requires a manual patch`) + const config = { ...configMapping(inherited[0]?.config), ...configMapping(row?.config) } + const changes = name === 'hide-official' ? { replaceTools: wanted.replaceTools } : ACTIVATION_KEYS[name] && wanted.enabled ? { [ACTIVATION_KEYS[name]]: true } : {} + for (const [key, value] of Object.entries(changes)) { + if (row?.config?.[key] !== undefined && JSON.stringify(row.config[key]) !== JSON.stringify(value) && !force) throw new Error(`${id}.${key} conflicts; use --force`) + config[key] = value + } + // Setting config replaces that entry's config in the host. Preserve every inherited key. + for (const [key, value] of Object.entries(config)) { + if (JSON.stringify(row?.config?.[key]) === JSON.stringify(value)) continue + const lines = next.split(/\r?\n/) + const start = lines.findIndex(line => line === `- id: ${id}`) + if (row && start < 0) throw new Error(`${id} requires a plain block row; edit it manually`) + const end = start < 0 ? -1 : lines.findIndex((line, i) => i > start && /^- /.test(line)) + const body = start < 0 ? [] : lines.slice(start, end < 0 ? lines.length : end) + if (body.some(line => new RegExp(`^\\s+${escapePattern(key)}:\\s*(?:#.*)?$`).test(line))) throw new Error(`${id}.${key} uses a multiline value; edit it manually`) + const plan = planEntryPatch(next, { id, config: { [key]: literal(value) } }, { force: true }) + if (!['write', 'noop'].includes(plan.action)) throw new Error(plan.note) + next = plan.text + } + if (!parseRows(next).some(item => item?.id === id)) next = `${next.replace(/^\s*\[\]\s*$/m, '').trimEnd()}\n- id: ${id}\n` + const lines = next.split(/\r?\n/), start = lines.findIndex(line => line === `- id: ${id}`) + if (start < 0) throw new Error(`${id} requires a plain block row; edit it manually`) + let end = lines.findIndex((line, i) => i > start && /^- /.test(line)); if (end < 0) end = lines.length + const disabled = lines.findIndex((line, i) => i > start && i < end && /^ disabled:/.test(line)) + if (disabled < 0) lines.splice(start + 1, 0, ` disabled: ${!wanted.enabled}`) + else if (parseRows(next).find(item => item?.id === id).disabled !== !wanted.enabled) lines[disabled] = ` disabled: ${!wanted.enabled}` + next = lines.join('\n') + } + parseRows(next) + return { text: next, changed: next !== text } +} + +export function configureSwitches(options, context, switches) { + if (!Object.keys(switches).length) return + const file = path.join(context.home, 'profiles', options.candProfile, 'cordis.patch.yml') + const before = fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : '[]\n' + const args = [...context.prefix, '--profile', options.candProfile] + const overlay = context.overlay ?? LLM_OVERLAY + if (fs.existsSync(overlay)) args.push('--patch', overlay) + const dump = () => { + const result = context.invoke([...args, '--dump-config']) + if (result.status !== 0) throw new Error('Candidate --dump-config failed; no sweep was started') + const rows = parseRows(result.stdout) + if (!Array.isArray(rows)) throw new Error('Candidate dump is not a component list') + return { rows, switches: switchesFromDump(result.stdout) } + } + const composed = options.dryRun ? [] : dump().rows + const plan = planSwitchPatch(before, switches, { force: options.force, composed }) + context.log(` candidate switches: ${JSON.stringify(switches)}`) + if (options.dryRun) { + context.log(` [dry-run] would merge effective component config into ${file}; baseline has no switch writes`) + context.log(' [dry-run] would verify --dump-config and run a bounded headless readiness turn (may call the configured model)') + return + } + if (plan.changed) fs.writeFileSync(file, plan.text, 'utf8') + const effective = dump().switches + for (const [name, wanted] of Object.entries(switches)) { + if (JSON.stringify(effective[name]) !== JSON.stringify(wanted)) throw new Error(`Composed switch ${name} differs from requested value`) + } + const result = context.invoke([...args, '--json', 'Reply only READY. Do not call any tools.']) + const output = `${result.stdout ?? ''}\n${result.stderr ?? ''}` + const events = String(result.stdout ?? '').split(/\r?\n/).flatMap(line => { try { return [JSON.parse(line)] } catch { return [] } }) + const turns = events.filter(event => event.type === 'status' && event.phase === 'turn_end') + if (result.status !== 0 || /READINESS_BLOCKED/.test(output) || events.some(event => event.type === 'error') + || !turns.length || turns.some(event => event.reason?.kind !== 'completed')) { + throw new Error('Candidate readiness probe failed; no sweep was started. Check host diagnostics and https://github.com/T-Auto/dsh-ops/issues/207 (READINESS_BLOCKED).') + } + context.log(' candidate readiness probe completed; no benchmark task was run') +} diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index daea4642..6f72240d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,9 @@ this package adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased] +- Add candidate-only benchmark component switches with conservative patch editing, composed-config verification and a bounded readiness turn. Archive effective switches, show them in reports, and reject comparisons across different candidate switch sets (#296). +- 为 benchmark 增加仅作用于 candidate 的组件开关，保守编辑补丁并核验合成配置及有时限的就绪 turn；归档实际开关、在报告中展示，并拒绝不同 candidate 开关集的混配（#296）。 + - Make benchmark model/reasoning selections actual shared routing conditions, archive composed configuration, and record the condition in sealed evidence. Repeated selections produce separate condition groups and reports; cross-condition pairing is refused, while historical runs remain readable as unknown (#294). - 将 benchmark 的模型与思考档位选择用于两臂实际路由，归档合成配置并在封存证据中记录条件。重复选项生成独立条件组和分组报告，拒绝跨条件配对；历史运行仍可按未知条件读取（#294）。 - Make benchmark setup skip Windows-only checks explicitly on POSIX, preserve platform/component configuration in summaries and sealed arms, reject cross-platform comparisons by default, and expand the optional tools smoke to Linux, Windows and macOS (#295). diff --git a/test/bench-compare.test.mjs b/test/bench-compare.test.mjs index 2c43b8c1..753e2a7b 100644 --- a/test/bench-compare.test.mjs +++ b/test/bench-compare.test.mjs @@ -167,7 +167,7 @@ await test('the same comparison renders byte-identical text twice: tables first, const text = renderReport(beyond) assert.equal(renderReport(beyond), text, 'rendering is a pure function of its inputs') assertEnglish(text) - assert.match(text, /^# DSH-OPS Benchmark Report \(beta\)\n\nCondition: unknown\n\nPlatform: base=unknown; cand=unknown\./, 'the header states condition and environment evidence before the data') + assert.match(text, /^# DSH-OPS Benchmark Report \(beta\)\n\nCondition: unknown\nCandidate switches: null \(null means unknown\)\n\nPlatform: base=unknown; cand=unknown\./, 'the header states condition, switches and environment evidence before the data') assert.match(text, /^## Per-task pairs$/m) assert.match(text, /^\| task \| base \| ops \(Δ%\) \| steps: base→ops \(Δ%\) \| tool calls: base→ops \(Δ%\) \| time: base→ops \(Δ%\) \| tool errors \| pass \|$/m, 'the first table carries cost, behaviour, wall-clock evidence and pass: no pairs, exclusion or direction column') assert.ok(text.includes(`| csv-sum | 1,000 | 926 (${green('\u22127.4%')}) | 4 → 4 (${green('0.0%')}) | 6 → 6 (${green('0.0%')}) | 60.0s → 55.0s (${green('\u22128.3%')}) | 0 → 0 | 2/2→2/2 |`), 'each cell keeps its own Δ% in parentheses, painted like every other percent — time included') diff --git a/test/bench-schedule.test.mjs b/test/bench-schedule.test.mjs index 4c356952..3441ccf0 100644 --- a/test/bench-schedule.test.mjs +++ b/test/bench-schedule.test.mjs @@ -536,6 +536,8 @@ await test('main: a sweep is unsealed by default and --seal opts in', async () = assert.ok(fs.existsSync(path.join(group, armDir, 'summary.json')), 'the per-arm summary is the artifact the report reads') const summary = JSON.parse(fs.readFileSync(path.join(group, armDir, 'summary.json'), 'utf8')) assert.equal(summary.components.shell, 'enabled', 'unsealed summaries still preserve component configuration') + assert.equal(summary.switches.shell.enabled, true) + assert.equal(summary.candidateSwitches.shell.enabled, true, 'both sides share the candidate conditions for pairing') assert.equal(summary.meta.platform, `${process.platform}-${process.arch}`) assert.equal(fs.existsSync(path.join(group, 'run.json')), false, 'nothing sealed this sweep') const reports = fs.readdirSync(group).filter((name) => name.startsWith('DSH-OPS Benchmark Report') && name.endsWith('.md')) diff --git a/test/bench-seal.test.mjs b/test/bench-seal.test.mjs index ca6fe77c..7f7dd53c 100644 --- a/test/bench-seal.test.mjs +++ b/test/bench-seal.test.mjs @@ -28,8 +28,8 @@ const SEALED_AT = new Date(NOW).toISOString() /** The two sides, with the fingerprints the contract requires. */ const ARMS = { - base: { label: 'bench-base', profile: 'bench-base', profileHash: 'a'.repeat(64) }, - cand: { label: 'bench-ops', profile: 'bench-ops', profileHash: 'b'.repeat(64) }, + base: { label: 'bench-base', profile: 'bench-base', profileHash: 'a'.repeat(64), switches: {} }, + cand: { label: 'bench-ops', profile: 'bench-ops', profileHash: 'b'.repeat(64), switches: { panel: { enabled: true } } }, } /** A string that exists only in a non-milestone trace lane, so "not inlined" is provable. */ @@ -163,6 +163,8 @@ await test('sealRun writes one schema-valid run.json and reports where it went', assert.equal(validation.ok, true) assert.equal(document.meta.arms.base.label, 'bench-base') assert.equal(document.meta.arms.cand.profileHash, ARMS.cand.profileHash) + assert.deepEqual(document.arms.base.switches, {}) + assert.deepEqual(document.arms.cand.switches, ARMS.cand.switches) assert.equal(document.verify.checks, 2) assert.equal(document.verify.passed, 2) assert.equal(document.verify.failed, 0) diff --git a/test/bench-switches.test.mjs b/test/bench-switches.test.mjs new file mode 100644 index 00000000..ae1136c8 --- /dev/null +++ b/test/bench-switches.test.mjs @@ -0,0 +1,151 @@ +import fs from 'node:fs' +import path from 'node:path' +import { parse } from 'yaml' +import { renderConfigDump } from '@deepseek-ai/dsh-app-boot' +import { JSDOM } from 'jsdom' +import { assert, report, test, withTempDir, waitFor } from './lib/harness.mjs' +import { extractSwitches, planSwitchPatch, configureSwitches } from '../bench/setup/switches.mjs' +import { main as setup } from '../bench/setup.mjs' +import { compareSummaries } from '../bench/lib/compare-core.mjs' +import { main as compareMain, renderReport } from '../bench/compare.mjs' +import { renderPanel } from '../bench/report/panel.mjs' +import { switchesFromDump } from '../bench/lib/switches.mjs' + +await test('switch parser validates repeatable names and replacement vocabulary', () => { + assert.deepEqual(extractSwitches(['--dry-run', '--switch', 'ptc=false', '--switch=hide-official=all']).switches, + { ptc: { enabled: false }, 'hide-official': { enabled: true, replaceTools: ['glob', 'grep', 'read'] } }) + for (const argv of [['--switch', 'oops'], ['--switch', 'ptc=maybe'], ['--switch', 'hide-official=write'], ['--switch', 'ptc', '--switch', 'ptc=false']]) assert.throws(() => extractSwitches(argv)) +}) + +await test('planning is idempotent and preserves foreign rows plus inherited component settings', () => { + const before = '# preserve me\n- id: unrelated\n config:\n value: 9\n' + const switches = extractSwitches(['--switch', 'discovery', '--switch', 'hide-official=all']).switches + const composed = [{ id: 'dsh-ops-discovery', config: { maxResults: 7, required: true, enabled: false } }] + const plan = planSwitchPatch(before, switches, { composed }) + assert.ok(plan.text.startsWith(before)) + assert.deepEqual(parse(plan.text).find(row => row.id === 'dsh-ops-discovery').config, { maxResults: 7, required: true, enabled: true }) + assert.equal(planSwitchPatch(plan.text, switches, { composed }).changed, false) + assert.throws(() => planSwitchPatch('- id: dsh-ops-ptc\n disabled: true\n', { ptc: { enabled: true } }), /force/) + assert.equal(parse(planSwitchPatch('- id: dsh-ops-ptc\n disabled: true\n', { ptc: { enabled: true } }, { force: true }).text)[0].disabled, false) + assert.throws(() => planSwitchPatch('- id: dsh-ops-ptc\n- id: dsh-ops-ptc\n', { ptc: { enabled: true } }), /Duplicate/) +}) + +await test('setup accepts switches in dry-run without launching a host or editing either profile', async () => { + await withTempDir('switch-dry-', async dir => { + const lines = [], example = path.join(dir, 'example.yml') + fs.writeFileSync(example, '[]\n') + const code = await setup(['--dsh-home', dir, '--plugin', 'none', '--no-pin', '--no-permission-mode', '--dry-run', '--switch', 'hide-official=all'], { + entry: { file: 'fixture-dsh', kind: 'exe' }, overlay: path.join(dir, 'llm.yml'), overlayExample: example, + invoke: () => { throw new Error('dry-run invoked host') }, out: line => lines.push(line), err: line => lines.push(line), + }) + assert.equal(code, 0, lines.join('\n')) + assert.match(lines.join('\n'), /candidate switches.*hide-official/) + assert.equal(fs.existsSync(path.join(dir, 'profiles')), false) + }) +}) + +await test('candidate preparation preserves baseline and refuses blocked or unproven readiness', async () => { + await withTempDir('switch-ready-', async dir => { + for (const profile of ['base', 'cand']) fs.mkdirSync(path.join(dir, 'profiles', profile), { recursive: true }) + const base = path.join(dir, 'profiles/base/cordis.patch.yml'), cand = path.join(dir, 'profiles/cand/cordis.patch.yml') + fs.writeFileSync(base, '# baseline sentinel\n[]\n'); fs.writeFileSync(cand, '[]\n') + const calls = [], options = { candProfile: 'cand', force: false, dryRun: false } + const context = { home: dir, prefix: [], overlay: path.join(dir, 'missing'), log: () => {}, invoke: args => { + calls.push(args) + assert.equal(args[args.indexOf('--profile') + 1], 'cand') + return args.includes('--dump-config') ? { status: 0, stdout: fs.readFileSync(cand, 'utf8') } + : { status: 0, stdout: '{"type":"status","phase":"turn_end","reason":{"kind":"completed"}}\n' } + } } + configureSwitches(options, context, { ptc: { enabled: true } }) + assert.equal(fs.readFileSync(base, 'utf8'), '# baseline sentinel\n[]\n') + assert.equal(calls.filter(args => args.includes('--json')).length, 1) + const good = context.invoke + for (const output of ['READINESS_BLOCKED', 'READY', '{"type":"turn_end","reason":"completed"}', '{"type":"status","phase":"turn_end","reason":{"kind":"failed"}}']) { + context.invoke = args => args.includes('--dump-config') ? good(args) : { status: 0, stdout: output } + assert.throws(() => configureSwitches(options, context, { ptc: { enabled: true } }), /issues\/207/) + } + }) +}) + +await test('candidate switch conditions differ across sweeps, never across the two arms of one sweep', () => { + const switches = { ptc: { enabled: true } } + const base = { results: [], switches: {}, candidateSwitches: switches } + const cand = { results: [], switches, candidateSwitches: switches } + assert.doesNotThrow(() => compareSummaries(base, cand)) + assert.throws(() => compareSummaries(base, { ...cand, candidateSwitches: { ptc: { enabled: false } } }), /Candidate switches differ/) + assert.match(renderReport(compareSummaries(base, cand)), /Candidate switches/) +}) + +await test('dynamic and non-mapping configuration is refused before candidate planning', () => { + for (const config of [{ expression: 'dynamic()' }, [true], 'unknown', 9]) { + assert.throws(() => planSwitchPatch('[]\n', { panel: { enabled: true } }, { + force: true, composed: [{ id: 'dsh-ops-panel', config }], + }), /manual|mapping|dynamic/) + } +}) + +await test('panel and shell activation respect legacy switches and explicit overrides', () => { + for (const [name, key] of [['panel', 'capture'], ['shell', 'publishBashTool']]) { + const before = `- id: dsh-ops-${name}\n disabled: false\n config:\n ${key}: false\n custom: 700\n` + assert.equal(switchesFromDump(before)[name].enabled, false) + assert.throws(() => planSwitchPatch(before, { [name]: { enabled: true } }), /force/) + const plan = planSwitchPatch(before, { [name]: { enabled: true } }, { force: true }) + assert.equal(parse(plan.text)[0].config[key], true) + assert.equal(parse(plan.text)[0].config.custom, 700) + assert.equal(switchesFromDump(plan.text)[name].enabled, true) + } +}) + +await test('unevaluated and malformed activation config remains unknown in archived facts', () => { + for (const name of ['panel', 'shell', 'discovery']) { + for (const config of ['!!js dynamic()', '[]', 'false']) { + assert.equal(switchesFromDump(`- id: dsh-ops-${name}\n config: ${config}\n`)[name].enabled, null) + } + assert.equal(switchesFromDump(`- id: dsh-ops-${name}\n disabled: true\n config: !!js dynamic()\n`)[name].enabled, false) + } +}) + +await test('official host composition retains inherited settings and applies only candidate switches', async () => { + await withTempDir('switch-compose-', async dir => { + const baseConfig = path.join(dir, 'base.yml'), candidatePatch = path.join(dir, 'candidate.yml') + fs.writeFileSync(baseConfig, JSON.stringify([ + { id: 'dsh-ops-panel', name: 'panel-fixture', disabled: true, config: { capture: false, maxFrames: 17 } }, + { id: 'dsh-ops-hide-official', name: 'hide-fixture', disabled: true, config: { replaceTools: [] } }, + ])) + const render = patches => renderConfigDump('dsh', baseConfig, patches, line => { throw new Error(line) }) + const baseline = render([]), wanted = extractSwitches(['--switch', 'panel', '--switch', 'hide-official=all']).switches + const plan = planSwitchPatch('[]\n', wanted, { composed: parse(baseline) }) + fs.writeFileSync(candidatePatch, plan.text) + const candidate = render([{ label: 'candidate', patches: parse(fs.readFileSync(candidatePatch, 'utf8')) }]) + assert.deepEqual(switchesFromDump(candidate), wanted) + assert.equal(parse(candidate).find(row => row.id === 'dsh-ops-panel').config.maxFrames, 17) + assert.equal(render([]), baseline) + }) +}) + +await test('CLI refuses differing candidate switches and the panel renders the archived set', async () => { + await withTempDir('switch-report-', async dir => { + const files = ['base', 'cand'].map(side => path.join(dir, `${side}.json`)) + const switches = { ptc: { enabled: true } }, lines = [] + for (const [index, file] of files.entries()) fs.writeFileSync(file, JSON.stringify({ format: 'dsh-ops-bench-summary-v1', results: [], candidateSwitches: { ptc: { enabled: index === 0 } } })) + assert.equal(await compareMain([...files, '--out', '-'], { out: line => lines.push(line), err: line => lines.push(line) }), 2) + assert.match(lines.join('\n'), /Candidate switches differ/) + assert.match(lines.join('\n'), /"enabled":true/) + assert.match(lines.join('\n'), /"enabled":false/) + const arms = { base: { label: 'base', profile: 'base', profileHash: 'a', switches: {} }, cand: { label: 'cand', profile: 'cand', profileHash: 'b', switches } } + const run = { schemaVersion: 1, meta: { arms }, arms, results: [] } + const file = path.join(dir, 'run.json'); fs.writeFileSync(file, JSON.stringify(run)) + const panel = renderPanel({ runJsonPath: file, outDir: dir }) + const dom = new JSDOM(fs.readFileSync(panel.htmlPath, 'utf8'), { runScripts: 'dangerously', beforeParse(window) { + window.fetch = async () => ({ ok: true, json: async () => run }) + } }) + try { await waitFor(() => { + const failure = dom.window.document.getElementById('fallback-error').textContent + assert.equal(failure, '') + return dom.window.document.body.textContent.includes('Candidate switches: {"ptc":{"enabled":true}}') + }) } + finally { dom.window.close() } + }) +}) + +report('bench-switches')