From 28a80a6253c7190b1a8ca9cc014b3974813c87b0 Mon Sep 17 00:00:00 2001
From: Shen Haoming <167516473+Haoming9527@users.noreply.github.com>
Date: Sun, 25 Oct 2026 00:40:54 +0800
Subject: [PATCH] fix(bench): record platform facts and guard incompatible
comparisons
---
.github/workflows/ci.yml | 21 +++++--
bench/README.md | 8 +++
bench/compare.mjs | 17 +++++-
bench/lib/compare-core.mjs | 7 ++-
bench/lib/environment.mjs | 49 +++++++++++++++
bench/report/panel.mjs | 4 ++
bench/run.mjs | 8 ++-
bench/run.schema.json | 1 +
bench/run/schedule.mjs | 4 ++
bench/setup.mjs | 2 +-
bench/setup/preflight.mjs | 7 ++-
bench/setup/profile.mjs | 13 ++--
docs/CHANGELOG.md | 3 +
test/bench-ci.test.mjs | 9 ++-
test/bench-compare.test.mjs | 3 +-
test/bench-platform.test.mjs | 95 +++++++++++++++++++++++++++++
test/bench-schedule.test.mjs | 8 ++-
test/bench-setup-preflight.test.mjs | 7 ++-
18 files changed, 240 insertions(+), 26 deletions(-)
create mode 100644 bench/lib/environment.mjs
create mode 100644 test/bench-platform.test.mjs
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 016df8d4..4d47ef08 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -238,7 +238,7 @@ jobs:
retention-days: 7
# The bench smoke: `workflow_dispatch` only, off by default, and informational (like `tools-trial`
- # above). One platform is enough, because it exercises the harness rather than the platform — the
+ # above). Three host platforms exercise the portable harness; the
# `validate-posix` matrix is the gate and `validate-windows` its non-blocking counterpart. Both
# sides are the *same* staged worker, so the run
# proves only what a harness self-check can prove: the corpus builds and is removed, the worker
@@ -248,7 +248,20 @@ jobs:
# **never** run in CI.
bench-smoke:
if: ${{ github.event_name == 'workflow_dispatch' && inputs.bench }}
- runs-on: ubuntu-24.04
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - os: ubuntu-24.04
+ target: linux-x64
+ worker: dsh-ops-worker
+ - os: windows-latest
+ target: win32-x64
+ worker: dsh-ops-worker.exe
+ - os: macos-latest
+ target: darwin-arm64
+ worker: dsh-ops-worker
+ runs-on: ${{ matrix.os }}
continue-on-error: true
timeout-minutes: 30
env:
@@ -293,7 +306,7 @@ jobs:
- name: Tool-level comparison, one binary on both sides
shell: bash
run: |
- worker="runtime/linux-x64/dsh-ops-worker"
+ worker="runtime/${{ matrix.target }}/${{ matrix.worker }}"
mkdir -p bench/runs/smoke/fixtures
node bench/cli.mjs tools \
--baseline "$worker" --candidate "$worker" \
@@ -309,7 +322,7 @@ jobs:
- uses: actions/upload-artifact@v7
if: always()
with:
- name: bench-smoke-report
+ name: bench-smoke-${{ matrix.target }}-report
path: |
bench/runs/smoke/**
bench/runs/*/run.json
diff --git a/bench/README.md b/bench/README.md
index a196fd86..506f2d0d 100644
--- a/bench/README.md
+++ b/bench/README.md
@@ -2,6 +2,14 @@
## 测试前的准备
+Linux / macOS：使用当前平台构建的 worker；`setup --dry-run` 会说明跳过 Windows PowerShell pin、PSModulePath、Windows Bash IPC 和随包 Windows shell 检查的原因。当前 shell 行已支持各平台，不要再从早期 Windows-only 默认值推断配置。模型 sweep 仍需自己的 OpenAI 兼容服务与凭据；`tools` 自检不调用模型。
+
+Linux / macOS: build the worker for the current host. `setup --dry-run` explains skipped Windows-only checks. `ab`, `compare`/`report`, `panel` and `tools` use the same entry point on every platform. A model sweep needs your own OpenAI-compatible endpoint; the tools self-check needs no model. Every sweep now fingerprints the composed profiles even without `--seal`, recording platform and component configuration in the arm summaries. `enabled` is a configuration fact, not proof of tool readiness; unresolved expressions remain `unknown`.
+
+`compare` 默认拒绝不同平台（或已知/未知平台）混配，退出码为 2；诊断时可加 `--allow-cross-platform`，报告会明确标记差异不是插件效应。旧数据两侧都缺平台时仍可读取，显示 `unknown`，不能据此声称同平台。CI 的可选 tools-only smoke 覆盖 Linux / Windows / macOS，各自上传 artifact；不运行模型 sweep，不产出跨平台结论。
+
+`compare` rejects mismatched platforms with exit 2. `--allow-cross-platform` is an explicit diagnostic override and adds a “not a plugin effect” warning. Both-unknown legacy archives remain readable. The optional, non-blocking tools smoke runs on Linux, Windows and macOS with separate artifacts.
+
```console
npm ci --ignore-scripts
npm run build && npm run build:ptc # lib/ + 原生 worker
diff --git a/bench/compare.mjs b/bench/compare.mjs
index dd5ccf01..69b5bdc8 100644
--- a/bench/compare.mjs
+++ b/bench/compare.mjs
@@ -40,6 +40,7 @@ import {
percentChange,
} from './lib/compare-core.mjs'
import { RUNS_DIR } from './lib/next-folder.mjs'
+import { environmentLines } from './lib/environment.mjs'
/** The sealed contract document inside a run directory. */
const SEALED_RUN = 'run.json'
@@ -78,6 +79,7 @@ const USAGE = `usage: node bench/compare.mjs [options]
--noise-floor-pct N pin the A/A noise floor; without a floor every cost
direction is withheld (noise-floor-unknown)
--noise measure the A/A noise floor from two baseline runs
+ --allow-cross-platform diagnostics only: allow different platforms, with a warning
--json print the comparison document as JSON instead of the report
--out where the rendered artifact goes (default: the sweep's group
folder, named "DSH-OPS Benchmark Report .md" — a run with no
@@ -143,7 +145,10 @@ export function parseArgs(argv) {
if (index >= argv.length) throw new Error(`${name} needs a value`)
return argv[index]
}
- if (name === '--json') {
+ if (name === '--allow-cross-platform') {
+ if (inline !== null) throw new Error('--allow-cross-platform takes no value')
+ options.allowCrossPlatform = true
+ } else if (name === '--json') {
if (inline !== null) throw new Error('--json takes no value; the comparison document goes to stdout and --out names a file')
options.json = true
} else if (name === '--aggregate') options.aggregate = take()
@@ -516,6 +521,7 @@ export function renderReport(comparison) {
const lines = [
REPORT_TITLE,
'',
+ ...environmentLines(comparison.environment),
...taskSection(comparison),
'',
...toolSection(comparison),
@@ -625,14 +631,19 @@ export async function main(argv, io = {}) {
return EXIT.usage
}
- const comparison = compareSummaries(base.doc, cand.doc, {
+ let comparison
+ try { comparison = compareSummaries(base.doc, cand.doc, {
+ allowCrossPlatform: options.allowCrossPlatform,
aggregate: options.aggregate,
tolerancePct: options.tolerancePct,
failOn: options.failOn,
regressionScope: options.regressionScope,
noiseFloorPct: options.noiseFloorPct ?? undefined,
noise: noise ?? undefined,
- })
+ }) } catch (error) {
+ err(`compare: ${String(error?.message ?? error)}`)
+ return EXIT.usage
+ }
const payload = options.json
? `${JSON.stringify(comparison, null, 2)}\n`
: `${renderReport(comparison)}\n`
diff --git a/bench/lib/compare-core.mjs b/bench/lib/compare-core.mjs
index 9197c1c4..700e4f39 100644
--- a/bench/lib/compare-core.mjs
+++ b/bench/lib/compare-core.mjs
@@ -33,6 +33,7 @@
*/
import { CORE_BUCKETS, median, minValue, pairMetrics } from './aggregate.mjs'
+import { compareEnvironment } from './environment.mjs'
export { median, minValue }
@@ -481,6 +482,7 @@ export function classifyExcluded(pairs, taskId = null) {
* @returns {object} the noise-floor document; `pct: null` when nothing could be paired.
*/
export function noiseFloor(baseA, baseB, options = {}) {
+ compareEnvironment(baseA, baseB, options)
const aggregate = AGGREGATES.includes(options.aggregate) ? options.aggregate : 'median'
const merged = pairMetrics(normalizeResults(baseA), normalizeResults(baseB))
const usablePairs = merged.pairs.filter((pair) => passed(pair.base) && passed(pair.cand))
@@ -547,7 +549,7 @@ export function resolveNoiseFloor(options = {}, base = null, cand = null) {
}
const noise = options.noise
if (noise && typeof noise === 'object' && !Array.isArray(noise) && (noise.base || noise.cand)) {
- return noiseFloor(noise.base ?? base, noise.cand ?? base, { aggregate: options.aggregate })
+ return noiseFloor(noise.base ?? base, noise.cand ?? base, { aggregate: options.aggregate, allowCrossPlatform: options.allowCrossPlatform })
}
return {
format: NOISE_FORMAT,
@@ -702,6 +704,8 @@ export function formatNumber(value) {
* @returns {object} the comparison structure, including the gate and the verdict.
*/
export function compareSummaries(base, cand, options = {}) {
+ const environment = compareEnvironment(base, cand, options)
+ if (options.noise) compareEnvironment(base, options.noise.base, options)
const aggregate = AGGREGATES.includes(options.aggregate) ? options.aggregate : 'median'
const tolerancePct = isNumber(options.tolerancePct) ? options.tolerancePct : 2
const failOn = FAIL_ON.includes(options.failOn) ? options.failOn : 'both'
@@ -853,6 +857,7 @@ export function compareSummaries(base, cand, options = {}) {
return {
format: COMPARE_FORMAT,
+ environment,
base: { label: sideLabel(base, 'base'), profile: sideProfile(base, 'base'), model: baseModel, dshVersion: sideDshVersion(base, 'base') },
cand: { label: sideLabel(cand, 'cand'), profile: sideProfile(cand, 'cand'), model: candModel, dshVersion: sideDshVersion(cand, 'cand') },
aggregate,
diff --git a/bench/lib/environment.mjs b/bench/lib/environment.mjs
new file mode 100644
index 00000000..f5629b91
--- /dev/null
+++ b/bench/lib/environment.mjs
@@ -0,0 +1,49 @@
+import { parse } from 'yaml'
+
+const COMPONENTS = ['shell', 'file', 'ptc', 'discovery', 'panel', 'tui', 'hide-official']
+
+/** Read configuration facts, never evaluate an expression or infer publication readiness. */
+export function componentFacts(text) {
+ const facts = Object.fromEntries(COMPONENTS.map(name => [name, 'unknown']))
+ let rows
+ try { rows = parse(text) } catch { return facts }
+ if (!Array.isArray(rows)) return facts
+ for (const name of COMPONENTS) {
+ const found = rows.filter(row => row?.id === `dsh-ops-${name}`)
+ if (!found.length) facts[name] = 'absent'
+ else if (found.length === 1) {
+ const row = found[0]
+ if (row.disabled === true || row.config?.enabled === false) facts[name] = 'disabled'
+ else if ((row.disabled === undefined || row.disabled === false) && (row.config?.enabled === undefined || row.config.enabled === true)) facts[name] = 'enabled'
+ }
+ }
+ return facts
+}
+
+export function platformOf(run) {
+ const value = run?.meta?.platform ?? run?.platform
+ return typeof value === 'string' && value.trim() ? value : 'unknown'
+}
+
+/** Old unknown/unknown archives remain readable; a known/unknown pair needs the override too. */
+export function compareEnvironment(base, cand, options = {}) {
+ const platforms = { base: platformOf(base), cand: platformOf(cand) }
+ const crossPlatform = platforms.base !== platforms.cand
+ if (crossPlatform && options.allowCrossPlatform !== true) {
+ throw new Error(`platform mismatch: base=${platforms.base}, cand=${platforms.cand}; use --allow-cross-platform only for diagnostics (not a plugin effect)`)
+ }
+ return { platforms, crossPlatform, components: {
+ base: base?.components ?? base?.arms?.base?.components ?? null,
+ cand: cand?.components ?? cand?.arms?.cand?.components ?? null,
+ } }
+}
+
+export function environmentLines(environment) {
+ if (!environment) return []
+ return [
+ `Platform: base=${environment.platforms.base}; cand=${environment.platforms.cand}.`,
+ ...(environment.crossPlatform ? ['WARNING: cross-platform diagnostic; differences are not a plugin effect.'] : []),
+ `Component rows (configuration, not a readiness receipt): base=${JSON.stringify(environment.components.base)}; cand=${JSON.stringify(environment.components.cand)}.`,
+ '',
+ ]
+}
diff --git a/bench/report/panel.mjs b/bench/report/panel.mjs
index a90d83cf..1a901cff 100644
--- a/bench/report/panel.mjs
+++ b/bench/report/panel.mjs
@@ -375,6 +375,10 @@ function renderConclusion() {
: '噪声底(A/A)±' + SUMMARY.noise.pct.toFixed(1) + '% → ' + (SUMMARY.readable ? '超过噪声 ⇒ 可读方向' : '落在噪声内 ⇒ 只报区间,不报方向')
}
function renderOverview(node) {
+ const environment = el('div', 'card')
+ environment.appendChild(el('h3', null, 'Platform / component configuration'))
+ environment.appendChild(el('pre', null, JSON.stringify({ platform: obj(RUN.meta).platform || 'unknown', components: Object.fromEntries(Object.entries(obj(RUN.arms)).map(([key, arm]) => [key, obj(arm).components || null])) }, null, 2)))
+ node.appendChild(environment)
const keys = pairKeys()
const taskRows = SUMMARY.tasks.map((task) => {
const base = task.arms[keys[0]]
diff --git a/bench/run.mjs b/bench/run.mjs
index 84640ec9..a1483f61 100644
--- a/bench/run.mjs
+++ b/bench/run.mjs
@@ -42,6 +42,7 @@ import { RUNS_DIR } from './lib/next-folder.mjs'
import { DEFAULT_REPEAT, armOrder, createInterrupt, planPairs, planSessions, repeatsById, runSweep } from './run/schedule.mjs'
import { sealRun } from './run/seal.mjs'
import { renderReport, reportFileName } from './compare.mjs'
+import { componentFacts } from './lib/environment.mjs'
/** The runner-only flags that take a value; `--task`/`--repeat`/`--folder`/`--base`/`--cand` are parsed by the CLI. */
const VALUE_FLAGS = ['--sleep-ms', '--out', '--heartbeat-ms', '--stall-ms', '--timeout', '--dsh', '--llm-patch', '--model']
@@ -233,7 +234,7 @@ export function dumpProfileConfig(entry, profile, context = {}) {
const detail = String(result.stderr ?? '').trim().split('\n').filter(Boolean).pop() ?? ''
throw new Error(`"dsh --profile ${profile} --dump-config" exited ${String(result.status)}${detail === '' ? '' : `: ${detail}`}`)
}
- return `${result.stdout ?? ''}\n${result.stderr ?? ''}`
+ return result.stdout ?? ''
}
/**
@@ -253,6 +254,7 @@ export function profileFingerprints(arms, context = {}) {
throw new Error(`"dsh --profile ${arm.profile} --dump-config" printed nothing, so the ${arm.side} arm has no profile fingerprint`)
}
hashes[arm.side] = createHash('sha256').update(text).digest('hex')
+ arm.components = componentFacts(text)
}
return hashes
}
@@ -478,12 +480,12 @@ export async function main(argv, io = {}) {
// fingerprint, and a sweep that cannot be sealed is not evidence. Refusing here costs
// nothing; refusing after the sweep would have spent the tokens first.
let hashes = io.profileHashes ?? null
- if (hashes === null && seal !== undefined) {
+ if (hashes === null) {
try {
hashes = profileFingerprints(arms, { entry, dump: io.dumpProfileConfig, env: io.env })
} catch (error) {
err(`bench ab: ${String(error?.message ?? error)}`)
- err('nothing was spawned and no token was spent; run "node bench/cli.mjs setup" if the profiles are not prepared yet, or pass --no-seal to sweep without sealing.')
+ err('nothing was spawned and no token was spent; run "node bench/cli.mjs setup" if the profiles are not prepared yet. Configuration facts are required even without --seal.')
return EXIT.failure
}
}
diff --git a/bench/run.schema.json b/bench/run.schema.json
index 494ee664..81538a7c 100644
--- a/bench/run.schema.json
+++ b/bench/run.schema.json
@@ -73,6 +73,7 @@
"$defs": {
"arm": { "description": "One arm of the comparison: what it was called, which profile contents it ran, and how it was launched.", "type": "object", "additionalProperties": false, "required": ["label", "profile", "profileHash"],
"properties": {
+ "components": { "description": "Component states read from composed configuration, not a publication/readiness receipt. Null for older or injected records.", "type": ["object", "null"], "additionalProperties": { "type": "string", "enum": ["enabled", "disabled", "absent", "unknown"] } },
"label": { "description": "Arm label, also the profile name it ran, e.g. bench-base.", "type": "string" },
"profile": { "description": "Profile name whose resolved configuration was measured.", "type": "string" },
"profileHash": { "description": "SHA-256 in hex of the arm's dumped configuration, the single-variable fingerprint.", "type": "string" },
diff --git a/bench/run/schedule.mjs b/bench/run/schedule.mjs
index bc821598..94ae9d5c 100644
--- a/bench/run/schedule.mjs
+++ b/bench/run/schedule.mjs
@@ -204,6 +204,9 @@ function sessionOptions(arm, pair, spec, options, ctx) {
/** One arm's summary: the shape `bench/compare.mjs` reads as a side. */
function buildSummary(arm, rows, ctx) {
return {
+ meta: { platform: ctx.spec.meta?.platform ?? `${process.platform}-${process.arch}` },
+ profileHash: arm.profileHash ?? null,
+ components: arm.components ?? null,
format: SUMMARY_FORMAT,
label: arm.label,
profile: arm.profile,
@@ -237,6 +240,7 @@ function noiseFor(spec, summaries, arms) {
/** The arm descriptor the seal step (P8a) lifts into `run.json`'s `arms`. */
function descriptor(arm) {
return {
+ components: arm.components ?? null,
label: arm.label,
profile: arm.profile,
profileHash: arm.profileHash ?? null,
diff --git a/bench/setup.mjs b/bench/setup.mjs
index fcadabc8..705fcf36 100644
--- a/bench/setup.mjs
+++ b/bench/setup.mjs
@@ -70,7 +70,7 @@ export async function main(argv, io = {}) {
const out = io.out ?? ((line) => console.log(line))
const err = io.err ?? ((line) => console.error(line))
let options
- try { options = parseArgs(argv, EXTRA_FLAGS) } catch (error) {
+ try { options = parseArgs(argv, { ...EXTRA_FLAGS, platform: io.platform }) } catch (error) {
err(`${error.message}\n\n${USAGE}`)
return EXIT.usage
}
diff --git a/bench/setup/preflight.mjs b/bench/setup/preflight.mjs
index a77486da..0f516df5 100644
--- a/bench/setup/preflight.mjs
+++ b/bench/setup/preflight.mjs
@@ -434,12 +434,15 @@ export function preflight(options, context = {}) {
else if (permissions.ok) log(` both sides will pin ${pairText(permissions.expected)}`)
log('\n— preflight: PSModulePath —')
- const modulePath = inspectModulePath({ modulePath: env.PSModulePath, windows: env.SystemRoot, isDirectory: context.isDirectory })
- outcomes.modulePath = modulePath ? 'poisoned' : 'clean'
+ const modulePath = platform === 'win32' ? inspectModulePath({ modulePath: env.PSModulePath, windows: env.SystemRoot, isDirectory: context.isDirectory }) : null
+ outcomes.modulePath = platform !== 'win32' ? 'skipped' : modulePath ? 'poisoned' : 'clean'
if (modulePath) {
log(` ✗ ${modulePath.entry} is ahead of the in-box 5.1 module directory`)
log(` search order: ${describeModulePath(env.PSModulePath, modulePath.entry)}`)
warnings.push(modulePath.warning)
+ } else if (platform !== 'win32') {
+ log(' skipped: Windows PSModulePath ordering does not apply on this platform')
+ log(' skipped: Windows Bash IPC proxy and bundled Windows shell payload checks do not apply')
} else {
log(' clean: no Core module root ahead of the in-box Windows PowerShell 5.1 directory')
}
diff --git a/bench/setup/profile.mjs b/bench/setup/profile.mjs
index ddb6ca75..da447a8f 100644
--- a/bench/setup/profile.mjs
+++ b/bench/setup/profile.mjs
@@ -152,12 +152,13 @@ export function defaultShellPath() {
/** The defaults, before any flag is read.
* @returns {Options} the defaults. */
-function defaultOptions() {
+function defaultOptions(platform = process.platform) {
return {
baseProfile: 'bench-base', candProfile: 'bench-ops', template: 'headless',
// There is no Windows PowerShell to pin off Windows, and `dsh-pwsh-sandbox` is
// Windows-gated in the composition anyway, so the default is "no pin" there.
- pinShell: process.platform === 'win32' ? defaultShellPath() : null,
+ pinShell: platform === 'win32' ? defaultShellPath() : null,
+ platform,
permissionMode: 'danger-full-access',
// The candidate measures the working tree: the default install is this checkout, so a
// sweep prepares itself from the code under edit without a publish or a version bump.
@@ -175,7 +176,7 @@ function defaultOptions() {
* @returns {Options} the options.
*/
export function parseArgs(argv, extra = {}) {
- const options = defaultOptions()
+ const options = defaultOptions(extra.platform)
const values = { ...VALUE_FLAGS, ...extra.value }
const booleans = { ...BOOLEAN_FLAGS, ...extra.boolean }
for (let i = 0; i < argv.length; i += 1) {
@@ -200,7 +201,7 @@ export function parseArgs(argv, extra = {}) {
}
}
if (options.baseProfile === options.candProfile) throw new Error('--base-profile and --cand-profile must differ')
- if (options.pinShell && process.platform === 'win32' && !path.isAbsolute(options.pinShell)) {
+ if (options.pinShell && options.platform === 'win32' && !path.isAbsolute(options.pinShell)) {
throw new Error(`--pin-shell must be an absolute path, got "${options.pinShell}"`)
}
if (!PERMISSION_MODES.includes(options.permissionMode)) {
@@ -590,7 +591,9 @@ function ensureProfile(name, context) {
*/
function pinBaselineShell(options, context) {
if (!options.pinShell) {
- context.log(' --no-pin: nothing was touched. The baseline then hits the PATH pwsh alias, so both sides run PowerShell 7')
+ context.log(options.platform !== 'win32'
+ ? ' skipped: Windows PowerShell pin is inapplicable on this platform; the host selects its configured shell'
+ : ' --no-pin: nothing was touched; the host selects its configured shell')
return 'skipped'
}
const file = path.join(context.home, 'profiles', options.baseProfile, 'cordis.patch.yml')
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 0ec16497..0c7b54a9 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -7,6 +7,9 @@ this package adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
## [Unreleased]
+- Make benchmark setup skip Windows-only checks explicitly on POSIX, preserve platform/component configuration in summaries and sealed arms, reject cross-platform comparisons by default, and expand the optional tools smoke to Linux, Windows and macOS (#295).
+- 基准准备在 POSIX 上明确跳过 Windows 专用检查，普通摘要和封存数据保留平台及组件配置，默认拒绝跨平台混配，可选 tools 自检覆盖 Linux、Windows、macOS（#295）。
+
- Make the Windows CI leg non-blocking while keeping it running on every event (#316): split `validate` into blocking `validate-posix` (Linux x64, macOS arm64) and `continue-on-error` `validate-windows`, and keep `ci-required` from depending on any `continue-on-error` job, so the aggregate neither waits for nor fails with the slowest leg. On `main` after #307 the Windows leg takes 527 / 529 s — up from 482 s before #307, because `npm ci` now downloads the 411 MB Windows Bash payload — against 136–139 s (Linux) and 168–182 s (macOS). `scripts/check-workflows.mjs` now enforces "every blocking job in the aggregate, no non-blocking job in it" and that the aggregate's failure loop names exactly the blocking jobs, so a job cannot stay in `needs` but escape judgement; the Windows result is a signal to act on, not a merge gate.
- 将 CI 的 Windows 腿改为非阻塞、但仍每次触发都运行（#316）：把 `validate` 拆成阻塞的 `validate-posix`（Linux x64、macOS arm64）与 `continue-on-error` 的 `validate-windows`，并让 `ci-required` 不依赖任何 `continue-on-error` job，聚合既不等待也不随最慢的一腿失败。`main` 在 #307 合入后实测 Windows 腿 527 / 529 秒（#307 之前 482 秒，因为 `npm ci` 现在要拉 411 MB 的 Windows Bash 载荷），Linux 136–139 秒、macOS 168–182 秒。`scripts/check-workflows.mjs` 现在强制“每个阻塞 job 都在聚合里、聚合里没有非阻塞 job”，并要求失败判定循环里的 job 集合与阻塞集合完全相等，避免 job 留在 `needs` 却逃过判定；Windows 结果是待处理的信号，不是合并门禁。
- Bound each `ops_replace` output to the same 32 MiB UTF-8 byte ceiling as file writes. Check expansion with overflow-safe arithmetic for the entire plan before constructing replacement content or publishing any file, including dry runs; an over-limit target returns `OUTPUT_LIMIT` with no file changes (#330).
diff --git a/test/bench-ci.test.mjs b/test/bench-ci.test.mjs
index 9811100d..b0640c6f 100644
--- a/test/bench-ci.test.mjs
+++ b/test/bench-ci.test.mjs
@@ -65,7 +65,13 @@ await test('"CI required" neither waits on nor judges the smoke', () => {
await test('the smoke drives the one CLI entry point with the same binary on both sides', () => {
assert.match(benchRuns, /node bench\/cli\.mjs tools\b/u, 'the smoke goes through the single cross-platform entry')
assert.match(benchRuns, /--baseline "\$worker" --candidate "\$worker"/u, 'both sides are the same staged worker')
- assert.match(benchRuns, /runtime\/linux-x64\/dsh-ops-worker/u, 'it uses the worker this job just built and staged')
+ assert.ok(benchRuns.includes('runtime/${{ matrix.target }}/${{ matrix.worker }}'), 'each runner uses its own staged worker')
+ assert.equal(benchJob['runs-on'], '${{ matrix.os }}')
+ assert.deepEqual(benchJob.strategy.matrix.include, [
+ { os: 'ubuntu-24.04', target: 'linux-x64', worker: 'dsh-ops-worker' },
+ { os: 'windows-latest', target: 'win32-x64', worker: 'dsh-ops-worker.exe' },
+ { os: 'macos-latest', target: 'darwin-arm64', worker: 'dsh-ops-worker' },
+ ])
assert.doesNotMatch(benchRuns, /bench\/cli\.mjs ab\b/u, 'the model-driven sweep is never run in CI')
for (const flag of ['--max-ms', '--budget', '--threshold']) {
assert.equal(benchRuns.includes(flag), false, `no timing gate in the smoke (${flag})`)
@@ -81,6 +87,7 @@ await test('the smoke uploads report.md / run.json / panel.html when they exist'
assert.equal(uploads.length, 1, 'the smoke uploads exactly one artifact')
const upload = uploads[0]
assert.match(String(upload.with?.name ?? ''), /-report$/u, 'the name follows the tools-trial-report convention')
+ assert.ok(upload.with.name.includes('${{ matrix.target }}'), 'parallel platforms never overwrite one another’s artifact')
assert.equal(upload.with?.['if-no-files-found'], 'warn', 'a tools-only smoke makes no run.json and must not fail on that')
assert.match(String(upload.if ?? ''), /always\(\)/u, 'evidence is uploaded even when the comparison is red')
const paths = String(upload.with?.path ?? '')
diff --git a/test/bench-compare.test.mjs b/test/bench-compare.test.mjs
index b23779ff..e3f5d0ae 100644
--- a/test/bench-compare.test.mjs
+++ b/test/bench-compare.test.mjs
@@ -167,7 +167,8 @@ await test('the same comparison renders byte-identical text twice: tables first,
const text = renderReport(beyond)
assert.equal(renderReport(beyond), text, 'rendering is a pure function of its inputs')
assertEnglish(text)
- assert.match(text, /^# DSH-OPS Benchmark Report \(beta\)\n\n## Per-task pairs$/m, 'the title leads straight into the data — no summary bar')
+ assert.match(text, /^# DSH-OPS Benchmark Report \(beta\)\n\nPlatform: base=unknown; cand=unknown\./, 'the title states environment evidence before the data')
+ assert.match(text, /^## Per-task pairs$/m)
assert.match(text, /^\| task \| base \| ops \(Δ%\) \| steps: base→ops \(Δ%\) \| tool calls: base→ops \(Δ%\) \| time: base→ops \(Δ%\) \| tool errors \| pass \|$/m, 'the first table carries cost, behaviour, wall-clock evidence and pass: no pairs, exclusion or direction column')
assert.ok(text.includes(`| csv-sum | 1,000 | 926 (${green('\u22127.4%')}) | 4 → 4 (${green('0.0%')}) | 6 → 6 (${green('0.0%')}) | 60.0s → 55.0s (${green('\u22128.3%')}) | 0 → 0 | 2/2→2/2 |`), 'each cell keeps its own Δ% in parentheses, painted like every other percent — time included')
assert.doesNotMatch(text, /geomean/, 'the table ends at TOTAL: the geomean row is gone')
diff --git a/test/bench-platform.test.mjs b/test/bench-platform.test.mjs
new file mode 100644
index 00000000..9f549d51
--- /dev/null
+++ b/test/bench-platform.test.mjs
@@ -0,0 +1,95 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import { assert, report, test, withTempDir } from './lib/harness.mjs'
+import { parseArgs as setupArgs } from '../bench/setup/profile.mjs'
+import { preflight } from '../bench/setup/preflight.mjs'
+import { compareSummaries } from '../bench/lib/compare-core.mjs'
+import { main as compare, renderReport } from '../bench/compare.mjs'
+import { componentFacts } from '../bench/lib/environment.mjs'
+import { dumpProfileConfig, profileFingerprints } from '../bench/run.mjs'
+import { main as setup } from '../bench/setup.mjs'
+
+await test('POSIX setup never probes Windows module directories or pins PowerShell', () => {
+ for (const platform of ['linux', 'darwin']) {
+ const options = setupArgs(['--plugin', 'none', '--dry-run'], { platform })
+ assert.equal(options.pinShell, null)
+ const lines = []
+ const result = preflight(options, { platform, log: line => lines.push(line),
+ env: { PSModulePath: 'C:\\Core;C:\\Windows', SystemRoot: 'C:\\Windows' },
+ isDirectory: () => { throw new Error('Windows probe on POSIX') } })
+ assert.equal(result.outcomes.modulePath, 'skipped')
+ assert.equal(result.problems.length, 0)
+ assert.match(lines.join('\n'), /skipped.*Windows/)
+ }
+})
+
+await test('platform mismatch refuses pairing before statistics, including A/A inputs', () => {
+ const base = { meta: { platform: 'linux-x64' }, results: [] }
+ const cand = { meta: { platform: 'darwin-arm64' }, results: [] }
+ assert.throws(() => compareSummaries(base, cand), /linux-x64.*darwin-arm64/)
+ assert.throws(() => compareSummaries(base, base, { noise: { base, cand } }), /platform/)
+ const diagnostic = compareSummaries(base, cand, { allowCrossPlatform: true })
+ assert.match(renderReport(diagnostic), /not a plugin effect/)
+ assert.equal(diagnostic.environment.crossPlatform, true)
+ assert.doesNotThrow(() => compareSummaries({ results: [] }, { results: [] }))
+})
+
+await test('compare CLI returns 2 for different platforms and requires an explicit override', async () => {
+ await withTempDir('bench-platform-', async dir => {
+ const files = ['base', 'cand'].map(name => path.join(dir, `${name}.json`))
+ for (const [i, file] of files.entries()) fs.writeFileSync(file, JSON.stringify({ format: 'dsh-ops-bench-summary-v1', meta: { platform: ['linux-x64', 'win32-x64'][i] }, results: [] }))
+ const lines = []
+ const io = { out: line => lines.push(line), err: line => lines.push(line) }
+ assert.equal(await compare([...files, '--out', '-'], io), 2)
+ assert.match(lines.join('\n'), /linux-x64.*win32-x64/)
+ lines.length = 0
+ assert.equal(await compare([...files, '--out', '-', '--allow-cross-platform'], io), 0)
+ assert.match(lines.join('\n'), /not a plugin effect/)
+ })
+})
+
+await test('composed component facts distinguish disabled, absent and unevaluated rows', () => {
+ const dump = '- id: dsh-ops-shell\n disabled: false\n- id: dsh-ops-discovery\n config:\n enabled: false\n- id: dsh-ops-panel\n disabled: process.platform !== "win32"\n'
+ const facts = componentFacts(dump)
+ assert.equal(facts.shell, 'enabled')
+ assert.equal(facts.discovery, 'disabled')
+ assert.equal(facts.panel, 'unknown')
+ assert.equal(facts.ptc, 'absent')
+ const arms = [{ side: 'base', profile: 'base' }, { side: 'cand', profile: 'cand' }]
+ profileFingerprints(arms, { dump: () => dump })
+ assert.deepEqual(arms[0].components, facts)
+ assert.deepEqual(arms[1].components, facts)
+ assert.equal(componentFacts('bad: [').shell, 'unknown')
+ assert.equal(componentFacts('- id: dsh-ops-shell\n- id: dsh-ops-shell\n').shell, 'unknown')
+})
+
+await test('Linux and macOS dry-run traverse setup without executing or writing profiles', async () => {
+ await withTempDir('bench-posix-setup-', async dir => {
+ const example = path.join(dir, 'example.yml')
+ fs.writeFileSync(example, '[]\n')
+ for (const platform of ['linux', 'darwin']) {
+ const home = path.join(dir, platform)
+ const lines = []
+ const code = await setup(['--dsh-home', home, '--plugin', 'none', '--dry-run'], {
+ platform, arch: 'x64', env: {}, entry: { kind: 'exe', file: 'fixture-dsh' },
+ overlay: path.join(dir, 'overlay.yml'), overlayExample: example,
+ invoke: () => { throw new Error('dry-run spawned the host') },
+ out: line => lines.push(line), err: line => lines.push(line),
+ })
+ assert.equal(code, 0, lines.join('\n'))
+ assert.match(lines.join('\n'), /skipped: Windows PowerShell pin/)
+ assert.match(lines.join('\n'), /skipped: Windows PSModulePath/)
+ assert.equal(fs.existsSync(home), false)
+ }
+ })
+})
+
+await test('host diagnostics cannot contaminate component configuration parsing', () => {
+ const text = dumpProfileConfig({ kind: 'exe', file: 'fixture-dsh' }, 'bench', { run: () => ({
+ status: 0, stdout: '- id: dsh-ops-shell\n', stderr: 'Warning: diagnostic, not configuration\n',
+ }) })
+ assert.equal(componentFacts(text).shell, 'enabled')
+ assert.doesNotMatch(text, /Warning/)
+})
+
+report('bench-platform')
diff --git a/test/bench-schedule.test.mjs b/test/bench-schedule.test.mjs
index 4aa37e0d..4c356952 100644
--- a/test/bench-schedule.test.mjs
+++ b/test/bench-schedule.test.mjs
@@ -524,7 +524,7 @@ await test('main: a sweep is unsealed by default and --seal opts in', async () =
resolveEntry: () => ENTRY,
installSignals: false,
now: NOW,
- dumpProfileConfig: () => { throw new Error('the default must not fingerprint anything') },
+ dumpProfileConfig: () => '- id: dsh-ops-shell\n disabled: false\n',
seal: () => { sealed += 1; return {} },
})
assert.equal(code, EXIT.ok)
@@ -534,6 +534,9 @@ await test('main: a sweep is unsealed by default and --seal opts in', async () =
const group = path.join(root, 'runs', fs.readdirSync(path.join(root, 'runs'))[0])
const armDir = fs.readdirSync(group).find((name) => name.endsWith('bench-base'))
assert.ok(fs.existsSync(path.join(group, armDir, 'summary.json')), 'the per-arm summary is the artifact the report reads')
+ const summary = JSON.parse(fs.readFileSync(path.join(group, armDir, 'summary.json'), 'utf8'))
+ assert.equal(summary.components.shell, 'enabled', 'unsealed summaries still preserve component configuration')
+ assert.equal(summary.meta.platform, `${process.platform}-${process.arch}`)
assert.equal(fs.existsSync(path.join(group, 'run.json')), false, 'nothing sealed this sweep')
const reports = fs.readdirSync(group).filter((name) => name.startsWith('DSH-OPS Benchmark Report') && name.endsWith('.md'))
assert.equal(reports.length, 1, 'a finished sweep writes its own report into the group folder')
@@ -555,6 +558,7 @@ await test('main: a sweep is unsealed by default and --seal opts in', async () =
installSignals: false,
now: NOW,
seal: () => { throw new Error('--no-seal must never seal') },
+ profileHashes: HASHES,
}), EXIT.ok)
assert.match(explicit.text(), /no — not requested/)
})
@@ -563,7 +567,7 @@ await test('main: a sweep is unsealed by default and --seal opts in', async () =
await test('main: a finished sweep writes the report by default and --no-report skips it', async () => {
await withTempDir('bench-main-report', async (root) => {
const runsRoot = path.join(root, 'runs')
- const base = { tasksDir: tasksDir(root, 1), runsDir: runsRoot, llmOverlay: overlayIn(root), resolveEntry: () => ENTRY, installSignals: false, now: NOW }
+ const base = { tasksDir: tasksDir(root, 1), runsDir: runsRoot, llmOverlay: overlayIn(root), resolveEntry: () => ENTRY, installSignals: false, now: NOW, profileHashes: HASHES }
const groups = () => fs.readdirSync(runsRoot)
const out = sink()
diff --git a/test/bench-setup-preflight.test.mjs b/test/bench-setup-preflight.test.mjs
index ebaf7e49..5a48a006 100644
--- a/test/bench-setup-preflight.test.mjs
+++ b/test/bench-setup-preflight.test.mjs
@@ -220,7 +220,7 @@ await test('the preflight reports the poisoned PSModulePath as a warning with th
const isDirectory = (file) => file === path.win32.join(CORE, 'Microsoft.PowerShell.Security')
const options = { baseProfile: 'bench-base', candProfile: 'bench-ops', permissionMode: 'danger-full-access', plugin: 'local', noBuildCheck: true }
const lines = []
- const dirty = preflight(options, { home: path.join(dir, 'home'), env: poisoned, platform: 'linux', arch: 'x64', log: (line) => lines.push(line), isDirectory })
+ const dirty = preflight(options, { home: path.join(dir, 'home'), env: poisoned, platform: 'win32', arch: 'x64', log: (line) => lines.push(line), isDirectory })
assert.equal(dirty.ok, true, 'a poisoned module path is a warning, not a refusal')
assert.equal(dirty.outcomes.modulePath, 'poisoned')
assert.equal(dirty.warnings.length, 1)
@@ -228,7 +228,7 @@ await test('the preflight reports the poisoned PSModulePath as a warning with th
assert.match(lines.join('\n'), /search order: 1\. C:\\Program Files\\PowerShell\\7\\Modules/)
assert.match(lines.join('\n'), /preflight: PSModulePath/)
- const clean = preflight(options, { home: path.join(dir, 'home'), env: { SystemRoot: 'C:\\Windows' }, platform: 'linux', arch: 'x64' })
+ const clean = preflight(options, { home: path.join(dir, 'home'), env: { SystemRoot: 'C:\\Windows' }, platform: 'win32', arch: 'x64' })
assert.equal(clean.outcomes.modulePath, 'clean')
assert.deepEqual(clean.warnings, [])
})
@@ -466,10 +466,11 @@ await test('the poisoned PSModulePath reaches main as a warning and never blocks
await withTempDir('bench-pre-main-module-', async (dir) => {
const fx = fixture(dir)
const io = fx.io({
+ platform: 'win32',
env: { SystemRoot: 'C:\\Windows', PSModulePath: `${CORE};${IN_BOX}` },
isDirectory: (file) => file === path.win32.join(CORE, 'Microsoft.PowerShell.Security'),
})
- const code = await main(['--dsh-home', fx.home, '--no-pin', '--dry-run'], io)
+ const code = await main(['--dsh-home', fx.home, '--no-pin', '--dry-run', '--no-build-check'], io)
assert.equal(code, 0, fx.err())
assert.match(fx.err(), /cmd\.exe/)
assert.match(fx.out(), /preflight: PSModulePath/)