From 688a5bf23bbd9c1aff7226b4336c1d0c151e959b Mon Sep 17 00:00:00 2001 From: Virtue039 Date: Thu, 8 Oct 2026 16:06:49 +0800 Subject: [PATCH 1/2] Keep empty SQL fields out of numeric benchmark scores Signed-off-by: Virtue039 --- .github/workflows/bench.yml | 38 ++++++++++++++++ scripts/bench/README.md | 10 ++++ scripts/bench/lib.mjs | 9 +++- scripts/bench/lib.pg.test.mjs | 24 ++++++++++ scripts/bench/lib.test.mjs | 86 +++++++++++++++++++++++++++++++++++ 5 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/bench.yml create mode 100644 scripts/bench/lib.pg.test.mjs create mode 100644 scripts/bench/lib.test.mjs diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 000000000..cb8b7367a --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,38 @@ +name: Bench parsers + +on: + push: + branches: [dev, main] + pull_request: + branches: [dev, main] + +permissions: + contents: read + +jobs: + bench: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: utopia + POSTGRES_PASSWORD: utopia + POSTGRES_DB: bench_test + ports: ["5432:5432"] + options: >- + --health-cmd pg_isready --health-interval 5s + --health-timeout 5s --health-retries 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + - name: Install PostgreSQL client + run: sudo apt-get update && sudo apt-get install -y postgresql-client + - name: Numeric SQL benchmark regressions + run: node --test scripts/bench/lib.test.mjs scripts/bench/lib.pg.test.mjs + env: + BENCH_TEST_PSQL: psql -X -h 127.0.0.1 -p 5432 -U utopia -tAc + BENCH_TEST_DB: bench_test + PGPASSWORD: utopia diff --git a/scripts/bench/README.md b/scripts/bench/README.md index 0006aa9c3..1331d6915 100644 --- a/scripts/bench/README.md +++ b/scripts/bench/README.md @@ -252,6 +252,16 @@ node scripts/bench/ask.mjs --kb --recall 8 # 只量检索：那条对 `lib.mjs` 是两个测量台共用的地基——**判等必须是同一份**，各写一份 `same()` 迟早漂移，而一旦漂移，「提议对了几条」与「答案对了几条」就不是同一把尺子量出来的。 +SQL 的空字段（psql 默认显示的 NULL、空字符串）与纯空白不参与数值判等，不能当成 `0`。 +`firstRow()` 仍取第一行各列的数；`value()` 只读第一列，第一列为空就返回空，不取后面的数。 +回归测试不需要模型服务；第二条额外验证真实 PostgreSQL 的输出，只执行 SELECT： + +```bash +node --test scripts/bench/lib.test.mjs +BENCH_TEST_PSQL="psql -X -h 127.0.0.1 -p 1517 -U utopia -tAc" BENCH_TEST_DB=utopia PGPASSWORD=utopia \ + node --test scripts/bench/lib.pg.test.mjs +``` + ## 本体的两个数（0061 决定 5） `competency.mjs` 把一个库**接受了的**能力问题按人在 chat 里问的方式问一遍（走 `/kbs/{id}/chat`， diff --git a/scripts/bench/lib.mjs b/scripts/bench/lib.mjs index 1e46c8be7..5c176a3a9 100644 --- a/scripts/bench/lib.mjs +++ b/scripts/bench/lib.mjs @@ -70,6 +70,10 @@ export const psql = (sql) => run(withDb(PSQL, APP_DB), sql); export const onDb = (db, sql) => run(withDb(PSQL, db), sql); export const num = (sql) => Number(psql(sql) || 0); +// psql 默认把 NULL 与空字符串都印成空字段；Number("") 却是 0， +// 让没有数的字段命中零值真值。两条读数路径都先排除空白，真实的 0 照常读。 +const numericField = (field) => field.trim() === "" ? undefined : Number(field); + /// 一条 SQL 第一行里的所有数字。 /// /// **模型不写单列查询。** 问「平均行金额」，它跑的是 @@ -81,7 +85,7 @@ export function firstRow(db, sql) { const out = onDb(db, `SET statement_timeout = '20s'; ${sql}`); const first = out.split("\n").map((l) => l.trim()).filter((l) => l !== "" && l !== "SET")[0]; if (first === undefined) return { empty: true }; - const ns = first.split("|").map((x) => Number(x)).filter((n) => Number.isFinite(n)); + const ns = first.split("|").map(numericField).filter((n) => Number.isFinite(n)); return { ns }; } catch (e) { return { error: String(e.stderr || e.message).split("\n").filter((l) => l.trim())[0]?.slice(0, 120) }; @@ -97,7 +101,8 @@ export function value(db, sql) { // 而 -tA 下数据行不带标签——不滤掉它，每条读到的第一行都是 `SET` const first = out.split("\n").map((l) => l.trim()).filter((l) => l !== "" && l !== "SET")[0]; if (first === undefined) return { empty: true }; - return { n: Number(String(first).split("|")[0]) }; + const n = numericField(first.split("|")[0]); + return n === undefined ? { empty: true } : { n }; } catch (e) { return { error: String(e.stderr || e.message).split("\n").filter((l) => l.trim())[0]?.slice(0, 120) }; } diff --git a/scripts/bench/lib.pg.test.mjs b/scripts/bench/lib.pg.test.mjs new file mode 100644 index 000000000..911b94526 --- /dev/null +++ b/scripts/bench/lib.pg.test.mjs @@ -0,0 +1,24 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +// BENCH_TEST_PSQL is a psql command with -tAc and connection options; BENCH_TEST_DB +// names an existing test database. These probes only SELECT, with no fixture writes. +test("real psql keeps blank SQL fields out of numeric benchmark scores", { + skip: !process.env.BENCH_TEST_PSQL, +}, async () => { + assert.ok(process.env.BENCH_TEST_DB, "BENCH_TEST_DB must name the test database"); + process.env.BENCH_PSQL = process.env.BENCH_TEST_PSQL; + const { firstRow, value, roughly } = await import("./lib.mjs"); + const db = process.env.BENCH_TEST_DB; + for (const field of ["NULL::numeric", "''::text", "' '::text"]) { + const sql = `SELECT 7, ${field}`; + const { ns } = firstRow(db, sql); + assert.deepEqual(ns, [7], sql); + assert.equal(ns.some((n) => roughly(n, 0)), false, sql); + assert.deepEqual(value(db, `SELECT ${field}, 7`), { empty: true }, field); + } + assert.deepEqual(firstRow(db, "SELECT 7, 0, -2.5, 0.25"), { ns: [7, 0, -2.5, 0.25] }); + assert.deepEqual(value(db, "SELECT 0"), { n: 0 }); + assert.deepEqual(firstRow(db, "SELECT NULL::numeric"), { empty: true }); + assert.deepEqual(firstRow(db, "SELECT 0 WHERE FALSE"), { empty: true }); +}); diff --git a/scripts/bench/lib.test.mjs b/scripts/bench/lib.test.mjs new file mode 100644 index 000000000..b9fc6659b --- /dev/null +++ b/scripts/bench/lib.test.mjs @@ -0,0 +1,86 @@ +import assert from "node:assert/strict"; +import childProcess from "node:child_process"; +import { syncBuiltinESMExports } from "node:module"; +import test from "node:test"; +import { firstRow, value, same, roughly } from "./lib.mjs"; + +// Stub only psql's stdout: the production command wrapper, parsers and comparison +// functions still run, without needing a database for these boundary cases. +function stdout(t, text) { + t.mock.method(childProcess, "execFileSync", () => text); + syncBuiltinESMExports(); + t.after(() => { + t.mock.restoreAll(); + syncBuiltinESMExports(); + }); +} + +for (const [label, output] of [ + ["trailing NULL or empty string", "SET\n7|\n"], + ["leading NULL or empty string", "SET\n|7\n"], + ["interior empty field", "SET\n7||9\n"], + ["whitespace-only field", "SET\n7| \t \n"], +]) { + test(`firstRow does not let ${label} match a zero answer`, (t) => { + stdout(t, output); + const ns = firstRow("fixture", "SELECT fixture").ns; + assert.deepEqual(ns, label === "interior empty field" ? [7, 9] : [7]); + // ask.mjs searches every returned number using roughly; mapping scores use same. + assert.equal(ns.some((n) => roughly(n, 0)), false); + assert.equal(ns.some((n) => same(n, 0)), false); + }); +} + +test("firstRow keeps real zero, negatives, decimals and numbers in later columns", (t) => { + stdout(t, "SET\ncontext|0|-7|0.25|-2.5|Infinity|NaN\n999\n"); + const { ns } = firstRow("fixture", "SELECT fixture"); + assert.deepEqual(ns, [0, -7, 0.25, -2.5]); + assert.equal(ns.some((n) => roughly(n, 0)), true); +}); + +test("a first row of empty fields contains no numbers", (t) => { + stdout(t, "SET\n| |\n"); + assert.deepEqual(firstRow("fixture", "SELECT fixture"), { ns: [] }); +}); + +for (const output of ["SET\n", "SET\n\n"]) { + test(`no rows or a single blank value stays empty (${JSON.stringify(output)})`, (t) => { + stdout(t, output); + assert.deepEqual(firstRow("fixture", "SELECT fixture"), { empty: true }); + assert.deepEqual(value("fixture", "SELECT fixture"), { empty: true }); + }); +} + +for (const [label, output] of [ + ["NULL or empty", "SET\n|7\n"], + ["whitespace-only", "SET\n \t |7\n"], +]) { + test(`value keeps a ${label} first column empty instead of selecting a later number`, (t) => { + stdout(t, output); + const got = value("fixture", "SELECT fixture"); + assert.deepEqual(got, { empty: true }); + assert.equal(same(got.n, 0), false); + assert.equal(roughly(got.n, 0), false); + }); +} + +for (const n of [0, -7, 0.25]) { + test(`value preserves a numeric first column (${n})`, (t) => { + stdout(t, `SET\n${n}|999\n`); + assert.deepEqual(value("fixture", "SELECT fixture"), { n }); + }); +} + +test("a nonnumeric first column does not become a later numeric value", (t) => { + stdout(t, "SET\ntext|0\n"); + assert.ok(Number.isNaN(value("fixture", "SELECT fixture").n)); +}); + +test("SQL errors retain their diagnostic instead of supplying a number", (t) => { + stdout(t, ""); + childProcess.execFileSync.mock.mockImplementation(() => { + throw Object.assign(new Error("command failed"), { stderr: "ERROR: bad SQL\n" }); + }); + assert.deepEqual(firstRow("fixture", "SELECT fixture"), { error: "ERROR: bad SQL" }); + assert.deepEqual(value("fixture", "SELECT fixture"), { error: "ERROR: bad SQL" }); +}); From f70c16b4726332d162305fd868a30b9ad5b1a431 Mon Sep 17 00:00:00 2001 From: Wayland Yang Date: Thu, 8 Oct 2026 17:00:19 +0800 Subject: [PATCH 2/2] The bench scoring tests run as a step of the web job, not as a workflow of their own with a Postgres service Co-Authored-By: Claude Opus 5.5 Signed-off-by: Wayland Yang --- .github/workflows/bench.yml | 38 ------------------------------------ .github/workflows/checks.yml | 3 +++ 2 files changed, 3 insertions(+), 38 deletions(-) delete mode 100644 .github/workflows/bench.yml diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml deleted file mode 100644 index cb8b7367a..000000000 --- a/.github/workflows/bench.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Bench parsers - -on: - push: - branches: [dev, main] - pull_request: - branches: [dev, main] - -permissions: - contents: read - -jobs: - bench: - runs-on: ubuntu-latest - services: - postgres: - image: postgres:16 - env: - POSTGRES_USER: utopia - POSTGRES_PASSWORD: utopia - POSTGRES_DB: bench_test - ports: ["5432:5432"] - options: >- - --health-cmd pg_isready --health-interval 5s - --health-timeout 5s --health-retries 10 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - - name: Install PostgreSQL client - run: sudo apt-get update && sudo apt-get install -y postgresql-client - - name: Numeric SQL benchmark regressions - run: node --test scripts/bench/lib.test.mjs scripts/bench/lib.pg.test.mjs - env: - BENCH_TEST_PSQL: psql -X -h 127.0.0.1 -p 5432 -U utopia -tAc - BENCH_TEST_DB: bench_test - PGPASSWORD: utopia diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 87890571d..cb6c2a72c 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -178,5 +178,8 @@ jobs: run: pnpm install --frozen-lockfile - name: Test run: pnpm test + # 测量台的判等不依赖任何包，借这个 job 现成的 node 跑，不另开一条流水线 + - name: Bench scoring tests + run: node --test ../scripts/bench/lib.test.mjs - name: Build (含类型检查) run: pnpm build