From 630220775ef487aac44f871f6d3002d0c8aa54a2 Mon Sep 17 00:00:00 2001 From: RowDaBoat Date: Mon, 7 Sep 2026 18:43:44 -0300 Subject: [PATCH 1/3] chore: start benchmark reliability review From 82956d6c65e397b603e694709ed098d8f5caa715 Mon Sep 17 00:00:00 2001 From: RowDaBoat Date: Mon, 7 Sep 2026 19:00:46 -0300 Subject: [PATCH 2/3] Make benchmark comparisons noise-aware --- .github/workflows/CI.yml | 24 +- .gitignore | 6 +- benchs/compare_benchmarks.nim | 29 ++ benchs/helpers/benchmarks.nim | 703 ++++++++++++++++++++++++++++------ benchs/vecsbench.nim | 10 +- tests/tbenchmarks.nim | 146 +++++++ 6 files changed, 783 insertions(+), 135 deletions(-) create mode 100644 benchs/compare_benchmarks.nim create mode 100644 tests/tbenchmarks.nim diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 403371c..bf111fa 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -58,12 +58,30 @@ jobs: echo "Baseline: $baseline ($(git rev-parse --short "$baseline"))" + baseline_ready=true git checkout "$baseline" -- src - nim c -r -d:danger --nimcache:nimcache/baseline "benchs/vecsbench.nim" baseline.csv || rm -f baseline.csv + if ! nim c -d:danger --nimcache:nimcache/baseline \ + -o:/tmp/vecs-benchmark-baseline "benchs/vecsbench.nim"; then + baseline_ready=false + fi git checkout HEAD -- src - if [ ! -f baseline.csv ]; then + nim c -d:danger --nimcache:nimcache/current \ + -o:/tmp/vecs-benchmark-current "benchs/vecsbench.nim" + + if [ "$baseline_ready" != true ]; then echo "Baseline src/ did not build against the current benchmarks, skipping comparison." + /tmp/vecs-benchmark-current current.csv + exit 0 fi - nim c -r -d:danger --nimcache:nimcache/current "benchs/vecsbench.nim" current.csv baseline.csv + nim c -d:danger --nimcache:nimcache/comparison \ + -o:/tmp/compare-benchmarks "benchs/compare_benchmarks.nim" + + /tmp/vecs-benchmark-current current-a.csv > /tmp/current-a.log + /tmp/vecs-benchmark-baseline baseline-a.csv > /tmp/baseline-a.log + /tmp/vecs-benchmark-baseline baseline-b.csv > /tmp/baseline-b.log + /tmp/vecs-benchmark-current current-b.csv > /tmp/current-b.log + + /tmp/compare-benchmarks \ + current-a.csv current-b.csv baseline-a.csv baseline-b.csv diff --git a/.gitignore b/.gitignore index 5166de1..c1bfe7e 100644 --- a/.gitignore +++ b/.gitignore @@ -15,9 +15,13 @@ tests/queries tests/world tests/events tests/manycomponents +tests/tbenchmarks benchs/vecs_bench benchs/vecs.csv baseline.csv current.csv - +baseline-a.csv +baseline-b.csv +current-a.csv +current-b.csv diff --git a/benchs/compare_benchmarks.nim b/benchs/compare_benchmarks.nim new file mode 100644 index 0000000..cb117a5 --- /dev/null +++ b/benchs/compare_benchmarks.nim @@ -0,0 +1,29 @@ +# ISC License +# Copyright (c) 2025 RowDaBoat +# `vecs` is a free open source ECS library for Nim. +import os +import helpers/benchmarks + + +proc loadMerged(firstPath, secondPath: string): BenchmarkSuite = + let first = loadBenchmarkSuiteFromCsv(firstPath) + let second = loadBenchmarkSuiteFromCsv(secondPath) + result = mergeBenchmarkSuites([first, second]) + + +proc usage(): string = + result = + "usage: compare_benchmarks " & + " " & + " " + + +if isMainModule: + if paramCount() != 4: + quit(usage(), QuitFailure) + + let candidate = loadMerged(paramStr(1), paramStr(2)) + let baseline = loadMerged(paramStr(3), paramStr(4)) + let baselineName = paramStr(3) & " + " & paramStr(4) + + echo compareBenchmarkSuites(baseline, candidate, baselineName) diff --git a/benchs/helpers/benchmarks.nim b/benchs/helpers/benchmarks.nim index f908ba1..810ffa1 100644 --- a/benchs/helpers/benchmarks.nim +++ b/benchs/helpers/benchmarks.nim @@ -1,13 +1,20 @@ # ISC License # Copyright (c) 2025 RowDaBoat # `vecs` is a free open source ECS library for Nim. -import times, math, algorithm, strutils, tables, unicode, std/monotimes, os +import math, algorithm, strutils, tables, std/monotimes, os + + +const + MinimumSampleTime = 0.001 + MaximumBatchSize = 10_000 type Parameters* = object samples*: int warmup*: int + batchSize*: int + runs*: int maxTime*: float maxMem*: float @@ -144,7 +151,9 @@ proc showSummary*(b: Benchmark) = proc showDetailed*(b: Benchmark) = echo "=".repeat(70) echo "Benchmark: ", b.name - echo "Samples: ", b.params.samples, " (warmup: ", b.params.warmup, ")" + echo "Samples: ", b.params.samples, + " (warmup: ", b.params.warmup, + ", batch: ", b.params.batchSize, ")" echo "" echo "Time Statistics:" @@ -195,7 +204,6 @@ proc showComparison*(cmp: Comparison) = echo "╠═", "═".repeat(66), "═╣" let timeIcon = if cmp.isFaster: "✓" else: "✗" - let timeColor = if cmp.isFaster: "" else: "" echo "║ Time : ", timeIcon, " ", (if cmp.isFaster: "FASTER" else: "SLOWER"), " by ", prettyPercent(abs(cmp.timeImprovement)), @@ -232,7 +240,8 @@ proc blackBox*[T](value: T) {.noinline.} = proc initBenchmark*(benchmarkName: string, sample, warm: int): Benchmark = result.name = benchmarkName - result.params = Parameters(samples: sample, warmup: warm) + result.params = + Parameters(samples: sample, warmup: warm, batchSize: 1, runs: 1) result.times = newSeqOfCap[float](sample) result.mems = newSeqOfCap[float](sample) @@ -246,6 +255,25 @@ template measure*(bench: var Benchmark, memBaseline: int, code: untyped) = bench.mems.add((getOccupiedMem() - memBaseline).float) +proc batchSizeFor(elapsed: float, target: float): int = + if elapsed <= 0.0: + return 1 + + result = ceil(target / elapsed).int + result = clamp(result, 1, MaximumBatchSize) + + +template measureRepeated(bench: var Benchmark, memBaseline, repetitions: int, + code: untyped) = + let t0 = getMonoTime() + for repetitionIndex in 0 ..< repetitions: + code + let elapsed = (getMonoTime() - t0).inNanoseconds.float / 1e9 + + bench.times.add(elapsed / repetitions.float) + bench.mems.add((getOccupiedMem() - memBaseline).float) + + template benchmark*(benchmarkName: string, sample, code: untyped): untyped = benchmark(benchmarkName, sample, 1, code) @@ -288,6 +316,37 @@ template benchmarkWithSetup*(benchmarkName: string, sample, warm, setup, code: u bench +template benchmarkRepeatedWithSetup*(benchmarkName: string, sample, warm, + setup, code: untyped): untyped = + var bench = initBenchmark(benchmarkName, sample, warm) + + block: + var repetitions = 1 + block: + setup + let calibrationStart = getMonoTime() + code + let calibrationTime = + (getMonoTime() - calibrationStart).inNanoseconds.float / 1e9 + repetitions = batchSizeFor(calibrationTime, MinimumSampleTime) + + bench.params.batchSize = repetitions + + for warmupIndex in 0 ..< warm: + setup + for repetitionIndex in 0 ..< repetitions: + code + + for sampleIndex in 0 ..< sample: + let memBaseline = getOccupiedMem() + setup + measureRepeated(bench, memBaseline, repetitions): + code + + finalize(bench) + bench + + proc initSuite*(name: string): BenchmarkSuite = result.name = name result.benchmarks = @[] @@ -312,182 +371,574 @@ proc showSummary*(suite: BenchmarkSuite) = echo "╚═", "═".repeat(60), "═╝" +const + DefaultComparisonConfidence* = 0.999 + DefaultMinimumRelativeChange* = 0.01 + + +proc serializeSamples(values: seq[float]): string = + var serialized = newSeq[string](values.len) + for index, value in values: + serialized[index] = value.formatFloat(ffScientific, 10) + + result = serialized.join(";") + + proc saveSummary*(suite: BenchmarkSuite, path: string) = var file = open(path, fmWrite) defer: file.close() - file.writeLine(suite.name & ",time_median,mem_median,time_seconds,mem_bytes") + file.writeLine( + suite.name & + ",time_median,mem_median,time_seconds,mem_bytes,time_samples,mem_samples,batch_size" + ) for bench in suite.benchmarks: - let mem = prettyMem(bench.memStats.median) + let memory = prettyMem(bench.memStats.median) let time = prettyTime(bench.timeStats.median) file.writeLine( - bench.name & "," & time & "," & mem & "," & + bench.name & "," & time & "," & memory & "," & bench.timeStats.median.formatFloat(ffScientific, 10) & "," & - bench.memStats.median.formatFloat(ffScientific, 10) + bench.memStats.median.formatFloat(ffScientific, 10) & "," & + serializeSamples(bench.times) & "," & + serializeSamples(bench.mems) & "," & + $bench.params.batchSize ) + type + ConfidenceInterval* = object + lower*: float + upper*: float + + ChangeStatus* = enum + ChangeUnchanged + ChangeInconclusive + ChangeImproved + ChangeRegressed + + MetricComparison = object + ratio: float + change: float + changeInterval: ConfidenceInterval + difference: float + status: ChangeStatus + hasRelativeChange: bool + BenchResult* = object name*: string timeRatio*: float memRatio*: float timeImprovement*: float memImprovement*: float + timeChangeInterval*: ConfidenceInterval + memChangeInterval*: ConfidenceInterval + timeDifference*: float + memDifference*: float + timeStatus*: ChangeStatus + memStatus*: ChangeStatus timeSignificant*: bool memSignificant*: bool timeBetter*: bool memBetter*: bool + timeHasRelativeChange*: bool + memHasRelativeChange*: bool missingInBaseline*: bool BenchComp* = object suiteName*: string baselineFile*: string margin*: float + confidence*: float + minimumRelativeChange*: float + baselineRuns*: int + candidateRuns*: int results*: seq[BenchResult] missingInCurrent*: seq[string] -proc loadBenchmarkSuiteFromCsv*(path: string): BenchmarkSuite = - result.benchmarks = @[] +proc parseNumber(value: string, number: var float): bool = + try: + number = parseFloat(value) + result = true + except ValueError: + result = false - if fileExists(path): - var file = open(path, fmRead) - defer: file.close() - - var isFirst = true - for line in file.lines: - if isFirst: - let headerParts = line.split(',', 1) - result.name = if headerParts.len > 0: headerParts[0] else: "Baseline" - isFirst = false - continue - - let parts = line.split(',') - if parts.len < 5: - continue - - var bench = initBenchmark(parts[0], 1, 0) - try: - let t = parseFloat(parts[3]) - let m = parseFloat(parts[4]) - bench.times.add(t) - bench.mems.add(m) - bench.timeStats = calculateStatistics(bench.times) - bench.memStats = calculateStatistics(bench.mems) - bench.totalTime = t - bench.totalMem = m - result.benchmarks.add(bench) - except ValueError: - continue +proc parseSamples(value: string): seq[float] = + for serialized in value.split(';'): + var number = 0.0 + if parseNumber(serialized, number): + result.add(number) -proc compareWithBaseline*(suite: BenchmarkSuite, csvPath: string, - margin: float = 0.05): BenchComp = - result.suiteName = suite.name - result.baselineFile = csvPath - result.margin = margin - result.results = @[] - result.missingInCurrent = @[] - let baseline = loadBenchmarkSuiteFromCsv(csvPath) +proc parseBatchSize(value: string): int = + try: + result = max(1, parseInt(value)) + except ValueError: + result = 1 - var baselineMap = initTable[string, Benchmark]() - for b in baseline.benchmarks: - baselineMap[b.name] = b - var currentNames = initTable[string, bool]() +proc benchmarkFromCsv(parts: seq[string], benchmark: var Benchmark): bool = + if parts.len < 5: + return false - for current in suite.benchmarks: - currentNames[current.name] = true - var res: BenchResult - res.name = current.name + var medianTime = 0.0 + var medianMemory = 0.0 + if not parseNumber(parts[3], medianTime): + return false + if not parseNumber(parts[4], medianMemory): + return false - if not baselineMap.hasKey(current.name): - res.missingInBaseline = true - result.results.add(res) - continue + benchmark = initBenchmark(parts[0], 1, 0) + if parts.len >= 7: + benchmark.times = parseSamples(parts[5]) + benchmark.mems = parseSamples(parts[6]) - let base = baselineMap[current.name] + if benchmark.times.len == 0: + benchmark.times.add(medianTime) + if benchmark.mems.len == 0: + benchmark.mems.add(medianMemory) + if parts.len >= 8: + benchmark.params.batchSize = parseBatchSize(parts[7]) - res.timeRatio = current.timeStats.median / base.timeStats.median - res.timeImprovement = - (current.timeStats.median - base.timeStats.median) / base.timeStats.median - res.timeBetter = res.timeImprovement < 0 - res.timeSignificant = abs(res.timeImprovement) > margin + benchmark.params.samples = benchmark.times.len + finalize(benchmark) + result = true - res.memRatio = current.memStats.median / base.memStats.median - res.memImprovement = - (current.memStats.median - base.memStats.median) / base.memStats.median - res.memBetter = res.memImprovement < 0 - res.memSignificant = abs(res.memImprovement) > margin - result.results.add(res) +proc loadBenchmarkSuiteFromCsv*(path: string): BenchmarkSuite = + result.benchmarks = @[] + if not fileExists(path): + return - for name, _ in baselineMap: - if not currentNames.hasKey(name): - result.missingInCurrent.add(name) + var file = open(path, fmRead) + defer: file.close() + var isHeader = true + for line in file.lines: + if isHeader: + let headerParts = line.split(',', 1) + result.name = if headerParts.len > 0: headerParts[0] else: "Baseline" + isHeader = false + else: + var benchmark: Benchmark + if benchmarkFromCsv(line.split(','), benchmark): + result.benchmarks.add(benchmark) + + +proc benchmarkIndex(suite: BenchmarkSuite, benchmarkName: string): int = + for index, benchmark in suite.benchmarks: + if benchmark.name == benchmarkName: + return index + + result = -1 + + +proc merge*(suite: var BenchmarkSuite, addition: BenchmarkSuite) = + if suite.name.len == 0: + suite.name = addition.name + + for addedBenchmark in addition.benchmarks: + let index = suite.benchmarkIndex(addedBenchmark.name) + if index < 0: + suite.benchmarks.add(addedBenchmark) + else: + suite.benchmarks[index].times.add(addedBenchmark.times) + suite.benchmarks[index].mems.add(addedBenchmark.mems) + suite.benchmarks[index].params.samples = suite.benchmarks[index].times.len + suite.benchmarks[index].params.runs += addedBenchmark.params.runs + suite.benchmarks[index].params.batchSize = + max(suite.benchmarks[index].params.batchSize, + addedBenchmark.params.batchSize) + finalize(suite.benchmarks[index]) + + +proc mergeBenchmarkSuites*(suites: openArray[BenchmarkSuite]): BenchmarkSuite = + for suite in suites: + result.merge(suite) + + +proc lowerConfidenceIndex(sampleCount: int, tailProbability: float): int = + if sampleCount <= 1: + return 0 + + let centerIndex = sampleCount div 2 + let centerProbability = exp( + lgamma((sampleCount + 1).float) - + lgamma((centerIndex + 1).float) - + lgamma((sampleCount - centerIndex + 1).float) - + sampleCount.float * ln(2.0) + ) + + var probabilities = newSeq[float](centerIndex + 1) + probabilities[centerIndex] = centerProbability + for index in countdown(centerIndex, 1): + let numerator = index.float + let denominator = (sampleCount - index + 1).float + probabilities[index - 1] = + probabilities[index] * numerator / denominator + + var cumulativeProbability = 0.0 + for index in 0 ..< centerIndex: + cumulativeProbability += probabilities[index] + if cumulativeProbability <= tailProbability: + result = index + else: + return + + +proc medianConfidenceInterval*(values: seq[float], + confidence: float): ConfidenceInterval = + if values.len == 0: + return -proc `$`*(comp: BenchComp): string = + var sorted = values + sorted.sort() + + let tailProbability = (1.0 - confidence) / 2.0 + let lowerIndex = lowerConfidenceIndex(sorted.len, tailProbability) + let upperIndex = sorted.high - lowerIndex + + result.lower = sorted[lowerIndex] + result.upper = sorted[upperIndex] + + +proc classifyRelativeChange(interval: ConfidenceInterval, + hasRepeatedRuns: bool, + minimumRelativeChange: float): ChangeStatus = + if interval.lower == 0.0 and interval.upper == 0.0: + return ChangeUnchanged + if not hasRepeatedRuns: + return ChangeInconclusive + if interval.upper < -minimumRelativeChange: + return ChangeImproved + if interval.lower > minimumRelativeChange: + return ChangeRegressed + if interval.lower >= -minimumRelativeChange and + interval.upper <= minimumRelativeChange: + return ChangeUnchanged + + result = ChangeInconclusive + + +proc classifyAbsoluteChange(interval: ConfidenceInterval, + hasRepeatedRuns: bool): ChangeStatus = + if interval.lower == 0.0 and interval.upper == 0.0: + return ChangeUnchanged + if not hasRepeatedRuns: + return ChangeInconclusive + if interval.upper < 0.0: + return ChangeImproved + if interval.lower > 0.0: + return ChangeRegressed + + result = ChangeInconclusive + + +proc compareMetric(baselineValues, candidateValues: seq[float], + baselineRuns, candidateRuns: int, + confidence, + minimumRelativeChange: float): MetricComparison = + if baselineValues.len == 0 or candidateValues.len == 0: + result.status = ChangeInconclusive + return + + let medianConfidence = 1.0 - (1.0 - confidence) / 2.0 + let baselineInterval = + medianConfidenceInterval(baselineValues, medianConfidence) + let candidateInterval = + medianConfidenceInterval(candidateValues, medianConfidence) + let baselineMedian = calculateStatistics(baselineValues).median + let candidateMedian = calculateStatistics(candidateValues).median + + let differenceInterval = ConfidenceInterval( + lower: candidateInterval.lower - baselineInterval.upper, + upper: candidateInterval.upper - baselineInterval.lower + ) + let hasRepeatedRuns = baselineRuns >= 2 and candidateRuns >= 2 + + result.difference = candidateMedian - baselineMedian + + if baselineMedian == 0.0 or baselineInterval.lower <= 0.0: + result.status = classifyAbsoluteChange( + differenceInterval, + hasRepeatedRuns + ) + return + + result.hasRelativeChange = true + result.ratio = candidateMedian / baselineMedian + result.change = result.ratio - 1.0 + result.changeInterval = ConfidenceInterval( + lower: candidateInterval.lower / baselineInterval.upper - 1.0, + upper: candidateInterval.upper / baselineInterval.lower - 1.0 + ) + result.status = classifyRelativeChange( + result.changeInterval, + hasRepeatedRuns, + minimumRelativeChange + ) + + +proc benchmarkResult(baseline, candidate: Benchmark, + confidence, + minimumRelativeChange: float): BenchResult = + result.name = candidate.name + + let timeComparison = compareMetric( + baseline.times, + candidate.times, + baseline.params.runs, + candidate.params.runs, + confidence, + minimumRelativeChange + ) + let memoryComparison = compareMetric( + baseline.mems, + candidate.mems, + baseline.params.runs, + candidate.params.runs, + confidence, + minimumRelativeChange + ) + + result.timeRatio = timeComparison.ratio + result.timeImprovement = timeComparison.change + result.timeChangeInterval = timeComparison.changeInterval + result.timeDifference = timeComparison.difference + result.timeStatus = timeComparison.status + result.timeSignificant = + timeComparison.status in {ChangeImproved, ChangeRegressed} + result.timeBetter = timeComparison.status == ChangeImproved + result.timeHasRelativeChange = timeComparison.hasRelativeChange + + result.memRatio = memoryComparison.ratio + result.memImprovement = memoryComparison.change + result.memChangeInterval = memoryComparison.changeInterval + result.memDifference = memoryComparison.difference + result.memStatus = memoryComparison.status + result.memSignificant = + memoryComparison.status in {ChangeImproved, ChangeRegressed} + result.memBetter = memoryComparison.status == ChangeImproved + result.memHasRelativeChange = memoryComparison.hasRelativeChange + + +proc suiteRuns(suite: BenchmarkSuite): int = + if suite.benchmarks.len == 0: + return 0 + + result = suite.benchmarks[0].params.runs + + +proc compareBenchmarkSuites*(baseline, candidate: BenchmarkSuite, + baselineName: string, + confidence: float = + DefaultComparisonConfidence, + minimumRelativeChange: float = + DefaultMinimumRelativeChange): BenchComp = + result.suiteName = candidate.name + result.baselineFile = baselineName + result.margin = minimumRelativeChange + result.confidence = confidence + result.minimumRelativeChange = minimumRelativeChange + result.baselineRuns = suiteRuns(baseline) + result.candidateRuns = suiteRuns(candidate) + result.results = @[] + result.missingInCurrent = @[] + + var baselineMap = initTable[string, Benchmark]() + for benchmark in baseline.benchmarks: + baselineMap[benchmark.name] = benchmark + + var candidateNames = initTable[string, bool]() + for candidateBenchmark in candidate.benchmarks: + candidateNames[candidateBenchmark.name] = true + if baselineMap.hasKey(candidateBenchmark.name): + let baselineBenchmark = baselineMap[candidateBenchmark.name] + result.results.add( + benchmarkResult( + baselineBenchmark, + candidateBenchmark, + confidence, + minimumRelativeChange + ) + ) + else: + result.results.add( + BenchResult( + name: candidateBenchmark.name, + missingInBaseline: true + ) + ) + + for benchmarkName, baselineBenchmark in baselineMap: + if not candidateNames.hasKey(benchmarkName): + result.missingInCurrent.add(baselineBenchmark.name) + + +proc compareWithBaseline*(suite: BenchmarkSuite, + csvPath: string): BenchComp = + let baseline = loadBenchmarkSuiteFromCsv(csvPath) + result = compareBenchmarkSuites( + baseline, + suite, + csvPath, + DefaultComparisonConfidence, + DefaultMinimumRelativeChange + ) + + +proc compareWithBaseline*(suite: BenchmarkSuite, csvPath: string, + margin: float): BenchComp = + let baseline = loadBenchmarkSuiteFromCsv(csvPath) + result = compareBenchmarkSuites( + baseline, + suite, + csvPath, + DefaultComparisonConfidence, + margin + ) + + +proc changeMarker(status: ChangeStatus): string = + case status + of ChangeImproved: + result = "▼" + of ChangeRegressed: + result = "▲" + of ChangeUnchanged: + result = "=" + of ChangeInconclusive: + result = "?" + + +proc timeChangeText(benchmark: BenchResult): string = + if benchmark.missingInBaseline: + return "N/A" + if benchmark.timeHasRelativeChange: + return changeMarker(benchmark.timeStatus) & " " & + prettyPercent(benchmark.timeImprovement) + + result = changeMarker(benchmark.timeStatus) & " " & + prettyTime(benchmark.timeDifference) + + +proc memoryChangeText(benchmark: BenchResult): string = + if benchmark.missingInBaseline: + return "N/A" + if benchmark.memHasRelativeChange: + return changeMarker(benchmark.memStatus) & " " & + prettyPercent(benchmark.memImprovement) + + result = changeMarker(benchmark.memStatus) & " " & + prettyMem(benchmark.memDifference) + + +proc comparisonStatus(benchmark: BenchResult): string = + if benchmark.missingInBaseline: + return "NEW (no baseline)" + + var statusParts: seq[string] = @[] + if benchmark.timeStatus == ChangeImproved: + statusParts.add("FASTER") + elif benchmark.timeStatus == ChangeRegressed: + statusParts.add("SLOWER") + + if benchmark.memStatus == ChangeImproved: + statusParts.add("LESS MEM") + elif benchmark.memStatus == ChangeRegressed: + statusParts.add("MORE MEM") + + if statusParts.len > 0: + return statusParts.join(" + ") + if benchmark.timeStatus == ChangeUnchanged and + benchmark.memStatus == ChangeUnchanged: + return "unchanged" + + result = "inconclusive" + + +proc `$`*(comparison: BenchComp): string = var lines: seq[string] = @[] - let innerWidth = 75 + let nameWidth = 30 + let metricWidth = 12 + let statusWidth = 24 + let innerWidth = + nameWidth + metricWidth + metricWidth + statusWidth + 9 + let confidenceText = + (comparison.confidence * 100.0).formatFloat(ffDecimal, 1) & + "% median intervals" + let minimumEffectText = + "Minimum directional effect: " & + (comparison.minimumRelativeChange * 100.0).formatFloat(ffDecimal, 1) & + "%" + let runsText = + "Runs: baseline " & $comparison.baselineRuns & + ", candidate " & $comparison.candidateRuns lines.add "" lines.add "╔═" & "═".repeat(innerWidth) & "═╗" - lines.add "║ " & ("Benchmark Comparison: " & comp.suiteName).alignLeft(innerWidth) & " ║" - lines.add "║ " & ("Baseline: " & comp.baselineFile).alignLeft(innerWidth) & " ║" - lines.add "║ " & ("Significance margin: " & prettyPercent(comp.margin)).alignLeft(innerWidth) & " ║" - lines.add "╠═" & "═".repeat(24) & "═╪" & "═".repeat(11) & "═╪" & "═".repeat(11) & "═╪" & "═".repeat(23) & "═╣" - lines.add "║ " & "Benchmark".alignLeft(24) & " │ " & "Time".alignLeft(10) & " │ " & "Memory".alignLeft(10) & " │ " & "Status".alignLeft(22) & " ║" - lines.add "╠═" & "═".repeat(24) & "═╪" & "═".repeat(11) & "═╪" & "═".repeat(11) & "═╪" & "═".repeat(23) & "═╣" - - for res in comp.results: - let name = (if res.missingInBaseline: res.name & "*" else: res.name).alignLeft(24) - - let timeTxt = - if res.missingInBaseline: - "N/A" - elif res.timeSignificant: - (if res.timeBetter: "▼ " else: "▲ ") & prettyPercent(res.timeImprovement) - else: - "≈ " & prettyPercent(abs(res.timeImprovement)) - - let memTxt = - if res.missingInBaseline: - "N/A" - elif res.memSignificant: - (if res.memBetter: "▼ " else: "▲ ") & prettyPercent(res.memImprovement) - else: - "≈ " & prettyPercent(abs(res.memImprovement)) - - var statusParts: seq[string] = @[] - if not res.missingInBaseline: - if res.timeSignificant: - statusParts.add(if res.timeBetter: "FASTER" else: "SLOWER") - if res.memSignificant: - statusParts.add(if res.memBetter: "LESS MEM" else: "MORE MEM") - - let status = - if res.missingInBaseline: - "NEW (no baseline)" - elif statusParts.len == 0: - "stable" - else: - statusParts.join(" + ") - - lines.add "║ " & name & " │ " & timeTxt.alignLeft(10) & " │ " & memTxt.alignLeft(10) & " │ " & status.alignLeft(22) & " ║" - - lines.add "╚═" & "═".repeat(24) & "═╧" & "═".repeat(11) & "═╧" & "═".repeat(11) & "═╧" & "═".repeat(23) & "═╝" - - if comp.missingInCurrent.len > 0: + lines.add "║ " & + ("Benchmark Comparison: " & comparison.suiteName).alignLeft(innerWidth) & + " ║" + lines.add "║ " & + ("Baseline: " & comparison.baselineFile).alignLeft(innerWidth) & + " ║" + lines.add "║ " & + ("Confidence: " & confidenceText).alignLeft(innerWidth) & + " ║" + lines.add "║ " & minimumEffectText.alignLeft(innerWidth) & " ║" + lines.add "║ " & runsText.alignLeft(innerWidth) & " ║" + lines.add "╠═" & "═".repeat(nameWidth) & + "═╪" & "═".repeat(metricWidth) & + "═╪" & "═".repeat(metricWidth) & + "═╪" & "═".repeat(statusWidth) & "═╣" + lines.add "║ " & "Benchmark".alignLeft(nameWidth) & + " │ " & "Time".alignLeft(metricWidth - 1) & + " │ " & "Memory".alignLeft(metricWidth - 1) & + " │ " & "Status".alignLeft(statusWidth - 1) & " ║" + lines.add "╠═" & "═".repeat(nameWidth) & + "═╪" & "═".repeat(metricWidth) & + "═╪" & "═".repeat(metricWidth) & + "═╪" & "═".repeat(statusWidth) & "═╣" + + for benchmark in comparison.results: + let benchmarkName = + (if benchmark.missingInBaseline: + benchmark.name & "*" + else: + benchmark.name) + .alignLeft(nameWidth) + let timeText = timeChangeText(benchmark).alignLeft(metricWidth - 1) + let memoryText = memoryChangeText(benchmark).alignLeft(metricWidth - 1) + let status = comparisonStatus(benchmark).alignLeft(statusWidth - 1) + + lines.add "║ " & benchmarkName & + " │ " & timeText & + " │ " & memoryText & + " │ " & status & " ║" + + lines.add "╚═" & "═".repeat(nameWidth) & + "═╧" & "═".repeat(metricWidth) & + "═╧" & "═".repeat(metricWidth) & + "═╧" & "═".repeat(statusWidth) & "═╝" + + if comparison.missingInCurrent.len > 0: lines.add "" lines.add "Removed from current suite (present in baseline only):" - for m in comp.missingInCurrent: - lines.add " • " & m + for benchmarkName in comparison.missingInCurrent: + lines.add " • " & benchmarkName lines.add "" - lines.add "Legend: ▼ = improvement ▲ = regression ≈ = within margin * = no baseline" + lines.add( + "Legend: ▼ = improvement ▲ = regression ? = inconclusive " & + "= = unchanged * = no baseline" + ) + if comparison.baselineRuns < 2 or comparison.candidateRuns < 2: + lines.add( + "At least two runs per revision are required for a directional result." + ) - return lines.join("\n") + result = lines.join("\n") diff --git a/benchs/vecsbench.nim b/benchs/vecsbench.nim index b2fb139..3088676 100644 --- a/benchs/vecsbench.nim +++ b/benchs/vecsbench.nim @@ -1,7 +1,7 @@ # ISC License # Copyright (c) 2025 RowDaBoat # `vecs` is a free open source ECS library for Nim. -import times, math, tables, random, os +import times, tables, random, os import helpers/[benchmarks, churncommon, common] import ../src/vecs @@ -225,7 +225,7 @@ proc runVecsBenchmarks(): BenchmarkSuite = ) showDetailed(suite.benchmarks[^1]) - suite.add benchmarkWithSetup( + suite.add benchmarkRepeatedWithSetup( "iteration", SAMPLE, WARMUP, @@ -248,7 +248,7 @@ proc runVecsBenchmarks(): BenchmarkSuite = showDetailed(suite.benchmarks[^1]) var s = 0'f32 - suite.add benchmarkWithSetup( + suite.add benchmarkRepeatedWithSetup( "read", SAMPLE, WARMUP, @@ -266,7 +266,7 @@ proc runVecsBenchmarks(): BenchmarkSuite = showDetailed(suite.benchmarks[^1]) blackBox(s) - suite.add benchmarkWithSetup( + suite.add benchmarkRepeatedWithSetup( "write", SAMPLE, WARMUP, @@ -287,7 +287,7 @@ proc runVecsBenchmarks(): BenchmarkSuite = blackBox(s) var rng = initRand(42) - suite.add benchmarkWithSetup( + suite.add benchmarkRepeatedWithSetup( "heterogeneous iter", SAMPLE, WARMUP, diff --git a/tests/tbenchmarks.nim b/tests/tbenchmarks.nim new file mode 100644 index 0000000..ccc22ec --- /dev/null +++ b/tests/tbenchmarks.nim @@ -0,0 +1,146 @@ +# ISC License +# Copyright (c) 2025 RowDaBoat +# `vecs` is a free open source ECS library for Nim. +import unittest, os +import ../benchs/helpers/benchmarks + + +proc repeatedValues(value: float, count: int): seq[float] = + result = newSeq[float](count) + for index in 0 ..< count: + result[index] = value + + +proc newSyntheticBenchmark(name: string, times, memories: seq[float], + runs: int = 1): Benchmark = + result = initBenchmark(name, times.len, 0) + result.times = times + result.mems = memories + result.params.runs = runs + finalize(result) + + +proc newSyntheticSuite(benchmark: Benchmark): BenchmarkSuite = + result = initSuite("Synthetic") + result.add(benchmark) + + +suite "Benchmark comparisons should": + test "preserve raw samples in CSV summaries": + let path = + getTempDir() / ("vecs-benchmarks-" & $getCurrentProcessId() & ".csv") + defer: + if fileExists(path): + removeFile(path) + + let benchmark = newSyntheticBenchmark( + "sample", + @[1.0, 2.0, 3.0, 4.0], + @[10.0, 20.0, 30.0, 40.0] + ) + let suite = newSyntheticSuite(benchmark) + + suite.saveSummary(path) + let loaded = loadBenchmarkSuiteFromCsv(path) + + check loaded.benchmarks.len == 1 + check loaded.benchmarks[0].times == benchmark.times + check loaded.benchmarks[0].mems == benchmark.mems + + + test "keep a single-run comparison inconclusive": + let baseline = newSyntheticSuite( + newSyntheticBenchmark( + "sample", + repeatedValues(100.0, 100), + repeatedValues(100.0, 100) + ) + ) + let candidate = newSyntheticSuite( + newSyntheticBenchmark( + "sample", + repeatedValues(80.0, 100), + repeatedValues(80.0, 100) + ) + ) + + let comparison = + compareBenchmarkSuites(baseline, candidate, "synthetic.csv") + + check comparison.results[0].timeStatus == ChangeInconclusive + check comparison.results[0].memStatus == ChangeInconclusive + + + test "detect a repeated improvement": + let baseline = newSyntheticSuite( + newSyntheticBenchmark( + "sample", + repeatedValues(100.0, 100), + repeatedValues(100.0, 100), + runs = 2 + ) + ) + let candidate = newSyntheticSuite( + newSyntheticBenchmark( + "sample", + repeatedValues(80.0, 100), + repeatedValues(80.0, 100), + runs = 2 + ) + ) + + let comparison = + compareBenchmarkSuites(baseline, candidate, "synthetic.csv") + + check comparison.results[0].timeStatus == ChangeImproved + check comparison.results[0].memStatus == ChangeImproved + + + test "treat sub-percent repeated changes as unchanged": + let baseline = newSyntheticSuite( + newSyntheticBenchmark( + "sample", + repeatedValues(100.0, 100), + repeatedValues(100.0, 100), + runs = 2 + ) + ) + let candidate = newSyntheticSuite( + newSyntheticBenchmark( + "sample", + repeatedValues(100.5, 100), + repeatedValues(100.5, 100), + runs = 2 + ) + ) + + let comparison = + compareBenchmarkSuites(baseline, candidate, "synthetic.csv") + + check comparison.results[0].timeStatus == ChangeUnchanged + check comparison.results[0].memStatus == ChangeUnchanged + + + test "leave overlapping repeated measurements inconclusive": + let baseline = newSyntheticSuite( + newSyntheticBenchmark( + "sample", + @[90.0, 100.0, 110.0, 90.0, 100.0, 110.0], + repeatedValues(100.0, 6), + runs = 2 + ) + ) + let candidate = newSyntheticSuite( + newSyntheticBenchmark( + "sample", + @[95.0, 105.0, 115.0, 95.0, 105.0, 115.0], + repeatedValues(100.0, 6), + runs = 2 + ) + ) + + let comparison = + compareBenchmarkSuites(baseline, candidate, "synthetic.csv") + + check comparison.results[0].timeStatus == ChangeInconclusive + check comparison.results[0].memStatus == ChangeUnchanged From 3bccd5b7cb525b654d41a09acee4ef1ca59c6c41 Mon Sep 17 00:00:00 2001 From: RowDaBoat Date: Mon, 7 Sep 2026 19:33:44 -0300 Subject: [PATCH 3/3] Compare benchmarks across process runs --- .github/workflows/CI.yml | 30 +++++--- .gitignore | 6 +- benchs/compare_benchmarks.nim | 52 +++++++++++--- benchs/helpers/benchmarks.nim | 129 ++++++++++++++++++++++------------ tests/tbenchmarks.nim | 121 ++++++++++++++++++++++++++----- 5 files changed, 252 insertions(+), 86 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index bf111fa..d385ab8 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -60,13 +60,13 @@ jobs: baseline_ready=true git checkout "$baseline" -- src - if ! nim c -d:danger --nimcache:nimcache/baseline \ + if ! nim c -d:danger -d:SAMPLE=100 --nimcache:nimcache/baseline \ -o:/tmp/vecs-benchmark-baseline "benchs/vecsbench.nim"; then baseline_ready=false fi git checkout HEAD -- src - nim c -d:danger --nimcache:nimcache/current \ + nim c -d:danger -d:SAMPLE=100 --nimcache:nimcache/current \ -o:/tmp/vecs-benchmark-current "benchs/vecsbench.nim" if [ "$baseline_ready" != true ]; then @@ -78,10 +78,24 @@ jobs: nim c -d:danger --nimcache:nimcache/comparison \ -o:/tmp/compare-benchmarks "benchs/compare_benchmarks.nim" - /tmp/vecs-benchmark-current current-a.csv > /tmp/current-a.log - /tmp/vecs-benchmark-baseline baseline-a.csv > /tmp/baseline-a.log - /tmp/vecs-benchmark-baseline baseline-b.csv > /tmp/baseline-b.log - /tmp/vecs-benchmark-current current-b.csv > /tmp/current-b.log + comparison_runs=15 + candidate_files=() + baseline_files=() - /tmp/compare-benchmarks \ - current-a.csv current-b.csv baseline-a.csv baseline-b.csv + for run_index in $(seq 1 "$comparison_runs"); do + current_file="current-${run_index}.csv" + baseline_file="baseline-${run_index}.csv" + + if (( run_index % 2 == 1 )); then + /tmp/vecs-benchmark-current "$current_file" > "/tmp/current-${run_index}.log" + /tmp/vecs-benchmark-baseline "$baseline_file" > "/tmp/baseline-${run_index}.log" + else + /tmp/vecs-benchmark-baseline "$baseline_file" > "/tmp/baseline-${run_index}.log" + /tmp/vecs-benchmark-current "$current_file" > "/tmp/current-${run_index}.log" + fi + + candidate_files+=("$current_file") + baseline_files+=("$baseline_file") + done + + /tmp/compare-benchmarks "${candidate_files[@]}" -- "${baseline_files[@]}" diff --git a/.gitignore b/.gitignore index c1bfe7e..301bcfc 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,5 @@ benchs/vecs_bench benchs/vecs.csv baseline.csv current.csv -baseline-a.csv -baseline-b.csv -current-a.csv -current-b.csv +baseline-*.csv +current-*.csv diff --git a/benchs/compare_benchmarks.nim b/benchs/compare_benchmarks.nim index cb117a5..0cff5e7 100644 --- a/benchs/compare_benchmarks.nim +++ b/benchs/compare_benchmarks.nim @@ -5,25 +5,55 @@ import os import helpers/benchmarks -proc loadMerged(firstPath, secondPath: string): BenchmarkSuite = - let first = loadBenchmarkSuiteFromCsv(firstPath) - let second = loadBenchmarkSuiteFromCsv(secondPath) - result = mergeBenchmarkSuites([first, second]) +proc loadMerged(paths: seq[string]): BenchmarkSuite = + for path in paths: + result.merge(loadBenchmarkSuiteFromCsv(path)) + + +proc splitPaths(arguments: seq[string], + candidatePaths, baselinePaths: var seq[string]): bool = + var foundSeparator = false + + for argument in arguments: + if argument == "--": + if foundSeparator: + return false + foundSeparator = true + elif foundSeparator: + baselinePaths.add argument + else: + candidatePaths.add argument + + if not foundSeparator and arguments.len == 4: + candidatePaths = arguments[0 .. 1] + baselinePaths = arguments[2 .. 3] + return true + + result = + foundSeparator and + candidatePaths.len > 0 and + candidatePaths.len == baselinePaths.len + + +proc describeRuns(paths: seq[string]): string = + result = paths[0] + if paths.len > 1: + result.add " + " & $(paths.len - 1) & " more runs" proc usage(): string = result = - "usage: compare_benchmarks " & - " " & - " " + "usage: compare_benchmarks ... -- ..." if isMainModule: - if paramCount() != 4: + var candidatePaths: seq[string] + var baselinePaths: seq[string] + if not splitPaths(commandLineParams(), candidatePaths, baselinePaths): quit(usage(), QuitFailure) - let candidate = loadMerged(paramStr(1), paramStr(2)) - let baseline = loadMerged(paramStr(3), paramStr(4)) - let baselineName = paramStr(3) & " + " & paramStr(4) + let candidate = loadMerged(candidatePaths) + let baseline = loadMerged(baselinePaths) + let baselineName = describeRuns(baselinePaths) echo compareBenchmarkSuites(baseline, candidate, baselineName) diff --git a/benchs/helpers/benchmarks.nim b/benchs/helpers/benchmarks.nim index 810ffa1..bd26839 100644 --- a/benchs/helpers/benchmarks.nim +++ b/benchs/helpers/benchmarks.nim @@ -33,6 +33,8 @@ type params*: Parameters times*: seq[float] mems*: seq[float] + runTimes*: seq[float] + runMems*: seq[float] timeStats*: Statistics memStats*: Statistics totalTime*: float @@ -128,6 +130,13 @@ proc finalize*(b: var Benchmark) = b.timeStats = calculateStatistics(b.times) b.memStats = calculateStatistics(b.mems) + if b.runTimes.len == 0 and b.times.len > 0: + b.runTimes.add b.timeStats.median + if b.runMems.len == 0 and b.mems.len > 0: + b.runMems.add b.memStats.median + + b.params.runs = min(b.runTimes.len, b.runMems.len) + b.totalTime = 0.0 for t in b.times: b.totalTime += t @@ -244,6 +253,8 @@ proc initBenchmark*(benchmarkName: string, sample, warm: int): Benchmark = Parameters(samples: sample, warmup: warm, batchSize: 1, runs: 1) result.times = newSeqOfCap[float](sample) result.mems = newSeqOfCap[float](sample) + result.runTimes = @[] + result.runMems = @[] template measure*(bench: var Benchmark, memBaseline: int, code: untyped) = @@ -374,6 +385,7 @@ proc showSummary*(suite: BenchmarkSuite) = const DefaultComparisonConfidence* = 0.999 DefaultMinimumRelativeChange* = 0.01 + MinimumComparisonRuns* = 15 proc serializeSamples(values: seq[float]): string = @@ -390,7 +402,8 @@ proc saveSummary*(suite: BenchmarkSuite, path: string) = file.writeLine( suite.name & - ",time_median,mem_median,time_seconds,mem_bytes,time_samples,mem_samples,batch_size" + ",time_median,mem_median,time_seconds,mem_bytes,time_samples,mem_samples," & + "batch_size,time_runs,mem_runs" ) for bench in suite.benchmarks: @@ -402,7 +415,9 @@ proc saveSummary*(suite: BenchmarkSuite, path: string) = bench.memStats.median.formatFloat(ffScientific, 10) & "," & serializeSamples(bench.times) & "," & serializeSamples(bench.mems) & "," & - $bench.params.batchSize + $bench.params.batchSize & "," & + serializeSamples(bench.runTimes) & "," & + serializeSamples(bench.runMems) ) @@ -501,6 +516,14 @@ proc benchmarkFromCsv(parts: seq[string], benchmark: var Benchmark): bool = benchmark.mems.add(medianMemory) if parts.len >= 8: benchmark.params.batchSize = parseBatchSize(parts[7]) + if parts.len >= 10: + benchmark.runTimes = parseSamples(parts[8]) + benchmark.runMems = parseSamples(parts[9]) + + if benchmark.runTimes.len == 0: + benchmark.runTimes.add(medianTime) + if benchmark.runMems.len == 0: + benchmark.runMems.add(medianMemory) benchmark.params.samples = benchmark.times.len finalize(benchmark) @@ -546,8 +569,9 @@ proc merge*(suite: var BenchmarkSuite, addition: BenchmarkSuite) = else: suite.benchmarks[index].times.add(addedBenchmark.times) suite.benchmarks[index].mems.add(addedBenchmark.mems) + suite.benchmarks[index].runTimes.add(addedBenchmark.runTimes) + suite.benchmarks[index].runMems.add(addedBenchmark.runMems) suite.benchmarks[index].params.samples = suite.benchmarks[index].times.len - suite.benchmarks[index].params.runs += addedBenchmark.params.runs suite.benchmarks[index].params.batchSize = max(suite.benchmarks[index].params.batchSize, addedBenchmark.params.batchSize) @@ -605,11 +629,11 @@ proc medianConfidenceInterval*(values: seq[float], proc classifyRelativeChange(interval: ConfidenceInterval, - hasRepeatedRuns: bool, + hasEnoughRuns: bool, minimumRelativeChange: float): ChangeStatus = if interval.lower == 0.0 and interval.upper == 0.0: return ChangeUnchanged - if not hasRepeatedRuns: + if not hasEnoughRuns: return ChangeInconclusive if interval.upper < -minimumRelativeChange: return ChangeImproved @@ -623,10 +647,10 @@ proc classifyRelativeChange(interval: ConfidenceInterval, proc classifyAbsoluteChange(interval: ConfidenceInterval, - hasRepeatedRuns: bool): ChangeStatus = + hasEnoughRuns: bool): ChangeStatus = if interval.lower == 0.0 and interval.upper == 0.0: return ChangeUnchanged - if not hasRepeatedRuns: + if not hasEnoughRuns: return ChangeInconclusive if interval.upper < 0.0: return ChangeImproved @@ -636,47 +660,59 @@ proc classifyAbsoluteChange(interval: ConfidenceInterval, result = ChangeInconclusive +proc allPositive(values: seq[float]): bool = + if values.len == 0: + return false + + for value in values: + if value <= 0.0: + return false + + result = true + + +proc pairedDifferences(baselineValues, + candidateValues: seq[float]): seq[float] = + result = newSeq[float](baselineValues.len) + for index in 0 ..< baselineValues.len: + result[index] = candidateValues[index] - baselineValues[index] + + +proc pairedRelativeChanges(baselineValues, + candidateValues: seq[float]): seq[float] = + result = newSeq[float](baselineValues.len) + for index in 0 ..< baselineValues.len: + result[index] = candidateValues[index] / baselineValues[index] - 1.0 + + proc compareMetric(baselineValues, candidateValues: seq[float], - baselineRuns, candidateRuns: int, confidence, minimumRelativeChange: float): MetricComparison = - if baselineValues.len == 0 or candidateValues.len == 0: + if baselineValues.len == 0 or baselineValues.len != candidateValues.len: result.status = ChangeInconclusive return - let medianConfidence = 1.0 - (1.0 - confidence) / 2.0 - let baselineInterval = - medianConfidenceInterval(baselineValues, medianConfidence) - let candidateInterval = - medianConfidenceInterval(candidateValues, medianConfidence) - let baselineMedian = calculateStatistics(baselineValues).median - let candidateMedian = calculateStatistics(candidateValues).median - - let differenceInterval = ConfidenceInterval( - lower: candidateInterval.lower - baselineInterval.upper, - upper: candidateInterval.upper - baselineInterval.lower - ) - let hasRepeatedRuns = baselineRuns >= 2 and candidateRuns >= 2 - - result.difference = candidateMedian - baselineMedian + let differences = pairedDifferences(baselineValues, candidateValues) + let hasEnoughRuns = baselineValues.len >= MinimumComparisonRuns + result.difference = calculateStatistics(differences).median - if baselineMedian == 0.0 or baselineInterval.lower <= 0.0: + if not baselineValues.allPositive or not candidateValues.allPositive: + let differenceInterval = + medianConfidenceInterval(differences, confidence) result.status = classifyAbsoluteChange( differenceInterval, - hasRepeatedRuns + hasEnoughRuns ) return + let changes = pairedRelativeChanges(baselineValues, candidateValues) result.hasRelativeChange = true - result.ratio = candidateMedian / baselineMedian - result.change = result.ratio - 1.0 - result.changeInterval = ConfidenceInterval( - lower: candidateInterval.lower / baselineInterval.upper - 1.0, - upper: candidateInterval.upper / baselineInterval.lower - 1.0 - ) + result.change = calculateStatistics(changes).median + result.ratio = result.change + 1.0 + result.changeInterval = medianConfidenceInterval(changes, confidence) result.status = classifyRelativeChange( result.changeInterval, - hasRepeatedRuns, + hasEnoughRuns, minimumRelativeChange ) @@ -687,18 +723,14 @@ proc benchmarkResult(baseline, candidate: Benchmark, result.name = candidate.name let timeComparison = compareMetric( - baseline.times, - candidate.times, - baseline.params.runs, - candidate.params.runs, + baseline.runTimes, + candidate.runTimes, confidence, minimumRelativeChange ) let memoryComparison = compareMetric( - baseline.mems, - candidate.mems, - baseline.params.runs, - candidate.params.runs, + baseline.runMems, + candidate.runMems, confidence, minimumRelativeChange ) @@ -728,7 +760,10 @@ proc suiteRuns(suite: BenchmarkSuite): int = if suite.benchmarks.len == 0: return 0 - result = suite.benchmarks[0].params.runs + result = min( + suite.benchmarks[0].runTimes.len, + suite.benchmarks[0].runMems.len + ) proc compareBenchmarkSuites*(baseline, candidate: BenchmarkSuite, @@ -869,13 +904,13 @@ proc `$`*(comparison: BenchComp): string = nameWidth + metricWidth + metricWidth + statusWidth + 9 let confidenceText = (comparison.confidence * 100.0).formatFloat(ffDecimal, 1) & - "% median intervals" + "% paired-run median intervals" let minimumEffectText = "Minimum directional effect: " & (comparison.minimumRelativeChange * 100.0).formatFloat(ffDecimal, 1) & "%" let runsText = - "Runs: baseline " & $comparison.baselineRuns & + "Process runs: baseline " & $comparison.baselineRuns & ", candidate " & $comparison.candidateRuns lines.add "" @@ -936,9 +971,11 @@ proc `$`*(comparison: BenchComp): string = "Legend: ▼ = improvement ▲ = regression ? = inconclusive " & "= = unchanged * = no baseline" ) - if comparison.baselineRuns < 2 or comparison.candidateRuns < 2: + if comparison.baselineRuns < MinimumComparisonRuns or + comparison.candidateRuns < MinimumComparisonRuns: lines.add( - "At least two runs per revision are required for a directional result." + "At least " & $MinimumComparisonRuns & + " matched process runs per revision are required for a directional result." ) result = lines.join("\n") diff --git a/tests/tbenchmarks.nim b/tests/tbenchmarks.nim index ccc22ec..6949c67 100644 --- a/tests/tbenchmarks.nim +++ b/tests/tbenchmarks.nim @@ -12,13 +12,20 @@ proc repeatedValues(value: float, count: int): seq[float] = proc newSyntheticBenchmark(name: string, times, memories: seq[float], - runs: int = 1): Benchmark = + runTimes: seq[float] = @[], + runMems: seq[float] = @[]): Benchmark = result = initBenchmark(name, times.len, 0) result.times = times result.mems = memories - result.params.runs = runs finalize(result) + if runTimes.len > 0: + result.runTimes = runTimes + if runMems.len > 0: + result.runMems = runMems + + result.params.runs = min(result.runTimes.len, result.runMems.len) + proc newSyntheticSuite(benchmark: Benchmark): BenchmarkSuite = result = initSuite("Synthetic") @@ -26,7 +33,7 @@ proc newSyntheticSuite(benchmark: Benchmark): BenchmarkSuite = suite "Benchmark comparisons should": - test "preserve raw samples in CSV summaries": + test "preserve samples and process runs in CSV summaries": let path = getTempDir() / ("vecs-benchmarks-" & $getCurrentProcessId() & ".csv") defer: @@ -36,7 +43,9 @@ suite "Benchmark comparisons should": let benchmark = newSyntheticBenchmark( "sample", @[1.0, 2.0, 3.0, 4.0], - @[10.0, 20.0, 30.0, 40.0] + @[10.0, 20.0, 30.0, 40.0], + @[2.0, 2.5, 3.0], + @[20.0, 25.0, 30.0] ) let suite = newSyntheticSuite(benchmark) @@ -46,6 +55,8 @@ suite "Benchmark comparisons should": check loaded.benchmarks.len == 1 check loaded.benchmarks[0].times == benchmark.times check loaded.benchmarks[0].mems == benchmark.mems + check loaded.benchmarks[0].runTimes == benchmark.runTimes + check loaded.benchmarks[0].runMems == benchmark.runMems test "keep a single-run comparison inconclusive": @@ -71,13 +82,80 @@ suite "Benchmark comparisons should": check comparison.results[0].memStatus == ChangeInconclusive - test "detect a repeated improvement": + test "ignore raw sample volume when process runs disagree": + let baseline = newSyntheticSuite( + newSyntheticBenchmark( + "sample", + repeatedValues(100.0, 100), + repeatedValues(100.0, 100), + repeatedValues(100.0, MinimumComparisonRuns), + repeatedValues(100.0, MinimumComparisonRuns) + ) + ) + let candidate = newSyntheticSuite( + newSyntheticBenchmark( + "sample", + repeatedValues(80.0, 100), + repeatedValues(80.0, 100), + @[ + 95.0, 105.0, 95.0, 105.0, 95.0, + 105.0, 95.0, 105.0, 95.0, 105.0, + 95.0, 105.0, 95.0, 105.0, 95.0 + ], + @[ + 95.0, 105.0, 95.0, 105.0, 95.0, + 105.0, 95.0, 105.0, 95.0, 105.0, + 95.0, 105.0, 95.0, 105.0, 95.0 + ] + ) + ) + + let comparison = + compareBenchmarkSuites(baseline, candidate, "synthetic.csv") + + check comparison.results[0].timeStatus == ChangeInconclusive + check comparison.results[0].memStatus == ChangeInconclusive + + + test "require enough independent process runs": + let baseline = newSyntheticSuite( + newSyntheticBenchmark( + "sample", + repeatedValues(100.0, 100), + repeatedValues(100.0, 100), + repeatedValues(100.0, MinimumComparisonRuns - 1), + repeatedValues(100.0, MinimumComparisonRuns - 1) + ) + ) + let candidate = newSyntheticSuite( + newSyntheticBenchmark( + "sample", + repeatedValues(80.0, 100), + repeatedValues(80.0, 100), + repeatedValues(80.0, MinimumComparisonRuns - 1), + repeatedValues(80.0, MinimumComparisonRuns - 1) + ) + ) + + let comparison = + compareBenchmarkSuites(baseline, candidate, "synthetic.csv") + + check comparison.results[0].timeStatus == ChangeInconclusive + check comparison.results[0].memStatus == ChangeInconclusive + + + test "detect an improvement across process runs": + let baselineRuns = repeatedValues(100.0, MinimumComparisonRuns) + var candidateRuns = repeatedValues(80.0, MinimumComparisonRuns) + candidateRuns[^1] = 105.0 + let baseline = newSyntheticSuite( newSyntheticBenchmark( "sample", repeatedValues(100.0, 100), repeatedValues(100.0, 100), - runs = 2 + baselineRuns, + baselineRuns ) ) let candidate = newSyntheticSuite( @@ -85,7 +163,8 @@ suite "Benchmark comparisons should": "sample", repeatedValues(80.0, 100), repeatedValues(80.0, 100), - runs = 2 + candidateRuns, + candidateRuns ) ) @@ -96,13 +175,14 @@ suite "Benchmark comparisons should": check comparison.results[0].memStatus == ChangeImproved - test "treat sub-percent repeated changes as unchanged": + test "treat sub-percent process changes as unchanged": let baseline = newSyntheticSuite( newSyntheticBenchmark( "sample", repeatedValues(100.0, 100), repeatedValues(100.0, 100), - runs = 2 + repeatedValues(100.0, MinimumComparisonRuns), + repeatedValues(100.0, MinimumComparisonRuns) ) ) let candidate = newSyntheticSuite( @@ -110,7 +190,8 @@ suite "Benchmark comparisons should": "sample", repeatedValues(100.5, 100), repeatedValues(100.5, 100), - runs = 2 + repeatedValues(100.5, MinimumComparisonRuns), + repeatedValues(100.5, MinimumComparisonRuns) ) ) @@ -121,21 +202,27 @@ suite "Benchmark comparisons should": check comparison.results[0].memStatus == ChangeUnchanged - test "leave overlapping repeated measurements inconclusive": + test "leave overlapping process measurements inconclusive": let baseline = newSyntheticSuite( newSyntheticBenchmark( "sample", - @[90.0, 100.0, 110.0, 90.0, 100.0, 110.0], - repeatedValues(100.0, 6), - runs = 2 + repeatedValues(100.0, 100), + repeatedValues(100.0, 100), + repeatedValues(100.0, MinimumComparisonRuns), + repeatedValues(100.0, MinimumComparisonRuns) ) ) let candidate = newSyntheticSuite( newSyntheticBenchmark( "sample", - @[95.0, 105.0, 115.0, 95.0, 105.0, 115.0], - repeatedValues(100.0, 6), - runs = 2 + repeatedValues(105.0, 100), + repeatedValues(100.0, 100), + @[ + 95.0, 105.0, 95.0, 105.0, 95.0, + 105.0, 95.0, 105.0, 95.0, 105.0, + 95.0, 105.0, 95.0, 105.0, 95.0 + ], + repeatedValues(100.0, MinimumComparisonRuns) ) )