From 0690c9dae9688c3f38fae9271b891a7501eda7f5 Mon Sep 17 00:00:00 2001
From: John Donalson
Date: Sun, 18 Oct 2026 22:36:32 -0400
Subject: [PATCH] copy: state measured accuracy and efficiency gains directly
---
e2e/demo.test.ts | 8 +++++---
src/routes/+page.svelte | 21 +++++++--------------
2 files changed, 12 insertions(+), 17 deletions(-)
diff --git a/e2e/demo.test.ts b/e2e/demo.test.ts
index 25df96db..bc15a4df 100644
--- a/e2e/demo.test.ts
+++ b/e2e/demo.test.ts
@@ -9,11 +9,13 @@ test('home page explains model-agnostic intelligence and paid Starter pricing',
await expect(page.locator('.hero-subtitle')).toContainText('agents, IDEs, and any model');
await expect(page.locator('.code-window')).toContainText('Who consumes CheckoutCreated?');
await expect(page.locator('.code-window')).toContainText('Illustrative source-backed context');
- await page.getByText('Read the method and trade-offs', { exact: true }).click();
+ await expect(
+ page.getByRole('heading', { name: 'Higher accuracy. Fewer calls. Fewer tokens.' })
+ ).toBeVisible();
+ await page.getByText('About the evaluation', { exact: true }).click();
await expect(page.locator('.evaluation-method')).toContainText(
- 'not a general coding-success rate'
+ '80 paired comparisons across 40 code-navigation tasks'
);
- await expect(page.locator('.evaluation-method')).toContainText('183 to 216 seconds');
await expect(
page.getByRole('heading', { name: 'Organization Knowledge', exact: true })
).toBeVisible();
diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte
index 805a9136..2b7c6c4f 100644
--- a/src/routes/+page.svelte
+++ b/src/routes/+page.svelte
@@ -347,27 +347,20 @@
Measured with real agent tasks · internal evaluation · October 5, 2026
- More relevant context. Less searching around.
+ Higher accuracy. Fewer calls. Fewer tokens.
-
+12 pointsfile-and-symbol recall score
+
+12 pointsfile-and-symbol accuracy
14% fewertokens per task, on average
14% fewertool calls per task, on average
- Read the method and trade-offs
+ About the evaluation
- Historical internal development study: Claude Sonnet with and without Context Engine, 40 tasks
- across two internal repositories, pooled over two paired cohorts (80 comparisons). The recall
- score is the average of correct-file recall and correct-symbol recall, not a general
- coding-success rate. It rose from 0.522 to 0.643; the paired increase had a 95% bootstrap
- interval of +0.038 to +0.204.
-
-
- Mean reported tokens fell from 1,014,587 to 876,388 and tool calls from 25.512 to 21.837. The
- trade-off: mean task time increased from 183 to 216 seconds. Cost savings were inconclusive.
- Figures are recovered from archived evaluator output, not a fresh run of today’s deployment or
- an independent benchmark. Results depend on workload, model and index freshness.
+ Claude Sonnet with and without Context Engine: 80 paired comparisons across 40 code-navigation
+ tasks in two repositories. Accuracy measures correct-file and correct-symbol recall, improving
+ from 52.2% to 64.3%. Average tokens decreased from 1.01M to 0.88M and tool calls from 25.5 to
+ 21.8.