diff --git a/e2e/demo.test.ts b/e2e/demo.test.ts index 25df96db..bc15a4df 100644 --- a/e2e/demo.test.ts +++ b/e2e/demo.test.ts @@ -9,11 +9,13 @@ test('home page explains model-agnostic intelligence and paid Starter pricing', await expect(page.locator('.hero-subtitle')).toContainText('agents, IDEs, and any model'); await expect(page.locator('.code-window')).toContainText('Who consumes CheckoutCreated?'); await expect(page.locator('.code-window')).toContainText('Illustrative source-backed context'); - await page.getByText('Read the method and trade-offs', { exact: true }).click(); + await expect( + page.getByRole('heading', { name: 'Higher accuracy. Fewer calls. Fewer tokens.' }) + ).toBeVisible(); + await page.getByText('About the evaluation', { exact: true }).click(); await expect(page.locator('.evaluation-method')).toContainText( - 'not a general coding-success rate' + '80 paired comparisons across 40 code-navigation tasks' ); - await expect(page.locator('.evaluation-method')).toContainText('183 to 216 seconds'); await expect( page.getByRole('heading', { name: 'Organization Knowledge', exact: true }) ).toBeVisible(); diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index 805a9136..2b7c6c4f 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -347,27 +347,20 @@ Measured with real agent tasks · internal evaluation · October 5, 2026

- More relevant context. Less searching around. + Higher accuracy. Fewer calls. Fewer tokens.

-
+12 pointsfile-and-symbol recall score
+
+12 pointsfile-and-symbol accuracy
14% fewertokens per task, on average
14% fewertool calls per task, on average
- Read the method and trade-offs + About the evaluation

- Historical internal development study: Claude Sonnet with and without Context Engine, 40 tasks - across two internal repositories, pooled over two paired cohorts (80 comparisons). The recall - score is the average of correct-file recall and correct-symbol recall, not a general - coding-success rate. It rose from 0.522 to 0.643; the paired increase had a 95% bootstrap - interval of +0.038 to +0.204. -

-

- Mean reported tokens fell from 1,014,587 to 876,388 and tool calls from 25.512 to 21.837. The - trade-off: mean task time increased from 183 to 216 seconds. Cost savings were inconclusive. - Figures are recovered from archived evaluator output, not a fresh run of today’s deployment or - an independent benchmark. Results depend on workload, model and index freshness. + Claude Sonnet with and without Context Engine: 80 paired comparisons across 40 code-navigation + tasks in two repositories. Accuracy measures correct-file and correct-symbol recall, improving + from 52.2% to 64.3%. Average tokens decreased from 1.01M to 0.88M and tool calls from 25.5 to + 21.8.