From 95fa36c1f9d9d36b1af52c7dedf5058b57c305cb Mon Sep 17 00:00:00 2001 From: John Donalson Date: Sun, 18 Oct 2026 22:35:05 -0400 Subject: [PATCH] feat: show cross-repo MCP evidence and scoped evaluation results --- e2e/demo.test.ts | 7 ++++++ src/app.scss | 32 ++++++++++++++++++++++++- src/routes/+page.svelte | 53 ++++++++++++++++++++++++++++++++++------- 3 files changed, 83 insertions(+), 9 deletions(-) diff --git a/e2e/demo.test.ts b/e2e/demo.test.ts index 0fc9a874..25df96db 100644 --- a/e2e/demo.test.ts +++ b/e2e/demo.test.ts @@ -7,6 +7,13 @@ test('home page explains model-agnostic intelligence and paid Starter pricing', await page.goto('/'); await expect(page.locator('h1')).toContainText('Codebase intelligence'); await expect(page.locator('.hero-subtitle')).toContainText('agents, IDEs, and any model'); + await expect(page.locator('.code-window')).toContainText('Who consumes CheckoutCreated?'); + await expect(page.locator('.code-window')).toContainText('Illustrative source-backed context'); + await page.getByText('Read the method and trade-offs', { exact: true }).click(); + await expect(page.locator('.evaluation-method')).toContainText( + 'not a general coding-success rate' + ); + await expect(page.locator('.evaluation-method')).toContainText('183 to 216 seconds'); await expect( page.getByRole('heading', { name: 'Organization Knowledge', exact: true }) ).toBeVisible(); diff --git a/src/app.scss b/src/app.scss index 4f476eae..07d906f4 100644 --- a/src/app.scss +++ b/src/app.scss @@ -1101,6 +1101,36 @@ body::before { font-weight: 500; } +.evaluation-results { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 24px; + border-block: 1px solid var(--border); + padding: 32px 0; +} + +.evaluation-results strong { + display: block; + color: var(--accent); + font-size: clamp(24px, 3vw, 40px); + font-weight: 500; +} + +.evaluation-results span, .evaluation-method { + color: var(--text-secondary); +} + +.evaluation-method { + margin-top: 24px; + max-width: 850px; + font-size: 14px; +} + +.evaluation-method summary { + cursor: pointer; + min-height: 44px; +} + .open-source-foundations { max-width: 1280px; margin: 0 auto 40px; @@ -1143,7 +1173,7 @@ a:focus-visible, button:focus-visible { } @media (max-width: 768px) { - #capabilities .features-grid, .open-source-links { + #capabilities .features-grid, .open-source-links, .evaluation-results { grid-template-columns: 1fr; } } diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index 53f3556c..805a9136 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -300,15 +300,22 @@ - mcp_tools.py + MCP · cross_repo_search -
# Find authentication flow
-result = repo_search(
-    query="user authentication",
-    limit=5,
-    compact=True
-)
-# Grounded source evidence
+
// Your agent asks across the business
+cross_repo_search({'{'}
+  "query": "Who consumes CheckoutCreated?",
+  "target_repos": ["storefront", "billing", "shared"],
+  "trace_boundary": true
+{'}'})
+
+// Illustrative source-backed context
+shared/events.ts       → event contract
+storefront/checkout.ts → publisher
+billing/invoices.ts    → consumer
@@ -335,6 +342,36 @@ result = repo_search( +
+

+ Measured with real agent tasks · internal evaluation · October 5, 2026 +

+

+ More relevant context. Less searching around. +

+
+
+12 pointsfile-and-symbol recall score
+
14% fewertokens per task, on average
+
14% fewertool calls per task, on average
+
+
+ Read the method and trade-offs +

+ Historical internal development study: Claude Sonnet with and without Context Engine, 40 tasks + across two internal repositories, pooled over two paired cohorts (80 comparisons). The recall + score is the average of correct-file recall and correct-symbol recall, not a general + coding-success rate. It rose from 0.522 to 0.643; the paired increase had a 95% bootstrap + interval of +0.038 to +0.204. +

+

+ Mean reported tokens fell from 1,014,587 to 876,388 and tool calls from 25.512 to 21.837. The + trade-off: mean task time increased from 183 to 216 seconds. Cost savings were inconclusive. + Figures are recovered from archived evaluator output, not a fresh run of today’s deployment or + an independent benchmark. Results depend on workload, model and index freshness. +

+
+
+

How It Works