From 6ee7e9d7959656dbc58ebc579f7ce37c1063b1cb Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 20 Sep 2026 20:16:15 +0000 Subject: [PATCH 1/7] =?UTF-8?q?fix:=20reach=20Supabase=20over=20PostgREST?= =?UTF-8?q?=20only=20=E2=80=94=20remove=20the=20ww-mcp=20edge=20function?= =?UTF-8?q?=20dependency=20(GIT-97)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit listRecords/getRecord/upsertRecord routed through /functions/v1/ww-mcp, an edge function that exists on nTEG's Supabase project and is not shipped with the package. On a customer-owned project every such read returned 404. - listRecords/getRecord now use PostgREST directly; upsertRecord delegates to directUpsert (it had no remaining callers). callMcp/getMcpUrl deleted. - Equality filters always emit an explicit operator, so a value containing '.' (a title, a version) is treated as data. - session_start loads session history and threads with Promise.allSettled, so a failed history read no longer discards a successful thread read and drops the panel to the local threads.json. - Transport tests pin reads to /rest/v1 and fail if an edge function returns. --- src/services/supabase-client.ts | 132 +++++++----------- src/tools/session-start.ts | 22 ++- .../services/supabase-client-rest.test.ts | 110 +++++++++++++++ 3 files changed, 183 insertions(+), 81 deletions(-) create mode 100644 tests/unit/services/supabase-client-rest.test.ts diff --git a/src/services/supabase-client.ts b/src/services/supabase-client.ts index 171629c..1d6bda1 100644 --- a/src/services/supabase-client.ts +++ b/src/services/supabase-client.ts @@ -1,10 +1,10 @@ /** - * Supabase MCP Client + * Supabase Client * - * HTTP client for ww-mcp Edge Function following the pattern from - * agents/coda/src/services/supabase-mcp.js + * Talks to Supabase over PostgREST only. GIT-97: every path here must work on + * a Supabase project provisioned from schema/setup.sql and nothing else. The + * package ships no edge function for data access, so none may be assumed. * - * Uses JSON-RPC 2.0 protocol over HTTPS. * Integrates with CacheService for performance. */ @@ -74,90 +74,65 @@ export function isConfigured(): boolean { } /** - * Get the ww-mcp Edge Function URL + * Build a PostgREST equality filter. + * + * Always emits an explicit `eq.` / `is.` operator. directQuery() guesses that a + * value containing "." already carries an operator, which misreads ordinary + * data such as a title ("Done != Deployed.") or a version ("1.8.0"). Callers of + * listRecords pass data, never operators, so nothing is guessed here. */ -function getMcpUrl(): string { - return `${SUPABASE_URL}/functions/v1/ww-mcp`; +function toEqualityFilter(value: unknown): string { + if (value === null || value === undefined) return "is.null"; + if (typeof value === "boolean") return `is.${value}`; + return `eq.${String(value)}`; } /** - * Call the ww-mcp Edge Function + * List records from a table with optional equality filters. + * + * GIT-97: previously routed through the ww-mcp edge function, which exists only + * on nTEG's Supabase project and 404s on every customer-owned one. */ -async function callMcp(toolName: string, args: Record): Promise { +export async function listRecords( + options: SupabaseListOptions +): Promise { if (!isConfigured()) { throw new Error("Supabase not configured - check SUPABASE_URL and SUPABASE_KEY/SUPABASE_SERVICE_ROLE_KEY"); } - const mcpUrl = getMcpUrl(); - - const response = await fetch(mcpUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${SUPABASE_KEY}`, - }, - body: JSON.stringify({ - jsonrpc: "2.0", - id: `${toolName}-${Date.now()}`, - method: "tools/call", - params: { - name: toolName, - arguments: args, - }, - }), - }); - - if (!response.ok) { - const text = await response.text(); - throw new Error(`MCP HTTP error: ${response.status} - ${text.slice(0, 200)}`); - } + const { table, columns, filters, limit = 50, orderBy } = options; - const result = await response.json() as { - error?: { message: string }; - result?: { - content?: Array<{ text: string }>; - }; - }; + const url = new URL(`${SUPABASE_REST_URL}/${table}`); + url.searchParams.set("select", columns || "*"); - if (result.error) { - throw new Error(`MCP error: ${JSON.stringify(result.error)}`); + for (const [key, value] of Object.entries(filters ?? {})) { + url.searchParams.set(key, toEqualityFilter(value)); } - // Parse the result content - if (result.result?.content?.[0]?.text) { - return JSON.parse(result.result.content[0].text) as T; + if (orderBy) { + url.searchParams.set("order", `${orderBy.column}.${orderBy.ascending ? "asc" : "desc"}`); } - return result.result as T; -} - -/** - * List records from a table with optional filters - */ -export async function listRecords( - options: SupabaseListOptions -): Promise { - const { table, columns, filters, limit = 50, orderBy } = options; - - const args: Record = { - table, - limit, - }; - - if (columns) { - args.columns = columns; - } + url.searchParams.set("limit", String(limit)); - if (filters) { - args.filters = filters; - } + const response = await fetch(url.toString(), { + method: "GET", + headers: { + apikey: SUPABASE_KEY, + Authorization: `Bearer ${SUPABASE_KEY}`, + "Content-Type": "application/json", + "Accept-Profile": "public", + }, + signal: AbortSignal.timeout(15_000), + }); - if (orderBy) { - args.orderBy = orderBy; + if (!response.ok) { + const text = await response.text(); + throw new Error(`Supabase REST error: ${response.status} - ${text.slice(0, 200)}`); } - const result = await callMcp<{ data: T[] }>("list_records", args); - return result.data || []; + const rows = (await response.json()) as T[]; + return rows || []; } /** @@ -167,24 +142,21 @@ export async function getRecord( table: string, id: string ): Promise { - const result = await callMcp<{ data: T }>("get_record", { table, id }); - return result.data || null; + const rows = await listRecords({ table, filters: { id }, limit: 1 }); + return rows[0] ?? null; } /** - * Upsert (insert or update) a record + * Upsert (insert or update) a record. + * + * Kept for API compatibility; delegates to directUpsert. Embeddings are + * generated client-side by the callers that need them (see embedding.ts). */ export async function upsertRecord( table: string, data: Record ): Promise { - const result = await callMcp<{ data: T; record?: T }>("upsert_record", { - table, - data, - }); - - // ww-mcp returns { data: record, operation: 'insert'|'update', embedding_generated: bool } - return result.data || (result as unknown as T); + return directUpsert(table, data); } /** @@ -243,7 +215,7 @@ export async function semanticSearch( } // ============================================================================ -// DIRECT SUPABASE QUERIES (bypass ww-mcp for bulk operations) +// DIRECT SUPABASE QUERIES // ============================================================================ /** diff --git a/src/tools/session-start.ts b/src/tools/session-start.ts index c6d034e..28ffedd 100644 --- a/src/tools/session-start.ts +++ b/src/tools/session-start.ts @@ -180,7 +180,13 @@ async function loadLastSession( try { // Parallel load: sessions + threads are independent queries // (was sequential — ~200-300ms saved by parallelizing) - const [sessions, supabaseThreads] = await Promise.all([ + // + // GIT-97: allSettled, not all. These reads are independent, and with + // Promise.all a failed session-history read rejected the pair and threw away + // a successful thread read — the panel then fell back to the local + // threads.json while list_threads showed the durable store. A failed half is + // named on stderr and degrades only itself. + const [sessionsResult, threadsResult] = await Promise.allSettled([ supabase.listRecords({ table: getTableName("sessions_lite"), filters: { agent, project }, @@ -190,6 +196,20 @@ async function loadLastSession( loadActiveThreadsFromSupabase(resolveThreadScope({ project })), ]); + let sessions: SessionRecord[] = []; + if (sessionsResult.status === "fulfilled") { + sessions = sessionsResult.value; + } else { + console.error("[session_start] Session history read failed (threads unaffected):", sessionsResult.reason); + } + + let supabaseThreads: Awaited> = null; + if (threadsResult.status === "fulfilled") { + supabaseThreads = threadsResult.value; + } else { + console.error("[session_start] Thread read failed (session history unaffected):", threadsResult.reason); + } + let aggregated_open_threads: ThreadObject[]; let displayInfo: ThreadDisplayInfo[] = []; let threadsFromSupabase = false; diff --git a/tests/unit/services/supabase-client-rest.test.ts b/tests/unit/services/supabase-client-rest.test.ts new file mode 100644 index 0000000..469fcf5 --- /dev/null +++ b/tests/unit/services/supabase-client-rest.test.ts @@ -0,0 +1,110 @@ +/** + * GIT-97: listRecords / getRecord must reach Supabase over PostgREST only. + * + * The package ships no data-access edge function, so a customer project + * provisioned from schema/setup.sql has none. These tests pin the transport: + * if a read ever goes back to /functions/v1/*, they fail. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +const SUPABASE_URL = "https://customer-project.supabase.co"; + +async function loadClient() { + vi.resetModules(); + process.env.SUPABASE_URL = SUPABASE_URL; + process.env.SUPABASE_SERVICE_ROLE_KEY = "test-service-role-key"; + return import("../../../src/services/supabase-client.js"); +} + +function okJson(body: unknown) { + return { ok: true, status: 200, json: async () => body, text: async () => JSON.stringify(body) } as unknown as Response; +} + +describe("supabase-client transport (GIT-97)", () => { + let fetchMock: ReturnType; + + beforeEach(() => { + fetchMock = vi.fn(async () => okJson([])); + vi.stubGlobal("fetch", fetchMock); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + delete process.env.SUPABASE_URL; + delete process.env.SUPABASE_SERVICE_ROLE_KEY; + }); + + it("listRecords calls PostgREST and never an edge function", async () => { + const client = await loadClient(); + await client.listRecords({ + table: "gitmem_sessions_lite", + filters: { agent: "cli", project: "dev-suite" }, + limit: 10, + orderBy: { column: "created_at", ascending: false }, + }); + + expect(fetchMock).toHaveBeenCalledTimes(1); + const [rawUrl, init] = fetchMock.mock.calls[0] as [string, RequestInit]; + const url = new URL(rawUrl); + + expect(url.pathname).toBe("/rest/v1/gitmem_sessions_lite"); + expect(rawUrl).not.toContain("/functions/v1/"); + expect(init.method).toBe("GET"); + expect(url.searchParams.get("agent")).toBe("eq.cli"); + expect(url.searchParams.get("project")).toBe("eq.dev-suite"); + expect(url.searchParams.get("order")).toBe("created_at.desc"); + expect(url.searchParams.get("limit")).toBe("10"); + expect(url.searchParams.get("select")).toBe("*"); + }); + + it("treats a filter value containing '.' as data, not as an operator", async () => { + const client = await loadClient(); + await client.listRecords({ + table: "gitmem_learnings_lite", + columns: "id,title", + filters: { title: "Done != Deployed. Verify the row." }, + }); + + const url = new URL((fetchMock.mock.calls[0] as [string])[0]); + expect(url.searchParams.get("title")).toBe("eq.Done != Deployed. Verify the row."); + expect(url.searchParams.get("select")).toBe("id,title"); + }); + + it("maps null and boolean filters to PostgREST is.*", async () => { + const client = await loadClient(); + await client.listRecords({ table: "gitmem_learnings_lite", filters: { archived_at: null, is_active: true } }); + + const url = new URL((fetchMock.mock.calls[0] as [string])[0]); + expect(url.searchParams.get("archived_at")).toBe("is.null"); + expect(url.searchParams.get("is_active")).toBe("is.true"); + }); + + it("getRecord filters by id over PostgREST and returns null when absent", async () => { + const client = await loadClient(); + const missing = await client.getRecord("gitmem_sessions", "07ac0fc5-13af-4f63-8ff9-6be604022aa4"); + expect(missing).toBeNull(); + + const url = new URL((fetchMock.mock.calls[0] as [string])[0]); + expect(url.pathname).toBe("/rest/v1/gitmem_sessions"); + expect(url.searchParams.get("id")).toBe("eq.07ac0fc5-13af-4f63-8ff9-6be604022aa4"); + expect(url.searchParams.get("limit")).toBe("1"); + + fetchMock.mockResolvedValueOnce(okJson([{ id: "abc", project: "gitmem" }])); + const found = await client.getRecord<{ id: string }>("gitmem_sessions", "abc"); + expect(found).toEqual({ id: "abc", project: "gitmem" }); + }); + + it("surfaces a REST failure as an error naming the status", async () => { + fetchMock.mockResolvedValueOnce({ ok: false, status: 404, text: async () => "relation does not exist" } as unknown as Response); + const client = await loadClient(); + await expect(client.listRecords({ table: "nope" })).rejects.toThrow(/Supabase REST error: 404/); + }); + + it("no source file references the ww-mcp edge function in code", async () => { + const { readFileSync } = await import("node:fs"); + const src = readFileSync(new URL("../../../src/services/supabase-client.ts", import.meta.url), "utf-8"); + const code = src.split("\n").filter((l) => !/^\s*(\/\/|\*|\/\*)/.test(l)).join("\n"); + expect(code).not.toContain("functions/v1"); + expect(code).not.toContain("callMcp"); + }); +}); From 3214426af10780200f4bc83825aca26d07a3bf8f Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 20 Sep 2026 20:38:12 +0000 Subject: [PATCH 2/7] fix: cache the embedding index on disk so a cold process start costs bytes, not megabytes (GIT-98) In local search mode every server process downloaded every learning with its 1536-dim embedding at startup and discarded it at exit. Short-lived processes (sub-agents, fan-outs, CI) paid full price each time; a customer exhausted a Supabase egress allowance this way. - getLearningsFingerprint(): one request, one row (Prefer: count=exact) gives row count + newest updated_at for ~60 bytes. - vector-disk-cache: rows persisted under that fingerprint, keyed by a hash of (store URL, table). Any mismatch, corruption or unknown fingerprint is a miss and falls back to the download. Atomic temp+rename writes, mode 0600. - Misses are single-flighted across processes with an O_EXCL lock: one leader downloads, followers wait (async, bounded) and read its cache. - cache-flush bypasses the disk cache and rewrites it. - Opt out with GITMEM_VECTOR_DISK_CACHE=0. --- src/services/startup.ts | 63 ++++- src/services/supabase-client.ts | 60 +++++ src/services/vector-disk-cache.ts | 235 ++++++++++++++++++ tests/unit/services/vector-disk-cache.test.ts | 214 ++++++++++++++++ 4 files changed, 565 insertions(+), 7 deletions(-) create mode 100644 src/services/vector-disk-cache.ts create mode 100644 tests/unit/services/vector-disk-cache.test.ts diff --git a/src/services/startup.ts b/src/services/startup.ts index ff08ae6..6a127ed 100644 --- a/src/services/startup.ts +++ b/src/services/startup.ts @@ -15,7 +15,8 @@ import * as fs from "fs"; import * as path from "path"; -import { isConfigured, loadScarsWithEmbeddings } from "./supabase-client.js"; +import { isConfigured, loadScarsWithEmbeddings, getLearningsFingerprint, getSupabaseUrl } from "./supabase-client.js"; +import { computeStoreKey, readVectorCache, writeVectorCache, tryAcquireDownloadLock, waitForLeaderCache } from "./vector-disk-cache.js"; import { getTableName } from "./tier.js"; import { getGitmemDir } from "./gitmem-dir.js"; import { @@ -106,13 +107,51 @@ interface ScarWithEmbedding { * NOTE: Now loads all learning types (scars, patterns, wins, anti-patterns), * not just scars. This fixes the issue where ~64 patterns were being ignored. */ -async function loadScarsFromSupabase(): Promise<{ +async function loadScarsFromSupabase(options: { bypassDiskCache?: boolean } = {}): Promise<{ scars: ScarWithEmbedding[]; latestUpdatedAt: string | null; + source: "disk" | "network"; }> { - console.error(`[startup] Loading ALL learnings with embeddings from Supabase (cross-project)`); const startTime = Date.now(); + // GIT-98: ask the store what it holds (one row, a few hundred bytes) before + // paying for the bulk download. Observed BEFORE the download on purpose: if + // the store changes mid-load the cache is merely stale and the next start + // reloads, which is the safe direction. + const storeKey = computeStoreKey(getSupabaseUrl(), getTableName("learnings")); + const fingerprint = await getLearningsFingerprint(); + + if (!options.bypassDiskCache) { + const cached = readVectorCache(storeKey, fingerprint); + if (cached.hit) { + console.error( + `[startup] Vector cache HIT: ${cached.rows.length} learnings from disk in ${Date.now() - startTime}ms ` + + `(${cached.bytesOnDisk} bytes on disk, bulk download skipped)` + ); + return { scars: cached.rows, latestUpdatedAt: fingerprint.latestUpdatedAt, source: "disk" }; + } + console.error(`[startup] Vector cache MISS: ${cached.reason}`); + } + + // Single-flight the download across processes. A fan-out starts N workers at + // once and a single store write invalidates the cache for all of them, so + // without this a miss costs N identical downloads. + let releaseLock = tryAcquireDownloadLock(storeKey); + if (!releaseLock && !options.bypassDiskCache && fingerprint.count >= 0) { + console.error("[startup] Another process is downloading the index — waiting for its cache"); + const fromLeader = await waitForLeaderCache(storeKey, fingerprint); + if (fromLeader) { + console.error( + `[startup] Vector cache HIT (via leader): ${fromLeader.rows.length} learnings in ${Date.now() - startTime}ms, bulk download skipped` + ); + return { scars: fromLeader.rows, latestUpdatedAt: fingerprint.latestUpdatedAt, source: "disk" }; + } + console.error("[startup] Leader did not produce a usable cache — downloading"); + } + releaseLock = releaseLock ?? (() => {}); + + console.error(`[startup] Loading ALL learnings with embeddings from Supabase (cross-project)`); + try { // Load ALL learnings across projects — semantic similarity handles relevance const learnings = await loadScarsWithEmbeddings(undefined, 500); @@ -127,10 +166,19 @@ async function loadScarsFromSupabase(): Promise<{ // Get latest updated_at for staleness tracking const latestUpdatedAt = learnings.length > 0 ? learnings[0].updated_at || null : null; - return { scars: learnings, latestUpdatedAt }; + // Never cache an empty result: an empty download is far more often a + // transient failure than a genuinely empty store, and caching it would make + // the failure sticky. + if (learnings.length > 0) { + writeVectorCache(storeKey, fingerprint, learnings); + } + + return { scars: learnings, latestUpdatedAt, source: "network" }; } catch (error) { console.error("[startup] Failed to load learnings:", error); - return { scars: [], latestUpdatedAt: null }; + return { scars: [], latestUpdatedAt: null, source: "network" }; + } finally { + releaseLock(); } } @@ -576,8 +624,9 @@ export async function flushCache(_project?: Project): Promise const previousCount = getLocalVectorSearch().getScarCount(); try { - // Load ALL fresh scars (cross-project) - const { scars, latestUpdatedAt } = await loadScarsFromSupabase(); + // Load ALL fresh scars (cross-project). A flush is the user saying "I do not + // trust what you have" — bypass the disk cache (GIT-98) and rewrite it. + const { scars, latestUpdatedAt } = await loadScarsFromSupabase({ bypassDiskCache: true }); // Reinitialize the unified index await reinitializeLocalSearch(scars, undefined, latestUpdatedAt || undefined); diff --git a/src/services/supabase-client.ts b/src/services/supabase-client.ts index 171629c..1e52576 100644 --- a/src/services/supabase-client.ts +++ b/src/services/supabase-client.ts @@ -579,6 +579,66 @@ export async function loadScarsWithEmbeddings( } } +/** + * Cheap fingerprint of the learnings the local index is built from (GIT-98). + * + * One request, one row: `Prefer: count=exact` puts the total in Content-Range + * ("0-0/273") and the single row carries the newest updated_at. A few hundred + * bytes, versus megabytes for the bulk load it lets a warm start skip. + * + * The filters MUST stay identical to loadScarsWithEmbeddings, or the + * fingerprint describes a different set from the one that was cached. + * + * Returns count -1 on any failure; callers treat that as "unknown", never as a match. + */ +export async function getLearningsFingerprint(): Promise<{ count: number; latestUpdatedAt: string | null }> { + if (!isConfigured()) return { count: -1, latestUpdatedAt: null }; + + try { + const url = new URL(`${SUPABASE_REST_URL}/${getTableName("learnings")}`); + url.searchParams.set("select", "updated_at"); + url.searchParams.set("learning_type", "in.(scar,pattern,win,anti_pattern)"); + url.searchParams.set("is_active", "eq.true"); + url.searchParams.set("order", "updated_at.desc"); + url.searchParams.set("limit", "1"); + + const response = await fetch(url.toString(), { + method: "GET", + headers: { + apikey: SUPABASE_KEY, + Authorization: `Bearer ${SUPABASE_KEY}`, + "Accept-Profile": "public", + Prefer: "count=exact", + }, + signal: AbortSignal.timeout(10_000), + }); + + if (!response.ok) { + console.error(`[supabase-direct] Fingerprint request failed: ${response.status}`); + return { count: -1, latestUpdatedAt: null }; + } + + // Content-Range: "0-0/273", or "*/0" for an empty set. + const range = response.headers.get("content-range") || ""; + const total = Number(range.split("/")[1]); + if (!Number.isInteger(total) || total < 0) { + console.error(`[supabase-direct] Fingerprint: unparseable Content-Range "${range}"`); + return { count: -1, latestUpdatedAt: null }; + } + + const rows = (await response.json()) as Array<{ updated_at?: string }>; + return { count: total, latestUpdatedAt: rows[0]?.updated_at ?? null }; + } catch (error) { + console.error("[supabase-direct] Fingerprint request error:", error instanceof Error ? error.message : error); + return { count: -1, latestUpdatedAt: null }; + } +} + +/** The configured Supabase URL, for keying per-store caches. Empty when unconfigured. */ +export function getSupabaseUrl(): string { + return SUPABASE_URL || ""; +} + /** * Scar search with severity weighting * diff --git a/src/services/vector-disk-cache.ts b/src/services/vector-disk-cache.ts new file mode 100644 index 0000000..91853b9 --- /dev/null +++ b/src/services/vector-disk-cache.ts @@ -0,0 +1,235 @@ +/** + * Vector Disk Cache (GIT-98) + * + * In local search mode every server process used to download every learning + * with its 1536-dim embedding at startup and throw it away at exit. That is a + * fair trade for one long interactive session and the worst possible trade for + * short-lived processes — sub-agents, fan-outs, CI, cron — which are now the + * common case. A customer exhausted a Supabase egress allowance this way. + * + * This module makes a cold process cheap by default: + * + * 1. Ask the store for a FINGERPRINT — row count + newest updated_at. One + * request, one row, a few hundred bytes (PostgREST `Prefer: count=exact`). + * 2. If a cache file with the same fingerprint exists, load vectors from disk. + * 3. Otherwise download as before and rewrite the cache. + * + * ## What the fingerprint can and cannot see + * + * It detects inserts, deactivations (count changes) and any edit that bumps + * updated_at. It does NOT detect an in-place edit that leaves updated_at and + * the row count unchanged. setup.sql maintains updated_at by trigger, so that + * requires a store whose trigger is missing. `cache-flush` always bypasses the + * disk cache, so there is a manual way out. + * + * ## Failure posture + * + * Every failure here degrades to the old behaviour (download), never to stale + * or empty results. An unreadable, corrupt, mismatched or foreign cache file is + * a miss. A failed fingerprint request is a miss. A failed write is logged and + * ignored. The cache is an optimisation and must never be load-bearing. + * + * ## Concurrency + * + * Writes go to a temp file and are renamed into place, so a reader sees either + * the old complete file or the new complete file, never a partial one. + * + * A fan-out starts N processes at the same instant, and any write to the store + * invalidates the cache for all of them at once. Without coordination that is N + * identical downloads — the exact cost this module exists to remove. So a miss + * is single-flighted across processes: one LEADER takes an O_EXCL lock file and + * downloads; FOLLOWERS wait for the cache to appear and read it. A follower + * that waits too long, or finds the leader's lock stale, downloads for itself. + * Waiting is async (never blocks the event loop) and bounded, so the worst case + * is today's behaviour plus a bounded delay — never a hang. + */ + +import * as fs from "fs"; +import * as path from "path"; +import * as crypto from "crypto"; +import { getGitmemDir } from "./gitmem-dir.js"; + +/** Bump when the on-disk shape changes; an older file is then a miss. */ +export const VECTOR_CACHE_FORMAT = 1; + +export interface StoreFingerprint { + /** Active learnings of the indexed types, as counted by the store. */ + count: number; + /** Newest updated_at among them, or null for an empty store. */ + latestUpdatedAt: string | null; +} + +interface VectorCacheFile { + format: number; + /** Identifies the store + table, so switching stores can never serve foreign rows. */ + storeKey: string; + fingerprint: StoreFingerprint; + savedAt: string; + rows: T[]; +} + +export type CacheReadResult = + | { hit: true; rows: T[]; bytesOnDisk: number } + | { hit: false; reason: string }; + +/** Opt-out: GITMEM_VECTOR_DISK_CACHE=0 (or "false") restores download-every-start. */ +export function isVectorDiskCacheEnabled(): boolean { + const v = process.env.GITMEM_VECTOR_DISK_CACHE; + return !(v === "0" || v === "false"); +} + +/** + * A stable, non-reversible key for (store URL, table). Hashed so the cache file + * name and contents never carry the project URL in clear. + */ +export function computeStoreKey(supabaseUrl: string, table: string): string { + return crypto.createHash("sha256").update(`${supabaseUrl}\n${table}`).digest("hex").slice(0, 16); +} + +export function getVectorCachePath(storeKey: string): string { + return path.join(getGitmemDir(), "cache", `learnings-vectors-${storeKey}.json`); +} + +export function fingerprintsMatch(a: StoreFingerprint, b: StoreFingerprint): boolean { + return a.count === b.count && a.latestUpdatedAt === b.latestUpdatedAt; +} + +/** + * Read the cache if, and only if, it was written for this store and this + * fingerprint. Anything else is a miss with a reason (logged by the caller). + */ +export function readVectorCache(storeKey: string, expected: StoreFingerprint): CacheReadResult { + if (!isVectorDiskCacheEnabled()) return { hit: false, reason: "disabled by GITMEM_VECTOR_DISK_CACHE" }; + + // A failed fingerprint request reports count -1. Never match on it: two + // failures would otherwise "agree" and serve whatever is on disk. + if (expected.count < 0) return { hit: false, reason: "store fingerprint unavailable" }; + + const file = getVectorCachePath(storeKey); + let raw: string; + try { + if (!fs.existsSync(file)) return { hit: false, reason: "no cache file" }; + raw = fs.readFileSync(file, "utf-8"); + } catch (error) { + return { hit: false, reason: `unreadable: ${error instanceof Error ? error.message : String(error)}` }; + } + + let parsed: VectorCacheFile; + try { + parsed = JSON.parse(raw) as VectorCacheFile; + } catch { + return { hit: false, reason: "corrupt JSON" }; + } + + if (!parsed || typeof parsed !== "object") return { hit: false, reason: "corrupt shape" }; + if (parsed.format !== VECTOR_CACHE_FORMAT) return { hit: false, reason: `format ${parsed.format} != ${VECTOR_CACHE_FORMAT}` }; + if (parsed.storeKey !== storeKey) return { hit: false, reason: "written for a different store" }; + if (!Array.isArray(parsed.rows)) return { hit: false, reason: "corrupt rows" }; + if (!parsed.fingerprint || !fingerprintsMatch(parsed.fingerprint, expected)) { + return { hit: false, reason: "store changed since cache was written" }; + } + + return { hit: true, rows: parsed.rows, bytesOnDisk: Buffer.byteLength(raw) }; +} + +/** + * Persist rows under the fingerprint that was observed BEFORE they were + * downloaded. If the store changed mid-download the file is merely stale and + * the next start misses and reloads — the safe direction. + */ +export function writeVectorCache(storeKey: string, fingerprint: StoreFingerprint, rows: T[]): boolean { + if (!isVectorDiskCacheEnabled()) return false; + if (fingerprint.count < 0) return false; // never persist under an unknown fingerprint + + const file = getVectorCachePath(storeKey); + const tmp = `${file}.${process.pid}.${Date.now()}.tmp`; + try { + fs.mkdirSync(path.dirname(file), { recursive: true }); + const payload: VectorCacheFile = { + format: VECTOR_CACHE_FORMAT, + storeKey, + fingerprint, + savedAt: new Date().toISOString(), + rows, + }; + fs.writeFileSync(tmp, JSON.stringify(payload), { mode: 0o600 }); + fs.renameSync(tmp, file); + return true; + } catch (error) { + console.error("[vector-cache] Failed to write cache (non-fatal):", error instanceof Error ? error.message : error); + try { if (fs.existsSync(tmp)) fs.unlinkSync(tmp); } catch { /* best effort */ } + return false; + } +} + +// --------------------------------------------------------------------------- +// Cross-process single-flight for cache misses +// --------------------------------------------------------------------------- + +/** A leader that has held the lock this long is presumed dead. A bulk load has a 15-30s budget. */ +const LOCK_STALE_MS = 60_000; +/** How long a follower waits for the leader before downloading for itself. */ +const FOLLOWER_WAIT_MS = 25_000; +const FOLLOWER_POLL_MS = 150; + +function lockPathFor(storeKey: string): string { + return `${getVectorCachePath(storeKey)}.lock`; +} + +/** Try to become the downloader. Returns a release function, or null if someone else is. */ +export function tryAcquireDownloadLock(storeKey: string): (() => void) | null { + if (!isVectorDiskCacheEnabled()) return () => {}; + const lock = lockPathFor(storeKey); + try { + fs.mkdirSync(path.dirname(lock), { recursive: true }); + } catch { + return () => {}; // cannot coordinate — behave as a lone downloader + } + + for (let attempt = 0; attempt < 2; attempt++) { + try { + const fd = fs.openSync(lock, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY, 0o600); + fs.writeSync(fd, JSON.stringify({ pid: process.pid, acquired_at: new Date().toISOString() })); + fs.closeSync(fd); + return () => { try { fs.unlinkSync(lock); } catch { /* already gone */ } }; + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "EEXIST") return () => {}; // odd fs — lone downloader + // Held by someone. Break it only if its holder is presumed dead, then retry once. + try { + if (Date.now() - fs.statSync(lock).mtimeMs > LOCK_STALE_MS) { + fs.unlinkSync(lock); + continue; + } + } catch { /* vanished between calls — retry */ continue; } + return null; + } + } + return null; +} + +/** + * Follower path: wait for the leader's cache to land. Resolves with the rows on + * a hit, or null when the caller should download for itself (leader gone, + * leader failed, or the wait budget ran out). + */ +export async function waitForLeaderCache( + storeKey: string, + expected: StoreFingerprint, + waitMs: number = FOLLOWER_WAIT_MS +): Promise & { hit: true } | null> { + const lock = lockPathFor(storeKey); + const deadline = Date.now() + waitMs; + + while (Date.now() < deadline) { + await new Promise((r) => setTimeout(r, FOLLOWER_POLL_MS)); + + const lockHeld = fs.existsSync(lock); + const cached = readVectorCache(storeKey, expected); + if (cached.hit) return cached; + + // Lock released and still no usable cache: the leader failed, or it cached a + // newer fingerprint than the one we observed. Either way, stop waiting. + if (!lockHeld) return null; + } + return null; +} diff --git a/tests/unit/services/vector-disk-cache.test.ts b/tests/unit/services/vector-disk-cache.test.ts new file mode 100644 index 0000000..b87d965 --- /dev/null +++ b/tests/unit/services/vector-disk-cache.test.ts @@ -0,0 +1,214 @@ +/** + * GIT-98: disk-backed vector cache. + * + * The contract under test: every failure is a MISS (download as before), never + * stale or foreign rows. Real filesystem, isolated GITMEM_DIR per test. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; + +let dir: string; + +async function load() { + vi.resetModules(); + return import("../../../src/services/vector-disk-cache.js"); +} + +const FP = { count: 3, latestUpdatedAt: "2026-09-20T15:25:16.480033+00:00" }; +const ROWS = [ + { id: "a", title: "one", embedding: [0.1, 0.2, 0.3] }, + { id: "b", title: "two", embedding: [0.4, 0.5, 0.6] }, + { id: "c", title: "three", embedding: [0.7, 0.8, 0.9] }, +]; + +describe("vector-disk-cache (GIT-98)", () => { + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), "gitmem-vcache-")); + process.env.GITMEM_DIR = dir; + delete process.env.GITMEM_VECTOR_DISK_CACHE; + }); + + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); + delete process.env.GITMEM_DIR; + delete process.env.GITMEM_VECTOR_DISK_CACHE; + }); + + it("round-trips rows when store and fingerprint match", async () => { + const c = await load(); + const key = c.computeStoreKey("https://x.supabase.co", "gitmem_learnings"); + expect(c.writeVectorCache(key, FP, ROWS)).toBe(true); + + const r = c.readVectorCache(key, FP); + expect(r.hit).toBe(true); + if (r.hit) { + expect(r.rows).toEqual(ROWS); + expect(r.bytesOnDisk).toBeGreaterThan(0); + } + }); + + it("misses when the row count changed (insert or deactivation)", async () => { + const c = await load(); + const key = c.computeStoreKey("https://x.supabase.co", "gitmem_learnings"); + c.writeVectorCache(key, FP, ROWS); + const r = c.readVectorCache(key, { ...FP, count: 4 }); + expect(r).toEqual({ hit: false, reason: "store changed since cache was written" }); + }); + + it("misses when the newest updated_at changed (edit)", async () => { + const c = await load(); + const key = c.computeStoreKey("https://x.supabase.co", "gitmem_learnings"); + c.writeVectorCache(key, FP, ROWS); + const r = c.readVectorCache(key, { ...FP, latestUpdatedAt: "2026-09-21T00:00:00+00:00" }); + expect(r.hit).toBe(false); + }); + + it("never serves another store's rows", async () => { + const c = await load(); + const mine = c.computeStoreKey("https://mine.supabase.co", "gitmem_learnings"); + const theirs = c.computeStoreKey("https://theirs.supabase.co", "gitmem_learnings"); + expect(mine).not.toBe(theirs); + c.writeVectorCache(mine, FP, ROWS); + expect(c.readVectorCache(theirs, FP).hit).toBe(false); + + // Same URL, different table prefix is a different store too. + const prefixed = c.computeStoreKey("https://mine.supabase.co", "orchestra_learnings"); + expect(prefixed).not.toBe(mine); + }); + + it("refuses a file that claims a different storeKey, even at the right path", async () => { + const c = await load(); + const key = c.computeStoreKey("https://x.supabase.co", "gitmem_learnings"); + c.writeVectorCache(key, FP, ROWS); + const file = c.getVectorCachePath(key); + const tampered = JSON.parse(fs.readFileSync(file, "utf-8")); + tampered.storeKey = "0000000000000000"; + fs.writeFileSync(file, JSON.stringify(tampered)); + expect(c.readVectorCache(key, FP)).toEqual({ hit: false, reason: "written for a different store" }); + }); + + it("an unknown fingerprint (count -1) never matches and is never persisted", async () => { + const c = await load(); + const key = c.computeStoreKey("https://x.supabase.co", "gitmem_learnings"); + const unknown = { count: -1, latestUpdatedAt: null }; + + expect(c.writeVectorCache(key, unknown, ROWS)).toBe(false); + expect(fs.existsSync(c.getVectorCachePath(key))).toBe(false); + + // Even if a file with an identical unknown fingerprint somehow exists, two failures must not "agree". + fs.mkdirSync(path.dirname(c.getVectorCachePath(key)), { recursive: true }); + fs.writeFileSync(c.getVectorCachePath(key), JSON.stringify({ + format: c.VECTOR_CACHE_FORMAT, storeKey: key, fingerprint: unknown, savedAt: "x", rows: ROWS, + })); + expect(c.readVectorCache(key, unknown)).toEqual({ hit: false, reason: "store fingerprint unavailable" }); + }); + + it("treats corrupt, truncated and wrong-format files as misses", async () => { + const c = await load(); + const key = c.computeStoreKey("https://x.supabase.co", "gitmem_learnings"); + const file = c.getVectorCachePath(key); + fs.mkdirSync(path.dirname(file), { recursive: true }); + + fs.writeFileSync(file, '{"format":1,"storeKey":"' + key + '","rows":[{"id":"a"'); + expect(c.readVectorCache(key, FP)).toEqual({ hit: false, reason: "corrupt JSON" }); + + fs.writeFileSync(file, "null"); + expect(c.readVectorCache(key, FP).hit).toBe(false); + + fs.writeFileSync(file, JSON.stringify({ format: 999, storeKey: key, fingerprint: FP, rows: ROWS })); + expect(c.readVectorCache(key, FP).hit).toBe(false); + + fs.writeFileSync(file, JSON.stringify({ format: c.VECTOR_CACHE_FORMAT, storeKey: key, fingerprint: FP, rows: "nope" })); + expect(c.readVectorCache(key, FP)).toEqual({ hit: false, reason: "corrupt rows" }); + }); + + it("GITMEM_VECTOR_DISK_CACHE=0 disables both read and write", async () => { + const c = await load(); + const key = c.computeStoreKey("https://x.supabase.co", "gitmem_learnings"); + c.writeVectorCache(key, FP, ROWS); + + process.env.GITMEM_VECTOR_DISK_CACHE = "0"; + expect(c.readVectorCache(key, FP).hit).toBe(false); + fs.rmSync(c.getVectorCachePath(key)); + expect(c.writeVectorCache(key, FP, ROWS)).toBe(false); + expect(fs.existsSync(c.getVectorCachePath(key))).toBe(false); + }); + + it("leaves no temp files behind and writes owner-only", async () => { + const c = await load(); + const key = c.computeStoreKey("https://x.supabase.co", "gitmem_learnings"); + c.writeVectorCache(key, FP, ROWS); + const cacheDir = path.dirname(c.getVectorCachePath(key)); + expect(fs.readdirSync(cacheDir).filter((f) => f.endsWith(".tmp"))).toEqual([]); + if (process.platform !== "win32") { + expect(fs.statSync(c.getVectorCachePath(key)).mode & 0o777).toBe(0o600); + } + }); + + it("the cache file does not contain the Supabase URL in clear", async () => { + const c = await load(); + const key = c.computeStoreKey("https://secret-project-ref.supabase.co", "gitmem_learnings"); + c.writeVectorCache(key, FP, ROWS); + const file = c.getVectorCachePath(key); + expect(path.basename(file)).not.toContain("secret-project-ref"); + expect(fs.readFileSync(file, "utf-8")).not.toContain("secret-project-ref"); + }); + + describe("single-flight download lock", () => { + it("only one caller becomes leader; release lets the next one in", async () => { + const c = await load(); + const key = c.computeStoreKey("https://x.supabase.co", "gitmem_learnings"); + const release = c.tryAcquireDownloadLock(key); + expect(release).toBeTypeOf("function"); + expect(c.tryAcquireDownloadLock(key)).toBeNull(); + release!(); + const again = c.tryAcquireDownloadLock(key); + expect(again).toBeTypeOf("function"); + again!(); + expect(fs.existsSync(`${c.getVectorCachePath(key)}.lock`)).toBe(false); + }); + + it("breaks a lock whose holder is presumed dead", async () => { + const c = await load(); + const key = c.computeStoreKey("https://x.supabase.co", "gitmem_learnings"); + c.tryAcquireDownloadLock(key); // leader "crashes" without releasing + const lock = `${c.getVectorCachePath(key)}.lock`; + const old = new Date(Date.now() - 5 * 60_000); + fs.utimesSync(lock, old, old); + const release = c.tryAcquireDownloadLock(key); + expect(release).toBeTypeOf("function"); + release!(); + }); + + it("a follower receives the leader's rows once the cache lands", async () => { + const c = await load(); + const key = c.computeStoreKey("https://x.supabase.co", "gitmem_learnings"); + const release = c.tryAcquireDownloadLock(key)!; + const follower = c.waitForLeaderCache(key, FP, 5_000); + setTimeout(() => { c.writeVectorCache(key, FP, ROWS); release(); }, 300); + const got = await follower; + expect(got?.hit).toBe(true); + expect(got?.rows).toEqual(ROWS); + }); + + it("a follower gives up (null) when the leader releases without a usable cache", async () => { + const c = await load(); + const key = c.computeStoreKey("https://x.supabase.co", "gitmem_learnings"); + const release = c.tryAcquireDownloadLock(key)!; + const follower = c.waitForLeaderCache(key, FP, 5_000); + setTimeout(() => release(), 300); // leader failed: no cache written + expect(await follower).toBeNull(); + }); + + it("a follower's wait is bounded when the leader hangs", async () => { + const c = await load(); + const key = c.computeStoreKey("https://x.supabase.co", "gitmem_learnings"); + c.tryAcquireDownloadLock(key); // never released, never writes + const t0 = Date.now(); + expect(await c.waitForLeaderCache(key, FP, 600)).toBeNull(); + expect(Date.now() - t0).toBeLessThan(2_000); + }); + }); +}); From d6a5ca0bc778f1bf42900e53ecbd57ebab4c1288 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 20 Sep 2026 20:38:48 -0700 Subject: [PATCH 3/7] test: blank-Supabase venue driver with an expected-failures gate (GIT-83) Drives the built MCP server over stdio against a Supabase project that has schema/setup.sql and nothing else: no edge functions, which is what every customer project looks like. Pro tier, with an isolated HOME and GITMEM_DIR per run. - flow: session_start -> list_threads -> recall -> confirm_scars (with relevance) -> create_thread -> list_threads -> session_close -> health. Seeds scars, a prior session and remote threads, plus a local threads.json that deliberately differs from the remote table. - egress: bytes received on cold and warm starts and around two session_starts (GIT-98). --usage N seeds scar usage so refresh_scar_behavioral_scores() has work to do. - blank-supabase-netlog.mjs is preloaded into the server and logs every fetch (method, path, status, body bytes). No request bodies or headers are logged. - Gate: exits 1 on any 4xx/5xx from /rest/v1/ or /functions/v1/ that is not on EXPECTED_FAILURES. Each entry names the ticket that removes it: GIT-73, GIT-105, GIT-106, GIT-84, GIT-97. - Credentials come only from a VENUE_ENV file outside the repo. Config problems exit 2 with a plain message. Production (cjptxyezuxdiinufgrrm), or any URL that is not https://.supabase.co, is refused before any request. - Embeddings come from a local fake Ollama endpoint, so no embedding API key is needed. Co-Authored-By: Claude Opus 5 --- tests/e2e/blank-supabase-netlog.mjs | 42 +++ tests/e2e/blank-supabase.mjs | 476 ++++++++++++++++++++++++++++ 2 files changed, 518 insertions(+) create mode 100644 tests/e2e/blank-supabase-netlog.mjs create mode 100644 tests/e2e/blank-supabase.mjs diff --git a/tests/e2e/blank-supabase-netlog.mjs b/tests/e2e/blank-supabase-netlog.mjs new file mode 100644 index 0000000..8010e9d --- /dev/null +++ b/tests/e2e/blank-supabase-netlog.mjs @@ -0,0 +1,42 @@ +/** + * Preload for tests/e2e/blank-supabase.mjs (loaded via NODE_OPTIONS=--import). + * + * Wraps globalThis.fetch in the gitmem server process and appends one JSON line + * per request to $GITMEM_NETLOG: method, host, path, status, and the size of the + * response body as the process received it (decoded — an upper bound on wire bytes + * when the response was gzip'd). Request bodies are never logged; auth headers + * are never logged. + */ +import { appendFileSync } from "node:fs"; + +const LOG = process.env.GITMEM_NETLOG; +const realFetch = globalThis.fetch; + +if (LOG && realFetch) { + globalThis.fetch = async function netlogFetch(input, init) { + const url = new URL(typeof input === "string" ? input : input.url ?? String(input)); + const method = (init?.method || (typeof input === "object" && input.method) || "GET").toUpperCase(); + const started = Date.now(); + let response; + try { + response = await realFetch(input, init); + } catch (err) { + appendFileSync(LOG, JSON.stringify({ t: started, method, host: url.host, path: url.pathname, status: 0, bytes: 0, error: String(err).slice(0, 200) }) + "\n"); + throw err; + } + const contentLength = response.headers.get("content-length"); + const encoding = response.headers.get("content-encoding"); + response + .clone() + .arrayBuffer() + .then((buf) => { + appendFileSync(LOG, JSON.stringify({ + t: started, ms: Date.now() - started, method, host: url.host, path: url.pathname, + query: url.search.slice(0, 160), status: response.status, bytes: buf.byteLength, + content_length: contentLength ? Number(contentLength) : null, encoding, + }) + "\n"); + }) + .catch(() => {}); + return response; + }; +} diff --git a/tests/e2e/blank-supabase.mjs b/tests/e2e/blank-supabase.mjs new file mode 100644 index 0000000..a3a37bb --- /dev/null +++ b/tests/e2e/blank-supabase.mjs @@ -0,0 +1,476 @@ +#!/usr/bin/env node +/** + * Blank-Supabase venue driver (GIT-97 / GIT-98). + * + * Runs the BUILT server (dist/index.js) over stdio, Pro tier, against a Supabase + * project that has schema/setup.sql and nothing else — no edge functions — which + * is what every customer project looks like. Each run gets an isolated HOME and + * GITMEM_DIR. Every fetch the server makes is logged by blank-supabase-netlog.mjs. + * + * Embeddings come from a local fake Ollama endpoint (deterministic bag-of-words + * hashing into 1536 dims), so no embedding API key is needed and all measured + * network traffic is venue traffic. + * + * Usage (build first: npm run build): + * VENUE_ENV=/path/outside/repo/.env node tests/e2e/blank-supabase.mjs flow --label main --out + * VENUE_ENV=/path/outside/repo/.env node tests/e2e/blank-supabase.mjs egress --label pr31 --out [--rows 250] [--usage 5] + * + * flow session_start -> list_threads -> recall -> confirm_scars -> create_thread + * -> list_threads -> session_close -> health, with a local threads.json that + * deliberately differs from the remote thread table. + * egress bytes received from the venue on a cold start, a warm start, and around + * two session_starts (GIT-98 disk cache). --usage N seeds N scars with + * >=3 usage rows so refresh_scar_behavioral_scores() has work to do. + * + * VENUE_ENV is a KEY=VALUE file OUTSIDE the repo defining SUPABASE_URL, + * SUPABASE_SERVICE_ROLE_KEY and VENUE_REF. Credentials are never committed. + * The driver WIPES the gitmem tables of that project before seeding. It refuses + * to run unless SUPABASE_URL is exactly https://.supabase.co and the + * ref is not on the deny list below. + * + * Exit status: 0 = every 4xx/5xx from /rest/v1/ or /functions/v1/ is on + * EXPECTED_FAILURES; 1 = at least one unexpected failure; 2 = usage/config error. + */ +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; +import { createServer } from "node:http"; +import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, dirname, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const DENY_REFS = new Set(["cjptxyezuxdiinufgrrm"]); // production GitMem — never touch + +/** + * Known venue failures. Any other 4xx/5xx from /rest/v1/ or /functions/v1/ + * fails the run. Each entry names the ticket that removes it — delete the entry + * when that ticket merges, so this list only shrinks. An entry allows a failure; + * it does not require one (e.g. the GIT-73 race is intermittent). + */ +const EXPECTED_FAILURES = [ + { ticket: "GIT-73", what: "metrics/session_start FK race", method: "POST", path: /^\/rest\/v1\/gitmem_query_metrics$/, status: 409 }, + { ticket: "GIT-105", what: "knowledge-triple thread id into a uuid column", method: "POST", path: /^\/rest\/v1\/knowledge_triples$/, status: 400 }, + { ticket: "GIT-106", what: "scar_enforcement_variants.active does not exist", method: "GET", path: /^\/rest\/v1\/scar_enforcement_variants$/, status: 400 }, + { ticket: "GIT-84", what: "unprefixed scar_usage table (remove once GIT-84 merges)", method: null, path: /^\/rest\/v1\/scar_usage$/, status: 404 }, + { ticket: "GIT-97", what: "ww-mcp edge function absent on customer projects (remove once PR #30 merges)", method: "POST", path: /^\/functions\/v1\/ww-mcp$/, status: 404 }, +]; +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO = resolve(HERE, "../.."); +const SERVER = join(REPO, "dist/index.js"); +const NETLOG_PRELOAD = join(HERE, "blank-supabase-netlog.mjs"); +const DIM = 1536; +const PROJECT = "blank-venue"; + +// ---------------------------------------------------------------- args / env +const mode = process.argv[2]; +const arg = (name, dflt) => { + const i = process.argv.indexOf(`--${name}`); + return i > 0 ? process.argv[i + 1] : dflt; +}; +const label = arg("label", "run"); +const outDir = resolve(arg("out", join(tmpdir(), `blank-supabase-${label}`))); +const seedRows = Number(arg("rows", "250")); +/** Configuration problems end the run with a plain message, never a stack trace. */ +function configError(message) { + console.error(`[driver] ${message}`); + process.exit(2); +} +if (!["flow", "egress"].includes(mode)) { + configError("usage: blank-supabase.mjs flow|egress --label --out [--rows N] [--usage N]"); +} + +const venueEnvPath = process.env.VENUE_ENV; +if (!venueEnvPath) { + configError("VENUE_ENV is not set. Point it at a KEY=VALUE file outside the repo with SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY and VENUE_REF."); +} +if (!existsSync(venueEnvPath)) configError(`VENUE_ENV file not found: ${venueEnvPath}`); +let venue; +try { + venue = Object.fromEntries( + readFileSync(venueEnvPath, "utf8") + .split("\n").filter((l) => l.includes("=") && !l.trimStart().startsWith("#")) + .map((l) => [l.slice(0, l.indexOf("=")).trim(), l.slice(l.indexOf("=") + 1).trim()]) + ); +} catch (err) { + configError(`Cannot read VENUE_ENV file ${venueEnvPath}: ${err.message}`); +} +const { SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, VENUE_REF } = venue; +const missingKeys = ["SUPABASE_URL", "SUPABASE_SERVICE_ROLE_KEY", "VENUE_REF"].filter((k) => !venue[k]); +if (missingKeys.length) configError(`VENUE_ENV file ${venueEnvPath} is missing: ${missingKeys.join(", ")}`); +if (DENY_REFS.has(VENUE_REF) || SUPABASE_URL !== `https://${VENUE_REF}.supabase.co`) { + configError(`Refusing to run against ${SUPABASE_URL} (ref ${VENUE_REF}). Only a disposable venue whose URL is https://.supabase.co is allowed; production is denied.`); +} +if (!existsSync(SERVER)) configError(`Build first (npm run build): ${SERVER} is missing`); +mkdirSync(outDir, { recursive: true }); +const VENUE_HOST = new URL(SUPABASE_URL).host; +console.log(`[driver] target venue: ${SUPABASE_URL} (ref ${VENUE_REF}) mode=${mode} label=${label}`); + +// ---------------------------------------------------------------- venue REST (driver-side seeding) +const rest = async (method, path, body, extraHeaders = {}) => { + const res = await fetch(`${SUPABASE_URL}/rest/v1/${path}`, { + method, + headers: { + apikey: SUPABASE_SERVICE_ROLE_KEY, + Authorization: `Bearer ${SUPABASE_SERVICE_ROLE_KEY}`, + "Content-Type": "application/json", + Prefer: "return=minimal", + ...extraHeaders, + }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + if (!res.ok) throw new Error(`${method} ${path} -> ${res.status} ${(await res.text()).slice(0, 300)}`); + return res; +}; +const WIPE_ORDER = [ + "gitmem_scar_usage", "gitmem_decisions", "gitmem_threads", "knowledge_triples", + "gitmem_query_metrics", "scar_enforcement_variants", "gitmem_learnings", "gitmem_sessions", +]; +async function wipeVenue() { + for (const t of WIPE_ORDER) await rest("DELETE", `${t}?id=not.is.null`); +} +async function count(table, filter = "") { + const res = await rest("HEAD", `${table}?select=id${filter}`, undefined, { Prefer: "count=exact", Range: "0-0" }); + return Number((res.headers.get("content-range") || "*/-1").split("/")[1]); +} + +// ---------------------------------------------------------------- fake embeddings (Ollama /api/embed shape) +function fnv(s) { + let h = 0x811c9dc5; + for (let i = 0; i < s.length; i++) { h ^= s.charCodeAt(i); h = Math.imul(h, 0x01000193) >>> 0; } + return h >>> 0; +} +function embed(text) { + const v = new Array(DIM).fill(0); + const words = String(text).toLowerCase().match(/[a-z0-9]+/g) || ["empty"]; + for (const w of words) for (let k = 0; k < 3; k++) { + const h = fnv(`${k}:${w}`); + v[h % DIM] += (h & 1) ? 1 : -1; + } + const n = Math.hypot(...v) || 1; + return v.map((x) => Math.round((x / n) * 1e6) / 1e6); +} +async function startFakeOllama() { + let calls = 0; + const server = createServer((req, res) => { + let body = ""; + req.on("data", (c) => (body += c)); + req.on("end", () => { + calls++; + const { input } = JSON.parse(body || "{}"); + const inputs = Array.isArray(input) ? input : [input]; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ model: "fake", embeddings: inputs.map(embed) })); + }); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + return { url: `http://127.0.0.1:${server.address().port}`, close: () => server.close(), calls: () => calls }; +} + +// ---------------------------------------------------------------- server process +function isolatedHome(tag) { + const home = mkdtempSync(join(tmpdir(), `gitmem-venue-${tag}-`)); + const gitmemDir = join(home, ".gitmem"); + mkdirSync(gitmemDir, { recursive: true }); + return { home, gitmemDir }; +} +const STRIP = /^(OPENAI_API_KEY|OPENROUTER_API_KEY|GITMEM_|SUPABASE_|OLLAMA_|NODE_OPTIONS$|CLAUDE_CODE_ENTRYPOINT$)/; +async function startServer({ home, gitmemDir, ollamaUrl, netlog, stderrSink }) { + const env = Object.fromEntries(Object.entries(process.env).filter(([k]) => !STRIP.test(k))); + Object.assign(env, { + HOME: home, + GITMEM_DIR: gitmemDir, + GITMEM_TIER: "pro", + SUPABASE_URL, + SUPABASE_SERVICE_ROLE_KEY, + GITMEM_EMBEDDING_PROVIDER: "ollama", + OLLAMA_URL: ollamaUrl, + GITMEM_OLLAMA_MODEL: "fake", + GITMEM_EMBEDDING_DIM: String(DIM), + GITMEM_DEFAULT_PROJECT: PROJECT, + GITMEM_NETLOG: netlog, + NODE_OPTIONS: `--import=${pathToFileURL(NETLOG_PRELOAD).href}`, + NO_COLOR: "1", + }); + const transport = new StdioClientTransport({ command: process.execPath, args: [SERVER], env, cwd: home, stderr: "pipe" }); + transport.stderr?.on("data", (d) => stderrSink.push(d.toString())); + const client = new Client({ name: "gitmem-blank-venue-driver", version: "1.0.0" }, { capabilities: {} }); + await client.connect(transport); + return { + call: async (name, args = {}) => { + const r = await client.callTool({ name, arguments: args }, undefined, { timeout: 120_000 }); + return (r.content || []).map((c) => c.text || "").join("\n"); + }, + close: async () => { try { await client.close(); } catch {} try { await transport.close(); } catch {} }, + }; +} +const readNet = (file) => (existsSync(file) ? readFileSync(file, "utf8").trim().split("\n").filter(Boolean).map((l) => JSON.parse(l)) : []); +async function waitQuiet(file, { minMs = 4000, quietMs = 4000, maxMs = 90_000 } = {}) { + const t0 = Date.now(); + let last = -1, lastChange = Date.now(); + while (Date.now() - t0 < maxMs) { + const n = readNet(file).length; + if (n !== last) { last = n; lastChange = Date.now(); } + if (Date.now() - t0 >= minMs && Date.now() - lastChange >= quietMs) return; + await new Promise((r) => setTimeout(r, 250)); + } +} +const venueReqs = (reqs) => reqs.filter((r) => r.host === VENUE_HOST); +const sumBytes = (reqs) => venueReqs(reqs).reduce((a, r) => a + (r.bytes || 0), 0); +const UUID = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi; + +// Every venue request from every server process in this run, for the failure check. +const allVenueRequests = []; +const expectedFor = (r) => EXPECTED_FAILURES.find((e) => + e.status === r.status && e.path.test(r.path) && (e.method === null || e.method === r.method)); +function checkFailures(reqs) { + const failures = reqs.filter((r) => r.status >= 400 && /^\/(rest|functions)\/v1\//.test(r.path)); + const unexpected = failures.filter((r) => !expectedFor(r)) + .map((r) => `${r.method} ${r.path}${r.query} -> ${r.status}`); + const expectedSeen = {}; + for (const r of failures) { + const e = expectedFor(r); + if (e) expectedSeen[`${e.ticket} ${r.method} ${r.path} ${r.status}`] = (expectedSeen[`${e.ticket} ${r.method} ${r.path} ${r.status}`] || 0) + 1; + } + return { unexpected, expected_seen: expectedSeen }; +} + +// ---------------------------------------------------------------- FLOW (runs 1 & 2) +async function flow() { + await wipeVenue(); + const now = Date.now(); + const iso = (msAgo) => new Date(now - msAgo).toISOString(); + + // A prior closed session, so there is a "last session" to load. + const priorSession = "11111111-1111-4111-8111-111111111111"; + await rest("POST", "gitmem_sessions", [{ + id: priorSession, session_title: "Seeded prior session", agent: "cli", project: PROJECT, + decisions: [], open_threads: [], closing_reflection: { what_worked: "seeded" }, + created_at: iso(3600_000), updated_at: iso(3600_000), + }]); + + // Learnings with embeddings, so recall has something to find. + const scars = [ + ["Supabase migrations must be dry-run before push", "Run db push --dry-run and read the plan before applying any migration to a shared database."], + ["Edge functions are not shipped to customer projects", "Anything under /functions/v1 exists only where someone deployed it; customer Supabase projects have none."], + ["Service role keys never go in logs", "Redact service role keys from stdout, stderr and CI logs; write them to a mode-600 env file."], + ["Verify deployment after merge", "Done is not deployed is not verified working: pull on the target, restart, and check it is running."], + ["Thread counts must agree across surfaces", "session_start and list_threads must read threads from the same source of truth."], + ]; + await rest("POST", "gitmem_learnings", scars.map(([title, description], i) => ({ + learning_type: "scar", title, description, severity: i < 2 ? "high" : "medium", + counter_arguments: ["You might think it is fine to skip — but it is not."], project: PROJECT, + embedding: JSON.stringify(embed(`${title} ${description}`)), is_active: true, + }))); + + // Remote threads (source of truth) — 3 open. + const remoteThreads = ["Remote thread A: rotate venue credentials", "Remote thread B: document blank-venue setup", "Remote thread C: follow up on per-row delta sync"]; + await rest("POST", "gitmem_threads", remoteThreads.map((text, i) => ({ + thread_id: `t-remote0${i + 1}`, text, status: "active", project: PROJECT, + source_session: priorSession, last_touched_at: iso(600_000 * (i + 1)), + }))); + + // Local threads.json that DIFFERS from the remote table: 2 local-only threads, none of the remote ones. + const { home, gitmemDir } = isolatedHome(`${label}-flow`); + const localThreads = [ + { id: "t-local001", text: "Stale local file: renew the TLS certificate on the staging box", status: "open", created_at: iso(86400_000) }, + { id: "t-local002", text: "Stale local file: benchmark ivfflat recall at ten thousand rows", status: "open", created_at: iso(86400_000) }, + ]; + writeFileSync(join(gitmemDir, "threads.json"), JSON.stringify(localThreads, null, 2)); + + const seeded = { + remote_threads_open: await count("gitmem_threads", "&status=eq.active"), + local_threads_json: localThreads.length, + learnings: await count("gitmem_learnings"), + sessions: await count("gitmem_sessions"), + }; + console.log("[driver] seeded:", seeded); + + const ollama = await startFakeOllama(); + const netlog = join(outDir, `netlog-${label}-flow.jsonl`); + writeFileSync(netlog, ""); + const stderr = []; + const transcript = []; + const srv = await startServer({ home, gitmemDir, ollamaUrl: ollama.url, netlog, stderrSink: stderr }); + const step = async (name, args) => { + const t0 = Date.now(); + let text, error; + try { text = await srv.call(name, args); } catch (e) { error = String(e); text = ""; } + transcript.push({ step: name, args, ms: Date.now() - t0, error, text }); + console.log(`[driver] ${name}: ${error ? "ERROR " + error : "ok"} (${Date.now() - t0}ms)`); + return text; + }; + + await waitQuiet(netlog, { minMs: 3000, quietMs: 3000 }); + const ss = await step("session_start", { project: PROJECT, agent_identity: "cli", force: true }); + const sessionId = (ss.match(UUID) || []).find((u) => u !== priorSession); + const ssThreadCount = Number((ss.match(/Threads \((\d+)\)/) || [])[1] ?? 0); + // Same-moment comparison: list_threads immediately after session_start, before anything writes a thread. + const lt0 = await step("list_threads", { project: PROJECT }); + const lt0Open = Number((lt0.match(/(\d+) open/) || [])[1] ?? -1); + + const rc = await step("recall", { plan: "Supabase migrations must be dry-run before push; verify deployment after merge, pull on the target, restart", project: PROJECT, match_count: 3 }); + // recall prints short ids ("id:448fede2"); resolve them to full UUIDs from the venue. + const allIds = await rest("GET", "gitmem_learnings?select=id", undefined, { Prefer: "" }).then((r) => r.json()); + const shortIds = [...new Set([...rc.matchAll(/id:([0-9a-f]{8})\b/g)].map((m) => m[1]))]; + const scarIds = shortIds.map((s) => allIds.find((r) => r.id.startsWith(s))?.id).filter(Boolean); + + const cs = await step("confirm_scars", { + confirmations: scarIds.map((id, i) => ({ + scar_id: id, + decision: i === 0 ? "APPLYING" : "N_A", + evidence: i === 0 + ? "Ran the dry-run first and read the migration plan before applying it to the blank venue project." + : "This scar concerns a different failure mode than the blank-venue verification being run here.", + relevance: i === 0 ? "high" : "low", + })), + }); + + const ct = await step("create_thread", { text: `Driver-created thread (${label}) ${new Date(now).toISOString()}` }); + const lt = await step("list_threads", { project: PROJECT }); + const ltOpen = Number((lt.match(/(\d+) open/) || [])[1] ?? -1); + + const sc = await step("session_close", { + session_id: sessionId, close_type: "standard", human_corrections: "none", + closing_reflection: { + what_broke: "nothing (driver)", what_took_longer: "n/a", do_differently: "n/a", what_worked: "driver", + wrong_assumption: "n/a", scars_applied: [], institutional_memory_items: "n/a", + collaborative_dynamic: "n/a", rapport_notes: "n/a", + }, + }); + await waitQuiet(netlog, { minMs: 2000, quietMs: 3000 }); + const health = await step("health", { failure_limit: 20 }); + await waitQuiet(netlog, { minMs: 1000, quietMs: 2000 }); + await srv.close(); + ollama.close(); + + const reqs = readNet(netlog); + const vr = venueReqs(reqs); + allVenueRequests.push(...vr); + const errText = stderr.join(""); + const failed = [...health.matchAll(/\((\d+) failed\)/g)].reduce((a, m) => a + Number(m[1]), 0); + const summary = { + label, mode, venue: SUPABASE_URL, seeded, session_id: sessionId, + session_start_thread_count: ssThreadCount, + list_threads_open_right_after_session_start: lt0Open, + list_threads_open_after_create_thread: ltOpen, + thread_counts_match: ssThreadCount === lt0Open, + session_start_matches_local_file: ssThreadCount === seeded.local_threads_json, + recall_scar_ids: scarIds.length, + remote_threads_after: await count("gitmem_threads"), + remote_sessions_after: await count("gitmem_sessions"), + remote_scar_usage_after: await count("gitmem_scar_usage"), + health_failed_total: failed, + health_text: health, + functions_v1_requests: vr.filter((r) => r.path.startsWith("/functions/v1/")).map((r) => `${r.method} ${r.path} -> ${r.status}`), + rest_requests: vr.length, + non_2xx_venue: vr.filter((r) => r.status < 200 || r.status >= 300).map((r) => `${r.method} ${r.path}${r.query} -> ${r.status}`), + stderr_has_mcp_404: /MCP HTTP error: 404/.test(errText), + stderr_has_failed_last_session: /\[session_start\] Failed to load last session/.test(errText), + stderr_error_lines: errText.split("\n").filter((l) => /error|fail|warn/i.test(l)).slice(0, 60), + embed_calls: ollama.calls(), + tool_errors: transcript.filter((s) => s.error || /^\{"error"/.test(s.text)).map((s) => `${s.step}: ${s.error || s.text.slice(0, 200)}`), + }; + writeFileSync(join(outDir, `stderr-${label}-flow.log`), errText); + writeFileSync(join(outDir, `transcript-${label}-flow.json`), JSON.stringify(transcript, null, 2)); + writeFileSync(join(outDir, `summary-${label}-flow.json`), JSON.stringify(summary, null, 2)); + return summary; +} + +// ---------------------------------------------------------------- EGRESS (run 3) +async function egress() { + await wipeVenue(); + const rows = []; + for (let i = 0; i < seedRows; i++) { + const title = `Egress learning ${i}: ${["deploy", "migration", "cache", "thread", "session"][i % 5]} behaviour ${i}`; + rows.push({ + learning_type: ["scar", "pattern", "win"][i % 3], title, + description: `Seeded learning ${i} for the GIT-98 egress measurement. `.repeat(3), + severity: "medium", project: PROJECT, is_active: true, + embedding: JSON.stringify(embed(title)), + }); + } + for (let i = 0; i < rows.length; i += 50) await rest("POST", "gitmem_learnings", rows.slice(i, i + 50)); + console.log(`[driver] seeded learnings: ${await count("gitmem_learnings")}`); + // --usage N: give N learnings >=3 recent gitmem_scar_usage rows each, the threshold at which + // refresh_scar_behavioral_scores() rewrites them (and bumps updated_at). A real Pro store with + // usage history looks like this; a store where usage never lands looks like --usage 0. + const usageScars = Number(arg("usage", "0")); + if (usageScars > 0) { + const ids = await rest("GET", `gitmem_learnings?select=id&learning_type=eq.scar&limit=${usageScars}`, undefined, { Prefer: "" }).then((r) => r.json()); + const surfaced = new Date(Date.now() - 86400_000).toISOString(); + await rest("POST", "gitmem_scar_usage", ids.flatMap(({ id }) => [0, 1, 2].map((k) => ({ + scar_id: id, agent: "cli", reference_type: k === 0 ? "none" : "acknowledged", surfaced_at: surfaced, + })))); + console.log(`[driver] seeded gitmem_scar_usage: ${await count("gitmem_scar_usage")} rows over ${ids.length} scars`); + } + + const { home, gitmemDir } = isolatedHome(`${label}-egress`); // shared across all starts: the disk cache must persist + const ollama = await startFakeOllama(); + const starts = []; + const run = async (name, { sessionStart = false } = {}) => { + const netlog = join(outDir, `netlog-${label}-${name}.jsonl`); + writeFileSync(netlog, ""); + const stderr = []; + const srv = await startServer({ home, gitmemDir, ollamaUrl: ollama.url, netlog, stderrSink: stderr }); + await waitQuiet(netlog); + const startupBytes = sumBytes(readNet(netlog)); + const startupReqs = venueReqs(readNet(netlog)).length; + if (sessionStart) { + await srv.call("session_start", { project: PROJECT, agent_identity: "cli", force: true }); + await waitQuiet(netlog, { minMs: 2000 }); + } + await srv.close(); + const reqs = venueReqs(readNet(netlog)); + allVenueRequests.push(...reqs); + const errText = stderr.join(""); + writeFileSync(join(outDir, `stderr-${label}-${name}.log`), errText); + const s = { + start: name, + startup_bytes: startupBytes, + startup_requests: startupReqs, + total_bytes: sumBytes(reqs), + total_requests: reqs.length, + largest: [...reqs].sort((a, b) => b.bytes - a.bytes).slice(0, 4).map((r) => `${r.method} ${r.path}${r.query.slice(0, 80)} ${r.status} ${r.bytes}B${r.encoding ? " (" + r.encoding + ")" : ""}`), + cache_log: errText.split("\n").filter((l) => /Vector cache|Loaded \d+ learnings|bulk download|Loading ALL/.test(l)), + }; + starts.push(s); + console.log(`[driver] ${name}: startup ${startupBytes} B / ${startupReqs} req; total ${s.total_bytes} B / ${s.total_requests} req`); + return s; + }; + // Which learnings did a session_start rewrite? (refresh_scar_behavioral_scores bumps updated_at) + const maxUpdated = () => rest("GET", "gitmem_learnings?select=updated_at&order=updated_at.desc&limit=1", undefined, { Prefer: "" }) + .then((r) => r.json()).then((rows) => rows[0]?.updated_at ?? null); + const changedSince = (ts) => rest("GET", `gitmem_learnings?select=id,title,decay_multiplier,updated_at&updated_at=gt.${encodeURIComponent(ts)}&order=updated_at.desc`, undefined, { Prefer: "" }) + .then((r) => r.json()); + const rounds = []; + await run("start1-cold"); + await run("start2-warm"); + for (const round of [1, 2]) { + const before = await maxUpdated(); + await run(`start${round * 2 + 1}-with-session_start-${round}`, { sessionStart: true }); + const changed = await changedSince(before); + rounds.push({ + round, + max_updated_at_before: before, + max_updated_at_after: await maxUpdated(), + rows_rewritten: changed.map((r) => ({ id: r.id.slice(0, 8), title: r.title.slice(0, 40), decay_multiplier: r.decay_multiplier })), + }); + await run(`start${round * 2 + 2}-after-session_start-${round}`); + } + ollama.close(); + const summary = { label, mode, venue: SUPABASE_URL, seeded_learnings: seedRows, rounds, starts }; + writeFileSync(join(outDir, `summary-${label}-egress.json`), JSON.stringify(summary, null, 2)); + return summary; +} + +const result = mode === "flow" ? await flow() : await egress(); +const failureCheck = checkFailures(allVenueRequests); +writeFileSync(join(outDir, `failure-check-${label}-${mode}.json`), JSON.stringify(failureCheck, null, 2)); +console.log(JSON.stringify({ ...result, health_text: undefined, stderr_error_lines: undefined, failure_check: failureCheck }, null, 2)); +if (failureCheck.unexpected.length > 0) { + console.error(`[driver] FAIL: ${failureCheck.unexpected.length} unexpected venue failure(s) — not on EXPECTED_FAILURES:`); + for (const u of failureCheck.unexpected) console.error(` ${u}`); + process.exit(1); +} +console.error(`[driver] PASS: no unexpected venue failures (${Object.keys(failureCheck.expected_seen).length} known kinds seen)`); +process.exit(0); From 28366f666dacf6095aeb2739b676cbf01a662152 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 20 Sep 2026 20:51:22 -0700 Subject: [PATCH 4/7] test: ww-mcp 404 now fails the venue gate; enforce no edge calls and matching thread counts (GIT-97) PR #30 removes the ww-mcp dependency, so its expected-failure entry comes out and any ww-mcp 404 fails the run. The driver also fails on: - any request to /functions/v1/, whatever its status - a session_start thread count that differs from list_threads in flow mode Co-Authored-By: Claude Opus 5 --- tests/e2e/blank-supabase.mjs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/e2e/blank-supabase.mjs b/tests/e2e/blank-supabase.mjs index a3a37bb..bdedc4d 100644 --- a/tests/e2e/blank-supabase.mjs +++ b/tests/e2e/blank-supabase.mjs @@ -29,7 +29,9 @@ * ref is not on the deny list below. * * Exit status: 0 = every 4xx/5xx from /rest/v1/ or /functions/v1/ is on - * EXPECTED_FAILURES; 1 = at least one unexpected failure; 2 = usage/config error. + * EXPECTED_FAILURES, no request reached /functions/v1/ at all, and (flow) the + * session_start thread count equals list_threads; 1 = any of those violated; + * 2 = usage/config error. */ import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; @@ -52,7 +54,6 @@ const EXPECTED_FAILURES = [ { ticket: "GIT-105", what: "knowledge-triple thread id into a uuid column", method: "POST", path: /^\/rest\/v1\/knowledge_triples$/, status: 400 }, { ticket: "GIT-106", what: "scar_enforcement_variants.active does not exist", method: "GET", path: /^\/rest\/v1\/scar_enforcement_variants$/, status: 400 }, { ticket: "GIT-84", what: "unprefixed scar_usage table (remove once GIT-84 merges)", method: null, path: /^\/rest\/v1\/scar_usage$/, status: 404 }, - { ticket: "GIT-97", what: "ww-mcp edge function absent on customer projects (remove once PR #30 merges)", method: "POST", path: /^\/functions\/v1\/ww-mcp$/, status: 404 }, ]; const HERE = dirname(fileURLToPath(import.meta.url)); const REPO = resolve(HERE, "../.."); @@ -465,6 +466,15 @@ async function egress() { const result = mode === "flow" ? await flow() : await egress(); const failureCheck = checkFailures(allVenueRequests); +// Invariants the edge-function removal (GIT-97) guarantees, whatever the status code. +const edgeCalls = allVenueRequests.filter((r) => r.path.startsWith("/functions/v1/")) + .map((r) => `${r.method} ${r.path} -> ${r.status}`); +if (edgeCalls.length) failureCheck.unexpected.push(...edgeCalls.map((c) => `edge function called (GIT-97): ${c}`)); +if (mode === "flow" && !result.thread_counts_match) { + failureCheck.unexpected.push( + `thread panel mismatch (GIT-97): session_start ${result.session_start_thread_count} != list_threads ${result.list_threads_open_right_after_session_start}` + ); +} writeFileSync(join(outDir, `failure-check-${label}-${mode}.json`), JSON.stringify(failureCheck, null, 2)); console.log(JSON.stringify({ ...result, health_text: undefined, stderr_error_lines: undefined, failure_check: failureCheck }, null, 2)); if (failureCheck.unexpected.length > 0) { From 9282ca7c2b155db3bd554b2f69e59022b245d2bd Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 20 Sep 2026 20:55:20 -0700 Subject: [PATCH 5/7] docs: draft CHANGELOG entry for Release A (GIT-97, GIT-98) Unversioned: the version is chosen at release (GIT-94). Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6547d41..cf07bbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,64 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] + + +**No schema change. You do not need to re-run `setup.sql`.** + +This release is about Pro stores, meaning your own Supabase project. It covers two things: gitmem now +works fully on a project that has only `setup.sql` in it, which is every customer project. And it +no longer downloads your whole memory every time a process starts. + +### Fixed + +- **gitmem no longer depends on an edge function your project doesn't have.** Some reads went + through `/functions/v1/ww-mcp`, an edge function that exists only on nTEG's own Supabase project + and was never shipped with the package. On your project every one of those reads returned 404, + and gitmem quietly carried on without the data. Every read and write now goes directly to your + project's REST API (PostgREST). What you'll notice: + - **The `session_start` thread panel now matches `list_threads`.** It used to fall back to the + local `threads.json` file, which can be stale, while `list_threads` read your Supabase table, + so the two could show different threads. Both now show the same threads from Supabase. A failed + session-history read also no longer throws away a successful thread read. + - **Your last session and recent decisions load at `session_start` again.** Before, that step + failed silently with `Failed to load last session`. + - Filter values containing a `.`, such as a title or a version string, are now always treated as + data. (GIT-97) + +### Added + +- **An on-disk copy of your embedding index, so short-lived processes stop re-downloading it.** In + local search mode, every gitmem process downloaded every learning with its 1536-number embedding + at startup and threw it away on exit. Sub-agents, parallel workers and CI jobs each paid that cost + again, and one user used up a Supabase egress allowance this way. + - **How it works now:** each start asks your store one small question, how many learnings it holds + and when the newest changed (about 100 bytes). If that matches the copy on disk, gitmem loads + from disk and skips the download. With 250 learnings, a second start went from about 920 KB + received to about 100 bytes. + - **Staleness:** any mismatch, corruption or unreadable file counts as a miss, and gitmem + downloads fresh. It never serves a copy it can't confirm is current. + - **Parallel starts:** when several processes start at once, one of them downloads and the others + wait for its copy, instead of all downloading at the same time. + - **`cache-flush`** skips the disk copy and rewrites it. + - **Where it lives:** `/cache/learnings-vectors-.json`. The gitmem dir is + `~/.gitmem`, or `GITMEM_DIR` if you set it. `` is derived from your Supabase URL, so + different stores never share a file. + - **What it contains:** a local copy of your learnings, including titles, descriptions and + embeddings. The file is written with mode `0600`, so only your user account can read it. + Deleting it is always safe; gitmem downloads again on the next start. + - **Opting out:** set `GITMEM_VECTOR_DISK_CACHE=0` (or `false`) to download on every start, as + before. (GIT-98) + +### Known issues + +- **A session that changes scar scores can still cause one re-download.** At `session_start`, gitmem + recalculates scar scores from their usage history. Any scar whose score changes gets a new + timestamp, and the next process start downloads the index again. Stores with no usage history + aren't affected. This goes away when per-row delta sync lands. +- **Scar usage and `confirm_scars` relevance aren't recorded on your project yet.** The usage write + still targets a table name that doesn't exist on customer stores. It's fixed separately (GIT-84) + and isn't part of this release. + ## [1.8.0] - 2026-08-09 **No one loses any information.** Nothing is deleted, nothing is moved, nothing is overwritten. From 8a1fa122985dc446ec69ea861fe0545e0ca2f615 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 20 Sep 2026 21:01:29 -0700 Subject: [PATCH 6/7] chore: clean dist before compiling in prepublishOnly tsc never deletes output whose source was removed, so a publish from a long-lived working tree shipped stale compiled files. The last pack included dist/commands/license.* and dist/tools/create-linear-issue.*, whose source is long gone. prepublishOnly now runs `rm -rf dist && tsc`. Co-Authored-By: Claude Opus 5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 53de28e..fbe36dc 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "test:all": "npm run test:unit && npm run test:smoke && npm run test:integration && npm run test:perf && npm run test:e2e", "test:watch": "vitest", "typecheck": "tsc --noEmit", - "prepublishOnly": "tsc", + "prepublishOnly": "rm -rf dist && tsc", "release-status": "bash scripts/release-status.sh" }, "dependencies": { From c14048e0eb13e71a1af4fa21fea645607134b58f Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 20 Sep 2026 21:02:17 -0700 Subject: [PATCH 7/7] chore: bump version to v1.9.0 Release A: ww-mcp removal (GIT-97) and the on-disk embedding index cache (GIT-98). No schema change. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 2 +- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf07bbd..3ffbf50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - +## [1.9.0] - 2026-09-20 **No schema change. You do not need to re-run `setup.sql`.** diff --git a/package-lock.json b/package-lock.json index 90405c9..9ce6c9b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "gitmem-mcp", - "version": "1.6.6", + "version": "1.9.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "gitmem-mcp", - "version": "1.6.6", + "version": "1.9.0", "license": "MIT", "dependencies": { "@modelcontextprotocol/sdk": "^1.0.0", diff --git a/package.json b/package.json index fbe36dc..05200c5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gitmem-mcp", - "version": "1.8.0", + "version": "1.9.0", "mcpName": "io.github.gitmem-dev/gitmem", "description": "Persistent learning memory for AI coding agents. Memory that compounds.", "type": "module",