Skip to content

fix: remote recall calls setup.sql's gitmem_scar_search (GIT-114) #50

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
3 commits merged into from
Sep 23, 2026
Merged

fix: remote recall calls setup.sql's gitmem_scar_search (GIT-114) #50

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4a6ad09
fix: remote recall calls setup.sql's gitmem_scar_search (GIT-114)
Sep 23, 2026
65cbba4
test: drift analyzer checks RPC names against v1.8.0's setup.sql (GIT…
Sep 23, 2026
ea7c6e8
test: venue flow recalls with GITMEM_SEARCH_MODE=remote and from a co…
Sep 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 97 additions & 94 deletions src/services/supabase-client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import type {
SupabaseSearchOptions,
} from "../types/index.js";
import { getCache } from "./cache.js";
import { getTableName } from "./tier.js";
import { getTableName, getTablePrefix } from "./tier.js";

// --- PostgREST Input Sanitization ---

Expand Down Expand Up @@ -159,61 +159,6 @@ export async function upsertRecord<T = unknown>(
return directUpsert<T>(table, data);
}

/**
* Semantic search across tables
*
* Generates an embedding for the query, then calls the match_<table>
* RPC function directly via PostgREST.
*/
export async function semanticSearch<T = unknown>(
options: SupabaseSearchOptions
): Promise<T[]> {
if (!isConfigured()) {
throw new Error("Supabase not configured");
}

const { query, match_count = 10 } = options;

// Generate embedding for the query
const { embed } = await import("./embedding.js");
const embedding = await embed(query);

if (!embedding) {
console.error("[semantic-search] No embedding provider configured — cannot run semantic search");
return [];
}

// GIT-93: same defect as scarSearch below — this built `${prefix}_semantic_search`,
// which exists under no prefix. The deployed function is match_<table>, and it
// takes similarity_threshold (unlike the _weighted variant), which is what the
// body already sends.
const rpcName = `match_${getTableName("learnings")}`;
const url = `${SUPABASE_URL}/rest/v1/rpc/${rpcName}`;

const response = await fetch(url, {
method: "POST",
headers: {
"Content-Type": "application/json",
apikey: SUPABASE_KEY,
Authorization: `Bearer ${SUPABASE_KEY}`,
},
body: JSON.stringify({
query_embedding: `[${embedding.join(",")}]`,
match_count,
similarity_threshold: 0.0,
}),
signal: AbortSignal.timeout(15_000),
});

if (!response.ok) {
const text = await response.text();
throw new Error(`Supabase RPC error: ${response.status} - ${text.slice(0, 200)}`);
}

const rows = (await response.json()) as T[];
return rows || [];
}

// ============================================================================
// DIRECT SUPABASE QUERIES
// ============================================================================
Expand Down Expand Up @@ -677,49 +622,107 @@ export async function scarSearch<T = unknown>(
return [];
}

// GIT-93: the RPC is named after the TABLE it searches, not after the prefix
// with a verb appended. This built `${prefix}_scar_search` — "orchestra_scar_search"
// under GITMEM_TABLE_PREFIX=orchestra_, and "gitmem_scar_search" by default.
// Neither exists: PostgREST exposes match_orchestra_learnings_weighted, and a
// survey of the deployed functions found no *_scar_search under any prefix. So
// this fallback returned PGRST202 on every call, on every deployment, since it
// was written — invisible because it is only reached while the local vector
// index is cold.
//
// _weighted is the right one of the pair: this function is documented as scar
// search WITH SEVERITY WEIGHTING, and it is the variant that returns
// decay_multiplier, which recall consumes. Note it takes match_threshold, not
// similarity_threshold — the unweighted variant takes the latter.
const rpcName = `match_${getTableName("learnings")}_weighted`;
const url = `${SUPABASE_URL}/rest/v1/rpc/${rpcName}`;
const rows = await callScarSearchRpc<T & { id: string; learning_type?: string }>(embedding, matchCount);
return (await withLearningColumns(rows)) as T[];
}

const response = await fetch(url, {
method: "POST",
headers: {
"Content-Type": "application/json",
apikey: SUPABASE_KEY,
Authorization: `Bearer ${SUPABASE_KEY}`,
},
body: JSON.stringify({
query_embedding: `[${embedding.join(",")}]`,
match_count: matchCount,
match_threshold: 0.0,
// project_filter is deliberately not sent. The primary path this falls back
// from is the unified CROSS-PROJECT vector cache, so filtering here would
// make the fallback return a different, narrower result set than the path
// it stands in for — a silent behaviour change on exactly the cold-start
// calls that are hardest to notice.
}),
signal: AbortSignal.timeout(15_000),
});
/**
* GIT-114: which scar-search function this store has.
*
* setup.sql (every version a customer can have) defines gitmem_scar_search,
* taking similarity_threshold. It does NOT define match_<table>_weighted —
* GIT-93 renamed the call to that after surveying nTEG's store, which has it,
* so every customer's cold-index recall got PGRST202.
*
* Order of preference:
* - default prefix: gitmem_scar_search (setup.sql), then
* match_gitmem_learnings_weighted for a store that predates or lacks it;
* - any other prefix: only match_<table>_weighted — gitmem_scar_search reads
* gitmem_learnings by name and would search the wrong table.
* The first function that exists is remembered for the process.
*/
interface ScarSearchRpc {
name: string;
thresholdParam: "similarity_threshold" | "match_threshold";
}

let resolvedScarSearchRpc: ScarSearchRpc | null = null;

/** For tests. */
export function resetScarSearchRpc(): void {
resolvedScarSearchRpc = null;
}

function scarSearchCandidates(): ScarSearchRpc[] {
const weighted: ScarSearchRpc = { name: `match_${getTableName("learnings")}_weighted`, thresholdParam: "match_threshold" };
return getTablePrefix() === "gitmem_"
? [{ name: "gitmem_scar_search", thresholdParam: "similarity_threshold" }, weighted]
: [weighted];
}

async function callScarSearchRpc<T>(embedding: number[], matchCount: number): Promise<T[]> {
const candidates = resolvedScarSearchRpc ? [resolvedScarSearchRpc] : scarSearchCandidates();
let lastError = "";
for (const rpc of candidates) {
const response = await fetch(`${SUPABASE_URL}/rest/v1/rpc/${rpc.name}`, {
method: "POST",
headers: {
"Content-Type": "application/json",
apikey: SUPABASE_KEY,
Authorization: `Bearer ${SUPABASE_KEY}`,
},
body: JSON.stringify({
query_embedding: `[${embedding.join(",")}]`,
match_count: matchCount,
[rpc.thresholdParam]: 0.0,
// project_filter is deliberately not sent. The primary path this falls
// back from is the unified CROSS-PROJECT vector cache, so filtering here
// would make the fallback return a narrower set than the path it stands
// in for — a silent behaviour change on the cold-start calls.
}),
signal: AbortSignal.timeout(15_000),
});

if (response.ok) {
resolvedScarSearchRpc = rpc;
const rows = (await response.json()) as T[];
return rows || [];
}

if (!response.ok) {
const text = await response.text();
throw new Error(`Supabase RPC error: ${response.status} - ${text.slice(0, 200)}`);
lastError = `Supabase RPC error: ${response.status} - ${text.slice(0, 200)}`;
// PGRST202: no such function (with these parameters) — try the next name.
// Anything else is a real failure of a function that exists.
if (!(response.status === 404 && text.includes("PGRST202"))) break;
console.error(`[scar-search] ${rpc.name} not on this store (PGRST202)`);
}
throw new Error(lastError);
}

const rows = (await response.json()) as T[];
return rows || [];
/**
* gitmem_scar_search returns id, title, description, severity, scar_type,
* counter_arguments, decay_multiplier and similarity — not the fields recall
* renders from (learning_type, applies_when, why_this_matters, …). Fetch them
* for the returned ids in one request. If that request fails the rows are
* still returned: recall renders what it has rather than losing the result.
*/
const RECALL_EXTRA_COLUMNS =
"id,learning_type,problem_context,solution_approach,applies_when,domain,keywords,source_linear_issue,why_this_matters,action_protocol,self_check_criteria,project";

async function withLearningColumns<T extends { id: string; learning_type?: string }>(rows: T[]): Promise<T[]> {
if (rows.length === 0 || rows.every((r) => r.learning_type !== undefined)) return rows;
try {
const extra = await directQuery<Record<string, unknown> & { id: string }>(getTableName("learnings"), {
select: RECALL_EXTRA_COLUMNS,
filters: { id: `in.(${rows.map((r) => r.id).join(",")})` },
});
const byId = new Map(extra.map((e) => [e.id, e]));
return rows.map((r) => ({ ...(byId.get(r.id) ?? {}), ...r }));
} catch (error) {
console.error("[scar-search] Could not fetch recall columns for the matched scars (returning without them):",
error instanceof Error ? error.message : error);
return rows;
}
}

/**
Expand Down
38 changes: 37 additions & 1 deletion tests/e2e/blank-supabase.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ function isolatedHome(tag) {
return { home, gitmemDir };
}
const STRIP = /^(OPENAI_API_KEY|OPENROUTER_API_KEY|GITMEM_|SUPABASE_|OLLAMA_|NODE_OPTIONS$|CLAUDE_CODE_ENTRYPOINT$)/;
async function startServer({ home, gitmemDir, ollamaUrl, netlog, stderrSink }) {
async function startServer({ home, gitmemDir, ollamaUrl, netlog, stderrSink, extraEnv = {} }) {
const env = Object.fromEntries(Object.entries(process.env).filter(([k]) => !STRIP.test(k)));
Object.assign(env, {
HOME: home,
Expand All @@ -216,6 +216,7 @@ async function startServer({ home, gitmemDir, ollamaUrl, netlog, stderrSink }) {
GITMEM_NETLOG: netlog,
NODE_OPTIONS: `--import=${pathToFileURL(NETLOG_PRELOAD).href}`,
NO_COLOR: "1",
...extraEnv,
});
const transport = new StdioClientTransport({ command: process.execPath, args: [SERVER], env, cwd: home, stderr: "pipe" });
transport.stderr?.on("data", (d) => stderrSink.push(d.toString()));
Expand Down Expand Up @@ -422,6 +423,33 @@ async function flow() {
const relevance = relevanceRows.map((m) => ({ tool: m.tool_name, memories_applied: m.metadata.memories_applied, memory_relevance: m.metadata.memory_relevance }));
await waitQuiet(netlog, { minMs: 1000, quietMs: 2000 });
await srv.close();

// GIT-114: recall served by the store, not the local index. (1) Forced with
// GITMEM_SEARCH_MODE=remote. (2) A fresh server asked before its index has
// loaded — the cold-start path every customer hits first. Both must return a
// seeded scar; the store must answer the RPC (a 404 fails the run below).
const recallProbe = async (tag, extraEnv) => {
const h = isolatedHome(`${label}-${tag}`);
const probeLog = join(outDir, `netlog-${label}-${tag}.jsonl`);
writeFileSync(probeLog, "");
const errs = [];
const p = await startServer({ home: h.home, gitmemDir: h.gitmemDir, ollamaUrl: ollama.url, netlog: probeLog, stderrSink: errs, extraEnv });
let text = "";
try {
text = await p.call("recall", { plan: "Supabase migrations must be dry-run before push", project: PROJECT, match_count: 3 });
} catch (e) { text = `ERROR ${e}`; }
await p.close();
allVenueRequests.push(...venueReqs(readNet(probeLog)));
const errText = errs.join("");
return {
found_seeded_scar: /Supabase migrations must be dry-run/.test(text),
used_store_rpc: venueReqs(readNet(probeLog)).some((r) => r.path.includes("/rpc/") && r.status < 300),
fell_back_from_local: /Local cache not ready/.test(errText),
text: text.slice(0, 300),
};
};
const recallRemote = await recallProbe("recall-remote", { GITMEM_SEARCH_MODE: "remote" });
const recallCold = await recallProbe("recall-cold", {});
ollama.close();

const reqs = readNet(netlog);
Expand All @@ -441,6 +469,8 @@ async function flow() {
remote_sessions_after: await count("gitmem_sessions"),
remote_scar_usage_after: await count("gitmem_scar_usage"),
session_close_persisted: closedRows.length === 1 && closedRows[0].closing_reflection != null,
recall_remote: recallRemote,
recall_cold: recallCold,
missing_payload_named: missingPayloadNamed,
resolve_thread_durable: resolveDurable,
close_warns: closeWarns,
Expand Down Expand Up @@ -648,6 +678,12 @@ if (mode === "flow" && !result.thread_triples_linked) {
if (mode === "flow" && result.false_stranded_notice) {
failureCheck.unexpected.push("session_start reported the store it reads as 'NOT being read' (GIT-107)");
}
if (mode === "flow" && !(result.recall_remote.found_seeded_scar && result.recall_remote.used_store_rpc)) {
failureCheck.unexpected.push(`recall with GITMEM_SEARCH_MODE=remote did not return a seeded scar from the store (GIT-114): ${result.recall_remote.text}`);
}
if (mode === "flow" && !result.recall_cold.found_seeded_scar) {
failureCheck.unexpected.push(`cold-start recall returned no seeded scar (GIT-114): ${result.recall_cold.text}`);
}
if (mode === "flow" && !result.session_close_persisted) {
failureCheck.unexpected.push(`session_close did not persist session ${result.session_id} (closing_reflection missing)`);
}
Expand Down
96 changes: 96 additions & 0 deletions tests/helpers/schema-drift.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -527,3 +527,99 @@ export function compare(scan: ScanResult, floor: Schema, current: Schema): Findi
for (const t of scan.tables) if (!floor.has(t.table)) note(t.table, null, "table", `${t.file}:${t.line}`);
return [...byKey.values()].sort((x, y) => `${x.table}.${x.column}`.localeCompare(`${y.table}.${y.column}`));
}

// ---------------------------------------------------------------- RPCs (GIT-114)

/** setup.sql -> function names (CREATE [OR REPLACE] FUNCTION name(...)). */
export function parseFunctions(sql: string): Set<string> {
const names = new Set<string>();
const re = /CREATE\s+(?:OR\s+REPLACE\s+)?FUNCTION\s+(?:public\.)?(\w+)\s*\(/gi;
let m: RegExpExecArray | null;
const clean = sql.replace(/--[^\n]*/g, "");
while ((m = re.exec(clean))) names.add(m[1]);
return names;
}

export interface RpcRef {
file: string;
line: number;
name: string;
/** A fixed https:// URL — another project (e.g. licensing), not the customer's store. */
external: boolean;
}

/**
* Text of a string or template literal, with getTableName("x") substitutions
* expanded to the default table name; any other substitution becomes "\0".
*/
function literalText(n: ts.Node): string | null {
if (ts.isStringLiteralLike(n)) return n.text;
if (ts.isTemplateExpression(n)) {
let out = n.head.text;
for (const span of n.templateSpans) {
const e = span.expression;
out += ts.isCallExpression(e) && ts.isIdentifier(e.expression) && e.expression.text === "getTableName" &&
e.arguments[0] && ts.isStringLiteralLike(e.arguments[0])
? DEFAULT_PREFIX + e.arguments[0].text
: "\0";
out += span.literal.text;
}
return out;
}
return null;
}

/**
* Every RPC name src/ can call. Two shapes:
* - a literal or template URL containing "/rpc/<name>";
* - in a file that builds "/rpc/${…}" dynamically, every `name:` property of
* an object literal (the candidate list the URL is built from).
* Returns the refs and the dynamic call sites whose names could not be resolved.
*/
export function scanRpcNames(srcDir: string): { refs: RpcRef[]; unresolved: Unresolved[] } {
const refs: RpcRef[] = [];
const unresolved: Unresolved[] = [];
const files: string[] = [];
const walk = (d: string) => {
for (const e of fs.readdirSync(d, { withFileTypes: true })) {
const p = path.join(d, e.name);
if (e.isDirectory()) walk(p);
else if (e.name.endsWith(".ts") && !e.name.endsWith(".test.ts") && !e.name.endsWith(".d.ts")) files.push(p);
}
};
walk(srcDir);
for (const file of files) {
const rel = path.relative(path.dirname(srcDir), file);
const sf = ts.createSourceFile(file, fs.readFileSync(file, "utf-8"), ts.ScriptTarget.Latest, true);
const lineOf = (n: ts.Node) => sf.getLineAndCharacterOfPosition(n.getStart()).line + 1;
const dynamicSites: ts.Node[] = [];
const nameProps: ts.PropertyAssignment[] = [];
const visit = (n: ts.Node): void => {
const text = literalText(n);
if (text !== null && text.includes("/rpc/")) {
const seg = text.slice(text.indexOf("/rpc/") + 5).split(/[/?"'\s]/)[0];
if (seg && !seg.includes("\0")) {
refs.push({ file: rel, line: lineOf(n), name: seg, external: /^https?:\/\//.test(text) });
} else {
dynamicSites.push(n);
}
}
if (ts.isPropertyAssignment(n) && propName(n) === "name") nameProps.push(n);
ts.forEachChild(n, visit);
};
visit(sf);
if (dynamicSites.length === 0) continue;
let found = 0;
for (const p of nameProps) {
const t = literalText(p.initializer);
if (t && /^[a-z][a-z0-9_]*$/.test(t)) {
refs.push({ file: rel, line: lineOf(p), name: t, external: false });
found++;
}
}
if (found === 0) {
for (const d of dynamicSites) unresolved.push({ file: rel, line: lineOf(d), call: "rpc", what: `dynamic RPC name ${d.getText().slice(0, 60)}` });
}
}
return { refs, unresolved };
}
Loading
Loading