fix(graph-merge): upgrade a pre-marker provenance sidecar on a database with the revision journal

.changeset/legacy-sidecar-revision-journal.md

@@ -0,0 +1,5 @@
+---
+"@nicia-ai/typegraph": patch
+---
+
+Upgrade a pre-marker merge-provenance sidecar on a database with the revision-change journal installed. The journal's triggers record every node write, so the sidecar's own `Provenance` rows left entries under its graph id and the upgrade refused them with `GRAPH_MERGE_PROVENANCE_ID_COLLISION` (`reason: "application-graph"`). A journal entry is now accepted when it records a `Provenance` node whose row is still stored and verifies; any other journal entry under that graph id still refuses.

apps/docs/src/content/docs/errors.md

@@ -966,7 +966,7 @@ The five `details.reason` values on `GRAPH_MERGE_PROVENANCE_ID_COLLISION`:
 
 | `details.reason` | The state that was found |
 | --- | --- |
-| `application-graph` | The id holds rows (in any per-graph table) or a schema that is not the sidecar's, so it belongs to an application. Rename the colliding graph or point the merge elsewhere. |
+| `application-graph` | The id holds rows (in any per-graph table) or a schema that is not the sidecar's, so it belongs to an application. When a pre-marker sidecar is classified, revision-change journal entries that record its own stored `Provenance` rows are not counted; every other journal entry is. Rename the colliding graph or point the merge elsewhere. |
 | `empty-legacy-sidecar` | A pre-marker sidecar with no rows at all, which carries no evidence of authorship and is indistinguishable from an application graph of the same shape. |
 | `unupgradeable-legacy-sidecar` | A pre-marker sidecar whose rows do not verify as provenance this library wrote for *this* target, so it cannot be upgraded to an owned sidecar. |
 | `unowned-exact-schema-graph` | The current sidecar schema with no ownership marker. Because the marker is written *first*, this library cannot have produced this state; contents are not consulted, so an empty or provenance-shaped occupant is refused too. |

packages/typegraph/src/backend/drizzle/postgres.ts

@@ -142,6 +142,7 @@ import {
 import { deriveBackend } from "../derive-backend";
 import { FIND_EDGES_ENDPOINT_FIXED_PARAM_COUNT } from "../edge-endpoint-sets";
 import { buildLiveNodeCandidates } from "../live-node-candidates";
+import { REVISION_JOURNAL_ENTITY } from "../revision-journal";
 import {
   createEdgeRowMapper,
   createNodeRowMapper,
@@ -455,9 +456,12 @@ const REVISION_CHANGE_FUNCTION = "typegraph_record_revision_change";
 
 function postgresRevisionChangeTargets(names: RevisionJournalTableNames) {
   return [
-    { entity: "node", table: names.nodes },
-    { entity: "edge", table: names.edges },
-    { entity: "identity", table: names.identityAssertions },
+    { entity: REVISION_JOURNAL_ENTITY.node, table: names.nodes },
+    { entity: REVISION_JOURNAL_ENTITY.edge, table: names.edges },
+    {
+      entity: REVISION_JOURNAL_ENTITY.identity,
+      table: names.identityAssertions,
+    },
   ] as const;
 }
 

packages/typegraph/src/backend/drizzle/sqlite.ts

@@ -105,6 +105,10 @@ import {
 } from "../capabilities/write-fence";
 import { FIND_EDGES_ENDPOINT_FIXED_PARAM_COUNT } from "../edge-endpoint-sets";
 import { buildLiveNodeCandidates } from "../live-node-candidates";
+import {
+  REVISION_JOURNAL_ENTITY,
+  type RevisionJournalEntity,
+} from "../revision-journal";
 import {
   createSerializedExecutionQueue,
   runWithSerializedQueue,
@@ -481,13 +485,17 @@ function sqliteRevisionChangeTriggerSpecs(
   name: string;
   table: string;
   action: "INSERT" | "UPDATE" | "DELETE";
-  entity: "node" | "edge" | "identity";
+  entity: RevisionJournalEntity;
   complete: 0 | 1;
 }>[] {
   const targets = [
-    { entity: "node", table: names.nodes, complete: 1 },
-    { entity: "edge", table: names.edges, complete: 1 },
-    { entity: "identity", table: names.identityAssertions, complete: 0 },
+    { entity: REVISION_JOURNAL_ENTITY.node, table: names.nodes, complete: 1 },
+    { entity: REVISION_JOURNAL_ENTITY.edge, table: names.edges, complete: 1 },
+    {
+      entity: REVISION_JOURNAL_ENTITY.identity,
+      table: names.identityAssertions,
+      complete: 0,
+    },
   ] as const;
   const actions = ["INSERT", "UPDATE", "DELETE"] as const;
   return targets.flatMap(({ entity, table, complete }) =>
@@ -507,8 +515,9 @@ function sqliteRevisionChangeTriggers(
   return sqliteRevisionChangeTriggerSpecs(names).map(
     ({ name, table, action, entity, complete }) => {
       const row = action === "DELETE" ? "OLD" : "NEW";
-      const kind = entity === "identity" ? "''" : `${row}.kind`;
-      const id = entity === "identity" ? "''" : `${row}.id`;
+      const recordsKey = entity !== REVISION_JOURNAL_ENTITY.identity;
+      const kind = recordsKey ? `${row}.kind` : "''";
+      const id = recordsKey ? `${row}.id` : "''";
       return `CREATE TRIGGER IF NOT EXISTS ${quoteRevisionJournalIdentifier(name)} AFTER ${action} ON ${quoteRevisionJournalIdentifier(table)} BEGIN INSERT INTO ${quoteRevisionJournalIdentifier(names.revisionChanges)} (entry_id, graph_id, revision, complete, entity, kind, id) SELECT lower(hex(randomblob(16))), ${row}.graph_id, COALESCE((SELECT revision FROM ${quoteRevisionJournalIdentifier(names.recordedClock)} WHERE graph_id = ${row}.graph_id), 0) + 1, ${complete}, '${entity}', ${kind}, ${id}; END`;
     },
   );

packages/typegraph/src/backend/revision-journal.ts

@@ -1,6 +1,21 @@
 import { ConfigurationError } from "../errors";
 import type { GraphBackend } from "./types";
 
+/**
+ * The `entity` a revision-journal entry records for a write to each journaled
+ * relation. The trigger builders pass these values as each trigger's target and
+ * the provenance sidecar's occupancy probe matches on them. The PostgreSQL
+ * trigger function body and the lineage reader still spell their own literals.
+ */
+export const REVISION_JOURNAL_ENTITY = {
+  node: "node",
+  edge: "edge",
+  identity: "identity",
+} as const;
+
+export type RevisionJournalEntity =
+  (typeof REVISION_JOURNAL_ENTITY)[keyof typeof REVISION_JOURNAL_ENTITY];
+
 /**
  * Install the revision-change journal on a privileged backend during schema
  * bootstrap or adoption. This is the only runtime API that invokes its DDL.

packages/typegraph/src/graph-merge/provenance-store.ts

@@ -160,6 +160,7 @@ import {
   requireWriteFence,
   resolveGraphRelationNames,
   resolveWriteFencePlan,
+  REVISION_JOURNAL_ENTITY,
   serializeSchema,
   sha256Hex,
   sql,
@@ -787,16 +788,19 @@ type SidecarProbeRow = Readonly<{ present: number }>;
  * tables happen to be empty is the same colonization refusing a stray node row
  * exists to prevent.
  *
- * `scope: "foreign"` narrows only the NODE table, to rows this module could not
- * have written: node rows of any other kind. Every other table is occupancy in
- * both scopes — a sidecar declares no edges, tracks no revisions, asserts no
- * identities, and projects no fulltext or unique keys, so a row in any of them
- * is not ours whatever the schema says. `ProvenanceOwner` rows are NOT excused
- * either: the marker probe has already classified every one of them, so a row
- * this raw probe can see and `findNodesByKind` cannot is unaccounted-for
- * occupancy and must refuse. It cannot be expressed through `findNodesByKind`,
- * which needs the kinds of a graph whose schema — by construction, in the case
- * that matters — was never registered.
+ * `scope: "foreign"` narrows the NODE table, to rows this module could not have
+ * written: node rows of any other kind. It narrows the revision journal the same
+ * way, because the journal is written by database-wide triggers rather than by
+ * the graph that owns the row: see {@link unaccountedJournalEntryFilter}. Every
+ * other table is occupancy in both scopes — a sidecar declares no edges, mints
+ * no revisions, asserts no identities, and projects no fulltext or unique keys,
+ * so a row in any of them is not ours whatever the schema says.
+ * `ProvenanceOwner` rows are NOT excused either: the marker probe has already
+ * classified every one of them, so a row this raw probe can see and
+ * `findNodesByKind` cannot is unaccounted-for occupancy and must refuse. It
+ * cannot be expressed through `findNodesByKind`, which needs the kinds of a
+ * graph whose schema — by construction, in the case that matters — was never
+ * registered.
  *
  * The probed set is every graph-scoped content relation of the shared
  * inventory, named through the backend's `tableNames` port. The bookkeeping
@@ -819,12 +823,48 @@ async function hasRowsUnderGraphId(
   if (await hasRowsInTable(port, schema.tables.edges, graphId, sql.empty())) {
     return true;
   }
+  const journal = resolveGraphRelationNames(schema.tables).revisionChanges;
   for (const tableName of secondaryRowTableNames(schema.tables)) {
-    if (await hasRowsInSecondaryTable(port, tableName, graphId)) return true;
+    const filter =
+      scope === "foreign" && tableName === journal ?
+        unaccountedJournalEntryFilter(journal, schema.tables.nodes)
+      : sql.empty();
+    if (await hasRowsInSecondaryTable(port, tableName, graphId, filter)) {
+      return true;
+    }
   }
   return false;
 }
 
+/**
+ * Narrows the revision journal to the entries a pre-marker sidecar's own rows
+ * cannot account for.
+ *
+ * The journal's triggers are installed per database, not per graph, so on a
+ * database that has them every `Provenance` row a pre-marker sidecar wrote left
+ * an entry under the sidecar's graph id. An entry is accounted for only when it
+ * records a node whose row is STILL STORED under that graph id and `(kind, id)`,
+ * so the entry says nothing the node table has not already said.
+ *
+ * The kind is restricted by that correlation alone. {@link hasRowsUnderGraphId}
+ * applies this filter only after its node probe found no stored row of another
+ * kind, so a stored row an entry can match is a `Provenance` row, verified by
+ * {@link isOwnedProvenanceRow} like every other.
+ *
+ * Everything else stays occupancy. An entry for an edge, an identity assertion
+ * or a node of another kind records content a sidecar never holds, and an entry
+ * for a `Provenance` id with no stored row records a hard-deleted row nothing
+ * can verify.
+ */
+function unaccountedJournalEntryFilter(
+  journalTable: string,
+  nodesTable: string,
+): ReturnType<typeof sql.empty> {
+  const journal = sql.identifier(journalTable);
+  const nodes = sql.identifier(nodesTable);
+  return sql` AND NOT (${journal}.entity = ${REVISION_JOURNAL_ENTITY.node} AND EXISTS (SELECT 1 FROM ${nodes} WHERE ${nodes}.graph_id = ${journal}.graph_id AND ${nodes}.kind = ${journal}.kind AND ${nodes}.id = ${journal}.id))`;
+}
+
 /**
  * The per-graph content tables beyond nodes and edges, deduplicated because a
  * backend may map two logical relations onto one physical name.
@@ -886,16 +926,17 @@ async function hasRowsInSecondaryTable(
   port: SidecarInspectionPort,
   tableName: string,
   graphId: string,
+  filter: ReturnType<typeof sql.empty>,
 ): Promise<boolean> {
   const tableExists = port.tableExists;
   if (tableExists !== undefined) {
     return (
       (await tableExists(tableName)) &&
-      (await hasRowsInTable(port, tableName, graphId, sql.empty()))
+      (await hasRowsInTable(port, tableName, graphId, filter))
     );
   }
   try {
-    return await hasRowsInTable(port, tableName, graphId, sql.empty());
+    return await hasRowsInTable(port, tableName, graphId, filter);
   } catch {
     return false;
   }

packages/typegraph/src/graph-merge/typegraph-internal.ts

@@ -16,6 +16,7 @@ export {
   GRAPH_RELATIONS,
   resolveGraphRelationNames,
 } from "../backend/graph-relations";
+export { REVISION_JOURNAL_ENTITY } from "../backend/revision-journal";
 export {
   sharesSerializedTransactionResource,
   snapshotExportContention,

packages/typegraph/tests/graph-merge/provenance-store.test.ts

@@ -75,6 +75,7 @@ import {
   projectBackendWithout,
 } from "../../src/backend/derive-backend";
 import { resolveGraphRelationNames } from "../../src/backend/graph-relations";
+import { installRevisionChangesJournal } from "../../src/backend/revision-journal";
 import type { NodeRow } from "../../src/backend/types";
 import { asEdgeId } from "../../src/core/types";
 import { branch } from "../../src/graph-merge/branch";
@@ -201,6 +202,21 @@ function revisionChangesTable(backend: GraphBackend): string {
   return resolveGraphRelationNames(backend.tableNames).revisionChanges;
 }
 
+/** What one revision-journal entry records about the write that produced it. */
+type JournalEntry = Readonly<{ entity: string; kind: string; id: string }>;
+
+/** Every revision-journal entry stored under `graphId`, in a stable order. */
+async function readJournalEntries(
+  backend: GraphBackend,
+  graphId: string,
+): Promise<readonly JournalEntry[]> {
+  return backend.execute<JournalEntry>(
+    asCompiledRowsSql(
+      sql`SELECT entity, kind, id FROM ${sql.identifier(revisionChangesTable(backend))} WHERE graph_id = ${graphId} ORDER BY entity, kind, id`,
+    ),
+  );
+}
+
 /** Reads the durable ownership marker row straight from the backend. */
 async function readOwnerMarker(
   backend: GraphBackend,
@@ -1645,6 +1661,139 @@ describe.each(backendMatrix())("provenance persistence [$name]", (entry) => {
     ).resolves.toBeDefined();
   });
 
+  it("upgrades a legacy sidecar whose own writes the revision triggers journaled", async () => {
+    const backend = await makeBackend();
+    const sidecarId = provenanceGraphId(careGraph.id);
+    // The triggers are database-wide: once installed they journal EVERY node
+    // write, the pre-marker sidecar's own `Provenance` rows included, under the
+    // sidecar's graph id. Those entries name rows the upgrade verifies one by
+    // one, so they are the sidecar's own history rather than an application's.
+    await installRevisionChangesJournal(backend);
+    const [legacyStore] = await createStoreWithSchema(
+      legacyShapedGraph(),
+      backend,
+    );
+    const legacyId = await provenanceNodeId(careGraph.id, SIDECAR_RECORD);
+    await legacyStore.nodes.Provenance.create(
+      { targetGraphId: careGraph.id, ...SIDECAR_RECORD },
+      { id: legacyId },
+    );
+    await expect(readJournalEntries(backend, sidecarId)).resolves.toEqual([
+      { entity: "node", kind: "Provenance", id: legacyId },
+    ]);
+
+    const upgraded = await openProvenanceStore(backend, careGraph.id);
+
+    await expect(
+      upgraded.nodes.Provenance.getById(legacyId as never),
+    ).resolves.toMatchObject({ id: legacyId });
+    await expect(readOwnerMarker(backend)).resolves.toMatchObject({
+      id: PROVENANCE_OWNER_ROW_ID,
+    });
+  });
+
+  // Each entry is one the stored `Provenance` row cannot account for. The first
+  // two differ from the row's own entry in exactly one column and reuse its id,
+  // so a rule that matched on the id alone would excuse them. The last names a
+  // `Provenance` row that IS stored, but under another graph id — what a copied
+  // sidecar or a colliding application row provides — so a rule that looked the
+  // row up without the graph id would excuse it.
+  it.each<
+    Readonly<{
+      evidence: string;
+      entryFor: (storedId: string) => JournalEntry;
+      storedUnderAnotherGraph?: true;
+    }>
+  >([
+    {
+      evidence: "a node of another kind under the stored row's id",
+      entryFor: (storedId: string) => ({
+        entity: "node",
+        kind: "Patient",
+        id: storedId,
+      }),
+    },
+    {
+      evidence: "an edge under the stored row's kind and id",
+      entryFor: (storedId: string) => ({
+        entity: "edge",
+        kind: "Provenance",
+        id: storedId,
+      }),
+    },
+    {
+      evidence: "a Provenance row that is no longer stored",
+      entryFor: () => ({
+        entity: "node",
+        kind: "Provenance",
+        id: "prov_hard-deleted",
+      }),
+    },
+    {
+      evidence: "an identity assertion",
+      entryFor: () => ({ entity: "identity", kind: "", id: "" }),
+    },
+    {
+      evidence: "a Provenance row stored only under another graph id",
+      entryFor: () => ({
+        entity: "node",
+        kind: "Provenance",
+        id: "prov_stored-elsewhere",
+      }),
+      storedUnderAnotherGraph: true,
+    },
+  ])(
+    "refuses a journaled legacy sidecar whose journal also records $evidence",
+    async ({ entryFor, storedUnderAnotherGraph }) => {
+      const backend = await makeBackend();
+      const sidecarId = provenanceGraphId(careGraph.id);
+      await installRevisionChangesJournal(backend);
+      const [legacyStore] = await createStoreWithSchema(
+        legacyShapedGraph(),
+        backend,
+      );
+      const legacyId = await provenanceNodeId(careGraph.id, SIDECAR_RECORD);
+      await legacyStore.nodes.Provenance.create(
+        { targetGraphId: careGraph.id, ...SIDECAR_RECORD },
+        { id: legacyId },
+      );
+      const entry = entryFor(legacyId);
+      if (storedUnderAnotherGraph === true) {
+        await backend.insertNode({
+          graphId: "another-graph",
+          kind: entry.kind,
+          id: entry.id,
+          props: { targetGraphId: careGraph.id, ...SIDECAR_RECORD },
+        });
+      }
+      // Plants one journal entry under the sidecar's graph id that no row
+      // stored under that graph id accounts for.
+      await requireDefined(backend.executeStatement)(
+        asCompiledStatementSql(
+          sql`
+            INSERT INTO ${sql.identifier(revisionChangesTable(backend))}
+                          (entry_id, graph_id, revision, complete, entity, kind, id)
+                        VALUES (${"journal-unaccounted"}, ${sidecarId}, ${1}, TRUE, ${entry.entity}, ${entry.kind}, ${entry.id})
+          `,
+        ),
+      );
+
+      await expect(
+        openProvenanceStore(backend, careGraph.id),
+      ).rejects.toMatchObject({
+        name: "ConfigurationError",
+        details: {
+          code: "GRAPH_MERGE_PROVENANCE_ID_COLLISION",
+          reason: "application-graph",
+        },
+      });
+      await expect(readOwnerMarker(backend)).resolves.toBeUndefined();
+      await expect(
+        legacyStore.nodes.Provenance.getById(legacyId as never),
+      ).resolves.toMatchObject({ id: legacyId });
+    },
+  );
+
   it("claims a FREE id inside the schema fence, before registering the schema", async () => {
     cleanups = [];
     const backend = await makeBackend();