diff --git a/.changeset/legacy-sidecar-revision-journal.md b/.changeset/legacy-sidecar-revision-journal.md new file mode 100644 index 000000000..18ddc3fef --- /dev/null +++ b/.changeset/legacy-sidecar-revision-journal.md @@ -0,0 +1,5 @@ +--- +"@nicia-ai/typegraph": patch +--- + +Upgrade a pre-marker merge-provenance sidecar on a database with the revision-change journal installed. The journal's triggers record every node write, so the sidecar's own `Provenance` rows left entries under its graph id and the upgrade refused them with `GRAPH_MERGE_PROVENANCE_ID_COLLISION` (`reason: "application-graph"`). A journal entry is now accepted when it records a `Provenance` node whose row is still stored and verifies; any other journal entry under that graph id still refuses. diff --git a/apps/docs/src/content/docs/errors.md b/apps/docs/src/content/docs/errors.md index af159f721..c293781b2 100644 --- a/apps/docs/src/content/docs/errors.md +++ b/apps/docs/src/content/docs/errors.md @@ -966,7 +966,7 @@ The five `details.reason` values on `GRAPH_MERGE_PROVENANCE_ID_COLLISION`: | `details.reason` | The state that was found | | --- | --- | -| `application-graph` | The id holds rows (in any per-graph table) or a schema that is not the sidecar's, so it belongs to an application. Rename the colliding graph or point the merge elsewhere. | +| `application-graph` | The id holds rows (in any per-graph table) or a schema that is not the sidecar's, so it belongs to an application. When a pre-marker sidecar is classified, revision-change journal entries that record its own stored `Provenance` rows are not counted; every other journal entry is. Rename the colliding graph or point the merge elsewhere. | | `empty-legacy-sidecar` | A pre-marker sidecar with no rows at all, which carries no evidence of authorship and is indistinguishable from an application graph of the same shape. | | `unupgradeable-legacy-sidecar` | A pre-marker sidecar whose rows do not verify as provenance this library wrote for *this* target, so it cannot be upgraded to an owned sidecar. | | `unowned-exact-schema-graph` | The current sidecar schema with no ownership marker. Because the marker is written *first*, this library cannot have produced this state; contents are not consulted, so an empty or provenance-shaped occupant is refused too. | diff --git a/packages/typegraph/src/backend/drizzle/postgres.ts b/packages/typegraph/src/backend/drizzle/postgres.ts index 0e310b2d8..aea44d148 100644 --- a/packages/typegraph/src/backend/drizzle/postgres.ts +++ b/packages/typegraph/src/backend/drizzle/postgres.ts @@ -142,6 +142,7 @@ import { import { deriveBackend } from "../derive-backend"; import { FIND_EDGES_ENDPOINT_FIXED_PARAM_COUNT } from "../edge-endpoint-sets"; import { buildLiveNodeCandidates } from "../live-node-candidates"; +import { REVISION_JOURNAL_ENTITY } from "../revision-journal"; import { createEdgeRowMapper, createNodeRowMapper, @@ -455,9 +456,12 @@ const REVISION_CHANGE_FUNCTION = "typegraph_record_revision_change"; function postgresRevisionChangeTargets(names: RevisionJournalTableNames) { return [ - { entity: "node", table: names.nodes }, - { entity: "edge", table: names.edges }, - { entity: "identity", table: names.identityAssertions }, + { entity: REVISION_JOURNAL_ENTITY.node, table: names.nodes }, + { entity: REVISION_JOURNAL_ENTITY.edge, table: names.edges }, + { + entity: REVISION_JOURNAL_ENTITY.identity, + table: names.identityAssertions, + }, ] as const; } diff --git a/packages/typegraph/src/backend/drizzle/sqlite.ts b/packages/typegraph/src/backend/drizzle/sqlite.ts index 35a0807af..683a915b1 100644 --- a/packages/typegraph/src/backend/drizzle/sqlite.ts +++ b/packages/typegraph/src/backend/drizzle/sqlite.ts @@ -105,6 +105,10 @@ import { } from "../capabilities/write-fence"; import { FIND_EDGES_ENDPOINT_FIXED_PARAM_COUNT } from "../edge-endpoint-sets"; import { buildLiveNodeCandidates } from "../live-node-candidates"; +import { + REVISION_JOURNAL_ENTITY, + type RevisionJournalEntity, +} from "../revision-journal"; import { createSerializedExecutionQueue, runWithSerializedQueue, @@ -481,13 +485,17 @@ function sqliteRevisionChangeTriggerSpecs( name: string; table: string; action: "INSERT" | "UPDATE" | "DELETE"; - entity: "node" | "edge" | "identity"; + entity: RevisionJournalEntity; complete: 0 | 1; }>[] { const targets = [ - { entity: "node", table: names.nodes, complete: 1 }, - { entity: "edge", table: names.edges, complete: 1 }, - { entity: "identity", table: names.identityAssertions, complete: 0 }, + { entity: REVISION_JOURNAL_ENTITY.node, table: names.nodes, complete: 1 }, + { entity: REVISION_JOURNAL_ENTITY.edge, table: names.edges, complete: 1 }, + { + entity: REVISION_JOURNAL_ENTITY.identity, + table: names.identityAssertions, + complete: 0, + }, ] as const; const actions = ["INSERT", "UPDATE", "DELETE"] as const; return targets.flatMap(({ entity, table, complete }) => @@ -507,8 +515,9 @@ function sqliteRevisionChangeTriggers( return sqliteRevisionChangeTriggerSpecs(names).map( ({ name, table, action, entity, complete }) => { const row = action === "DELETE" ? "OLD" : "NEW"; - const kind = entity === "identity" ? "''" : `${row}.kind`; - const id = entity === "identity" ? "''" : `${row}.id`; + const recordsKey = entity !== REVISION_JOURNAL_ENTITY.identity; + const kind = recordsKey ? `${row}.kind` : "''"; + const id = recordsKey ? `${row}.id` : "''"; return `CREATE TRIGGER IF NOT EXISTS ${quoteRevisionJournalIdentifier(name)} AFTER ${action} ON ${quoteRevisionJournalIdentifier(table)} BEGIN INSERT INTO ${quoteRevisionJournalIdentifier(names.revisionChanges)} (entry_id, graph_id, revision, complete, entity, kind, id) SELECT lower(hex(randomblob(16))), ${row}.graph_id, COALESCE((SELECT revision FROM ${quoteRevisionJournalIdentifier(names.recordedClock)} WHERE graph_id = ${row}.graph_id), 0) + 1, ${complete}, '${entity}', ${kind}, ${id}; END`; }, ); diff --git a/packages/typegraph/src/backend/revision-journal.ts b/packages/typegraph/src/backend/revision-journal.ts index bc5e85589..fdbe58c07 100644 --- a/packages/typegraph/src/backend/revision-journal.ts +++ b/packages/typegraph/src/backend/revision-journal.ts @@ -1,6 +1,21 @@ import { ConfigurationError } from "../errors"; import type { GraphBackend } from "./types"; +/** + * The `entity` a revision-journal entry records for a write to each journaled + * relation. The trigger builders pass these values as each trigger's target and + * the provenance sidecar's occupancy probe matches on them. The PostgreSQL + * trigger function body and the lineage reader still spell their own literals. + */ +export const REVISION_JOURNAL_ENTITY = { + node: "node", + edge: "edge", + identity: "identity", +} as const; + +export type RevisionJournalEntity = + (typeof REVISION_JOURNAL_ENTITY)[keyof typeof REVISION_JOURNAL_ENTITY]; + /** * Install the revision-change journal on a privileged backend during schema * bootstrap or adoption. This is the only runtime API that invokes its DDL. diff --git a/packages/typegraph/src/graph-merge/provenance-store.ts b/packages/typegraph/src/graph-merge/provenance-store.ts index ff3d11d2a..4374a7400 100644 --- a/packages/typegraph/src/graph-merge/provenance-store.ts +++ b/packages/typegraph/src/graph-merge/provenance-store.ts @@ -160,6 +160,7 @@ import { requireWriteFence, resolveGraphRelationNames, resolveWriteFencePlan, + REVISION_JOURNAL_ENTITY, serializeSchema, sha256Hex, sql, @@ -787,16 +788,19 @@ type SidecarProbeRow = Readonly<{ present: number }>; * tables happen to be empty is the same colonization refusing a stray node row * exists to prevent. * - * `scope: "foreign"` narrows only the NODE table, to rows this module could not - * have written: node rows of any other kind. Every other table is occupancy in - * both scopes — a sidecar declares no edges, tracks no revisions, asserts no - * identities, and projects no fulltext or unique keys, so a row in any of them - * is not ours whatever the schema says. `ProvenanceOwner` rows are NOT excused - * either: the marker probe has already classified every one of them, so a row - * this raw probe can see and `findNodesByKind` cannot is unaccounted-for - * occupancy and must refuse. It cannot be expressed through `findNodesByKind`, - * which needs the kinds of a graph whose schema — by construction, in the case - * that matters — was never registered. + * `scope: "foreign"` narrows the NODE table, to rows this module could not have + * written: node rows of any other kind. It narrows the revision journal the same + * way, because the journal is written by database-wide triggers rather than by + * the graph that owns the row: see {@link unaccountedJournalEntryFilter}. Every + * other table is occupancy in both scopes — a sidecar declares no edges, mints + * no revisions, asserts no identities, and projects no fulltext or unique keys, + * so a row in any of them is not ours whatever the schema says. + * `ProvenanceOwner` rows are NOT excused either: the marker probe has already + * classified every one of them, so a row this raw probe can see and + * `findNodesByKind` cannot is unaccounted-for occupancy and must refuse. It + * cannot be expressed through `findNodesByKind`, which needs the kinds of a + * graph whose schema — by construction, in the case that matters — was never + * registered. * * The probed set is every graph-scoped content relation of the shared * inventory, named through the backend's `tableNames` port. The bookkeeping @@ -819,12 +823,48 @@ async function hasRowsUnderGraphId( if (await hasRowsInTable(port, schema.tables.edges, graphId, sql.empty())) { return true; } + const journal = resolveGraphRelationNames(schema.tables).revisionChanges; for (const tableName of secondaryRowTableNames(schema.tables)) { - if (await hasRowsInSecondaryTable(port, tableName, graphId)) return true; + const filter = + scope === "foreign" && tableName === journal ? + unaccountedJournalEntryFilter(journal, schema.tables.nodes) + : sql.empty(); + if (await hasRowsInSecondaryTable(port, tableName, graphId, filter)) { + return true; + } } return false; } +/** + * Narrows the revision journal to the entries a pre-marker sidecar's own rows + * cannot account for. + * + * The journal's triggers are installed per database, not per graph, so on a + * database that has them every `Provenance` row a pre-marker sidecar wrote left + * an entry under the sidecar's graph id. An entry is accounted for only when it + * records a node whose row is STILL STORED under that graph id and `(kind, id)`, + * so the entry says nothing the node table has not already said. + * + * The kind is restricted by that correlation alone. {@link hasRowsUnderGraphId} + * applies this filter only after its node probe found no stored row of another + * kind, so a stored row an entry can match is a `Provenance` row, verified by + * {@link isOwnedProvenanceRow} like every other. + * + * Everything else stays occupancy. An entry for an edge, an identity assertion + * or a node of another kind records content a sidecar never holds, and an entry + * for a `Provenance` id with no stored row records a hard-deleted row nothing + * can verify. + */ +function unaccountedJournalEntryFilter( + journalTable: string, + nodesTable: string, +): ReturnType { + const journal = sql.identifier(journalTable); + const nodes = sql.identifier(nodesTable); + return sql` AND NOT (${journal}.entity = ${REVISION_JOURNAL_ENTITY.node} AND EXISTS (SELECT 1 FROM ${nodes} WHERE ${nodes}.graph_id = ${journal}.graph_id AND ${nodes}.kind = ${journal}.kind AND ${nodes}.id = ${journal}.id))`; +} + /** * The per-graph content tables beyond nodes and edges, deduplicated because a * backend may map two logical relations onto one physical name. @@ -886,16 +926,17 @@ async function hasRowsInSecondaryTable( port: SidecarInspectionPort, tableName: string, graphId: string, + filter: ReturnType, ): Promise { const tableExists = port.tableExists; if (tableExists !== undefined) { return ( (await tableExists(tableName)) && - (await hasRowsInTable(port, tableName, graphId, sql.empty())) + (await hasRowsInTable(port, tableName, graphId, filter)) ); } try { - return await hasRowsInTable(port, tableName, graphId, sql.empty()); + return await hasRowsInTable(port, tableName, graphId, filter); } catch { return false; } diff --git a/packages/typegraph/src/graph-merge/typegraph-internal.ts b/packages/typegraph/src/graph-merge/typegraph-internal.ts index 2e78d638a..fb0260411 100644 --- a/packages/typegraph/src/graph-merge/typegraph-internal.ts +++ b/packages/typegraph/src/graph-merge/typegraph-internal.ts @@ -16,6 +16,7 @@ export { GRAPH_RELATIONS, resolveGraphRelationNames, } from "../backend/graph-relations"; +export { REVISION_JOURNAL_ENTITY } from "../backend/revision-journal"; export { sharesSerializedTransactionResource, snapshotExportContention, diff --git a/packages/typegraph/tests/graph-merge/provenance-store.test.ts b/packages/typegraph/tests/graph-merge/provenance-store.test.ts index e0f0d877a..0d63d3349 100644 --- a/packages/typegraph/tests/graph-merge/provenance-store.test.ts +++ b/packages/typegraph/tests/graph-merge/provenance-store.test.ts @@ -75,6 +75,7 @@ import { projectBackendWithout, } from "../../src/backend/derive-backend"; import { resolveGraphRelationNames } from "../../src/backend/graph-relations"; +import { installRevisionChangesJournal } from "../../src/backend/revision-journal"; import type { NodeRow } from "../../src/backend/types"; import { asEdgeId } from "../../src/core/types"; import { branch } from "../../src/graph-merge/branch"; @@ -201,6 +202,21 @@ function revisionChangesTable(backend: GraphBackend): string { return resolveGraphRelationNames(backend.tableNames).revisionChanges; } +/** What one revision-journal entry records about the write that produced it. */ +type JournalEntry = Readonly<{ entity: string; kind: string; id: string }>; + +/** Every revision-journal entry stored under `graphId`, in a stable order. */ +async function readJournalEntries( + backend: GraphBackend, + graphId: string, +): Promise { + return backend.execute( + asCompiledRowsSql( + sql`SELECT entity, kind, id FROM ${sql.identifier(revisionChangesTable(backend))} WHERE graph_id = ${graphId} ORDER BY entity, kind, id`, + ), + ); +} + /** Reads the durable ownership marker row straight from the backend. */ async function readOwnerMarker( backend: GraphBackend, @@ -1645,6 +1661,139 @@ describe.each(backendMatrix())("provenance persistence [$name]", (entry) => { ).resolves.toBeDefined(); }); + it("upgrades a legacy sidecar whose own writes the revision triggers journaled", async () => { + const backend = await makeBackend(); + const sidecarId = provenanceGraphId(careGraph.id); + // The triggers are database-wide: once installed they journal EVERY node + // write, the pre-marker sidecar's own `Provenance` rows included, under the + // sidecar's graph id. Those entries name rows the upgrade verifies one by + // one, so they are the sidecar's own history rather than an application's. + await installRevisionChangesJournal(backend); + const [legacyStore] = await createStoreWithSchema( + legacyShapedGraph(), + backend, + ); + const legacyId = await provenanceNodeId(careGraph.id, SIDECAR_RECORD); + await legacyStore.nodes.Provenance.create( + { targetGraphId: careGraph.id, ...SIDECAR_RECORD }, + { id: legacyId }, + ); + await expect(readJournalEntries(backend, sidecarId)).resolves.toEqual([ + { entity: "node", kind: "Provenance", id: legacyId }, + ]); + + const upgraded = await openProvenanceStore(backend, careGraph.id); + + await expect( + upgraded.nodes.Provenance.getById(legacyId as never), + ).resolves.toMatchObject({ id: legacyId }); + await expect(readOwnerMarker(backend)).resolves.toMatchObject({ + id: PROVENANCE_OWNER_ROW_ID, + }); + }); + + // Each entry is one the stored `Provenance` row cannot account for. The first + // two differ from the row's own entry in exactly one column and reuse its id, + // so a rule that matched on the id alone would excuse them. The last names a + // `Provenance` row that IS stored, but under another graph id — what a copied + // sidecar or a colliding application row provides — so a rule that looked the + // row up without the graph id would excuse it. + it.each< + Readonly<{ + evidence: string; + entryFor: (storedId: string) => JournalEntry; + storedUnderAnotherGraph?: true; + }> + >([ + { + evidence: "a node of another kind under the stored row's id", + entryFor: (storedId: string) => ({ + entity: "node", + kind: "Patient", + id: storedId, + }), + }, + { + evidence: "an edge under the stored row's kind and id", + entryFor: (storedId: string) => ({ + entity: "edge", + kind: "Provenance", + id: storedId, + }), + }, + { + evidence: "a Provenance row that is no longer stored", + entryFor: () => ({ + entity: "node", + kind: "Provenance", + id: "prov_hard-deleted", + }), + }, + { + evidence: "an identity assertion", + entryFor: () => ({ entity: "identity", kind: "", id: "" }), + }, + { + evidence: "a Provenance row stored only under another graph id", + entryFor: () => ({ + entity: "node", + kind: "Provenance", + id: "prov_stored-elsewhere", + }), + storedUnderAnotherGraph: true, + }, + ])( + "refuses a journaled legacy sidecar whose journal also records $evidence", + async ({ entryFor, storedUnderAnotherGraph }) => { + const backend = await makeBackend(); + const sidecarId = provenanceGraphId(careGraph.id); + await installRevisionChangesJournal(backend); + const [legacyStore] = await createStoreWithSchema( + legacyShapedGraph(), + backend, + ); + const legacyId = await provenanceNodeId(careGraph.id, SIDECAR_RECORD); + await legacyStore.nodes.Provenance.create( + { targetGraphId: careGraph.id, ...SIDECAR_RECORD }, + { id: legacyId }, + ); + const entry = entryFor(legacyId); + if (storedUnderAnotherGraph === true) { + await backend.insertNode({ + graphId: "another-graph", + kind: entry.kind, + id: entry.id, + props: { targetGraphId: careGraph.id, ...SIDECAR_RECORD }, + }); + } + // Plants one journal entry under the sidecar's graph id that no row + // stored under that graph id accounts for. + await requireDefined(backend.executeStatement)( + asCompiledStatementSql( + sql` + INSERT INTO ${sql.identifier(revisionChangesTable(backend))} + (entry_id, graph_id, revision, complete, entity, kind, id) + VALUES (${"journal-unaccounted"}, ${sidecarId}, ${1}, TRUE, ${entry.entity}, ${entry.kind}, ${entry.id}) + `, + ), + ); + + await expect( + openProvenanceStore(backend, careGraph.id), + ).rejects.toMatchObject({ + name: "ConfigurationError", + details: { + code: "GRAPH_MERGE_PROVENANCE_ID_COLLISION", + reason: "application-graph", + }, + }); + await expect(readOwnerMarker(backend)).resolves.toBeUndefined(); + await expect( + legacyStore.nodes.Provenance.getById(legacyId as never), + ).resolves.toMatchObject({ id: legacyId }); + }, + ); + it("claims a FREE id inside the schema fence, before registering the schema", async () => { cleanups = []; const backend = await makeBackend();