From 133883d572834f67e4c25252db7022e3c0c6306f Mon Sep 17 00:00:00 2001 From: ggn Date: Mon, 12 Oct 2026 15:26:03 -0400 Subject: [PATCH 1/2] feat(storage): add portfolio forge observation contract --- CHANGELOG.md | 10 ++++ VERSION | 2 +- pyproject.toml | 2 +- src/traust_contracts/v1/storage/sql.py | 3 +- storage/v1/README.md | 31 +++++++++++ storage/v1/postgres/migrations/001_to_002.sql | 5 +- storage/v1/postgres/migrations/002_to_003.sql | 1 + .../postgres/schema/portfolio_forge_facts.sql | 27 ++++++++++ storage/v1/sqlite/migrations/001_to_002.sql | 5 +- storage/v1/sqlite/migrations/002_to_003.sql | 1 + .../sqlite/schema/portfolio_forge_facts.sql | 27 ++++++++++ tests/test_forge_facts.py | 52 +++++++++++++++++++ tests/test_storage_postgres.py | 1 + tests/test_storage_registry.py | 6 +-- tests/test_storage_sql.py | 3 +- tests/test_storage_store.py | 3 +- uv.lock | 4 +- 17 files changed, 165 insertions(+), 18 deletions(-) create mode 100644 storage/v1/postgres/migrations/002_to_003.sql create mode 100644 storage/v1/postgres/schema/portfolio_forge_facts.sql create mode 100644 storage/v1/sqlite/migrations/002_to_003.sql create mode 100644 storage/v1/sqlite/schema/portfolio_forge_facts.sql create mode 100644 tests/test_forge_facts.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ff8a27..c9f1768 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ All notable changes to traust-contracts are documented here. +## [0.54.0] + +### Added + +- Storage revision 2: `portfolio_forge_facts` in SQLite and PostgreSQL, with + an explicit revision-1 migration that preserves existing records. Caches + forge observations and their audit-baseline context for XWING-2214; derived + repository state is never stored. See storage/v1/README.md for sources and + migration instructions. + ## [0.53.0] ### Added diff --git a/VERSION b/VERSION index 7f422a1..524456c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.53.0 +0.54.0 diff --git a/pyproject.toml b/pyproject.toml index 9db6b0f..4ac59a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "traust-contracts" -version = "0.53.0" +version = "0.54.0" license = "Apache-2.0" description = "Contract surface for the AI security ledger ecosystem: schemas and shared enums" requires-python = ">=3.11" diff --git a/src/traust_contracts/v1/storage/sql.py b/src/traust_contracts/v1/storage/sql.py index 6b0da31..6dd781b 100644 --- a/src/traust_contracts/v1/storage/sql.py +++ b/src/traust_contracts/v1/storage/sql.py @@ -17,7 +17,8 @@ #: and every earlier change are part of revision 1. Databases created under any #: earlier schema are recreated, not migrated. #: Each later step adds migrations/NNN_to_NNN+1.sql; Store.migrate() runs them. -REVISION = 1 +#: 2: cached Portfolio forge observations (XWING-2214). +REVISION = 2 @cache diff --git a/storage/v1/README.md b/storage/v1/README.md index 69c9d0e..cf1d729 100644 --- a/storage/v1/README.md +++ b/storage/v1/README.md @@ -391,3 +391,34 @@ make test PostgreSQL tests use the fixed local test DSN. They run when `psycopg` and the database are available; otherwise they warn and skip. + +## Portfolio forge observations + +`portfolio_forge_facts` caches the latest slow/rate-limited forge observation. +Core supplies a canonical URL identity (`url:...`), or a filing identity (`key:...`) +when no supported URL exists. This is an observation lookup key, not a replacement +for the registered repository/CodeLine UUID model. Missing-URL filings remain distinct. +The audited SHA, audit date and audit-presence flag record the comparison context; +Core must reject reuse when that context changes. Unknown measurements stay NULL. + +Audit age, churn ratio, live finding counts, risk tier, exposure and `RepoState` +are deliberately absent: Core derives them from their owning domains on each read. +There is no new database-file location. The table belongs in the configured storage. + +Storage revision 2 adds this table. Back up an existing revision-1 database and +invoke the existing `Store.migrate()` operation explicitly; `Store.init()` refuses +an older revision. The migration creates the table and preserves existing data. +No conversion of the unmerged traust PR #58 `security_repo_state` table is shipped. + +Sources for this move from [traust#58](https://github.com/traust-security/traust/pull/58): +- [XWING-2214, ownership table and acceptance criteria 4–5](https://redhat.atlassian.net/browse/XWING-2214): + Contracts owns forge-facts DDL; RepoState must never be stored. +- [XWING-2205, Where code goes](https://redhat.atlassian.net/browse/XWING-2205): + all table definitions belong in Contracts, domain rules in Core. +- [Shaun's PR review](https://github.com/traust-security/traust/pull/58#issuecomment-6069412172): + move the pack's storage to Core/Contracts. The newer 2214 requirements narrow + this to forge facts rather than moving the old whole-state table unchanged. + +The lookup-key encoding, baseline guards, table/column names and revision migration +are implementation choices for review, not names prescribed by those sources. +Comparison-status vocabulary remains in Core for now, as 2214 explicitly requires. diff --git a/storage/v1/postgres/migrations/001_to_002.sql b/storage/v1/postgres/migrations/001_to_002.sql index b88291b..61e76cd 100644 --- a/storage/v1/postgres/migrations/001_to_002.sql +++ b/storage/v1/postgres/migrations/001_to_002.sql @@ -1,4 +1 @@ --- Placeholder for storage v1 revision 1 -> 2. --- --- Deltas only (ALTER / DROP / data UPDATE). New tables, indexes and views --- belong in schema/ and views/; Store.migrate() re-runs those files. +-- Revision 2 adds portfolio_forge_facts through schema bootstrap; no existing rows change. diff --git a/storage/v1/postgres/migrations/002_to_003.sql b/storage/v1/postgres/migrations/002_to_003.sql new file mode 100644 index 0000000..5b514e4 --- /dev/null +++ b/storage/v1/postgres/migrations/002_to_003.sql @@ -0,0 +1 @@ +-- Placeholder for revision 2 -> 3; deltas only. diff --git a/storage/v1/postgres/schema/portfolio_forge_facts.sql b/storage/v1/postgres/schema/portfolio_forge_facts.sql new file mode 100644 index 0000000..106044d --- /dev/null +++ b/storage/v1/postgres/schema/portfolio_forge_facts.sql @@ -0,0 +1,27 @@ +-- Latest forge observation only; derived RepoState is never stored. +-- identity is a canonical URL key or a distinct missing-URL filing key, supplied by Core. +-- audited_sha, audit_date and has_audit identify the baseline used for the observation. +CREATE TABLE IF NOT EXISTS traust_storage.portfolio_forge_facts ( + identity TEXT NOT NULL PRIMARY KEY, + repo_key TEXT NOT NULL UNIQUE, + repo_url TEXT, + source_url TEXT, + audited_sha TEXT, + audit_date TEXT, + has_audit BOOLEAN NOT NULL, + status TEXT NOT NULL, + pushed_at TEXT, + archived BOOLEAN, + default_branch TEXT, + visibility TEXT, + fork BOOLEAN, + parent TEXT, + ahead_by BIGINT, + changed_lines BIGINT, + sensitive_changed BOOLEAN, + sensitive_lines BIGINT, + deps_only BOOLEAN, + compare_truncated BOOLEAN, + observed_at TIMESTAMPTZ NOT NULL, + error TEXT +); diff --git a/storage/v1/sqlite/migrations/001_to_002.sql b/storage/v1/sqlite/migrations/001_to_002.sql index b88291b..61e76cd 100644 --- a/storage/v1/sqlite/migrations/001_to_002.sql +++ b/storage/v1/sqlite/migrations/001_to_002.sql @@ -1,4 +1 @@ --- Placeholder for storage v1 revision 1 -> 2. --- --- Deltas only (ALTER / DROP / data UPDATE). New tables, indexes and views --- belong in schema/ and views/; Store.migrate() re-runs those files. +-- Revision 2 adds portfolio_forge_facts through schema bootstrap; no existing rows change. diff --git a/storage/v1/sqlite/migrations/002_to_003.sql b/storage/v1/sqlite/migrations/002_to_003.sql new file mode 100644 index 0000000..5b514e4 --- /dev/null +++ b/storage/v1/sqlite/migrations/002_to_003.sql @@ -0,0 +1 @@ +-- Placeholder for revision 2 -> 3; deltas only. diff --git a/storage/v1/sqlite/schema/portfolio_forge_facts.sql b/storage/v1/sqlite/schema/portfolio_forge_facts.sql new file mode 100644 index 0000000..436a3b2 --- /dev/null +++ b/storage/v1/sqlite/schema/portfolio_forge_facts.sql @@ -0,0 +1,27 @@ +-- Latest forge observation only; derived RepoState is never stored. +-- identity is a canonical URL key or a distinct missing-URL filing key, supplied by Core. +-- audited_sha, audit_date and has_audit identify the baseline used for the observation. +CREATE TABLE IF NOT EXISTS portfolio_forge_facts ( + identity TEXT NOT NULL PRIMARY KEY, + repo_key TEXT NOT NULL UNIQUE, + repo_url TEXT, + source_url TEXT, + audited_sha TEXT, + audit_date TEXT, + has_audit BOOLEAN NOT NULL, + status TEXT NOT NULL, + pushed_at TEXT, + archived BOOLEAN, + default_branch TEXT, + visibility TEXT, + fork BOOLEAN, + parent TEXT, + ahead_by BIGINT, + changed_lines BIGINT, + sensitive_changed BOOLEAN, + sensitive_lines BIGINT, + deps_only BOOLEAN, + compare_truncated BOOLEAN, + observed_at TEXT NOT NULL, + error TEXT +); diff --git a/tests/test_forge_facts.py b/tests/test_forge_facts.py new file mode 100644 index 0000000..d08dd5a --- /dev/null +++ b/tests/test_forge_facts.py @@ -0,0 +1,52 @@ +"""Upgrade safety and the persisted/derived boundary for XWING-2214.""" + +import sqlite3 + +import pytest + +from traust_contracts.v1.storage import IngestError, Store +from traust_contracts.v1.storage.sql import REVISION + + +def test_revision_one_upgrade_adds_facts_without_changing_existing_rows(): + conn = sqlite3.connect(":memory:") + store = Store(conn) + store.init() + product = store.register_product("example") + conn.execute("DROP TABLE portfolio_forge_facts") + conn.execute("UPDATE traust_storage_meta SET revision = 1") + conn.commit() + with pytest.raises(IngestError, match="migration"): + store.init() + assert store.migrate() == 1 + assert conn.execute("SELECT revision FROM traust_storage_meta").fetchone() == (REVISION,) + assert store.register_product("example") == product + assert conn.execute("SELECT count(*) FROM portfolio_forge_facts").fetchone() == (0,) + + +def test_unknown_measurements_and_missing_url_filings_remain_distinct(): + conn = sqlite3.connect(":memory:") + Store(conn).init() + for key in ("filing-a", "filing-b"): + conn.execute( + """INSERT INTO portfolio_forge_facts + (identity, repo_key, has_audit, status, observed_at) + VALUES (?, ?, ?, ?, ?)""", + (f"key:{key}", key, True, "no-repo-url", "2026-10-01T00:00:00+00:00"), + ) + assert conn.execute("SELECT repo_url, changed_lines FROM portfolio_forge_facts").fetchall() == [ + (None, None), + (None, None), + ] + columns = {row[1] for row in conn.execute("PRAGMA table_info(portfolio_forge_facts)")} + assert not columns & { + "risk_tier", + "exposure", + "audit_age_days", + "churn_ratio", + "live_crit_high", + } + assert {"audited_sha", "audit_date", "has_audit"} <= columns + assert not conn.execute( + "SELECT name FROM sqlite_master WHERE name='security_repo_state'" + ).fetchall() diff --git a/tests/test_storage_postgres.py b/tests/test_storage_postgres.py index bb258b6..0428ae3 100644 --- a/tests/test_storage_postgres.py +++ b/tests/test_storage_postgres.py @@ -33,6 +33,7 @@ from traust_contracts.v1.storage.sql import REVISION TABLES = { + "portfolio_forge_facts", *REGISTRY_TABLES, *INVENTORY_TABLES, "artifact_binding", diff --git a/tests/test_storage_registry.py b/tests/test_storage_registry.py index 4ba0d96..0fce8dd 100644 --- a/tests/test_storage_registry.py +++ b/tests/test_storage_registry.py @@ -331,12 +331,12 @@ def test_migrate_runs_a_step_through_its_delta_file(monkeypatch: pytest.MonkeyPa conn.execute("CREATE VIEW current_binding AS SELECT binding_id FROM artifact_binding") conn.commit() for module in ("traust_contracts.v1.storage.store", "traust_contracts.v1.storage.sql"): - monkeypatch.setattr(f"{module}.REVISION", 2) + monkeypatch.setattr(f"{module}.REVISION", REVISION + 1) with pytest.raises(IngestError, match="explicit migration"): store.init() - assert store.migrate() == 1 + assert store.migrate() == REVISION store.init() - assert conn.execute("SELECT revision FROM traust_storage_meta").fetchone() == (2,) + assert conn.execute("SELECT revision FROM traust_storage_meta").fetchone() == (REVISION + 1,) assert _schema(conn) == _schema(_fresh()) assert store.find_product_repo("rhacm", ACM_CLI) == owner diff --git a/tests/test_storage_sql.py b/tests/test_storage_sql.py index cfb6b2e..8ad7f68 100644 --- a/tests/test_storage_sql.py +++ b/tests/test_storage_sql.py @@ -25,6 +25,7 @@ "artifact_binding", "artifact_location", "traust_storage_meta", + "portfolio_forge_facts", *PROJECTION_TABLES.values(), *ALL_SECONDARY_PROJECTION_TABLES, } @@ -35,7 +36,7 @@ def test_storage_revision_matches_migration_placeholder() -> None: # Revision 1 is the rebaselined schema (0.50.0). Every later step needs a # delta file so Store.migrate() can build up to REVISION; the next step's # file exists as a placeholder. - assert REVISION == 1 + assert REVISION == 2 for dialect in ("sqlite", "postgres"): migrations = storage_dir() / dialect / "migrations" assert sorted(p.name for p in migrations.glob("*.sql")) == [ diff --git a/tests/test_storage_store.py b/tests/test_storage_store.py index dede655..839f3f8 100644 --- a/tests/test_storage_store.py +++ b/tests/test_storage_store.py @@ -38,6 +38,7 @@ TABLES = [ *REGISTRY_TABLES, *INVENTORY_TABLES, + "portfolio_forge_facts", "artifact_binding", "artifact_evidence", "artifact_location", @@ -121,7 +122,7 @@ def test_init_revision_and_dependency_shape(store: Store) -> None: store.init() assert conn.execute("SELECT * FROM traust_storage_meta").fetchall() == original conn.execute("ALTER TABLE report_finding DROP COLUMN blocked_external") - conn.execute("UPDATE traust_storage_meta SET revision=?", (2,)) + conn.execute("UPDATE traust_storage_meta SET revision=?", (REVISION + 1,)) conn.commit() legacy_dump = tuple(conn.iterdump()) with pytest.raises(IngestError, match=r"explicit migration"): diff --git a/uv.lock b/uv.lock index 73ece87..890565f 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.11" [[package]] @@ -641,7 +641,7 @@ wheels = [ [[package]] name = "traust-contracts" -version = "0.53.0" +version = "0.54.0" source = { editable = "." } dependencies = [ { name = "jsonschema", extra = ["format-nongpl"] }, From 2759c35fa6dc2d7bb9be9424a062ebc5444eee2d Mon Sep 17 00:00:00 2001 From: ggn Date: Mon, 12 Oct 2026 19:18:34 -0400 Subject: [PATCH 2/2] docs(storage): keep essential forge observation guidance --- CHANGELOG.md | 7 +++---- storage/v1/README.md | 39 +++++++++++---------------------------- 2 files changed, 14 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9f1768..fb914d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,9 @@ All notable changes to traust-contracts are documented here. ### Added - Storage revision 2: `portfolio_forge_facts` in SQLite and PostgreSQL, with - an explicit revision-1 migration that preserves existing records. Caches - forge observations and their audit-baseline context for XWING-2214; derived - repository state is never stored. See storage/v1/README.md for sources and - migration instructions. + an explicit revision-1 migration that preserves existing records. Stores forge + observations and their audit-baseline context; derived repository state is not + stored. See [migration instructions](storage/v1/README.md#migrations). ## [0.53.0] diff --git a/storage/v1/README.md b/storage/v1/README.md index cf1d729..069f099 100644 --- a/storage/v1/README.md +++ b/storage/v1/README.md @@ -394,31 +394,14 @@ database are available; otherwise they warn and skip. ## Portfolio forge observations -`portfolio_forge_facts` caches the latest slow/rate-limited forge observation. -Core supplies a canonical URL identity (`url:...`), or a filing identity (`key:...`) -when no supported URL exists. This is an observation lookup key, not a replacement -for the registered repository/CodeLine UUID model. Missing-URL filings remain distinct. -The audited SHA, audit date and audit-presence flag record the comparison context; -Core must reject reuse when that context changes. Unknown measurements stay NULL. - -Audit age, churn ratio, live finding counts, risk tier, exposure and `RepoState` -are deliberately absent: Core derives them from their owning domains on each read. -There is no new database-file location. The table belongs in the configured storage. - -Storage revision 2 adds this table. Back up an existing revision-1 database and -invoke the existing `Store.migrate()` operation explicitly; `Store.init()` refuses -an older revision. The migration creates the table and preserves existing data. -No conversion of the unmerged traust PR #58 `security_repo_state` table is shipped. - -Sources for this move from [traust#58](https://github.com/traust-security/traust/pull/58): -- [XWING-2214, ownership table and acceptance criteria 4–5](https://redhat.atlassian.net/browse/XWING-2214): - Contracts owns forge-facts DDL; RepoState must never be stored. -- [XWING-2205, Where code goes](https://redhat.atlassian.net/browse/XWING-2205): - all table definitions belong in Contracts, domain rules in Core. -- [Shaun's PR review](https://github.com/traust-security/traust/pull/58#issuecomment-6069412172): - move the pack's storage to Core/Contracts. The newer 2214 requirements narrow - this to forge facts rather than moving the old whole-state table unchanged. - -The lookup-key encoding, baseline guards, table/column names and revision migration -are implementation choices for review, not names prescribed by those sources. -Comparison-status vocabulary remains in Core for now, as 2214 explicitly requires. +`portfolio_forge_facts` stores the latest forge observation, keyed by canonical URL +(`url:...`) or filing key (`key:...`) when no supported URL exists. Missing-URL +filings remain distinct; these lookup keys do not replace repository/CodeLine UUIDs. +The audited SHA, audit date and audit-presence flag identify the comparison +baseline; Core rejects reuse when it changes. Unknown measurements stay NULL. +Core derives age, churn, finding counts, risk and exposure on read; `RepoState` +is not stored. + +Storage revision 2 adds this table to the configured database. For existing +revision-1 storage, back up and explicitly call `Store.migrate()`; `Store.init()` +refuses older revisions. See [Migrations](#migrations).