diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ff8a27..fb914d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to traust-contracts are documented here. +## [0.54.0] + +### Added + +- Storage revision 2: `portfolio_forge_facts` in SQLite and PostgreSQL, with + an explicit revision-1 migration that preserves existing records. Stores forge + observations and their audit-baseline context; derived repository state is not + stored. See [migration instructions](storage/v1/README.md#migrations). + ## [0.53.0] ### Added diff --git a/VERSION b/VERSION index 7f422a1..524456c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.53.0 +0.54.0 diff --git a/pyproject.toml b/pyproject.toml index 9db6b0f..4ac59a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "traust-contracts" -version = "0.53.0" +version = "0.54.0" license = "Apache-2.0" description = "Contract surface for the AI security ledger ecosystem: schemas and shared enums" requires-python = ">=3.11" diff --git a/src/traust_contracts/v1/storage/sql.py b/src/traust_contracts/v1/storage/sql.py index 6b0da31..6dd781b 100644 --- a/src/traust_contracts/v1/storage/sql.py +++ b/src/traust_contracts/v1/storage/sql.py @@ -17,7 +17,8 @@ #: and every earlier change are part of revision 1. Databases created under any #: earlier schema are recreated, not migrated. #: Each later step adds migrations/NNN_to_NNN+1.sql; Store.migrate() runs them. -REVISION = 1 +#: 2: cached Portfolio forge observations (XWING-2214). +REVISION = 2 @cache diff --git a/storage/v1/README.md b/storage/v1/README.md index 69c9d0e..069f099 100644 --- a/storage/v1/README.md +++ b/storage/v1/README.md @@ -391,3 +391,17 @@ make test PostgreSQL tests use the fixed local test DSN. They run when `psycopg` and the database are available; otherwise they warn and skip. + +## Portfolio forge observations + +`portfolio_forge_facts` stores the latest forge observation, keyed by canonical URL +(`url:...`) or filing key (`key:...`) when no supported URL exists. Missing-URL +filings remain distinct; these lookup keys do not replace repository/CodeLine UUIDs. +The audited SHA, audit date and audit-presence flag identify the comparison +baseline; Core rejects reuse when it changes. Unknown measurements stay NULL. +Core derives age, churn, finding counts, risk and exposure on read; `RepoState` +is not stored. + +Storage revision 2 adds this table to the configured database. For existing +revision-1 storage, back up and explicitly call `Store.migrate()`; `Store.init()` +refuses older revisions. See [Migrations](#migrations). diff --git a/storage/v1/postgres/migrations/001_to_002.sql b/storage/v1/postgres/migrations/001_to_002.sql index b88291b..61e76cd 100644 --- a/storage/v1/postgres/migrations/001_to_002.sql +++ b/storage/v1/postgres/migrations/001_to_002.sql @@ -1,4 +1 @@ --- Placeholder for storage v1 revision 1 -> 2. --- --- Deltas only (ALTER / DROP / data UPDATE). New tables, indexes and views --- belong in schema/ and views/; Store.migrate() re-runs those files. +-- Revision 2 adds portfolio_forge_facts through schema bootstrap; no existing rows change. diff --git a/storage/v1/postgres/migrations/002_to_003.sql b/storage/v1/postgres/migrations/002_to_003.sql new file mode 100644 index 0000000..5b514e4 --- /dev/null +++ b/storage/v1/postgres/migrations/002_to_003.sql @@ -0,0 +1 @@ +-- Placeholder for revision 2 -> 3; deltas only. diff --git a/storage/v1/postgres/schema/portfolio_forge_facts.sql b/storage/v1/postgres/schema/portfolio_forge_facts.sql new file mode 100644 index 0000000..106044d --- /dev/null +++ b/storage/v1/postgres/schema/portfolio_forge_facts.sql @@ -0,0 +1,27 @@ +-- Latest forge observation only; derived RepoState is never stored. +-- identity is a canonical URL key or a distinct missing-URL filing key, supplied by Core. +-- audited_sha, audit_date and has_audit identify the baseline used for the observation. +CREATE TABLE IF NOT EXISTS traust_storage.portfolio_forge_facts ( + identity TEXT NOT NULL PRIMARY KEY, + repo_key TEXT NOT NULL UNIQUE, + repo_url TEXT, + source_url TEXT, + audited_sha TEXT, + audit_date TEXT, + has_audit BOOLEAN NOT NULL, + status TEXT NOT NULL, + pushed_at TEXT, + archived BOOLEAN, + default_branch TEXT, + visibility TEXT, + fork BOOLEAN, + parent TEXT, + ahead_by BIGINT, + changed_lines BIGINT, + sensitive_changed BOOLEAN, + sensitive_lines BIGINT, + deps_only BOOLEAN, + compare_truncated BOOLEAN, + observed_at TIMESTAMPTZ NOT NULL, + error TEXT +); diff --git a/storage/v1/sqlite/migrations/001_to_002.sql b/storage/v1/sqlite/migrations/001_to_002.sql index b88291b..61e76cd 100644 --- a/storage/v1/sqlite/migrations/001_to_002.sql +++ b/storage/v1/sqlite/migrations/001_to_002.sql @@ -1,4 +1 @@ --- Placeholder for storage v1 revision 1 -> 2. --- --- Deltas only (ALTER / DROP / data UPDATE). New tables, indexes and views --- belong in schema/ and views/; Store.migrate() re-runs those files. +-- Revision 2 adds portfolio_forge_facts through schema bootstrap; no existing rows change. diff --git a/storage/v1/sqlite/migrations/002_to_003.sql b/storage/v1/sqlite/migrations/002_to_003.sql new file mode 100644 index 0000000..5b514e4 --- /dev/null +++ b/storage/v1/sqlite/migrations/002_to_003.sql @@ -0,0 +1 @@ +-- Placeholder for revision 2 -> 3; deltas only. diff --git a/storage/v1/sqlite/schema/portfolio_forge_facts.sql b/storage/v1/sqlite/schema/portfolio_forge_facts.sql new file mode 100644 index 0000000..436a3b2 --- /dev/null +++ b/storage/v1/sqlite/schema/portfolio_forge_facts.sql @@ -0,0 +1,27 @@ +-- Latest forge observation only; derived RepoState is never stored. +-- identity is a canonical URL key or a distinct missing-URL filing key, supplied by Core. +-- audited_sha, audit_date and has_audit identify the baseline used for the observation. +CREATE TABLE IF NOT EXISTS portfolio_forge_facts ( + identity TEXT NOT NULL PRIMARY KEY, + repo_key TEXT NOT NULL UNIQUE, + repo_url TEXT, + source_url TEXT, + audited_sha TEXT, + audit_date TEXT, + has_audit BOOLEAN NOT NULL, + status TEXT NOT NULL, + pushed_at TEXT, + archived BOOLEAN, + default_branch TEXT, + visibility TEXT, + fork BOOLEAN, + parent TEXT, + ahead_by BIGINT, + changed_lines BIGINT, + sensitive_changed BOOLEAN, + sensitive_lines BIGINT, + deps_only BOOLEAN, + compare_truncated BOOLEAN, + observed_at TEXT NOT NULL, + error TEXT +); diff --git a/tests/test_forge_facts.py b/tests/test_forge_facts.py new file mode 100644 index 0000000..d08dd5a --- /dev/null +++ b/tests/test_forge_facts.py @@ -0,0 +1,52 @@ +"""Upgrade safety and the persisted/derived boundary for XWING-2214.""" + +import sqlite3 + +import pytest + +from traust_contracts.v1.storage import IngestError, Store +from traust_contracts.v1.storage.sql import REVISION + + +def test_revision_one_upgrade_adds_facts_without_changing_existing_rows(): + conn = sqlite3.connect(":memory:") + store = Store(conn) + store.init() + product = store.register_product("example") + conn.execute("DROP TABLE portfolio_forge_facts") + conn.execute("UPDATE traust_storage_meta SET revision = 1") + conn.commit() + with pytest.raises(IngestError, match="migration"): + store.init() + assert store.migrate() == 1 + assert conn.execute("SELECT revision FROM traust_storage_meta").fetchone() == (REVISION,) + assert store.register_product("example") == product + assert conn.execute("SELECT count(*) FROM portfolio_forge_facts").fetchone() == (0,) + + +def test_unknown_measurements_and_missing_url_filings_remain_distinct(): + conn = sqlite3.connect(":memory:") + Store(conn).init() + for key in ("filing-a", "filing-b"): + conn.execute( + """INSERT INTO portfolio_forge_facts + (identity, repo_key, has_audit, status, observed_at) + VALUES (?, ?, ?, ?, ?)""", + (f"key:{key}", key, True, "no-repo-url", "2026-10-01T00:00:00+00:00"), + ) + assert conn.execute("SELECT repo_url, changed_lines FROM portfolio_forge_facts").fetchall() == [ + (None, None), + (None, None), + ] + columns = {row[1] for row in conn.execute("PRAGMA table_info(portfolio_forge_facts)")} + assert not columns & { + "risk_tier", + "exposure", + "audit_age_days", + "churn_ratio", + "live_crit_high", + } + assert {"audited_sha", "audit_date", "has_audit"} <= columns + assert not conn.execute( + "SELECT name FROM sqlite_master WHERE name='security_repo_state'" + ).fetchall() diff --git a/tests/test_storage_postgres.py b/tests/test_storage_postgres.py index bb258b6..0428ae3 100644 --- a/tests/test_storage_postgres.py +++ b/tests/test_storage_postgres.py @@ -33,6 +33,7 @@ from traust_contracts.v1.storage.sql import REVISION TABLES = { + "portfolio_forge_facts", *REGISTRY_TABLES, *INVENTORY_TABLES, "artifact_binding", diff --git a/tests/test_storage_registry.py b/tests/test_storage_registry.py index 4ba0d96..0fce8dd 100644 --- a/tests/test_storage_registry.py +++ b/tests/test_storage_registry.py @@ -331,12 +331,12 @@ def test_migrate_runs_a_step_through_its_delta_file(monkeypatch: pytest.MonkeyPa conn.execute("CREATE VIEW current_binding AS SELECT binding_id FROM artifact_binding") conn.commit() for module in ("traust_contracts.v1.storage.store", "traust_contracts.v1.storage.sql"): - monkeypatch.setattr(f"{module}.REVISION", 2) + monkeypatch.setattr(f"{module}.REVISION", REVISION + 1) with pytest.raises(IngestError, match="explicit migration"): store.init() - assert store.migrate() == 1 + assert store.migrate() == REVISION store.init() - assert conn.execute("SELECT revision FROM traust_storage_meta").fetchone() == (2,) + assert conn.execute("SELECT revision FROM traust_storage_meta").fetchone() == (REVISION + 1,) assert _schema(conn) == _schema(_fresh()) assert store.find_product_repo("rhacm", ACM_CLI) == owner diff --git a/tests/test_storage_sql.py b/tests/test_storage_sql.py index cfb6b2e..8ad7f68 100644 --- a/tests/test_storage_sql.py +++ b/tests/test_storage_sql.py @@ -25,6 +25,7 @@ "artifact_binding", "artifact_location", "traust_storage_meta", + "portfolio_forge_facts", *PROJECTION_TABLES.values(), *ALL_SECONDARY_PROJECTION_TABLES, } @@ -35,7 +36,7 @@ def test_storage_revision_matches_migration_placeholder() -> None: # Revision 1 is the rebaselined schema (0.50.0). Every later step needs a # delta file so Store.migrate() can build up to REVISION; the next step's # file exists as a placeholder. - assert REVISION == 1 + assert REVISION == 2 for dialect in ("sqlite", "postgres"): migrations = storage_dir() / dialect / "migrations" assert sorted(p.name for p in migrations.glob("*.sql")) == [ diff --git a/tests/test_storage_store.py b/tests/test_storage_store.py index dede655..839f3f8 100644 --- a/tests/test_storage_store.py +++ b/tests/test_storage_store.py @@ -38,6 +38,7 @@ TABLES = [ *REGISTRY_TABLES, *INVENTORY_TABLES, + "portfolio_forge_facts", "artifact_binding", "artifact_evidence", "artifact_location", @@ -121,7 +122,7 @@ def test_init_revision_and_dependency_shape(store: Store) -> None: store.init() assert conn.execute("SELECT * FROM traust_storage_meta").fetchall() == original conn.execute("ALTER TABLE report_finding DROP COLUMN blocked_external") - conn.execute("UPDATE traust_storage_meta SET revision=?", (2,)) + conn.execute("UPDATE traust_storage_meta SET revision=?", (REVISION + 1,)) conn.commit() legacy_dump = tuple(conn.iterdump()) with pytest.raises(IngestError, match=r"explicit migration"): diff --git a/uv.lock b/uv.lock index 73ece87..890565f 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.11" [[package]] @@ -641,7 +641,7 @@ wheels = [ [[package]] name = "traust-contracts" -version = "0.53.0" +version = "0.54.0" source = { editable = "." } dependencies = [ { name = "jsonschema", extra = ["format-nongpl"] },