From bfd52739ce38ad5fc332e801f54ac0d5fa6617ad Mon Sep 17 00:00:00 2001 From: Shaun Smith <51304449+smith-xyz@users.noreply.github.com> Date: Wed, 21 Oct 2026 09:09:19 -0400 Subject: [PATCH 01/10] chore(contracts): pin the one-owner db contract; load DDL per namespace; park the retired traust_storage lane --- pyproject.toml | 2 +- src/traust_core/v1/contracts.py | 92 ++++++------------- tests/v1/conftest.py | 27 +++++- tests/v1/repositories/test_triage.py | 2 + tests/v1/services/test_artifact_publishing.py | 11 ++- tests/v1/services/test_storage.py | 2 + tests/v1/test_contracts.py | 14 +-- tests/v1/test_layering.py | 2 +- uv.lock | 6 +- 9 files changed, 81 insertions(+), 77 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b2790bc0..57a48c3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dev = [ ] [tool.uv.sources] -traust-contracts = { git = "https://github.com/traust-security/traust-contracts.git", rev = "104519fd18f724e121c492c160e3dc2d67408f90" } +traust-contracts = { git = "https://github.com/traust-security/traust-contracts.git", rev = "1da48c06493e5b88b99445299e025bfc7c46673a" } [tool.hatch.build.targets.wheel] packages = ["src/traust_core"] diff --git a/src/traust_core/v1/contracts.py b/src/traust_core/v1/contracts.py index f62a03a9..f721ad8f 100644 --- a/src/traust_core/v1/contracts.py +++ b/src/traust_core/v1/contracts.py @@ -1,56 +1,27 @@ from __future__ import annotations import json -import sqlite3 -from collections.abc import Iterator, Mapping +from collections.abc import Mapping from dataclasses import dataclass from enum import StrEnum from functools import cache from importlib import resources from importlib.metadata import distribution from pathlib import Path -from types import MappingProxyType +from types import MappingProxyType, ModuleType from typing import Any, Literal from jsonschema import Draft202012Validator, FormatChecker from referencing import Registry, Resource +from traust_contracts.v1.ledger import sql as ledger_sql +from traust_contracts.v1.ops import sql as ops_sql +from traust_contracts.v1.storage import sql as storage_sql from traust_core.v1.errors import ConfigError, DocumentError, Issue Dialect = Literal["sqlite", "postgres"] -Area = Literal["storage", "ledger"] - - -@dataclass(frozen=True, slots=True) -class _Bootstrap: - first_tables: tuple[str, ...] - view_order: tuple[str, ...] = () - - -# Bootstrap order is in traust-contracts' Python today; to move into its data. -_BOOTSTRAP: dict[Area, _Bootstrap] = { - "storage": _Bootstrap( - first_tables=("artifact_evidence", "artifact_binding", "artifact_location"), - view_order=( - "binding_current", - "report_current", - "ownership_current", - "current_finding", - "threat_current", - "validation_current", - "finding_first_seen", - "finding_timeline", - "pqc_posture", - "sla_clock", - "sla_threshold", - "pattern_exposure", - "attack_coverage", - ), - ), - "ledger": _Bootstrap( - first_tables=("schema_revision", "layers", "events", "materialized_findings") - ), -} +Area = Literal["traust", "traust_ops", "traust_ledger"] +AREAS: tuple[Area, ...] = ("traust", "traust_ops", "traust_ledger") @cache @@ -69,8 +40,12 @@ def _read_json(path: Path) -> dict[str, Any]: raise ConfigError(f"traust-contracts has no {path.relative_to(_root())}") from None +def _schema_dir() -> Path: + return _root() / "artifacts_schema/v1" + + def schema_path(name: str) -> Path: - return _root() / "schemas/v1" / f"{name}.schema.json" + return _schema_dir() / f"{name}.schema.json" def schema(name: str) -> dict[str, Any]: @@ -78,9 +53,7 @@ def schema(name: str) -> dict[str, Any]: def schema_names() -> list[str]: - return sorted( - p.name.removesuffix(".schema.json") for p in (_root() / "schemas/v1").glob("*.json") - ) + return sorted(p.name.removesuffix(".schema.json") for p in _schema_dir().glob("*.json")) def enum_values(name: str) -> tuple[str, ...]: @@ -162,7 +135,7 @@ def storage_profile(name: str) -> dict[str, Any]: @cache def _registry() -> Registry: resources_ = [] - for path in (_root() / "schemas/v1").glob("*.schema.json"): + for path in _schema_dir().glob("*.schema.json"): resource = Resource.from_contents(json.loads(path.read_text())) resources_ += [(path.name, resource), (resource.id() or path.name, resource)] return Registry().with_resources(resources_) @@ -190,29 +163,22 @@ def validate_json(name: str, payload: bytes) -> Any: return document +def _loader(area: Area) -> ModuleType: + # One db contract, three namespaces; bootstrap order: traust, then ops, then ledger. + return {"traust": storage_sql, "traust_ops": ops_sql, "traust_ledger": ledger_sql}[area] + + def ddl_files(area: Area, dialect: Dialect) -> list[Path]: - plan = _BOOTSTRAP[area] - root = _root() / f"{area}/v1" / dialect - tables = {p.stem: p for p in (root / "schema").glob("*.sql")} - first = [tables.pop(t) for t in plan.first_tables] - views = {p.stem: p for p in (root / "views").glob("*.sql")} if (root / "views").is_dir() else {} - ordered_views = [views.pop(v) for v in plan.view_order if v in views] - namespace = [root / "namespace.sql"] if dialect == "postgres" else [] - return [*namespace, *first, *sorted(tables.values()), *ordered_views, *sorted(views.values())] - - -def _statements(dialect: Dialect, path: Path) -> Iterator[str]: - sql = path.read_text(encoding="utf-8") - if dialect == "postgres": - yield sql - return - statement = "" - for line in sql.splitlines(keepends=True): - statement += line - if sqlite3.complete_statement(statement): - yield statement - statement = "" + return _loader(area).bootstrap_files(dialect) def ddl(area: Area, dialect: Dialect) -> list[str]: - return [s for path in ddl_files(area, dialect) for s in _statements(dialect, path)] + loader = _loader(area) + return [ + s for path in ddl_files(area, dialect) for s in loader.bootstrap_statements(dialect, path) + ] + + +def database_ddl(dialect: Dialect) -> list[str]: + """Every namespace, in bootstrap order: the whole db contract.""" + return [s for area in AREAS for s in ddl(area, dialect)] diff --git a/tests/v1/conftest.py b/tests/v1/conftest.py index 2e45b260..22312ec0 100644 --- a/tests/v1/conftest.py +++ b/tests/v1/conftest.py @@ -25,6 +25,10 @@ def apply_contract_ddl(engine: Engine, area: contracts.Area) -> None: apply_schema(engine, contracts.ddl(area, contract_dialect(engine))) +def apply_contract_db(engine: Engine) -> None: + apply_schema(engine, contracts.database_ddl(contract_dialect(engine))) + + def _engine(kind: str, tmp_path: Path) -> Engine: if kind == "postgres" and not PG_URL: pytest.skip("TRAUST_TEST_DATABASE_URL not set") @@ -47,7 +51,7 @@ def database( @contextmanager def storage_db(kind: str, tmp_path: Path) -> Generator[Engine]: eng = _engine(kind, tmp_path) - apply_contract_ddl(eng, "storage") + apply_contract_ddl(eng, "storage") # type: ignore[arg-type] # parked: storage-retire try: yield eng finally: @@ -57,6 +61,27 @@ def storage_db(kind: str, tmp_path: Path) -> Generator[Engine]: eng.dispose() +@contextmanager +def contract_db(kind: str, tmp_path: Path) -> Generator[Engine]: + """A fresh database bootstrapped from the whole contracts db (every namespace).""" + eng = _engine(kind, tmp_path) + if kind == "postgres": + _drop_namespaces(eng) + apply_contract_db(eng) + try: + yield eng + finally: + if kind == "postgres": + _drop_namespaces(eng) + eng.dispose() + + +def _drop_namespaces(engine: Engine) -> None: + with engine.begin() as conn: + for area in reversed(contracts.AREAS): + conn.exec_driver_sql(f"DROP SCHEMA IF EXISTS {area} CASCADE") + + def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: for item in items: if mark := item.get_closest_marker("todo"): diff --git a/tests/v1/repositories/test_triage.py b/tests/v1/repositories/test_triage.py index 76c9cf88..6d0b97b8 100644 --- a/tests/v1/repositories/test_triage.py +++ b/tests/v1/repositories/test_triage.py @@ -18,6 +18,8 @@ ) from traust_core.v1.services.findings import record_triage +pytestmark = pytest.mark.todo("storage-retire: traust_storage DDL and profiles left contracts") + BACKENDS = ["memory", *SQL_BACKENDS] diff --git a/tests/v1/services/test_artifact_publishing.py b/tests/v1/services/test_artifact_publishing.py index a0509a55..cf16cc18 100644 --- a/tests/v1/services/test_artifact_publishing.py +++ b/tests/v1/services/test_artifact_publishing.py @@ -17,6 +17,8 @@ from traust_core.v1.services.artifact_publishing import ArtifactPublisher, StorageIndex from traust_core.v1.services.storage import get_binding +STORAGE_RETIRED = pytest.mark.todo("storage-retire: traust_storage DDL and profiles left contracts") + SUBJECT = Subject(tree="findings", product="openshift", repo_dir="oc", base="oc") FINDING = { "id": "f001", @@ -34,7 +36,13 @@ def triage_with(*findings: dict) -> bytes: return json.dumps(doc).encode() -@pytest.fixture(params=["files-only", "memory-index", "sqlite-index"]) +@pytest.fixture( + params=[ + "files-only", + pytest.param("memory-index", marks=STORAGE_RETIRED), + pytest.param("sqlite-index", marks=STORAGE_RETIRED), + ] +) def setup( request: pytest.FixtureRequest, tmp_path: Path ) -> Iterator[tuple[ArtifactPublisher, object]]: @@ -152,6 +160,7 @@ def test_resubmitting_is_idempotent(setup: tuple[ArtifactPublisher, object]) -> assert (again.written, again.binding_id) == (first.written, first.binding_id) +@STORAGE_RETIRED def test_context_builds_files_only_or_indexed_from_config(tmp_path: Path) -> None: files_only = Context(Config.model_validate({"artifacts": {"path": str(tmp_path / "ar")}})) assert ( diff --git a/tests/v1/services/test_storage.py b/tests/v1/services/test_storage.py index f3095155..f04ff88d 100644 --- a/tests/v1/services/test_storage.py +++ b/tests/v1/services/test_storage.py @@ -17,6 +17,8 @@ ) from traust_core.v1.services.storage import get_binding, record_artifact +pytestmark = pytest.mark.todo("storage-retire: traust_storage DDL and profiles left contracts") + DIGEST = "a" * 64 RUN = BindingContext(subject_id="org/repo", run_id="r1") TRIAGE = json.dumps( diff --git a/tests/v1/test_contracts.py b/tests/v1/test_contracts.py index 4fb944d9..d68fb9ef 100644 --- a/tests/v1/test_contracts.py +++ b/tests/v1/test_contracts.py @@ -4,7 +4,7 @@ import pytest import sqlalchemy as sa -from tests.v1.conftest import apply_contract_ddl +from tests.v1.conftest import apply_contract_db from traust_core.v1 import contracts from traust_core.v1.errors import DocumentError from traust_core.v1.repositories import create_database_engine, schema_drift @@ -24,18 +24,18 @@ def test_schema_gate_is_exact_and_reports_paths() -> None: assert "$" in paths or any(p.startswith("findings/0") for p in paths) -@pytest.mark.parametrize("area", ["storage", "ledger"]) -def test_contract_ddl_applies_on_sqlite(area: str, tmp_path: Path) -> None: +def test_database_ddl_applies_every_namespace_on_sqlite(tmp_path: Path) -> None: engine = create_database_engine(f"sqlite:///{tmp_path}/c.db") - apply_contract_ddl(engine, area) - assert sa.inspect(engine).get_table_names() + apply_contract_db(engine) + names = set(sa.inspect(engine).get_table_names()) + assert {"repository", "ops_run_order", "ledger_events"} <= names def test_schema_drift_flags_disagreements(tmp_path: Path) -> None: engine = create_database_engine(f"sqlite:///{tmp_path}/c.db") - apply_contract_ddl(engine, "ledger") + apply_contract_db(engine) wrong = sa.MetaData() - sa.Table("layers", wrong, sa.Column("not_in_ddl", sa.Text)) + sa.Table("ledger_layers", wrong, sa.Column("not_in_ddl", sa.Text)) problems = schema_drift(engine, wrong) assert any("not_in_ddl: missing in database" in p for p in problems) assert any("not declared in Table" in p for p in problems) diff --git a/tests/v1/test_layering.py b/tests/v1/test_layering.py index 4d2ed959..5d9eed00 100644 --- a/tests/v1/test_layering.py +++ b/tests/v1/test_layering.py @@ -18,7 +18,7 @@ ] THIRD_PARTY = { "errors": set(), - "contracts": {"jsonschema", "referencing"}, + "contracts": {"jsonschema", "referencing", "traust_contracts"}, "models": {"pydantic"}, "providers": {"pydantic"}, "repositories": {"sqlalchemy", "psycopg", "pydantic"}, diff --git a/uv.lock b/uv.lock index 52e657b0..9a0ae5f4 100644 --- a/uv.lock +++ b/uv.lock @@ -937,8 +937,8 @@ wheels = [ [[package]] name = "traust-contracts" -version = "0.48.1" -source = { git = "https://github.com/traust-security/traust-contracts.git?rev=104519fd18f724e121c492c160e3dc2d67408f90#104519fd18f724e121c492c160e3dc2d67408f90" } +version = "0.0.0" +source = { git = "https://github.com/traust-security/traust-contracts.git?rev=1da48c06493e5b88b99445299e025bfc7c46673a#1da48c06493e5b88b99445299e025bfc7c46673a" } dependencies = [ { name = "jsonschema", extra = ["format-nongpl"] }, { name = "pydantic" }, @@ -978,7 +978,7 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6.0" }, { name = "referencing", specifier = ">=0.30" }, { name = "sqlalchemy", specifier = ">=2.0" }, - { name = "traust-contracts", git = "https://github.com/traust-security/traust-contracts.git?rev=104519fd18f724e121c492c160e3dc2d67408f90" }, + { name = "traust-contracts", git = "https://github.com/traust-security/traust-contracts.git?rev=1da48c06493e5b88b99445299e025bfc7c46673a" }, ] provides-extras = ["postgres"] From b1103a13f841fd690ce01c0018ea2c20b876156d Mon Sep 17 00:00:00 2001 From: Shaun Smith <51304449+smith-xyz@users.noreply.github.com> Date: Wed, 21 Oct 2026 09:16:51 -0400 Subject: [PATCH 02/10] feat(repositories): one repository per contract table, built per dialect by a TableFactory (XWING-2363) --- src/traust_core/v1/context/context.py | 12 + src/traust_core/v1/models/records/__init__.py | 78 +++++ .../v1/models/records/dependencies.py | 45 +++ src/traust_core/v1/models/records/findings.py | 110 ++++++ .../v1/models/records/inventory.py | 49 +++ src/traust_core/v1/models/records/ledger.py | 25 ++ src/traust_core/v1/models/records/ops.py | 54 +++ src/traust_core/v1/repositories/contract.py | 32 ++ src/traust_core/v1/repositories/sql.py | 26 +- src/traust_core/v1/repositories/storage.py | 2 +- src/traust_core/v1/repositories/tables.py | 179 ++++++++++ .../v1/repositories/traust/__init__.py | 1 + .../traust/assessments/__init__.py | 61 ++++ .../traust/assessments/artifact.py | 51 +++ .../traust/assessments/assessment.py | 67 ++++ .../traust/assessments/location.py | 47 +++ .../traust/assessments/occurrence.py | 55 +++ .../traust/dependencies/__init__.py | 79 +++++ .../traust/dependencies/advisory.py | 41 +++ .../traust/dependencies/advisory_package.py | 42 +++ .../traust/dependencies/dependency.py | 45 +++ .../repositories/traust/dependencies/image.py | 50 +++ .../traust/dependencies/package.py | 50 +++ .../repositories/traust/dependencies/sbom.py | 32 ++ .../repositories/traust/findings/__init__.py | 92 +++++ .../v1/repositories/traust/findings/cwe.py | 35 ++ .../repositories/traust/findings/decision.py | 75 ++++ .../repositories/traust/findings/finding.py | 61 ++++ .../traust/findings/finding_cwe.py | 33 ++ .../traust/findings/finding_external_ref.py | 37 ++ .../traust/findings/finding_label.py | 37 ++ .../traust/findings/review_item.py | 46 +++ .../repositories/traust/inventory/__init__.py | 88 +++++ .../traust/inventory/component.py | 44 +++ .../traust/inventory/ownership.py | 57 ++++ .../repositories/traust/inventory/product.py | 26 ++ .../repositories/traust/inventory/release.py | 33 ++ .../traust/inventory/repository.py | 37 ++ .../traust/inventory/repository_ref.py | 69 ++++ .../v1/repositories/traust/inventory/team.py | 26 ++ .../v1/repositories/traust_ledger/__init__.py | 46 +++ .../v1/repositories/traust_ledger/events.py | 50 +++ .../v1/repositories/traust_ledger/layers.py | 43 +++ .../v1/repositories/traust_ops/__init__.py | 73 ++++ .../traust_ops/assessment_input.py | 39 +++ .../repositories/traust_ops/feed_snapshot.py | 46 +++ .../v1/repositories/traust_ops/run_batch.py | 27 ++ .../v1/repositories/traust_ops/run_order.py | 102 ++++++ .../v1/repositories/traust_ops/spend_event.py | 57 ++++ src/traust_core/v1/repositories/triage.py | 3 +- tests/v1/repositories/tables/__init__.py | 0 tests/v1/repositories/tables/test_factory.py | 83 +++++ .../repositories/tables/test_repositories.py | 322 ++++++++++++++++++ 53 files changed, 2908 insertions(+), 12 deletions(-) create mode 100644 src/traust_core/v1/models/records/__init__.py create mode 100644 src/traust_core/v1/models/records/dependencies.py create mode 100644 src/traust_core/v1/models/records/findings.py create mode 100644 src/traust_core/v1/models/records/inventory.py create mode 100644 src/traust_core/v1/models/records/ledger.py create mode 100644 src/traust_core/v1/models/records/ops.py create mode 100644 src/traust_core/v1/repositories/contract.py create mode 100644 src/traust_core/v1/repositories/tables.py create mode 100644 src/traust_core/v1/repositories/traust/__init__.py create mode 100644 src/traust_core/v1/repositories/traust/assessments/__init__.py create mode 100644 src/traust_core/v1/repositories/traust/assessments/artifact.py create mode 100644 src/traust_core/v1/repositories/traust/assessments/assessment.py create mode 100644 src/traust_core/v1/repositories/traust/assessments/location.py create mode 100644 src/traust_core/v1/repositories/traust/assessments/occurrence.py create mode 100644 src/traust_core/v1/repositories/traust/dependencies/__init__.py create mode 100644 src/traust_core/v1/repositories/traust/dependencies/advisory.py create mode 100644 src/traust_core/v1/repositories/traust/dependencies/advisory_package.py create mode 100644 src/traust_core/v1/repositories/traust/dependencies/dependency.py create mode 100644 src/traust_core/v1/repositories/traust/dependencies/image.py create mode 100644 src/traust_core/v1/repositories/traust/dependencies/package.py create mode 100644 src/traust_core/v1/repositories/traust/dependencies/sbom.py create mode 100644 src/traust_core/v1/repositories/traust/findings/__init__.py create mode 100644 src/traust_core/v1/repositories/traust/findings/cwe.py create mode 100644 src/traust_core/v1/repositories/traust/findings/decision.py create mode 100644 src/traust_core/v1/repositories/traust/findings/finding.py create mode 100644 src/traust_core/v1/repositories/traust/findings/finding_cwe.py create mode 100644 src/traust_core/v1/repositories/traust/findings/finding_external_ref.py create mode 100644 src/traust_core/v1/repositories/traust/findings/finding_label.py create mode 100644 src/traust_core/v1/repositories/traust/findings/review_item.py create mode 100644 src/traust_core/v1/repositories/traust/inventory/__init__.py create mode 100644 src/traust_core/v1/repositories/traust/inventory/component.py create mode 100644 src/traust_core/v1/repositories/traust/inventory/ownership.py create mode 100644 src/traust_core/v1/repositories/traust/inventory/product.py create mode 100644 src/traust_core/v1/repositories/traust/inventory/release.py create mode 100644 src/traust_core/v1/repositories/traust/inventory/repository.py create mode 100644 src/traust_core/v1/repositories/traust/inventory/repository_ref.py create mode 100644 src/traust_core/v1/repositories/traust/inventory/team.py create mode 100644 src/traust_core/v1/repositories/traust_ledger/__init__.py create mode 100644 src/traust_core/v1/repositories/traust_ledger/events.py create mode 100644 src/traust_core/v1/repositories/traust_ledger/layers.py create mode 100644 src/traust_core/v1/repositories/traust_ops/__init__.py create mode 100644 src/traust_core/v1/repositories/traust_ops/assessment_input.py create mode 100644 src/traust_core/v1/repositories/traust_ops/feed_snapshot.py create mode 100644 src/traust_core/v1/repositories/traust_ops/run_batch.py create mode 100644 src/traust_core/v1/repositories/traust_ops/run_order.py create mode 100644 src/traust_core/v1/repositories/traust_ops/spend_event.py create mode 100644 tests/v1/repositories/tables/__init__.py create mode 100644 tests/v1/repositories/tables/test_factory.py create mode 100644 tests/v1/repositories/tables/test_repositories.py diff --git a/src/traust_core/v1/context/context.py b/src/traust_core/v1/context/context.py index 76282271..eae14abd 100644 --- a/src/traust_core/v1/context/context.py +++ b/src/traust_core/v1/context/context.py @@ -22,7 +22,9 @@ create_database_engine, ) from traust_core.v1.repositories.analysis_results import AnalysisResultsRepository +from traust_core.v1.repositories.sql import Dialect from traust_core.v1.repositories.storage import SqlStorageUnitOfWork +from traust_core.v1.repositories.tables import TableFactory from traust_core.v1.services.artifact_publishing import ( ArtifactPublisher, PublishSpecs, @@ -77,7 +79,17 @@ def process(self) -> ProcessRunner: def database(self) -> Engine: return create_database_engine(self.config.database.engine_url()) + @cached_property + def dialect(self) -> Dialect: + return Dialect.of(self.database) + + @cached_property + def tables(self) -> TableFactory: + """Contract tables for the configured database: schemas on Postgres, prefixes on SQLite.""" + return TableFactory.for_dialect(self.dialect) + def unit_of_work(self, factory: Callable[[Engine], U]) -> U: + """e.g. unit_of_work(SqlInventoryUnitOfWork); its repositories use this engine's tables.""" return factory(self.database) def artifact_publisher(self, specs: PublishSpecs) -> ArtifactPublisher: diff --git a/src/traust_core/v1/models/records/__init__.py b/src/traust_core/v1/models/records/__init__.py new file mode 100644 index 00000000..c5d39eeb --- /dev/null +++ b/src/traust_core/v1/models/records/__init__.py @@ -0,0 +1,78 @@ +"""One row model per contracts db table (traust, traust_ops, traust_ledger). + +Records mirror the DDL column for column; domain models and services build on +them. id / created_at / updated_at are None until the database writes them. +""" + +from traust_core.v1.models.records.dependencies import ( + AdvisoryPackageRecord, + AdvisoryRecord, + DependencyRecord, + ImageRecord, + PackageRecord, + SbomRecord, +) +from traust_core.v1.models.records.findings import ( + ArtifactRecord, + AssessmentRecord, + CweRecord, + DecisionRecord, + FindingCweRecord, + FindingExternalRefRecord, + FindingLabelRecord, + FindingRecord, + LocationRecord, + OccurrenceRecord, + ReviewItemRecord, +) +from traust_core.v1.models.records.inventory import ( + ComponentRecord, + OwnershipRecord, + ProductRecord, + ReleaseRecord, + RepositoryRecord, + RepositoryRefRecord, + TeamRecord, +) +from traust_core.v1.models.records.ledger import EventRecord, LayerRecord +from traust_core.v1.models.records.ops import ( + AssessmentInputRecord, + FeedSnapshotRecord, + RunBatchRecord, + RunOrderRecord, + SpendEventRecord, +) + +__all__ = [ + "AdvisoryPackageRecord", + "AdvisoryRecord", + "ArtifactRecord", + "AssessmentInputRecord", + "AssessmentRecord", + "ComponentRecord", + "CweRecord", + "DecisionRecord", + "DependencyRecord", + "EventRecord", + "FeedSnapshotRecord", + "FindingCweRecord", + "FindingExternalRefRecord", + "FindingLabelRecord", + "FindingRecord", + "ImageRecord", + "LayerRecord", + "LocationRecord", + "OccurrenceRecord", + "OwnershipRecord", + "PackageRecord", + "ProductRecord", + "ReleaseRecord", + "RepositoryRecord", + "RepositoryRefRecord", + "ReviewItemRecord", + "RunBatchRecord", + "RunOrderRecord", + "SbomRecord", + "SpendEventRecord", + "TeamRecord", +] diff --git a/src/traust_core/v1/models/records/dependencies.py b/src/traust_core/v1/models/records/dependencies.py new file mode 100644 index 00000000..e8483d75 --- /dev/null +++ b/src/traust_core/v1/models/records/dependencies.py @@ -0,0 +1,45 @@ +"""Rows of the traust namespace, 2 · Dependencies - one model per contract table.""" + +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from traust_core.v1.models.base import AppendOnly, Entity + + +class PackageRecord(Entity): + source_repository_id: UUID | None = None + sbom_id: UUID | None = None + ecosystem: str + name: str + + +class SbomRecord(AppendOnly): + digest: str + artifact_id: UUID | None = None + + +class DependencyRecord(AppendOnly): + sbom_id: UUID + package_id: UUID + version: str + direct: bool + + +class ImageRecord(AppendOnly): + digest: str + platform: str | None = None + repository_ref_id: UUID | None = None + sbom_id: UUID | None = None + + +class AdvisoryRecord(Entity): + alias_of_id: UUID | None = None + identifier: str + published_at: datetime | None = None + + +class AdvisoryPackageRecord(Entity): + advisory_id: UUID + package_id: UUID diff --git a/src/traust_core/v1/models/records/findings.py b/src/traust_core/v1/models/records/findings.py new file mode 100644 index 00000000..27554f4d --- /dev/null +++ b/src/traust_core/v1/models/records/findings.py @@ -0,0 +1,110 @@ +"""Rows of the traust namespace, 3 · Assessments & findings - one model per contract table. + +Classifying columns (kind, status, severity, validity, ...) are str here: their +values come from contracts registries, checked at intake by services. +""" + +from __future__ import annotations + +from datetime import datetime +from decimal import Decimal +from uuid import UUID + +from traust_core.v1.models.base import AppendOnly, Entity + + +class AssessmentRecord(AppendOnly): + repository_ref_id: UUID + image_id: UUID | None = None + baseline_assessment_id: UUID | None = None + run_order_id: UUID | None = None + kind: str + status: str + assessor: str + assessor_version: str + occurred_at: datetime + + +class ArtifactRecord(AppendOnly): + assessment_id: UUID | None = None + digest: str + kind: str + uri: str + + +class OccurrenceRecord(AppendOnly): + finding_id: UUID + assessment_id: UUID + severity: str + cvss_vector: str | None = None + cvss_score: Decimal | None = None + rule: str | None = None + + +class LocationRecord(AppendOnly): + occurrence_id: UUID + path: str + line_start: int | None = None + line_end: int | None = None + language: str | None = None + symbol: str | None = None + + +class FindingRecord(Entity): + repository_id: UUID + advisory_id: UUID | None = None + advisory_match: str | None = None + fingerprint_algo: str + fingerprint: str + kind: str | None = None + title: str + + +class CweRecord(Entity): + identifier: str + abstraction: str | None = None + mapping_usage: str | None = None + catalog_version: str | None = None + + +class FindingCweRecord(Entity): + finding_id: UUID + cwe_id: UUID + + +class FindingLabelRecord(Entity): + finding_id: UUID + taxonomy: str + label: str + + +class FindingExternalRefRecord(Entity): + finding_id: UUID + system: str + ref: str + + +class DecisionRecord(AppendOnly): + finding_id: UUID + assessment_id: UUID | None = None + fixed_repository_ref_id: UUID | None = None + repository_ref_id: UUID | None = None + restates_id: UUID | None = None + artifact_id: UUID | None = None + statement_key: str + validity: str | None = None + resolution: str | None = None + evidence_grade: str | None = None + reason: str | None = None + severity: str | None = None + source_channel: str | None = None + actor: str | None = None + occurred_at: datetime + + +class ReviewItemRecord(Entity): + finding_id: UUID + decision_id: UUID | None = None + resolved_by_decision_id: UUID | None = None + reason: str | None = None + status: str diff --git a/src/traust_core/v1/models/records/inventory.py b/src/traust_core/v1/models/records/inventory.py new file mode 100644 index 00000000..288f0109 --- /dev/null +++ b/src/traust_core/v1/models/records/inventory.py @@ -0,0 +1,49 @@ +"""Rows of the traust namespace, 1 · Inventory - one model per contract table.""" + +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from traust_core.v1.models.base import Entity + + +class ProductRecord(Entity): + slug: str + name: str + + +class ReleaseRecord(Entity): + product_id: UUID + version: str + + +class ComponentRecord(Entity): + release_id: UUID + repository_ref_id: UUID + + +class RepositoryRecord(Entity): + upstream_id: UUID | None = None + url: str + public: bool + + +class RepositoryRefRecord(Entity): + repository_id: UUID + name: str | None = None + kind: str | None = None + is_default: bool = False + sha: str + manifest_sbom_id: UUID | None = None + + +class TeamRecord(Entity): + name: str + + +class OwnershipRecord(Entity): + repository_id: UUID + team_id: UUID + valid_from: datetime + valid_to: datetime | None = None diff --git a/src/traust_core/v1/models/records/ledger.py b/src/traust_core/v1/models/records/ledger.py new file mode 100644 index 00000000..4b825218 --- /dev/null +++ b/src/traust_core/v1/models/records/ledger.py @@ -0,0 +1,25 @@ +"""Rows of the traust_ledger namespace, 5 · Ledger - one model per contract table.""" + +from __future__ import annotations + +from uuid import UUID + +from traust_core.v1.models.base import AppendOnly, Entity + + +class LayerRecord(Entity): + repository_id: UUID + line: str | None = None + merkle_root: str | None = None + merkle_epoch: int | None = None + merkle_size: int | None = None + merkle_root_signature: str | None = None + merkle_signing_method: str + merkle_signature_format: int | None = None + + +class EventRecord(AppendOnly): + layer_id: UUID + decision_id: UUID + seq: int + event_payload: bytes diff --git a/src/traust_core/v1/models/records/ops.py b/src/traust_core/v1/models/records/ops.py new file mode 100644 index 00000000..e6eb2fab --- /dev/null +++ b/src/traust_core/v1/models/records/ops.py @@ -0,0 +1,54 @@ +"""Rows of the traust_ops namespace, 4 · Operations - one model per contract table.""" + +from __future__ import annotations + +from datetime import datetime +from decimal import Decimal +from uuid import UUID + +from traust_core.v1.models.base import AppendOnly, Entity + + +class RunBatchRecord(Entity): + kind: str + period: str + + +class RunOrderRecord(Entity): + run_batch_id: UUID | None = None + repository_id: UUID | None = None + artifact_id: UUID | None = None + rule: str | None = None + event_source: str | None = None + lane: str + state: str + priority: int + reason: str | None = None + requested_by: str + lease_holder: str | None = None + lease_expires_at: datetime | None = None + started_at: datetime | None = None + finished_at: datetime | None = None + + +class SpendEventRecord(AppendOnly): + run_order_id: UUID | None = None + provider: str + model: str + skill: str | None = None + tokens_in: int + tokens_out: int + cost_usd: Decimal + occurred_at: datetime + + +class FeedSnapshotRecord(AppendOnly): + run_order_id: UUID | None = None + artifact_id: UUID + feed: str + retrieved_at: datetime + + +class AssessmentInputRecord(AppendOnly): + assessment_id: UUID + feed_snapshot_id: UUID diff --git a/src/traust_core/v1/repositories/contract.py b/src/traust_core/v1/repositories/contract.py new file mode 100644 index 00000000..f2edde18 --- /dev/null +++ b/src/traust_core/v1/repositories/contract.py @@ -0,0 +1,32 @@ +"""The whole contracts db as table specs: every namespace, every table, once.""" + +from __future__ import annotations + +from sqlalchemy.engine import Engine + +from traust_core.v1.repositories import traust_ledger, traust_ops +from traust_core.v1.repositories.sql import Dialect, schema_drift +from traust_core.v1.repositories.tables import TableFactory, TableSpec +from traust_core.v1.repositories.traust import assessments, dependencies, findings, inventory + +CONTRACT_TABLES: tuple[TableSpec, ...] = ( + *inventory.TABLES, + *dependencies.TABLES, + *assessments.TABLES, + *findings.TABLES, + *traust_ops.TABLES, + *traust_ledger.TABLES, +) + + +def contract_tables(dialect: Dialect) -> TableFactory: + """The dialect's factory with every contract table built onto its MetaData.""" + tables = TableFactory.for_dialect(dialect) + for spec in CONTRACT_TABLES: + tables.table(spec) + return tables + + +def contract_drift(engine: Engine) -> list[str]: + """Where the database and the declared tables disagree; empty = in step.""" + return schema_drift(engine, contract_tables(Dialect.of(engine)).metadata) diff --git a/src/traust_core/v1/repositories/sql.py b/src/traust_core/v1/repositories/sql.py index 4e2627dc..c3b8b84a 100644 --- a/src/traust_core/v1/repositories/sql.py +++ b/src/traust_core/v1/repositories/sql.py @@ -3,8 +3,9 @@ import json from collections.abc import Iterable, Mapping, Sequence from datetime import UTC, datetime +from enum import StrEnum from types import TracebackType -from typing import Any, Literal, Self +from typing import Any, Self from uuid import UUID, uuid4 import sqlalchemy as sa @@ -14,8 +15,16 @@ from traust_core.v1.errors import ConflictError, NotFoundError, RepositoryError -STORAGE_SCHEMA = "traust_storage" -LEDGER_SCHEMA = "traust_ledger" + +class Dialect(StrEnum): + """The two databases the contracts DDL ships for; names match its directories.""" + + SQLITE = "sqlite" + POSTGRES = "postgres" + + @classmethod + def of(cls, bind: Engine | Connection) -> Dialect: + return cls.POSTGRES if bind.dialect.name == "postgresql" else cls.SQLITE def _compact_json(value: Any) -> str: @@ -33,13 +42,11 @@ def _sqlite_functions(dbapi_conn: Any, _: Any) -> None: # Contracts' uuid(): ext/misc/uuid.c in other drivers, registered here. dbapi_conn.create_function("uuid", 0, lambda: str(uuid4())) - return engine.execution_options( - schema_translate_map={STORAGE_SCHEMA: None, LEDGER_SCHEMA: None} - ) + return engine -def contract_dialect(engine: Engine | Connection) -> Literal["sqlite", "postgres"]: - return "postgres" if engine.dialect.name == "postgresql" else "sqlite" +def contract_dialect(engine: Engine | Connection) -> Dialect: + return Dialect.of(engine) class UtcTimestamp(sa.TypeDecorator[datetime]): @@ -247,10 +254,9 @@ def _execute( def schema_drift(engine: Engine, metadata: sa.MetaData) -> list[str]: inspector = sa.inspect(engine) - translate = engine.get_execution_options().get("schema_translate_map", {}) problems: list[str] = [] for table in metadata.sorted_tables: - schema = translate.get(table.schema, table.schema) + schema = table.schema if not inspector.has_table(table.name, schema=schema): problems.append(f"{table.name}: missing in database") continue diff --git a/src/traust_core/v1/repositories/storage.py b/src/traust_core/v1/repositories/storage.py index f887bdd5..b1574c29 100644 --- a/src/traust_core/v1/repositories/storage.py +++ b/src/traust_core/v1/repositories/storage.py @@ -10,13 +10,13 @@ from traust_core.v1.models.storage import ArtifactBinding, BindingContext, Evidence from traust_core.v1.repositories.memory import InMemoryUnitOfWork from traust_core.v1.repositories.sql import ( - STORAGE_SCHEMA, SqlRepository, SqlUnitOfWork, UtcTimestamp, ) from traust_core.v1.repositories.unit_of_work import UnitOfWork +STORAGE_SCHEMA = "traust_storage" # retired by the one-owner contract; parked: storage-retire metadata = sa.MetaData(schema=STORAGE_SCHEMA) artifact_evidence = sa.Table( diff --git a/src/traust_core/v1/repositories/tables.py b/src/traust_core/v1/repositories/tables.py new file mode 100644 index 00000000..42303f08 --- /dev/null +++ b/src/traust_core/v1/repositories/tables.py @@ -0,0 +1,179 @@ +"""Contract tables, built per dialect. + +The contracts DDL places the three namespaces differently per database: +Postgres schemas (`traust_ops.run_order`), SQLite table prefixes in one file +(`ops_run_order`) because FKs do not work across ATTACHed files. A TableSpec +declares a table once; the TableFactory for a dialect builds it with the +right schema and name. Repositories ask the factory for their table, so one +repository class serves both databases. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from functools import cache +from typing import Any, ClassVar, Generic, Protocol, TypeVar +from uuid import UUID + +import sqlalchemy as sa +from sqlalchemy.engine import Connection + +from traust_core.v1.errors import NotFoundError +from traust_core.v1.models.base import AppendOnly, Entity, Model +from traust_core.v1.repositories.sql import ( + Dialect, + RecordUuid, + SqlRepository, + SqlUnitOfWork, + UtcTimestamp, + append_only_columns, + record_columns, +) + + +@dataclass(frozen=True, slots=True) +class Namespace: + schema: str + sqlite_prefix: str = "" + + def locate(self, name: str, dialect: Dialect) -> tuple[str | None, str]: + if dialect is Dialect.POSTGRES: + return self.schema, name + return None, f"{self.sqlite_prefix}{name}" + + +TRAUST = Namespace("traust") +TRAUST_OPS = Namespace("traust_ops", "ops_") +TRAUST_LEDGER = Namespace("traust_ledger", "ledger_") + + +@dataclass(frozen=True, slots=True, eq=False) +class TableSpec: + """One contract table: domain columns only; base columns follow append_only.""" + + namespace: Namespace + name: str + columns: Callable[[], Sequence[sa.Column[Any]]] = field(repr=False) + append_only: bool = False + + def build(self, metadata: sa.MetaData, dialect: Dialect) -> sa.Table: + schema, name = self.namespace.locate(self.name, dialect) + base = append_only_columns() if self.append_only else record_columns() + return sa.Table(name, metadata, *base, *self.columns(), schema=schema) + + +class TableFactory: + """Every contract table for one dialect, on one MetaData (used by schema_drift).""" + + def __init__(self, dialect: Dialect) -> None: + self.dialect = dialect + self.metadata = sa.MetaData() + self._tables: dict[int, sa.Table] = {} + + @classmethod + @cache + def for_dialect(cls, dialect: Dialect) -> TableFactory: + return cls(dialect) + + @classmethod + def for_bind(cls, bind: Connection | sa.Engine) -> TableFactory: + return cls.for_dialect(Dialect.of(bind)) + + def table(self, spec: TableSpec) -> sa.Table: + if (table := self._tables.get(id(spec))) is None: + table = self._tables[id(spec)] = spec.build(self.metadata, self.dialect) + return table + + +def uuid_column(name: str, *, nullable: bool = False) -> sa.Column[UUID]: + return sa.Column(name, RecordUuid, nullable=nullable) + + +def text_column(name: str, *, nullable: bool = False) -> sa.Column[str]: + return sa.Column(name, sa.Text, nullable=nullable) + + +def timestamp_column(name: str, *, nullable: bool = False) -> sa.Column[Any]: + return sa.Column(name, UtcTimestamp, nullable=nullable) + + +A = TypeVar("A", bound=AppendOnly) +E = TypeVar("E", bound=Entity) +R = TypeVar("R", bound=Model) + +_BASE = frozenset({"id", "created_at", "updated_at"}) + + +class RecordRepository(Protocol[R]): + def add(self, record: R) -> R: ... + def get(self, record_id: UUID) -> R: ... + + +class MutableRecordRepository(RecordRepository[E], Protocol[E]): + def update(self, record: E) -> E: ... + + +class SqlTableRepository(SqlRepository, Generic[R]): + """add / get for one contract table; subclasses set spec and record.""" + + spec: ClassVar[TableSpec] + record: ClassVar[type[Any]] + + def __init__(self, connection: Connection, tables: TableFactory | None = None) -> None: + super().__init__(connection) + self.table = (tables or TableFactory.for_bind(connection)).table(self.spec) + + def add(self, record: R) -> R: + saved = self._insert_record(f"add {self.spec.name}", self.table, self._values(record)) + return record.model_copy(update=saved) + + def get(self, record_id: UUID) -> R: + found = self._one(f"get {record_id}", self.table.c.id == record_id) + if found is None: + raise NotFoundError(f"{self.spec.name} {record_id}") + return found + + def _values(self, record: R) -> dict[str, Any]: + return record.model_dump(exclude=_BASE & set(type(record).model_fields)) + + def _to_record(self, row: Mapping[str, Any]) -> R: + return self.record.model_validate(dict(row)) # type: ignore[no-any-return] + + def _select(self) -> sa.Select[Any]: + return sa.select(self.table) + + def _one(self, operation: str, *where: sa.ColumnElement[bool]) -> R | None: + row = self._execute(operation, self._select().where(*where)).mappings().one_or_none() + return None if row is None else self._to_record(row) + + def _many( + self, + operation: str, + *where: sa.ColumnElement[bool], + order_by: Sequence[sa.ColumnElement[Any]] = (), + ) -> list[R]: + stmt = self._select().where(*where).order_by(*order_by, self.table.c.created_at) + return [self._to_record(r) for r in self._execute(operation, stmt).mappings()] + + +class SqlAppendOnlyRepository(SqlTableRepository[A]): + """Append-only table: add and read; the contracts trigger rejects mutation.""" + + +class SqlMutableRepository(SqlTableRepository[E]): + def update(self, record: E) -> E: + if record.id is None: + raise NotFoundError(f"{self.spec.name}: update needs a saved record") + saved = self._update_record( + f"update {record.id}", self.table, record.id, self._values(record) + ) + return record.model_copy(update=saved) + + +class SqlTablesUnitOfWork(SqlUnitOfWork): + """A unit of work whose repositories share the engine dialect's TableFactory.""" + + def __init__(self, engine: sa.Engine) -> None: + super().__init__(engine) + self.tables = TableFactory.for_bind(engine) diff --git a/src/traust_core/v1/repositories/traust/__init__.py b/src/traust_core/v1/repositories/traust/__init__.py new file mode 100644 index 00000000..cad17b00 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/__init__.py @@ -0,0 +1 @@ +"""The traust namespace: domain tables (inventory, dependencies, assessments & findings).""" diff --git a/src/traust_core/v1/repositories/traust/assessments/__init__.py b/src/traust_core/v1/repositories/traust/assessments/__init__.py new file mode 100644 index 00000000..824c4e03 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/assessments/__init__.py @@ -0,0 +1,61 @@ +"""3a · Assessments - what was examined. One repository per table.""" + +from __future__ import annotations + +from typing import Protocol + +from traust_core.v1.repositories.tables import SqlTablesUnitOfWork +from traust_core.v1.repositories.traust.assessments.artifact import ( + ARTIFACT, + ArtifactRepository, + SqlArtifactRepository, +) +from traust_core.v1.repositories.traust.assessments.assessment import ( + ASSESSMENT, + AssessmentRepository, + SqlAssessmentRepository, +) +from traust_core.v1.repositories.traust.assessments.location import ( + LOCATION, + LocationRepository, + SqlLocationRepository, +) +from traust_core.v1.repositories.traust.assessments.occurrence import ( + OCCURRENCE, + OccurrenceRepository, + SqlOccurrenceRepository, +) +from traust_core.v1.repositories.unit_of_work import UnitOfWork + +TABLES = (ASSESSMENT, ARTIFACT, OCCURRENCE, LOCATION) + + +class AssessmentsUnitOfWork(UnitOfWork, Protocol): + assessments: AssessmentRepository + artifacts: ArtifactRepository + occurrences: OccurrenceRepository + locations: LocationRepository + + +class SqlAssessmentsUnitOfWork(SqlTablesUnitOfWork): + def _open_repositories(self) -> None: + conn, tables = self.connection, self.tables + self.assessments = SqlAssessmentRepository(conn, tables) + self.artifacts = SqlArtifactRepository(conn, tables) + self.occurrences = SqlOccurrenceRepository(conn, tables) + self.locations = SqlLocationRepository(conn, tables) + + +__all__ = [ + "TABLES", + "ArtifactRepository", + "AssessmentRepository", + "AssessmentsUnitOfWork", + "LocationRepository", + "OccurrenceRepository", + "SqlArtifactRepository", + "SqlAssessmentRepository", + "SqlAssessmentsUnitOfWork", + "SqlLocationRepository", + "SqlOccurrenceRepository", +] diff --git a/src/traust_core/v1/repositories/traust/assessments/artifact.py b/src/traust_core/v1/repositories/traust/assessments/artifact.py new file mode 100644 index 00000000..b520f14f --- /dev/null +++ b/src/traust_core/v1/repositories/traust/assessments/artifact.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +from traust_core.v1.models.records import ArtifactRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + RecordRepository, + SqlAppendOnlyRepository, + TableSpec, + text_column, + uuid_column, +) + +ARTIFACT = TableSpec( + TRAUST, + "artifact", + lambda: ( + uuid_column("assessment_id", nullable=True), + text_column("digest"), + text_column("kind"), + text_column("uri"), + ), + append_only=True, +) + + +class ArtifactRepository(RecordRepository[ArtifactRecord], Protocol): + def find(self, assessment_id: UUID | None, digest: str) -> ArtifactRecord | None: + """By (assessment, digest); assessment None = an imported artifact.""" + ... + + def list_for_assessment(self, assessment_id: UUID) -> list[ArtifactRecord]: ... + + +class SqlArtifactRepository(SqlAppendOnlyRepository[ArtifactRecord]): + spec = ARTIFACT + record = ArtifactRecord + + def find(self, assessment_id: UUID | None, digest: str) -> ArtifactRecord | None: + c = self.table.c + produced = ( + c.assessment_id.is_(None) + if assessment_id is None + else (c.assessment_id == assessment_id) + ) + return self._one(f"find {digest}", produced, c.digest == digest) + + def list_for_assessment(self, assessment_id: UUID) -> list[ArtifactRecord]: + return self._many(f"list for {assessment_id}", self.table.c.assessment_id == assessment_id) diff --git a/src/traust_core/v1/repositories/traust/assessments/assessment.py b/src/traust_core/v1/repositories/traust/assessments/assessment.py new file mode 100644 index 00000000..dfb21381 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/assessments/assessment.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +from traust_core.v1.models.records import AssessmentRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + RecordRepository, + SqlAppendOnlyRepository, + TableSpec, + text_column, + timestamp_column, + uuid_column, +) + +ASSESSMENT = TableSpec( + TRAUST, + "assessment", + lambda: ( + uuid_column("repository_ref_id"), + uuid_column("image_id", nullable=True), + uuid_column("baseline_assessment_id", nullable=True), + uuid_column("run_order_id", nullable=True), + text_column("kind"), + text_column("status"), + text_column("assessor"), + text_column("assessor_version"), + timestamp_column("occurred_at"), + ), + append_only=True, +) + + +class AssessmentRepository(RecordRepository[AssessmentRecord], Protocol): + def latest(self, repository_ref_id: UUID, kind: str) -> AssessmentRecord | None: ... + def list_for_ref(self, repository_ref_id: UUID) -> list[AssessmentRecord]: ... + def list_for_run_order(self, run_order_id: UUID) -> list[AssessmentRecord]: ... + + +class SqlAssessmentRepository(SqlAppendOnlyRepository[AssessmentRecord]): + spec = ASSESSMENT + record = AssessmentRecord + + def latest(self, repository_ref_id: UUID, kind: str) -> AssessmentRecord | None: + c = self.table.c + stmt = ( + self._select() + .where(c.repository_ref_id == repository_ref_id, c.kind == kind) + .order_by(c.occurred_at.desc(), c.created_at.desc()) + .limit(1) + ) + row = self._execute(f"latest {kind} for {repository_ref_id}", stmt).mappings().first() + return None if row is None else self._to_record(row) + + def list_for_ref(self, repository_ref_id: UUID) -> list[AssessmentRecord]: + c = self.table.c + return self._many( + f"list for {repository_ref_id}", + c.repository_ref_id == repository_ref_id, + order_by=(c.occurred_at,), + ) + + def list_for_run_order(self, run_order_id: UUID) -> list[AssessmentRecord]: + return self._many( + f"list for order {run_order_id}", self.table.c.run_order_id == run_order_id + ) diff --git a/src/traust_core/v1/repositories/traust/assessments/location.py b/src/traust_core/v1/repositories/traust/assessments/location.py new file mode 100644 index 00000000..96b01471 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/assessments/location.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +import sqlalchemy as sa + +from traust_core.v1.models.records import LocationRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + RecordRepository, + SqlAppendOnlyRepository, + TableSpec, + text_column, + uuid_column, +) + +LOCATION = TableSpec( + TRAUST, + "location", + lambda: ( + uuid_column("occurrence_id"), + text_column("path"), + sa.Column("line_start", sa.Integer, nullable=True), + sa.Column("line_end", sa.Integer, nullable=True), + text_column("language", nullable=True), + text_column("symbol", nullable=True), + ), + append_only=True, +) + + +class LocationRepository(RecordRepository[LocationRecord], Protocol): + def list_for_occurrence(self, occurrence_id: UUID) -> list[LocationRecord]: ... + + +class SqlLocationRepository(SqlAppendOnlyRepository[LocationRecord]): + spec = LOCATION + record = LocationRecord + + def list_for_occurrence(self, occurrence_id: UUID) -> list[LocationRecord]: + c = self.table.c + return self._many( + f"list for {occurrence_id}", + c.occurrence_id == occurrence_id, + order_by=(c.path, c.line_start), + ) diff --git a/src/traust_core/v1/repositories/traust/assessments/occurrence.py b/src/traust_core/v1/repositories/traust/assessments/occurrence.py new file mode 100644 index 00000000..714a52fc --- /dev/null +++ b/src/traust_core/v1/repositories/traust/assessments/occurrence.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +import sqlalchemy as sa + +from traust_core.v1.models.records import OccurrenceRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + RecordRepository, + SqlAppendOnlyRepository, + TableSpec, + text_column, + uuid_column, +) + +OCCURRENCE = TableSpec( + TRAUST, + "occurrence", + lambda: ( + uuid_column("finding_id"), + uuid_column("assessment_id"), + text_column("severity"), + text_column("cvss_vector", nullable=True), + sa.Column("cvss_score", sa.Numeric, nullable=True), + text_column("rule", nullable=True), + ), + append_only=True, +) + + +class OccurrenceRepository(RecordRepository[OccurrenceRecord], Protocol): + def find(self, finding_id: UUID, assessment_id: UUID) -> OccurrenceRecord | None: ... + def list_for_assessment(self, assessment_id: UUID) -> list[OccurrenceRecord]: ... + def list_for_finding(self, finding_id: UUID) -> list[OccurrenceRecord]: ... + + +class SqlOccurrenceRepository(SqlAppendOnlyRepository[OccurrenceRecord]): + spec = OCCURRENCE + record = OccurrenceRecord + + def find(self, finding_id: UUID, assessment_id: UUID) -> OccurrenceRecord | None: + c = self.table.c + return self._one( + f"find {finding_id}/{assessment_id}", + c.finding_id == finding_id, + c.assessment_id == assessment_id, + ) + + def list_for_assessment(self, assessment_id: UUID) -> list[OccurrenceRecord]: + return self._many(f"list for {assessment_id}", self.table.c.assessment_id == assessment_id) + + def list_for_finding(self, finding_id: UUID) -> list[OccurrenceRecord]: + return self._many(f"list for finding {finding_id}", self.table.c.finding_id == finding_id) diff --git a/src/traust_core/v1/repositories/traust/dependencies/__init__.py b/src/traust_core/v1/repositories/traust/dependencies/__init__.py new file mode 100644 index 00000000..e067bef4 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/dependencies/__init__.py @@ -0,0 +1,79 @@ +"""2 · Dependencies - third-party code and known vulnerabilities. One repository per table.""" + +from __future__ import annotations + +from typing import Protocol + +from traust_core.v1.repositories.tables import SqlTablesUnitOfWork +from traust_core.v1.repositories.traust.dependencies.advisory import ( + ADVISORY, + AdvisoryRepository, + SqlAdvisoryRepository, +) +from traust_core.v1.repositories.traust.dependencies.advisory_package import ( + ADVISORY_PACKAGE, + AdvisoryPackageRepository, + SqlAdvisoryPackageRepository, +) +from traust_core.v1.repositories.traust.dependencies.dependency import ( + DEPENDENCY, + DependencyRepository, + SqlDependencyRepository, +) +from traust_core.v1.repositories.traust.dependencies.image import ( + IMAGE, + ImageRepository, + SqlImageRepository, +) +from traust_core.v1.repositories.traust.dependencies.package import ( + PACKAGE, + PackageRepository, + SqlPackageRepository, +) +from traust_core.v1.repositories.traust.dependencies.sbom import ( + SBOM, + SbomRepository, + SqlSbomRepository, +) +from traust_core.v1.repositories.unit_of_work import UnitOfWork + +TABLES = (PACKAGE, SBOM, DEPENDENCY, IMAGE, ADVISORY, ADVISORY_PACKAGE) + + +class DependenciesUnitOfWork(UnitOfWork, Protocol): + packages: PackageRepository + sboms: SbomRepository + dependencies: DependencyRepository + images: ImageRepository + advisories: AdvisoryRepository + advisory_packages: AdvisoryPackageRepository + + +class SqlDependenciesUnitOfWork(SqlTablesUnitOfWork): + def _open_repositories(self) -> None: + conn, tables = self.connection, self.tables + self.packages = SqlPackageRepository(conn, tables) + self.sboms = SqlSbomRepository(conn, tables) + self.dependencies = SqlDependencyRepository(conn, tables) + self.images = SqlImageRepository(conn, tables) + self.advisories = SqlAdvisoryRepository(conn, tables) + self.advisory_packages = SqlAdvisoryPackageRepository(conn, tables) + + +__all__ = [ + "TABLES", + "AdvisoryPackageRepository", + "AdvisoryRepository", + "DependenciesUnitOfWork", + "DependencyRepository", + "ImageRepository", + "PackageRepository", + "SbomRepository", + "SqlAdvisoryPackageRepository", + "SqlAdvisoryRepository", + "SqlDependenciesUnitOfWork", + "SqlDependencyRepository", + "SqlImageRepository", + "SqlPackageRepository", + "SqlSbomRepository", +] diff --git a/src/traust_core/v1/repositories/traust/dependencies/advisory.py b/src/traust_core/v1/repositories/traust/dependencies/advisory.py new file mode 100644 index 00000000..60800366 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/dependencies/advisory.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +from traust_core.v1.models.records import AdvisoryRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + text_column, + timestamp_column, + uuid_column, +) + +ADVISORY = TableSpec( + TRAUST, + "advisory", + lambda: ( + uuid_column("alias_of_id", nullable=True), + text_column("identifier"), + timestamp_column("published_at", nullable=True), + ), +) + + +class AdvisoryRepository(MutableRecordRepository[AdvisoryRecord], Protocol): + def find_by_identifier(self, identifier: str) -> AdvisoryRecord | None: ... + def list_aliases(self, advisory_id: UUID) -> list[AdvisoryRecord]: ... + + +class SqlAdvisoryRepository(SqlMutableRepository[AdvisoryRecord]): + spec = ADVISORY + record = AdvisoryRecord + + def find_by_identifier(self, identifier: str) -> AdvisoryRecord | None: + return self._one(f"find {identifier}", self.table.c.identifier == identifier) + + def list_aliases(self, advisory_id: UUID) -> list[AdvisoryRecord]: + return self._many(f"aliases of {advisory_id}", self.table.c.alias_of_id == advisory_id) diff --git a/src/traust_core/v1/repositories/traust/dependencies/advisory_package.py b/src/traust_core/v1/repositories/traust/dependencies/advisory_package.py new file mode 100644 index 00000000..faf68042 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/dependencies/advisory_package.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +from traust_core.v1.models.records import AdvisoryPackageRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + uuid_column, +) + +ADVISORY_PACKAGE = TableSpec( + TRAUST, "advisory_package", lambda: (uuid_column("advisory_id"), uuid_column("package_id")) +) + + +class AdvisoryPackageRepository(MutableRecordRepository[AdvisoryPackageRecord], Protocol): + def find(self, advisory_id: UUID, package_id: UUID) -> AdvisoryPackageRecord | None: ... + def list_for_advisory(self, advisory_id: UUID) -> list[AdvisoryPackageRecord]: ... + def list_for_package(self, package_id: UUID) -> list[AdvisoryPackageRecord]: ... + + +class SqlAdvisoryPackageRepository(SqlMutableRepository[AdvisoryPackageRecord]): + spec = ADVISORY_PACKAGE + record = AdvisoryPackageRecord + + def find(self, advisory_id: UUID, package_id: UUID) -> AdvisoryPackageRecord | None: + c = self.table.c + return self._one( + f"find {advisory_id}/{package_id}", + c.advisory_id == advisory_id, + c.package_id == package_id, + ) + + def list_for_advisory(self, advisory_id: UUID) -> list[AdvisoryPackageRecord]: + return self._many(f"list for {advisory_id}", self.table.c.advisory_id == advisory_id) + + def list_for_package(self, package_id: UUID) -> list[AdvisoryPackageRecord]: + return self._many(f"list for package {package_id}", self.table.c.package_id == package_id) diff --git a/src/traust_core/v1/repositories/traust/dependencies/dependency.py b/src/traust_core/v1/repositories/traust/dependencies/dependency.py new file mode 100644 index 00000000..719a6bc2 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/dependencies/dependency.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +import sqlalchemy as sa + +from traust_core.v1.models.records import DependencyRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + RecordRepository, + SqlAppendOnlyRepository, + TableSpec, + text_column, + uuid_column, +) + +DEPENDENCY = TableSpec( + TRAUST, + "dependency", + lambda: ( + uuid_column("sbom_id"), + uuid_column("package_id"), + text_column("version"), + sa.Column("direct", sa.Boolean, nullable=False), + ), + append_only=True, +) + + +class DependencyRepository(RecordRepository[DependencyRecord], Protocol): + def list_for_sbom(self, sbom_id: UUID) -> list[DependencyRecord]: ... + def list_for_package(self, package_id: UUID) -> list[DependencyRecord]: ... + + +class SqlDependencyRepository(SqlAppendOnlyRepository[DependencyRecord]): + spec = DEPENDENCY + record = DependencyRecord + + def list_for_sbom(self, sbom_id: UUID) -> list[DependencyRecord]: + return self._many(f"list for sbom {sbom_id}", self.table.c.sbom_id == sbom_id) + + def list_for_package(self, package_id: UUID) -> list[DependencyRecord]: + """SBOMs that include the package (advisory sweep, usage query).""" + return self._many(f"list for package {package_id}", self.table.c.package_id == package_id) diff --git a/src/traust_core/v1/repositories/traust/dependencies/image.py b/src/traust_core/v1/repositories/traust/dependencies/image.py new file mode 100644 index 00000000..9dd212d0 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/dependencies/image.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +from traust_core.v1.models.records import ImageRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + RecordRepository, + SqlAppendOnlyRepository, + TableSpec, + text_column, + uuid_column, +) + +IMAGE = TableSpec( + TRAUST, + "image", + lambda: ( + text_column("digest"), + text_column("platform", nullable=True), + uuid_column("repository_ref_id", nullable=True), + uuid_column("sbom_id", nullable=True), + ), + append_only=True, +) + + +class ImageRepository(RecordRepository[ImageRecord], Protocol): + def find_by_digest(self, digest: str) -> ImageRecord | None: ... + def list_built_from(self, repository_ref_id: UUID) -> list[ImageRecord]: ... + def list_with_sbom(self, sbom_id: UUID) -> list[ImageRecord]: ... + + +class SqlImageRepository(SqlAppendOnlyRepository[ImageRecord]): + spec = IMAGE + record = ImageRecord + + def find_by_digest(self, digest: str) -> ImageRecord | None: + return self._one(f"find {digest}", self.table.c.digest == digest) + + def list_built_from(self, repository_ref_id: UUID) -> list[ImageRecord]: + """Images whose declared provenance names this commit.""" + return self._many( + f"list built from {repository_ref_id}", + self.table.c.repository_ref_id == repository_ref_id, + ) + + def list_with_sbom(self, sbom_id: UUID) -> list[ImageRecord]: + return self._many(f"list with sbom {sbom_id}", self.table.c.sbom_id == sbom_id) diff --git a/src/traust_core/v1/repositories/traust/dependencies/package.py b/src/traust_core/v1/repositories/traust/dependencies/package.py new file mode 100644 index 00000000..67114684 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/dependencies/package.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +from traust_core.v1.models.records import PackageRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + text_column, + uuid_column, +) + +PACKAGE = TableSpec( + TRAUST, + "package", + lambda: ( + uuid_column("source_repository_id", nullable=True), + uuid_column("sbom_id", nullable=True), + text_column("ecosystem"), + text_column("name"), + ), +) + + +class PackageRepository(MutableRecordRepository[PackageRecord], Protocol): + def find(self, ecosystem: str, name: str) -> PackageRecord | None: ... + def list_from_repository(self, repository_id: UUID) -> list[PackageRecord]: ... + def list_with_sbom(self, sbom_id: UUID) -> list[PackageRecord]: ... + + +class SqlPackageRepository(SqlMutableRepository[PackageRecord]): + spec = PACKAGE + record = PackageRecord + + def find(self, ecosystem: str, name: str) -> PackageRecord | None: + c = self.table.c + return self._one(f"find {ecosystem}/{name}", c.ecosystem == ecosystem, c.name == name) + + def list_from_repository(self, repository_id: UUID) -> list[PackageRecord]: + """Internal libraries the repository produces.""" + return self._many( + f"list from {repository_id}", self.table.c.source_repository_id == repository_id + ) + + def list_with_sbom(self, sbom_id: UUID) -> list[PackageRecord]: + """Image packages (base images) carrying this SBOM.""" + return self._many(f"list with sbom {sbom_id}", self.table.c.sbom_id == sbom_id) diff --git a/src/traust_core/v1/repositories/traust/dependencies/sbom.py b/src/traust_core/v1/repositories/traust/dependencies/sbom.py new file mode 100644 index 00000000..45ff7b29 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/dependencies/sbom.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from typing import Protocol + +from traust_core.v1.models.records import SbomRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + RecordRepository, + SqlAppendOnlyRepository, + TableSpec, + text_column, + uuid_column, +) + +SBOM = TableSpec( + TRAUST, + "sbom", + lambda: (text_column("digest"), uuid_column("artifact_id", nullable=True)), + append_only=True, +) + + +class SbomRepository(RecordRepository[SbomRecord], Protocol): + def find_by_digest(self, digest: str) -> SbomRecord | None: ... + + +class SqlSbomRepository(SqlAppendOnlyRepository[SbomRecord]): + spec = SBOM + record = SbomRecord + + def find_by_digest(self, digest: str) -> SbomRecord | None: + return self._one(f"find {digest}", self.table.c.digest == digest) diff --git a/src/traust_core/v1/repositories/traust/findings/__init__.py b/src/traust_core/v1/repositories/traust/findings/__init__.py new file mode 100644 index 00000000..1e8cf5c6 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/findings/__init__.py @@ -0,0 +1,92 @@ +"""3b · Findings & decisions - what was found and judged. One repository per table.""" + +from __future__ import annotations + +from typing import Protocol + +from traust_core.v1.repositories.tables import SqlTablesUnitOfWork +from traust_core.v1.repositories.traust.findings.cwe import CWE, CweRepository, SqlCweRepository +from traust_core.v1.repositories.traust.findings.decision import ( + DECISION, + DecisionRepository, + SqlDecisionRepository, +) +from traust_core.v1.repositories.traust.findings.finding import ( + FINDING, + FindingRepository, + SqlFindingRepository, +) +from traust_core.v1.repositories.traust.findings.finding_cwe import ( + FINDING_CWE, + FindingCweRepository, + SqlFindingCweRepository, +) +from traust_core.v1.repositories.traust.findings.finding_external_ref import ( + FINDING_EXTERNAL_REF, + FindingExternalRefRepository, + SqlFindingExternalRefRepository, +) +from traust_core.v1.repositories.traust.findings.finding_label import ( + FINDING_LABEL, + FindingLabelRepository, + SqlFindingLabelRepository, +) +from traust_core.v1.repositories.traust.findings.review_item import ( + REVIEW_ITEM, + ReviewItemRepository, + SqlReviewItemRepository, +) +from traust_core.v1.repositories.unit_of_work import UnitOfWork + +TABLES = ( + FINDING, + CWE, + FINDING_CWE, + FINDING_LABEL, + FINDING_EXTERNAL_REF, + DECISION, + REVIEW_ITEM, +) + + +class FindingsUnitOfWork(UnitOfWork, Protocol): + findings: FindingRepository + cwes: CweRepository + finding_cwes: FindingCweRepository + finding_labels: FindingLabelRepository + finding_external_refs: FindingExternalRefRepository + decisions: DecisionRepository + review_items: ReviewItemRepository + + +class SqlFindingsUnitOfWork(SqlTablesUnitOfWork): + def _open_repositories(self) -> None: + conn, tables = self.connection, self.tables + self.findings = SqlFindingRepository(conn, tables) + self.cwes = SqlCweRepository(conn, tables) + self.finding_cwes = SqlFindingCweRepository(conn, tables) + self.finding_labels = SqlFindingLabelRepository(conn, tables) + self.finding_external_refs = SqlFindingExternalRefRepository(conn, tables) + self.decisions = SqlDecisionRepository(conn, tables) + self.review_items = SqlReviewItemRepository(conn, tables) + + +__all__ = [ + "TABLES", + "CweRepository", + "DecisionRepository", + "FindingCweRepository", + "FindingExternalRefRepository", + "FindingLabelRepository", + "FindingRepository", + "FindingsUnitOfWork", + "ReviewItemRepository", + "SqlCweRepository", + "SqlDecisionRepository", + "SqlFindingCweRepository", + "SqlFindingExternalRefRepository", + "SqlFindingLabelRepository", + "SqlFindingRepository", + "SqlFindingsUnitOfWork", + "SqlReviewItemRepository", +] diff --git a/src/traust_core/v1/repositories/traust/findings/cwe.py b/src/traust_core/v1/repositories/traust/findings/cwe.py new file mode 100644 index 00000000..5f288a68 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/findings/cwe.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from typing import Protocol + +from traust_core.v1.models.records import CweRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + text_column, +) + +CWE = TableSpec( + TRAUST, + "cwe", + lambda: ( + text_column("identifier"), + text_column("abstraction", nullable=True), + text_column("mapping_usage", nullable=True), + text_column("catalog_version", nullable=True), + ), +) + + +class CweRepository(MutableRecordRepository[CweRecord], Protocol): + def find_by_identifier(self, identifier: str) -> CweRecord | None: ... + + +class SqlCweRepository(SqlMutableRepository[CweRecord]): + spec = CWE + record = CweRecord + + def find_by_identifier(self, identifier: str) -> CweRecord | None: + return self._one(f"find {identifier}", self.table.c.identifier == identifier) diff --git a/src/traust_core/v1/repositories/traust/findings/decision.py b/src/traust_core/v1/repositories/traust/findings/decision.py new file mode 100644 index 00000000..29dc5dcd --- /dev/null +++ b/src/traust_core/v1/repositories/traust/findings/decision.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Protocol +from uuid import UUID + +from traust_core.v1.models.records import DecisionRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + RecordRepository, + SqlAppendOnlyRepository, + TableSpec, + text_column, + timestamp_column, + uuid_column, +) + +DECISION = TableSpec( + TRAUST, + "decision", + lambda: ( + uuid_column("finding_id"), + uuid_column("assessment_id", nullable=True), + uuid_column("fixed_repository_ref_id", nullable=True), + uuid_column("repository_ref_id", nullable=True), + uuid_column("restates_id", nullable=True), + uuid_column("artifact_id", nullable=True), + text_column("statement_key"), + text_column("validity", nullable=True), + text_column("resolution", nullable=True), + text_column("evidence_grade", nullable=True), + text_column("reason", nullable=True), + text_column("severity", nullable=True), + text_column("source_channel", nullable=True), + text_column("actor", nullable=True), + timestamp_column("occurred_at"), + ), + append_only=True, +) + + +class DecisionRepository(RecordRepository[DecisionRecord], Protocol): + def find_by_statement(self, finding_id: UUID, statement_key: str) -> DecisionRecord | None: + """The idempotency key: a retried or re-spelled statement is the same decision.""" + ... + + def list_for_finding(self, finding_id: UUID) -> list[DecisionRecord]: ... + def list_since(self, since: datetime, limit: int) -> list[DecisionRecord]: ... + + +class SqlDecisionRepository(SqlAppendOnlyRepository[DecisionRecord]): + spec = DECISION + record = DecisionRecord + + def find_by_statement(self, finding_id: UUID, statement_key: str) -> DecisionRecord | None: + c = self.table.c + return self._one( + f"find {statement_key}", c.finding_id == finding_id, c.statement_key == statement_key + ) + + def list_for_finding(self, finding_id: UUID) -> list[DecisionRecord]: + c = self.table.c + return self._many( + f"list for {finding_id}", c.finding_id == finding_id, order_by=(c.occurred_at,) + ) + + def list_since(self, since: datetime, limit: int) -> list[DecisionRecord]: + c = self.table.c + stmt = ( + self._select() + .where(c.occurred_at > since) + .order_by(c.occurred_at, c.created_at) + .limit(limit) + ) + return [self._to_record(r) for r in self._execute(f"since {since}", stmt).mappings()] diff --git a/src/traust_core/v1/repositories/traust/findings/finding.py b/src/traust_core/v1/repositories/traust/findings/finding.py new file mode 100644 index 00000000..ad695c0e --- /dev/null +++ b/src/traust_core/v1/repositories/traust/findings/finding.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +from traust_core.v1.models.records import FindingRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + text_column, + uuid_column, +) + +FINDING = TableSpec( + TRAUST, + "finding", + lambda: ( + uuid_column("repository_id"), + uuid_column("advisory_id", nullable=True), + text_column("advisory_match", nullable=True), + text_column("fingerprint_algo"), + text_column("fingerprint"), + text_column("kind", nullable=True), + text_column("title"), + ), +) + + +class FindingRepository(MutableRecordRepository[FindingRecord], Protocol): + def find_by_fingerprint( + self, repository_id: UUID, fingerprint_algo: str, fingerprint: str + ) -> FindingRecord | None: ... + + def list_for_repository(self, repository_id: UUID) -> list[FindingRecord]: ... + def list_for_advisory(self, advisory_id: UUID) -> list[FindingRecord]: ... + + +class SqlFindingRepository(SqlMutableRepository[FindingRecord]): + spec = FINDING + record = FindingRecord + + def find_by_fingerprint( + self, repository_id: UUID, fingerprint_algo: str, fingerprint: str + ) -> FindingRecord | None: + c = self.table.c + return self._one( + f"find {fingerprint_algo}:{fingerprint}", + c.repository_id == repository_id, + c.fingerprint_algo == fingerprint_algo, + c.fingerprint == fingerprint, + ) + + def list_for_repository(self, repository_id: UUID) -> list[FindingRecord]: + return self._many(f"list for {repository_id}", self.table.c.repository_id == repository_id) + + def list_for_advisory(self, advisory_id: UUID) -> list[FindingRecord]: + return self._many( + f"list for advisory {advisory_id}", self.table.c.advisory_id == advisory_id + ) diff --git a/src/traust_core/v1/repositories/traust/findings/finding_cwe.py b/src/traust_core/v1/repositories/traust/findings/finding_cwe.py new file mode 100644 index 00000000..0de2dc87 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/findings/finding_cwe.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +from traust_core.v1.models.records import FindingCweRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + uuid_column, +) + +FINDING_CWE = TableSpec( + TRAUST, "finding_cwe", lambda: (uuid_column("finding_id"), uuid_column("cwe_id")) +) + + +class FindingCweRepository(MutableRecordRepository[FindingCweRecord], Protocol): + def list_for_finding(self, finding_id: UUID) -> list[FindingCweRecord]: ... + def list_for_cwe(self, cwe_id: UUID) -> list[FindingCweRecord]: ... + + +class SqlFindingCweRepository(SqlMutableRepository[FindingCweRecord]): + spec = FINDING_CWE + record = FindingCweRecord + + def list_for_finding(self, finding_id: UUID) -> list[FindingCweRecord]: + return self._many(f"list for {finding_id}", self.table.c.finding_id == finding_id) + + def list_for_cwe(self, cwe_id: UUID) -> list[FindingCweRecord]: + return self._many(f"list for cwe {cwe_id}", self.table.c.cwe_id == cwe_id) diff --git a/src/traust_core/v1/repositories/traust/findings/finding_external_ref.py b/src/traust_core/v1/repositories/traust/findings/finding_external_ref.py new file mode 100644 index 00000000..fcb82c3d --- /dev/null +++ b/src/traust_core/v1/repositories/traust/findings/finding_external_ref.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +from traust_core.v1.models.records import FindingExternalRefRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + text_column, + uuid_column, +) + +FINDING_EXTERNAL_REF = TableSpec( + TRAUST, + "finding_external_ref", + lambda: (uuid_column("finding_id"), text_column("system"), text_column("ref")), +) + + +class FindingExternalRefRepository(MutableRecordRepository[FindingExternalRefRecord], Protocol): + def list_for_finding(self, finding_id: UUID) -> list[FindingExternalRefRecord]: ... + def list_for_ref(self, system: str, ref: str) -> list[FindingExternalRefRecord]: ... + + +class SqlFindingExternalRefRepository(SqlMutableRepository[FindingExternalRefRecord]): + spec = FINDING_EXTERNAL_REF + record = FindingExternalRefRecord + + def list_for_finding(self, finding_id: UUID) -> list[FindingExternalRefRecord]: + return self._many(f"list for {finding_id}", self.table.c.finding_id == finding_id) + + def list_for_ref(self, system: str, ref: str) -> list[FindingExternalRefRecord]: + c = self.table.c + return self._many(f"list {system}:{ref}", c.system == system, c.ref == ref) diff --git a/src/traust_core/v1/repositories/traust/findings/finding_label.py b/src/traust_core/v1/repositories/traust/findings/finding_label.py new file mode 100644 index 00000000..d9e62d82 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/findings/finding_label.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +from traust_core.v1.models.records import FindingLabelRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + text_column, + uuid_column, +) + +FINDING_LABEL = TableSpec( + TRAUST, + "finding_label", + lambda: (uuid_column("finding_id"), text_column("taxonomy"), text_column("label")), +) + + +class FindingLabelRepository(MutableRecordRepository[FindingLabelRecord], Protocol): + def list_for_finding(self, finding_id: UUID) -> list[FindingLabelRecord]: ... + def list_with(self, taxonomy: str, label: str) -> list[FindingLabelRecord]: ... + + +class SqlFindingLabelRepository(SqlMutableRepository[FindingLabelRecord]): + spec = FINDING_LABEL + record = FindingLabelRecord + + def list_for_finding(self, finding_id: UUID) -> list[FindingLabelRecord]: + return self._many(f"list for {finding_id}", self.table.c.finding_id == finding_id) + + def list_with(self, taxonomy: str, label: str) -> list[FindingLabelRecord]: + c = self.table.c + return self._many(f"list {taxonomy}:{label}", c.taxonomy == taxonomy, c.label == label) diff --git a/src/traust_core/v1/repositories/traust/findings/review_item.py b/src/traust_core/v1/repositories/traust/findings/review_item.py new file mode 100644 index 00000000..b9fae938 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/findings/review_item.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +from traust_core.v1.models.records import ReviewItemRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + text_column, + uuid_column, +) + +REVIEW_ITEM = TableSpec( + TRAUST, + "review_item", + lambda: ( + uuid_column("finding_id"), + uuid_column("decision_id", nullable=True), + uuid_column("resolved_by_decision_id", nullable=True), + text_column("reason", nullable=True), + text_column("status"), + ), +) + +PENDING = "pending" # classifier plan: ReviewItem.status; the partial index's predicate + + +class ReviewItemRepository(MutableRecordRepository[ReviewItemRecord], Protocol): + def list_for_finding(self, finding_id: UUID) -> list[ReviewItemRecord]: ... + def pending(self, limit: int) -> list[ReviewItemRecord]: ... + + +class SqlReviewItemRepository(SqlMutableRepository[ReviewItemRecord]): + spec = REVIEW_ITEM + record = ReviewItemRecord + + def list_for_finding(self, finding_id: UUID) -> list[ReviewItemRecord]: + return self._many(f"list for {finding_id}", self.table.c.finding_id == finding_id) + + def pending(self, limit: int) -> list[ReviewItemRecord]: + c = self.table.c + stmt = self._select().where(c.status == PENDING).order_by(c.created_at).limit(limit) + return [self._to_record(r) for r in self._execute("pending", stmt).mappings()] diff --git a/src/traust_core/v1/repositories/traust/inventory/__init__.py b/src/traust_core/v1/repositories/traust/inventory/__init__.py new file mode 100644 index 00000000..604ae590 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/inventory/__init__.py @@ -0,0 +1,88 @@ +"""1 · Inventory - what exists and who owns it. One repository per table.""" + +from __future__ import annotations + +from typing import Protocol + +from traust_core.v1.repositories.tables import SqlTablesUnitOfWork +from traust_core.v1.repositories.traust.inventory.component import ( + COMPONENT, + ComponentRepository, + SqlComponentRepository, +) +from traust_core.v1.repositories.traust.inventory.ownership import ( + OWNERSHIP, + OwnershipRepository, + SqlOwnershipRepository, +) +from traust_core.v1.repositories.traust.inventory.product import ( + PRODUCT, + ProductRepository, + SqlProductRepository, +) +from traust_core.v1.repositories.traust.inventory.release import ( + RELEASE, + ReleaseRepository, + SqlReleaseRepository, +) +from traust_core.v1.repositories.traust.inventory.repository import ( + REPOSITORY, + RepositoryRepository, + SqlRepositoryRepository, +) +from traust_core.v1.repositories.traust.inventory.repository_ref import ( + REPOSITORY_REF, + RepositoryRefRepository, + SqlRepositoryRefRepository, +) +from traust_core.v1.repositories.traust.inventory.team import ( + TEAM, + SqlTeamRepository, + TeamRepository, +) +from traust_core.v1.repositories.unit_of_work import UnitOfWork + +TABLES = (PRODUCT, RELEASE, COMPONENT, REPOSITORY, REPOSITORY_REF, TEAM, OWNERSHIP) + + +class InventoryUnitOfWork(UnitOfWork, Protocol): + products: ProductRepository + releases: ReleaseRepository + components: ComponentRepository + repositories: RepositoryRepository + repository_refs: RepositoryRefRepository + teams: TeamRepository + ownerships: OwnershipRepository + + +class SqlInventoryUnitOfWork(SqlTablesUnitOfWork): + def _open_repositories(self) -> None: + conn, tables = self.connection, self.tables + self.products = SqlProductRepository(conn, tables) + self.releases = SqlReleaseRepository(conn, tables) + self.components = SqlComponentRepository(conn, tables) + self.repositories = SqlRepositoryRepository(conn, tables) + self.repository_refs = SqlRepositoryRefRepository(conn, tables) + self.teams = SqlTeamRepository(conn, tables) + self.ownerships = SqlOwnershipRepository(conn, tables) + + +__all__ = [ + "TABLES", + "ComponentRepository", + "InventoryUnitOfWork", + "OwnershipRepository", + "ProductRepository", + "ReleaseRepository", + "RepositoryRefRepository", + "RepositoryRepository", + "SqlComponentRepository", + "SqlInventoryUnitOfWork", + "SqlOwnershipRepository", + "SqlProductRepository", + "SqlReleaseRepository", + "SqlRepositoryRefRepository", + "SqlRepositoryRepository", + "SqlTeamRepository", + "TeamRepository", +] diff --git a/src/traust_core/v1/repositories/traust/inventory/component.py b/src/traust_core/v1/repositories/traust/inventory/component.py new file mode 100644 index 00000000..232c75f7 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/inventory/component.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +from traust_core.v1.models.records import ComponentRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + uuid_column, +) + +COMPONENT = TableSpec( + TRAUST, "component", lambda: (uuid_column("release_id"), uuid_column("repository_ref_id")) +) + + +class ComponentRepository(MutableRecordRepository[ComponentRecord], Protocol): + def find(self, release_id: UUID, repository_ref_id: UUID) -> ComponentRecord | None: ... + def list_for_release(self, release_id: UUID) -> list[ComponentRecord]: ... + def list_for_ref(self, repository_ref_id: UUID) -> list[ComponentRecord]: ... + + +class SqlComponentRepository(SqlMutableRepository[ComponentRecord]): + spec = COMPONENT + record = ComponentRecord + + def find(self, release_id: UUID, repository_ref_id: UUID) -> ComponentRecord | None: + c = self.table.c + return self._one( + f"find {release_id}/{repository_ref_id}", + c.release_id == release_id, + c.repository_ref_id == repository_ref_id, + ) + + def list_for_release(self, release_id: UUID) -> list[ComponentRecord]: + return self._many(f"list for {release_id}", self.table.c.release_id == release_id) + + def list_for_ref(self, repository_ref_id: UUID) -> list[ComponentRecord]: + return self._many( + f"list for ref {repository_ref_id}", self.table.c.repository_ref_id == repository_ref_id + ) diff --git a/src/traust_core/v1/repositories/traust/inventory/ownership.py b/src/traust_core/v1/repositories/traust/inventory/ownership.py new file mode 100644 index 00000000..9b6fdb7b --- /dev/null +++ b/src/traust_core/v1/repositories/traust/inventory/ownership.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Protocol +from uuid import UUID + +import sqlalchemy as sa + +from traust_core.v1.models.records import OwnershipRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + timestamp_column, + uuid_column, +) + +OWNERSHIP = TableSpec( + TRAUST, + "ownership", + lambda: ( + uuid_column("repository_id"), + uuid_column("team_id"), + timestamp_column("valid_from"), + timestamp_column("valid_to", nullable=True), + ), +) + + +class OwnershipRepository(MutableRecordRepository[OwnershipRecord], Protocol): + def current(self, repository_id: UUID) -> list[OwnershipRecord]: ... + def as_of(self, repository_id: UUID, at: datetime) -> list[OwnershipRecord]: ... + def list_for_team(self, team_id: UUID) -> list[OwnershipRecord]: ... + + +class SqlOwnershipRepository(SqlMutableRepository[OwnershipRecord]): + spec = OWNERSHIP + record = OwnershipRecord + + def current(self, repository_id: UUID) -> list[OwnershipRecord]: + c = self.table.c + return self._many( + f"current for {repository_id}", c.repository_id == repository_id, c.valid_to.is_(None) + ) + + def as_of(self, repository_id: UUID, at: datetime) -> list[OwnershipRecord]: + c = self.table.c + return self._many( + f"as of {at} for {repository_id}", + c.repository_id == repository_id, + c.valid_from <= at, + sa.or_(c.valid_to.is_(None), c.valid_to > at), + ) + + def list_for_team(self, team_id: UUID) -> list[OwnershipRecord]: + return self._many(f"list for team {team_id}", self.table.c.team_id == team_id) diff --git a/src/traust_core/v1/repositories/traust/inventory/product.py b/src/traust_core/v1/repositories/traust/inventory/product.py new file mode 100644 index 00000000..8ebe221b --- /dev/null +++ b/src/traust_core/v1/repositories/traust/inventory/product.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from typing import Protocol + +from traust_core.v1.models.records import ProductRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + text_column, +) + +PRODUCT = TableSpec(TRAUST, "product", lambda: (text_column("slug"), text_column("name"))) + + +class ProductRepository(MutableRecordRepository[ProductRecord], Protocol): + def find_by_slug(self, slug: str) -> ProductRecord | None: ... + + +class SqlProductRepository(SqlMutableRepository[ProductRecord]): + spec = PRODUCT + record = ProductRecord + + def find_by_slug(self, slug: str) -> ProductRecord | None: + return self._one(f"find {slug}", self.table.c.slug == slug) diff --git a/src/traust_core/v1/repositories/traust/inventory/release.py b/src/traust_core/v1/repositories/traust/inventory/release.py new file mode 100644 index 00000000..d56aa3b6 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/inventory/release.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +from traust_core.v1.models.records import ReleaseRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + text_column, + uuid_column, +) + +RELEASE = TableSpec(TRAUST, "release", lambda: (uuid_column("product_id"), text_column("version"))) + + +class ReleaseRepository(MutableRecordRepository[ReleaseRecord], Protocol): + def find(self, product_id: UUID, version: str) -> ReleaseRecord | None: ... + def list_for_product(self, product_id: UUID) -> list[ReleaseRecord]: ... + + +class SqlReleaseRepository(SqlMutableRepository[ReleaseRecord]): + spec = RELEASE + record = ReleaseRecord + + def find(self, product_id: UUID, version: str) -> ReleaseRecord | None: + c = self.table.c + return self._one(f"find {version}", c.product_id == product_id, c.version == version) + + def list_for_product(self, product_id: UUID) -> list[ReleaseRecord]: + return self._many(f"list for {product_id}", self.table.c.product_id == product_id) diff --git a/src/traust_core/v1/repositories/traust/inventory/repository.py b/src/traust_core/v1/repositories/traust/inventory/repository.py new file mode 100644 index 00000000..e139c6e5 --- /dev/null +++ b/src/traust_core/v1/repositories/traust/inventory/repository.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from typing import Protocol + +import sqlalchemy as sa + +from traust_core.v1.models.records import RepositoryRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + text_column, + uuid_column, +) + +REPOSITORY = TableSpec( + TRAUST, + "repository", + lambda: ( + uuid_column("upstream_id", nullable=True), + text_column("url"), + sa.Column("public", sa.Boolean, nullable=False), + ), +) + + +class RepositoryRepository(MutableRecordRepository[RepositoryRecord], Protocol): + def find_by_url(self, url: str) -> RepositoryRecord | None: ... + + +class SqlRepositoryRepository(SqlMutableRepository[RepositoryRecord]): + spec = REPOSITORY + record = RepositoryRecord + + def find_by_url(self, url: str) -> RepositoryRecord | None: + return self._one(f"find {url}", self.table.c.url == url) diff --git a/src/traust_core/v1/repositories/traust/inventory/repository_ref.py b/src/traust_core/v1/repositories/traust/inventory/repository_ref.py new file mode 100644 index 00000000..a7df684a --- /dev/null +++ b/src/traust_core/v1/repositories/traust/inventory/repository_ref.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +import sqlalchemy as sa + +from traust_core.v1.models.records import RepositoryRefRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + text_column, + uuid_column, +) + +REPOSITORY_REF = TableSpec( + TRAUST, + "repository_ref", + lambda: ( + uuid_column("repository_id"), + text_column("name", nullable=True), + text_column("kind", nullable=True), + sa.Column("is_default", sa.Boolean, nullable=False), + text_column("sha"), + uuid_column("manifest_sbom_id", nullable=True), + ), +) + + +class RepositoryRefRepository(MutableRecordRepository[RepositoryRefRecord], Protocol): + def find(self, repository_id: UUID, name: str | None, sha: str) -> RepositoryRefRecord | None: + """A named ref by (repository, name, sha); name None = the bare commit.""" + ... + + def list_for(self, repository_id: UUID) -> list[RepositoryRefRecord]: ... + def default(self, repository_id: UUID) -> RepositoryRefRecord | None: ... + def list_for_manifest_sbom(self, sbom_id: UUID) -> list[RepositoryRefRecord]: ... + + +class SqlRepositoryRefRepository(SqlMutableRepository[RepositoryRefRecord]): + spec = REPOSITORY_REF + record = RepositoryRefRecord + + def find(self, repository_id: UUID, name: str | None, sha: str) -> RepositoryRefRecord | None: + c = self.table.c + named = c.name.is_(None) if name is None else c.name == name + return self._one( + f"find {name or 'bare'}@{sha}", c.repository_id == repository_id, named, c.sha == sha + ) + + def list_for(self, repository_id: UUID) -> list[RepositoryRefRecord]: + return self._many(f"list for {repository_id}", self.table.c.repository_id == repository_id) + + def default(self, repository_id: UUID) -> RepositoryRefRecord | None: + """The latest ref recorded as the default branch.""" + c = self.table.c + stmt = ( + self._select() + .where(c.repository_id == repository_id, c.is_default.is_(True)) + .order_by(c.created_at.desc()) + .limit(1) + ) + row = self._execute(f"default for {repository_id}", stmt).mappings().first() + return None if row is None else self._to_record(row) + + def list_for_manifest_sbom(self, sbom_id: UUID) -> list[RepositoryRefRecord]: + return self._many(f"list for sbom {sbom_id}", self.table.c.manifest_sbom_id == sbom_id) diff --git a/src/traust_core/v1/repositories/traust/inventory/team.py b/src/traust_core/v1/repositories/traust/inventory/team.py new file mode 100644 index 00000000..ed74f01a --- /dev/null +++ b/src/traust_core/v1/repositories/traust/inventory/team.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from typing import Protocol + +from traust_core.v1.models.records import TeamRecord +from traust_core.v1.repositories.tables import ( + TRAUST, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + text_column, +) + +TEAM = TableSpec(TRAUST, "team", lambda: (text_column("name"),)) + + +class TeamRepository(MutableRecordRepository[TeamRecord], Protocol): + def find_by_name(self, name: str) -> TeamRecord | None: ... + + +class SqlTeamRepository(SqlMutableRepository[TeamRecord]): + spec = TEAM + record = TeamRecord + + def find_by_name(self, name: str) -> TeamRecord | None: + return self._one(f"find {name}", self.table.c.name == name) diff --git a/src/traust_core/v1/repositories/traust_ledger/__init__.py b/src/traust_core/v1/repositories/traust_ledger/__init__.py new file mode 100644 index 00000000..80fa43a2 --- /dev/null +++ b/src/traust_core/v1/repositories/traust_ledger/__init__.py @@ -0,0 +1,46 @@ +"""5 · Ledger (traust_ledger) - app-owned proof tables; traust-ledger (library) computes. + +One repository per table. SQLite carries the namespace as the `ledger_` prefix. +""" + +from __future__ import annotations + +from typing import Protocol + +from traust_core.v1.repositories.tables import SqlTablesUnitOfWork +from traust_core.v1.repositories.traust_ledger.events import ( + EVENTS, + EventRepository, + SqlEventRepository, +) +from traust_core.v1.repositories.traust_ledger.layers import ( + LAYERS, + LayerRepository, + SqlLayerRepository, +) +from traust_core.v1.repositories.unit_of_work import UnitOfWork + +TABLES = (LAYERS, EVENTS) + + +class LedgerUnitOfWork(UnitOfWork, Protocol): + layers: LayerRepository + events: EventRepository + + +class SqlLedgerUnitOfWork(SqlTablesUnitOfWork): + def _open_repositories(self) -> None: + conn, tables = self.connection, self.tables + self.layers = SqlLayerRepository(conn, tables) + self.events = SqlEventRepository(conn, tables) + + +__all__ = [ + "TABLES", + "EventRepository", + "LayerRepository", + "LedgerUnitOfWork", + "SqlEventRepository", + "SqlLayerRepository", + "SqlLedgerUnitOfWork", +] diff --git a/src/traust_core/v1/repositories/traust_ledger/events.py b/src/traust_core/v1/repositories/traust_ledger/events.py new file mode 100644 index 00000000..56fd2ae6 --- /dev/null +++ b/src/traust_core/v1/repositories/traust_ledger/events.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +import sqlalchemy as sa + +from traust_core.v1.models.records import EventRecord +from traust_core.v1.repositories.tables import ( + TRAUST_LEDGER, + RecordRepository, + SqlAppendOnlyRepository, + TableSpec, + uuid_column, +) + +EVENTS = TableSpec( + TRAUST_LEDGER, + "events", + lambda: ( + uuid_column("layer_id"), + uuid_column("decision_id"), + sa.Column("seq", sa.Integer, nullable=False), + sa.Column("event_payload", sa.LargeBinary, nullable=False), + ), + append_only=True, +) + + +class EventRepository(RecordRepository[EventRecord], Protocol): + def find_by_decision(self, decision_id: UUID) -> EventRecord | None: ... + def list_for_layer(self, layer_id: UUID) -> list[EventRecord]: ... + def last_seq(self, layer_id: UUID) -> int | None: ... + + +class SqlEventRepository(SqlAppendOnlyRepository[EventRecord]): + spec = EVENTS + record = EventRecord + + def find_by_decision(self, decision_id: UUID) -> EventRecord | None: + return self._one(f"find for {decision_id}", self.table.c.decision_id == decision_id) + + def list_for_layer(self, layer_id: UUID) -> list[EventRecord]: + c = self.table.c + return self._many(f"list for {layer_id}", c.layer_id == layer_id, order_by=(c.seq,)) + + def last_seq(self, layer_id: UUID) -> int | None: + c = self.table.c + stmt = sa.select(sa.func.max(c.seq)).where(c.layer_id == layer_id) + return self._execute(f"last seq {layer_id}", stmt).scalar_one() # type: ignore[no-any-return] diff --git a/src/traust_core/v1/repositories/traust_ledger/layers.py b/src/traust_core/v1/repositories/traust_ledger/layers.py new file mode 100644 index 00000000..949b10d2 --- /dev/null +++ b/src/traust_core/v1/repositories/traust_ledger/layers.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +import sqlalchemy as sa + +from traust_core.v1.models.records import LayerRecord +from traust_core.v1.repositories.tables import ( + TRAUST_LEDGER, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + text_column, + uuid_column, +) + +LAYERS = TableSpec( + TRAUST_LEDGER, + "layers", + lambda: ( + uuid_column("repository_id"), + text_column("line", nullable=True), + text_column("merkle_root", nullable=True), + sa.Column("merkle_epoch", sa.Integer, nullable=True), + sa.Column("merkle_size", sa.Integer, nullable=True), + text_column("merkle_root_signature", nullable=True), + text_column("merkle_signing_method"), + sa.Column("merkle_signature_format", sa.Integer, nullable=True), + ), +) + + +class LayerRepository(MutableRecordRepository[LayerRecord], Protocol): + def list_for_repository(self, repository_id: UUID) -> list[LayerRecord]: ... + + +class SqlLayerRepository(SqlMutableRepository[LayerRecord]): + spec = LAYERS + record = LayerRecord + + def list_for_repository(self, repository_id: UUID) -> list[LayerRecord]: + return self._many(f"list for {repository_id}", self.table.c.repository_id == repository_id) diff --git a/src/traust_core/v1/repositories/traust_ops/__init__.py b/src/traust_core/v1/repositories/traust_ops/__init__.py new file mode 100644 index 00000000..e9a1395a --- /dev/null +++ b/src/traust_core/v1/repositories/traust_ops/__init__.py @@ -0,0 +1,73 @@ +"""4 · Operations (traust_ops) - what traust ran, with what, and what it cost. + +One repository per table. SQLite carries the namespace as the `ops_` prefix. +""" + +from __future__ import annotations + +from typing import Protocol + +from traust_core.v1.repositories.tables import SqlTablesUnitOfWork +from traust_core.v1.repositories.traust_ops.assessment_input import ( + ASSESSMENT_INPUT, + AssessmentInputRepository, + SqlAssessmentInputRepository, +) +from traust_core.v1.repositories.traust_ops.feed_snapshot import ( + FEED_SNAPSHOT, + FeedSnapshotRepository, + SqlFeedSnapshotRepository, +) +from traust_core.v1.repositories.traust_ops.run_batch import ( + RUN_BATCH, + RunBatchRepository, + SqlRunBatchRepository, +) +from traust_core.v1.repositories.traust_ops.run_order import ( + RUN_ORDER, + RunOrderRepository, + SqlRunOrderRepository, +) +from traust_core.v1.repositories.traust_ops.spend_event import ( + SPEND_EVENT, + SpendEventRepository, + SqlSpendEventRepository, +) +from traust_core.v1.repositories.unit_of_work import UnitOfWork + +TABLES = (RUN_BATCH, RUN_ORDER, SPEND_EVENT, FEED_SNAPSHOT, ASSESSMENT_INPUT) + + +class OpsUnitOfWork(UnitOfWork, Protocol): + run_batches: RunBatchRepository + run_orders: RunOrderRepository + spend_events: SpendEventRepository + feed_snapshots: FeedSnapshotRepository + assessment_inputs: AssessmentInputRepository + + +class SqlOpsUnitOfWork(SqlTablesUnitOfWork): + def _open_repositories(self) -> None: + conn, tables = self.connection, self.tables + self.run_batches = SqlRunBatchRepository(conn, tables) + self.run_orders = SqlRunOrderRepository(conn, tables) + self.spend_events = SqlSpendEventRepository(conn, tables) + self.feed_snapshots = SqlFeedSnapshotRepository(conn, tables) + self.assessment_inputs = SqlAssessmentInputRepository(conn, tables) + + +__all__ = [ + "TABLES", + "AssessmentInputRepository", + "FeedSnapshotRepository", + "OpsUnitOfWork", + "RunBatchRepository", + "RunOrderRepository", + "SpendEventRepository", + "SqlAssessmentInputRepository", + "SqlFeedSnapshotRepository", + "SqlOpsUnitOfWork", + "SqlRunBatchRepository", + "SqlRunOrderRepository", + "SqlSpendEventRepository", +] diff --git a/src/traust_core/v1/repositories/traust_ops/assessment_input.py b/src/traust_core/v1/repositories/traust_ops/assessment_input.py new file mode 100644 index 00000000..e0f3d90e --- /dev/null +++ b/src/traust_core/v1/repositories/traust_ops/assessment_input.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +from traust_core.v1.models.records import AssessmentInputRecord +from traust_core.v1.repositories.tables import ( + TRAUST_OPS, + RecordRepository, + SqlAppendOnlyRepository, + TableSpec, + uuid_column, +) + +ASSESSMENT_INPUT = TableSpec( + TRAUST_OPS, + "assessment_input", + lambda: (uuid_column("assessment_id"), uuid_column("feed_snapshot_id")), + append_only=True, +) + + +class AssessmentInputRepository(RecordRepository[AssessmentInputRecord], Protocol): + def list_for_assessment(self, assessment_id: UUID) -> list[AssessmentInputRecord]: ... + def list_for_snapshot(self, feed_snapshot_id: UUID) -> list[AssessmentInputRecord]: ... + + +class SqlAssessmentInputRepository(SqlAppendOnlyRepository[AssessmentInputRecord]): + spec = ASSESSMENT_INPUT + record = AssessmentInputRecord + + def list_for_assessment(self, assessment_id: UUID) -> list[AssessmentInputRecord]: + return self._many(f"list for {assessment_id}", self.table.c.assessment_id == assessment_id) + + def list_for_snapshot(self, feed_snapshot_id: UUID) -> list[AssessmentInputRecord]: + return self._many( + f"list for snapshot {feed_snapshot_id}", + self.table.c.feed_snapshot_id == feed_snapshot_id, + ) diff --git a/src/traust_core/v1/repositories/traust_ops/feed_snapshot.py b/src/traust_core/v1/repositories/traust_ops/feed_snapshot.py new file mode 100644 index 00000000..aad37724 --- /dev/null +++ b/src/traust_core/v1/repositories/traust_ops/feed_snapshot.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from typing import Protocol + +from traust_core.v1.models.records import FeedSnapshotRecord +from traust_core.v1.repositories.tables import ( + TRAUST_OPS, + RecordRepository, + SqlAppendOnlyRepository, + TableSpec, + text_column, + timestamp_column, + uuid_column, +) + +FEED_SNAPSHOT = TableSpec( + TRAUST_OPS, + "feed_snapshot", + lambda: ( + uuid_column("run_order_id", nullable=True), + uuid_column("artifact_id"), + text_column("feed"), + timestamp_column("retrieved_at"), + ), + append_only=True, +) + + +class FeedSnapshotRepository(RecordRepository[FeedSnapshotRecord], Protocol): + def latest(self, feed: str) -> FeedSnapshotRecord | None: ... + + +class SqlFeedSnapshotRepository(SqlAppendOnlyRepository[FeedSnapshotRecord]): + spec = FEED_SNAPSHOT + record = FeedSnapshotRecord + + def latest(self, feed: str) -> FeedSnapshotRecord | None: + c = self.table.c + stmt = ( + self._select() + .where(c.feed == feed) + .order_by(c.retrieved_at.desc(), c.created_at.desc()) + .limit(1) + ) + row = self._execute(f"latest {feed}", stmt).mappings().first() + return None if row is None else self._to_record(row) diff --git a/src/traust_core/v1/repositories/traust_ops/run_batch.py b/src/traust_core/v1/repositories/traust_ops/run_batch.py new file mode 100644 index 00000000..680807a2 --- /dev/null +++ b/src/traust_core/v1/repositories/traust_ops/run_batch.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import Protocol + +from traust_core.v1.models.records import RunBatchRecord +from traust_core.v1.repositories.tables import ( + TRAUST_OPS, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + text_column, +) + +RUN_BATCH = TableSpec(TRAUST_OPS, "run_batch", lambda: (text_column("kind"), text_column("period"))) + + +class RunBatchRepository(MutableRecordRepository[RunBatchRecord], Protocol): + def find(self, kind: str, period: str) -> RunBatchRecord | None: ... + + +class SqlRunBatchRepository(SqlMutableRepository[RunBatchRecord]): + spec = RUN_BATCH + record = RunBatchRecord + + def find(self, kind: str, period: str) -> RunBatchRecord | None: + c = self.table.c + return self._one(f"find {kind}/{period}", c.kind == kind, c.period == period) diff --git a/src/traust_core/v1/repositories/traust_ops/run_order.py b/src/traust_core/v1/repositories/traust_ops/run_order.py new file mode 100644 index 00000000..60f19ad1 --- /dev/null +++ b/src/traust_core/v1/repositories/traust_ops/run_order.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +from datetime import datetime, timedelta +from typing import Protocol +from uuid import UUID + +import sqlalchemy as sa + +from traust_core.v1.models.records import RunOrderRecord +from traust_core.v1.repositories.tables import ( + TRAUST_OPS, + MutableRecordRepository, + SqlMutableRepository, + TableSpec, + text_column, + timestamp_column, + uuid_column, +) + +RUN_ORDER = TableSpec( + TRAUST_OPS, + "run_order", + lambda: ( + uuid_column("run_batch_id", nullable=True), + uuid_column("repository_id", nullable=True), + uuid_column("artifact_id", nullable=True), + text_column("rule", nullable=True), + text_column("event_source", nullable=True), + text_column("lane"), + text_column("state"), + sa.Column("priority", sa.Integer, nullable=False), + text_column("reason", nullable=True), + text_column("requested_by"), + text_column("lease_holder", nullable=True), + timestamp_column("lease_expires_at", nullable=True), + timestamp_column("started_at", nullable=True), + timestamp_column("finished_at", nullable=True), + ), +) + +# run_order.state values the queue mechanics depend on (contracts enum, to add). +QUEUED = "queued" +LEASED = "leased" + + +class RunOrderRepository(MutableRecordRepository[RunOrderRecord], Protocol): + def claim(self, worker: str, now: datetime, lease_for: timedelta) -> RunOrderRecord | None: + """Lease the next queued order (priority, then age); None when the queue is empty.""" + ... + + def release_expired(self, now: datetime) -> int: ... + def list_for_batch(self, run_batch_id: UUID) -> list[RunOrderRecord]: ... + def list_for_repository(self, repository_id: UUID, state: str) -> list[RunOrderRecord]: ... + + +class SqlRunOrderRepository(SqlMutableRepository[RunOrderRecord]): + spec = RUN_ORDER + record = RunOrderRecord + + def claim(self, worker: str, now: datetime, lease_for: timedelta) -> RunOrderRecord | None: + c = self.table.c + # SKIP LOCKED on Postgres; SQLite serialises writers and drops FOR UPDATE. + next_id = ( + sa.select(c.id) + .where(c.state == QUEUED) + .order_by(c.priority, c.created_at) + .limit(1) + .with_for_update(skip_locked=True) + .scalar_subquery() + ) + stmt = ( + sa.update(self.table) + .where(c.id == next_id, c.state == QUEUED) + .values(state=LEASED, lease_holder=worker, lease_expires_at=now + lease_for) + .returning(c.id) + ) + claimed = self._execute(f"claim for {worker}", stmt).scalar_one_or_none() + return None if claimed is None else self.get(claimed) + + def release_expired(self, now: datetime) -> int: + """Return expired leases to the queue; the count released.""" + c = self.table.c + stmt = ( + sa.update(self.table) + .where(c.state == LEASED, c.lease_expires_at < now) + .values(state=QUEUED, lease_holder=None, lease_expires_at=None) + ) + return self._execute(f"release expired at {now}", stmt).rowcount + + def list_for_batch(self, run_batch_id: UUID) -> list[RunOrderRecord]: + return self._many( + f"list for batch {run_batch_id}", self.table.c.run_batch_id == run_batch_id + ) + + def list_for_repository(self, repository_id: UUID, state: str) -> list[RunOrderRecord]: + c = self.table.c + return self._many( + f"list {state} for {repository_id}", + c.repository_id == repository_id, + c.state == state, + order_by=(c.priority,), + ) diff --git a/src/traust_core/v1/repositories/traust_ops/spend_event.py b/src/traust_core/v1/repositories/traust_ops/spend_event.py new file mode 100644 index 00000000..4535f9c6 --- /dev/null +++ b/src/traust_core/v1/repositories/traust_ops/spend_event.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Protocol +from uuid import UUID + +import sqlalchemy as sa + +from traust_core.v1.models.records import SpendEventRecord +from traust_core.v1.repositories.tables import ( + TRAUST_OPS, + RecordRepository, + SqlAppendOnlyRepository, + TableSpec, + text_column, + timestamp_column, + uuid_column, +) + +SPEND_EVENT = TableSpec( + TRAUST_OPS, + "spend_event", + lambda: ( + uuid_column("run_order_id", nullable=True), + text_column("provider"), + text_column("model"), + text_column("skill", nullable=True), + sa.Column("tokens_in", sa.BigInteger, nullable=False), + sa.Column("tokens_out", sa.BigInteger, nullable=False), + sa.Column("cost_usd", sa.Numeric, nullable=False), + timestamp_column("occurred_at"), + ), + append_only=True, +) + + +class SpendEventRepository(RecordRepository[SpendEventRecord], Protocol): + def list_for_order(self, run_order_id: UUID) -> list[SpendEventRecord]: ... + def list_between(self, start: datetime, end: datetime) -> list[SpendEventRecord]: ... + + +class SqlSpendEventRepository(SqlAppendOnlyRepository[SpendEventRecord]): + spec = SPEND_EVENT + record = SpendEventRecord + + def list_for_order(self, run_order_id: UUID) -> list[SpendEventRecord]: + return self._many(f"list for {run_order_id}", self.table.c.run_order_id == run_order_id) + + def list_between(self, start: datetime, end: datetime) -> list[SpendEventRecord]: + """Spend with start <= occurred_at < end.""" + c = self.table.c + return self._many( + f"list {start}..{end}", + c.occurred_at >= start, + c.occurred_at < end, + order_by=(c.occurred_at,), + ) diff --git a/src/traust_core/v1/repositories/triage.py b/src/traust_core/v1/repositories/triage.py index b859b567..06ca7baa 100644 --- a/src/traust_core/v1/repositories/triage.py +++ b/src/traust_core/v1/repositories/triage.py @@ -8,8 +8,9 @@ from traust_core.v1.models.findings import TriageVerdict from traust_core.v1.models.vocabulary import Verdict from traust_core.v1.repositories.memory import InMemoryUnitOfWork -from traust_core.v1.repositories.sql import STORAGE_SCHEMA, SqlRepository, SqlUnitOfWork +from traust_core.v1.repositories.sql import SqlRepository, SqlUnitOfWork from traust_core.v1.repositories.storage import ( + STORAGE_SCHEMA, EvidenceRepository, InMemoryEvidenceRepository, SqlEvidenceRepository, diff --git a/tests/v1/repositories/tables/__init__.py b/tests/v1/repositories/tables/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/v1/repositories/tables/test_factory.py b/tests/v1/repositories/tables/test_factory.py new file mode 100644 index 00000000..e8309b67 --- /dev/null +++ b/tests/v1/repositories/tables/test_factory.py @@ -0,0 +1,83 @@ +from pathlib import Path + +import pytest +import sqlalchemy as sa + +from tests.v1.conftest import SQL_BACKENDS, contract_db +from traust_core.v1 import contracts +from traust_core.v1.repositories.contract import CONTRACT_TABLES, contract_drift, contract_tables +from traust_core.v1.repositories.sql import Dialect +from traust_core.v1.repositories.tables import ( + TRAUST, + TRAUST_LEDGER, + TRAUST_OPS, + TableFactory, +) +from traust_core.v1.repositories.traust.inventory import REPOSITORY +from traust_core.v1.repositories.traust_ledger import EVENTS +from traust_core.v1.repositories.traust_ops import RUN_ORDER + + +@pytest.mark.parametrize( + ("namespace", "postgres", "sqlite"), + [ + (TRAUST, ("traust", "finding"), (None, "finding")), + (TRAUST_OPS, ("traust_ops", "finding"), (None, "ops_finding")), + (TRAUST_LEDGER, ("traust_ledger", "finding"), (None, "ledger_finding")), + ], +) +def test_namespace_places_a_table_per_dialect(namespace, postgres, sqlite) -> None: + assert namespace.locate("finding", Dialect.POSTGRES) == postgres + assert namespace.locate("finding", Dialect.SQLITE) == sqlite + + +@pytest.mark.parametrize( + ("dialect", "names"), + [ + (Dialect.POSTGRES, ["traust.repository", "traust_ops.run_order", "traust_ledger.events"]), + (Dialect.SQLITE, ["repository", "ops_run_order", "ledger_events"]), + ], +) +def test_factory_builds_contract_names(dialect: Dialect, names: list[str]) -> None: + tables = TableFactory.for_dialect(dialect) + assert [tables.table(s).fullname for s in (REPOSITORY, RUN_ORDER, EVENTS)] == names + + +def test_factory_is_one_per_dialect_and_builds_each_table_once() -> None: + tables = TableFactory.for_dialect(Dialect.SQLITE) + assert TableFactory.for_dialect(Dialect.SQLITE) is tables + assert TableFactory.for_dialect(Dialect.POSTGRES) is not tables + assert tables.table(REPOSITORY) is tables.table(REPOSITORY) + + +def test_factory_follows_the_engine(tmp_path: Path) -> None: + engine = sa.create_engine(f"sqlite:///{tmp_path}/f.db") + assert TableFactory.for_bind(engine) is TableFactory.for_dialect(Dialect.SQLITE) + + +def test_append_only_tables_have_no_updated_at() -> None: + tables = contract_tables(Dialect.SQLITE) + for spec in CONTRACT_TABLES: + assert ("updated_at" in tables.table(spec).c) is not spec.append_only, spec.name + + +@pytest.mark.parametrize("dialect", list(Dialect)) +def test_every_contract_table_has_exactly_one_spec(dialect: Dialect) -> None: + files = [p.stem for a in contracts.AREAS for p in contracts.ddl_files(a, dialect)] + contract = sorted(f for f in files if f != "namespace" and not _is_view(f, dialect)) + specs = sorted(spec.name for spec in CONTRACT_TABLES) + assert specs == contract + + +def _is_view(stem: str, dialect: Dialect) -> bool: + return any( + p.stem == stem and p.parent.name == "views" + for a in contracts.AREAS + for p in contracts.ddl_files(a, dialect) + ) + + +@pytest.mark.parametrize("kind", SQL_BACKENDS) +def test_declared_tables_match_the_contracts_ddl(kind: str, tmp_path: Path) -> None: + with contract_db(kind, tmp_path) as engine: + assert contract_drift(engine) == [] diff --git a/tests/v1/repositories/tables/test_repositories.py b/tests/v1/repositories/tables/test_repositories.py new file mode 100644 index 00000000..1feb3616 --- /dev/null +++ b/tests/v1/repositories/tables/test_repositories.py @@ -0,0 +1,322 @@ +from collections.abc import Iterator +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from decimal import Decimal +from pathlib import Path +from time import sleep + +import pytest +import sqlalchemy as sa +from sqlalchemy.engine import Engine + +from tests.v1.conftest import SQL_BACKENDS, contract_db +from traust_core.v1.errors import ConflictError, RepositoryError +from traust_core.v1.models.records import ( + AdvisoryPackageRecord, + AdvisoryRecord, + ArtifactRecord, + AssessmentRecord, + ComponentRecord, + CweRecord, + DecisionRecord, + DependencyRecord, + EventRecord, + FeedSnapshotRecord, + FindingCweRecord, + FindingRecord, + LayerRecord, + LocationRecord, + OccurrenceRecord, + OwnershipRecord, + PackageRecord, + ProductRecord, + ReleaseRecord, + RepositoryRecord, + RepositoryRefRecord, + ReviewItemRecord, + RunOrderRecord, + SbomRecord, + SpendEventRecord, + TeamRecord, +) +from traust_core.v1.repositories.traust.assessments import SqlAssessmentsUnitOfWork +from traust_core.v1.repositories.traust.dependencies import SqlDependenciesUnitOfWork +from traust_core.v1.repositories.traust.findings import SqlFindingsUnitOfWork +from traust_core.v1.repositories.traust.inventory import SqlInventoryUnitOfWork +from traust_core.v1.repositories.traust_ledger import SqlLedgerUnitOfWork +from traust_core.v1.repositories.traust_ops import SqlOpsUnitOfWork + +T0 = datetime(2026, 10, 5, tzinfo=UTC) +SHA = "a" * 40 + + +@pytest.fixture(params=SQL_BACKENDS) +def engine(request: pytest.FixtureRequest, tmp_path: Path) -> Iterator[Engine]: + with contract_db(request.param, tmp_path) as eng: + yield eng + + +@dataclass +class Scanned: + repository: RepositoryRecord + ref: RepositoryRefRecord + assessment: AssessmentRecord + finding: FindingRecord + + +@pytest.fixture +def scanned(engine: Engine) -> Scanned: + with SqlInventoryUnitOfWork(engine) as inv: + repo = inv.repositories.add(RepositoryRecord(url="https://github.com/o/r", public=True)) + assert repo.id is not None + ref = inv.repository_refs.add( + RepositoryRefRecord( + repository_id=repo.id, name="main", kind="branch", is_default=True, sha=SHA + ) + ) + inv.commit() + assert ref.id is not None + with SqlAssessmentsUnitOfWork(engine) as a: + assessment = a.assessments.add( + AssessmentRecord( + repository_ref_id=ref.id, + kind="audit", + status="completed", + assessor="skill:audit", + assessor_version="1.0", + occurred_at=T0, + ) + ) + a.commit() + with SqlFindingsUnitOfWork(engine) as f: + finding = f.findings.add( + FindingRecord( + repository_id=repo.id, fingerprint_algo="v3", fingerprint="fp1", title="SQLi" + ) + ) + f.commit() + return Scanned(repo, ref, assessment, finding) + + +def test_inventory_round_trip_and_natural_keys(engine: Engine, scanned: Scanned) -> None: + repo, ref = scanned.repository, scanned.ref + assert repo.id is not None and ref.id is not None + with SqlInventoryUnitOfWork(engine) as inv: + product = inv.products.add(ProductRecord(slug="ocp", name="OpenShift")) + assert product.id is not None + release = inv.releases.add(ReleaseRecord(product_id=product.id, version="4.22")) + assert release.id is not None + component = inv.components.add( + ComponentRecord(release_id=release.id, repository_ref_id=ref.id) + ) + team = inv.teams.add(TeamRecord(name="core")) + assert team.id is not None + owned = inv.ownerships.add( + OwnershipRecord(repository_id=repo.id, team_id=team.id, valid_from=T0) + ) + inv.commit() + + assert inv.products.find_by_slug("ocp") == product + assert inv.releases.find(product.id, "4.22") == release + assert inv.components.list_for_ref(ref.id) == [component] + assert inv.repositories.find_by_url("https://github.com/o/r") == repo + assert inv.repository_refs.find(repo.id, "main", SHA) == ref + assert inv.repository_refs.find(repo.id, None, SHA) is None + assert inv.repository_refs.default(repo.id) == ref + assert inv.teams.find_by_name("core") == team + assert inv.ownerships.current(repo.id) == [owned] + assert inv.ownerships.as_of(repo.id, T0 - timedelta(days=1)) == [] + + +def test_closing_ownership_is_an_update_the_database_touches( + engine: Engine, scanned: Scanned +) -> None: + repo = scanned.repository + assert repo.id is not None + with SqlInventoryUnitOfWork(engine) as inv: + team = inv.teams.add(TeamRecord(name="core")) + assert team.id is not None + owned = inv.ownerships.add( + OwnershipRecord(repository_id=repo.id, team_id=team.id, valid_from=T0) + ) + inv.commit() + sleep(0.01) + with SqlInventoryUnitOfWork(engine) as inv: + closed = inv.ownerships.update(owned.model_copy(update={"valid_to": T0 + timedelta(1)})) + inv.commit() + assert inv.ownerships.current(repo.id) == [] + assert inv.ownerships.as_of(repo.id, T0) == [closed] + assert closed.updated_at > owned.updated_at # type: ignore[operator] + + +def test_duplicate_natural_key_is_a_conflict(engine: Engine, scanned: Scanned) -> None: + with SqlInventoryUnitOfWork(engine) as inv, pytest.raises(ConflictError): + inv.repositories.add(RepositoryRecord(url=scanned.repository.url, public=False)) + + +def test_dependencies_trace_a_package_to_the_refs_that_ship_it( + engine: Engine, scanned: Scanned +) -> None: + with SqlDependenciesUnitOfWork(engine) as d: + sbom = d.sboms.add(SbomRecord(digest="sha256:sbom")) + package = d.packages.add(PackageRecord(ecosystem="go", name="golang.org/x/crypto")) + assert sbom.id is not None and package.id is not None + dep = d.dependencies.add( + DependencyRecord(sbom_id=sbom.id, package_id=package.id, version="0.1.0", direct=True) + ) + advisory = d.advisories.add(AdvisoryRecord(identifier="CVE-2026-0001")) + assert advisory.id is not None + alias = d.advisories.add(AdvisoryRecord(identifier="GHSA-x", alias_of_id=advisory.id)) + hit = d.advisory_packages.add( + AdvisoryPackageRecord(advisory_id=advisory.id, package_id=package.id) + ) + d.commit() + + assert d.sboms.find_by_digest("sha256:sbom") == sbom + assert d.packages.find("go", "golang.org/x/crypto") == package + assert d.dependencies.list_for_package(package.id) == [dep] + assert d.advisories.list_aliases(advisory.id) == [alias] + assert d.advisory_packages.list_for_package(package.id) == [hit] + + +def test_findings_chain_and_decisions_are_idempotent_by_statement( + engine: Engine, scanned: Scanned +) -> None: + finding, assessment = scanned.finding, scanned.assessment + assert finding.id is not None and assessment.id is not None + with SqlAssessmentsUnitOfWork(engine) as a: + occurrence = a.occurrences.add( + OccurrenceRecord( + finding_id=finding.id, + assessment_id=assessment.id, + severity="high", + cvss_score=Decimal("7.5"), + ) + ) + assert occurrence.id is not None + location = a.locations.add( + LocationRecord(occurrence_id=occurrence.id, path="a.go", line_start=3, line_end=4) + ) + artifact = a.artifacts.add( + ArtifactRecord(assessment_id=assessment.id, digest="sha256:r", kind="report", uri="f") + ) + a.commit() + assert a.occurrences.list_for_assessment(assessment.id) == [occurrence] + assert a.occurrences.find(finding.id, assessment.id) == occurrence + assert a.locations.list_for_occurrence(occurrence.id) == [location] + assert a.artifacts.find(assessment.id, "sha256:r") == artifact + assert a.assessments.latest(scanned.ref.id, "audit") == assessment # type: ignore[arg-type] + + with SqlFindingsUnitOfWork(engine) as f: + cwe = f.cwes.add(CweRecord(identifier="CWE-89", abstraction="base")) + assert cwe.id is not None + f.finding_cwes.add(FindingCweRecord(finding_id=finding.id, cwe_id=cwe.id)) + decision = f.decisions.add( + DecisionRecord( + finding_id=finding.id, + assessment_id=assessment.id, + statement_key="k1", + validity="true_positive", + occurred_at=T0, + ) + ) + review = f.review_items.add( + ReviewItemRecord(finding_id=finding.id, decision_id=decision.id, status="pending") + ) + f.commit() + assert f.findings.find_by_fingerprint(finding.repository_id, "v3", "fp1") == finding + assert f.finding_cwes.list_for_cwe(cwe.id) == f.finding_cwes.list_for_finding(finding.id) + assert f.decisions.find_by_statement(finding.id, "k1") == decision + assert f.decisions.list_since(T0 - timedelta(1), limit=10) == [decision] + assert f.review_items.pending(limit=10) == [review] + with pytest.raises(ConflictError): + f.decisions.add(decision.model_copy(update={"id": None, "created_at": None})) + + +def test_append_only_rows_reject_mutation_in_the_database(engine: Engine, scanned: Scanned) -> None: + with SqlAssessmentsUnitOfWork(engine) as a: + table = a.assessments.table + with pytest.raises((RepositoryError, ConflictError)): + a.assessments._execute("raw update", sa.update(table).values(status="failed")) + + +def test_run_orders_claim_in_priority_order_and_leases_expire( + engine: Engine, scanned: Scanned +) -> None: + def order(priority: int) -> RunOrderRecord: + return RunOrderRecord( + repository_id=scanned.repository.id, + lane="audit", + state="queued", + priority=priority, + requested_by="user:me", + ) + + with SqlOpsUnitOfWork(engine) as ops: + later = ops.run_orders.add(order(5)) + sooner = ops.run_orders.add(order(1)) + ops.commit() + + claimed = ops.run_orders.claim("w1", T0, timedelta(minutes=5)) + assert claimed is not None and claimed.id == sooner.id + assert claimed.state == "leased" and claimed.lease_holder == "w1" + next_claim = ops.run_orders.claim("w2", T0, timedelta(minutes=5)) + assert next_claim is not None and next_claim.id == later.id + assert ops.run_orders.claim("w3", T0, timedelta(minutes=5)) is None + + assert ops.run_orders.release_expired(T0 + timedelta(minutes=10)) == 2 + assert ops.run_orders.claim("w4", T0, timedelta(minutes=5)).id == sooner.id # type: ignore[union-attr] + ops.commit() + + +def test_ops_spend_and_feed_snapshots(engine: Engine, scanned: Scanned) -> None: + with SqlAssessmentsUnitOfWork(engine) as a: + artifact = a.artifacts.add(ArtifactRecord(digest="sha256:osv", kind="feed", uri="f")) + a.commit() + assert artifact.id is not None + with SqlOpsUnitOfWork(engine) as ops: + spend = ops.spend_events.add( + SpendEventRecord( + provider="anthropic", + model="m", + tokens_in=10, + tokens_out=5, + cost_usd=Decimal("0.25"), + occurred_at=T0, + ) + ) + old = ops.feed_snapshots.add( + FeedSnapshotRecord(artifact_id=artifact.id, feed="osv", retrieved_at=T0) + ) + new = ops.feed_snapshots.add( + FeedSnapshotRecord(artifact_id=artifact.id, feed="osv", retrieved_at=T0 + timedelta(1)) + ) + ops.commit() + assert ops.spend_events.list_between(T0, T0 + timedelta(1)) == [spend] + assert ops.feed_snapshots.latest("osv") == new != old + + +def test_ledger_events_prove_one_decision_each(engine: Engine, scanned: Scanned) -> None: + finding = scanned.finding + assert finding.id is not None and scanned.repository.id is not None + with SqlFindingsUnitOfWork(engine) as f: + decision = f.decisions.add( + DecisionRecord( + finding_id=finding.id, actor="user:me", statement_key="k", occurred_at=T0 + ) + ) + f.commit() + assert decision.id is not None + with SqlLedgerUnitOfWork(engine) as led: + layer = led.layers.add( + LayerRecord(repository_id=scanned.repository.id, merkle_signing_method="none") + ) + assert layer.id is not None + assert led.events.last_seq(layer.id) is None + event = led.events.add( + EventRecord(layer_id=layer.id, decision_id=decision.id, seq=0, event_payload=b"\x00") + ) + led.commit() + assert led.events.find_by_decision(decision.id) == event + assert led.events.last_seq(layer.id) == 0 + assert led.layers.list_for_repository(scanned.repository.id) == [layer] From d28fc3e2285db56c153939316e8c168b5b030e77 Mon Sep 17 00:00:00 2001 From: Shaun Smith <51304449+smith-xyz@users.noreply.github.com> Date: Wed, 21 Oct 2026 09:28:41 -0400 Subject: [PATCH 03/10] refactor(core): models are the contract tables; one SqlTraustUnitOfWork spans every repository; retire the stub repositories and traust_storage lane (XWING-2363) --- src/traust_core/v1/context/context.py | 8 +- src/traust_core/v1/contracts.py | 9 - src/traust_core/v1/models/assessment.py | 43 ++-- src/traust_core/v1/models/attestation.py | 35 +-- src/traust_core/v1/models/code_facts.py | 10 +- src/traust_core/v1/models/common.py | 4 +- src/traust_core/v1/models/compliance.py | 10 +- src/traust_core/v1/models/dependencies.py | 98 +++++++++ .../v1/models/detection_quality.py | 16 +- src/traust_core/v1/models/exposure.py | 129 ----------- src/traust_core/v1/models/findings.py | 143 +++++++----- src/traust_core/v1/models/insight.py | 4 +- src/traust_core/v1/models/inventory.py | 121 +++++++++++ src/traust_core/v1/models/ledger.py | 29 ++- src/traust_core/v1/models/onboarding.py | 13 +- src/traust_core/v1/models/operations.py | 75 ++++--- src/traust_core/v1/models/portfolio.py | 163 -------------- src/traust_core/v1/models/pqc.py | 6 +- src/traust_core/v1/models/records/__init__.py | 78 ------- .../v1/models/records/dependencies.py | 45 ---- src/traust_core/v1/models/records/findings.py | 110 ---------- .../v1/models/records/inventory.py | 49 ----- src/traust_core/v1/models/records/ledger.py | 25 --- src/traust_core/v1/models/records/ops.py | 54 ----- src/traust_core/v1/models/remediation.py | 12 +- src/traust_core/v1/models/storage.py | 64 ------ src/traust_core/v1/models/validation.py | 6 +- src/traust_core/v1/providers/analysis.py | 2 +- src/traust_core/v1/providers/feeds.py | 2 +- src/traust_core/v1/providers/sources.py | 2 +- src/traust_core/v1/repositories/artifacts.py | 32 --- src/traust_core/v1/repositories/assessment.py | 47 ---- .../v1/repositories/attestation.py | 27 --- src/traust_core/v1/repositories/code_facts.py | 22 -- src/traust_core/v1/repositories/compliance.py | 50 ----- src/traust_core/v1/repositories/contract.py | 67 +++++- .../v1/repositories/detection_quality.py | 44 ---- src/traust_core/v1/repositories/exposure.py | 55 ----- src/traust_core/v1/repositories/findings.py | 53 ----- src/traust_core/v1/repositories/migrations.py | 20 -- src/traust_core/v1/repositories/operations.py | 48 ----- src/traust_core/v1/repositories/portfolio.py | 71 ------ .../v1/repositories/remediation.py | 40 ---- src/traust_core/v1/repositories/storage.py | 171 --------------- .../traust/assessments/__init__.py | 11 - .../traust/assessments/artifact.py | 16 +- .../traust/assessments/assessment.py | 20 +- .../traust/assessments/location.py | 12 +- .../traust/assessments/occurrence.py | 20 +- .../traust/dependencies/__init__.py | 13 -- .../traust/dependencies/advisory.py | 16 +- .../traust/dependencies/advisory_package.py | 20 +- .../traust/dependencies/dependency.py | 16 +- .../repositories/traust/dependencies/image.py | 20 +- .../traust/dependencies/package.py | 20 +- .../repositories/traust/dependencies/sbom.py | 12 +- .../repositories/traust/findings/__init__.py | 14 -- .../v1/repositories/traust/findings/cwe.py | 12 +- .../repositories/traust/findings/decision.py | 20 +- .../repositories/traust/findings/finding.py | 20 +- .../traust/findings/finding_cwe.py | 16 +- .../traust/findings/finding_external_ref.py | 16 +- .../traust/findings/finding_label.py | 16 +- .../traust/findings/review_item.py | 16 +- .../repositories/traust/inventory/__init__.py | 14 -- .../traust/inventory/component.py | 20 +- .../traust/inventory/ownership.py | 20 +- .../repositories/traust/inventory/product.py | 12 +- .../repositories/traust/inventory/release.py | 16 +- .../traust/inventory/repository.py | 12 +- .../traust/inventory/repository_ref.py | 24 +-- .../v1/repositories/traust/inventory/team.py | 12 +- .../v1/repositories/traust_ledger/__init__.py | 9 - .../v1/repositories/traust_ledger/events.py | 16 +- .../v1/repositories/traust_ledger/layers.py | 12 +- .../v1/repositories/traust_ops/__init__.py | 12 -- .../traust_ops/assessment_input.py | 16 +- .../repositories/traust_ops/feed_snapshot.py | 12 +- .../v1/repositories/traust_ops/run_batch.py | 12 +- .../v1/repositories/traust_ops/run_order.py | 20 +- .../v1/repositories/traust_ops/spend_event.py | 16 +- src/traust_core/v1/repositories/triage.py | 115 ---------- src/traust_core/v1/repositories/validation.py | 26 --- src/traust_core/v1/services/__init__.py | 18 +- .../v1/services/artifact_publishing.py | 45 ++-- src/traust_core/v1/services/assessment.py | 18 +- src/traust_core/v1/services/attestation.py | 22 +- src/traust_core/v1/services/code_facts.py | 12 +- src/traust_core/v1/services/compliance.py | 4 +- .../v1/services/detection_quality.py | 4 +- src/traust_core/v1/services/exposure.py | 33 ++- .../v1/services/findings/__init__.py | 3 +- .../v1/services/findings/decisions.py | 68 ++---- .../v1/services/findings/intake.py | 6 +- .../v1/services/findings/precedents.py | 4 +- .../v1/services/findings/review.py | 15 +- .../v1/services/findings/service.py | 4 +- .../v1/services/findings/taxonomy.py | 4 +- src/traust_core/v1/services/insight.py | 4 +- src/traust_core/v1/services/ledger.py | 6 +- src/traust_core/v1/services/operations.py | 31 ++- src/traust_core/v1/services/portfolio.py | 53 ++--- src/traust_core/v1/services/pqc.py | 8 +- src/traust_core/v1/services/remediation.py | 8 +- src/traust_core/v1/services/storage.py | 105 --------- src/traust_core/v1/services/validation.py | 4 +- tests/v1/conftest.py | 13 -- tests/v1/fixtures.py | 29 +++ tests/v1/models/test_artifacts.py | 2 +- .../repositories/tables/test_repositories.py | 203 +++++++++--------- .../v1/repositories/test_analysis_results.py | 2 +- tests/v1/repositories/test_triage.py | 108 ---------- tests/v1/services/test_artifact_publishing.py | 64 +++--- tests/v1/services/test_storage.py | 148 ------------- tests/v1/test_stubs.py | 3 +- 115 files changed, 1090 insertions(+), 2795 deletions(-) create mode 100644 src/traust_core/v1/models/dependencies.py delete mode 100644 src/traust_core/v1/models/exposure.py create mode 100644 src/traust_core/v1/models/inventory.py delete mode 100644 src/traust_core/v1/models/portfolio.py delete mode 100644 src/traust_core/v1/models/records/__init__.py delete mode 100644 src/traust_core/v1/models/records/dependencies.py delete mode 100644 src/traust_core/v1/models/records/findings.py delete mode 100644 src/traust_core/v1/models/records/inventory.py delete mode 100644 src/traust_core/v1/models/records/ledger.py delete mode 100644 src/traust_core/v1/models/records/ops.py delete mode 100644 src/traust_core/v1/models/storage.py delete mode 100644 src/traust_core/v1/repositories/artifacts.py delete mode 100644 src/traust_core/v1/repositories/assessment.py delete mode 100644 src/traust_core/v1/repositories/attestation.py delete mode 100644 src/traust_core/v1/repositories/code_facts.py delete mode 100644 src/traust_core/v1/repositories/compliance.py delete mode 100644 src/traust_core/v1/repositories/detection_quality.py delete mode 100644 src/traust_core/v1/repositories/exposure.py delete mode 100644 src/traust_core/v1/repositories/findings.py delete mode 100644 src/traust_core/v1/repositories/migrations.py delete mode 100644 src/traust_core/v1/repositories/operations.py delete mode 100644 src/traust_core/v1/repositories/portfolio.py delete mode 100644 src/traust_core/v1/repositories/remediation.py delete mode 100644 src/traust_core/v1/repositories/storage.py delete mode 100644 src/traust_core/v1/repositories/triage.py delete mode 100644 src/traust_core/v1/repositories/validation.py delete mode 100644 src/traust_core/v1/services/storage.py create mode 100644 tests/v1/fixtures.py delete mode 100644 tests/v1/repositories/test_triage.py delete mode 100644 tests/v1/services/test_storage.py diff --git a/src/traust_core/v1/context/context.py b/src/traust_core/v1/context/context.py index eae14abd..4bc40782 100644 --- a/src/traust_core/v1/context/context.py +++ b/src/traust_core/v1/context/context.py @@ -22,13 +22,13 @@ create_database_engine, ) from traust_core.v1.repositories.analysis_results import AnalysisResultsRepository +from traust_core.v1.repositories.contract import SqlTraustUnitOfWork from traust_core.v1.repositories.sql import Dialect -from traust_core.v1.repositories.storage import SqlStorageUnitOfWork from traust_core.v1.repositories.tables import TableFactory from traust_core.v1.services.artifact_publishing import ( ArtifactPublisher, PublishSpecs, - StorageIndex, + SqlArtifactIndex, ) from traust_core.v1.services.operations import Materializer @@ -89,14 +89,14 @@ def tables(self) -> TableFactory: return TableFactory.for_dialect(self.dialect) def unit_of_work(self, factory: Callable[[Engine], U]) -> U: - """e.g. unit_of_work(SqlInventoryUnitOfWork); its repositories use this engine's tables.""" + """e.g. unit_of_work(SqlTraustUnitOfWork): every repository, one transaction.""" return factory(self.database) def artifact_publisher(self, specs: PublishSpecs) -> ArtifactPublisher: index = None if self.config.database.url is not None: engine = self.database - index = StorageIndex(lambda: SqlStorageUnitOfWork(engine), self.clock) + index = SqlArtifactIndex(lambda: SqlTraustUnitOfWork(engine)) return ArtifactPublisher( specs, AnalysisResultsRepository(self.object_store, ARTIFACT_FOR_KIND), index ) diff --git a/src/traust_core/v1/contracts.py b/src/traust_core/v1/contracts.py index f721ad8f..be1f96ac 100644 --- a/src/traust_core/v1/contracts.py +++ b/src/traust_core/v1/contracts.py @@ -123,15 +123,6 @@ def resolve(schema_name: str, node: Mapping[str, Any]) -> list[tuple[str, Mappin return [(schema_name, own), *(r for b in branches for r in resolve(schema_name, b))] -@cache -def _profiles() -> dict[str, dict[str, Any]]: - return _read_json(_root() / "storage/v1/profiles.json")["artifacts"] - - -def storage_profile(name: str) -> dict[str, Any]: - return _profiles().get(name, {}) - - @cache def _registry() -> Registry: resources_ = [] diff --git a/src/traust_core/v1/models/assessment.py b/src/traust_core/v1/models/assessment.py index 262d3d9c..1544a901 100644 --- a/src/traust_core/v1/models/assessment.py +++ b/src/traust_core/v1/models/assessment.py @@ -5,11 +5,9 @@ from typing import NewType from uuid import UUID -from traust_core.v1.models.base import Entity, Model -from traust_core.v1.models.common import ArtifactRef +from traust_core.v1.models.base import AppendOnly, Model from traust_core.v1.models.findings import Label, SkillInput from traust_core.v1.models.vocabulary import ( - AuditProfile, ChainSeverity, IsolationPosture, PeachBoundaryType, @@ -64,7 +62,7 @@ class AssessmentFilter(Model): ... class ThreatFilter(Model): ... -class AssessmentInput(SkillInput): ... +class AssessmentSkillInput(SkillInput): ... class Coverage(Model): @@ -76,16 +74,31 @@ def ratio(self) -> float: raise NotImplementedError("stub: replaces build_coverage_gap_rollup.build_rollup") -class Assessment(Entity): - revision_id: UUID - kind: AssessmentKind - profile: AuditProfile | None = None - artifact: ArtifactRef - assessed_at: datetime - coverage: Coverage | None = None +class Assessment(AppendOnly): + """3a · what was examined: a ref (and image), by an assessor, with what outcome.""" + repository_ref_id: UUID + image_id: UUID | None = None + baseline_assessment_id: UUID | None = None + run_order_id: UUID | None = None + kind: str + status: str + assessor: str + assessor_version: str + occurred_at: datetime -class Threat(Entity): + +class Artifact(AppendOnly): + """A document an assessment produced (or imported); bytes live at uri.""" + + assessment_id: UUID | None = None + digest: str + kind: str + uri: str + + +# Artifact-lane content: read from the assessment's artifacts, never tables. +class Threat(Model): assessment_id: UUID label: Label title: str @@ -97,7 +110,7 @@ class Threat(Entity): techniques: tuple[AttackTechnique, ...] = () -class AttackChain(Entity): +class AttackChain(Model): assessment_id: UUID threat_ids: tuple[UUID, ...] severity: ChainSeverity @@ -116,7 +129,7 @@ class IsolationResult(Model): class Freshness(Model): - code_line_id: UUID + repository_ref_id: UUID last_assessed_at: datetime | None = None churn_ratio: float | None = None @@ -127,4 +140,4 @@ def age_days(self, now: datetime) -> int | None: class Population(Model): as_of: datetime rows: tuple[PopulationRow, ...] = () - never_assessed_code_line_ids: tuple[UUID, ...] = () + never_assessed_repository_ids: tuple[UUID, ...] = () diff --git a/src/traust_core/v1/models/attestation.py b/src/traust_core/v1/models/attestation.py index 57b27512..fa0f19d3 100644 --- a/src/traust_core/v1/models/attestation.py +++ b/src/traust_core/v1/models/attestation.py @@ -3,25 +3,15 @@ from datetime import datetime from enum import StrEnum from typing import NewType -from uuid import UUID -from traust_core.v1.models.base import Entity, Model +from traust_core.v1.models.base import Model from traust_core.v1.models.common import Principal -from traust_core.v1.models.vocabulary import ReviewItemStatus, ReviewReason, SourceType +from traust_core.v1.models.vocabulary import SourceType CommentSourceName = NewType("CommentSourceName", str) ExternalHandle = NewType("ExternalHandle", str) -class ReviewSubjectKind(StrEnum): - FINDING = "finding" - WORK_ITEM = "work-item" - ORDER = "order" - FLEET_FIX = "fleet-fix" - RULE_DRAFT = "rule-draft" - IDENTITY_MOVE = "identity-move" - - class AttestationDecision(StrEnum): CONFIRM = "confirm" REJECT = "reject" @@ -46,11 +36,6 @@ class Comment(Model): url: str | None = None -class ReviewSubject(Model): - kind: ReviewSubjectKind - id: UUID - - class Statement(Model): source: SourceType author: Principal | None = None @@ -58,19 +43,3 @@ class Statement(Model): url: str | None = None at: datetime trust: TrustTier - - -class Attestation(Entity): - review_item_id: UUID - principal: Principal - decision: AttestationDecision - statement: Statement | None = None - - -class ReviewItem(Entity): - subject: ReviewSubject - reason: ReviewReason - status: ReviewItemStatus - - def resolve(self, attestation: Attestation) -> ReviewItem: - raise NotImplementedError("stub: replaces ledger resolve_handler.resolve_review_item") diff --git a/src/traust_core/v1/models/code_facts.py b/src/traust_core/v1/models/code_facts.py index e558eacd..faa7e7c8 100644 --- a/src/traust_core/v1/models/code_facts.py +++ b/src/traust_core/v1/models/code_facts.py @@ -3,8 +3,8 @@ from enum import StrEnum from uuid import UUID -from traust_core.v1.models.base import Entity, Model -from traust_core.v1.models.common import Location +from traust_core.v1.models.base import Model +from traust_core.v1.models.common import CodeSpan class CodeFactKind(StrEnum): @@ -76,8 +76,8 @@ class SymbolQuery(Model): ... class CollectResult(Model): ... -class CodeFact(Entity): - revision_id: UUID +class CodeFact(Model): + repository_ref_id: UUID kind: CodeFactKind - location: Location | None = None + location: CodeSpan | None = None detail: CodeFactDetail diff --git a/src/traust_core/v1/models/common.py b/src/traust_core/v1/models/common.py index e0cec01e..212e980f 100644 --- a/src/traust_core/v1/models/common.py +++ b/src/traust_core/v1/models/common.py @@ -67,12 +67,12 @@ def has_role(self, role: Role) -> bool: raise NotImplementedError("stub: replaces ledger require_admin / restatement authority") -class Location(Model): +class CodeSpan(Model): path: RepoPath line: int | None = None @classmethod - def parse(cls, raw: str) -> Location: + def parse(cls, raw: str) -> CodeSpan: raise NotImplementedError("stub: replaces check_location_paths.py") diff --git a/src/traust_core/v1/models/compliance.py b/src/traust_core/v1/models/compliance.py index 4d6352d3..7d322ab1 100644 --- a/src/traust_core/v1/models/compliance.py +++ b/src/traust_core/v1/models/compliance.py @@ -4,7 +4,7 @@ from typing import NewType from uuid import UUID -from traust_core.v1.models.base import Entity, Model +from traust_core.v1.models.base import Model from traust_core.v1.models.common import ArtifactRef, Principal, Sha256 from traust_core.v1.models.findings import SkillInput from traust_core.v1.models.vocabulary import ( @@ -86,7 +86,7 @@ def evaluate(self, evidence: ComplianceEvidence) -> AssertionResult: raise NotImplementedError("stub: replaces compliance_assert.eval_assertion") -class ComplianceScope(Entity): +class ComplianceScope(Model): product_id: UUID frameworks: tuple[ComplianceFramework, ...] = () targets: tuple[ComplianceTarget, ...] = () @@ -104,7 +104,7 @@ class ControlResult(Model): evidence: tuple[EvidenceId, ...] = () -class ComplianceAssessment(Entity): +class ComplianceAssessment(Model): scope_id: UUID framework: ComplianceFramework assessed_at: datetime @@ -115,7 +115,7 @@ def posture(self) -> CompliancePosture: raise NotImplementedError("stub: replaces compliance/dashboard.latest_per_target_framework") -class Adr(Entity): +class Adr(Model): product_id: UUID number: int title: str @@ -123,7 +123,7 @@ class Adr(Entity): artifact: ArtifactRef -class DocVariance(Entity): +class DocVariance(Model): product_id: UUID kind: DocVarianceKind disposition: DocVarianceDisposition diff --git a/src/traust_core/v1/models/dependencies.py b/src/traust_core/v1/models/dependencies.py new file mode 100644 index 00000000..4437b76c --- /dev/null +++ b/src/traust_core/v1/models/dependencies.py @@ -0,0 +1,98 @@ +"""2 · Dependencies - third-party code and known vulnerabilities. + +Table models mirror the traust contracts DDL; ranges, symbols and impact +detail live in artifacts (scanner output, impact analysis), not columns. +""" + +from __future__ import annotations + +from datetime import datetime +from enum import StrEnum +from typing import NewType +from uuid import UUID + +from traust_core.v1.models.base import AppendOnly, Entity, Model +from traust_core.v1.models.findings import SkillInput + +AdvisoryIdentifier = NewType("AdvisoryIdentifier", str) +Version = NewType("Version", str) +FeedName = NewType("FeedName", str) + + +class Package(Entity): + source_repository_id: UUID | None = None + sbom_id: UUID | None = None + ecosystem: str + name: str + + +class Sbom(AppendOnly): + digest: str + artifact_id: UUID | None = None + + +class Dependency(AppendOnly): + sbom_id: UUID + package_id: UUID + version: str + direct: bool + + +class Image(AppendOnly): + digest: str + platform: str | None = None + repository_ref_id: UUID | None = None + sbom_id: UUID | None = None + + +class Advisory(Entity): + alias_of_id: UUID | None = None + identifier: str + published_at: datetime | None = None + + +class AdvisoryPackage(Entity): + advisory_id: UUID + package_id: UUID + + +class VexStatus(StrEnum): + NOT_AFFECTED = "not_affected" + AFFECTED = "affected" + FIXED = "fixed" + UNDER_INVESTIGATION = "under_investigation" + + +# Shapes land with their implementation slice. +class ImpactEvidence(Model): ... + + +class DependencyIngest(Model): ... + + +class DependencyProvenance(Model): ... + + +class ProvenanceCheck(Model): ... + + +class AffectedDependency(Model): ... + + +class ImpactFilter(Model): ... + + +class ExposureFilter(Model): ... + + +class ExposureDimension(Model): ... + + +class ExposureRollup(Model): ... + + +class ImpactAnalysis(Model): + """Read from the impact-analysis artifact of an assessment.""" + + +class ImpactInput(SkillInput): ... diff --git a/src/traust_core/v1/models/detection_quality.py b/src/traust_core/v1/models/detection_quality.py index ecc7abf5..607936af 100644 --- a/src/traust_core/v1/models/detection_quality.py +++ b/src/traust_core/v1/models/detection_quality.py @@ -3,9 +3,9 @@ from datetime import datetime from uuid import UUID -from traust_core.v1.models.base import Entity, Model +from traust_core.v1.models.base import Model from traust_core.v1.models.common import ArtifactRef -from traust_core.v1.models.exposure import AdvisoryIdentifier +from traust_core.v1.models.dependencies import AdvisoryIdentifier from traust_core.v1.models.vocabulary import ( BenchmarkTargetEmbargo, BenchmarkTargetEvidenceKind, @@ -49,10 +49,10 @@ class BenchmarkFilter(Model): ... class MissFilter(Model): ... -class BenchmarkTarget(Entity): +class BenchmarkTarget(Model): advisory: AdvisoryIdentifier repository_id: UUID - fix_revision_id: UUID + fix_repository_ref_id: UUID provenance: BenchmarkTargetProvenance embargo: BenchmarkTargetEmbargo evidence: BenchmarkTargetEvidenceKind @@ -66,7 +66,7 @@ class BenchmarkMatch(Model): injection_score: float | None = None -class BenchmarkRun(Entity): +class BenchmarkRun(Model): rule_pack_id: UUID started_at: datetime matches: tuple[BenchmarkMatch, ...] = () @@ -75,13 +75,13 @@ def scorecard(self) -> Scorecard: raise NotImplementedError("stub: replaces match_benchmark.score, sweep/benchmark.cmd_score") -class RulePack(Entity): +class RulePack(Model): name: str version: str rules: tuple[RuleRef, ...] = () -class RuleDraft(Entity): +class RuleDraft(Model): source: RuleDraftSource language: Language artifact: ArtifactRef @@ -95,7 +95,7 @@ class Calibration(Model): delta: CalibrationDelta -class CriticalMiss(Entity): +class CriticalMiss(Model): advisory: AdvisoryIdentifier repository_id: UUID reason: MissReason diff --git a/src/traust_core/v1/models/exposure.py b/src/traust_core/v1/models/exposure.py deleted file mode 100644 index 50642847..00000000 --- a/src/traust_core/v1/models/exposure.py +++ /dev/null @@ -1,129 +0,0 @@ -from __future__ import annotations - -from datetime import datetime -from enum import StrEnum -from typing import NewType -from uuid import UUID - -from traust_core.v1.models.base import Entity, Model -from traust_core.v1.models.common import ArtifactRef -from traust_core.v1.models.findings import SkillInput -from traust_core.v1.models.vocabulary import ( - DependencyEvidenceLevel, - Ecosystem, - ImpactClassification, - ImpactTier, - Severity, -) - -AdvisoryIdentifier = NewType("AdvisoryIdentifier", str) -Version = NewType("Version", str) -QualifiedSymbol = NewType("QualifiedSymbol", str) -FeedName = NewType("FeedName", str) - - -class VexStatus(StrEnum): - NOT_AFFECTED = "not_affected" - AFFECTED = "affected" - FIXED = "fixed" - UNDER_INVESTIGATION = "under_investigation" - - -# Shapes land with their implementation slice. -class SymbolSource(Model): ... - - -class ImpactEvidence(Model): ... - - -class DependencyIngest(Model): ... - - -class DependencyProvenance(Model): ... - - -class ProvenanceCheck(Model): ... - - -class AffectedDependency(Model): ... - - -class ImpactFilter(Model): ... - - -class ExposureFilter(Model): ... - - -class ExposureDimension(Model): ... - - -class ExposureRollup(Model): ... - - -class ImpactInput(SkillInput): ... - - -class FeedRun(Entity): - feed: FeedName - error: str | None = None - - -class Package(Model): - ecosystem: Ecosystem - name: str - - -class AffectedRange(Model): - package: Package - introduced: Version | None = None - fixed: Version | None = None - - def contains(self, version: Version) -> bool: - raise NotImplementedError("stub: replaces impact/analyzer.version_in_range") - - -class Advisory(Entity): - identifier: AdvisoryIdentifier - aliases: tuple[AdvisoryIdentifier, ...] = () - summary: str - severity: Severity | None = None - affected: tuple[AffectedRange, ...] = () - published_at: datetime | None = None - kev: bool = False - epss: float | None = None - - def affects(self, package: Package, version: Version) -> bool: - raise NotImplementedError("stub: replaces dependency-watch/fleet_sweep.fixed_range") - - def fixed_in(self, package: Package) -> Version | None: - raise NotImplementedError("stub: replaces adapters/osv.fixed_version") - - -class AdvisorySymbols(Model): - advisory_id: UUID - symbols: tuple[QualifiedSymbol, ...] = () - entry_points: tuple[QualifiedSymbol, ...] = () - source: SymbolSource - - -class Dependency(Entity): - revision_id: UUID - package: Package - version: Version - direct: bool - evidence: DependencyEvidenceLevel - - -class ImpactAnalysis(Entity): - advisory_id: UUID - revision_id: UUID - classification: ImpactClassification - tier: ImpactTier - evidence: tuple[ImpactEvidence, ...] = () - artifact: ArtifactRef - - -class VexStatement(Model): - advisory: AdvisoryIdentifier - product_id: UUID - status: VexStatus diff --git a/src/traust_core/v1/models/findings.py b/src/traust_core/v1/models/findings.py index 1e5f21db..cb786998 100644 --- a/src/traust_core/v1/models/findings.py +++ b/src/traust_core/v1/models/findings.py @@ -2,25 +2,23 @@ from collections.abc import Mapping, Sequence from datetime import datetime +from decimal import Decimal from enum import StrEnum from typing import NewType from uuid import UUID -from traust_core.v1.models.base import Entity, Model -from traust_core.v1.models.common import ArtifactRef, Location, Principal, Provenance -from traust_core.v1.models.integrity import Fingerprint +from traust_core.v1.models.base import AppendOnly, Entity, Model +from traust_core.v1.models.common import CodeSpan from traust_core.v1.models.vocabulary import ( DispositionAssurance, - DispositionEmbargo, DispositionResolution, - EvidenceGrade, FindingOrigin, Severity, Validity, ) Label = NewType("Label", str) -Cwe = NewType("Cwe", str) +CweId = NewType("CweId", str) Category = NewType("Category", str) Confidence = NewType("Confidence", float) Rationale = NewType("Rationale", str) @@ -49,12 +47,6 @@ class FindingDimension(StrEnum): class WeaknessLabels(Model): ... -class SeverityOverride(Model): ... - - -class ReviewContext(Model): ... - - class PrecedentReason(Model): ... @@ -67,9 +59,6 @@ class IntakeResult(Model): ... class RecordDecision(Model): ... -class TriageRecorded(Model): ... - - class ReviewCase(Model): ... @@ -92,9 +81,9 @@ class FindingCandidate(Model): label: Label repository_id: UUID title: str - cwes: tuple[Cwe, ...] = () + cwes: tuple[CweId, ...] = () category: Category | None = None - locations: tuple[Location, ...] = () + locations: tuple[CodeSpan, ...] = () reported_severity: Severity confidence: Confidence origin: FindingOrigin @@ -105,50 +94,99 @@ def parse_many(cls, raw: bytes, format: CandidateFormat) -> list[FindingCandidat class Finding(Entity): + """3b · one weakness or vulnerability in a repository, identified by fingerprint.""" + repository_id: UUID - match_key: Fingerprint + advisory_id: UUID | None = None + advisory_match: str | None = None + fingerprint_algo: str + fingerprint: str + kind: str | None = None title: str - cwes: tuple[Cwe, ...] = () - category: Category | None = None - labels: WeaknessLabels - origin: FindingOrigin - first_seen_at: datetime -class Occurrence(Entity): +class Occurrence(AppendOnly): + """A finding as one assessment reported it.""" + finding_id: UUID assessment_id: UUID - label: Label - locations: tuple[Location, ...] = () - reported_severity: Severity - confidence: Confidence - artifact: ArtifactRef + severity: str + cvss_vector: str | None = None + cvss_score: Decimal | None = None + rule: str | None = None + + +class Location(AppendOnly): + """The location table: where an occurrence was reported, relative to its commit.""" + occurrence_id: UUID + path: str + line_start: int | None = None + line_end: int | None = None + language: str | None = None + symbol: str | None = None -class Decision(Entity): + +class Cwe(Entity): + """MITRE CWE catalog entry - reference data, loaded read-only.""" + + identifier: str + abstraction: str | None = None + mapping_usage: str | None = None + catalog_version: str | None = None + + +class FindingCwe(Entity): finding_id: UUID - code_line_id: UUID | None = None - validity: Validity | None = None - resolution: DispositionResolution | None = None - severity_override: SeverityOverride | None = None - evidence_grade: EvidenceGrade - evidence: tuple[EvidenceRef, ...] = () - embargo: DispositionEmbargo | None = None - actor: Principal - provenance: Provenance - rationale: Rationale - review: ReviewContext | None = None + cwe_id: UUID + + +class FindingLabel(Entity): + finding_id: UUID + taxonomy: str + label: str + + +class FindingExternalRef(Entity): + finding_id: UUID + system: str + ref: str + + +class Decision(AppendOnly): + """A judgement on a finding; corrections are new decisions (restates_id).""" + + finding_id: UUID + assessment_id: UUID | None = None + fixed_repository_ref_id: UUID | None = None + repository_ref_id: UUID | None = None restates_id: UUID | None = None + artifact_id: UUID | None = None + statement_key: str + validity: str | None = None + resolution: str | None = None + evidence_grade: str | None = None + reason: str | None = None + severity: str | None = None + source_channel: str | None = None + actor: str | None = None + occurred_at: datetime - def statement_key(self) -> StatementKey: - raise NotImplementedError("stub: replaces ledger compute_event_id (classifier steps 9/12)") +class ReviewItem(Entity): + finding_id: UUID + decision_id: UUID | None = None + resolved_by_decision_id: UUID | None = None + reason: str | None = None + status: str + +# Read models over the views (current_decision, current_resolution, finding_status). class Disposition(Model): finding_id: UUID - code_line_id: UUID - validity: Validity - resolution: DispositionResolution + line: str | None = None + validity: Validity | None = None + resolution: DispositionResolution | None = None assurance: DispositionAssurance conflict: bool = False fp_overridden: bool = False @@ -209,16 +247,3 @@ class FindingSeriesPoint(Model): regressions: int mttr_days: float | None = None risk_index: float - - -# Legacy triage row; becomes a Decision in the Findings slice. -class TriageVerdict(Model): - binding_id: str - artifact_digest: str - finding_id: str - source_finding_id: str | None = None - triage_completed: str - verdict: str - severity: str | None = None - vote_breakdown: Mapping[str, int] | None = None - rationale: str | None = None diff --git a/src/traust_core/v1/models/insight.py b/src/traust_core/v1/models/insight.py index bc8d5b49..47ba4b5f 100644 --- a/src/traust_core/v1/models/insight.py +++ b/src/traust_core/v1/models/insight.py @@ -6,7 +6,6 @@ from traust_core.v1.models.base import Model from traust_core.v1.models.common import TimeWindow -from traust_core.v1.models.portfolio import BusinessUnit, TeamName class ViewName(StrEnum): @@ -34,7 +33,6 @@ class ViewModel(Model): ... class ViewFilter(Model): product_ids: tuple[UUID, ...] = () - business_units: tuple[BusinessUnit, ...] = () - teams: tuple[TeamName, ...] = () + team_ids: tuple[UUID, ...] = () as_of: datetime | None = None window: TimeWindow | None = None diff --git a/src/traust_core/v1/models/inventory.py b/src/traust_core/v1/models/inventory.py new file mode 100644 index 00000000..a228fa72 --- /dev/null +++ b/src/traust_core/v1/models/inventory.py @@ -0,0 +1,121 @@ +"""1 · Inventory - what exists and who owns it. + +Table models mirror the traust contracts DDL column for column; the rest are +service types for questions answered at read time (RepoState, filters). +""" + +from __future__ import annotations + +from datetime import datetime +from enum import StrEnum +from typing import NewType +from uuid import UUID + +from traust_core.v1.models.assessment import Freshness +from traust_core.v1.models.base import Entity, Model + +ProductSlug = NewType("ProductSlug", str) +RefName = NewType("RefName", str) +VersionLabel = NewType("VersionLabel", str) +TeamName = NewType("TeamName", str) +OrgName = NewType("OrgName", str) + + +class Product(Entity): + slug: str + name: str + + +class Release(Entity): + product_id: UUID + version: str + + +class Component(Entity): + release_id: UUID + repository_ref_id: UUID + + +class Repository(Entity): + upstream_id: UUID | None = None + url: str + public: bool + + +class RepositoryRef(Entity): + """A commit, optionally on a named branch or tag; a line is (repository_id, name).""" + + repository_id: UUID + name: str | None = None + kind: str | None = None + is_default: bool = False + sha: str + manifest_sbom_id: UUID | None = None + + +class Team(Entity): + name: str + + +class Ownership(Entity): + repository_id: UUID + team_id: UUID + valid_from: datetime + valid_to: datetime | None = None + + +class ExposureClass(StrEnum): + PUBLIC_EXTERNAL = "public-external" + PUBLIC_INTERNAL = "public-internal" + PRIVATE_EXTERNAL = "private-external" + PRIVATE_INTERNAL = "private-internal" + + +class RiskTier(StrEnum): + P0 = "P0" + P1 = "P1" + P2 = "P2" + P3 = "P3" + + +class ForgeHost(StrEnum): + GITHUB = "github" + GITLAB = "gitlab" + + +# Shapes land with their implementation slice. +class OwnershipSource(Model): ... + + +class SyncReport(Model): ... + + +class RepositoryFilter(Model): ... + + +class ComponentFilter(Model): ... + + +class ReleaseChange(Model): ... + + +class CrownJewelTier(Model): ... + + +class Liveness(Model): + """Forge facts fetched live at routing time - never stored.""" + + archived: bool + public: bool + pushed_at: datetime | None = None + observed_at: datetime + + +class RepoState(Model): + repository_id: UUID + exposure: ExposureClass + risk_tier: RiskTier + crown_jewel: CrownJewelTier | None = None + freshness: Freshness | None = None + open_critical_high: int + liveness: Liveness | None = None diff --git a/src/traust_core/v1/models/ledger.py b/src/traust_core/v1/models/ledger.py index f9cffe1a..1fc95740 100644 --- a/src/traust_core/v1/models/ledger.py +++ b/src/traust_core/v1/models/ledger.py @@ -4,12 +4,33 @@ from enum import StrEnum from uuid import UUID -from traust_core.v1.models.attestation import Attestation -from traust_core.v1.models.base import Model +from traust_core.v1.models.base import AppendOnly, Entity, Model from traust_core.v1.models.common import Principal from traust_core.v1.models.findings import Decision +class Layer(Entity): + """5 · one Merkle tree of decision events for a repository (or one line), signed per root.""" + + repository_id: UUID + line: str | None = None + merkle_root: str | None = None + merkle_epoch: int | None = None + merkle_size: int | None = None + merkle_root_signature: str | None = None + merkle_signing_method: str + merkle_signature_format: int | None = None + + +class Event(AppendOnly): + """5 · the proof of one decision: its leaf in a layer.""" + + layer_id: UUID + decision_id: UUID + seq: int + event_payload: bytes + + # Members land with the Ledger slice (from traust-ledger _internal/kinds.py). class LedgerEventKind(StrEnum): ... @@ -34,10 +55,6 @@ class LedgerEvent(Model): def from_decision(cls, decision: Decision, stream_id: UUID) -> LedgerEvent: raise NotImplementedError("stub: replaces ledger events/builders.py, lib/event_time.py") - @classmethod - def from_attestation(cls, attestation: Attestation, stream_id: UUID) -> LedgerEvent: - raise NotImplementedError("stub: replaces ledger events/builders.build_human_event") - class Checkpoint(Model): stream_id: UUID diff --git a/src/traust_core/v1/models/onboarding.py b/src/traust_core/v1/models/onboarding.py index faca097a..4d142446 100644 --- a/src/traust_core/v1/models/onboarding.py +++ b/src/traust_core/v1/models/onboarding.py @@ -4,15 +4,8 @@ from traust_core.v1.models.assessment import AssessmentKind from traust_core.v1.models.base import Model +from traust_core.v1.models.inventory import OrgName, ProductSlug, RefName, TeamName from traust_core.v1.models.operations import AssetRequest -from traust_core.v1.models.portfolio import ( - Designation, - OrgName, - Ownership, - ProductSlug, - RefName, - Segment, -) from traust_core.v1.models.values import HttpsRepoUrl @@ -22,13 +15,11 @@ class Gap(Model): ... class RepositorySpec(Model): url: HttpsRepoUrl refs: tuple[RefName, ...] = () - designation: Designation | None = None - ownership: Ownership | None = None + team: TeamName | None = None class OnboardingRequest(Model): product: ProductSlug - segment: Segment | None = None repositories: tuple[RepositorySpec, ...] = () org: OrgName | None = None baseline: tuple[AssessmentKind, ...] = () diff --git a/src/traust_core/v1/models/operations.py b/src/traust_core/v1/models/operations.py index 55d8d8dc..d481d4b1 100644 --- a/src/traust_core/v1/models/operations.py +++ b/src/traust_core/v1/models/operations.py @@ -9,8 +9,7 @@ from pydantic import Field -from traust_core.v1.models.base import Dto, Entity, Model -from traust_core.v1.models.common import Principal +from traust_core.v1.models.base import AppendOnly, Dto, Entity, Model from traust_core.v1.models.vocabulary import BudgetEnforcement AssetName = NewType("AssetName", str) @@ -72,13 +71,10 @@ class Route(Dto): reason: str -# Members land with the Operations slice (traust-engine run loop). +# Members land with the Operations slice (contracts run-order-state, to add). class OrderState(StrEnum): ... -class OrderEventKind(StrEnum): ... - - # Shapes land with their implementation slice. class OrderProgress(Model): ... @@ -113,40 +109,55 @@ class SpendRollup(Model): ... class StaleItem(Model): ... -class Order(Entity): - request: AssetRequest - state: OrderState - requested_by: Principal - depends_on_ids: tuple[UUID, ...] = () - progress: OrderProgress | None = None +class RunBatch(Entity): + """4a · a tranche: one per (kind, period), e.g. drain_tranche / 2026-W31.""" - def transition(self, event: OrderEventKind, at: datetime) -> Order: - raise NotImplementedError("stub: replaces cron/subprocess orchestration state") + kind: str + period: str -class Lease(Model): - order_id: UUID - worker: WorkerId - expires_at: datetime +class RunOrder(Entity): + """4a · one unit of work; queued, leased by a worker, finished with an outcome.""" - def expired(self, now: datetime) -> bool: - raise NotImplementedError("stub: order lease for the traust-engine run loop") + run_batch_id: UUID | None = None + repository_id: UUID | None = None + artifact_id: UUID | None = None + rule: str | None = None + event_source: str | None = None + lane: str + state: str + priority: int + reason: str | None = None + requested_by: str + lease_holder: str | None = None + lease_expires_at: datetime | None = None + started_at: datetime | None = None + finished_at: datetime | None = None + + +class SpendEvent(AppendOnly): + run_order_id: UUID | None = None + provider: str + model: str + skill: str | None = None + tokens_in: int + tokens_out: int + cost_usd: Decimal + occurred_at: datetime -class OrderEvent(Entity): - order_id: UUID - kind: OrderEventKind - detail: str | None = None +class FeedSnapshot(AppendOnly): + run_order_id: UUID | None = None + artifact_id: UUID + feed: str + retrieved_at: datetime -class SpendEvent(Entity): - order_id: UUID | None = None - provider: ProviderName - model: ModelName | None = None - skill: SkillName | None = None - tokens: int | None = None - cost_usd: Decimal - at: datetime +class AssessmentInput(AppendOnly): + """Which feed snapshot an assessment used.""" + + assessment_id: UUID + feed_snapshot_id: UUID class Budget(Model): diff --git a/src/traust_core/v1/models/portfolio.py b/src/traust_core/v1/models/portfolio.py deleted file mode 100644 index 02893f06..00000000 --- a/src/traust_core/v1/models/portfolio.py +++ /dev/null @@ -1,163 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timedelta -from enum import StrEnum -from typing import NewType -from uuid import UUID - -from traust_core.v1.models.assessment import Freshness -from traust_core.v1.models.base import Entity, Model -from traust_core.v1.models.values import GitSha, HttpsRepoUrl -from traust_core.v1.models.vocabulary import OwnershipClass, RefKind - -ProductSlug = NewType("ProductSlug", str) -RefName = NewType("RefName", str) -VersionLabel = NewType("VersionLabel", str) -TeamName = NewType("TeamName", str) -BusinessUnit = NewType("BusinessUnit", str) -OrgName = NewType("OrgName", str) -ImageRef = NewType("ImageRef", str) - - -class Designation(StrEnum): - EXTERNAL = "external" - INTERNAL_TOOLING = "internal-tooling" - - -class ExposureClass(StrEnum): - PUBLIC_EXTERNAL = "public-external" - PUBLIC_INTERNAL = "public-internal" - PRIVATE_EXTERNAL = "private-external" - PRIVATE_INTERNAL = "private-internal" - - -class RiskTier(StrEnum): - P0 = "P0" - P1 = "P1" - P2 = "P2" - P3 = "P3" - - -class Visibility(StrEnum): - PUBLIC = "public" - PRIVATE = "private" - - -class ForgeHost(StrEnum): - GITHUB = "github" - GITLAB = "gitlab" - - -# Shapes land with their implementation slice. -class Segment(Model): ... - - -class ProductLifecycle(Model): ... - - -class Contact(Model): ... - - -class TrackerTarget(Model): ... - - -class OwnershipSource(Model): ... - - -class ResourceType(Model): ... - - -class ComponentSpec(Model): ... - - -class ReleaseChange(Model): ... - - -class SyncReport(Model): ... - - -class RepositoryFilter(Model): ... - - -class ComponentFilter(Model): ... - - -class Liveness(Model): - archived: bool - pushed_at: datetime | None = None - observed_at: datetime - - -class Ownership(Model): - team: TeamName - business_unit: BusinessUnit - ownership: OwnershipClass - manager: str | None = None - tracker: TrackerTarget | None = None - source: OwnershipSource - - -class Product(Entity): - slug: ProductSlug - segment: Segment | None = None - lifecycle: ProductLifecycle | None = None - contacts: tuple[Contact, ...] = () - - -class Repository(Entity): - url: HttpsRepoUrl - visibility: Visibility | None = None - has_public_upstream: bool = False - designation: Designation | None = None - liveness: Liveness | None = None - - def exposure_class(self) -> ExposureClass: - raise NotImplementedError("stub: replaces build_rescan_worklist.exposure_class") - - def is_dormant(self, now: datetime, after: timedelta) -> bool: - raise NotImplementedError("stub: replaces check_repo_liveness.classify") - - def risk_tier(self, open_critical_high: int, now: datetime) -> RiskTier: - raise NotImplementedError("stub: replaces build_rescan_worklist.risk_tier") - - -class CodeLine(Entity): - repository_id: UUID - ref: RefName - kind: RefKind - - -class Revision(Entity): - repository_id: UUID - sha: GitSha - - -class Component(Entity): - product_id: UUID - code_line_id: UUID - sub_service: str | None = None - resource_type: ResourceType | None = None - ownership: Ownership | None = None - - -class Release(Entity): - component_id: UUID - version: VersionLabel - category: str | None = None - images: tuple[ImageRef, ...] = () - - def diff(self, previous: Release) -> ReleaseChange: - raise NotImplementedError("stub: replaces build_release_events.diff_container") - - -class CrownJewelTier(Model): ... - - -class RepoState(Model): - repository_id: UUID - exposure: ExposureClass - risk_tier: RiskTier - crown_jewel: CrownJewelTier | None = None - freshness: Freshness | None = None - open_critical_high: int - liveness: Liveness | None = None diff --git a/src/traust_core/v1/models/pqc.py b/src/traust_core/v1/models/pqc.py index 6f1c2427..8f1203b8 100644 --- a/src/traust_core/v1/models/pqc.py +++ b/src/traust_core/v1/models/pqc.py @@ -2,9 +2,9 @@ from uuid import UUID -from traust_core.v1.models.base import Entity, Model +from traust_core.v1.models.base import Model from traust_core.v1.models.code_facts import CryptoFact -from traust_core.v1.models.exposure import Package +from traust_core.v1.models.dependencies import Package from traust_core.v1.models.findings import SkillInput from traust_core.v1.models.vocabulary import ( PqcBlastRadius, @@ -32,7 +32,7 @@ class PqcBlocker(Model): vendor: str | None = None -class PqcReadiness(Entity): +class PqcReadiness(Model): assessment_id: UUID bucket: PqcReadinessBucket classification: PqcClassification diff --git a/src/traust_core/v1/models/records/__init__.py b/src/traust_core/v1/models/records/__init__.py deleted file mode 100644 index c5d39eeb..00000000 --- a/src/traust_core/v1/models/records/__init__.py +++ /dev/null @@ -1,78 +0,0 @@ -"""One row model per contracts db table (traust, traust_ops, traust_ledger). - -Records mirror the DDL column for column; domain models and services build on -them. id / created_at / updated_at are None until the database writes them. -""" - -from traust_core.v1.models.records.dependencies import ( - AdvisoryPackageRecord, - AdvisoryRecord, - DependencyRecord, - ImageRecord, - PackageRecord, - SbomRecord, -) -from traust_core.v1.models.records.findings import ( - ArtifactRecord, - AssessmentRecord, - CweRecord, - DecisionRecord, - FindingCweRecord, - FindingExternalRefRecord, - FindingLabelRecord, - FindingRecord, - LocationRecord, - OccurrenceRecord, - ReviewItemRecord, -) -from traust_core.v1.models.records.inventory import ( - ComponentRecord, - OwnershipRecord, - ProductRecord, - ReleaseRecord, - RepositoryRecord, - RepositoryRefRecord, - TeamRecord, -) -from traust_core.v1.models.records.ledger import EventRecord, LayerRecord -from traust_core.v1.models.records.ops import ( - AssessmentInputRecord, - FeedSnapshotRecord, - RunBatchRecord, - RunOrderRecord, - SpendEventRecord, -) - -__all__ = [ - "AdvisoryPackageRecord", - "AdvisoryRecord", - "ArtifactRecord", - "AssessmentInputRecord", - "AssessmentRecord", - "ComponentRecord", - "CweRecord", - "DecisionRecord", - "DependencyRecord", - "EventRecord", - "FeedSnapshotRecord", - "FindingCweRecord", - "FindingExternalRefRecord", - "FindingLabelRecord", - "FindingRecord", - "ImageRecord", - "LayerRecord", - "LocationRecord", - "OccurrenceRecord", - "OwnershipRecord", - "PackageRecord", - "ProductRecord", - "ReleaseRecord", - "RepositoryRecord", - "RepositoryRefRecord", - "ReviewItemRecord", - "RunBatchRecord", - "RunOrderRecord", - "SbomRecord", - "SpendEventRecord", - "TeamRecord", -] diff --git a/src/traust_core/v1/models/records/dependencies.py b/src/traust_core/v1/models/records/dependencies.py deleted file mode 100644 index e8483d75..00000000 --- a/src/traust_core/v1/models/records/dependencies.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Rows of the traust namespace, 2 · Dependencies - one model per contract table.""" - -from __future__ import annotations - -from datetime import datetime -from uuid import UUID - -from traust_core.v1.models.base import AppendOnly, Entity - - -class PackageRecord(Entity): - source_repository_id: UUID | None = None - sbom_id: UUID | None = None - ecosystem: str - name: str - - -class SbomRecord(AppendOnly): - digest: str - artifact_id: UUID | None = None - - -class DependencyRecord(AppendOnly): - sbom_id: UUID - package_id: UUID - version: str - direct: bool - - -class ImageRecord(AppendOnly): - digest: str - platform: str | None = None - repository_ref_id: UUID | None = None - sbom_id: UUID | None = None - - -class AdvisoryRecord(Entity): - alias_of_id: UUID | None = None - identifier: str - published_at: datetime | None = None - - -class AdvisoryPackageRecord(Entity): - advisory_id: UUID - package_id: UUID diff --git a/src/traust_core/v1/models/records/findings.py b/src/traust_core/v1/models/records/findings.py deleted file mode 100644 index 27554f4d..00000000 --- a/src/traust_core/v1/models/records/findings.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Rows of the traust namespace, 3 · Assessments & findings - one model per contract table. - -Classifying columns (kind, status, severity, validity, ...) are str here: their -values come from contracts registries, checked at intake by services. -""" - -from __future__ import annotations - -from datetime import datetime -from decimal import Decimal -from uuid import UUID - -from traust_core.v1.models.base import AppendOnly, Entity - - -class AssessmentRecord(AppendOnly): - repository_ref_id: UUID - image_id: UUID | None = None - baseline_assessment_id: UUID | None = None - run_order_id: UUID | None = None - kind: str - status: str - assessor: str - assessor_version: str - occurred_at: datetime - - -class ArtifactRecord(AppendOnly): - assessment_id: UUID | None = None - digest: str - kind: str - uri: str - - -class OccurrenceRecord(AppendOnly): - finding_id: UUID - assessment_id: UUID - severity: str - cvss_vector: str | None = None - cvss_score: Decimal | None = None - rule: str | None = None - - -class LocationRecord(AppendOnly): - occurrence_id: UUID - path: str - line_start: int | None = None - line_end: int | None = None - language: str | None = None - symbol: str | None = None - - -class FindingRecord(Entity): - repository_id: UUID - advisory_id: UUID | None = None - advisory_match: str | None = None - fingerprint_algo: str - fingerprint: str - kind: str | None = None - title: str - - -class CweRecord(Entity): - identifier: str - abstraction: str | None = None - mapping_usage: str | None = None - catalog_version: str | None = None - - -class FindingCweRecord(Entity): - finding_id: UUID - cwe_id: UUID - - -class FindingLabelRecord(Entity): - finding_id: UUID - taxonomy: str - label: str - - -class FindingExternalRefRecord(Entity): - finding_id: UUID - system: str - ref: str - - -class DecisionRecord(AppendOnly): - finding_id: UUID - assessment_id: UUID | None = None - fixed_repository_ref_id: UUID | None = None - repository_ref_id: UUID | None = None - restates_id: UUID | None = None - artifact_id: UUID | None = None - statement_key: str - validity: str | None = None - resolution: str | None = None - evidence_grade: str | None = None - reason: str | None = None - severity: str | None = None - source_channel: str | None = None - actor: str | None = None - occurred_at: datetime - - -class ReviewItemRecord(Entity): - finding_id: UUID - decision_id: UUID | None = None - resolved_by_decision_id: UUID | None = None - reason: str | None = None - status: str diff --git a/src/traust_core/v1/models/records/inventory.py b/src/traust_core/v1/models/records/inventory.py deleted file mode 100644 index 288f0109..00000000 --- a/src/traust_core/v1/models/records/inventory.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Rows of the traust namespace, 1 · Inventory - one model per contract table.""" - -from __future__ import annotations - -from datetime import datetime -from uuid import UUID - -from traust_core.v1.models.base import Entity - - -class ProductRecord(Entity): - slug: str - name: str - - -class ReleaseRecord(Entity): - product_id: UUID - version: str - - -class ComponentRecord(Entity): - release_id: UUID - repository_ref_id: UUID - - -class RepositoryRecord(Entity): - upstream_id: UUID | None = None - url: str - public: bool - - -class RepositoryRefRecord(Entity): - repository_id: UUID - name: str | None = None - kind: str | None = None - is_default: bool = False - sha: str - manifest_sbom_id: UUID | None = None - - -class TeamRecord(Entity): - name: str - - -class OwnershipRecord(Entity): - repository_id: UUID - team_id: UUID - valid_from: datetime - valid_to: datetime | None = None diff --git a/src/traust_core/v1/models/records/ledger.py b/src/traust_core/v1/models/records/ledger.py deleted file mode 100644 index 4b825218..00000000 --- a/src/traust_core/v1/models/records/ledger.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Rows of the traust_ledger namespace, 5 · Ledger - one model per contract table.""" - -from __future__ import annotations - -from uuid import UUID - -from traust_core.v1.models.base import AppendOnly, Entity - - -class LayerRecord(Entity): - repository_id: UUID - line: str | None = None - merkle_root: str | None = None - merkle_epoch: int | None = None - merkle_size: int | None = None - merkle_root_signature: str | None = None - merkle_signing_method: str - merkle_signature_format: int | None = None - - -class EventRecord(AppendOnly): - layer_id: UUID - decision_id: UUID - seq: int - event_payload: bytes diff --git a/src/traust_core/v1/models/records/ops.py b/src/traust_core/v1/models/records/ops.py deleted file mode 100644 index e6eb2fab..00000000 --- a/src/traust_core/v1/models/records/ops.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Rows of the traust_ops namespace, 4 · Operations - one model per contract table.""" - -from __future__ import annotations - -from datetime import datetime -from decimal import Decimal -from uuid import UUID - -from traust_core.v1.models.base import AppendOnly, Entity - - -class RunBatchRecord(Entity): - kind: str - period: str - - -class RunOrderRecord(Entity): - run_batch_id: UUID | None = None - repository_id: UUID | None = None - artifact_id: UUID | None = None - rule: str | None = None - event_source: str | None = None - lane: str - state: str - priority: int - reason: str | None = None - requested_by: str - lease_holder: str | None = None - lease_expires_at: datetime | None = None - started_at: datetime | None = None - finished_at: datetime | None = None - - -class SpendEventRecord(AppendOnly): - run_order_id: UUID | None = None - provider: str - model: str - skill: str | None = None - tokens_in: int - tokens_out: int - cost_usd: Decimal - occurred_at: datetime - - -class FeedSnapshotRecord(AppendOnly): - run_order_id: UUID | None = None - artifact_id: UUID - feed: str - retrieved_at: datetime - - -class AssessmentInputRecord(AppendOnly): - assessment_id: UUID - feed_snapshot_id: UUID diff --git a/src/traust_core/v1/models/remediation.py b/src/traust_core/v1/models/remediation.py index fa2085a2..68b1ffff 100644 --- a/src/traust_core/v1/models/remediation.py +++ b/src/traust_core/v1/models/remediation.py @@ -3,10 +3,10 @@ from enum import StrEnum from uuid import UUID -from traust_core.v1.models.base import Entity, Model +from traust_core.v1.models.base import Model from traust_core.v1.models.common import ArtifactRef from traust_core.v1.models.findings import SkillInput -from traust_core.v1.models.portfolio import OrgName +from traust_core.v1.models.inventory import OrgName from traust_core.v1.models.values import HttpsRepoUrl from traust_core.v1.models.vocabulary import ( CrossRepoPropagation, @@ -77,7 +77,7 @@ class Fork(Model): visibility: ForkVisibility -class WorkItem(Entity): +class WorkItem(Model): finding_ids: tuple[UUID, ...] component_id: UUID status: RemediationStatus @@ -94,16 +94,16 @@ def is_done(self) -> bool: raise NotImplementedError("stub: replaces next_pending_remediation.is_done") -class FleetFix(Entity): +class FleetFix(Model): matcher: FleetFixMatcher spec: FleetFixSpec target_ids: tuple[UUID, ...] = () approval_id: UUID | None = None -class Verification(Entity): +class Verification(Model): work_item_id: UUID - revision_id: UUID + repository_ref_id: UUID verdict: VerificationVerdict regression_ids: tuple[UUID, ...] = () artifact: ArtifactRef diff --git a/src/traust_core/v1/models/storage.py b/src/traust_core/v1/models/storage.py deleted file mode 100644 index 0fa64ced..00000000 --- a/src/traust_core/v1/models/storage.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import annotations - -import hashlib -from datetime import datetime - -from traust_core.v1.errors import ValidationError -from traust_core.v1.models.base import Model - -BINDING_DOMAIN = b"traust-binding-v1\x00" - - -class BindingContext(Model): - scope_id: str = "local" - subject_id: str | None = None - run_id: str | None = None - layer_id: str | None = None - role: str | None = None - supersedes_binding_id: str | None = None - - -class Evidence(Model): - digest: str - byte_size: int - first_ingested_at: datetime - - -class ArtifactBinding(Model): - binding_id: str - artifact_digest: str - artifact_name: str - context: BindingContext - bound_at: datetime - references: tuple[str, ...] = () - byte_size: int | None = None - - -def identifier_bytes(field: str, value: str) -> bytes: - if "\x00" in value: - raise ValidationError(f"{field}: NUL is not allowed") - try: - return value.encode("utf-8") - except UnicodeEncodeError: - raise ValidationError(f"{field}: invalid Unicode") from None - - -def binding_id(artifact_digest: str, artifact_name: str, context: BindingContext) -> str: - encoded = bytearray(BINDING_DOMAIN) - for field, value in ( - ("artifact_digest", artifact_digest), - ("artifact_name", artifact_name), - ("scope_id", context.scope_id), - ): - encoded += identifier_bytes(field, value) + b"\x00" - for field, optional in ( - ("subject_id", context.subject_id), - ("run_id", context.run_id), - ("layer_id", context.layer_id), - ): - encoded += ( - b"\x00" if optional is None else b"\x01" + identifier_bytes(field, optional) + b"\x00" - ) - if context.role is not None: - encoded += b"\x01" + identifier_bytes("role", context.role) + b"\x00" - return hashlib.sha256(encoded).hexdigest() diff --git a/src/traust_core/v1/models/validation.py b/src/traust_core/v1/models/validation.py index c4ad117d..5b631066 100644 --- a/src/traust_core/v1/models/validation.py +++ b/src/traust_core/v1/models/validation.py @@ -3,7 +3,7 @@ from datetime import datetime from uuid import UUID -from traust_core.v1.models.base import Entity, Model +from traust_core.v1.models.base import Model from traust_core.v1.models.findings import EvidenceRef, SkillInput from traust_core.v1.models.vocabulary import ( ValidationAdapterKind, @@ -81,7 +81,7 @@ def permits(self, action: ValidationAction, target: ValidationTarget) -> bool: raise NotImplementedError("stub: replaces validation/scope.py x2, browser scope.py") -class ValidationPlan(Entity): +class ValidationPlan(Model): finding_id: UUID target: ValidationTarget technique: ValidationTechnique @@ -94,7 +94,7 @@ def requires_approval(self) -> bool: raise NotImplementedError("stub: replaces validate-findings approval mode") -class ValidationAttempt(Entity): +class ValidationAttempt(Model): plan_id: UUID verdict: ValidationVerdict evidence: tuple[EvidenceRef, ...] = () diff --git a/src/traust_core/v1/providers/analysis.py b/src/traust_core/v1/providers/analysis.py index 492cf0ff..fba92eb8 100644 --- a/src/traust_core/v1/providers/analysis.py +++ b/src/traust_core/v1/providers/analysis.py @@ -2,7 +2,7 @@ from typing import Protocol -from traust_core.v1.models.exposure import Advisory, Dependency, ImpactEvidence +from traust_core.v1.models.dependencies import Advisory, Dependency, ImpactEvidence from traust_core.v1.providers.base import Readiness, Result from traust_core.v1.providers.tools import ToolRequest diff --git a/src/traust_core/v1/providers/feeds.py b/src/traust_core/v1/providers/feeds.py index 3edee9ae..e6c79ac1 100644 --- a/src/traust_core/v1/providers/feeds.py +++ b/src/traust_core/v1/providers/feeds.py @@ -4,7 +4,7 @@ from typing import Protocol, TypeVar from traust_core.v1.models.base import Dto -from traust_core.v1.models.exposure import Advisory, Package, Version +from traust_core.v1.models.dependencies import Advisory, Package, Version from traust_core.v1.providers.base import Readiness, Result Rec = TypeVar("Rec", covariant=True) diff --git a/src/traust_core/v1/providers/sources.py b/src/traust_core/v1/providers/sources.py index 8eee0a81..f56908ca 100644 --- a/src/traust_core/v1/providers/sources.py +++ b/src/traust_core/v1/providers/sources.py @@ -5,7 +5,7 @@ from traust_core.v1.models.base import Dto from traust_core.v1.models.code_facts import LanguageStats -from traust_core.v1.models.portfolio import OrgName +from traust_core.v1.models.inventory import OrgName from traust_core.v1.models.values import GitSha, HttpsRepoUrl from traust_core.v1.providers.base import Readiness, Result diff --git a/src/traust_core/v1/repositories/artifacts.py b/src/traust_core/v1/repositories/artifacts.py deleted file mode 100644 index 64a87872..00000000 --- a/src/traust_core/v1/repositories/artifacts.py +++ /dev/null @@ -1,32 +0,0 @@ -from __future__ import annotations - -from typing import Protocol -from uuid import UUID - -from traust_core.v1.models.base import Model -from traust_core.v1.models.common import ( - ArtifactName, - ArtifactRef, - Page, - PageRequest, - Provenance, -) - - -class ArtifactFilter(Model): ... - - -# NEED(storage): local mode = SQLite as the index + files for blobs; server mode = Postgres -# + object storage. Same IDs both ways: rows point at blobs by UUID, never by path. -# NEED(query-first): a document read whole stays a blob a service loads on demand; a field -# becomes a column only when a service filters, groups, sorts or joins on it (D1). -class ArtifactStore(Protocol): - def put(self, name: ArtifactName, payload: bytes) -> ArtifactRef: ... - def get(self, artifact_id: UUID) -> bytes: ... - def stat(self, artifact_id: UUID) -> ArtifactRef: ... - - -class ArtifactIndex(Protocol): - def record(self, ref: ArtifactRef, provenance: Provenance) -> None: ... - def get(self, artifact_id: UUID) -> ArtifactRef: ... - def list(self, filter: ArtifactFilter, page: PageRequest) -> Page[ArtifactRef]: ... diff --git a/src/traust_core/v1/repositories/assessment.py b/src/traust_core/v1/repositories/assessment.py deleted file mode 100644 index 487d6d67..00000000 --- a/src/traust_core/v1/repositories/assessment.py +++ /dev/null @@ -1,47 +0,0 @@ -from __future__ import annotations - -from collections.abc import Sequence -from datetime import datetime -from typing import Protocol -from uuid import UUID - -from traust_core.v1.models.assessment import ( - Assessment, - AssessmentFilter, - AssessmentKind, - Population, - Threat, - ThreatFilter, -) -from traust_core.v1.models.common import Page, PageRequest -from traust_core.v1.models.pqc import PqcReadiness -from traust_core.v1.repositories.unit_of_work import UnitOfWork - - -class AssessmentRepository(Protocol): - def get(self, assessment_id: UUID) -> Assessment: ... - def add(self, assessment: Assessment) -> Assessment: ... - def latest(self, code_line_id: UUID, kind: AssessmentKind) -> Assessment | None: ... - def list(self, filter: AssessmentFilter, page: PageRequest) -> Page[Assessment]: ... - def population(self, as_of: datetime, filter: AssessmentFilter) -> Population: ... - - -class ThreatRepository(Protocol): - def add_many(self, threats: Sequence[Threat]) -> list[Threat]: ... - def list(self, filter: ThreatFilter, page: PageRequest) -> Page[Threat]: ... - - -class PqcRepository(Protocol): - def add(self, readiness: PqcReadiness) -> PqcReadiness: ... - def update(self, readiness: PqcReadiness) -> PqcReadiness: ... - def get(self, assessment_id: UUID) -> PqcReadiness: ... - def list(self, page: PageRequest) -> Page[PqcReadiness]: ... - - -class AssessmentUnitOfWork(UnitOfWork, Protocol): - assessments: AssessmentRepository - threats: ThreatRepository - - -class PqcUnitOfWork(UnitOfWork, Protocol): - pqc: PqcRepository diff --git a/src/traust_core/v1/repositories/attestation.py b/src/traust_core/v1/repositories/attestation.py deleted file mode 100644 index 8316926d..00000000 --- a/src/traust_core/v1/repositories/attestation.py +++ /dev/null @@ -1,27 +0,0 @@ -from __future__ import annotations - -from typing import Protocol -from uuid import UUID - -from traust_core.v1.models.attestation import Attestation, ReviewItem, ReviewSubject -from traust_core.v1.models.common import Page, PageRequest -from traust_core.v1.models.findings import ReviewFilter -from traust_core.v1.repositories.unit_of_work import UnitOfWork - - -class ReviewItemRepository(Protocol): - def get(self, review_item_id: UUID) -> ReviewItem: ... - def add(self, review_item: ReviewItem) -> ReviewItem: ... - def update(self, review_item: ReviewItem) -> ReviewItem: ... - def list(self, filter: ReviewFilter, page: PageRequest) -> Page[ReviewItem]: ... - - -class AttestationRepository(Protocol): - def get(self, attestation_id: UUID) -> Attestation: ... - def add(self, attestation: Attestation) -> Attestation: ... - def list_for(self, subject: ReviewSubject) -> list[Attestation]: ... - - -class AttestationUnitOfWork(UnitOfWork, Protocol): - review_items: ReviewItemRepository - attestations: AttestationRepository diff --git a/src/traust_core/v1/repositories/code_facts.py b/src/traust_core/v1/repositories/code_facts.py deleted file mode 100644 index 54a86de7..00000000 --- a/src/traust_core/v1/repositories/code_facts.py +++ /dev/null @@ -1,22 +0,0 @@ -from __future__ import annotations - -from collections.abc import Sequence -from typing import Protocol -from uuid import UUID - -from traust_core.v1.models.code_facts import CodeFact, CodeFactKind, SymbolFact, SymbolQuery -from traust_core.v1.models.common import Location, Page, PageRequest -from traust_core.v1.repositories.unit_of_work import UnitOfWork - - -class CodeFactRepository(Protocol): - def add_many(self, facts: Sequence[CodeFact]) -> list[CodeFact]: ... - def list( - self, revision_id: UUID, kinds: Sequence[CodeFactKind], page: PageRequest - ) -> Page[CodeFact]: ... - def find_symbols(self, revision_id: UUID, query: SymbolQuery) -> list[SymbolFact]: ... - def references(self, revision_id: UUID, query: SymbolQuery) -> list[Location]: ... - - -class CodeFactsUnitOfWork(UnitOfWork, Protocol): - code_facts: CodeFactRepository diff --git a/src/traust_core/v1/repositories/compliance.py b/src/traust_core/v1/repositories/compliance.py deleted file mode 100644 index d39de12d..00000000 --- a/src/traust_core/v1/repositories/compliance.py +++ /dev/null @@ -1,50 +0,0 @@ -from __future__ import annotations - -from collections.abc import Sequence -from typing import Protocol -from uuid import UUID - -from traust_core.v1.models.common import Page, PageRequest -from traust_core.v1.models.compliance import ( - Adr, - ComplianceAssessment, - ComplianceFilter, - ComplianceScope, - DocVariance, - DocVarianceFilter, - ScopeFilter, -) -from traust_core.v1.models.vocabulary import ComplianceFramework -from traust_core.v1.repositories.unit_of_work import UnitOfWork - - -class ComplianceScopeRepository(Protocol): - def get(self, scope_id: UUID) -> ComplianceScope: ... - def add(self, scope: ComplianceScope) -> ComplianceScope: ... - def update(self, scope: ComplianceScope) -> ComplianceScope: ... - def list(self, filter: ScopeFilter, page: PageRequest) -> Page[ComplianceScope]: ... - - -class ComplianceAssessmentRepository(Protocol): - def add(self, assessment: ComplianceAssessment) -> ComplianceAssessment: ... - def latest( - self, scope_id: UUID, framework: ComplianceFramework - ) -> ComplianceAssessment | None: ... - def list(self, filter: ComplianceFilter, page: PageRequest) -> Page[ComplianceAssessment]: ... - - -class AdrRepository(Protocol): - def replace_for(self, product_id: UUID, adrs: Sequence[Adr]) -> list[Adr]: ... - def list_for(self, product_id: UUID) -> list[Adr]: ... - - -class DocVarianceRepository(Protocol): - def add_many(self, variances: Sequence[DocVariance]) -> list[DocVariance]: ... - def list(self, filter: DocVarianceFilter, page: PageRequest) -> Page[DocVariance]: ... - - -class ComplianceUnitOfWork(UnitOfWork, Protocol): - scopes: ComplianceScopeRepository - assessments: ComplianceAssessmentRepository - adrs: AdrRepository - doc_variances: DocVarianceRepository diff --git a/src/traust_core/v1/repositories/contract.py b/src/traust_core/v1/repositories/contract.py index f2edde18..407b07c4 100644 --- a/src/traust_core/v1/repositories/contract.py +++ b/src/traust_core/v1/repositories/contract.py @@ -1,13 +1,28 @@ -"""The whole contracts db as table specs: every namespace, every table, once.""" +"""The whole contracts db: every table spec, and one unit of work over all of them. + +Section protocols (InventoryUnitOfWork, FindingsUnitOfWork, ...) are what a +service declares it needs; SqlTraustUnitOfWork satisfies every one of them on +a single connection, so work that spans sections - intake writes an +assessment, occurrences, findings and decisions - commits or rolls back as +one transaction. Services that collaborate share the same instance. +""" from __future__ import annotations +from typing import Protocol + from sqlalchemy.engine import Engine from traust_core.v1.repositories import traust_ledger, traust_ops from traust_core.v1.repositories.sql import Dialect, schema_drift -from traust_core.v1.repositories.tables import TableFactory, TableSpec +from traust_core.v1.repositories.tables import SqlTablesUnitOfWork, TableFactory, TableSpec from traust_core.v1.repositories.traust import assessments, dependencies, findings, inventory +from traust_core.v1.repositories.traust.assessments import AssessmentsUnitOfWork +from traust_core.v1.repositories.traust.dependencies import DependenciesUnitOfWork +from traust_core.v1.repositories.traust.findings import FindingsUnitOfWork +from traust_core.v1.repositories.traust.inventory import InventoryUnitOfWork +from traust_core.v1.repositories.traust_ledger import LedgerUnitOfWork +from traust_core.v1.repositories.traust_ops import OpsUnitOfWork CONTRACT_TABLES: tuple[TableSpec, ...] = ( *inventory.TABLES, @@ -30,3 +45,51 @@ def contract_tables(dialect: Dialect) -> TableFactory: def contract_drift(engine: Engine) -> list[str]: """Where the database and the declared tables disagree; empty = in step.""" return schema_drift(engine, contract_tables(Dialect.of(engine)).metadata) + + +class TraustUnitOfWork( + InventoryUnitOfWork, + DependenciesUnitOfWork, + AssessmentsUnitOfWork, + FindingsUnitOfWork, + OpsUnitOfWork, + LedgerUnitOfWork, + Protocol, +): + """Every repository, one transaction.""" + + +class SqlTraustUnitOfWork(SqlTablesUnitOfWork): + def _open_repositories(self) -> None: + conn, tables = self.connection, self.tables + self.products = inventory.SqlProductRepository(conn, tables) + self.releases = inventory.SqlReleaseRepository(conn, tables) + self.components = inventory.SqlComponentRepository(conn, tables) + self.repositories = inventory.SqlRepositoryRepository(conn, tables) + self.repository_refs = inventory.SqlRepositoryRefRepository(conn, tables) + self.teams = inventory.SqlTeamRepository(conn, tables) + self.ownerships = inventory.SqlOwnershipRepository(conn, tables) + self.packages = dependencies.SqlPackageRepository(conn, tables) + self.sboms = dependencies.SqlSbomRepository(conn, tables) + self.dependencies = dependencies.SqlDependencyRepository(conn, tables) + self.images = dependencies.SqlImageRepository(conn, tables) + self.advisories = dependencies.SqlAdvisoryRepository(conn, tables) + self.advisory_packages = dependencies.SqlAdvisoryPackageRepository(conn, tables) + self.assessments = assessments.SqlAssessmentRepository(conn, tables) + self.artifacts = assessments.SqlArtifactRepository(conn, tables) + self.occurrences = assessments.SqlOccurrenceRepository(conn, tables) + self.locations = assessments.SqlLocationRepository(conn, tables) + self.findings = findings.SqlFindingRepository(conn, tables) + self.cwes = findings.SqlCweRepository(conn, tables) + self.finding_cwes = findings.SqlFindingCweRepository(conn, tables) + self.finding_labels = findings.SqlFindingLabelRepository(conn, tables) + self.finding_external_refs = findings.SqlFindingExternalRefRepository(conn, tables) + self.decisions = findings.SqlDecisionRepository(conn, tables) + self.review_items = findings.SqlReviewItemRepository(conn, tables) + self.run_batches = traust_ops.SqlRunBatchRepository(conn, tables) + self.run_orders = traust_ops.SqlRunOrderRepository(conn, tables) + self.spend_events = traust_ops.SqlSpendEventRepository(conn, tables) + self.feed_snapshots = traust_ops.SqlFeedSnapshotRepository(conn, tables) + self.assessment_inputs = traust_ops.SqlAssessmentInputRepository(conn, tables) + self.layers = traust_ledger.SqlLayerRepository(conn, tables) + self.events = traust_ledger.SqlEventRepository(conn, tables) diff --git a/src/traust_core/v1/repositories/detection_quality.py b/src/traust_core/v1/repositories/detection_quality.py deleted file mode 100644 index 33748ec6..00000000 --- a/src/traust_core/v1/repositories/detection_quality.py +++ /dev/null @@ -1,44 +0,0 @@ -from __future__ import annotations - -from collections.abc import Sequence -from typing import Protocol -from uuid import UUID - -from traust_core.v1.models.common import Page, PageRequest -from traust_core.v1.models.detection_quality import ( - BenchmarkFilter, - BenchmarkRun, - BenchmarkTarget, - CriticalMiss, - MissFilter, - RuleDraft, - RuleDraftStatus, - RulePack, -) -from traust_core.v1.repositories.unit_of_work import UnitOfWork - - -class BenchmarkRepository(Protocol): - def add_targets(self, targets: Sequence[BenchmarkTarget]) -> list[BenchmarkTarget]: ... - def targets(self, target_ids: Sequence[UUID]) -> list[BenchmarkTarget]: ... - def add_run(self, run: BenchmarkRun) -> BenchmarkRun: ... - def runs(self, filter: BenchmarkFilter) -> list[BenchmarkRun]: ... - - -class RuleRepository(Protocol): - def add_pack(self, pack: RulePack) -> RulePack: ... - def packs(self) -> list[RulePack]: ... - def add_draft(self, draft: RuleDraft) -> RuleDraft: ... - def update_draft(self, draft: RuleDraft) -> RuleDraft: ... - def drafts(self, status: RuleDraftStatus) -> list[RuleDraft]: ... - - -class MissRepository(Protocol): - def add_many(self, misses: Sequence[CriticalMiss]) -> list[CriticalMiss]: ... - def list(self, filter: MissFilter, page: PageRequest) -> Page[CriticalMiss]: ... - - -class DetectionQualityUnitOfWork(UnitOfWork, Protocol): - benchmarks: BenchmarkRepository - rules: RuleRepository - misses: MissRepository diff --git a/src/traust_core/v1/repositories/exposure.py b/src/traust_core/v1/repositories/exposure.py deleted file mode 100644 index cc12e1dd..00000000 --- a/src/traust_core/v1/repositories/exposure.py +++ /dev/null @@ -1,55 +0,0 @@ -from __future__ import annotations - -from collections.abc import Sequence -from typing import Protocol -from uuid import UUID - -from traust_core.v1.models.common import Page, PageRequest -from traust_core.v1.models.exposure import ( - Advisory, - AdvisoryIdentifier, - AdvisorySymbols, - Dependency, - FeedName, - FeedRun, - ImpactAnalysis, - ImpactFilter, - Package, - Version, -) -from traust_core.v1.repositories.unit_of_work import UnitOfWork - - -class AdvisoryRepository(Protocol): - def get(self, advisory_id: UUID) -> Advisory: ... - def find_by_identifier(self, identifier: AdvisoryIdentifier) -> Advisory | None: ... - def add(self, advisory: Advisory) -> Advisory: ... - def update(self, advisory: Advisory) -> Advisory: ... - def list_affecting(self, package: Package, version: Version) -> list[Advisory]: ... - def put_symbols(self, symbols: AdvisorySymbols) -> None: ... - def symbols(self, advisory_id: UUID) -> AdvisorySymbols | None: ... - - -class FeedRunRepository(Protocol): - def last(self, feed: FeedName) -> FeedRun | None: ... - def add(self, run: FeedRun) -> FeedRun: ... - - -class DependencyRepository(Protocol): - def replace_for( - self, revision_id: UUID, dependencies: Sequence[Dependency] - ) -> list[Dependency]: ... - def list_for(self, revision_id: UUID) -> list[Dependency]: ... - def dependents(self, package: Package) -> list[UUID]: ... - - -class ImpactRepository(Protocol): - def add(self, analysis: ImpactAnalysis) -> ImpactAnalysis: ... - def list(self, filter: ImpactFilter, page: PageRequest) -> Page[ImpactAnalysis]: ... - - -class ExposureUnitOfWork(UnitOfWork, Protocol): - advisories: AdvisoryRepository - feed_runs: FeedRunRepository - dependencies: DependencyRepository - impacts: ImpactRepository diff --git a/src/traust_core/v1/repositories/findings.py b/src/traust_core/v1/repositories/findings.py deleted file mode 100644 index 0b06f099..00000000 --- a/src/traust_core/v1/repositories/findings.py +++ /dev/null @@ -1,53 +0,0 @@ -from __future__ import annotations - -from collections.abc import Sequence -from datetime import datetime -from typing import Protocol -from uuid import UUID - -from traust_core.v1.models.common import Page, PageRequest -from traust_core.v1.models.findings import ( - Decision, - Disposition, - Finding, - FindingFilter, - FindingView, - Occurrence, -) -from traust_core.v1.models.integrity import Fingerprint -from traust_core.v1.models.vocabulary import Severity -from traust_core.v1.repositories.unit_of_work import UnitOfWork - - -class FindingRepository(Protocol): - def get(self, finding_id: UUID) -> Finding: ... - def find_by_match_key(self, repository_id: UUID, key: Fingerprint) -> Finding | None: ... - def add(self, finding: Finding) -> Finding: ... - def update(self, finding: Finding) -> Finding: ... - - -class OccurrenceRepository(Protocol): - def add(self, occurrence: Occurrence) -> Occurrence: ... - def list_for_finding(self, finding_id: UUID) -> list[Occurrence]: ... - def list_for_assessment(self, assessment_id: UUID) -> list[Occurrence]: ... - - -class DecisionRepository(Protocol): - def add(self, decision: Decision) -> Decision: ... - def list_for_finding(self, finding_id: UUID) -> list[Decision]: ... - def list_since(self, since: datetime, page: PageRequest) -> Page[Decision]: ... - - -# Severity/SLA are stored at write time (repos never run policies); rebuild_dispositions refreshes. -class DispositionView(Protocol): - def get(self, finding_id: UUID, code_line_id: UUID) -> Disposition: ... - def put(self, disposition: Disposition) -> None: ... - def list(self, filter: FindingFilter, page: PageRequest) -> Page[FindingView]: ... - def open_count(self, repository_id: UUID, severities: Sequence[Severity]) -> int: ... - - -class FindingUnitOfWork(UnitOfWork, Protocol): - findings: FindingRepository - occurrences: OccurrenceRepository - decisions: DecisionRepository - dispositions: DispositionView diff --git a/src/traust_core/v1/repositories/migrations.py b/src/traust_core/v1/repositories/migrations.py deleted file mode 100644 index 0154d6e4..00000000 --- a/src/traust_core/v1/repositories/migrations.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import annotations - -from typing import NewType - -from sqlalchemy.engine import Engine - -SchemaVersion = NewType("SchemaVersion", str) - - -# NEED(local->server): moving from local (SQLite + dir) to Postgres + object storage is one -# step: copy rows and blobs as-is. UUID keys mean nothing is re-keyed or re-linked. -class Migrations: - def __init__(self, engine: Engine) -> None: - self._engine = engine - - def current(self) -> SchemaVersion: - raise NotImplementedError("stub: replaces traust_storage_meta / schema_revision") - - def apply(self) -> SchemaVersion: - raise NotImplementedError("stub: replaces contracts migrations/*.sql runners") diff --git a/src/traust_core/v1/repositories/operations.py b/src/traust_core/v1/repositories/operations.py deleted file mode 100644 index b2dad70f..00000000 --- a/src/traust_core/v1/repositories/operations.py +++ /dev/null @@ -1,48 +0,0 @@ -from __future__ import annotations - -from datetime import datetime -from typing import Protocol -from uuid import UUID - -from traust_core.v1.models.common import Page, PageRequest, TimeWindow -from traust_core.v1.models.operations import ( - Lease, - Order, - OrderEvent, - OrderFilter, - SpendDimension, - SpendEvent, - SpendFilter, - SpendRollup, - WorkerId, -) -from traust_core.v1.repositories.unit_of_work import UnitOfWork - - -class OrderRepository(Protocol): - def get(self, order_id: UUID) -> Order: ... - def add(self, order: Order) -> Order: ... - def update(self, order: Order) -> Order: ... - def claim(self, worker: WorkerId, now: datetime) -> Lease | None: ... - def list(self, filter: OrderFilter, page: PageRequest) -> Page[Order]: ... - - -class OrderEventRepository(Protocol): - def append(self, event: OrderEvent) -> OrderEvent: ... - def list_for(self, order_id: UUID) -> list[OrderEvent]: ... - - -class SpendRepository(Protocol): - def append(self, event: SpendEvent) -> SpendEvent: ... - def list(self, filter: SpendFilter, page: PageRequest) -> Page[SpendEvent]: ... - def aggregate(self, filter: SpendFilter, group_by: SpendDimension) -> list[SpendRollup]: ... - def unattributed(self, window: TimeWindow) -> list[SpendEvent]: ... - - -class OpsUnitOfWork(UnitOfWork, Protocol): - orders: OrderRepository - order_events: OrderEventRepository - - -class SpendUnitOfWork(UnitOfWork, Protocol): - spend: SpendRepository diff --git a/src/traust_core/v1/repositories/portfolio.py b/src/traust_core/v1/repositories/portfolio.py deleted file mode 100644 index ea89b17f..00000000 --- a/src/traust_core/v1/repositories/portfolio.py +++ /dev/null @@ -1,71 +0,0 @@ -from __future__ import annotations - -from typing import Protocol -from uuid import UUID - -from traust_core.v1.models.common import Page, PageRequest -from traust_core.v1.models.portfolio import ( - CodeLine, - Component, - ComponentFilter, - Product, - ProductSlug, - RefName, - Release, - Repository, - RepositoryFilter, - Revision, -) -from traust_core.v1.models.values import GitSha, HttpsRepoUrl -from traust_core.v1.repositories.unit_of_work import UnitOfWork - - -class ProductRepository(Protocol): - def get(self, product_id: UUID) -> Product: ... - def find_by_slug(self, slug: ProductSlug) -> Product | None: ... - def add(self, product: Product) -> Product: ... - def update(self, product: Product) -> Product: ... - def list(self, page: PageRequest) -> Page[Product]: ... - - -class RepositoryRepository(Protocol): - def get(self, repository_id: UUID) -> Repository: ... - def find_by_url(self, url: HttpsRepoUrl) -> Repository | None: ... - def add(self, repository: Repository) -> Repository: ... - def update(self, repository: Repository) -> Repository: ... - def list(self, filter: RepositoryFilter, page: PageRequest) -> Page[Repository]: ... - - -class CodeLineRepository(Protocol): - def get(self, code_line_id: UUID) -> CodeLine: ... - def find_by_ref(self, repository_id: UUID, ref: RefName) -> CodeLine | None: ... - def add(self, code_line: CodeLine) -> CodeLine: ... - def list_for(self, repository_id: UUID) -> list[CodeLine]: ... - - -class RevisionRepository(Protocol): - def get(self, revision_id: UUID) -> Revision: ... - def find_by_sha(self, repository_id: UUID, sha: GitSha) -> Revision | None: ... - def add(self, revision: Revision) -> Revision: ... - - -class ComponentRepository(Protocol): - def get(self, component_id: UUID) -> Component: ... - def find(self, product_id: UUID, code_line_id: UUID) -> Component | None: ... - def add(self, component: Component) -> Component: ... - def update(self, component: Component) -> Component: ... - def list(self, filter: ComponentFilter, page: PageRequest) -> Page[Component]: ... - - -class ReleaseRepository(Protocol): - def add(self, release: Release) -> Release: ... - def list_for(self, component_id: UUID) -> list[Release]: ... - - -class PortfolioUnitOfWork(UnitOfWork, Protocol): - products: ProductRepository - repositories: RepositoryRepository - code_lines: CodeLineRepository - revisions: RevisionRepository - components: ComponentRepository - releases: ReleaseRepository diff --git a/src/traust_core/v1/repositories/remediation.py b/src/traust_core/v1/repositories/remediation.py deleted file mode 100644 index 8cec6ba6..00000000 --- a/src/traust_core/v1/repositories/remediation.py +++ /dev/null @@ -1,40 +0,0 @@ -from __future__ import annotations - -from typing import Protocol -from uuid import UUID - -from traust_core.v1.models.common import Page, PageRequest -from traust_core.v1.models.remediation import ( - FleetFix, - Verification, - VerificationFilter, - WorkItem, - WorkItemFilter, -) -from traust_core.v1.repositories.unit_of_work import UnitOfWork - - -class WorkItemRepository(Protocol): - def get(self, work_item_id: UUID) -> WorkItem: ... - def add(self, work_item: WorkItem) -> WorkItem: ... - def update(self, work_item: WorkItem) -> WorkItem: ... - def list(self, filter: WorkItemFilter, page: PageRequest) -> Page[WorkItem]: ... - def next_pending(self, filter: WorkItemFilter) -> WorkItem | None: ... - - -class FleetFixRepository(Protocol): - def get(self, fleet_fix_id: UUID) -> FleetFix: ... - def add(self, fleet_fix: FleetFix) -> FleetFix: ... - def update(self, fleet_fix: FleetFix) -> FleetFix: ... - - -class VerificationRepository(Protocol): - def add(self, verification: Verification) -> Verification: ... - def list_for(self, work_item_id: UUID) -> list[Verification]: ... - def list(self, filter: VerificationFilter, page: PageRequest) -> Page[Verification]: ... - - -class RemediationUnitOfWork(UnitOfWork, Protocol): - work_items: WorkItemRepository - fleet_fixes: FleetFixRepository - verifications: VerificationRepository diff --git a/src/traust_core/v1/repositories/storage.py b/src/traust_core/v1/repositories/storage.py deleted file mode 100644 index b1574c29..00000000 --- a/src/traust_core/v1/repositories/storage.py +++ /dev/null @@ -1,171 +0,0 @@ -from __future__ import annotations - -from collections.abc import Sequence -from datetime import datetime -from typing import Any, Protocol - -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql, sqlite - -from traust_core.v1.models.storage import ArtifactBinding, BindingContext, Evidence -from traust_core.v1.repositories.memory import InMemoryUnitOfWork -from traust_core.v1.repositories.sql import ( - SqlRepository, - SqlUnitOfWork, - UtcTimestamp, -) -from traust_core.v1.repositories.unit_of_work import UnitOfWork - -STORAGE_SCHEMA = "traust_storage" # retired by the one-owner contract; parked: storage-retire -metadata = sa.MetaData(schema=STORAGE_SCHEMA) - -artifact_evidence = sa.Table( - "artifact_evidence", - metadata, - sa.Column("digest", sa.Text, primary_key=True), - sa.Column("byte_size", sa.BigInteger, nullable=False), - sa.Column("first_ingested_at", UtcTimestamp, nullable=False), -) - -artifact_binding = sa.Table( - "artifact_binding", - metadata, - sa.Column("binding_id", sa.Text, primary_key=True), - sa.Column("artifact_digest", sa.Text, nullable=False), - sa.Column("artifact_name", sa.Text, nullable=False), - sa.Column("artifact_role", sa.Text), - sa.Column("scope_id", sa.Text, nullable=False), - sa.Column("subject_id", sa.Text), - sa.Column("run_id", sa.Text), - sa.Column("layer_id", sa.Text), - sa.Column("supersedes_binding_id", sa.Text), - sa.Column("bound_at", UtcTimestamp, nullable=False), -) - -artifact_location = sa.Table( - "artifact_location", - metadata, - sa.Column("binding_id", sa.Text, primary_key=True), - sa.Column("reference", sa.Text, primary_key=True), - sa.Column("registered_at", UtcTimestamp, nullable=False), -) - - -class EvidenceRepository(Protocol): - def find_binding(self, binding_id: str) -> ArtifactBinding | None: ... - def add(self, evidence: Evidence, binding: ArtifactBinding) -> None: ... - def add_locations(self, binding_id: str, references: Sequence[str], at: datetime) -> None: ... - - -class StorageUnitOfWork(UnitOfWork, Protocol): - evidence: EvidenceRepository - - -class SqlEvidenceRepository(SqlRepository): - def _insert_ignore(self, table: sa.Table, rows: list[dict[str, Any]]) -> None: - if not rows: - return - insert = ( - postgresql.insert if self._connection.dialect.name == "postgresql" else sqlite.insert - ) - self._execute(f"add {table.name}", insert(table).values(rows).on_conflict_do_nothing()) - - def find_binding(self, binding_id: str) -> ArtifactBinding | None: - stmt = ( - sa.select(artifact_binding, artifact_evidence.c.byte_size) - .join( - artifact_evidence, artifact_evidence.c.digest == artifact_binding.c.artifact_digest - ) - .where(artifact_binding.c.binding_id == binding_id) - ) - row = self._execute(f"get binding {binding_id}", stmt).mappings().first() - if row is None: - return None - refs = self._execute( - f"get locations {binding_id}", - sa.select(artifact_location.c.reference) - .where(artifact_location.c.binding_id == binding_id) - .order_by(artifact_location.c.registered_at, artifact_location.c.reference), - ).scalars() - return _to_binding(row, tuple(refs)) - - def add(self, evidence: Evidence, binding: ArtifactBinding) -> None: - self._insert_ignore(artifact_evidence, [evidence.model_dump()]) - ctx = binding.context - self._execute( - f"add binding {binding.binding_id}", - sa.insert(artifact_binding).values( - binding_id=binding.binding_id, - artifact_digest=binding.artifact_digest, - artifact_name=binding.artifact_name, - artifact_role=ctx.role, - scope_id=ctx.scope_id, - subject_id=ctx.subject_id, - run_id=ctx.run_id, - layer_id=ctx.layer_id, - supersedes_binding_id=ctx.supersedes_binding_id, - bound_at=binding.bound_at, - ), - ) - - def add_locations(self, binding_id: str, references: Sequence[str], at: datetime) -> None: - self._insert_ignore( - artifact_location, - [{"binding_id": binding_id, "reference": r, "registered_at": at} for r in references], - ) - - -def _to_binding(row: Any, references: tuple[str, ...]) -> ArtifactBinding: - return ArtifactBinding( - binding_id=row["binding_id"], - artifact_digest=row["artifact_digest"], - artifact_name=row["artifact_name"], - context=BindingContext( - scope_id=row["scope_id"], - subject_id=row["subject_id"], - run_id=row["run_id"], - layer_id=row["layer_id"], - role=row["artifact_role"], - supersedes_binding_id=row["supersedes_binding_id"], - ), - bound_at=row["bound_at"], - references=references, - byte_size=row["byte_size"], - ) - - -class SqlStorageUnitOfWork(SqlUnitOfWork): - evidence: SqlEvidenceRepository - - def _open_repositories(self) -> None: - self.evidence = SqlEvidenceRepository(self.connection) - - -class InMemoryEvidenceRepository: - def __init__(self) -> None: - self._evidence: dict[str, Evidence] = {} - self._bindings: dict[str, ArtifactBinding] = {} - self._locations: dict[str, dict[str, datetime]] = {} - - def find_binding(self, binding_id: str) -> ArtifactBinding | None: - binding = self._bindings.get(binding_id) - if binding is None: - return None - locations = self._locations.get(binding_id, {}) - refs = tuple(r for r, _ in sorted(locations.items(), key=lambda kv: (kv[1], kv[0]))) - size = self._evidence[binding.artifact_digest].byte_size - return binding.model_copy(update={"references": refs, "byte_size": size}) - - def add(self, evidence: Evidence, binding: ArtifactBinding) -> None: - self._evidence.setdefault(evidence.digest, evidence) - self._bindings[binding.binding_id] = binding - - def add_locations(self, binding_id: str, references: Sequence[str], at: datetime) -> None: - for ref in references: - self._locations.setdefault(binding_id, {}).setdefault(ref, at) - - -class InMemoryStorageUnitOfWork(InMemoryUnitOfWork): - def __init__(self) -> None: - super().__init__() - self.evidence = InMemoryEvidenceRepository() diff --git a/src/traust_core/v1/repositories/traust/assessments/__init__.py b/src/traust_core/v1/repositories/traust/assessments/__init__.py index 824c4e03..8c929d72 100644 --- a/src/traust_core/v1/repositories/traust/assessments/__init__.py +++ b/src/traust_core/v1/repositories/traust/assessments/__init__.py @@ -4,7 +4,6 @@ from typing import Protocol -from traust_core.v1.repositories.tables import SqlTablesUnitOfWork from traust_core.v1.repositories.traust.assessments.artifact import ( ARTIFACT, ArtifactRepository, @@ -37,15 +36,6 @@ class AssessmentsUnitOfWork(UnitOfWork, Protocol): locations: LocationRepository -class SqlAssessmentsUnitOfWork(SqlTablesUnitOfWork): - def _open_repositories(self) -> None: - conn, tables = self.connection, self.tables - self.assessments = SqlAssessmentRepository(conn, tables) - self.artifacts = SqlArtifactRepository(conn, tables) - self.occurrences = SqlOccurrenceRepository(conn, tables) - self.locations = SqlLocationRepository(conn, tables) - - __all__ = [ "TABLES", "ArtifactRepository", @@ -55,7 +45,6 @@ def _open_repositories(self) -> None: "OccurrenceRepository", "SqlArtifactRepository", "SqlAssessmentRepository", - "SqlAssessmentsUnitOfWork", "SqlLocationRepository", "SqlOccurrenceRepository", ] diff --git a/src/traust_core/v1/repositories/traust/assessments/artifact.py b/src/traust_core/v1/repositories/traust/assessments/artifact.py index b520f14f..935cec1d 100644 --- a/src/traust_core/v1/repositories/traust/assessments/artifact.py +++ b/src/traust_core/v1/repositories/traust/assessments/artifact.py @@ -3,7 +3,7 @@ from typing import Protocol from uuid import UUID -from traust_core.v1.models.records import ArtifactRecord +from traust_core.v1.models.assessment import Artifact from traust_core.v1.repositories.tables import ( TRAUST, RecordRepository, @@ -26,19 +26,19 @@ ) -class ArtifactRepository(RecordRepository[ArtifactRecord], Protocol): - def find(self, assessment_id: UUID | None, digest: str) -> ArtifactRecord | None: +class ArtifactRepository(RecordRepository[Artifact], Protocol): + def find(self, assessment_id: UUID | None, digest: str) -> Artifact | None: """By (assessment, digest); assessment None = an imported artifact.""" ... - def list_for_assessment(self, assessment_id: UUID) -> list[ArtifactRecord]: ... + def list_for_assessment(self, assessment_id: UUID) -> list[Artifact]: ... -class SqlArtifactRepository(SqlAppendOnlyRepository[ArtifactRecord]): +class SqlArtifactRepository(SqlAppendOnlyRepository[Artifact]): spec = ARTIFACT - record = ArtifactRecord + record = Artifact - def find(self, assessment_id: UUID | None, digest: str) -> ArtifactRecord | None: + def find(self, assessment_id: UUID | None, digest: str) -> Artifact | None: c = self.table.c produced = ( c.assessment_id.is_(None) @@ -47,5 +47,5 @@ def find(self, assessment_id: UUID | None, digest: str) -> ArtifactRecord | None ) return self._one(f"find {digest}", produced, c.digest == digest) - def list_for_assessment(self, assessment_id: UUID) -> list[ArtifactRecord]: + def list_for_assessment(self, assessment_id: UUID) -> list[Artifact]: return self._many(f"list for {assessment_id}", self.table.c.assessment_id == assessment_id) diff --git a/src/traust_core/v1/repositories/traust/assessments/assessment.py b/src/traust_core/v1/repositories/traust/assessments/assessment.py index dfb21381..f08dcc40 100644 --- a/src/traust_core/v1/repositories/traust/assessments/assessment.py +++ b/src/traust_core/v1/repositories/traust/assessments/assessment.py @@ -3,7 +3,7 @@ from typing import Protocol from uuid import UUID -from traust_core.v1.models.records import AssessmentRecord +from traust_core.v1.models.assessment import Assessment from traust_core.v1.repositories.tables import ( TRAUST, RecordRepository, @@ -32,17 +32,17 @@ ) -class AssessmentRepository(RecordRepository[AssessmentRecord], Protocol): - def latest(self, repository_ref_id: UUID, kind: str) -> AssessmentRecord | None: ... - def list_for_ref(self, repository_ref_id: UUID) -> list[AssessmentRecord]: ... - def list_for_run_order(self, run_order_id: UUID) -> list[AssessmentRecord]: ... +class AssessmentRepository(RecordRepository[Assessment], Protocol): + def latest(self, repository_ref_id: UUID, kind: str) -> Assessment | None: ... + def list_for_ref(self, repository_ref_id: UUID) -> list[Assessment]: ... + def list_for_run_order(self, run_order_id: UUID) -> list[Assessment]: ... -class SqlAssessmentRepository(SqlAppendOnlyRepository[AssessmentRecord]): +class SqlAssessmentRepository(SqlAppendOnlyRepository[Assessment]): spec = ASSESSMENT - record = AssessmentRecord + record = Assessment - def latest(self, repository_ref_id: UUID, kind: str) -> AssessmentRecord | None: + def latest(self, repository_ref_id: UUID, kind: str) -> Assessment | None: c = self.table.c stmt = ( self._select() @@ -53,7 +53,7 @@ def latest(self, repository_ref_id: UUID, kind: str) -> AssessmentRecord | None: row = self._execute(f"latest {kind} for {repository_ref_id}", stmt).mappings().first() return None if row is None else self._to_record(row) - def list_for_ref(self, repository_ref_id: UUID) -> list[AssessmentRecord]: + def list_for_ref(self, repository_ref_id: UUID) -> list[Assessment]: c = self.table.c return self._many( f"list for {repository_ref_id}", @@ -61,7 +61,7 @@ def list_for_ref(self, repository_ref_id: UUID) -> list[AssessmentRecord]: order_by=(c.occurred_at,), ) - def list_for_run_order(self, run_order_id: UUID) -> list[AssessmentRecord]: + def list_for_run_order(self, run_order_id: UUID) -> list[Assessment]: return self._many( f"list for order {run_order_id}", self.table.c.run_order_id == run_order_id ) diff --git a/src/traust_core/v1/repositories/traust/assessments/location.py b/src/traust_core/v1/repositories/traust/assessments/location.py index 96b01471..0d5074e5 100644 --- a/src/traust_core/v1/repositories/traust/assessments/location.py +++ b/src/traust_core/v1/repositories/traust/assessments/location.py @@ -5,7 +5,7 @@ import sqlalchemy as sa -from traust_core.v1.models.records import LocationRecord +from traust_core.v1.models.findings import Location from traust_core.v1.repositories.tables import ( TRAUST, RecordRepository, @@ -30,15 +30,15 @@ ) -class LocationRepository(RecordRepository[LocationRecord], Protocol): - def list_for_occurrence(self, occurrence_id: UUID) -> list[LocationRecord]: ... +class LocationRepository(RecordRepository[Location], Protocol): + def list_for_occurrence(self, occurrence_id: UUID) -> list[Location]: ... -class SqlLocationRepository(SqlAppendOnlyRepository[LocationRecord]): +class SqlLocationRepository(SqlAppendOnlyRepository[Location]): spec = LOCATION - record = LocationRecord + record = Location - def list_for_occurrence(self, occurrence_id: UUID) -> list[LocationRecord]: + def list_for_occurrence(self, occurrence_id: UUID) -> list[Location]: c = self.table.c return self._many( f"list for {occurrence_id}", diff --git a/src/traust_core/v1/repositories/traust/assessments/occurrence.py b/src/traust_core/v1/repositories/traust/assessments/occurrence.py index 714a52fc..c23ee8c2 100644 --- a/src/traust_core/v1/repositories/traust/assessments/occurrence.py +++ b/src/traust_core/v1/repositories/traust/assessments/occurrence.py @@ -5,7 +5,7 @@ import sqlalchemy as sa -from traust_core.v1.models.records import OccurrenceRecord +from traust_core.v1.models.findings import Occurrence from traust_core.v1.repositories.tables import ( TRAUST, RecordRepository, @@ -30,17 +30,17 @@ ) -class OccurrenceRepository(RecordRepository[OccurrenceRecord], Protocol): - def find(self, finding_id: UUID, assessment_id: UUID) -> OccurrenceRecord | None: ... - def list_for_assessment(self, assessment_id: UUID) -> list[OccurrenceRecord]: ... - def list_for_finding(self, finding_id: UUID) -> list[OccurrenceRecord]: ... +class OccurrenceRepository(RecordRepository[Occurrence], Protocol): + def find(self, finding_id: UUID, assessment_id: UUID) -> Occurrence | None: ... + def list_for_assessment(self, assessment_id: UUID) -> list[Occurrence]: ... + def list_for_finding(self, finding_id: UUID) -> list[Occurrence]: ... -class SqlOccurrenceRepository(SqlAppendOnlyRepository[OccurrenceRecord]): +class SqlOccurrenceRepository(SqlAppendOnlyRepository[Occurrence]): spec = OCCURRENCE - record = OccurrenceRecord + record = Occurrence - def find(self, finding_id: UUID, assessment_id: UUID) -> OccurrenceRecord | None: + def find(self, finding_id: UUID, assessment_id: UUID) -> Occurrence | None: c = self.table.c return self._one( f"find {finding_id}/{assessment_id}", @@ -48,8 +48,8 @@ def find(self, finding_id: UUID, assessment_id: UUID) -> OccurrenceRecord | None c.assessment_id == assessment_id, ) - def list_for_assessment(self, assessment_id: UUID) -> list[OccurrenceRecord]: + def list_for_assessment(self, assessment_id: UUID) -> list[Occurrence]: return self._many(f"list for {assessment_id}", self.table.c.assessment_id == assessment_id) - def list_for_finding(self, finding_id: UUID) -> list[OccurrenceRecord]: + def list_for_finding(self, finding_id: UUID) -> list[Occurrence]: return self._many(f"list for finding {finding_id}", self.table.c.finding_id == finding_id) diff --git a/src/traust_core/v1/repositories/traust/dependencies/__init__.py b/src/traust_core/v1/repositories/traust/dependencies/__init__.py index e067bef4..012914d7 100644 --- a/src/traust_core/v1/repositories/traust/dependencies/__init__.py +++ b/src/traust_core/v1/repositories/traust/dependencies/__init__.py @@ -4,7 +4,6 @@ from typing import Protocol -from traust_core.v1.repositories.tables import SqlTablesUnitOfWork from traust_core.v1.repositories.traust.dependencies.advisory import ( ADVISORY, AdvisoryRepository, @@ -49,17 +48,6 @@ class DependenciesUnitOfWork(UnitOfWork, Protocol): advisory_packages: AdvisoryPackageRepository -class SqlDependenciesUnitOfWork(SqlTablesUnitOfWork): - def _open_repositories(self) -> None: - conn, tables = self.connection, self.tables - self.packages = SqlPackageRepository(conn, tables) - self.sboms = SqlSbomRepository(conn, tables) - self.dependencies = SqlDependencyRepository(conn, tables) - self.images = SqlImageRepository(conn, tables) - self.advisories = SqlAdvisoryRepository(conn, tables) - self.advisory_packages = SqlAdvisoryPackageRepository(conn, tables) - - __all__ = [ "TABLES", "AdvisoryPackageRepository", @@ -71,7 +59,6 @@ def _open_repositories(self) -> None: "SbomRepository", "SqlAdvisoryPackageRepository", "SqlAdvisoryRepository", - "SqlDependenciesUnitOfWork", "SqlDependencyRepository", "SqlImageRepository", "SqlPackageRepository", diff --git a/src/traust_core/v1/repositories/traust/dependencies/advisory.py b/src/traust_core/v1/repositories/traust/dependencies/advisory.py index 60800366..f98c9892 100644 --- a/src/traust_core/v1/repositories/traust/dependencies/advisory.py +++ b/src/traust_core/v1/repositories/traust/dependencies/advisory.py @@ -3,7 +3,7 @@ from typing import Protocol from uuid import UUID -from traust_core.v1.models.records import AdvisoryRecord +from traust_core.v1.models.dependencies import Advisory from traust_core.v1.repositories.tables import ( TRAUST, MutableRecordRepository, @@ -25,17 +25,17 @@ ) -class AdvisoryRepository(MutableRecordRepository[AdvisoryRecord], Protocol): - def find_by_identifier(self, identifier: str) -> AdvisoryRecord | None: ... - def list_aliases(self, advisory_id: UUID) -> list[AdvisoryRecord]: ... +class AdvisoryRepository(MutableRecordRepository[Advisory], Protocol): + def find_by_identifier(self, identifier: str) -> Advisory | None: ... + def list_aliases(self, advisory_id: UUID) -> list[Advisory]: ... -class SqlAdvisoryRepository(SqlMutableRepository[AdvisoryRecord]): +class SqlAdvisoryRepository(SqlMutableRepository[Advisory]): spec = ADVISORY - record = AdvisoryRecord + record = Advisory - def find_by_identifier(self, identifier: str) -> AdvisoryRecord | None: + def find_by_identifier(self, identifier: str) -> Advisory | None: return self._one(f"find {identifier}", self.table.c.identifier == identifier) - def list_aliases(self, advisory_id: UUID) -> list[AdvisoryRecord]: + def list_aliases(self, advisory_id: UUID) -> list[Advisory]: return self._many(f"aliases of {advisory_id}", self.table.c.alias_of_id == advisory_id) diff --git a/src/traust_core/v1/repositories/traust/dependencies/advisory_package.py b/src/traust_core/v1/repositories/traust/dependencies/advisory_package.py index faf68042..ea84fecc 100644 --- a/src/traust_core/v1/repositories/traust/dependencies/advisory_package.py +++ b/src/traust_core/v1/repositories/traust/dependencies/advisory_package.py @@ -3,7 +3,7 @@ from typing import Protocol from uuid import UUID -from traust_core.v1.models.records import AdvisoryPackageRecord +from traust_core.v1.models.dependencies import AdvisoryPackage from traust_core.v1.repositories.tables import ( TRAUST, MutableRecordRepository, @@ -17,17 +17,17 @@ ) -class AdvisoryPackageRepository(MutableRecordRepository[AdvisoryPackageRecord], Protocol): - def find(self, advisory_id: UUID, package_id: UUID) -> AdvisoryPackageRecord | None: ... - def list_for_advisory(self, advisory_id: UUID) -> list[AdvisoryPackageRecord]: ... - def list_for_package(self, package_id: UUID) -> list[AdvisoryPackageRecord]: ... +class AdvisoryPackageRepository(MutableRecordRepository[AdvisoryPackage], Protocol): + def find(self, advisory_id: UUID, package_id: UUID) -> AdvisoryPackage | None: ... + def list_for_advisory(self, advisory_id: UUID) -> list[AdvisoryPackage]: ... + def list_for_package(self, package_id: UUID) -> list[AdvisoryPackage]: ... -class SqlAdvisoryPackageRepository(SqlMutableRepository[AdvisoryPackageRecord]): +class SqlAdvisoryPackageRepository(SqlMutableRepository[AdvisoryPackage]): spec = ADVISORY_PACKAGE - record = AdvisoryPackageRecord + record = AdvisoryPackage - def find(self, advisory_id: UUID, package_id: UUID) -> AdvisoryPackageRecord | None: + def find(self, advisory_id: UUID, package_id: UUID) -> AdvisoryPackage | None: c = self.table.c return self._one( f"find {advisory_id}/{package_id}", @@ -35,8 +35,8 @@ def find(self, advisory_id: UUID, package_id: UUID) -> AdvisoryPackageRecord | N c.package_id == package_id, ) - def list_for_advisory(self, advisory_id: UUID) -> list[AdvisoryPackageRecord]: + def list_for_advisory(self, advisory_id: UUID) -> list[AdvisoryPackage]: return self._many(f"list for {advisory_id}", self.table.c.advisory_id == advisory_id) - def list_for_package(self, package_id: UUID) -> list[AdvisoryPackageRecord]: + def list_for_package(self, package_id: UUID) -> list[AdvisoryPackage]: return self._many(f"list for package {package_id}", self.table.c.package_id == package_id) diff --git a/src/traust_core/v1/repositories/traust/dependencies/dependency.py b/src/traust_core/v1/repositories/traust/dependencies/dependency.py index 719a6bc2..ca1fd41f 100644 --- a/src/traust_core/v1/repositories/traust/dependencies/dependency.py +++ b/src/traust_core/v1/repositories/traust/dependencies/dependency.py @@ -5,7 +5,7 @@ import sqlalchemy as sa -from traust_core.v1.models.records import DependencyRecord +from traust_core.v1.models.dependencies import Dependency from traust_core.v1.repositories.tables import ( TRAUST, RecordRepository, @@ -28,18 +28,18 @@ ) -class DependencyRepository(RecordRepository[DependencyRecord], Protocol): - def list_for_sbom(self, sbom_id: UUID) -> list[DependencyRecord]: ... - def list_for_package(self, package_id: UUID) -> list[DependencyRecord]: ... +class DependencyRepository(RecordRepository[Dependency], Protocol): + def list_for_sbom(self, sbom_id: UUID) -> list[Dependency]: ... + def list_for_package(self, package_id: UUID) -> list[Dependency]: ... -class SqlDependencyRepository(SqlAppendOnlyRepository[DependencyRecord]): +class SqlDependencyRepository(SqlAppendOnlyRepository[Dependency]): spec = DEPENDENCY - record = DependencyRecord + record = Dependency - def list_for_sbom(self, sbom_id: UUID) -> list[DependencyRecord]: + def list_for_sbom(self, sbom_id: UUID) -> list[Dependency]: return self._many(f"list for sbom {sbom_id}", self.table.c.sbom_id == sbom_id) - def list_for_package(self, package_id: UUID) -> list[DependencyRecord]: + def list_for_package(self, package_id: UUID) -> list[Dependency]: """SBOMs that include the package (advisory sweep, usage query).""" return self._many(f"list for package {package_id}", self.table.c.package_id == package_id) diff --git a/src/traust_core/v1/repositories/traust/dependencies/image.py b/src/traust_core/v1/repositories/traust/dependencies/image.py index 9dd212d0..2ae853bb 100644 --- a/src/traust_core/v1/repositories/traust/dependencies/image.py +++ b/src/traust_core/v1/repositories/traust/dependencies/image.py @@ -3,7 +3,7 @@ from typing import Protocol from uuid import UUID -from traust_core.v1.models.records import ImageRecord +from traust_core.v1.models.dependencies import Image from traust_core.v1.repositories.tables import ( TRAUST, RecordRepository, @@ -26,25 +26,25 @@ ) -class ImageRepository(RecordRepository[ImageRecord], Protocol): - def find_by_digest(self, digest: str) -> ImageRecord | None: ... - def list_built_from(self, repository_ref_id: UUID) -> list[ImageRecord]: ... - def list_with_sbom(self, sbom_id: UUID) -> list[ImageRecord]: ... +class ImageRepository(RecordRepository[Image], Protocol): + def find_by_digest(self, digest: str) -> Image | None: ... + def list_built_from(self, repository_ref_id: UUID) -> list[Image]: ... + def list_with_sbom(self, sbom_id: UUID) -> list[Image]: ... -class SqlImageRepository(SqlAppendOnlyRepository[ImageRecord]): +class SqlImageRepository(SqlAppendOnlyRepository[Image]): spec = IMAGE - record = ImageRecord + record = Image - def find_by_digest(self, digest: str) -> ImageRecord | None: + def find_by_digest(self, digest: str) -> Image | None: return self._one(f"find {digest}", self.table.c.digest == digest) - def list_built_from(self, repository_ref_id: UUID) -> list[ImageRecord]: + def list_built_from(self, repository_ref_id: UUID) -> list[Image]: """Images whose declared provenance names this commit.""" return self._many( f"list built from {repository_ref_id}", self.table.c.repository_ref_id == repository_ref_id, ) - def list_with_sbom(self, sbom_id: UUID) -> list[ImageRecord]: + def list_with_sbom(self, sbom_id: UUID) -> list[Image]: return self._many(f"list with sbom {sbom_id}", self.table.c.sbom_id == sbom_id) diff --git a/src/traust_core/v1/repositories/traust/dependencies/package.py b/src/traust_core/v1/repositories/traust/dependencies/package.py index 67114684..ce68ac96 100644 --- a/src/traust_core/v1/repositories/traust/dependencies/package.py +++ b/src/traust_core/v1/repositories/traust/dependencies/package.py @@ -3,7 +3,7 @@ from typing import Protocol from uuid import UUID -from traust_core.v1.models.records import PackageRecord +from traust_core.v1.models.dependencies import Package from traust_core.v1.repositories.tables import ( TRAUST, MutableRecordRepository, @@ -25,26 +25,26 @@ ) -class PackageRepository(MutableRecordRepository[PackageRecord], Protocol): - def find(self, ecosystem: str, name: str) -> PackageRecord | None: ... - def list_from_repository(self, repository_id: UUID) -> list[PackageRecord]: ... - def list_with_sbom(self, sbom_id: UUID) -> list[PackageRecord]: ... +class PackageRepository(MutableRecordRepository[Package], Protocol): + def find(self, ecosystem: str, name: str) -> Package | None: ... + def list_from_repository(self, repository_id: UUID) -> list[Package]: ... + def list_with_sbom(self, sbom_id: UUID) -> list[Package]: ... -class SqlPackageRepository(SqlMutableRepository[PackageRecord]): +class SqlPackageRepository(SqlMutableRepository[Package]): spec = PACKAGE - record = PackageRecord + record = Package - def find(self, ecosystem: str, name: str) -> PackageRecord | None: + def find(self, ecosystem: str, name: str) -> Package | None: c = self.table.c return self._one(f"find {ecosystem}/{name}", c.ecosystem == ecosystem, c.name == name) - def list_from_repository(self, repository_id: UUID) -> list[PackageRecord]: + def list_from_repository(self, repository_id: UUID) -> list[Package]: """Internal libraries the repository produces.""" return self._many( f"list from {repository_id}", self.table.c.source_repository_id == repository_id ) - def list_with_sbom(self, sbom_id: UUID) -> list[PackageRecord]: + def list_with_sbom(self, sbom_id: UUID) -> list[Package]: """Image packages (base images) carrying this SBOM.""" return self._many(f"list with sbom {sbom_id}", self.table.c.sbom_id == sbom_id) diff --git a/src/traust_core/v1/repositories/traust/dependencies/sbom.py b/src/traust_core/v1/repositories/traust/dependencies/sbom.py index 45ff7b29..eaf4453d 100644 --- a/src/traust_core/v1/repositories/traust/dependencies/sbom.py +++ b/src/traust_core/v1/repositories/traust/dependencies/sbom.py @@ -2,7 +2,7 @@ from typing import Protocol -from traust_core.v1.models.records import SbomRecord +from traust_core.v1.models.dependencies import Sbom from traust_core.v1.repositories.tables import ( TRAUST, RecordRepository, @@ -20,13 +20,13 @@ ) -class SbomRepository(RecordRepository[SbomRecord], Protocol): - def find_by_digest(self, digest: str) -> SbomRecord | None: ... +class SbomRepository(RecordRepository[Sbom], Protocol): + def find_by_digest(self, digest: str) -> Sbom | None: ... -class SqlSbomRepository(SqlAppendOnlyRepository[SbomRecord]): +class SqlSbomRepository(SqlAppendOnlyRepository[Sbom]): spec = SBOM - record = SbomRecord + record = Sbom - def find_by_digest(self, digest: str) -> SbomRecord | None: + def find_by_digest(self, digest: str) -> Sbom | None: return self._one(f"find {digest}", self.table.c.digest == digest) diff --git a/src/traust_core/v1/repositories/traust/findings/__init__.py b/src/traust_core/v1/repositories/traust/findings/__init__.py index 1e8cf5c6..72a63ebc 100644 --- a/src/traust_core/v1/repositories/traust/findings/__init__.py +++ b/src/traust_core/v1/repositories/traust/findings/__init__.py @@ -4,7 +4,6 @@ from typing import Protocol -from traust_core.v1.repositories.tables import SqlTablesUnitOfWork from traust_core.v1.repositories.traust.findings.cwe import CWE, CweRepository, SqlCweRepository from traust_core.v1.repositories.traust.findings.decision import ( DECISION, @@ -59,18 +58,6 @@ class FindingsUnitOfWork(UnitOfWork, Protocol): review_items: ReviewItemRepository -class SqlFindingsUnitOfWork(SqlTablesUnitOfWork): - def _open_repositories(self) -> None: - conn, tables = self.connection, self.tables - self.findings = SqlFindingRepository(conn, tables) - self.cwes = SqlCweRepository(conn, tables) - self.finding_cwes = SqlFindingCweRepository(conn, tables) - self.finding_labels = SqlFindingLabelRepository(conn, tables) - self.finding_external_refs = SqlFindingExternalRefRepository(conn, tables) - self.decisions = SqlDecisionRepository(conn, tables) - self.review_items = SqlReviewItemRepository(conn, tables) - - __all__ = [ "TABLES", "CweRepository", @@ -87,6 +74,5 @@ def _open_repositories(self) -> None: "SqlFindingExternalRefRepository", "SqlFindingLabelRepository", "SqlFindingRepository", - "SqlFindingsUnitOfWork", "SqlReviewItemRepository", ] diff --git a/src/traust_core/v1/repositories/traust/findings/cwe.py b/src/traust_core/v1/repositories/traust/findings/cwe.py index 5f288a68..6cc93ce3 100644 --- a/src/traust_core/v1/repositories/traust/findings/cwe.py +++ b/src/traust_core/v1/repositories/traust/findings/cwe.py @@ -2,7 +2,7 @@ from typing import Protocol -from traust_core.v1.models.records import CweRecord +from traust_core.v1.models.findings import Cwe from traust_core.v1.repositories.tables import ( TRAUST, MutableRecordRepository, @@ -23,13 +23,13 @@ ) -class CweRepository(MutableRecordRepository[CweRecord], Protocol): - def find_by_identifier(self, identifier: str) -> CweRecord | None: ... +class CweRepository(MutableRecordRepository[Cwe], Protocol): + def find_by_identifier(self, identifier: str) -> Cwe | None: ... -class SqlCweRepository(SqlMutableRepository[CweRecord]): +class SqlCweRepository(SqlMutableRepository[Cwe]): spec = CWE - record = CweRecord + record = Cwe - def find_by_identifier(self, identifier: str) -> CweRecord | None: + def find_by_identifier(self, identifier: str) -> Cwe | None: return self._one(f"find {identifier}", self.table.c.identifier == identifier) diff --git a/src/traust_core/v1/repositories/traust/findings/decision.py b/src/traust_core/v1/repositories/traust/findings/decision.py index 29dc5dcd..11752dde 100644 --- a/src/traust_core/v1/repositories/traust/findings/decision.py +++ b/src/traust_core/v1/repositories/traust/findings/decision.py @@ -4,7 +4,7 @@ from typing import Protocol from uuid import UUID -from traust_core.v1.models.records import DecisionRecord +from traust_core.v1.models.findings import Decision from traust_core.v1.repositories.tables import ( TRAUST, RecordRepository, @@ -39,32 +39,32 @@ ) -class DecisionRepository(RecordRepository[DecisionRecord], Protocol): - def find_by_statement(self, finding_id: UUID, statement_key: str) -> DecisionRecord | None: +class DecisionRepository(RecordRepository[Decision], Protocol): + def find_by_statement(self, finding_id: UUID, statement_key: str) -> Decision | None: """The idempotency key: a retried or re-spelled statement is the same decision.""" ... - def list_for_finding(self, finding_id: UUID) -> list[DecisionRecord]: ... - def list_since(self, since: datetime, limit: int) -> list[DecisionRecord]: ... + def list_for_finding(self, finding_id: UUID) -> list[Decision]: ... + def list_since(self, since: datetime, limit: int) -> list[Decision]: ... -class SqlDecisionRepository(SqlAppendOnlyRepository[DecisionRecord]): +class SqlDecisionRepository(SqlAppendOnlyRepository[Decision]): spec = DECISION - record = DecisionRecord + record = Decision - def find_by_statement(self, finding_id: UUID, statement_key: str) -> DecisionRecord | None: + def find_by_statement(self, finding_id: UUID, statement_key: str) -> Decision | None: c = self.table.c return self._one( f"find {statement_key}", c.finding_id == finding_id, c.statement_key == statement_key ) - def list_for_finding(self, finding_id: UUID) -> list[DecisionRecord]: + def list_for_finding(self, finding_id: UUID) -> list[Decision]: c = self.table.c return self._many( f"list for {finding_id}", c.finding_id == finding_id, order_by=(c.occurred_at,) ) - def list_since(self, since: datetime, limit: int) -> list[DecisionRecord]: + def list_since(self, since: datetime, limit: int) -> list[Decision]: c = self.table.c stmt = ( self._select() diff --git a/src/traust_core/v1/repositories/traust/findings/finding.py b/src/traust_core/v1/repositories/traust/findings/finding.py index ad695c0e..409ccec4 100644 --- a/src/traust_core/v1/repositories/traust/findings/finding.py +++ b/src/traust_core/v1/repositories/traust/findings/finding.py @@ -3,7 +3,7 @@ from typing import Protocol from uuid import UUID -from traust_core.v1.models.records import FindingRecord +from traust_core.v1.models.findings import Finding from traust_core.v1.repositories.tables import ( TRAUST, MutableRecordRepository, @@ -28,22 +28,22 @@ ) -class FindingRepository(MutableRecordRepository[FindingRecord], Protocol): +class FindingRepository(MutableRecordRepository[Finding], Protocol): def find_by_fingerprint( self, repository_id: UUID, fingerprint_algo: str, fingerprint: str - ) -> FindingRecord | None: ... + ) -> Finding | None: ... - def list_for_repository(self, repository_id: UUID) -> list[FindingRecord]: ... - def list_for_advisory(self, advisory_id: UUID) -> list[FindingRecord]: ... + def list_for_repository(self, repository_id: UUID) -> list[Finding]: ... + def list_for_advisory(self, advisory_id: UUID) -> list[Finding]: ... -class SqlFindingRepository(SqlMutableRepository[FindingRecord]): +class SqlFindingRepository(SqlMutableRepository[Finding]): spec = FINDING - record = FindingRecord + record = Finding def find_by_fingerprint( self, repository_id: UUID, fingerprint_algo: str, fingerprint: str - ) -> FindingRecord | None: + ) -> Finding | None: c = self.table.c return self._one( f"find {fingerprint_algo}:{fingerprint}", @@ -52,10 +52,10 @@ def find_by_fingerprint( c.fingerprint == fingerprint, ) - def list_for_repository(self, repository_id: UUID) -> list[FindingRecord]: + def list_for_repository(self, repository_id: UUID) -> list[Finding]: return self._many(f"list for {repository_id}", self.table.c.repository_id == repository_id) - def list_for_advisory(self, advisory_id: UUID) -> list[FindingRecord]: + def list_for_advisory(self, advisory_id: UUID) -> list[Finding]: return self._many( f"list for advisory {advisory_id}", self.table.c.advisory_id == advisory_id ) diff --git a/src/traust_core/v1/repositories/traust/findings/finding_cwe.py b/src/traust_core/v1/repositories/traust/findings/finding_cwe.py index 0de2dc87..53c04c84 100644 --- a/src/traust_core/v1/repositories/traust/findings/finding_cwe.py +++ b/src/traust_core/v1/repositories/traust/findings/finding_cwe.py @@ -3,7 +3,7 @@ from typing import Protocol from uuid import UUID -from traust_core.v1.models.records import FindingCweRecord +from traust_core.v1.models.findings import FindingCwe from traust_core.v1.repositories.tables import ( TRAUST, MutableRecordRepository, @@ -17,17 +17,17 @@ ) -class FindingCweRepository(MutableRecordRepository[FindingCweRecord], Protocol): - def list_for_finding(self, finding_id: UUID) -> list[FindingCweRecord]: ... - def list_for_cwe(self, cwe_id: UUID) -> list[FindingCweRecord]: ... +class FindingCweRepository(MutableRecordRepository[FindingCwe], Protocol): + def list_for_finding(self, finding_id: UUID) -> list[FindingCwe]: ... + def list_for_cwe(self, cwe_id: UUID) -> list[FindingCwe]: ... -class SqlFindingCweRepository(SqlMutableRepository[FindingCweRecord]): +class SqlFindingCweRepository(SqlMutableRepository[FindingCwe]): spec = FINDING_CWE - record = FindingCweRecord + record = FindingCwe - def list_for_finding(self, finding_id: UUID) -> list[FindingCweRecord]: + def list_for_finding(self, finding_id: UUID) -> list[FindingCwe]: return self._many(f"list for {finding_id}", self.table.c.finding_id == finding_id) - def list_for_cwe(self, cwe_id: UUID) -> list[FindingCweRecord]: + def list_for_cwe(self, cwe_id: UUID) -> list[FindingCwe]: return self._many(f"list for cwe {cwe_id}", self.table.c.cwe_id == cwe_id) diff --git a/src/traust_core/v1/repositories/traust/findings/finding_external_ref.py b/src/traust_core/v1/repositories/traust/findings/finding_external_ref.py index fcb82c3d..7c186412 100644 --- a/src/traust_core/v1/repositories/traust/findings/finding_external_ref.py +++ b/src/traust_core/v1/repositories/traust/findings/finding_external_ref.py @@ -3,7 +3,7 @@ from typing import Protocol from uuid import UUID -from traust_core.v1.models.records import FindingExternalRefRecord +from traust_core.v1.models.findings import FindingExternalRef from traust_core.v1.repositories.tables import ( TRAUST, MutableRecordRepository, @@ -20,18 +20,18 @@ ) -class FindingExternalRefRepository(MutableRecordRepository[FindingExternalRefRecord], Protocol): - def list_for_finding(self, finding_id: UUID) -> list[FindingExternalRefRecord]: ... - def list_for_ref(self, system: str, ref: str) -> list[FindingExternalRefRecord]: ... +class FindingExternalRefRepository(MutableRecordRepository[FindingExternalRef], Protocol): + def list_for_finding(self, finding_id: UUID) -> list[FindingExternalRef]: ... + def list_for_ref(self, system: str, ref: str) -> list[FindingExternalRef]: ... -class SqlFindingExternalRefRepository(SqlMutableRepository[FindingExternalRefRecord]): +class SqlFindingExternalRefRepository(SqlMutableRepository[FindingExternalRef]): spec = FINDING_EXTERNAL_REF - record = FindingExternalRefRecord + record = FindingExternalRef - def list_for_finding(self, finding_id: UUID) -> list[FindingExternalRefRecord]: + def list_for_finding(self, finding_id: UUID) -> list[FindingExternalRef]: return self._many(f"list for {finding_id}", self.table.c.finding_id == finding_id) - def list_for_ref(self, system: str, ref: str) -> list[FindingExternalRefRecord]: + def list_for_ref(self, system: str, ref: str) -> list[FindingExternalRef]: c = self.table.c return self._many(f"list {system}:{ref}", c.system == system, c.ref == ref) diff --git a/src/traust_core/v1/repositories/traust/findings/finding_label.py b/src/traust_core/v1/repositories/traust/findings/finding_label.py index d9e62d82..b26a6603 100644 --- a/src/traust_core/v1/repositories/traust/findings/finding_label.py +++ b/src/traust_core/v1/repositories/traust/findings/finding_label.py @@ -3,7 +3,7 @@ from typing import Protocol from uuid import UUID -from traust_core.v1.models.records import FindingLabelRecord +from traust_core.v1.models.findings import FindingLabel from traust_core.v1.repositories.tables import ( TRAUST, MutableRecordRepository, @@ -20,18 +20,18 @@ ) -class FindingLabelRepository(MutableRecordRepository[FindingLabelRecord], Protocol): - def list_for_finding(self, finding_id: UUID) -> list[FindingLabelRecord]: ... - def list_with(self, taxonomy: str, label: str) -> list[FindingLabelRecord]: ... +class FindingLabelRepository(MutableRecordRepository[FindingLabel], Protocol): + def list_for_finding(self, finding_id: UUID) -> list[FindingLabel]: ... + def list_with(self, taxonomy: str, label: str) -> list[FindingLabel]: ... -class SqlFindingLabelRepository(SqlMutableRepository[FindingLabelRecord]): +class SqlFindingLabelRepository(SqlMutableRepository[FindingLabel]): spec = FINDING_LABEL - record = FindingLabelRecord + record = FindingLabel - def list_for_finding(self, finding_id: UUID) -> list[FindingLabelRecord]: + def list_for_finding(self, finding_id: UUID) -> list[FindingLabel]: return self._many(f"list for {finding_id}", self.table.c.finding_id == finding_id) - def list_with(self, taxonomy: str, label: str) -> list[FindingLabelRecord]: + def list_with(self, taxonomy: str, label: str) -> list[FindingLabel]: c = self.table.c return self._many(f"list {taxonomy}:{label}", c.taxonomy == taxonomy, c.label == label) diff --git a/src/traust_core/v1/repositories/traust/findings/review_item.py b/src/traust_core/v1/repositories/traust/findings/review_item.py index b9fae938..dda16e50 100644 --- a/src/traust_core/v1/repositories/traust/findings/review_item.py +++ b/src/traust_core/v1/repositories/traust/findings/review_item.py @@ -3,7 +3,7 @@ from typing import Protocol from uuid import UUID -from traust_core.v1.models.records import ReviewItemRecord +from traust_core.v1.models.findings import ReviewItem from traust_core.v1.repositories.tables import ( TRAUST, MutableRecordRepository, @@ -28,19 +28,19 @@ PENDING = "pending" # classifier plan: ReviewItem.status; the partial index's predicate -class ReviewItemRepository(MutableRecordRepository[ReviewItemRecord], Protocol): - def list_for_finding(self, finding_id: UUID) -> list[ReviewItemRecord]: ... - def pending(self, limit: int) -> list[ReviewItemRecord]: ... +class ReviewItemRepository(MutableRecordRepository[ReviewItem], Protocol): + def list_for_finding(self, finding_id: UUID) -> list[ReviewItem]: ... + def pending(self, limit: int) -> list[ReviewItem]: ... -class SqlReviewItemRepository(SqlMutableRepository[ReviewItemRecord]): +class SqlReviewItemRepository(SqlMutableRepository[ReviewItem]): spec = REVIEW_ITEM - record = ReviewItemRecord + record = ReviewItem - def list_for_finding(self, finding_id: UUID) -> list[ReviewItemRecord]: + def list_for_finding(self, finding_id: UUID) -> list[ReviewItem]: return self._many(f"list for {finding_id}", self.table.c.finding_id == finding_id) - def pending(self, limit: int) -> list[ReviewItemRecord]: + def pending(self, limit: int) -> list[ReviewItem]: c = self.table.c stmt = self._select().where(c.status == PENDING).order_by(c.created_at).limit(limit) return [self._to_record(r) for r in self._execute("pending", stmt).mappings()] diff --git a/src/traust_core/v1/repositories/traust/inventory/__init__.py b/src/traust_core/v1/repositories/traust/inventory/__init__.py index 604ae590..1a2f8c51 100644 --- a/src/traust_core/v1/repositories/traust/inventory/__init__.py +++ b/src/traust_core/v1/repositories/traust/inventory/__init__.py @@ -4,7 +4,6 @@ from typing import Protocol -from traust_core.v1.repositories.tables import SqlTablesUnitOfWork from traust_core.v1.repositories.traust.inventory.component import ( COMPONENT, ComponentRepository, @@ -55,18 +54,6 @@ class InventoryUnitOfWork(UnitOfWork, Protocol): ownerships: OwnershipRepository -class SqlInventoryUnitOfWork(SqlTablesUnitOfWork): - def _open_repositories(self) -> None: - conn, tables = self.connection, self.tables - self.products = SqlProductRepository(conn, tables) - self.releases = SqlReleaseRepository(conn, tables) - self.components = SqlComponentRepository(conn, tables) - self.repositories = SqlRepositoryRepository(conn, tables) - self.repository_refs = SqlRepositoryRefRepository(conn, tables) - self.teams = SqlTeamRepository(conn, tables) - self.ownerships = SqlOwnershipRepository(conn, tables) - - __all__ = [ "TABLES", "ComponentRepository", @@ -77,7 +64,6 @@ def _open_repositories(self) -> None: "RepositoryRefRepository", "RepositoryRepository", "SqlComponentRepository", - "SqlInventoryUnitOfWork", "SqlOwnershipRepository", "SqlProductRepository", "SqlReleaseRepository", diff --git a/src/traust_core/v1/repositories/traust/inventory/component.py b/src/traust_core/v1/repositories/traust/inventory/component.py index 232c75f7..7fbd4056 100644 --- a/src/traust_core/v1/repositories/traust/inventory/component.py +++ b/src/traust_core/v1/repositories/traust/inventory/component.py @@ -3,7 +3,7 @@ from typing import Protocol from uuid import UUID -from traust_core.v1.models.records import ComponentRecord +from traust_core.v1.models.inventory import Component from traust_core.v1.repositories.tables import ( TRAUST, MutableRecordRepository, @@ -17,17 +17,17 @@ ) -class ComponentRepository(MutableRecordRepository[ComponentRecord], Protocol): - def find(self, release_id: UUID, repository_ref_id: UUID) -> ComponentRecord | None: ... - def list_for_release(self, release_id: UUID) -> list[ComponentRecord]: ... - def list_for_ref(self, repository_ref_id: UUID) -> list[ComponentRecord]: ... +class ComponentRepository(MutableRecordRepository[Component], Protocol): + def find(self, release_id: UUID, repository_ref_id: UUID) -> Component | None: ... + def list_for_release(self, release_id: UUID) -> list[Component]: ... + def list_for_ref(self, repository_ref_id: UUID) -> list[Component]: ... -class SqlComponentRepository(SqlMutableRepository[ComponentRecord]): +class SqlComponentRepository(SqlMutableRepository[Component]): spec = COMPONENT - record = ComponentRecord + record = Component - def find(self, release_id: UUID, repository_ref_id: UUID) -> ComponentRecord | None: + def find(self, release_id: UUID, repository_ref_id: UUID) -> Component | None: c = self.table.c return self._one( f"find {release_id}/{repository_ref_id}", @@ -35,10 +35,10 @@ def find(self, release_id: UUID, repository_ref_id: UUID) -> ComponentRecord | N c.repository_ref_id == repository_ref_id, ) - def list_for_release(self, release_id: UUID) -> list[ComponentRecord]: + def list_for_release(self, release_id: UUID) -> list[Component]: return self._many(f"list for {release_id}", self.table.c.release_id == release_id) - def list_for_ref(self, repository_ref_id: UUID) -> list[ComponentRecord]: + def list_for_ref(self, repository_ref_id: UUID) -> list[Component]: return self._many( f"list for ref {repository_ref_id}", self.table.c.repository_ref_id == repository_ref_id ) diff --git a/src/traust_core/v1/repositories/traust/inventory/ownership.py b/src/traust_core/v1/repositories/traust/inventory/ownership.py index 9b6fdb7b..aff4537c 100644 --- a/src/traust_core/v1/repositories/traust/inventory/ownership.py +++ b/src/traust_core/v1/repositories/traust/inventory/ownership.py @@ -6,7 +6,7 @@ import sqlalchemy as sa -from traust_core.v1.models.records import OwnershipRecord +from traust_core.v1.models.inventory import Ownership from traust_core.v1.repositories.tables import ( TRAUST, MutableRecordRepository, @@ -28,23 +28,23 @@ ) -class OwnershipRepository(MutableRecordRepository[OwnershipRecord], Protocol): - def current(self, repository_id: UUID) -> list[OwnershipRecord]: ... - def as_of(self, repository_id: UUID, at: datetime) -> list[OwnershipRecord]: ... - def list_for_team(self, team_id: UUID) -> list[OwnershipRecord]: ... +class OwnershipRepository(MutableRecordRepository[Ownership], Protocol): + def current(self, repository_id: UUID) -> list[Ownership]: ... + def as_of(self, repository_id: UUID, at: datetime) -> list[Ownership]: ... + def list_for_team(self, team_id: UUID) -> list[Ownership]: ... -class SqlOwnershipRepository(SqlMutableRepository[OwnershipRecord]): +class SqlOwnershipRepository(SqlMutableRepository[Ownership]): spec = OWNERSHIP - record = OwnershipRecord + record = Ownership - def current(self, repository_id: UUID) -> list[OwnershipRecord]: + def current(self, repository_id: UUID) -> list[Ownership]: c = self.table.c return self._many( f"current for {repository_id}", c.repository_id == repository_id, c.valid_to.is_(None) ) - def as_of(self, repository_id: UUID, at: datetime) -> list[OwnershipRecord]: + def as_of(self, repository_id: UUID, at: datetime) -> list[Ownership]: c = self.table.c return self._many( f"as of {at} for {repository_id}", @@ -53,5 +53,5 @@ def as_of(self, repository_id: UUID, at: datetime) -> list[OwnershipRecord]: sa.or_(c.valid_to.is_(None), c.valid_to > at), ) - def list_for_team(self, team_id: UUID) -> list[OwnershipRecord]: + def list_for_team(self, team_id: UUID) -> list[Ownership]: return self._many(f"list for team {team_id}", self.table.c.team_id == team_id) diff --git a/src/traust_core/v1/repositories/traust/inventory/product.py b/src/traust_core/v1/repositories/traust/inventory/product.py index 8ebe221b..e4e570a5 100644 --- a/src/traust_core/v1/repositories/traust/inventory/product.py +++ b/src/traust_core/v1/repositories/traust/inventory/product.py @@ -2,7 +2,7 @@ from typing import Protocol -from traust_core.v1.models.records import ProductRecord +from traust_core.v1.models.inventory import Product from traust_core.v1.repositories.tables import ( TRAUST, MutableRecordRepository, @@ -14,13 +14,13 @@ PRODUCT = TableSpec(TRAUST, "product", lambda: (text_column("slug"), text_column("name"))) -class ProductRepository(MutableRecordRepository[ProductRecord], Protocol): - def find_by_slug(self, slug: str) -> ProductRecord | None: ... +class ProductRepository(MutableRecordRepository[Product], Protocol): + def find_by_slug(self, slug: str) -> Product | None: ... -class SqlProductRepository(SqlMutableRepository[ProductRecord]): +class SqlProductRepository(SqlMutableRepository[Product]): spec = PRODUCT - record = ProductRecord + record = Product - def find_by_slug(self, slug: str) -> ProductRecord | None: + def find_by_slug(self, slug: str) -> Product | None: return self._one(f"find {slug}", self.table.c.slug == slug) diff --git a/src/traust_core/v1/repositories/traust/inventory/release.py b/src/traust_core/v1/repositories/traust/inventory/release.py index d56aa3b6..15d0f10a 100644 --- a/src/traust_core/v1/repositories/traust/inventory/release.py +++ b/src/traust_core/v1/repositories/traust/inventory/release.py @@ -3,7 +3,7 @@ from typing import Protocol from uuid import UUID -from traust_core.v1.models.records import ReleaseRecord +from traust_core.v1.models.inventory import Release from traust_core.v1.repositories.tables import ( TRAUST, MutableRecordRepository, @@ -16,18 +16,18 @@ RELEASE = TableSpec(TRAUST, "release", lambda: (uuid_column("product_id"), text_column("version"))) -class ReleaseRepository(MutableRecordRepository[ReleaseRecord], Protocol): - def find(self, product_id: UUID, version: str) -> ReleaseRecord | None: ... - def list_for_product(self, product_id: UUID) -> list[ReleaseRecord]: ... +class ReleaseRepository(MutableRecordRepository[Release], Protocol): + def find(self, product_id: UUID, version: str) -> Release | None: ... + def list_for_product(self, product_id: UUID) -> list[Release]: ... -class SqlReleaseRepository(SqlMutableRepository[ReleaseRecord]): +class SqlReleaseRepository(SqlMutableRepository[Release]): spec = RELEASE - record = ReleaseRecord + record = Release - def find(self, product_id: UUID, version: str) -> ReleaseRecord | None: + def find(self, product_id: UUID, version: str) -> Release | None: c = self.table.c return self._one(f"find {version}", c.product_id == product_id, c.version == version) - def list_for_product(self, product_id: UUID) -> list[ReleaseRecord]: + def list_for_product(self, product_id: UUID) -> list[Release]: return self._many(f"list for {product_id}", self.table.c.product_id == product_id) diff --git a/src/traust_core/v1/repositories/traust/inventory/repository.py b/src/traust_core/v1/repositories/traust/inventory/repository.py index e139c6e5..422bb90a 100644 --- a/src/traust_core/v1/repositories/traust/inventory/repository.py +++ b/src/traust_core/v1/repositories/traust/inventory/repository.py @@ -4,7 +4,7 @@ import sqlalchemy as sa -from traust_core.v1.models.records import RepositoryRecord +from traust_core.v1.models.inventory import Repository from traust_core.v1.repositories.tables import ( TRAUST, MutableRecordRepository, @@ -25,13 +25,13 @@ ) -class RepositoryRepository(MutableRecordRepository[RepositoryRecord], Protocol): - def find_by_url(self, url: str) -> RepositoryRecord | None: ... +class RepositoryRepository(MutableRecordRepository[Repository], Protocol): + def find_by_url(self, url: str) -> Repository | None: ... -class SqlRepositoryRepository(SqlMutableRepository[RepositoryRecord]): +class SqlRepositoryRepository(SqlMutableRepository[Repository]): spec = REPOSITORY - record = RepositoryRecord + record = Repository - def find_by_url(self, url: str) -> RepositoryRecord | None: + def find_by_url(self, url: str) -> Repository | None: return self._one(f"find {url}", self.table.c.url == url) diff --git a/src/traust_core/v1/repositories/traust/inventory/repository_ref.py b/src/traust_core/v1/repositories/traust/inventory/repository_ref.py index a7df684a..7e04c465 100644 --- a/src/traust_core/v1/repositories/traust/inventory/repository_ref.py +++ b/src/traust_core/v1/repositories/traust/inventory/repository_ref.py @@ -5,7 +5,7 @@ import sqlalchemy as sa -from traust_core.v1.models.records import RepositoryRefRecord +from traust_core.v1.models.inventory import RepositoryRef from traust_core.v1.repositories.tables import ( TRAUST, MutableRecordRepository, @@ -29,31 +29,31 @@ ) -class RepositoryRefRepository(MutableRecordRepository[RepositoryRefRecord], Protocol): - def find(self, repository_id: UUID, name: str | None, sha: str) -> RepositoryRefRecord | None: +class RepositoryRefRepository(MutableRecordRepository[RepositoryRef], Protocol): + def find(self, repository_id: UUID, name: str | None, sha: str) -> RepositoryRef | None: """A named ref by (repository, name, sha); name None = the bare commit.""" ... - def list_for(self, repository_id: UUID) -> list[RepositoryRefRecord]: ... - def default(self, repository_id: UUID) -> RepositoryRefRecord | None: ... - def list_for_manifest_sbom(self, sbom_id: UUID) -> list[RepositoryRefRecord]: ... + def list_for(self, repository_id: UUID) -> list[RepositoryRef]: ... + def default(self, repository_id: UUID) -> RepositoryRef | None: ... + def list_for_manifest_sbom(self, sbom_id: UUID) -> list[RepositoryRef]: ... -class SqlRepositoryRefRepository(SqlMutableRepository[RepositoryRefRecord]): +class SqlRepositoryRefRepository(SqlMutableRepository[RepositoryRef]): spec = REPOSITORY_REF - record = RepositoryRefRecord + record = RepositoryRef - def find(self, repository_id: UUID, name: str | None, sha: str) -> RepositoryRefRecord | None: + def find(self, repository_id: UUID, name: str | None, sha: str) -> RepositoryRef | None: c = self.table.c named = c.name.is_(None) if name is None else c.name == name return self._one( f"find {name or 'bare'}@{sha}", c.repository_id == repository_id, named, c.sha == sha ) - def list_for(self, repository_id: UUID) -> list[RepositoryRefRecord]: + def list_for(self, repository_id: UUID) -> list[RepositoryRef]: return self._many(f"list for {repository_id}", self.table.c.repository_id == repository_id) - def default(self, repository_id: UUID) -> RepositoryRefRecord | None: + def default(self, repository_id: UUID) -> RepositoryRef | None: """The latest ref recorded as the default branch.""" c = self.table.c stmt = ( @@ -65,5 +65,5 @@ def default(self, repository_id: UUID) -> RepositoryRefRecord | None: row = self._execute(f"default for {repository_id}", stmt).mappings().first() return None if row is None else self._to_record(row) - def list_for_manifest_sbom(self, sbom_id: UUID) -> list[RepositoryRefRecord]: + def list_for_manifest_sbom(self, sbom_id: UUID) -> list[RepositoryRef]: return self._many(f"list for sbom {sbom_id}", self.table.c.manifest_sbom_id == sbom_id) diff --git a/src/traust_core/v1/repositories/traust/inventory/team.py b/src/traust_core/v1/repositories/traust/inventory/team.py index ed74f01a..8b02dd4c 100644 --- a/src/traust_core/v1/repositories/traust/inventory/team.py +++ b/src/traust_core/v1/repositories/traust/inventory/team.py @@ -2,7 +2,7 @@ from typing import Protocol -from traust_core.v1.models.records import TeamRecord +from traust_core.v1.models.inventory import Team from traust_core.v1.repositories.tables import ( TRAUST, MutableRecordRepository, @@ -14,13 +14,13 @@ TEAM = TableSpec(TRAUST, "team", lambda: (text_column("name"),)) -class TeamRepository(MutableRecordRepository[TeamRecord], Protocol): - def find_by_name(self, name: str) -> TeamRecord | None: ... +class TeamRepository(MutableRecordRepository[Team], Protocol): + def find_by_name(self, name: str) -> Team | None: ... -class SqlTeamRepository(SqlMutableRepository[TeamRecord]): +class SqlTeamRepository(SqlMutableRepository[Team]): spec = TEAM - record = TeamRecord + record = Team - def find_by_name(self, name: str) -> TeamRecord | None: + def find_by_name(self, name: str) -> Team | None: return self._one(f"find {name}", self.table.c.name == name) diff --git a/src/traust_core/v1/repositories/traust_ledger/__init__.py b/src/traust_core/v1/repositories/traust_ledger/__init__.py index 80fa43a2..7add4d99 100644 --- a/src/traust_core/v1/repositories/traust_ledger/__init__.py +++ b/src/traust_core/v1/repositories/traust_ledger/__init__.py @@ -7,7 +7,6 @@ from typing import Protocol -from traust_core.v1.repositories.tables import SqlTablesUnitOfWork from traust_core.v1.repositories.traust_ledger.events import ( EVENTS, EventRepository, @@ -28,13 +27,6 @@ class LedgerUnitOfWork(UnitOfWork, Protocol): events: EventRepository -class SqlLedgerUnitOfWork(SqlTablesUnitOfWork): - def _open_repositories(self) -> None: - conn, tables = self.connection, self.tables - self.layers = SqlLayerRepository(conn, tables) - self.events = SqlEventRepository(conn, tables) - - __all__ = [ "TABLES", "EventRepository", @@ -42,5 +34,4 @@ def _open_repositories(self) -> None: "LedgerUnitOfWork", "SqlEventRepository", "SqlLayerRepository", - "SqlLedgerUnitOfWork", ] diff --git a/src/traust_core/v1/repositories/traust_ledger/events.py b/src/traust_core/v1/repositories/traust_ledger/events.py index 56fd2ae6..019b2907 100644 --- a/src/traust_core/v1/repositories/traust_ledger/events.py +++ b/src/traust_core/v1/repositories/traust_ledger/events.py @@ -5,7 +5,7 @@ import sqlalchemy as sa -from traust_core.v1.models.records import EventRecord +from traust_core.v1.models.ledger import Event from traust_core.v1.repositories.tables import ( TRAUST_LEDGER, RecordRepository, @@ -27,20 +27,20 @@ ) -class EventRepository(RecordRepository[EventRecord], Protocol): - def find_by_decision(self, decision_id: UUID) -> EventRecord | None: ... - def list_for_layer(self, layer_id: UUID) -> list[EventRecord]: ... +class EventRepository(RecordRepository[Event], Protocol): + def find_by_decision(self, decision_id: UUID) -> Event | None: ... + def list_for_layer(self, layer_id: UUID) -> list[Event]: ... def last_seq(self, layer_id: UUID) -> int | None: ... -class SqlEventRepository(SqlAppendOnlyRepository[EventRecord]): +class SqlEventRepository(SqlAppendOnlyRepository[Event]): spec = EVENTS - record = EventRecord + record = Event - def find_by_decision(self, decision_id: UUID) -> EventRecord | None: + def find_by_decision(self, decision_id: UUID) -> Event | None: return self._one(f"find for {decision_id}", self.table.c.decision_id == decision_id) - def list_for_layer(self, layer_id: UUID) -> list[EventRecord]: + def list_for_layer(self, layer_id: UUID) -> list[Event]: c = self.table.c return self._many(f"list for {layer_id}", c.layer_id == layer_id, order_by=(c.seq,)) diff --git a/src/traust_core/v1/repositories/traust_ledger/layers.py b/src/traust_core/v1/repositories/traust_ledger/layers.py index 949b10d2..9fbcd990 100644 --- a/src/traust_core/v1/repositories/traust_ledger/layers.py +++ b/src/traust_core/v1/repositories/traust_ledger/layers.py @@ -5,7 +5,7 @@ import sqlalchemy as sa -from traust_core.v1.models.records import LayerRecord +from traust_core.v1.models.ledger import Layer from traust_core.v1.repositories.tables import ( TRAUST_LEDGER, MutableRecordRepository, @@ -31,13 +31,13 @@ ) -class LayerRepository(MutableRecordRepository[LayerRecord], Protocol): - def list_for_repository(self, repository_id: UUID) -> list[LayerRecord]: ... +class LayerRepository(MutableRecordRepository[Layer], Protocol): + def list_for_repository(self, repository_id: UUID) -> list[Layer]: ... -class SqlLayerRepository(SqlMutableRepository[LayerRecord]): +class SqlLayerRepository(SqlMutableRepository[Layer]): spec = LAYERS - record = LayerRecord + record = Layer - def list_for_repository(self, repository_id: UUID) -> list[LayerRecord]: + def list_for_repository(self, repository_id: UUID) -> list[Layer]: return self._many(f"list for {repository_id}", self.table.c.repository_id == repository_id) diff --git a/src/traust_core/v1/repositories/traust_ops/__init__.py b/src/traust_core/v1/repositories/traust_ops/__init__.py index e9a1395a..e955ea2d 100644 --- a/src/traust_core/v1/repositories/traust_ops/__init__.py +++ b/src/traust_core/v1/repositories/traust_ops/__init__.py @@ -7,7 +7,6 @@ from typing import Protocol -from traust_core.v1.repositories.tables import SqlTablesUnitOfWork from traust_core.v1.repositories.traust_ops.assessment_input import ( ASSESSMENT_INPUT, AssessmentInputRepository, @@ -46,16 +45,6 @@ class OpsUnitOfWork(UnitOfWork, Protocol): assessment_inputs: AssessmentInputRepository -class SqlOpsUnitOfWork(SqlTablesUnitOfWork): - def _open_repositories(self) -> None: - conn, tables = self.connection, self.tables - self.run_batches = SqlRunBatchRepository(conn, tables) - self.run_orders = SqlRunOrderRepository(conn, tables) - self.spend_events = SqlSpendEventRepository(conn, tables) - self.feed_snapshots = SqlFeedSnapshotRepository(conn, tables) - self.assessment_inputs = SqlAssessmentInputRepository(conn, tables) - - __all__ = [ "TABLES", "AssessmentInputRepository", @@ -66,7 +55,6 @@ def _open_repositories(self) -> None: "SpendEventRepository", "SqlAssessmentInputRepository", "SqlFeedSnapshotRepository", - "SqlOpsUnitOfWork", "SqlRunBatchRepository", "SqlRunOrderRepository", "SqlSpendEventRepository", diff --git a/src/traust_core/v1/repositories/traust_ops/assessment_input.py b/src/traust_core/v1/repositories/traust_ops/assessment_input.py index e0f3d90e..00a4d28b 100644 --- a/src/traust_core/v1/repositories/traust_ops/assessment_input.py +++ b/src/traust_core/v1/repositories/traust_ops/assessment_input.py @@ -3,7 +3,7 @@ from typing import Protocol from uuid import UUID -from traust_core.v1.models.records import AssessmentInputRecord +from traust_core.v1.models.operations import AssessmentInput from traust_core.v1.repositories.tables import ( TRAUST_OPS, RecordRepository, @@ -20,19 +20,19 @@ ) -class AssessmentInputRepository(RecordRepository[AssessmentInputRecord], Protocol): - def list_for_assessment(self, assessment_id: UUID) -> list[AssessmentInputRecord]: ... - def list_for_snapshot(self, feed_snapshot_id: UUID) -> list[AssessmentInputRecord]: ... +class AssessmentInputRepository(RecordRepository[AssessmentInput], Protocol): + def list_for_assessment(self, assessment_id: UUID) -> list[AssessmentInput]: ... + def list_for_snapshot(self, feed_snapshot_id: UUID) -> list[AssessmentInput]: ... -class SqlAssessmentInputRepository(SqlAppendOnlyRepository[AssessmentInputRecord]): +class SqlAssessmentInputRepository(SqlAppendOnlyRepository[AssessmentInput]): spec = ASSESSMENT_INPUT - record = AssessmentInputRecord + record = AssessmentInput - def list_for_assessment(self, assessment_id: UUID) -> list[AssessmentInputRecord]: + def list_for_assessment(self, assessment_id: UUID) -> list[AssessmentInput]: return self._many(f"list for {assessment_id}", self.table.c.assessment_id == assessment_id) - def list_for_snapshot(self, feed_snapshot_id: UUID) -> list[AssessmentInputRecord]: + def list_for_snapshot(self, feed_snapshot_id: UUID) -> list[AssessmentInput]: return self._many( f"list for snapshot {feed_snapshot_id}", self.table.c.feed_snapshot_id == feed_snapshot_id, diff --git a/src/traust_core/v1/repositories/traust_ops/feed_snapshot.py b/src/traust_core/v1/repositories/traust_ops/feed_snapshot.py index aad37724..d4cf559b 100644 --- a/src/traust_core/v1/repositories/traust_ops/feed_snapshot.py +++ b/src/traust_core/v1/repositories/traust_ops/feed_snapshot.py @@ -2,7 +2,7 @@ from typing import Protocol -from traust_core.v1.models.records import FeedSnapshotRecord +from traust_core.v1.models.operations import FeedSnapshot from traust_core.v1.repositories.tables import ( TRAUST_OPS, RecordRepository, @@ -26,15 +26,15 @@ ) -class FeedSnapshotRepository(RecordRepository[FeedSnapshotRecord], Protocol): - def latest(self, feed: str) -> FeedSnapshotRecord | None: ... +class FeedSnapshotRepository(RecordRepository[FeedSnapshot], Protocol): + def latest(self, feed: str) -> FeedSnapshot | None: ... -class SqlFeedSnapshotRepository(SqlAppendOnlyRepository[FeedSnapshotRecord]): +class SqlFeedSnapshotRepository(SqlAppendOnlyRepository[FeedSnapshot]): spec = FEED_SNAPSHOT - record = FeedSnapshotRecord + record = FeedSnapshot - def latest(self, feed: str) -> FeedSnapshotRecord | None: + def latest(self, feed: str) -> FeedSnapshot | None: c = self.table.c stmt = ( self._select() diff --git a/src/traust_core/v1/repositories/traust_ops/run_batch.py b/src/traust_core/v1/repositories/traust_ops/run_batch.py index 680807a2..5eac00f1 100644 --- a/src/traust_core/v1/repositories/traust_ops/run_batch.py +++ b/src/traust_core/v1/repositories/traust_ops/run_batch.py @@ -2,7 +2,7 @@ from typing import Protocol -from traust_core.v1.models.records import RunBatchRecord +from traust_core.v1.models.operations import RunBatch from traust_core.v1.repositories.tables import ( TRAUST_OPS, MutableRecordRepository, @@ -14,14 +14,14 @@ RUN_BATCH = TableSpec(TRAUST_OPS, "run_batch", lambda: (text_column("kind"), text_column("period"))) -class RunBatchRepository(MutableRecordRepository[RunBatchRecord], Protocol): - def find(self, kind: str, period: str) -> RunBatchRecord | None: ... +class RunBatchRepository(MutableRecordRepository[RunBatch], Protocol): + def find(self, kind: str, period: str) -> RunBatch | None: ... -class SqlRunBatchRepository(SqlMutableRepository[RunBatchRecord]): +class SqlRunBatchRepository(SqlMutableRepository[RunBatch]): spec = RUN_BATCH - record = RunBatchRecord + record = RunBatch - def find(self, kind: str, period: str) -> RunBatchRecord | None: + def find(self, kind: str, period: str) -> RunBatch | None: c = self.table.c return self._one(f"find {kind}/{period}", c.kind == kind, c.period == period) diff --git a/src/traust_core/v1/repositories/traust_ops/run_order.py b/src/traust_core/v1/repositories/traust_ops/run_order.py index 60f19ad1..6773f136 100644 --- a/src/traust_core/v1/repositories/traust_ops/run_order.py +++ b/src/traust_core/v1/repositories/traust_ops/run_order.py @@ -6,7 +6,7 @@ import sqlalchemy as sa -from traust_core.v1.models.records import RunOrderRecord +from traust_core.v1.models.operations import RunOrder from traust_core.v1.repositories.tables import ( TRAUST_OPS, MutableRecordRepository, @@ -43,21 +43,21 @@ LEASED = "leased" -class RunOrderRepository(MutableRecordRepository[RunOrderRecord], Protocol): - def claim(self, worker: str, now: datetime, lease_for: timedelta) -> RunOrderRecord | None: +class RunOrderRepository(MutableRecordRepository[RunOrder], Protocol): + def claim(self, worker: str, now: datetime, lease_for: timedelta) -> RunOrder | None: """Lease the next queued order (priority, then age); None when the queue is empty.""" ... def release_expired(self, now: datetime) -> int: ... - def list_for_batch(self, run_batch_id: UUID) -> list[RunOrderRecord]: ... - def list_for_repository(self, repository_id: UUID, state: str) -> list[RunOrderRecord]: ... + def list_for_batch(self, run_batch_id: UUID) -> list[RunOrder]: ... + def list_for_repository(self, repository_id: UUID, state: str) -> list[RunOrder]: ... -class SqlRunOrderRepository(SqlMutableRepository[RunOrderRecord]): +class SqlRunOrderRepository(SqlMutableRepository[RunOrder]): spec = RUN_ORDER - record = RunOrderRecord + record = RunOrder - def claim(self, worker: str, now: datetime, lease_for: timedelta) -> RunOrderRecord | None: + def claim(self, worker: str, now: datetime, lease_for: timedelta) -> RunOrder | None: c = self.table.c # SKIP LOCKED on Postgres; SQLite serialises writers and drops FOR UPDATE. next_id = ( @@ -87,12 +87,12 @@ def release_expired(self, now: datetime) -> int: ) return self._execute(f"release expired at {now}", stmt).rowcount - def list_for_batch(self, run_batch_id: UUID) -> list[RunOrderRecord]: + def list_for_batch(self, run_batch_id: UUID) -> list[RunOrder]: return self._many( f"list for batch {run_batch_id}", self.table.c.run_batch_id == run_batch_id ) - def list_for_repository(self, repository_id: UUID, state: str) -> list[RunOrderRecord]: + def list_for_repository(self, repository_id: UUID, state: str) -> list[RunOrder]: c = self.table.c return self._many( f"list {state} for {repository_id}", diff --git a/src/traust_core/v1/repositories/traust_ops/spend_event.py b/src/traust_core/v1/repositories/traust_ops/spend_event.py index 4535f9c6..abd90d21 100644 --- a/src/traust_core/v1/repositories/traust_ops/spend_event.py +++ b/src/traust_core/v1/repositories/traust_ops/spend_event.py @@ -6,7 +6,7 @@ import sqlalchemy as sa -from traust_core.v1.models.records import SpendEventRecord +from traust_core.v1.models.operations import SpendEvent from traust_core.v1.repositories.tables import ( TRAUST_OPS, RecordRepository, @@ -34,19 +34,19 @@ ) -class SpendEventRepository(RecordRepository[SpendEventRecord], Protocol): - def list_for_order(self, run_order_id: UUID) -> list[SpendEventRecord]: ... - def list_between(self, start: datetime, end: datetime) -> list[SpendEventRecord]: ... +class SpendEventRepository(RecordRepository[SpendEvent], Protocol): + def list_for_order(self, run_order_id: UUID) -> list[SpendEvent]: ... + def list_between(self, start: datetime, end: datetime) -> list[SpendEvent]: ... -class SqlSpendEventRepository(SqlAppendOnlyRepository[SpendEventRecord]): +class SqlSpendEventRepository(SqlAppendOnlyRepository[SpendEvent]): spec = SPEND_EVENT - record = SpendEventRecord + record = SpendEvent - def list_for_order(self, run_order_id: UUID) -> list[SpendEventRecord]: + def list_for_order(self, run_order_id: UUID) -> list[SpendEvent]: return self._many(f"list for {run_order_id}", self.table.c.run_order_id == run_order_id) - def list_between(self, start: datetime, end: datetime) -> list[SpendEventRecord]: + def list_between(self, start: datetime, end: datetime) -> list[SpendEvent]: """Spend with start <= occurred_at < end.""" c = self.table.c return self._many( diff --git a/src/traust_core/v1/repositories/triage.py b/src/traust_core/v1/repositories/triage.py deleted file mode 100644 index 06ca7baa..00000000 --- a/src/traust_core/v1/repositories/triage.py +++ /dev/null @@ -1,115 +0,0 @@ -from __future__ import annotations - -from collections.abc import Sequence -from typing import Any, Protocol - -import sqlalchemy as sa - -from traust_core.v1.models.findings import TriageVerdict -from traust_core.v1.models.vocabulary import Verdict -from traust_core.v1.repositories.memory import InMemoryUnitOfWork -from traust_core.v1.repositories.sql import SqlRepository, SqlUnitOfWork -from traust_core.v1.repositories.storage import ( - STORAGE_SCHEMA, - EvidenceRepository, - InMemoryEvidenceRepository, - SqlEvidenceRepository, -) -from traust_core.v1.repositories.unit_of_work import UnitOfWork - - -class TriageVerdictRepository(Protocol): - def add_all(self, verdicts: Sequence[TriageVerdict]) -> None: ... - def for_binding(self, binding_id: str) -> list[TriageVerdict]: ... - def for_source_finding(self, source_finding_id: str) -> list[TriageVerdict]: ... - def true_positives(self, binding_id: str) -> list[TriageVerdict]: ... - - -class TriageUnitOfWork(UnitOfWork, Protocol): - evidence: EvidenceRepository - triage_verdicts: TriageVerdictRepository - - -triage_verdict = sa.Table( - "triage_verdict", - sa.MetaData(schema=STORAGE_SCHEMA), - sa.Column("binding_id", sa.Text, primary_key=True), - sa.Column("artifact_digest", sa.Text, nullable=False), - sa.Column("finding_id", sa.Text, primary_key=True), - sa.Column("source_finding_id", sa.Text), - sa.Column("triage_completed", sa.Text, nullable=False), - sa.Column("verdict", sa.Text, nullable=False), - sa.Column("severity", sa.Text), - sa.Column("vote_breakdown", sa.JSON(none_as_null=True)), - sa.Column("rationale", sa.Text), -) - - -class SqlTriageVerdictRepository(SqlRepository): - def add_all(self, verdicts: Sequence[TriageVerdict]) -> None: - if verdicts: - rows = [v.model_dump() for v in verdicts] - self._execute(f"add {len(rows)} verdicts", sa.insert(triage_verdict), rows) - - def _select(self, operation: str, *where: sa.ColumnElement[bool]) -> list[TriageVerdict]: - stmt = sa.select(triage_verdict).where(*where).order_by(triage_verdict.c.finding_id) - return [_to_verdict(r) for r in self._execute(operation, stmt).mappings()] - - def for_binding(self, binding_id: str) -> list[TriageVerdict]: - return self._select(f"for binding {binding_id}", triage_verdict.c.binding_id == binding_id) - - def for_source_finding(self, source_finding_id: str) -> list[TriageVerdict]: - return self._select( - f"for source finding {source_finding_id}", - triage_verdict.c.source_finding_id == source_finding_id, - ) - - def true_positives(self, binding_id: str) -> list[TriageVerdict]: - return self._select( - f"true positives in {binding_id}", - triage_verdict.c.binding_id == binding_id, - triage_verdict.c.verdict == Verdict.TRUE_POSITIVE, - ) - - -def _to_verdict(row: Any) -> TriageVerdict: - return TriageVerdict.model_validate(dict(row)) - - -class SqlTriageUnitOfWork(SqlUnitOfWork): - evidence: SqlEvidenceRepository - triage_verdicts: SqlTriageVerdictRepository - - def _open_repositories(self) -> None: - self.evidence = SqlEvidenceRepository(self.connection) - self.triage_verdicts = SqlTriageVerdictRepository(self.connection) - - -class InMemoryTriageVerdictRepository: - def __init__(self) -> None: - self._rows: dict[tuple[str, str], TriageVerdict] = {} - - def add_all(self, verdicts: Sequence[TriageVerdict]) -> None: - for v in verdicts: - self._rows[(v.binding_id, v.finding_id)] = v - - def _select(self, keep: Any) -> list[TriageVerdict]: - return sorted((v for v in self._rows.values() if keep(v)), key=lambda v: v.finding_id) - - def for_binding(self, binding_id: str) -> list[TriageVerdict]: - return self._select(lambda v: v.binding_id == binding_id) - - def for_source_finding(self, source_finding_id: str) -> list[TriageVerdict]: - return self._select(lambda v: v.source_finding_id == source_finding_id) - - def true_positives(self, binding_id: str) -> list[TriageVerdict]: - return self._select( - lambda v: v.binding_id == binding_id and v.verdict == Verdict.TRUE_POSITIVE - ) - - -class InMemoryTriageUnitOfWork(InMemoryUnitOfWork): - def __init__(self) -> None: - super().__init__() - self.evidence = InMemoryEvidenceRepository() - self.triage_verdicts = InMemoryTriageVerdictRepository() diff --git a/src/traust_core/v1/repositories/validation.py b/src/traust_core/v1/repositories/validation.py deleted file mode 100644 index cd3d5668..00000000 --- a/src/traust_core/v1/repositories/validation.py +++ /dev/null @@ -1,26 +0,0 @@ -from __future__ import annotations - -from typing import Protocol -from uuid import UUID - -from traust_core.v1.models.common import Page, PageRequest -from traust_core.v1.models.validation import ValidationAttempt, ValidationFilter, ValidationPlan -from traust_core.v1.repositories.unit_of_work import UnitOfWork - - -class ValidationPlanRepository(Protocol): - def get(self, plan_id: UUID) -> ValidationPlan: ... - def add(self, plan: ValidationPlan) -> ValidationPlan: ... - def update(self, plan: ValidationPlan) -> ValidationPlan: ... - def list_for(self, finding_id: UUID) -> list[ValidationPlan]: ... - - -class ValidationAttemptRepository(Protocol): - def add(self, attempt: ValidationAttempt) -> ValidationAttempt: ... - def list_for(self, plan_id: UUID) -> list[ValidationAttempt]: ... - def list(self, filter: ValidationFilter, page: PageRequest) -> Page[ValidationAttempt]: ... - - -class ValidationUnitOfWork(UnitOfWork, Protocol): - plans: ValidationPlanRepository - attempts: ValidationAttemptRepository diff --git a/src/traust_core/v1/services/__init__.py b/src/traust_core/v1/services/__init__.py index 2f474810..513492de 100644 --- a/src/traust_core/v1/services/__init__.py +++ b/src/traust_core/v1/services/__init__.py @@ -1,33 +1,23 @@ from traust_core.v1.services.artifact_publishing import ( + ArtifactIndex, ArtifactPublisher, - BindingIndex, Companion, InputFormat, JsonInput, PublishResult, PublishSpec, PublishSpecs, - StorageIndex, -) -from traust_core.v1.services.storage import ( - RecordResult, - bind_artifact, - get_binding, - record_artifact, + SqlArtifactIndex, ) __all__ = [ + "ArtifactIndex", "ArtifactPublisher", - "BindingIndex", "Companion", "InputFormat", "JsonInput", "PublishResult", "PublishSpec", "PublishSpecs", - "RecordResult", - "StorageIndex", - "bind_artifact", - "get_binding", - "record_artifact", + "SqlArtifactIndex", ] diff --git a/src/traust_core/v1/services/artifact_publishing.py b/src/traust_core/v1/services/artifact_publishing.py index f299281b..07c0df65 100644 --- a/src/traust_core/v1/services/artifact_publishing.py +++ b/src/traust_core/v1/services/artifact_publishing.py @@ -8,15 +8,11 @@ from traust_core.v1.errors import ConfigError, DocumentError, Issue, TraustError from traust_core.v1.models.analysis_results import ResultKind, Subject from traust_core.v1.models.artifacts import Artifact +from traust_core.v1.models.assessment import Artifact as ArtifactRow from traust_core.v1.models.base import Dto -from traust_core.v1.models.common import ArtifactRef, Page, PageRequest -from traust_core.v1.models.storage import BindingContext -from traust_core.v1.providers.clock import Clock from traust_core.v1.repositories.analysis_results import AnalysisResultsRepository, artifact_kind -from traust_core.v1.repositories.artifacts import ArtifactFilter from traust_core.v1.repositories.object_store import ObjectRef -from traust_core.v1.repositories.storage import StorageUnitOfWork -from traust_core.v1.services.storage import record_artifact +from traust_core.v1.repositories.traust.assessments import AssessmentsUnitOfWork A = TypeVar("A", bound=Artifact) @@ -72,27 +68,30 @@ def names(self) -> list[str]: return sorted(self._specs) -class BindingIndex(Protocol): - def record(self, artifact: Artifact, subject: Subject, ref: ObjectRef, run_id: str) -> str: ... +class ArtifactIndex(Protocol): + def record(self, artifact: Artifact, subject: Subject, ref: ObjectRef, run_id: str) -> UUID: ... -class StorageIndex: - def __init__(self, storage: Callable[[], StorageUnitOfWork], clock: Clock) -> None: - self._storage = storage - self._clock = clock +class SqlArtifactIndex: + """Records each published document as an artifact row; same bytes, same row.""" - def record(self, artifact: Artifact, subject: Subject, ref: ObjectRef, run_id: str) -> str: - context = BindingContext(subject_id=subject.key, run_id=run_id, role=artifact.role) - result = record_artifact( - self._storage(), artifact.schema, artifact.payload, self._clock, context, [str(ref.key)] - ) - return result.binding_id + def __init__(self, uow: Callable[[], AssessmentsUnitOfWork]) -> None: + self._uow = uow + + def record(self, artifact: Artifact, subject: Subject, ref: ObjectRef, run_id: str) -> UUID: + with self._uow() as u: + row = u.artifacts.find(None, ref.sha256) or u.artifacts.add( + ArtifactRow(digest=ref.sha256, kind=artifact.schema, uri=str(ref.key)) + ) + u.commit() + assert row.id is not None + return row.id class PublishResult(Dto): artifact: str written: tuple[str, ...] - binding_id: str | None = None + artifact_id: UUID | None = None index_error: str | None = None @property @@ -105,7 +104,7 @@ def __init__( self, specs: PublishSpecs, results: AnalysisResultsRepository, - index: BindingIndex | None = None, + index: ArtifactIndex | None = None, ) -> None: self._specs = specs self._results = results @@ -127,10 +126,10 @@ def publish_artifact(self, artifact: Artifact, subject: Subject, run_id: str) -> if self._index is None: return result try: - binding = self._index.record(artifact, subject, primary, run_id) + artifact_id = self._index.record(artifact, subject, primary, run_id) except TraustError as e: return result.model_copy(update={"index_error": str(e)}) - return result.model_copy(update={"binding_id": binding}) + return result.model_copy(update={"artifact_id": artifact_id}) def get(self, artifact_id: UUID) -> Artifact: raise NotImplementedError("stub: replaces AnalysisResultsRepository.read by path subject") @@ -138,7 +137,7 @@ def get(self, artifact_id: UUID) -> Artifact: def verify(self, artifact_id: UUID) -> DigestCheck: raise NotImplementedError("stub: replaces verify_report_digests.py, ledger report digests") - def list(self, filter: ArtifactFilter, page: PageRequest) -> Page[ArtifactRef]: + def list_for_assessment(self, assessment_id: UUID) -> list[ArtifactRow]: raise NotImplementedError("stub: replaces corpus/report_store listing") diff --git a/src/traust_core/v1/services/assessment.py b/src/traust_core/v1/services/assessment.py index 7a025675..e3ee9ce5 100644 --- a/src/traust_core/v1/services/assessment.py +++ b/src/traust_core/v1/services/assessment.py @@ -8,8 +8,8 @@ from traust_core.v1.models.assessment import ( Assessment, AssessmentFilter, - AssessmentInput, AssessmentKind, + AssessmentSkillInput, CoverageRow, Freshness, Population, @@ -20,34 +20,34 @@ ToolRun, ) from traust_core.v1.models.common import Page, PageRequest, Provenance -from traust_core.v1.repositories.assessment import AssessmentUnitOfWork +from traust_core.v1.repositories.traust.assessments import AssessmentsUnitOfWork from traust_core.v1.services.findings.intake import FindingIntake class AssessmentService: - def __init__(self, uow: AssessmentUnitOfWork, intake: FindingIntake) -> None: + def __init__(self, uow: AssessmentsUnitOfWork, intake: FindingIntake) -> None: self._uow = uow self._intake = intake - def run_tools(self, revision_id: UUID, tools: Sequence[ToolName]) -> ToolRun: + def run_tools(self, repository_ref_id: UUID, tools: Sequence[ToolName]) -> ToolRun: raise NotImplementedError("stub: replaces run_checkov.py, adapters _contract_bridge") def prepare( - self, revision_id: UUID, kind: AssessmentKind, base_id: UUID | None - ) -> AssessmentInput: + self, repository_ref_id: UUID, kind: AssessmentKind, base_id: UUID | None + ) -> AssessmentSkillInput: raise NotImplementedError( "stub: replaces build_diff_packet.py, resolve_baseline.py, threat-model inputs" ) def record( - self, doc: AssessmentArtifact, revision_id: UUID, provenance: Provenance + self, doc: AssessmentArtifact, repository_ref_id: UUID, provenance: Provenance ) -> Assessment: raise NotImplementedError("stub: replaces corpus/store_ingest.py, threat_model.py") def get(self, assessment_id: UUID) -> Assessment: raise NotImplementedError("stub: assessment read") - def latest(self, code_line_id: UUID, kind: AssessmentKind) -> Assessment | None: + def latest(self, repository_ref_id: UUID, kind: AssessmentKind) -> Assessment | None: raise NotImplementedError("stub: replaces corpus/resolver latest report") def list(self, filter: AssessmentFilter, page: PageRequest) -> Page[Assessment]: @@ -56,7 +56,7 @@ def list(self, filter: AssessmentFilter, page: PageRequest) -> Page[Assessment]: def population(self, as_of: datetime, filter: AssessmentFilter) -> Population: raise NotImplementedError("stub: replaces census/build_census.py, corpus/resolver") - def freshness(self, code_line_id: UUID) -> Freshness: + def freshness(self, repository_ref_id: UUID) -> Freshness: raise NotImplementedError("stub: replaces build_rescan_worklist audit_age / churn") def threats(self, filter: ThreatFilter, page: PageRequest) -> Page[Threat]: diff --git a/src/traust_core/v1/services/attestation.py b/src/traust_core/v1/services/attestation.py index 232ef3e7..659e3d9f 100644 --- a/src/traust_core/v1/services/attestation.py +++ b/src/traust_core/v1/services/attestation.py @@ -3,24 +3,21 @@ from uuid import UUID from traust_core.v1.models.attestation import ( - Attestation, AttestationDecision, Comment, CommentSourceName, Credential, - ReviewItem, - ReviewSubject, Statement, TrustTier, ) from traust_core.v1.models.common import Page, PageRequest, Principal -from traust_core.v1.models.findings import ReviewFilter -from traust_core.v1.models.portfolio import Ownership +from traust_core.v1.models.findings import Decision, ReviewFilter, ReviewItem +from traust_core.v1.models.inventory import Ownership from traust_core.v1.models.remediation import ExternalRef from traust_core.v1.models.vocabulary import ReviewReason from traust_core.v1.providers.collaboration import CommentSource from traust_core.v1.providers.identity import IdentityProvider -from traust_core.v1.repositories.attestation import AttestationUnitOfWork +from traust_core.v1.repositories.traust.findings import FindingsUnitOfWork from traust_core.v1.services.ledger import LedgerService @@ -34,7 +31,7 @@ def trust( class AttestationService: def __init__( self, - uow: AttestationUnitOfWork, + uow: FindingsUnitOfWork, ledger: LedgerService | None, comments: CommentSource, policy: StatementPolicy, @@ -44,7 +41,9 @@ def __init__( self._comments = comments self._policy = policy - def request(self, subject: ReviewSubject, reason: ReviewReason) -> ReviewItem: + def request( + self, finding_id: UUID, decision_id: UUID | None, reason: ReviewReason + ) -> ReviewItem: raise NotImplementedError("stub: replaces countersign inbox writes, fleet-fix batch review") def attest( @@ -53,16 +52,17 @@ def attest( principal: Principal, decision: AttestationDecision, statement: Statement | None, - ) -> Attestation: + ) -> Decision: + """A human answer is a decision (actor set) that resolves the review item.""" raise NotImplementedError("stub: replaces countersign.py, ledger cli countersign") - def import_statements(self, source: CommentSourceName, ref: ExternalRef) -> list[Attestation]: + def import_statements(self, source: CommentSourceName, ref: ExternalRef) -> list[Decision]: raise NotImplementedError("stub: replaces track-findings MR/Jira comment reads") def pending(self, filter: ReviewFilter, page: PageRequest) -> Page[ReviewItem]: raise NotImplementedError("stub: replaces countersign.discover_pending") - def for_subject(self, subject: ReviewSubject) -> list[Attestation]: + def for_finding(self, finding_id: UUID) -> list[ReviewItem]: raise NotImplementedError("stub: replaces countersign baseline_for") diff --git a/src/traust_core/v1/services/code_facts.py b/src/traust_core/v1/services/code_facts.py index b199683c..fd33f207 100644 --- a/src/traust_core/v1/services/code_facts.py +++ b/src/traust_core/v1/services/code_facts.py @@ -14,13 +14,13 @@ from traust_core.v1.models.common import Page, PageRequest from traust_core.v1.providers.sources import SourceProvider from traust_core.v1.providers.tools import FactEnumerator -from traust_core.v1.repositories.code_facts import CodeFactsUnitOfWork +from traust_core.v1.repositories.traust.assessments import AssessmentsUnitOfWork class CodeFactsService: def __init__( self, - uow: CodeFactsUnitOfWork, + uow: AssessmentsUnitOfWork, sources: SourceProvider, enumerators: Sequence[FactEnumerator], ) -> None: @@ -28,16 +28,16 @@ def __init__( self._sources = sources self._enumerators = enumerators - def collect(self, revision_id: UUID, kinds: Sequence[CodeFactKind]) -> CollectResult: + def collect(self, repository_ref_id: UUID, kinds: Sequence[CodeFactKind]) -> CollectResult: raise NotImplementedError("stub: replaces build_symbol_index, secure-code enumerators") def list( - self, revision_id: UUID, kinds: Sequence[CodeFactKind], page: PageRequest + self, repository_ref_id: UUID, kinds: Sequence[CodeFactKind], page: PageRequest ) -> Page[CodeFact]: raise NotImplementedError("stub: code facts read") - def symbols(self, revision_id: UUID, query: SymbolQuery) -> list[SymbolFact]: + def symbols(self, repository_ref_id: UUID, query: SymbolQuery) -> list[SymbolFact]: raise NotImplementedError("stub: replaces query_index.py, build_diff_packet.callers_for") - def language_stats(self, revision_id: UUID) -> LanguageStats: + def language_stats(self, repository_ref_id: UUID) -> LanguageStats: raise NotImplementedError("stub: replaces loc-dashboard GitHub language fetch") diff --git a/src/traust_core/v1/services/compliance.py b/src/traust_core/v1/services/compliance.py index 45ca55d5..1b76819b 100644 --- a/src/traust_core/v1/services/compliance.py +++ b/src/traust_core/v1/services/compliance.py @@ -24,11 +24,11 @@ ScopeFilter, ) from traust_core.v1.models.vocabulary import ComplianceFramework -from traust_core.v1.repositories.compliance import ComplianceUnitOfWork +from traust_core.v1.repositories.traust.assessments import AssessmentsUnitOfWork class ComplianceService: - def __init__(self, uow: ComplianceUnitOfWork) -> None: + def __init__(self, uow: AssessmentsUnitOfWork) -> None: self._uow = uow def set_scope(self, doc: ComplianceScopeArtifact, principal: Principal) -> ComplianceScope: diff --git a/src/traust_core/v1/services/detection_quality.py b/src/traust_core/v1/services/detection_quality.py index 62d9a037..3b8e39b9 100644 --- a/src/traust_core/v1/services/detection_quality.py +++ b/src/traust_core/v1/services/detection_quality.py @@ -22,11 +22,11 @@ Scorecard, ) from traust_core.v1.models.vocabulary import ComplianceFramework -from traust_core.v1.repositories.detection_quality import DetectionQualityUnitOfWork +from traust_core.v1.repositories.traust.assessments import AssessmentsUnitOfWork class DetectionQualityService: - def __init__(self, uow: DetectionQualityUnitOfWork) -> None: + def __init__(self, uow: AssessmentsUnitOfWork) -> None: self._uow = uow def record(self, doc: BenchmarkTargetArtifact, provenance: Provenance) -> BenchmarkTarget: diff --git a/src/traust_core/v1/services/exposure.py b/src/traust_core/v1/services/exposure.py index d22d55b8..1117f35b 100644 --- a/src/traust_core/v1/services/exposure.py +++ b/src/traust_core/v1/services/exposure.py @@ -4,10 +4,10 @@ from traust_core.v1.models.artifacts import ImpactAnalysisArtifact from traust_core.v1.models.common import Page, PageRequest, Provenance -from traust_core.v1.models.exposure import ( +from traust_core.v1.models.dependencies import ( Advisory, AdvisoryIdentifier, - AdvisorySymbols, + AdvisoryPackage, AffectedDependency, Dependency, DependencyIngest, @@ -21,36 +21,51 @@ ImpactInput, Package, ProvenanceCheck, + Sbom, Version, VexStatus, ) from traust_core.v1.providers.feeds import FeedStatus -from traust_core.v1.repositories.exposure import ExposureUnitOfWork +from traust_core.v1.repositories.contract import TraustUnitOfWork from traust_core.v1.services.findings.intake import FindingIntake class ExposureService: - def __init__(self, uow: ExposureUnitOfWork, intake: FindingIntake) -> None: + def __init__(self, uow: TraustUnitOfWork, intake: FindingIntake) -> None: self._uow = uow self._intake = intake def refresh_feed(self, feed: FeedName) -> FeedStatus: raise NotImplementedError("stub: replaces fetch_feeds.py, registry/feeds.py") - def ingest_dependencies(self, revision_id: UUID) -> DependencyIngest: + def ingest_sbom(self, artifact_id: UUID) -> Sbom: + """CycloneDX / SPDX from standard tools; deduplicated by normalized package-list digest.""" raise NotImplementedError("stub: replaces portfolio/graph.build_deps, parsers.py") - def resolve_symbols(self, advisory_id: UUID) -> AdvisorySymbols: + def ingest_dependencies(self, repository_ref_id: UUID) -> DependencyIngest: + raise NotImplementedError("stub: replaces portfolio/graph.build_deps, parsers.py") + + def resolve_symbols(self, advisory_id: UUID) -> UUID: + """Symbols land as an artifact; returns its artifact id.""" raise NotImplementedError( "stub: replaces resolve_advisory_symbols.py, expand_advisory_entry_points.py" ) - def prepare(self, advisory_id: UUID, revision_id: UUID) -> ImpactInput: + def prepare(self, advisory_id: UUID, repository_ref_id: UUID) -> ImpactInput: raise NotImplementedError("stub: replaces impact/analyzer evidence collection") def record(self, doc: ImpactAnalysisArtifact, provenance: Provenance) -> ImpactAnalysis: raise NotImplementedError("stub: replaces impact/analyzer.emit_impact_analysis") + def affects(self, advisory_id: UUID, package_id: UUID, version: Version) -> bool: + raise NotImplementedError("stub: replaces dependency-watch/fleet_sweep.fixed_range") + + def fixed_in(self, advisory_id: UUID, package_id: UUID) -> Version | None: + raise NotImplementedError("stub: replaces adapters/osv.fixed_version") + + def packages_affected(self, advisory_id: UUID) -> list[AdvisoryPackage]: + raise NotImplementedError("stub: advisory -> package links from feeds") + def verify_dependency(self, package: Package, version: Version) -> DependencyProvenance: raise NotImplementedError("stub: replaces verify_dep_provenance.py") @@ -69,10 +84,10 @@ def advisories_for(self, finding_id: UUID) -> list[Advisory]: def affected(self, advisory_id: UUID) -> list[AffectedDependency]: raise NotImplementedError("stub: replaces fleet_sweep.py, run_impact_sweep.py") - def dependencies(self, revision_id: UUID) -> list[Dependency]: + def dependencies(self, repository_ref_id: UUID) -> list[Dependency]: raise NotImplementedError("stub: replaces portfolio-graph dependency reads") - def dependents(self, package: Package) -> list[UUID]: + def dependents(self, package_id: UUID) -> list[UUID]: raise NotImplementedError("stub: replaces portfolio/graph dependents, smoke_deps_multi") def impacts(self, filter: ImpactFilter, page: PageRequest) -> Page[ImpactAnalysis]: diff --git a/src/traust_core/v1/services/findings/__init__.py b/src/traust_core/v1/services/findings/__init__.py index 3b2fa9e8..025616bc 100644 --- a/src/traust_core/v1/services/findings/__init__.py +++ b/src/traust_core/v1/services/findings/__init__.py @@ -1,4 +1,4 @@ -from traust_core.v1.services.findings.decisions import FindingDecisions, record_triage +from traust_core.v1.services.findings.decisions import FindingDecisions from traust_core.v1.services.findings.identity import IdentityPolicy from traust_core.v1.services.findings.intake import FindingIntake from traust_core.v1.services.findings.precedents import FindingPrecedents @@ -17,5 +17,4 @@ "RatingPolicy", "ReviewPolicy", "TaxonomyPolicy", - "record_triage", ] diff --git a/src/traust_core/v1/services/findings/decisions.py b/src/traust_core/v1/services/findings/decisions.py index b88e78c0..fa349eda 100644 --- a/src/traust_core/v1/services/findings/decisions.py +++ b/src/traust_core/v1/services/findings/decisions.py @@ -1,35 +1,34 @@ from __future__ import annotations -from collections.abc import Sequence -from typing import Any from uuid import UUID from traust_core.v1.models.artifacts import TriageArtifact from traust_core.v1.models.common import Principal, Provenance from traust_core.v1.models.findings import ( + Decision, Disposition, RecordDecision, + StatementKey, TriageInput, - TriageRecorded, - TriageVerdict, ) -from traust_core.v1.models.storage import BindingContext from traust_core.v1.models.vocabulary import RestatementReason -from traust_core.v1.providers.clock import Clock -from traust_core.v1.repositories.findings import FindingUnitOfWork -from traust_core.v1.repositories.triage import TriageUnitOfWork +from traust_core.v1.repositories.contract import TraustUnitOfWork from traust_core.v1.services.ledger import LedgerService -from traust_core.v1.services.storage import RecordResult, bind_artifact class FindingDecisions: - def __init__(self, uow: FindingUnitOfWork, ledger: LedgerService | None) -> None: + """Decisions are the only writes to a finding's state; every read is a view over them.""" + + def __init__(self, uow: TraustUnitOfWork, ledger: LedgerService | None) -> None: self._uow = uow self._ledger = ledger def decide(self, command: RecordDecision) -> Disposition: raise NotImplementedError("stub: replaces emit_*_ledger_events.py, ledger event_handler") + def statement_key(self, decision: Decision) -> StatementKey: + raise NotImplementedError("stub: replaces ledger compute_event_id (classifier steps 9/12)") + def restate( self, decision_id: UUID, reason: RestatementReason, principal: Principal ) -> Disposition: @@ -38,49 +37,6 @@ def restate( def prepare(self, assessment_id: UUID) -> TriageInput: raise NotImplementedError("stub: replaces the triage skill's query_index / precedent CLIs") - def record(self, doc: TriageArtifact, provenance: Provenance) -> TriageRecorded: - raise NotImplementedError("stub: wraps record_triage; replaces emit_triage_ledger_events") - - def rebuild_dispositions(self, repository_id: UUID | None) -> int: - raise NotImplementedError("stub: replaces ledger materialize, corpus findings_db") - - -def verdicts_from(triage: TriageArtifact, binding_id: str, digest: str) -> list[TriageVerdict]: - return [ - TriageVerdict( - binding_id=binding_id, - artifact_digest=digest, - finding_id=f.id, - source_finding_id=f.orig_id, - triage_completed=triage.triage_completed, - verdict=f.verdict, - severity=f.severity, - vote_breakdown=_votes(f.vote_breakdown), - rationale=f.rationale, - ) - for f in triage.findings - ] - - -def _votes(view: Any) -> dict[str, int] | None: - if view is None: - return None - return { - k: getattr(view, k) - for k in ("true_positive", "hardening", "false_positive", "cannot_verify") - } - - -def record_triage( - uow: TriageUnitOfWork, - triage: TriageArtifact, - clock: Clock, - context: BindingContext, - references: Sequence[str] = (), -) -> RecordResult: - with uow: - result = bind_artifact(uow, triage.schema, triage.payload, clock, context, references) - if not result.already_bound: - uow.triage_verdicts.add_all(verdicts_from(triage, result.binding_id, result.digest)) - uow.commit() - return result + def record(self, doc: TriageArtifact, provenance: Provenance) -> list[Decision]: + """A triage is an assessment (kind triage) and one decision per verdict, one transaction.""" + raise NotImplementedError("stub: replaces record_triage, emit_triage_ledger_events") diff --git a/src/traust_core/v1/services/findings/intake.py b/src/traust_core/v1/services/findings/intake.py index 30226fa7..50c42955 100644 --- a/src/traust_core/v1/services/findings/intake.py +++ b/src/traust_core/v1/services/findings/intake.py @@ -5,14 +5,14 @@ from traust_core.v1.models.common import Provenance from traust_core.v1.models.findings import Finding, FindingCandidate, IdentityMove, IntakeResult -from traust_core.v1.repositories.findings import FindingUnitOfWork +from traust_core.v1.repositories.contract import TraustUnitOfWork from traust_core.v1.services.findings.identity import IdentityPolicy from traust_core.v1.services.findings.taxonomy import TaxonomyPolicy class FindingIntake: def __init__( - self, uow: FindingUnitOfWork, identity: IdentityPolicy, taxonomy: TaxonomyPolicy + self, uow: TraustUnitOfWork, identity: IdentityPolicy, taxonomy: TaxonomyPolicy ) -> None: self._uow = uow self._identity = identity @@ -28,5 +28,5 @@ def accept( "stub: replaces route_impact_findings, route_regressions, emit_fuzz_events" ) - def rebaseline(self, moves: Sequence[IdentityMove], attestation_id: UUID) -> list[Finding]: + def rebaseline(self, moves: Sequence[IdentityMove], decision_id: UUID) -> list[Finding]: raise NotImplementedError("stub: replaces finding_identity.rebaseline") diff --git a/src/traust_core/v1/services/findings/precedents.py b/src/traust_core/v1/services/findings/precedents.py index 2293bbd1..ba0dec9d 100644 --- a/src/traust_core/v1/services/findings/precedents.py +++ b/src/traust_core/v1/services/findings/precedents.py @@ -1,11 +1,11 @@ from __future__ import annotations from traust_core.v1.models.findings import FindingCandidate, Precedent -from traust_core.v1.repositories.findings import FindingUnitOfWork +from traust_core.v1.repositories.contract import TraustUnitOfWork class FindingPrecedents: - def __init__(self, uow: FindingUnitOfWork) -> None: + def __init__(self, uow: TraustUnitOfWork) -> None: self._uow = uow def for_candidate(self, candidate: FindingCandidate, limit: int) -> list[Precedent]: diff --git a/src/traust_core/v1/services/findings/review.py b/src/traust_core/v1/services/findings/review.py index 05f86923..a86bf3b6 100644 --- a/src/traust_core/v1/services/findings/review.py +++ b/src/traust_core/v1/services/findings/review.py @@ -2,11 +2,16 @@ from uuid import UUID -from traust_core.v1.models.attestation import ReviewItem from traust_core.v1.models.common import Page, PageRequest -from traust_core.v1.models.findings import Decision, Disposition, ReviewCase, ReviewFilter +from traust_core.v1.models.findings import ( + Decision, + Disposition, + ReviewCase, + ReviewFilter, + ReviewItem, +) from traust_core.v1.models.vocabulary import ReviewReason, TenancyProfile -from traust_core.v1.repositories.findings import FindingUnitOfWork +from traust_core.v1.repositories.contract import TraustUnitOfWork class ReviewPolicy: @@ -19,7 +24,7 @@ def needs_signoff( class FindingReview: - def __init__(self, uow: FindingUnitOfWork, policy: ReviewPolicy) -> None: + def __init__(self, uow: TraustUnitOfWork, policy: ReviewPolicy) -> None: self._uow = uow self._policy = policy @@ -29,5 +34,5 @@ def queue(self, filter: ReviewFilter, page: PageRequest) -> Page[ReviewCase]: def request_signoff(self, finding_id: UUID, reason: ReviewReason) -> ReviewItem: raise NotImplementedError("stub: replaces ledger needs_review payloads") - def apply_attestation(self, attestation_id: UUID) -> Disposition: + def resolve(self, review_item_id: UUID, decision_id: UUID) -> ReviewItem: raise NotImplementedError("stub: replaces ledger resolve_handler, cli countersign/resolve") diff --git a/src/traust_core/v1/services/findings/service.py b/src/traust_core/v1/services/findings/service.py index 598da952..20496bf6 100644 --- a/src/traust_core/v1/services/findings/service.py +++ b/src/traust_core/v1/services/findings/service.py @@ -14,7 +14,7 @@ FindingView, ) from traust_core.v1.models.vocabulary import Severity -from traust_core.v1.repositories.findings import FindingUnitOfWork +from traust_core.v1.repositories.contract import TraustUnitOfWork from traust_core.v1.services.findings.decisions import FindingDecisions from traust_core.v1.services.findings.intake import FindingIntake from traust_core.v1.services.findings.precedents import FindingPrecedents @@ -24,7 +24,7 @@ class FindingService: def __init__( self, - uow: FindingUnitOfWork, + uow: TraustUnitOfWork, intake: FindingIntake, decisions: FindingDecisions, review: FindingReview, diff --git a/src/traust_core/v1/services/findings/taxonomy.py b/src/traust_core/v1/services/findings/taxonomy.py index 7cf5319f..2ea7556a 100644 --- a/src/traust_core/v1/services/findings/taxonomy.py +++ b/src/traust_core/v1/services/findings/taxonomy.py @@ -2,11 +2,11 @@ from collections.abc import Sequence -from traust_core.v1.models.findings import Cwe, WeaknessLabels +from traust_core.v1.models.findings import CweId, WeaknessLabels class TaxonomyPolicy: - def labels(self, cwes: Sequence[Cwe]) -> WeaknessLabels: + def labels(self, cwes: Sequence[CweId]) -> WeaknessLabels: raise NotImplementedError( "stub: replaces asvs_for, classify_axis, infer_techniques, ATT&CK mapping" ) diff --git a/src/traust_core/v1/services/insight.py b/src/traust_core/v1/services/insight.py index a0450c6a..acd0a165 100644 --- a/src/traust_core/v1/services/insight.py +++ b/src/traust_core/v1/services/insight.py @@ -5,7 +5,7 @@ from traust_core.v1.models.common import Provenance from traust_core.v1.models.insight import ViewFilter, ViewModel, ViewName -from traust_core.v1.models.operations import Order +from traust_core.v1.models.operations import RunOrder from traust_core.v1.services.artifact_publishing import ArtifactPublisher, PublishResult from traust_core.v1.services.operations import OpsService @@ -30,7 +30,7 @@ def build(self, view: ViewName, filter: ViewFilter) -> ViewModel: def publish(self, view: ViewName, filter: ViewFilter, provenance: Provenance) -> PublishResult: raise NotImplementedError("stub: replaces dashboards' md/html/json writes") - def refresh(self, views: Sequence[ViewName] | None) -> list[Order]: + def refresh(self, views: Sequence[ViewName] | None) -> list[RunOrder]: raise NotImplementedError("stub: replaces cli/refresh_dashboards.py") diff --git a/src/traust_core/v1/services/ledger.py b/src/traust_core/v1/services/ledger.py index 5c6330ca..c452f5e5 100644 --- a/src/traust_core/v1/services/ledger.py +++ b/src/traust_core/v1/services/ledger.py @@ -2,14 +2,13 @@ from uuid import UUID -from traust_core.v1.models.attestation import Attestation from traust_core.v1.models.findings import Decision from traust_core.v1.models.ledger import AppendReceipt, Checkpoint, VerifyResult from traust_core.v1.providers.clock import Clock from traust_core.v1.providers.ledger import LedgerClient -# NEED(ledger-optional): the integrity chain is optional; decisions live in DecisionRepository +# NEED(ledger-optional): the integrity chain is optional; decisions live in the decision table # either way. When on, it always works: its tables sit in the configured record store # (SQLite locally, Postgres on a server). No file-backed ledger, no snapshot mode. class LedgerService: @@ -20,9 +19,6 @@ def __init__(self, client: LedgerClient, clock: Clock) -> None: def append_decision(self, decision: Decision, owner_id: UUID) -> AppendReceipt: raise NotImplementedError("stub: replaces emit_*_ledger_events append_to_layer") - def append_attestation(self, attestation: Attestation, owner_id: UUID) -> AppendReceipt: - raise NotImplementedError("stub: replaces ledger cli countersign event append") - def checkpoint(self, owner_id: UUID) -> Checkpoint: raise NotImplementedError("stub: replaces engine ledger/service checkpoint, layer_ref") diff --git a/src/traust_core/v1/services/operations.py b/src/traust_core/v1/services/operations.py index 4a1bfd7d..380411c7 100644 --- a/src/traust_core/v1/services/operations.py +++ b/src/traust_core/v1/services/operations.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Sequence +from datetime import timedelta from decimal import Decimal from typing import Protocol, TypeVar from uuid import UUID @@ -13,13 +14,11 @@ BudgetVerdict, JobResult, LaneName, - Lease, - Order, - OrderEvent, OrderFilter, OrderProgress, Refusal, Route, + RunOrder, SchedulePlan, SpendAttribution, SpendDimension, @@ -33,7 +32,7 @@ ) from traust_core.v1.providers.clock import Clock from traust_core.v1.providers.feeds import FeedProvider -from traust_core.v1.repositories.operations import OpsUnitOfWork, SpendUnitOfWork +from traust_core.v1.repositories.traust_ops import OpsUnitOfWork from traust_core.v1.services.provider_catalog import ProviderCatalog S = TypeVar("S", contravariant=True) @@ -55,42 +54,40 @@ def __init__(self, uow: OpsUnitOfWork, clock: Clock) -> None: self._uow = uow self._clock = clock - def submit(self, request: AssetRequest, principal: Principal) -> Order: + def submit(self, request: AssetRequest, principal: Principal) -> RunOrder: raise NotImplementedError("stub: replaces cron/subprocess orchestration, */ops.py facades") - def claim(self, worker: WorkerId) -> Lease | None: + def claim(self, worker: WorkerId, lease_for: timedelta) -> RunOrder | None: raise NotImplementedError("stub: order lease for the traust-engine run loop") - def heartbeat(self, lease: Lease) -> Lease: + def heartbeat(self, order_id: UUID, worker: WorkerId, lease_for: timedelta) -> RunOrder: raise NotImplementedError("stub: order lease renewal for the traust-engine run loop") - def save_progress(self, lease: Lease, progress: OrderProgress) -> Order: + def save_progress(self, order_id: UUID, progress: OrderProgress) -> UUID: + """Progress goes to the run manifest artifact; returns its artifact id.""" raise NotImplementedError("stub: replaces cli/checkpoint.py skill phase state") - def complete(self, lease: Lease, result: JobResult) -> Order: + def complete(self, order_id: UUID, result: JobResult) -> RunOrder: raise NotImplementedError("stub: replaces per-CLI exit codes and progress files") - def approve(self, order_id: UUID, attestation_id: UUID) -> Order: + def approve(self, order_id: UUID, decision_id: UUID) -> RunOrder: raise NotImplementedError("stub: replaces remediation push gates") - def schedule(self, plan: SchedulePlan) -> list[Order]: + def schedule(self, plan: SchedulePlan) -> list[RunOrder]: raise NotImplementedError("stub: replaces refresh_dashboards.build_stages") - def get(self, order_id: UUID) -> Order: + def get(self, order_id: UUID) -> RunOrder: raise NotImplementedError("stub: order read") - def list(self, filter: OrderFilter, page: PageRequest) -> Page[Order]: + def list(self, filter: OrderFilter, page: PageRequest) -> Page[RunOrder]: raise NotImplementedError("stub: order read") - def events(self, order_id: UUID) -> list[OrderEvent]: - raise NotImplementedError("stub: order event read") - def timeline(self, filter: OrderFilter, bucket: TimeBucket) -> list[ThroughputPoint]: raise NotImplementedError("stub: replaces metrics/history.py, collect_harness_metrics") class SpendService: - def __init__(self, uow: SpendUnitOfWork, budgets: Sequence[Budget]) -> None: + def __init__(self, uow: OpsUnitOfWork, budgets: Sequence[Budget]) -> None: self._uow = uow self._budgets = budgets diff --git a/src/traust_core/v1/services/portfolio.py b/src/traust_core/v1/services/portfolio.py index 44887f83..a031a02a 100644 --- a/src/traust_core/v1/services/portfolio.py +++ b/src/traust_core/v1/services/portfolio.py @@ -1,17 +1,14 @@ from __future__ import annotations -from collections.abc import Sequence +from datetime import datetime, timedelta from uuid import UUID from traust_core.v1.models.common import Page, PageRequest -from traust_core.v1.models.portfolio import ( - CodeLine, +from traust_core.v1.models.inventory import ( Component, ComponentFilter, - ComponentSpec, - Designation, + ExposureClass, ForgeHost, - ImageRef, Liveness, OrgName, Ownership, @@ -20,54 +17,58 @@ ProductSlug, RefName, Release, + ReleaseChange, Repository, RepositoryFilter, + RepositoryRef, RepoState, - Revision, - Segment, + RiskTier, SyncReport, VersionLabel, ) from traust_core.v1.models.values import GitSha, HttpsRepoUrl from traust_core.v1.models.vocabulary import RefKind from traust_core.v1.providers.sources import SourceProvider -from traust_core.v1.repositories.portfolio import PortfolioUnitOfWork +from traust_core.v1.repositories.traust.inventory import InventoryUnitOfWork from traust_core.v1.services.assessment import AssessmentService from traust_core.v1.services.findings.service import FindingService class PortfolioService: - def __init__(self, uow: PortfolioUnitOfWork) -> None: + def __init__(self, uow: InventoryUnitOfWork) -> None: self._uow = uow - def register_product(self, slug: ProductSlug, segment: Segment | None) -> Product: + def register_product(self, slug: ProductSlug, name: str) -> Product: raise NotImplementedError("stub: replaces registry/products.py, inventory.py") - def register_repository(self, url: HttpsRepoUrl) -> Repository: + def register_repository(self, url: HttpsRepoUrl, public: bool) -> Repository: raise NotImplementedError("stub: replaces corpus_intake.py, add-inputs") - def register_code_line(self, repository_id: UUID, ref: RefName, kind: RefKind) -> CodeLine: + def register_ref( + self, repository_id: UUID, sha: GitSha, name: RefName | None, kind: RefKind | None + ) -> RepositoryRef: raise NotImplementedError("stub: replaces product_repo (product, repo, ref) rows") - def register_revision(self, repository_id: UUID, sha: GitSha) -> Revision: - raise NotImplementedError("stub: replaces corpus/store_ingest repo@sha keys") + def record_release(self, product_id: UUID, version: VersionLabel) -> Release: + raise NotImplementedError("stub: replaces product_repo_version, build_release_events") - def add_component(self, product_id: UUID, code_line_id: UUID, spec: ComponentSpec) -> Component: + def add_component(self, release_id: UUID, repository_ref_id: UUID) -> Component: raise NotImplementedError("stub: replaces product_repo upserts") - def record_release( - self, component_id: UUID, version: VersionLabel, images: Sequence[ImageRef] - ) -> Release: - raise NotImplementedError("stub: replaces product_repo_version, build_release_events") + def release_changes(self, release_id: UUID, previous_id: UUID) -> ReleaseChange: + raise NotImplementedError("stub: replaces build_release_events.diff_container") - def assign_ownership(self, component_id: UUID, ownership: Ownership) -> Component: + def assign_owner(self, repository_id: UUID, team: str, at: datetime) -> Ownership: raise NotImplementedError("stub: replaces repo_owner, subject_ownership") - def set_designation(self, repository_id: UUID, designation: Designation) -> Repository: - raise NotImplementedError("stub: replaces build_rescan_worklist.load_designations") + def exposure_class(self, repository_id: UUID, liveness: Liveness) -> ExposureClass: + raise NotImplementedError("stub: replaces build_rescan_worklist.exposure_class") + + def is_dormant(self, liveness: Liveness, now: datetime, after: timedelta) -> bool: + raise NotImplementedError("stub: replaces check_repo_liveness.classify") - def record_liveness(self, repository_id: UUID, liveness: Liveness) -> Repository: - raise NotImplementedError("stub: replaces check_repo_liveness.py") + def risk_tier(self, repository_id: UUID, open_critical_high: int, now: datetime) -> RiskTier: + raise NotImplementedError("stub: replaces build_rescan_worklist.risk_tier") def get_repository(self, repository_id: UUID) -> Repository: raise NotImplementedError("stub: portfolio read") @@ -93,7 +94,7 @@ def __init__(self, portfolio: PortfolioService, sources: SourceProvider) -> None self._portfolio = portfolio self._sources = sources - def refresh(self, repository_id: UUID) -> Repository: + def liveness(self, repository_id: UUID) -> Liveness: raise NotImplementedError("stub: replaces check_repo_liveness.gh_repo_info") def import_org(self, org: OrgName, host: ForgeHost) -> SyncReport: diff --git a/src/traust_core/v1/services/pqc.py b/src/traust_core/v1/services/pqc.py index 71b37f9b..6ab752d8 100644 --- a/src/traust_core/v1/services/pqc.py +++ b/src/traust_core/v1/services/pqc.py @@ -13,19 +13,19 @@ PqcReadinessInput, PqcRollup, ) -from traust_core.v1.repositories.assessment import PqcUnitOfWork +from traust_core.v1.repositories.traust.assessments import AssessmentsUnitOfWork from traust_core.v1.services.code_facts import CodeFactsService class PqcService: - def __init__(self, uow: PqcUnitOfWork, code_facts: CodeFactsService) -> None: + def __init__(self, uow: AssessmentsUnitOfWork, code_facts: CodeFactsService) -> None: self._uow = uow self._code_facts = code_facts - def prescan(self, revision_id: UUID) -> PqcReadiness: + def prescan(self, repository_ref_id: UUID) -> PqcReadiness: raise NotImplementedError("stub: replaces bulk_prescan, l2_prepass, scan_xcrypto_usage") - def prepare(self, revision_id: UUID) -> PqcReadinessInput: + def prepare(self, repository_ref_id: UUID) -> PqcReadinessInput: raise NotImplementedError("stub: replaces pqc-readiness skill inputs") @overload diff --git a/src/traust_core/v1/services/remediation.py b/src/traust_core/v1/services/remediation.py index e6148b1a..0010e639 100644 --- a/src/traust_core/v1/services/remediation.py +++ b/src/traust_core/v1/services/remediation.py @@ -26,12 +26,12 @@ WorkItemFilter, ) from traust_core.v1.models.vocabulary import FixStrategy, RemediationStatus -from traust_core.v1.repositories.remediation import RemediationUnitOfWork +from traust_core.v1.repositories.contract import TraustUnitOfWork from traust_core.v1.services.findings.service import FindingService class RemediationService: - def __init__(self, uow: RemediationUnitOfWork, findings: FindingService) -> None: + def __init__(self, uow: TraustUnitOfWork, findings: FindingService) -> None: self._uow = uow self._findings = findings @@ -80,14 +80,14 @@ def list(self, filter: WorkItemFilter, page: PageRequest) -> Page[WorkItem]: class VerificationService: - def __init__(self, uow: RemediationUnitOfWork, findings: FindingService) -> None: + def __init__(self, uow: TraustUnitOfWork, findings: FindingService) -> None: self._uow = uow self._findings = findings def prepare(self, work_item_id: UUID) -> VerificationInput: raise NotImplementedError("stub: replaces verify-remediation/build_verify_sweep.py") - def differential(self, work_item_id: UUID, revision_id: UUID) -> EvidenceItem: + def differential(self, work_item_id: UUID, repository_ref_id: UUID) -> EvidenceItem: raise NotImplementedError("stub: replaces verify-remediation/scanner_differential.py") def propagation(self, work_item_id: UUID, component_id: UUID) -> Propagation: diff --git a/src/traust_core/v1/services/storage.py b/src/traust_core/v1/services/storage.py deleted file mode 100644 index 958bb287..00000000 --- a/src/traust_core/v1/services/storage.py +++ /dev/null @@ -1,105 +0,0 @@ -from __future__ import annotations - -from collections.abc import Sequence - -from traust_core.v1 import contracts -from traust_core.v1.errors import ConflictError, NotFoundError, ValidationError -from traust_core.v1.models.base import Dto -from traust_core.v1.models.integrity import sha256_hex -from traust_core.v1.models.storage import ( - ArtifactBinding, - BindingContext, - Evidence, - binding_id, - identifier_bytes, -) -from traust_core.v1.providers.clock import Clock -from traust_core.v1.repositories.storage import StorageUnitOfWork - - -class RecordResult(Dto): - digest: str - binding_id: str - already_bound: bool - - -def _check_context(name: str, context: BindingContext) -> None: - profile = contracts.storage_profile(name) - if not context.scope_id: - raise ValidationError("scope_id: required") - if missing := [f for f in profile.get("required", ()) if getattr(context, f) in (None, "")]: - raise ValidationError(f"artifact {name}: requires {', '.join(missing)}") - if context.role is not None and context.role not in profile.get("roles", ()): - raise ValidationError(f"artifact {name}: role {context.role!r} not allowed") - - -def _references(references: Sequence[str]) -> tuple[str, ...]: - for ref in references: - if not ref: - raise ValidationError("references: empty reference") - identifier_bytes("references", ref) - return tuple(dict.fromkeys(references)) - - -def bind_artifact( - uow: StorageUnitOfWork, - name: str, - payload: bytes, - clock: Clock, - context: BindingContext | None = None, - references: Sequence[str] = (), -) -> RecordResult: - context = context or BindingContext() - contracts.validate_json(name, payload) - _check_context(name, context) - refs = _references(references) - digest = sha256_hex(payload) - bid = binding_id(digest, name, context) - now = clock.now() - if uow.evidence.find_binding(bid) is not None: - uow.evidence.add_locations(bid, refs, now) - return RecordResult(digest=digest, binding_id=bid, already_bound=True) - if context.supersedes_binding_id is not None: - previous = uow.evidence.find_binding(context.supersedes_binding_id) - if previous is None or (previous.artifact_name, previous.context.scope_id) != ( - name, - context.scope_id, - ): - raise ConflictError( - f"artifact {name}: supersedes unknown or unrelated binding " - f"{context.supersedes_binding_id}" - ) - uow.evidence.add( - Evidence(digest=digest, byte_size=len(payload), first_ingested_at=now), - ArtifactBinding( - binding_id=bid, - artifact_digest=digest, - artifact_name=name, - context=context, - bound_at=now, - ), - ) - uow.evidence.add_locations(bid, refs, now) - return RecordResult(digest=digest, binding_id=bid, already_bound=False) - - -def record_artifact( - uow: StorageUnitOfWork, - name: str, - payload: bytes, - clock: Clock, - context: BindingContext | None = None, - references: Sequence[str] = (), -) -> RecordResult: - with uow: - result = bind_artifact(uow, name, payload, clock, context, references) - uow.commit() - return result - - -def get_binding(uow: StorageUnitOfWork, binding_id_value: str) -> ArtifactBinding: - with uow: - binding = uow.evidence.find_binding(binding_id_value) - if binding is None: - raise NotFoundError(f"binding {binding_id_value}") - return binding diff --git a/src/traust_core/v1/services/validation.py b/src/traust_core/v1/services/validation.py index 1ddc7ca9..6a60dc38 100644 --- a/src/traust_core/v1/services/validation.py +++ b/src/traust_core/v1/services/validation.py @@ -22,14 +22,14 @@ ) from traust_core.v1.models.vocabulary import ValidationTechnique from traust_core.v1.providers.validation import ValidationAdapter -from traust_core.v1.repositories.validation import ValidationUnitOfWork +from traust_core.v1.repositories.contract import TraustUnitOfWork from traust_core.v1.services.findings.decisions import FindingDecisions class ValidationService: def __init__( self, - uow: ValidationUnitOfWork, + uow: TraustUnitOfWork, decisions: FindingDecisions, adapters: Sequence[ValidationAdapter], ) -> None: diff --git a/tests/v1/conftest.py b/tests/v1/conftest.py index 22312ec0..5fcf016a 100644 --- a/tests/v1/conftest.py +++ b/tests/v1/conftest.py @@ -48,19 +48,6 @@ def database( eng.dispose() -@contextmanager -def storage_db(kind: str, tmp_path: Path) -> Generator[Engine]: - eng = _engine(kind, tmp_path) - apply_contract_ddl(eng, "storage") # type: ignore[arg-type] # parked: storage-retire - try: - yield eng - finally: - if kind == "postgres": - with eng.begin() as conn: - conn.exec_driver_sql("DROP SCHEMA traust_storage CASCADE") - eng.dispose() - - @contextmanager def contract_db(kind: str, tmp_path: Path) -> Generator[Engine]: """A fresh database bootstrapped from the whole contracts db (every namespace).""" diff --git a/tests/v1/fixtures.py b/tests/v1/fixtures.py new file mode 100644 index 00000000..2658de9c --- /dev/null +++ b/tests/v1/fixtures.py @@ -0,0 +1,29 @@ +import json + +TRIAGE = json.dumps( + { + "triage_completed": "2025-01-01", + "triage_context": { + "mode": "synthetic", + "environment": "synthetic test environment", + "scoring": "synthetic test scoring", + "noise_tolerance": "synthetic", + "votes_per_finding": 1, + "repo": "example.invalid/synthetic/repository", + "harness_version": "0.0.0", + "reemitted": "2025-01-02", + "reemit_method": "synthetic fixture generation", + }, + "summary": { + "input_count": 0, + "true_positives": 0, + "hardening": 0, + "false_positives": 0, + "undetermined": 0, + "duplicates": 0, + "needs_manual_test": 0, + "by_severity": {"critical": 0, "high": 0, "medium": 0, "low": 0, "informational": 0}, + }, + "findings": [], + } +).encode() diff --git a/tests/v1/models/test_artifacts.py b/tests/v1/models/test_artifacts.py index e0f59ee3..07ab5541 100644 --- a/tests/v1/models/test_artifacts.py +++ b/tests/v1/models/test_artifacts.py @@ -2,7 +2,7 @@ import pytest -from tests.v1.services.test_storage import TRIAGE +from tests.v1.fixtures import TRIAGE from traust_core.v1 import contracts from traust_core.v1.errors import DocumentError from traust_core.v1.models.artifacts import ANALYSIS_ARTIFACTS, Artifact, TriageArtifact diff --git a/tests/v1/repositories/tables/test_repositories.py b/tests/v1/repositories/tables/test_repositories.py index 1feb3616..423512f2 100644 --- a/tests/v1/repositories/tables/test_repositories.py +++ b/tests/v1/repositories/tables/test_repositories.py @@ -11,40 +11,29 @@ from tests.v1.conftest import SQL_BACKENDS, contract_db from traust_core.v1.errors import ConflictError, RepositoryError -from traust_core.v1.models.records import ( - AdvisoryPackageRecord, - AdvisoryRecord, - ArtifactRecord, - AssessmentRecord, - ComponentRecord, - CweRecord, - DecisionRecord, - DependencyRecord, - EventRecord, - FeedSnapshotRecord, - FindingCweRecord, - FindingRecord, - LayerRecord, - LocationRecord, - OccurrenceRecord, - OwnershipRecord, - PackageRecord, - ProductRecord, - ReleaseRecord, - RepositoryRecord, - RepositoryRefRecord, - ReviewItemRecord, - RunOrderRecord, - SbomRecord, - SpendEventRecord, - TeamRecord, +from traust_core.v1.models.assessment import Artifact, Assessment +from traust_core.v1.models.dependencies import Advisory, AdvisoryPackage, Dependency, Package, Sbom +from traust_core.v1.models.findings import ( + Cwe, + Decision, + Finding, + FindingCwe, + Location, + Occurrence, + ReviewItem, ) -from traust_core.v1.repositories.traust.assessments import SqlAssessmentsUnitOfWork -from traust_core.v1.repositories.traust.dependencies import SqlDependenciesUnitOfWork -from traust_core.v1.repositories.traust.findings import SqlFindingsUnitOfWork -from traust_core.v1.repositories.traust.inventory import SqlInventoryUnitOfWork -from traust_core.v1.repositories.traust_ledger import SqlLedgerUnitOfWork -from traust_core.v1.repositories.traust_ops import SqlOpsUnitOfWork +from traust_core.v1.models.inventory import ( + Component, + Ownership, + Product, + Release, + Repository, + RepositoryRef, + Team, +) +from traust_core.v1.models.ledger import Event, Layer +from traust_core.v1.models.operations import FeedSnapshot, RunOrder, SpendEvent +from traust_core.v1.repositories.contract import SqlTraustUnitOfWork T0 = datetime(2026, 10, 5, tzinfo=UTC) SHA = "a" * 40 @@ -58,27 +47,27 @@ def engine(request: pytest.FixtureRequest, tmp_path: Path) -> Iterator[Engine]: @dataclass class Scanned: - repository: RepositoryRecord - ref: RepositoryRefRecord - assessment: AssessmentRecord - finding: FindingRecord + repository: Repository + ref: RepositoryRef + assessment: Assessment + finding: Finding @pytest.fixture def scanned(engine: Engine) -> Scanned: - with SqlInventoryUnitOfWork(engine) as inv: - repo = inv.repositories.add(RepositoryRecord(url="https://github.com/o/r", public=True)) + with SqlTraustUnitOfWork(engine) as inv: + repo = inv.repositories.add(Repository(url="https://github.com/o/r", public=True)) assert repo.id is not None ref = inv.repository_refs.add( - RepositoryRefRecord( + RepositoryRef( repository_id=repo.id, name="main", kind="branch", is_default=True, sha=SHA ) ) inv.commit() assert ref.id is not None - with SqlAssessmentsUnitOfWork(engine) as a: + with SqlTraustUnitOfWork(engine) as a: assessment = a.assessments.add( - AssessmentRecord( + Assessment( repository_ref_id=ref.id, kind="audit", status="completed", @@ -88,11 +77,9 @@ def scanned(engine: Engine) -> Scanned: ) ) a.commit() - with SqlFindingsUnitOfWork(engine) as f: + with SqlTraustUnitOfWork(engine) as f: finding = f.findings.add( - FindingRecord( - repository_id=repo.id, fingerprint_algo="v3", fingerprint="fp1", title="SQLi" - ) + Finding(repository_id=repo.id, fingerprint_algo="v3", fingerprint="fp1", title="SQLi") ) f.commit() return Scanned(repo, ref, assessment, finding) @@ -101,19 +88,15 @@ def scanned(engine: Engine) -> Scanned: def test_inventory_round_trip_and_natural_keys(engine: Engine, scanned: Scanned) -> None: repo, ref = scanned.repository, scanned.ref assert repo.id is not None and ref.id is not None - with SqlInventoryUnitOfWork(engine) as inv: - product = inv.products.add(ProductRecord(slug="ocp", name="OpenShift")) + with SqlTraustUnitOfWork(engine) as inv: + product = inv.products.add(Product(slug="ocp", name="OpenShift")) assert product.id is not None - release = inv.releases.add(ReleaseRecord(product_id=product.id, version="4.22")) + release = inv.releases.add(Release(product_id=product.id, version="4.22")) assert release.id is not None - component = inv.components.add( - ComponentRecord(release_id=release.id, repository_ref_id=ref.id) - ) - team = inv.teams.add(TeamRecord(name="core")) + component = inv.components.add(Component(release_id=release.id, repository_ref_id=ref.id)) + team = inv.teams.add(Team(name="core")) assert team.id is not None - owned = inv.ownerships.add( - OwnershipRecord(repository_id=repo.id, team_id=team.id, valid_from=T0) - ) + owned = inv.ownerships.add(Ownership(repository_id=repo.id, team_id=team.id, valid_from=T0)) inv.commit() assert inv.products.find_by_slug("ocp") == product @@ -133,15 +116,13 @@ def test_closing_ownership_is_an_update_the_database_touches( ) -> None: repo = scanned.repository assert repo.id is not None - with SqlInventoryUnitOfWork(engine) as inv: - team = inv.teams.add(TeamRecord(name="core")) + with SqlTraustUnitOfWork(engine) as inv: + team = inv.teams.add(Team(name="core")) assert team.id is not None - owned = inv.ownerships.add( - OwnershipRecord(repository_id=repo.id, team_id=team.id, valid_from=T0) - ) + owned = inv.ownerships.add(Ownership(repository_id=repo.id, team_id=team.id, valid_from=T0)) inv.commit() sleep(0.01) - with SqlInventoryUnitOfWork(engine) as inv: + with SqlTraustUnitOfWork(engine) as inv: closed = inv.ownerships.update(owned.model_copy(update={"valid_to": T0 + timedelta(1)})) inv.commit() assert inv.ownerships.current(repo.id) == [] @@ -150,25 +131,25 @@ def test_closing_ownership_is_an_update_the_database_touches( def test_duplicate_natural_key_is_a_conflict(engine: Engine, scanned: Scanned) -> None: - with SqlInventoryUnitOfWork(engine) as inv, pytest.raises(ConflictError): - inv.repositories.add(RepositoryRecord(url=scanned.repository.url, public=False)) + with SqlTraustUnitOfWork(engine) as inv, pytest.raises(ConflictError): + inv.repositories.add(Repository(url=scanned.repository.url, public=False)) def test_dependencies_trace_a_package_to_the_refs_that_ship_it( engine: Engine, scanned: Scanned ) -> None: - with SqlDependenciesUnitOfWork(engine) as d: - sbom = d.sboms.add(SbomRecord(digest="sha256:sbom")) - package = d.packages.add(PackageRecord(ecosystem="go", name="golang.org/x/crypto")) + with SqlTraustUnitOfWork(engine) as d: + sbom = d.sboms.add(Sbom(digest="sha256:sbom")) + package = d.packages.add(Package(ecosystem="go", name="golang.org/x/crypto")) assert sbom.id is not None and package.id is not None dep = d.dependencies.add( - DependencyRecord(sbom_id=sbom.id, package_id=package.id, version="0.1.0", direct=True) + Dependency(sbom_id=sbom.id, package_id=package.id, version="0.1.0", direct=True) ) - advisory = d.advisories.add(AdvisoryRecord(identifier="CVE-2026-0001")) + advisory = d.advisories.add(Advisory(identifier="CVE-2026-0001")) assert advisory.id is not None - alias = d.advisories.add(AdvisoryRecord(identifier="GHSA-x", alias_of_id=advisory.id)) + alias = d.advisories.add(Advisory(identifier="GHSA-x", alias_of_id=advisory.id)) hit = d.advisory_packages.add( - AdvisoryPackageRecord(advisory_id=advisory.id, package_id=package.id) + AdvisoryPackage(advisory_id=advisory.id, package_id=package.id) ) d.commit() @@ -184,9 +165,9 @@ def test_findings_chain_and_decisions_are_idempotent_by_statement( ) -> None: finding, assessment = scanned.finding, scanned.assessment assert finding.id is not None and assessment.id is not None - with SqlAssessmentsUnitOfWork(engine) as a: + with SqlTraustUnitOfWork(engine) as a: occurrence = a.occurrences.add( - OccurrenceRecord( + Occurrence( finding_id=finding.id, assessment_id=assessment.id, severity="high", @@ -195,10 +176,10 @@ def test_findings_chain_and_decisions_are_idempotent_by_statement( ) assert occurrence.id is not None location = a.locations.add( - LocationRecord(occurrence_id=occurrence.id, path="a.go", line_start=3, line_end=4) + Location(occurrence_id=occurrence.id, path="a.go", line_start=3, line_end=4) ) artifact = a.artifacts.add( - ArtifactRecord(assessment_id=assessment.id, digest="sha256:r", kind="report", uri="f") + Artifact(assessment_id=assessment.id, digest="sha256:r", kind="report", uri="f") ) a.commit() assert a.occurrences.list_for_assessment(assessment.id) == [occurrence] @@ -207,12 +188,12 @@ def test_findings_chain_and_decisions_are_idempotent_by_statement( assert a.artifacts.find(assessment.id, "sha256:r") == artifact assert a.assessments.latest(scanned.ref.id, "audit") == assessment # type: ignore[arg-type] - with SqlFindingsUnitOfWork(engine) as f: - cwe = f.cwes.add(CweRecord(identifier="CWE-89", abstraction="base")) + with SqlTraustUnitOfWork(engine) as f: + cwe = f.cwes.add(Cwe(identifier="CWE-89", abstraction="base")) assert cwe.id is not None - f.finding_cwes.add(FindingCweRecord(finding_id=finding.id, cwe_id=cwe.id)) + f.finding_cwes.add(FindingCwe(finding_id=finding.id, cwe_id=cwe.id)) decision = f.decisions.add( - DecisionRecord( + Decision( finding_id=finding.id, assessment_id=assessment.id, statement_key="k1", @@ -221,7 +202,7 @@ def test_findings_chain_and_decisions_are_idempotent_by_statement( ) ) review = f.review_items.add( - ReviewItemRecord(finding_id=finding.id, decision_id=decision.id, status="pending") + ReviewItem(finding_id=finding.id, decision_id=decision.id, status="pending") ) f.commit() assert f.findings.find_by_fingerprint(finding.repository_id, "v3", "fp1") == finding @@ -234,7 +215,7 @@ def test_findings_chain_and_decisions_are_idempotent_by_statement( def test_append_only_rows_reject_mutation_in_the_database(engine: Engine, scanned: Scanned) -> None: - with SqlAssessmentsUnitOfWork(engine) as a: + with SqlTraustUnitOfWork(engine) as a: table = a.assessments.table with pytest.raises((RepositoryError, ConflictError)): a.assessments._execute("raw update", sa.update(table).values(status="failed")) @@ -243,8 +224,8 @@ def test_append_only_rows_reject_mutation_in_the_database(engine: Engine, scanne def test_run_orders_claim_in_priority_order_and_leases_expire( engine: Engine, scanned: Scanned ) -> None: - def order(priority: int) -> RunOrderRecord: - return RunOrderRecord( + def order(priority: int) -> RunOrder: + return RunOrder( repository_id=scanned.repository.id, lane="audit", state="queued", @@ -252,7 +233,7 @@ def order(priority: int) -> RunOrderRecord: requested_by="user:me", ) - with SqlOpsUnitOfWork(engine) as ops: + with SqlTraustUnitOfWork(engine) as ops: later = ops.run_orders.add(order(5)) sooner = ops.run_orders.add(order(1)) ops.commit() @@ -270,13 +251,13 @@ def order(priority: int) -> RunOrderRecord: def test_ops_spend_and_feed_snapshots(engine: Engine, scanned: Scanned) -> None: - with SqlAssessmentsUnitOfWork(engine) as a: - artifact = a.artifacts.add(ArtifactRecord(digest="sha256:osv", kind="feed", uri="f")) + with SqlTraustUnitOfWork(engine) as a: + artifact = a.artifacts.add(Artifact(digest="sha256:osv", kind="feed", uri="f")) a.commit() assert artifact.id is not None - with SqlOpsUnitOfWork(engine) as ops: + with SqlTraustUnitOfWork(engine) as ops: spend = ops.spend_events.add( - SpendEventRecord( + SpendEvent( provider="anthropic", model="m", tokens_in=10, @@ -286,10 +267,10 @@ def test_ops_spend_and_feed_snapshots(engine: Engine, scanned: Scanned) -> None: ) ) old = ops.feed_snapshots.add( - FeedSnapshotRecord(artifact_id=artifact.id, feed="osv", retrieved_at=T0) + FeedSnapshot(artifact_id=artifact.id, feed="osv", retrieved_at=T0) ) new = ops.feed_snapshots.add( - FeedSnapshotRecord(artifact_id=artifact.id, feed="osv", retrieved_at=T0 + timedelta(1)) + FeedSnapshot(artifact_id=artifact.id, feed="osv", retrieved_at=T0 + timedelta(1)) ) ops.commit() assert ops.spend_events.list_between(T0, T0 + timedelta(1)) == [spend] @@ -299,24 +280,52 @@ def test_ops_spend_and_feed_snapshots(engine: Engine, scanned: Scanned) -> None: def test_ledger_events_prove_one_decision_each(engine: Engine, scanned: Scanned) -> None: finding = scanned.finding assert finding.id is not None and scanned.repository.id is not None - with SqlFindingsUnitOfWork(engine) as f: + with SqlTraustUnitOfWork(engine) as f: decision = f.decisions.add( - DecisionRecord( - finding_id=finding.id, actor="user:me", statement_key="k", occurred_at=T0 - ) + Decision(finding_id=finding.id, actor="user:me", statement_key="k", occurred_at=T0) ) f.commit() assert decision.id is not None - with SqlLedgerUnitOfWork(engine) as led: + with SqlTraustUnitOfWork(engine) as led: layer = led.layers.add( - LayerRecord(repository_id=scanned.repository.id, merkle_signing_method="none") + Layer(repository_id=scanned.repository.id, merkle_signing_method="none") ) assert layer.id is not None assert led.events.last_seq(layer.id) is None event = led.events.add( - EventRecord(layer_id=layer.id, decision_id=decision.id, seq=0, event_payload=b"\x00") + Event(layer_id=layer.id, decision_id=decision.id, seq=0, event_payload=b"\x00") ) led.commit() assert led.events.find_by_decision(decision.id) == event assert led.events.last_seq(layer.id) == 0 assert led.layers.list_for_repository(scanned.repository.id) == [layer] + + +def test_one_unit_of_work_spans_sections_in_one_transaction(engine: Engine) -> None: + with SqlTraustUnitOfWork(engine) as uow: + repo = uow.repositories.add(Repository(url="https://github.com/o/tx", public=True)) + assert repo.id is not None + ref = uow.repository_refs.add(RepositoryRef(repository_id=repo.id, sha=SHA)) + assert ref.id is not None + assessment = uow.assessments.add( + Assessment( + repository_ref_id=ref.id, + kind="audit", + status="completed", + assessor="skill:audit", + assessor_version="1.0", + occurred_at=T0, + ) + ) + finding = uow.findings.add( + Finding(repository_id=repo.id, fingerprint_algo="v3", fingerprint="t", title="t") + ) + assert assessment.id is not None and finding.id is not None + uow.occurrences.add( + Occurrence(finding_id=finding.id, assessment_id=assessment.id, severity="low") + ) + uow.rollback() # e.g. a later decision failed: nothing from any section lands + + with SqlTraustUnitOfWork(engine) as uow: + assert uow.repositories.find_by_url("https://github.com/o/tx") is None + assert uow.findings.list_for_repository(repo.id) == [] diff --git a/tests/v1/repositories/test_analysis_results.py b/tests/v1/repositories/test_analysis_results.py index d3624290..734b8c33 100644 --- a/tests/v1/repositories/test_analysis_results.py +++ b/tests/v1/repositories/test_analysis_results.py @@ -3,7 +3,7 @@ import pytest -from tests.v1.services.test_storage import TRIAGE +from tests.v1.fixtures import TRIAGE from traust_core.v1.errors import ValidationError from traust_core.v1.models.analysis_results import ResultKind, Subject from traust_core.v1.models.artifacts import ARTIFACT_FOR_KIND diff --git a/tests/v1/repositories/test_triage.py b/tests/v1/repositories/test_triage.py deleted file mode 100644 index 6d0b97b8..00000000 --- a/tests/v1/repositories/test_triage.py +++ /dev/null @@ -1,108 +0,0 @@ -import json -from collections.abc import Iterator -from pathlib import Path - -import pytest -import sqlalchemy as sa - -from tests.v1.conftest import CLOCK, SQL_BACKENDS, storage_db -from tests.v1.services.test_storage import RUN, TRIAGE -from traust_core.v1.errors import ValidationError -from traust_core.v1.models.artifacts import TriageArtifact -from traust_core.v1.repositories import schema_drift -from traust_core.v1.repositories.triage import ( - InMemoryTriageUnitOfWork, - SqlTriageUnitOfWork, - TriageUnitOfWork, - triage_verdict, -) -from traust_core.v1.services.findings import record_triage - -pytestmark = pytest.mark.todo("storage-retire: traust_storage DDL and profiles left contracts") - -BACKENDS = ["memory", *SQL_BACKENDS] - - -def triage(*findings: dict) -> TriageArtifact: - doc = json.loads(TRIAGE) - doc["findings"] = list(findings) - return TriageArtifact.from_document(doc) - - -VOTES = {"true_positive": 3, "hardening": 0, "false_positive": 0, "cannot_verify": 0} -TP = { - "id": "f001", - "orig_id": "SCA-1", - "title": "SQL injection here", - "verdict": "true_positive", - "severity": "high", - "vote_breakdown": VOTES, -} -FP = { - "id": "f002", - "orig_id": "SCA-2", - "title": "Not reachable at all", - "verdict": "false_positive", -} - - -@pytest.fixture(params=BACKENDS) -def make_uow(request: pytest.FixtureRequest, tmp_path: Path) -> Iterator[object]: - if request.param == "memory": - uow = InMemoryTriageUnitOfWork() - yield lambda: uow - return - with storage_db(request.param, tmp_path) as engine: - yield lambda: SqlTriageUnitOfWork(engine) - - -def read(make_uow: object, query: str, arg: str) -> list: - uow: TriageUnitOfWork = make_uow() - with uow: - return getattr(uow.triage_verdicts, query)(arg) - - -def test_recording_a_triage_projects_one_verdict_per_finding(make_uow: object) -> None: - result = record_triage(make_uow(), triage(TP, FP), CLOCK, RUN) - rows = read(make_uow, "for_binding", result.binding_id) - assert [(v.finding_id, v.verdict, v.source_finding_id) for v in rows] == [ - ("f001", "true_positive", "SCA-1"), - ("f002", "false_positive", "SCA-2"), - ] - assert rows[0].vote_breakdown == VOTES - assert rows[1].vote_breakdown is None - - -def test_domain_questions(make_uow: object) -> None: - result = record_triage(make_uow(), triage(TP, FP), CLOCK, RUN) - assert [v.finding_id for v in read(make_uow, "true_positives", result.binding_id)] == ["f001"] - assert [v.verdict for v in read(make_uow, "for_source_finding", "SCA-2")] == ["false_positive"] - - -def test_recording_again_is_idempotent(make_uow: object) -> None: - first = record_triage(make_uow(), triage(TP), CLOCK, RUN) - again = record_triage(make_uow(), triage(TP), CLOCK, RUN) - assert again.already_bound and again.binding_id == first.binding_id - assert len(read(make_uow, "for_binding", first.binding_id)) == 1 - - -def test_binding_context_must_fit_the_storage_profile(make_uow: object) -> None: - from traust_core.v1.models.storage import BindingContext - - with pytest.raises(ValidationError): - record_triage(make_uow(), triage(TP), CLOCK, BindingContext()) - - -def test_table_matches_contracts_ddl(tmp_path: Path) -> None: - with storage_db("sqlite", tmp_path) as engine: - assert schema_drift(engine, triage_verdict.metadata) == [] - with engine.connect() as conn: - assert conn.execute(sa.text("SELECT count(*) FROM findings_summary")).scalar() == 0 - - -def test_missing_votes_are_sql_null_not_json_null(tmp_path: Path) -> None: - with storage_db("sqlite", tmp_path) as engine: - record_triage(SqlTriageUnitOfWork(engine), triage(FP), CLOCK, RUN) - with engine.connect() as conn: - votes = conn.execute(sa.text("SELECT vote_breakdown FROM triage_verdict")).scalar() - assert votes is None diff --git a/tests/v1/services/test_artifact_publishing.py b/tests/v1/services/test_artifact_publishing.py index cf16cc18..27f42b8f 100644 --- a/tests/v1/services/test_artifact_publishing.py +++ b/tests/v1/services/test_artifact_publishing.py @@ -1,23 +1,21 @@ import json from collections.abc import Iterator from pathlib import Path +from uuid import UUID import pytest -from tests.v1.conftest import CLOCK, apply_contract_ddl, storage_db +from tests.v1.conftest import apply_contract_db, contract_db from tests.v1.example.artifact_publishing import example_specs as default_specs -from tests.v1.services.test_storage import TRIAGE +from tests.v1.fixtures import TRIAGE from traust_core.v1.context import Config, Context from traust_core.v1.errors import ConfigError, DocumentError, RepositoryError from traust_core.v1.models.analysis_results import ResultKind, Subject from traust_core.v1.models.artifacts import TriageArtifact from traust_core.v1.repositories import InMemoryObjectStore from traust_core.v1.repositories.analysis_results import AnalysisResultsRepository -from traust_core.v1.repositories.storage import InMemoryStorageUnitOfWork, SqlStorageUnitOfWork -from traust_core.v1.services.artifact_publishing import ArtifactPublisher, StorageIndex -from traust_core.v1.services.storage import get_binding - -STORAGE_RETIRED = pytest.mark.todo("storage-retire: traust_storage DDL and profiles left contracts") +from traust_core.v1.repositories.contract import SqlTraustUnitOfWork +from traust_core.v1.services.artifact_publishing import ArtifactPublisher, SqlArtifactIndex SUBJECT = Subject(tree="findings", product="openshift", repo_dir="oc", base="oc") FINDING = { @@ -36,32 +34,20 @@ def triage_with(*findings: dict) -> bytes: return json.dumps(doc).encode() -@pytest.fixture( - params=[ - "files-only", - pytest.param("memory-index", marks=STORAGE_RETIRED), - pytest.param("sqlite-index", marks=STORAGE_RETIRED), - ] -) +@pytest.fixture(params=["files-only", "sqlite-index"]) def setup( request: pytest.FixtureRequest, tmp_path: Path ) -> Iterator[tuple[ArtifactPublisher, object]]: results = AnalysisResultsRepository(InMemoryObjectStore()) if request.param == "files-only": yield ArtifactPublisher(default_specs(), results), None - elif request.param == "memory-index": - uow = InMemoryStorageUnitOfWork() - yield ( - ArtifactPublisher(default_specs(), results, StorageIndex(lambda: uow, CLOCK)), - lambda: uow, - ) - else: - with storage_db("sqlite", tmp_path) as engine: + return + with contract_db("sqlite", tmp_path) as engine: - def make() -> SqlStorageUnitOfWork: - return SqlStorageUnitOfWork(engine) + def make() -> SqlTraustUnitOfWork: + return SqlTraustUnitOfWork(engine) - yield ArtifactPublisher(default_specs(), results, StorageIndex(make, CLOCK)), make + yield ArtifactPublisher(default_specs(), results, SqlArtifactIndex(make)), make def test_submit_writes_json_then_rendered_markdown(setup: tuple[ArtifactPublisher, object]) -> None: @@ -85,17 +71,19 @@ def test_markdown_is_rendered_from_the_validated_document() -> None: ) -def test_index_records_a_binding_that_points_at_the_json_file( +def test_index_records_an_artifact_row_that_points_at_the_json_file( setup: tuple[ArtifactPublisher, object], ) -> None: - service, storage = setup + service, uow = setup result = service.publish("triage", SUBJECT, triage_with(FINDING), run_id="r1") - if storage is None: - assert result.binding_id is None + if uow is None: + assert result.artifact_id is None return - binding = get_binding(storage(), result.binding_id) - assert binding.references == ("findings/openshift/oc/oc-triage.json",) - assert binding.context.subject_id == SUBJECT.key + assert result.artifact_id is not None + with uow() as u: # type: ignore[operator] + row = u.artifacts.get(result.artifact_id) + assert row.uri == "findings/openshift/oc/oc-triage.json" + assert row.kind == "triage" and row.assessment_id is None def test_invalid_input_writes_nothing_and_returns_every_issue() -> None: @@ -142,7 +130,7 @@ def test_reader_returns_the_named_artifact(tmp_path: Path) -> None: def test_index_failure_keeps_files_and_reports_degraded() -> None: class BrokenIndex: - def record(self, *_: object) -> str: + def record(self, *_: object) -> UUID: raise RepositoryError("database down") results = AnalysisResultsRepository(InMemoryObjectStore()) @@ -157,16 +145,15 @@ def test_resubmitting_is_idempotent(setup: tuple[ArtifactPublisher, object]) -> service, _ = setup first = service.publish("triage", SUBJECT, triage_with(FINDING), run_id="r1") again = service.publish("triage", SUBJECT, triage_with(FINDING), run_id="r1") - assert (again.written, again.binding_id) == (first.written, first.binding_id) + assert (again.written, again.artifact_id) == (first.written, first.artifact_id) -@STORAGE_RETIRED def test_context_builds_files_only_or_indexed_from_config(tmp_path: Path) -> None: files_only = Context(Config.model_validate({"artifacts": {"path": str(tmp_path / "ar")}})) assert ( files_only.artifact_publisher(default_specs()) .publish("triage", SUBJECT, TRIAGE, "r1") - .binding_id + .artifact_id is None ) @@ -177,9 +164,8 @@ def test_context_builds_files_only_or_indexed_from_config(tmp_path: Path) -> Non "database": {"url": f"sqlite:///{tmp_path}/db.sqlite"}, } ), - clock=CLOCK, ) - apply_contract_ddl(indexed.database, "storage") + apply_contract_db(indexed.database) published = indexed.artifact_publisher(default_specs()).publish("triage", SUBJECT, TRIAGE, "r1") - assert published.binding_id is not None + assert published.artifact_id is not None assert (tmp_path / "ar2/findings/openshift/oc/oc-triage.md").is_file() diff --git a/tests/v1/services/test_storage.py b/tests/v1/services/test_storage.py deleted file mode 100644 index f04ff88d..00000000 --- a/tests/v1/services/test_storage.py +++ /dev/null @@ -1,148 +0,0 @@ -import json -from collections.abc import Iterator -from pathlib import Path - -import pytest -from sqlalchemy.engine import Engine - -from tests.v1.conftest import CLOCK, SQL_BACKENDS, storage_db -from traust_core.v1.errors import ConflictError, DocumentError, NotFoundError, ValidationError -from traust_core.v1.models.storage import BindingContext, binding_id -from traust_core.v1.repositories import schema_drift -from traust_core.v1.repositories.storage import ( - InMemoryStorageUnitOfWork, - SqlStorageUnitOfWork, - StorageUnitOfWork, - metadata, -) -from traust_core.v1.services.storage import get_binding, record_artifact - -pytestmark = pytest.mark.todo("storage-retire: traust_storage DDL and profiles left contracts") - -DIGEST = "a" * 64 -RUN = BindingContext(subject_id="org/repo", run_id="r1") -TRIAGE = json.dumps( - { - "triage_completed": "2025-01-01", - "triage_context": { - "mode": "synthetic", - "environment": "synthetic test environment", - "scoring": "synthetic test scoring", - "noise_tolerance": "synthetic", - "votes_per_finding": 1, - "repo": "example.invalid/synthetic/repository", - "harness_version": "0.0.0", - "reemitted": "2025-01-02", - "reemit_method": "synthetic fixture generation", - }, - "summary": { - "input_count": 0, - "true_positives": 0, - "hardening": 0, - "false_positives": 0, - "undetermined": 0, - "duplicates": 0, - "needs_manual_test": 0, - "by_severity": {"critical": 0, "high": 0, "medium": 0, "low": 0, "informational": 0}, - }, - "findings": [], - } -).encode() - - -@pytest.fixture(params=SQL_BACKENDS) -def engine(request: pytest.FixtureRequest, tmp_path: Path) -> Iterator[Engine]: - with storage_db(request.param, tmp_path) as eng: - yield eng - - -@pytest.fixture(params=["memory", *SQL_BACKENDS]) -def uow(request: pytest.FixtureRequest, tmp_path: Path) -> Iterator[StorageUnitOfWork]: - if request.param == "memory": - yield InMemoryStorageUnitOfWork() - return - with storage_db(request.param, tmp_path) as eng: - yield SqlStorageUnitOfWork(eng) - - -@pytest.mark.parametrize( - ("name", "context", "expected"), - [ - ( - "triage", - BindingContext(), - "9949f898c09fbf4a2b823e7dc840a0882a222614a14df064e170c8dd4cabfe30", - ), - ("triage", RUN, "9786c5efd29f35bdffb4324d2cafc389867ab9ca356b6eb92f042e96c31d0ee6"), - ( - "report", - BindingContext(scope_id="team-a", subject_id="org/repo", run_id="r1", role="baseline"), - "1dd74cb4859ba1aa3070e65f2177a23f3a79b2aa9838a7b4138a1bee44eb42e8", - ), - ( - "layer", - BindingContext(layer_id="L1"), - "849e32b2f0e43a169733395a436fc954dca2afa20e3da63120c1b31a66ef16b7", - ), - ( - "report", - BindingContext(subject_id="", run_id="r1"), - "d0f816254bef13e04684f8739200eb0e623572fbbe70107b4e65dce000834b7d", - ), - ], -) -def test_binding_id_matches_traust_contracts_reference( - name: str, context: BindingContext, expected: str -) -> None: - assert binding_id(DIGEST, name, context) == expected - - -def test_binding_id_rejects_nul() -> None: - with pytest.raises(ValidationError): - binding_id(DIGEST, "triage", BindingContext(subject_id="a\x00b")) - - -def test_tables_match_contracts_ddl(engine: Engine) -> None: - assert schema_drift(engine, metadata) == [] - - -def test_record_new_artifact(uow: StorageUnitOfWork) -> None: - result = record_artifact(uow, "triage", TRIAGE, CLOCK, RUN, ["s3://bucket/triage.json"]) - assert not result.already_bound - binding = get_binding(uow, result.binding_id) - assert binding.artifact_digest == result.digest - assert binding.byte_size == len(TRIAGE) - assert binding.references == ("s3://bucket/triage.json",) - assert binding.context == RUN - assert binding.bound_at == CLOCK.now() - - -def test_recording_again_is_idempotent_and_adds_references(uow: StorageUnitOfWork) -> None: - first = record_artifact(uow, "triage", TRIAGE, CLOCK, RUN, ["a"]) - again = record_artifact(uow, "triage", TRIAGE, CLOCK, RUN, ["a", "b"]) - assert again.already_bound and again.binding_id == first.binding_id - assert get_binding(uow, first.binding_id).references == ("a", "b") - - -def test_invalid_document_is_rejected_before_writing(uow: StorageUnitOfWork) -> None: - with pytest.raises(DocumentError): - record_artifact(uow, "triage", b'{"findings": []}', CLOCK, RUN) - with pytest.raises(NotFoundError): - get_binding(uow, binding_id(DIGEST, "triage", RUN)) - - -@pytest.mark.parametrize( - "context", - [BindingContext(), BindingContext(subject_id="org/repo", run_id="r1", role="nope")], -) -def test_binding_context_must_fit_the_storage_profile( - uow: StorageUnitOfWork, context: BindingContext -) -> None: - with pytest.raises(ValidationError): - record_artifact(uow, "triage", TRIAGE, CLOCK, context) - - -def test_supersedes_must_name_a_known_binding(uow: StorageUnitOfWork) -> None: - context = RUN.model_copy(update={"supersedes_binding_id": "f" * 64}) - with pytest.raises(ConflictError): - record_artifact(uow, "triage", TRIAGE, CLOCK, context) diff --git a/tests/v1/test_stubs.py b/tests/v1/test_stubs.py index d59c30e0..5cd6e0bf 100644 --- a/tests/v1/test_stubs.py +++ b/tests/v1/test_stubs.py @@ -88,4 +88,5 @@ def test_stub_signature_is_fully_typed(stub: tuple[str, Any]) -> None: def test_every_domain_has_stubs() -> None: roles = {owner.split(".")[2] for owner, _ in STUBS} - assert {"models", "repositories", "services", "context"} <= roles + assert {"models", "services", "context"} <= roles + assert "repositories" not in roles # one real repository per contract table, no stubs From f04f618c96c12e28a0d2fd7c400617029953e581 Mon Sep 17 00:00:00 2001 From: Shaun Smith <51304449+smith-xyz@users.noreply.github.com> Date: Wed, 21 Oct 2026 09:30:11 -0400 Subject: [PATCH 04/10] refactor(repositories): object storage in its own package; test each table model against its columns --- src/traust_core/v1/context/context.py | 2 +- src/traust_core/v1/rendering/markdown.py | 2 +- src/traust_core/v1/repositories/__init__.py | 2 +- .../v1/repositories/objects/__init__.py | 28 +++++++++++++++++++ .../{ => objects}/analysis_results.py | 2 +- .../{object_store.py => objects/store.py} | 0 .../v1/services/artifact_publishing.py | 7 +++-- tests/v1/repositories/objects/__init__.py | 0 .../{ => objects}/test_analysis_results.py | 2 +- .../test_store.py} | 0 tests/v1/repositories/tables/test_factory.py | 22 +++++++++++++++ tests/v1/services/test_artifact_publishing.py | 2 +- 12 files changed, 61 insertions(+), 8 deletions(-) create mode 100644 src/traust_core/v1/repositories/objects/__init__.py rename src/traust_core/v1/repositories/{ => objects}/analysis_results.py (97%) rename src/traust_core/v1/repositories/{object_store.py => objects/store.py} (100%) create mode 100644 tests/v1/repositories/objects/__init__.py rename tests/v1/repositories/{ => objects}/test_analysis_results.py (97%) rename tests/v1/repositories/{test_object_store.py => objects/test_store.py} (100%) diff --git a/src/traust_core/v1/context/context.py b/src/traust_core/v1/context/context.py index 4bc40782..9bab856f 100644 --- a/src/traust_core/v1/context/context.py +++ b/src/traust_core/v1/context/context.py @@ -21,8 +21,8 @@ UnitOfWork, create_database_engine, ) -from traust_core.v1.repositories.analysis_results import AnalysisResultsRepository from traust_core.v1.repositories.contract import SqlTraustUnitOfWork +from traust_core.v1.repositories.objects.analysis_results import AnalysisResultsRepository from traust_core.v1.repositories.sql import Dialect from traust_core.v1.repositories.tables import TableFactory from traust_core.v1.services.artifact_publishing import ( diff --git a/src/traust_core/v1/rendering/markdown.py b/src/traust_core/v1/rendering/markdown.py index 248a2103..265d67e9 100644 --- a/src/traust_core/v1/rendering/markdown.py +++ b/src/traust_core/v1/rendering/markdown.py @@ -5,7 +5,7 @@ from typing import Protocol from traust_core.v1.models.base import Model -from traust_core.v1.repositories.object_store import ObjectKey, ObjectRef, ObjectStore +from traust_core.v1.repositories.objects.store import ObjectKey, ObjectRef, ObjectStore class Table(Model): diff --git a/src/traust_core/v1/repositories/__init__.py b/src/traust_core/v1/repositories/__init__.py index fcc1cf33..bf354a80 100644 --- a/src/traust_core/v1/repositories/__init__.py +++ b/src/traust_core/v1/repositories/__init__.py @@ -1,5 +1,5 @@ from traust_core.v1.repositories.memory import InMemoryUnitOfWork -from traust_core.v1.repositories.object_store import ( +from traust_core.v1.repositories.objects.store import ( InMemoryObjectStore, LocalObjectStore, ObjectKey, diff --git a/src/traust_core/v1/repositories/objects/__init__.py b/src/traust_core/v1/repositories/objects/__init__.py new file mode 100644 index 00000000..09906893 --- /dev/null +++ b/src/traust_core/v1/repositories/objects/__init__.py @@ -0,0 +1,28 @@ +"""Object storage: document bytes by key (local disk, S3-compatible), never database rows. + +Rows that point at these bytes live in the artifact table (uri + digest). +""" + +from traust_core.v1.repositories.objects.analysis_results import ( + AnalysisResultsRepository, + ResultsLayout, + artifact_kind, +) +from traust_core.v1.repositories.objects.store import ( + InMemoryObjectStore, + LocalObjectStore, + ObjectKey, + ObjectRef, + ObjectStore, +) + +__all__ = [ + "AnalysisResultsRepository", + "InMemoryObjectStore", + "LocalObjectStore", + "ObjectKey", + "ObjectRef", + "ObjectStore", + "ResultsLayout", + "artifact_kind", +] diff --git a/src/traust_core/v1/repositories/analysis_results.py b/src/traust_core/v1/repositories/objects/analysis_results.py similarity index 97% rename from src/traust_core/v1/repositories/analysis_results.py rename to src/traust_core/v1/repositories/objects/analysis_results.py index cb1486f8..389bdf6c 100644 --- a/src/traust_core/v1/repositories/analysis_results.py +++ b/src/traust_core/v1/repositories/objects/analysis_results.py @@ -6,7 +6,7 @@ from traust_core.v1.errors import ConfigError from traust_core.v1.models.analysis_results import ResultKind, Subject from traust_core.v1.models.artifacts import Artifact -from traust_core.v1.repositories.object_store import ObjectKey, ObjectRef, ObjectStore +from traust_core.v1.repositories.objects.store import ObjectKey, ObjectRef, ObjectStore A = TypeVar("A", bound=Artifact) diff --git a/src/traust_core/v1/repositories/object_store.py b/src/traust_core/v1/repositories/objects/store.py similarity index 100% rename from src/traust_core/v1/repositories/object_store.py rename to src/traust_core/v1/repositories/objects/store.py diff --git a/src/traust_core/v1/services/artifact_publishing.py b/src/traust_core/v1/services/artifact_publishing.py index 07c0df65..8633783f 100644 --- a/src/traust_core/v1/services/artifact_publishing.py +++ b/src/traust_core/v1/services/artifact_publishing.py @@ -10,8 +10,11 @@ from traust_core.v1.models.artifacts import Artifact from traust_core.v1.models.assessment import Artifact as ArtifactRow from traust_core.v1.models.base import Dto -from traust_core.v1.repositories.analysis_results import AnalysisResultsRepository, artifact_kind -from traust_core.v1.repositories.object_store import ObjectRef +from traust_core.v1.repositories.objects.analysis_results import ( + AnalysisResultsRepository, + artifact_kind, +) +from traust_core.v1.repositories.objects.store import ObjectRef from traust_core.v1.repositories.traust.assessments import AssessmentsUnitOfWork A = TypeVar("A", bound=Artifact) diff --git a/tests/v1/repositories/objects/__init__.py b/tests/v1/repositories/objects/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/v1/repositories/test_analysis_results.py b/tests/v1/repositories/objects/test_analysis_results.py similarity index 97% rename from tests/v1/repositories/test_analysis_results.py rename to tests/v1/repositories/objects/test_analysis_results.py index 734b8c33..98ddadfe 100644 --- a/tests/v1/repositories/test_analysis_results.py +++ b/tests/v1/repositories/objects/test_analysis_results.py @@ -8,7 +8,7 @@ from traust_core.v1.models.analysis_results import ResultKind, Subject from traust_core.v1.models.artifacts import ARTIFACT_FOR_KIND from traust_core.v1.repositories import InMemoryObjectStore, LocalObjectStore, ObjectKey -from traust_core.v1.repositories.analysis_results import ( +from traust_core.v1.repositories.objects.analysis_results import ( AnalysisResultsRepository, ResultsLayout, ResultsRepository, diff --git a/tests/v1/repositories/test_object_store.py b/tests/v1/repositories/objects/test_store.py similarity index 100% rename from tests/v1/repositories/test_object_store.py rename to tests/v1/repositories/objects/test_store.py diff --git a/tests/v1/repositories/tables/test_factory.py b/tests/v1/repositories/tables/test_factory.py index e8309b67..4fac1c66 100644 --- a/tests/v1/repositories/tables/test_factory.py +++ b/tests/v1/repositories/tables/test_factory.py @@ -11,6 +11,7 @@ TRAUST, TRAUST_LEDGER, TRAUST_OPS, + SqlTableRepository, TableFactory, ) from traust_core.v1.repositories.traust.inventory import REPOSITORY @@ -81,3 +82,24 @@ def _is_view(stem: str, dialect: Dialect) -> bool: def test_declared_tables_match_the_contracts_ddl(kind: str, tmp_path: Path) -> None: with contract_db(kind, tmp_path) as engine: assert contract_drift(engine) == [] + + +def _table_repositories() -> list[type[SqlTableRepository]]: + pending, found = list(SqlTableRepository.__subclasses__()), [] + while pending: + cls = pending.pop() + pending += cls.__subclasses__() + if "spec" in vars(cls): + found.append(cls) + return found + + +def test_each_table_has_one_repository_and_its_model_matches_the_columns() -> None: + import traust_core.v1.repositories.contract # noqa: F401 (registers every repository) + + repos = _table_repositories() + assert sorted(r.spec.name for r in repos) == sorted(s.name for s in CONTRACT_TABLES) + tables = contract_tables(Dialect.SQLITE) + for repo in repos: + columns = set(tables.table(repo.spec).c.keys()) + assert set(repo.record.model_fields) == columns, repo.spec.name diff --git a/tests/v1/services/test_artifact_publishing.py b/tests/v1/services/test_artifact_publishing.py index 27f42b8f..479b087b 100644 --- a/tests/v1/services/test_artifact_publishing.py +++ b/tests/v1/services/test_artifact_publishing.py @@ -13,8 +13,8 @@ from traust_core.v1.models.analysis_results import ResultKind, Subject from traust_core.v1.models.artifacts import TriageArtifact from traust_core.v1.repositories import InMemoryObjectStore -from traust_core.v1.repositories.analysis_results import AnalysisResultsRepository from traust_core.v1.repositories.contract import SqlTraustUnitOfWork +from traust_core.v1.repositories.objects.analysis_results import AnalysisResultsRepository from traust_core.v1.services.artifact_publishing import ArtifactPublisher, SqlArtifactIndex SUBJECT = Subject(tree="findings", product="openshift", repo_dir="oc", base="oc") From 34b4cbe73f007f89b723d6512372ca7c06ddccf5 Mon Sep 17 00:00:00 2001 From: Shaun Smith <51304449+smith-xyz@users.noreply.github.com> Date: Wed, 21 Oct 2026 09:40:19 -0400 Subject: [PATCH 05/10] test(repositories): table contract, constraint, per-section and unit-of-work suites on SQLite and Postgres; template databases per test --- Makefile | 6 +- src/traust_core/v1/repositories/tables.py | 6 +- .../v1/repositories/traust_ops/run_order.py | 2 +- tests/v1/conftest.py | 78 +++- tests/v1/repositories/tables/conftest.py | 437 ++++++++++++++++++ .../repositories/tables/test_assessments.py | 68 +++ .../repositories/tables/test_constraints.py | 136 ++++++ .../repositories/tables/test_dependencies.py | 62 +++ tests/v1/repositories/tables/test_findings.py | 116 +++++ .../v1/repositories/tables/test_inventory.py | 89 ++++ tests/v1/repositories/tables/test_ledger.py | 54 +++ tests/v1/repositories/tables/test_ops.py | 112 +++++ .../repositories/tables/test_repositories.py | 331 ------------- .../tables/test_table_contract.py | 106 +++++ .../repositories/tables/test_unit_of_work.py | 95 ++++ 15 files changed, 1355 insertions(+), 343 deletions(-) create mode 100644 tests/v1/repositories/tables/conftest.py create mode 100644 tests/v1/repositories/tables/test_assessments.py create mode 100644 tests/v1/repositories/tables/test_constraints.py create mode 100644 tests/v1/repositories/tables/test_dependencies.py create mode 100644 tests/v1/repositories/tables/test_findings.py create mode 100644 tests/v1/repositories/tables/test_inventory.py create mode 100644 tests/v1/repositories/tables/test_ledger.py create mode 100644 tests/v1/repositories/tables/test_ops.py delete mode 100644 tests/v1/repositories/tables/test_repositories.py create mode 100644 tests/v1/repositories/tables/test_table_contract.py create mode 100644 tests/v1/repositories/tables/test_unit_of_work.py diff --git a/Makefile b/Makefile index 770493f9..e4946793 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ export TRAUST_TEST_DATABASE_URL PYTHON ?= python3 -.PHONY: help setup sync hooks lint lint-fix test test-integration coverage coverage-html coverage-all db-up db-down +.PHONY: help setup sync hooks lint lint-fix test test-integration test-repositories coverage coverage-html coverage-all db-up db-down help: @echo "Targets ($(notdir $(CURDIR))):" @@ -19,6 +19,7 @@ help: @echo " make lint-fix — ruff --fix + format" @echo " make test — unit tests (no integration)" @echo " make test-integration — integration tests against Postgres (make db-up first)" + @echo " make test-repositories — repository layer on SQLite and Postgres (make db-up first)" @echo " make coverage — unit tests with coverage (term), fails under 80%" @echo " make coverage-html — unit tests with coverage (term + htmlcov/)" @echo " make coverage-all — unit + integration with coverage (needs make db-up)" @@ -48,6 +49,9 @@ test: test-integration: uv run --extra postgres pytest tests/ -q -m integration +test-repositories: + uv run --extra postgres pytest tests/v1/repositories/tables -q -m "integration or not integration" + coverage: uv run pytest tests/ --cov=traust_core --cov-report=term-missing -q -m "not integration" diff --git a/src/traust_core/v1/repositories/tables.py b/src/traust_core/v1/repositories/tables.py index 42303f08..63f565b6 100644 --- a/src/traust_core/v1/repositories/tables.py +++ b/src/traust_core/v1/repositories/tables.py @@ -153,7 +153,11 @@ def _many( *where: sa.ColumnElement[bool], order_by: Sequence[sa.ColumnElement[Any]] = (), ) -> list[R]: - stmt = self._select().where(*where).order_by(*order_by, self.table.c.created_at) + stmt = ( + self._select() + .where(*where) + .order_by(*order_by, self.table.c.created_at, self.table.c.id) + ) return [self._to_record(r) for r in self._execute(operation, stmt).mappings()] diff --git a/src/traust_core/v1/repositories/traust_ops/run_order.py b/src/traust_core/v1/repositories/traust_ops/run_order.py index 6773f136..1de260fd 100644 --- a/src/traust_core/v1/repositories/traust_ops/run_order.py +++ b/src/traust_core/v1/repositories/traust_ops/run_order.py @@ -63,7 +63,7 @@ def claim(self, worker: str, now: datetime, lease_for: timedelta) -> RunOrder | next_id = ( sa.select(c.id) .where(c.state == QUEUED) - .order_by(c.priority, c.created_at) + .order_by(c.priority, c.created_at, c.id) .limit(1) .with_for_update(skip_locked=True) .scalar_subquery() diff --git a/tests/v1/conftest.py b/tests/v1/conftest.py index 5fcf016a..c3cd5ea4 100644 --- a/tests/v1/conftest.py +++ b/tests/v1/conftest.py @@ -1,10 +1,14 @@ from __future__ import annotations import os +import shutil +import tempfile from collections.abc import Generator, Iterator from contextlib import contextmanager from datetime import UTC, datetime +from functools import cache from pathlib import Path +from uuid import uuid4 import pytest import sqlalchemy as sa @@ -50,23 +54,79 @@ def database( @contextmanager def contract_db(kind: str, tmp_path: Path) -> Generator[Engine]: - """A fresh database bootstrapped from the whole contracts db (every namespace).""" - eng = _engine(kind, tmp_path) + """A fresh database holding the whole contracts db (every namespace). + + The DDL runs once per session into a template; each test gets a copy - a file + copy on SQLite, CREATE DATABASE ... TEMPLATE on Postgres - so tests stay + isolated without re-bootstrapping 31 tables and their triggers every time. + """ if kind == "postgres": - _drop_namespaces(eng) + with _postgres_copy() as eng: + yield eng + return + target = tmp_path / "contract.db" + shutil.copyfile(_sqlite_template(), target) + eng = create_database_engine(f"sqlite:///{target}") + try: + yield eng + finally: + eng.dispose() + + +@cache +def _sqlite_template() -> Path: + path = Path(tempfile.mkdtemp(prefix="traust-contract-")) / "template.db" + eng = create_database_engine(f"sqlite:///{path}") apply_contract_db(eng) + eng.dispose() + return path + + +_PG_TEMPLATE = f"traust_contract_tmpl_{os.getpid()}" + + +def _pg_admin() -> Engine: + if not PG_URL: + pytest.skip("TRAUST_TEST_DATABASE_URL not set") + return sa.create_engine(PG_URL, isolation_level="AUTOCOMMIT") + + +@cache +def _postgres_template() -> str: + admin = _pg_admin() + with admin.connect() as conn: + conn.exec_driver_sql(f"DROP DATABASE IF EXISTS {_PG_TEMPLATE} WITH (FORCE)") + conn.exec_driver_sql(f"CREATE DATABASE {_PG_TEMPLATE}") + admin.dispose() + eng = create_database_engine(sa.make_url(PG_URL).set(database=_PG_TEMPLATE)) + apply_contract_db(eng) + eng.dispose() # a template database must have no open connections to be copied + return _PG_TEMPLATE + + +@contextmanager +def _postgres_copy() -> Generator[Engine]: + template = _postgres_template() + name = f"traust_contract_{uuid4().hex[:12]}" + admin = _pg_admin() + with admin.connect() as conn: + conn.exec_driver_sql(f"CREATE DATABASE {name} TEMPLATE {template}") + eng = create_database_engine(sa.make_url(PG_URL).set(database=name)) try: yield eng finally: - if kind == "postgres": - _drop_namespaces(eng) eng.dispose() + with admin.connect() as conn: + conn.exec_driver_sql(f"DROP DATABASE IF EXISTS {name} WITH (FORCE)") + admin.dispose() -def _drop_namespaces(engine: Engine) -> None: - with engine.begin() as conn: - for area in reversed(contracts.AREAS): - conn.exec_driver_sql(f"DROP SCHEMA IF EXISTS {area} CASCADE") +def pytest_sessionfinish(session: pytest.Session) -> None: + if PG_URL and _postgres_template.cache_info().currsize: + admin = _pg_admin() + with admin.connect() as conn: + conn.exec_driver_sql(f"DROP DATABASE IF EXISTS {_PG_TEMPLATE} WITH (FORCE)") + admin.dispose() def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: diff --git a/tests/v1/repositories/tables/conftest.py b/tests/v1/repositories/tables/conftest.py new file mode 100644 index 00000000..e9857203 --- /dev/null +++ b/tests/v1/repositories/tables/conftest.py @@ -0,0 +1,437 @@ +"""Repository-layer harness: a fresh contracts database per test, on SQLite and Postgres. + +Every test that takes `engine`, `uow` or `seed` runs once per backend; Postgres +runs when TRAUST_TEST_DATABASE_URL is set (make db-up; make test-integration). +Seed builds valid rows with their parents, so a test states only what it is +about: `seed.decision(seed.finding(), severity="high")`. +""" + +from __future__ import annotations + +from collections.abc import Iterator, Sequence +from datetime import UTC, datetime +from decimal import Decimal +from itertools import count +from pathlib import Path +from time import sleep +from typing import Any, TypeVar + +import pytest +from sqlalchemy.engine import Engine + +from tests.v1.conftest import SQL_BACKENDS, contract_db +from traust_core.v1.models.assessment import Artifact, Assessment +from traust_core.v1.models.base import Model +from traust_core.v1.models.dependencies import ( + Advisory, + AdvisoryPackage, + Dependency, + Image, + Package, + Sbom, +) +from traust_core.v1.models.findings import ( + Cwe, + Decision, + Finding, + FindingCwe, + FindingExternalRef, + FindingLabel, + Location, + Occurrence, + ReviewItem, +) +from traust_core.v1.models.inventory import ( + Component, + Ownership, + Product, + Release, + Repository, + RepositoryRef, + Team, +) +from traust_core.v1.models.ledger import Event, Layer +from traust_core.v1.models.operations import ( + AssessmentInput, + FeedSnapshot, + RunBatch, + RunOrder, + SpendEvent, +) +from traust_core.v1.repositories.contract import SqlTraustUnitOfWork +from traust_core.v1.repositories.tables import SqlTableRepository + +T0 = datetime(2026, 10, 5, tzinfo=UTC) +SHA = "a" * 40 +M = TypeVar("M", bound=Model) + + +@pytest.fixture(params=SQL_BACKENDS) +def engine(request: pytest.FixtureRequest, tmp_path: Path) -> Iterator[Engine]: + with contract_db(request.param, tmp_path) as eng: + yield eng + + +@pytest.fixture +def uow(engine: Engine) -> Iterator[SqlTraustUnitOfWork]: + """An open unit of work; uncommitted work rolls back when the test ends.""" + with SqlTraustUnitOfWork(engine) as u: + yield u + + +@pytest.fixture +def seed(uow: SqlTraustUnitOfWork) -> Seed: + return Seed(uow) + + +def repository_for(uow: SqlTraustUnitOfWork, table: str) -> SqlTableRepository[Any]: + for value in vars(uow).values(): + if isinstance(value, SqlTableRepository) and value.spec.name == table: + return value + raise KeyError(table) + + +def same_rows(actual: Sequence[Model], expected: Sequence[Model]) -> bool: + """Equal as sets of rows. Rows written in one Postgres transaction share created_at + (now() is the transaction's start), so list order is only asserted where a + query defines it (occurred_at, seq, priority).""" + key = lambda m: str(m.id) # type: ignore[attr-defined] # noqa: E731 + return sorted(actual, key=key) == sorted(expected, key=key) + + +def apart() -> None: + """Separate created_at: SQLite stamps milliseconds per statement.""" + sleep(0.002) + + +def saved_id(model: Model) -> Any: + assert getattr(model, "id", None) is not None, f"{type(model).__name__} is not saved" + return model.id # type: ignore[attr-defined] + + +class Seed: + """Valid rows with their parents: new_() is unsaved,
() is added.""" + + def __init__(self, uow: SqlTraustUnitOfWork) -> None: + self.uow = uow + self._n = count(1) + + def _next(self) -> int: + return next(self._n) + + def _add(self, table: str, model: M) -> M: + return repository_for(self.uow, table).add(model) # type: ignore[no-any-return] + + def new(self, table: str) -> Model: + """An unsaved, valid row for any contract table (parents saved).""" + return getattr(self, f"new_{table}")() # type: ignore[no-any-return] + + # 1 · Inventory + def new_product(self, **kw: Any) -> Product: + n = self._next() + return Product(**{"slug": f"product-{n}", "name": f"Product {n}", **kw}) + + def product(self, **kw: Any) -> Product: + return self._add("product", self.new_product(**kw)) + + def new_release(self, product: Product | None = None, **kw: Any) -> Release: + parent = product or self.product() + return Release(**{"product_id": saved_id(parent), "version": f"{self._next()}.0", **kw}) + + def release(self, product: Product | None = None, **kw: Any) -> Release: + return self._add("release", self.new_release(product, **kw)) + + def new_repository(self, **kw: Any) -> Repository: + return Repository( + **{"url": f"https://github.com/org/repo-{self._next()}", "public": True, **kw} + ) + + def repository(self, **kw: Any) -> Repository: + return self._add("repository", self.new_repository(**kw)) + + def new_repository_ref(self, repository: Repository | None = None, **kw: Any) -> RepositoryRef: + parent = repository or self.repository() + values = {"repository_id": saved_id(parent), "name": "main", "kind": "branch", "sha": SHA} + return RepositoryRef(**{**values, **kw}) + + def repository_ref(self, repository: Repository | None = None, **kw: Any) -> RepositoryRef: + return self._add("repository_ref", self.new_repository_ref(repository, **kw)) + + def new_component( + self, release: Release | None = None, ref: RepositoryRef | None = None + ) -> Component: + return Component( + release_id=saved_id(release or self.release()), + repository_ref_id=saved_id(ref or self.repository_ref()), + ) + + def component( + self, release: Release | None = None, ref: RepositoryRef | None = None + ) -> Component: + return self._add("component", self.new_component(release, ref)) + + def new_team(self, **kw: Any) -> Team: + return Team(**{"name": f"team-{self._next()}", **kw}) + + def team(self, **kw: Any) -> Team: + return self._add("team", self.new_team(**kw)) + + def new_ownership( + self, repository: Repository | None = None, team: Team | None = None, **kw: Any + ) -> Ownership: + values = { + "repository_id": saved_id(repository or self.repository()), + "team_id": saved_id(team or self.team()), + "valid_from": T0, + } + return Ownership(**{**values, **kw}) + + def ownership( + self, repository: Repository | None = None, team: Team | None = None, **kw: Any + ) -> Ownership: + return self._add("ownership", self.new_ownership(repository, team, **kw)) + + # 2 · Dependencies + def new_sbom(self, **kw: Any) -> Sbom: + return Sbom(**{"digest": f"sha256:sbom-{self._next()}", **kw}) + + def sbom(self, **kw: Any) -> Sbom: + return self._add("sbom", self.new_sbom(**kw)) + + def new_package(self, **kw: Any) -> Package: + return Package(**{"ecosystem": "go", "name": f"example.com/pkg-{self._next()}", **kw}) + + def package(self, **kw: Any) -> Package: + return self._add("package", self.new_package(**kw)) + + def new_dependency( + self, sbom: Sbom | None = None, package: Package | None = None, **kw: Any + ) -> Dependency: + values = { + "sbom_id": saved_id(sbom or self.sbom()), + "package_id": saved_id(package or self.package()), + "version": "1.0.0", + "direct": True, + } + return Dependency(**{**values, **kw}) + + def dependency( + self, sbom: Sbom | None = None, package: Package | None = None, **kw: Any + ) -> Dependency: + return self._add("dependency", self.new_dependency(sbom, package, **kw)) + + def new_image(self, **kw: Any) -> Image: + return Image(**{"digest": f"sha256:image-{self._next()}", "platform": "linux/amd64", **kw}) + + def image(self, **kw: Any) -> Image: + return self._add("image", self.new_image(**kw)) + + def new_advisory(self, **kw: Any) -> Advisory: + return Advisory(**{"identifier": f"CVE-2026-{self._next():04d}", **kw}) + + def advisory(self, **kw: Any) -> Advisory: + return self._add("advisory", self.new_advisory(**kw)) + + def new_advisory_package( + self, advisory: Advisory | None = None, package: Package | None = None + ) -> AdvisoryPackage: + return AdvisoryPackage( + advisory_id=saved_id(advisory or self.advisory()), + package_id=saved_id(package or self.package()), + ) + + def advisory_package( + self, advisory: Advisory | None = None, package: Package | None = None + ) -> AdvisoryPackage: + return self._add("advisory_package", self.new_advisory_package(advisory, package)) + + # 3a · Assessments + def new_assessment(self, ref: RepositoryRef | None = None, **kw: Any) -> Assessment: + values = { + "repository_ref_id": saved_id(ref or self.repository_ref()), + "kind": "audit", + "status": "completed", + "assessor": "skill:audit", + "assessor_version": "1.0", + "occurred_at": T0, + } + return Assessment(**{**values, **kw}) + + def assessment(self, ref: RepositoryRef | None = None, **kw: Any) -> Assessment: + return self._add("assessment", self.new_assessment(ref, **kw)) + + def new_artifact(self, assessment: Assessment | None = None, **kw: Any) -> Artifact: + n = self._next() + values = { + "assessment_id": saved_id(assessment or self.assessment()), + "digest": f"sha256:doc-{n}", + "kind": "report", + "uri": f"file:///artifacts/{n}.json", + } + return Artifact(**{**values, **kw}) + + def artifact(self, assessment: Assessment | None = None, **kw: Any) -> Artifact: + return self._add("artifact", self.new_artifact(assessment, **kw)) + + def new_occurrence( + self, finding: Finding | None = None, assessment: Assessment | None = None, **kw: Any + ) -> Occurrence: + values = { + "finding_id": saved_id(finding or self.finding()), + "assessment_id": saved_id(assessment or self.assessment()), + "severity": "high", + } + return Occurrence(**{**values, **kw}) + + def occurrence( + self, finding: Finding | None = None, assessment: Assessment | None = None, **kw: Any + ) -> Occurrence: + return self._add("occurrence", self.new_occurrence(finding, assessment, **kw)) + + def new_location(self, occurrence: Occurrence | None = None, **kw: Any) -> Location: + values = { + "occurrence_id": saved_id(occurrence or self.occurrence()), + "path": "pkg/api/handler.go", + "line_start": 10, + "line_end": 12, + } + return Location(**{**values, **kw}) + + def location(self, occurrence: Occurrence | None = None, **kw: Any) -> Location: + return self._add("location", self.new_location(occurrence, **kw)) + + # 3b · Findings + def new_finding(self, repository: Repository | None = None, **kw: Any) -> Finding: + values = { + "repository_id": saved_id(repository or self.repository()), + "fingerprint_algo": "v3", + "fingerprint": f"fp-{self._next()}", + "title": "SQL injection in handler", + } + return Finding(**{**values, **kw}) + + def finding(self, repository: Repository | None = None, **kw: Any) -> Finding: + return self._add("finding", self.new_finding(repository, **kw)) + + def new_cwe(self, **kw: Any) -> Cwe: + return Cwe(**{"identifier": f"CWE-{self._next()}", "abstraction": "base", **kw}) + + def cwe(self, **kw: Any) -> Cwe: + return self._add("cwe", self.new_cwe(**kw)) + + def new_finding_cwe(self, finding: Finding | None = None, cwe: Cwe | None = None) -> FindingCwe: + return FindingCwe( + finding_id=saved_id(finding or self.finding()), cwe_id=saved_id(cwe or self.cwe()) + ) + + def finding_cwe(self, finding: Finding | None = None, cwe: Cwe | None = None) -> FindingCwe: + return self._add("finding_cwe", self.new_finding_cwe(finding, cwe)) + + def new_finding_label(self, finding: Finding | None = None, **kw: Any) -> FindingLabel: + values = {"finding_id": saved_id(finding or self.finding()), "taxonomy": "attack"} + return FindingLabel(**{**values, "label": "T1190", **kw}) + + def finding_label(self, finding: Finding | None = None, **kw: Any) -> FindingLabel: + return self._add("finding_label", self.new_finding_label(finding, **kw)) + + def new_finding_external_ref( + self, finding: Finding | None = None, **kw: Any + ) -> FindingExternalRef: + values = {"finding_id": saved_id(finding or self.finding()), "system": "jira"} + return FindingExternalRef(**{**values, "ref": f"SEC-{self._next()}", **kw}) + + def finding_external_ref(self, finding: Finding | None = None, **kw: Any) -> FindingExternalRef: + return self._add("finding_external_ref", self.new_finding_external_ref(finding, **kw)) + + def new_decision(self, finding: Finding | None = None, **kw: Any) -> Decision: + values = { + "finding_id": saved_id(finding or self.finding()), + "statement_key": f"stmt-{self._next()}", + "actor": "user:analyst", + "validity": "true_positive", + "occurred_at": T0, + } + return Decision(**{**values, **kw}) + + def decision(self, finding: Finding | None = None, **kw: Any) -> Decision: + return self._add("decision", self.new_decision(finding, **kw)) + + def new_review_item(self, finding: Finding | None = None, **kw: Any) -> ReviewItem: + values = {"finding_id": saved_id(finding or self.finding()), "status": "pending"} + return ReviewItem(**{**values, "reason": "auto_accept_blocked", **kw}) + + def review_item(self, finding: Finding | None = None, **kw: Any) -> ReviewItem: + return self._add("review_item", self.new_review_item(finding, **kw)) + + # 4 · Operations + def new_run_batch(self, **kw: Any) -> RunBatch: + return RunBatch(**{"kind": "drain_tranche", "period": f"2026-W{self._next():02d}", **kw}) + + def run_batch(self, **kw: Any) -> RunBatch: + return self._add("run_batch", self.new_run_batch(**kw)) + + def new_run_order(self, **kw: Any) -> RunOrder: + values = {"lane": "audit", "state": "queued", "priority": 5, "requested_by": "user:me"} + return RunOrder(**{**values, **kw}) + + def run_order(self, **kw: Any) -> RunOrder: + return self._add("run_order", self.new_run_order(**kw)) + + def new_spend_event(self, **kw: Any) -> SpendEvent: + values = { + "provider": "anthropic", + "model": "claude", + "tokens_in": 100, + "tokens_out": 20, + "cost_usd": Decimal("0.25"), + "occurred_at": T0, + } + return SpendEvent(**{**values, **kw}) + + def spend_event(self, **kw: Any) -> SpendEvent: + return self._add("spend_event", self.new_spend_event(**kw)) + + def new_feed_snapshot(self, **kw: Any) -> FeedSnapshot: + doc = self.artifact(assessment_id=None, kind="feed") + values = {"artifact_id": saved_id(doc), "feed": "osv", "retrieved_at": T0} + return FeedSnapshot(**{**values, **kw}) + + def feed_snapshot(self, **kw: Any) -> FeedSnapshot: + return self._add("feed_snapshot", self.new_feed_snapshot(**kw)) + + def new_assessment_input( + self, assessment: Assessment | None = None, snapshot: FeedSnapshot | None = None + ) -> AssessmentInput: + return AssessmentInput( + assessment_id=saved_id(assessment or self.assessment()), + feed_snapshot_id=saved_id(snapshot or self.feed_snapshot()), + ) + + def assessment_input( + self, assessment: Assessment | None = None, snapshot: FeedSnapshot | None = None + ) -> AssessmentInput: + return self._add("assessment_input", self.new_assessment_input(assessment, snapshot)) + + # 5 · Ledger + def new_layers(self, repository: Repository | None = None, **kw: Any) -> Layer: + values = {"repository_id": saved_id(repository or self.repository())} + return Layer(**{**values, "merkle_signing_method": "none", **kw}) + + def layer(self, repository: Repository | None = None, **kw: Any) -> Layer: + return self._add("layers", self.new_layers(repository, **kw)) + + def new_events( + self, layer: Layer | None = None, decision: Decision | None = None, **kw: Any + ) -> Event: + values = { + "layer_id": saved_id(layer or self.layer()), + "decision_id": saved_id(decision or self.decision()), + "seq": 0, + "event_payload": b"\x00\x01", + } + return Event(**{**values, **kw}) + + def event( + self, layer: Layer | None = None, decision: Decision | None = None, **kw: Any + ) -> Event: + return self._add("events", self.new_events(layer, decision, **kw)) diff --git a/tests/v1/repositories/tables/test_assessments.py b/tests/v1/repositories/tables/test_assessments.py new file mode 100644 index 00000000..e7dd57df --- /dev/null +++ b/tests/v1/repositories/tables/test_assessments.py @@ -0,0 +1,68 @@ +"""3a · Assessment repositories: latest-of-kind, artifacts, occurrences and locations.""" + +from __future__ import annotations + +from datetime import timedelta +from decimal import Decimal + +from tests.v1.repositories.tables.conftest import T0, Seed, same_rows + + +def test_latest_assessment_per_ref_and_kind(seed: Seed) -> None: + u = seed.uow + ref = seed.repository_ref() + assert u.assessments.latest(ref.id, "audit") is None + old = seed.assessment(ref, occurred_at=T0) + new = seed.assessment(ref, occurred_at=T0 + timedelta(days=7)) + seed.assessment(ref, kind="threat_model", occurred_at=T0 + timedelta(days=30)) + seed.assessment(occurred_at=T0 + timedelta(days=60)) # another ref + assert u.assessments.latest(ref.id, "audit") == new + assert u.assessments.list_for_ref(ref.id)[:2] == [old, new] # by occurred_at + + +def test_latest_is_by_occurred_at_not_insert_order(seed: Seed) -> None: + ref = seed.repository_ref() + newer = seed.assessment(ref, occurred_at=T0 + timedelta(days=1)) + seed.assessment(ref, occurred_at=T0) # imported later, examined earlier + assert seed.uow.assessments.latest(ref.id, "audit") == newer + + +def test_assessments_produced_by_a_run_order(seed: Seed) -> None: + order = seed.run_order() + produced = seed.assessment(run_order_id=order.id) + seed.assessment() + assert seed.uow.assessments.list_for_run_order(order.id) == [produced] + + +def test_artifacts_of_an_assessment_and_imported_ones(seed: Seed) -> None: + u = seed.uow + a = seed.assessment() + report = seed.artifact(a, digest="sha256:report") + sarif = seed.artifact(a, digest="sha256:sarif", kind="sarif") + imported = seed.artifact(assessment_id=None, digest="sha256:report") + assert same_rows(u.artifacts.list_for_assessment(a.id), [report, sarif]) + assert u.artifacts.find(a.id, "sha256:report") == report + assert u.artifacts.find(None, "sha256:report") == imported + assert u.artifacts.find(None, "sha256:sarif") is None + + +def test_occurrences_by_scan_and_by_finding(seed: Seed) -> None: + u = seed.uow + a1, a2 = seed.assessment(), seed.assessment() + f1, f2 = seed.finding(), seed.finding() + o11 = seed.occurrence(f1, a1, cvss_vector="CVSS:3.1/AV:N", cvss_score=Decimal("9.8")) + o21 = seed.occurrence(f2, a1) + o12 = seed.occurrence(f1, a2) + assert same_rows(u.occurrences.list_for_assessment(a1.id), [o11, o21]) + assert same_rows(u.occurrences.list_for_finding(f1.id), [o11, o12]) + assert u.occurrences.find(f1.id, a2.id) == o12 + assert u.occurrences.get(o11.id).cvss_score == Decimal("9.8") + + +def test_locations_ordered_by_path_then_line(seed: Seed) -> None: + occurrence = seed.occurrence() + b = seed.location(occurrence, path="b.go", line_start=1, line_end=1) + a20 = seed.location(occurrence, path="a.go", line_start=20, line_end=21) + a3 = seed.location(occurrence, path="a.go", line_start=3, line_end=3) + whole = seed.location(occurrence, path="c.go", line_start=None, line_end=None) + assert seed.uow.locations.list_for_occurrence(occurrence.id) == [a3, a20, b, whole] diff --git a/tests/v1/repositories/tables/test_constraints.py b/tests/v1/repositories/tables/test_constraints.py new file mode 100644 index 00000000..1425e05f --- /dev/null +++ b/tests/v1/repositories/tables/test_constraints.py @@ -0,0 +1,136 @@ +"""The contracts' cross-column and referential rules hold on both backends. + +Each case is a row the DDL must refuse (plan Appendix B, plus the DDL's own +CHECKs); the repository surfaces the refusal as ConflictError. +""" + +from __future__ import annotations + +from collections.abc import Callable +from datetime import timedelta +from decimal import Decimal +from uuid import uuid4 + +import pytest +import sqlalchemy as sa + +from tests.v1.repositories.tables.conftest import T0, Seed, repository_for +from traust_core.v1.errors import ConflictError +from traust_core.v1.models.base import Model + +Case = Callable[[Seed], tuple[str, Model]] + +INVALID: dict[str, Case] = { + # 1 · Inventory + "default ref must be a branch": lambda s: ( + "repository_ref", + s.new_repository_ref(kind="tag", is_default=True), + ), + "named ref needs a kind": lambda s: ("repository_ref", s.new_repository_ref(kind=None)), + "bare commit has no kind": lambda s: ("repository_ref", s.new_repository_ref(name=None)), + "ownership ends after it starts": lambda s: ( + "ownership", + s.new_ownership(valid_to=T0 - timedelta(days=1)), + ), + # 3a · Assessments + "location range starts at line 1": lambda s: ( + "location", + s.new_location(line_start=0, line_end=1), + ), + "location range is ordered": lambda s: ("location", s.new_location(line_start=9, line_end=3)), + "location lines are both set or both null": lambda s: ( + "location", + s.new_location(line_start=None, line_end=4), + ), + "cvss score is 0-10": lambda s: ("occurrence", s.new_occurrence(cvss_score=Decimal("10.5"))), + # 3b · Findings + "advisory match only with an advisory": lambda s: ( + "finding", + s.new_finding(advisory_match="confirmed"), + ), + "advisory needs a match grade": lambda s: ( + "finding", + s.new_finding(advisory_id=s.advisory().id), + ), + "decision has a who": lambda s: ("decision", s.new_decision(actor=None)), + "decision has one who": lambda s: ( + "decision", + s.new_decision(assessment_id=s.assessment().id), + ), + "assessed decision takes the assessment's commit": lambda s: ( + "decision", + s.new_decision( + actor=None, assessment_id=s.assessment().id, repository_ref_id=s.repository_ref().id + ), + ), + # 4 · Operations + "routed or injected, never both": lambda s: ( + "run_order", + s.new_run_order(rule="r1", event_source="push"), + ), + "leased exactly when a holder is set": lambda s: ("run_order", s.new_run_order(state="leased")), + "lease holder and expiry go together": lambda s: ( + "run_order", + s.new_run_order(state="leased", lease_holder="w1"), + ), + "finished after started": lambda s: ( + "run_order", + s.new_run_order(started_at=T0, finished_at=T0 - timedelta(minutes=1)), + ), + "spend is never negative": lambda s: ("spend_event", s.new_spend_event(tokens_in=-1)), + # 5 · Ledger + "event seq starts at 0": lambda s: ("events", s.new_events(seq=-1)), + # referential integrity + "release needs a product": lambda s: ( + "release", + s.new_release().model_copy(update={"product_id": uuid4()}), + ), + "occurrence needs its assessment": lambda s: ( + "occurrence", + s.new_occurrence().model_copy(update={"assessment_id": uuid4()}), + ), + "event proves a real decision": lambda s: ( + "events", + s.new_events().model_copy(update={"decision_id": uuid4()}), + ), +} + + +@pytest.mark.parametrize("case", list(INVALID), ids=list(INVALID)) +def test_database_refuses(seed: Seed, case: str) -> None: + table, row = INVALID[case](seed) + with pytest.raises(ConflictError): + repository_for(seed.uow, table).add(row) + + +def test_one_open_ownership_per_repository_and_team(seed: Seed) -> None: + repo, team = seed.repository(), seed.team() + closed = seed.ownership(repo, team, valid_to=T0 + timedelta(days=1)) + seed.ownership(repo, team, valid_from=T0 + timedelta(days=1)) # reopened after closing + assert closed.valid_to is not None + with pytest.raises(ConflictError): + seed.ownership(repo, team, valid_from=T0 + timedelta(days=2)) + + +def test_bare_commits_are_unique_per_repository(seed: Seed) -> None: + repo = seed.repository() + seed.repository_ref(repo, name=None, kind=None) + seed.repository_ref(repo, name="main") # same sha on a branch is a different ref + with pytest.raises(ConflictError): + seed.repository_ref(repo, name=None, kind=None) + + +def test_imported_artifacts_are_unique_by_digest(seed: Seed) -> None: + seed.artifact(assessment_id=None, digest="sha256:same") + seed.artifact(digest="sha256:same") # produced by an assessment: its own key + with pytest.raises(ConflictError): + seed.artifact(assessment_id=None, digest="sha256:same") + + +def test_parent_rows_cannot_be_deleted_while_referenced(seed: Seed) -> None: + product = seed.product() + seed.release(product) + repository = repository_for(seed.uow, "product") + stmt = sa.delete(repository.table).where(repository.table.c.id == product.id) + with pytest.raises(ConflictError): + repository._execute("raw delete", stmt) diff --git a/tests/v1/repositories/tables/test_dependencies.py b/tests/v1/repositories/tables/test_dependencies.py new file mode 100644 index 00000000..e0da4b06 --- /dev/null +++ b/tests/v1/repositories/tables/test_dependencies.py @@ -0,0 +1,62 @@ +"""2 · Dependencies repositories: the joins the blast-radius queries walk.""" + +from __future__ import annotations + +from tests.v1.repositories.tables.conftest import Seed, same_rows + + +def test_package_lookups(seed: Seed) -> None: + u = seed.uow + lib_repo = seed.repository() + base = seed.sbom() + internal = seed.package( + ecosystem="go", name="example.com/lib", source_repository_id=lib_repo.id + ) + image_pkg = seed.package(ecosystem="docker", name="ubi9", sbom_id=base.id) + seed.package() + assert u.packages.find("go", "example.com/lib") == internal + assert u.packages.find("npm", "example.com/lib") is None + assert u.packages.list_from_repository(lib_repo.id) == [internal] + assert u.packages.list_with_sbom(base.id) == [image_pkg] + + +def test_dependencies_by_sbom_and_by_package(seed: Seed) -> None: + u = seed.uow + s1, s2 = seed.sbom(), seed.sbom() + pkg, other = seed.package(), seed.package() + d1 = seed.dependency(s1, pkg, version="1.0.0") + d2 = seed.dependency(s2, pkg, version="1.2.0", direct=False) + d3 = seed.dependency(s1, other) + assert same_rows(u.dependencies.list_for_sbom(s1.id), [d1, d3]) + assert same_rows(u.dependencies.list_for_package(pkg.id), [d1, d2]) + assert d2.direct is False + + +def test_sbom_is_deduplicated_by_digest(seed: Seed) -> None: + sbom = seed.sbom(digest="sha256:pkgs") + assert seed.uow.sboms.find_by_digest("sha256:pkgs") == sbom + assert seed.uow.sboms.find_by_digest("sha256:other") is None + + +def test_images_by_digest_provenance_and_sbom(seed: Seed) -> None: + u = seed.uow + ref, sbom = seed.repository_ref(), seed.sbom() + built = seed.image(digest="sha256:img", repository_ref_id=ref.id, sbom_id=sbom.id) + seed.image() # no declared provenance, no SBOM yet + assert u.images.find_by_digest("sha256:img") == built + assert u.images.list_built_from(ref.id) == [built] + assert u.images.list_with_sbom(sbom.id) == [built] + + +def test_advisory_aliases_and_affected_packages(seed: Seed) -> None: + u = seed.uow + cve = seed.advisory(identifier="CVE-2026-1000") + ghsa = seed.advisory(identifier="GHSA-aaaa", alias_of_id=cve.id) + pkg = seed.package() + hit = seed.advisory_package(cve, pkg) + assert u.advisories.find_by_identifier("GHSA-aaaa") == ghsa + assert u.advisories.list_aliases(cve.id) == [ghsa] + assert u.advisories.list_aliases(ghsa.id) == [] + assert u.advisory_packages.find(cve.id, pkg.id) == hit + assert u.advisory_packages.list_for_advisory(cve.id) == [hit] + assert u.advisory_packages.list_for_package(pkg.id) == [hit] diff --git a/tests/v1/repositories/tables/test_findings.py b/tests/v1/repositories/tables/test_findings.py new file mode 100644 index 00000000..f244c6c9 --- /dev/null +++ b/tests/v1/repositories/tables/test_findings.py @@ -0,0 +1,116 @@ +"""3b · Finding repositories: identity, taxonomy links, decisions and the review queue.""" + +from __future__ import annotations + +from datetime import timedelta + +import pytest + +from tests.v1.repositories.tables.conftest import T0, Seed, apart, same_rows +from traust_core.v1.errors import ConflictError + + +def test_finding_identity_is_per_repository_and_recipe(seed: Seed) -> None: + u = seed.uow + repo, other = seed.repository(), seed.repository() + finding = seed.finding(repo, fingerprint="abc") + seed.finding(other, fingerprint="abc") # same fingerprint, another repository + seed.finding(repo, fingerprint="abc", fingerprint_algo="v4") # next recipe + assert u.findings.find_by_fingerprint(repo.id, "v3", "abc") == finding + assert u.findings.find_by_fingerprint(repo.id, "v2", "abc") is None + assert len(u.findings.list_for_repository(repo.id)) == 2 + + +def test_findings_known_as_an_advisory(seed: Seed) -> None: + advisory = seed.advisory() + known = seed.finding(advisory_id=advisory.id, advisory_match="confirmed") + seed.finding() + assert seed.uow.findings.list_for_advisory(advisory.id) == [known] + + +def test_finding_title_can_be_corrected(seed: Seed) -> None: + finding = seed.finding(title="SQLi") + updated = seed.uow.findings.update(finding.model_copy(update={"title": "SQL injection"})) + assert seed.uow.findings.get(finding.id).title == updated.title == "SQL injection" + + +def test_cwe_catalog_and_finding_links(seed: Seed) -> None: + u = seed.uow + cwe89, cwe20 = seed.cwe(identifier="CWE-89"), seed.cwe(identifier="CWE-20") + finding = seed.finding() + l89, l20 = seed.finding_cwe(finding, cwe89), seed.finding_cwe(finding, cwe20) + other = seed.finding_cwe(seed.finding(), cwe89) + assert u.cwes.find_by_identifier("CWE-89") == cwe89 + assert same_rows(u.finding_cwes.list_for_finding(finding.id), [l89, l20]) + assert same_rows(u.finding_cwes.list_for_cwe(cwe89.id), [l89, other]) + + +def test_labels_and_external_refs(seed: Seed) -> None: + u = seed.uow + finding = seed.finding() + attack = seed.finding_label(finding, taxonomy="attack", label="T1190") + asvs = seed.finding_label(finding, taxonomy="asvs", label="V5.3.4") + seed.finding_label(taxonomy="attack", label="T1059") + ticket = seed.finding_external_ref(finding, system="jira", ref="SEC-1") + shared = seed.finding_external_ref(system="jira", ref="SEC-1") # one ticket, two findings + assert same_rows(u.finding_labels.list_for_finding(finding.id), [attack, asvs]) + assert u.finding_labels.list_with("attack", "T1190") == [attack] + assert u.finding_external_refs.list_for_finding(finding.id) == [ticket] + assert same_rows(u.finding_external_refs.list_for_ref("jira", "SEC-1"), [ticket, shared]) + + +def test_decisions_are_idempotent_by_statement_key(seed: Seed) -> None: + u = seed.uow + finding = seed.finding() + decision = seed.decision(finding, statement_key="k1") + assert u.decisions.find_by_statement(finding.id, "k1") == decision + with pytest.raises(ConflictError): + seed.decision(finding, statement_key="k1", validity="false_positive") + + +def test_decision_history_is_in_judgement_order(seed: Seed) -> None: + finding = seed.finding() + later = seed.decision(finding, occurred_at=T0 + timedelta(days=2), resolution="resolved") + first = seed.decision(finding, occurred_at=T0) + correction = seed.decision(finding, occurred_at=T0 + timedelta(days=3), restates_id=later.id) + assert seed.uow.decisions.list_for_finding(finding.id) == [first, later, correction] + + +def test_assessor_and_human_decisions(seed: Seed) -> None: + assessment, ref = seed.assessment(), seed.repository_ref() + by_assessor = seed.decision(actor=None, assessment_id=assessment.id) + by_human = seed.decision(actor="user:me", repository_ref_id=ref.id, source_channel="jira") + assert by_assessor.actor is None and by_assessor.assessment_id == assessment.id + assert by_human.repository_ref_id == ref.id + + +def test_decisions_since_are_paged_in_time_order(seed: Seed) -> None: + days = [seed.decision(occurred_at=T0 + timedelta(days=d)) for d in (3, 1, 2)] + since = seed.uow.decisions.list_since(T0, limit=2) + assert since == [days[1], days[2]] + assert seed.uow.decisions.list_since(T0 + timedelta(days=3), limit=10) == [] + + +def test_review_queue_holds_only_pending_items_oldest_first(seed: Seed) -> None: + u = seed.uow + finding = seed.finding() + first = seed.review_item(finding) + done = seed.review_item(finding, status="accepted") + seed.uow.commit() # separate transactions: distinct created_at on Postgres + apart() + second = seed.review_item() + assert u.review_items.pending(limit=10) == [first, second] + assert u.review_items.pending(limit=1) == [first] + assert same_rows(u.review_items.list_for_finding(finding.id), [first, done]) + + +def test_resolving_a_review_item(seed: Seed) -> None: + u = seed.uow + item = seed.review_item() + answer = seed.decision(item_finding := u.findings.get(item.finding_id)) + resolved = u.review_items.update( + item.model_copy(update={"status": "accepted", "resolved_by_decision_id": answer.id}) + ) + assert item_finding.id == item.finding_id + assert u.review_items.pending(limit=10) == [] + assert u.review_items.get(item.id) == resolved diff --git a/tests/v1/repositories/tables/test_inventory.py b/tests/v1/repositories/tables/test_inventory.py new file mode 100644 index 00000000..ae2f6b3f --- /dev/null +++ b/tests/v1/repositories/tables/test_inventory.py @@ -0,0 +1,89 @@ +"""1 · Inventory repositories: natural-key lookups and the temporal ownership reads.""" + +from __future__ import annotations + +from datetime import timedelta + +from tests.v1.repositories.tables.conftest import SHA, T0, Seed, apart, same_rows + +B = "b" * 40 + + +def test_lookups_by_natural_key(seed: Seed) -> None: + u = seed.uow + product = seed.product(slug="ocp") + release = seed.release(product, version="4.22") + repo = seed.repository(url="https://github.com/org/x") + team = seed.team(name="core") + assert u.products.find_by_slug("ocp") == product + assert u.releases.find(product.id, "4.22") == release + assert u.repositories.find_by_url("https://github.com/org/x") == repo + assert u.teams.find_by_name("core") == team + assert u.products.find_by_slug("nope") is None + assert u.releases.find(product.id, "9.9") is None + + +def test_releases_and_components_list_by_parent(seed: Seed) -> None: + u = seed.uow + product, other = seed.product(), seed.product() + r1, r2 = seed.release(product), seed.release(product) + seed.release(other) + ref = seed.repository_ref() + c1, c2 = seed.component(r1, ref), seed.component(r2, ref) + assert same_rows(u.releases.list_for_product(product.id), [r1, r2]) + assert u.components.list_for_release(r1.id) == [c1] + assert same_rows(u.components.list_for_ref(ref.id), [c1, c2]) + assert u.components.find(r2.id, ref.id) == c2 + + +def test_ref_lookup_separates_named_and_bare_commits(seed: Seed) -> None: + u = seed.uow + repo = seed.repository() + main = seed.repository_ref(repo, name="main", sha=SHA) + bare = seed.repository_ref(repo, name=None, kind=None, sha=B) + assert u.repository_refs.find(repo.id, "main", SHA) == main + assert u.repository_refs.find(repo.id, None, B) == bare + assert u.repository_refs.find(repo.id, None, SHA) is None + assert u.repository_refs.find(repo.id, "main", B) is None + assert same_rows(u.repository_refs.list_for(repo.id), [main, bare]) + + +def test_default_ref_is_the_latest_recorded_default(seed: Seed) -> None: + u = seed.uow + repo = seed.repository() + assert u.repository_refs.default(repo.id) is None + seed.repository_ref(repo, name="dev", sha=SHA) + first = seed.repository_ref(repo, name="main", sha=SHA, is_default=True) + assert u.repository_refs.default(repo.id) == first + u.commit() # a later scan, a later transaction + apart() + later = seed.repository_ref(repo, name="main", sha=B, is_default=True) + assert u.repository_refs.default(repo.id) == later + + +def test_refs_carrying_a_manifest_sbom(seed: Seed) -> None: + sbom = seed.sbom() + ref = seed.repository_ref(manifest_sbom_id=sbom.id) + seed.repository_ref() + assert seed.uow.repository_refs.list_for_manifest_sbom(sbom.id) == [ref] + + +def test_ownership_current_and_as_of(seed: Seed) -> None: + u = seed.uow + repo, platform, core = seed.repository(), seed.team(), seed.team() + day = timedelta(days=1) + old = seed.ownership(repo, platform, valid_from=T0, valid_to=T0 + 10 * day) + new = seed.ownership(repo, core, valid_from=T0 + 10 * day) + assert u.ownerships.current(repo.id) == [new] + assert u.ownerships.as_of(repo.id, T0 - day) == [] + assert u.ownerships.as_of(repo.id, T0) == [old] # valid_from is inclusive + assert u.ownerships.as_of(repo.id, T0 + 10 * day) == [new] # valid_to is exclusive + assert u.ownerships.list_for_team(platform.id) == [old] + + +def test_closing_ownership_moves_it_out_of_current(seed: Seed) -> None: + u = seed.uow + owned = seed.ownership() + closed = u.ownerships.update(owned.model_copy(update={"valid_to": T0 + timedelta(days=3)})) + assert u.ownerships.current(owned.repository_id) == [] + assert u.ownerships.as_of(owned.repository_id, T0 + timedelta(days=1)) == [closed] diff --git a/tests/v1/repositories/tables/test_ledger.py b/tests/v1/repositories/tables/test_ledger.py new file mode 100644 index 00000000..e82b51b6 --- /dev/null +++ b/tests/v1/repositories/tables/test_ledger.py @@ -0,0 +1,54 @@ +"""5 · Ledger repositories: layers per repository and one event per decision.""" + +from __future__ import annotations + +import pytest + +from tests.v1.repositories.tables.conftest import Seed, same_rows +from traust_core.v1.errors import ConflictError + + +def test_layers_per_repository_and_line(seed: Seed) -> None: + repo = seed.repository() + repo_layer = seed.layer(repo) + branch = seed.layer(repo, line="release-4.22") + seed.layer() + assert same_rows(seed.uow.layers.list_for_repository(repo.id), [repo_layer, branch]) + + +def test_signing_a_root_updates_the_layer(seed: Seed) -> None: + layer = seed.layer() + signed = seed.uow.layers.update( + layer.model_copy( + update={"merkle_root": "ab12", "merkle_size": 1, "merkle_root_signature": "sig"} + ) + ) + assert seed.uow.layers.get(layer.id) == signed + + +def test_events_append_in_sequence(seed: Seed) -> None: + events = seed.uow.events + layer = seed.layer() + assert events.last_seq(layer.id) is None + e0 = seed.event(layer, seq=0) + e1 = seed.event(layer, seq=1, event_payload=b"\xff" * 64) + assert events.last_seq(layer.id) == 1 + assert events.list_for_layer(layer.id) == [e0, e1] + assert events.get(e1.id).event_payload == b"\xff" * 64 + + +def test_each_decision_is_proved_once_and_seq_is_unique(seed: Seed) -> None: + events = seed.uow.events + layer, decision = seed.layer(), seed.decision() + proof = seed.event(layer, decision, seq=0) + assert events.find_by_decision(decision.id) == proof + with pytest.raises(ConflictError): + seed.event(seed.layer(), decision) # same decision, another layer + seed.uow.rollback() + + +def test_seq_is_unique_per_layer(seed: Seed) -> None: + layer = seed.layer() + seed.event(layer, seq=0) + with pytest.raises(ConflictError): + seed.event(layer, seq=0) diff --git a/tests/v1/repositories/tables/test_ops.py b/tests/v1/repositories/tables/test_ops.py new file mode 100644 index 00000000..e0cac585 --- /dev/null +++ b/tests/v1/repositories/tables/test_ops.py @@ -0,0 +1,112 @@ +"""4 · Operations repositories: the run-order queue, spend windows and feed freshness.""" + +from __future__ import annotations + +from datetime import timedelta + +import pytest +from sqlalchemy.engine import Engine + +from tests.v1.repositories.tables.conftest import T0, Seed, apart +from traust_core.v1.repositories.contract import SqlTraustUnitOfWork + +LEASE = timedelta(minutes=5) + + +def test_claim_takes_priority_then_age_and_skips_non_queued(seed: Seed) -> None: + orders = seed.uow.run_orders + seed.run_order(state="planned", priority=0) + seed.run_order(state="succeeded", priority=0) + later = seed.run_order(priority=5) + soon_old = seed.run_order(priority=1) + seed.uow.commit() # separate transactions: distinct created_at on Postgres + apart() + soon_new = seed.run_order(priority=1) + claimed = [orders.claim(f"w{i}", T0, LEASE) for i in range(4)] + assert [c.id if c else None for c in claimed] == [soon_old.id, soon_new.id, later.id, None] + + +def test_claim_records_the_lease(seed: Seed) -> None: + order = seed.run_order() + claimed = seed.uow.run_orders.claim("worker-1", T0, LEASE) + assert claimed is not None and claimed.id == order.id + assert (claimed.state, claimed.lease_holder) == ("leased", "worker-1") + assert claimed.lease_expires_at == T0 + LEASE + assert claimed.updated_at >= order.updated_at + + +def test_expired_leases_return_to_the_queue(seed: Seed) -> None: + orders = seed.uow.run_orders + expired, live = seed.run_order(priority=1), seed.run_order(priority=2) + orders.claim("w1", T0, LEASE) + orders.claim("w2", T0 + LEASE, LEASE) + assert orders.release_expired(T0 + LEASE + timedelta(seconds=1)) == 1 + released = orders.get(expired.id) + assert (released.state, released.lease_holder, released.lease_expires_at) == ( + "queued", + None, + None, + ) + assert orders.get(live.id).state == "leased" + + +def test_orders_by_batch_and_by_repository_state(seed: Seed) -> None: + orders = seed.uow.run_orders + batch, repo = seed.run_batch(), seed.repository() + in_batch = seed.run_order(run_batch_id=batch.id) + planned = seed.run_order(repository_id=repo.id, state="planned", priority=2) + urgent = seed.run_order(repository_id=repo.id, state="planned", priority=1) + seed.run_order(repository_id=repo.id) + assert orders.list_for_batch(batch.id) == [in_batch] + assert orders.list_for_repository(repo.id, "planned") == [urgent, planned] + + +def test_one_batch_per_kind_and_period(seed: Seed) -> None: + batch = seed.run_batch(kind="drain_tranche", period="2026-W31") + assert seed.uow.run_batches.find("drain_tranche", "2026-W31") == batch + assert seed.uow.run_batches.find("router_run", "2026-W31") is None + + +def test_spend_by_order_and_half_open_window(seed: Seed) -> None: + spend = seed.uow.spend_events + order = seed.run_order() + start = seed.spend_event(run_order_id=order.id, occurred_at=T0) + inside = seed.spend_event(occurred_at=T0 + timedelta(hours=23)) + seed.spend_event(occurred_at=T0 + timedelta(days=1)) # end is exclusive + assert spend.list_for_order(order.id) == [start] + assert spend.list_between(T0, T0 + timedelta(days=1)) == [start, inside] + + +def test_latest_feed_snapshot_by_retrieval_time(seed: Seed) -> None: + feeds = seed.uow.feed_snapshots + assert feeds.latest("osv") is None + new = seed.feed_snapshot(feed="osv", retrieved_at=T0 + timedelta(days=1)) + seed.feed_snapshot(feed="osv", retrieved_at=T0) + seed.feed_snapshot(feed="epss", retrieved_at=T0 + timedelta(days=9)) + assert feeds.latest("osv") == new + + +def test_assessment_inputs_link_results_to_snapshots(seed: Seed) -> None: + inputs = seed.uow.assessment_inputs + snapshot, assessment = seed.feed_snapshot(), seed.assessment() + link = seed.assessment_input(assessment, snapshot) + assert inputs.list_for_assessment(assessment.id) == [link] + assert inputs.list_for_snapshot(snapshot.id) == [link] + + +@pytest.mark.integration +def test_concurrent_workers_never_claim_the_same_order(engine: Engine) -> None: + """Postgres row locks: an uncommitted lease is skipped, not waited on or doubled.""" + if engine.dialect.name != "postgresql": + pytest.skip("SQLite serialises writers; concurrency is a Postgres property") + with SqlTraustUnitOfWork(engine) as setup: + seed = Seed(setup) + first, second = seed.run_order(priority=1), seed.run_order(priority=2) + setup.commit() + with SqlTraustUnitOfWork(engine) as a, SqlTraustUnitOfWork(engine) as b: + got_a = a.run_orders.claim("a", T0, LEASE) + got_b = b.run_orders.claim("b", T0, LEASE) # a's lease is not committed yet + assert got_a is not None and got_b is not None + assert {got_a.id, got_b.id} == {first.id, second.id} + a.commit() + b.commit() diff --git a/tests/v1/repositories/tables/test_repositories.py b/tests/v1/repositories/tables/test_repositories.py deleted file mode 100644 index 423512f2..00000000 --- a/tests/v1/repositories/tables/test_repositories.py +++ /dev/null @@ -1,331 +0,0 @@ -from collections.abc import Iterator -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from decimal import Decimal -from pathlib import Path -from time import sleep - -import pytest -import sqlalchemy as sa -from sqlalchemy.engine import Engine - -from tests.v1.conftest import SQL_BACKENDS, contract_db -from traust_core.v1.errors import ConflictError, RepositoryError -from traust_core.v1.models.assessment import Artifact, Assessment -from traust_core.v1.models.dependencies import Advisory, AdvisoryPackage, Dependency, Package, Sbom -from traust_core.v1.models.findings import ( - Cwe, - Decision, - Finding, - FindingCwe, - Location, - Occurrence, - ReviewItem, -) -from traust_core.v1.models.inventory import ( - Component, - Ownership, - Product, - Release, - Repository, - RepositoryRef, - Team, -) -from traust_core.v1.models.ledger import Event, Layer -from traust_core.v1.models.operations import FeedSnapshot, RunOrder, SpendEvent -from traust_core.v1.repositories.contract import SqlTraustUnitOfWork - -T0 = datetime(2026, 10, 5, tzinfo=UTC) -SHA = "a" * 40 - - -@pytest.fixture(params=SQL_BACKENDS) -def engine(request: pytest.FixtureRequest, tmp_path: Path) -> Iterator[Engine]: - with contract_db(request.param, tmp_path) as eng: - yield eng - - -@dataclass -class Scanned: - repository: Repository - ref: RepositoryRef - assessment: Assessment - finding: Finding - - -@pytest.fixture -def scanned(engine: Engine) -> Scanned: - with SqlTraustUnitOfWork(engine) as inv: - repo = inv.repositories.add(Repository(url="https://github.com/o/r", public=True)) - assert repo.id is not None - ref = inv.repository_refs.add( - RepositoryRef( - repository_id=repo.id, name="main", kind="branch", is_default=True, sha=SHA - ) - ) - inv.commit() - assert ref.id is not None - with SqlTraustUnitOfWork(engine) as a: - assessment = a.assessments.add( - Assessment( - repository_ref_id=ref.id, - kind="audit", - status="completed", - assessor="skill:audit", - assessor_version="1.0", - occurred_at=T0, - ) - ) - a.commit() - with SqlTraustUnitOfWork(engine) as f: - finding = f.findings.add( - Finding(repository_id=repo.id, fingerprint_algo="v3", fingerprint="fp1", title="SQLi") - ) - f.commit() - return Scanned(repo, ref, assessment, finding) - - -def test_inventory_round_trip_and_natural_keys(engine: Engine, scanned: Scanned) -> None: - repo, ref = scanned.repository, scanned.ref - assert repo.id is not None and ref.id is not None - with SqlTraustUnitOfWork(engine) as inv: - product = inv.products.add(Product(slug="ocp", name="OpenShift")) - assert product.id is not None - release = inv.releases.add(Release(product_id=product.id, version="4.22")) - assert release.id is not None - component = inv.components.add(Component(release_id=release.id, repository_ref_id=ref.id)) - team = inv.teams.add(Team(name="core")) - assert team.id is not None - owned = inv.ownerships.add(Ownership(repository_id=repo.id, team_id=team.id, valid_from=T0)) - inv.commit() - - assert inv.products.find_by_slug("ocp") == product - assert inv.releases.find(product.id, "4.22") == release - assert inv.components.list_for_ref(ref.id) == [component] - assert inv.repositories.find_by_url("https://github.com/o/r") == repo - assert inv.repository_refs.find(repo.id, "main", SHA) == ref - assert inv.repository_refs.find(repo.id, None, SHA) is None - assert inv.repository_refs.default(repo.id) == ref - assert inv.teams.find_by_name("core") == team - assert inv.ownerships.current(repo.id) == [owned] - assert inv.ownerships.as_of(repo.id, T0 - timedelta(days=1)) == [] - - -def test_closing_ownership_is_an_update_the_database_touches( - engine: Engine, scanned: Scanned -) -> None: - repo = scanned.repository - assert repo.id is not None - with SqlTraustUnitOfWork(engine) as inv: - team = inv.teams.add(Team(name="core")) - assert team.id is not None - owned = inv.ownerships.add(Ownership(repository_id=repo.id, team_id=team.id, valid_from=T0)) - inv.commit() - sleep(0.01) - with SqlTraustUnitOfWork(engine) as inv: - closed = inv.ownerships.update(owned.model_copy(update={"valid_to": T0 + timedelta(1)})) - inv.commit() - assert inv.ownerships.current(repo.id) == [] - assert inv.ownerships.as_of(repo.id, T0) == [closed] - assert closed.updated_at > owned.updated_at # type: ignore[operator] - - -def test_duplicate_natural_key_is_a_conflict(engine: Engine, scanned: Scanned) -> None: - with SqlTraustUnitOfWork(engine) as inv, pytest.raises(ConflictError): - inv.repositories.add(Repository(url=scanned.repository.url, public=False)) - - -def test_dependencies_trace_a_package_to_the_refs_that_ship_it( - engine: Engine, scanned: Scanned -) -> None: - with SqlTraustUnitOfWork(engine) as d: - sbom = d.sboms.add(Sbom(digest="sha256:sbom")) - package = d.packages.add(Package(ecosystem="go", name="golang.org/x/crypto")) - assert sbom.id is not None and package.id is not None - dep = d.dependencies.add( - Dependency(sbom_id=sbom.id, package_id=package.id, version="0.1.0", direct=True) - ) - advisory = d.advisories.add(Advisory(identifier="CVE-2026-0001")) - assert advisory.id is not None - alias = d.advisories.add(Advisory(identifier="GHSA-x", alias_of_id=advisory.id)) - hit = d.advisory_packages.add( - AdvisoryPackage(advisory_id=advisory.id, package_id=package.id) - ) - d.commit() - - assert d.sboms.find_by_digest("sha256:sbom") == sbom - assert d.packages.find("go", "golang.org/x/crypto") == package - assert d.dependencies.list_for_package(package.id) == [dep] - assert d.advisories.list_aliases(advisory.id) == [alias] - assert d.advisory_packages.list_for_package(package.id) == [hit] - - -def test_findings_chain_and_decisions_are_idempotent_by_statement( - engine: Engine, scanned: Scanned -) -> None: - finding, assessment = scanned.finding, scanned.assessment - assert finding.id is not None and assessment.id is not None - with SqlTraustUnitOfWork(engine) as a: - occurrence = a.occurrences.add( - Occurrence( - finding_id=finding.id, - assessment_id=assessment.id, - severity="high", - cvss_score=Decimal("7.5"), - ) - ) - assert occurrence.id is not None - location = a.locations.add( - Location(occurrence_id=occurrence.id, path="a.go", line_start=3, line_end=4) - ) - artifact = a.artifacts.add( - Artifact(assessment_id=assessment.id, digest="sha256:r", kind="report", uri="f") - ) - a.commit() - assert a.occurrences.list_for_assessment(assessment.id) == [occurrence] - assert a.occurrences.find(finding.id, assessment.id) == occurrence - assert a.locations.list_for_occurrence(occurrence.id) == [location] - assert a.artifacts.find(assessment.id, "sha256:r") == artifact - assert a.assessments.latest(scanned.ref.id, "audit") == assessment # type: ignore[arg-type] - - with SqlTraustUnitOfWork(engine) as f: - cwe = f.cwes.add(Cwe(identifier="CWE-89", abstraction="base")) - assert cwe.id is not None - f.finding_cwes.add(FindingCwe(finding_id=finding.id, cwe_id=cwe.id)) - decision = f.decisions.add( - Decision( - finding_id=finding.id, - assessment_id=assessment.id, - statement_key="k1", - validity="true_positive", - occurred_at=T0, - ) - ) - review = f.review_items.add( - ReviewItem(finding_id=finding.id, decision_id=decision.id, status="pending") - ) - f.commit() - assert f.findings.find_by_fingerprint(finding.repository_id, "v3", "fp1") == finding - assert f.finding_cwes.list_for_cwe(cwe.id) == f.finding_cwes.list_for_finding(finding.id) - assert f.decisions.find_by_statement(finding.id, "k1") == decision - assert f.decisions.list_since(T0 - timedelta(1), limit=10) == [decision] - assert f.review_items.pending(limit=10) == [review] - with pytest.raises(ConflictError): - f.decisions.add(decision.model_copy(update={"id": None, "created_at": None})) - - -def test_append_only_rows_reject_mutation_in_the_database(engine: Engine, scanned: Scanned) -> None: - with SqlTraustUnitOfWork(engine) as a: - table = a.assessments.table - with pytest.raises((RepositoryError, ConflictError)): - a.assessments._execute("raw update", sa.update(table).values(status="failed")) - - -def test_run_orders_claim_in_priority_order_and_leases_expire( - engine: Engine, scanned: Scanned -) -> None: - def order(priority: int) -> RunOrder: - return RunOrder( - repository_id=scanned.repository.id, - lane="audit", - state="queued", - priority=priority, - requested_by="user:me", - ) - - with SqlTraustUnitOfWork(engine) as ops: - later = ops.run_orders.add(order(5)) - sooner = ops.run_orders.add(order(1)) - ops.commit() - - claimed = ops.run_orders.claim("w1", T0, timedelta(minutes=5)) - assert claimed is not None and claimed.id == sooner.id - assert claimed.state == "leased" and claimed.lease_holder == "w1" - next_claim = ops.run_orders.claim("w2", T0, timedelta(minutes=5)) - assert next_claim is not None and next_claim.id == later.id - assert ops.run_orders.claim("w3", T0, timedelta(minutes=5)) is None - - assert ops.run_orders.release_expired(T0 + timedelta(minutes=10)) == 2 - assert ops.run_orders.claim("w4", T0, timedelta(minutes=5)).id == sooner.id # type: ignore[union-attr] - ops.commit() - - -def test_ops_spend_and_feed_snapshots(engine: Engine, scanned: Scanned) -> None: - with SqlTraustUnitOfWork(engine) as a: - artifact = a.artifacts.add(Artifact(digest="sha256:osv", kind="feed", uri="f")) - a.commit() - assert artifact.id is not None - with SqlTraustUnitOfWork(engine) as ops: - spend = ops.spend_events.add( - SpendEvent( - provider="anthropic", - model="m", - tokens_in=10, - tokens_out=5, - cost_usd=Decimal("0.25"), - occurred_at=T0, - ) - ) - old = ops.feed_snapshots.add( - FeedSnapshot(artifact_id=artifact.id, feed="osv", retrieved_at=T0) - ) - new = ops.feed_snapshots.add( - FeedSnapshot(artifact_id=artifact.id, feed="osv", retrieved_at=T0 + timedelta(1)) - ) - ops.commit() - assert ops.spend_events.list_between(T0, T0 + timedelta(1)) == [spend] - assert ops.feed_snapshots.latest("osv") == new != old - - -def test_ledger_events_prove_one_decision_each(engine: Engine, scanned: Scanned) -> None: - finding = scanned.finding - assert finding.id is not None and scanned.repository.id is not None - with SqlTraustUnitOfWork(engine) as f: - decision = f.decisions.add( - Decision(finding_id=finding.id, actor="user:me", statement_key="k", occurred_at=T0) - ) - f.commit() - assert decision.id is not None - with SqlTraustUnitOfWork(engine) as led: - layer = led.layers.add( - Layer(repository_id=scanned.repository.id, merkle_signing_method="none") - ) - assert layer.id is not None - assert led.events.last_seq(layer.id) is None - event = led.events.add( - Event(layer_id=layer.id, decision_id=decision.id, seq=0, event_payload=b"\x00") - ) - led.commit() - assert led.events.find_by_decision(decision.id) == event - assert led.events.last_seq(layer.id) == 0 - assert led.layers.list_for_repository(scanned.repository.id) == [layer] - - -def test_one_unit_of_work_spans_sections_in_one_transaction(engine: Engine) -> None: - with SqlTraustUnitOfWork(engine) as uow: - repo = uow.repositories.add(Repository(url="https://github.com/o/tx", public=True)) - assert repo.id is not None - ref = uow.repository_refs.add(RepositoryRef(repository_id=repo.id, sha=SHA)) - assert ref.id is not None - assessment = uow.assessments.add( - Assessment( - repository_ref_id=ref.id, - kind="audit", - status="completed", - assessor="skill:audit", - assessor_version="1.0", - occurred_at=T0, - ) - ) - finding = uow.findings.add( - Finding(repository_id=repo.id, fingerprint_algo="v3", fingerprint="t", title="t") - ) - assert assessment.id is not None and finding.id is not None - uow.occurrences.add( - Occurrence(finding_id=finding.id, assessment_id=assessment.id, severity="low") - ) - uow.rollback() # e.g. a later decision failed: nothing from any section lands - - with SqlTraustUnitOfWork(engine) as uow: - assert uow.repositories.find_by_url("https://github.com/o/tx") is None - assert uow.findings.list_for_repository(repo.id) == [] diff --git a/tests/v1/repositories/tables/test_table_contract.py b/tests/v1/repositories/tables/test_table_contract.py new file mode 100644 index 00000000..c45425d3 --- /dev/null +++ b/tests/v1/repositories/tables/test_table_contract.py @@ -0,0 +1,106 @@ +"""Every contract table, both backends: what the database guarantees for each row. + +Parametrized over CONTRACT_TABLES, so a new table is covered as soon as it has a +spec, a repository and a Seed builder. +""" + +from __future__ import annotations + +from time import sleep +from uuid import UUID, uuid4 + +import pytest +import sqlalchemy as sa + +from tests.v1.repositories.tables.conftest import Seed, repository_for +from traust_core.v1.errors import ConflictError, NotFoundError, RepositoryError +from traust_core.v1.repositories.contract import CONTRACT_TABLES, SqlTraustUnitOfWork + +TABLES = [spec.name for spec in CONTRACT_TABLES] +MUTABLE = [spec.name for spec in CONTRACT_TABLES if not spec.append_only] +APPEND_ONLY = [spec.name for spec in CONTRACT_TABLES if spec.append_only] +# Tables whose rows have no natural key: two identical rows are two facts. +NO_NATURAL_KEY = { + "assessment", + "review_item", + "run_order", + "spend_event", + "feed_snapshot", + "layers", +} + + +@pytest.mark.parametrize("table", TABLES) +def test_database_generates_id_and_timestamps(seed: Seed, table: str) -> None: + saved = repository_for(seed.uow, table).add(seed.new(table)) + assert isinstance(saved.id, UUID) and saved.id.version == 4 + assert saved.created_at is not None and saved.created_at.tzinfo is not None + if table in MUTABLE: + assert saved.updated_at is not None and saved.updated_at >= saved.created_at + else: + assert "updated_at" not in type(saved).model_fields + + +@pytest.mark.parametrize("table", TABLES) +def test_saved_row_round_trips_after_commit(seed: Seed, engine: sa.Engine, table: str) -> None: + saved = repository_for(seed.uow, table).add(seed.new(table)) + seed.uow.commit() + with SqlTraustUnitOfWork(engine) as other: + assert repository_for(other, table).get(saved.id) == saved + + +@pytest.mark.parametrize("table", TABLES) +def test_get_missing_row_raises_not_found(uow: SqlTraustUnitOfWork, table: str) -> None: + with pytest.raises(NotFoundError): + repository_for(uow, table).get(uuid4()) + + +@pytest.mark.parametrize("table", TABLES) +def test_natural_key_rejects_a_duplicate(seed: Seed, table: str) -> None: + repository = repository_for(seed.uow, table) + row = seed.new(table) + first = repository.add(row) + if table in NO_NATURAL_KEY: + assert repository.add(row).id != first.id + return + with pytest.raises(ConflictError): + repository.add(row) + + +@pytest.mark.parametrize("table", MUTABLE) +def test_update_saves_and_the_database_touches_updated_at(seed: Seed, table: str) -> None: + repository = repository_for(seed.uow, table) + saved = repository.add(seed.new(table)) + seed.uow.commit() + sleep(0.01) # the touch trigger has millisecond precision on SQLite + updated = repository.update(saved) + assert updated.id == saved.id and updated.created_at == saved.created_at + assert updated.updated_at > saved.updated_at + assert repository.get(saved.id) == updated + + +@pytest.mark.parametrize("table", MUTABLE) +def test_update_of_a_missing_row_raises_not_found(seed: Seed, table: str) -> None: + repository = repository_for(seed.uow, table) + with pytest.raises(NotFoundError): + repository.update(seed.new(table).model_copy(update={"id": uuid4()})) + + +@pytest.mark.parametrize("table", APPEND_ONLY) +def test_append_only_repository_has_no_update(uow: SqlTraustUnitOfWork, table: str) -> None: + assert not hasattr(repository_for(uow, table), "update") + + +@pytest.mark.parametrize("table", APPEND_ONLY) +@pytest.mark.parametrize("statement", ["update", "delete"]) +def test_database_rejects_mutating_append_only_rows(seed: Seed, table: str, statement: str) -> None: + repository = repository_for(seed.uow, table) + saved = repository.add(seed.new(table)) + where = repository.table.c.id == saved.id + stmt = ( + sa.update(repository.table).where(where).values(created_at=saved.created_at) + if statement == "update" + else sa.delete(repository.table).where(where) + ) + with pytest.raises((RepositoryError, ConflictError), match="append-only"): + repository._execute(f"raw {statement}", stmt) diff --git a/tests/v1/repositories/tables/test_unit_of_work.py b/tests/v1/repositories/tables/test_unit_of_work.py new file mode 100644 index 00000000..2b66fedf --- /dev/null +++ b/tests/v1/repositories/tables/test_unit_of_work.py @@ -0,0 +1,95 @@ +"""SqlTraustUnitOfWork: one transaction over every repository, on both backends.""" + +from __future__ import annotations + +import pytest +from sqlalchemy.engine import Engine + +from tests.v1.repositories.tables.conftest import Seed +from traust_core.v1.errors import ConflictError, RepositoryError +from traust_core.v1.repositories.contract import SqlTraustUnitOfWork + + +def _intake(seed: Seed) -> dict[str, object]: + """What intake writes for one scan: rows in inventory, 3a and 3b.""" + repo = seed.repository(url="https://github.com/org/tx") + ref = seed.repository_ref(repo) + assessment = seed.assessment(ref) + finding = seed.finding(repo) + occurrence = seed.occurrence(finding, assessment) + decision = seed.decision(finding, actor=None, assessment_id=assessment.id) + return {"repo": repo, "finding": finding, "occurrence": occurrence, "decision": decision} + + +def test_commit_makes_every_section_visible_together(engine: Engine) -> None: + with SqlTraustUnitOfWork(engine) as uow: + rows = _intake(Seed(uow)) + uow.commit() + with SqlTraustUnitOfWork(engine) as other: + assert other.repositories.find_by_url("https://github.com/org/tx") == rows["repo"] + assert other.decisions.list_for_finding(rows["finding"].id) == [rows["decision"]] # type: ignore[attr-defined] + + +def test_rollback_discards_every_section(engine: Engine) -> None: + with SqlTraustUnitOfWork(engine) as uow: + rows = _intake(Seed(uow)) + uow.rollback() + with SqlTraustUnitOfWork(engine) as other: + assert other.repositories.find_by_url("https://github.com/org/tx") is None + assert other.occurrences.list_for_finding(rows["finding"].id) == [] # type: ignore[attr-defined] + + +def test_leaving_without_commit_rolls_back(engine: Engine) -> None: + with SqlTraustUnitOfWork(engine) as uow: + _intake(Seed(uow)) + with SqlTraustUnitOfWork(engine) as other: + assert other.repositories.find_by_url("https://github.com/org/tx") is None + + +def test_an_exception_rolls_back_the_whole_unit(engine: Engine) -> None: + with pytest.raises(RuntimeError), SqlTraustUnitOfWork(engine) as uow: + _intake(Seed(uow)) + raise RuntimeError("a later step failed") + with SqlTraustUnitOfWork(engine) as other: + assert other.repositories.find_by_url("https://github.com/org/tx") is None + + +def test_a_conflict_midway_leaves_nothing_after_rollback(engine: Engine) -> None: + with SqlTraustUnitOfWork(engine) as uow: + seed = Seed(uow) + rows = _intake(seed) + with pytest.raises(ConflictError): + seed.decision(rows["finding"], statement_key=rows["decision"].statement_key) # type: ignore[arg-type, attr-defined] + uow.rollback() + with SqlTraustUnitOfWork(engine) as other: + assert other.repositories.find_by_url("https://github.com/org/tx") is None + + +def test_uncommitted_work_is_invisible_to_another_unit(engine: Engine) -> None: + with SqlTraustUnitOfWork(engine) as writer: + Seed(writer).repository(url="https://github.com/org/pending") + with SqlTraustUnitOfWork(engine) as reader: + assert reader.repositories.find_by_url("https://github.com/org/pending") is None + writer.commit() + with SqlTraustUnitOfWork(engine) as reader: + assert reader.repositories.find_by_url("https://github.com/org/pending") is not None + + +def test_a_unit_of_work_can_be_reused_for_another_transaction(engine: Engine) -> None: + uow = SqlTraustUnitOfWork(engine) + with uow: + Seed(uow).repository(url="https://github.com/org/one") + uow.commit() + with uow: + assert uow.repositories.find_by_url("https://github.com/org/one") is not None + + +def test_repositories_are_unusable_outside_with(engine: Engine) -> None: + with pytest.raises(RepositoryError, match="outside a 'with' block"): + SqlTraustUnitOfWork(engine).connection # noqa: B018 + + +def test_every_repository_shares_the_units_connection(uow: SqlTraustUnitOfWork) -> None: + connections = {id(r._connection) for r in vars(uow).values() if hasattr(r, "spec")} + assert connections == {id(uow.connection)} + assert len([r for r in vars(uow).values() if hasattr(r, "spec")]) == 31 From 2b7c761a0009af0fbb667c507776ee70d3d2b58d Mon Sep 17 00:00:00 2001 From: Shaun Smith <51304449+smith-xyz@users.noreply.github.com> Date: Wed, 21 Oct 2026 09:51:07 -0400 Subject: [PATCH 06/10] test(repositories): behavior only - CRUD per table, queries per section, unit of work; drop DDL/drift checks owned by traust-contracts --- src/traust_core/v1/repositories/__init__.py | 2 - src/traust_core/v1/repositories/contract.py | 20 +-- src/traust_core/v1/repositories/sql.py | 22 --- src/traust_core/v1/repositories/tables.py | 2 +- .../repositories/tables/test_constraints.py | 136 ------------------ tests/v1/repositories/tables/test_crud.py | 70 +++++++++ tests/v1/repositories/tables/test_factory.py | 105 -------------- tests/v1/repositories/tables/test_findings.py | 8 +- tests/v1/repositories/tables/test_ledger.py | 19 +-- .../tables/test_table_contract.py | 106 -------------- tests/v1/test_contracts.py | 12 +- 11 files changed, 79 insertions(+), 423 deletions(-) delete mode 100644 tests/v1/repositories/tables/test_constraints.py create mode 100644 tests/v1/repositories/tables/test_crud.py delete mode 100644 tests/v1/repositories/tables/test_factory.py delete mode 100644 tests/v1/repositories/tables/test_table_contract.py diff --git a/src/traust_core/v1/repositories/__init__.py b/src/traust_core/v1/repositories/__init__.py index bf354a80..ea8551c8 100644 --- a/src/traust_core/v1/repositories/__init__.py +++ b/src/traust_core/v1/repositories/__init__.py @@ -13,7 +13,6 @@ apply_schema, create_database_engine, record_columns, - schema_drift, ) from traust_core.v1.repositories.unit_of_work import UnitOfWork @@ -31,5 +30,4 @@ "apply_schema", "create_database_engine", "record_columns", - "schema_drift", ] diff --git a/src/traust_core/v1/repositories/contract.py b/src/traust_core/v1/repositories/contract.py index 407b07c4..56decda1 100644 --- a/src/traust_core/v1/repositories/contract.py +++ b/src/traust_core/v1/repositories/contract.py @@ -1,4 +1,4 @@ -"""The whole contracts db: every table spec, and one unit of work over all of them. +"""Every contract table spec, and one unit of work over all of them. Section protocols (InventoryUnitOfWork, FindingsUnitOfWork, ...) are what a service declares it needs; SqlTraustUnitOfWork satisfies every one of them on @@ -11,11 +11,8 @@ from typing import Protocol -from sqlalchemy.engine import Engine - from traust_core.v1.repositories import traust_ledger, traust_ops -from traust_core.v1.repositories.sql import Dialect, schema_drift -from traust_core.v1.repositories.tables import SqlTablesUnitOfWork, TableFactory, TableSpec +from traust_core.v1.repositories.tables import SqlTablesUnitOfWork, TableSpec from traust_core.v1.repositories.traust import assessments, dependencies, findings, inventory from traust_core.v1.repositories.traust.assessments import AssessmentsUnitOfWork from traust_core.v1.repositories.traust.dependencies import DependenciesUnitOfWork @@ -34,19 +31,6 @@ ) -def contract_tables(dialect: Dialect) -> TableFactory: - """The dialect's factory with every contract table built onto its MetaData.""" - tables = TableFactory.for_dialect(dialect) - for spec in CONTRACT_TABLES: - tables.table(spec) - return tables - - -def contract_drift(engine: Engine) -> list[str]: - """Where the database and the declared tables disagree; empty = in step.""" - return schema_drift(engine, contract_tables(Dialect.of(engine)).metadata) - - class TraustUnitOfWork( InventoryUnitOfWork, DependenciesUnitOfWork, diff --git a/src/traust_core/v1/repositories/sql.py b/src/traust_core/v1/repositories/sql.py index c3b8b84a..271c1639 100644 --- a/src/traust_core/v1/repositories/sql.py +++ b/src/traust_core/v1/repositories/sql.py @@ -250,25 +250,3 @@ def _execute( raise ConflictError(f"{name}: {operation}: {e.orig}") from e except SQLAlchemyError as e: raise RepositoryError(f"{name}: {operation} failed: {e}") from e - - -def schema_drift(engine: Engine, metadata: sa.MetaData) -> list[str]: - inspector = sa.inspect(engine) - problems: list[str] = [] - for table in metadata.sorted_tables: - schema = table.schema - if not inspector.has_table(table.name, schema=schema): - problems.append(f"{table.name}: missing in database") - continue - actual = {c["name"]: c for c in inspector.get_columns(table.name, schema=schema)} - for column in table.columns: - found = actual.pop(column.name, None) - if found is None: - problems.append(f"{table.name}.{column.name}: missing in database") - elif found["nullable"] != column.nullable and not column.primary_key: - problems.append( - f"{table.name}.{column.name}: nullable {found['nullable']} in database, " - f"{column.nullable} in Table" - ) - problems += [f"{table.name}.{name}: not declared in Table" for name in sorted(actual)] - return problems diff --git a/src/traust_core/v1/repositories/tables.py b/src/traust_core/v1/repositories/tables.py index 63f565b6..35884364 100644 --- a/src/traust_core/v1/repositories/tables.py +++ b/src/traust_core/v1/repositories/tables.py @@ -64,7 +64,7 @@ def build(self, metadata: sa.MetaData, dialect: Dialect) -> sa.Table: class TableFactory: - """Every contract table for one dialect, on one MetaData (used by schema_drift).""" + """Every contract table for one dialect, built once on one MetaData.""" def __init__(self, dialect: Dialect) -> None: self.dialect = dialect diff --git a/tests/v1/repositories/tables/test_constraints.py b/tests/v1/repositories/tables/test_constraints.py deleted file mode 100644 index 1425e05f..00000000 --- a/tests/v1/repositories/tables/test_constraints.py +++ /dev/null @@ -1,136 +0,0 @@ -"""The contracts' cross-column and referential rules hold on both backends. - -Each case is a row the DDL must refuse (plan Appendix B, plus the DDL's own -CHECKs); the repository surfaces the refusal as ConflictError. -""" - -from __future__ import annotations - -from collections.abc import Callable -from datetime import timedelta -from decimal import Decimal -from uuid import uuid4 - -import pytest -import sqlalchemy as sa - -from tests.v1.repositories.tables.conftest import T0, Seed, repository_for -from traust_core.v1.errors import ConflictError -from traust_core.v1.models.base import Model - -Case = Callable[[Seed], tuple[str, Model]] - -INVALID: dict[str, Case] = { - # 1 · Inventory - "default ref must be a branch": lambda s: ( - "repository_ref", - s.new_repository_ref(kind="tag", is_default=True), - ), - "named ref needs a kind": lambda s: ("repository_ref", s.new_repository_ref(kind=None)), - "bare commit has no kind": lambda s: ("repository_ref", s.new_repository_ref(name=None)), - "ownership ends after it starts": lambda s: ( - "ownership", - s.new_ownership(valid_to=T0 - timedelta(days=1)), - ), - # 3a · Assessments - "location range starts at line 1": lambda s: ( - "location", - s.new_location(line_start=0, line_end=1), - ), - "location range is ordered": lambda s: ("location", s.new_location(line_start=9, line_end=3)), - "location lines are both set or both null": lambda s: ( - "location", - s.new_location(line_start=None, line_end=4), - ), - "cvss score is 0-10": lambda s: ("occurrence", s.new_occurrence(cvss_score=Decimal("10.5"))), - # 3b · Findings - "advisory match only with an advisory": lambda s: ( - "finding", - s.new_finding(advisory_match="confirmed"), - ), - "advisory needs a match grade": lambda s: ( - "finding", - s.new_finding(advisory_id=s.advisory().id), - ), - "decision has a who": lambda s: ("decision", s.new_decision(actor=None)), - "decision has one who": lambda s: ( - "decision", - s.new_decision(assessment_id=s.assessment().id), - ), - "assessed decision takes the assessment's commit": lambda s: ( - "decision", - s.new_decision( - actor=None, assessment_id=s.assessment().id, repository_ref_id=s.repository_ref().id - ), - ), - # 4 · Operations - "routed or injected, never both": lambda s: ( - "run_order", - s.new_run_order(rule="r1", event_source="push"), - ), - "leased exactly when a holder is set": lambda s: ("run_order", s.new_run_order(state="leased")), - "lease holder and expiry go together": lambda s: ( - "run_order", - s.new_run_order(state="leased", lease_holder="w1"), - ), - "finished after started": lambda s: ( - "run_order", - s.new_run_order(started_at=T0, finished_at=T0 - timedelta(minutes=1)), - ), - "spend is never negative": lambda s: ("spend_event", s.new_spend_event(tokens_in=-1)), - # 5 · Ledger - "event seq starts at 0": lambda s: ("events", s.new_events(seq=-1)), - # referential integrity - "release needs a product": lambda s: ( - "release", - s.new_release().model_copy(update={"product_id": uuid4()}), - ), - "occurrence needs its assessment": lambda s: ( - "occurrence", - s.new_occurrence().model_copy(update={"assessment_id": uuid4()}), - ), - "event proves a real decision": lambda s: ( - "events", - s.new_events().model_copy(update={"decision_id": uuid4()}), - ), -} - - -@pytest.mark.parametrize("case", list(INVALID), ids=list(INVALID)) -def test_database_refuses(seed: Seed, case: str) -> None: - table, row = INVALID[case](seed) - with pytest.raises(ConflictError): - repository_for(seed.uow, table).add(row) - - -def test_one_open_ownership_per_repository_and_team(seed: Seed) -> None: - repo, team = seed.repository(), seed.team() - closed = seed.ownership(repo, team, valid_to=T0 + timedelta(days=1)) - seed.ownership(repo, team, valid_from=T0 + timedelta(days=1)) # reopened after closing - assert closed.valid_to is not None - with pytest.raises(ConflictError): - seed.ownership(repo, team, valid_from=T0 + timedelta(days=2)) - - -def test_bare_commits_are_unique_per_repository(seed: Seed) -> None: - repo = seed.repository() - seed.repository_ref(repo, name=None, kind=None) - seed.repository_ref(repo, name="main") # same sha on a branch is a different ref - with pytest.raises(ConflictError): - seed.repository_ref(repo, name=None, kind=None) - - -def test_imported_artifacts_are_unique_by_digest(seed: Seed) -> None: - seed.artifact(assessment_id=None, digest="sha256:same") - seed.artifact(digest="sha256:same") # produced by an assessment: its own key - with pytest.raises(ConflictError): - seed.artifact(assessment_id=None, digest="sha256:same") - - -def test_parent_rows_cannot_be_deleted_while_referenced(seed: Seed) -> None: - product = seed.product() - seed.release(product) - repository = repository_for(seed.uow, "product") - stmt = sa.delete(repository.table).where(repository.table.c.id == product.id) - with pytest.raises(ConflictError): - repository._execute("raw delete", stmt) diff --git a/tests/v1/repositories/tables/test_crud.py b/tests/v1/repositories/tables/test_crud.py new file mode 100644 index 00000000..d8739c22 --- /dev/null +++ b/tests/v1/repositories/tables/test_crud.py @@ -0,0 +1,70 @@ +"""Every table repository, on SQLite and Postgres: save a row, get it back, update it. + +Parametrized over CONTRACT_TABLES, so a new table is covered once it has a spec, +a repository and a Seed builder. +""" + +from __future__ import annotations + +from uuid import UUID, uuid4 + +import pytest +import sqlalchemy as sa + +from tests.v1.repositories.tables.conftest import Seed, apart, repository_for +from traust_core.v1.errors import ConflictError, NotFoundError +from traust_core.v1.repositories.contract import CONTRACT_TABLES, SqlTraustUnitOfWork + +TABLES = [spec.name for spec in CONTRACT_TABLES] +MUTABLE = [spec.name for spec in CONTRACT_TABLES if not spec.append_only] + + +@pytest.mark.parametrize("table", TABLES) +def test_add_returns_the_saved_row(seed: Seed, table: str) -> None: + row = seed.new(table) + saved = repository_for(seed.uow, table).add(row) + assert isinstance(saved.id, UUID) + assert saved.created_at is not None + assert saved.model_copy(update={"id": None, "created_at": None, **_no_updated_at(row)}) == row + + +@pytest.mark.parametrize("table", TABLES) +def test_get_returns_the_committed_row(seed: Seed, engine: sa.Engine, table: str) -> None: + saved = repository_for(seed.uow, table).add(seed.new(table)) + seed.uow.commit() + with SqlTraustUnitOfWork(engine) as other: + assert repository_for(other, table).get(saved.id) == saved + + +@pytest.mark.parametrize("table", TABLES) +def test_get_missing_raises_not_found(uow: SqlTraustUnitOfWork, table: str) -> None: + with pytest.raises(NotFoundError): + repository_for(uow, table).get(uuid4()) + + +@pytest.mark.parametrize("table", MUTABLE) +def test_update_saves_the_row(seed: Seed, table: str) -> None: + repository = repository_for(seed.uow, table) + saved = repository.add(seed.new(table)) + seed.uow.commit() + apart() + updated = repository.update(saved) + assert (updated.id, updated.created_at) == (saved.id, saved.created_at) + assert updated.updated_at > saved.updated_at + assert repository.get(saved.id) == updated + + +@pytest.mark.parametrize("table", MUTABLE) +def test_update_missing_raises_not_found(seed: Seed, table: str) -> None: + with pytest.raises(NotFoundError): + repository_for(seed.uow, table).update(seed.new(table).model_copy(update={"id": uuid4()})) + + +def test_a_rejected_write_raises_conflict(seed: Seed) -> None: + repository = seed.repository(url="https://github.com/org/dup") + with pytest.raises(ConflictError): + seed.repository(url=repository.url) + + +def _no_updated_at(row: object) -> dict[str, None]: + return {"updated_at": None} if "updated_at" in type(row).model_fields else {} diff --git a/tests/v1/repositories/tables/test_factory.py b/tests/v1/repositories/tables/test_factory.py deleted file mode 100644 index 4fac1c66..00000000 --- a/tests/v1/repositories/tables/test_factory.py +++ /dev/null @@ -1,105 +0,0 @@ -from pathlib import Path - -import pytest -import sqlalchemy as sa - -from tests.v1.conftest import SQL_BACKENDS, contract_db -from traust_core.v1 import contracts -from traust_core.v1.repositories.contract import CONTRACT_TABLES, contract_drift, contract_tables -from traust_core.v1.repositories.sql import Dialect -from traust_core.v1.repositories.tables import ( - TRAUST, - TRAUST_LEDGER, - TRAUST_OPS, - SqlTableRepository, - TableFactory, -) -from traust_core.v1.repositories.traust.inventory import REPOSITORY -from traust_core.v1.repositories.traust_ledger import EVENTS -from traust_core.v1.repositories.traust_ops import RUN_ORDER - - -@pytest.mark.parametrize( - ("namespace", "postgres", "sqlite"), - [ - (TRAUST, ("traust", "finding"), (None, "finding")), - (TRAUST_OPS, ("traust_ops", "finding"), (None, "ops_finding")), - (TRAUST_LEDGER, ("traust_ledger", "finding"), (None, "ledger_finding")), - ], -) -def test_namespace_places_a_table_per_dialect(namespace, postgres, sqlite) -> None: - assert namespace.locate("finding", Dialect.POSTGRES) == postgres - assert namespace.locate("finding", Dialect.SQLITE) == sqlite - - -@pytest.mark.parametrize( - ("dialect", "names"), - [ - (Dialect.POSTGRES, ["traust.repository", "traust_ops.run_order", "traust_ledger.events"]), - (Dialect.SQLITE, ["repository", "ops_run_order", "ledger_events"]), - ], -) -def test_factory_builds_contract_names(dialect: Dialect, names: list[str]) -> None: - tables = TableFactory.for_dialect(dialect) - assert [tables.table(s).fullname for s in (REPOSITORY, RUN_ORDER, EVENTS)] == names - - -def test_factory_is_one_per_dialect_and_builds_each_table_once() -> None: - tables = TableFactory.for_dialect(Dialect.SQLITE) - assert TableFactory.for_dialect(Dialect.SQLITE) is tables - assert TableFactory.for_dialect(Dialect.POSTGRES) is not tables - assert tables.table(REPOSITORY) is tables.table(REPOSITORY) - - -def test_factory_follows_the_engine(tmp_path: Path) -> None: - engine = sa.create_engine(f"sqlite:///{tmp_path}/f.db") - assert TableFactory.for_bind(engine) is TableFactory.for_dialect(Dialect.SQLITE) - - -def test_append_only_tables_have_no_updated_at() -> None: - tables = contract_tables(Dialect.SQLITE) - for spec in CONTRACT_TABLES: - assert ("updated_at" in tables.table(spec).c) is not spec.append_only, spec.name - - -@pytest.mark.parametrize("dialect", list(Dialect)) -def test_every_contract_table_has_exactly_one_spec(dialect: Dialect) -> None: - files = [p.stem for a in contracts.AREAS for p in contracts.ddl_files(a, dialect)] - contract = sorted(f for f in files if f != "namespace" and not _is_view(f, dialect)) - specs = sorted(spec.name for spec in CONTRACT_TABLES) - assert specs == contract - - -def _is_view(stem: str, dialect: Dialect) -> bool: - return any( - p.stem == stem and p.parent.name == "views" - for a in contracts.AREAS - for p in contracts.ddl_files(a, dialect) - ) - - -@pytest.mark.parametrize("kind", SQL_BACKENDS) -def test_declared_tables_match_the_contracts_ddl(kind: str, tmp_path: Path) -> None: - with contract_db(kind, tmp_path) as engine: - assert contract_drift(engine) == [] - - -def _table_repositories() -> list[type[SqlTableRepository]]: - pending, found = list(SqlTableRepository.__subclasses__()), [] - while pending: - cls = pending.pop() - pending += cls.__subclasses__() - if "spec" in vars(cls): - found.append(cls) - return found - - -def test_each_table_has_one_repository_and_its_model_matches_the_columns() -> None: - import traust_core.v1.repositories.contract # noqa: F401 (registers every repository) - - repos = _table_repositories() - assert sorted(r.spec.name for r in repos) == sorted(s.name for s in CONTRACT_TABLES) - tables = contract_tables(Dialect.SQLITE) - for repo in repos: - columns = set(tables.table(repo.spec).c.keys()) - assert set(repo.record.model_fields) == columns, repo.spec.name diff --git a/tests/v1/repositories/tables/test_findings.py b/tests/v1/repositories/tables/test_findings.py index f244c6c9..8c8f8783 100644 --- a/tests/v1/repositories/tables/test_findings.py +++ b/tests/v1/repositories/tables/test_findings.py @@ -4,10 +4,7 @@ from datetime import timedelta -import pytest - from tests.v1.repositories.tables.conftest import T0, Seed, apart, same_rows -from traust_core.v1.errors import ConflictError def test_finding_identity_is_per_repository_and_recipe(seed: Seed) -> None: @@ -59,13 +56,12 @@ def test_labels_and_external_refs(seed: Seed) -> None: assert same_rows(u.finding_external_refs.list_for_ref("jira", "SEC-1"), [ticket, shared]) -def test_decisions_are_idempotent_by_statement_key(seed: Seed) -> None: +def test_decision_by_statement_key(seed: Seed) -> None: u = seed.uow finding = seed.finding() decision = seed.decision(finding, statement_key="k1") assert u.decisions.find_by_statement(finding.id, "k1") == decision - with pytest.raises(ConflictError): - seed.decision(finding, statement_key="k1", validity="false_positive") + assert u.decisions.find_by_statement(finding.id, "k2") is None def test_decision_history_is_in_judgement_order(seed: Seed) -> None: diff --git a/tests/v1/repositories/tables/test_ledger.py b/tests/v1/repositories/tables/test_ledger.py index e82b51b6..38ced284 100644 --- a/tests/v1/repositories/tables/test_ledger.py +++ b/tests/v1/repositories/tables/test_ledger.py @@ -2,10 +2,7 @@ from __future__ import annotations -import pytest - from tests.v1.repositories.tables.conftest import Seed, same_rows -from traust_core.v1.errors import ConflictError def test_layers_per_repository_and_line(seed: Seed) -> None: @@ -37,18 +34,8 @@ def test_events_append_in_sequence(seed: Seed) -> None: assert events.get(e1.id).event_payload == b"\xff" * 64 -def test_each_decision_is_proved_once_and_seq_is_unique(seed: Seed) -> None: - events = seed.uow.events +def test_event_for_a_decision(seed: Seed) -> None: layer, decision = seed.layer(), seed.decision() proof = seed.event(layer, decision, seq=0) - assert events.find_by_decision(decision.id) == proof - with pytest.raises(ConflictError): - seed.event(seed.layer(), decision) # same decision, another layer - seed.uow.rollback() - - -def test_seq_is_unique_per_layer(seed: Seed) -> None: - layer = seed.layer() - seed.event(layer, seq=0) - with pytest.raises(ConflictError): - seed.event(layer, seq=0) + assert seed.uow.events.find_by_decision(decision.id) == proof + assert seed.uow.events.find_by_decision(seed.decision().id) is None diff --git a/tests/v1/repositories/tables/test_table_contract.py b/tests/v1/repositories/tables/test_table_contract.py deleted file mode 100644 index c45425d3..00000000 --- a/tests/v1/repositories/tables/test_table_contract.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Every contract table, both backends: what the database guarantees for each row. - -Parametrized over CONTRACT_TABLES, so a new table is covered as soon as it has a -spec, a repository and a Seed builder. -""" - -from __future__ import annotations - -from time import sleep -from uuid import UUID, uuid4 - -import pytest -import sqlalchemy as sa - -from tests.v1.repositories.tables.conftest import Seed, repository_for -from traust_core.v1.errors import ConflictError, NotFoundError, RepositoryError -from traust_core.v1.repositories.contract import CONTRACT_TABLES, SqlTraustUnitOfWork - -TABLES = [spec.name for spec in CONTRACT_TABLES] -MUTABLE = [spec.name for spec in CONTRACT_TABLES if not spec.append_only] -APPEND_ONLY = [spec.name for spec in CONTRACT_TABLES if spec.append_only] -# Tables whose rows have no natural key: two identical rows are two facts. -NO_NATURAL_KEY = { - "assessment", - "review_item", - "run_order", - "spend_event", - "feed_snapshot", - "layers", -} - - -@pytest.mark.parametrize("table", TABLES) -def test_database_generates_id_and_timestamps(seed: Seed, table: str) -> None: - saved = repository_for(seed.uow, table).add(seed.new(table)) - assert isinstance(saved.id, UUID) and saved.id.version == 4 - assert saved.created_at is not None and saved.created_at.tzinfo is not None - if table in MUTABLE: - assert saved.updated_at is not None and saved.updated_at >= saved.created_at - else: - assert "updated_at" not in type(saved).model_fields - - -@pytest.mark.parametrize("table", TABLES) -def test_saved_row_round_trips_after_commit(seed: Seed, engine: sa.Engine, table: str) -> None: - saved = repository_for(seed.uow, table).add(seed.new(table)) - seed.uow.commit() - with SqlTraustUnitOfWork(engine) as other: - assert repository_for(other, table).get(saved.id) == saved - - -@pytest.mark.parametrize("table", TABLES) -def test_get_missing_row_raises_not_found(uow: SqlTraustUnitOfWork, table: str) -> None: - with pytest.raises(NotFoundError): - repository_for(uow, table).get(uuid4()) - - -@pytest.mark.parametrize("table", TABLES) -def test_natural_key_rejects_a_duplicate(seed: Seed, table: str) -> None: - repository = repository_for(seed.uow, table) - row = seed.new(table) - first = repository.add(row) - if table in NO_NATURAL_KEY: - assert repository.add(row).id != first.id - return - with pytest.raises(ConflictError): - repository.add(row) - - -@pytest.mark.parametrize("table", MUTABLE) -def test_update_saves_and_the_database_touches_updated_at(seed: Seed, table: str) -> None: - repository = repository_for(seed.uow, table) - saved = repository.add(seed.new(table)) - seed.uow.commit() - sleep(0.01) # the touch trigger has millisecond precision on SQLite - updated = repository.update(saved) - assert updated.id == saved.id and updated.created_at == saved.created_at - assert updated.updated_at > saved.updated_at - assert repository.get(saved.id) == updated - - -@pytest.mark.parametrize("table", MUTABLE) -def test_update_of_a_missing_row_raises_not_found(seed: Seed, table: str) -> None: - repository = repository_for(seed.uow, table) - with pytest.raises(NotFoundError): - repository.update(seed.new(table).model_copy(update={"id": uuid4()})) - - -@pytest.mark.parametrize("table", APPEND_ONLY) -def test_append_only_repository_has_no_update(uow: SqlTraustUnitOfWork, table: str) -> None: - assert not hasattr(repository_for(uow, table), "update") - - -@pytest.mark.parametrize("table", APPEND_ONLY) -@pytest.mark.parametrize("statement", ["update", "delete"]) -def test_database_rejects_mutating_append_only_rows(seed: Seed, table: str, statement: str) -> None: - repository = repository_for(seed.uow, table) - saved = repository.add(seed.new(table)) - where = repository.table.c.id == saved.id - stmt = ( - sa.update(repository.table).where(where).values(created_at=saved.created_at) - if statement == "update" - else sa.delete(repository.table).where(where) - ) - with pytest.raises((RepositoryError, ConflictError), match="append-only"): - repository._execute(f"raw {statement}", stmt) diff --git a/tests/v1/test_contracts.py b/tests/v1/test_contracts.py index d68fb9ef..29d66228 100644 --- a/tests/v1/test_contracts.py +++ b/tests/v1/test_contracts.py @@ -7,7 +7,7 @@ from tests.v1.conftest import apply_contract_db from traust_core.v1 import contracts from traust_core.v1.errors import DocumentError -from traust_core.v1.repositories import create_database_engine, schema_drift +from traust_core.v1.repositories import create_database_engine ROOT = Path(__file__).resolve().parents[2] @@ -29,13 +29,3 @@ def test_database_ddl_applies_every_namespace_on_sqlite(tmp_path: Path) -> None: apply_contract_db(engine) names = set(sa.inspect(engine).get_table_names()) assert {"repository", "ops_run_order", "ledger_events"} <= names - - -def test_schema_drift_flags_disagreements(tmp_path: Path) -> None: - engine = create_database_engine(f"sqlite:///{tmp_path}/c.db") - apply_contract_db(engine) - wrong = sa.MetaData() - sa.Table("ledger_layers", wrong, sa.Column("not_in_ddl", sa.Text)) - problems = schema_drift(engine, wrong) - assert any("not_in_ddl: missing in database" in p for p in problems) - assert any("not declared in Table" in p for p in problems) From cee784c8041e06c7ba770a3eb46ada13670175e9 Mon Sep 17 00:00:00 2001 From: Shaun Smith <51304449+smith-xyz@users.noreply.github.com> Date: Wed, 21 Oct 2026 09:56:35 -0400 Subject: [PATCH 07/10] chore(repositories): base-column default compiles to statement_timestamp() on Postgres, matching contracts --- src/traust_core/v1/repositories/sql.py | 2 +- tests/v1/repositories/test_sql.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/traust_core/v1/repositories/sql.py b/src/traust_core/v1/repositories/sql.py index 271c1639..e88508a5 100644 --- a/src/traust_core/v1/repositories/sql.py +++ b/src/traust_core/v1/repositories/sql.py @@ -111,7 +111,7 @@ class _DbNow(sa.FunctionElement[datetime]): @compiles(_DbNow, "postgresql") def _postgres_now(element: Any, compiler: Any, **kw: Any) -> str: - return "now()" + return "statement_timestamp()" @compiles(_DbNow, "sqlite") diff --git a/tests/v1/repositories/test_sql.py b/tests/v1/repositories/test_sql.py index 02271a6f..7ac5fbf8 100644 --- a/tests/v1/repositories/test_sql.py +++ b/tests/v1/repositories/test_sql.py @@ -105,7 +105,7 @@ def _open_repositories(self) -> None: _TOUCH_POSTGRES = """ CREATE OR REPLACE FUNCTION base_records_touch_updated_at() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN - NEW.updated_at := now(); + NEW.updated_at := statement_timestamp(); RETURN NEW; END $$; From 533d8d057f0d5b46b327f8e1f504b2f7277411de Mon Sep 17 00:00:00 2001 From: Shaun Smith <51304449+smith-xyz@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:03:49 -0400 Subject: [PATCH 08/10] docs: agent context and README match the table repositories, one unit of work, and behaviour-only repository tests --- .agents/skills/traust-core-patterns/SKILL.md | 4 +- .../skills/traust-core-patterns/contracts.md | 5 +- .../skills/traust-core-patterns/repository.md | 56 +++++++-------- AGENTS.md | 20 +++--- README.md | 71 +++++++++---------- 5 files changed, 79 insertions(+), 77 deletions(-) diff --git a/.agents/skills/traust-core-patterns/SKILL.md b/.agents/skills/traust-core-patterns/SKILL.md index e843b561..89d759c8 100644 --- a/.agents/skills/traust-core-patterns/SKILL.md +++ b/.agents/skills/traust-core-patterns/SKILL.md @@ -9,8 +9,8 @@ Rules live in `AGENTS.md` → **Design rules**: one owner per capability, one se | Adding | Recipe | Worked example | |---|---|---| -| An aggregate repository over a contracts table | [repository.md](repository.md#aggregate-repository) | `src/traust_core/v1/repositories/triage.py` (mechanics only; legacy ids) | -| A table and its repository | [repository.md](repository.md) | `tests/v1/example/repository.py`; over a contracts table: `src/traust_core/v1/repositories/storage.py` | +| A repository (or a query) over a contracts table | [repository.md](repository.md) | `src/traust_core/v1/repositories/traust/inventory/repository.py`; tests in `tests/v1/repositories/tables/` | +| Cross-section writes in one transaction | [repository.md](repository.md#unit-of-work) | `src/traust_core/v1/repositories/contract.py` (`SqlTraustUnitOfWork`) | | A domain service, policy or interface | [service.md](service.md) | `src/traust_core/v1/services/portfolio.py` (shape) | | A tool, source or feed provider | [provider.md](provider.md) | `tests/v1/example/provider.py` | | An artifact type the harness/LLM produces | [artifacts.md](artifacts.md) | `tests/v1/example/artifact_publishing.py` (`TRIAGE`) | diff --git a/.agents/skills/traust-core-patterns/contracts.md b/.agents/skills/traust-core-patterns/contracts.md index 107c5db1..c8e3a401 100644 --- a/.agents/skills/traust-core-patterns/contracts.md +++ b/.agents/skills/traust-core-patterns/contracts.md @@ -10,7 +10,7 @@ Bump the pin: Write (exact): - `contracts.validate(name, document)` / `contracts.validate_json(name, payload)` → `DocumentError` with `{path, message}` issues -- `ArtifactPublisher` and `record_artifact` already call it +- `ArtifactPublisher` already calls it Read: @@ -22,4 +22,5 @@ Read: Database: -- `apply_schema(engine, contracts.ddl("storage", dialect))`; check repositories with `schema_drift` +- `apply_schema(engine, contracts.database_ddl(dialect))` creates every namespace (traust, traust_ops, traust_ledger) in bootstrap order +- The DDL is contracts' to own and test; core declares each table once as a `TableSpec` and tests repository behaviour (see [repository.md](repository.md)) diff --git a/.agents/skills/traust-core-patterns/repository.md b/.agents/skills/traust-core-patterns/repository.md index c963fd6b..d946d11a 100644 --- a/.agents/skills/traust-core-patterns/repository.md +++ b/.agents/skills/traust-core-patterns/repository.md @@ -1,39 +1,39 @@ # Repository -1. Model in `models/.py`: `Entity`/`Model` subclass or frozen dataclass. No database types. -2. Interface: a `Protocol` over one aggregate, in `repositories/.py`, using the shared verbs only as needed: `get(id) -> T` (raises `NotFoundError`), `find_*(...) -> T | None`, `list(filter, page) -> Page[T]`, `add(T) -> T` and `update(T) -> T` returning the saved record (database-generated `id`, `created_at`, `updated_at` via `RETURNING`). IDs are `UUID`, named `_id`. No generic CRUD base, no cross-domain tables. -3. Unit-of-work protocol: `class XUnitOfWork(UnitOfWork, Protocol): xs: XRepository`. -4. SQL implementation: - - `sa.Table(...)` describing the existing table (DDL is owned by traust-contracts) - - `class SqlXRepository(SqlRepository)`; every statement goes through `self._execute(" ", stmt)` - - `_to_row(model)` / `_to_model(row)` private functions; rows never leave the repository - - missing row → `NotFoundError`; duplicates surface as `ConflictError` from `_execute` -5. `class SqlXUnitOfWork(SqlUnitOfWork)` attaching the repository in `_open_repositories`. -6. In-memory double implementing the same interface, raising the same errors. -7. Contract tests parametrized over `["memory", *SQL_BACKENDS]` using `tests/v1/conftest.py::database`. The Postgres case is marked `integration` and runs with `make db-up && make test-integration`. - -The domain service takes the unit of work in its constructor and opens it per use case: +One repository per contract table. The DDL is owned by traust-contracts; you never write DDL, and a table that isn't in the pinned contracts yet is a gap to report, not one to invent. + +## Add a table repository + +Files mirror the contracts namespaces: `repositories/traust/
/
.py`, `repositories/traust_ops/
.py`, `repositories/traust_ledger/
.py`. Copy `repositories/traust/inventory/repository.py`. + +1. **Model** in `models/
.py`: subclass `Entity` (mutable table) or `AppendOnly` (append-only table); fields are the columns, named as in the DDL; no behaviour, no database types. +2. **Spec**: `TableSpec(TRAUST | TRAUST_OPS | TRAUST_LEDGER, "
", lambda: (), append_only=...)` using `uuid_column` / `text_column` / `timestamp_column` (or `sa.Column` for booleans, integers, numerics). Base columns come from the spec; don't declare them. `TableFactory` builds it per dialect (Postgres schema or SQLite `ops_`/`ledger_` prefix). +3. **Protocol**: `class XRepository(MutableRecordRepository[X] | RecordRepository[X], Protocol)` listing only this table's queries. +4. **SQL class**: `class SqlXRepository(SqlMutableRepository[X] | SqlAppendOnlyRepository[X])` with `spec = X_SPEC`, `record = X`. `add`/`get`/`update` are inherited; each query uses `self._one(...)` / `self._many(..., order_by=...)` over `self.table.c`. Missing row → `NotFoundError`; database refusals surface as `ConflictError` from `_execute`. +5. **Register**: add the spec to the section `__init__.py` `TABLES`, the repository to its section unit-of-work protocol, and the attribute to `SqlTraustUnitOfWork` in `repositories/contract.py`. +6. **Seed + tests**: add `new_
()` / `
()` to `Seed` in `tests/v1/repositories/tables/conftest.py` — the per-table CRUD tests then cover it automatically — and a test per query in the section's test module. + +New reads go on the owning table's module: no duplicate queries of the same data, no speculative helpers, never reshape a table to fit a caller. + +## Unit of work + +Services declare the narrow protocol they need (`InventoryUnitOfWork`, `FindingsUnitOfWork`, …, `TraustUnitOfWork` across sections). `SqlTraustUnitOfWork(engine)` opens every repository on one connection, so one `commit()` or `rollback()` covers every section. Services that write together share one instance. ```python -class TaskService: - def __init__(self, uow: TaskUnitOfWork) -> None: +class FindingIntake: + def __init__(self, uow: TraustUnitOfWork) -> None: self._uow = uow - def complete(self, task_id: UUID) -> Task: + def accept(self, assessment_id: UUID, candidates: Sequence[FindingCandidate]) -> IntakeResult: with self._uow as uow: - done = uow.tasks.update(uow.tasks.get(task_id).complete()) # rule on the model - uow.commit() - return done + ... # occurrences, findings, decisions + uow.commit() # all or nothing ``` -## Aggregate repository +## Tests -One repository per aggregate (the thing the domain asks questions about), not per table. `repositories/triage.py` shows the SQL mechanics (table, mapping, `schema_drift`, idempotency); its string ids and `record_triage` free function are legacy and get replaced in the Findings slice — copy the mechanics, not the shape. +Behaviour only, in `tests/v1/repositories/tables/`, each test running on SQLite and Postgres (`make test-repositories`; Postgres needs `make db-up`). Use the `uow` and `seed` fixtures; a test states only what it is about: `seed.decision(seed.finding(), severity="high")`. -1. Domain model in `models/.py` (`TriageVerdict` in `models/findings.py`): fields = the projection's columns, typed. -2. Protocol with the domain questions only (`for_binding`, `for_source_finding`, `true_positives`); no generic CRUD. -3. `sa.Table` describing the contracts table (DDL from traust-contracts); match JSON columns' encoding to existing writers (`JSON(none_as_null=True)`, compact serializer from `create_database_engine`). -4. SQL + in-memory implementations; a unit of work that exposes it alongside any repository it must write with (`evidence` + `triage_verdicts`). -5. Domain service writes in one transaction: `bind_artifact(...)` then the projection, then `commit()`. -6. Tests: contract tests on memory/sqlite (+ Postgres `integration`), `schema_drift` against the DDL, idempotency. -7. Files go by role then subject: `models/.py`, `repositories/.py`, `services/.py`; tests mirror under `tests/v1//`. +- Per query: the right rows come back (and `None` / `[]` when they shouldn't). +- Assert list order only where the query defines it (`occurred_at`, `seq`, `priority`); otherwise compare with `same_rows`. +- Never restate the DDL (columns, constraints, triggers, indexes): contracts owns and tests that. diff --git a/AGENTS.md b/AGENTS.md index 93b08f5d..e80a3d9c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,14 +7,14 @@ Python 3.11+ library. The platform API lives in `traust_core.v1`. Overview: [REA - **One owner per capability.** Never implement a capability twice. Inconsistent data always means a second owner: remove it, never sync it. - **Less is the goal.** One service per domain owns its use cases (reads and writes). Split a service only when it holds two responsibilities (e.g. `services/findings/`). No speculative helpers, no generic bases. - **Place behaviour by responsibility:** - - model: rules about one entity's own state (`Disposition.admit`, `Repository.risk_tier`) + - model: what a row is (one model per contract table, fields = columns) and rules about one value's own state (`Disposition.admit`); table models carry no behaviour - policy: rules needing config or spanning domains (`RatingPolicy`, `IdentityPolicy`) - service: use-case orchestration - - repository: persistence, reads and writes, its own domain's tables only + - repository: one per contract table — save, get, update and the named queries on that table - provider: anything outside the process (tools, forges, feeds, ledger, identity) - **Domain boundaries.** A service never touches another domain's repositories; it calls that domain's service. Several repositories of your own domain in one service are fine — the line is whose repositories, not how many. Repositories never join across domains. Callers compose across domains by typed ID; composite reads are their own query object (`RepoStateQuery`). -- **The database owns identity and time.** Every record has `id` (UUIDv4), `created_at`, `updated_at`, generated by the database on insert (`RETURNING`); on update the contracts `
_touch` trigger keeps `updated_at` — the app never sets it (no SQLAlchemy `onupdate`). On SQLite, `uuid()` is registered by `create_database_engine`. Append-only tables (ledger events) stop at `id` (UUIDv4) and `created_at` — rows are never updated, so there is no `updated_at`. `id is None` = not saved yet; `add` returns the saved record. The app never mints IDs or timestamps. IDs are plain `UUID`, named `_id` (fields and parameters, matching the DDL's `
_id`); never a `str` ID, never a path as an ID, never a path in a signature. Don't re-declare `id` or add your own created/updated timestamps on an `Entity` or `AppendOnly`. Order comes from `created_at`, never from the id. -- **Same shapes everywhere.** Repositories: `get`, `find_*`, `list`, `add`, `update` (`add`/`update` return the saved record). Skill doors: `prepare(...) -> Input` and `record(doc, provenance)` (`@overload` per document). Rollups: `summary(filter, group_by)`; series: `timeline(filter, bucket)`. Deterministic document checks: `Lint`. +- **The database owns identity and time.** Every record has `id` (UUIDv4), `created_at`, `updated_at`, generated by the database on insert (`RETURNING`); on update the contracts `
_touch` trigger keeps `updated_at` — the app never sets it (no SQLAlchemy `onupdate`). On SQLite, `uuid()` is registered by `create_database_engine`. Append-only tables (ledger events) stop at `id` (UUIDv4) and `created_at` — rows are never updated, so there is no `updated_at`. `id is None` = not saved yet; `add` returns the saved record. The app never mints IDs or timestamps. IDs are plain `UUID`, named `_id` (fields and parameters, matching the DDL's `
_id`); never a `str` ID, never a path as an ID, never a path in a signature. Don't re-declare `id` or add your own created/updated timestamps on an `Entity` or `AppendOnly`. Order comes from `created_at` (stamped per statement on both engines); ties break by `id`, which carries no order. +- **Same shapes everywhere.** Repositories: `get`, `find_*`, `list_*`, `add`, `update` (`add`/`update` return the saved record; append-only tables have no `update`). Skill doors: `prepare(...) -> Input` and `record(doc, provenance)` (`@overload` per document). Rollups: `summary(filter, group_by)`; series: `timeline(filter, bucket)`. Deterministic document checks: `Lint`. - **Services return domain results.** `JobResult` exists only at the engine/CLI adapter boundary. - **One code path.** Solo (`traust …`, in-process), small group (`traust serve --with-engine`, one process) and scale (REST API, engine and runners as separate processes) run the same services; only the transport (in-process or REST) and the storage (SQLite + dir, or Postgres + object storage) differ. Never add a mode-specific branch of domain logic; the REST API holds none. CI runs service tests in-process and through the REST adapter, plus one end-to-end skill on SQLite with no server - **Before adding a symbol, find the existing owner.** Extend it; if it truly can't own the new thing, say why in the PR. `grep -rn "NEED(" src` first. @@ -45,7 +45,7 @@ Python 3.11+ library. The platform API lives in `traust_core.v1`. Overview: [REA - `errors` → `contracts` → `models` → `providers` → `repositories` → `rendering` → `services` → `context`; each imports only earlier ones (`tests/v1/test_layering.py`) - One question per folder: `models` what it is and the rules about its own state, `providers` how we reach outside (tools, sources, feeds, process, clock), `repositories` how it is stored, `services` what we do with it (use cases + pure policies), `context` how it is wired -- Organised by role, then subject: `models/.py`, `repositories/.py`, `services/.py`; a subject becomes a package only when it needs several modules (`services/findings/`) +- Organised by role, then subject: `models/
.py` (table models + that section's service types), `repositories/traust/
/
.py`, `repositories/traust_ops/
.py`, `repositories/traust_ledger/
.py` (mirroring the contracts namespaces), `repositories/objects/` (bytes), `services/.py`; a service becomes a package only when it needs several modules (`services/findings/`) - `models/base.py`: `Model` (strict, internal), `Dto` (tolerant, boundary data), `Entity` (`id`, `created_at`, `updated_at` — `None` until the database saves it), `AppendOnly` (`id`, `created_at` — rows are never updated) - Tests mirror `src`: `tests/v1//test_.py` - Pack policy (routing rules, cadences, lanes, skills, CLIs) stays in traust @@ -65,8 +65,8 @@ Python 3.11+ library. The platform API lives in `traust_core.v1`. Overview: [REA - Nothing from traust-contracts is copied or generated into this repo - Artifacts are named types in `models/artifacts.py` (`TriageArtifact`, ...); their fields come from the installed schema, never a hand-written copy - Write with `ArtifactPublisher` (exact schema gate); read with `ArtifactPublisher.get(ArtifactId)`. `AnalysisResultsRepository.read(XArtifact, subject)` is legacy (path-keyed) until the Artifacts slice. No `json.load`/`.get()`/`json.dump` on artifacts -- Names: `models/artifacts.py` = what an artifact is; `repositories/object_store.py` = raw bytes; `services/artifact_publishing.py` = the publish use case. Modules name the subject; the package names the role (no `_service` suffix) -- Every repository `sa.Table` over a contracts table has a `schema_drift` test +- Names: `models/artifacts.py` = what an artifact document is; `models/assessment.py::Artifact` = its `artifact` table row; `repositories/objects/store.py` = raw bytes; `services/artifact_publishing.py` = the publish use case. Modules name the subject; the package names the role (no `_service` suffix) +- Core never tests the DDL: traust-contracts owns and tests its schema. Core tests repository behaviour against it - Artifacts produced by the harness or an LLM go through `ArtifactPublisher`; nothing writes `analysis-results` files directly - Protocols implemented outside core (`Materializer`, `Router` in `services/operations.py`) exist only for a decided pattern or backlog item; each gets an implementation in the repo that owns it - Work is a service. Services return domain results; a `Materializer` (for traust-engine) and a CLI command (for LLMs) are thin adapters that validate input, call one service and turn its result into a `JobResult`; no logic in either @@ -96,9 +96,10 @@ Python 3.11+ library. The platform API lives in `traust_core.v1`. Overview: [REA - Unknown or unmeasured is `None`, never `0` **Structure** -- Repository pattern: one repository per aggregate, written on SQLAlchemy Core; no ORM mapped classes, no generic CRUD base +- Repository pattern: one repository per contract table: a `TableSpec` (columns once, built per dialect by `TableFactory`) and a `SqlTableRepository` subclass, on SQLAlchemy Core; no ORM mapped classes. The shared base gives only `add`/`get`/`update`; every other method is a named query on its table's module - Domain objects never know about the database; rows map to models only inside repositories - Services receive units of work, other services and providers through their constructor; they never open connections, build paths, read env vars or start processes +- A service declares the narrow section protocol it needs (`InventoryUnitOfWork`, `FindingsUnitOfWork`, …, or `TraustUnitOfWork` when it spans sections); `SqlTraustUnitOfWork` satisfies them all on one connection, so services that must write together share one instance and one transaction - `Context` only at entry points; pass the specific dependency, not `Context` - No `subprocess` outside `ProcessRunner`, no clock reads outside `Clock` - Interfaces are `Protocol`s defined where they are consumed @@ -109,7 +110,8 @@ Python 3.11+ library. The platform API lives in `traust_core.v1`. Overview: [REA - Wrap driver and library errors with `raise ... from e`; never swallow them **Tests** -- Every repository, store and provider has contract tests that run against each implementation (memory, sqlite, Postgres when `TRAUST_TEST_DATABASE_URL` is set) +- Table repositories: behaviour tests in `tests/v1/repositories/tables/` — save, get, update and each query — on SQLite and Postgres through the `uow`/`seed` fixtures (`make test-repositories`). No tests that restate the DDL +- Stores and providers: contract tests that run against each implementation - Pure functions get table tests - Test names say the behaviour they protect - A ported legacy scenario that belongs to a later slice is still written now, in its owner's test module, marked `@pytest.mark.todo("", legacy="/tests/::")` (skipped until that slice removes the marker). Never drop a legacy scenario silently diff --git a/README.md b/README.md index ca9936dc..686c6c42 100644 --- a/README.md +++ b/README.md @@ -6,15 +6,15 @@ All modules live under `traust_core.v1` (see [Versioning](#versioning)). | API | Module | Use it for | Instead of | |---|---|---|---| -| **Contracts** | `contracts` | the pinned traust-contracts dependency, read as data: `validate(name, doc)` (exact JSON Schema gate), `schema()`, `enum_values()`, `normalize_enum(name, value)` (registry read view), `ddl()`, `storage_profile()` | skills remembering to run `reporting validate` | +| **Contracts** | `contracts` | the pinned traust-contracts dependency, read as data: `validate(name, doc)` (exact JSON Schema gate), `schema()`, `enum_values()`, `normalize_enum(name, value)` (registry read view), `database_ddl(dialect)` (every namespace, bootstrap order) | skills remembering to run `reporting validate` | | **Errors** | `errors` | the exception hierarchy every layer raises (`ValidationError`, `NotFoundError`, …) | ad hoc exceptions | -| **Models** | `models` | what things are and the rules about their own state: `base` (`Model` strict, `Dto` tolerant, `Entity` with a database-generated UUIDv4 id, `AppendOnly` for tables that are never updated), `values` (`HttpsRepoUrl`, `GitSha`, `CveId`), `integrity`, `storage`, `operations` (`JobResult`, `AssetRequest`, `Order`), and one module per subject (`findings`, `portfolio`, …) | dicts passed between scripts | +| **Models** | `models` | one model per contract table, grouped by section — `inventory`, `dependencies`, `assessment`, `findings`, `operations`, `ledger` (fields = columns) — next to that section's service types; `base` (`Model` strict, `Dto` tolerant, `Entity` with a database-generated UUIDv4 id, `AppendOnly` for tables that are never updated), `values` (`HttpsRepoUrl`, `GitSha`, `CveId`), `integrity` | dicts passed between scripts | | **Artifacts** | `models.artifacts` | the `Artifact` base + `SchemaView` (fields from the installed schema, `doc.findings[0].verdict`, unknown names raise) and the named documents (`TriageArtifact`, `SecurityAuditArtifact`, …, `Verdict`, `Severity`) | `json.load` + `.get("key")` | -| **Artifact publishing** | `services.artifact_publishing` | `ctx.artifact_publisher(specs).publish(name, subject, raw, run_id)` (LLM) or `.publish_artifact(artifact, subject, run_id)` (code): exact gate → files (always) → index (when `database.url` is set) | writing `analysis-results` files by hand | -| **Analysis results** | `repositories.analysis_results` | `ctx.analysis_results().read(TriageArtifact, subject)` / `.read_all(TriageArtifact, tree)`; raw `put/get/find(subject, kind)`; today's naming convention in one place | `json.load` + `"-security-audit.json"` path building | -| **Storage records** | `repositories.storage`, `services.storage` | `record_artifact()` / `get_binding()` over contracts storage/v1 tables | contracts `Store.ingest/get_binding` | -| **Records (pattern)** | `repositories.sql` | `SqlUnitOfWork` + one `SqlRepository` per aggregate on SQLAlchemy Core; sqlite or Postgres by URL. `record_columns()` (mutable: UUIDv4 id + `created_at` + `updated_at`) / `append_only_columns()` (UUIDv4 id + `created_at`, no `updated_at`) + `_insert_record`/`_update_record`, database-generated via `RETURNING`; `updated_at` kept by the contracts touch trigger, SQLite `uuid()` registered by `create_database_engine` | `sqlite3.connect`, inline SQL, client-minted ids/timestamps | -| **Object store** | `repositories.object_store` | `ObjectStore.put/get/list` raw bytes by key, digest-verified (local dir; S3 later) | `write_text`, `json.dump` | +| **Artifact publishing** | `services.artifact_publishing` | `ctx.artifact_publisher(specs).publish(name, subject, raw, run_id)` (LLM) or `.publish_artifact(artifact, subject, run_id)` (code): exact gate → files (always) → an `artifact` row (when `database.url` is set) | writing `analysis-results` files by hand | +| **Analysis results** | `repositories.objects.analysis_results` | `ctx.analysis_results().read(TriageArtifact, subject)` / `.read_all(TriageArtifact, tree)`; raw `put/get/find(subject, kind)`; today's naming convention in one place | `json.load` + `"-security-audit.json"` path building | +| **Table repositories** | `repositories.traust.*`, `repositories.traust_ops`, `repositories.traust_ledger`, `repositories.contract` | one repository per contract table (`uow.findings.find_by_fingerprint(...)`, `uow.run_orders.claim(...)`); `SqlTraustUnitOfWork(engine)` opens all 31 on one connection — one transaction across sections | `sqlite3.connect`, inline SQL, per-script tables | +| **Records (pattern)** | `repositories.sql`, `repositories.tables` | `TableSpec` (columns once) + `TableFactory` (Postgres schemas or SQLite `ops_`/`ledger_` prefixes, by engine) + `SqlTableRepository` (`add`/`get`/`update`) on SQLAlchemy Core; sqlite or Postgres by URL. `record_columns()` (mutable: UUIDv4 id + `created_at` + `updated_at`) / `append_only_columns()` (UUIDv4 id + `created_at`, no `updated_at`) + `_insert_record`/`_update_record`, database-generated via `RETURNING`; `updated_at` kept by the contracts touch trigger, SQLite `uuid()` registered by `create_database_engine` | `sqlite3.connect`, inline SQL, client-minted ids/timestamps | +| **Object store** | `repositories.objects.store` | `ObjectStore.put/get/list` raw bytes by key, digest-verified (local dir; S3 later) | `write_text`, `json.dump` | | **Rendering** | `rendering` | `Report` → `MarkdownRenderer` → `publish()`; `markdown_table()` | 18 copies of `render_md` | | **Jobs** | `models.operations`, `services.operations` | `Materializer.materialize(AssetRequest) -> JobResult` (succeeded · degraded · refused · failed): the seam traust-engine calls; `Router` for continuous-ops `route()` | each CLI inventing its own exit codes | | **Providers** | `providers` | the outside world: `base` (`Provider`, `Readiness`, `Provenance`, registry), `tools`, `sources`, `feeds`, `process` (`ProcessRunner`, the only subprocess path), `clock` | ad hoc `subprocess`/`git`/fetch code, 22 private `run` wrappers | @@ -54,9 +54,9 @@ traust, traust-engine and traust-ledger import core and call services in-process | | Local user | SCI / team | |---|---|---| | Config | `artifacts.path` only | + `database.url` (Postgres), object store | -| `ArtifactPublisher.publish` | validate → files | validate → files → storage binding | +| `ArtifactPublisher.publish` | validate → files | validate → files → `artifact` row | -The database schema is created from the traust-contracts repo by whoever provisions the database; traust-core reads and writes rows and checks its tables with `schema_drift()`. +The database schema is created from the traust-contracts DDL by whoever provisions the database (`apply_schema(engine, contracts.database_ddl(dialect))`); traust-core reads and writes rows. Contracts owns and tests the schema; core tests repository behaviour. ## Usage @@ -78,23 +78,23 @@ All in [`tests/v1/example/`](tests/v1/example): | File | Shows | |---|---| -| `model.py`, `repository.py`, `services.py` | domain model, repository (SQL + in-memory), unit of work, service (example predates the one-service-per-domain rule) | +| `model.py`, `repository.py`, `services.py` | SQL + in-memory repository mechanics over a non-contract table, unit of work, service (predates one-service-per-domain; for contract tables copy `repositories/traust/inventory/repository.py`) | | `artifact_publishing.py` | a publish spec (`TRIAGE` over `TriageArtifact`) with a rendered markdown companion | | `report.py` | read through a repository → pure `build_report` → render → publish | | `router.py` | a pure router, table-tested | | `provider.py` | a tool provider: request DTO, `check()`, `acquire()` with provenance | | `continuous_ops/` | engine → materializer → service → repository → `JobResult`, then a pure router reads the stored state; the same service reached from a CLI | -The real (non-example) aggregate repository to copy is `src/traust_core/v1/repositories/triage.py` (`TriageVerdictRepository`), written by `services/findings/decisions.py::record_triage`. +The real table repository to copy is `src/traust_core/v1/repositories/traust/inventory/repository.py` (a `TableSpec`, a narrow protocol, a `SqlTableRepository` subclass); its tests are in `tests/v1/repositories/tables/`. ## Contracts -traust-contracts is a pinned dependency (`[tool.uv.sources]` git rev in `pyproject.toml`). traust-core reads its **data files** (schemas, enums, DDL, profiles) from the installed package and never imports its Python. Nothing is copied or generated. +traust-contracts is a pinned dependency (`[tool.uv.sources]` git rev in `pyproject.toml`). traust-core reads its **data files** (schemas, enums, DDL) from the installed package, through `traust_core.v1.contracts` only. Nothing is copied or generated. ```mermaid flowchart LR C["traust-contracts @ pinned rev
(installed dependency: schemas · enums · DDL)"] --> G["contracts.validate()
write gate, exact"] - C --> D["contracts.ddl()
create tables"] + C --> D["contracts.database_ddl()
create tables"] C --> R["models.artifacts.TriageArtifact, ...
fields from the schema"] ``` @@ -143,49 +143,47 @@ sibling checkout. No dependency pin or historical payload changes are made. | Kind | Answers | Lives in | |---|---|---| -| **Core** | *what* the data means and *how* it is stored, read and run: models, repositories, services, policies | this repo, by role: `models/.py`, `repositories/.py`, `services/.py` (`services/findings/` is a package) | +| **Core** | *what* the data means and *how* it is stored, read and run: models, repositories, services, policies | this repo, by role: `models/
.py`, `repositories/traust/
/
.py` (+ `traust_ops/`, `traust_ledger/`), `services/.py` (`services/findings/` is a package) | | **Pack policy** | *when / which*: routing rules, cadences, lanes, skills, CLIs | traust (`traust_pack_security`), not this repo | -Everything here is `v1` API, reshaped in place until it is frozen. A concrete repository appears per **aggregate** when there is a domain question to answer; its methods are those questions (see `repositories/triage.py`). Tests mirror `src`: `tests/v1//test_.py`. +Everything here is `v1` API, reshaped in place until it is frozen. Each contract table has one repository; beyond `add`/`get`/`update` its methods are the questions asked of that table. Tests mirror `src`: `tests/v1//test_.py`. ## Rules - Services never open connections, build paths, start processes or read env vars; they receive what they need through their constructor. - `Context` at entry points only. - Data is parsed into a model once, at the edge. No bare `dict` across a function boundary. -- SQLAlchemy Core only: no ORM mapped classes, no generic CRUD base. Domain objects never know about the database. +- SQLAlchemy Core only: no ORM mapped classes. Models never know about the database; the shared repository base gives only `add`/`get`/`update`. - Unknown or unmeasured values are `None`, never `0`. -- Every repository, store and provider has contract tests. +- Every table repository has behaviour tests on SQLite and Postgres; stores and providers have contract tests. - traust-core is the hub: other traust repos depend on it, not the reverse. The only code dependency is the traust-ledger client, inside the ledger provider adapter; traust-contracts is pinned data. HTTP only in providers (outbound) and the HTTP adapter (inbound). -## Domain stubs (not implemented) +## Domain services (stubs) -Per subject: `models/.py`, `repositories/.py` (`Protocol` + unit of work) and `services/.py`. Every method raises `NotImplementedError("stub: replaces …")` naming the code it replaces. Rules: every repository and service lives here; traust-engine only runs work; the LLM reads and writes documents only; identity is a database-generated UUID with `created_at` / `updated_at` (`None` until saved), the fingerprint is only a match key. +Repositories are real; services are mostly stubs. Every stub raises `NotImplementedError("stub: replaces …")` naming the code it replaces. A service declares the section unit of work it needs; `SqlTraustUnitOfWork` satisfies every one on one connection. -| Domain | Module | Service | Repository | -|---|---|---|---| -| Portfolio: what we protect | `services.portfolio` | `PortfolioService` + `PortfolioPolicy` (`exposure_class`, `risk_tier`) | `PortfolioRepository` | -| Assessment: what we looked at | `services.assessment` | `AssessmentService` | `AssessmentRepository` | -| Code facts: deterministic facts per commit | `services.code_facts` | `CodeFactsService` | `CodeFactsRepository` | -| Exposure: known vulns reaching us | `services.exposure` | `ExposureService` | `AdvisoryRepository`, `DependencyGraphRepository` | -| Findings: what's wrong and every decision on it | `services.findings` | `FindingService` (`services/findings/service.py`) wires `intake`, `decisions`, `review`, `precedents` and the stateless `IdentityPolicy`, `TransitionPolicy`, `RatingPolicy` | `FindingRepository` (reads: `current`, `history`, `disposition`, `baseline`, `open_count`) | -| Validation: prove or refute live | `services.validation` | `ValidationService` | `ValidationRepository` | -| Remediation: fix and verify the fix | `services.remediation` | `RemediationService` | `RemediationRepository` | -| Compliance: obligations met | `services.compliance` | `ComplianceService` | `ComplianceRepository` | -| Detection quality: how good detection is | `services.detection_quality` | `DetectionQualityService` | `DetectionQualityRepository` | -| Operations: order, lease, approval, spend state | `services.operations` | `OpsService` | `OpsRepository`, `SpendRepository` | -| Attestation: human sign-off | `services.attestation` | `AttestationService` | `AttestationRepository` | - -Insight (dashboards) has no service: views read repositories and render. Repository protocols are empty until a service implementation needs a question; `FindingRepository` already lists its known reads. Model fields beyond identity are pending the open modelling questions (one Finding, one Decision, Subject/CodeLine). +| Domain | Service | Unit of work (tables) | +|---|---|---| +| Portfolio: what we protect | `services.portfolio` · `PortfolioService`, `PortfolioSync`, `RepoStateQuery` | `InventoryUnitOfWork` (product, release, component, repository, repository_ref, team, ownership) | +| Exposure: known vulns reaching us | `services.exposure` · `ExposureService` | `TraustUnitOfWork` (package, sbom, dependency, image, advisory, advisory_package + findings) | +| Assessment: what we looked at | `services.assessment` · `AssessmentService` | `AssessmentsUnitOfWork` (assessment, artifact, occurrence, location) | +| Findings: what's wrong and every decision on it | `services.findings` · `FindingService` + `intake`, `decisions`, `review`, `precedents`; `IdentityPolicy`, `RatingPolicy` | `TraustUnitOfWork` (finding, cwe, finding_cwe, finding_label, finding_external_ref, decision, review_item); dispositions are SQL views over `decision` | +| Attestation: human sign-off | `services.attestation` · `AttestationService` | `FindingsUnitOfWork` — a human answer is a `decision` that resolves a `review_item` | +| Operations: orders, leases, spend | `services.operations` · `OpsService`, `SpendService` | `OpsUnitOfWork` (run_batch, run_order, spend_event, feed_snapshot, assessment_input) | +| Ledger: proof of decisions | `services.ledger` · `LedgerService` | `LedgerUnitOfWork` (layers, events) | +| PQC, code facts, compliance, detection quality | `services.pqc`, `services.code_facts`, `services.compliance`, `services.detection_quality` | `AssessmentsUnitOfWork` — content is read from the assessment's artifacts, never tables | +| Validation, remediation | `services.validation`, `services.remediation` | `TraustUnitOfWork` — assessments + decisions + external refs + artifacts | + +Insight (dashboards) has no repository: views read services and render. ## Not built yet | Item | Where | |---|---| -| Layout contract (path template + kinds) replacing today's hard-coded convention | traust-contracts, `repositories.analysis_results` | +| Layout contract (path template + kinds) replacing today's hard-coded convention | traust-contracts, `repositories.objects.analysis_results` | | Fingerprint rules (shape fixed; raises until the D5 spec lands) | `models.integrity`, `services.findings.identity` | | First real providers (git, forges, feeds) | packages | -| S3 object store backend | `repositories.object_store` | +| S3 object store backend | `repositories.objects.store` | ## Versioning @@ -201,5 +199,6 @@ make coverage # + coverage (coverage-html for htmlcov/) make db-up # shared traust-postgres container (same as contracts and ledger) make test-integration # Postgres tests (marked `integration`) +make test-repositories # repository layer on SQLite and Postgres make db-down ``` From 1845f9b8502d4648f04b2f89088b5b5a0a5e0e63 Mon Sep 17 00:00:00 2001 From: Shaun Smith <51304449+smith-xyz@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:07:59 -0400 Subject: [PATCH 09/10] refactor(models): rename the schema-backed document base to ArtifactDocument; Artifact is the artifact table row --- .../skills/traust-core-patterns/contracts.md | 2 +- AGENTS.md | 2 +- README.md | 4 +- src/traust_core/v1/models/artifacts.py | 61 +++++++++++-------- .../repositories/objects/analysis_results.py | 8 +-- .../v1/services/artifact_publishing.py | 24 +++++--- src/traust_core/v1/services/lint.py | 8 +-- tests/v1/models/test_artifacts.py | 4 +- 8 files changed, 66 insertions(+), 47 deletions(-) diff --git a/.agents/skills/traust-core-patterns/contracts.md b/.agents/skills/traust-core-patterns/contracts.md index c8e3a401..7291a882 100644 --- a/.agents/skills/traust-core-patterns/contracts.md +++ b/.agents/skills/traust-core-patterns/contracts.md @@ -14,7 +14,7 @@ Write (exact): Read: -- Named artifact types live in `src/traust_core/v1/models/artifacts.py`; add one line per new artifact: `class XArtifact(Artifact, name="x", schema="x", kind=ResultKind.X): ...` +- Named artifact types live in `src/traust_core/v1/models/artifacts.py`; add one line per new artifact: `class XArtifact(ArtifactDocument, name="x", schema="x", kind=ResultKind.X): ...` - Fields come from the installed schema; never declare them by hand - To see what fields exist: `XArtifact.describe()` (types, required/optional) or open `XArtifact.schema_path()` - `ctx.analysis_results().read(XArtifact, subject)` / `.read_all(XArtifact, tree)`; `XArtifact.parse(raw)` when you have bytes diff --git a/AGENTS.md b/AGENTS.md index e80a3d9c..6cc65d4e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -65,7 +65,7 @@ Python 3.11+ library. The platform API lives in `traust_core.v1`. Overview: [REA - Nothing from traust-contracts is copied or generated into this repo - Artifacts are named types in `models/artifacts.py` (`TriageArtifact`, ...); their fields come from the installed schema, never a hand-written copy - Write with `ArtifactPublisher` (exact schema gate); read with `ArtifactPublisher.get(ArtifactId)`. `AnalysisResultsRepository.read(XArtifact, subject)` is legacy (path-keyed) until the Artifacts slice. No `json.load`/`.get()`/`json.dump` on artifacts -- Names: `models/artifacts.py` = what an artifact document is; `models/assessment.py::Artifact` = its `artifact` table row; `repositories/objects/store.py` = raw bytes; `services/artifact_publishing.py` = the publish use case. Modules name the subject; the package names the role (no `_service` suffix) +- Names: `models/artifacts.py` = what an artifact document is (`ArtifactDocument` and its named types); `models/assessment.py::Artifact` = the `artifact` table row; `repositories/objects/store.py` = raw bytes; `services/artifact_publishing.py` = the publish use case. Modules name the subject; the package names the role (no `_service` suffix) - Core never tests the DDL: traust-contracts owns and tests its schema. Core tests repository behaviour against it - Artifacts produced by the harness or an LLM go through `ArtifactPublisher`; nothing writes `analysis-results` files directly - Protocols implemented outside core (`Materializer`, `Router` in `services/operations.py`) exist only for a decided pattern or backlog item; each gets an implementation in the repo that owns it diff --git a/README.md b/README.md index 686c6c42..53e12b58 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ All modules live under `traust_core.v1` (see [Versioning](#versioning)). | **Contracts** | `contracts` | the pinned traust-contracts dependency, read as data: `validate(name, doc)` (exact JSON Schema gate), `schema()`, `enum_values()`, `normalize_enum(name, value)` (registry read view), `database_ddl(dialect)` (every namespace, bootstrap order) | skills remembering to run `reporting validate` | | **Errors** | `errors` | the exception hierarchy every layer raises (`ValidationError`, `NotFoundError`, …) | ad hoc exceptions | | **Models** | `models` | one model per contract table, grouped by section — `inventory`, `dependencies`, `assessment`, `findings`, `operations`, `ledger` (fields = columns) — next to that section's service types; `base` (`Model` strict, `Dto` tolerant, `Entity` with a database-generated UUIDv4 id, `AppendOnly` for tables that are never updated), `values` (`HttpsRepoUrl`, `GitSha`, `CveId`), `integrity` | dicts passed between scripts | -| **Artifacts** | `models.artifacts` | the `Artifact` base + `SchemaView` (fields from the installed schema, `doc.findings[0].verdict`, unknown names raise) and the named documents (`TriageArtifact`, `SecurityAuditArtifact`, …, `Verdict`, `Severity`) | `json.load` + `.get("key")` | +| **Artifacts** | `models.artifacts` | the `ArtifactDocument` base + `SchemaView` (fields from the installed schema, `doc.findings[0].verdict`, unknown names raise) and the named documents (`TriageArtifact`, `SecurityAuditArtifact`, …, `Verdict`, `Severity`) | `json.load` + `.get("key")` | | **Artifact publishing** | `services.artifact_publishing` | `ctx.artifact_publisher(specs).publish(name, subject, raw, run_id)` (LLM) or `.publish_artifact(artifact, subject, run_id)` (code): exact gate → files (always) → an `artifact` row (when `database.url` is set) | writing `analysis-results` files by hand | | **Analysis results** | `repositories.objects.analysis_results` | `ctx.analysis_results().read(TriageArtifact, subject)` / `.read_all(TriageArtifact, tree)`; raw `put/get/find(subject, kind)`; today's naming convention in one place | `json.load` + `"-security-audit.json"` path building | | **Table repositories** | `repositories.traust.*`, `repositories.traust_ops`, `repositories.traust_ledger`, `repositories.contract` | one repository per contract table (`uow.findings.find_by_fingerprint(...)`, `uow.run_orders.claim(...)`); `SqlTraustUnitOfWork(engine)` opens all 31 on one connection — one transaction across sections | `sqlite3.connect`, inline SQL, per-script tables | @@ -102,7 +102,7 @@ flowchart LR |---|---| | Move to a new contracts version | change the `rev` in `pyproject.toml`, `uv sync`, `make test` | | Try an unreleased contracts change | `uv add --editable ../traust-contracts` locally (don't commit) | -| Add a named artifact | one line in `models/artifacts.py`: `class XArtifact(Artifact, name=…, schema=…, kind=…)` | +| Add a named artifact | one line in `models/artifacts.py`: `class XArtifact(ArtifactDocument, name=…, schema=…, kind=…)` | | See an artifact's fields | `TriageArtifact.describe()` (field tree with types, required/optional); `TriageArtifact.schema_path()` | ### Registry reader review candidate diff --git a/src/traust_core/v1/models/artifacts.py b/src/traust_core/v1/models/artifacts.py index 30a10143..19b1eac4 100644 --- a/src/traust_core/v1/models/artifacts.py +++ b/src/traust_core/v1/models/artifacts.py @@ -101,7 +101,7 @@ def describe(nodes: Sequence[Node], depth: int = 0, max_depth: int = 4) -> list[ return lines -class Artifact(ABC): +class ArtifactDocument(ABC): name: ClassVar[str] schema: ClassVar[str] kind: ClassVar[ResultKind | None] @@ -159,7 +159,7 @@ def __dir__(self) -> list[str]: class SecurityAuditArtifact( - Artifact, + ArtifactDocument, name="security-audit", schema="report", kind=ResultKind.SECURITY_AUDIT, @@ -168,7 +168,7 @@ class SecurityAuditArtifact( class FindingsCurrentArtifact( - Artifact, + ArtifactDocument, name="findings-current", schema="report", kind=ResultKind.FINDINGS_CURRENT, @@ -177,25 +177,25 @@ class FindingsCurrentArtifact( class FindingsLayerArtifact( - Artifact, name="findings-layer", schema="layer", kind=ResultKind.FINDINGS_LAYER + ArtifactDocument, name="findings-layer", schema="layer", kind=ResultKind.FINDINGS_LAYER ): ... -class TriageArtifact(Artifact, name="triage", schema="triage", kind=ResultKind.TRIAGE): ... +class TriageArtifact(ArtifactDocument, name="triage", schema="triage", kind=ResultKind.TRIAGE): ... class ThreatModelArtifact( - Artifact, name="threat-model", schema="threat-model", kind=ResultKind.THREAT_MODEL + ArtifactDocument, name="threat-model", schema="threat-model", kind=ResultKind.THREAT_MODEL ): ... class ValidationArtifact( - Artifact, name="validation", schema="validation", kind=ResultKind.VALIDATION + ArtifactDocument, name="validation", schema="validation", kind=ResultKind.VALIDATION ): ... class VerificationArtifact( - Artifact, + ArtifactDocument, name="remediation-verification", schema="verification", kind=ResultKind.REMEDIATION_VERIFICATION, @@ -203,7 +203,7 @@ class VerificationArtifact( class CloudConfigAuditArtifact( - Artifact, + ArtifactDocument, name="cloud-config-audit", schema="cloud-config-audit", kind=ResultKind.CLOUD_CONFIG_AUDIT, @@ -211,50 +211,63 @@ class CloudConfigAuditArtifact( class PrivProfileArtifact( - Artifact, name="priv-profile", schema="operator-priv-profile", kind=ResultKind.PRIV_PROFILE + ArtifactDocument, + name="priv-profile", + schema="operator-priv-profile", + kind=ResultKind.PRIV_PROFILE, ): ... # Skill documents recorded through the domain services (R13); no analysis-results layout. -class AdapterResultArtifact(Artifact, name="adapter-result", schema="adapter-result"): ... +class AdapterResultArtifact(ArtifactDocument, name="adapter-result", schema="adapter-result"): ... -class IsolationReviewArtifact(Artifact, name="isolation-review", schema="isolation-review"): ... +class IsolationReviewArtifact( + ArtifactDocument, name="isolation-review", schema="isolation-review" +): ... -class ImpactAnalysisArtifact(Artifact, name="impact-analysis", schema="impact-analysis"): ... +class ImpactAnalysisArtifact( + ArtifactDocument, name="impact-analysis", schema="impact-analysis" +): ... -class PqcFactsArtifact(Artifact, name="pqc-facts", schema="pqc-facts"): ... +class PqcFactsArtifact(ArtifactDocument, name="pqc-facts", schema="pqc-facts"): ... -class PqcReadinessArtifact(Artifact, name="pqc-readiness", schema="pqc-readiness"): ... +class PqcReadinessArtifact(ArtifactDocument, name="pqc-readiness", schema="pqc-readiness"): ... -class PqcBlockersArtifact(Artifact, name="pqc-blockers", schema="pqc-blockers"): ... +class PqcBlockersArtifact(ArtifactDocument, name="pqc-blockers", schema="pqc-blockers"): ... -class PqcDecisionTreeArtifact(Artifact, name="pqc-decision-tree", schema="pqc-decision-tree"): ... +class PqcDecisionTreeArtifact( + ArtifactDocument, name="pqc-decision-tree", schema="pqc-decision-tree" +): ... -class RemediationArtifact(Artifact, name="remediation", schema="remediation"): ... +class RemediationArtifact(ArtifactDocument, name="remediation", schema="remediation"): ... -class FleetFixArtifact(Artifact, name="fleet-fix", schema="fleet-fix"): ... +class FleetFixArtifact(ArtifactDocument, name="fleet-fix", schema="fleet-fix"): ... -class ComplianceScopeArtifact(Artifact, name="compliance-scope", schema="compliance-scope"): ... +class ComplianceScopeArtifact( + ArtifactDocument, name="compliance-scope", schema="compliance-scope" +): ... class ComplianceAssessmentArtifact( - Artifact, name="compliance-assessment", schema="compliance-assessment" + ArtifactDocument, name="compliance-assessment", schema="compliance-assessment" ): ... -class DocVarianceArtifact(Artifact, name="doc-variance", schema="doc-variance"): ... +class DocVarianceArtifact(ArtifactDocument, name="doc-variance", schema="doc-variance"): ... -class BenchmarkTargetArtifact(Artifact, name="benchmark-target", schema="benchmark-target"): ... +class BenchmarkTargetArtifact( + ArtifactDocument, name="benchmark-target", schema="benchmark-target" +): ... AssessmentArtifact = ( @@ -266,7 +279,7 @@ class BenchmarkTargetArtifact(Artifact, name="benchmark-target", schema="benchma ) PqcArtifact = PqcFactsArtifact | PqcReadinessArtifact | PqcDecisionTreeArtifact -ANALYSIS_ARTIFACTS: tuple[type[Artifact], ...] = ( +ANALYSIS_ARTIFACTS: tuple[type[ArtifactDocument], ...] = ( SecurityAuditArtifact, FindingsCurrentArtifact, FindingsLayerArtifact, diff --git a/src/traust_core/v1/repositories/objects/analysis_results.py b/src/traust_core/v1/repositories/objects/analysis_results.py index 389bdf6c..ac6f0c50 100644 --- a/src/traust_core/v1/repositories/objects/analysis_results.py +++ b/src/traust_core/v1/repositories/objects/analysis_results.py @@ -5,13 +5,13 @@ from traust_core.v1.errors import ConfigError from traust_core.v1.models.analysis_results import ResultKind, Subject -from traust_core.v1.models.artifacts import Artifact +from traust_core.v1.models.artifacts import ArtifactDocument from traust_core.v1.repositories.objects.store import ObjectKey, ObjectRef, ObjectStore -A = TypeVar("A", bound=Artifact) +A = TypeVar("A", bound=ArtifactDocument) -def artifact_kind(artifact: type[Artifact]) -> ResultKind: +def artifact_kind(artifact: type[ArtifactDocument]) -> ResultKind: if artifact.kind is None: raise ConfigError(f"artifact {artifact.name!r} has no analysis-results file kind") return artifact.kind @@ -47,7 +47,7 @@ class AnalysisResultsRepository: def __init__( self, objects: ObjectStore, - artifacts: Mapping[ResultKind, type[Artifact]] | None = None, + artifacts: Mapping[ResultKind, type[ArtifactDocument]] | None = None, layout: ResultsLayout | None = None, ) -> None: self._objects = objects diff --git a/src/traust_core/v1/services/artifact_publishing.py b/src/traust_core/v1/services/artifact_publishing.py index 8633783f..74961d5b 100644 --- a/src/traust_core/v1/services/artifact_publishing.py +++ b/src/traust_core/v1/services/artifact_publishing.py @@ -7,8 +7,8 @@ from traust_core.v1.errors import ConfigError, DocumentError, Issue, TraustError from traust_core.v1.models.analysis_results import ResultKind, Subject -from traust_core.v1.models.artifacts import Artifact -from traust_core.v1.models.assessment import Artifact as ArtifactRow +from traust_core.v1.models.artifacts import ArtifactDocument +from traust_core.v1.models.assessment import Artifact from traust_core.v1.models.base import Dto from traust_core.v1.repositories.objects.analysis_results import ( AnalysisResultsRepository, @@ -17,7 +17,7 @@ from traust_core.v1.repositories.objects.store import ObjectRef from traust_core.v1.repositories.traust.assessments import AssessmentsUnitOfWork -A = TypeVar("A", bound=Artifact) +A = TypeVar("A", bound=ArtifactDocument) class InputFormat(Protocol): @@ -72,7 +72,9 @@ def names(self) -> list[str]: class ArtifactIndex(Protocol): - def record(self, artifact: Artifact, subject: Subject, ref: ObjectRef, run_id: str) -> UUID: ... + def record( + self, artifact: ArtifactDocument, subject: Subject, ref: ObjectRef, run_id: str + ) -> UUID: ... class SqlArtifactIndex: @@ -81,10 +83,12 @@ class SqlArtifactIndex: def __init__(self, uow: Callable[[], AssessmentsUnitOfWork]) -> None: self._uow = uow - def record(self, artifact: Artifact, subject: Subject, ref: ObjectRef, run_id: str) -> UUID: + def record( + self, artifact: ArtifactDocument, subject: Subject, ref: ObjectRef, run_id: str + ) -> UUID: with self._uow() as u: row = u.artifacts.find(None, ref.sha256) or u.artifacts.add( - ArtifactRow(digest=ref.sha256, kind=artifact.schema, uri=str(ref.key)) + Artifact(digest=ref.sha256, kind=artifact.schema, uri=str(ref.key)) ) u.commit() assert row.id is not None @@ -118,7 +122,9 @@ def publish(self, name: str, subject: Subject, raw: bytes, run_id: str) -> Publi artifact = spec.artifact.parse(spec.input.to_json(name, raw)) return self.publish_artifact(artifact, subject, run_id) - def publish_artifact(self, artifact: Artifact, subject: Subject, run_id: str) -> PublishResult: + def publish_artifact( + self, artifact: ArtifactDocument, subject: Subject, run_id: str + ) -> PublishResult: spec = self._specs.get(artifact.name) primary = self._results.put(subject, artifact_kind(spec.artifact), artifact.payload) written = [ @@ -134,13 +140,13 @@ def publish_artifact(self, artifact: Artifact, subject: Subject, run_id: str) -> return result.model_copy(update={"index_error": str(e)}) return result.model_copy(update={"artifact_id": artifact_id}) - def get(self, artifact_id: UUID) -> Artifact: + def get(self, artifact_id: UUID) -> ArtifactDocument: raise NotImplementedError("stub: replaces AnalysisResultsRepository.read by path subject") def verify(self, artifact_id: UUID) -> DigestCheck: raise NotImplementedError("stub: replaces verify_report_digests.py, ledger report digests") - def list_for_assessment(self, assessment_id: UUID) -> list[ArtifactRow]: + def list_for_assessment(self, assessment_id: UUID) -> list[Artifact]: raise NotImplementedError("stub: replaces corpus/report_store listing") diff --git a/src/traust_core/v1/services/lint.py b/src/traust_core/v1/services/lint.py index bdfbc592..c764d104 100644 --- a/src/traust_core/v1/services/lint.py +++ b/src/traust_core/v1/services/lint.py @@ -3,7 +3,7 @@ from collections.abc import Mapping, Sequence from typing import Protocol -from traust_core.v1.models.artifacts import Artifact +from traust_core.v1.models.artifacts import ArtifactDocument from traust_core.v1.models.base import Model @@ -23,14 +23,14 @@ class LintReport(Model): ... class LintRule(Protocol): id: LintRuleId - def check(self, subject: Artifact, context: LintContext) -> list[Violation]: ... + def check(self, subject: ArtifactDocument, context: LintContext) -> list[Violation]: ... class Lint: - def __init__(self, rules: Mapping[type[Artifact], Sequence[LintRule]]) -> None: + def __init__(self, rules: Mapping[type[ArtifactDocument], Sequence[LintRule]]) -> None: self._rules = rules - def run(self, subject: Artifact, context: LintContext) -> LintReport: + def run(self, subject: ArtifactDocument, context: LintContext) -> LintReport: raise NotImplementedError( "stub: replaces check_citations, lint_verdict_citations," " validate_compliance_assessment, validate_isolation_review," diff --git a/tests/v1/models/test_artifacts.py b/tests/v1/models/test_artifacts.py index 07ab5541..dbdde98f 100644 --- a/tests/v1/models/test_artifacts.py +++ b/tests/v1/models/test_artifacts.py @@ -5,7 +5,7 @@ from tests.v1.fixtures import TRIAGE from traust_core.v1 import contracts from traust_core.v1.errors import DocumentError -from traust_core.v1.models.artifacts import ANALYSIS_ARTIFACTS, Artifact, TriageArtifact +from traust_core.v1.models.artifacts import ANALYSIS_ARTIFACTS, ArtifactDocument, TriageArtifact from traust_core.v1.models.vocabulary import Severity, Verdict FINDING = { @@ -62,7 +62,7 @@ def test_cross_file_refs_resolve_into_the_target_schema() -> None: def test_artifact_subclasses_must_declare_a_schema() -> None: with pytest.raises(TypeError): - class Broken(Artifact): ... + class Broken(ArtifactDocument): ... def test_schema_path_points_at_the_installed_contract() -> None: From 6ffc88110a1a683c5dbc258d24b606e4e76cc8ee Mon Sep 17 00:00:00 2001 From: Shaun Smith <51304449+smith-xyz@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:10:37 -0400 Subject: [PATCH 10/10] chore(contracts): pin 42a6367 (statement_timestamp() base columns) --- pyproject.toml | 2 +- tests/v1/repositories/tables/conftest.py | 4 +--- uv.lock | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 57a48c3c..849a28f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dev = [ ] [tool.uv.sources] -traust-contracts = { git = "https://github.com/traust-security/traust-contracts.git", rev = "1da48c06493e5b88b99445299e025bfc7c46673a" } +traust-contracts = { git = "https://github.com/traust-security/traust-contracts.git", rev = "42a6367906bd935481a48f4877ed0f9de7ba5c79" } [tool.hatch.build.targets.wheel] packages = ["src/traust_core"] diff --git a/tests/v1/repositories/tables/conftest.py b/tests/v1/repositories/tables/conftest.py index e9857203..6ac71b88 100644 --- a/tests/v1/repositories/tables/conftest.py +++ b/tests/v1/repositories/tables/conftest.py @@ -92,9 +92,7 @@ def repository_for(uow: SqlTraustUnitOfWork, table: str) -> SqlTableRepository[A def same_rows(actual: Sequence[Model], expected: Sequence[Model]) -> bool: - """Equal as sets of rows. Rows written in one Postgres transaction share created_at - (now() is the transaction's start), so list order is only asserted where a - query defines it (occurred_at, seq, priority).""" + """Equal as sets of rows; list order is asserted only where a query defines it.""" key = lambda m: str(m.id) # type: ignore[attr-defined] # noqa: E731 return sorted(actual, key=key) == sorted(expected, key=key) diff --git a/uv.lock b/uv.lock index 9a0ae5f4..3eb80888 100644 --- a/uv.lock +++ b/uv.lock @@ -938,7 +938,7 @@ wheels = [ [[package]] name = "traust-contracts" version = "0.0.0" -source = { git = "https://github.com/traust-security/traust-contracts.git?rev=1da48c06493e5b88b99445299e025bfc7c46673a#1da48c06493e5b88b99445299e025bfc7c46673a" } +source = { git = "https://github.com/traust-security/traust-contracts.git?rev=42a6367906bd935481a48f4877ed0f9de7ba5c79#42a6367906bd935481a48f4877ed0f9de7ba5c79" } dependencies = [ { name = "jsonschema", extra = ["format-nongpl"] }, { name = "pydantic" }, @@ -978,7 +978,7 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6.0" }, { name = "referencing", specifier = ">=0.30" }, { name = "sqlalchemy", specifier = ">=2.0" }, - { name = "traust-contracts", git = "https://github.com/traust-security/traust-contracts.git?rev=1da48c06493e5b88b99445299e025bfc7c46673a" }, + { name = "traust-contracts", git = "https://github.com/traust-security/traust-contracts.git?rev=42a6367906bd935481a48f4877ed0f9de7ba5c79" }, ] provides-extras = ["postgres"]