diff --git a/AGENTS.md b/AGENTS.md index 5b2aa042859..771804ef71c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,6 +89,11 @@ The project uses `poetry` for Python dependency management, `pnpm` for website f ```bash make build-protos ``` +- **PostgreSQL Codegen (`sqlc`)**: + If you modify [`go/internal/database/postgres/queries.sql`](go/internal/database/postgres/queries.sql) or [`go/internal/database/postgres/migrations/`](go/internal/database/postgres/migrations/), regenerate the Go query code (`go install github.com/sqlc-dev/sqlc/cmd/sqlc@latest` if not installed): + ```bash + cd go/internal/database/postgres && sqlc generate + ``` --- @@ -301,7 +306,7 @@ All Go microservices are compiled using a single, unified multi-target Dockerfil - *Design Pattern*: Models here **mirror** the Datastore models defined in the Python library ([`osv/models.py`](osv/models.py)). - *Consistency Testing*: To prevent synchronization drift between Go and Python database models, a database validation test is maintained under [`go/internal/database/datastore/internal/validate/`](go/internal/database/datastore/internal/validate/) (run via `run_validate.sh`). - *Schema Quirks (Crucial for Agents)*: In the Datastore database, the legacy term `bug` was used for vulnerabilities. Consequently, many Datastore fields still use names like `bug_id` or `bug_ids`. In the Go codebase, these are mapped to Go struct fields like `VulnID` or `VulnIDs` (e.g., `AffectedCommits` has `VulnID string datastore:"bug_id"`). Pay close attention to the `datastore:` tag when writing queries or defining new fields! - - **`postgres/`**: PostgreSQL implementation of the domain interfaces using `sqlc` and `pgx/v5` ([`go/internal/database/postgres/`](go/internal/database/postgres/)). + - **`postgres/`**: PostgreSQL implementation of the domain interfaces using `sqlc` and `pgx/v5` ([`go/internal/database/postgres/`](go/internal/database/postgres/)). Schema migrations are managed with `goose` under [`go/internal/database/postgres/migrations/`](go/internal/database/postgres/migrations/) and applied automatically in CI/CD via `gcr.io/oss-vdb/goose` ([`docker/goose/`](docker/goose/)). - **`dualread/`**: Shadow dual-reader wrapper ([`go/internal/database/dualread/`](go/internal/database/dualread/)) that serves primary reads from Datastore/GCS while probabilistically shadowing queries (`POSTGRES_SHADOW_RATE`) to PostgreSQL in the background for latency comparison. - **`gitter/`**: Client logic to interface with the Gitter caching service. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 401b284d71d..6feca7f324a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,6 +69,8 @@ You must install: 1. [Go](https://go.dev/doc/install) >= 1.26 1. `protobuf-compiler` (e.g. `protoc` command, install via `sudo apt install protobuf-compiler` on Debian/Ubuntu) 1. `protoc-gen-go` and `protoc-gen-go-grpc` (Go proto plugins, install via `go install google.golang.org/protobuf/cmd/protoc-gen-go@latest` and `go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest`. Ensure your Go bin directory, e.g. `~/go/bin`, is in your `PATH`) +1. [sqlc](https://docs.sqlc.dev/en/latest/overview/install.html) (optional, only if modifying PostgreSQL queries/schema; install via `go install github.com/sqlc-dev/sqlc/cmd/sqlc@latest`) +1. `cloud-sql-proxy` (optional, maintainers only for connecting to Cloud SQL; install via `gcloud components install cloud-sql-proxy`, `go install github.com/GoogleCloudPlatform/cloud-sql-proxy/v2@latest`, or the [standalone binary](https://cloud.google.com/sql/docs/postgres/connect-instance-auth-proxy#install)) Then you can set up the development environment by cloning the OSV repo and installing the Poetry dependencies. diff --git a/Makefile b/Makefile index 27c04a53541..fe023b760e0 100644 --- a/Makefile +++ b/Makefile @@ -127,6 +127,12 @@ run-api-server-test: run-grafana-local: ## Run local Grafana for testing ./tools/run_grafana_local.sh $(ARGS) +run-cloud-sql-proxy: ## Run Cloud SQL Auth Proxy against prod (override via INSTANCE=project:region:instance) + cloud-sql-proxy "$${INSTANCE:-oss-vdb:us-central1:osv-db}" --auto-iam-authn + +run-cloud-sql-proxy-staging: ## Run Cloud SQL Auth Proxy against staging (oss-vdb-test) + cloud-sql-proxy "oss-vdb-test:us-central1:osv-db" --auto-iam-authn + # TODO: API integration tests. all-tests: lib-tests vulnfeed-tests bindings-tests go-tests ## Run all tests diff --git a/db_test/20261001064316_test.sql b/db_test/20261001064316_test.sql deleted file mode 100644 index 6fdfd1a6546..00000000000 --- a/db_test/20261001064316_test.sql +++ /dev/null @@ -1,49 +0,0 @@ --- +goose Up -CREATE TABLE vulnerabilities ( - id TEXT PRIMARY KEY, -- NB: This should have COLLATE "C" - data JSONB NOT NULL, - proto BYTEA NOT NULL -); --- vuln_id does not reference vulnerabilities(id) because there is no index on (vuln_id) --- (to save space/writes), which would cause a full table scan on vulnerability deletes. --- Use vuln_commits to look up commits and delete by (commit_sha, vuln_id). -CREATE TABLE affected_commits ( - commit_sha BYTEA NOT NULL, - vuln_id TEXT NOT NULL, - PRIMARY KEY (commit_sha, vuln_id) -); -CREATE TABLE vuln_commits ( - vuln_id TEXT PRIMARY KEY REFERENCES vulnerabilities(id) ON DELETE CASCADE, - commits BYTEA[] -); - -CREATE TABLE affected_versions ( - id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - vuln_id TEXT NOT NULL REFERENCES vulnerabilities(id) ON DELETE CASCADE, - name TEXT NOT NULL, - base_ecosystem TEXT NOT NULL, - ecosystems TEXT[] NOT NULL, - coarse_min TEXT COLLATE "C" NOT NULL, - coarse_max TEXT COLLATE "C" NOT NULL, - versions TEXT[], - events JSONB -); - --- Primary index for typical queries (with ecosystem provided) -CREATE INDEX idx_affected_versions_match -ON affected_versions (name, base_ecosystem, vuln_id, id, coarse_min, coarse_max) -INCLUDE (ecosystems); - --- Secondary index for queries without ecosystems -CREATE INDEX idx_affected_versions_name_only -ON affected_versions (name, vuln_id, id); - --- Index for worker updates/deletes by vuln_id -CREATE INDEX idx_affected_versions_vuln_id ON affected_versions (vuln_id); - - --- +goose Down -DROP TABLE affected_versions; -DROP TABLE vuln_commits; -DROP TABLE affected_commits; -DROP TABLE vulnerabilities; diff --git a/deployment/README.md b/deployment/README.md index d8be59b80ef..11ef55243c6 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -85,6 +85,38 @@ printf "%s" "$PG_RO_PASS" | gcloud secrets versions add osv-db-readonly-password unset PG_RO_PASS ``` +### Cloud SQL PostgreSQL Schema Migrations & IAM Access + +Database schema migrations live in `go/internal/database/postgres/migrations/` and are managed with [goose](https://github.com/pressly/goose). + +* **Staging (`oss-vdb-test`)**: Applied automatically by `build-and-stage.yaml` right after `terraform apply` using the `gcr.io/oss-vdb/goose` Cloud Builder image (see `docker/goose/`). +* **Production (`oss-vdb` & `private-osv`)**: Applied automatically by `deploy-prod.yaml` at the release `$COMMIT_SHA` right before promoting the Cloud Deploy pipelines. + +To connect or run `goose` manually from your workstation via IAM authentication: + +1. Install `cloud-sql-proxy` (via `gcloud components install cloud-sql-proxy`, `go install github.com/GoogleCloudPlatform/cloud-sql-proxy/v2@latest`, or the [standalone binary](https://cloud.google.com/sql/docs/postgres/connect-instance-auth-proxy#install)). +2. Create a Cloud SQL IAM user for your Google account (once per instance): + ```bash + gcloud sql users create "$(gcloud config get-value account)" \ + --instance=osv-db \ + --project="$PROJECT_ID" \ + --type=CLOUD_IAM_USER \ + --database-roles=cloudsqlsuperuser + ``` +3. Start `cloud-sql-proxy` with automatic IAM authentication (or via `make run-cloud-sql-proxy-staging` / `make run-cloud-sql-proxy`): + ```bash + # Use us-central1 for oss-vdb / oss-vdb-test, or us-west2 for private-osv + cloud-sql-proxy "${PROJECT_ID}:us-central1:osv-db" --auto-iam-authn + ``` +4. Run `goose` (if needed, setting `role=cloudsqlsuperuser` so any created/altered objects remain owned by `cloudsqlsuperuser`): + ```bash + goose -dir go/internal/database/postgres/migrations postgres \ + "host=127.0.0.1 port=5432 user=$(gcloud config get-value account) dbname=osv sslmode=disable options=-crole=cloudsqlsuperuser" \ + status + ``` + +> **Reverting migrations**: In production, prefer pushing a new forward-fix migration PR rather than rolling back history. For immediate rollbacks of an additive migration, you can connect via `cloud-sql-proxy` and run `goose ... down` (which executes the `-- +goose Down` block of the latest version), then revert the migration file in git. + ## Setting up auto-scaler diff --git a/deployment/build-and-stage.yaml b/deployment/build-and-stage.yaml index 96efa853456..cf8cda458d0 100644 --- a/deployment/build-and-stage.yaml +++ b/deployment/build-and-stage.yaml @@ -541,6 +541,12 @@ steps: args: ['apply', '-no-color', '-auto-approve'] dir: deployment/terraform/environments/oss-vdb-test +# Apply PostgreSQL schema migrations to staging environment +- name: gcr.io/oss-vdb/goose + env: ['CLOUDSQL_INSTANCE=oss-vdb-test:us-central1:osv-db'] + args: ['up'] + dir: go/internal/database/postgres/migrations + # Cloud Deploy # GKE Workers - name: 'gcr.io/cloud-builders/gcloud' diff --git a/deployment/clouddeploy/osv-api/run-prod.yaml b/deployment/clouddeploy/osv-api/run-prod.yaml index e1ad9dc1b64..611872184eb 100644 --- a/deployment/clouddeploy/osv-api/run-prod.yaml +++ b/deployment/clouddeploy/osv-api/run-prod.yaml @@ -25,9 +25,9 @@ spec: - name: OSV_NORMALIZE_MATCH_NAMES value: "true" - name: POSTGRES_SHADOW_RATE - value: "1.0" + value: "0.0" - name: POSTGRES_HOST - value: "10.25.0.3" + value: "10.25.1.7" - name: POSTGRES_PORT value: "6432" - name: POSTGRES_USER @@ -39,7 +39,7 @@ spec: - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: - name: postgres-readonly-password + name: osv-db-readonly-password key: latest resources: limits: diff --git a/deployment/clouddeploy/osv-api/run-staging.yaml b/deployment/clouddeploy/osv-api/run-staging.yaml index 159fbe1702c..90441bfcbd6 100644 --- a/deployment/clouddeploy/osv-api/run-staging.yaml +++ b/deployment/clouddeploy/osv-api/run-staging.yaml @@ -27,9 +27,9 @@ spec: - name: OSV_NORMALIZE_MATCH_NAMES value: "true" - name: POSTGRES_SHADOW_RATE - value: "1.0" + value: "0.0" - name: POSTGRES_HOST - value: "172.24.0.3" + value: "172.24.1.6" - name: POSTGRES_PORT value: "6432" - name: POSTGRES_USER @@ -41,7 +41,7 @@ spec: - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: - name: postgres-readonly-password + name: osv-db-readonly-password key: latest resources: limits: diff --git a/deployment/deploy-prod.yaml b/deployment/deploy-prod.yaml index 6843a2b222a..394211a08e8 100644 --- a/deployment/deploy-prod.yaml +++ b/deployment/deploy-prod.yaml @@ -13,6 +13,27 @@ # limitations under the License. steps: +# Fetch PostgreSQL migrations at the release commit and apply to prod and private-osv +- name: gcr.io/cloud-builders/git + entrypoint: bash + env: ['COMMIT_SHA=$COMMIT_SHA'] + args: + - '-c' + - | + set -euo pipefail + git clone --filter=blob:none --no-checkout https://github.com/google/osv.dev.git repo + cd repo + git sparse-checkout set go/internal/database/postgres/migrations + git checkout "$COMMIT_SHA" +- name: gcr.io/oss-vdb/goose + env: ['CLOUDSQL_INSTANCE=oss-vdb:us-central1:osv-db'] + args: ['up'] + dir: repo/go/internal/database/postgres/migrations +- name: gcr.io/oss-vdb/goose + env: ['CLOUDSQL_INSTANCE=private-osv:us-west2:osv-db'] + args: ['up'] + dir: repo/go/internal/database/postgres/migrations + # Promote Cloud Deploy - name: gcr.io/cloud-builders/gcloud args: ['deploy', 'releases', 'promote', '--quiet', '--release=osv-$SHORT_SHA', '--region=us-central1', '--delivery-pipeline=gke-workers', '--to-target=production-workers', '--annotations=tag=$TAG_NAME'] diff --git a/deployment/terraform/environments/private-osv/main.tf b/deployment/terraform/environments/private-osv/main.tf index 548215828e5..9f3cc8bd209 100644 --- a/deployment/terraform/environments/private-osv/main.tf +++ b/deployment/terraform/environments/private-osv/main.tf @@ -26,7 +26,6 @@ module "osv_pipeline" { postgres_region = "us-west2" postgres_tier = "db-perf-optimized-N-2" postgres_availability_type = "ZONAL" - postgres_ipv4_enabled = false } diff --git a/deployment/terraform/modules/osv_pipeline/database.tf b/deployment/terraform/modules/osv_pipeline/database.tf index 8c8e6313480..1131c80ddc9 100644 --- a/deployment/terraform/modules/osv_pipeline/database.tf +++ b/deployment/terraform/modules/osv_pipeline/database.tf @@ -115,6 +115,16 @@ resource "google_sql_user" "worker" { depends_on = [google_sql_database.osv] } +resource "google_sql_user" "deployment" { + project = var.project_id + instance = google_sql_database_instance.postgres.name + name = "deployment@oss-vdb.iam" + type = "CLOUD_IAM_SERVICE_ACCOUNT" + database_roles = ["cloudsqlsuperuser"] + + depends_on = [google_sql_user.worker] +} + resource "google_secret_manager_secret" "postgres_worker_password" { project = var.project_id secret_id = "osv-db-worker-password" diff --git a/deployment/terraform/modules/osv_pipeline/iam.tf b/deployment/terraform/modules/osv_pipeline/iam.tf index 0119a6dbdc4..6d6e0ee9393 100644 --- a/deployment/terraform/modules/osv_pipeline/iam.tf +++ b/deployment/terraform/modules/osv_pipeline/iam.tf @@ -107,3 +107,15 @@ resource "google_secret_manager_secret_iam_member" "worker_postgres_secret_acces member = "serviceAccount:${google_service_account.worker_sa.email}" } +# Cloud SQL IAM authentication permissions for the Cloud Build deployment service account (goose migrations) +resource "google_project_iam_member" "deployment_cloudsql_roles" { + for_each = toset([ + "roles/cloudsql.client", + "roles/cloudsql.instanceUser", + ]) + + project = var.project_id + role = each.value + member = "serviceAccount:deployment@oss-vdb.iam.gserviceaccount.com" +} + diff --git a/deployment/terraform/modules/osv_pipeline/outputs.tf b/deployment/terraform/modules/osv_pipeline/outputs.tf index f542a0e6a5d..5b701aaa811 100644 --- a/deployment/terraform/modules/osv_pipeline/outputs.tf +++ b/deployment/terraform/modules/osv_pipeline/outputs.tf @@ -74,6 +74,7 @@ output "postgres_instance_name" { depends_on = [ google_sql_database.osv, google_sql_user.worker, + google_sql_user.deployment, ] } diff --git a/docker/goose/Dockerfile b/docker/goose/Dockerfile new file mode 100644 index 00000000000..200fc2de4ae --- /dev/null +++ b/docker/goose/Dockerfile @@ -0,0 +1,19 @@ +FROM golang:1.27.1-alpine@sha256:8a5910f31396cd4d89662f56c68b3ae31d374308270a1c3bd96672ee5ed43414 AS GO_BUILD + +ARG GOOSE_VERSION=3.28.0 +ARG CLOUD_SQL_PROXY_VERSION=2.26.0 +WORKDIR /build/ +RUN GOBIN=$(pwd) go install \ + -tags='no_clickhouse no_libsql no_mssql no_mysql no_redshift no_sqlite3 no_turso no_vertica no_ydb' \ + github.com/pressly/goose/v3/cmd/goose@v${GOOSE_VERSION} +RUN GOBIN=$(pwd) go install \ + github.com/GoogleCloudPlatform/cloud-sql-proxy/v2@v${CLOUD_SQL_PROXY_VERSION} + +FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:786a1537df13c54289a46507b22a411f6c71154ae0293d12adce4b9358acca63 + +COPY --from=GO_BUILD /build/goose /usr/bin/goose +COPY --from=GO_BUILD /build/cloud-sql-proxy /usr/bin/cloud-sql-proxy +COPY entrypoint.bash /builder/entrypoint.bash +RUN chmod +x /builder/entrypoint.bash + +ENTRYPOINT ["/builder/entrypoint.bash"] diff --git a/docker/goose/README.md b/docker/goose/README.md new file mode 100644 index 00000000000..575413f6886 --- /dev/null +++ b/docker/goose/README.md @@ -0,0 +1,19 @@ +# Goose Cloud Builder Image + +Used for running PostgreSQL schema migrations (`goose`) via Cloud SQL Auth Proxy (`cloud-sql-proxy`) in Cloud Build (`build-and-stage.yaml` and `deploy-prod.yaml`). Contains `goose`, `cloud-sql-proxy`, and `gcloud`. + +To build and push the builder image, run from this directory: +```bash +gcloud builds submit --project=oss-vdb --config=cloudbuild.yaml +``` + +Afterwards, it can be used in Cloud Build as: +```yaml +steps: +- name: gcr.io/oss-vdb/goose + env: ['CLOUDSQL_INSTANCE=oss-vdb-test:us-central1:osv-db'] + args: ['up'] + dir: go/internal/database/postgres/migrations +``` + +When `CLOUDSQL_INSTANCE` is set, the entrypoint starts `cloud-sql-proxy --auto-iam-authn` in the background, waits for its readiness probe, connects as the active Cloud Build IAM user (`deployment@oss-vdb.iam`) with `role=cloudsqlsuperuser`, and runs `goose`. diff --git a/docker/goose/cloudbuild.yaml b/docker/goose/cloudbuild.yaml new file mode 100644 index 00000000000..33f9d0ce9d0 --- /dev/null +++ b/docker/goose/cloudbuild.yaml @@ -0,0 +1,24 @@ +# In this directory, run the following command to build this builder: +# $ gcloud builds submit . --project=oss-vdb --config=cloudbuild.yaml +substitutions: + _GOOSE_VERSION: 3.28.0 + _CLOUD_SQL_PROXY_VERSION: 2.26.0 +steps: +- name: 'gcr.io/cloud-builders/docker' + args: + - build + - --build-arg + - GOOSE_VERSION=${_GOOSE_VERSION} + - --build-arg + - CLOUD_SQL_PROXY_VERSION=${_CLOUD_SQL_PROXY_VERSION} + - --tag + - gcr.io/${PROJECT_ID}/goose:${_GOOSE_VERSION} + - --tag + - gcr.io/${PROJECT_ID}/goose:latest + - . +- name: 'gcr.io/${PROJECT_ID}/goose:${_GOOSE_VERSION}' + args: ['version'] +images: + - 'gcr.io/${PROJECT_ID}/goose:${_GOOSE_VERSION}' + - 'gcr.io/${PROJECT_ID}/goose:latest' +tags: ['cloud-builders-community'] diff --git a/docker/goose/entrypoint.bash b/docker/goose/entrypoint.bash new file mode 100644 index 00000000000..d709a6d2b70 --- /dev/null +++ b/docker/goose/entrypoint.bash @@ -0,0 +1,55 @@ +#!/bin/bash +# +# Goose + Cloud SQL Auth Proxy cloud builder entry point. +set -euo pipefail + +if [[ "${1:-}" == "version" || "${1:-}" == "-version" || "${1:-}" == "--version" ]]; then + goose -version + cloud-sql-proxy --version + exit 0 +fi + +if [[ -n "${CLOUDSQL_INSTANCE:-}" ]]; then + active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2> /dev/null) + if [[ -z "${active_account}" ]]; then + echo "No active gcloud account found for Cloud SQL IAM authentication." + exit 1 + fi + + echo "Starting cloud-sql-proxy for ${CLOUDSQL_INSTANCE}..." + cloud-sql-proxy "${CLOUDSQL_INSTANCE}" \ + --port=5432 \ + --auto-iam-authn \ + --health-check \ + --http-port=9090 & + PROXY_PID=$! + trap 'kill "${PROXY_PID}" 2>/dev/null || true' EXIT + + ready=false + for _ in {1..30}; do + if ! kill -0 "${PROXY_PID}" 2>/dev/null; then + echo "cloud-sql-proxy exited unexpectedly." + wait "${PROXY_PID}" || true + exit 1 + fi + if curl -sf http://127.0.0.1:9090/readiness >/dev/null 2>&1; then + ready=true + break + fi + sleep 1 + done + + if [[ "${ready}" != "true" ]]; then + echo "Timed out waiting for cloud-sql-proxy readiness." + exit 1 + fi + + # Cloud SQL IAM service account usernames omit the .gserviceaccount.com suffix. + db_user="${PGUSER:-${active_account%.gserviceaccount.com}}" + db_name="${PGDATABASE:-osv}" + export GOOSE_DRIVER="${GOOSE_DRIVER:-postgres}" + export GOOSE_DBSTRING="${GOOSE_DBSTRING:-host=127.0.0.1 port=5432 user=${db_user} dbname=${db_name} sslmode=disable options=-crole=cloudsqlsuperuser}" +fi + +echo "Running: goose $*" +goose "$@" diff --git a/go/cloudbuild.yaml b/go/cloudbuild.yaml index c35584439e6..b11cfdb6017 100644 --- a/go/cloudbuild.yaml +++ b/go/cloudbuild.yaml @@ -19,6 +19,8 @@ # go/run_tests.sh # go/*.go # go/**/*.go +# go/**/*.sql +# go/internal/database/postgres/sqlc.yaml steps: - name: 'gcr.io/cloud-builders/git' diff --git a/go/internal/database/postgres/migrations/00001_initial_core_tables.sql b/go/internal/database/postgres/migrations/00001_initial_core_tables.sql new file mode 100644 index 00000000000..16327fd4f41 --- /dev/null +++ b/go/internal/database/postgres/migrations/00001_initial_core_tables.sql @@ -0,0 +1,103 @@ +-- +goose Up +CREATE TABLE raw_vulnerabilities ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + vuln_id TEXT COLLATE "C", + source_id TEXT COLLATE "C" NOT NULL, + source_path TEXT COLLATE "C" NOT NULL, + content_sha256 BYTEA NOT NULL, + git_commit_hash TEXT COLLATE "C", + gcs_generation BIGINT, + raw_data JSONB NOT NULL, + fetched_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uq_raw_vuln_source_hash UNIQUE (source_id, source_path, content_sha256) +); + +CREATE INDEX idx_raw_vuln_lookup ON raw_vulnerabilities (source_id, source_path, fetched_at DESC); + +CREATE TABLE vulnerabilities ( + id TEXT COLLATE "C" NOT NULL, + revision INT NOT NULL DEFAULT 1, + source_id TEXT COLLATE "C" NOT NULL, + source_path TEXT COLLATE "C" NOT NULL, + raw_vuln_id BIGINT REFERENCES raw_vulnerabilities(id), + modified TIMESTAMPTZ NOT NULL, + source_modified TIMESTAMPTZ, + withdrawn TIMESTAMPTZ, + alias_raw TEXT[] COLLATE "C", + related_raw TEXT[] COLLATE "C", + upstream_raw TEXT[] COLLATE "C", + data JSONB NOT NULL, + proto BYTEA NOT NULL, + PRIMARY KEY (id) INCLUDE (modified) +); + +-- Fast covering lookup by source and path for reconciliation and ListBySource +CREATE INDEX idx_vulns_source ON vulnerabilities (source_id, source_path) INCLUDE (id, source_modified, withdrawn); + +-- Modified time index for feed sync and incremental exporters +CREATE INDEX idx_vulns_modified ON vulnerabilities (modified DESC); + +-- Partial index for fast filtering of withdrawn records +CREATE INDEX idx_vulns_withdrawn ON vulnerabilities (withdrawn) WHERE withdrawn IS NOT NULL; + +-- Partial GIN indexes for relations computation (ignoring empty arrays and withdrawn records) +CREATE INDEX idx_vulns_alias_raw ON vulnerabilities USING GIN (alias_raw) WHERE cardinality(alias_raw) > 0 AND withdrawn IS NULL; +CREATE INDEX idx_vulns_related_raw ON vulnerabilities USING GIN (related_raw) WHERE cardinality(related_raw) > 0 AND withdrawn IS NULL; +CREATE INDEX idx_vulns_upstream_raw ON vulnerabilities USING GIN (upstream_raw) WHERE cardinality(upstream_raw) > 0 AND withdrawn IS NULL; + +-- vuln_id does not reference vulnerabilities(id) so that if a vulnerability row +-- ever needs to be hard-deleted, its historical audit trail is preserved. +CREATE TABLE historic_vulnerabilities ( + vuln_id TEXT COLLATE "C" NOT NULL, + revision INT NOT NULL, + raw_vuln_id BIGINT REFERENCES raw_vulnerabilities(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + data JSONB NOT NULL, + PRIMARY KEY (vuln_id, revision) +); + +-- vuln_id does not reference vulnerabilities(id) because there is no index on (vuln_id) +-- (to save space/writes), which would cause a full table scan on vulnerability deletes. +-- Use vuln_commits to look up commits and delete by (commit_sha, vuln_id). +CREATE TABLE affected_commits ( + commit_sha BYTEA NOT NULL, + vuln_id TEXT COLLATE "C" NOT NULL, + PRIMARY KEY (commit_sha, vuln_id) +); + +CREATE TABLE vuln_commits ( + vuln_id TEXT COLLATE "C" PRIMARY KEY REFERENCES vulnerabilities(id) ON DELETE CASCADE, + commits BYTEA[] +); + +CREATE TABLE affected_versions ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + vuln_id TEXT COLLATE "C" NOT NULL REFERENCES vulnerabilities(id) ON DELETE CASCADE, + name TEXT COLLATE "C" NOT NULL, + base_ecosystem TEXT COLLATE "C" NOT NULL, + ecosystems TEXT[] COLLATE "C" NOT NULL, + coarse_min TEXT COLLATE "C" NOT NULL, + coarse_max TEXT COLLATE "C" NOT NULL, + versions TEXT[] COLLATE "C", + events JSONB +); + +-- Primary index for typical queries (with ecosystem provided) +CREATE INDEX idx_affected_versions_match +ON affected_versions (name, base_ecosystem, vuln_id, id, coarse_min, coarse_max) +INCLUDE (ecosystems); + +-- Secondary index for queries without ecosystems +CREATE INDEX idx_affected_versions_name_only +ON affected_versions (name, vuln_id, id); + +-- Index for worker updates/deletes by vuln_id +CREATE INDEX idx_affected_versions_vuln_id ON affected_versions (vuln_id); + +-- +goose Down +DROP TABLE affected_versions; +DROP TABLE vuln_commits; +DROP TABLE affected_commits; +DROP TABLE historic_vulnerabilities; +DROP TABLE vulnerabilities; +DROP TABLE raw_vulnerabilities; diff --git a/go/internal/database/postgres/models.go b/go/internal/database/postgres/models.go index f5ecce053a5..d059ca837e4 100644 --- a/go/internal/database/postgres/models.go +++ b/go/internal/database/postgres/models.go @@ -4,6 +4,12 @@ package postgres +import ( + "time" + + "github.com/jackc/pgx/v5/pgtype" +) + type AffectedCommit struct { CommitSha []byte VulnID string @@ -21,13 +27,43 @@ type AffectedVersion struct { Events []byte } +type HistoricVulnerability struct { + VulnID string + Revision int32 + RawVulnID pgtype.Int8 + CreatedAt time.Time + Data []byte +} + +type RawVulnerability struct { + ID int64 + VulnID pgtype.Text + SourceID string + SourcePath string + ContentSha256 []byte + GitCommitHash pgtype.Text + GcsGeneration pgtype.Int8 + RawData []byte + FetchedAt time.Time +} + type VulnCommit struct { VulnID string Commits [][]byte } type Vulnerability struct { - ID string - Data []byte - Proto []byte + ID string + Revision int32 + SourceID string + SourcePath string + RawVulnID pgtype.Int8 + Modified time.Time + SourceModified *time.Time + Withdrawn *time.Time + AliasRaw []string + RelatedRaw []string + UpstreamRaw []string + Data []byte + Proto []byte } diff --git a/go/internal/database/postgres/queries.sql b/go/internal/database/postgres/queries.sql index 6c70e016532..350d8e5e780 100644 --- a/go/internal/database/postgres/queries.sql +++ b/go/internal/database/postgres/queries.sql @@ -4,7 +4,7 @@ FROM vulnerabilities WHERE id = ANY(@ids::text[]); -- name: GetVulnerabilityModified :many -SELECT id, (data->>'modified')::timestamptz AS modified +SELECT id, modified FROM vulnerabilities WHERE id = ANY(@ids::text[]); diff --git a/go/internal/database/postgres/queries.sql.go b/go/internal/database/postgres/queries.sql.go index dc0de80b982..14e0320bacc 100644 --- a/go/internal/database/postgres/queries.sql.go +++ b/go/internal/database/postgres/queries.sql.go @@ -11,7 +11,7 @@ import ( ) const getVulnerabilityModified = `-- name: GetVulnerabilityModified :many -SELECT id, (data->>'modified')::timestamptz AS modified +SELECT id, modified FROM vulnerabilities WHERE id = ANY($1::text[]) ` diff --git a/go/internal/database/postgres/sqlc.yaml b/go/internal/database/postgres/sqlc.yaml index 70489d6814c..01257033b51 100644 --- a/go/internal/database/postgres/sqlc.yaml +++ b/go/internal/database/postgres/sqlc.yaml @@ -1,7 +1,7 @@ version: "2" sql: - engine: "postgresql" - schema: "../../../../db_test" + schema: "migrations" queries: "queries.sql" gen: go: @@ -11,6 +11,11 @@ sql: overrides: - db_type: "timestamptz" go_type: "time.Time" + - db_type: "timestamptz" + nullable: true + go_type: + type: "time.Time" + pointer: true - db_type: "jsonb" go_type: type: "[]byte" diff --git a/go/run_tests.sh b/go/run_tests.sh index 1d315ba425b..4ac66f7a3dc 100755 --- a/go/run_tests.sh +++ b/go/run_tests.sh @@ -2,5 +2,8 @@ go test ./... +# Verify sqlc generated files are up to date +(cd ./internal/database/postgres && go run github.com/sqlc-dev/sqlc/cmd/sqlc@v1.31.1 diff) + # Run the validation for the go/python datastore models cd ./internal/database/datastore/internal/validate/ && ./run_validate.sh