From fd6d8c13882d4e0b0a8b71c673643357a533db31 Mon Sep 17 00:00:00 2001 From: darshpatel1052 Date: Tue, 3 Nov 2026 11:46:26 +0530 Subject: [PATCH 1/2] fix: read triage metrics from metadata paths --- go/internal/gcs/triage.go | 8 ++-- go/internal/gcs/triage_test.go | 88 ++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 4 deletions(-) diff --git a/go/internal/gcs/triage.go b/go/internal/gcs/triage.go index 01931a28579..f8a2cc15807 100644 --- a/go/internal/gcs/triage.go +++ b/go/internal/gcs/triage.go @@ -29,13 +29,13 @@ var gcsSources = map[string]sourceConfig{ "test-nvd": {Bucket: "osv-test-cve-osv-conversion", PathTemplate: "nvd-osv/%s.json"}, "test-cve5": {Bucket: "osv-test-cve-osv-conversion", PathTemplate: "cve5/%s.json"}, "test-osv": {Bucket: "osv-test-cve-osv-conversion", PathTemplate: "osv-output/%s.json"}, - "test-nvd-metrics": {Bucket: "osv-test-cve-osv-conversion", PathTemplate: "nvd-osv/%s.metrics.json"}, - "test-cve5-metrics": {Bucket: "osv-test-cve-osv-conversion", PathTemplate: "cve5/%s.metrics.json"}, + "test-nvd-metrics": {Bucket: "osv-test-cve-osv-conversion", PathTemplate: "metadata/nvd/metrics/%s.metrics.json"}, + "test-cve5-metrics": {Bucket: "osv-test-cve-osv-conversion", PathTemplate: "metadata/cve5/metrics/%s.metrics.json"}, "prod-nvd": {Bucket: "cve-osv-conversion", PathTemplate: "nvd-osv/%s.json"}, "prod-cve5": {Bucket: "cve-osv-conversion", PathTemplate: "cve5/%s.json"}, "prod-osv": {Bucket: "cve-osv-conversion", PathTemplate: "osv-output/%s.json"}, - "prod-nvd-metrics": {Bucket: "cve-osv-conversion", PathTemplate: "nvd-osv/%s.metrics.json"}, - "prod-cve5-metrics": {Bucket: "cve-osv-conversion", PathTemplate: "cve5/%s.metrics.json"}, + "prod-nvd-metrics": {Bucket: "cve-osv-conversion", PathTemplate: "metadata/nvd/metrics/%s.metrics.json"}, + "prod-cve5-metrics": {Bucket: "cve-osv-conversion", PathTemplate: "metadata/cve5/metrics/%s.metrics.json"}, } // TriageStore implements models.TriageStore for reading CVE data from GCS and upstream APIs. diff --git a/go/internal/gcs/triage_test.go b/go/internal/gcs/triage_test.go index be1177be410..0b127ce605c 100644 --- a/go/internal/gcs/triage_test.go +++ b/go/internal/gcs/triage_test.go @@ -3,10 +3,14 @@ package gcs_test import ( "context" "errors" + "net/http" + "net/http/httptest" "testing" + "cloud.google.com/go/storage" "github.com/google/osv.dev/go/internal/gcs" "github.com/google/osv.dev/go/internal/models" + "google.golang.org/api/option" ) func TestNewTriageStore(t *testing.T) { @@ -50,3 +54,87 @@ func TestTriageStore_GetFile_InvalidArgument(t *testing.T) { }) } } + +func TestTriageStore_GetFile_GCS(t *testing.T) { + t.Parallel() + + tests := []struct { + source string + bucket string + object string + }{ + {"test-nvd-metrics", "osv-test-cve-osv-conversion", "metadata/nvd/metrics/CVE-2024-1234.metrics.json"}, + {"test-cve5-metrics", "osv-test-cve-osv-conversion", "metadata/cve5/metrics/CVE-2024-1234.metrics.json"}, + {"prod-nvd-metrics", "cve-osv-conversion", "metadata/nvd/metrics/CVE-2024-1234.metrics.json"}, + {"prod-cve5-metrics", "cve-osv-conversion", "metadata/cve5/metrics/CVE-2024-1234.metrics.json"}, + {"test-nvd", "osv-test-cve-osv-conversion", "nvd-osv/CVE-2024-1234.json"}, + {"test-cve5", "osv-test-cve-osv-conversion", "cve5/CVE-2024-1234.json"}, + {"test-osv", "osv-test-cve-osv-conversion", "osv-output/CVE-2024-1234.json"}, + {"prod-nvd", "cve-osv-conversion", "nvd-osv/CVE-2024-1234.json"}, + {"prod-cve5", "cve-osv-conversion", "cve5/CVE-2024-1234.json"}, + {"prod-osv", "cve-osv-conversion", "osv-output/CVE-2024-1234.json"}, + } + + for _, tc := range tests { + t.Run(tc.source, func(t *testing.T) { + t.Parallel() + + for _, missing := range []bool{false, true} { + name := "found" + if missing { + name = "missing" + } + + t.Run(name, func(t *testing.T) { + t.Parallel() + + const content = `{"id":"CVE-2024-1234"}` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + wantPath := "/b/" + tc.bucket + "/o/" + tc.object + if r.URL.Path != wantPath { + t.Errorf("requested GCS path %q, want %q", r.URL.Path, wantPath) + http.NotFound(w, r) + + return + } + if missing { + http.NotFound(w, r) + + return + } + if _, err := w.Write([]byte(content)); err != nil { + t.Errorf("failed to write GCS response: %v", err) + } + })) + t.Cleanup(server.Close) + + client, err := storage.NewClient(t.Context(), option.WithEndpoint(server.URL), option.WithoutAuthentication(), storage.WithJSONReads()) + if err != nil { + t.Fatalf("failed to create storage client: %v", err) + } + t.Cleanup(func() { + if err := client.Close(); err != nil { + t.Errorf("failed to close storage client: %v", err) + } + }) + + store := gcs.NewTriageStore(client) + data, err := store.GetFile(t.Context(), tc.source, "cve-2024-1234") + if missing { + if !errors.Is(err, models.ErrNotFound) { + t.Errorf("expected ErrNotFound, got %v", err) + } + + return + } + if err != nil { + t.Fatalf("GetFile failed: %v", err) + } + if string(data) != content { + t.Errorf("GetFile returned %q, want %q", data, content) + } + }) + } + }) + } +} From 45e04152370472140d850ed9d39d0bae2b5a20db Mon Sep 17 00:00:00 2001 From: darshpatel1052 Date: Wed, 4 Nov 2026 10:16:39 +0530 Subject: [PATCH 2/2] test: share triage GCS test server and client --- go/internal/gcs/triage_test.go | 73 +++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 32 deletions(-) diff --git a/go/internal/gcs/triage_test.go b/go/internal/gcs/triage_test.go index 0b127ce605c..21ff247e194 100644 --- a/go/internal/gcs/triage_test.go +++ b/go/internal/gcs/triage_test.go @@ -3,8 +3,10 @@ package gcs_test import ( "context" "errors" + "fmt" "net/http" "net/http/httptest" + "strings" "testing" "cloud.google.com/go/storage" @@ -75,51 +77,57 @@ func TestTriageStore_GetFile_GCS(t *testing.T) { {"prod-osv", "cve-osv-conversion", "osv-output/CVE-2024-1234.json"}, } + objects := make(map[string]string, len(tests)*2) + for _, tc := range tests { + path := "/b/" + tc.bucket + "/o/" + tc.object + objects[path] = fmt.Sprintf(`{"source":%q}`, tc.source) + // A separate CVE keeps missing reads independent of parallel successful reads. + objects[strings.Replace(path, "CVE-2024-1234", "CVE-2024-5678", 1)] = "" + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + content, ok := objects[r.URL.Path] + if !ok { + t.Errorf("unexpected GCS path %q", r.URL.Path) + } + if content == "" { + http.NotFound(w, r) + + return + } + if _, err := w.Write([]byte(content)); err != nil { + t.Errorf("failed to write GCS response: %v", err) + } + })) + t.Cleanup(server.Close) + + client, err := storage.NewClient(t.Context(), option.WithEndpoint(server.URL), option.WithoutAuthentication(), storage.WithJSONReads()) + if err != nil { + t.Fatalf("failed to create storage client: %v", err) + } + t.Cleanup(func() { + if err := client.Close(); err != nil { + t.Errorf("failed to close storage client: %v", err) + } + }) + store := gcs.NewTriageStore(client) + for _, tc := range tests { t.Run(tc.source, func(t *testing.T) { t.Parallel() for _, missing := range []bool{false, true} { name := "found" + cveID := "cve-2024-1234" if missing { name = "missing" + cveID = "cve-2024-5678" } t.Run(name, func(t *testing.T) { t.Parallel() - const content = `{"id":"CVE-2024-1234"}` - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - wantPath := "/b/" + tc.bucket + "/o/" + tc.object - if r.URL.Path != wantPath { - t.Errorf("requested GCS path %q, want %q", r.URL.Path, wantPath) - http.NotFound(w, r) - - return - } - if missing { - http.NotFound(w, r) - - return - } - if _, err := w.Write([]byte(content)); err != nil { - t.Errorf("failed to write GCS response: %v", err) - } - })) - t.Cleanup(server.Close) - - client, err := storage.NewClient(t.Context(), option.WithEndpoint(server.URL), option.WithoutAuthentication(), storage.WithJSONReads()) - if err != nil { - t.Fatalf("failed to create storage client: %v", err) - } - t.Cleanup(func() { - if err := client.Close(); err != nil { - t.Errorf("failed to close storage client: %v", err) - } - }) - - store := gcs.NewTriageStore(client) - data, err := store.GetFile(t.Context(), tc.source, "cve-2024-1234") + data, err := store.GetFile(t.Context(), tc.source, cveID) if missing { if !errors.Is(err, models.ErrNotFound) { t.Errorf("expected ErrNotFound, got %v", err) @@ -130,6 +138,7 @@ func TestTriageStore_GetFile_GCS(t *testing.T) { if err != nil { t.Fatalf("GetFile failed: %v", err) } + content := fmt.Sprintf(`{"source":%q}`, tc.source) if string(data) != content { t.Errorf("GetFile returned %q, want %q", data, content) }