From 0417cbb7bf13f264b91af5d57177c47c25cd1d86 Mon Sep 17 00:00:00 2001 From: MountainOwl Date: Tue, 20 Oct 2026 16:56:15 +0000 Subject: [PATCH] perf(analytics): cache the project profile per project per day Every review ran a git ls-tree, read up to 100 manifests and made an SCM metadata call to build the analytics project profile. The scan and the metadata are now cached per project per UTC day in the existing analytics_projects.sqlite, since each review runs in its own worker process. A failed metadata read is not cached, and the declared domain and project type are still read from config on every review. --- src/bubo/project_analytics.py | 62 +++++++++++++++++++++-- tests/test_profile_cache.py | 93 +++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 tests/test_profile_cache.py diff --git a/src/bubo/project_analytics.py b/src/bubo/project_analytics.py index faa2355..0ec61be 100644 --- a/src/bubo/project_analytics.py +++ b/src/bubo/project_analytics.py @@ -9,6 +9,8 @@ import sqlite3 import subprocess import uuid +from contextlib import closing +from datetime import UTC, datetime from pathlib import Path from typing import Any from urllib.parse import quote, urlsplit @@ -299,16 +301,68 @@ def collect_profile(cfg: Any, token: str, project: str, repo: Path) -> dict[str, configured = cfg.analytics_config.profiles.get(f"{cfg.provider}:{project}", {}) domain = configured.get("domain", "unknown") project_type = configured.get("project_type", "unknown") - technologies, coverage = detect_stack(repo) + day = _utc_day() + scanned = _cached_scan(identifier, day) + if scanned is None: + technologies, coverage = detect_stack(repo) + scanned = { + "stack_coverage": coverage, + "technologies": technologies, + **fetch_metadata(cfg.provider, base, token, project), + } + if scanned.get("metadata_status") != "unavailable": + _store_scan(identifier, day, scanned) return { "project_id": identifier, "domain": domain if domain in DOMAINS else "unknown", "project_type": project_type if project_type in PROJECT_TYPES else "unknown", "profile_source": "declared" if configured else "unknown", "stack_source": "tracked_paths_and_manifests", - "stack_coverage": coverage, - "technologies": technologies, - **fetch_metadata(cfg.provider, base, token, project), + **scanned, } except Exception: return {} + + +# The stack scan and SCM metadata read are cached per project per UTC day. +# Each review is its own worker process, so the cache lives beside the +# project-ID table. Declared domain/type are read from config every time. + + +def _utc_day() -> str: + return datetime.now(UTC).date().isoformat() + + +def _profile_db() -> sqlite3.Connection: + db = sqlite3.connect(paths.DB.parent / "analytics_projects.sqlite", timeout=0.2) + db.execute( + "CREATE TABLE IF NOT EXISTS profiles (id TEXT PRIMARY KEY, day TEXT NOT NULL, " + "scan TEXT NOT NULL)" + ) + return db + + +def _cached_scan(identifier: str, day: str) -> dict[str, Any] | None: + try: + with closing(_profile_db()) as db: + row = db.execute( + "SELECT scan FROM profiles WHERE id = ? AND day = ?", (identifier, day) + ).fetchone() + if row is None: + return None + scan = json.loads(row[0]) + scan["technologies"] = [tuple(item) for item in scan["technologies"]] + return scan + except OSError, sqlite3.Error, ValueError, KeyError, TypeError: + return None + + +def _store_scan(identifier: str, day: str, scan: dict[str, Any]) -> None: + try: + with closing(_profile_db()) as db, db: + db.execute( + "INSERT OR REPLACE INTO profiles VALUES (?, ?, ?)", + (identifier, day, json.dumps(scan)), + ) + except OSError, sqlite3.Error: + return diff --git a/tests/test_profile_cache.py b/tests/test_profile_cache.py new file mode 100644 index 0000000..6fc5fbc --- /dev/null +++ b/tests/test_profile_cache.py @@ -0,0 +1,93 @@ +"""The project profile is computed once per project per day, not per review. + +Each review runs in its own worker process, so the cache must outlive the +process. Drives the real ``collect_profile`` against a real git checkout and +counts the stack scans and SCM metadata reads it does. +""" + +import subprocess +from pathlib import Path +from unittest.mock import patch + +import pytest + +from bubo import analytics +from bubo import project_analytics as profiles +from bubo.analytics_config import AnalyticsConfig +from bubo.review_config import ReviewConfig + + +@pytest.fixture(autouse=True) +def isolated(tmp_path, monkeypatch): + monkeypatch.setattr(profiles.paths, "DB", tmp_path / "state" / "reviewer.sqlite") + monkeypatch.setattr(analytics, "_install_id", None) + monkeypatch.setattr(analytics, "_install_path", None) + + +def _repo(tmp_path: Path) -> Path: + repo = tmp_path / "repo" + repo.mkdir() + (repo / "app.py").write_text("import fastapi\n") + (repo / "requirements.txt").write_text("fastapi\n") + git = ["git", "-C", str(repo), "-c", "user.name=t", "-c", "user.email=t@t"] + subprocess.run([*git, "init", "-q"], check=True) + subprocess.run([*git, "add", "."], check=True) + subprocess.run([*git, "commit", "-qm", "init"], check=True) + return repo + + +CFG = ReviewConfig(provider="github", analytics_config=AnalyticsConfig(enabled=True)) +METADATA = {"metadata_status": "verified", "visibility": "public", "namespace_kind": "org"} + + +def _collect(repo: Path, days: list[str], metadata=METADATA, project="acme/app"): + profiles_seen = [] + with ( + patch.object(profiles, "detect_stack", wraps=profiles.detect_stack) as scan, + patch.object(profiles, "fetch_metadata", return_value=metadata) as meta, + patch.object(profiles, "_utc_day", side_effect=days), + ): + for _ in days: + profiles_seen.append(profiles.collect_profile(CFG, "tok", project, repo)) + return profiles_seen, scan.call_count, meta.call_count + + +def test_reviews_of_one_project_on_one_day_scan_once(tmp_path): + seen, scans, reads = _collect(_repo(tmp_path), ["2026-10-20"] * 3) + assert (scans, reads) == (1, 1) + assert seen[0] == seen[1] == seen[2] + assert ("framework", "fastapi") in seen[0]["technologies"] + + +def test_profile_is_refreshed_the_next_day(tmp_path): + _, scans, reads = _collect(_repo(tmp_path), ["2026-10-20", "2026-10-20", "2026-10-21"]) + assert (scans, reads) == (2, 2) + + +def test_each_project_has_its_own_profile(tmp_path): + repo = _repo(tmp_path) + _collect(repo, ["2026-10-20"], project="acme/app") + _, scans, _ = _collect(repo, ["2026-10-20"], project="acme/other") + assert scans == 1 + + +def test_unavailable_metadata_is_retried_next_review(tmp_path): + down = {"metadata_status": "unavailable", "visibility": "unknown"} + _, _, reads = _collect(_repo(tmp_path), ["2026-10-20"] * 2, metadata=down) + assert reads == 2 + + +def test_declared_profile_changes_apply_without_waiting_a_day(tmp_path): + repo = _repo(tmp_path) + _collect(repo, ["2026-10-20"]) + declared = ReviewConfig( + provider="github", + analytics_config=AnalyticsConfig( + enabled=True, profiles={"github:acme/app": {"domain": "fintech"}} + ), + ) + with ( + patch.object(profiles, "fetch_metadata", return_value=METADATA), + patch.object(profiles, "_utc_day", return_value="2026-10-20"), + ): + assert profiles.collect_profile(declared, "tok", "acme/app", repo)["domain"] == "fintech"