perf(analytics): cache the project profile per project per day

src/bubo/project_analytics.py

@@ -9,6 +9,8 @@
 import sqlite3
 import subprocess
 import uuid
+from contextlib import closing
+from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
 from urllib.parse import quote, urlsplit
@@ -299,16 +301,68 @@ def collect_profile(cfg: Any, token: str, project: str, repo: Path) -> dict[str,
         configured = cfg.analytics_config.profiles.get(f"{cfg.provider}:{project}", {})
         domain = configured.get("domain", "unknown")
         project_type = configured.get("project_type", "unknown")
-        technologies, coverage = detect_stack(repo)
+        day = _utc_day()
+        scanned = _cached_scan(identifier, day)
+        if scanned is None:
+            technologies, coverage = detect_stack(repo)
+            scanned = {
+                "stack_coverage": coverage,
+                "technologies": technologies,
+                **fetch_metadata(cfg.provider, base, token, project),
+            }
+            if scanned.get("metadata_status") != "unavailable":
+                _store_scan(identifier, day, scanned)
         return {
             "project_id": identifier,
             "domain": domain if domain in DOMAINS else "unknown",
             "project_type": project_type if project_type in PROJECT_TYPES else "unknown",
             "profile_source": "declared" if configured else "unknown",
             "stack_source": "tracked_paths_and_manifests",
-            "stack_coverage": coverage,
-            "technologies": technologies,
-            **fetch_metadata(cfg.provider, base, token, project),
+            **scanned,
         }
     except Exception:
         return {}
+
+
+# The stack scan and SCM metadata read are cached per project per UTC day.
+# Each review is its own worker process, so the cache lives beside the
+# project-ID table. Declared domain/type are read from config every time.
+
+
+def _utc_day() -> str:
+    return datetime.now(UTC).date().isoformat()
+
+
+def _profile_db() -> sqlite3.Connection:
+    db = sqlite3.connect(paths.DB.parent / "analytics_projects.sqlite", timeout=0.2)
+    db.execute(
+        "CREATE TABLE IF NOT EXISTS profiles (id TEXT PRIMARY KEY, day TEXT NOT NULL, "
+        "scan TEXT NOT NULL)"
+    )
+    return db
+
+
+def _cached_scan(identifier: str, day: str) -> dict[str, Any] | None:
+    try:
+        with closing(_profile_db()) as db:
+            row = db.execute(
+                "SELECT scan FROM profiles WHERE id = ? AND day = ?", (identifier, day)
+            ).fetchone()
+        if row is None:
+            return None
+        scan = json.loads(row[0])
+        scan["technologies"] = [tuple(item) for item in scan["technologies"]]
+        return scan
+    except OSError, sqlite3.Error, ValueError, KeyError, TypeError:
+        return None
+
+
+def _store_scan(identifier: str, day: str, scan: dict[str, Any]) -> None:
+    try:
+        with closing(_profile_db()) as db, db:
+            db.execute(
+                "INSERT OR REPLACE INTO profiles VALUES (?, ?, ?)",
+                (identifier, day, json.dumps(scan)),
+            )
+    except OSError, sqlite3.Error:
+        return

tests/test_profile_cache.py

@@ -0,0 +1,93 @@
+"""The project profile is computed once per project per day, not per review.
+
+Each review runs in its own worker process, so the cache must outlive the
+process. Drives the real ``collect_profile`` against a real git checkout and
+counts the stack scans and SCM metadata reads it does.
+"""
+
+import subprocess
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from bubo import analytics
+from bubo import project_analytics as profiles
+from bubo.analytics_config import AnalyticsConfig
+from bubo.review_config import ReviewConfig
+
+
+@pytest.fixture(autouse=True)
+def isolated(tmp_path, monkeypatch):
+    monkeypatch.setattr(profiles.paths, "DB", tmp_path / "state" / "reviewer.sqlite")
+    monkeypatch.setattr(analytics, "_install_id", None)
+    monkeypatch.setattr(analytics, "_install_path", None)
+
+
+def _repo(tmp_path: Path) -> Path:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "app.py").write_text("import fastapi\n")
+    (repo / "requirements.txt").write_text("fastapi\n")
+    git = ["git", "-C", str(repo), "-c", "user.name=t", "-c", "user.email=t@t"]
+    subprocess.run([*git, "init", "-q"], check=True)
+    subprocess.run([*git, "add", "."], check=True)
+    subprocess.run([*git, "commit", "-qm", "init"], check=True)
+    return repo
+
+
+CFG = ReviewConfig(provider="github", analytics_config=AnalyticsConfig(enabled=True))
+METADATA = {"metadata_status": "verified", "visibility": "public", "namespace_kind": "org"}
+
+
+def _collect(repo: Path, days: list[str], metadata=METADATA, project="acme/app"):
+    profiles_seen = []
+    with (
+        patch.object(profiles, "detect_stack", wraps=profiles.detect_stack) as scan,
+        patch.object(profiles, "fetch_metadata", return_value=metadata) as meta,
+        patch.object(profiles, "_utc_day", side_effect=days),
+    ):
+        for _ in days:
+            profiles_seen.append(profiles.collect_profile(CFG, "tok", project, repo))
+    return profiles_seen, scan.call_count, meta.call_count
+
+
+def test_reviews_of_one_project_on_one_day_scan_once(tmp_path):
+    seen, scans, reads = _collect(_repo(tmp_path), ["2026-10-20"] * 3)
+    assert (scans, reads) == (1, 1)
+    assert seen[0] == seen[1] == seen[2]
+    assert ("framework", "fastapi") in seen[0]["technologies"]
+
+
+def test_profile_is_refreshed_the_next_day(tmp_path):
+    _, scans, reads = _collect(_repo(tmp_path), ["2026-10-20", "2026-10-20", "2026-10-21"])
+    assert (scans, reads) == (2, 2)
+
+
+def test_each_project_has_its_own_profile(tmp_path):
+    repo = _repo(tmp_path)
+    _collect(repo, ["2026-10-20"], project="acme/app")
+    _, scans, _ = _collect(repo, ["2026-10-20"], project="acme/other")
+    assert scans == 1
+
+
+def test_unavailable_metadata_is_retried_next_review(tmp_path):
+    down = {"metadata_status": "unavailable", "visibility": "unknown"}
+    _, _, reads = _collect(_repo(tmp_path), ["2026-10-20"] * 2, metadata=down)
+    assert reads == 2
+
+
+def test_declared_profile_changes_apply_without_waiting_a_day(tmp_path):
+    repo = _repo(tmp_path)
+    _collect(repo, ["2026-10-20"])
+    declared = ReviewConfig(
+        provider="github",
+        analytics_config=AnalyticsConfig(
+            enabled=True, profiles={"github:acme/app": {"domain": "fintech"}}
+        ),
+    )
+    with (
+        patch.object(profiles, "fetch_metadata", return_value=METADATA),
+        patch.object(profiles, "_utc_day", return_value="2026-10-20"),
+    ):
+        assert profiles.collect_profile(declared, "tok", "acme/app", repo)["domain"] == "fintech"