Fix leaderboard 500s and slow ranking queries

docs/operations/leaderboard-query-indexes.sql

@@ -0,0 +1,18 @@
+-- Run each statement outside a transaction. These indexes were built
+-- concurrently on production during the 2026-10-23 timeout incident.
+-- Match the eligibility predicates in both kernelboard and kernelbot.
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_runs_secret_leaderboard_pass
+    ON leaderboard.runs (submission_id, runner)
+    WHERE secret AND passed AND mode = 'leaderboard';
+
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_runs_secret_fail
+    ON leaderboard.runs (submission_id, runner)
+    WHERE secret AND passed = FALSE;
+
+-- Keep insert-heavy run metadata eligible for index-only scans between vacuums.
+-- The default insert scale factor of 0.2 allowed hundreds of thousands of new
+-- runs before vacuum; recent leaderboards then required heap reads per run.
+ALTER TABLE leaderboard.runs SET (
+    autovacuum_vacuum_scale_factor = 0.02,
+    autovacuum_vacuum_insert_scale_factor = 0.02
+);

kernelboard/api/leaderboard.py

@@ -239,11 +239,30 @@ def _get_query():
             WHERE leaderboard_id = %(leaderboard_id)s
         ),
 
+        -- Read only this leaderboard's submissions and run metadata once.
+        -- The lateral fence prevents a runner filter from scanning runs belonging
+        -- to every other leaderboard. No run payloads are needed for rankings.
+        scoped_submissions AS MATERIALIZED (
+            SELECT id, user_id, submission_time, file_name, status, leaderboard_id
+            FROM leaderboard.submission
+            WHERE leaderboard_id = %(leaderboard_id)s
+        ),
+        scoped_runs AS MATERIALIZED (
+            SELECT r.*
+            FROM scoped_submissions s
+            JOIN LATERAL (
+                SELECT submission_id, runner, score, secret, passed
+                FROM leaderboard.runs
+                WHERE submission_id = s.id
+                OFFSET 0
+            ) r ON TRUE
+        ),
+
         -- Total submission count per user per GPU type for this leaderboard.
-        submission_counts AS (
+        submission_counts AS MATERIALIZED (
             SELECT s.user_id, r.runner, COUNT(DISTINCT s.id) AS submission_count
-            FROM leaderboard.submission s
-            JOIN leaderboard.runs r ON r.submission_id = s.id
+            FROM scoped_submissions s
+            JOIN scoped_runs r ON r.submission_id = s.id
             WHERE s.leaderboard_id = %(leaderboard_id)s
                 AND s.status <> 'hacked'
                 AND NOT EXISTS (
@@ -265,38 +284,11 @@ def _get_query():
                 s.file_name AS file_name,
                 r.submission_id AS submission_id,
                 COALESCE(sc.submission_count, 0) AS submission_count,
-                validation.status AS validation_status,
-                validation.passed_shapes AS validation_shapes_passed,
-                validation.total_shapes AS validation_shapes_total,
-                validation.fully_validated AS validation_fully_validated,
-                validation.geomean_sync_wall_speedup AS validation_geomean_speedup,
-                validation.contract_version AS validation_contract_version,
-                validation.checked_at AS validation_checked_at,
                 RANK() OVER (PARTITION BY r.runner, u.id ORDER BY r.score ASC) AS rank
-            FROM leaderboard.runs r
-                JOIN leaderboard.submission s ON r.submission_id = s.id
+            FROM scoped_runs r
+                JOIN scoped_submissions s ON r.submission_id = s.id
                 LEFT JOIN leaderboard.user_info u ON s.user_id = u.id
                 LEFT JOIN submission_counts sc ON s.user_id = sc.user_id AND r.runner = sc.runner
-                LEFT JOIN LATERAL (
-                    SELECT
-                        status,
-                        passed_shapes,
-                        total_shapes,
-                        fully_validated,
-                        geomean_sync_wall_speedup,
-                        contract_version,
-                        checked_at
-                    FROM leaderboard.submission_validation
-                    WHERE submission_id = s.id
-                        AND gpu_type = r.runner
-                        AND contract_version = (
-                            SELECT task->'validation'->>'version'
-                            FROM leaderboard.leaderboard
-                            WHERE id = %(leaderboard_id)s
-                        )
-                    ORDER BY checked_at DESC
-                    LIMIT 1
-                ) validation ON TRUE
             WHERE NOT r.secret
                 AND r.score IS NOT NULL
                 AND r.passed
@@ -328,7 +320,38 @@ def _get_query():
         ),
 
         -- From ranked_runs, keep only the top run per user.
-        top_runs AS (SELECT * FROM ranked_runs WHERE rank = 1)
+        top_runs AS (
+            SELECT r.*,
+                validation.status AS validation_status,
+                validation.passed_shapes AS validation_shapes_passed,
+                validation.total_shapes AS validation_shapes_total,
+                validation.fully_validated AS validation_fully_validated,
+                validation.geomean_sync_wall_speedup AS validation_geomean_speedup,
+                validation.contract_version AS validation_contract_version,
+                validation.checked_at AS validation_checked_at
+            FROM ranked_runs r
+                LEFT JOIN LATERAL (
+                    SELECT
+                        status,
+                        passed_shapes,
+                        total_shapes,
+                        fully_validated,
+                        geomean_sync_wall_speedup,
+                        contract_version,
+                        checked_at
+                    FROM leaderboard.submission_validation
+                    WHERE submission_id = r.submission_id
+                        AND gpu_type = r.runner
+                        AND contract_version = (
+                            SELECT task->'validation'->>'version'
+                            FROM leaderboard.leaderboard
+                            WHERE id = %(leaderboard_id)s
+                        )
+                    ORDER BY checked_at DESC
+                    LIMIT 1
+                ) validation ON TRUE
+            WHERE r.rank = 1
+        )
 
         SELECT jsonb_build_object(
             'rankings', (SELECT jsonb_object_agg(g.gpu_type, (

kernelboard/api/leaderboard_summaries.py

@@ -48,7 +48,7 @@ def _get_cached_top_users(redis_conn, leaderboard_ids: list[int]) -> dict[int, l
         return {}
 
 
-def _set_cached_top_users(redis_conn, leaderboard_id: int, top_users: list):
+def _set_cached_top_users(redis_conn, leaderboard_id: int, top_users: list | None):
     """Cache top_users for ended leaderboard (no expiry)."""
     if not redis_conn:
         return
@@ -179,8 +179,7 @@ def _get_leaderboards_cached(total_start: float, force_refresh: bool = False):
 
         # 5. Cache newly computed ended leaderboards
         for lb_id in uncached_ended_ids:
-            if lb_id in computed_results:
-                _set_cached_top_users(redis_conn, lb_id, computed_results[lb_id])
+            _set_cached_top_users(redis_conn, lb_id, computed_results.get(lb_id))
 
         # 6. Get metadata for all leaderboards
         cur.execute(_get_leaderboard_metadata_query())

kernelboard/lib/db.py

@@ -16,7 +16,6 @@ def get_db_connection() -> psycopg2.extensions.connection:
             raise RuntimeError(
                 "DATABASE_URL is not set in the application configuration."
             )
-        print("database_url", database_url)
         g.db_connection = psycopg2.connect(database_url)
     return g.db_connection
 

tests/api/test_leaderboard_api.py

@@ -428,3 +428,33 @@ def test_hacked_submissions_are_hidden_from_public_leaderboard(client, app):
     assert "hidden_submission_status_hacked.py" not in ranked_files
     assert "hidden_job_status_hacked.py" not in ranked_files
     assert "visible_clean_status.py" in ranked_files
+
+
+def test_equal_best_runs_preserve_ties_and_distinct_submission_count(client, app):
+    before = client.get("/api/leaderboard/339").get_json()["data"]["rankings"]
+    gpu, entries = next(iter(before.items()))
+    winner = entries[0]
+    with app.app_context():
+        conn = get_db_connection()
+        with conn.cursor() as cur:
+            cur.execute(
+                """
+                INSERT INTO leaderboard.runs (
+                    id, submission_id, start_time, end_time, mode, secret,
+                    runner, score, passed, compilation, meta, result, system_info
+                )
+                SELECT 990001, submission_id, start_time, end_time, mode, secret,
+                       runner, score, passed, compilation, meta, result, system_info
+                FROM leaderboard.runs
+                WHERE submission_id = %s AND runner = %s
+                    AND NOT secret AND passed AND score = %s
+                LIMIT 1
+                """,
+                (winner["submission_id"], gpu, winner["score"]),
+            )
+        conn.commit()
+    after = client.get("/api/leaderboard/339").get_json()["data"]["rankings"][gpu]
+    tied = [item for item in after if item["submission_id"] == winner["submission_id"]]
+    assert len(tied) == 2
+    assert all(item["score"] == winner["score"] for item in tied)
+    assert all(item["submission_count"] == winner["submission_count"] for item in tied)

tests/api/test_leaderboard_summaries_api.py

@@ -24,3 +24,30 @@ def test_leaderboard_metadata(client):
     assert all(
         ids[i] > ids[i + 1] for i in range(len(ids) - 1)
     ), f"Leaderboard IDs are not in decreasing order: {ids}"
+
+
+def test_empty_ended_leaderboard_is_cached(client, app, monkeypatch):
+    from kernelboard.api import leaderboard_summaries as summaries
+    from kernelboard.lib.db import get_db_connection
+
+    with app.app_context():
+        conn = get_db_connection()
+        with conn.cursor() as cur:
+            cur.execute("UPDATE leaderboard.leaderboard SET deadline = NOW() - INTERVAL '1 day'")
+            cur.execute("UPDATE leaderboard.runs SET passed = FALSE")
+            cur.execute("SELECT id FROM leaderboard.leaderboard")
+            leaderboard_ids = [row[0] for row in cur.fetchall()]
+        summaries._delete_cached_top_users(summaries._get_redis(), leaderboard_ids)
+        conn.commit()
+
+    first = client.get("/api/leaderboard-summaries")
+    assert first.status_code == 200
+    assert all(lb["top_users"] is None for lb in first.get_json()["data"]["leaderboards"])
+
+    def unexpected_recompute():
+        raise AssertionError("Empty ended leaderboards should be served from cache")
+
+    monkeypatch.setattr(summaries, "_get_query_for_ids", unexpected_recompute)
+    second = client.get("/api/leaderboard-summaries")
+    assert second.status_code == 200
+    assert second.get_json() == first.get_json()