From 70010d63096243a2a2a4e5b812f44f5b66edb088 Mon Sep 17 00:00:00 2001 From: Mark Saroufim Date: Fri, 23 Oct 2026 17:10:07 -0700 Subject: [PATCH] Fix leaderboard timeouts with scoped run queries and empty-result caching --- docs/operations/leaderboard-query-indexes.sql | 18 ++++ kernelboard/api/leaderboard.py | 89 ++++++++++++------- kernelboard/api/leaderboard_summaries.py | 5 +- kernelboard/lib/db.py | 1 - tests/api/test_leaderboard_api.py | 30 +++++++ tests/api/test_leaderboard_summaries_api.py | 27 ++++++ 6 files changed, 133 insertions(+), 37 deletions(-) create mode 100644 docs/operations/leaderboard-query-indexes.sql diff --git a/docs/operations/leaderboard-query-indexes.sql b/docs/operations/leaderboard-query-indexes.sql new file mode 100644 index 00000000..bb5af818 --- /dev/null +++ b/docs/operations/leaderboard-query-indexes.sql @@ -0,0 +1,18 @@ +-- Run each statement outside a transaction. These indexes were built +-- concurrently on production during the 2026-10-23 timeout incident. +-- Match the eligibility predicates in both kernelboard and kernelbot. +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_runs_secret_leaderboard_pass + ON leaderboard.runs (submission_id, runner) + WHERE secret AND passed AND mode = 'leaderboard'; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_runs_secret_fail + ON leaderboard.runs (submission_id, runner) + WHERE secret AND passed = FALSE; + +-- Keep insert-heavy run metadata eligible for index-only scans between vacuums. +-- The default insert scale factor of 0.2 allowed hundreds of thousands of new +-- runs before vacuum; recent leaderboards then required heap reads per run. +ALTER TABLE leaderboard.runs SET ( + autovacuum_vacuum_scale_factor = 0.02, + autovacuum_vacuum_insert_scale_factor = 0.02 +); diff --git a/kernelboard/api/leaderboard.py b/kernelboard/api/leaderboard.py index 59b6434c..4b86149c 100644 --- a/kernelboard/api/leaderboard.py +++ b/kernelboard/api/leaderboard.py @@ -239,11 +239,30 @@ def _get_query(): WHERE leaderboard_id = %(leaderboard_id)s ), + -- Read only this leaderboard's submissions and run metadata once. + -- The lateral fence prevents a runner filter from scanning runs belonging + -- to every other leaderboard. No run payloads are needed for rankings. + scoped_submissions AS MATERIALIZED ( + SELECT id, user_id, submission_time, file_name, status, leaderboard_id + FROM leaderboard.submission + WHERE leaderboard_id = %(leaderboard_id)s + ), + scoped_runs AS MATERIALIZED ( + SELECT r.* + FROM scoped_submissions s + JOIN LATERAL ( + SELECT submission_id, runner, score, secret, passed + FROM leaderboard.runs + WHERE submission_id = s.id + OFFSET 0 + ) r ON TRUE + ), + -- Total submission count per user per GPU type for this leaderboard. - submission_counts AS ( + submission_counts AS MATERIALIZED ( SELECT s.user_id, r.runner, COUNT(DISTINCT s.id) AS submission_count - FROM leaderboard.submission s - JOIN leaderboard.runs r ON r.submission_id = s.id + FROM scoped_submissions s + JOIN scoped_runs r ON r.submission_id = s.id WHERE s.leaderboard_id = %(leaderboard_id)s AND s.status <> 'hacked' AND NOT EXISTS ( @@ -265,38 +284,11 @@ def _get_query(): s.file_name AS file_name, r.submission_id AS submission_id, COALESCE(sc.submission_count, 0) AS submission_count, - validation.status AS validation_status, - validation.passed_shapes AS validation_shapes_passed, - validation.total_shapes AS validation_shapes_total, - validation.fully_validated AS validation_fully_validated, - validation.geomean_sync_wall_speedup AS validation_geomean_speedup, - validation.contract_version AS validation_contract_version, - validation.checked_at AS validation_checked_at, RANK() OVER (PARTITION BY r.runner, u.id ORDER BY r.score ASC) AS rank - FROM leaderboard.runs r - JOIN leaderboard.submission s ON r.submission_id = s.id + FROM scoped_runs r + JOIN scoped_submissions s ON r.submission_id = s.id LEFT JOIN leaderboard.user_info u ON s.user_id = u.id LEFT JOIN submission_counts sc ON s.user_id = sc.user_id AND r.runner = sc.runner - LEFT JOIN LATERAL ( - SELECT - status, - passed_shapes, - total_shapes, - fully_validated, - geomean_sync_wall_speedup, - contract_version, - checked_at - FROM leaderboard.submission_validation - WHERE submission_id = s.id - AND gpu_type = r.runner - AND contract_version = ( - SELECT task->'validation'->>'version' - FROM leaderboard.leaderboard - WHERE id = %(leaderboard_id)s - ) - ORDER BY checked_at DESC - LIMIT 1 - ) validation ON TRUE WHERE NOT r.secret AND r.score IS NOT NULL AND r.passed @@ -328,7 +320,38 @@ def _get_query(): ), -- From ranked_runs, keep only the top run per user. - top_runs AS (SELECT * FROM ranked_runs WHERE rank = 1) + top_runs AS ( + SELECT r.*, + validation.status AS validation_status, + validation.passed_shapes AS validation_shapes_passed, + validation.total_shapes AS validation_shapes_total, + validation.fully_validated AS validation_fully_validated, + validation.geomean_sync_wall_speedup AS validation_geomean_speedup, + validation.contract_version AS validation_contract_version, + validation.checked_at AS validation_checked_at + FROM ranked_runs r + LEFT JOIN LATERAL ( + SELECT + status, + passed_shapes, + total_shapes, + fully_validated, + geomean_sync_wall_speedup, + contract_version, + checked_at + FROM leaderboard.submission_validation + WHERE submission_id = r.submission_id + AND gpu_type = r.runner + AND contract_version = ( + SELECT task->'validation'->>'version' + FROM leaderboard.leaderboard + WHERE id = %(leaderboard_id)s + ) + ORDER BY checked_at DESC + LIMIT 1 + ) validation ON TRUE + WHERE r.rank = 1 + ) SELECT jsonb_build_object( 'rankings', (SELECT jsonb_object_agg(g.gpu_type, ( diff --git a/kernelboard/api/leaderboard_summaries.py b/kernelboard/api/leaderboard_summaries.py index a18c2bbc..a10f2b6a 100644 --- a/kernelboard/api/leaderboard_summaries.py +++ b/kernelboard/api/leaderboard_summaries.py @@ -48,7 +48,7 @@ def _get_cached_top_users(redis_conn, leaderboard_ids: list[int]) -> dict[int, l return {} -def _set_cached_top_users(redis_conn, leaderboard_id: int, top_users: list): +def _set_cached_top_users(redis_conn, leaderboard_id: int, top_users: list | None): """Cache top_users for ended leaderboard (no expiry).""" if not redis_conn: return @@ -179,8 +179,7 @@ def _get_leaderboards_cached(total_start: float, force_refresh: bool = False): # 5. Cache newly computed ended leaderboards for lb_id in uncached_ended_ids: - if lb_id in computed_results: - _set_cached_top_users(redis_conn, lb_id, computed_results[lb_id]) + _set_cached_top_users(redis_conn, lb_id, computed_results.get(lb_id)) # 6. Get metadata for all leaderboards cur.execute(_get_leaderboard_metadata_query()) diff --git a/kernelboard/lib/db.py b/kernelboard/lib/db.py index f0b4f7be..a97f2a04 100644 --- a/kernelboard/lib/db.py +++ b/kernelboard/lib/db.py @@ -16,7 +16,6 @@ def get_db_connection() -> psycopg2.extensions.connection: raise RuntimeError( "DATABASE_URL is not set in the application configuration." ) - print("database_url", database_url) g.db_connection = psycopg2.connect(database_url) return g.db_connection diff --git a/tests/api/test_leaderboard_api.py b/tests/api/test_leaderboard_api.py index fce17306..47ebfca2 100644 --- a/tests/api/test_leaderboard_api.py +++ b/tests/api/test_leaderboard_api.py @@ -428,3 +428,33 @@ def test_hacked_submissions_are_hidden_from_public_leaderboard(client, app): assert "hidden_submission_status_hacked.py" not in ranked_files assert "hidden_job_status_hacked.py" not in ranked_files assert "visible_clean_status.py" in ranked_files + + +def test_equal_best_runs_preserve_ties_and_distinct_submission_count(client, app): + before = client.get("/api/leaderboard/339").get_json()["data"]["rankings"] + gpu, entries = next(iter(before.items())) + winner = entries[0] + with app.app_context(): + conn = get_db_connection() + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO leaderboard.runs ( + id, submission_id, start_time, end_time, mode, secret, + runner, score, passed, compilation, meta, result, system_info + ) + SELECT 990001, submission_id, start_time, end_time, mode, secret, + runner, score, passed, compilation, meta, result, system_info + FROM leaderboard.runs + WHERE submission_id = %s AND runner = %s + AND NOT secret AND passed AND score = %s + LIMIT 1 + """, + (winner["submission_id"], gpu, winner["score"]), + ) + conn.commit() + after = client.get("/api/leaderboard/339").get_json()["data"]["rankings"][gpu] + tied = [item for item in after if item["submission_id"] == winner["submission_id"]] + assert len(tied) == 2 + assert all(item["score"] == winner["score"] for item in tied) + assert all(item["submission_count"] == winner["submission_count"] for item in tied) diff --git a/tests/api/test_leaderboard_summaries_api.py b/tests/api/test_leaderboard_summaries_api.py index fda4ac25..6307f150 100644 --- a/tests/api/test_leaderboard_summaries_api.py +++ b/tests/api/test_leaderboard_summaries_api.py @@ -24,3 +24,30 @@ def test_leaderboard_metadata(client): assert all( ids[i] > ids[i + 1] for i in range(len(ids) - 1) ), f"Leaderboard IDs are not in decreasing order: {ids}" + + +def test_empty_ended_leaderboard_is_cached(client, app, monkeypatch): + from kernelboard.api import leaderboard_summaries as summaries + from kernelboard.lib.db import get_db_connection + + with app.app_context(): + conn = get_db_connection() + with conn.cursor() as cur: + cur.execute("UPDATE leaderboard.leaderboard SET deadline = NOW() - INTERVAL '1 day'") + cur.execute("UPDATE leaderboard.runs SET passed = FALSE") + cur.execute("SELECT id FROM leaderboard.leaderboard") + leaderboard_ids = [row[0] for row in cur.fetchall()] + summaries._delete_cached_top_users(summaries._get_redis(), leaderboard_ids) + conn.commit() + + first = client.get("/api/leaderboard-summaries") + assert first.status_code == 200 + assert all(lb["top_users"] is None for lb in first.get_json()["data"]["leaderboards"]) + + def unexpected_recompute(): + raise AssertionError("Empty ended leaderboards should be served from cache") + + monkeypatch.setattr(summaries, "_get_query_for_ids", unexpected_recompute) + second = client.get("/api/leaderboard-summaries") + assert second.status_code == 200 + assert second.get_json() == first.get_json()