diff --git a/kernelboard/api/leaderboard.py b/kernelboard/api/leaderboard.py index 6f33bf71..e1ed9ba6 100644 --- a/kernelboard/api/leaderboard.py +++ b/kernelboard/api/leaderboard.py @@ -166,7 +166,18 @@ def _get_query(): JOIN leaderboard.submission s ON r.submission_id = s.id LEFT JOIN leaderboard.user_info u ON s.user_id = u.id LEFT JOIN submission_counts sc ON s.user_id = sc.user_id AND r.runner = sc.runner - WHERE NOT r.secret AND r.score IS NOT NULL AND r.passed AND s.leaderboard_id = %(leaderboard_id)s + WHERE NOT r.secret + AND r.score IS NOT NULL + AND r.passed + AND s.leaderboard_id = %(leaderboard_id)s + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.runs sr + WHERE sr.submission_id = s.id + AND sr.secret + AND sr.runner = r.runner + AND sr.passed = FALSE + ) ), -- From ranked_runs, keep only the top run per user. @@ -248,6 +259,14 @@ def get_custom_trend(leaderboard_id: int): AND r.score IS NOT NULL AND r.passed = true AND NOT r.secret + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.runs sr + WHERE sr.submission_id = s.id + AND sr.secret + AND sr.runner = r.runner + AND sr.passed = FALSE + ) ORDER BY s.submission_time ASC """ cur.execute(sql, (HARDCODED_USER_ID, leaderboard_id)) @@ -369,6 +388,14 @@ def get_user_trend(leaderboard_id: int): AND r.score IS NOT NULL AND r.passed = true AND NOT r.secret + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.runs sr + WHERE sr.submission_id = s.id + AND sr.secret + AND sr.runner = r.runner + AND sr.passed = FALSE + ) ORDER BY s.submission_time ASC """ cur.execute(sql, (*user_id_list, leaderboard_id)) @@ -455,6 +482,14 @@ def get_fastest_trend(leaderboard_id: int): AND r.score IS NOT NULL AND r.passed = true AND NOT r.secret + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.runs sr + WHERE sr.submission_id = s.id + AND sr.secret + AND sr.runner = r.runner + AND sr.passed = FALSE + ) ORDER BY s.submission_time ASC """ cur.execute(sql, (leaderboard_id,)) diff --git a/kernelboard/api/leaderboard_summaries.py b/kernelboard/api/leaderboard_summaries.py index f0a2f82f..a220e5c5 100644 --- a/kernelboard/api/leaderboard_summaries.py +++ b/kernelboard/api/leaderboard_summaries.py @@ -370,6 +370,14 @@ def _get_query_for_ids(): AND r.score IS NOT NULL AND r.passed AND s.leaderboard_id IN %s + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.runs sr + WHERE sr.submission_id = s.id + AND sr.secret + AND sr.runner = r.runner + AND sr.passed = FALSE + ) ), personal_best_runs AS ( SELECT * FROM personal_best_candidates @@ -466,6 +474,14 @@ def _get_query(): WHERE NOT r.secret AND r.score IS NOT NULL AND r.passed + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.runs sr + WHERE sr.submission_id = s.id + AND sr.secret + AND sr.runner = r.runner + AND sr.passed = FALSE + ) ), -- Step 2: Select only the best run for each user diff --git a/kernelboard/index.py b/kernelboard/index.py index 691fcfb4..c77f8faa 100644 --- a/kernelboard/index.py +++ b/kernelboard/index.py @@ -83,7 +83,17 @@ def index(): JOIN priority_gpu_types p on p.leaderboard_id = a.id AND p.gpu_type = r.runner LEFT JOIN leaderboard.user_info u ON s.user_id = u.id - WHERE NOT r.secret AND r.score IS NOT NULL AND r.passed + WHERE NOT r.secret + AND r.score IS NOT NULL + AND r.passed + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.runs sr + WHERE sr.submission_id = s.id + AND sr.secret + AND sr.runner = r.runner + AND sr.passed = FALSE + ) ), -- Select only the best run for each user and GPU type. diff --git a/kernelboard/leaderboard.py b/kernelboard/leaderboard.py index 7f50506e..353a0b5b 100644 --- a/kernelboard/leaderboard.py +++ b/kernelboard/leaderboard.py @@ -42,7 +42,18 @@ def leaderboard(leaderboard_id: int): FROM leaderboard.runs r JOIN leaderboard.submission s ON r.submission_id = s.id LEFT JOIN leaderboard.user_info u ON s.user_id = u.id - WHERE NOT r.secret AND r.score IS NOT NULL AND r.passed AND s.leaderboard_id = %(leaderboard_id)s + WHERE NOT r.secret + AND r.score IS NOT NULL + AND r.passed + AND s.leaderboard_id = %(leaderboard_id)s + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.runs sr + WHERE sr.submission_id = s.id + AND sr.secret + AND sr.runner = r.runner + AND sr.passed = FALSE + ) ), -- From ranked_runs, keep only the top run per user. diff --git a/ranking_worker.py b/ranking_worker.py index 2ed1e588..e9bfb5cc 100644 --- a/ranking_worker.py +++ b/ranking_worker.py @@ -130,6 +130,14 @@ def ensure_snapshot_table(conn): WHERE NOT r.secret AND r.score IS NOT NULL AND r.passed + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.runs sr + WHERE sr.submission_id = s.id + AND sr.secret + AND sr.runner = r.runner + AND sr.passed = FALSE + ) ), personal_best_runs AS ( diff --git a/tests/api/test_leaderboard_api.py b/tests/api/test_leaderboard_api.py index 2b92cd3b..a27e5a2a 100644 --- a/tests/api/test_leaderboard_api.py +++ b/tests/api/test_leaderboard_api.py @@ -26,3 +26,95 @@ def test_leaderboard_no_submissions(client, app): res = response.get_json() assert res["data"]["rankings"] == {} + + +def test_failed_secret_benchmark_hides_public_leaderboard_run(client, app): + with app.app_context(): + conn = get_db_connection() + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO leaderboard.submission + (id, leaderboard_id, file_name, user_id, code_id, submission_time, done) + VALUES + (900001, 339, 'hidden_secret_fail.py', '123456789012345', 13, NOW(), TRUE), + (900002, 339, 'visible_public_pass.py', '234567890123456', 13, NOW(), TRUE) + """ + ) + cur.execute( + """ + INSERT INTO leaderboard.runs + ( + id, + submission_id, + start_time, + end_time, + mode, + secret, + runner, + score, + passed, + compilation, + meta, + result, + system_info + ) + VALUES + ( + 900001, + 900001, + NOW(), + NOW(), + 'leaderboard', + FALSE, + 'H100', + -999, + TRUE, + '{}', + '{}', + '{}', + '{}' + ), + ( + 900002, + 900001, + NOW(), + NOW(), + 'benchmark', + TRUE, + 'H100', + NULL, + FALSE, + '{}', + '{}', + '{}', + '{}' + ), + ( + 900003, + 900002, + NOW(), + NOW(), + 'leaderboard', + FALSE, + 'H100', + -998, + TRUE, + '{}', + '{}', + '{}', + '{}' + ) + """ + ) + conn.commit() + + response = client.get("/api/leaderboard/339") + assert response.status_code == 200 + + payload = response.get_json() + h100_rankings = payload["data"]["rankings"]["H100"] + ranked_files = {row["file_name"] for row in h100_rankings} + + assert "hidden_secret_fail.py" not in ranked_files + assert "visible_public_pass.py" in ranked_files