diff --git a/src/libkernelbot/leaderboard_db.py b/src/libkernelbot/leaderboard_db.py index 5ab239245..c01507537 100644 --- a/src/libkernelbot/leaderboard_db.py +++ b/src/libkernelbot/leaderboard_db.py @@ -884,6 +884,15 @@ def get_leaderboard_submissions( AND r.score IS NOT NULL AND r.passed AND s.user_id = %s + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.runs sr + WHERE sr.submission_id = s.id + AND sr.secret + AND sr.runner = r.runner + AND sr.mode = r.mode + AND sr.passed = FALSE + ) ORDER BY r.score ASC LIMIT %s OFFSET %s """ @@ -905,6 +914,15 @@ def get_leaderboard_submissions( JOIN leaderboard.user_info ui ON s.user_id = ui.id WHERE l.name = %s AND r.runner = %s AND NOT r.secret AND r.score IS NOT NULL AND r.passed + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.runs sr + WHERE sr.submission_id = s.id + AND sr.secret + AND sr.runner = r.runner + AND sr.mode = r.mode + AND sr.passed = FALSE + ) ORDER BY s.user_id, r.score ASC ) SELECT @@ -1244,8 +1262,19 @@ def get_user_submissions( submission_ids = [row[0] for row in submissions] runs_query = """ SELECT submission_id, runner as gpu_type, score - FROM leaderboard.runs - WHERE submission_id = ANY(%s) AND NOT secret AND passed + FROM leaderboard.runs r + WHERE submission_id = ANY(%s) + AND NOT secret + AND passed + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.runs sr + WHERE sr.submission_id = r.submission_id + AND sr.secret + AND sr.runner = r.runner + AND sr.mode = r.mode + AND sr.passed = FALSE + ) """ self.cursor.execute(runs_query, (submission_ids,)) runs_by_submission: dict = {} @@ -1384,6 +1413,15 @@ def get_leaderboard_submission_count( AND r.score IS NOT NULL AND r.passed AND s.user_id = %s + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.runs sr + WHERE sr.submission_id = s.id + AND sr.secret + AND sr.runner = r.runner + AND sr.mode = r.mode + AND sr.passed = FALSE + ) """ args = (leaderboard_name, gpu_name, user_id) else: @@ -1397,6 +1435,15 @@ def get_leaderboard_submission_count( AND NOT r.secret AND r.score IS NOT NULL AND r.passed + AND NOT EXISTS ( + SELECT 1 + FROM leaderboard.runs sr + WHERE sr.submission_id = s.id + AND sr.secret + AND sr.runner = r.runner + AND sr.mode = r.mode + AND sr.passed = FALSE + ) """ args = (leaderboard_name, gpu_name) diff --git a/tests/test_leaderboard_db.py b/tests/test_leaderboard_db.py index 1ebf3b290..e50cd9c03 100644 --- a/tests/test_leaderboard_db.py +++ b/tests/test_leaderboard_db.py @@ -368,6 +368,75 @@ def test_leaderboard_submission_ranked(database, submit_leaderboard): }, ] + +def test_failed_secret_run_hides_submission_from_rankings(database, submit_leaderboard): + submit_time = datetime.datetime.now(tz=datetime.timezone.utc) + failed_secret = dataclasses.replace(sample_run_result(), passed=False) + + with database as db: + hacked = db.create_submission( + "submit-leaderboard", "fast.py", 5, "fast", submit_time, user_name="user5" + ) + _create_submission_run(db, hacked, mode="leaderboard", runner="A100", score=1.0) + _create_submission_run( + db, + hacked, + mode="leaderboard", + secret=True, + runner="A100", + score=None, + result=failed_secret, + ) + db.mark_submission_done(hacked) + + valid = db.create_submission( + "submit-leaderboard", "valid.py", 6, "valid", submit_time, user_name="user6" + ) + _create_submission_run(db, valid, mode="leaderboard", runner="A100", score=2.0) + _create_submission_run( + db, + valid, + mode="leaderboard", + secret=True, + runner="A100", + score=None, + ) + db.mark_submission_done(valid) + + with database as db: + ranked = db.get_leaderboard_submissions("submit-leaderboard", "A100") + assert [row["submission_id"] for row in ranked] == [valid] + assert db.get_leaderboard_submission_count("submit-leaderboard", "A100") == 1 + assert db.get_leaderboard_submission_count("submit-leaderboard", "A100", "5") == 0 + + +def test_failed_secret_run_hides_user_submission_scores(database, submit_leaderboard): + submit_time = datetime.datetime.now(tz=datetime.timezone.utc) + failed_secret = dataclasses.replace(sample_run_result(), passed=False) + + with database as db: + sub_id = db.create_submission( + "submit-leaderboard", "fast.py", 5, "fast", submit_time, user_name="user5" + ) + _create_submission_run(db, sub_id, mode="leaderboard", runner="A100", score=1.0) + _create_submission_run( + db, + sub_id, + mode="leaderboard", + secret=True, + runner="A100", + score=None, + result=failed_secret, + ) + db.mark_submission_done(sub_id) + + with database as db: + submissions = db.get_user_submissions("5", leaderboard_name="submit-leaderboard") + assert len(submissions) == 1 + assert submissions[0]["id"] == sub_id + assert submissions[0]["runs"] == [] + + def test_validate_identity_web_auth_happy_path(database, submit_leaderboard): with database as db: db.cursor.execute(