diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6ffbcea6c..c1860cffc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Node uses: actions/setup-node@v4 with: - node-version: 22 + node-version: "24" cache: yarn - name: Install dependencies run: yarn install --frozen-lockfile @@ -45,7 +45,7 @@ jobs: - name: Set up Node uses: actions/setup-node@v4 with: - node-version: 22 + node-version: "24" cache: yarn - name: Install dependencies (pre-pulling the database image) run: | diff --git a/.nvmrc b/.nvmrc index 8fdd954df..a45fd52cc 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -22 \ No newline at end of file +24 diff --git a/Dockerfile b/Dockerfile index 6e1511d1e..cde81b375 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:22-alpine AS deps +FROM node:24-alpine AS deps WORKDIR /build COPY package*.json ./ @@ -6,7 +6,7 @@ COPY yarn.lock ./ RUN yarn install -FROM node:22-alpine AS builder +FROM node:24-alpine AS builder WORKDIR /build COPY --from=deps /build/node_modules ./node_modules @@ -14,7 +14,7 @@ COPY . . RUN yarn build -FROM node:22-alpine +FROM node:24-alpine WORKDIR /opt/5stack diff --git a/Dockerfile.dev b/Dockerfile.dev index 700fc2720..a7b2261da 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM node:22-alpine AS deps +FROM node:24-alpine AS deps WORKDIR /build COPY package*.json ./ @@ -6,7 +6,7 @@ COPY yarn.lock ./ RUN yarn install -FROM node:22-alpine +FROM node:24-alpine WORKDIR /opt/5stack diff --git a/codepier.yaml b/codepier.yaml index 4451d2d21..aa24ea11f 100644 --- a/codepier.yaml +++ b/codepier.yaml @@ -1,5 +1,5 @@ namespaces: [5stack] -image: node:22 +image: node:24 deployment: api workdir: /opt/5stack sync: diff --git a/hasura/functions/leaderboard/get_leaderboard.sql b/hasura/functions/leaderboard/get_leaderboard.sql index c69c5a1f4..f8c7798b9 100644 --- a/hasura/functions/leaderboard/get_leaderboard.sql +++ b/hasura/functions/leaderboard/get_leaderboard.sql @@ -795,8 +795,8 @@ $$; -- ============================================================ -- Awards leaderboard -- value = gold count, secondary = silver count, tertiary = bronze count --- matches_played = total medals. Olympic medal-table ordering. Only tournament --- placements count; hand-granted awards must not move rankings. +-- matches_played = mvp count. Olympic medal-table ordering, MVP first. +-- Hand-granted awards count the same as calculated ones. -- ============================================================ CREATE OR REPLACE FUNCTION public._leaderboard_awards( _window_days INT, @@ -825,23 +825,35 @@ BEGIN END IF; RETURN QUERY - WITH counts AS ( - SELECT + -- A tier counts once per player per tournament, season, event or league season, + -- however often it was granted there. Only unscoped grants, which need the grant + -- role, count every row. + WITH medals AS ( + SELECT DISTINCT ar.player_steam_id, - SUM(CASE WHEN ar.placement = 0 THEN 1 ELSE 0 END)::int as mvp, - SUM(CASE WHEN ar.placement = 1 THEN 1 ELSE 0 END)::int as gold, - SUM(CASE WHEN ar.placement = 2 THEN 1 ELSE 0 END)::int as silver, - SUM(CASE WHEN ar.placement = 3 THEN 1 ELSE 0 END)::int as bronze, - COUNT(*)::int as total + medal.tier, + COALESCE(ar.tournament_id, ar.season_id, ar.event_id, ar.league_season_id, ar.id) AS occasion FROM award_recipients ar + JOIN awards a ON a.id = ar.award_id + CROSS JOIN LATERAL ( + SELECT COALESCE(ar.placement_tier, a.tier) AS tier + ) medal -- Season placements have no match_options, so a match-type filter has to -- exclude them rather than drop them for a missing join. LEFT JOIN tournaments t ON t.id = ar.tournament_id LEFT JOIN match_options mo ON mo.id = t.match_options_id LEFT JOIN seasons s ON s.id = ar.season_id WHERE ar.player_steam_id IS NOT NULL - AND ar.placement IS NOT NULL - AND (ar.tournament_id IS NOT NULL OR ar.season_id IS NOT NULL) + AND medal.tier IN ('mvp', 'gold', 'silver', 'bronze') + AND ( + ar.tournament_id IS NOT NULL + OR ar.season_id IS NOT NULL + OR ( + _match_type IS NULL + AND (_from IS NULL OR ar.created_at >= _from) + AND (_to IS NULL OR ar.created_at < _to) + ) + ) -- Awards hang off a tournament, so there is no matchmaking bucket: any -- non-Overall source keeps only tournament-backed awards of that kind, -- and Matchmaking correctly comes back empty. @@ -875,7 +887,17 @@ BEGIN ) ) ) - GROUP BY ar.player_steam_id + ), + counts AS ( + SELECT + m.player_steam_id, + COUNT(*) FILTER (WHERE m.tier = 'mvp')::int as mvp, + COUNT(*) FILTER (WHERE m.tier = 'gold')::int as gold, + COUNT(*) FILTER (WHERE m.tier = 'silver')::int as silver, + COUNT(*) FILTER (WHERE m.tier = 'bronze')::int as bronze, + COUNT(*)::int as total + FROM medals m + GROUP BY m.player_steam_id ) SELECT c.player_steam_id::text as player_steam_id, @@ -938,7 +960,14 @@ BEGIN SELECT le.player_steam_id, le.value, - (RANK() OVER (ORDER BY le.value DESC))::int AS rank, + -- Must match web/pages/leaderboard.vue's order_by or jump-to-player opens the wrong page. + (RANK() OVER ( + ORDER BY + CASE WHEN _category = 'awards' THEN le.matches_played END DESC, + le.value DESC, + CASE WHEN _category = 'awards' THEN le.secondary_value END DESC, + CASE WHEN _category = 'awards' THEN le.tertiary_value END DESC + ))::int AS rank, (COUNT(*) OVER ())::int AS total -- Pass all 7 args explicitly. A shorter call binds ambiguously if a stale -- overload still exists; exact arity always resolves the 7-arg one. diff --git a/hasura/functions/match/match_players_per_lineup.sql b/hasura/functions/match/match_players_per_lineup.sql index 0aa36adf3..ad745f473 100644 --- a/hasura/functions/match/match_players_per_lineup.sql +++ b/hasura/functions/match/match_players_per_lineup.sql @@ -1,9 +1,23 @@ +-- Inlined, not is_tournament_match(): functions/tournaments/ is applied after +-- this directory and LANGUAGE sql bodies are validated at create. CREATE OR REPLACE FUNCTION public.match_max_players_per_lineup(match matches) RETURNS integer LANGUAGE sql STABLE AS $$ - SELECT get_match_type_min_players(mo.type) + COALESCE(mo.number_of_substitutes, 0) + SELECT get_match_type_min_players(mo.type) + CASE + -- A computed field on every match list row: without substitutes there is nothing to look up. + WHEN COALESCE(mo.number_of_substitutes, 0) = 0 THEN 0 + WHEN EXISTS ( + SELECT 1 + FROM tournament_brackets tb + INNER JOIN tournament_stages ts ON ts.id = tb.tournament_stage_id + INNER JOIN tournaments t ON t.id = ts.tournament_id + WHERE tb.match_id = match.id + AND (mo.type = 'Duel' OR NOT t.substitutes_enabled) + ) THEN 0 + ELSE mo.number_of_substitutes + END FROM match_options mo WHERE mo.id = match.match_options_id; $$; @@ -18,12 +32,16 @@ AS $$ WHERE mo.id = match.match_options_id; $$; +-- Duel is checked per tournament: draft games and one-off Duels still take substitutes. CREATE OR REPLACE FUNCTION public.tournament_max_players_per_lineup(tournament tournaments) RETURNS integer LANGUAGE sql STABLE AS $$ - SELECT get_match_type_min_players(mo.type) + COALESCE(mo.number_of_substitutes, 0) + SELECT get_match_type_min_players(mo.type) + CASE + WHEN mo.type = 'Duel' OR NOT tournament.substitutes_enabled THEN 0 + ELSE COALESCE(mo.number_of_substitutes, 0) + END FROM match_options mo WHERE mo.id = tournament.match_options_id; $$; diff --git a/hasura/functions/tournaments/check_team_eligibility.sql b/hasura/functions/tournaments/check_team_eligibility.sql index f8e52d3da..9d7ff4a92 100644 --- a/hasura/functions/tournaments/check_team_eligibility.sql +++ b/hasura/functions/tournaments/check_team_eligibility.sql @@ -1,4 +1,7 @@ -CREATE OR REPLACE FUNCTION public.check_team_eligibility(roster tournament_team_roster) RETURNS VOID +-- Without the drop the one-argument version survives as an ambiguous overload. +DROP FUNCTION IF EXISTS public.check_team_eligibility(tournament_team_roster); + +CREATE OR REPLACE FUNCTION public.check_team_eligibility(roster tournament_team_roster, enforce_max_players boolean DEFAULT true) RETURNS VOID LANGUAGE plpgsql AS $$ DECLARE @@ -18,10 +21,12 @@ BEGIN RETURN; END IF; - max_players := tournament_max_players_per_lineup(tournament); + IF enforce_max_players THEN + max_players := tournament_max_players_per_lineup(tournament); - IF roster_count > max_players THEN - RAISE EXCEPTION USING ERRCODE = '22000', MESSAGE = 'Roster has too many players'; + IF roster_count > max_players THEN + RAISE EXCEPTION USING ERRCODE = '22000', MESSAGE = 'Roster has too many players'; + END IF; END IF; min_players := tournament_min_players_per_lineup(tournament); @@ -38,4 +43,4 @@ BEGIN SET eligible_at = NOW() WHERE id = roster.tournament_team_id; END; -$$; \ No newline at end of file +$$; diff --git a/hasura/functions/tournaments/get_stage_qualifier_seeds.sql b/hasura/functions/tournaments/get_stage_qualifier_seeds.sql new file mode 100644 index 000000000..5aa8a3d30 --- /dev/null +++ b/hasura/functions/tournaments/get_stage_qualifier_seeds.sql @@ -0,0 +1,89 @@ +DROP FUNCTION IF EXISTS public.get_team_at_stage_rank(uuid, int, int); + +-- Wildcards rank by win rate, not wins (uneven groups), and never head-to-head (they never met). +CREATE OR REPLACE FUNCTION public.get_stage_qualifier_seeds( + _stage_id uuid, + _seeds int +) RETURNS TABLE(seed int, tournament_team_id uuid) +LANGUAGE plpgsql STABLE +AS $$ +#variable_conflict use_column +DECLARE + _groups int; + _whole_placements int; +BEGIN + SELECT GREATEST(COALESCE(ts.groups, 1), 1) + INTO _groups + FROM tournament_stages ts + WHERE ts.id = _stage_id; + + _whole_placements := _seeds / _groups; + + RETURN QUERY + WITH eligible AS ( + SELECT + vtsr.tournament_team_id, + vtsr.group_number, + vtsr.wins, + vtsr.losses, + vtsr.maps_won, + vtsr.maps_lost, + vtsr.rounds_won, + vtsr.rounds_lost, + vtsr.team_kdr, + tt.seed AS tournament_seed, + ROW_NUMBER() OVER ( + PARTITION BY vtsr.group_number + ORDER BY vtsr.rank + ) AS placement + FROM v_team_stage_results vtsr + INNER JOIN tournament_teams tt + ON tt.id = vtsr.tournament_team_id + AND tt.eligible_at IS NOT NULL + WHERE vtsr.tournament_stage_id = _stage_id + ), + wildcards AS ( + SELECT + e.tournament_team_id, + ROW_NUMBER() OVER ( + ORDER BY + e.placement, + CASE WHEN e.wins + e.losses > 0 + THEN e.wins::float / (e.wins + e.losses) + ELSE 0 + END DESC, + CASE WHEN e.maps_lost > 0 + THEN e.maps_won::float / e.maps_lost + ELSE e.maps_won::float + END DESC, + CASE WHEN e.rounds_lost > 0 + THEN e.rounds_won::float / e.rounds_lost + ELSE e.rounds_won::float + END DESC, + e.team_kdr DESC, + e.tournament_seed ASC NULLS LAST, + e.tournament_team_id ASC + ) AS position + FROM eligible e + WHERE e.placement > _whole_placements + ), + -- Numbered in order rather than by slot: a group short of its guaranteed places leaves its seed to the wildcards. + qualifiers AS ( + SELECT 0 AS pool, (e.placement - 1) * _groups + e.group_number AS slot, e.tournament_team_id + FROM eligible e + WHERE e.placement <= _whole_placements + UNION ALL + SELECT 1, w.position, w.tournament_team_id + FROM wildcards w + ), + seeded AS ( + SELECT + ROW_NUMBER() OVER (ORDER BY q.pool, q.slot) AS seed, + q.tournament_team_id + FROM qualifiers q + ) + SELECT s.seed::int, s.tournament_team_id + FROM seeded s + WHERE s.seed <= _seeds; +END; +$$; diff --git a/hasura/functions/tournaments/get_stage_team_counts.sql b/hasura/functions/tournaments/get_stage_team_counts.sql index 2f799f773..0a2290ddc 100644 --- a/hasura/functions/tournaments/get_stage_team_counts.sql +++ b/hasura/functions/tournaments/get_stage_team_counts.sql @@ -36,16 +36,24 @@ BEGIN DECLARE previous_stage_type text; previous_stage_max_teams int; + previous_stage_no_elimination boolean; + previous_stage_teams int; BEGIN - SELECT type, max_teams INTO previous_stage_type, previous_stage_max_teams + SELECT type, max_teams, swiss_no_elimination + INTO previous_stage_type, previous_stage_max_teams, previous_stage_no_elimination FROM tournament_stages WHERE tournament_id = _tournament_id AND "order" = _stage_order - 1; - -- For RoundRobin stages, only the top teams advance, capped by this - -- stage's max_teams. Using the previous stage's max_teams here would - -- size brackets for every team in the RR, not just the qualifiers. - IF previous_stage_type = 'RoundRobin' THEN + IF previous_stage_type = 'RoundRobin' + OR (previous_stage_type = 'Swiss' AND previous_stage_no_elimination) THEN effective_teams := LEAST(stage_max_teams, previous_stage_max_teams); + ELSIF previous_stage_type = 'Swiss' THEN + SELECT previous.effective_teams + INTO previous_stage_teams + FROM get_stage_team_counts(_tournament_id, _stage_order - 1, _tournament_status) previous; + + -- Half the field reaches 3 wins, rounded up: an odd field's bye is a free win. + effective_teams := LEAST(stage_max_teams, CEIL(previous_stage_teams / 2.0)::int); ELSE -- get the number of matches from the last round of the previous stage SELECT COUNT(*) INTO effective_teams diff --git a/hasura/functions/tournaments/get_team_at_stage_rank.sql b/hasura/functions/tournaments/get_team_at_stage_rank.sql deleted file mode 100644 index e1b837b0f..000000000 --- a/hasura/functions/tournaments/get_team_at_stage_rank.sql +++ /dev/null @@ -1,39 +0,0 @@ --- Returns the tournament_team_id of the team ranked `_rank` (1-indexed) within --- the given stage group, considering only eligible teams (`eligible_at IS NOT --- NULL`). The tiebreaker chain itself lives in v_team_stage_results so the UI --- standings and bracket-progression seeding never disagree -- this function --- just re-ranks within the eligible subset so disqualified teams don't take up --- a seed slot in the next stage. -CREATE OR REPLACE FUNCTION public.get_team_at_stage_rank( - _stage_id uuid, - _group int, - _rank int -) RETURNS uuid -LANGUAGE plpgsql STABLE -AS $$ -DECLARE - result_team_id uuid; -BEGIN - WITH eligible_ranked AS ( - SELECT - vtsr.tournament_team_id, - vtsr.group_number, - ROW_NUMBER() OVER ( - PARTITION BY vtsr.group_number - ORDER BY vtsr.rank - ) as eligible_rank - FROM v_team_stage_results vtsr - INNER JOIN tournament_teams tt - ON tt.id = vtsr.tournament_team_id - AND tt.eligible_at IS NOT NULL - WHERE vtsr.tournament_stage_id = _stage_id - ) - SELECT tournament_team_id - INTO result_team_id - FROM eligible_ranked - WHERE group_number = _group - AND eligible_rank = _rank; - - RETURN result_team_id; -END; -$$; diff --git a/hasura/functions/tournaments/seed_stage.sql b/hasura/functions/tournaments/seed_stage.sql index aba48acd1..ba58b8fd1 100644 --- a/hasura/functions/tournaments/seed_stage.sql +++ b/hasura/functions/tournaments/seed_stage.sql @@ -11,6 +11,8 @@ DECLARE team_1_seed_val int; team_2_seed_val int; teams_assigned_count int; + _seed_count int; + _qualifiers uuid[]; BEGIN RAISE NOTICE '=== STARTING STAGE SEEDING ==='; RAISE NOTICE 'Stage ID: %', stage_id; @@ -90,6 +92,20 @@ BEGIN PERFORM public.seed_swiss_stage(stage_id); RETURN; ELSE + IF previous_stage.id IS NOT NULL AND (previous_stage.type = 'RoundRobin' OR previous_stage.type = 'Swiss') THEN + SELECT MAX(GREATEST(COALESCE(tb.team_1_seed, 0), COALESCE(tb.team_2_seed, 0))) + INTO _seed_count + FROM tournament_brackets tb + WHERE tb.tournament_stage_id = stage.id + AND COALESCE(tb.path, 'WB') = 'WB'; + + SELECT array_agg(q.tournament_team_id ORDER BY s.n) + INTO _qualifiers + FROM generate_series(1, COALESCE(_seed_count, 0)) AS s(n) + LEFT JOIN public.get_stage_qualifier_seeds(previous_stage.id, _seed_count) q + ON q.seed = s.n; + END IF; + -- Process first-round brackets for elimination tournaments -- For elimination: process first-round winners brackets FOR bracket IN @@ -105,33 +121,14 @@ BEGIN team_1_seed_val := bracket.team_1_seed; team_2_seed_val := bracket.team_2_seed; - -- For elimination brackets coming from RoundRobin/Swiss stages, use stage results - -- Otherwise, lookup teams by seed IF previous_stage.id IS NOT NULL AND (previous_stage.type = 'RoundRobin' OR previous_stage.type = 'Swiss') THEN - -- Map next-stage seed N to (group, rank_in_group) of the previous - -- stage so groups cross-seed: with G groups, seed 1 = top of group 1, - -- seed 2 = top of group 2, seed G+1 = 2nd of group 1, and so on. - -- Falls back to a single group for Swiss (or RR with groups = 1), - -- which degenerates to plain top-N ordering. - DECLARE - prev_groups int; - group_idx int; - rank_in_group int; - BEGIN - prev_groups := GREATEST(COALESCE(previous_stage.groups, 1), 1); - - IF team_1_seed_val IS NOT NULL THEN - group_idx := ((team_1_seed_val - 1) % prev_groups) + 1; - rank_in_group := ((team_1_seed_val - 1) / prev_groups) + 1; - team_1_id := get_team_at_stage_rank(previous_stage.id, group_idx, rank_in_group); - END IF; + IF team_1_seed_val IS NOT NULL THEN + team_1_id := _qualifiers[team_1_seed_val]; + END IF; - IF team_2_seed_val IS NOT NULL THEN - group_idx := ((team_2_seed_val - 1) % prev_groups) + 1; - rank_in_group := ((team_2_seed_val - 1) / prev_groups) + 1; - team_2_id := get_team_at_stage_rank(previous_stage.id, group_idx, rank_in_group); - END IF; - END; + IF team_2_seed_val IS NOT NULL THEN + team_2_id := _qualifiers[team_2_seed_val]; + END IF; ELSE -- Find team with matching seed for position 1 IF team_1_seed_val IS NOT NULL THEN diff --git a/hasura/metadata/databases/default/tables/public_tournaments.yaml b/hasura/metadata/databases/default/tables/public_tournaments.yaml index 7598b5adb..d527f13af 100644 --- a/hasura/metadata/databases/default/tables/public_tournaments.yaml +++ b/hasura/metadata/databases/default/tables/public_tournaments.yaml @@ -247,6 +247,7 @@ insert_permissions: - name - scheduling_mode - start + - substitutes_enabled - check_in_closes_before_minutes - check_in_opens_before_minutes - check_in_required @@ -303,6 +304,7 @@ insert_permissions: - name - scheduling_mode - start + - substitutes_enabled - check_in_closes_before_minutes - check_in_opens_before_minutes - check_in_required @@ -334,6 +336,7 @@ select_permissions: - organizer_steam_id - start - status + - substitutes_enabled - awards_enabled - check_in_closes_before_minutes - check_in_ends_at @@ -407,6 +410,7 @@ select_permissions: - organizer_steam_id - start - status + - substitutes_enabled - awards_enabled - check_in_closes_before_minutes - check_in_ends_at @@ -466,6 +470,7 @@ select_permissions: - organizer_steam_id - start - status + - substitutes_enabled - awards_enabled - check_in_closes_before_minutes - check_in_ends_at @@ -539,6 +544,7 @@ update_permissions: - scheduling_mode - start - status + - substitutes_enabled - awards_enabled - check_in_closes_before_minutes - check_in_opens_before_minutes @@ -572,6 +578,7 @@ update_permissions: - scheduling_mode - start - status + - substitutes_enabled - awards_enabled - check_in_closes_before_minutes - check_in_opens_before_minutes diff --git a/hasura/migrations/default/1886000000619_tournament_substitutes_enabled/down.sql b/hasura/migrations/default/1886000000619_tournament_substitutes_enabled/down.sql new file mode 100644 index 000000000..51d0781ca --- /dev/null +++ b/hasura/migrations/default/1886000000619_tournament_substitutes_enabled/down.sql @@ -0,0 +1,2 @@ +ALTER TABLE public.tournaments + DROP COLUMN IF EXISTS substitutes_enabled; diff --git a/hasura/migrations/default/1886000000619_tournament_substitutes_enabled/up.sql b/hasura/migrations/default/1886000000619_tournament_substitutes_enabled/up.sql new file mode 100644 index 000000000..64ab6df3f --- /dev/null +++ b/hasura/migrations/default/1886000000619_tournament_substitutes_enabled/up.sql @@ -0,0 +1,4 @@ +ALTER TABLE public.tournaments + ADD COLUMN IF NOT EXISTS substitutes_enabled boolean NOT NULL DEFAULT true; + +COMMENT ON COLUMN public.tournaments.substitutes_enabled IS 'Whether teams may roster and field substitutes beyond the starting lineup'; diff --git a/hasura/triggers/match_maps.sql b/hasura/triggers/match_maps.sql index 02a9b3d81..978feb3d4 100644 --- a/hasura/triggers/match_maps.sql +++ b/hasura/triggers/match_maps.sql @@ -36,13 +36,14 @@ DECLARE _auto_cancel_duration_override integer; _live_match_timeout_override integer; _live_match_timeout text; + _tv_delay integer; BEGIN SELECT auto_cancellation, auto_cancel_duration INTO _auto_cancellation, _auto_cancel_duration_override FROM resolve_match_auto_cancel(NEW.match_id); - SELECT mo.live_match_timeout - INTO _live_match_timeout_override + SELECT mo.live_match_timeout, mo.tv_delay + INTO _live_match_timeout_override, _tv_delay FROM matches m INNER JOIN match_options mo ON mo.id = m.match_options_id WHERE m.id = NEW.match_id; @@ -73,8 +74,33 @@ BEGIN END IF; END IF; + -- Re-armed so tv_delay + demo upload can't outlast a timeout counted from map start. + -- Live only: re-arming an ended match lets CancelExpiredMatches overwrite its result. + IF NEW.status IN ('WaitingForTV', 'UploadingDemo') AND OLD.status IS DISTINCT FROM NEW.status THEN + IF _auto_cancellation THEN + UPDATE matches + SET cancels_at = NOW() + (COALESCE(_tv_delay, 0) || ' seconds')::interval + (_live_match_timeout)::interval + WHERE id = NEW.match_id + AND status = 'Live'; + END IF; + END IF; + IF NEW.status = 'Finished' AND OLD.status IS DISTINCT FROM NEW.status THEN NEW.ended_at = NOW(); + + -- The next map gets a warmup window, not the end-of-map deadline that may have expired. + IF _auto_cancellation AND EXISTS ( + SELECT 1 + FROM match_maps mm + WHERE mm.match_id = NEW.match_id + AND mm.id <> NEW.id + AND mm.status NOT IN ('Finished', 'Surrendered', 'Canceled') + ) THEN + UPDATE matches + SET cancels_at = NOW() + (_auto_cancel_duration)::interval + WHERE id = NEW.match_id + AND status = 'Live'; + END IF; END IF; -- Server-side anchor for how long demo recording/upload has been running, diff --git a/hasura/triggers/tournament_stages.sql b/hasura/triggers/tournament_stages.sql index c2f024c59..cc02c570c 100644 --- a/hasura/triggers/tournament_stages.sql +++ b/hasura/triggers/tournament_stages.sql @@ -1,8 +1,12 @@ +DROP FUNCTION IF EXISTS public.validate_tournament_stage(text, integer, integer, integer, uuid, uuid); + -- Shared validation function for tournament stages CREATE OR REPLACE FUNCTION public.validate_tournament_stage( p_stage_type text, p_groups integer, p_min_teams integer, + p_max_teams integer, + p_swiss_no_elimination boolean, p_stage_order integer, p_tournament_id uuid, p_stage_id uuid @@ -12,8 +16,10 @@ LANGUAGE plpgsql AS $$ DECLARE prev_stage_record RECORD; + next_stage_record RECORD; max_teams_advancing int; last_round_matches int; + _groups int := GREATEST(COALESCE(p_groups, 1), 1); BEGIN -- Validate Swiss tournament requirements IF p_stage_type = 'Swiss' THEN @@ -25,8 +31,12 @@ BEGIN -- Note: Odd numbers are handled by pairing with adjacent pools END IF; - -- Validate first stage minimum teams (must be at least 4 * number of groups) - IF p_stage_order = 1 AND p_groups IS NOT NULL AND p_groups > 0 AND p_stage_type != 'Swiss' THEN + IF p_stage_type = 'RoundRobin' THEN + IF p_min_teams < 3 * _groups THEN + RAISE EXCEPTION 'Stage % must have at least % teams given % groups (minimum 3 teams per group)', + p_stage_order, 3 * _groups, _groups USING ERRCODE = '22000'; + END IF; + ELSIF p_stage_order = 1 AND p_groups IS NOT NULL AND p_groups > 0 AND p_stage_type != 'Swiss' THEN IF p_min_teams < 4 * p_groups THEN RAISE EXCEPTION 'First stage must have at least % teams given % groups (minimum 4 teams per group)', 4 * p_groups, p_groups USING ERRCODE = '22000'; @@ -40,7 +50,31 @@ BEGIN FROM tournament_stages WHERE tournament_id = p_tournament_id AND "order" = p_stage_order - 1; - IF prev_stage_record.id IS NOT NULL THEN + IF prev_stage_record.id IS NOT NULL AND prev_stage_record.type IN ('RoundRobin', 'Swiss') THEN + -- Valve Swiss only advances its 3-win teams: half the field, rounded up for an odd + -- field's bye. Its last round is just the 2-2 pool, so it never bounds this stage's minimum. + IF prev_stage_record.type = 'Swiss' AND NOT prev_stage_record.swiss_no_elimination THEN + max_teams_advancing := (prev_stage_record.max_teams + 1) / 2; + ELSE + max_teams_advancing := prev_stage_record.max_teams; + END IF; + + IF p_max_teams < 2 THEN + RAISE EXCEPTION 'Stage % must have at least 2 teams', p_stage_order USING ERRCODE = '22000'; + END IF; + + IF p_max_teams > max_teams_advancing THEN + IF prev_stage_record.type = 'Swiss' AND NOT prev_stage_record.swiss_no_elimination THEN + RAISE EXCEPTION 'Stage % takes % teams but only % teams can reach 3 wins in stage %', + p_stage_order, p_max_teams, max_teams_advancing, p_stage_order - 1 + USING ERRCODE = '22000'; + END IF; + + RAISE EXCEPTION 'Stage % advances % teams but stage % holds at most % teams', + p_stage_order, p_max_teams, p_stage_order - 1, max_teams_advancing + USING ERRCODE = '22000'; + END IF; + ELSIF prev_stage_record.id IS NOT NULL THEN -- Calculate max teams that can advance from previous stage -- Count matches in the last round of the previous stage (each match produces 1 winner) SELECT COUNT(*) INTO last_round_matches @@ -63,12 +97,34 @@ BEGIN -- This stage must be able to accommodate the advancing teams IF p_min_teams < max_teams_advancing THEN - RAISE EXCEPTION 'Stage % cannot accommodate % teams advancing from stage % (min_teams: %)', - p_stage_order, max_teams_advancing, p_stage_order - 1, p_min_teams + RAISE EXCEPTION 'Stage % cannot accommodate % teams advancing from stage % (min_teams: %)', + p_stage_order, max_teams_advancing, p_stage_order - 1, p_min_teams USING ERRCODE = '22000'; END IF; END IF; END IF; + + IF p_stage_type = 'RoundRobin' OR (p_stage_type = 'Swiss' AND COALESCE(p_swiss_no_elimination, false)) THEN + SELECT * INTO next_stage_record + FROM tournament_stages + WHERE tournament_id = p_tournament_id AND "order" = p_stage_order + 1; + + IF next_stage_record.id IS NOT NULL AND next_stage_record.max_teams > p_max_teams THEN + RAISE EXCEPTION 'Stage % advances % teams but stage % holds at most % teams', + p_stage_order + 1, next_stage_record.max_teams, p_stage_order, p_max_teams + USING ERRCODE = '22000'; + END IF; + ELSIF p_stage_type = 'Swiss' THEN + SELECT * INTO next_stage_record + FROM tournament_stages + WHERE tournament_id = p_tournament_id AND "order" = p_stage_order + 1; + + IF next_stage_record.id IS NOT NULL AND next_stage_record.max_teams > (p_max_teams + 1) / 2 THEN + RAISE EXCEPTION 'Stage % takes % teams but only % teams can reach 3 wins in stage %', + p_stage_order + 1, next_stage_record.max_teams, (p_max_teams + 1) / 2, p_stage_order + USING ERRCODE = '22000'; + END IF; + END IF; END; $$; @@ -125,6 +181,8 @@ BEGIN NEW.type, NEW.groups, NEW.min_teams, + NEW.max_teams, + NEW.swiss_no_elimination, current_order, NEW.tournament_id, NEW.id @@ -136,6 +194,18 @@ BEGIN AND id != NEW.id ORDER BY "order" ASC LOOP + -- Halving only chains elimination stages; a ranked stage in between breaks the chain. + IF EXISTS ( + SELECT 1 + FROM tournament_stages ranked + WHERE ranked.tournament_id = NEW.tournament_id + AND ranked.type IN ('RoundRobin', 'Swiss') + AND ranked."order" >= LEAST(stage_record."order", current_order) + AND ranked."order" < GREATEST(stage_record."order", current_order) + ) THEN + CONTINUE; + END IF; + IF stage_record."order" < current_order THEN next_min_teams := _min_teams * (2 ^ (current_order - stage_record."order")); ELSE @@ -161,7 +231,7 @@ BEGIN -- Validate groups number can divide next stage's min_teams with remainder 0 -- This check happens after all stages have been updated - IF NEW.groups IS NOT NULL AND NEW.groups > 1 THEN + IF NEW.groups IS NOT NULL AND NEW.groups > 1 AND NEW.type != 'RoundRobin' THEN -- Get the next stage in sequence SELECT * INTO next_stage_record FROM tournament_stages @@ -268,6 +338,8 @@ BEGIN NEW.type, NEW.groups, NEW.min_teams, + NEW.max_teams, + NEW.swiss_no_elimination, NEW."order", NEW.tournament_id, NEW.id diff --git a/hasura/triggers/tournament_team_roster.sql b/hasura/triggers/tournament_team_roster.sql index 8255bf8cf..edb926bbc 100644 --- a/hasura/triggers/tournament_team_roster.sql +++ b/hasura/triggers/tournament_team_roster.sql @@ -4,12 +4,15 @@ CREATE OR REPLACE FUNCTION public.taiud_tournament_team_roster() RETURNS TRIGGER DECLARE _team_id uuid; BEGIN + -- Max players only on joining: a lowered cap must not block a roster shedding players. IF TG_OP = 'DELETE' THEN - PERFORM check_team_eligibility(OLD); + PERFORM check_team_eligibility(OLD, false); + ELSIF TG_OP = 'INSERT' THEN + PERFORM check_team_eligibility(NEW, true); ELSE - PERFORM check_team_eligibility(NEW); + PERFORM check_team_eligibility(NEW, NEW.tournament_team_id IS DISTINCT FROM OLD.tournament_team_id); END IF; - + RETURN NEW; END; $$; diff --git a/hasura/triggers/tournaments.sql b/hasura/triggers/tournaments.sql index 994c4275f..b999578fe 100644 --- a/hasura/triggers/tournaments.sql +++ b/hasura/triggers/tournaments.sql @@ -138,6 +138,14 @@ BEGIN MESSAGE = 'Check-in has already started; the schedule can no longer be changed'; END IF; + -- The bracket is seeded once registration closes, and the roster lock then keeps a team at + -- its minimum from swapping anyone out, so taking substitutes away would strand it. + IF OLD.substitutes_enabled AND NOT NEW.substitutes_enabled + AND OLD.status NOT IN ('Setup', 'RegistrationOpen') THEN + RAISE EXCEPTION USING ERRCODE = '22000', + MESSAGE = 'Substitutes can only be turned off before registration closes'; + END IF; + IF NEW.status IS DISTINCT FROM OLD.status THEN -- CheckInReview is a HOLD, not a lifecycle step: entering it is only -- legal out of RegistrationOpen with check-in actually in force, and diff --git a/hasura/views/v_team_stage_results_compute.sql b/hasura/views/v_team_stage_results_compute.sql index 4fb7234db..ef815e82b 100644 --- a/hasura/views/v_team_stage_results_compute.sql +++ b/hasura/views/v_team_stage_results_compute.sql @@ -1,14 +1,14 @@ -- Tracks: matches played, matches remaining, wins, losses, rounds won, rounds lost -- Also exposes the team's group within the stage (taken from their winner-bracket -- brackets) and their rank within that group, computed with the same tiebreaker --- chain that advance_round_robin_teams / get_team_at_stage_rank / seed_stage use +-- chain that advance_round_robin_teams / get_stage_qualifier_seeds / seed_stage use -- to promote teams to the next stage. The UI reads `rank` directly so it never -- disagrees with bracket progression. -- -- Two ordering columns are exposed and share the SAME ORDER BY so they can never -- disagree: -- - `rank` : ROW_NUMBER, unique & deterministic. Used by the UI display --- and by get_team_at_stage_rank() for OFFSET-based seeding. +-- and by get_stage_qualifier_seeds() for next-stage seeding. -- - `placement` : RANK, ties allowed. Used by calculate_tournament_awards() -- so multiple teams sharing a final-stage placement suppress -- the bronze award when appropriate. @@ -374,7 +374,7 @@ stage_rows AS NOT MATERIALIZED ( AND te.tournament_stage_id = ass.tournament_stage_id ) -- Column order MUST keep the original 16 columns first (tournament_team_id .. --- group_number) and `rank` next so existing consumers (UI, get_team_at_stage_rank) +-- group_number) and `rank` next so existing consumers (UI, get_stage_qualifier_seeds) -- keep working. `placement` is appended at the end for the award calculator. SELECT sr.tournament_team_id, diff --git a/src/awards/awards.controller.ts b/src/awards/awards.controller.ts index 44436f028..6f0e0d5be 100644 --- a/src/awards/awards.controller.ts +++ b/src/awards/awards.controller.ts @@ -87,7 +87,7 @@ export class AwardsController { note?: string | null; user?: User; }) { - const user = await this.assertCanGrant(data.user, data); + const user = await this.assertCanGrant(data.user, data, data); return await this.awards.grantAward({ award_id: data.award_id, player_steam_id: data.player_steam_id, @@ -241,6 +241,8 @@ export class AwardsController { // Tournament organizers can always hand out awards inside their own // tournament, regardless of where the global grant floor sits. No other scope // has a delegated role, so an event/season/league grant needs the floor. + // Medals count on the leaderboard, so that delegation stops short of the + // organizer's own name and team. private async assertCanGrant( user: User | undefined, scope: { @@ -249,6 +251,10 @@ export class AwardsController { season_id?: string | null; league_season_id?: string | null; }, + recipient?: { + player_steam_id?: string | null; + team_id?: string | null; + }, ): Promise { const resolved = this.requireUser(user); @@ -270,6 +276,16 @@ export class AwardsController { } await this.awards.requireOrganizer(scope.tournament_id, resolved); + + if ( + recipient && + (await this.awards.isOwnRecipient(resolved.steam_id, recipient)) + ) { + throw new ForbiddenException( + "Tournament organizers cannot grant awards to themselves or their own team", + ); + } + return resolved; } diff --git a/src/awards/awards.service.ts b/src/awards/awards.service.ts index 430c6eeda..a4527e2ba 100644 --- a/src/awards/awards.service.ts +++ b/src/awards/awards.service.ts @@ -550,6 +550,31 @@ export class AwardsService { throw new ForbiddenException("Not the tournament organizer"); } + public async isOwnRecipient( + steamId: string, + recipient: { player_steam_id?: string | null; team_id?: string | null }, + ): Promise { + if (recipient.player_steam_id) { + return String(recipient.player_steam_id) === String(steamId); + } + + if (!recipient.team_id) { + return false; + } + + const rows = await this.postgres.query>( + `SELECT player_steam_id AS steam_id + FROM public.team_roster + WHERE team_id = $1 + AND player_steam_id = $2 + AND role <> 'Invite' + LIMIT 1`, + [recipient.team_id, steamId], + ); + + return rows.length > 0; + } + private async requireAward(awardId: string): Promise { const [award] = await this.postgres.query( `SELECT * FROM public.awards WHERE id = $1 LIMIT 1`, diff --git a/src/cache/cache.service.ts b/src/cache/cache.service.ts index bf26ba13e..4ab7839c1 100644 --- a/src/cache/cache.service.ts +++ b/src/cache/cache.service.ts @@ -67,9 +67,13 @@ export class CacheService { return await this.connection.get(key); } - public async forget(key: string) { + public async forget(...keys: Array) { + if (keys.length === 0) { + return true; + } + try { - await this.connection.del(key); + await this.connection.del(...keys); return true; } catch (error) { this.logger.error("unable to remove value from redis", error); diff --git a/src/matches/enums/MatchJobs.ts b/src/matches/enums/MatchJobs.ts index e412b0da4..bebd80648 100644 --- a/src/matches/enums/MatchJobs.ts +++ b/src/matches/enums/MatchJobs.ts @@ -1,4 +1,5 @@ export const MatchJobs = { CheckOnDemandServerJob: `CheckOnDemandServerJob`, AutoPickExpiredVeto: `AutoPickExpiredVeto`, + StopOnDemandServer: `StopOnDemandServer`, } as const; diff --git a/src/matches/events/MatchMapStatusEvent.spec.ts b/src/matches/events/MatchMapStatusEvent.spec.ts new file mode 100644 index 000000000..9f719c1fb --- /dev/null +++ b/src/matches/events/MatchMapStatusEvent.spec.ts @@ -0,0 +1,129 @@ +import MatchMapStatusEvent from "./MatchMapStatusEvent"; + +describe("MatchMapStatusEvent", () => { + const logger = { log: jest.fn(), warn: jest.fn(), error: jest.fn() }; + const hasura = { query: jest.fn(), mutation: jest.fn() }; + const matchAssistant = { sendServerMatchId: jest.fn() }; + const notifications = { + sendMatchMapPauseNotification: jest.fn(), + resolveMatchAlerts: jest.fn(), + }; + + let matchMaps: Array<{ id: string; status: string }>; + let currentMatchMapId: string | null; + + const process = async (status: string, winningLineupId?: string) => { + const event = new MatchMapStatusEvent( + logger as any, + hasura as any, + matchAssistant as any, + {} as any, + notifications as any, + ); + event.setData("match-1", { + status: status as any, + winning_lineup_id: winningLineupId, + }); + await event.process(); + }; + + const mapUpdates = () => + hasura.mutation.mock.calls + .map(([mutation]: [any]) => mutation.update_match_maps_by_pk) + .filter(Boolean); + + beforeEach(() => { + jest.clearAllMocks(); + hasura.query.mockImplementation(async (query: any) => { + if (query.match_map_rounds) { + return { + match_map_rounds: [{ lineup_1_score: 13, lineup_2_score: 7 }], + }; + } + return { + matches_by_pk: { + current_match_map_id: currentMatchMapId, + lineup_1_id: "lineup-1", + lineup_2_id: "lineup-2", + status: "Live", + match_maps: matchMaps, + }, + }; + }); + hasura.mutation.mockResolvedValue({ + update_match_maps_by_pk: { + id: "map", + match: { current_match_map_id: "map-2" }, + }, + }); + }); + + describe("a late end-of-map status after the map was already finished", () => { + beforeEach(() => { + matchMaps = [ + { id: "map-1", status: "Finished" }, + { id: "map-2", status: "Scheduled" }, + ]; + currentMatchMapId = "map-2"; + }); + + it.each(["WaitingForTV", "UploadingDemo", "Finished"])( + "does not write %s onto the next, unplayed map", + async (status) => { + await process(status, "lineup-1"); + + expect(mapUpdates()).toEqual([]); + expect(matchAssistant.sendServerMatchId).not.toHaveBeenCalled(); + }, + ); + }); + + it("moves a live map into WaitingForTV", async () => { + matchMaps = [{ id: "map-1", status: "Live" }]; + currentMatchMapId = "map-1"; + + await process("WaitingForTV", "lineup-1"); + + expect(mapUpdates()).toEqual([ + expect.objectContaining({ + __args: { + pk_columns: { id: "map-1" }, + _set: { status: "WaitingForTV", winning_lineup_id: "lineup-1" }, + }, + }), + ]); + }); + + it("finishes a map that is uploading its demo", async () => { + matchMaps = [ + { id: "map-1", status: "UploadingDemo" }, + { id: "map-2", status: "Scheduled" }, + ]; + currentMatchMapId = "map-1"; + + await process("Finished", "lineup-1"); + + expect(mapUpdates()).toEqual([ + expect.objectContaining({ + __args: { + pk_columns: { id: "map-1" }, + _set: { status: "Finished", winning_lineup_id: "lineup-1" }, + }, + }), + ]); + expect(matchAssistant.sendServerMatchId).toHaveBeenCalledWith("match-1"); + }); + + it("still applies statuses that start a map", async () => { + matchMaps = [{ id: "map-1", status: "Warmup" }]; + currentMatchMapId = "map-1"; + + await process("Live"); + + expect(mapUpdates()).toEqual([ + expect.objectContaining({ + __args: { pk_columns: { id: "map-1" }, _set: { status: "Live" } }, + }), + ]); + }); +}); diff --git a/src/matches/events/MatchMapStatusEvent.ts b/src/matches/events/MatchMapStatusEvent.ts index e8b37e68e..076d92b98 100644 --- a/src/matches/events/MatchMapStatusEvent.ts +++ b/src/matches/events/MatchMapStatusEvent.ts @@ -5,6 +5,21 @@ export default class MatchMapStatusEvent extends MatchEventProcessor<{ status: e_match_map_status_enum; winning_lineup_id?: string; }> { + private static readonly END_OF_MAP_STATUSES = [ + "WaitingForTV", + "UploadingDemo", + "Finished", + ]; + + private static readonly IN_PLAY_MAP_STATUSES = [ + "Knife", + "Live", + "Overtime", + "Paused", + "WaitingForTV", + "UploadingDemo", + ]; + public async process() { const { matches_by_pk: match } = await this.hasura.query({ matches_by_pk: { @@ -14,6 +29,10 @@ export default class MatchMapStatusEvent extends MatchEventProcessor<{ current_match_map_id: true, lineup_1_id: true, lineup_2_id: true, + match_maps: { + id: true, + status: true, + }, }, }); @@ -21,6 +40,22 @@ export default class MatchMapStatusEvent extends MatchEventProcessor<{ return; } + // Statuses hit whichever map is current, so a late one would land on a map that never started. + const currentMapStatus = match.match_maps?.find( + (matchMap) => matchMap.id === match.current_match_map_id, + )?.status as string | undefined; + + if ( + MatchMapStatusEvent.END_OF_MAP_STATUSES.includes(this.data.status) && + !MatchMapStatusEvent.IN_PLAY_MAP_STATUSES.includes(currentMapStatus) + ) { + this.logger.warn( + `MatchMapStatusEvent ignoring ${this.data.status} match=${this.matchId} ` + + `match_map=${match.current_match_map_id} is ${currentMapStatus}, not in play`, + ); + return; + } + const isFinished = this.data.status === "Finished"; let resolvedWinningLineupId: string | undefined = diff --git a/src/matches/game-streamer/game-streamer.service.spec.ts b/src/matches/game-streamer/game-streamer.service.spec.ts index d1361eccd..2855b0575 100644 --- a/src/matches/game-streamer/game-streamer.service.spec.ts +++ b/src/matches/game-streamer/game-streamer.service.spec.ts @@ -13,8 +13,15 @@ describe("GameStreamerService", () => { let logger: { log: jest.Mock; warn: jest.Mock; error: jest.Mock }; const config = { - get: (key: string) => - key === "gameServers" ? { namespace: "test" } : ({} as any), + get: (key: string) => { + if (key === "gameServers") { + return { namespace: "test" }; + } + if (key === "app") { + return { relayDomain: "https://tv.example.test" }; + } + return {} as any; + }, }; beforeEach(() => { @@ -171,6 +178,24 @@ describe("GameStreamerService", () => { }); }); + describe("buildConnectEnv", () => { + const server = { host: "game.example.test", port: 27015, tv_port: 27020 }; + + it("streams a playcast match from this install's relay", async () => { + const env = await (service as any).buildConnectEnv( + "m-1", + server, + "secret", + true, + "tv", + ); + + expect(env.find((e: any) => e.name === "PLAYCAST_URL")?.value).toBe( + "https://tv.example.test/m-1", + ); + }); + }); + describe("reportStatus", () => { const streamSetOf = () => hasura.mutation.mock.calls[0][0].update_match_streams.__args._set; diff --git a/src/matches/game-streamer/game-streamer.service.ts b/src/matches/game-streamer/game-streamer.service.ts index 19e28f33c..46d057338 100644 --- a/src/matches/game-streamer/game-streamer.service.ts +++ b/src/matches/game-streamer/game-streamer.service.ts @@ -2830,7 +2830,10 @@ export class GameStreamerService { if (mode === "tv") { if (usePlaycast) { return [ - { name: "PLAYCAST_URL", value: `https://tv.5stack.gg/${matchId}` }, + { + name: "PLAYCAST_URL", + value: `${this.appConfig.relayDomain}/${matchId}`, + }, { name: "PLAYCAST_PASSWORD", value: "" }, ]; } diff --git a/src/matches/jobs/CancelExpiredMatches.spec.ts b/src/matches/jobs/CancelExpiredMatches.spec.ts index c2b4db093..56064d735 100644 --- a/src/matches/jobs/CancelExpiredMatches.spec.ts +++ b/src/matches/jobs/CancelExpiredMatches.spec.ts @@ -52,6 +52,9 @@ describe("CancelExpiredMatches", () => { const rcon = { connect: jest.fn(), }; + const matchAssistant = { + sendServerMatchId: jest.fn(), + }; let job: CancelExpiredMatches; let tournamentMatches: any[]; @@ -85,6 +88,7 @@ describe("CancelExpiredMatches", () => { notifications as any, configService as any, rcon as any, + matchAssistant as any, ); }); @@ -325,9 +329,7 @@ describe("CancelExpiredMatches", () => { lineup_2: { id: "lineup-2", is_ready: false, - lineup_players: [ - { steam_id: "no-show-b", is_connected: false }, - ], + lineup_players: [{ steam_id: "no-show-b", is_connected: false }], }, }), ]; @@ -491,7 +493,9 @@ describe("CancelExpiredMatches", () => { ); expect(order.indexOf("forfeit")).toBeGreaterThanOrEqual(0); - expect(order.indexOf("abandon")).toBeGreaterThan(order.indexOf("forfeit")); + expect(order.indexOf("abandon")).toBeGreaterThan( + order.indexOf("forfeit"), + ); }); }); @@ -583,6 +587,293 @@ describe("CancelExpiredMatches", () => { ); }); + describe("a map the server decided but never finished", () => { + const playedLineup = ( + id: string, + steamId: string, + isConnected = false, + ) => ({ + id, + is_ready: true, + lineup_players: [{ steam_id: steamId, is_connected: isConnected }], + }); + + const mutationsOf = (key: string) => + hasura.mutation.mock.calls + .map(([arg]: [any]) => arg?.[key]) + .filter(Boolean); + + const matchStatusSets = () => + mutationsOf("update_matches_by_pk") + .map((mutation: any) => mutation.__args._set.status) + .filter(Boolean); + + it("finishes a map left in WaitingForTV instead of cancelling the match", async () => { + tournamentMatches = [ + expiredTournamentMatch({ + is_tournament_match: false, + match_maps: [ + { + id: "map-1", + status: "WaitingForTV", + winning_lineup_id: "lineup-2", + }, + ], + lineup_1: playedLineup("lineup-1", "1"), + lineup_2: playedLineup("lineup-2", "2"), + }), + ]; + + await expect(job.process()).resolves.toBe(1); + + expect(mutationsOf("update_match_maps_by_pk")).toEqual([ + expect.objectContaining({ + __args: { + pk_columns: { id: "map-1" }, + _set: { status: "Finished" }, + }, + }), + ]); + expect(matchStatusSets()).toEqual([]); + expect(mutationsOf("insert_abandoned_matches")).toEqual([]); + expect(rconClient.send).not.toHaveBeenCalled(); + }); + + it("finishes a tournament map still uploading its demo instead of forfeiting", async () => { + tournamentMatches = [ + expiredTournamentMatch({ + match_maps: [ + { + id: "map-1", + status: "UploadingDemo", + winning_lineup_id: "lineup-1", + }, + ], + lineup_1: playedLineup("lineup-1", "1"), + lineup_2: playedLineup("lineup-2", "2"), + }), + ]; + + await job.process(); + + expect(mutationsOf("update_match_maps_by_pk")).toHaveLength(1); + expect(matchStatusSets()).toEqual([]); + expect(mutationsOf("insert_abandoned_matches")).toEqual([]); + }); + + const finishedMapLeaves = (match: Record) => + hasura.mutation.mockImplementation(async (mutation: any) => + mutation.update_match_maps_by_pk + ? { update_match_maps_by_pk: { match } } + : {}, + ); + + const stalledSeries = (overrides: Record = {}) => + expiredTournamentMatch({ + match_maps: [ + { id: "map-1", status: "Finished", winning_lineup_id: "lineup-2" }, + { id: "map-2", status: "Scheduled", winning_lineup_id: null }, + ], + lineup_1: playedLineup("lineup-1", "1"), + lineup_2: playedLineup("lineup-2", "2"), + ...overrides, + }); + + it("asks the server for the next map when the series is not over", async () => { + finishedMapLeaves({ status: "Live", current_match_map_id: "map-2" }); + tournamentMatches = [ + expiredTournamentMatch({ + is_tournament_match: false, + match_maps: [ + { + id: "map-1", + status: "WaitingForTV", + winning_lineup_id: "lineup-1", + }, + { id: "map-2", status: "Scheduled", winning_lineup_id: null }, + ], + lineup_1: playedLineup("lineup-1", "1"), + lineup_2: playedLineup("lineup-2", "2"), + }), + ]; + + await job.process(); + + expect(matchAssistant.sendServerMatchId).toHaveBeenCalledWith("match-1"); + expect(matchStatusSets()).toEqual([]); + }); + + it("leaves the server alone when finishing the map ended the series", async () => { + // A Bo3 won 2-0 still has map 3 as its current map. + finishedMapLeaves({ status: "Finished", current_match_map_id: "map-3" }); + tournamentMatches = [ + expiredTournamentMatch({ + is_tournament_match: false, + match_maps: [ + { id: "map-1", status: "Finished", winning_lineup_id: "lineup-1" }, + { + id: "map-2", + status: "UploadingDemo", + winning_lineup_id: "lineup-1", + }, + { id: "map-3", status: "Scheduled", winning_lineup_id: null }, + ], + lineup_1: playedLineup("lineup-1", "1"), + lineup_2: playedLineup("lineup-2", "2"), + }), + ]; + + await job.process(); + + expect(matchAssistant.sendServerMatchId).not.toHaveBeenCalled(); + }); + + it("cancels a series whose server never came back without penalising anyone", async () => { + tournamentMatches = [stalledSeries({ is_tournament_match: false })]; + + await job.process(); + + expect(matchStatusSets()).toEqual(["Canceled"]); + expect(mutationsOf("insert_abandoned_matches")).toEqual([]); + }); + + it.each([true, false])( + "asks an organizer about a stalled tournament series rather than forfeiting it (is_ready=%s)", + async (isReady) => { + // A forfeit would go to lineup 1 despite lineup 2 winning (is_ready) or coin-toss (not). + jest.spyOn(Math, "random").mockReturnValue(0.25); + tournamentMatches = [ + stalledSeries({ + lineup_1: { ...playedLineup("lineup-1", "1"), is_ready: isReady }, + lineup_2: { ...playedLineup("lineup-2", "2"), is_ready: isReady }, + }), + ]; + + await job.process(); + + expect(matchStatusSets()).toEqual([]); + expect(mutationsOf("update_matches_by_pk")).toEqual([ + expect.objectContaining({ + __args: expect.objectContaining({ _set: { cancels_at: null } }), + }), + ]); + expect(notifications.send).toHaveBeenCalledWith( + "MatchSupport", + expect.objectContaining({ entity_id: "match-1" }), + undefined, + DISCORD_COLORS.RED, + ); + expect(mutationsOf("insert_abandoned_matches")).toEqual([]); + }, + ); + + it.each(["Live", "Paused", "WaitingForTV"])( + "asks an organizer about a tournament match whose server died with map 1 %s and no winner", + async (mapStatus) => { + tournamentMatches = [ + expiredTournamentMatch({ + match_maps: [ + { id: "map-1", status: mapStatus, winning_lineup_id: null }, + { id: "map-2", status: "Scheduled", winning_lineup_id: null }, + ], + lineup_1: { ...playedLineup("lineup-1", "1"), is_ready: true }, + lineup_2: { ...playedLineup("lineup-2", "2"), is_ready: true }, + }), + ]; + + await job.process(); + + expect(matchStatusSets()).toEqual([]); + expect(notifications.send).toHaveBeenCalledWith( + "MatchSupport", + expect.objectContaining({ entity_id: "match-1" }), + undefined, + DISCORD_COLORS.RED, + ); + expect(mutationsOf("insert_abandoned_matches")).toEqual([]); + }, + ); + + it("does not force start a match waiting on the TV broadcast", async () => { + tournamentMatches = [ + expiredTournamentMatch({ + is_tournament_match: false, + cancels_at: new Date(Date.now() + 30 * 1000).toISOString(), + match_maps: [ + { + id: "map-1", + status: "WaitingForTV", + winning_lineup_id: "lineup-1", + }, + ], + lineup_1: playedLineup("lineup-1", "1", true), + lineup_2: playedLineup("lineup-2", "2", true), + }), + ]; + + await expect(job.process()).resolves.toBe(0); + + expect(rconClient.send).not.toHaveBeenCalled(); + expect(hasura.mutation).not.toHaveBeenCalled(); + }); + + it("cancels a WaitingForTV map with no winner, but penalises nobody", async () => { + tournamentMatches = [ + expiredTournamentMatch({ + is_tournament_match: false, + match_maps: [ + { id: "map-1", status: "WaitingForTV", winning_lineup_id: null }, + ], + lineup_1: playedLineup("lineup-1", "1"), + lineup_2: playedLineup("lineup-2", "2"), + }), + ]; + + await job.process(); + + expect(mutationsOf("update_match_maps_by_pk")).toEqual([]); + expect(matchStatusSets()).toEqual(["Canceled"]); + expect(mutationsOf("insert_abandoned_matches")).toEqual([]); + }); + + it("penalises nobody when every map was already played", async () => { + tournamentMatches = [ + expiredTournamentMatch({ + is_tournament_match: false, + match_maps: [ + { id: "map-1", status: "Finished", winning_lineup_id: null }, + ], + lineup_1: playedLineup("lineup-1", "1"), + lineup_2: playedLineup("lineup-2", "2"), + }), + ]; + + await job.process(); + + expect(rconClient.send).not.toHaveBeenCalledWith("force_ready"); + expect(mutationsOf("insert_abandoned_matches")).toEqual([]); + }); + + it("still force starts the next map of a series once everyone is back", async () => { + tournamentMatches = [ + expiredTournamentMatch({ + is_tournament_match: false, + match_maps: [ + { id: "map-1", status: "Finished", winning_lineup_id: "lineup-1" }, + { id: "map-2", status: "Warmup", winning_lineup_id: null }, + ], + lineup_1: playedLineup("lineup-1", "1", true), + lineup_2: playedLineup("lineup-2", "2", true), + }), + ]; + + await job.process(); + + expect(rconClient.send).toHaveBeenCalledWith("force_ready"); + expect(mutationsOf("update_match_maps_by_pk")).toEqual([]); + }); + }); + it("falls back to the normal expiry path when force start fails", async () => { rconClient.send.mockRejectedValue(new Error("rcon down")); tournamentMatches = [ diff --git a/src/matches/jobs/CancelExpiredMatches.ts b/src/matches/jobs/CancelExpiredMatches.ts index 206f2f8d5..7a955d980 100644 --- a/src/matches/jobs/CancelExpiredMatches.ts +++ b/src/matches/jobs/CancelExpiredMatches.ts @@ -8,12 +8,29 @@ import { NotificationsService } from "../../notifications/notifications.service" import { AppConfig } from "../../configs/types/AppConfig"; import { RconService } from "../../rcon/rcon.service"; import { DISCORD_COLORS } from "../../notifications/utilities/constants"; +import { MatchAssistantService } from "../match-assistant/match-assistant.service"; @UseQueue("Matches", MatchQueues.ScheduledMatches) export class CancelExpiredMatches extends WorkerHost { // How far ahead of cancels_at a full lobby is allowed to be force started. private static readonly FORCE_START_LEAD_MS = 60 * 1000; + private static readonly IN_PLAY_MAP_STATUSES = [ + "Knife", + "Live", + "Overtime", + "Paused", + "WaitingForTV", + "UploadingDemo", + ]; + + private static readonly END_OF_MAP_STATUSES = [ + "WaitingForTV", + "UploadingDemo", + ]; + + private static readonly PLAYED_MAP_STATUSES = ["Finished", "Surrendered"]; + private readonly appConfig: AppConfig; constructor( @@ -22,6 +39,7 @@ export class CancelExpiredMatches extends WorkerHost { private readonly notifications: NotificationsService, private readonly configService: ConfigService, private readonly rcon: RconService, + private readonly matchAssistant: MatchAssistantService, ) { super(); this.appConfig = this.configService.get("app"); @@ -54,6 +72,13 @@ export class CancelExpiredMatches extends WorkerHost { continue; } + const decidedMap = this.getDecidedUnfinishedMap(match); + if (decidedMap) { + await this.finishDecidedMap(match, decidedMap); + handled++; + continue; + } + if (match.is_tournament_match) { await this.handleExpiredTournamentMatch(match); } else { @@ -79,13 +104,99 @@ export class CancelExpiredMatches extends WorkerHost { private isAwaitingWarmup( match: Awaited>[number], ) { - const started = ["Knife", "Live", "Overtime", "Paused"]; + const matchMaps = match.match_maps ?? []; + + if ( + matchMaps.some((matchMap) => + CancelExpiredMatches.IN_PLAY_MAP_STATUSES.includes( + matchMap.status as string, + ), + ) + ) { + return false; + } + + return ( + matchMaps.length === 0 || + matchMaps.some( + (matchMap) => + !CancelExpiredMatches.PLAYED_MAP_STATUSES.includes( + matchMap.status as string, + ), + ) + ); + } + + private hasPlayedMap( + match: Awaited>[number], + ) { + return (match.match_maps ?? []).some((matchMap) => + CancelExpiredMatches.PLAYED_MAP_STATUSES.includes( + matchMap.status as string, + ), + ); + } - return !(match.match_maps ?? []).some((matchMap) => - started.includes(matchMap.status as string), + private hasStartedMap( + match: Awaited>[number], + ) { + return (match.match_maps ?? []).some((matchMap) => + [ + ...CancelExpiredMatches.IN_PLAY_MAP_STATUSES, + ...CancelExpiredMatches.PLAYED_MAP_STATUSES, + ].includes(matchMap.status as string), ); } + // The winner was published but the server died before Finished; nothing else will finish it. + private getDecidedUnfinishedMap( + match: Awaited>[number], + ) { + return (match.match_maps ?? []).find( + (matchMap) => + CancelExpiredMatches.END_OF_MAP_STATUSES.includes( + matchMap.status as string, + ) && matchMap.winning_lineup_id, + ); + } + + // The map, not the match: update_match_state then settles the series from the rounds. + private async finishDecidedMap( + match: Awaited>[number], + matchMap: Awaited< + ReturnType + >[number]["match_maps"][number], + ) { + const { update_match_maps_by_pk } = await this.hasura.mutation({ + update_match_maps_by_pk: { + __args: { + pk_columns: { + id: matchMap.id, + }, + _set: { + status: "Finished", + }, + }, + match: { + status: true, + current_match_map_id: true, + }, + }, + }); + + this.logger.log( + `finished map ${matchMap.id} stuck in ${matchMap.status} on expired match ${match.id}`, + ); + + // A series that just ended still has its unplayed maps as the current one. + if ( + update_match_maps_by_pk?.match?.status === "Live" && + update_match_maps_by_pk.match.current_match_map_id + ) { + await this.matchAssistant.sendServerMatchId(match.id); + } + } + // A no-show is a rostered player who never connected to the server at all. // Placeholder slots have no steam_id and cannot connect, so they never count. private hasNoShow( @@ -189,7 +300,7 @@ export class CancelExpiredMatches extends WorkerHost { // out and then stalled lands here too. Nobody no-showed that one -- they // all turned up -- and by the time it expires they have long since // disconnected, so is_connected would read every one of them as absent. - if (!this.isAwaitingWarmup(match)) { + if (!this.isAwaitingWarmup(match) || this.hasPlayedMap(match)) { return; } @@ -250,7 +361,7 @@ export class CancelExpiredMatches extends WorkerHost { } await rcon.send( - 'say Match canceled - not everyone showed up before the deadline.', + "say Match canceled - not everyone showed up before the deadline.", ); } catch (error) { // Never let this stop the cancellation itself. @@ -267,7 +378,8 @@ export class CancelExpiredMatches extends WorkerHost { const hasReadyLineup = match.lineup_1.is_ready || match.lineup_2.is_ready; const isAdminMode = match.options?.match_mode === "admin"; - if (!hasReadyLineup && isAdminMode) { + // Once a map has started, a forfeit would pick the series winner by readiness. + if (this.hasStartedMap(match) || (!hasReadyLineup && isAdminMode)) { await this.requestOrganizerAttention(match.id); return; } @@ -405,7 +517,9 @@ export class CancelExpiredMatches extends WorkerHost { server_id: true, is_tournament_match: true, match_maps: { + id: true, status: true, + winning_lineup_id: true, }, options: { match_mode: true, diff --git a/src/matches/jobs/ReconcileOnDemandServerJobs.spec.ts b/src/matches/jobs/ReconcileOnDemandServerJobs.spec.ts new file mode 100644 index 000000000..8093a9309 --- /dev/null +++ b/src/matches/jobs/ReconcileOnDemandServerJobs.spec.ts @@ -0,0 +1,40 @@ +jest.mock("@kubernetes/client-node", () => ({ + BatchV1Api: class BatchV1Api {}, + CoreV1Api: class CoreV1Api {}, + KubeConfig: class KubeConfig {}, + Exec: class Exec {}, +})); + +import { ReconcileOnDemandServerJobs } from "./ReconcileOnDemandServerJobs"; + +describe("ReconcileOnDemandServerJobs", () => { + let matchAssistant: { reconcileOnDemandServerJobs: jest.Mock }; + let logger: { error: jest.Mock }; + let processor: ReconcileOnDemandServerJobs; + + beforeEach(() => { + matchAssistant = { reconcileOnDemandServerJobs: jest.fn() }; + logger = { error: jest.fn() }; + processor = new ReconcileOnDemandServerJobs( + logger as any, + matchAssistant as any, + ); + }); + + it("runs the sweep", async () => { + await processor.process(); + + expect(matchAssistant.reconcileOnDemandServerJobs).toHaveBeenCalledTimes(1); + }); + + // A repeatable job: the next tick is the retry. + it("logs a failed sweep instead of failing the repeatable job", async () => { + matchAssistant.reconcileOnDemandServerJobs.mockRejectedValue( + new Error("hasura unavailable"), + ); + + await expect(processor.process()).resolves.toBeUndefined(); + + expect(logger.error).toHaveBeenCalled(); + }); +}); diff --git a/src/matches/jobs/ReconcileOnDemandServerJobs.ts b/src/matches/jobs/ReconcileOnDemandServerJobs.ts new file mode 100644 index 000000000..b12ac5b1a --- /dev/null +++ b/src/matches/jobs/ReconcileOnDemandServerJobs.ts @@ -0,0 +1,26 @@ +import { Logger } from "@nestjs/common"; +import { WorkerHost } from "@nestjs/bullmq"; +import { MatchQueues } from "../enums/MatchQueues"; +import { UseQueue } from "../../utilities/QueueProcessors"; +import { MatchAssistantService } from "../match-assistant/match-assistant.service"; + +@UseQueue("Matches", MatchQueues.ScheduledMatches) +export class ReconcileOnDemandServerJobs extends WorkerHost { + constructor( + private readonly logger: Logger, + private readonly matchAssistant: MatchAssistantService, + ) { + super(); + } + + async process(): Promise { + try { + await this.matchAssistant.reconcileOnDemandServerJobs(); + } catch (error) { + this.logger.error( + `ReconcileOnDemandServerJobs failed: ${(error as Error)?.message}`, + (error as Error)?.stack, + ); + } + } +} diff --git a/src/matches/jobs/StopOnDemandServer.spec.ts b/src/matches/jobs/StopOnDemandServer.spec.ts new file mode 100644 index 000000000..1e7483ada --- /dev/null +++ b/src/matches/jobs/StopOnDemandServer.spec.ts @@ -0,0 +1,53 @@ +jest.mock("@kubernetes/client-node", () => ({ + BatchV1Api: class BatchV1Api {}, + CoreV1Api: class CoreV1Api {}, + KubeConfig: class KubeConfig {}, + Exec: class Exec {}, +})); + +import { StopOnDemandServer } from "./StopOnDemandServer"; + +describe("StopOnDemandServer", () => { + let matchAssistant: { + stopEndedMatchServer: jest.Mock; + removeUnstoppedOnDemandServer: jest.Mock; + }; + let processor: StopOnDemandServer; + + beforeEach(() => { + matchAssistant = { + stopEndedMatchServer: jest.fn(), + removeUnstoppedOnDemandServer: jest.fn(), + }; + processor = new StopOnDemandServer(matchAssistant as any); + }); + + it("stops the match's server", async () => { + await processor.process({ data: { matchId: "match-1" } } as any); + + expect(matchAssistant.stopEndedMatchServer).toHaveBeenCalledWith("match-1"); + expect(matchAssistant.removeUnstoppedOnDemandServer).not.toHaveBeenCalled(); + }); + + it("checks back on a server it already signalled", async () => { + await processor.process({ + data: { matchId: "match-1", jobUid: "uid-1" }, + } as any); + + expect(matchAssistant.removeUnstoppedOnDemandServer).toHaveBeenCalledWith( + "match-1", + "uid-1", + ); + expect(matchAssistant.stopEndedMatchServer).not.toHaveBeenCalled(); + }); + + it("fails the job when the teardown fails, so BullMQ retries it", async () => { + matchAssistant.stopEndedMatchServer.mockRejectedValue( + new Error("k8s down"), + ); + + await expect( + processor.process({ data: { matchId: "match-1" } } as any), + ).rejects.toThrow("k8s down"); + }); +}); diff --git a/src/matches/jobs/StopOnDemandServer.ts b/src/matches/jobs/StopOnDemandServer.ts index 770c32c1c..50626c4b9 100644 --- a/src/matches/jobs/StopOnDemandServer.ts +++ b/src/matches/jobs/StopOnDemandServer.ts @@ -13,9 +13,16 @@ export class StopOnDemandServer extends WorkerHost { async process( job: Job<{ matchId: string; + jobUid?: string; }>, ): Promise { - const { matchId } = job.data; - await this.matchAssistant.stopOnDemandServer(matchId); + const { matchId, jobUid } = job.data; + + if (jobUid) { + await this.matchAssistant.removeUnstoppedOnDemandServer(matchId, jobUid); + return; + } + + await this.matchAssistant.stopEndedMatchServer(matchId); } } diff --git a/src/matches/match-assistant/match-assistant.on-demand-teardown.spec.ts b/src/matches/match-assistant/match-assistant.on-demand-teardown.spec.ts new file mode 100644 index 000000000..b8ace931a --- /dev/null +++ b/src/matches/match-assistant/match-assistant.on-demand-teardown.spec.ts @@ -0,0 +1,694 @@ +const listNamespacedPod = jest.fn(); +const deleteNamespacedPod = jest.fn(); +const createNamespacedJob = jest.fn(); +const readNamespacedJob = jest.fn(); +const deleteNamespacedJob = jest.fn(); +const exec = jest.fn(); + +jest.mock("@kubernetes/client-node", () => ({ + BatchV1Api: class BatchV1Api { + createNamespacedJob = createNamespacedJob; + readNamespacedJob = readNamespacedJob; + deleteNamespacedJob = deleteNamespacedJob; + }, + CoreV1Api: class CoreV1Api { + listNamespacedPod = listNamespacedPod; + deleteNamespacedPod = deleteNamespacedPod; + }, + KubeConfig: class KubeConfig { + loadFromDefault() {} + makeApiClient(ctor: new () => unknown) { + return new ctor(); + } + }, + Exec: class Exec { + exec = exec; + }, +})); + +import { MatchAssistantService } from "./match-assistant.service"; +import { StopOnDemandServer } from "../jobs/StopOnDemandServer"; +import { FailedToCreateOnDemandServer } from "../errors/FailedToCreateOnDemandServer"; + +describe("MatchAssistantService — on-demand server teardown", () => { + let service: MatchAssistantService; + let hasura: { query: jest.Mock; mutation: jest.Mock }; + let cache: { lock: jest.Mock }; + let queue: { add: jest.Mock }; + let scheduledMatchesQueue: { add: jest.Mock; getDelayed: jest.Mock }; + let logger: { + warn: jest.Mock; + log: jest.Mock; + error: jest.Mock; + verbose: jest.Mock; + }; + + const notFound = { code: 404 }; + + const activeJob = (uid = "uid-1") => ({ + metadata: { name: "m-match-1", uid }, + status: { active: 1 }, + }); + + const pod = ( + phase: string, + state: Record, + jobUid = "uid-1", + name = "m-match-1-abcde", + ) => ({ + metadata: { name, ownerReferences: [{ kind: "Job", uid: jobUid }] }, + spec: { containers: [{ name: "game-server" }] }, + status: { + phase, + containerStatuses: [{ name: "game-server", state }], + }, + }); + + const runningPod = (jobUid = "uid-1", name?: string) => + pod("Running", { running: {} }, jobUid, name); + const creatingPod = (jobUid = "uid-1", name?: string) => + pod("Pending", { waiting: { reason: "ContainerCreating" } }, jobUid, name); + + const serverReleases = () => + hasura.mutation.mock.calls + .map(([mutation]) => mutation?.update_servers) + .filter(Boolean); + + beforeEach(() => { + for (const fn of [ + listNamespacedPod, + deleteNamespacedPod, + createNamespacedJob, + readNamespacedJob, + deleteNamespacedJob, + exec, + ]) { + fn.mockReset(); + } + + listNamespacedPod.mockResolvedValue({ items: [] }); + deleteNamespacedPod.mockResolvedValue({}); + createNamespacedJob.mockResolvedValue({}); + readNamespacedJob.mockRejectedValue(notFound); + deleteNamespacedJob.mockResolvedValue({}); + exec.mockResolvedValue({}); + + hasura = { + query: jest.fn(async (request: any) => { + if (request.matches_by_pk) { + return { matches_by_pk: { server_error: null } }; + } + return {}; + }), + mutation: jest.fn(async () => ({})), + }; + cache = { + lock: jest.fn(async (_key: string, fn: () => Promise) => fn()), + }; + queue = { add: jest.fn() }; + scheduledMatchesQueue = { + add: jest.fn(), + getDelayed: jest.fn(async (): Promise => []), + }; + logger = { + warn: jest.fn(), + log: jest.fn(), + error: jest.fn(), + verbose: jest.fn(), + }; + + service = new MatchAssistantService( + logger as any, + {} as any, + cache as any, + { + get: jest.fn((key: string) => + key === "gameServers" ? { namespace: "test" } : {}, + ), + } as any, + hasura as any, + { decrypt: jest.fn(async () => "rcon") } as any, + { getJobBootDiagnostics: jest.fn() } as any, + { + resolveGameServerPluginImage: jest.fn( + async () => "ghcr.io/5stackgg/game-server-sw:latest", + ), + } as any, + { + resolveForServer: jest.fn(async (): Promise => null), + environmentFor: jest.fn((): unknown[] => []), + } as any, + queue as any, + scheduledMatchesQueue as any, + ); + }); + + describe("the graceful stop", () => { + it("removes the Job when its pod never got to Running", async () => { + readNamespacedJob + .mockResolvedValueOnce(activeJob()) + .mockRejectedValue(notFound); + listNamespacedPod + .mockResolvedValueOnce({ items: [creatingPod()] }) + .mockResolvedValueOnce({ items: [creatingPod()] }) + .mockResolvedValue({ items: [] }); + + await service.stopOnDemandServer("match-1"); + + expect(exec).not.toHaveBeenCalled(); + expect(deleteNamespacedJob).toHaveBeenCalledWith( + expect.objectContaining({ name: "m-match-1", namespace: "test" }), + ); + }); + + // The scheduled-matches queue has one worker, and nothing reuses the name afterwards. + it("does not hold the queue waiting for the Job to disappear", async () => { + readNamespacedJob + .mockResolvedValueOnce(activeJob()) + .mockRejectedValue(notFound); + listNamespacedPod + .mockResolvedValueOnce({ items: [creatingPod()] }) + .mockResolvedValueOnce({ items: [creatingPod()] }) + .mockResolvedValue({ items: [] }); + + await service.stopOnDemandServer("match-1"); + + expect(readNamespacedJob).toHaveBeenCalledTimes(1); + }); + + it("signals a running game server and checks back that it stopped", async () => { + readNamespacedJob.mockResolvedValue(activeJob("uid-1")); + listNamespacedPod.mockResolvedValue({ items: [runningPod()] }); + + await service.stopOnDemandServer("match-1"); + + expect(exec).toHaveBeenCalledWith( + "test", + "m-match-1-abcde", + "game-server", + ["kill", "-SIGUSR1", "1"], + expect.anything(), + expect.anything(), + expect.anything(), + false, + ); + expect(deleteNamespacedJob).not.toHaveBeenCalled(); + expect(scheduledMatchesQueue.add).toHaveBeenCalledWith( + "StopOnDemandServer", + { matchId: "match-1", jobUid: "uid-1" }, + expect.objectContaining({ + delay: MatchAssistantService.ON_DEMAND_SERVER_STOP_CHECK_DELAY_MS, + }), + ); + }); + + it("leaves a Job that already finished alone, logs and all", async () => { + readNamespacedJob.mockResolvedValue({ + metadata: { name: "m-match-1", uid: "uid-1" }, + status: { + succeeded: 1, + conditions: [{ type: "Complete", status: "True" }], + }, + }); + listNamespacedPod.mockResolvedValue({ + items: [pod("Succeeded", { terminated: { exitCode: 0 } })], + }); + + await service.stopOnDemandServer("match-1"); + + expect(exec).not.toHaveBeenCalled(); + expect(deleteNamespacedJob).not.toHaveBeenCalled(); + expect(scheduledMatchesQueue.add).not.toHaveBeenCalled(); + }); + + // BullMQ only retries a job that throws. + it("rethrows when Kubernetes cannot be reached, and keeps the reservation", async () => { + const unavailable = Object.assign(new Error("connect ECONNREFUSED"), { + code: 500, + }); + readNamespacedJob.mockRejectedValue(unavailable); + listNamespacedPod.mockRejectedValue(unavailable); + + await expect(service.stopOnDemandServer("match-1")).rejects.toThrow( + "connect ECONNREFUSED", + ); + + expect(serverReleases()).toHaveLength(0); + }); + }); + + describe("checking back on a signalled server", () => { + beforeEach(() => { + hasura.query.mockImplementation(async (request: any) => { + if (request.matches) { + return { + matches: [ + { + id: "match-1", + status: "Finished", + server_id: null, + options: { tv_delay: 0 }, + }, + ], + }; + } + if (request.servers) { + return { servers: [] }; + } + return {}; + }); + }); + + it("removes the Job it signalled when it is still running", async () => { + readNamespacedJob + .mockResolvedValueOnce(activeJob("uid-1")) + .mockRejectedValue(notFound); + + await service.removeUnstoppedOnDemandServer("match-1", "uid-1"); + + expect(deleteNamespacedJob).toHaveBeenCalledWith( + expect.objectContaining({ name: "m-match-1" }), + ); + }); + + it("deletes the Job only while it is still the one it signalled", async () => { + readNamespacedJob + .mockResolvedValueOnce(activeJob("uid-1")) + .mockRejectedValue(notFound); + + await service.removeUnstoppedOnDemandServer("match-1", "uid-1"); + + expect(deleteNamespacedJob).toHaveBeenCalledWith( + expect.objectContaining({ + name: "m-match-1", + body: expect.objectContaining({ + preconditions: { uid: "uid-1" }, + propagationPolicy: "Background", + }), + }), + ); + }); + + it("leaves a Job that replaced it between the read and the delete", async () => { + readNamespacedJob.mockResolvedValueOnce(activeJob("uid-1")); + deleteNamespacedJob.mockRejectedValue({ code: 409 }); + listNamespacedPod.mockResolvedValue({ + items: [creatingPod("uid-2", "m-match-1-fghij")], + }); + + await expect( + service.removeUnstoppedOnDemandServer("match-1", "uid-1"), + ).resolves.toBeUndefined(); + + expect(deleteNamespacedPod).not.toHaveBeenCalled(); + }); + + it("removes only the pods that belonged to the Job it removed", async () => { + readNamespacedJob + .mockResolvedValueOnce(activeJob("uid-1")) + .mockRejectedValue(notFound); + listNamespacedPod + .mockResolvedValueOnce({ + items: [ + runningPod("uid-1", "m-match-1-abcde"), + creatingPod("uid-2", "m-match-1-fghij"), + ], + }) + .mockResolvedValue({ items: [] }); + + await service.removeUnstoppedOnDemandServer("match-1", "uid-1"); + + expect( + deleteNamespacedPod.mock.calls.map(([request]) => request.name), + ).toEqual(["m-match-1-abcde"]); + }); + + it("does not hold the queue waiting for the Job to disappear", async () => { + readNamespacedJob + .mockResolvedValueOnce(activeJob("uid-1")) + .mockRejectedValue(notFound); + + await service.removeUnstoppedOnDemandServer("match-1", "uid-1"); + + expect(readNamespacedJob).toHaveBeenCalledTimes(1); + }); + + it("leaves a Job created for a later assignment alone", async () => { + readNamespacedJob.mockResolvedValue(activeJob("uid-2")); + + await service.removeUnstoppedOnDemandServer("match-1", "uid-1"); + + expect(deleteNamespacedJob).not.toHaveBeenCalled(); + }); + + it("leaves a Job that stopped the way it was asked to", async () => { + readNamespacedJob.mockResolvedValue({ + metadata: { name: "m-match-1", uid: "uid-1" }, + status: { conditions: [{ type: "Complete", status: "True" }] }, + }); + + await service.removeUnstoppedOnDemandServer("match-1", "uid-1"); + + expect(deleteNamespacedJob).not.toHaveBeenCalled(); + }); + }); + + describe("a queued stop that runs late", () => { + let matchStatus: string; + + beforeEach(() => { + matchStatus = "Live"; + hasura.query.mockImplementation(async (request: any) => { + if (request.matches_by_pk) { + return { + matches_by_pk: { status: matchStatus, server_error: null }, + }; + } + return {}; + }); + }); + + const runStop = () => + new StopOnDemandServer(service).process({ + data: { matchId: "match-1" }, + } as any); + + it("leaves the server of a match that was started again alone", async () => { + readNamespacedJob + .mockResolvedValueOnce(activeJob("uid-2")) + .mockRejectedValue(notFound); + listNamespacedPod + .mockResolvedValueOnce({ items: [creatingPod("uid-2")] }) + .mockResolvedValue({ items: [] }); + + await runStop(); + + expect(deleteNamespacedJob).not.toHaveBeenCalled(); + expect(serverReleases()).toHaveLength(0); + }); + + it("frees only rows still held by a match that has ended", async () => { + matchStatus = "Canceled"; + + await runStop(); + + const [release] = serverReleases(); + + expect(release.__args.where).toEqual({ + reserved_by_match_id: { _eq: "match-1" }, + current_match: { + status: { _in: expect.arrayContaining(["Canceled", "Finished"]) }, + }, + }); + }); + }); + + describe("removing a server outright", () => { + it("deletes the Job before its pods", async () => { + listNamespacedPod + .mockResolvedValueOnce({ items: [runningPod()] }) + .mockResolvedValue({ items: [] }); + + await service.stopOnDemandServer("match-1", { remove: true }); + + expect(deleteNamespacedJob).toHaveBeenCalled(); + expect(deleteNamespacedPod).toHaveBeenCalled(); + expect(deleteNamespacedJob.mock.invocationCallOrder[0]).toBeLessThan( + deleteNamespacedPod.mock.invocationCallOrder[0], + ); + }); + + it("waits for the pods to be gone, not just the Job", async () => { + listNamespacedPod + .mockResolvedValueOnce({ items: [runningPod()] }) + .mockResolvedValueOnce({ items: [runningPod()] }) + .mockResolvedValue({ items: [] }); + + await service.stopOnDemandServer("match-1", { remove: true }); + + expect(listNamespacedPod.mock.calls.length).toBeGreaterThanOrEqual(3); + }); + + it("releases only the server it was given", async () => { + await service.stopOnDemandServer("match-1", { + remove: true, + serverId: "server-1", + }); + + const [release] = serverReleases(); + + expect(release.__args.where).toEqual({ + reserved_by_match_id: { _eq: "match-1" }, + id: { _eq: "server-1" }, + }); + }); + }); + + describe("assigning an on-demand server", () => { + let matchStatus: string; + let serverIdWriteAffectedRows: number; + + beforeEach(() => { + matchStatus = "Live"; + serverIdWriteAffectedRows = 1; + + hasura.query.mockImplementation(async (request: any) => { + if (request.matches_by_pk?.match_maps) { + return { + matches_by_pk: { + region: "USE", + password: "secret", + server_id: null, + source: "5stack", + max_players_per_lineup: 5, + is_tournament_match: false, + options: { type: "Competitive" }, + match_maps: [ + { + order: 1, + map: { name: "de_inferno", workshop_map_id: null }, + }, + ], + }, + }; + } + if (request.matches_by_pk?.server_error) { + return { matches_by_pk: { server_error: null } }; + } + if (request.matches_by_pk) { + return { + matches_by_pk: { + id: "match-1", + status: matchStatus, + region: "USE", + source: "5stack", + options: { prefer_dedicated_server: false }, + }, + }; + } + if (request.game_server_nodes) { + return { game_server_nodes: [{ id: "node-1" }] }; + } + if (request.servers) { + return { + servers: [ + { + id: "server-1", + label: "node-1:27015", + host: "10.0.0.1", + port: 27015, + tv_port: 27020, + api_password: "api", + rcon_password: "encrypted", + game_server_node: { + id: "node-1", + pin_plugin_version: null, + pin_plugin_runtime: null, + supports_cpu_pinning: false, + }, + server_region: { is_lan: false, steam_relay: false }, + }, + ], + }; + } + if (request.settings_by_pk) { + return { settings_by_pk: null }; + } + return {}; + }); + + hasura.mutation.mockImplementation(async (request: any) => { + if (request.update_matches) { + return { + update_matches: { affected_rows: serverIdWriteAffectedRows }, + }; + } + return {}; + }); + }); + + it("does not look for a server for a match that already ended", async () => { + matchStatus = "Canceled"; + + const assignOnDemandServer = jest.spyOn( + service as any, + "assignOnDemandServer", + ); + + await service.assignServer("match-1"); + + expect(assignOnDemandServer).not.toHaveBeenCalled(); + expect(createNamespacedJob).not.toHaveBeenCalled(); + }); + + it("does not fall back to a dedicated server for a match that ended during the on-demand attempt", async () => { + jest + .spyOn(service as any, "assignOnDemandServer") + .mockImplementation(async () => { + matchStatus = "Canceled"; + return false; + }); + const assignDedicatedServer = jest + .spyOn(service as any, "assignDedicatedServer") + .mockResolvedValue(true); + + await service.assignServer("match-1"); + + expect(assignDedicatedServer).not.toHaveBeenCalled(); + }); + + it("never creates a Job for a match that ended before the pool lock was taken", async () => { + matchStatus = "Canceled"; + + await expect( + (service as any).assignOnDemandServer("match-1"), + ).resolves.toBe(false); + + expect(createNamespacedJob).not.toHaveBeenCalled(); + }); + + it("removes the Job it just created when the match ended mid-assignment", async () => { + serverIdWriteAffectedRows = 0; + + await expect( + (service as any).assignOnDemandServer("match-1"), + ).resolves.toBe(false); + + const serverIdWrite = hasura.mutation.mock.calls + .map(([mutation]) => mutation?.update_matches) + .find((update) => update?.__args?._set?.server_id === "server-1"); + + expect(serverIdWrite.__args.where.status._nin).toEqual( + expect.arrayContaining(["Canceled", "Finished"]), + ); + + const created = createNamespacedJob.mock.invocationCallOrder[0]; + + expect( + deleteNamespacedJob.mock.invocationCallOrder.some( + (order) => order > created, + ), + ).toBe(true); + expect(serverReleases()).toContainEqual( + expect.objectContaining({ + __args: expect.objectContaining({ + where: expect.objectContaining({ + reserved_by_match_id: { _eq: "match-1" }, + }), + }), + }), + ); + expect(queue.add).not.toHaveBeenCalled(); + }); + + it("keeps the server the match already points at when the boot check cannot be queued", async () => { + queue.add.mockRejectedValue(new Error("redis unavailable")); + + await expect( + (service as any).assignOnDemandServer("match-1"), + ).rejects.toBeInstanceOf(FailedToCreateOnDemandServer); + + const created = createNamespacedJob.mock.invocationCallOrder[0]; + + expect( + deleteNamespacedJob.mock.invocationCallOrder.some( + (order) => order > created, + ), + ).toBe(false); + expect( + serverReleases().filter((release) => + hasura.mutation.mock.invocationCallOrder.some( + (order, index) => + order > created && + hasura.mutation.mock.calls[index][0]?.update_servers === release, + ), + ), + ).toEqual([]); + }); + + it("labels the Job so a sweep can find it", async () => { + await expect( + (service as any).assignOnDemandServer("match-1"), + ).resolves.toBe(true); + + const body = createNamespacedJob.mock.calls[0][0].body; + const labels = { + app: "game-server", + role: "match", + "match-id": "match-1", + }; + + expect(body.metadata.labels).toEqual(labels); + expect(body.spec.template.metadata.labels).toEqual({ + job: "m-match-1", + ...labels, + }); + }); + + it("holds the pool lock past a teardown but not past the retries", async () => { + await (service as any).assignOnDemandServer("match-1"); + + const [key, , expires] = cache.lock.mock.calls[0]; + + expect(key).toBe("get-on-demand-server:USE"); + expect(expires).toBeGreaterThanOrEqual(30); + expect(expires).toBeLessThanOrEqual(45); + }); + + it("retries when another assignment holds the pool lock", async () => { + cache.lock.mockRejectedValue( + new Error( + "Failed to acquire lock for get-on-demand-server:USE after 10 attempts", + ), + ); + + await expect( + (service as any).assignOnDemandServer("match-1"), + ).rejects.toBeInstanceOf(FailedToCreateOnDemandServer); + }); + + it("does not mistake a failure inside the lock for contention", async () => { + hasura.query.mockImplementation(async (request: any) => { + if (request.servers) { + throw new Error("hasura unavailable"); + } + if (request.game_server_nodes) { + return { game_server_nodes: [{ id: "node-1" }] }; + } + if (request.matches_by_pk?.match_maps) { + return { + matches_by_pk: { + region: "USE", + match_maps: [{ order: 1, map: { name: "de_inferno" } }], + }, + }; + } + return { matches_by_pk: { status: "Live", server_error: null } }; + }); + + await expect( + (service as any).assignOnDemandServer("match-1"), + ).rejects.toThrow("hasura unavailable"); + }); + }); +}); diff --git a/src/matches/match-assistant/match-assistant.reconcile-jobs.spec.ts b/src/matches/match-assistant/match-assistant.reconcile-jobs.spec.ts new file mode 100644 index 000000000..ca9f1ede9 --- /dev/null +++ b/src/matches/match-assistant/match-assistant.reconcile-jobs.spec.ts @@ -0,0 +1,472 @@ +const listNamespacedJob = jest.fn(); +const readNamespacedJob = jest.fn(); +const deleteNamespacedJob = jest.fn(); +const listNamespacedPod = jest.fn(); +const deleteNamespacedPod = jest.fn(); + +jest.mock("@kubernetes/client-node", () => ({ + BatchV1Api: class BatchV1Api { + listNamespacedJob = listNamespacedJob; + readNamespacedJob = readNamespacedJob; + deleteNamespacedJob = deleteNamespacedJob; + }, + CoreV1Api: class CoreV1Api { + listNamespacedPod = listNamespacedPod; + deleteNamespacedPod = deleteNamespacedPod; + }, + KubeConfig: class KubeConfig { + loadFromDefault() {} + makeApiClient(ctor: new () => unknown) { + return new ctor(); + } + }, + Exec: class Exec {}, +})); + +import { MatchAssistantService } from "./match-assistant.service"; + +describe("MatchAssistantService — reconciling on-demand server Jobs", () => { + const NOW = Date.parse("2026-09-16T12:00:00Z"); + const MINUTE = 60 * 1000; + const MATCH_ID = "0b6f3c9e-4a57-4d2f-9f0e-3c2b1a0d9e8f"; + const JOB_NAME = `m-${MATCH_ID}`; + + let service: MatchAssistantService; + let hasura: { query: jest.Mock; mutation: jest.Mock }; + let store: Map; + let cache: { + get: jest.Mock; + put: jest.Mock; + forget: jest.Mock; + lock: jest.Mock; + }; + let logger: { + warn: jest.Mock; + log: jest.Mock; + error: jest.Mock; + verbose: jest.Mock; + }; + let now: number; + let matches: Array>; + let reservedServers: Array<{ id: string; reserved_by_match_id: string }>; + + const job = (overrides: Record = {}) => ({ + metadata: { + name: JOB_NAME, + uid: "uid-1", + creationTimestamp: new Date(NOW - 60 * MINUTE), + ...(overrides.metadata ?? {}), + }, + status: { active: 1, ...(overrides.status ?? {}) }, + }); + + const match = (overrides: Record = {}) => ({ + id: MATCH_ID, + status: "Live", + ended_at: null as string | null, + cancels_at: null as string | null, + server_id: "server-1", + options: { tv_delay: 120 }, + ...overrides, + }); + + const deletedJobs = () => + deleteNamespacedJob.mock.calls.map(([request]) => request.name); + + beforeEach(() => { + for (const fn of [ + listNamespacedJob, + readNamespacedJob, + deleteNamespacedJob, + listNamespacedPod, + deleteNamespacedPod, + ]) { + fn.mockReset(); + } + + now = NOW; + jest.spyOn(Date, "now").mockImplementation(() => now); + + listNamespacedJob.mockResolvedValue({ items: [job()] }); + readNamespacedJob.mockRejectedValue({ code: 404 }); + deleteNamespacedJob.mockResolvedValue({}); + listNamespacedPod.mockResolvedValue({ items: [] }); + deleteNamespacedPod.mockResolvedValue({}); + + matches = []; + reservedServers = []; + + hasura = { + query: jest.fn(async (request: any) => { + if (request.matches) { + return { matches }; + } + if (request.servers) { + return { servers: reservedServers }; + } + return {}; + }), + mutation: jest.fn(async () => ({})), + }; + + store = new Map(); + cache = { + get: jest.fn(async (key: string) => store.get(key)), + put: jest.fn(async (key: string, value: unknown) => { + store.set(key, value); + return true; + }), + forget: jest.fn(async (...keys: Array) => { + keys.forEach((key) => store.delete(key)); + return true; + }), + lock: jest.fn(), + }; + logger = { + warn: jest.fn(), + log: jest.fn(), + error: jest.fn(), + verbose: jest.fn(), + }; + + service = new MatchAssistantService( + logger as any, + {} as any, + cache as any, + { + get: jest.fn((key: string) => + key === "gameServers" ? { namespace: "test" } : {}, + ), + } as any, + hasura as any, + {} as any, + {} as any, + {} as any, + {} as any, + { add: jest.fn() } as any, + { add: jest.fn(), getDelayed: jest.fn() } as any, + ); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it("deletes a running Job whose match row is gone", async () => { + await service.reconcileOnDemandServerJobs(); + + expect(deletedJobs()).toEqual([JOB_NAME]); + }); + + it("deletes a labelled Job whose match row is gone", async () => { + listNamespacedJob.mockResolvedValue({ + items: [ + job({ + metadata: { + labels: { + app: "game-server", + role: "match", + "match-id": MATCH_ID, + }, + }, + }), + ], + }); + + await service.reconcileOnDemandServerJobs(); + + expect(deletedJobs()).toEqual([JOB_NAME]); + }); + + it("keeps a Live match's Job while it holds its on-demand server", async () => { + matches = [match()]; + reservedServers = [{ id: "server-1", reserved_by_match_id: MATCH_ID }]; + + await service.reconcileOnDemandServerJobs(); + now += 60 * MINUTE; + await service.reconcileOnDemandServerJobs(); + + expect(deleteNamespacedJob).not.toHaveBeenCalled(); + }); + + // The reservation, the Job and the server_id write are separate statements a sweep can land between. + it("keeps a Job younger than the create grace", async () => { + listNamespacedJob.mockResolvedValue({ + items: [job({ metadata: { creationTimestamp: new Date(NOW - MINUTE) } })], + }); + + await service.reconcileOnDemandServerJobs(); + + expect(deleteNamespacedJob).not.toHaveBeenCalled(); + }); + + it("ignores Jobs that already completed or failed", async () => { + listNamespacedJob.mockResolvedValue({ + items: [ + job({ + status: { + active: 0, + conditions: [{ type: "Complete", status: "True" }], + }, + }), + job({ + metadata: { + name: "m-1c0e2d3f-5b6a-4c7d-8e9f-0a1b2c3d4e5f", + }, + status: { + active: 0, + conditions: [{ type: "Failed", status: "True" }], + }, + }), + ], + }); + + await service.reconcileOnDemandServerJobs(); + + expect(deleteNamespacedJob).not.toHaveBeenCalled(); + expect(hasura.query).not.toHaveBeenCalled(); + }); + + it("ignores Jobs that are not match servers", async () => { + listNamespacedJob.mockResolvedValue({ + items: [ + job({ + metadata: { + name: `gs-live-${MATCH_ID}`, + labels: { + app: "game-streamer", + role: "live", + "match-id": MATCH_ID, + }, + }, + }), + job({ metadata: { name: "gs-demo-0b6f3c9e4a57" } }), + job({ metadata: { name: "update-cs-server-node-1" } }), + job({ metadata: { name: "validate-gamedata-12345-public" } }), + job({ metadata: { name: "m-notauuid" } }), + ], + }); + + await service.reconcileOnDemandServerJobs(); + + expect(deleteNamespacedJob).not.toHaveBeenCalled(); + expect(hasura.query).not.toHaveBeenCalled(); + }); + + it("lists only labelled Jobs once no unlabelled match server Job is left", async () => { + const labelled = job({ + metadata: { + labels: { app: "game-server", role: "match", "match-id": MATCH_ID }, + }, + }); + matches = [match()]; + reservedServers = [{ id: "server-1", reserved_by_match_id: MATCH_ID }]; + + await service.reconcileOnDemandServerJobs(); + await service.reconcileOnDemandServerJobs(); + + listNamespacedJob.mockResolvedValue({ items: [labelled] }); + await service.reconcileOnDemandServerJobs(); + await service.reconcileOnDemandServerJobs(); + + expect( + listNamespacedJob.mock.calls.map(([request]) => request.labelSelector), + ).toEqual([undefined, undefined, undefined, "app=game-server,role=match"]); + }); + + it("clears the orphan timers of every match holding its server in one call", async () => { + const other = "1c0e2d3f-5b6a-4c7d-8e9f-0a1b2c3d4e5f"; + listNamespacedJob.mockResolvedValue({ + items: [job(), job({ metadata: { name: `m-${other}`, uid: "uid-2" } })], + }); + matches = [match(), match({ id: other, server_id: "server-2" })]; + reservedServers = [ + { id: "server-1", reserved_by_match_id: MATCH_ID }, + { id: "server-2", reserved_by_match_id: other }, + ]; + + await service.reconcileOnDemandServerJobs(); + + expect(cache.forget.mock.calls).toEqual([ + [ + "match-server-job:orphaned-since:uid-1", + "match-server-job:orphaned-since:uid-2", + ], + ]); + }); + + it("deletes nothing when the Jobs cannot be listed", async () => { + listNamespacedJob.mockRejectedValue(new Error("forbidden")); + + await expect( + service.reconcileOnDemandServerJobs(), + ).resolves.toBeUndefined(); + + expect(deleteNamespacedJob).not.toHaveBeenCalled(); + expect(hasura.query).not.toHaveBeenCalled(); + expect(logger.error).toHaveBeenCalled(); + }); + + it("deletes a Job only while it is still the one it listed", async () => { + await service.reconcileOnDemandServerJobs(); + + expect(deleteNamespacedJob).toHaveBeenCalledWith( + expect.objectContaining({ + name: JOB_NAME, + body: expect.objectContaining({ preconditions: { uid: "uid-1" } }), + }), + ); + }); + + it("leaves a Job replaced after it was listed, and the rows its match holds", async () => { + matches = [ + match({ + status: "Canceled", + server_id: null, + cancels_at: new Date(NOW - 60 * MINUTE).toISOString(), + }), + ]; + reservedServers = [{ id: "server-2", reserved_by_match_id: MATCH_ID }]; + deleteNamespacedJob.mockRejectedValue({ code: 409 }); + + await service.reconcileOnDemandServerJobs(); + + expect(hasura.mutation).not.toHaveBeenCalled(); + expect(logger.error).not.toHaveBeenCalled(); + }); + + // Shares the one-worker scheduled-matches queue, and never reuses the name it removes. + it("does not wait for a removed Job to disappear", async () => { + await service.reconcileOnDemandServerJobs(); + + expect(deletedJobs()).toEqual([JOB_NAME]); + expect(readNamespacedJob).not.toHaveBeenCalled(); + }); + + it("treats a Job that is already gone as deleted", async () => { + deleteNamespacedJob.mockRejectedValue({ code: 404 }); + + await expect( + service.reconcileOnDemandServerJobs(), + ).resolves.toBeUndefined(); + + expect(deletedJobs()).toEqual([JOB_NAME]); + expect(logger.error).not.toHaveBeenCalled(); + }); + + describe("a match that has ended", () => { + it("deletes a Canceled match's Job once the grace has passed", async () => { + matches = [ + match({ + status: "Canceled", + server_id: null, + cancels_at: new Date(NOW - MINUTE).toISOString(), + }), + ]; + + await service.reconcileOnDemandServerJobs(); + expect(deleteNamespacedJob).not.toHaveBeenCalled(); + + now += 15 * MINUTE; + await service.reconcileOnDemandServerJobs(); + expect(deletedJobs()).toEqual([JOB_NAME]); + }); + + it("waits out tv_delay before deleting a Finished match's Job", async () => { + matches = [ + match({ + status: "Finished", + server_id: null, + ended_at: new Date(NOW - 11 * MINUTE).toISOString(), + options: { tv_delay: 5 * 60 }, + }), + ]; + + await service.reconcileOnDemandServerJobs(); + expect(deleteNamespacedJob).not.toHaveBeenCalled(); + + now += 5 * MINUTE; + await service.reconcileOnDemandServerJobs(); + expect(deletedJobs()).toEqual([JOB_NAME]); + }); + + it("counts the grace from when it first saw a match with no end time", async () => { + matches = [match({ status: "Surrendered", server_id: null })]; + + await service.reconcileOnDemandServerJobs(); + now += 5 * MINUTE; + await service.reconcileOnDemandServerJobs(); + expect(deleteNamespacedJob).not.toHaveBeenCalled(); + + now += 10 * MINUTE; + await service.reconcileOnDemandServerJobs(); + expect(deletedJobs()).toEqual([JOB_NAME]); + }); + + // tau_matches only frees the server_id it can see. + it("releases the on-demand server the match still held", async () => { + matches = [ + match({ + status: "Canceled", + server_id: null, + cancels_at: new Date(NOW - 60 * MINUTE).toISOString(), + }), + ]; + reservedServers = [{ id: "server-2", reserved_by_match_id: MATCH_ID }]; + + await service.reconcileOnDemandServerJobs(); + + expect(deletedJobs()).toEqual([JOB_NAME]); + expect(hasura.mutation).toHaveBeenCalledWith( + expect.objectContaining({ + update_servers: expect.objectContaining({ + __args: expect.objectContaining({ + where: { + reserved_by_match_id: { _eq: MATCH_ID }, + id: { _eq: "server-2" }, + _or: [ + { + current_match: { + status: { + _in: expect.arrayContaining(["Canceled", "Finished"]), + }, + }, + }, + { _not: { matches: { id: { _eq: MATCH_ID } } } }, + ], + }, + }), + }), + }), + ); + }); + }); + + it("deletes a live match's Job once the match no longer holds its server", async () => { + matches = [match({ status: "Live", server_id: null })]; + + await service.reconcileOnDemandServerJobs(); + expect(deleteNamespacedJob).not.toHaveBeenCalled(); + + now += 10 * MINUTE; + await service.reconcileOnDemandServerJobs(); + expect(deletedJobs()).toEqual([JOB_NAME]); + }); + + it("forgets a match that went back to holding its server", async () => { + matches = [match({ status: "Live", server_id: null })]; + await service.reconcileOnDemandServerJobs(); + + matches = [match()]; + reservedServers = [{ id: "server-1", reserved_by_match_id: MATCH_ID }]; + now += 10 * MINUTE; + await service.reconcileOnDemandServerJobs(); + + matches = [match({ status: "Live", server_id: null })]; + reservedServers = []; + now += MINUTE; + await service.reconcileOnDemandServerJobs(); + + expect(deleteNamespacedJob).not.toHaveBeenCalled(); + }); +}); diff --git a/src/matches/match-assistant/match-assistant.service.ts b/src/matches/match-assistant/match-assistant.service.ts index eb00feb43..62efbef1f 100644 --- a/src/matches/match-assistant/match-assistant.service.ts +++ b/src/matches/match-assistant/match-assistant.service.ts @@ -7,6 +7,8 @@ import { CoreV1Api, KubeConfig, Exec, + V1Job, + V1Pod, } from "@kubernetes/client-node"; import { RconService } from "../../rcon/rcon.service"; import { User } from "../../auth/types/User"; @@ -22,6 +24,7 @@ import { e_match_status_enum, e_match_types_enum, e_timeout_settings_enum, + servers_bool_exp, } from "../../../generated"; import { CacheService } from "../../cache/cache.service"; import { EncryptionService } from "../../encryption/encryption.service"; @@ -37,6 +40,8 @@ export class MatchAssistantService { private gameServerConfig: GameServersConfig; private readonly namespace: string; + // Jobs created before the match labels only match by name, so the sweep lists everything until none is left. + private unlabelledServerJobsRemain = true; private static readonly REBOOTABLE_ON_DEMAND_STATUSES: readonly e_match_status_enum[] = [ "Scheduled", @@ -62,6 +67,20 @@ export class MatchAssistantService { "practice", ]; public static readonly ON_DEMAND_SERVER_BOOT_CHECK_DELAY_MS = 15 * 1000; + public static readonly ON_DEMAND_SERVER_STOP_CHECK_DELAY_MS = 2 * 60 * 1000; + private static readonly ON_DEMAND_SERVER_TEARDOWN_TIMEOUT_MS = 15 * 1000; + // Must outlast the previous Job's teardown (cache.lock releases by key, not owner), + // yet expire well inside assignServer's ~55s of retries. + private static readonly ON_DEMAND_ASSIGNMENT_LOCK_SECONDS = 45; + private static readonly ORPHANED_JOB_CREATE_GRACE_MS = 5 * 60 * 1000; + private static readonly ORPHANED_JOB_UNASSIGNED_GRACE_MS = 5 * 60 * 1000; + private static readonly ORPHANED_JOB_ENDED_GRACE_MS = 10 * 60 * 1000; + private static readonly MATCH_SERVER_JOB_SELECTOR = + "app=game-server,role=match"; + private static readonly MATCH_SERVER_JOB_NAME = + /^m-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i; + private static readonly UUID = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; private static readonly INITIAL_BOOT_STATUS_DETAIL = "Waiting for Kubernetes to create the match server pod."; @@ -88,6 +107,14 @@ export class MatchAssistantService { return `m-${matchId}`; } + public static GetMatchServerJobLabels(matchId: string) { + return { + app: "game-server", + role: "match", + "match-id": matchId, + }; + } + public async sendServerMatchId(matchId: string) { try { await this.command(matchId, `get_match`); @@ -418,6 +445,7 @@ export class MatchAssistantService { id: matchId, }, id: true, + status: true, region: true, source: true, options: { @@ -426,6 +454,14 @@ export class MatchAssistantService { }, }); + if ( + !match || + MatchAssistantService.TERMINAL_MATCH_STATUSES.includes(match.status) + ) { + this.logger.log(`[${matchId}] match has ended, not assigning a server`); + return; + } + const onDemandOnly = MatchAssistantService.ON_DEMAND_ONLY_SOURCES.includes(match.source); @@ -474,6 +510,11 @@ export class MatchAssistantService { } } + // The fallbacks end in startMatch, which would write Live over a match canceled meanwhile. + if (await this.hasMatchEnded(matchId)) { + return; + } + // No pod, and no second pool to fall back on. Saying so is what turns the // session Failed -- match_events reads WaitingForServer off a practice // match as "no practice server was available" -- rather than leaving it @@ -869,9 +910,21 @@ export class MatchAssistantService { const map = match.match_maps.at(0).map; - return this.cache.lock( + let locked = false; + + const assigned = this.cache.lock( `get-on-demand-server:${match.region}`, async () => { + locked = true; + + // A cancel during the lock wait saw no server_id, so nothing would stop a Job made now. + if (await this.hasMatchEnded(matchId)) { + this.logger.log( + `[${matchId}] match ended before an on demand server was assigned`, + ); + return false; + } + this.logger.log(`[${matchId}] assigning on demand server`); // Always tear down any existing k8s job for this match before creating @@ -879,7 +932,15 @@ export class MatchAssistantService { // (b) server_id was cleared but a stale job is left over from a prior // assignment, (c) delete propagation is slow — the wait-until-gone loop // inside stopOnDemandServer(remove=true) ensures the name is free. - await this.stopOnDemandServer(matchId, true); + try { + await this.stopOnDemandServer(matchId, { remove: true }); + } catch (error) { + this.logger.error( + `[${matchId}] unable to remove the previous on demand server`, + error?.response?.body?.message || error, + ); + throw new FailedToCreateOnDemandServer(); + } const kc = new KubeConfig(); kc.loadFromDefault(); @@ -1083,13 +1144,14 @@ export class MatchAssistantService { const gameModeEnvironment = this.gameModesService.environmentFor(gameMode); - await batch.createNamespacedJob({ + const createdJob = await batch.createNamespacedJob({ namespace: this.namespace, body: { apiVersion: "batch/v1", kind: "Job", metadata: { name: jobName, + labels: MatchAssistantService.GetMatchServerJobLabels(matchId), }, spec: { ttlSecondsAfterFinished: 60 * 60 * 24, @@ -1098,6 +1160,7 @@ export class MatchAssistantService { name: jobName, labels: { job: jobName, + ...MatchAssistantService.GetMatchServerJobLabels(matchId), }, }, spec: { @@ -1257,25 +1320,47 @@ export class MatchAssistantService { `[${matchId}] create service for on demand server`, ); - await this.hasura.mutation({ - update_matches_by_pk: { + const { update_matches } = await this.hasura.mutation({ + update_matches: { __args: { - pk_columns: { - id: matchId, + where: { + id: { + _eq: matchId, + }, + status: { + _nin: [...MatchAssistantService.TERMINAL_MATCH_STATUSES], + }, }, _set: { server_id: server.id, }, }, - __typename: true, + affected_rows: true, }, }); - await this.delayCheckOnDemandServer(matchId); - - return true; + if (!update_matches?.affected_rows) { + this.logger.warn( + `[${matchId}] match ended while its on demand server was being created, removing it`, + ); + await this.removeOnDemandServerJob(matchId, { + uid: createdJob?.metadata?.uid, + }); + await this.releaseOnDemandServer(matchId, server.id); + return false; + } } catch (error) { - await this.stopOnDemandServer(matchId, true); + try { + await this.removeOnDemandServerJob(matchId); + } catch (teardownError) { + this.logger.error( + `[${matchId}] unable to remove the failed on demand server`, + teardownError?.response?.body?.message || teardownError, + ); + } + + // No matches.server_id points at this row yet, so no trigger would free it. + await this.releaseOnDemandServer(matchId, server.id); this.logger.error( `[${matchId}] unable to create on demand server`, @@ -1284,8 +1369,51 @@ export class MatchAssistantService { throw new FailedToCreateOnDemandServer(); } + + // The match holds the server now, so the retry goes through a reboot instead of the teardown above. + try { + await this.delayCheckOnDemandServer(matchId); + } catch (error) { + this.logger.error( + `[${matchId}] unable to schedule the on demand server boot check`, + error, + ); + throw new FailedToCreateOnDemandServer(); + } + + return true; }, - 10, + MatchAssistantService.ON_DEMAND_ASSIGNMENT_LOCK_SECONDS, + ); + + // cache.lock throws a plain Error after ~1s of contention; assignServer's retries handle it. + return assigned.catch((error) => { + if (locked) { + throw error; + } + + this.logger.warn( + `[${matchId}] on demand server pool is busy, retrying: ${error?.message}`, + ); + throw new FailedToCreateOnDemandServer(); + }); + } + + private async hasMatchEnded(matchId: string): Promise { + const { matches_by_pk } = await this.hasura.query({ + matches_by_pk: { + __args: { + id: matchId, + }, + status: true, + }, + }); + + return ( + !matches_by_pk || + MatchAssistantService.TERMINAL_MATCH_STATUSES.includes( + matches_by_pk.status, + ) ); } @@ -1625,124 +1753,584 @@ export class MatchAssistantService { ); } - public async stopOnDemandServer(matchId: string, remove = false) { + public async stopOnDemandServer( + matchId: string, + options: { + remove?: boolean; + serverId?: string; + releaseOnlyIfEnded?: boolean; + } = {}, + ) { this.logger.log(`[${matchId}] stopping match servers`); - const jobName = MatchAssistantService.GetMatchServerJobId(matchId); - try { - const kc = new KubeConfig(); - kc.loadFromDefault(); + if (options.remove) { + await this.removeOnDemandServerJob(matchId, { wait: true }); + } else { + await this.signalOnDemandServerStop(matchId); + } + } catch (error) { + this.logger.error( + `[${matchId}] unable to stop on demand server`, + error?.response?.body?.message || error, + ); + throw error; + } + + await this.releaseOnDemandServer( + matchId, + options.serverId, + options.releaseOnlyIfEnded + ? MatchAssistantService.reservedByEndedMatch() + : undefined, + ); + + await this.setServerError(matchId, null); + } - const core = kc.makeApiClient(CoreV1Api); - const batch = kc.makeApiClient(BatchV1Api); + // The queued stop can run after a restart, when the Job under this name is the new server's. + public async stopEndedMatchServer(matchId: string) { + if (!(await this.hasMatchEnded(matchId))) { + this.logger.log( + `[${matchId}] match was started again, leaving its server running`, + ); + return; + } - const podList = await core.listNamespacedPod({ + await this.stopOnDemandServer(matchId, { releaseOnlyIfEnded: true }); + } + + public async releaseOnDemandServer( + matchId: string, + serverId?: string, + condition?: servers_bool_exp, + ) { + await this.hasura.mutation({ + update_servers: { + __args: { + where: { + reserved_by_match_id: { + _eq: matchId, + }, + ...(serverId + ? { + id: { + _eq: serverId, + }, + } + : {}), + ...(condition ?? {}), + }, + _set: { + boot_status: null, + boot_status_detail: null, + connected: false, + reserved_by_match_id: null, + }, + }, + __typename: true, + }, + }); + } + + // A SIGUSR1 sent before setup.sh hands PID 1 to server.sh is silently ignored. + public async removeUnstoppedOnDemandServer(matchId: string, jobUid: string) { + const kc = new KubeConfig(); + kc.loadFromDefault(); + const batch = kc.makeApiClient(BatchV1Api); + + const job = await this.readOnDemandServerJob( + batch, + MatchAssistantService.GetMatchServerJobId(matchId), + ); + + if ( + !job || + job.metadata?.uid !== jobUid || + MatchAssistantService.isJobFinished(job) + ) { + return; + } + + const { matches, servers } = await this.getOnDemandServerJobMatches([ + matchId, + ]); + + if (MatchAssistantService.holdsOnDemandServer(matches.at(0), servers)) { + return; + } + + this.logger.warn( + `[${matchId}] on demand server did not stop after it was signalled, removing it`, + ); + + await this.removeOnDemandServerJob(matchId, { uid: jobUid }); + } + + public async reconcileOnDemandServerJobs(): Promise { + const kc = new KubeConfig(); + kc.loadFromDefault(); + const batch = kc.makeApiClient(BatchV1Api); + + let jobs: Array; + try { + ({ items: jobs } = await batch.listNamespacedJob({ namespace: this.namespace, - labelSelector: `job-name=${jobName}`, - }); + ...(this.unlabelledServerJobsRemain + ? {} + : { labelSelector: MatchAssistantService.MATCH_SERVER_JOB_SELECTOR }), + })); + } catch (error) { + this.logger.error( + `[match-server-reaper] listJobs failed: ${error?.message}`, + ); + return; + } - for (const pod of podList.items) { - this.logger.verbose(`[${matchId}] remove pod`); + const now = Date.now(); + const candidates = new Map(); + let unlabelledServerJobsRemain = false; - if (!remove) { - try { - await new Exec(kc).exec( - this.namespace, - pod.metadata!.name!, - pod.spec!.containers?.at(0)?.name, - ["kill", "-SIGUSR1", "1"], - process.stdout, - process.stderr, - process.stdin, - false, - ); - } catch (error) { - this.logger.warn( - `[${matchId}] graceful shutdown signal failed: ${error?.message || "exec error"}`, - ); - } + for (const job of jobs) { + const matchId = MatchAssistantService.getServerJobMatchId(job); + + if (!matchId || MatchAssistantService.isJobFinished(job)) { + continue; + } + + if (job.metadata?.labels?.role !== "match") { + unlabelledServerJobsRemain = true; + } + + const createdAt = new Date( + job.metadata?.creationTimestamp ?? now, + ).getTime(); + + if ( + now - createdAt < + MatchAssistantService.ORPHANED_JOB_CREATE_GRACE_MS + ) { + continue; + } + + candidates.set(matchId, job); + } + + this.unlabelledServerJobsRemain = unlabelledServerJobsRemain; + + if (candidates.size === 0) { + return; + } + + const { matches, servers } = await this.getOnDemandServerJobMatches([ + ...candidates.keys(), + ]); + + const heldKeys: Array = []; + + for (const [matchId, job] of candidates) { + const match = matches.find(({ id }) => id === matchId); + const orphanedKey = `match-server-job:orphaned-since:${job.metadata?.uid ?? job.metadata?.name}`; + + if (MatchAssistantService.holdsOnDemandServer(match, servers)) { + heldKeys.push(orphanedKey); + continue; + } + + if (match) { + const ended = MatchAssistantService.TERMINAL_MATCH_STATUSES.includes( + match.status, + ); + const since = + (ended && MatchAssistantService.getMatchEndedAt(match)) || + (await this.getOrphanedSince(orphanedKey, now)); + const grace = ended + ? (match.options?.tv_delay ?? 0) * 1000 + + MatchAssistantService.ORPHANED_JOB_ENDED_GRACE_MS + : MatchAssistantService.ORPHANED_JOB_UNASSIGNED_GRACE_MS; + + if (now - since < grace) { continue; } - await core - .deleteNamespacedPod({ - name: pod.metadata!.name!, - namespace: this.namespace, - gracePeriodSeconds: 0, - }) - .catch((error) => { - if (error.code.toString() !== "404") { - throw error; + } + + this.logger.warn( + `[${matchId}] removing orphaned on demand server job ${job.metadata?.name} (${match ? `match is ${match.status}` : "match was deleted"})`, + ); + + try { + const removed = await this.removeOnDemandServerJob(matchId, { + uid: job.metadata?.uid, + }); + + // Rows were read before the loop; the write-time condition keeps a reassignment's row. + if (removed) { + for (const server of servers) { + if (server.reserved_by_match_id === matchId) { + await this.releaseOnDemandServer( + matchId, + server.id, + MatchAssistantService.reservationNotHeld(matchId), + ); } - }); + } + } + + await this.cache.forget(orphanedKey); + } catch (error) { + this.logger.error( + `[${matchId}] unable to remove orphaned on demand server: ${error?.message}`, + ); + } + } + + await this.cache.forget(...heldKeys); + } + + private async getOnDemandServerJobMatches(matchIds: Array) { + const { matches } = await this.hasura.query({ + matches: { + __args: { + where: { + id: { + _in: matchIds, + }, + }, + }, + id: true, + status: true, + ended_at: true, + cancels_at: true, + server_id: true, + options: { + tv_delay: true, + }, + }, + }); + + const { servers } = await this.hasura.query({ + servers: { + __args: { + where: { + reserved_by_match_id: { + _in: matchIds, + }, + is_dedicated: { + _eq: false, + }, + }, + }, + id: true, + reserved_by_match_id: true, + }, + }); + + return { matches, servers }; + } + + private static holdsOnDemandServer( + match: { id: string; status: e_match_status_enum; server_id?: string }, + servers: Array<{ id: string; reserved_by_match_id?: string }>, + ) { + return ( + !!match?.server_id && + !MatchAssistantService.TERMINAL_MATCH_STATUSES.includes(match.status) && + servers.some( + (server) => + server.id === match.server_id && + server.reserved_by_match_id === match.id, + ) + ); + } + + // Canceled clears ended_at; tbu_matches stamps cancels_at at cancel time instead. + private static getMatchEndedAt(match: { + status: e_match_status_enum; + ended_at?: string; + cancels_at?: string; + }): number | null { + const endedAt = + match.ended_at ?? (match.status === "Canceled" ? match.cancels_at : null); + + return endedAt ? new Date(endedAt).getTime() : null; + } + + private async getOrphanedSince(key: string, now: number): Promise { + const since = await this.cache.get(key); + + if (typeof since === "number") { + return since; + } + + await this.cache.put(key, now, 24 * 60 * 60); + + return now; + } + + private static reservedByEndedMatch(): servers_bool_exp { + return { + current_match: { + status: { + _in: [...MatchAssistantService.TERMINAL_MATCH_STATUSES], + }, + }, + }; + } + + // holdsOnDemandServer, negated, for the update to evaluate at write time. + private static reservationNotHeld(matchId: string): servers_bool_exp { + return { + _or: [ + MatchAssistantService.reservedByEndedMatch(), + { + _not: { + matches: { + id: { + _eq: matchId, + }, + }, + }, + }, + ], + }; + } + + private static getServerJobMatchId(job: V1Job): string | null { + const labels = job.metadata?.labels ?? {}; + + const matchId = + labels.app === "game-server" && labels.role === "match" + ? labels["match-id"] + : MatchAssistantService.MATCH_SERVER_JOB_NAME.exec( + job.metadata?.name ?? "", + )?.[1]; + + return matchId && MatchAssistantService.UUID.test(matchId) ? matchId : null; + } + + private static isJobFinished(job: V1Job) { + return (job.status?.conditions ?? []).some( + (condition) => + condition.status === "True" && + ["Complete", "Failed", "SuccessCriteriaMet", "FailureTarget"].includes( + condition.type, + ), + ); + } + + private static isGameServerRunning(pod: V1Pod) { + if (pod.status?.phase !== "Running" || pod.metadata?.deletionTimestamp) { + return false; + } + + const container = pod.spec?.containers?.at(0)?.name; + + return (pod.status.containerStatuses ?? []).some( + (status) => status.name === container && !!status.state?.running, + ); + } + + private static isNotFound(error: { code?: number | string }) { + return error?.code?.toString() === "404"; + } + + private static isConflict(error: { code?: number | string }) { + return error?.code?.toString() === "409"; + } + + private static isOwnedBy(pod: V1Pod, jobUid?: string) { + return ( + !jobUid || + (pod.metadata?.ownerReferences ?? []).some( + (owner) => owner.uid === jobUid, + ) + ); + } + + private async readOnDemandServerJob( + batch: BatchV1Api, + jobName: string, + ): Promise { + try { + return await batch.readNamespacedJob({ + name: jobName, + namespace: this.namespace, + }); + } catch (error) { + if (MatchAssistantService.isNotFound(error)) { + return null; } + throw error; + } + } + + private async signalOnDemandServerStop(matchId: string) { + const jobName = MatchAssistantService.GetMatchServerJobId(matchId); + + const kc = new KubeConfig(); + kc.loadFromDefault(); + + const core = kc.makeApiClient(CoreV1Api); + const batch = kc.makeApiClient(BatchV1Api); + + const job = await this.readOnDemandServerJob(batch, jobName); - if (!remove) { + if (!job || MatchAssistantService.isJobFinished(job)) { + return; + } + + const { items: pods } = await core.listNamespacedPod({ + namespace: this.namespace, + labelSelector: `job-name=${jobName}`, + }); + + const running = pods.filter((pod) => + MatchAssistantService.isGameServerRunning(pod), + ); + + if (running.length === 0) { + if (pods.some((pod) => pod.status?.phase === "Succeeded")) { return; } - this.logger.verbose(`[${matchId}] remove job`); + // Nothing to signal: a pending pod boots anyway and a failed one is replaced (backoffLimit). + await this.removeOnDemandServerJob(matchId, { uid: job.metadata?.uid }); + return; + } + + for (const pod of running) { + this.logger.verbose(`[${matchId}] signal pod to stop`); - await batch - .deleteNamespacedJob({ - name: jobName, - namespace: this.namespace, + try { + await new Exec(kc).exec( + this.namespace, + pod.metadata!.name!, + pod.spec!.containers?.at(0)?.name, + ["kill", "-SIGUSR1", "1"], + process.stdout, + process.stderr, + process.stdin, + false, + ); + } catch (error) { + this.logger.warn( + `[${matchId}] graceful shutdown signal failed: ${error?.message || "exec error"}`, + ); + } + } + + await this.scheduledMatchesQueue.add( + MatchJobs.StopOnDemandServer, + { + matchId, + jobUid: job.metadata?.uid, + }, + { + delay: MatchAssistantService.ON_DEMAND_SERVER_STOP_CHECK_DELAY_MS, + attempts: 3, + backoff: { + type: "exponential", + delay: 10 * 1000, + }, + removeOnFail: true, + removeOnComplete: true, + jobId: `match.${matchId}.server-stop-check.${job.metadata?.uid}`, + }, + ); + } + + // A uid pins the Job and its pods (409 = already replaced): a reassignment reuses the name. + private async removeOnDemandServerJob( + matchId: string, + options: { uid?: string; wait?: boolean } = {}, + ): Promise { + const jobName = MatchAssistantService.GetMatchServerJobId(matchId); + + const kc = new KubeConfig(); + kc.loadFromDefault(); + + const core = kc.makeApiClient(CoreV1Api); + const batch = kc.makeApiClient(BatchV1Api); + + this.logger.verbose(`[${matchId}] remove job`); + + let removed = true; + + // Job before pods, or the Job controller replaces the deleted pods. + // The API server ignores query params once a body is sent, so options go in the body. + await batch + .deleteNamespacedJob({ + name: jobName, + namespace: this.namespace, + body: { propagationPolicy: "Background", gracePeriodSeconds: 0, + ...(options.uid ? { preconditions: { uid: options.uid } } : {}), + }, + }) + .catch((error) => { + if ( + !MatchAssistantService.isNotFound(error) && + !(options.uid && MatchAssistantService.isConflict(error)) + ) { + throw error; + } + removed = false; + }); + + const { items: pods } = await core.listNamespacedPod({ + namespace: this.namespace, + labelSelector: `job-name=${jobName}`, + }); + + for (const pod of pods) { + if (!MatchAssistantService.isOwnedBy(pod, options.uid)) { + continue; + } + + this.logger.verbose(`[${matchId}] remove pod`); + + await core + .deleteNamespacedPod({ + name: pod.metadata!.name!, + namespace: this.namespace, + gracePeriodSeconds: 0, }) .catch((error) => { - if (error.code.toString() !== "404") { + if (!MatchAssistantService.isNotFound(error)) { throw error; } }); + } - // Wait for the job to be fully gone from the k8s API before returning. - // Without this, a subsequent createNamespacedJob with the same name races - // against delete propagation and gets HTTP 409 AlreadyExists. - const deadline = Date.now() + 15_000; - while (Date.now() < deadline) { - try { - await batch.readNamespacedJob({ - name: jobName, - namespace: this.namespace, - }); - } catch (error) { - if (error.code?.toString() === "404") { - break; - } - throw error; + if (!options.wait) { + return removed; + } + + // A same-name create 409s until the Job is gone, and its pods hold the next server's host ports. + const deadline = + Date.now() + MatchAssistantService.ON_DEMAND_SERVER_TEARDOWN_TIMEOUT_MS; + while (Date.now() < deadline) { + if (!(await this.readOnDemandServerJob(batch, jobName))) { + const { items: remaining } = await core.listNamespacedPod({ + namespace: this.namespace, + labelSelector: `job-name=${jobName}`, + }); + + if (remaining.length === 0) { + this.logger.verbose(`[${matchId}] stopped on demand server`); + return removed; } - await new Promise((r) => setTimeout(r, 200)); } - this.logger.verbose(`[${matchId}] stopped on demand server`); - } catch (error) { - this.logger.error( - `[${matchId}] unable to stop on demand server`, - error?.response?.body?.message || error, - ); + await new Promise((resolve) => setTimeout(resolve, 200)); } - await this.hasura.mutation({ - update_servers: { - __args: { - where: { - reserved_by_match_id: { - _eq: matchId, - }, - }, - _set: { - boot_status: null, - boot_status_detail: null, - connected: false, - reserved_by_match_id: null, - }, - }, - __typename: true, - }, - }); + this.logger.warn( + `[${matchId}] on demand server was still shutting down after ${MatchAssistantService.ON_DEMAND_SERVER_TEARDOWN_TIMEOUT_MS / 1000}s`, + ); - await this.setServerError(matchId, null); + return removed; } public async getAvailableMaps(matchId: string) { diff --git a/src/matches/match-relay/match-relay.service.spec.ts b/src/matches/match-relay/match-relay.service.spec.ts new file mode 100644 index 000000000..b65d28fc1 --- /dev/null +++ b/src/matches/match-relay/match-relay.service.spec.ts @@ -0,0 +1,194 @@ +import { EventEmitter } from "events"; +import { MatchRelayService } from "./match-relay.service"; + +const fakeResponse = () => { + let resolveEnded: () => void; + const ended = new Promise((resolve) => { + resolveEnded = resolve; + }); + + const response = { + statusCode: undefined as number | undefined, + headers: {} as Record, + body: undefined as unknown, + ended, + writeHead(code: number, headers?: unknown) { + response.statusCode = code; + if (headers && typeof headers === "object") { + Object.assign(response.headers, headers); + } + return response; + }, + setHeader(name: string, value: unknown) { + response.headers[name] = value; + }, + end(body?: unknown) { + response.body = body; + resolveEnded(); + return response; + }, + }; + + return response; +}; + +describe("MatchRelayService", () => { + const matchId = "match-1"; + const token = "s845489096165654t8799308478907"; + + let service: MatchRelayService; + + beforeEach(() => { + service = new MatchRelayService({ + log: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + } as any); + }); + + const openPost = ( + field: "start" | "full" | "delta", + fragment: number, + query: Record = {}, + ) => { + const request = Object.assign(new EventEmitter(), { query }); + const response = fakeResponse(); + + service.postField( + request as any, + response as any, + token, + field, + matchId, + fragment, + ); + + return { + response, + finish: async () => { + request.emit("data", Buffer.from(`${field}-${fragment}`)); + request.emit("end"); + await response.ended; + return response; + }, + }; + }; + + const post = ( + field: "start" | "full" | "delta", + fragment: number, + query: Record = {}, + ) => openPost(field, fragment, query).finish(); + + const sync = (query: Record = {}) => { + const response = fakeResponse(); + service.getSyncInfo({ query } as any, response as any, matchId); + return response; + }; + + const getStart = (fragment: number) => { + const response = fakeResponse(); + service.getStart(response as any, matchId, fragment); + return response; + }; + + const startBroadcastAt = async (fragment: number) => { + await post("start", fragment, { + tick: "100", + tps: "64", + map: "de_inferno", + keyframe_interval: "3", + protocol: "5", + }); + await post("full", fragment, { tick: "100" }); + await post("delta", fragment, { endtick: "292" }); + }; + + it("reports the fragment the broadcast signed up at, with numeric fields", async () => { + await startBroadcastAt(42); + + const response = sync({ fragment: "0" }); + + expect(response.statusCode).toBe(200); + expect(JSON.parse(response.body as string)).toEqual( + expect.objectContaining({ + fragment: 42, + signup_fragment: 42, + tick: 100, + endtick: 292, + maxtick: 292, + tps: 64, + keyframe_interval: 3, + map: "de_inferno", + protocol: 5, + }), + ); + }); + + // CS2 sends tps as a decimal (64.0), not an integer, so the coercion has to + // accept a fractional part or clients get tps back as a string. + it("reports a fractional tps as a number", async () => { + await post("start", 42, { + tick: "100", + tps: "64.0", + map: "de_inferno", + }); + await post("full", 42, { tick: "100" }); + await post("delta", 42, { endtick: "292" }); + + const response = sync({ fragment: "0" }); + + expect(JSON.parse(response.body as string)).toEqual( + expect.objectContaining({ tps: 64 }), + ); + }); + + it("keeps fields that are not numeric in /sync as the strings the server sent", async () => { + await post("start", 42, { + tick: "100", + tps: "64", + map: "3070284539", + protocol: "5", + }); + await post("full", 42, { tick: "100" }); + await post("delta", 42, { endtick: "292" }); + + const body = JSON.parse(sync({ fragment: "0" }).body as string); + + expect(body.map).toBe("3070284539"); + expect(body.tick).toBe(100); + }); + + it("serves start only at the fragment the broadcast signed up at", async () => { + await startBroadcastAt(42); + + expect(getStart(42).statusCode).toBe(200); + expect(getStart(0).statusCode).toBe(404); + }); + + it("moves the signup fragment when the game server re-sends start", async () => { + await startBroadcastAt(42); + await post("start", 50, { tick: "900", tps: "64", map: "de_inferno" }); + + expect(getStart(50).statusCode).toBe(200); + expect(getStart(42).statusCode).toBe(404); + }); + + it("asks for start again when a fragment arrives before any start", async () => { + const response = await post("full", 7, { tick: "100" }); + + expect(response.statusCode).toBe(205); + }); + + it("asks for start again when a fragment arrives before the start data has", async () => { + const start = openPost("start", 42, { tick: "100", tps: "64" }); + + const early = await post("full", 42, { tick: "100" }); + expect(early.statusCode).toBe(205); + + await start.finish(); + + const late = await post("full", 43, { tick: "292" }); + expect(late.statusCode).toBe(200); + }); +}); diff --git a/src/matches/match-relay/match-relay.service.ts b/src/matches/match-relay/match-relay.service.ts index ef67bb3fe..7e4f91317 100644 --- a/src/matches/match-relay/match-relay.service.ts +++ b/src/matches/match-relay/match-relay.service.ts @@ -11,6 +11,14 @@ import { @Injectable() export class MatchRelayService { + private static readonly NUMERIC_QUERY_FIELDS = [ + "tick", + "endtick", + "tps", + "keyframe_interval", + "protocol", + ]; + private readonly gzip = promisify(zlib.gzip); private readonly broadcasts: { @@ -198,11 +206,14 @@ export class MatchRelayService { broadcast.fragments.clear(); } + const signupFragment = fragmentIndex; + if (field == "start") { fragmentIndex = 0; } - if (field != "start" && !broadcast.fragments.has(0)) { + // 205 makes the server re-send start, so it also covers a start whose body hasn't landed. + if (field != "start" && broadcast.fragments.get(0)?.start?.data == null) { response.writeHead(205); response.end(); return; @@ -216,13 +227,15 @@ export class MatchRelayService { const fragment = broadcast.fragments.get(fragmentIndex)!; if (fragment[field] == null) { - fragment[field] = { - ...(field === "start" ? { signup_fragment: fragmentIndex } : {}), - }; + fragment[field] = {}; + } + + if (field === "start") { + fragment.start!.signup_fragment = signupFragment; } Object.entries(request.query).forEach(([key, value]) => { - fragment[field]![key] = value; + fragment[field]![key] = MatchRelayService.parseQueryValue(key, value); }); const body: Buffer[] = []; @@ -255,6 +268,19 @@ export class MatchRelayService { }); } + // Clients read /sync's tick, tps, etc. as JSON numbers, as Valve's reference relay sends them. + private static parseQueryValue(key: string, value: unknown) { + if ( + typeof value !== "string" || + !MatchRelayService.NUMERIC_QUERY_FIELDS.includes(key) || + !/^\d+(?:\.\d+)?$/.test(value) + ) { + return value; + } + + return Number(value); + } + private relayError( response: Response, code: number, diff --git a/src/matches/matches.controller.match-events.spec.ts b/src/matches/matches.controller.match-events.spec.ts new file mode 100644 index 000000000..3632937e0 --- /dev/null +++ b/src/matches/matches.controller.match-events.spec.ts @@ -0,0 +1,402 @@ +jest.mock("@kubernetes/client-node", () => ({ + BatchV1Api: class BatchV1Api {}, + CoreV1Api: class CoreV1Api {}, + KubeConfig: class KubeConfig {}, + Exec: class Exec {}, +})); + +import { MatchesController } from "./matches.controller"; + +describe("MatchesController — match_events on-demand servers", () => { + let controller: MatchesController; + let hasura: { query: jest.Mock; mutation: jest.Mock }; + let matchAssistant: Record; + let scheduledMatchesQueue: { add: jest.Mock }; + let discordBotMessaging: { removeMatchChannel: jest.Mock }; + let utilityPractice: Record; + let servers: Record< + string, + { + is_dedicated: boolean; + reserved_by_match_id?: string | null; + game_server_node?: { region: string } | null; + } | null + >; + let currentMatch: Record; + + const stopJobs = () => + scheduledMatchesQueue.add.mock.calls.filter( + ([name]) => name === "StopOnDemandServer", + ); + + const row = (overrides: Record = {}) => ({ + id: "match-1", + source: "5stack", + status: "Live", + region: "USE", + server_id: "server-1", + match_options_id: "options-1", + lineup_1_id: "lineup-1", + lineup_2_id: "lineup-2", + ...overrides, + }); + + beforeEach(() => { + servers = { + "server-1": { + is_dedicated: false, + reserved_by_match_id: "match-1", + game_server_node: { region: "USE" }, + }, + "server-2": { + is_dedicated: false, + reserved_by_match_id: "match-1", + game_server_node: { region: "USE" }, + }, + "server-3": { + is_dedicated: false, + reserved_by_match_id: null, + game_server_node: { region: "USE" }, + }, + "dedicated-1": { is_dedicated: true, game_server_node: null }, + }; + currentMatch = { + id: "match-1", + options: { prefer_dedicated_server: false }, + server: { + id: "server-2", + is_dedicated: false, + reserved_by_match_id: "match-1", + game_server_node_id: "node-1", + }, + }; + + hasura = { + query: jest.fn(async (request: any) => { + if (request.servers_by_pk) { + return { + servers_by_pk: servers[request.servers_by_pk.__args.id] ?? null, + }; + } + if (request.match_options_by_pk) { + return { match_options_by_pk: { tv_delay: 30 } }; + } + if (request.matches_by_pk) { + return { matches_by_pk: currentMatch }; + } + return {}; + }), + mutation: jest.fn(async (request: any) => + request.update_matches ? { update_matches: { affected_rows: 1 } } : {}, + ), + }; + matchAssistant = { + removeVetoPickTimeout: jest.fn(), + scheduleVetoPickTimeout: jest.fn(), + stopOnDemandServer: jest.fn(), + releaseOnDemandServer: jest.fn(), + assignServer: jest.fn(), + reserveDedicatedServer: jest.fn(), + }; + scheduledMatchesQueue = { add: jest.fn() }; + discordBotMessaging = { removeMatchChannel: jest.fn() }; + utilityPractice = { + evictForMatch: jest.fn(async (): Promise => undefined), + markEndedForMatch: jest.fn(), + }; + + controller = new MatchesController( + { log: jest.fn(), warn: jest.fn(), error: jest.fn() } as any, + hasura as any, + { query: jest.fn(async (): Promise => []) } as any, + { get: jest.fn(() => ({})) } as any, + { cancelMatchMakingByMatchId: jest.fn() } as any, + matchAssistant as any, + discordBotMessaging as any, + { updateMatchOverview: jest.fn() } as any, + { removeTeamChannels: jest.fn() } as any, + { + resolveMatchAlerts: jest.fn(async (): Promise => undefined), + sendMatchWaitingForServerNotification: jest.fn( + async (): Promise => undefined, + ), + } as any, + { removeLobby: jest.fn() } as any, + { add: jest.fn() } as any, + {} as any, + {} as any, + {} as any, + {} as any, + { add: jest.fn(async (): Promise => undefined) } as any, + scheduledMatchesQueue as any, + {} as any, + { removeBroadcast: jest.fn() } as any, + { + createMatchVoiceChannels: jest.fn(), + movePlayersToMatchChannels: jest.fn(), + } as any, + { + stopLive: jest.fn(), + stopLiveIfRunning: jest.fn(), + promotePendingLiveStreams: jest.fn(async () => ({ + promoted: [] as string[], + })), + } as any, + {} as any, + { resumeAllPausedBatches: jest.fn() } as any, + {} as any, + {} as any, + { clearMatch: jest.fn() } as any, + { closeChannel: jest.fn(), graceOnMatchEnd: jest.fn() } as any, + utilityPractice as any, + {} as any, + ); + }); + + it("stops the on-demand server of a match that is deleted", async () => { + await controller.match_events({ + op: "DELETE", + old: row(), + new: {}, + } as any); + + expect(stopJobs()).toHaveLength(1); + + const [[, data, options]] = stopJobs(); + + expect(data).toEqual({ matchId: "match-1" }); + expect(options?.delay ?? 0).toBe(0); + expect(options?.attempts).toBeGreaterThan(1); + }); + + it("stops the server even when an end-of-match side effect throws", async () => { + discordBotMessaging.removeMatchChannel.mockRejectedValue( + new Error("discord unavailable"), + ); + + await controller + .match_events({ + op: "UPDATE", + old: row({ status: "Live" }), + new: row({ status: "Finished" }), + } as any) + .catch((): void => undefined); + + expect(stopJobs()).toHaveLength(1); + expect(stopJobs()[0][2]).toEqual( + expect.objectContaining({ delay: 30 * 1000 }), + ); + }); + + it("stops a canceled match's server straight away", async () => { + await controller.match_events({ + op: "UPDATE", + old: row({ status: "WaitingForServer" }), + new: row({ status: "Canceled" }), + } as any); + + expect(stopJobs()).toHaveLength(1); + expect(stopJobs()[0][2]?.delay ?? 0).toBe(0); + }); + + it("stops the server when its row is already gone", async () => { + servers["server-1"] = null; + + await controller.match_events({ + op: "UPDATE", + old: row({ status: "Live" }), + new: row({ status: "Canceled" }), + } as any); + + expect(stopJobs()).toHaveLength(1); + }); + + it("leaves a dedicated server running", async () => { + await controller.match_events({ + op: "UPDATE", + old: row({ status: "Live", server_id: "dedicated-1" }), + new: row({ status: "Finished", server_id: "dedicated-1" }), + } as any); + + expect(stopJobs()).toHaveLength(0); + }); + + it("does not stop again when its own server_id write re-fires the event", async () => { + await controller.match_events({ + op: "UPDATE", + old: row({ status: "Finished" }), + new: row({ status: "Finished", server_id: null }), + } as any); + + expect(stopJobs()).toHaveLength(0); + }); + + it("releases only the old row when a reboot moves the match onto a new on-demand server", async () => { + await controller.match_events({ + op: "UPDATE", + old: row({ server_id: "server-1" }), + new: row({ server_id: "server-2" }), + } as any); + + expect(matchAssistant.stopOnDemandServer).not.toHaveBeenCalled(); + expect(matchAssistant.releaseOnDemandServer).toHaveBeenCalledWith( + "match-1", + "server-1", + ); + }); + + it("stops the old on-demand server when the match moves onto a dedicated one", async () => { + currentMatch = { + ...currentMatch, + server: { + id: "dedicated-1", + is_dedicated: true, + reserved_by_match_id: "match-1", + game_server_node_id: null, + }, + }; + + await controller.match_events({ + op: "UPDATE", + old: row({ status: "WaitingForServer", server_id: "server-1" }), + new: row({ status: "WaitingForServer", server_id: "dedicated-1" }), + } as any); + + expect(matchAssistant.stopOnDemandServer).toHaveBeenCalledWith("match-1", { + serverId: "server-1", + }); + expect(matchAssistant.releaseOnDemandServer).not.toHaveBeenCalled(); + }); + + it("still stops the server when it is taken off the match", async () => { + currentMatch = { ...currentMatch, server: null }; + + await controller.match_events({ + op: "UPDATE", + old: row({ status: "WaitingForServer", server_id: "server-1" }), + new: row({ status: "WaitingForServer", server_id: null }), + } as any); + + expect(matchAssistant.stopOnDemandServer).toHaveBeenCalledWith("match-1"); + }); + + it("leaves the server alone when only the region changes", async () => { + currentMatch = { ...currentMatch, server: null }; + + await controller.match_events({ + op: "UPDATE", + old: row({ status: "Live", region: null, server_id: null }), + new: row({ status: "Live", region: "USE", server_id: null }), + } as any); + + expect(matchAssistant.stopOnDemandServer).not.toHaveBeenCalled(); + }); + + const serverClears = () => + hasura.mutation.mock.calls + .map(([request]) => request.update_matches?.__args) + .filter(Boolean); + + it("hands a server picked by hand back to assignment, since nothing booted it", async () => { + await controller.match_events({ + op: "UPDATE", + old: row({ server_id: "server-1" }), + new: row({ server_id: "server-3" }), + } as any); + + expect(serverClears()).toEqual([ + { + where: { id: { _eq: "match-1" }, server_id: { _eq: "server-3" } }, + _set: { server_id: null }, + }, + ]); + expect(matchAssistant.stopOnDemandServer).not.toHaveBeenCalled(); + expect(matchAssistant.releaseOnDemandServer).not.toHaveBeenCalled(); + expect(matchAssistant.assignServer).not.toHaveBeenCalled(); + }); + + it("moves an on-demand server off a region the match left", async () => { + await controller.match_events({ + op: "UPDATE", + old: row({ region: "USE", server_id: "server-1" }), + new: row({ region: "EUW", server_id: "server-1" }), + } as any); + + expect(serverClears()).toEqual([ + { + where: { id: { _eq: "match-1" }, server_id: { _eq: "server-1" } }, + _set: { server_id: null }, + }, + ]); + expect(matchAssistant.stopOnDemandServer).not.toHaveBeenCalled(); + expect(matchAssistant.releaseOnDemandServer).not.toHaveBeenCalled(); + expect(matchAssistant.assignServer).not.toHaveBeenCalled(); + }); + + it.each([ + ["an on-demand server already in the new region", null, "server-1"], + ["a dedicated server", "EUW", "dedicated-1"], + ])( + "keeps %s when only the region changes", + async (_label, oldRegion, serverId) => { + currentMatch = { + ...currentMatch, + server: { + id: serverId, + is_dedicated: serverId === "dedicated-1", + reserved_by_match_id: "match-1", + game_server_node_id: null, + }, + }; + + await controller.match_events({ + op: "UPDATE", + old: row({ region: oldRegion, server_id: serverId }), + new: row({ region: "USE", server_id: serverId }), + } as any); + + expect(serverClears()).toEqual([]); + expect(matchAssistant.stopOnDemandServer).not.toHaveBeenCalled(); + expect(matchAssistant.releaseOnDemandServer).not.toHaveBeenCalled(); + }, + ); + + describe("a practice match", () => { + it("stops the server even when ending the session throws", async () => { + utilityPractice.markEndedForMatch.mockRejectedValue( + new Error("session update failed"), + ); + + await controller + .match_events({ + op: "UPDATE", + old: row({ source: "practice", status: "Live" }), + new: row({ source: "practice", status: "Canceled" }), + } as any) + .catch((): void => undefined); + + expect(stopJobs()).toHaveLength(1); + expect(stopJobs()[0][2]?.attempts).toBeGreaterThan(1); + }); + + it("does not stop again when its own server_id write re-fires the event", async () => { + await controller.match_events({ + op: "UPDATE", + old: row({ source: "practice", status: "Canceled" }), + new: row({ source: "practice", status: "Canceled", server_id: null }), + } as any); + + expect(stopJobs()).toHaveLength(0); + }); + + it("stops the server of a practice match that is deleted", async () => { + await controller.match_events({ + op: "DELETE", + old: row({ source: "practice", status: "Live" }), + new: {}, + } as any); + + expect(stopJobs()).toHaveLength(1); + }); + }); +}); diff --git a/src/matches/matches.controller.ts b/src/matches/matches.controller.ts index 85fe3f18c..598f44579 100644 --- a/src/matches/matches.controller.ts +++ b/src/matches/matches.controller.ts @@ -684,6 +684,13 @@ export class MatchesController { return; } + const endedServerId = MatchesController.endedMatchServerId(data); + + // First: any later side effect can throw, and nothing else would stop the server. + if (endedServerId) { + await this.stopEndedMatchServer(data, matchId, endedServerId); + } + if ( data.op === "UPDATE" && data.new.status === "WaitingForServer" && @@ -814,61 +821,25 @@ export class MatchesController { matchId, }); - const serverId = data.new.server_id; - - if (!serverId) { + if (!endedServerId) { return; } - const { servers_by_pk: server } = await this.hasura.query({ - servers_by_pk: { - __args: { - id: serverId, - }, - is_dedicated: true, - }, - }); - - const { match_options_by_pk: matchOptions } = await this.hasura.query({ - match_options_by_pk: { - __args: { - id: data.new.match_options_id, - }, - tv_delay: true, - }, - }); - - let delay = matchOptions?.tv_delay || 1; - - if (status === "Canceled" || data.op === "DELETE") { - delay = 0; - } - - this.logger.log( - `[${matchId}] adding stop / restart server job in ${delay} seconds`, - ); - - if (!server.is_dedicated) { - await this.scheduledMatchesQueue.add( - StopOnDemandServer.name, - { matchId }, - delay ? { delay: delay * 1000 } : undefined, - ); - } - - await this.hasura.mutation({ - update_matches_by_pk: { - __args: { - pk_columns: { - id: data.new.id || data.old.id, - }, - _set: { - server_id: null, + if (data.op !== "DELETE") { + await this.hasura.mutation({ + update_matches_by_pk: { + __args: { + pk_columns: { + id: matchId, + }, + _set: { + server_id: null, + }, }, + __typename: true, }, - __typename: true, - }, - }); + }); + } await this.handleGpuFreed(); @@ -883,7 +854,9 @@ export class MatchesController { data.old.region !== data.new.region ) { try { - await this.matchAssistant.stopOnDemandServer(matchId); + if (await this.stopReplacedServer(data, matchId)) { + return; + } } catch (error) { this.logger.error( `[${matchId}] unable to stop on demand server`, @@ -950,6 +923,161 @@ export class MatchesController { await this.discordMatchOverview.updateMatchOverview(matchId); } + // An UPDATE reads only `new`: the server_id-nulling write re-fires with the server still in `old`. + private static endedMatchServerId( + data: HasuraEventData, + ): string | null { + if (data.op === "DELETE") { + return (data.old.server_id as string) ?? null; + } + + if (MatchesController.TERMINAL_STATUSES.includes(data.new.status)) { + return (data.new.server_id as string) ?? null; + } + + return null; + } + + private static stopOnDemandServerJobOptions(delaySeconds = 0) { + return { + ...(delaySeconds ? { delay: delaySeconds * 1000 } : {}), + attempts: 5, + backoff: { + type: "exponential", + delay: 10 * 1000, + }, + }; + } + + private async stopEndedMatchServer( + data: HasuraEventData, + matchId: string, + serverId: string, + ) { + const { servers_by_pk: server } = await this.hasura.query({ + servers_by_pk: { + __args: { + id: serverId, + }, + is_dedicated: true, + }, + }); + + if (server?.is_dedicated) { + return; + } + + let delay = 0; + + if (data.op !== "DELETE" && data.new.status !== "Canceled") { + const { match_options_by_pk: matchOptions } = await this.hasura.query({ + match_options_by_pk: { + __args: { + id: data.new.match_options_id, + }, + tv_delay: true, + }, + }); + + delay = matchOptions?.tv_delay || 1; + } + + this.logger.log( + `[${matchId}] adding stop / restart server job in ${delay} seconds`, + ); + + await this.scheduledMatchesQueue.add( + StopOnDemandServer.name, + { matchId }, + MatchesController.stopOnDemandServerJobOptions(delay), + ); + } + + // True when it cleared server_id instead: that write's own event stops the Job and reassigns. + private async stopReplacedServer( + data: HasuraEventData, + matchId: string, + ): Promise { + const oldServerId = data.old.server_id as string | undefined; + const newServerId = data.new.server_id as string | undefined; + + // No old server: a Job under this match's name is an assignment still in flight. + if (!oldServerId) { + return false; + } + + if (!newServerId) { + await this.matchAssistant.stopOnDemandServer(matchId); + return false; + } + + const { servers_by_pk: server } = await this.hasura.query({ + servers_by_pk: { + __args: { + id: newServerId, + }, + is_dedicated: true, + reserved_by_match_id: true, + game_server_node: { + region: true, + }, + }, + }); + + if (newServerId === oldServerId) { + if ( + !server || + server.is_dedicated || + !data.new.region || + server.game_server_node?.region === data.new.region + ) { + return false; + } + + return await this.clearMatchServer(matchId, oldServerId); + } + + if (server && !server.is_dedicated) { + // assignOnDemandServer reserves the row before writing server_id, and the Job is named per match. + if (server.reserved_by_match_id === matchId) { + await this.matchAssistant.releaseOnDemandServer(matchId, oldServerId); + return false; + } + + // Picked by hand, so nothing booted a Job for it. + return await this.clearMatchServer(matchId, newServerId); + } + + await this.matchAssistant.stopOnDemandServer(matchId, { + serverId: oldServerId, + }); + + return false; + } + + private async clearMatchServer(matchId: string, serverId: string) { + const { update_matches } = await this.hasura.mutation({ + update_matches: { + __args: { + where: { + id: { + _eq: matchId, + }, + server_id: { + _eq: serverId, + }, + }, + _set: { + server_id: null, + }, + }, + affected_rows: true, + }, + }); + + return (update_matches?.affected_rows ?? 0) > 0; + } + private async utilityPracticeMatchEvents( data: HasuraEventData, matchId: string, @@ -960,20 +1088,22 @@ export class MatchesController { data.op === "DELETE" || MatchesController.TERMINAL_STATUSES.includes(status) ) { - await this.utilityPractice.markEndedForMatch(matchId); + const serverId = MatchesController.endedMatchServerId(data); + + if (serverId) { + await this.scheduledMatchesQueue.add( + StopOnDemandServer.name, + { matchId }, + MatchesController.stopOnDemandServerJobOptions(), + ); + } - const serverId = (data.new.server_id ?? data.old.server_id) as - | string - | null; + await this.utilityPractice.markEndedForMatch(matchId); if (!serverId) { return; } - await this.scheduledMatchesQueue.add(StopOnDemandServer.name, { - matchId, - }); - if (data.op !== "DELETE") { await this.hasura.mutation({ update_matches_by_pk: { diff --git a/src/matches/matches.module.ts b/src/matches/matches.module.ts index 286563bc9..d4174b536 100644 --- a/src/matches/matches.module.ts +++ b/src/matches/matches.module.ts @@ -64,6 +64,7 @@ import { BackfillSeasonElo } from "./jobs/BackfillSeasonElo"; import { SeasonEloBackfillService } from "./season-elo-backfill.service"; import { PostgresService } from "src/postgres/postgres.service"; import { StopOnDemandServer } from "./jobs/StopOnDemandServer"; +import { ReconcileOnDemandServerJobs } from "./jobs/ReconcileOnDemandServerJobs"; import { MatchRelayController } from "./match-relay/match-relay.controller"; import { MatchRelayService } from "./match-relay/match-relay.service"; import { MatchRelayAuthMiddleware } from "./match-relay/match-relay-auth-middleware"; @@ -187,6 +188,7 @@ import { CameraMonitorService } from "./camera/camera-monitor.service"; CheckForScheduledMatches, RemoveCancelledMatches, StopOnDemandServer, + ReconcileOnDemandServerJobs, CancelInvalidTournaments, CleanAbandonedMatches, ReapIdleDemoSessions, @@ -374,6 +376,16 @@ export class MatchesModule implements NestModule { }, ); + void scheduleMatchQueue.add( + ReconcileOnDemandServerJobs.name, + {}, + { + repeat: { + pattern: "* * * * *", + }, + }, + ); + void scheduleMatchQueue.add( PollMediaMtxViewers.name, {}, diff --git a/test/awards.spec.ts b/test/awards.spec.ts index bd942e81b..3da7cd7d9 100644 --- a/test/awards.spec.ts +++ b/test/awards.spec.ts @@ -75,6 +75,27 @@ describe("awards (SQL-driven)", () => { return award.id; }; + const user = (steam_id: string, role: string) => + ({ steam_id, role }) as never; + + const awardsController = ( + getSetting: (name: string) => Promise = async () => "administrator", + ) => + new AwardsController( + new AwardsService( + new Logger("AwardsActionTest"), + // No test touches artwork, so a stub that fails loudly is enough. + { + put: jest.fn(), + remove: jest.fn(), + has: jest.fn().mockResolvedValue(false), + } as never, + postgres, + { notifyPlayers: jest.fn() } as never, + ), + { getSetting } as never, + ); + describe("catalog", () => { it("ships the four tournament system awards", async () => { const rows = await postgres.query< @@ -450,26 +471,300 @@ describe("awards (SQL-driven)", () => { expect(Number(counts.calculated)).toBeGreaterThan(0); expect(Number(counts.manual)).toBe(1); }); + }); + + describe("medal leaderboard", () => { + type Medals = { mvp: number; gold: number; silver: number; bronze: number }; + + const daysAgo = (days: number) => + new Date(Date.now() - days * 86_400_000).toISOString(); + + const medalBoard = async ( + filters: { + windowDays?: number; + matchType?: string; + seasonId?: string; + } = {}, + ) => { + const rows = await postgres.query< + Array<{ + player_steam_id: string; + value: number; + secondary_value: number; + tertiary_value: number; + matches_played: number; + }> + >("SELECT * FROM get_leaderboard('awards', $1, $2, false, NULL, $3)", [ + filters.windowDays ?? 0, + filters.matchType ?? null, + filters.seasonId ?? null, + ]); + return new Map( + rows.map((row) => [ + String(row.player_steam_id), + { + mvp: Number(row.matches_played), + gold: Number(row.value), + silver: Number(row.secondary_value), + bronze: Number(row.tertiary_value), + }, + ]), + ); + }; + + const grant = async (input: { + award_id: string; + player_steam_id?: string; + team_id?: string; + season_id?: string; + }) => + awardsController().grantAward({ + ...input, + user: user(await fx.player(), "administrator"), + }); + + it("counts hand-granted awards by their award tier", async () => { + const steam = await fx.player(); + + for (const tier of ["mvp", "gold", "silver", "bronze"]) { + await grant({ + award_id: await createAward(`Hand ${tier}`, tier), + player_steam_id: steam, + }); + } + + expect((await medalBoard()).get(steam)).toEqual({ + mvp: 1, + gold: 1, + silver: 1, + bronze: 1, + }); + }); + + it("counts a team grant once per rostered player and never for the team", async () => { + const team = await fx.team(2); + const roster = await postgres.query>( + "SELECT player_steam_id FROM team_roster WHERE team_id = $1 AND role <> 'Invite'", + [team.id], + ); + expect(roster).toHaveLength(3); - it("keeps hand-granted awards out of the medal leaderboard", async () => { + await grant({ + award_id: await createAward("Squad Gold", "gold"), + team_id: team.id, + }); + + const board = await medalBoard(); + expect([...board.keys()].sort()).toEqual( + roster.map((r) => String(r.player_steam_id)).sort(), + ); + for (const medals of board.values()) { + expect(medals).toEqual({ mvp: 0, gold: 1, silver: 0, bronze: 0 }); + } + }); + + it("windows hand grants by the date they were granted", async () => { + const [recent, old] = await fx.players(2); + const awardId = await createAward("Community Gold", "gold"); + + await grant({ award_id: awardId, player_steam_id: recent }); + const stale = await grant({ award_id: awardId, player_steam_id: old }); + await postgres.query( + "UPDATE award_recipients SET created_at = $2 WHERE id = $1", + [stale.id, daysAgo(90)], + ); + + expect([...(await medalBoard({ windowDays: 30 })).keys()]).toEqual([ + recent, + ]); + expect([...(await medalBoard()).keys()].sort()).toEqual( + [recent, old].sort(), + ); + }); + + it("counts a season-scoped hand grant on its own season, not the one it was granted during", async () => { + const past = await fx.season(daysAgo(60), daysAgo(31)); + const current = await fx.season(daysAgo(30)); + const steam = await fx.player(); + + await grant({ + award_id: await createAward("Season Gold", "gold"), + player_steam_id: steam, + season_id: past, + }); + + expect((await medalBoard({ seasonId: past })).get(steam)).toEqual({ + mvp: 0, + gold: 1, + silver: 0, + bronze: 0, + }); + expect((await medalBoard({ seasonId: current })).has(steam)).toBe(false); + }); + + it("counts an unscoped hand grant on the season it was granted in", async () => { + const past = await fx.season(daysAgo(60), daysAgo(31)); + const current = await fx.season(daysAgo(30)); + const steam = await fx.player(); + + const granted = await grant({ + award_id: await createAward("Community Gold", "gold"), + player_steam_id: steam, + }); + await postgres.query( + "UPDATE award_recipients SET created_at = $2 WHERE id = $1", + [granted.id, daysAgo(45)], + ); + + expect((await medalBoard({ seasonId: past })).has(steam)).toBe(true); + expect((await medalBoard({ seasonId: current })).has(steam)).toBe(false); + }); + + it("leaves special-tier awards off the board", async () => { + const [holder, specialOnly] = await fx.players(2); + + await grant({ + award_id: await createAward("Community Gold", "gold"), + player_steam_id: holder, + }); + await grant({ + award_id: await createAward("Good Sport"), + player_steam_id: holder, + }); + await grant({ + award_id: await createAward("Fan Favourite"), + player_steam_id: specialOnly, + }); + + const board = await medalBoard(); + expect([...board.keys()]).toEqual([holder]); + expect(board.get(holder)).toEqual({ + mvp: 0, + gold: 1, + silver: 0, + bronze: 0, + }); + }); + + it("drops unscoped hand grants once a match type is picked", async () => { + const steam = await fx.player(); + + await grant({ + award_id: await createAward("Community Gold", "gold"), + player_steam_id: steam, + }); + + expect((await medalBoard()).has(steam)).toBe(true); + expect((await medalBoard({ matchType: "Competitive" })).has(steam)).toBe( + false, + ); + }); + + it("counts an overridden tournament placement by placement, not by the award's tier", async () => { const t = await playedOutCup(); - const awardId = await createAward("Community MVP"); - const steamId = await fx.player(); + const specialId = await createAward("Summer Cup Champion"); await postgres.query( - `INSERT INTO award_recipients (award_id, player_steam_id, source) - VALUES ($1, $2, 'manual')`, - [awardId, steamId], + `INSERT INTO tournament_awards (tournament_id, placement, award_id) + VALUES ($1, 1, $2)`, + [t.id, specialId], ); + await postgres.query("SELECT recalculate_tournament_awards($1)", [t.id]); - const rows = await postgres.query>( - "SELECT player_steam_id FROM get_leaderboard('awards', 0)", + const champions = await postgres.query< + Array<{ player_steam_id: string; award_id: string }> + >( + `SELECT player_steam_id, award_id FROM award_recipients + WHERE tournament_id = $1 AND placement = 1 + AND player_steam_id IS NOT NULL`, + [t.id], ); - expect(rows.length).toBeGreaterThan(0); - expect(rows.map((r) => String(r.player_steam_id))).not.toContain( - String(steamId), + expect(champions.length).toBeGreaterThan(0); + expect(new Set(champions.map((c) => c.award_id))).toEqual( + new Set([specialId]), ); - void t; + + const board = await medalBoard(); + for (const champion of champions) { + expect(board.get(String(champion.player_steam_id))?.gold).toBe(1); + } + }); + + it("counts a tier once per tournament however often it was granted there", async () => { + const t = await playedOutCup(); + const [{ player_steam_id: steam }] = await postgres.query< + Array<{ player_steam_id: string }> + >( + `SELECT r.player_steam_id::text FROM tournament_team_roster r + WHERE r.tournament_id = $1 + AND NOT EXISTS ( + SELECT 1 FROM award_recipients ar + WHERE ar.tournament_id = r.tournament_id + AND ar.player_steam_id = r.player_steam_id + AND ar.placement IN (0, 1) + ) + LIMIT 1`, + [t.id], + ); + const before = (await medalBoard()).get(steam) ?? { + mvp: 0, + gold: 0, + silver: 0, + bronze: 0, + }; + const gold = await systemAward("tournament_gold"); + + for (let i = 0; i < 3; i++) { + await awardsController().grantAward({ + award_id: gold, + player_steam_id: steam, + tournament_id: t.id, + user: user(await fx.player(), "administrator"), + }); + } + await grant({ award_id: gold, player_steam_id: steam }); + + expect((await medalBoard()).get(steam)).toEqual({ + ...before, + gold: before.gold + 2, + }); + }); + + it("ranks a player where the board's default order places them", async () => { + const [mvp, doubleGold, goldSilver, gold] = await fx.players(4); + const mvpAward = await createAward("Hand MVP", "mvp"); + const goldAward = await createAward("Hand Gold", "gold", true); + const silverAward = await createAward("Hand Silver", "silver"); + + await grant({ award_id: mvpAward, player_steam_id: mvp }); + await grant({ award_id: goldAward, player_steam_id: doubleGold }); + await grant({ award_id: goldAward, player_steam_id: doubleGold }); + await grant({ award_id: goldAward, player_steam_id: goldSilver }); + await grant({ award_id: silverAward, player_steam_id: goldSilver }); + await grant({ award_id: goldAward, player_steam_id: gold }); + + // The awards order_by in web/pages/leaderboard.vue. + const board = await postgres.query>( + `SELECT player_steam_id FROM get_leaderboard('awards', 0) + ORDER BY matches_played DESC, value DESC, + secondary_value DESC, tertiary_value DESC`, + ); + expect(board.map((r) => r.player_steam_id)).toEqual([ + mvp, + doubleGold, + goldSilver, + gold, + ]); + + const ranks: number[] = []; + for (const steam of [mvp, doubleGold, goldSilver, gold]) { + const [row] = await postgres.query>( + "SELECT rank FROM get_player_leaderboard_rank('awards', 0, $1)", + [steam], + ); + ranks.push(Number(row?.rank)); + } + expect(ranks).toEqual([1, 2, 3, 4]); }); }); @@ -736,27 +1031,11 @@ describe("awards (SQL-driven)", () => { createFloor = "administrator"; grantFloor = "administrator"; - const service = new AwardsService( - new Logger("AwardsActionTest"), - // No test touches artwork, so a stub that fails loudly is enough. - { - put: jest.fn(), - remove: jest.fn(), - has: jest.fn().mockResolvedValue(false), - } as never, - postgres, - { notifyPlayers: jest.fn() } as never, + controller = awardsController(async (name: string) => + name.includes("create") ? createFloor : grantFloor, ); - - controller = new AwardsController(service, { - getSetting: async (name: string) => - name.includes("create") ? createFloor : grantFloor, - } as never); }); - const user = (steam_id: string, role: string) => - ({ steam_id, role }) as never; - const rosterOf = async (tournamentId: string) => { const [row] = await postgres.query< Array<{ player_steam_id: string; team_id: string }> @@ -886,6 +1165,42 @@ describe("awards (SQL-driven)", () => { ).rejects.toThrow("Not the tournament organizer"); }); + it("refuses an organizer below the floor granting to themselves or their own team", async () => { + const t = await playedOutCup(); + const awardId = await systemAward("tournament_gold"); + const { player_steam_id, team_id } = await rosterOf(t.id); + const teammate = String(player_steam_id); + await postgres.query( + "UPDATE tournaments SET organizer_steam_id = $2 WHERE id = $1", + [t.id, teammate], + ); + + await expect( + controller.grantAward({ + award_id: awardId, + player_steam_id: teammate, + tournament_id: t.id, + user: user(teammate, "user"), + }), + ).rejects.toThrow("cannot grant awards to themselves"); + await expect( + controller.grantAward({ + award_id: awardId, + team_id, + tournament_id: t.id, + user: user(teammate, "user"), + }), + ).rejects.toThrow("cannot grant awards to themselves"); + + const granted = await controller.grantAward({ + award_id: awardId, + player_steam_id: teammate, + tournament_id: t.id, + user: user(teammate, "administrator"), + }); + expect(granted.id).toBeTruthy(); + }); + it("refuses a recipient who never played in the tournament", async () => { const t = await playedOutCup(); const awardId = await createAward("Outsider"); @@ -934,6 +1249,70 @@ describe("awards (SQL-driven)", () => { ); expect(row.season_id).toBe(season); }); + + it("lets a granter hand an award to themselves", async () => { + const granter = await fx.player(); + const awardId = await createAward("Self Nominated"); + + const granted = await controller.grantAward({ + award_id: awardId, + player_steam_id: granter, + user: user(granter, "administrator"), + }); + + const [row] = await postgres.query< + Array<{ player_steam_id: string; awarded_by_steam_id: string }> + >( + `SELECT player_steam_id, awarded_by_steam_id + FROM award_recipients WHERE id = $1`, + [granted.id], + ); + expect(String(row.player_steam_id)).toBe(granter); + expect(String(row.awarded_by_steam_id)).toBe(granter); + }); + + it("stores a team grant on the team itself when no tournament is named", async () => { + const team = await fx.team(2); + const awardId = await createAward("Squad Honour"); + const [{ roster }] = await postgres.query>( + `SELECT count(*) AS roster FROM team_roster + WHERE team_id = $1 AND role <> 'Invite'`, + [team.id], + ); + + await controller.grantAward({ + award_id: awardId, + team_id: team.id, + user: user(await fx.player(), "administrator"), + }); + + const teamRows = await postgres.query< + Array<{ + player_steam_id: string | null; + tournament_id: string | null; + tournament_team_id: string | null; + }> + >( + `SELECT player_steam_id, tournament_id, tournament_team_id + FROM award_recipients + WHERE award_id = $1 AND team_id = $2`, + [awardId, team.id], + ); + expect(teamRows).toEqual([ + { + player_steam_id: null, + tournament_id: null, + tournament_team_id: null, + }, + ]); + + const [{ players }] = await postgres.query>( + `SELECT count(*) AS players FROM award_recipients + WHERE award_id = $1 AND player_steam_id IS NOT NULL`, + [awardId], + ); + expect(Number(players)).toBe(Number(roster)); + }); }); describe("revokeAward", () => { diff --git a/test/match-scoring.spec.ts b/test/match-scoring.spec.ts index 0cfea283d..996423242 100644 --- a/test/match-scoring.spec.ts +++ b/test/match-scoring.spec.ts @@ -339,6 +339,109 @@ describe("match scoring from rounds (SQL-driven)", () => { expect(paused.cancels_at).toBeNull(); }); + describe("the end-of-map window (WaitingForTV / UploadingDemo)", () => { + const setMapStatus = (mapId: string, status: string) => + postgres.query("UPDATE match_maps SET status = $2 WHERE id = $1", [ + mapId, + status, + ]); + + const cancelsAt = async (matchId: string) => { + const [row] = await postgres.query>( + "SELECT cancels_at FROM matches WHERE id = $1", + [matchId], + ); + return row.cancels_at; + }; + + it("pushes the live-match timeout past the tv_delay the broadcast still needs", async () => { + const match = await createLiveMatch(1); + await postgres.query( + "UPDATE match_options SET live_match_timeout = 30, tv_delay = 115 WHERE id = $1", + [match.options_id], + ); + await setMapStatus(match.mapIds[0], "Live"); + + for (const status of ["WaitingForTV", "UploadingDemo"]) { + await postgres.query( + "UPDATE matches SET cancels_at = NOW() - interval '1 minute' WHERE id = $1", + [match.id], + ); + + await setMapStatus(match.mapIds[0], status); + + const deadline = await cancelsAt(match.id); + expect(deadline).not.toBeNull(); + const minutesOut = (deadline!.getTime() - Date.now()) / 60_000; + // live_match_timeout (30m) + tv_delay (115s) + expect(minutesOut).toBeGreaterThan(31.5); + expect(minutesOut).toBeLessThan(32.5); + } + }); + + it("does not re-arm a match that already ended", async () => { + const match = await createLiveMatch(1); + await setMapStatus(match.mapIds[0], "Live"); + await postgres.query( + "UPDATE matches SET status = 'Forfeit' WHERE id = $1", + [match.id], + ); + + await setMapStatus(match.mapIds[0], "WaitingForTV"); + + expect(await cancelsAt(match.id)).toBeNull(); + }); + + it("finishing a map with more of the series to play gives the next map the warmup window", async () => { + const match = await createLiveMatch(3); + await postgres.query( + "UPDATE match_options SET auto_cancel_duration = 20 WHERE id = $1", + [match.options_id], + ); + await setMapStatus(match.mapIds[0], "Live"); + await setMapStatus(match.mapIds[0], "WaitingForTV"); + await postgres.query( + "UPDATE matches SET cancels_at = NOW() - interval '1 minute' WHERE id = $1", + [match.id], + ); + + await recordScore(match.mapIds[0], 13, 7); + await finishMap(match.mapIds[0]); + + expect((await matchRow(match.id)).status).toBe("Live"); + const deadline = await cancelsAt(match.id); + expect(deadline).not.toBeNull(); + const minutesOut = (deadline!.getTime() - Date.now()) / 60_000; + expect(minutesOut).toBeGreaterThan(19); + expect(minutesOut).toBeLessThan(21); + + await setMapStatus(match.mapIds[1], "Live"); + await recordScore(match.mapIds[1], 13, 5); + await finishMap(match.mapIds[1]); + + expect((await matchRow(match.id)).status).toBe("Finished"); + expect(await cancelsAt(match.id)).toBeNull(); + }); + + it("finishing a map left in WaitingForTV finishes the match for its winner", async () => { + const match = await createLiveMatch(1); + await recordScore(match.mapIds[0], 13, 7); + await setMapStatus(match.mapIds[0], "Live"); + await postgres.query( + "UPDATE match_maps SET status = 'WaitingForTV', winning_lineup_id = $2 WHERE id = $1", + [match.mapIds[0], match.lineup_1_id], + ); + + await finishMap(match.mapIds[0]); + + const after = await matchRow(match.id); + expect(after.status).toBe("Finished"); + expect(after.winning_lineup_id).toBe(match.lineup_1_id); + expect(after.ended_at).not.toBeNull(); + expect(await cancelsAt(match.id)).toBeNull(); + }); + }); + it("finishing the map stamps the map's ended_at", async () => { const match = await createLiveMatch(1); await recordScore(match.mapIds[0], 13, 7); diff --git a/test/tournament-group-stages.spec.ts b/test/tournament-group-stages.spec.ts new file mode 100644 index 000000000..99987fc7c --- /dev/null +++ b/test/tournament-group-stages.spec.ts @@ -0,0 +1,424 @@ +import { PostgresService } from "./../src/postgres/postgres.service"; +import { Fixtures } from "./utils/fixtures"; +import { BracketRow, TournamentFixtures } from "./utils/tournament-fixtures"; +import { + bootMigratedDb, + seedRegionWithServer, + SqlTestDb, +} from "./utils/sql-test-db"; + +describe("tournament stages: grouped RoundRobin advancement (SQL-driven)", () => { + let db: SqlTestDb; + let postgres: PostgresService; + let fx: Fixtures; + let tfx: TournamentFixtures; + + beforeAll(async () => { + db = await bootMigratedDb("TournamentGroupStagesTest"); + postgres = db.postgres; + fx = new Fixtures(postgres, 76561199320000000n); + tfx = new TournamentFixtures(postgres, fx); + await seedRegionWithServer(postgres, "TestA"); + }, 600_000); + + afterAll(async () => { + await db?.stop(); + }); + + beforeEach(async () => { + await postgres.query("DELETE FROM matches"); + await postgres.query("DELETE FROM tournaments"); + await postgres.query("DELETE FROM match_options"); + await postgres.query("DELETE FROM teams"); + await postgres.query("DELETE FROM players"); + }); + + type Standing = { + tournament_team_id: string; + group_number: number; + rank: number; + wins: number; + losses: number; + }; + + const stageRow = async (stageId: string) => + ( + await postgres.query>( + "SELECT min_teams, max_teams FROM tournament_stages WHERE id = $1", + [stageId], + ) + )[0]; + + const insertStage = ( + tournamentId: string, + type: string, + order: number, + minTeams: number, + maxTeams: number, + groups = 1, + ) => + postgres.query( + `INSERT INTO tournament_stages (tournament_id, type, "order", min_teams, max_teams, groups) + VALUES ($1, $2, $3, $4, $5, $6)`, + [tournamentId, type, order, minTeams, maxTeams, groups], + ); + + const groupSizes = async (stageId: string) => + ( + await postgres.query>( + `SELECT count(DISTINCT team)::int AS teams FROM ( + SELECT "group", tournament_team_id_1 AS team FROM tournament_brackets WHERE tournament_stage_id = $1 + UNION ALL + SELECT "group", tournament_team_id_2 FROM tournament_brackets WHERE tournament_stage_id = $1 + ) slots + WHERE team IS NOT NULL + GROUP BY "group" ORDER BY "group"`, + [stageId], + ) + ).map((row) => Number(row.teams)); + + const standings = (stageId: string) => + postgres.query>( + `SELECT tournament_team_id, group_number, rank, wins, losses + FROM v_team_stage_results WHERE tournament_stage_id = $1 + ORDER BY group_number, rank`, + [stageId], + ); + + const teamAt = (rows: Array, group: number, rank: number) => + rows.find( + (row) => Number(row.group_number) === group && Number(row.rank) === rank, + )!.tournament_team_id; + + const teamsInRound = async (stageId: string, round: number) => + (await tfx.getBrackets(stageId)) + .filter((bracket) => bracket.round === round) + .flatMap((bracket) => [ + bracket.tournament_team_id_1, + bracket.tournament_team_id_2, + ]) + .filter((team): team is string => team !== null) + .sort(); + + // Outside `strictGroups` slots 1-3 beat each other in a cycle, so a runner-up only goes 1-2. + const scriptedResults = async ( + tournamentId: string, + groups: number, + strictGroups: Set, + ) => { + const seeds = new Map( + ( + await postgres.query>( + "SELECT id, seed FROM tournament_teams WHERE tournament_id = $1", + [tournamentId], + ) + ).map((row) => [row.id, Number(row.seed)]), + ); + return (bracket: BracketRow): "lineup_1_id" | "lineup_2_id" => { + const group = Number(bracket.group); + const slot = (team: string) => (seeds.get(team)! - group) / groups; + const a = slot(bracket.tournament_team_id_1!); + const b = slot(bracket.tournament_team_id_2!); + let firstWins: boolean; + if (a === 0 || b === 0) { + firstWins = a === 0; + } else if (strictGroups.has(group)) { + firstWins = a < b; + } else { + firstWins = (b - a + 3) % 3 === 1; + } + return firstWins ? "lineup_1_id" : "lineup_2_id"; + }; + }; + + describe("#507 advancement is sized by the next stage, not by group size", () => { + it("resizing a 4-group round robin to 20 teams leaves the 8-team playoff alone", async () => { + const t = await tfx.createTournament([ + { type: "RoundRobin", order: 1, minTeams: 16, maxTeams: 16, groups: 4 }, + { type: "DoubleElimination", order: 2, minTeams: 8, maxTeams: 8 }, + ]); + + await expect( + postgres.query( + "UPDATE tournament_stages SET min_teams = 20, max_teams = 20 WHERE id = $1", + [t.stageIds[0]], + ), + ).resolves.toBeDefined(); + + expect(await stageRow(t.stageIds[1])).toEqual({ + min_teams: 8, + max_teams: 8, + }); + }); + + it("a later stage after a round robin doesn't inflate the round robin", async () => { + const t = await tfx.createTournament([ + { type: "RoundRobin", order: 1, minTeams: 4, maxTeams: 8 }, + { type: "SingleElimination", order: 2, minTeams: 4, maxTeams: 4 }, + ]); + + expect(await stageRow(t.stageIds[0])).toEqual({ + min_teams: 4, + max_teams: 8, + }); + }); + + it("a round robin can't advance more teams than it holds", async () => { + const grouped = await tfx.createTournament([ + { type: "RoundRobin", order: 1, minTeams: 8, maxTeams: 8, groups: 2 }, + ]); + + await expect( + insertStage(grouped.id, "SingleElimination", 2, 16, 16), + ).rejects.toThrow(/holds at most 8 teams/i); + expect(await stageRow(grouped.stageIds[0])).toEqual({ + min_teams: 8, + max_teams: 8, + }); + + const single = await tfx.createTournament([ + { type: "RoundRobin", order: 1, minTeams: 8, maxTeams: 8 }, + { type: "SingleElimination", order: 2, minTeams: 8, maxTeams: 8 }, + ]); + + await expect( + postgres.query( + "UPDATE tournament_stages SET min_teams = 6, max_teams = 6 WHERE id = $1", + [single.stageIds[0]], + ), + ).rejects.toThrow(/holds at most 6 teams/i); + }); + + it("six groups feeding eight: every group winner plus the two best runners-up", async () => { + const t = await tfx.launch( + [ + { + type: "RoundRobin", + order: 1, + minTeams: 24, + maxTeams: 24, + groups: 6, + }, + { type: "SingleElimination", order: 2, minTeams: 8, maxTeams: 8 }, + ], + 24, + ); + const [roundRobin, playoff] = t.stageIds; + + await tfx.playStage( + roundRobin, + await scriptedResults(t.id, 6, new Set([5, 6])), + ); + + const table = await standings(roundRobin); + expect( + table + .filter((row) => Number(row.rank) === 2) + .map((row) => `${row.group_number}:${row.wins}-${row.losses}`), + ).toEqual(["1:1-2", "2:1-2", "3:1-2", "4:1-2", "5:2-1", "6:2-1"]); + + expect(await teamsInRound(playoff, 1)).toEqual( + [ + ...[1, 2, 3, 4, 5, 6].map((group) => teamAt(table, group, 1)), + teamAt(table, 5, 2), + teamAt(table, 6, 2), + ].sort(), + ); + + await tfx.playStage(playoff); + expect(await tfx.tournamentStatus(t.id)).toBe("Finished"); + }, 180_000); + }); + + describe("a group left short of its guaranteed places", () => { + it("gives the empty seed to the best wildcard instead of a bye", async () => { + const t = await tfx.launch( + [ + { + type: "RoundRobin", + order: 1, + minTeams: 10, + maxTeams: 10, + groups: 2, + }, + { type: "SingleElimination", order: 2, minTeams: 8, maxTeams: 8 }, + ], + 10, + ); + const [roundRobin] = t.stageIds; + + await tfx.playStage( + roundRobin, + await scriptedResults(t.id, 2, new Set([1, 2])), + ); + + const table = await standings(roundRobin); + const disqualified = [teamAt(table, 2, 4), teamAt(table, 2, 5)]; + await postgres.query( + "UPDATE tournament_teams SET eligible_at = NULL WHERE id = ANY($1)", + [disqualified], + ); + + const seeds = await postgres.query< + Array<{ seed: number; tournament_team_id: string }> + >( + "SELECT seed, tournament_team_id FROM get_stage_qualifier_seeds($1, 8) ORDER BY seed", + [roundRobin], + ); + + expect(seeds.map((row) => Number(row.seed))).toEqual([ + 1, 2, 3, 4, 5, 6, 7, 8, + ]); + expect(seeds.map((row) => row.tournament_team_id).sort()).toEqual( + [ + ...[1, 2, 3, 4, 5].map((rank) => teamAt(table, 1, rank)), + ...[1, 2, 3].map((rank) => teamAt(table, 2, rank)), + ].sort(), + ); + }, 120_000); + }); + + describe("#617 any number of teams can advance", () => { + it("an 8-team round robin (min 4) feeds a 2-seat playoff, the shape start_league_season builds", async () => { + const t = await tfx.createTournament([ + { type: "RoundRobin", order: 1, minTeams: 4, maxTeams: 8 }, + ]); + + await expect( + insertStage(t.id, "SingleElimination", 2, 2, 4), + ).resolves.toBeDefined(); + }); + + it("a 10-team no-elimination Swiss (min 4) feeds a 2-seat playoff, the other shape start_league_season builds", async () => { + const t = await tfx.createTournament([]); + const [swiss] = await postgres.query>( + `INSERT INTO tournament_stages (tournament_id, type, "order", min_teams, max_teams, groups, max_rounds, swiss_no_elimination) + VALUES ($1, 'Swiss', 1, 4, 10, 1, 4, true) RETURNING id`, + [t.id], + ); + + await expect( + insertStage(t.id, "SingleElimination", 2, 2, 4), + ).resolves.toBeDefined(); + await expect( + postgres.query( + "UPDATE tournament_stages SET min_teams = 3, max_teams = 3 WHERE id = $1", + [swiss.id], + ), + ).rejects.toThrow(/holds at most 3 teams/i); + }); + + it("two group winners advance into a single final", async () => { + const t = await tfx.launch( + [ + { type: "RoundRobin", order: 1, minTeams: 8, maxTeams: 8, groups: 2 }, + { type: "SingleElimination", order: 2, minTeams: 2, maxTeams: 2 }, + ], + 8, + ); + const [roundRobin, final] = t.stageIds; + + expect((await tfx.getBrackets(final)).length).toBe(1); + + await tfx.playStage(roundRobin); + const table = await standings(roundRobin); + + const [bracket] = await tfx.getBrackets(final); + expect( + [bracket.tournament_team_id_1, bracket.tournament_team_id_2].sort(), + ).toEqual([teamAt(table, 1, 1), teamAt(table, 2, 1)].sort()); + expect(bracket.match_id).not.toBeNull(); + + await tfx.playStage(final); + expect(await tfx.tournamentStatus(t.id)).toBe("Finished"); + }, 120_000); + }); + + describe("#618 a group needs three teams, not four", () => { + it("a 4-group round robin takes 15 teams and plays groups of 4, 4, 4 and 3", async () => { + const t = await tfx.launch( + [ + { + type: "RoundRobin", + order: 1, + minTeams: 15, + maxTeams: 16, + groups: 4, + }, + { type: "SingleElimination", order: 2, minTeams: 4, maxTeams: 4 }, + ], + 15, + ); + const [roundRobin, playoff] = t.stageIds; + + expect(await groupSizes(roundRobin)).toEqual([4, 4, 4, 3]); + + await tfx.playStage(roundRobin); + expect( + (await tfx.getBrackets(roundRobin)).every( + (bracket) => bracket.finished, + ), + ).toBe(true); + + const table = await standings(roundRobin); + expect(await teamsInRound(playoff, 1)).toEqual( + [1, 2, 3, 4].map((group) => teamAt(table, group, 1)).sort(), + ); + + await tfx.playStage(playoff); + expect(await tfx.tournamentStatus(t.id)).toBe("Finished"); + }, 120_000); + + it("wildcards compare win rate, so a 3-team group's runner-up isn't punished for playing fewer games", async () => { + const t = await tfx.launch( + [ + { + type: "RoundRobin", + order: 1, + minTeams: 15, + maxTeams: 16, + groups: 4, + }, + { type: "SingleElimination", order: 2, minTeams: 6, maxTeams: 6 }, + ], + 15, + ); + const [roundRobin, playoff] = t.stageIds; + + await tfx.playStage( + roundRobin, + await scriptedResults(t.id, 4, new Set([4])), + ); + + const table = await standings(roundRobin); + expect( + table + .filter((row) => Number(row.rank) === 2) + .map((row) => `${row.group_number}:${row.wins}-${row.losses}`), + ).toEqual(["1:1-2", "2:1-2", "3:1-2", "4:1-1"]); + + // Seeds 1 and 2 of a 6-team bracket take byes straight into round 2. + const field = new Set([ + ...(await teamsInRound(playoff, 1)), + ...(await teamsInRound(playoff, 2)), + ]); + + expect(field.size).toBe(6); + for (const group of [1, 2, 3, 4]) { + expect(field.has(teamAt(table, group, 1))).toBe(true); + } + expect(field.has(teamAt(table, 4, 2))).toBe(true); + }, 120_000); + + it("groups smaller than three teams are still rejected", async () => { + const t = await tfx.createTournament([]); + + await expect( + insertStage(t.id, "RoundRobin", 1, 11, 16, 4), + ).rejects.toThrow(/3 teams per group/i); + await expect( + insertStage(t.id, "RoundRobin", 1, 3, 3, 1), + ).resolves.toBeDefined(); + }); + }); +}); diff --git a/test/tournament-stage-regions.spec.ts b/test/tournament-stage-regions.spec.ts new file mode 100644 index 000000000..b5aacd617 --- /dev/null +++ b/test/tournament-stage-regions.spec.ts @@ -0,0 +1,143 @@ +import { PostgresService } from "./../src/postgres/postgres.service"; +import { Fixtures } from "./utils/fixtures"; +import { TournamentFixtures } from "./utils/tournament-fixtures"; +import { + bootMigratedDb, + runAsUser, + seedRegionWithServer, + SqlTestDb, +} from "./utils/sql-test-db"; + +// A stage can override the tournament's region settings, including picking a +// LAN region. Covers the clone path stage options take on their way into the +// scheduled matches, and the role gate that keeps players off LAN regions. +describe("tournament stage region overrides (SQL-driven)", () => { + let db: SqlTestDb; + let postgres: PostgresService; + let fx: Fixtures; + let tfx: TournamentFixtures; + + const SE4: Array<{ + type: string; + order: number; + minTeams: number; + maxTeams: number; + }> = [{ type: "SingleElimination", order: 1, minTeams: 4, maxTeams: 4 }]; + + beforeAll(async () => { + db = await bootMigratedDb("TournamentStageRegionsTest"); + postgres = db.postgres; + fx = new Fixtures(postgres, 76561199400000000n); + tfx = new TournamentFixtures(postgres, fx); + await seedRegionWithServer(postgres, "TestA", 27015); + // Two Ranked regions keeps tbi_match_options out of its single-region + // branch, which rewrites regions to every enabled server region. + await postgres.query( + `INSERT INTO server_regions (value, description, is_lan) + VALUES ('TestLan', 'TestLan', true) ON CONFLICT (value) DO NOTHING`, + ); + await postgres.query( + `INSERT INTO servers (host, label, rcon_password, port, region, type, is_dedicated, enabled) + VALUES ('127.0.0.1', 'TestLan', $1, 27016, 'TestLan', 'Ranked', true, true)`, + [Buffer.from("password")], + ); + }, 600_000); + + afterAll(async () => { + await db?.stop(); + }); + + beforeEach(async () => { + await postgres.query("DELETE FROM matches"); + await postgres.query("DELETE FROM tournaments"); + await postgres.query("DELETE FROM match_options"); + await postgres.query("DELETE FROM teams"); + await postgres.query("DELETE FROM players"); + }); + + const stageOptions = async ( + stageId: string, + regions: Array, + ): Promise => { + const [options] = await postgres.query>( + `INSERT INTO match_options (mr, best_of, type, map_pool_id, map_veto, region_veto, regions) + SELECT 8, 1, 'Wingman', id, false, false, $1 + FROM map_pools WHERE type = 'Wingman' AND seed = true RETURNING id`, + [regions], + ); + await postgres.query( + `UPDATE tournament_stages SET match_options_id = $1 WHERE id = $2`, + [options.id, stageId], + ); + return options.id; + }; + + const launchWithStageRegions = async (regions: Array) => { + const tournament = await tfx.createTournament(SE4); + await stageOptions(tournament.stageIds[0], regions); + + await tfx.setStatus( + tournament.id, + tournament.organizer, + "RegistrationOpen", + ); + for (let i = 0; i < 4; i++) { + await tfx.registerTeam(tournament.id, await fx.team(1)); + } + await tfx.setStatus( + tournament.id, + tournament.organizer, + "RegistrationClosed", + ); + await tfx.setStatus(tournament.id, tournament.organizer, "Live"); + + return tournament; + }; + + const matchRegions = (stageId: string) => + postgres.query; region_veto: boolean }>>( + `SELECT mo.regions, mo.region_veto + FROM tournament_brackets tb + INNER JOIN matches m ON m.id = tb.match_id + INNER JOIN match_options mo ON mo.id = m.match_options_id + WHERE tb.tournament_stage_id = $1`, + [stageId], + ); + + it("carries a stage's LAN region into every match it schedules", async () => { + const tournament = await launchWithStageRegions(["TestLan"]); + + const matches = await matchRegions(tournament.stageIds[0]); + + expect(matches.length).toBeGreaterThan(0); + for (const match of matches) { + expect(match.regions).toEqual(["TestLan"]); + expect(match.region_veto).toBe(false); + } + }); + + it("leaves the tournament's own region alone when the stage overrides nothing", async () => { + const tournament = await tfx.launch(SE4, 4); + + const matches = await matchRegions(tournament.stageIds[0]); + + expect(matches.length).toBeGreaterThan(0); + for (const match of matches) { + expect(match.regions).toEqual(["TestA"]); + } + }); + + it("refuses a LAN region on stage options written by a player", async () => { + const tournament = await tfx.createTournament(SE4); + + await expect( + runAsUser(postgres, tournament.organizer, "user", (query) => + query( + `INSERT INTO match_options (mr, best_of, type, map_pool_id, map_veto, region_veto, regions) + SELECT 8, 1, 'Wingman', id, false, false, '{TestLan}' + FROM map_pools WHERE type = 'Wingman' AND seed = true`, + ), + ), + ).rejects.toThrow(/Cannot assign the Lan region/); + }); +}); diff --git a/test/tournament-stages.spec.ts b/test/tournament-stages.spec.ts index 39024959b..037e23aa2 100644 --- a/test/tournament-stages.spec.ts +++ b/test/tournament-stages.spec.ts @@ -344,7 +344,7 @@ describe("tournament stages: Swiss and RoundRobin (SQL-driven)", () => { describe("multi-stage advancement", () => { it("adding a later stage raises earlier stage minimums (halving rule)", async () => { const t = await tfx.createTournament([ - { type: "RoundRobin", order: 1, minTeams: 4, maxTeams: 8 }, + { type: "SingleElimination", order: 1, minTeams: 4, maxTeams: 8 }, ]); await postgres.query( `INSERT INTO tournament_stages (tournament_id, type, "order", min_teams, max_teams) @@ -444,5 +444,172 @@ describe("tournament stages: Swiss and RoundRobin (SQL-driven)", () => { await tfx.playRound(t.stageIds[1], 2); expect(await tfx.tournamentStatus(t.id)).toBe("Finished"); }, 120_000); + + it("a Valve Swiss stage sends all eight 3-win teams into an 8-team playoff", async () => { + const t = await tfx.launch( + [ + { type: "Swiss", order: 1, minTeams: 16, maxTeams: 16 }, + { type: "SingleElimination", order: 2, minTeams: 8, maxTeams: 8 }, + ], + 16, + ); + const [swiss, playoff] = t.stageIds; + + for (let round = 1; round <= 5; round++) { + await tfx.playRound(swiss, round); + } + + const qualified = (await tfx.stageResults(swiss)) + .filter((row) => Number(row.wins) === 3) + .map((row) => row.tournament_team_id) + .sort(); + expect(qualified.length).toBe(8); + + const quarterfinals = (await tfx.getBrackets(playoff)).filter( + (b) => b.round === 1, + ); + expect(quarterfinals.length).toBe(4); + expect( + quarterfinals + .flatMap((b) => [b.tournament_team_id_1, b.tournament_team_id_2]) + .sort(), + ).toEqual(qualified); + + await tfx.playStage(playoff); + expect(await tfx.tournamentStatus(t.id)).toBe("Finished"); + }, 180_000); + + // One team takes a bye, a free win, every round, so fifteen teams send eight on. + it("an odd Valve Swiss seats every 3-win team, the bye's included", async () => { + const t = await tfx.launch( + [ + { type: "Swiss", order: 1, minTeams: 15, maxTeams: 15 }, + { type: "SingleElimination", order: 2, minTeams: 2, maxTeams: 8 }, + ], + 15, + ); + const [swiss, playoff] = t.stageIds; + + await tfx.playStage(swiss); + + const qualified = (await tfx.stageResults(swiss)) + .filter((row) => Number(row.wins) >= 3) + .map((row) => row.tournament_team_id) + .sort(); + expect(qualified.length).toBe(8); + + const quarterfinals = (await tfx.getBrackets(playoff)).filter( + (b) => b.round === 1, + ); + expect( + quarterfinals + .flatMap((b) => [b.tournament_team_id_1, b.tournament_team_id_2]) + .filter(Boolean) + .sort(), + ).toEqual(qualified); + }, 180_000); + + it("a stage after a Valve Swiss can't take more teams than reach 3 wins", async () => { + const t = await tfx.createTournament([ + { type: "Swiss", order: 1, minTeams: 16, maxTeams: 16 }, + ]); + const insertPlayoff = (maxTeams: number) => + postgres.query( + `INSERT INTO tournament_stages (tournament_id, type, "order", min_teams, max_teams) + VALUES ($1, 'SingleElimination', 2, 8, $2)`, + [t.id, maxTeams], + ); + + await expect(insertPlayoff(16)).rejects.toThrow( + /only 8 teams can reach 3 wins/i, + ); + await expect(insertPlayoff(8)).resolves.toBeDefined(); + + const odd = await tfx.createTournament([ + { type: "Swiss", order: 1, minTeams: 15, maxTeams: 15 }, + ]); + await expect( + postgres.query( + `INSERT INTO tournament_stages (tournament_id, type, "order", min_teams, max_teams) + VALUES ($1, 'SingleElimination', 2, 8, 9)`, + [odd.id], + ), + ).rejects.toThrow(/only 8 teams can reach 3 wins/i); + await expect( + postgres.query( + `INSERT INTO tournament_stages (tournament_id, type, "order", min_teams, max_teams) + VALUES ($1, 'SingleElimination', 2, 8, 8)`, + [odd.id], + ), + ).resolves.toBeDefined(); + + const wide = await tfx.createTournament([ + { type: "Swiss", order: 1, minTeams: 32, maxTeams: 32 }, + { type: "SingleElimination", order: 2, minTeams: 16, maxTeams: 16 }, + ]); + await expect( + postgres.query( + "UPDATE tournament_stages SET min_teams = 16, max_teams = 16 WHERE id = $1", + [wide.stageIds[0]], + ), + ).rejects.toThrow(/only 8 teams can reach 3 wins/i); + }); + + it("a Valve Swiss that fills below its max only sizes the playoff for the half that reaches 3 wins", async () => { + const t = await tfx.launch( + [ + { type: "Swiss", order: 1, minTeams: 16, maxTeams: 32 }, + { type: "SingleElimination", order: 2, minTeams: 8, maxTeams: 16 }, + ], + 16, + ); + + const quarterfinals = (await tfx.getBrackets(t.stageIds[1])).filter( + (b) => b.round === 1, + ); + expect(quarterfinals.length).toBe(4); + }); + + it("a stage after a Valve Swiss only has to fit under its 3-win half, not its 2-2 last round", async () => { + const insertPlayoff = ( + tournamentId: string, + minTeams: number, + maxTeams: number, + ) => + postgres.query( + `INSERT INTO tournament_stages (tournament_id, type, "order", min_teams, max_teams) + VALUES ($1, 'SingleElimination', 2, $2, $3)`, + [tournamentId, minTeams, maxTeams], + ); + + const wide = await tfx.createTournament([ + { type: "Swiss", order: 1, minTeams: 32, maxTeams: 32 }, + ]); + await expect(insertPlayoff(wide.id, 4, 8)).resolves.toBeDefined(); + + const final = await tfx.createTournament([ + { type: "Swiss", order: 1, minTeams: 16, maxTeams: 16 }, + ]); + await expect(insertPlayoff(final.id, 2, 2)).resolves.toBeDefined(); + }); + + it("resizing a Valve Swiss leaves the stage after it alone", async () => { + const t = await tfx.createTournament([ + { type: "Swiss", order: 1, minTeams: 16, maxTeams: 16 }, + { type: "SingleElimination", order: 2, minTeams: 4, maxTeams: 4 }, + ]); + + await postgres.query( + "UPDATE tournament_stages SET min_teams = 12 WHERE id = $1", + [t.stageIds[0]], + ); + + const [playoff] = await postgres.query< + Array<{ min_teams: number; max_teams: number }> + >("SELECT min_teams, max_teams FROM tournament_stages WHERE id = $1", [ + t.stageIds[1], + ]); + expect(playoff).toEqual({ min_teams: 4, max_teams: 4 }); + }); }); }); diff --git a/test/tournament-substitutes.spec.ts b/test/tournament-substitutes.spec.ts new file mode 100644 index 000000000..50620acd2 --- /dev/null +++ b/test/tournament-substitutes.spec.ts @@ -0,0 +1,403 @@ +import { PostgresService } from "./../src/postgres/postgres.service"; +import { Fixtures } from "./utils/fixtures"; +import { + bootMigratedDb, + runAsUser, + seedRegionWithServer, + SqlTestDb, +} from "./utils/sql-test-db"; + +describe("tournament substitutes (SQL-driven)", () => { + let db: SqlTestDb; + let postgres: PostgresService; + let fx: Fixtures; + + beforeAll(async () => { + db = await bootMigratedDb("TournamentSubstitutesTest"); + postgres = db.postgres; + fx = new Fixtures(postgres, 76561199990000000n); + await seedRegionWithServer(postgres, "TestA"); + }, 600_000); + + afterAll(async () => { + await db?.stop(); + }); + + beforeEach(async () => { + await postgres.query("DELETE FROM matches"); + await postgres.query("DELETE FROM tournaments"); + await postgres.query("DELETE FROM match_options"); + await postgres.query("DELETE FROM teams"); + await postgres.query("DELETE FROM players"); + }); + + const createTournament = async ({ + type, + substitutes, + }: { + type: "Duel" | "Wingman"; + substitutes: number; + }) => { + const organizer = await fx.player(); + const [options] = await postgres.query>( + `INSERT INTO match_options (mr, best_of, type, map_pool_id, map_veto, region_veto, regions, number_of_substitutes) + SELECT 8, 1, $1, id, false, true, '{TestA}', $2 + FROM map_pools WHERE type = $1 AND seed = true RETURNING id`, + [type, substitutes], + ); + const [tournament] = await postgres.query>( + `INSERT INTO tournaments (name, start, organizer_steam_id, match_options_id, status) + VALUES ($1, now() + interval '1 day', $2, $3, 'Setup') RETURNING id`, + [fx.nextName("cup"), organizer, options.id], + ); + await postgres.query( + `INSERT INTO tournament_stages (tournament_id, type, "order", min_teams, max_teams) + VALUES ($1, 'SingleElimination', 1, 4, 8)`, + [tournament.id], + ); + return { id: tournament.id, organizer, optionsId: options.id }; + }; + + const setStatus = (tournamentId: string, organizer: string, status: string) => + runAsUser(postgres, organizer, "admin", (query) => + query("UPDATE tournaments SET status = $1 WHERE id = $2", [ + status, + tournamentId, + ]), + ); + + const setSubstitutesEnabled = (tournamentId: string, enabled: boolean) => + postgres.query( + "UPDATE tournaments SET substitutes_enabled = $1 WHERE id = $2", + [enabled, tournamentId], + ); + + const registerTeam = ( + tournamentId: string, + team: { id: string; owner: string }, + ) => + runAsUser(postgres, team.owner, "admin", async (query) => { + const [row] = (await query( + `INSERT INTO tournament_teams (tournament_id, team_id, name) + SELECT $1, id, name FROM teams WHERE id = $2 RETURNING id`, + [tournamentId, team.id], + )) as Array<{ id: string }>; + return row.id; + }); + + const addRosterPlayer = ( + tournamentId: string, + tournamentTeamId: string, + owner: string, + ) => + runAsUser(postgres, owner, "admin", async (query) => { + const player = await fx.player(); + await query( + `INSERT INTO tournament_team_roster (tournament_team_id, player_steam_id, tournament_id) + VALUES ($1, $2, $3)`, + [tournamentTeamId, player, tournamentId], + ); + }); + + const rosterSteamIds = async (tournamentTeamId: string) => { + const rows = await postgres.query>( + "SELECT player_steam_id FROM tournament_team_roster WHERE tournament_team_id = $1", + [tournamentTeamId], + ); + return rows.map((row) => row.player_steam_id); + }; + + const lineupSizes = async (tournamentId: string) => { + const [row] = await postgres.query< + Array<{ min_players: number; max_players: number }> + >( + `SELECT tournament_min_players_per_lineup(t) AS min_players, + tournament_max_players_per_lineup(t) AS max_players + FROM tournaments t WHERE t.id = $1`, + [tournamentId], + ); + return row; + }; + + const seedBracket = async ( + tournament: { id: string; organizer: string }, + mates: number, + ) => { + await setStatus(tournament.id, tournament.organizer, "RegistrationOpen"); + for (let i = 0; i < 4; i++) { + await registerTeam(tournament.id, await fx.team(mates)); + } + await setStatus(tournament.id, tournament.organizer, "RegistrationClosed"); + + return postgres.query< + Array<{ + id: string; + match_options_id: string; + lineup_1_id: string; + lineup_2_id: string; + max_players: number; + }> + >( + `SELECT m.id, m.match_options_id, m.lineup_1_id, m.lineup_2_id, + match_max_players_per_lineup(m) AS max_players + FROM tournament_brackets tb + INNER JOIN tournament_stages ts ON ts.id = tb.tournament_stage_id + INNER JOIN matches m ON m.id = tb.match_id + WHERE ts.tournament_id = $1 + ORDER BY tb.round, tb.match_number`, + [tournament.id], + ); + }; + + const seatedCount = async (lineupId: string) => { + const [row] = await postgres.query>( + "SELECT count(*)::int AS count FROM match_lineup_players WHERE match_lineup_id = $1", + [lineupId], + ); + return row.count; + }; + + describe("Duel tournaments", () => { + it("caps the lineup at one player whatever the substitute count", async () => { + const tournament = await createTournament({ + type: "Duel", + substitutes: 2, + }); + + expect(await lineupSizes(tournament.id)).toEqual({ + min_players: 1, + max_players: 1, + }); + }); + + it("registering a team rosters only the captain", async () => { + const tournament = await createTournament({ + type: "Duel", + substitutes: 2, + }); + await setStatus(tournament.id, tournament.organizer, "RegistrationOpen"); + const team = await fx.team(2); + + const tournamentTeamId = await registerTeam(tournament.id, team); + + expect(await rosterSteamIds(tournamentTeamId)).toEqual([team.owner]); + }); + + it("rejects a stand-in added to the roster", async () => { + const tournament = await createTournament({ + type: "Duel", + substitutes: 2, + }); + await setStatus(tournament.id, tournament.organizer, "RegistrationOpen"); + const team = await fx.team(0); + const tournamentTeamId = await registerTeam(tournament.id, team); + + await expect( + addRosterPlayer(tournament.id, tournamentTeamId, team.owner), + ).rejects.toThrow(/too many players/i); + }); + + it("seats one player per side and refuses a stand-in on the match", async () => { + const tournament = await createTournament({ + type: "Duel", + substitutes: 2, + }); + + const matches = await seedBracket(tournament, 2); + + expect(matches.length).toBe(2); + for (const match of matches) { + expect(match.max_players).toBe(1); + expect(await seatedCount(match.lineup_1_id)).toBe(1); + expect(await seatedCount(match.lineup_2_id)).toBe(1); + } + + await expect(fx.lineupPlayer(matches[0].lineup_1_id)).rejects.toThrow( + /Max number of players/i, + ); + }); + + it("raising a scheduled match's substitutes does not reopen stand-in slots", async () => { + const tournament = await createTournament({ + type: "Duel", + substitutes: 0, + }); + const [match] = await seedBracket(tournament, 0); + + await postgres.query( + "UPDATE match_options SET number_of_substitutes = 3 WHERE id = $1", + [match.match_options_id], + ); + + const [row] = await postgres.query>( + "SELECT match_max_players_per_lineup(m) AS max_players FROM matches m WHERE m.id = $1", + [match.id], + ); + expect(row.max_players).toBe(1); + await expect(fx.lineupPlayer(match.lineup_1_id)).rejects.toThrow( + /Max number of players/i, + ); + }); + }); + + describe("substitutes_enabled", () => { + it("defaults to on", async () => { + const tournament = await createTournament({ + type: "Wingman", + substitutes: 2, + }); + + const [row] = await postgres.query< + Array<{ substitutes_enabled: boolean }> + >("SELECT substitutes_enabled FROM tournaments WHERE id = $1", [ + tournament.id, + ]); + expect(row.substitutes_enabled).toBe(true); + expect(await lineupSizes(tournament.id)).toEqual({ + min_players: 2, + max_players: 4, + }); + }); + + it("turning it off caps a Wingman tournament at the starting lineup, and back on restores it", async () => { + const tournament = await createTournament({ + type: "Wingman", + substitutes: 2, + }); + + await setSubstitutesEnabled(tournament.id, false); + expect(await lineupSizes(tournament.id)).toEqual({ + min_players: 2, + max_players: 2, + }); + + await setSubstitutesEnabled(tournament.id, true); + expect(await lineupSizes(tournament.id)).toEqual({ + min_players: 2, + max_players: 4, + }); + }); + + it("turned off, rosters and scheduled matches only take the starting lineup", async () => { + const tournament = await createTournament({ + type: "Wingman", + substitutes: 2, + }); + await setSubstitutesEnabled(tournament.id, false); + + const matches = await seedBracket(tournament, 3); + + const teams = await postgres.query>( + "SELECT id FROM tournament_teams WHERE tournament_id = $1", + [tournament.id], + ); + for (const team of teams) { + expect((await rosterSteamIds(team.id)).length).toBe(2); + } + for (const match of matches) { + expect(match.max_players).toBe(2); + expect(await seatedCount(match.lineup_1_id)).toBe(2); + expect(await seatedCount(match.lineup_2_id)).toBe(2); + } + }); + + it("can be turned back on but not off once registration has closed", async () => { + const tournament = await createTournament({ + type: "Wingman", + substitutes: 2, + }); + await seedBracket(tournament, 1); + + await expect(setSubstitutesEnabled(tournament.id, false)).rejects.toThrow( + /only be turned off before registration closes/i, + ); + + const withoutSubstitutes = await createTournament({ + type: "Wingman", + substitutes: 2, + }); + await setSubstitutesEnabled(withoutSubstitutes.id, false); + await seedBracket(withoutSubstitutes, 1); + + await expect( + setSubstitutesEnabled(withoutSubstitutes.id, true), + ).resolves.toBeDefined(); + }); + + it("a roster already over the new cap can still check in and drop its stand-ins", async () => { + const tournament = await createTournament({ + type: "Wingman", + substitutes: 1, + }); + await setStatus(tournament.id, tournament.organizer, "RegistrationOpen"); + const team = await fx.team(2); + const tournamentTeamId = await registerTeam(tournament.id, team); + expect((await rosterSteamIds(tournamentTeamId)).length).toBe(3); + + await setSubstitutesEnabled(tournament.id, false); + + await runAsUser(postgres, team.owner, "admin", (query) => + query( + `UPDATE tournament_team_roster SET checked_in_at = now() + WHERE tournament_team_id = $1 AND player_steam_id = $2`, + [tournamentTeamId, team.owner], + ), + ); + + await runAsUser(postgres, team.owner, "admin", (query) => + query( + `DELETE FROM tournament_team_roster + WHERE tournament_team_id = $1 AND player_steam_id <> $2`, + [tournamentTeamId, team.owner], + ), + ); + expect(await rosterSteamIds(tournamentTeamId)).toEqual([team.owner]); + }); + + it("lowering the substitute count below a roster does not lock that roster", async () => { + const tournament = await createTournament({ + type: "Wingman", + substitutes: 1, + }); + await setStatus(tournament.id, tournament.organizer, "RegistrationOpen"); + const team = await fx.team(2); + const tournamentTeamId = await registerTeam(tournament.id, team); + + await postgres.query( + "UPDATE match_options SET number_of_substitutes = 0 WHERE id = $1", + [tournament.optionsId], + ); + + await runAsUser(postgres, team.owner, "admin", (query) => + query( + `UPDATE tournament_team_roster SET checked_in_at = now() + WHERE tournament_team_id = $1`, + [tournamentTeamId], + ), + ); + + await expect( + addRosterPlayer(tournament.id, tournamentTeamId, team.owner), + ).rejects.toThrow(/too many players/i); + }); + }); + + describe("outside tournaments", () => { + it("a Duel match keeps its configured substitute slots", async () => { + const match = await fx.match({ type: "Duel", substitutes: 2 }); + + const [row] = await postgres.query>( + "SELECT match_max_players_per_lineup(m) AS max_players FROM matches m WHERE m.id = $1", + [match.id], + ); + expect(row.max_players).toBe(3); + + for (let i = 0; i < 3; i++) { + await fx.lineupPlayer(match.lineup_1_id); + } + await expect(fx.lineupPlayer(match.lineup_1_id)).rejects.toThrow( + /Max number of players/i, + ); + }); + }); +}); diff --git a/test/utils/tournament-fixtures.ts b/test/utils/tournament-fixtures.ts index 4cfe8996a..cd3994a74 100644 --- a/test/utils/tournament-fixtures.ts +++ b/test/utils/tournament-fixtures.ts @@ -10,6 +10,7 @@ export type StageSpec = { order: number; minTeams: number; maxTeams: number; + groups?: number; }; export type BracketRow = { @@ -48,9 +49,16 @@ export class TournamentFixtures { const stageIds: Array = []; for (const stage of stages) { const [row] = await this.postgres.query>( - `INSERT INTO tournament_stages (tournament_id, type, "order", min_teams, max_teams) - VALUES ($1, $2, $3, $4, $5) RETURNING id`, - [tournament.id, stage.type, stage.order, stage.minTeams, stage.maxTeams], + `INSERT INTO tournament_stages (tournament_id, type, "order", min_teams, max_teams, groups) + VALUES ($1, $2, $3, $4, $5, $6) RETURNING id`, + [ + tournament.id, + stage.type, + stage.order, + stage.minTeams, + stage.maxTeams, + stage.groups ?? 1, + ], ); stageIds.push(row.id); } @@ -142,6 +150,31 @@ export class TournamentFixtures { return brackets.length; } + // Uneven groups run different round counts, so this sweeps open matches, not rounds. + async playStage( + stageId: string, + pickWinner: ( + bracket: BracketRow, + ) => "lineup_1_id" | "lineup_2_id" = () => "lineup_1_id", + ): Promise { + for (;;) { + const open = await this.postgres.query>( + `SELECT id, round, match_number, "group", match_id, + tournament_team_id_1, tournament_team_id_2, finished + FROM tournament_brackets + WHERE tournament_stage_id = $1 AND match_id IS NOT NULL AND finished = false + ORDER BY round, "group", match_number`, + [stageId], + ); + if (open.length === 0) { + return; + } + for (const bracket of open) { + await this.winMatch(bracket.match_id!, pickWinner(bracket)); + } + } + } + async tournamentStatus(id: string): Promise { const [row] = await this.postgres.query>( "SELECT status FROM tournaments WHERE id = $1",