Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions .github/scripts/slow_test_durations.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
python .github/scripts/slow_test_durations.py junit-*/junit-*.xml

Use a green ``main`` run: it has a ``junit-*`` artifact for every shard of the slow tier, the gateway tests, the unit
tier and the installer tier, which ``gh run download`` puts in a folder each. Pass them all, since the table is
rewritten whole. Each test's time counts toward its file, or toward the test itself for a file the table names in
``split_by_test``; that list is kept as it is.
tier and the installer tier, which ``gh run download`` puts in a folder each. Each test's time counts toward its file,
or toward the test itself for a file the table names in ``split_by_test``; that list is kept as it is. A file the
reports have no test of, such as one of a tier the run skipped, keeps the times the table had for it while it exists.
"""

from __future__ import annotations
Expand Down Expand Up @@ -43,9 +43,13 @@ def main() -> int:
if not seconds:
print("slow test durations: the reports have no test cases; the table is unchanged")
return 1
table["seconds"] = {unit: round(value, 1) for unit, value in sorted(seconds.items())}
reported = {unit.split("::", 1)[0] for unit in seconds}
kept = {unit: value for unit, value in table.get("seconds", {}).items()
if unit.split("::", 1)[0] not in reported and (args.root / unit.split("::", 1)[0]).is_file()}
table["seconds"] = {unit: round(value, 1) for unit, value in sorted({**kept, **seconds}.items())}
args.table.write_text(json.dumps(table, indent=1) + "\n")
print(f"slow test durations: {len(seconds)} units, {sum(seconds.values()) / 60:.1f} min, written to {args.table}")
print(f"slow test durations: {len(seconds)} units, {sum(seconds.values()) / 60:.1f} min, and {len(kept)} kept "
f"from files the reports have no test of, written to {args.table}")
return 0


Expand Down
26 changes: 26 additions & 0 deletions tst/unit/test_slow_test_durations_script.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,3 +67,29 @@ def test_reports_with_no_cases_leave_the_table_as_it_was(tmp_path):
result = _refresh(table, tmp_path, _report(tmp_path, "empty.xml", []))
assert result.returncode == 1
assert table.read_text() == before


def test_a_file_the_reports_have_no_test_of_keeps_its_times(tmp_path):
table = _checkout(tmp_path)
installer = "tst/installer/plugin_add_test.py"
(tmp_path / installer).parent.mkdir(parents=True)
(tmp_path / installer).write_text("")
table.write_text(json.dumps({"split_by_test": [SPLIT], "seconds": {
installer: 120.0, f"{SPLIT}::test_old": 50.0, "tst/integration/cli/cli_test.py": 80.0,
"tst/integration/gone_test.py": 9}}))
report = _report(tmp_path, "r.xml", [("tst.integration.cli.cli_test", "test_deploy", 5.0)])
result = _refresh(table, tmp_path, report)
assert result.returncode == 0, result.stdout
assert json.loads(table.read_text())["seconds"] == {
"tst/installer/plugin_add_test.py": 120.0,
"tst/integration/cli/cli_test.py": 5.0,
f"{SPLIT}::test_old": 50.0,
}


def test_a_reported_split_file_drops_its_tests_the_reports_dont_have(tmp_path):
table = _checkout(tmp_path)
table.write_text(json.dumps({"split_by_test": [SPLIT], "seconds": {f"{SPLIT}::test_renamed": 50.0}}))
report = _report(tmp_path, "r.xml", [("tst.integration.env.gateway.gateway_test", "test_new", 7.0)])
assert _refresh(table, tmp_path, report).returncode == 0
assert json.loads(table.read_text())["seconds"] == {f"{SPLIT}::test_new": 7.0}