From 3e4829e1e6aceccb996a7094b7e93a1ef7f3237d Mon Sep 17 00:00:00 2001 From: Hysann <1932819331hy@gmail.com> Date: Tue, 27 Oct 2026 13:05:16 +0900 Subject: [PATCH] fix: validate exact error output in example checker Signed-off-by: YU HUANG <343001481+1932819331hy-debug@users.noreply.github.com> --- .github/workflows/check-examples.yml | 2 + CONTRIBUTING.md | 10 +++ scripts/check_examples.py | 6 +- tests/test_check_examples.py | 112 +++++++++++++++++++++++++++ 4 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 tests/test_check_examples.py diff --git a/.github/workflows/check-examples.yml b/.github/workflows/check-examples.yml index 1d29d1c..57c94c9 100644 --- a/.github/workflows/check-examples.yml +++ b/.github/workflows/check-examples.yml @@ -22,6 +22,8 @@ jobs: - uses: actions/setup-python@751276eafca55f10f252026015f43c488f2322ba with: python-version: '3.13' + - name: Test the example checker + run: python -m unittest discover -s tests -v - uses: actions/setup-node@94196ee1d15439c1b6651cd87ef14e88ec435966 with: node-version: '24' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 66b9a74..baf98ca 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,6 +9,16 @@ Keep each example a small standard-library terminal program. All nine must: ## Checks +Run the checker's standard-library regression tests from the repository root: + +```sh +python3 -m unittest discover -s tests -v +``` + +These tests require only Python and check exact output validation, including +the known `exit status 1` trailer from `go run`. Unexpected child output is +rejected without being printed in failure diagnostics. + With all listed runtimes and CMake installed, run from the repository root: ```sh diff --git a/scripts/check_examples.py b/scripts/check_examples.py index cd7b780..e107d83 100644 --- a/scripts/check_examples.py +++ b/scripts/check_examples.py @@ -49,7 +49,11 @@ def main(): if value: passed = result.returncode == 0 and result.stdout == SUCCESS and result.stderr == "" else: - passed = result.returncode == 1 and result.stdout == "" and ERROR in result.stderr + expected_errors = [ERROR + "\n"] + if language == "go": + # `go run` can append its own diagnostic after the demo exits. + expected_errors.append(ERROR + "\nexit status 1\n") + passed = result.returncode == 1 and result.stdout == "" and result.stderr in expected_errors if not passed: raise SystemExit(f"FAIL {language}: {label}; unexpected exit code or output (output withheld)") print(f"PASS {language}: {label}") diff --git a/tests/test_check_examples.py b/tests/test_check_examples.py new file mode 100644 index 0000000..3924059 --- /dev/null +++ b/tests/test_check_examples.py @@ -0,0 +1,112 @@ +"""Check output validation without requiring all nine language runtimes.""" + +from contextlib import redirect_stderr, redirect_stdout +import io +from pathlib import Path +import subprocess +import sys +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +import check_examples + + +class CheckExamplesTests(unittest.TestCase): + def run_checker(self, language="python", *, error_stderr=None, + error_stdout="", error_code=1, failure_case=None, + success_stdout=check_examples.SUCCESS, success_stderr="", + success_code=0): + if error_stderr is None: + error_stderr = check_examples.ERROR + "\n" + + def run(command, **kwargs): + env = kwargs.get("env") + if env is None: # Build commands do not exercise output validation. + return subprocess.CompletedProcess(command, 0) + value = env.get("DEMO_SECRET") + if value: + return subprocess.CompletedProcess( + command, success_code, success_stdout, success_stderr) + if failure_case is not None and value != failure_case: + return subprocess.CompletedProcess( + command, 1, "", check_examples.ERROR + "\n") + return subprocess.CompletedProcess( + command, error_code, error_stdout, error_stderr) + + output = io.StringIO() + errors = io.StringIO() + with patch.object(sys, "argv", ["check_examples.py", language]), \ + patch.object(check_examples.subprocess, "run", side_effect=run), \ + redirect_stdout(output), redirect_stderr(errors): + check_examples.main() + return output.getvalue(), errors.getvalue() + + def test_exact_output_passes_all_cases(self): + for language in check_examples.COMMANDS: + with self.subTest(language=language): + output, errors = self.run_checker(language) + self.assertEqual(output, "".join( + f"PASS {language}: {case}\n" + for case in ("missing", "empty", "zero", "present"))) + self.assertEqual(errors, "") + + def test_extra_error_prefix_or_suffix_fails(self): + for language in check_examples.COMMANDS: + for stderr in ("unexpected\n" + check_examples.ERROR + "\n", + check_examples.ERROR + "\nunexpected\n"): + for case in (None, ""): + with self.subTest(language=language, stderr=stderr, case=case): + with self.assertRaisesRegex(SystemExit, "output withheld"): + self.run_checker(language, error_stderr=stderr, + failure_case=case) + + def test_missing_error_newline_fails(self): + with self.assertRaises(SystemExit): + self.run_checker(error_stderr=check_examples.ERROR) + + def test_wrong_error_status_fails(self): + for code in (0, 2, -1): + with self.subTest(code=code), self.assertRaises(SystemExit): + self.run_checker(error_code=code) + + def test_unexpected_error_stdout_fails_without_disclosure(self): + with self.assertRaises(SystemExit) as raised: + self.run_checker(error_stdout="synthetic-private-output") + self.assertNotIn("synthetic-private-output", str(raised.exception)) + self.assertIn("output withheld", str(raised.exception)) + + def test_go_launcher_trailer_passes(self): + output, _ = self.run_checker( + "go", error_stderr=check_examples.ERROR + "\nexit status 1\n") + self.assertIn("PASS go: empty", output) + + def test_go_trailer_is_not_allowed_for_other_languages(self): + for language in check_examples.COMMANDS: + if language == "go": + continue + with self.subTest(language=language), self.assertRaises(SystemExit): + self.run_checker( + language, error_stderr=check_examples.ERROR + "\nexit status 1\n") + + def test_go_rejects_modified_or_repeated_trailer(self): + for trailer in ("exit status 2\n", "exit status 1\nextra\n", + "exit status 1\nexit status 1\n", "exit status 1"): + with self.subTest(trailer=trailer), self.assertRaises(SystemExit): + self.run_checker("go", error_stderr=check_examples.ERROR + "\n" + trailer) + + def test_error_diagnostics_withhold_stderr(self): + with self.assertRaises(SystemExit) as raised: + self.run_checker(error_stderr="synthetic-private-error") + self.assertNotIn("synthetic-private-error", str(raised.exception)) + self.assertIn("output withheld", str(raised.exception)) + + def test_invalid_success_result_fails(self): + for changes in ({"success_code": 1}, {"success_stdout": "wrong\n"}, + {"success_stderr": "unexpected\n"}): + with self.subTest(changes=changes), self.assertRaises(SystemExit): + self.run_checker(**changes) + + +if __name__ == "__main__": + unittest.main()