From 6c3e88db4cb89713afac5ba00510174fa4477273 Mon Sep 17 00:00:00 2001 From: m96-chan Date: Sun, 6 Sep 2026 15:23:13 +0900 Subject: [PATCH] test: Run the exporters in CI instead of skipping them The pandas and pyarrow exporters are a documented feature and CI never ran a line of them. It installed the wheel with no extras, so the tests that needed those packages skipped, and the report said "194 passed, 2 skipped" without saying that two of the four export formats were untested. Install the wheel as `[export]`, which takes the extra's contents from pyproject.toml rather than repeating pandas and pyarrow here, and assert afterwards that both imported. A skip is the failure mode this is meant to remove, so it should not be able to come back quietly. The tests needed work first. Every one of them was wrapped in try: from pyetwkit.export import to_csv ... except ImportError: pytest.skip("pyetwkit.export module not available") which is wrong twice over. The module always imports -- pandas and pyarrow are imported inside the functions that need them -- so eight of the ten guards could only ever have fired on a genuine ImportError from inside the exporters, reporting a broken module as an absent optional dependency. That is the same shape as #77, where a wrong import name skipped 112 tests for months. Decide once, up front, whether each optional dependency is installed, and mark only the tests that actually need one. Everything else now asserts. With the extra installed all eighteen run; without it, twelve still run and the six that skip say which package is missing and how to get it, rather than claiming the module is unavailable. Also added round trips for parquet and arrow. Both had only a `callable()` check, so installing pyarrow in CI would otherwise have bought nothing. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HhPcm483PrDmuQEUSALBFn --- .github/workflows/ci.yml | 15 ++- tests/test_export.py | 242 ++++++++++++++++++++++++--------------- 2 files changed, 163 insertions(+), 94 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8e7a12..faec374 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,8 +198,19 @@ jobs: Write-Host "ERROR: No wheel found for $pyVer" exit 1 } - Write-Host "Installing wheel: $($wheel.FullName)" - uv pip install "$($wheel.FullName)" + # Install with the export extra, so the pandas/pyarrow exporters are + # actually exercised rather than skipped. Taking the extra from the + # wheel keeps its contents defined in pyproject.toml alone. + Write-Host "Installing wheel: $($wheel.FullName)[export]" + uv pip install "$($wheel.FullName)[export]" + shell: pwsh + + - name: Confirm the export extra is present + # The exporters are a documented feature, and their tests skip + # themselves when pandas or pyarrow is missing. Fail here rather than + # let that turn into silent non-coverage. + run: | + & .\.venv\Scripts\python.exe -c "import pandas, pyarrow; print('pandas', pandas.__version__, '/ pyarrow', pyarrow.__version__)" shell: pwsh - name: Run tests diff --git a/tests/test_export.py b/tests/test_export.py index bc90648..78c4d9f 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -1,7 +1,12 @@ """Tests for data export functionality (v0.3.0 - #14, #33).""" +from __future__ import annotations + +import importlib.util +import json import os import tempfile +from pathlib import Path import pytest @@ -23,6 +28,39 @@ def check_extension_available() -> bool: ) +def _installed(module: str) -> bool: + return importlib.util.find_spec(module) is not None + + +# The exporters keep pandas and pyarrow optional, importing them inside the +# functions that need them, so `pyetwkit.export` itself always imports. Deciding +# up front which optional dependency is present is what lets everything else +# assert rather than skip: wrapping a call in `except ImportError: skip` would +# also swallow a genuine ImportError raised inside the module and report a +# broken exporter as a missing extra. +requires_pandas = pytest.mark.skipif( + not _installed("pandas"), reason="pandas is not installed (pip install pyetwkit[export])" +) +requires_pyarrow = pytest.mark.skipif( + not _installed("pyarrow"), reason="pyarrow is not installed (pip install pyetwkit[export])" +) + +SAMPLE_EVENTS = [ + { + "provider_name": "Microsoft-Windows-DNS-Client", + "event_id": 3020, + "process_id": 4104, + "properties": {"QueryName": "example.com", "QueryType": 1}, + }, + { + "provider_name": "Microsoft-Windows-DNS-Client", + "event_id": 3006, + "process_id": 4104, + "properties": {"QueryName": "github.com", "QueryType": 28}, + }, +] + + class TestEventToDict: """Tests for event to dict conversion.""" @@ -42,133 +80,153 @@ def test_event_to_dict_returns_dict(self) -> None: assert hasattr(event_class, "to_dict") +class TestExportModule: + """Tests for export module structure.""" + + def test_export_module_exists(self) -> None: + """The module must import with no optional dependency present.""" + from pyetwkit import export # noqa: F401 + + def test_export_all_functions(self) -> None: + """Test that all export functions are available.""" + from pyetwkit.export import ( + to_arrow, + to_csv, + to_dataframe, + to_json, + to_jsonl, + to_parquet, + ) + + assert all( + callable(f) for f in [to_dataframe, to_csv, to_json, to_jsonl, to_parquet, to_arrow] + ) + + +class TestToJSON: + """JSON export needs no optional dependency.""" + + def test_to_json_function_exists(self) -> None: + from pyetwkit.export import to_json + + assert callable(to_json) + + def test_to_jsonl_function_exists(self) -> None: + from pyetwkit.export import to_jsonl + + assert callable(to_jsonl) + + def test_to_json_round_trips(self) -> None: + from pyetwkit.export import to_json + + text = to_json(SAMPLE_EVENTS) + assert text is not None + assert [e["event_id"] for e in json.loads(text)] == [3020, 3006] + + def test_to_jsonl_writes_one_object_per_line(self, tmp_path: Path) -> None: + from pyetwkit.export import to_jsonl + + path = tmp_path / "events.jsonl" + to_jsonl(SAMPLE_EVENTS, path) + lines = path.read_text(encoding="utf-8").splitlines() + assert len(lines) == 2 + assert json.loads(lines[0])["event_id"] == 3020 + + class TestToDataFrame: """Tests for DataFrame conversion.""" def test_to_dataframe_function_exists(self) -> None: - """Test that to_dataframe function exists in export module.""" - try: - from pyetwkit.export import to_dataframe + """Available whether or not pandas is: the import is done lazily.""" + from pyetwkit.export import to_dataframe - assert callable(to_dataframe) - except ImportError: - pytest.skip("pyetwkit.export module not available") + assert callable(to_dataframe) + @requires_pandas def test_to_dataframe_empty_list(self) -> None: """Test converting empty list to dataframe.""" - try: - from pyetwkit.export import to_dataframe + from pyetwkit.export import to_dataframe - df = to_dataframe([]) - assert len(df) == 0 - except ImportError: - pytest.skip("pyetwkit.export module not available") + df = to_dataframe([]) + assert len(df) == 0 + + @requires_pandas + def test_to_dataframe_carries_the_events(self) -> None: + from pyetwkit.export import to_dataframe + + df = to_dataframe(SAMPLE_EVENTS) + assert len(df) == 2 + assert list(df["event_id"]) == [3020, 3006] class TestToCSV: - """Tests for CSV export.""" + """CSV export goes through pandas.""" def test_to_csv_function_exists(self) -> None: - """Test that to_csv function exists.""" - try: - from pyetwkit.export import to_csv + from pyetwkit.export import to_csv - assert callable(to_csv) - except ImportError: - pytest.skip("pyetwkit.export module not available") + assert callable(to_csv) + @requires_pandas def test_to_csv_creates_file(self) -> None: """Test that to_csv creates a file.""" - try: - from pyetwkit.export import to_csv + from pyetwkit.export import to_csv - with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as f: - temp_path = f.name + with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as f: + temp_path = f.name - try: - to_csv([], temp_path) - assert os.path.exists(temp_path) - finally: - if os.path.exists(temp_path): - os.unlink(temp_path) - except ImportError: - pytest.skip("pyetwkit.export module not available") - - -class TestToJSON: - """Tests for JSON export.""" - - def test_to_json_function_exists(self) -> None: - """Test that to_json function exists.""" try: - from pyetwkit.export import to_json + to_csv([], temp_path) + assert os.path.exists(temp_path) + finally: + if os.path.exists(temp_path): + os.unlink(temp_path) - assert callable(to_json) - except ImportError: - pytest.skip("pyetwkit.export module not available") + @requires_pandas + def test_to_csv_writes_the_events(self, tmp_path: Path) -> None: + from pyetwkit.export import to_csv - def test_to_jsonl_function_exists(self) -> None: - """Test that to_jsonl function exists.""" - try: - from pyetwkit.export import to_jsonl - - assert callable(to_jsonl) - except ImportError: - pytest.skip("pyetwkit.export module not available") + path = tmp_path / "events.csv" + to_csv(SAMPLE_EVENTS, path) + text = path.read_text(encoding="utf-8") + assert "event_id" in text + assert "3020" in text class TestToParquet: - """Tests for Parquet export.""" + """Parquet export needs pyarrow.""" def test_to_parquet_function_exists(self) -> None: - """Test that to_parquet function exists.""" - try: - from pyetwkit.export import to_parquet + from pyetwkit.export import to_parquet - assert callable(to_parquet) - except ImportError: - pytest.skip("pyetwkit.export module not available") + assert callable(to_parquet) + @requires_pandas + @requires_pyarrow + def test_to_parquet_round_trips(self, tmp_path: Path) -> None: + import pandas as pd -class TestToArrow: - """Tests for Arrow export.""" + from pyetwkit.export import to_parquet - def test_to_arrow_function_exists(self) -> None: - """Test that to_arrow function exists.""" - try: - from pyetwkit.export import to_arrow + path = tmp_path / "events.parquet" + to_parquet(SAMPLE_EVENTS, path) + assert path.exists() + assert list(pd.read_parquet(path)["event_id"]) == [3020, 3006] - assert callable(to_arrow) - except ImportError: - pytest.skip("pyetwkit.export module not available") +class TestToArrow: + """Arrow export needs pyarrow.""" -class TestExportModule: - """Tests for export module structure.""" + def test_to_arrow_function_exists(self) -> None: + from pyetwkit.export import to_arrow - def test_export_module_exists(self) -> None: - """Test that pyetwkit.export module exists.""" - try: - from pyetwkit import export # noqa: F401 + assert callable(to_arrow) - assert True - except ImportError: - pytest.skip("pyetwkit.export module not available") + @requires_pandas + @requires_pyarrow + def test_to_arrow_returns_a_table_of_the_events(self) -> None: + from pyetwkit.export import to_arrow - def test_export_all_functions(self) -> None: - """Test that all export functions are available.""" - try: - from pyetwkit.export import ( - to_arrow, - to_csv, - to_dataframe, - to_json, - to_jsonl, - to_parquet, - ) - - assert all( - callable(f) for f in [to_dataframe, to_csv, to_json, to_jsonl, to_parquet, to_arrow] - ) - except ImportError: - pytest.skip("pyetwkit.export module not available") + table = to_arrow(SAMPLE_EVENTS) + assert table.num_rows == 2 + assert table.column("event_id").to_pylist() == [3020, 3006]