From 53cda0c18ce704c90c54b7795ff36e73939525c8 Mon Sep 17 00:00:00 2001 From: sammist2 Date: Sat, 17 Oct 2026 15:10:14 -0700 Subject: [PATCH] Note when proxy_hints has too few dimensions to test (#1044) proxy_hints compares pairs of dimensions, so a dataset with fewer than two testable dimensions has no pair to test and returns {"hints": []}. With no explanation an agent can read that as "tested, found nothing" rather than "nothing could be tested." _proxy_hints_impl now adds a 'notes' entry naming the testable dimensions and stating that at least two are needed, when the empty result is due to there being fewer than two. A dataset that does have a pair to test is unaffected. Adds a regression test. --- faircode/mcp_server.py | 16 +++++++++++++++- tests/test_mcp_server.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/faircode/mcp_server.py b/faircode/mcp_server.py index 0d6b4b2..46639cb 100644 --- a/faircode/mcp_server.py +++ b/faircode/mcp_server.py @@ -391,12 +391,26 @@ def _proxy_hints_impl(path, overrides=None, held_out_with=None, alpha=None, kw.update(_exact_kwargs(exact_permutations, exact_max_rows)) kw["max_age"] = resolved["max_age"] kw["age_reference_year"] = resolved["age_reference_year"] - output = {"hints": compute_proxy_hints(df, result["dimensions"], held_out=held_out, **kw)} + hints = compute_proxy_hints(df, result["dimensions"], held_out=held_out, **kw) + output = {"hints": hints} sheet_notes = [n for n in (_sheet_note(path),) if n] sheet_notes += [n for spec in (held_out_with or []) for n in (_sheet_note(partition_held_out_path(spec)[0]),) if n] if sheet_notes: output["sheet_notes"] = sheet_notes # deprecated alias of the sheet entries in `notes` (#950) notes = sheet_notes + [n for n in (encoding_ignored_note(path, encoding),) if n] + held_notes + # Proxy hints compare *pairs* of dimensions, so fewer than two testable + # dimensions (detected dimensions plus any held-out columns) yields no pairs + # and an empty `hints` list. Say so, or an empty result reads as "tested, + # found nothing" rather than "nothing could be tested" (#1044). + if not hints: + testable = [d["name"] for d in result["dimensions"]] + list(held_out or {}) + if len(testable) < 2: + shown = ", ".join(testable) if testable else "none" + notes = notes + [ + f"no proxy pairs were tested: proxy hints compare pairs of " + f"dimensions and this dataset has {len(testable)} ({shown}); " + f"at least two are needed." + ] if notes: output["notes"] = notes _flag_deprecated_aliases(output) diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 3c4d289..ccf90ed 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -367,6 +367,41 @@ def test_proxy_hints_with_no_significant_pairs_returns_an_empty_list_not_an_erro assert "provenance" in result +def test_proxy_hints_notes_when_too_few_dimensions_to_test(tmp_path): + # Proxy hints compare pairs of dimensions, so a single-dimension dataset has + # no pair to test and returns an empty `hints` list. A note says why, so the + # empty result is not read as "tested, found nothing" (#1041 is the sibling + # fix for benchmark filters; this is #1044). + path = tmp_path / "one.csv" + path.write_text("sex\nM\nF\nM\nF\n", encoding="utf-8") + + result = _proxy_hints_impl(str(path)) + + assert result["hints"] == [] + assert any( + "no proxy pairs were tested" in note and "at least two are needed" in note + for note in result.get("notes", []) + ), result.get("notes") + + # A dataset with two associated dimensions is tested, so it carries no such note. + two = tmp_path / "two.csv" + two.write_text( + "sex,occupation\n" + + "\n".join( + f"{'male' if i % 2 == 0 else 'female'}," + f"{'engineer' if i % 2 == 0 else 'nurse'}" + for i in range(100) + ) + + "\n", + encoding="utf-8", + ) + tested = _proxy_hints_impl(str(two)) + assert tested["hints"] + assert not any( + "no proxy pairs were tested" in note for note in tested.get("notes", []) + ) + + def test_proxy_hints_runtime_error_propagates_with_a_clean_message(tmp_path, monkeypatch): import faircode.mcp_server as mcp_server