Name valid values when a benchmark filter matches nothing (#1041)

faircode/mcp_server.py

@@ -496,6 +496,8 @@ def _get_benchmark_results_impl(kind="fairness", audit=None, model=None, strateg
             f"{path} not found - restore the package mirror from paper/results-frozen/ "
             "with `scripts.freeze_paper_results.mirror_for_mcp()`")
     df = pd.read_csv(path)
+    source = df
+    notes = []
     for column, value in (("audit", audit), ("model", model), ("strategy", strategy),
                           ("metric", metric), ("protected_attribute", protected_attribute)):
         if value is None:
@@ -508,14 +510,32 @@ def _get_benchmark_results_impl(kind="fairness", audit=None, model=None, strateg
                 f"(available: {', '.join(df.columns)}); it cannot be used as a "
                 f"filter for this kind"
             )
+        # A value that matches none of the column's values (e.g. the lowercase
+        # audit names mean audit="COMPAS" finds nothing) otherwise returns an
+        # empty result an agent can misread as "no data" (#1041). Name the valid
+        # values, and point out a case-only mismatch, without failing the call.
+        valid = source[column].dropna().unique().tolist()
+        if value not in valid:
+            available = ", ".join(str(v) for v in sorted(valid, key=str))
+            hint = ""
+            if isinstance(value, str):
+                case_match = [v for v in valid
+                              if isinstance(v, str) and v.lower() == value.lower()]
+                if case_match:
+                    hint = (f" (values are case-sensitive; "
+                            f"did you mean {case_match[0]!r}?)")
+            notes.append(f"{column} {value!r} not found; available: {available}{hint}")
         df = df[df[column] == value]
     total = len(df)
     for column in _RESULTS_NUMERIC_COLUMNS:
         if column in df.columns:
             df[column] = df[column].round(_RESULTS_ROUND_DECIMALS)
     head = df.head(_RESULTS_ROW_LIMIT).astype(object)
     rows = head.where(head.notna(), None).to_dict(orient="records")
-    return {"results": rows, "total_matches": total, "truncated": total > len(rows)}
+    result = {"results": rows, "total_matches": total, "truncated": total > len(rows)}
+    if notes:
+        result["notes"] = notes
+    return result
 
 
 def build_server():
@@ -802,6 +822,11 @@ def get_benchmark_results(kind: str = "fairness", audit: str | None = None,
         ignored, so a mistaken filter can't come back looking like an
         unfiltered result.
 
+        Filter values are matched case-sensitively and the audit names are
+        lowercase (e.g. "compas"), so a value that matches nothing returns an
+        empty result with a `notes` entry naming that column's valid values,
+        rather than a bare empty result a caller could misread as "no data."
+
         Returns {"results": [...], "total_matches": N, "truncated": bool} -
         results are capped at 200 rows even when more match, so narrow the
         query with more filters if `truncated` is true rather than assuming

tests/test_mcp_server.py

@@ -550,7 +550,32 @@ def test_get_benchmark_results_rounds_float_precision():
 def test_get_benchmark_results_no_match_returns_empty_not_an_error():
     result = _get_benchmark_results_impl(audit="not_a_real_audit")
 
-    assert result == {"results": [], "total_matches": 0, "truncated": False}
+    assert result["results"] == []
+    assert result["total_matches"] == 0
+    assert result["truncated"] is False
+    # An unmatched filter value names that column's valid values so the empty
+    # result can't be misread as "no data" (#1041).
+    assert result["notes"] == [
+        "audit 'not_a_real_audit' not found; available: ai_fair_recruitment, "
+        "benefits_denial, compas, german_credit_lending, healthcare_readmission, "
+        "insurance_denial, tenant_screening"
+    ]
+
+
+def test_get_benchmark_results_case_mismatch_hints_the_right_value():
+    # Audit names are lowercase, so audit="COMPAS" matches nothing; the note
+    # should point out the case-only mismatch rather than looking like no data (#1041).
+    upper = _get_benchmark_results_impl(audit="COMPAS")
+    assert upper["total_matches"] == 0
+    assert len(upper["notes"]) == 1
+    note = upper["notes"][0]
+    assert note.startswith("audit 'COMPAS' not found; available:")
+    assert "did you mean 'compas'?" in note
+
+    # The exact (lowercase) value still matches and carries no note.
+    lower = _get_benchmark_results_impl(audit="compas")
+    assert lower["total_matches"] > 0
+    assert "notes" not in lower
 
 
 def test_get_benchmark_results_invalid_kind_raises():