diff --git a/faircode/mcp_server.py b/faircode/mcp_server.py index 0d6b4b2..0bcd8e3 100644 --- a/faircode/mcp_server.py +++ b/faircode/mcp_server.py @@ -496,6 +496,8 @@ def _get_benchmark_results_impl(kind="fairness", audit=None, model=None, strateg f"{path} not found - restore the package mirror from paper/results-frozen/ " "with `scripts.freeze_paper_results.mirror_for_mcp()`") df = pd.read_csv(path) + source = df + notes = [] for column, value in (("audit", audit), ("model", model), ("strategy", strategy), ("metric", metric), ("protected_attribute", protected_attribute)): if value is None: @@ -508,6 +510,21 @@ def _get_benchmark_results_impl(kind="fairness", audit=None, model=None, strateg f"(available: {', '.join(df.columns)}); it cannot be used as a " f"filter for this kind" ) + # A value that matches none of the column's values (e.g. the lowercase + # audit names mean audit="COMPAS" finds nothing) otherwise returns an + # empty result an agent can misread as "no data" (#1041). Name the valid + # values, and point out a case-only mismatch, without failing the call. + valid = source[column].dropna().unique().tolist() + if value not in valid: + available = ", ".join(str(v) for v in sorted(valid, key=str)) + hint = "" + if isinstance(value, str): + case_match = [v for v in valid + if isinstance(v, str) and v.lower() == value.lower()] + if case_match: + hint = (f" (values are case-sensitive; " + f"did you mean {case_match[0]!r}?)") + notes.append(f"{column} {value!r} not found; available: {available}{hint}") df = df[df[column] == value] total = len(df) for column in _RESULTS_NUMERIC_COLUMNS: @@ -515,7 +532,10 @@ def _get_benchmark_results_impl(kind="fairness", audit=None, model=None, strateg df[column] = df[column].round(_RESULTS_ROUND_DECIMALS) head = df.head(_RESULTS_ROW_LIMIT).astype(object) rows = head.where(head.notna(), None).to_dict(orient="records") - return {"results": rows, "total_matches": total, "truncated": total > len(rows)} + result = {"results": rows, "total_matches": total, "truncated": total > len(rows)} + if notes: + result["notes"] = notes + return result def build_server(): @@ -802,6 +822,11 @@ def get_benchmark_results(kind: str = "fairness", audit: str | None = None, ignored, so a mistaken filter can't come back looking like an unfiltered result. + Filter values are matched case-sensitively and the audit names are + lowercase (e.g. "compas"), so a value that matches nothing returns an + empty result with a `notes` entry naming that column's valid values, + rather than a bare empty result a caller could misread as "no data." + Returns {"results": [...], "total_matches": N, "truncated": bool} - results are capped at 200 rows even when more match, so narrow the query with more filters if `truncated` is true rather than assuming diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 3c4d289..954d1cd 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -550,7 +550,32 @@ def test_get_benchmark_results_rounds_float_precision(): def test_get_benchmark_results_no_match_returns_empty_not_an_error(): result = _get_benchmark_results_impl(audit="not_a_real_audit") - assert result == {"results": [], "total_matches": 0, "truncated": False} + assert result["results"] == [] + assert result["total_matches"] == 0 + assert result["truncated"] is False + # An unmatched filter value names that column's valid values so the empty + # result can't be misread as "no data" (#1041). + assert result["notes"] == [ + "audit 'not_a_real_audit' not found; available: ai_fair_recruitment, " + "benefits_denial, compas, german_credit_lending, healthcare_readmission, " + "insurance_denial, tenant_screening" + ] + + +def test_get_benchmark_results_case_mismatch_hints_the_right_value(): + # Audit names are lowercase, so audit="COMPAS" matches nothing; the note + # should point out the case-only mismatch rather than looking like no data (#1041). + upper = _get_benchmark_results_impl(audit="COMPAS") + assert upper["total_matches"] == 0 + assert len(upper["notes"]) == 1 + note = upper["notes"][0] + assert note.startswith("audit 'COMPAS' not found; available:") + assert "did you mean 'compas'?" in note + + # The exact (lowercase) value still matches and carries no note. + lower = _get_benchmark_results_impl(audit="compas") + assert lower["total_matches"] > 0 + assert "notes" not in lower def test_get_benchmark_results_invalid_kind_raises():