diff --git a/assets/profiler-engine.js b/assets/profiler-engine.js index eefb8dd..7bb5a7b 100644 --- a/assets/profiler-engine.js +++ b/assets/profiler-engine.js @@ -269,8 +269,11 @@ // Handles quoted fields, escaped quotes (""), and newlines inside quotes. // pandas renames a repeated header to name.1, name.2, ... (skipping any name // already taken); without this the second copy overwrote the first in every - // row object and both dimensions read the LAST column's values (#834). + // row object and both dimensions read the LAST column's values (#834). A blank + // header cell is named "Unnamed: N" (N = zero-based column index) first, as pandas + // does, so the column has the same name in both engines (#1046). function dedupeHeaders(columns) { + columns = columns.map(function (c, i) { return c === '' ? 'Unnamed: ' + i : c; }); var used = Object.create(null), seen = Object.create(null); columns.forEach(function (c) { used[c] = (used[c] || 0) + 1; }); var taken = Object.create(null); diff --git a/faircode/SPEC.md b/faircode/SPEC.md index 2b71007..3609fcb 100644 --- a/faircode/SPEC.md +++ b/faircode/SPEC.md @@ -23,7 +23,7 @@ that is what the `unfair.py` / `fair.py` audits do. The Profiler answers a diffe ## 1. Column auto-detection -Repeated header names are made unique the way pandas does (`name`, `name.1`, `name.2`, ... skipping names already taken) before anything else; the browser parser does the same so two same-named columns are two dimensions, never one overwriting the other (#834). +A blank header cell is named `Unnamed: N` (N is the zero-based column index), as pandas does. Repeated header names are then made unique the same way (`name`, `name.1`, `name.2`, ... skipping names already taken) before anything else; the browser parser does the same so two same-named columns are two dimensions, never one overwriting the other (#834). **Tokenize** the column name: strip accents (Unicode NFD, then drop U+0300-U+036F, so `Género` becomes `Genero`), split on separators **and** camelCase boundaries, then lower-case. diff --git a/tests/test_js_parity_profile_parsing.py b/tests/test_js_parity_profile_parsing.py index 66b172e..9bb081a 100644 --- a/tests/test_js_parity_profile_parsing.py +++ b/tests/test_js_parity_profile_parsing.py @@ -223,13 +223,17 @@ def test_python_js_sample_dataset_is_byte_identical(): @pytest.mark.parametrize("name,text", [ ("duplicate_headers", "sex,sex,race\nM,F,A\nM,F,B\nM,F,A\nF,M,B\n"), ("duplicate_headers_colliding_suffix", "a,a,a,a.1\n1,2,3,4\n"), + ("blank_header_cell", "sex,,race\nM,a,x\nF,b,y\nM,a,x\nF,b,y\n"), + ("all_blank_header_row", ",,\nM,a,x\nF,b,y\nM,a,x\nF,b,y\n"), + ("blank_header_next_to_unnamed_name", "Unnamed: 1,,race\nM,a,x\nF,b,y\nM,a,x\nF,b,y\n"), ("quoted_empty_single_column_row", 'region\r\n""\r\nN\r\nS\r\n'), ("quoted_empty_last_line", 'region\r\nN\r\n""'), ("non_ascii_digit_ages", "age\n٣٠\n25\n40\n٣٠\n"), ]) def test_python_js_parity_on_parser_edge_cases(tmp_path, name, text): """#834 (pandas-style header de-duplication), #835 (a quoted-empty row is a - missing value, not a blank line) and #836 (only ASCII digits are ages): + missing value, not a blank line), #836 (only ASCII digits are ages) and #1046 + (a blank header cell is "Unnamed: N", not an empty name): the CSV parsers and the profile of the result agree on each.""" path = tmp_path / f"{name}.csv" path.write_text(text, encoding="utf-8", newline="")