Skip to content

Commit 029d9bb

Browse files
committed
perf: skip NaN restore for float columns without NaN
Signed-off-by: Maha Rana Yadavalli <271375718+maharanay22@users.noreply.github.com>
1 parent 2546c2f commit 029d9bb

1 file changed

Lines changed: 8 additions & 3 deletions

File tree

‎src/databricks/sql/result_set.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -135,9 +135,14 @@ def _convert_arrow_table(self, table):
135135
# The nullable float dtypes turn IEEE NaN into NA, which would make it
136136
# indistinguishable from SQL NULL, so restore NaN values from Arrow.
137137
for i, field in enumerate(table_renamed.schema):
138-
if pyarrow.types.is_floating(field.type):
139-
is_nan = pyarrow.compute.is_nan(table_renamed.column(i))
140-
res[is_nan.fill_null(False).to_numpy(), i] = float("nan")
138+
if not pyarrow.types.is_floating(field.type):
139+
continue
140+
nan_mask = pyarrow.compute.is_nan(table_renamed.column(i))
141+
if not pyarrow.compute.any(nan_mask).as_py():
142+
continue
143+
if nan_mask.null_count:
144+
nan_mask = nan_mask.fill_null(False)
145+
res[nan_mask.to_numpy(), i] = float("nan")
141146

142147
return [ResultRow(*v) for v in res]
143148

0 commit comments

Comments
 (0)