Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions crates/pyspark-rs/src/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -955,6 +955,14 @@ pub(crate) fn py_to_value(obj: &Bound<'_, PyAny>) -> PyResult<Value> {
return Ok(Value::Null);
}

// Exact built-in strings cannot implement numeric conversion protocols.
// Keep subclasses and strings that fail UTF-8 extraction on the generic path.
if let Ok(s) = obj.cast_exact::<pyo3::types::PyString>() {
if let Ok(s) = s.extract::<String>() {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: when UTF-8 extraction fails on an exact str, the generic path can't succeed. It ends with "unsupported value type for createDataFrame: str", which is misleading. This isn't new and doesn't block the PR, but returning the extraction error here would be simpler and clearer.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed that preserving the extraction error would be clearer. I left the existing TypeError fallback unchanged to keep this optimization behavior-preserving; the surrogate cases check that compatibility. Propagating the original Unicode error can be a separate behavior change.

return Ok(Value::String(s));
}
}

// Check for bool (before int because bool is a subclass of int in Python)
if let Ok(b) = obj.extract::<bool>() {
return Ok(Value::Bool(b));
Expand Down
32 changes: 31 additions & 1 deletion python/tests/test_dropin_offline.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
import pytest

from _mock_connect_server import message_field, mock_connect_session, struct_schema
from pyspark.sql import SparkSession, types as T
from pyspark.sql import Row, SparkSession, types as T


# --------------------------------------------------------------------------- types
Expand Down Expand Up @@ -3085,3 +3085,33 @@ def allow_response_after_request():
else:
assert result == [("id", "bigint")]
assert service.calls == 1


# Exact built-in string conversion guards.


@pytest.mark.parametrize("text", [
"", "1.25", "-42", "NaN", "a\x00b", "é雪😀", "e\u0301", "\uffff",
])
def test_exact_string_roundtrip(text):
actual = Row(value=text).value
assert type(actual) is str
assert actual == text


@pytest.mark.parametrize("text", ["\ud800", "\udfff", "a\ud800b"])
def test_exact_string_surrogate_error_preserves_generic_fallback(text):
with pytest.raises(TypeError):
Row(value=text)


def test_exact_string_subclass_uses_generic_conversion():
# Pin existing subclass behavior; treating subclasses as text is a separate
# behavior change.
class IndexString(str):
def __index__(self):
return 17

actual = Row(value=IndexString("x")).value
assert type(actual) is int
assert actual == 17
Loading