[SPARK-59432][CONNECT][INFRA] Build free-threaded (cp3xxt) wheels

.github/workflows/build_python_connect.yml

@@ -57,7 +57,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.11"]
+        # 3.14t is the free-threaded (no-GIL) build, covering the free-threaded wheels.
+        python-version: ["3.11", "3.14t"]
     steps:
       - uses: actions/checkout@v4
 
@@ -87,15 +88,18 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             target
-          key: ${{ runner.os }}-cargo-parity-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: ${{ runner.os }}-cargo-parity-
+          key: ${{ runner.os }}-cargo-parity-py${{ matrix.python-version }}-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo-parity-py${{ matrix.python-version }}-
 
       - name: Install system dependencies
         run: |
           sudo apt-get update
           sudo apt-get install -y protobuf-compiler
 
       - name: Install Python dependencies
+        env:
+          # grpcio and zstandard build from source on the free-threaded interpreter.
+          GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS: "4"
         run: |
           python -m pip install --upgrade pip
           python -m pip install pytest pandas pyarrow numpy grpcio grpcio-status googleapis-common-protos cloudpickle zstandard py4j
@@ -109,6 +113,28 @@ jobs:
           cargo build -p pyspark-rs --release
           cp target/release/lib_pyspark.so python/pyspark/_pyspark.so
 
+      - name: Check the extension keeps the GIL disabled
+        # Runs WITHOUT PYTHON_GIL, so the interpreter decides whether to re-enable the GIL
+        # when a module that has not declared free-threading support is imported. Our
+        # extension and the free-threaded wheels must leave it disabled on their own.
+        if: endsWith(matrix.python-version, 't')
+        run: |
+          python - <<'PY'
+          import importlib.util, sys, sysconfig
+          assert sysconfig.get_config_var("Py_GIL_DISABLED"), "not a free-threaded interpreter"
+          import numpy, pandas, pyarrow, zstandard  # noqa: F401
+          spec = importlib.util.spec_from_file_location("_pyspark", "python/pyspark/_pyspark.so")
+          spec.loader.exec_module(importlib.util.module_from_spec(spec))
+          assert not sys._is_gil_enabled(), "the GIL was re-enabled by our extension or a test dependency"
+          print("GIL still disabled after loading _pyspark and the test dependencies")
+          # grpcio (imported by the reference client that drives the parity suite) has not
+          # declared free-threading support, which is why the parity steps set PYTHON_GIL=0.
+          # Flag it if that changes, so the override can be dropped.
+          import grpc, grpc_status  # noqa: F401
+          if not sys._is_gil_enabled():
+              print("::notice::grpcio no longer re-enables the GIL; PYTHON_GIL=0 may be unnecessary")
+          PY
+
       - name: Cache Spark ${{ env.SPARK_VERSION }} distribution
         uses: actions/cache@v4
         with:
@@ -154,6 +180,7 @@ jobs:
           # need pandas + pyarrow (installed above) plus pyspark (from SPARK_HOME/python).
           PYSPARK_PYTHON: python
           PYSPARK_DRIVER_PYTHON: python
+          PYTHON_GIL: ${{ endsWith(matrix.python-version, 't') && '0' || '' }}  # see the parity step
         run: |
           SPARK_HOME="${{ steps.spark-download.outputs.SPARK_HOME }}"
           echo "Starting Connect server on ${SPARK_CONNECT_TESTING_REMOTE}"
@@ -183,6 +210,10 @@ jobs:
         env:
           PYTHONUNBUFFERED: "1"   # stream per-file progress into the live log
           RUST_PYSPARK_SO: ${{ github.workspace }}/python/pyspark/_pyspark.so
+          # Free-threaded leg only: force the GIL off. The parity suite drives the reference
+          # client, which imports grpcio; grpcio has not declared free-threading support, so
+          # without this the GIL would be re-enabled and the leg would not test free-threading.
+          PYTHON_GIL: ${{ endsWith(matrix.python-version, 't') && '0' || '' }}
         run: |
           # pipefail so the script's exit code (nonzero on any unexpected failure) gates
           # the job - otherwise `| tee` would mask it and the gate would be falsely green.
@@ -203,11 +234,14 @@ jobs:
           # in the separate local-cluster phase below (mirroring Apache's
           # build_python_connect.yml, which runs exactly these under local-cluster) and
           # are excluded here where the server is plain local[*].
+          # The free-threaded leg adds scripts/parity_known_failures_freethreaded.txt on top
+          # of the shared manifest (environmental failures specific to free-threaded CPython).
           python scripts/run_official_tests.py \
             --spark "${{ steps.spark-source.outputs.SPARK_SOURCE }}" \
             --remote "${SPARK_CONNECT_TESTING_REMOTE}" \
             --jobs 1 \
             --exclude "test_artifact.py,test_artifact_localcluster.py,test_parity_resources.py" \
+            ${{ endsWith(matrix.python-version, 't') && '--manifest scripts/parity_known_failures_freethreaded.txt' || '' }} \
             2>&1 | tee /tmp/parity_ours.log
 
       - name: Restart Connect server in local-cluster mode
@@ -218,6 +252,7 @@ jobs:
         env:
           PYSPARK_PYTHON: python
           PYSPARK_DRIVER_PYTHON: python
+          PYTHON_GIL: ${{ endsWith(matrix.python-version, 't') && '0' || '' }}  # see the parity step
         run: |
           SPARK_HOME="${{ steps.spark-download.outputs.SPARK_HOME }}"
           echo "Stopping the local[*] Connect server..."
@@ -252,6 +287,7 @@ jobs:
         env:
           PYTHONUNBUFFERED: "1"
           RUST_PYSPARK_SO: ${{ github.workspace }}/python/pyspark/_pyspark.so
+          PYTHON_GIL: ${{ endsWith(matrix.python-version, 't') && '0' || '' }}  # see the parity step
         run: |
           set -o pipefail
           # Mirror Apache's local-cluster testnames: resource profiles + artifact

.github/workflows/freethreaded.yml

@@ -0,0 +1,54 @@
+name: Free-threaded build
+
+# Builds the pyspark-rs extension against a free-threaded (no-GIL, PEP 703)
+# CPython. The extension is excluded from the workspace default-members, so the
+# regular Rust CI never compiles it against a free-threaded interpreter -- and
+# compiling against Py_GIL_DISABLED is exactly what enforces that every #[pyclass]
+# is Sync. This workflow is that missing check.
+
+on:
+  push:
+    branches: [master, "branch-*"]
+  pull_request:
+    branches: [master, "branch-*"]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  freethreaded:
+    name: Build the extension against the free-threaded ABI
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13t"
+      - name: Install Rust toolchain
+        shell: bash
+        run: |
+          rustup toolchain install stable --profile minimal
+          rustup default stable
+      - name: Install protoc
+        shell: bash
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+      - name: Build pyspark-rs against the free-threaded interpreter
+        # If any #[pyclass] were not Sync, this build would fail: pyo3 requires
+        # Sync pyclasses on the free-threaded ABI. So a green build proves the
+        # extension is free-threading-safe.
+        env:
+          PYO3_PYTHON: python
+        shell: bash
+        run: cargo build -p pyspark-rs --release
+      - name: Import the extension on the free-threaded interpreter
+        # Load the built extension standalone (avoiding the Python skin's heavier
+        # imports) to confirm it initializes on a free-threaded interpreter, and
+        # report whether the GIL is disabled. Under the pinned pyo3 0.28 a plain
+        # #[pymodule] is free-threading by default (it emits Py_MOD_GIL_NOT_USED),
+        # so the GIL is expected to stay off here (gil_enabled: False).
+        shell: bash
+        run: |
+          cp target/release/lib_pyspark.so /tmp/_pyspark.so
+          cd /tmp
+          python -c "import sys, _pyspark; print('loaded:', _pyspark.__name__); print('gil_enabled:', sys._is_gil_enabled())"

.github/workflows/release.yml

@@ -292,7 +292,37 @@ jobs:
         shell: bash
         run: |
           python -m pip install --upgrade maturin
-          args=(--release --out dist --target "${{ matrix.target }}")
+          # abi3 is passed here (not in pyproject) so the free-threaded leg can build
+          # without it — abi3 and the free-threaded ABI are mutually exclusive.
+          args=(--release --out dist --target "${{ matrix.target }}" --features pyo3/abi3-py39)
+          if [[ "${{ matrix.zig }}" == "true" ]]; then
+            python -m pip install ziglang
+            args+=(--zig)
+          fi
+          maturin build "${args[@]}"
+      - name: Set up free-threaded interpreters
+        if: matrix.target == 'x86_64-unknown-linux-gnu'
+        uses: actions/setup-python@v5
+        with:
+          python-version: |
+            3.13t
+            3.14t
+      - name: Build free-threaded wheels with maturin
+        # The free-threaded (PEP 703) ABI is incompatible with abi3, so these are
+        # separate, version-specific cp313t/cp314t wheels built without the abi3
+        # feature (dropped from pyproject; the abi3 leg passes it explicitly). They
+        # land in the same dist/ and are uploaded alongside the abi3 wheel. x86_64
+        # Linux for now -- the aarch64 free-threaded cross-compile needs pyo3 to
+        # detect the target's free-threaded ABI and is a follow-up.
+        if: matrix.target == 'x86_64-unknown-linux-gnu'
+        shell: bash
+        run: |
+          # The preceding setup-python made a free-threaded interpreter the default
+          # `python`, but maturin and ziglang were pip-installed into 3.11 by the abi3
+          # step. Reinstall them here so maturin runs and can locate zig via
+          # `python -m ziglang` under the now-default free-threaded interpreter.
+          python -m pip install --upgrade maturin
+          args=(--release --out dist --target "${{ matrix.target }}" --interpreter python3.13t python3.14t)
           if [[ "${{ matrix.zig }}" == "true" ]]; then
             python -m pip install ziglang
             args+=(--zig)

pyproject.toml

@@ -69,8 +69,10 @@ manifest-path = "crates/pyspark-rs/Cargo.toml"
 # the sdist with "License-File LICENSE does not exist in distribution file". Ship the
 # LICENSE in the sdist so it satisfies the declared metadata.
 include = [{ path = "LICENSE", format = "sdist" }]
-# Use abi3 for a stable ABI compatible across Python 3.9+.
-features = ["pyo3/abi3-py39", "pyo3/extension-module"]
+# abi3 is driven per build leg, not here: it is incompatible with the free-threaded
+# ABI, so the free-threaded (cp3xxt) wheel leg must build without it, while the
+# stable-ABI leg passes `--features pyo3/abi3-py39` explicitly (see release.yml).
+features = ["pyo3/extension-module"]
 # Keep tests out of the built wheel/sdist: our own offline suite lives outside the package
 # (python/tests), but the vendored pyspark tree carries upstream test modules
 # (e.g. pyspark/pipelines/tests, pyspark/testing/tests) that end users don't need.

scripts/parity_known_failures_freethreaded.txt

@@ -0,0 +1,10 @@
+# Known environmental failures on free-threaded CPython only, applied on top of
+# parity_known_failures.txt by the free-threaded parity leg. Same format and drift check.
+#
+# protobuf publishes its C (upb) backend only as an abi3 wheel, which free-threaded
+# CPython cannot load, so pip installs the pure-Python backend there. The reference
+# client builds every request as a Python protobuf message and serializes it with
+# SerializeToString(); the pure-Python backend serializes recursively and overflows the
+# default recursion limit on very deep plans, where upb does not. The reference client
+# over grpcio hits the same overflow, so this is not a client regression.
+pyspark/sql/tests/connect/test_connect_error.py::SparkConnectErrorTests::test_recursion_handling_for_plan_logging  # 400-deep plan: pure-Python protobuf (no free-threaded upb wheel) overflows the recursion limit serializing the request

scripts/run_official_tests.py

@@ -199,7 +199,13 @@ def main():
     ap.add_argument(
         "--remote", default=os.environ.get("SPARK_CONNECT_TESTING_REMOTE", "sc://localhost:15002")
     )
-    ap.add_argument("--manifest", default=str(DEFAULT_MANIFEST))
+    ap.add_argument(
+        "--manifest",
+        action="append",
+        default=[str(DEFAULT_MANIFEST)],
+        help="Additional known-failures manifest, applied on top of "
+        "scripts/parity_known_failures.txt. May be repeated.",
+    )
     ap.add_argument("--jobs", type=int, default=1)
     ap.add_argument("--timeout", type=int, default=360)
     ap.add_argument(
@@ -257,7 +263,12 @@ def main():
         print("!! no connect test files found under", spark_py)
         return 2
 
-    per_file, whole_file = load_manifest(Path(args.manifest))
+    per_file, whole_file = {}, set()
+    for manifest in args.manifest:
+        m_per_file, m_whole_file = load_manifest(Path(manifest))
+        for relpath, suffixes in m_per_file.items():
+            per_file.setdefault(relpath, []).extend(suffixes)
+        whole_file |= m_whole_file
     n_skip = sum(len(v) for v in per_file.values())
     print(
         f"Discovered {len(files)} connect test files; manifest skips "