diff --git a/.github/workflows/build_python_connect.yml b/.github/workflows/build_python_connect.yml index 831eb285..25ebaab2 100644 --- a/.github/workflows/build_python_connect.yml +++ b/.github/workflows/build_python_connect.yml @@ -57,7 +57,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11"] + # 3.14t is the free-threaded (no-GIL) build, covering the free-threaded wheels. + python-version: ["3.11", "3.14t"] steps: - uses: actions/checkout@v4 @@ -87,8 +88,8 @@ jobs: ~/.cargo/registry ~/.cargo/git target - key: ${{ runner.os }}-cargo-parity-${{ hashFiles('**/Cargo.lock') }} - restore-keys: ${{ runner.os }}-cargo-parity- + key: ${{ runner.os }}-cargo-parity-py${{ matrix.python-version }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-cargo-parity-py${{ matrix.python-version }}- - name: Install system dependencies run: | @@ -96,6 +97,9 @@ jobs: sudo apt-get install -y protobuf-compiler - name: Install Python dependencies + env: + # grpcio and zstandard build from source on the free-threaded interpreter. + GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS: "4" run: | python -m pip install --upgrade pip python -m pip install pytest pandas pyarrow numpy grpcio grpcio-status googleapis-common-protos cloudpickle zstandard py4j @@ -109,6 +113,28 @@ jobs: cargo build -p pyspark-rs --release cp target/release/lib_pyspark.so python/pyspark/_pyspark.so + - name: Check the extension keeps the GIL disabled + # Runs WITHOUT PYTHON_GIL, so the interpreter decides whether to re-enable the GIL + # when a module that has not declared free-threading support is imported. Our + # extension and the free-threaded wheels must leave it disabled on their own. + if: endsWith(matrix.python-version, 't') + run: | + python - <<'PY' + import importlib.util, sys, sysconfig + assert sysconfig.get_config_var("Py_GIL_DISABLED"), "not a free-threaded interpreter" + import numpy, pandas, pyarrow, zstandard # noqa: F401 + spec = importlib.util.spec_from_file_location("_pyspark", "python/pyspark/_pyspark.so") + spec.loader.exec_module(importlib.util.module_from_spec(spec)) + assert not sys._is_gil_enabled(), "the GIL was re-enabled by our extension or a test dependency" + print("GIL still disabled after loading _pyspark and the test dependencies") + # grpcio (imported by the reference client that drives the parity suite) has not + # declared free-threading support, which is why the parity steps set PYTHON_GIL=0. + # Flag it if that changes, so the override can be dropped. + import grpc, grpc_status # noqa: F401 + if not sys._is_gil_enabled(): + print("::notice::grpcio no longer re-enables the GIL; PYTHON_GIL=0 may be unnecessary") + PY + - name: Cache Spark ${{ env.SPARK_VERSION }} distribution uses: actions/cache@v4 with: @@ -154,6 +180,7 @@ jobs: # need pandas + pyarrow (installed above) plus pyspark (from SPARK_HOME/python). PYSPARK_PYTHON: python PYSPARK_DRIVER_PYTHON: python + PYTHON_GIL: ${{ endsWith(matrix.python-version, 't') && '0' || '' }} # see the parity step run: | SPARK_HOME="${{ steps.spark-download.outputs.SPARK_HOME }}" echo "Starting Connect server on ${SPARK_CONNECT_TESTING_REMOTE}" @@ -183,6 +210,10 @@ jobs: env: PYTHONUNBUFFERED: "1" # stream per-file progress into the live log RUST_PYSPARK_SO: ${{ github.workspace }}/python/pyspark/_pyspark.so + # Free-threaded leg only: force the GIL off. The parity suite drives the reference + # client, which imports grpcio; grpcio has not declared free-threading support, so + # without this the GIL would be re-enabled and the leg would not test free-threading. + PYTHON_GIL: ${{ endsWith(matrix.python-version, 't') && '0' || '' }} run: | # pipefail so the script's exit code (nonzero on any unexpected failure) gates # the job - otherwise `| tee` would mask it and the gate would be falsely green. @@ -203,11 +234,14 @@ jobs: # in the separate local-cluster phase below (mirroring Apache's # build_python_connect.yml, which runs exactly these under local-cluster) and # are excluded here where the server is plain local[*]. + # The free-threaded leg adds scripts/parity_known_failures_freethreaded.txt on top + # of the shared manifest (environmental failures specific to free-threaded CPython). python scripts/run_official_tests.py \ --spark "${{ steps.spark-source.outputs.SPARK_SOURCE }}" \ --remote "${SPARK_CONNECT_TESTING_REMOTE}" \ --jobs 1 \ --exclude "test_artifact.py,test_artifact_localcluster.py,test_parity_resources.py" \ + ${{ endsWith(matrix.python-version, 't') && '--manifest scripts/parity_known_failures_freethreaded.txt' || '' }} \ 2>&1 | tee /tmp/parity_ours.log - name: Restart Connect server in local-cluster mode @@ -218,6 +252,7 @@ jobs: env: PYSPARK_PYTHON: python PYSPARK_DRIVER_PYTHON: python + PYTHON_GIL: ${{ endsWith(matrix.python-version, 't') && '0' || '' }} # see the parity step run: | SPARK_HOME="${{ steps.spark-download.outputs.SPARK_HOME }}" echo "Stopping the local[*] Connect server..." @@ -252,6 +287,7 @@ jobs: env: PYTHONUNBUFFERED: "1" RUST_PYSPARK_SO: ${{ github.workspace }}/python/pyspark/_pyspark.so + PYTHON_GIL: ${{ endsWith(matrix.python-version, 't') && '0' || '' }} # see the parity step run: | set -o pipefail # Mirror Apache's local-cluster testnames: resource profiles + artifact diff --git a/.github/workflows/freethreaded.yml b/.github/workflows/freethreaded.yml new file mode 100644 index 00000000..33354e4c --- /dev/null +++ b/.github/workflows/freethreaded.yml @@ -0,0 +1,54 @@ +name: Free-threaded build + +# Builds the pyspark-rs extension against a free-threaded (no-GIL, PEP 703) +# CPython. The extension is excluded from the workspace default-members, so the +# regular Rust CI never compiles it against a free-threaded interpreter -- and +# compiling against Py_GIL_DISABLED is exactly what enforces that every #[pyclass] +# is Sync. This workflow is that missing check. + +on: + push: + branches: [master, "branch-*"] + pull_request: + branches: [master, "branch-*"] + workflow_dispatch: + +permissions: + contents: read + +jobs: + freethreaded: + name: Build the extension against the free-threaded ABI + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13t" + - name: Install Rust toolchain + shell: bash + run: | + rustup toolchain install stable --profile minimal + rustup default stable + - name: Install protoc + shell: bash + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + - name: Build pyspark-rs against the free-threaded interpreter + # If any #[pyclass] were not Sync, this build would fail: pyo3 requires + # Sync pyclasses on the free-threaded ABI. So a green build proves the + # extension is free-threading-safe. + env: + PYO3_PYTHON: python + shell: bash + run: cargo build -p pyspark-rs --release + - name: Import the extension on the free-threaded interpreter + # Load the built extension standalone (avoiding the Python skin's heavier + # imports) to confirm it initializes on a free-threaded interpreter, and + # report whether the GIL is disabled. Under the pinned pyo3 0.28 a plain + # #[pymodule] is free-threading by default (it emits Py_MOD_GIL_NOT_USED), + # so the GIL is expected to stay off here (gil_enabled: False). + shell: bash + run: | + cp target/release/lib_pyspark.so /tmp/_pyspark.so + cd /tmp + python -c "import sys, _pyspark; print('loaded:', _pyspark.__name__); print('gil_enabled:', sys._is_gil_enabled())" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8a9d5ca6..f6986a67 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -292,7 +292,37 @@ jobs: shell: bash run: | python -m pip install --upgrade maturin - args=(--release --out dist --target "${{ matrix.target }}") + # abi3 is passed here (not in pyproject) so the free-threaded leg can build + # without it — abi3 and the free-threaded ABI are mutually exclusive. + args=(--release --out dist --target "${{ matrix.target }}" --features pyo3/abi3-py39) + if [[ "${{ matrix.zig }}" == "true" ]]; then + python -m pip install ziglang + args+=(--zig) + fi + maturin build "${args[@]}" + - name: Set up free-threaded interpreters + if: matrix.target == 'x86_64-unknown-linux-gnu' + uses: actions/setup-python@v5 + with: + python-version: | + 3.13t + 3.14t + - name: Build free-threaded wheels with maturin + # The free-threaded (PEP 703) ABI is incompatible with abi3, so these are + # separate, version-specific cp313t/cp314t wheels built without the abi3 + # feature (dropped from pyproject; the abi3 leg passes it explicitly). They + # land in the same dist/ and are uploaded alongside the abi3 wheel. x86_64 + # Linux for now -- the aarch64 free-threaded cross-compile needs pyo3 to + # detect the target's free-threaded ABI and is a follow-up. + if: matrix.target == 'x86_64-unknown-linux-gnu' + shell: bash + run: | + # The preceding setup-python made a free-threaded interpreter the default + # `python`, but maturin and ziglang were pip-installed into 3.11 by the abi3 + # step. Reinstall them here so maturin runs and can locate zig via + # `python -m ziglang` under the now-default free-threaded interpreter. + python -m pip install --upgrade maturin + args=(--release --out dist --target "${{ matrix.target }}" --interpreter python3.13t python3.14t) if [[ "${{ matrix.zig }}" == "true" ]]; then python -m pip install ziglang args+=(--zig) diff --git a/pyproject.toml b/pyproject.toml index e6f77f83..023e8414 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,8 +69,10 @@ manifest-path = "crates/pyspark-rs/Cargo.toml" # the sdist with "License-File LICENSE does not exist in distribution file". Ship the # LICENSE in the sdist so it satisfies the declared metadata. include = [{ path = "LICENSE", format = "sdist" }] -# Use abi3 for a stable ABI compatible across Python 3.9+. -features = ["pyo3/abi3-py39", "pyo3/extension-module"] +# abi3 is driven per build leg, not here: it is incompatible with the free-threaded +# ABI, so the free-threaded (cp3xxt) wheel leg must build without it, while the +# stable-ABI leg passes `--features pyo3/abi3-py39` explicitly (see release.yml). +features = ["pyo3/extension-module"] # Keep tests out of the built wheel/sdist: our own offline suite lives outside the package # (python/tests), but the vendored pyspark tree carries upstream test modules # (e.g. pyspark/pipelines/tests, pyspark/testing/tests) that end users don't need. diff --git a/scripts/parity_known_failures_freethreaded.txt b/scripts/parity_known_failures_freethreaded.txt new file mode 100644 index 00000000..f17f70df --- /dev/null +++ b/scripts/parity_known_failures_freethreaded.txt @@ -0,0 +1,10 @@ +# Known environmental failures on free-threaded CPython only, applied on top of +# parity_known_failures.txt by the free-threaded parity leg. Same format and drift check. +# +# protobuf publishes its C (upb) backend only as an abi3 wheel, which free-threaded +# CPython cannot load, so pip installs the pure-Python backend there. The reference +# client builds every request as a Python protobuf message and serializes it with +# SerializeToString(); the pure-Python backend serializes recursively and overflows the +# default recursion limit on very deep plans, where upb does not. The reference client +# over grpcio hits the same overflow, so this is not a client regression. +pyspark/sql/tests/connect/test_connect_error.py::SparkConnectErrorTests::test_recursion_handling_for_plan_logging # 400-deep plan: pure-Python protobuf (no free-threaded upb wheel) overflows the recursion limit serializing the request diff --git a/scripts/run_official_tests.py b/scripts/run_official_tests.py index 09ad8d44..dc56fc85 100755 --- a/scripts/run_official_tests.py +++ b/scripts/run_official_tests.py @@ -199,7 +199,13 @@ def main(): ap.add_argument( "--remote", default=os.environ.get("SPARK_CONNECT_TESTING_REMOTE", "sc://localhost:15002") ) - ap.add_argument("--manifest", default=str(DEFAULT_MANIFEST)) + ap.add_argument( + "--manifest", + action="append", + default=[str(DEFAULT_MANIFEST)], + help="Additional known-failures manifest, applied on top of " + "scripts/parity_known_failures.txt. May be repeated.", + ) ap.add_argument("--jobs", type=int, default=1) ap.add_argument("--timeout", type=int, default=360) ap.add_argument( @@ -257,7 +263,12 @@ def main(): print("!! no connect test files found under", spark_py) return 2 - per_file, whole_file = load_manifest(Path(args.manifest)) + per_file, whole_file = {}, set() + for manifest in args.manifest: + m_per_file, m_whole_file = load_manifest(Path(manifest)) + for relpath, suffixes in m_per_file.items(): + per_file.setdefault(relpath, []).extend(suffixes) + whole_file |= m_whole_file n_skip = sum(len(v) for v in per_file.values()) print( f"Discovered {len(files)} connect test files; manifest skips "