From 3cc3fed5e2300a52f9788808a6560ab4b75b109d Mon Sep 17 00:00:00 2001 From: jameswillis Date: Thu, 10 Sep 2026 21:25:01 -0700 Subject: [PATCH 1/8] [SPARK-59432][CONNECT][INFRA] Build free-threaded (cp3xxt) wheels abi3 and the free-threaded (PEP 703) ABI are mutually exclusive, so a free-threaded interpreter cannot load the current cp39-abi3 wheels. Add a Linux cp313t/cp314t wheel build leg to the release workflow (alongside the abi3 leg), and add a PR-time workflow that compiles pyspark-rs against a free-threaded interpreter -- which enforces that every #[pyclass] is Sync -- and imports the built extension. The GIL is actually disabled once the module declares gil_used = false (the companion change); this leg makes a loadable free-threaded wheel exist. --- .github/workflows/freethreaded.yml | 54 ++++++++++++++++++++++++++++++ .github/workflows/release.yml | 21 ++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 .github/workflows/freethreaded.yml diff --git a/.github/workflows/freethreaded.yml b/.github/workflows/freethreaded.yml new file mode 100644 index 0000000..d4f65e6 --- /dev/null +++ b/.github/workflows/freethreaded.yml @@ -0,0 +1,54 @@ +name: Free-threaded build + +# Builds the pyspark-rs extension against a free-threaded (no-GIL, PEP 703) +# CPython. The extension is excluded from the workspace default-members, so the +# regular Rust CI never compiles it against a free-threaded interpreter -- and +# compiling against Py_GIL_DISABLED is exactly what enforces that every #[pyclass] +# is Sync. This workflow is that missing check. + +on: + push: + branches: [master, "branch-*"] + pull_request: + branches: [master, "branch-*"] + workflow_dispatch: + +permissions: + contents: read + +jobs: + freethreaded: + name: Build the extension against the free-threaded ABI + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13t" + - name: Install Rust toolchain + shell: bash + run: | + rustup toolchain install stable --profile minimal + rustup default stable + - name: Install protoc + shell: bash + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + - name: Build pyspark-rs against the free-threaded interpreter + # If any #[pyclass] were not Sync, this build would fail: pyo3 requires + # Sync pyclasses on the free-threaded ABI. So a green build proves the + # extension is free-threading-safe. + env: + PYO3_PYTHON: python + shell: bash + run: cargo build -p pyspark-rs --release + - name: Import the extension on the free-threaded interpreter + # Load the built extension standalone (avoiding the Python skin's heavier + # imports) to confirm it initializes on a free-threaded interpreter, and + # report whether the GIL is disabled. The GIL only actually stays off once + # the module declares gil_used = false (SPARK-59432); until then CPython + # re-enables it on import, which is expected and not a failure here. + shell: bash + run: | + cp target/release/lib_pyspark.so /tmp/_pyspark.so + cd /tmp + python -c "import sys, _pyspark; print('loaded:', _pyspark.__name__); print('gil_enabled:', sys._is_gil_enabled())" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8a9d5ca..65daf72 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -298,6 +298,27 @@ jobs: args+=(--zig) fi maturin build "${args[@]}" + - name: Set up free-threaded interpreters + if: runner.os == 'Linux' + uses: actions/setup-python@v5 + with: + python-version: | + 3.13t + 3.14t + - name: Build free-threaded wheels with maturin + # The free-threaded (PEP 703) ABI is incompatible with abi3, so these are + # separate, version-specific cp313t/cp314t wheels; pyo3 ignores the abi3-py39 + # feature when it detects a free-threaded interpreter. They land in the same + # dist/ and are uploaded alongside the abi3 wheel. Linux only for now -- + # macOS/Windows free-threaded legs can follow. + if: runner.os == 'Linux' + shell: bash + run: | + args=(--release --out dist --target "${{ matrix.target }}" --interpreter python3.13t python3.14t) + if [[ "${{ matrix.zig }}" == "true" ]]; then + args+=(--zig) + fi + maturin build "${args[@]}" - uses: actions/upload-artifact@v4 with: name: wheels-${{ matrix.target }} From 3d0aa2027ba2597f3af05abbda0f2177fd7c9833 Mon Sep 17 00:00:00 2001 From: jameswillis Date: Thu, 10 Sep 2026 23:26:17 -0700 Subject: [PATCH 2/8] [SPARK-59432][CONNECT][INFRA] Fix --zig in the free-threaded leg; correct pyo3 0.28 free-threading note Install maturin and ziglang into the free-threaded interpreter that setup-python makes the default before building, so maturin --zig can locate zig via python -m ziglang (the abi3 step installs them into 3.11, which is no longer the default). Also correct the freethreaded.yml comment: under the pinned pyo3 0.28 a plain #[pymodule] is free-threading by default (Py_MOD_GIL_NOT_USED), so the GIL stays off without any gil_used declaration. --- .github/workflows/freethreaded.yml | 6 +++--- .github/workflows/release.yml | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/freethreaded.yml b/.github/workflows/freethreaded.yml index d4f65e6..33354e4 100644 --- a/.github/workflows/freethreaded.yml +++ b/.github/workflows/freethreaded.yml @@ -44,9 +44,9 @@ jobs: - name: Import the extension on the free-threaded interpreter # Load the built extension standalone (avoiding the Python skin's heavier # imports) to confirm it initializes on a free-threaded interpreter, and - # report whether the GIL is disabled. The GIL only actually stays off once - # the module declares gil_used = false (SPARK-59432); until then CPython - # re-enables it on import, which is expected and not a failure here. + # report whether the GIL is disabled. Under the pinned pyo3 0.28 a plain + # #[pymodule] is free-threading by default (it emits Py_MOD_GIL_NOT_USED), + # so the GIL is expected to stay off here (gil_enabled: False). shell: bash run: | cp target/release/lib_pyspark.so /tmp/_pyspark.so diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 65daf72..ce37eb0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -314,8 +314,14 @@ jobs: if: runner.os == 'Linux' shell: bash run: | + # The preceding setup-python made a free-threaded interpreter the default + # `python`, but maturin and ziglang were pip-installed into 3.11 by the abi3 + # step. Reinstall them here so maturin runs and can locate zig via + # `python -m ziglang` under the now-default free-threaded interpreter. + python -m pip install --upgrade maturin args=(--release --out dist --target "${{ matrix.target }}" --interpreter python3.13t python3.14t) if [[ "${{ matrix.zig }}" == "true" ]]; then + python -m pip install ziglang args+=(--zig) fi maturin build "${args[@]}" From 852215e95413e6ecd0fb915143393e30bc463314 Mon Sep 17 00:00:00 2001 From: jameswillis Date: Fri, 11 Sep 2026 00:06:47 -0700 Subject: [PATCH 3/8] [SPARK-59432][CONNECT][INFRA] Build the free-threaded leg without abi3 Under pyo3 0.28 the abi3 and free-threaded ABIs are mutually exclusive: building the extension with pyo3/abi3-py39 against a free-threaded interpreter fails to compile (no fastcall pymethods). Move abi3-py39 out of the pyproject [tool.maturin] features and pass it only on the stable-ABI leg's maturin --features, so the free-threaded cp3xxt leg builds without it. --- .github/workflows/release.yml | 4 +++- pyproject.toml | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ce37eb0..33aa434 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -292,7 +292,9 @@ jobs: shell: bash run: | python -m pip install --upgrade maturin - args=(--release --out dist --target "${{ matrix.target }}") + # abi3 is passed here (not in pyproject) so the free-threaded leg can build + # without it — abi3 and the free-threaded ABI are mutually exclusive. + args=(--release --out dist --target "${{ matrix.target }}" --features pyo3/abi3-py39) if [[ "${{ matrix.zig }}" == "true" ]]; then python -m pip install ziglang args+=(--zig) diff --git a/pyproject.toml b/pyproject.toml index e6f77f8..023e841 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,8 +69,10 @@ manifest-path = "crates/pyspark-rs/Cargo.toml" # the sdist with "License-File LICENSE does not exist in distribution file". Ship the # LICENSE in the sdist so it satisfies the declared metadata. include = [{ path = "LICENSE", format = "sdist" }] -# Use abi3 for a stable ABI compatible across Python 3.9+. -features = ["pyo3/abi3-py39", "pyo3/extension-module"] +# abi3 is driven per build leg, not here: it is incompatible with the free-threaded +# ABI, so the free-threaded (cp3xxt) wheel leg must build without it, while the +# stable-ABI leg passes `--features pyo3/abi3-py39` explicitly (see release.yml). +features = ["pyo3/extension-module"] # Keep tests out of the built wheel/sdist: our own offline suite lives outside the package # (python/tests), but the vendored pyspark tree carries upstream test modules # (e.g. pyspark/pipelines/tests, pyspark/testing/tests) that end users don't need. From 2bec0ae6f70e991a8efecb6d0793ecd104c9836c Mon Sep 17 00:00:00 2001 From: jameswillis Date: Fri, 11 Sep 2026 00:18:57 -0700 Subject: [PATCH 4/8] [SPARK-59432][CONNECT][INFRA] Limit the free-threaded wheel to x86_64 for now The aarch64 free-threaded leg cross-compiles with the host x86_64 3.13t interpreter and pyo3 0.28 mis-detects the target's free-threaded ABI, failing to compile (no fastcall pymethods). Build the free-threaded (cp313t/cp314t) wheel on x86_64 Linux only; the aarch64 free-threaded cross-compile is a follow-up. --- .github/workflows/release.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 33aa434..f6986a6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -301,7 +301,7 @@ jobs: fi maturin build "${args[@]}" - name: Set up free-threaded interpreters - if: runner.os == 'Linux' + if: matrix.target == 'x86_64-unknown-linux-gnu' uses: actions/setup-python@v5 with: python-version: | @@ -309,11 +309,12 @@ jobs: 3.14t - name: Build free-threaded wheels with maturin # The free-threaded (PEP 703) ABI is incompatible with abi3, so these are - # separate, version-specific cp313t/cp314t wheels; pyo3 ignores the abi3-py39 - # feature when it detects a free-threaded interpreter. They land in the same - # dist/ and are uploaded alongside the abi3 wheel. Linux only for now -- - # macOS/Windows free-threaded legs can follow. - if: runner.os == 'Linux' + # separate, version-specific cp313t/cp314t wheels built without the abi3 + # feature (dropped from pyproject; the abi3 leg passes it explicitly). They + # land in the same dist/ and are uploaded alongside the abi3 wheel. x86_64 + # Linux for now -- the aarch64 free-threaded cross-compile needs pyo3 to + # detect the target's free-threaded ABI and is a follow-up. + if: matrix.target == 'x86_64-unknown-linux-gnu' shell: bash run: | # The preceding setup-python made a free-threaded interpreter the default From 00a1ddd8b3deed056d4ccaaec3ced2556e5fd88a Mon Sep 17 00:00:00 2001 From: jameswillis Date: Thu, 24 Sep 2026 17:11:50 -0700 Subject: [PATCH 5/8] [SPARK-59432][CONNECT][INFRA] Run the connect parity suite on free-threaded Python 3.14t Add a 3.14t leg to the connect parity job so the official connect suite runs through the Rust client on a free-threaded interpreter. The suite drives the reference client, which imports grpcio; grpcio does not declare free-threading support and would re-enable the GIL on import, so the leg forces the GIL off with PYTHON_GIL=0 and asserts it stays off. --- .github/workflows/build_python_connect.yml | 34 ++++++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_python_connect.yml b/.github/workflows/build_python_connect.yml index 831eb28..0bcf23a 100644 --- a/.github/workflows/build_python_connect.yml +++ b/.github/workflows/build_python_connect.yml @@ -57,7 +57,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11"] + # 3.14t is the free-threaded (no-GIL) build, covering the free-threaded wheels. + python-version: ["3.11", "3.14t"] steps: - uses: actions/checkout@v4 @@ -87,15 +88,27 @@ jobs: ~/.cargo/registry ~/.cargo/git target - key: ${{ runner.os }}-cargo-parity-${{ hashFiles('**/Cargo.lock') }} - restore-keys: ${{ runner.os }}-cargo-parity- + key: ${{ runner.os }}-cargo-parity-py${{ matrix.python-version }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-cargo-parity-py${{ matrix.python-version }}- - name: Install system dependencies run: | sudo apt-get update sudo apt-get install -y protobuf-compiler + - name: Keep the GIL disabled on the free-threaded interpreter + # The official suite drives the reference pyspark client, which imports grpcio. + # grpcio has no free-threaded wheel and does not declare free-threading support, + # so importing it would silently re-enable the GIL and this leg would no longer + # test free-threaded execution. Force the GIL off for every process in the job, + # including the Connect server's Python workers. + if: endsWith(matrix.python-version, 't') + run: echo "PYTHON_GIL=0" >> "$GITHUB_ENV" + - name: Install Python dependencies + env: + # grpcio and zstandard build from source on the free-threaded interpreter. + GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS: "4" run: | python -m pip install --upgrade pip python -m pip install pytest pandas pyarrow numpy grpcio grpcio-status googleapis-common-protos cloudpickle zstandard py4j @@ -109,6 +122,21 @@ jobs: cargo build -p pyspark-rs --release cp target/release/lib_pyspark.so python/pyspark/_pyspark.so + - name: Check the free-threaded interpreter keeps the GIL disabled + # Fails the leg if the interpreter is not free-threaded, or if loading the test + # dependencies and our extension turns the GIL back on. + if: endsWith(matrix.python-version, 't') + run: | + python - <<'PY' + import importlib.util, sys, sysconfig + assert sysconfig.get_config_var("Py_GIL_DISABLED"), "not a free-threaded interpreter" + import numpy, pandas, pyarrow, zstandard, grpc, grpc_status # noqa: F401 + spec = importlib.util.spec_from_file_location("_pyspark", "python/pyspark/_pyspark.so") + spec.loader.exec_module(importlib.util.module_from_spec(spec)) + assert not sys._is_gil_enabled(), "the GIL was re-enabled" + print("free-threaded, GIL disabled:", sys.version) + PY + - name: Cache Spark ${{ env.SPARK_VERSION }} distribution uses: actions/cache@v4 with: From 7ea218b38b13d40d8572d0a9020d3a9b3cdca09b Mon Sep 17 00:00:00 2001 From: jameswillis Date: Thu, 24 Sep 2026 17:16:50 -0700 Subject: [PATCH 6/8] [SPARK-59432][CONNECT][INFRA] Force the GIL off only for the parity suite Scope PYTHON_GIL=0 to the parity-suite and Connect-server steps instead of the whole job, so the free-threaded check runs without it and verifies that the extension and test dependencies leave the GIL disabled on their own. --- .github/workflows/build_python_connect.yml | 35 ++++++++++++---------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build_python_connect.yml b/.github/workflows/build_python_connect.yml index 0bcf23a..712deea 100644 --- a/.github/workflows/build_python_connect.yml +++ b/.github/workflows/build_python_connect.yml @@ -96,15 +96,6 @@ jobs: sudo apt-get update sudo apt-get install -y protobuf-compiler - - name: Keep the GIL disabled on the free-threaded interpreter - # The official suite drives the reference pyspark client, which imports grpcio. - # grpcio has no free-threaded wheel and does not declare free-threading support, - # so importing it would silently re-enable the GIL and this leg would no longer - # test free-threaded execution. Force the GIL off for every process in the job, - # including the Connect server's Python workers. - if: endsWith(matrix.python-version, 't') - run: echo "PYTHON_GIL=0" >> "$GITHUB_ENV" - - name: Install Python dependencies env: # grpcio and zstandard build from source on the free-threaded interpreter. @@ -122,19 +113,26 @@ jobs: cargo build -p pyspark-rs --release cp target/release/lib_pyspark.so python/pyspark/_pyspark.so - - name: Check the free-threaded interpreter keeps the GIL disabled - # Fails the leg if the interpreter is not free-threaded, or if loading the test - # dependencies and our extension turns the GIL back on. + - name: Check the extension keeps the GIL disabled + # Runs WITHOUT PYTHON_GIL, so the interpreter decides whether to re-enable the GIL + # when a module that has not declared free-threading support is imported. Our + # extension and the free-threaded wheels must leave it disabled on their own. if: endsWith(matrix.python-version, 't') run: | python - <<'PY' import importlib.util, sys, sysconfig assert sysconfig.get_config_var("Py_GIL_DISABLED"), "not a free-threaded interpreter" - import numpy, pandas, pyarrow, zstandard, grpc, grpc_status # noqa: F401 + import numpy, pandas, pyarrow, zstandard # noqa: F401 spec = importlib.util.spec_from_file_location("_pyspark", "python/pyspark/_pyspark.so") spec.loader.exec_module(importlib.util.module_from_spec(spec)) - assert not sys._is_gil_enabled(), "the GIL was re-enabled" - print("free-threaded, GIL disabled:", sys.version) + assert not sys._is_gil_enabled(), "the GIL was re-enabled by our extension or a test dependency" + print("GIL still disabled after loading _pyspark and the test dependencies") + # grpcio (imported by the reference client that drives the parity suite) has not + # declared free-threading support, which is why the parity steps set PYTHON_GIL=0. + # Flag it if that changes, so the override can be dropped. + import grpc, grpc_status # noqa: F401 + if not sys._is_gil_enabled(): + print("::notice::grpcio no longer re-enables the GIL; PYTHON_GIL=0 may be unnecessary") PY - name: Cache Spark ${{ env.SPARK_VERSION }} distribution @@ -182,6 +180,7 @@ jobs: # need pandas + pyarrow (installed above) plus pyspark (from SPARK_HOME/python). PYSPARK_PYTHON: python PYSPARK_DRIVER_PYTHON: python + PYTHON_GIL: ${{ endsWith(matrix.python-version, 't') && '0' || '' }} # see the parity step run: | SPARK_HOME="${{ steps.spark-download.outputs.SPARK_HOME }}" echo "Starting Connect server on ${SPARK_CONNECT_TESTING_REMOTE}" @@ -211,6 +210,10 @@ jobs: env: PYTHONUNBUFFERED: "1" # stream per-file progress into the live log RUST_PYSPARK_SO: ${{ github.workspace }}/python/pyspark/_pyspark.so + # Free-threaded leg only: force the GIL off. The parity suite drives the reference + # client, which imports grpcio; grpcio has not declared free-threading support, so + # without this the GIL would be re-enabled and the leg would not test free-threading. + PYTHON_GIL: ${{ endsWith(matrix.python-version, 't') && '0' || '' }} run: | # pipefail so the script's exit code (nonzero on any unexpected failure) gates # the job - otherwise `| tee` would mask it and the gate would be falsely green. @@ -246,6 +249,7 @@ jobs: env: PYSPARK_PYTHON: python PYSPARK_DRIVER_PYTHON: python + PYTHON_GIL: ${{ endsWith(matrix.python-version, 't') && '0' || '' }} # see the parity step run: | SPARK_HOME="${{ steps.spark-download.outputs.SPARK_HOME }}" echo "Stopping the local[*] Connect server..." @@ -280,6 +284,7 @@ jobs: env: PYTHONUNBUFFERED: "1" RUST_PYSPARK_SO: ${{ github.workspace }}/python/pyspark/_pyspark.so + PYTHON_GIL: ${{ endsWith(matrix.python-version, 't') && '0' || '' }} # see the parity step run: | set -o pipefail # Mirror Apache's local-cluster testnames: resource profiles + artifact From 88c466eb2622c4f0d61c819cebfd3351de10e7f8 Mon Sep 17 00:00:00 2001 From: jameswillis Date: Thu, 24 Sep 2026 21:59:36 -0700 Subject: [PATCH 7/8] [SPARK-59432][CONNECT][INFRA] Add a free-threaded known-failures manifest On free-threaded CPython pip installs protobuf's pure-Python backend (protobuf ships upb only as an abi3 wheel), which overflows the recursion limit serializing the 400-deep plan in test_recursion_handling_for_plan_logging. The reference client hits the same overflow, so list it in a manifest applied only on the free-threaded leg; run_official_tests.py now accepts --manifest more than once. --- .github/workflows/build_python_connect.yml | 3 +++ scripts/parity_known_failures_freethreaded.txt | 10 ++++++++++ scripts/run_official_tests.py | 15 +++++++++++++-- 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 scripts/parity_known_failures_freethreaded.txt diff --git a/.github/workflows/build_python_connect.yml b/.github/workflows/build_python_connect.yml index 712deea..c5dd5b5 100644 --- a/.github/workflows/build_python_connect.yml +++ b/.github/workflows/build_python_connect.yml @@ -234,11 +234,14 @@ jobs: # in the separate local-cluster phase below (mirroring Apache's # build_python_connect.yml, which runs exactly these under local-cluster) and # are excluded here where the server is plain local[*]. + # The free-threaded leg adds scripts/parity_known_failures_freethreaded.txt on top + # of the shared manifest (environmental failures specific to free-threaded CPython). python scripts/run_official_tests.py \ --spark "${{ steps.spark-source.outputs.SPARK_SOURCE }}" \ --remote "${SPARK_CONNECT_TESTING_REMOTE}" \ --jobs 1 \ --exclude "test_artifact.py,test_artifact_localcluster.py,test_parity_resources.py" \ + ${{ endsWith(matrix.python-version, 't') && '--manifest scripts/parity_known_failures.txt --manifest scripts/parity_known_failures_freethreaded.txt' || '' }} \ 2>&1 | tee /tmp/parity_ours.log - name: Restart Connect server in local-cluster mode diff --git a/scripts/parity_known_failures_freethreaded.txt b/scripts/parity_known_failures_freethreaded.txt new file mode 100644 index 0000000..f17f70d --- /dev/null +++ b/scripts/parity_known_failures_freethreaded.txt @@ -0,0 +1,10 @@ +# Known environmental failures on free-threaded CPython only, applied on top of +# parity_known_failures.txt by the free-threaded parity leg. Same format and drift check. +# +# protobuf publishes its C (upb) backend only as an abi3 wheel, which free-threaded +# CPython cannot load, so pip installs the pure-Python backend there. The reference +# client builds every request as a Python protobuf message and serializes it with +# SerializeToString(); the pure-Python backend serializes recursively and overflows the +# default recursion limit on very deep plans, where upb does not. The reference client +# over grpcio hits the same overflow, so this is not a client regression. +pyspark/sql/tests/connect/test_connect_error.py::SparkConnectErrorTests::test_recursion_handling_for_plan_logging # 400-deep plan: pure-Python protobuf (no free-threaded upb wheel) overflows the recursion limit serializing the request diff --git a/scripts/run_official_tests.py b/scripts/run_official_tests.py index 09ad8d4..fd9f20a 100755 --- a/scripts/run_official_tests.py +++ b/scripts/run_official_tests.py @@ -199,7 +199,13 @@ def main(): ap.add_argument( "--remote", default=os.environ.get("SPARK_CONNECT_TESTING_REMOTE", "sc://localhost:15002") ) - ap.add_argument("--manifest", default=str(DEFAULT_MANIFEST)) + ap.add_argument( + "--manifest", + action="append", + default=None, + help="Known-failures manifest. Repeat to combine several (e.g. the shared list plus " + "an interpreter-specific one). Defaults to scripts/parity_known_failures.txt.", + ) ap.add_argument("--jobs", type=int, default=1) ap.add_argument("--timeout", type=int, default=360) ap.add_argument( @@ -257,7 +263,12 @@ def main(): print("!! no connect test files found under", spark_py) return 2 - per_file, whole_file = load_manifest(Path(args.manifest)) + per_file, whole_file = {}, set() + for manifest in args.manifest or [str(DEFAULT_MANIFEST)]: + m_per_file, m_whole_file = load_manifest(Path(manifest)) + for relpath, suffixes in m_per_file.items(): + per_file.setdefault(relpath, []).extend(suffixes) + whole_file |= m_whole_file n_skip = sum(len(v) for v in per_file.values()) print( f"Discovered {len(files)} connect test files; manifest skips " From 23b290d4953e79509a80e4e4596608a0a8ee3b1b Mon Sep 17 00:00:00 2001 From: jameswillis Date: Fri, 25 Sep 2026 10:56:36 -0700 Subject: [PATCH 8/8] [SPARK-59432][CONNECT][INFRA] Keep the shared manifest as the --manifest default Extra --manifest values append to the default list, so the free-threaded leg only passes its own manifest. --- .github/workflows/build_python_connect.yml | 2 +- scripts/run_official_tests.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_python_connect.yml b/.github/workflows/build_python_connect.yml index c5dd5b5..25ebaab 100644 --- a/.github/workflows/build_python_connect.yml +++ b/.github/workflows/build_python_connect.yml @@ -241,7 +241,7 @@ jobs: --remote "${SPARK_CONNECT_TESTING_REMOTE}" \ --jobs 1 \ --exclude "test_artifact.py,test_artifact_localcluster.py,test_parity_resources.py" \ - ${{ endsWith(matrix.python-version, 't') && '--manifest scripts/parity_known_failures.txt --manifest scripts/parity_known_failures_freethreaded.txt' || '' }} \ + ${{ endsWith(matrix.python-version, 't') && '--manifest scripts/parity_known_failures_freethreaded.txt' || '' }} \ 2>&1 | tee /tmp/parity_ours.log - name: Restart Connect server in local-cluster mode diff --git a/scripts/run_official_tests.py b/scripts/run_official_tests.py index fd9f20a..dc56fc8 100755 --- a/scripts/run_official_tests.py +++ b/scripts/run_official_tests.py @@ -202,9 +202,9 @@ def main(): ap.add_argument( "--manifest", action="append", - default=None, - help="Known-failures manifest. Repeat to combine several (e.g. the shared list plus " - "an interpreter-specific one). Defaults to scripts/parity_known_failures.txt.", + default=[str(DEFAULT_MANIFEST)], + help="Additional known-failures manifest, applied on top of " + "scripts/parity_known_failures.txt. May be repeated.", ) ap.add_argument("--jobs", type=int, default=1) ap.add_argument("--timeout", type=int, default=360) @@ -264,7 +264,7 @@ def main(): return 2 per_file, whole_file = {}, set() - for manifest in args.manifest or [str(DEFAULT_MANIFEST)]: + for manifest in args.manifest: m_per_file, m_whole_file = load_manifest(Path(manifest)) for relpath, suffixes in m_per_file.items(): per_file.setdefault(relpath, []).extend(suffixes)