docker: add arm64 (sbsa) nccl2.30 build variant and wire into release CI

.github/workflows/release.yml

@@ -11,18 +11,28 @@ permissions:
 jobs:
   build-and-release:
     name: build-run (${{ matrix.name }})
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.runs-on }}
 
     strategy:
       fail-fast: false
       matrix:
         include:
           - name: cuda12.9-ubuntu22.04
             dockerfile: docker/Dockerfile.cuda12.9.ubuntu22.04
+            runs-on: ubuntu-22.04
+            platform: linux/amd64
           - name: cuda13-ubuntu22.04
             dockerfile: docker/Dockerfile.cuda13.x.ubuntu22.04
+            runs-on: ubuntu-22.04
+            platform: linux/amd64
           - name: cuda12.x-rockylinux8
             dockerfile: docker/Dockerfile.cuda12.x.rockylinux8
+            runs-on: ubuntu-22.04
+            platform: linux/amd64
+          - name: cuda13-ubuntu22.04-nccl2.30-arm64
+            dockerfile: docker/Dockerfile.cuda13.x.ubuntu22.04.nccl2.30.arm64
+            runs-on: ubuntu-22.04-arm      # native arm64 runner (do NOT use amd64 + QEMU)
+            platform: linux/arm64
 
     steps:
       - name: Checkout source
@@ -53,7 +63,7 @@ jobs:
           BUILD_DATE=$(date +%Y%m%d)
 
           docker buildx build \
-            --platform linux/amd64 \
+            --platform ${{ matrix.platform }} \
             --progress=plain \
             -f ${{ matrix.dockerfile }} \
             --build-arg BUILD_DATE=${BUILD_DATE} \

docker/Dockerfile.cuda13.x.ubuntu22.04.nccl2.30

@@ -0,0 +1,183 @@
+###########################
+# Build-time configuration
+###########################
+
+# Base OS and CUDA versions
+ARG UBUNTU_VERSION=22.04
+ARG CUDA_VERSION=13.1.0
+ARG CUDART_VERSION=13.1.80
+ARG CUDART_MAJOR_VERSION=13
+
+# NCCL versions (built from source and patched — see step 3)
+ARG NCCL_PACKAGE_VERSION=2.30.7-1
+ARG NCCL_SO_VERSION=2.30.7
+# NCCL source build (patched) configuration
+#  - NCCL_SRC_REF     : git tag/branch of NVIDIA/nccl to build
+#  - NCCL_MAX_IB_DEVS : device cap (raised so dense multi-port hosts use all ports)
+#  - NVCC_GENCODE     : optional gencode override to cut build time (empty = NCCL default)
+ARG NCCL_SRC_REF=v2.30.7-1
+ARG NCCL_MAX_IB_DEVS=400
+ARG NVCC_GENCODE=
+
+# OpenMPI versions
+# - MPI_VERSION: full OpenMPI version
+# - MPI_SERIES: major.minor series used in download URL
+ARG MPI_VERSION=4.1.8
+ARG MPI_SERIES=4.1
+
+# Build date (override at build time)
+ARG BUILD_DATE=20260128
+
+###########################
+# Build Stage
+###########################
+FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} AS build
+
+# Re-declare build args for this stage (values are inherited)
+ARG UBUNTU_VERSION
+ARG CUDA_VERSION
+ARG CUDART_VERSION
+ARG NCCL_PACKAGE_VERSION
+ARG NCCL_SO_VERSION
+ARG NCCL_SRC_REF
+ARG NCCL_MAX_IB_DEVS
+ARG NVCC_GENCODE
+ARG MPI_VERSION
+ARG MPI_SERIES
+ARG BUILD_DATE
+
+ENV DEBIAN_FRONTEND=noninteractive
+WORKDIR /workspace
+
+# -------------------------
+# 1. Base build dependencies
+# -------------------------
+RUN { apt-get -o Acquire::http::No-Cache=true update > build.log 2>&1 && \
+    apt-get install -y --no-install-recommends \
+        build-essential gcc g++ curl git wget ca-certificates \
+        make automake autoconf libtool pkg-config \
+        python3 python3-pip gzip xz-utils >> build.log 2>&1 && \
+    rm -rf /var/lib/apt/lists/* && rm -f build.log; } || (cat build.log && false)
+
+# -------------------------
+# 2. Install CUDA keyring and restore NVIDIA repository
+# -------------------------
+RUN { wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb > build.log 2>&1 && \
+    dpkg -i cuda-keyring_1.1-1_all.deb >> build.log 2>&1 && \
+    apt-get update >> build.log 2>&1 && rm -f build.log; } || (cat build.log && false)
+
+# -------------------------
+# 3. Build NCCL from source (patched: bound IB device enumeration to MAX_IB_DEVS)
+#    Fixes the out-of-bounds write / crash on hosts whose active IB/RoCE ports
+#    exceed MAX_IB_DEVS. Installs to /usr/local/nccl (lib + headers).
+# -------------------------
+RUN { git clone --depth 1 --single-branch -b ${NCCL_SRC_REF} https://github.com/NVIDIA/nccl.git nccl-src > build.log 2>&1 && \
+    cd nccl-src && \
+    sed -i 's/for (int port_num = 1; port_num <= devAttr.phys_port_cnt; port_num++) {/for (int port_num = 1; port_num <= devAttr.phys_port_cnt \&\& ncclNIbDevs < MAX_IB_DEVS; port_num++) {/' src/transport/net_ib/init.cc && \
+    sed -i 's/for (int dev = devOffset; dev < devCount; ++dev) {/for (int dev = devOffset; dev < devCount \&\& ncclNIbDevs < MAX_IB_DEVS; ++dev) {/' src/transport/net_ib/init.cc && \
+    { grep -q 'phys_port_cnt && ncclNIbDevs < MAX_IB_DEVS' src/transport/net_ib/init.cc && \
+      grep -q 'devCount && ncclNIbDevs < MAX_IB_DEVS' src/transport/net_ib/init.cc; } || \
+      { echo "ERROR: NCCL IB-devs guard patch did not apply to ${NCCL_SRC_REF}" >&2; exit 1; } && \
+    sed -i -E "s/#define[[:space:]]+MAX_IB_DEVS[[:space:]]+32/#define MAX_IB_DEVS ${NCCL_MAX_IB_DEVS}/" src/transport/net_ib/common.h && \
+    { [ -n "${NVCC_GENCODE}" ] || unset NVCC_GENCODE; } && \
+    make -j"$(nproc)" src.build ${NVCC_GENCODE:+NVCC_GENCODE="${NVCC_GENCODE}"} >> build.log 2>&1 && \
+    mkdir -p /usr/local/nccl/lib /usr/local/nccl/include && \
+    cp -P build/lib/libnccl.so* /usr/local/nccl/lib/ && \
+    cp -a build/include/. /usr/local/nccl/include/ && \
+    echo /usr/local/nccl/lib > /etc/ld.so.conf.d/nccl.conf && ldconfig && \
+    cd /workspace && rm -rf nccl-src && rm -f build.log; } || (cat build.log && false)
+
+# -------------------------
+# 4. Build OpenMPI from source
+# -------------------------
+RUN wget https://download.open-mpi.org/release/open-mpi/v${MPI_SERIES}/openmpi-${MPI_VERSION}.tar.gz > /dev/null 2>&1 && \
+    tar zxvf openmpi-${MPI_VERSION}.tar.gz > /dev/null 2>&1 && \
+    cd openmpi-${MPI_VERSION} && \
+    ./configure --prefix=/usr/local/sihpc --with-cuda=/usr/local/cuda > /dev/null  2>&1 && \
+    make -j$(nproc) > /dev/null 2>&1 && make install > /dev/null 2>&1 && \
+    rm -rf /workspace/openmpi-${MPI_VERSION} /workspace/openmpi-${MPI_VERSION}.tar.gz
+
+# -------------------------
+# 5. Build nccl-tests
+# -------------------------
+RUN git clone --depth 1 --single-branch -b sync/upstream-20251216 https://github.com/scitix/nccl-tests.git > /dev/null 2>&1 && \
+    cd nccl-tests && \
+    { [ -n "${NVCC_GENCODE}" ] || unset NVCC_GENCODE; } && \
+    { make MPI=1 MPI_HOME=/usr/local/sihpc NCCL_HOME=/usr/local/nccl ${NVCC_GENCODE:+NVCC_GENCODE="${NVCC_GENCODE}"} > build.log 2>&1 && rm -f build.log || (cat build.log && false); } && \
+    mkdir -p /usr/local/sihpc/libexec/nccl-tests && \
+    cp -rf build/*_perf /usr/local/sihpc/libexec/nccl-tests/ && \
+    mkdir -p /usr/local/sihpc/bin && \
+    cp scripts/nccl_perf /usr/local/sihpc/bin/nccl_perf && \
+    cp scripts/nccl_test /usr/local/sihpc/libexec/nccl-tests/nccl_test && \
+    cp scripts/env.sh /usr/local/sihpc/env.sh && \
+    cp scripts/install_sihpc /usr/local/sihpc/bin/install_sihpc && \
+    cp scripts/uninstall_sihpc /usr/local/sihpc/bin/uninstall_sihpc && \
+    rm -rf /workspace/nccl-tests
+
+# -------------------------
+# 6. Collect runtime libraries (strict selection)
+# -------------------------
+RUN { set -e && \
+    mkdir -p /usr/local/sihpc/lib > build.log 2>&1 && \
+    cp /usr/local/cuda/lib64/libcudart* /usr/local/sihpc/lib/ >> build.log 2>&1 && \
+    cp -P /usr/local/nccl/lib/libnccl.so* /usr/local/sihpc/lib/ >> build.log 2>&1 && \
+    rm -f build.log; } || (cat build.log && false)
+#    cp /lib/x86_64-linux-gnu/libltdl.so.7.3.1 /usr/local/sihpc/lib/ && \
+#    cp /usr/lib/x86_64-linux-gnu/libhwloc.so* /usr/local/sihpc/lib/ && \
+#    cp /usr/lib/x86_64-linux-gnu/libevent_core* /usr/local/sihpc/lib/ && \
+#    cp /usr/lib/x86_64-linux-gnu/libevent_pthreads* /usr/local/sihpc/lib/
+
+# -------------------------
+# 7. Fix library symlinks
+# -------------------------
+RUN cd /usr/local/sihpc/lib && \
+    rm -f libcudart.so libcudart.so.${CUDART_MAJOR_VERSION} && \
+    ln -sf libnccl.so.${NCCL_SO_VERSION} libnccl.so.2 && \
+    ln -sf libnccl.so.2 libnccl.so && \
+    ln -sf libcudart.so.${CUDART_VERSION} libcudart.so.${CUDART_MAJOR_VERSION} && \
+    ln -sf libcudart.so.${CUDART_MAJOR_VERSION} libcudart.so
+#    rm -f libevent_core-2.1.so.7 && \
+#    ln -sf libhwloc.so.15.1.0 libhwloc.so.15 && \
+#    ln -sf libhwloc.so.15.1.0 libhwloc.so && \
+#    ln -sf libevent_core-2.1.so.7.0.0 libevent_core-2.1.so.7 && \
+#    ln -sf libevent_core-2.1.so.7 libevent_core-2.1.so && \
+#    ln -sf libevent_pthreads-2.1.so.7.0.0 libevent_pthreads-2.1.so.7 && \
+#    ln -sf libevent_pthreads-2.1.so.7 libevent_pthreads-2.1.so && \
+#    ln -sf libltdl.so.7.3.1 libltdl.so.7 && \
+#    ln -sf libltdl.so.7 libltdl.so
+
+###########################
+# Package Stage
+###########################
+FROM ubuntu:${UBUNTU_VERSION} AS package
+
+# Re-declare args for this stage (values are inherited)
+ARG UBUNTU_VERSION
+ARG NCCL_PACKAGE_VERSION
+ARG MPI_VERSION
+ARG BUILD_DATE
+
+# Expose versions/date as environment variables for runtime shell expansion
+ENV NCCL_PACKAGE_VERSION=${NCCL_PACKAGE_VERSION} \
+    MPI_VERSION=${MPI_VERSION} \
+    BUILD_DATE=${BUILD_DATE}
+
+COPY --from=build /usr/local/sihpc /usr/local/sihpc
+
+WORKDIR /
+RUN apt-get update && apt-get install -y --no-install-recommends makeself && \
+    SAFE_NCCL_PKG=$(printf '%s\n' "${NCCL_PACKAGE_VERSION}" | tr '+' '-') && \
+    PACKAGE_FILENAME="sicl-nccl${SAFE_NCCL_PKG}-ompi${MPI_VERSION}-ubuntu${UBUNTU_VERSION}-${BUILD_DATE}.run" && \
+    { makeself --gzip /usr/local/sihpc \
+      "${PACKAGE_FILENAME}" \
+      "SiHPC MPI + NCCL + NCCL-tests Portable Installer" \
+      ./bin/install_sihpc > build.log 2>&1 && rm -f build.log; } || (cat build.log && false)
+
+###########################
+# Export Stage
+###########################
+# Use a scratch image so that --output type=local only exports the .run file.
+# This avoids permission errors caused by root-owned OS files being written
+# to the non-root GitHub Actions runner's workspace.
+FROM scratch AS export
+COPY --from=package /*.run /