diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3080e992..08f77f85 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ permissions: jobs: build-and-release: name: build-run (${{ matrix.name }}) - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.runs-on }} strategy: fail-fast: false @@ -19,10 +19,20 @@ jobs: include: - name: cuda12.9-ubuntu22.04 dockerfile: docker/Dockerfile.cuda12.9.ubuntu22.04 + runs-on: ubuntu-22.04 + platform: linux/amd64 - name: cuda13-ubuntu22.04 dockerfile: docker/Dockerfile.cuda13.x.ubuntu22.04 + runs-on: ubuntu-22.04 + platform: linux/amd64 - name: cuda12.x-rockylinux8 dockerfile: docker/Dockerfile.cuda12.x.rockylinux8 + runs-on: ubuntu-22.04 + platform: linux/amd64 + - name: cuda13-ubuntu22.04-nccl2.30-arm64 + dockerfile: docker/Dockerfile.cuda13.x.ubuntu22.04.nccl2.30.arm64 + runs-on: ubuntu-22.04-arm # native arm64 runner (do NOT use amd64 + QEMU) + platform: linux/arm64 steps: - name: Checkout source @@ -53,7 +63,7 @@ jobs: BUILD_DATE=$(date +%Y%m%d) docker buildx build \ - --platform linux/amd64 \ + --platform ${{ matrix.platform }} \ --progress=plain \ -f ${{ matrix.dockerfile }} \ --build-arg BUILD_DATE=${BUILD_DATE} \ diff --git a/docker/Dockerfile.cuda13.x.ubuntu22.04.nccl2.30 b/docker/Dockerfile.cuda13.x.ubuntu22.04.nccl2.30 new file mode 100644 index 00000000..2f4562b7 --- /dev/null +++ b/docker/Dockerfile.cuda13.x.ubuntu22.04.nccl2.30 @@ -0,0 +1,183 @@ +########################### +# Build-time configuration +########################### + +# Base OS and CUDA versions +ARG UBUNTU_VERSION=22.04 +ARG CUDA_VERSION=13.1.0 +ARG CUDART_VERSION=13.1.80 +ARG CUDART_MAJOR_VERSION=13 + +# NCCL versions (built from source and patched — see step 3) +ARG NCCL_PACKAGE_VERSION=2.30.7-1 +ARG NCCL_SO_VERSION=2.30.7 +# NCCL source build (patched) configuration +# - NCCL_SRC_REF : git tag/branch of NVIDIA/nccl to build +# - NCCL_MAX_IB_DEVS : device cap (raised so dense multi-port hosts use all ports) +# - NVCC_GENCODE : optional gencode override to cut build time (empty = NCCL default) +ARG NCCL_SRC_REF=v2.30.7-1 +ARG NCCL_MAX_IB_DEVS=400 +ARG NVCC_GENCODE= + +# OpenMPI versions +# - MPI_VERSION: full OpenMPI version +# - MPI_SERIES: major.minor series used in download URL +ARG MPI_VERSION=4.1.8 +ARG MPI_SERIES=4.1 + +# Build date (override at build time) +ARG BUILD_DATE=20260128 + +########################### +# Build Stage +########################### +FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} AS build + +# Re-declare build args for this stage (values are inherited) +ARG UBUNTU_VERSION +ARG CUDA_VERSION +ARG CUDART_VERSION +ARG NCCL_PACKAGE_VERSION +ARG NCCL_SO_VERSION +ARG NCCL_SRC_REF +ARG NCCL_MAX_IB_DEVS +ARG NVCC_GENCODE +ARG MPI_VERSION +ARG MPI_SERIES +ARG BUILD_DATE + +ENV DEBIAN_FRONTEND=noninteractive +WORKDIR /workspace + +# ------------------------- +# 1. Base build dependencies +# ------------------------- +RUN { apt-get -o Acquire::http::No-Cache=true update > build.log 2>&1 && \ + apt-get install -y --no-install-recommends \ + build-essential gcc g++ curl git wget ca-certificates \ + make automake autoconf libtool pkg-config \ + python3 python3-pip gzip xz-utils >> build.log 2>&1 && \ + rm -rf /var/lib/apt/lists/* && rm -f build.log; } || (cat build.log && false) + +# ------------------------- +# 2. Install CUDA keyring and restore NVIDIA repository +# ------------------------- +RUN { wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb > build.log 2>&1 && \ + dpkg -i cuda-keyring_1.1-1_all.deb >> build.log 2>&1 && \ + apt-get update >> build.log 2>&1 && rm -f build.log; } || (cat build.log && false) + +# ------------------------- +# 3. Build NCCL from source (patched: bound IB device enumeration to MAX_IB_DEVS) +# Fixes the out-of-bounds write / crash on hosts whose active IB/RoCE ports +# exceed MAX_IB_DEVS. Installs to /usr/local/nccl (lib + headers). +# ------------------------- +RUN { git clone --depth 1 --single-branch -b ${NCCL_SRC_REF} https://github.com/NVIDIA/nccl.git nccl-src > build.log 2>&1 && \ + cd nccl-src && \ + sed -i 's/for (int port_num = 1; port_num <= devAttr.phys_port_cnt; port_num++) {/for (int port_num = 1; port_num <= devAttr.phys_port_cnt \&\& ncclNIbDevs < MAX_IB_DEVS; port_num++) {/' src/transport/net_ib/init.cc && \ + sed -i 's/for (int dev = devOffset; dev < devCount; ++dev) {/for (int dev = devOffset; dev < devCount \&\& ncclNIbDevs < MAX_IB_DEVS; ++dev) {/' src/transport/net_ib/init.cc && \ + { grep -q 'phys_port_cnt && ncclNIbDevs < MAX_IB_DEVS' src/transport/net_ib/init.cc && \ + grep -q 'devCount && ncclNIbDevs < MAX_IB_DEVS' src/transport/net_ib/init.cc; } || \ + { echo "ERROR: NCCL IB-devs guard patch did not apply to ${NCCL_SRC_REF}" >&2; exit 1; } && \ + sed -i -E "s/#define[[:space:]]+MAX_IB_DEVS[[:space:]]+32/#define MAX_IB_DEVS ${NCCL_MAX_IB_DEVS}/" src/transport/net_ib/common.h && \ + { [ -n "${NVCC_GENCODE}" ] || unset NVCC_GENCODE; } && \ + make -j"$(nproc)" src.build ${NVCC_GENCODE:+NVCC_GENCODE="${NVCC_GENCODE}"} >> build.log 2>&1 && \ + mkdir -p /usr/local/nccl/lib /usr/local/nccl/include && \ + cp -P build/lib/libnccl.so* /usr/local/nccl/lib/ && \ + cp -a build/include/. /usr/local/nccl/include/ && \ + echo /usr/local/nccl/lib > /etc/ld.so.conf.d/nccl.conf && ldconfig && \ + cd /workspace && rm -rf nccl-src && rm -f build.log; } || (cat build.log && false) + +# ------------------------- +# 4. Build OpenMPI from source +# ------------------------- +RUN wget https://download.open-mpi.org/release/open-mpi/v${MPI_SERIES}/openmpi-${MPI_VERSION}.tar.gz > /dev/null 2>&1 && \ + tar zxvf openmpi-${MPI_VERSION}.tar.gz > /dev/null 2>&1 && \ + cd openmpi-${MPI_VERSION} && \ + ./configure --prefix=/usr/local/sihpc --with-cuda=/usr/local/cuda > /dev/null 2>&1 && \ + make -j$(nproc) > /dev/null 2>&1 && make install > /dev/null 2>&1 && \ + rm -rf /workspace/openmpi-${MPI_VERSION} /workspace/openmpi-${MPI_VERSION}.tar.gz + +# ------------------------- +# 5. Build nccl-tests +# ------------------------- +RUN git clone --depth 1 --single-branch -b sync/upstream-20251216 https://github.com/scitix/nccl-tests.git > /dev/null 2>&1 && \ + cd nccl-tests && \ + { [ -n "${NVCC_GENCODE}" ] || unset NVCC_GENCODE; } && \ + { make MPI=1 MPI_HOME=/usr/local/sihpc NCCL_HOME=/usr/local/nccl ${NVCC_GENCODE:+NVCC_GENCODE="${NVCC_GENCODE}"} > build.log 2>&1 && rm -f build.log || (cat build.log && false); } && \ + mkdir -p /usr/local/sihpc/libexec/nccl-tests && \ + cp -rf build/*_perf /usr/local/sihpc/libexec/nccl-tests/ && \ + mkdir -p /usr/local/sihpc/bin && \ + cp scripts/nccl_perf /usr/local/sihpc/bin/nccl_perf && \ + cp scripts/nccl_test /usr/local/sihpc/libexec/nccl-tests/nccl_test && \ + cp scripts/env.sh /usr/local/sihpc/env.sh && \ + cp scripts/install_sihpc /usr/local/sihpc/bin/install_sihpc && \ + cp scripts/uninstall_sihpc /usr/local/sihpc/bin/uninstall_sihpc && \ + rm -rf /workspace/nccl-tests + +# ------------------------- +# 6. Collect runtime libraries (strict selection) +# ------------------------- +RUN { set -e && \ + mkdir -p /usr/local/sihpc/lib > build.log 2>&1 && \ + cp /usr/local/cuda/lib64/libcudart* /usr/local/sihpc/lib/ >> build.log 2>&1 && \ + cp -P /usr/local/nccl/lib/libnccl.so* /usr/local/sihpc/lib/ >> build.log 2>&1 && \ + rm -f build.log; } || (cat build.log && false) +# cp /lib/x86_64-linux-gnu/libltdl.so.7.3.1 /usr/local/sihpc/lib/ && \ +# cp /usr/lib/x86_64-linux-gnu/libhwloc.so* /usr/local/sihpc/lib/ && \ +# cp /usr/lib/x86_64-linux-gnu/libevent_core* /usr/local/sihpc/lib/ && \ +# cp /usr/lib/x86_64-linux-gnu/libevent_pthreads* /usr/local/sihpc/lib/ + +# ------------------------- +# 7. Fix library symlinks +# ------------------------- +RUN cd /usr/local/sihpc/lib && \ + rm -f libcudart.so libcudart.so.${CUDART_MAJOR_VERSION} && \ + ln -sf libnccl.so.${NCCL_SO_VERSION} libnccl.so.2 && \ + ln -sf libnccl.so.2 libnccl.so && \ + ln -sf libcudart.so.${CUDART_VERSION} libcudart.so.${CUDART_MAJOR_VERSION} && \ + ln -sf libcudart.so.${CUDART_MAJOR_VERSION} libcudart.so +# rm -f libevent_core-2.1.so.7 && \ +# ln -sf libhwloc.so.15.1.0 libhwloc.so.15 && \ +# ln -sf libhwloc.so.15.1.0 libhwloc.so && \ +# ln -sf libevent_core-2.1.so.7.0.0 libevent_core-2.1.so.7 && \ +# ln -sf libevent_core-2.1.so.7 libevent_core-2.1.so && \ +# ln -sf libevent_pthreads-2.1.so.7.0.0 libevent_pthreads-2.1.so.7 && \ +# ln -sf libevent_pthreads-2.1.so.7 libevent_pthreads-2.1.so && \ +# ln -sf libltdl.so.7.3.1 libltdl.so.7 && \ +# ln -sf libltdl.so.7 libltdl.so + +########################### +# Package Stage +########################### +FROM ubuntu:${UBUNTU_VERSION} AS package + +# Re-declare args for this stage (values are inherited) +ARG UBUNTU_VERSION +ARG NCCL_PACKAGE_VERSION +ARG MPI_VERSION +ARG BUILD_DATE + +# Expose versions/date as environment variables for runtime shell expansion +ENV NCCL_PACKAGE_VERSION=${NCCL_PACKAGE_VERSION} \ + MPI_VERSION=${MPI_VERSION} \ + BUILD_DATE=${BUILD_DATE} + +COPY --from=build /usr/local/sihpc /usr/local/sihpc + +WORKDIR / +RUN apt-get update && apt-get install -y --no-install-recommends makeself && \ + SAFE_NCCL_PKG=$(printf '%s\n' "${NCCL_PACKAGE_VERSION}" | tr '+' '-') && \ + PACKAGE_FILENAME="sicl-nccl${SAFE_NCCL_PKG}-ompi${MPI_VERSION}-ubuntu${UBUNTU_VERSION}-${BUILD_DATE}.run" && \ + { makeself --gzip /usr/local/sihpc \ + "${PACKAGE_FILENAME}" \ + "SiHPC MPI + NCCL + NCCL-tests Portable Installer" \ + ./bin/install_sihpc > build.log 2>&1 && rm -f build.log; } || (cat build.log && false) + +########################### +# Export Stage +########################### +# Use a scratch image so that --output type=local only exports the .run file. +# This avoids permission errors caused by root-owned OS files being written +# to the non-root GitHub Actions runner's workspace. +FROM scratch AS export +COPY --from=package /*.run / \ No newline at end of file diff --git a/docker/Dockerfile.cuda13.x.ubuntu22.04.nccl2.30.arm64 b/docker/Dockerfile.cuda13.x.ubuntu22.04.nccl2.30.arm64 new file mode 100644 index 00000000..eda00b6f --- /dev/null +++ b/docker/Dockerfile.cuda13.x.ubuntu22.04.nccl2.30.arm64 @@ -0,0 +1,178 @@ +########################### +# Build-time configuration +########################### + +# Base OS and CUDA versions +ARG UBUNTU_VERSION=22.04 +ARG CUDA_VERSION=13.1.0 +ARG CUDART_VERSION=13.1.80 +ARG CUDART_MAJOR_VERSION=13 + +# CUDA apt repo architecture: +# x86_64 for amd64, sbsa for datacenter ARM (Grace / GH200 / Ampere Altra). +ARG CUDA_REPO_ARCH=sbsa + +# NCCL versions (built from source and patched — see step 3) +ARG NCCL_PACKAGE_VERSION=2.30.7-1 +ARG NCCL_SO_VERSION=2.30.7 +# NCCL source build (patched) configuration +# - NCCL_SRC_REF : git tag/branch of NVIDIA/nccl to build +# - NCCL_MAX_IB_DEVS : device cap (raised so dense multi-port hosts use all ports) +# - NVCC_GENCODE : optional gencode override to cut build time (empty = NCCL default) +ARG NCCL_SRC_REF=v2.30.7-1 +ARG NCCL_MAX_IB_DEVS=400 +ARG NVCC_GENCODE= + +# OpenMPI versions +# - MPI_VERSION: full OpenMPI version +# - MPI_SERIES: major.minor series used in download URL +ARG MPI_VERSION=4.1.8 +ARG MPI_SERIES=4.1 + +# Build date (override at build time) +ARG BUILD_DATE=20260128 + +########################### +# Build Stage +########################### +FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} AS build + +# Re-declare build args for this stage (values are inherited) +ARG UBUNTU_VERSION +ARG CUDA_VERSION +ARG CUDART_VERSION +ARG CUDA_REPO_ARCH +ARG NCCL_PACKAGE_VERSION +ARG NCCL_SO_VERSION +ARG NCCL_SRC_REF +ARG NCCL_MAX_IB_DEVS +ARG NVCC_GENCODE +ARG MPI_VERSION +ARG MPI_SERIES +ARG BUILD_DATE + +ENV DEBIAN_FRONTEND=noninteractive +WORKDIR /workspace + +# ------------------------- +# 1. Base build dependencies +# ------------------------- +RUN { apt-get -o Acquire::http::No-Cache=true update > build.log 2>&1 && \ + apt-get install -y --no-install-recommends \ + build-essential gcc g++ curl git wget ca-certificates \ + make automake autoconf libtool pkg-config \ + python3 python3-pip gzip xz-utils >> build.log 2>&1 && \ + rm -rf /var/lib/apt/lists/* && rm -f build.log; } || (cat build.log && false) + +# ------------------------- +# 2. Install CUDA keyring and restore NVIDIA repository +# (cuda-keyring is arch-independent; the repo path selects x86_64 vs sbsa) +# ------------------------- +RUN { wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/${CUDA_REPO_ARCH}/cuda-keyring_1.1-1_all.deb > build.log 2>&1 && \ + dpkg -i cuda-keyring_1.1-1_all.deb >> build.log 2>&1 && \ + apt-get update >> build.log 2>&1 && rm -f build.log; } || (cat build.log && false) + +# ------------------------- +# 3. Build NCCL from source (patched: bound IB device enumeration to MAX_IB_DEVS) +# Fixes the out-of-bounds write / crash on hosts whose active IB/RoCE ports +# exceed MAX_IB_DEVS. Installs to /usr/local/nccl (lib + headers). +# ------------------------- +RUN { git clone --depth 1 --single-branch -b ${NCCL_SRC_REF} https://github.com/NVIDIA/nccl.git nccl-src > build.log 2>&1 && \ + cd nccl-src && \ + sed -i 's/for (int port_num = 1; port_num <= devAttr.phys_port_cnt; port_num++) {/for (int port_num = 1; port_num <= devAttr.phys_port_cnt \&\& ncclNIbDevs < MAX_IB_DEVS; port_num++) {/' src/transport/net_ib/init.cc && \ + sed -i 's/for (int dev = devOffset; dev < devCount; ++dev) {/for (int dev = devOffset; dev < devCount \&\& ncclNIbDevs < MAX_IB_DEVS; ++dev) {/' src/transport/net_ib/init.cc && \ + { grep -q 'phys_port_cnt && ncclNIbDevs < MAX_IB_DEVS' src/transport/net_ib/init.cc && \ + grep -q 'devCount && ncclNIbDevs < MAX_IB_DEVS' src/transport/net_ib/init.cc; } || \ + { echo "ERROR: NCCL IB-devs guard patch did not apply to ${NCCL_SRC_REF}" >&2; exit 1; } && \ + sed -i -E "s/#define[[:space:]]+MAX_IB_DEVS[[:space:]]+32/#define MAX_IB_DEVS ${NCCL_MAX_IB_DEVS}/" src/transport/net_ib/common.h && \ + { [ -n "${NVCC_GENCODE}" ] || unset NVCC_GENCODE; } && \ + make -j"$(nproc)" src.build ${NVCC_GENCODE:+NVCC_GENCODE="${NVCC_GENCODE}"} >> build.log 2>&1 && \ + mkdir -p /usr/local/nccl/lib /usr/local/nccl/include && \ + cp -P build/lib/libnccl.so* /usr/local/nccl/lib/ && \ + cp -a build/include/. /usr/local/nccl/include/ && \ + echo /usr/local/nccl/lib > /etc/ld.so.conf.d/nccl.conf && ldconfig && \ + cd /workspace && rm -rf nccl-src && rm -f build.log; } || (cat build.log && false) + +# ------------------------- +# 4. Build OpenMPI from source +# ------------------------- +RUN wget https://download.open-mpi.org/release/open-mpi/v${MPI_SERIES}/openmpi-${MPI_VERSION}.tar.gz > /dev/null 2>&1 && \ + tar zxvf openmpi-${MPI_VERSION}.tar.gz > /dev/null 2>&1 && \ + cd openmpi-${MPI_VERSION} && \ + ./configure --prefix=/usr/local/sihpc --with-cuda=/usr/local/cuda > /dev/null 2>&1 && \ + make -j$(nproc) > /dev/null 2>&1 && make install > /dev/null 2>&1 && \ + rm -rf /workspace/openmpi-${MPI_VERSION} /workspace/openmpi-${MPI_VERSION}.tar.gz + +# ------------------------- +# 5. Build nccl-tests +# ------------------------- +RUN git clone --depth 1 --single-branch -b sync/upstream-20251216 https://github.com/scitix/nccl-tests.git > /dev/null 2>&1 && \ + cd nccl-tests && \ + { [ -n "${NVCC_GENCODE}" ] || unset NVCC_GENCODE; } && \ + { make MPI=1 MPI_HOME=/usr/local/sihpc NCCL_HOME=/usr/local/nccl ${NVCC_GENCODE:+NVCC_GENCODE="${NVCC_GENCODE}"} > build.log 2>&1 && rm -f build.log || (cat build.log && false); } && \ + mkdir -p /usr/local/sihpc/libexec/nccl-tests && \ + cp -rf build/*_perf /usr/local/sihpc/libexec/nccl-tests/ && \ + mkdir -p /usr/local/sihpc/bin && \ + cp scripts/nccl_perf /usr/local/sihpc/bin/nccl_perf && \ + cp scripts/nccl_test /usr/local/sihpc/libexec/nccl-tests/nccl_test && \ + cp scripts/env.sh /usr/local/sihpc/env.sh && \ + cp scripts/install_sihpc /usr/local/sihpc/bin/install_sihpc && \ + cp scripts/uninstall_sihpc /usr/local/sihpc/bin/uninstall_sihpc && \ + rm -rf /workspace/nccl-tests + +# ------------------------- +# 6. Collect runtime libraries (strict selection) +# Note: libcudart lives in /usr/local/cuda/lib64 on both x86_64 and sbsa. +# ------------------------- +RUN { set -e && \ + mkdir -p /usr/local/sihpc/lib > build.log 2>&1 && \ + cp /usr/local/cuda/lib64/libcudart* /usr/local/sihpc/lib/ >> build.log 2>&1 && \ + cp -P /usr/local/nccl/lib/libnccl.so* /usr/local/sihpc/lib/ >> build.log 2>&1 && \ + rm -f build.log; } || (cat build.log && false) + +# ------------------------- +# 7. Fix library symlinks +# ------------------------- +RUN cd /usr/local/sihpc/lib && \ + rm -f libcudart.so libcudart.so.${CUDART_MAJOR_VERSION} && \ + ln -sf libnccl.so.${NCCL_SO_VERSION} libnccl.so.2 && \ + ln -sf libnccl.so.2 libnccl.so && \ + ln -sf libcudart.so.${CUDART_VERSION} libcudart.so.${CUDART_MAJOR_VERSION} && \ + ln -sf libcudart.so.${CUDART_MAJOR_VERSION} libcudart.so + +########################### +# Package Stage +########################### +FROM ubuntu:${UBUNTU_VERSION} AS package + +# Re-declare args for this stage (values are inherited) +ARG UBUNTU_VERSION +ARG NCCL_PACKAGE_VERSION +ARG MPI_VERSION +ARG BUILD_DATE + +# Expose versions/date as environment variables for runtime shell expansion +ENV NCCL_PACKAGE_VERSION=${NCCL_PACKAGE_VERSION} \ + MPI_VERSION=${MPI_VERSION} \ + BUILD_DATE=${BUILD_DATE} + +COPY --from=build /usr/local/sihpc /usr/local/sihpc + +WORKDIR / +RUN apt-get update && apt-get install -y --no-install-recommends makeself && \ + SAFE_NCCL_PKG=$(printf '%s\n' "${NCCL_PACKAGE_VERSION}" | tr '+' '-') && \ + ARCH=$(uname -m) && \ + PACKAGE_FILENAME="sicl-nccl${SAFE_NCCL_PKG}-ompi${MPI_VERSION}-ubuntu${UBUNTU_VERSION}-${ARCH}-${BUILD_DATE}.run" && \ + { makeself --gzip /usr/local/sihpc \ + "${PACKAGE_FILENAME}" \ + "SiHPC MPI + NCCL + NCCL-tests Portable Installer" \ + ./bin/install_sihpc > build.log 2>&1 && rm -f build.log; } || (cat build.log && false) + +########################### +# Export Stage +########################### +# Use a scratch image so that --output type=local only exports the .run file. +# This avoids permission errors caused by root-owned OS files being written +# to the non-root GitHub Actions runner's workspace. +FROM scratch AS export +COPY --from=package /*.run /