diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 00000000..06dd8455 --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,163 @@ +name: CD + +on: + push: + tags: + - '*' + workflow_dispatch: + +jobs: + Linux-OpenCL: + name: Linux OpenCL + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + fetch-tags: true + - name: Script + run: | + docker run --rm -i -v "$GITHUB_WORKSPACE:/workspace" -w /workspace centos:7 bash -s <<'EOF' + set -e -o pipefail + sed -i -e '/^mirrorlist/d;/^#baseurl=/{s,^#,,;s,/mirror,/vault,;}' /etc/yum.repos.d/CentOS*.repo + # yum update -y + yum install -y centos-release-scl epel-release + sed -i -e '/^mirrorlist/d;/^# *baseurl=/{s,^# *,,;s,/mirror,/vault,;}' /etc/yum.repos.d/CentOS*.repo + yum install -y devtoolset-11-gcc-c++ devtoolset-11-libstdc++-static make git ocl-icd-devel + + source /opt/rh/devtoolset-11/enable + g++ --version + ldd --version + + make STATIC_RUNTIME=1 -j "$(nproc)" + cp -vr README.* LICENSE tools/ build-release/ + cd build-release + rm -f -- *.o + ./prpll -h + EOF + - uses: actions/upload-artifact@v7 + with: + name: PRPLL-NTT_linux_x64_opencl + path: build-release/* + + Linux-CUDA: + name: Linux CUDA + + runs-on: ubuntu-latest + strategy: + matrix: + include: + - cuda: '13.2.1' + container: 'rockylinux8' + - cuda: '12.9.2' + container: 'rockylinux8' + - cuda: '11.8.0' + container: 'centos7' + - cuda: '10.2' + container: 'centos7' + fail-fast: false + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + fetch-tags: true + - name: Script + run: | + docker run --rm -i -v "$GITHUB_WORKSPACE:/workspace" -w /workspace "nvcr.io/nvidia/cuda:${{ matrix.cuda }}-devel-${{ matrix.container }}" bash -s <<'EOF' + set -e -o pipefail + if [[ "${{ matrix.container }}" == centos* ]]; then + sed -i -e '/^mirrorlist/d;/^#baseurl=/{s,^#,,;s,/mirror,/vault,;}' /etc/yum.repos.d/CentOS*.repo + # yum update -y + yum install -y centos-release-scl epel-release + sed -i -e '/^mirrorlist/d;/^# *baseurl=/{s,^# *,,;s,/mirror,/vault,;}' /etc/yum.repos.d/CentOS*.repo + yum install -y devtoolset-11-gcc-c++ devtoolset-11-libstdc++-static make git + source /opt/rh/devtoolset-11/enable + else + # dnf update -y + dnf install -y gcc-toolset-11-gcc-c++ make git + source /opt/rh/gcc-toolset-11/enable + fi + + g++ --version + ldd --version + + make CUDA=1 STATIC_RUNTIME=1 STATIC_CUDA=${{ matrix.cuda != '10.2' && '1' || '0' }} -j "$(nproc)" + cp -vr README.* LICENSE tools/ build-cuda/ + cd build-cuda + rm -f -- *.o + if [[ "${{ matrix.cuda }}" == "10.2" ]]; then + cp -v /usr/local/cuda/lib64/{libnvrtc.so.10.2,libnvrtc-builtins.so.10.2} . + fi + # ./prpll -h + EOF + - uses: actions/upload-artifact@v7 + with: + name: PRPLL-NTT_linux_x64_cuda_${{ matrix.cuda }} + path: build-cuda/* + + Windows-OpenCL: + name: Windows OpenCL + + runs-on: windows-2022 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + fetch-tags: true + - uses: step-security/msvc-dev-cmd@v1 + - name: Install OpenCL + run: | + vcpkg install opencl + - name: Before Script + shell: bash + run: | + bash genbundle.sh src/cuda/*.cuh src/cl/*.cl > src/bundle.cpp + printf '"%s"\n' "$(basename "$(git describe --tags --long --always)")" > src/version.inc + - name: Script + run: | + msbuild PRPLL.sln /m /p:Configuration=OpenCL-Release /p:StaticRuntime=true /p:OpenCLRoot=C:\vcpkg\installed\x64-windows + Copy-Item -Recurse README.*, LICENSE, tools\ build-msvc\OpenCL\Release\ + cd build-msvc\OpenCL\Release\ + & .\prpll -h + - uses: actions/upload-artifact@v7 + with: + name: PRPLL-NTT_win_x64_opencl + path: build-msvc/OpenCL/Release/* + + Windows-CUDA: + name: Windows CUDA + + runs-on: windows-2022 + strategy: + matrix: + cuda: ['13.2.1', '12.9.2', '11.8.0', '10.2.89'] + fail-fast: false + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + fetch-tags: true + - uses: step-security/msvc-dev-cmd@v1 + - name: Install CUDA Toolkit + uses: N-Storm/cuda-toolkit@v0.2.34 + with: + cuda: ${{ matrix.cuda }} + - name: Before Script + shell: bash + run: | + bash genbundle.sh src/cuda/*.cuh src/cl/*.cl > src/bundle.cpp + printf '"%s"\n' "$(basename "$(git describe --tags --long --always)")" > src/version.inc + - name: Script + run: | + msbuild PRPLL.sln /m /p:Configuration=CUDA-Release /p:StaticRuntime=true /p:StaticCUDA=${{ matrix.cuda != '10.2.89' && 'true' || 'false' }} + Copy-Item -Recurse README.*, LICENSE, tools\ build-msvc\CUDA\Release\ + cd build-msvc\CUDA\Release\ + if ("${{ matrix.cuda }}" -eq "10.2.89") { + Copy-Item "$env:CUDA_PATH\bin\nvrtc64_102_0.dll", "$env:CUDA_PATH\bin\nvrtc-builtins64_102.dll" . + } + # & .\prpll -h + - uses: actions/upload-artifact@v7 + with: + name: PRPLL-NTT_win_x64_cuda_${{ matrix.cuda }} + path: build-msvc/CUDA/Release/* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3947c982..8037acef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,60 +4,227 @@ on: push: pull_request: schedule: - - cron: '0 0 1 * *' + - cron: '0 0 1 * *' jobs: - Linux: - name: Linux + Kernel-Smoke: + name: Kernel compile + Gerbicz check (POCL CPU, ${{ matrix.os }}) + # The kernels in src/cl are compiled at first use, on the device, so the build jobs below never see them. + # This compiles the FP64 kernel set on POCL's CPU device -- a non-NVIDIA OpenCL target -- and ends in a + # Gerbicz check. Two runners: the arm64 entry puts the kernels through LLVM's AArch64 backend and runs + # the host code on arm64, which the build-only arm jobs above never do. Debug build so the runtime + # asserts are exercised too. The integer-NTT types are left out (far too slow on a CPU device). + # + # POCL_WORK_GROUP_METHOD=loops: with the default (vectorizing) work-group method POCL spends 15-25 minutes + # compiling these kernels on the AVX-512 runner CPUs; with "loops" and POCL 7.2 the whole check takes about + # 10 seconds. Ubuntu's own POCL packages (5.0 on 24.04, 6.0 on 26.04) are slow even with "loops" and the + # conda-forge 7.1 build computes wrong results, so POCL 7.2 is built from source against the distro LLVM + # and cached; a cache hit costs seconds. + # Both runners are handled by the same commands: 26.04-arm has the same LLVM 21 as 26.04. + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-26.04, ubuntu-26.04-arm] + env: + POCL_WORK_GROUP_METHOD: loops + POCL_VERSION: "7.2" + steps: + - uses: actions/checkout@v7 + - name: Install + # Installed unconditionally: the cached libpocl links against the LLVM runtime libraries, so they are + # needed on a cache hit too, not only to build POCL. The 26.04 images already carry llvm-21-dev and + # clang-21, but not libclang-cpp21-dev (which POCL needs); naming them all keeps this independent of + # what a future image happens to ship. + run: | + sudo apt-get update -y + sudo apt-get install -y ocl-icd-opencl-dev cmake ninja-build llvm-21-dev libclang-21-dev libclang-cpp21-dev clang-21 libhwloc-dev + - name: Restore POCL + id: pocl-cache + uses: actions/cache@v4 + with: + path: ~/pocl + key: pocl-${{ env.POCL_VERSION }}-${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-llvm21 + - name: Build POCL + if: steps.pocl-cache.outputs.cache-hit != 'true' + run: | + curl -sSLf "https://github.com/pocl/pocl/archive/refs/tags/v$POCL_VERSION.tar.gz" -o pocl.tar.gz + tar -xzf pocl.tar.gz + cmake -G Ninja -S "pocl-$POCL_VERSION" -B pocl-build -DCMAKE_BUILD_TYPE=Release -DWITH_LLVM_CONFIG=/usr/bin/llvm-config-21 \ + -DENABLE_ICD=ON -DENABLE_TESTS=OFF -DENABLE_EXAMPLES=OFF -DCMAKE_INSTALL_PREFIX="$HOME/pocl" -DPOCL_INSTALL_ICD_VENDORDIR="$HOME/pocl/etc/OpenCL/vendors" + ninja -C pocl-build install + - name: Use POCL + run: echo "OCL_ICD_VENDORS=$HOME/pocl/etc/OpenCL/vendors" >> "$GITHUB_ENV" + - name: Build + run: make DEBUG=1 -O -j "$(nproc)" + - name: Compile the kernels and pass a Gerbicz check + # Step-level bound (not job-level, so a slow apt mirror cannot fail the job): a correct run takes + # well under a minute each. prpll ignores SIGTERM while inside an OpenCL compile, hence -s KILL. + # + # The same check runs under a few -use settings and the residues must all agree. A -use knob is a + # performance setting, so it cannot change the answer: after a fixed number of iterations the residue + # depends only on the exponent. That makes a one-line invariant out of the failure this code is most + # prone to -- a kernel that is correct in the shipped configuration and computes the wrong thing in + # another -- which a single-configuration run cannot see at all. TAIL_KERNELS=0 swaps the double-wide + # tail kernels for single-wide ones, and WMUL=1 changes how carryFused partitions its workgroup. + timeout-minutes: 25 + run: | + cd build-debug + ./prpll -h + want="" + for cfg in "" "-use TAIL_KERNELS=0" "-use WMUL=1"; do + # Start each run from scratch: left alone, prpll would resume from the previous run's savefile. + rm -rf gpuowl-0.log 5000011 + # Dump the log whatever happened, so a timeout or crash still shows how far the kernels got. + # Steps run under "bash -e", so the run has to be guarded: unguarded, a failing run ends the step + # right there and the dump never happens. prpll's own stdout is block-buffered into the runner's + # pipe and is lost when it is killed, so the log file is the only record. + rc=0 + timeout -s KILL 6m ./prpll -device 0 -prp 5000011 -fft 256:2:256 -block 200 -iters 400 $cfg || rc=$? + cat gpuowl-0.log || true + test "$rc" -eq 0 + res=$(grep -aoE 'OK +400 +[0-9a-f]{16}' gpuowl-0.log | grep -oE '[0-9a-f]{16}$') + test -n "$res" + echo "residue with '${cfg:-defaults}': $res" + if [ -z "$want" ]; then + want=$res + elif [ "$res" != "$want" ]; then + echo "::error::-use changed the residue: '$cfg' gives $res, defaults give $want" + exit 1 + fi + done + + - name: Verify a proof under every FFT type + # The Gerbicz check above never reaches tailMulLow: that kernel is used only when a proof is built, + # so it can be wrong while everything above stays green. That is not hypothetical -- the double-wide + # tailMul read a whole line with the wrong lane and no test here noticed, because a PRP run and its + # Gerbicz check are both perfectly happy without it. + # + # Verifying a known-good proof closes that hole cheaply: it runs tailSquare and tailMulLow for a few + # thousand iterations, once per FFT type, and the six here are the ones allShapes() enumerates, so + # every number type (FP64, FP32, GF31, GF61) and every hybrid of them gets exercised. The proof is + # type-agnostic: it is a list of residues, so one file checks all six. + timeout-minutes: 30 + run: | + cd build-debug + for t in 0 1 2 3 4 51; do + rm -rf gpuowl-0.log 786433 + rc=0 + timeout -s KILL 10m ./prpll -device 0 -verify ../test/786433-10.proof -fft "$t:256:2:256" || rc=$? + cat gpuowl-0.log || true + test "$rc" -eq 0 + done + + Linux-OpenCL: + name: Linux OpenCL runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04, ubuntu-24.04] + os: [ubuntu-22.04, ubuntu-24.04, ubuntu-26.04, ubuntu-22.04-arm, ubuntu-24.04-arm, ubuntu-26.04-arm] cxx: [g++, clang++] + exclude: + - os: ubuntu-22.04-arm + cxx: clang++ fail-fast: false env: CXX: ${{ matrix.cxx }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Install run: | sudo apt-get update -y - sudo apt-get install -y cppcheck ocl-icd-opencl-dev pocl-opencl-icd + sudo apt-get install -y ocl-icd-opencl-dev pocl-opencl-icd $CXX --version - name: Script run: | - make prpll -O -j "$(nproc)" - cd build-release + make DEBUG=1 -O -j "$(nproc)" + cd build-debug rm -f -- *.o ./prpll -h - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 if: always() with: - name: ${{ matrix.os }}_${{ matrix.cxx }}_prpll - path: ${{ github.workspace }} - - name: Cppcheck + name: ${{ matrix.os }}_${{ endsWith(matrix.os, '-arm') && 'arm' || 'x86' }}_${{ matrix.cxx }}_opencl_prpll + path: build-debug/ + + Linux-CUDA: + name: Linux CUDA + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-22.04, ubuntu-24.04, ubuntu-26.04] + cxx: [g++, clang++] + fail-fast: false + env: + CXX: ${{ matrix.cxx }} + steps: + - uses: actions/checkout@v7 + - name: Install + run: | + sudo apt-get update -y + sudo apt-get install -y nvidia-cuda-toolkit + $CXX --version + - name: Script + run: | + make DEBUG=1 CUDA=1 -O -j "$(nproc)" + cd build-debug + rm -f -- *.o + ./prpll -h + - uses: actions/upload-artifact@v7 + if: always() + with: + name: ${{ matrix.os }}_${{ endsWith(matrix.os, '-arm') && 'arm' || 'x86' }}_${{ matrix.cxx }}_cuda_prpll + path: build-debug/ + + Cppcheck: + name: Cppcheck + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Install + run: | + sudo apt-get update -y + sudo apt-get install -y cppcheck + - name: Script run: cppcheck --enable=all --force . - - name: Clang-Tidy - if: ${{ matrix.cxx == 'clang++' }} + + Clang-Tidy: + name: Clang-Tidy + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Script run: clang-tidy -checks='bugprone-*,-bugprone-reserved-identifier,cert-*,-cert-dcl37-c,-cert-dcl51-cpp,clang-analyzer-*,concurrency-*,misc-*,-misc-no-recursion,modernize-*,-modernize-use-trailing-return-type,performance-*,portability-*,readability-const-return-type,readability-container-*,readability-duplicate-include,readability-else-after-return,readability-make-member-function-cons,readability-non-const-parameter,readability-redundant-*,readability-simplify-*,readability-string-compare,readability-use-*' -header-filter='.*' src/*.cpp -- -Wall -O3 -std=gnu++20 continue-on-error: true - - name: ShellCheck - run: shopt -s globstar; shellcheck -o avoid-nullary-conditions,check-extra-masked-returns,check-set-e-suppressed,deprecate-which,quote-safe-variables,require-double-brackets -s bash **/*.sh + + ShellCheck: + name: ShellCheck + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Script + run: shopt -s globstar; shellcheck -o avoid-nullary-conditions,check-set-e-suppressed,deprecate-which,quote-safe-variables,require-double-brackets -s bash **/*.sh continue-on-error: true - Windows: - name: Windows + Windows-MSYS2: + name: Windows MSYS2 - runs-on: windows-latest + runs-on: ${{ matrix.os }} strategy: matrix: + os: [windows-latest] # windows-11-arm cxx: [g++, clang++] fail-fast: false env: CXX: ${{ matrix.cxx }} + PACKAGE_PREFIX: mingw-w64-${{ endsWith(matrix.os, '-arm') && 'clang-aarch64' || 'x86_64' }}- steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Before Install run: | echo "C:\msys64\mingw64\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append @@ -66,42 +233,99 @@ jobs: echo "LIBPATH=-LC:\msys64\mingw64\lib" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Install run: | - pacman -S --noconfirm mingw-w64-x86_64-gmp mingw-w64-x86_64-opencl-icd + pacman -S --noconfirm "${env:PACKAGE_PREFIX}opencl-icd" & $env:CXX --version - name: Install Clang if: ${{ matrix.cxx == 'clang++' }} run: | - pacman -S --noconfirm mingw-w64-x86_64-clang + pacman -S --noconfirm "${env:PACKAGE_PREFIX}clang" & $env:CXX --version - name: Script - run: | # Cannot use `make exe`, as the OpenCL ICD Loader does not support static linking - make prpll -O -j $env:NUMBER_OF_PROCESSORS - cd build-release + run: | + make DEBUG=1 -O -j $env:NUMBER_OF_PROCESSORS + cd build-debug rm *.o .\prpll.exe -h - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 + if: always() + with: + name: win_${{ endsWith(matrix.os, '-arm') && 'arm' || 'x86' }}_${{ matrix.cxx }}_prpll + path: build-debug/ + + Windows-MSVC-OpenCL: + name: Windows MSVC OpenCL + + runs-on: windows-2022 + steps: + - uses: actions/checkout@v7 + - uses: step-security/msvc-dev-cmd@v1 + - name: Install OpenCL + run: | + vcpkg install opencl + - name: Before Script + shell: bash + run: | + bash genbundle.sh src/cuda/*.cuh src/cl/*.cl > src/bundle.cpp + printf '"%s"\n' "$(basename "$(git describe --tags --long --always)")" > src/version.inc + - name: Script + run: | + msbuild PRPLL.sln /m /p:Configuration=OpenCL-Debug /p:OpenCLRoot=C:\vcpkg\installed\x64-windows + cd build-msvc\OpenCL\Debug\ + & .\prpll -h + - uses: actions/upload-artifact@v7 if: always() with: - name: win_${{ matrix.cxx }}_prpll - path: ${{ github.workspace }} + name: win_x86_msvc_opencl_prpll + path: build-msvc/ + + Windows-MSVC-CUDA: + name: Windows MSVC CUDA + + runs-on: windows-2022 + steps: + - uses: actions/checkout@v7 + - uses: step-security/msvc-dev-cmd@v1 + - name: Install CUDA Toolkit + uses: N-Storm/cuda-toolkit@v0.2.34 + - name: Before Script + shell: bash + run: | + bash genbundle.sh src/cuda/*.cuh src/cl/*.cl > src/bundle.cpp + printf '"%s"\n' "$(basename "$(git describe --tags --long --always)")" > src/version.inc + - name: Script + run: | + msbuild PRPLL.sln /m /p:Configuration=CUDA-Debug + # cd build-msvc\CUDA\Debug\ + # & .\prpll -h + - uses: actions/upload-artifact@v7 + if: always() + with: + name: win_x86_msvc_cuda_prpll + path: build-msvc/ macOS: name: macOS - runs-on: macos-13 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-26-intel, macos-latest] + fail-fast: false + env: + CXX: g++-15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Install run: | - brew install gcc@14 + $CXX --version - name: Script run: | - make prpll -j "$(sysctl -n hw.ncpu)" - cd build-release + make DEBUG=1 -j "$(sysctl -n hw.ncpu)" + cd build-debug rm -f -- *.o ./prpll -h - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 if: always() with: - name: macos_prpll - path: ${{ github.workspace }} + name: macos_${{ endsWith(matrix.os, '-intel') && 'x86' || 'arm' }}_prpll + path: build-debug/ diff --git a/Makefile b/Makefile index 26bffeda..3d15b836 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,14 @@ -# Use "make DEBUG=1" for a debug build +# Use "make CUDA=1" for a CUDA build, use "make DEBUG=1" for a debug build # The build artifacts are put in the "build-release" subfolder (or "build-debug" for a debug build). # On Windows invoke with "make exe" or "make all" +DEBUG = 0 +CUDA = 0 +STATIC_RUNTIME = 0 +STATIC_CUDA = 0 + # Uncomment below as desired to set a particular compiler or force a debug build: # CXX = g++-12 # DEBUG = 1 @@ -12,41 +17,54 @@ HOST_OS = $(shell uname -s) -ifeq ($(HOST_OS), Darwin) -# Real GCC (not clang), needed for 128-bit floats and std::filesystem::path -CXX = g++-14 +CXX ?= g++ + +ifeq ($(CUDA), 1) + BIN=build-cuda + CUDASRCS1 = clwrap_cuda.cpp cudawrap.cpp + CUDAFLAGS = -DCUDA_BACKEND -Isrc/cuda -I/usr/local/cuda/include + CUDAOBJS = $(CUDASRCS1:%.cpp=$(BIN)/%.o) + ifeq ($(STATIC_CUDA), 1) + OPENCL_LIBS = -L/usr/local/cuda/lib64 -Wl,--start-group -lnvrtc_static -lnvrtc-builtins_static -lnvptxcompiler_static -Wl,--end-group -lcuda -lpthread -ldl + else + OPENCL_LIBS = -L/usr/local/cuda/lib64 -Wl,-rpath,'$$ORIGIN' -lnvrtc -lcuda -lpthread + endif else -CXX = g++ + BIN=build-release + CUDAFLAGS = + CUDAOBJS = + ifeq ($(HOST_OS), Darwin) + OPENCL_LIBS = -framework OpenCL + else + OPENCL_LIBS = -lOpenCL -lpthread + endif endif -ifneq ($(findstring MINGW, $(HOST_OS)), MINGW) -COMMON_FLAGS = -Wall -std=c++20 -static-libstdc++ -static-libgcc -else +COMMON_FLAGS = -Wall -Wextra $(CUDAFLAGS) -std=c++20 + +ifeq ($(STATIC_RUNTIME),1) + LDFLAGS += -static-libstdc++ -static-libgcc + + ifeq ($(findstring MINGW, $(HOST_OS)), MINGW) # For mingw-64 use this: -COMMON_FLAGS = -Wall -std=c++20 -static-libstdc++ -static-libgcc -static + LDFLAGS += -static + endif endif -# -fext-numeric-literals -ifeq ($(HOST_OS), Darwin) -OPENCL_LIBS = -framework OpenCL -else -OPENCL_LIBS = -lOpenCL +ifeq ($(findstring MINGW, $(HOST_OS)), MINGW) + CPPFLAGS += -DWINVER=0x0601 -D_WIN32_WINNT=0x0601 + LDFLAGS += -Wl,--subsystem,console:6.01 endif - +# -fext-numeric-literals ifeq ($(DEBUG), 1) BIN=build-debug - -CXXFLAGS = -g $(COMMON_FLAGS) -STRIP= +CXXFLAGS = -g -Og $(COMMON_FLAGS) else -BIN=build-release - -CXXFLAGS = -O2 -DNDEBUG $(COMMON_FLAGS) -STRIP=-s +CXXFLAGS = -O3 -flto -DNDEBUG $(COMMON_FLAGS) endif @@ -56,7 +74,7 @@ SRCS2 = test.cpp # SRCS=$(addprefix src/, $(SRCS1)) -OBJS = $(SRCS1:%.cpp=$(BIN)/%.o) +OBJS = $(CUDAOBJS) $(SRCS1:%.cpp=$(BIN)/%.o) DEPDIR := $(BIN)/.d $(shell mkdir -p $(DEPDIR) >/dev/null) DEPFLAGS = -MT $@ -MMD -MP -MF $(DEPDIR)/$*.Td @@ -70,35 +88,44 @@ prpll: $(BIN)/prpll amd: $(BIN)/prpll-amd #$(BIN)/test: $(BIN)/test.o -# $(CXX) $(CXXFLAGS) -o $@ $< $(LIBPATH) ${STRIP} +# $(CXX) $(CXXFLAGS) -o $@ $< $(LIBPATH) $(BIN)/prpll: ${OBJS} - $(CXX) $(CXXFLAGS) -o $@ ${OBJS} $(LIBPATH) $(OPENCL_LIBS) ${STRIP} + $(CXX) $(LDFLAGS) $(CXXFLAGS) -o $@ ${OBJS} $(LIBPATH) $(OPENCL_LIBS) # Instead of linking with libOpenCL, link with libamdocl64 $(BIN)/prpll-amd: ${OBJS} - $(CXX) $(CXXFLAGS) -o $@ ${OBJS} $(LIBPATH) -lamdocl64 -L/opt/rocm/lib ${STRIP} + $(CXX) $(LDFLAGS) $(CXXFLAGS) -o $@ ${OBJS} $(LIBPATH) -lamdocl64 -L/opt/rocm/lib clean: - rm -rf build-debug build-release + rm -rf build-debug build-release build-cuda $(BIN)/%.o : src/%.cpp $(DEPDIR)/%.d $(COMPILE.cc) $(OUTPUT_OPTION) $< $(POSTCOMPILE) +$(BIN)/%.o : src/cuda/%.cpp $(DEPDIR)/%.d + $(COMPILE.cc) $(OUTPUT_OPTION) $< + $(POSTCOMPILE) -# src/bundle.cpp is just a wrapping of the OpenCL sources (*.cl) as a C string. +# src/bundle.cpp is just a wrapping of the OpenCL sources (*.cl) as a C string (as well as the CUDA OpenCL translation code) -src/bundle.cpp: genbundle.sh src/cl/*.cl - ./genbundle.sh $^ > src/bundle.cpp +src/bundle.cpp: genbundle.sh src/cuda/*.cuh src/cl/*.cl + bash genbundle.sh $^ > src/bundle.cpp $(DEPDIR)/%.d: ; .PRECIOUS: $(DEPDIR)/%.d src/version.cpp : src/version.inc +# The version string compiled into the binary and reported to PrimeNet in +# every result. Defaults to `git describe` of the checkout; a build from an +# exported tree (no .git) or a packager that wants the upstream string passes +# it explicitly: make VERSION=v8.0-57-g6cb4c12 +VERSION ?= $(shell basename `git describe --tags --long --dirty --always --match 'v/prpll/*'`) + src/version.inc: FORCE - echo \"`basename \`git describe --tags --long --dirty --always --match v/prpll/*\``\" > $(BIN)/version.new + echo \"$(VERSION)\" > $(BIN)/version.new diff -q -N $(BIN)/version.new $@ >/dev/null || mv $(BIN)/version.new $@ echo Version: `cat $@` diff --git a/PRPLL.sln b/PRPLL.sln new file mode 100644 index 00000000..6eaecdd1 --- /dev/null +++ b/PRPLL.sln @@ -0,0 +1,30 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PRPLL", "PRPLL.vcxproj", "{2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + OpenCL-Debug|x64 = OpenCL-Debug|x64 + OpenCL-Release|x64 = OpenCL-Release|x64 + CUDA-Debug|x64 = CUDA-Debug|x64 + CUDA-Release|x64 = CUDA-Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.OpenCL-Debug|x64.ActiveCfg = OpenCL-Debug|x64 + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.OpenCL-Debug|x64.Build.0 = OpenCL-Debug|x64 + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.OpenCL-Release|x64.ActiveCfg = OpenCL-Release|x64 + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.OpenCL-Release|x64.Build.0 = OpenCL-Release|x64 + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.CUDA-Debug|x64.ActiveCfg = CUDA-Debug|x64 + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.CUDA-Debug|x64.Build.0 = CUDA-Debug|x64 + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.CUDA-Release|x64.ActiveCfg = CUDA-Release|x64 + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.CUDA-Release|x64.Build.0 = CUDA-Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {5185D8CA-7025-4E3B-8190-85F2B7B4A526} + EndGlobalSection +EndGlobal diff --git a/PRPLL.vcxproj b/PRPLL.vcxproj new file mode 100644 index 00000000..743cb33d --- /dev/null +++ b/PRPLL.vcxproj @@ -0,0 +1,198 @@ + + + + + OpenCL-Debug + x64 + + + OpenCL-Release + x64 + + + CUDA-Debug + x64 + + + CUDA-Release + x64 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + true + + + + 17.0 + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A} + Win32Proj + PRPLL + PRPLL + 10.0 + + + + Application + true + MultiByte + v143 + + + Application + false + true + MultiByte + v143 + + + + + + + + + true + false + + + $(ProjectDir)build-msvc\OpenCL\Debug\ + $(ProjectDir)build-msvc\obj\OpenCL\Debug\ + prpll + true + + + $(ProjectDir)build-msvc\OpenCL\Release\ + $(ProjectDir)build-msvc\obj\OpenCL\Release\ + prpll + + + $(ProjectDir)build-msvc\CUDA\Debug\ + $(ProjectDir)build-msvc\obj\CUDA\Debug\ + prpll + true + + + $(ProjectDir)build-msvc\CUDA\Release\ + $(ProjectDir)build-msvc\obj\CUDA\Release\ + prpll + + + + Level4 + WIN32;WIN64;_CONSOLE;_CRT_SECURE_NO_WARNINGS;NOMINMAX;%(PreprocessorDefinitions) + stdcpp20 + Sync + true + + + Console + + + + + Disabled + _DEBUG;%(PreprocessorDefinitions) + MultiThreadedDebug + MultiThreadedDebugDLL + ProgramDatabase + EnableFastChecks + + + true + + + + + MaxSpeed + true + true + NDEBUG;%(PreprocessorDefinitions) + MultiThreaded + MultiThreadedDLL + + + true + true + true + + + + + WINVER=0x0601;_WIN32_WINNT=0x0601;%(PreprocessorDefinitions) + + + /SUBSYSTEM:CONSOLE,6.01 %(AdditionalOptions) + + + + + $(OpenCLRoot)\debug\lib;%(AdditionalLibraryDirectories) + OpenCL.lib;%(AdditionalDependencies) + + + + + $(OpenCLRoot)\lib;%(AdditionalLibraryDirectories) + OpenCL.lib;%(AdditionalDependencies) + + + + + CUDA_BACKEND;%(PreprocessorDefinitions) + $(ProjectDir)\src\cuda;$(CUDA_PATH)\include;%(AdditionalIncludeDirectories) + + + $(CUDA_PATH)\lib\x64;%(AdditionalLibraryDirectories) + + + + + cuda.lib;nvrtc.lib;Ws2_32.lib;%(AdditionalDependencies) + + + + + cuda.lib;nvrtc_static.lib;nvrtc-builtins_static.lib;nvptxcompiler_static.lib;User32.lib;Ws2_32.lib;%(AdditionalDependencies) + + + + + diff --git a/README.md b/README.md index 5ae916af..ba6b9ada 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ PRPLL implements two primality tests for Mersenne numbers: PRP ("PRobable Prime") and LL ("Lucas-Lehmer") as the name suggests. -PRPLL is an OpenCL (GPU) program for primality testing Mersenne numbers. +PRPLL is an OpenCL (GPU) and CUDA program for primality testing Mersenne numbers. ## Build @@ -32,12 +32,21 @@ Invoke `make` in the source directory. See `prpll -h` for the command line options. +## License + +This project is licensed under the **GNU General Public License v3.0** - see [LICENSE](LICENSE) for details. + + +## Credits + +[PRPLL](https://github.com/preda/gpuowl) was originally authored (as gpuowl) by **Mihai Preda**. **George Woltman** authored optimizations and NTT code contributions. The CUDA backend was authored by **"Sherpa"** in honor of John Allen Frey. + + ## Why LL For Mersenne primes search, the PRP test is by far preferred over LL, such that LL is not used anymore for search. But LL is still used to verify a prime found by PRP (which is a very rare occurence). - ### Lucas-Lehmer (LL) This is a test that proves whether a Mersenne number is prime or not, but without providing a factor in the case where it is not prime. The Lucas-Lehmer test is very simple to describe: iterate the function f(x)=(x^2 - 2) modulo M(p) starting with the number 4. If diff --git a/README.txt b/README.txt index 64b8dcd9..825fb6c3 100644 --- a/README.txt +++ b/README.txt @@ -9,6 +9,7 @@ In the gpuowl project directory (where the file Makefile is located) run make. This will produce a file "prpll" in the build-debug or build-release subdirectory. Use "make" to do a release build in the "build-release" subdirectory. +Use "make CUDA=1" to produce a CUDA version in the "build-cuda" subdirectory. Use "make DEBUG=1" to produce a debug build in the "build-debug" subdirectory. Use "make exe" for a Windows build. Use "make clean" to remove the "build-debug" and "build-release" directories. @@ -28,3 +29,6 @@ Run you need to fix your OpenCL installation first. 2. run "prpll -h", and verify that it displays a list of devices towards the end. + +3. run "prpll -tune", to tune various PRPLL options for your GPU. + diff --git a/genbundle.sh b/genbundle.sh index ec042bb2..d13521f8 100755 --- a/genbundle.sh +++ b/genbundle.sh @@ -1,3 +1,4 @@ +#!/bin/bash cat < CL_FILE_NAMES\{${names}\}\; +echo "static const std::vector CL_FILE_NAMES{${names}};" cat <& getClFileNames() { return CL_FILE_NAMES; } diff --git a/src/AllocTrac.cpp b/src/AllocTrac.cpp index b951bcd8..1995b719 100644 --- a/src/AllocTrac.cpp +++ b/src/AllocTrac.cpp @@ -1,7 +1,6 @@ // Copyright (C) Mihai Preda. #include "AllocTrac.h" -#include std::atomic AllocTrac::totalAlloc = 0; size_t AllocTrac::maxAlloc = size_t(15) * 1024 * 1024 * 1024; // 15 GB diff --git a/src/AllocTrac.h b/src/AllocTrac.h index 2bcedb4c..0c3169c4 100644 --- a/src/AllocTrac.h +++ b/src/AllocTrac.h @@ -36,8 +36,8 @@ class AllocTrac { AllocTrac(const AllocTrac&) = delete; void operator=(const AllocTrac&) = delete; - AllocTrac(AllocTrac&& rhs) : size(rhs.size) { rhs.size = 0; } - AllocTrac& operator=(AllocTrac&& rhs) { + AllocTrac(AllocTrac&& rhs) noexcept : size(rhs.size) { rhs.size = 0; } + AllocTrac& operator=(AllocTrac&& rhs) noexcept { AllocTrac tmp{std::move(rhs)}; swap(*this, tmp); return *this; diff --git a/src/Args.cpp b/src/Args.cpp index 041202ad..bb6bbe54 100644 --- a/src/Args.cpp +++ b/src/Args.cpp @@ -2,13 +2,14 @@ #include "Args.h" #include "File.h" -#include "FFTConfig.h" #include "clwrap.h" #include "gpuid.h" #include "Proof.h" +#include "version.h" #include #include +#include #include #include #include @@ -16,6 +17,9 @@ #include #include +// This is a copy of the args.verbose level. It allows the CUDA wrapper to access the value. +int prpll_verbose = 0; + int Args::value(const string& key, int valNotFound) const { auto it = flags.find(key); if (it == flags.end()) { return valNotFound; } @@ -50,7 +54,7 @@ vector Args::splitArgLine(const string& inputLine) { ret.push_back({prev, {}}); prev = s; } else { - ret.push_back({prev, s}); + ret.emplace_back(prev, s); prev.clear(); } } @@ -65,14 +69,14 @@ vector Args::splitArgLine(const string& inputLine) { // Splits a string of the form "Foo=bar,C,D=1" into key=value pairs, with value defaulting to "1". vector Args::splitUses(string ss) { // pass by value is intentional vector ret; - std::replace(ss.begin(), ss.end(), ',', ' '); + std::ranges::replace(ss, ',', ' '); std::istringstream iss{ss}; - vector uses{std::istream_iterator{iss}, std::istream_iterator{}}; + vector const uses{std::istream_iterator{iss}, std::istream_iterator{}}; for (const string &s : uses) { auto pos = s.find('='); - string key = (pos == string::npos) ? s : s.substr(0, pos); - string val = (pos == string::npos) ? "1"s : s.substr(pos+1); - ret.push_back({key, val}); + string const key = (pos == string::npos) ? s : s.substr(0, pos); + string const val = (pos == string::npos) ? "1"s : s.substr(pos+1); + ret.emplace_back(key, val); } return ret; } @@ -86,7 +90,7 @@ void Args::readConfig(const fs::path& path) { } } -u32 Args::getProofPow(u32 exponent) const { +u32 Args::getProofPow(u64 exponent) const { if (proofPow == -1) { return ProofSet::bestPower(exponent); } assert(proofPow >= 1); return proofPow; @@ -94,19 +98,18 @@ u32 Args::getProofPow(u32 exponent) const { string Args::tailDir() const { return fs::path{dir}.filename().string(); } -bool Args::hasFlag(const string& key) const { return flags.find(key) != flags.end(); } +bool Args::hasFlag(const string& key) const { return flags.contains(key); } void Args::printHelp() { printf(R"( PRPLL is "PRobable Prime and Lucas-Lehmer Categorizer", AKA "Purple-cat" -PRPLL is under active development and not ready for production use. -PRPLL is an OpenCL (GPU) program for primality testing Mersenne numbers (of the form 2^n - 1). +PRPLL is an OpenCL/CUDA (GPU) program for primality testing Mersenne numbers (of the form 2^n - 1). To check that OpenCL is installed correctly use the command "clinfo". If clinfo does not find any devices or otherwise fails, this program will not run. -This program is tested on Linux/ROCm (AMD GPUs); it may also run on Windows and on Nvidia GPUs. +This program is tested on Linux/ROCm (AMD GPUs); it also runs on Windows and on Nvidia GPUs. For information about Mersenne primes search see https://www.mersenne.org/ @@ -117,8 +120,7 @@ and should be able to run. Worktodo: PRPLL keeps the active tasks in per-worker files worktodo-0.txt, worktodo-1.txt etc in the local directory. These per-worker files are supplied from the global worktodo.txt file if -pool is used. -In turn the global worktodo.txt can be supplied through the primenet.py script, -either the one located at gpuowl/tools/primenet.py or https://download.mersenne.ca/primenet.py +In turn the work files can be supplied through AutoPrimeNet, located at https://download.mersenne.ca/AutoPrimeNet It is also possible to manually add exponents by adding lines of the form "PRP=118063003" to worktodo-.txt @@ -167,17 +169,12 @@ named "config.txt" in the prpll run directory. -cache : use binary kernel cache; useful with repeated use of -roeTune and -tune -roe : measure the Round-Off Error (Z) for more iterations (slow) --use : comma separated list of defines for configuring gpuowl.cl, such as: - -use FAST_BARRIER: on AMD Radeon VII and older AMD GPUs, use a faster barrier(). Do not use - this option on Nvidia GPUs or on RDNA AMD GPUs where it produces errors +-use : comma separated list of defines for configuring openCL code, such as: + -use FAST_BARRIER: on AMD Radeon VII and older AMD GPUs, use a faster barrier(). This option + may not work on Nvidia GPUs or on RDNA AMD GPUs where it produces errors (which are nevertheless detected). -use NO_ASM : do not use __asm() blocks (inline assembly) - -use STATS= : enable carry statistics collection & logging, for the kernel according to : - 1 = CarryFused - 2 = CarryFusedMul - 4 = CarryA - 8 = CarryMul - -use TAIL_KERNELS= : change how tailSquare operates according to : + -use TAIL_KERNELS= : change how tailSquare and tailMul operate according to : 0 = single wide, single kernel 1 = single wide, two kernels 2 = double wide, single kernel @@ -196,15 +193,21 @@ named "config.txt" in the prpll run directory. 1 = All trig values are pre-computed and read from memmory. -use DEBUG : enable asserts in OpenCL kernels (slow, developers) + -use STATS= : enable carry statistics collection & logging (developers), for the kernel according to : + 1 = CarryFused, 2 = CarryFusedMul, 4 = CarryA, 8 = CarryMul -tune : Looks for best settings to include in config.txt. Times many FFTs to find fastest one to test exponents -- written to tune.txt. An -fft can be given on the command line to limit which FFTs are timed. Options are not required. If present, the options are a comma separated list from below. - noconfig - Skip timings to find best config.txt settings + noconfig - Skip timings to find best config.txt settings. + inplace - Skip timings for not-in-place FFTs and NTTs. All nVidia GPUs seem to prefer in-place FFTs and NTTs. fp64 - Tune for settings that affect FP64 FFTs. Time FP64 FFTs for tune.txt. ntt - Tune for settings that affect integer NTTs. Time integer NTTs for tune.txt. + nofp32 - Do not tune for settings that affect FP32 FFTs. Some openCL compilers have trouble with FP32. minexp= - Time FFTs to find the best one for exponents greater than . maxexp= - Time FFTs to find the best one for exponents less than . + fp6431 - Time FP64+M31 FFTs for tune.txt. Only GPUs with great FP64 performance will find this beneficial. + quick= - Use higher values for a quicker, potentially less accurate tune. Val ranges from 1 to 10. -device : select the GPU at position N in the list of devices -uid : select the GPU with the given UID (on ROCm/AMDGPU, Linux) -pci : select the GPU with the given PCI BDF, e.g. "0c:00.0" @@ -219,7 +222,7 @@ Device selection : use one of -uid , -pci , -device , see the list } for (unsigned i = 0; i < deviceIds.size(); ++i) { cl_device_id id = deviceIds[i]; - string bdf = getBdfFromDevice(id); + string const bdf = getBdfFromDevice(id); printf("%2u : %7s | %16s | %-24s | %s | %s\n", i, bdf.c_str(), @@ -238,7 +241,7 @@ Device selection : use one of -uid , -pci , -device , see the list u32 activeSize = 0; float maxBpw = 0; string variants; - for (enum FFT_TYPES type : {FFT64, FFT3161, FFT3261, FFT61}) { + for (enum FFT_TYPES const type : {FFT64, FFT3161, FFT3261, FFT61}) { for (auto c : configs) { if (c.fft_type != type) continue; if (c.size() != activeSize) { @@ -268,9 +271,12 @@ void Args::parse(const string& line) { // conditional defines predicated on a FFT char fftBuf[32]; char configBuf[256]; - sscanf(line.c_str(), "! %31s %255s", fftBuf, configBuf); - string fft = fftBuf; - string config = configBuf; + if (sscanf(line.c_str(), "! %31s %255s", fftBuf, configBuf) != 2) { // otherwise the buffers are uninitialised + log("config line ignored (expected \"! \"): \"%s\"\n", line.c_str()); + return; + } + string const fft = fftBuf; + string const config = configBuf; perFftConfig[fft] = splitUses(config); return; } @@ -284,10 +290,14 @@ void Args::parse(const string& line) { if (key == "-h" || key == "--help") { printHelp(); throw "help"; - } else if (key == "-version") { - // log("PRPLL %s\n", VERSION); + } if (key == "-version") { + // Plain stdout, no log prefix: the flag exists for scripts and launchers + // that record which build wrote a result (Task.cpp reports VERSION to + // PrimeNet), so the one line must be the version and nothing else. + printf("%s\n", (VERSION[0] == 'v') ? VERSION + 1 : VERSION); + fflush(stdout); throw "version"; - } else if (key == "-info") { + } if (key == "-info") { if (s.empty()) { log("-info expects an FFT spec, e.g. -info 1K:13:256\n"); throw "-info "; @@ -296,12 +306,12 @@ void Args::parse(const string& line) { for (const FFTShape& shape : FFTShape::multiSpec(s)) { for (u32 variant = 0; variant <= LAST_VARIANT; variant = next_variant (variant)) { if (variant != LAST_VARIANT && shape.fft_type != FFT64) continue; - FFTConfig fft{shape, variant, CARRY_AUTO}; + FFTConfig const fft{shape, variant, CARRY_AUTO}; log("%12s | %.2f | %5.1f\n", fft.spec().c_str(), fft.maxBpw(), fft.maxExp() / 1'000'000.0); } } throw "info"; - } else if (key == "-od") { + } if (key == "-od") { double od = stod(s); fftOverdrive = 1 + od / 1000; } else if (key == "-roe") { @@ -310,15 +320,17 @@ void Args::parse(const string& line) { } else if (key == "-tune") { doTune = true; if (!s.empty()) { tune = s; } - } else if (key == "-ctune") { - doCtune = true; - if (!s.empty()) { ctune.push_back(s); } +// } else if (key == "-ctune") { +// doCtune = true; +// if (!s.empty()) { ctune.push_back(s); } } else if (key == "-ztune") { doZtune = true; } else if (key == "-carryTune") { carryTune = true; } else if (key == "-verbose" || key == "-v") { - verbose = true; + if (s.empty()) verbose = 1; + else verbose = stoi(s); + prpll_verbose = verbose; } else if (key == "-time") { profile = true; } else if (key == "-workers") { @@ -363,11 +375,14 @@ void Args::parse(const string& line) { } } else if (key == "-maxAlloc" || key == "-maxalloc") { - assert(!s.empty()); + if (s.empty()) { // s.back() below would be undefined + log("-maxAlloc expects a value, e.g. -maxAlloc 4G\n"); + throw "-maxAlloc "; + } u32 multiple = (s.back() == 'G') ? (1u << 30) : (1u << 20); maxAlloc = size_t(stod(s) * multiple + .5); } - else if (key == "-iters") { iters = stoi(s); assert(iters && (iters % 10000 == 0)); } + else if (key == "-iters") { iters = stoi(s); assert(iters > 0); } // any positive count; release never enforced the old multiple-of-10000 rule else if (key == "-prp" || key == "-PRP") { prpExp = stoll(s); } else if (key == "-ll" || key == "-LL") { llExp = stoll(s); } else if (key == "-smallest") { smallest = true; } @@ -380,7 +395,7 @@ void Args::parse(const string& line) { else if (key == "-dir") { dir = s; } else if (key == "-carry") { if (s == "short" || s == "long") { - carry = s == "short" ? CARRY_SHORT : CARRY_LONG; + carry = s == "short" ? CARRY_32 : CARRY_64; } else { log("-carry expects short|long\n"); throw "-carry expects short|long"; @@ -393,8 +408,8 @@ void Args::parse(const string& line) { } } else if (key == "-log") { logStep = stoi(s); - if (logStep % 1000 != 0) { - log("-log must be a multiple of 1000\n"); + if (logStep == 0 || logStep % 1000 != 0) { // 0 would divide by zero in the PRP loop + log("-log must be a positive multiple of 1000\n"); throw "invalid log size"; } } else if (key == "-use") { @@ -408,7 +423,12 @@ void Args::parse(const string& line) { } else if (key == "-unsafeMath") { safeMath = false; } else if (key == "-save") { - nSavefiles = stoi(s); + int const n = stoi(s); + if (n < 1) { // 0 makes Saver::trimFiles index v[-1] + log("-save must be at least 1\n"); + throw "invalid -save value"; + } + nSavefiles = n; } else { log("Argument '%s' '%s' not understood\n", key.c_str(), s.c_str()); throw "args"; @@ -418,7 +438,9 @@ void Args::parse(const string& line) { void Args::setDefaults() { uid = getUidFromPos(device); - log("device %d, OpenCL %s, unique id '%s'\n", device, getDriverVersionByPos(device).c_str(), uid.c_str()); + cl_device_id dev = getDevice(device); + log("device %d, OpenCL %s, %s, unique id '%s'\n", device, getDriverVersionByPos(device).c_str(), + isAmdGpu(dev) ? getBoardName(dev).c_str() : getDeviceName(dev).c_str(), uid.c_str()); if (!masterDir.empty()) { assert(masterDir.is_absolute()); diff --git a/src/Args.h b/src/Args.h index 795cd99c..5f1a1a0c 100644 --- a/src/Args.h +++ b/src/Args.h @@ -3,6 +3,7 @@ #pragma once #include "common.h" +#include "FFTConfig.h" #include #include @@ -21,19 +22,17 @@ class Args { static vector splitUses(std::string ss); static std::string mergeArgs(int argc, char **argv); - enum {CARRY_AUTO = 0, CARRY_SHORT, CARRY_LONG}; - explicit Args(bool silent = false) : silent{silent} {} - + void parse(const string& line); void setDefaults(); - bool uses(const std::string& key) const { return flags.find(key) != flags.end(); } - int value(const std::string& key, int valNotFound = -1) const; + [[nodiscard]] bool uses(const std::string& key) const { return flags.contains(key); } + [[nodiscard]] int value(const std::string& key, int valNotFound = -1) const; void readConfig(const fs::path& path); - u32 getProofPow(u32 exponent) const; - string tailDir() const; + [[nodiscard]] u32 getProofPow(u64 exponent) const; + [[nodiscard]] string tailDir() const; - bool hasFlag(const string& key) const; + [[nodiscard]] bool hasFlag(const string& key) const; bool silent; string user; @@ -56,10 +55,10 @@ class Args { std::map> perFftConfig; int device = 0; - + bool safeMath = true; bool clean = true; - bool verbose = false; + int verbose = 0; bool useCache = false; bool profile = false; bool smallest = false; @@ -72,14 +71,14 @@ class Args { bool keepProof = false; - int carry = CARRY_AUTO; + enum CARRY_KIND carry = CARRY_AUTO; u32 workers = 1; u32 blockSize = 1000; u32 logStep = 20000; string fftSpec; - u32 prpExp = 0; - u32 llExp = 0; + u64 prpExp = 0; + u64 llExp = 0; size_t maxAlloc = 0; diff --git a/src/Background.h b/src/Background.h index 96cbe804..c93bf17f 100644 --- a/src/Background.h +++ b/src/Background.h @@ -18,7 +18,7 @@ class Background { std::deque > tasks; std::mutex mut; std::condition_variable cond; - bool stopRequested; + bool stopRequested{false}; std::jthread thread; void run() { @@ -30,9 +30,8 @@ class Background { while (tasks.empty()) { if (stopRequested) { return; - } else { - cond.wait(lock); - } + } cond.wait(lock); + } task = tasks.front(); } @@ -48,7 +47,7 @@ class Background { } { - std::unique_lock lock(mut); + std::unique_lock const lock(mut); assert(!tasks.empty()); tasks.pop_front(); if (tasks.size() == maxSize - 1 || tasks.empty()) { cond.notify_all(); } @@ -59,12 +58,12 @@ class Background { public: Background(unsigned size = 2) : maxSize{size}, - stopRequested(false), + thread{&Background::run, this} { } ~Background() { - std::lock_guard lock(mut); + std::scoped_lock const lock(mut); stopRequested = true; cond.notify_all(); } @@ -74,7 +73,7 @@ class Background { while (!tasks.empty()) { cond.wait(lock); } } - template void operator()(T task) { + template void operator()(const T& task) { std::unique_lock lock(mut); while (tasks.size() >= maxSize) { cond.wait(lock); diff --git a/src/Buffer.h b/src/Buffer.h index 7866975d..b7796261 100644 --- a/src/Buffer.h +++ b/src/Buffer.h @@ -28,16 +28,17 @@ class Buffer { TimeInfo *tInfo; Buffer(cl_context context, TimeInfo *tInfo, Queue* queue, size_t size, unsigned flags, const T* ptr = nullptr) - : ptr{size == 0 ? NULL : makeBuf_(context, flags, size * sizeof(T), ptr)} + : ptr{size == 0 ? nullptr : makeBuf_(context, flags, size * sizeof(T), ptr)} , size{size} , allocTrac(size * sizeof(T)) , queue{queue} , tInfo{tInfo} {} - void fill(T value, u32 len) { - assert(len <= size); - queue->fillBuf(get(), value, (len ? len : size) * sizeof(T), tInfo); + void fill(T value, size_t sizeOrFull = 0) { + assert(sizeOrFull <= size); + auto fillSize = sizeOrFull ? sizeOrFull : size; + queue->fillBuf(get(), value, fillSize * sizeof(T), tInfo); } public: @@ -49,15 +50,15 @@ class Buffer { Buffer(TimeInfo *tInfo, Queue* queue, size_t size) : Buffer(queue->context->get(), tInfo, queue, size, CL_MEM_READ_WRITE /*| CL_MEM_HOST_NO_ACCESS*/) {} - Buffer(Buffer&& rhs) = default; + Buffer(Buffer&& rhs) noexcept = default; - Buffer& operator=(Buffer&& rhs) { + Buffer& operator=(Buffer&& rhs) noexcept { assert(size == rhs.size); std::swap(ptr, rhs.ptr); return *this; } - cl_mem get() const { return ptr.get(); } + [[nodiscard]] cl_mem get() const { return ptr.get(); } void read(T* out, size_t readSize) const { assert(readSize && readSize <= size); @@ -68,7 +69,7 @@ class Buffer { void read(vector& v) const { read(v.data(), v.size()); } // sync read - vector read(size_t sizeOrFull = 0) const { + [[nodiscard]] vector read(size_t sizeOrFull = 0) const { auto readSize = sizeOrFull ? sizeOrFull : size; vector ret(readSize); read(ret); @@ -84,8 +85,8 @@ class Buffer { void write(const vector& vect) { queue->write(get(), vect, tInfo); } - void zero(size_t len = 0) { - fill(0, len); + void zero(size_t sizeOrFull = 0) { + fill(0, sizeOrFull); } void set(T value) { diff --git a/src/Context.h b/src/Context.h index 3f281061..ec14002d 100644 --- a/src/Context.h +++ b/src/Context.h @@ -11,5 +11,5 @@ class Context : public std::unique_ptr { public: explicit Context(cl_device_id id): unique_ptr{createContext(id)}, id{id} {} - cl_device_id deviceId() const { return id; } + [[nodiscard]] cl_device_id deviceId() const { return id; } }; diff --git a/src/CycleFile.cpp b/src/CycleFile.cpp index 0b1db8fd..c2488093 100644 --- a/src/CycleFile.cpp +++ b/src/CycleFile.cpp @@ -3,6 +3,9 @@ #include "CycleFile.h" #include "fs.h" +#include +#include + CycleFile::CycleFile(const fs::path& name) : name{name}, f{File::openWrite(name + ".new")} @@ -12,6 +15,12 @@ CycleFile::CycleFile(const fs::path& name) : CycleFile::~CycleFile() { if (!f) { return; } f.reset(); + if (std::uncaught_exceptions() > uncaughtAtStart) { + // The write threw (WriteError on a full disk, say): keep the previous file, drop the partial one. + std::error_code ec; + fs::remove(name + ".new", ec); + return; + } fancyRename(name + ".new", name); } diff --git a/src/CycleFile.h b/src/CycleFile.h index 816c3456..e02e8163 100644 --- a/src/CycleFile.h +++ b/src/CycleFile.h @@ -2,6 +2,8 @@ #pragma once +#include + #include "File.h" #include @@ -21,4 +23,8 @@ class CycleFile { // Cancel the rename void reset(); + + // Exceptions in flight when this object was created; if more are in flight when it is destroyed, the write + // is being unwound (e.g. disk full) and the partial .new file must not replace the previous good file. + int uncaughtAtStart = std::uncaught_exceptions(); }; diff --git a/src/Event.cpp b/src/Event.cpp index f6098085..bd235177 100644 --- a/src/Event.cpp +++ b/src/Event.cpp @@ -4,6 +4,7 @@ #include "TimeInfo.h" #include +#include Event::Event(EventHolder&& e, TimeInfo* tInfo) : event{std::move(e)}, @@ -13,7 +14,7 @@ Event::Event(EventHolder&& e, TimeInfo* tInfo) : } Event::~Event() { - [[maybe_unused]] bool done = isComplete(); + [[maybe_unused]] bool const done = isComplete(); assert(done); } diff --git a/src/Event.h b/src/Event.h index 98810a5b..479391a7 100644 --- a/src/Event.h +++ b/src/Event.h @@ -17,7 +17,7 @@ class Event { Event(Event&& oth) = default; ~Event(); - cl_event get() const { return event.get(); } + [[nodiscard]] cl_event get() const { return event.get(); } bool isComplete(); bool isRunning(); diff --git a/src/FFTConfig.cpp b/src/FFTConfig.cpp index 2308a037..80f4a5a9 100644 --- a/src/FFTConfig.cpp +++ b/src/FFTConfig.cpp @@ -23,7 +23,7 @@ struct FftBpw { array bpw; }; -map> BPW { +static map> BPW { #include "fftbpw.h" }; @@ -32,9 +32,9 @@ namespace { u32 parseInt(const string& s) { // if (s.empty()) { return 1; } assert(!s.empty()); - char c = s.back(); - u32 multiple = c == 'k' || c == 'K' ? 1024 : c == 'm' || c == 'M' ? 1024 * 1024 : 1; - return strtod(s.c_str(), nullptr) * multiple; + char const c = s.back(); + u32 const multiple = c == 'k' || c == 'K' ? 1024 : c == 'm' || c == 'M' ? 1024 * 1024 : 1; + return u32(strtod(s.c_str(), nullptr) * multiple); } } // namespace @@ -52,26 +52,31 @@ vector FFTShape::multiSpec(const string& iniSpec) { for (const string &spec : split(iniSpec, ',')) { enum FFT_TYPES fft_type = FFT64; + bool hasTypePrefix = false; auto parts = split(spec, ':'); if (parseInt(parts[0]) < 60) { // Look for a prefix specifying the FFT type fft_type = (enum FFT_TYPES) parseInt(parts[0]); parts = vector(next(parts.begin()), parts.end()); + hasTypePrefix = true; } assert(parts.size() <= 3); if (parts.size() == 3) { - u32 width = parseInt(parts[0]); - u32 middle = parseInt(parts[1]); - u32 height = parseInt(parts[2]); - ret.push_back({fft_type, width, middle, height}); + u32 const width = parseInt(parts[0]); + u32 const middle = parseInt(parts[1]); + u32 const height = parseInt(parts[2]); + ret.emplace_back(fft_type, width, middle, height); continue; } assert(parts.size() == 1); - parts = split(spec, '-'); - assert(parts.size() >= 1 && parts.size() <= 2); - u32 sizeFrom = parseInt(parts[0]); - u32 sizeTo = parts.size() == 2 ? parseInt(parts[1]) : sizeFrom; + // Parse the size range from the part after the type prefix (splitting the whole spec would read "1:8M" as 1). + auto range = split(parts[0], '-'); + assert(!range.empty() && range.size() <= 2); + u32 const sizeFrom = parseInt(range[0]); + u32 const sizeTo = range.size() == 2 ? parseInt(range[1]) : sizeFrom; auto shapes = allShapes(sizeFrom, sizeTo); + // allShapes() enumerates every FFT type; an explicit prefix asks for one of them. + if (hasTypePrefix) { std::erase_if(shapes, [fft_type](const FFTShape& sh) { return sh.fft_type != fft_type; }); } if (shapes.empty()) { log("Could not find a FFT config for '%s'\n", spec.c_str()); throw "Invalid FFT spec"; @@ -83,21 +88,20 @@ vector FFTShape::multiSpec(const string& iniSpec) { vector FFTShape::allShapes(u32 sizeFrom, u32 sizeTo) { vector configs; - for (enum FFT_TYPES type : {FFT64, FFT3161, FFT3261, FFT61, FFT323161}) { - for (u32 width : {256, 512, 1024, 4096}) { - for (u32 height : {256, 512, 1024}) { - if (width == 256 && height == 1024) { continue; } // Skip because we prefer width >= height - for (u32 middle : {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}) { - if (type != FFT64 && (middle & (middle - 1))) continue; // Reject non-power-of-two NTTs - u32 sz = width * height * middle * 2; + for (enum FFT_TYPES const type : {FFT64, FFT6431, FFT3161, FFT3261, FFT61, FFT323161}) { + for (u32 const width : {256, 512, 1024, 4096}) { + for (u32 const height : {256, 512, 1024}) { + for (u32 const middle : {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}) { + if (type != FFT64 && type != FFT32 && (middle & (middle - 1))) continue; // Reject non-power-of-two NTTs + u32 const sz = width * height * middle * 2; if (sizeFrom <= sz && sz <= sizeTo) { - configs.push_back({type, width, middle, height}); + configs.emplace_back(type, width, middle, height); } } } } } - std::sort(configs.begin(), configs.end(), + std::ranges::sort(configs, [](const FFTShape &a, const FFTShape &b) { if (a.size() != b.size()) { return (a.size() < b.size()); } if (a.width != b.width) { @@ -126,10 +130,6 @@ FFTShape::FFTShape(enum FFT_TYPES t, const string& w, const string& m, const str FFTShape{t, parseInt(w), parseInt(m), parseInt(h)} {} -FFTShape::FFTShape(u32 w, u32 m, u32 h) : - FFTShape(FFT64, w, m, h) { -} - FFTShape::FFTShape(enum FFT_TYPES t, u32 w, u32 m, u32 h) : fft_type{t}, width{w}, middle{m}, height{h} { assert(w && m && h); @@ -137,31 +137,45 @@ FFTShape::FFTShape(enum FFT_TYPES t, u32 w, u32 m, u32 h) : // Un-initialized shape, don't set BPW if (w == 1 && m == 1 && h == 1) { return; } - string s = spec(); + // Same limits FFTConfig applies to a full spec. Shapes can also arrive here from -tune / -info / size ranges, and a + // too-small one (e.g. 128:2:128) drives middle to 0 in the fallback below and then loops forever. + if ((w != 256 && w != 512 && w != 1024 && w != 4096) || m < 2 || m > 16 || (h != 256 && h != 512 && h != 1024)) { + log("Invalid FFT shape %u:%u:%u (width 256/512/1024/4096, middle 2..16, height 256/512/1024)\n", w, m, h); + throw "Invalid FFT shape"; + } + + string const s = spec(); if (auto it = BPW.find(s); it != BPW.end()) { bpw = it->second; } else { if (height > width) { - bpw = FFTShape{h, m, w}.bpw; + bpw = FFTShape{t, h, m, w}.bpw; } else { - // Make up some defaults - - //double d = 0.275 * (log2(size()) - log2(256 * 13 * 1024 * 2)); - //bpw = {18.1-d, 18.2-d, 18.2-d, 18.3-d}; - //log("BPW info for %s not found, defaults={%.2f, %.2f, %.2f, %.2f}\n", s.c_str(), bpw[0], bpw[1], bpw[2], bpw[3]); - // Manipulate the shape into something that was likely pre-computed + u32 const orig_w = w; + u32 const orig_m = m; + u32 const orig_h = h; while (m < 9) { m *= 2; w /= 2; } while (w >= 4*h) { w /= 2; h *= 2; } while (w < h || w < 256 || w == 2048) { w *= 2; h /= 2; } while (h < 256) { h *= 2; m /= 2; } - if (m == 1) m = 2; - bpw = FFTShape{w, m, h}.bpw; - for (u32 j = 0; j < NUM_BPW_ENTRIES; ++j) bpw[j] -= 0.05; // Assume this fft spec is worse than measured fft specs - if (this->isFavoredShape()) { // Don't output this warning message for non-favored shapes (we expect the BPW info to be missing) - printf("BPW info for %s not found, defaults={", s.c_str()); - for (u32 j = 0; j < NUM_BPW_ENTRIES; ++j) printf("%s%.2f", j ? ", " : "", (double) bpw[j]); - printf("}\n"); + if (m < 2) m = 2; + + // Make up some defaults (should only happen for experimental FFT types (t >= 52) + if (w == orig_w && m == orig_m && h == orig_h) { + bpw = {18.1f, 18.1f, 18.1f, 18.1f, 18.1f, 18.1f}; + log("ERROR: BPW info for %s not found, using default of 18.1.\n", s.c_str()); + } + + // Try the modified shape + else { + bpw = FFTShape{t, w, m, h}.bpw; + for (u32 j = 0; j < NUM_BPW_ENTRIES; ++j) bpw[j] -= 0.05f; // Assume this fft spec is worse than measured fft specs + if (this->isFavoredShape()) { // Don't output this warning message for non-favored shapes (we expect the BPW info to be missing) + printf("BPW info for %s not found, defaults={", s.c_str()); + for (u32 j = 0; j < NUM_BPW_ENTRIES; ++j) printf("%s%.2f", j ? ", " : "", (double) bpw[j]); + printf("}\n"); + } } } } @@ -175,31 +189,40 @@ float FFTShape::carry32BPW() const { // We model carry with a Gumbel distrib similar to the one used for ROE, and measure carry with // -use STATS=1. See -carryTune -//GW: I have no idea why this is needed. Without it, -tune fails on FFT sizes from 256K to 1M -// Perhaps it has something to do with RNDVALdoubleToLong in carryutil -if (18.35 + 0.5 * (log2(13 * 1024 * 512) - log2(size())) > 19.0) return 19.0; - - return 18.35 + 0.5 * (log2(13 * 1024 * 512) - log2(size())); + // The 19.0 cap is a hard limit of the CARRY32 code, not an empirical one, which is why the formula + // above must be clamped for the smaller FFTs (without it, -tune failed on FFT sizes from 256K to 1M). + // + // In the CARRY32 case weightAndCarryOne() returns the raw bits of RNDVAL + value rather than stripping + // RNDVAL off (carryutil.cl:217-219), so bit 51 is set iff value >= 0 and bits 52+ hold the exponent. + // carryStep(i64, i32*) then takes the carry as xtract32(x, nBits), i.e. bits [nBits, nBits+32). That + // window must stay strictly below bit 51 for the sign fill to work, so nBits <= 19, and since a big word + // has nBits = EXP / NWORDS + 1 that means EXP / NWORDS <= 18, i.e. bpw < 19.0. At nBits = 20 the window + // includes bit 51 and every big word with a non-negative value yields a large negative carry. + if (18.35 + 0.5 * (log2(13 * 1024 * 512) - log2(size())) > 19.0) return 19.0f; + + return float(18.35 + 0.5 * (log2(13 * 1024 * 512) - log2(size()))); } -bool FFTShape::needsLargeCarry(u32 E) const { +bool FFTShape::needsLargeCarry(u64 E) const { + // carry32BPW() caps at 19.0 and the comparison below is a strict >, so E == 19 * size() would still + // select CARRY32 with EXP / NWORDS == 19, which the CARRY32 code cannot handle (see carry32BPW()). + // Test the kernel's own EXP / NWORDS expression to close that off-by-one. + if (E / size() >= 19) { return true; } return E / double(size()) > carry32BPW(); } // Return TRUE for "favored" shapes. That is, those that are most likely to be useful. To save time in generating bpw data, only these favored // shapes have their bpw data pre-computed. Bpw for non-favored shapes is guessed from the bpw data we do have. Also. -tune will normally only // time favored shapes. These are the rules for deciding favored shapes: -// WIDTH >= HEIGHT // WIDTH=4K: HEIGHT>=512, MIDDLE>=9 (2*8 combos) // WIDTH=1K: MIDDLE>=5 (3*12 combos) // WIDTH=512: MIDDLE>=4 (2*13 combos) // WIDTH=256: MIDDLE>=1 (16 combos) bool FFTShape::isFavoredShape() const { - return width >= height && - ((width == 4096 && height >= 512 && middle >= 9) || - (width == 1024 && middle >= 5) || - (width == 512 && middle >= 4) || - (width == 256 && middle >= 1)); + return ((width == 4096 && height >= 512 && middle >= 9) || + (width == 1024 && middle >= 5) || + (width == 512 && middle >= 4) || + (width == 256 && middle >= 1)); } FFTConfig::FFTConfig(const string& spec) { @@ -212,14 +235,45 @@ FFTConfig::FFTConfig(const string& spec) { v.resize(v.size() - 1); } + // Sanity check the spec + if (v.size() >= 3) { + u32 const w = parseInt(v[0]); + u32 const m = parseInt(v[1]); + u32 const h = parseInt(v[2]); + if (w != 256 && w != 512 && w != 1024 && w != 4096) { + log("Width must be 256, 512, 1024, or 4096.\n"); + throw "Invalid FFT spec"; + } + if (m < 2 || m > 16) { + log("Middle must be between 2 and 16.\n"); + throw "Invalid FFT spec"; + } + if (h != 256 && h != 512 && h != 1024) { + log("Height must be 256, 512, 1024.\n"); + throw "Invalid FFT spec"; + } + if (fft_type != FFT64 && fft_type != FFT32 && (m & (m - 1))) { + log("NTT middle must be a power of two.\n"); + throw "Invalid FFT spec"; + } + } + if (v.size() == 1) { - *this = {FFTShape::multiSpec(spec).front(), LAST_VARIANT, CARRY_AUTO}; - } if (v.size() == 3) { + // A bare size ("8M") means an FFT of that size of the requested type -- FP64 unless a prefix says otherwise. + // multiSpec() returns every type for an unprefixed size, sorted without regard to type, so pick ours explicitly. + auto shapes = FFTShape::multiSpec(spec); + std::erase_if(shapes, [fft_type](const FFTShape& sh) { return sh.fft_type != fft_type; }); + if (shapes.empty()) { + log("No FFT of type %d with size '%s'\n", int(fft_type), v[0].c_str()); + throw "Invalid FFT spec"; + } + *this = {shapes.front(), LAST_VARIANT, CARRY_AUTO}; + } else if (v.size() == 3) { *this = {FFTShape{fft_type, v[0], v[1], v[2]}, LAST_VARIANT, CARRY_AUTO}; } else if (v.size() == 4) { *this = {FFTShape{fft_type, v[0], v[1], v[2]}, parseInt(v[3]), CARRY_AUTO}; } else if (v.size() == 5) { - int c = parseInt(v[4]); + int const c = parseInt(v[4]); assert(c == 0 || c == 1); *this = {FFTShape{fft_type, v[0], v[1], v[2]}, parseInt(v[3]), c == 0 ? CARRY_32 : CARRY_64}; } else { @@ -227,29 +281,32 @@ FFTConfig::FFTConfig(const string& spec) { } } -FFTConfig::FFTConfig(FFTShape shape, u32 variant, u32 carry) : +FFTConfig::FFTConfig(FFTShape shape, u32 variant, enum CARRY_KIND carry) : shape{shape}, variant{variant}, carry{carry} { - assert(variant_W(variant) < N_VARIANT_W); - assert(variant_M(variant) < N_VARIANT_M); - assert(variant_H(variant) < N_VARIANT_H); - - if (shape.fft_type == FFT64) FFT_FP64 = 1, FFT_FP32 = 0, NTT_GF31 = 0, NTT_GF61 = 0, WordSize = 4; - else if (shape.fft_type == FFT3161) FFT_FP64 = 0, FFT_FP32 = 0, NTT_GF31 = 1, NTT_GF61 = 1, WordSize = 8; - else if (shape.fft_type == FFT3261) FFT_FP64 = 0, FFT_FP32 = 1, NTT_GF31 = 0, NTT_GF61 = 1, WordSize = 8; - else if (shape.fft_type == FFT61) FFT_FP64 = 0, FFT_FP32 = 0, NTT_GF31 = 0, NTT_GF61 = 1, WordSize = 4; - else if (shape.fft_type == FFT323161) FFT_FP64 = 0, FFT_FP32 = 1, NTT_GF31 = 1, NTT_GF61 = 1, WordSize = 8; - else if (shape.fft_type == FFT3231) FFT_FP64 = 0, FFT_FP32 = 1, NTT_GF31 = 1, NTT_GF61 = 0, WordSize = 4; - else if (shape.fft_type == FFT6431) FFT_FP64 = 1, FFT_FP32 = 0, NTT_GF31 = 1, NTT_GF61 = 0, WordSize = 8; - else if (shape.fft_type == FFT31) FFT_FP64 = 0, FFT_FP32 = 0, NTT_GF31 = 1, NTT_GF61 = 0, WordSize = 4; - else if (shape.fft_type == FFT32) FFT_FP64 = 0, FFT_FP32 = 1, NTT_GF31 = 0, NTT_GF61 = 0, WordSize = 4; + // Checked at runtime, not only asserted: an out-of-range digit indexes past bpw[] in maxBpw() and selects kernel + // variants that do not exist (the shipped tune.txt predates this encoding and has such rows). + if (variant_W(variant) >= N_VARIANT_W || variant_M(variant) >= N_VARIANT_M || variant_H(variant) >= N_VARIANT_H) { + log("Invalid FFT variant %u for %s (digits must be < %u%u%u)\n", variant, shape.spec().c_str(), N_VARIANT_W, N_VARIANT_M, N_VARIANT_H); + throw "Invalid FFT variant"; + } + + if (shape.fft_type == FFT64) FFT_FP64 = true, FFT_FP32 = false, NTT_GF31 = false, NTT_GF61 = false, WordSize = 4; + else if (shape.fft_type == FFT3161) FFT_FP64 = false, FFT_FP32 = false, NTT_GF31 = true, NTT_GF61 = true, WordSize = 8; + else if (shape.fft_type == FFT3261) FFT_FP64 = false, FFT_FP32 = true, NTT_GF31 = false, NTT_GF61 = true, WordSize = 8; + else if (shape.fft_type == FFT61) FFT_FP64 = false, FFT_FP32 = false, NTT_GF31 = false, NTT_GF61 = true, WordSize = 4; + else if (shape.fft_type == FFT323161) FFT_FP64 = false, FFT_FP32 = true, NTT_GF31 = true, NTT_GF61 = true, WordSize = 8; + else if (shape.fft_type == FFT3231) FFT_FP64 = false, FFT_FP32 = true, NTT_GF31 = true, NTT_GF61 = false, WordSize = 4; + else if (shape.fft_type == FFT6431) FFT_FP64 = true, FFT_FP32 = false, NTT_GF31 = true, NTT_GF61 = false, WordSize = 8; + else if (shape.fft_type == FFT31) FFT_FP64 = false, FFT_FP32 = false, NTT_GF31 = true, NTT_GF61 = false, WordSize = 4; + else if (shape.fft_type == FFT32) FFT_FP64 = false, FFT_FP32 = true, NTT_GF31 = false, NTT_GF61 = false, WordSize = 4; else throw "FFT type"; } string FFTConfig::spec() const { - string s = shape.spec() + ":" + to_string(variant_W(variant)) + to_string(variant_M(variant)) + to_string(variant_H(variant)); + string const s = shape.spec() + ":" + to_string(variant_W(variant)) + to_string(variant_M(variant)) + to_string(variant_H(variant)); return carry == CARRY_AUTO ? s : (s + (carry == CARRY_32 ? ":0" : ":1")); } @@ -263,48 +320,48 @@ float FFTConfig::maxBpw() const { } // Interpolate for the maximum bpw. This might could be improved upon. However, I doubt people will use these variants often. else { - float b1 = shape.bpw[variant_M(variant) * 3 + variant_W(variant)]; - float b2 = shape.bpw[variant_M(variant) * 3 + variant_H(variant)]; - b = (b1 + b2) / 2.0; + float const b1 = shape.bpw[variant_M(variant) * 3 + variant_W(variant)]; + float const b2 = shape.bpw[variant_M(variant) * 3 + variant_H(variant)]; + b = (b1 + b2) / 2.0f; } // Only some FFTs support both 32 and 64 bit carries. return (carry == CARRY_32 && (shape.fft_type == FFT64 || shape.fft_type == FFT3231)) ? std::min(shape.carry32BPW(), b) : b; } -FFTConfig FFTConfig::bestFit(const Args& args, u32 E, const string& spec) { +FFTConfig FFTConfig::bestFit(const Args& args, u64 E, const string& spec) { // A FFT-spec was given, simply take the first FFT from the spec that can handle E if (!spec.empty()) { FFTConfig fft{spec}; if (fft.maxExp() * args.fftOverdrive < E) { - log("Warning: %s (max %" PRIu64 ") may be too small for %u\n", fft.spec().c_str(), fft.maxExp(), E); + log("Warning: %s (max %" PRIu64 ") may be too small for %" PRIu64 "\n", fft.spec().c_str(), fft.maxExp(), E); } return fft; } // No FFT-spec given, so choose from tune.txt the fastest FFT that can handle E - vector tunes = TuneEntry::readTuneFile(args); + vector const tunes = TuneEntry::readTuneFile(args); for (const TuneEntry& e : tunes) { // The first acceptable is the best as they're sorted by cost if (E <= e.fft.maxExp() * args.fftOverdrive) { return e.fft; } } - log("No FFTs found in tune.txt that can handle %u. Consider tuning with -tune\n", E); + log("No FFTs found in tune.txt that can handle %" PRIu64 ". Consider tuning with -tune\n", E); // Take the first FFT that can handle E for (const FFTShape& shape : FFTShape::allShapes()) { - for (u32 v : {101, 202}) { + for (u32 const v : {101, 202}) { if (FFTConfig fft{shape, v, CARRY_AUTO}; fft.maxExp() * args.fftOverdrive >= E) { return fft; } } } - log("No FFT found for %u\n", E); + log("No FFT found for %" PRIu64 "\n", E); throw "No FFT"; } string numberK(u64 n) { - u32 K = 1024; - u32 M = K * K; + u32 const K = 1024; + u32 const M = K * K; if (n % M == 0) { return to_string(n / M) + 'M'; } @@ -312,10 +369,10 @@ string numberK(u64 n) { if (n >= M && (n * u64(100)) % M == 0) { snprintf(buf, sizeof(buf), "%.2f", float(n) / M); return string(buf) + 'M'; - } else if (n >= K) { + } if (n >= K) { snprintf(buf, sizeof(buf), "%g", float(n) / K); return string(buf) + 'K'; - } else { + } return to_string(n); - } + } diff --git a/src/FFTConfig.h b/src/FFTConfig.h index c873eb8c..1fac5f86 100644 --- a/src/FFTConfig.h +++ b/src/FFTConfig.h @@ -12,7 +12,9 @@ // We pre-calculate the maximum BPW for a number of fft specs. From these entries we can either look up or interpolate to get the // maximum BPW for all variants of an FFT spec. The variants for which maximum bpw are precomputed are 000, 101, 202, 010, 111, 212. -#define NUM_BPW_ENTRIES 6 +enum { +NUM_BPW_ENTRIES = 6 +}; class Args; @@ -27,8 +29,6 @@ class FFTShape { public: static std::vector allShapes(u32 from=0, u32 to = -1); - static tuple getChainLengths(u32 fftSize, u32 exponent, u32 middle); - static vector multiSpec(const string& spec); enum FFT_TYPES fft_type; @@ -37,22 +37,22 @@ class FFTShape { u32 height = 0; array bpw; - FFTShape(u32 w = 1, u32 m = 1, u32 h = 1); - FFTShape(enum FFT_TYPES t, u32 w, u32 m, u32 h); + FFTShape(enum FFT_TYPES t = FFT64, u32 w = 1, u32 m = 1, u32 h = 1); FFTShape(enum FFT_TYPES t, const string& w, const string& m, const string& h); explicit FFTShape(const string& spec); - u32 size() const { return width * height * middle * 2; } - u32 nW() const { return (width == 1024 || width == 256 /*|| width == 4096*/) ? 4 : 8; } - u32 nH() const { return (height == 1024 || height == 256 /*|| height == 4096*/) ? 4 : 8; } + [[nodiscard]] u32 size() const { return width * height * middle * 2; } + [[nodiscard]] u32 nW() const { return (/*width == 1024 ||*/ width == 256) ? 4 : 8; } + [[nodiscard]] u32 nH() const { return (/*height == 1024 ||*/ height == 256) ? 4 : 8; } - float minBpw() const { return fft_type != FFT32 ? 3.0f : 1.0f; } - float maxBpw() const { return *max_element(bpw.begin(), bpw.end()); } - std::string spec() const { return (fft_type ? to_string(fft_type) + ':' : "") + numberK(width) + ':' + numberK(middle) + ':' + numberK(height); } + [[nodiscard]] float minBpw() const { return fft_type != FFT32 ? 3.0f : 1.0f; } + [[nodiscard]] float maxBpw() const { return *std::ranges::max_element(bpw); } + [[nodiscard]] u64 maxExp() const { return u64(maxBpw() * size()); } + [[nodiscard]] std::string spec() const { return (fft_type ? to_string(fft_type) + ':' : "") + numberK(width) + ':' + numberK(middle) + ':' + numberK(height); } - float carry32BPW() const; - bool needsLargeCarry(u32 E) const; - bool isFavoredShape() const; + [[nodiscard]] float carry32BPW() const; + [[nodiscard]] bool needsLargeCarry(u64 E) const; + [[nodiscard]] bool isFavoredShape() const; }; static const u32 N_VARIANT_W = 3; @@ -73,7 +73,7 @@ enum CARRY_KIND {CARRY_32=0, CARRY_64=1, CARRY_AUTO=2}; struct FFTConfig { public: - static FFTConfig bestFit(const Args& args, u32 E, const std::string& spec); + static FFTConfig bestFit(const Args& args, u64 E, const std::string& spec); // Which FP and NTT primes are involved in the FFT bool FFT_FP64; @@ -85,17 +85,17 @@ struct FFTConfig { // Size (in bytes) of integer data passed to FFTs/NTTs on the GPU u32 WordSize; - FFTShape shape{}; + FFTShape shape; u32 variant; - u32 carry; + enum CARRY_KIND carry; explicit FFTConfig(const string& spec); - FFTConfig(FFTShape shape, u32 variant, u32 carry); + FFTConfig(FFTShape shape, u32 variant, enum CARRY_KIND carry); - std::string spec() const; - u64 size() const { return shape.size(); } - u64 maxExp() const { return maxBpw() * shape.size(); } + [[nodiscard]] std::string spec() const; + [[nodiscard]] u32 size() const { return shape.size(); } + [[nodiscard]] u64 maxExp() const { return u64(maxBpw() * shape.size()); } // this config's variant and carry, not the shape's best - float minBpw() const { return shape.minBpw(); } - float maxBpw() const; + [[nodiscard]] float minBpw() const { return shape.minBpw(); } + [[nodiscard]] float maxBpw() const; }; diff --git a/src/File.cpp b/src/File.cpp index 3aaad2e0..fd04256e 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -3,10 +3,11 @@ #include "File.h" #include #include +#include using namespace std; -File::File(const std::filesystem::__cxx11::path& path, const string& mode, bool throwOnError) +File::File(const std::filesystem::path& path, const string& mode, bool throwOnError) : readOnly{mode == "rb"}, name{path.string()} { assert(readOnly || throwOnError); @@ -18,9 +19,11 @@ File::File(const std::filesystem::__cxx11::path& path, const string& mode, bool if (mode == "ab") { assert(f); -#if HAS_SETLINEBUF - setlinebuf(f); -#endif +//#if HAS_SETLINEBUF +// setlinebuf(f); +//#endif + // tdulcet's suggested portable replacement for the lines above + setvbuf(f, nullptr, _IONBF, 0); } } @@ -33,12 +36,12 @@ File::~File() { f = nullptr; } -i64 File::size(const fs::path &name) { +u64 File::size(const fs::path &name) { error_code dummy; return filesystem::file_size(name, dummy); } -File& File::operator=(File&& other) { +File& File::operator=(File&& other) noexcept { assert(this != &other); this->~File(); new (this) File(std::move(other)); diff --git a/src/File.h b/src/File.h index 1e1766e4..0b1a3a56 100644 --- a/src/File.h +++ b/src/File.h @@ -8,8 +8,12 @@ #include #include #include +#ifndef _MSC_VER // unistd.h does not exist for MSVC #include +#endif #include +#include +#include #include #include #include @@ -18,7 +22,7 @@ #include #endif -#if defined(__APPLE__) +#ifdef __APPLE__ #include #endif @@ -28,19 +32,26 @@ #define HAS_SETLINEBUF 0 #endif -namespace fs = std::filesystem; - -struct CRCError { - std::string name; -}; +//! Macros for __attribute__ compiler/crossplatform support +#ifndef _MSC_VER +#define FORMAT_PRINTF(fmt_idx, arg_idx) __attribute__((format(printf, fmt_idx, arg_idx))) +#define FORMAT_SCANF(fmt_idx, arg_idx) __attribute__((format(scanf, fmt_idx, arg_idx))) +#else +#define FORMAT_PRINTF(fmt_idx, arg_idx) +#define FORMAT_SCANF(fmt_idx, arg_idx) +#endif -struct ReadError { - std::string name; -}; +namespace fs = std::filesystem; -struct WriteError { +// File errors derive from std::exception so that the top-level handlers in main(), gpuWorker() and +// Background::run() log them and carry on instead of letting them reach std::terminate. +struct FileError : std::runtime_error { std::string name; + FileError(const char* kind, std::string n) : std::runtime_error(std::string(kind) + ": " + n), name(std::move(n)) {} }; +struct CRCError : FileError { explicit CRCError(std::string n) : FileError("CRC error", std::move(n)) {} }; +struct ReadError : FileError { explicit ReadError(std::string n) : FileError("read error", std::move(n)) {} }; +struct WriteError : FileError { explicit WriteError(std::string n) : FileError("write error", std::move(n)) {} }; class File { FILE* f = nullptr; @@ -48,15 +59,18 @@ class File { File(const fs::path &path, const string& mode, bool throwOnError); - bool readNoThrow(void* data, u32 nBytes) const { return fread(data, nBytes, 1, get()); } + bool readNoThrow(void* data, size_t nBytes) const { return fread(data, nBytes, 1, this->get()); } - void read(void* data, u32 nBytes) const { + void read(void* data, size_t nBytes) const { if (!readNoThrow(data, nBytes)) { throw ReadError{name}; } } void datasync() { fflush(f); -#if defined(_WIN32) || defined(__WIN32__) +#ifdef _MSC_VER +// We'd really like to use FlushFileBuffers(h), but we do not have easy access to the Windows file handle. +// We might could get that by getting the pathname and opening the file with native Windows routines. +#elif defined(_WIN32) || defined(__WIN32__) _commit(fileno(f)); #elif defined(__APPLE__) fcntl(fileno(f), F_FULLFSYNC, 0); @@ -74,7 +88,7 @@ class File { public: const std::string name; - static i64 size(const fs::path& name); + static u64 size(const fs::path& name); static File openRead(const fs::path& name) { return File{name, "rb", false}; } static File openReadThrow(const fs::path& name) { return File{name, "rb", true}; } @@ -85,13 +99,13 @@ class File { static void append(const fs::path& name, std::string_view text) { File::openAppend(name).write(text); } - File() : f{}, readOnly{true} {} + File() : readOnly{true} {} - File(FILE* f, const string& name) : f{f}, readOnly{false}, name{name} {} + File(FILE* f, string name) : f{f}, readOnly{false}, name{std::move(name)} {} - File(File&& other) : f{other.f}, readOnly{other.readOnly}, name{other.name} { other.f = nullptr; } + File(File&& other) noexcept : f{other.f}, readOnly{other.readOnly}, name{other.name} { other.f = nullptr; } - File& operator=(File&& other); + File& operator=(File&& other) noexcept ; File(const File& other) = delete; File& operator=(const File& other) = delete; @@ -127,22 +141,22 @@ class File { template void write(const T& x) const { write(&x, sizeof(T)); } - void write(const void* data, u32 nBytes) const { - if (!fwrite(data, nBytes, 1, get())) { throw WriteError{name}; } + void write(const void* data, size_t nBytes) const { + if (!fwrite(data, nBytes, 1, this->get())) { throw WriteError{name}; } } void seek(long offset, int whence = SEEK_SET) { - int ret = fseek(get(), offset, whence); + int const ret = fseek(this->get(), offset, whence); if (ret) { throw ReadError{name}; } // throw(std::ios_base::failure(("fseek: "s + to_string(ret)).c_str())); } - void flush() { fflush(get()); } + void flush() { fflush(this->get()); } - int printf(const char *fmt, ...) const __attribute__((format(printf, 2, 3))) { + int printf(const char *fmt, ...) const FORMAT_PRINTF(2, 3) { va_list va; va_start(va, fmt); - int ret = vfprintf(f, fmt, va); + int const ret = vfprintf(f, fmt, va); va_end(va); #if !HAS_LINEBUF @@ -152,10 +166,10 @@ class File { return ret; } - int scanf(const char *fmt, ...) __attribute__((format(scanf, 2, 3))) { + int scanf(const char *fmt, ...) FORMAT_SCANF(2, 3) { va_list va; va_start(va, fmt); - int ret = vfscanf(f, fmt, va); + int const ret = vfscanf(f, fmt, va); va_end(va); return ret; } @@ -165,10 +179,10 @@ class File { void write(string_view s) { write(s.data(), s.size()); } operator bool() const { return f != nullptr; } - FILE* get() const { return f; } + [[nodiscard]] FILE* get() const { return f; } - long ftell() const { - long pos = ::ftell(get()); + [[nodiscard]] long ftell() const { + long const pos = ::ftell(this->get()); assert(pos >= 0); return pos; } @@ -179,8 +193,8 @@ class File { } long size() { - long savePos = ftell(); - long retSize = seekEnd(); + long const savePos = ftell(); + long const retSize = seekEnd(); seek(savePos); return retSize; } @@ -191,7 +205,7 @@ class File { std::string readLine() { char buf[1024]; buf[0] = 0; - bool ok = fgets(buf, sizeof(buf), get()); + bool const ok = fgets(buf, sizeof(buf), this->get()); if (!ok) { return ""; } // EOF or error string line = buf; if (line.empty() || line.back() != '\n') { @@ -208,7 +222,7 @@ class File { } template - std::vector read(u32 nWords) const { + [[nodiscard]] std::vector read(size_t nWords) const { vector ret; ret.resize(nWords); read(ret.data(), nWords * sizeof(T)); @@ -216,8 +230,8 @@ class File { } template - std::vector readChecked(u32 nWords) const { - u32 expectedCRC = read(1)[0]; + [[nodiscard]] std::vector readChecked(size_t nWords) const { + u32 const expectedCRC = read(1)[0]; return readWithCRC(nWords, expectedCRC); } @@ -228,7 +242,7 @@ class File { } template - std::vector readWithCRC(u32 nWords, u32 crc) const { + [[nodiscard]] std::vector readWithCRC(size_t nWords, u32 crc) const { auto data = read(nWords); if (crc != crc32(data)) { log("File '%s' : CRC: expected %u, actual %u\n", name.c_str(), crc, crc32(data)); @@ -237,18 +251,19 @@ class File { return data; } - std::vector readBytesLE(u32 nBytes) { + std::vector readBytesLE(size_t nBytes) { assert(nBytes > 0); - u32 nWords = (nBytes - 1) / 4 + 1; + size_t const nWords = (nBytes - 1) / 4 + 1; vector data(nWords); read(data.data(), nBytes); return data; } - - u32 readUpTo(void* data, u32 nUpToBytes) { return fread(data, 1, nUpToBytes, get()); } - + + size_t readUpTo(void* data, size_t nUpToBytes) { return fread(data, 1, nUpToBytes, this->get()); } + string readAll() { - size_t sz = size(); + u64 const sz = size(); + if (sz == 0) { return {}; } // fread of 0 bytes would be reported as a ReadError return {read(sz).data(), sz}; } }; diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 44656fa1..62e01dbf 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -18,16 +18,16 @@ #include "Sha3Hash.h" #include -#include #include #include #include #include #include -#include #define _USE_MATH_DEFINES #include +#include +#include #ifndef M_PIl #define M_PIl 3.141592653589793238462643383279502884L @@ -37,68 +37,69 @@ #define M_PI 3.141592653589793238462643383279502884 #endif -#define CARRY_LEN 8 +#ifndef M_LN2l +#define M_LN2l 0.69314718055994530941723212145818L +#endif + +#ifndef M_LN2 +#define M_LN2 0.69314718055994530941723212145818 +#endif + +enum { +CARRY_LEN = 8 +}; namespace { u32 kAt(u32 H, u32 line, u32 col) { return (line + col * H) * 2; } -double weight(u32 N, u32 E, u32 H, u32 line, u32 col, u32 rep) { +double weight(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { return exp2l((long double)(extra(N, E, kAt(H, line, col) + rep)) / N); } -double invWeight(u32 N, u32 E, u32 H, u32 line, u32 col, u32 rep) { +double invWeight(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { return exp2l(-(long double)(extra(N, E, kAt(H, line, col) + rep)) / N); } -double weightM1(u32 N, u32 E, u32 H, u32 line, u32 col, u32 rep) { - return exp2l((long double)(extra(N, E, kAt(H, line, col) + rep)) / N) - 1; +// MSVC does not truly support long double. Use expm1 rather than exp2 and subtracting one. +double weightM1(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { + return expm1l(M_LN2l * (long double)(extra(N, E, kAt(H, line, col) + rep)) / N); } -double invWeightM1(u32 N, u32 E, u32 H, u32 line, u32 col, u32 rep) { - return exp2l(- (long double)(extra(N, E, kAt(H, line, col) + rep)) / N) - 1; +double invWeightM1(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { + return expm1l(M_LN2l * - (long double)(extra(N, E, kAt(H, line, col) + rep)) / N); } double boundUnderOne(double x) { return std::min(x, nexttoward(1, 0)); } -float weight32(u32 N, u32 E, u32 H, u32 line, u32 col, u32 rep) { - return exp2((double)(extra(N, E, kAt(H, line, col) + rep)) / N); +float weight32(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { + return float(exp2((double)(extra(N, E, kAt(H, line, col) + rep)) / N)); } -float invWeight32(u32 N, u32 E, u32 H, u32 line, u32 col, u32 rep) { - return exp2(-(double)(extra(N, E, kAt(H, line, col) + rep)) / N); +float invWeight32(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { + return float(exp2(-(double)(extra(N, E, kAt(H, line, col) + rep)) / N)); } -float weightM132(u32 N, u32 E, u32 H, u32 line, u32 col, u32 rep) { - return exp2((double)(extra(N, E, kAt(H, line, col) + rep)) / N) - 1; +float weightM132(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { + return float(expm1(M_LN2 * (double)(extra(N, E, kAt(H, line, col) + rep)) / N)); } -float invWeightM132(u32 N, u32 E, u32 H, u32 line, u32 col, u32 rep) { - return exp2(- (double)(extra(N, E, kAt(H, line, col) + rep)) / N) - 1; +float invWeightM132(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { + return float(expm1(M_LN2 * - (double)(extra(N, E, kAt(H, line, col) + rep)) / N)); } -float boundUnderOne(float x) { return std::min(x, nexttowardf(1, 0)); } - -Weights genWeights(FFTConfig fft, u32 E, u32 W, u32 H, u32 nW, bool AmdGpu) { - u32 N = 2u * W * H; - u32 groupWidth = W / nW; +Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool nvidiaGpu) { + u32 const N = 2u * W * H; + u32 const groupWidth = W / nW; vector weightsConstIF; vector weightsIF; - vector bits; if (fft.FFT_FP64) { // Inverse + Forward for (u32 thread = 0; thread < groupWidth; ++thread) { auto iw = invWeight(N, E, H, 0, thread, 0); auto w = weight(N, E, H, 0, thread, 0); - // nVidia GPUs have a constant cache that only works on buffer sizes less than 64KB. Create a smaller buffer - // that is a copy of the first part of weightsIF. There are several kernels that need the combined weightsIF - // buffer, so there is an unfortunate duplication of these weights. - if (!AmdGpu) { - weightsConstIF.push_back(2 * boundUnderOne(iw)); - weightsConstIF.push_back(2 * w); - } weightsIF.push_back(2 * boundUnderOne(iw)); weightsIF.push_back(2 * w); } @@ -108,6 +109,19 @@ Weights genWeights(FFTConfig fft, u32 E, u32 W, u32 H, u32 nW, bool AmdGpu) { weightsIF.push_back(invWeightM1(N, E, H, gy, 0, 0)); weightsIF.push_back(weightM1(N, E, H, gy, 0, 0)); } + + // nVidia GPUs have a fast constant cache that only works on buffer sizes less than 64KB. Create two smaller buffers + // that can be used to create the large group order buffer with a single multiply. + if (nvidiaGpu) { + for (u32 gy = 0; gy < 64; ++gy) { + weightsConstIF.push_back(invWeightM1(N, E, H, gy, 0, 0)); + weightsConstIF.push_back(weightM1(N, E, H, gy, 0, 0)); + } + for (u32 gy = 0; gy < H; gy += 64) { + weightsConstIF.push_back(invWeightM1(N, E, H, gy, 0, 0)); + weightsConstIF.push_back(weightM1(N, E, H, gy, 0, 0)); + } + } } else if (fft.FFT_FP32) { @@ -115,17 +129,13 @@ Weights genWeights(FFTConfig fft, u32 E, u32 W, u32 H, u32 nW, bool AmdGpu) { vector weightsIF32; // Inverse + Forward for (u32 thread = 0; thread < groupWidth; ++thread) { - auto iw = invWeight32(N, E, H, 0, thread, 0); - auto w = weight32(N, E, H, 0, thread, 0); - // nVidia GPUs have a constant cache that only works on buffer sizes less than 64KB. Create a smaller buffer - // that is a copy of the first part of weightsIF. There are several kernels that need the combined weightsIF - // buffer, so there is an unfortunate duplication of these weights. - if (!AmdGpu) { - weightsConstIF32.push_back(2 * boundUnderOne(iw)); - weightsConstIF32.push_back(2 * w); - } - weightsIF32.push_back(2 * boundUnderOne(iw)); - weightsIF32.push_back(2 * w); + auto iw = invWeight32(N, E, H, 0, thread, 0) ; + auto w = weight32(N, E, H, 0, thread, 0) ; + // Weights are scaled by 2^-24 and 2^48 so that multiplicaton by 1/epsilon does not generate infinty results (width and height variant 2). + iw = iw * 281474976710656.0f; + w = w * 0.000000059604644775390625f; + weightsIF32.push_back(iw); + weightsIF32.push_back(w); } // the group order matches CarryA/M (not fftP/CarryFused). @@ -134,39 +144,35 @@ Weights genWeights(FFTConfig fft, u32 E, u32 W, u32 H, u32 nW, bool AmdGpu) { weightsIF32.push_back(weightM132(N, E, H, gy, 0, 0)); } + // nVidia GPUs have a fast constant cache that only works on buffer sizes less than 64KB. Create two smaller buffers + // that can be used to create the large group order buffer with a single multiply. + if (nvidiaGpu) { + for (u32 gy = 0; gy < 64; ++gy) { + weightsConstIF32.push_back(invWeightM132(N, E, H, gy, 0, 0)); + weightsConstIF32.push_back(weightM132(N, E, H, gy, 0, 0)); + } + for (u32 gy = 0; gy < H; gy += 64) { + weightsConstIF32.push_back(invWeightM132(N, E, H, gy, 0, 0)); + weightsConstIF32.push_back(weightM132(N, E, H, gy, 0, 0)); + } + } + // Copy the float vectors to the double vectors - weightsConstIF.resize(weightsConstIF32.size() / 2); - memcpy((double *) weightsConstIF.data(), weightsConstIF32.data(), weightsConstIF32.size() * sizeof(float)); weightsIF.resize(weightsIF32.size() / 2); memcpy((double *) weightsIF.data(), weightsIF32.data(), weightsIF32.size() * sizeof(float)); + weightsConstIF.resize(weightsConstIF32.size() / 2); + memcpy((double *) weightsConstIF.data(), weightsConstIF32.data(), weightsConstIF32.size() * sizeof(float)); } - if (fft.FFT_FP64 || fft.FFT_FP32) { - for (u32 line = 0; line < H; ++line) { - for (u32 thread = 0; thread < groupWidth; ) { - std::bitset<32> b; - for (u32 bitoffset = 0; bitoffset < 32; bitoffset += nW*2, ++thread) { - for (u32 block = 0; block < nW; ++block) { - for (u32 rep = 0; rep < 2; ++rep) { - if (isBigWord(N, E, kAt(H, line, block * groupWidth + thread) + rep)) { b.set(bitoffset + block * 2 + rep); } - } - } - } - bits.push_back(b.to_ulong()); - } - } - assert(bits.size() == N / 32); - } - - return Weights{weightsConstIF, weightsIF, bits}; + return Weights{.weightsConstIF=weightsConstIF, .weightsIF=weightsIF}; } string toLiteral(i32 value) { return to_string(value); } string toLiteral(u32 value) { return to_string(value) + 'u'; } -[[maybe_unused]] string toLiteral(long value) { return to_string(value) + "l"; } -[[maybe_unused]] string toLiteral(unsigned long value) { return to_string(value) + "ul"; } -[[maybe_unused]] string toLiteral(long long value) { return to_string(value) + "l"; } // Yes, this looks wrong. The Mingw64 C compiler uses -[[maybe_unused]] string toLiteral(unsigned long long value) { return to_string(value) + "ul"; } // long long for 64-bits, while openCL uses long for 64 bits. +[[maybe_unused]] string toLiteral(long value) { return to_string(value) + "ll"; } +[[maybe_unused]] string toLiteral(unsigned long value) { return to_string(value) + "ull"; } +[[maybe_unused]] string toLiteral(long long value) { return to_string(value) + "ll"; } +[[maybe_unused]] string toLiteral(unsigned long long value) { return to_string(value) + "ull"; } template string toLiteral(F value) { @@ -213,7 +219,7 @@ string toLiteral(const string& s) { return s; } [[maybe_unused]] string toLiteral(ulong2 cs) { return "U2("s + toLiteral(cs.first) + ',' + toLiteral(cs.second) + ')'; } template -string toDefine(const string& k, T v) { return " -D"s + k + '=' + toLiteral(v); } +string toDefine(const string& k, const T& v) { return " -D"s + k + '=' + toLiteral(v); } template string toDefine(const T& vect) { @@ -227,8 +233,14 @@ constexpr bool isInList(const string& s, initializer_list list) { return false; } -string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector& extraConf, u32 E, bool doLog, - bool &tail_single_wide, bool &tail_single_kernel, u32 &in_place, u32 &pad_size) { +// Capacity of the ROE and carry statistics sample buffers; passed to the kernels as STATS_SIZE so they stop recording when full. +enum { +ROE_SIZE = 100000, +CARRY_SIZE = 100000 +}; + +string clDefines(Args& args, cl_device_id id, FFTConfig fft, const vector& extraConf, u64 E, bool doLog, + bool &tail_single_wide, bool &tail_single_kernel, u32 &in_place, u32 &pad_size, u32 &wmul) { map config; // Highest priority is the requested "extra" conf @@ -244,13 +256,14 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< } // Default value for -use options that must also be parsed in C++ code - tail_single_wide = 0, tail_single_kernel = 1; // Default tailSquare is double-wide in one kernel - in_place = 0; // Default is not in-place + tail_single_wide = false, tail_single_kernel = true; // Default tailSquare is double-wide in one kernel + in_place = isNvidiaGpu(id) ? 1 : 0; // Default is in-place for nVidia, not in-place for others (must match base.cl) + wmul = 2; // Default is carryFused processes two lines at a time pad_size = isAmdGpu(id) ? 256 : 0; // Default is 256 bytes for AMD, 0 for others // Validate -use options for (const auto& [k, v] : config) { - bool isValid = isInList(k, { + bool const isValid = isInList(k, { "FAST_BARRIER", "STATS", "IN_SIZEX", @@ -264,8 +277,8 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< "NO_ASM", "DEBUG", "CARRY64", - "BIGLIT", - "NONTEMPORAL", + "BIGLIT", // Deprecated + "NONTEMPORAL", // Deprecated "INPLACE", "PAD", "MIDDLE_IN_LDS_TRANSPOSE", @@ -279,7 +292,14 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< "TABMUL_CHAIN31", "TABMUL_CHAIN32", "TABMUL_CHAIN61", - "MODM31" + "MODM31", + "LOADS","STORES", + "NOREG", // CUDA - experimental + "WMUL", + "MULTI_Q", + "GRAPHS", + "L1CUDA", + "PDL" // CUDA, sm_90+: programmatic dependent launch }); if (!isValid) { log("Warning: unrecognized -use key '%s'\n", k.c_str()); @@ -287,20 +307,110 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< // Some -use options are needed in both OpenCL code and C++ initialization code if (k == "TAIL_KERNELS") { - if (atoi(v.c_str()) == 0) tail_single_wide = 1, tail_single_kernel = 1; - if (atoi(v.c_str()) == 1) tail_single_wide = 1, tail_single_kernel = 0; - if (atoi(v.c_str()) == 2) tail_single_wide = 0, tail_single_kernel = 1; - if (atoi(v.c_str()) == 3) tail_single_wide = 0, tail_single_kernel = 0; + if (atoi(v.c_str()) == 0) tail_single_wide = true, tail_single_kernel = true; + if (atoi(v.c_str()) == 1) tail_single_wide = true, tail_single_kernel = false; + if (atoi(v.c_str()) == 2) tail_single_wide = false, tail_single_kernel = true; + if (atoi(v.c_str()) == 3) tail_single_wide = false, tail_single_kernel = false; } if (k == "INPLACE") in_place = atoi(v.c_str()); + if (k == "WMUL") wmul = atoi(v.c_str()); if (k == "PAD") pad_size = atoi(v.c_str()); } + // Maximum WMUL is 32KB / (WIDTH * SHUFL_BYTES_W). If using the 32KB maximum, LDS padding must be disabled. + // Furthermore, I've seen the CUDA compiler refuse to create a kernel with 1024 threads. Thus, we limit WMUL to 2 for a 1K width and to 1 for a 4K width. + { + u32 const shufl_bytes_w = args.value("SHUFL_BYTES_W", 8); + u32 max_wmul = 32768 / (fft.shape.width * shufl_bytes_w); + if (max_wmul > 2 && fft.shape.width >= 1024) max_wmul = 2; + if (max_wmul > 1 && fft.shape.width >= 4096) max_wmul = 1; + if (wmul > max_wmul) { + wmul = max_wmul; + config["WMUL"] = to_string(wmul); + log("WMUL setting too large for this FFT width. Changing to WMUL=%d\n", wmul); + } + if (fft.shape.width * shufl_bytes_w * wmul >= 32768) { + log("Local shared memory limit of 32KB exceeded. Changing to LDSPAD_W=0\n"); + config["LDSPAD_W"] = to_string(0); + } + } + + // MULTI_Q is not allowed when profiling with -time + if (args.profile && args.value("MULTI_Q", 0)) { + args.flags["MULTI_Q"] = to_string(0); + // config was copied out of args.flags above, so it needs the same treatment: it is what the kernels are + // compiled from, and the L2_STRIPING limit a few lines below reads args. Leaving config alone builds + // kernels that still believe in the second queue, with an L2_STRIPING allowed only without it. + config["MULTI_Q"] = to_string(0); + log("MULTI_Q is disabled when profiling with -time.\n"); + } + // The !OLD_FENCE carry hand-off in carryFused coordinates the lanes of a wavefront with sync() and nothing + // else, from inside a divergent branch, so a workgroup barrier is not a substitute. sync() is bar.warp.sync + // on nVidia and free on AMD, where a wavefront really does advance in lock-step. Anywhere else it compiles + // to nothing, the hand-off races, and the result is silently wrong: on an Intel iGPU -use OLD_FENCE=0 + // returns a wrong residue inside 400 iterations. base.cl already defaults OLD_FENCE to 1 off AMD; make + // that hold when it is asked for explicitly too. + if (!isAmdGpu(id) && !isNvidiaGpu(id)) { + if (auto it = config.find("OLD_FENCE"); it != config.end() && atoi(it->second.c_str()) == 0) { + it->second = to_string(1); + log("OLD_FENCE=0 needs an AMD or nVidia device; using OLD_FENCE=1.\n"); + } + } + + // GRAPHS are not allowed when profiling with -time. GRAPH replays the four bottom-half + // kernels without per-kernel events, and the events recorded while capturing the graph never execute, so the + // profile would show those kernels -- most of an iteration -- as one call of ~0 ns. + if (args.profile && args.value("GRAPHS", 1)) { + args.flags["GRAPHS"] = to_string(0); +#if CUDA_BACKEND + log("GRAPHS are disabled when profiling with -time.\n"); +#endif + } + + // L2_STRIPING is not allowed if INPLACE=0. Maximum L2_STRIPING is WIDTH/64 if MULTI_Q=0 and WIDTH/128 if MULTI_Q=1. + // Technically, L2_STRIPING of WIDTH/32, MULTI_Q=0 could be allowed but that is just a more complicated way to implement L2_STRIPING=0. + // Also, WIDTH/64, MULTI_Q=1 could be allowed with some marker/sync code changes but that is very similar to L2_STRIPING=0. + { + u32 l2_striping = args.value("L2_STRIPING", 0); + u32 multi_q = args.value("MULTI_Q", 0); + if (l2_striping && !in_place) { + config["L2_STRIPING"] = to_string(0); + args.flags["L2_STRIPING"] = to_string(0); + log("L2_STRIPING is only allowed if INPLACE=1. Changing to L2_STRIPING=0.\n"); + } + else if (multi_q == 0 && l2_striping > fft.shape.width/64) { + config["L2_STRIPING"] = to_string(fft.shape.width/64); + args.flags["L2_STRIPING"] = to_string(fft.shape.width/64); + log("Max L2_STRIPING when MULTI_Q=0 exceeded. Changing to L2_STRIPING=%u.\n", fft.shape.width/64); + } + else if (multi_q > 0 && l2_striping > fft.shape.width/128) { + config["L2_STRIPING"] = to_string(fft.shape.width/128); + args.flags["L2_STRIPING"] = to_string(fft.shape.width/128); + log("Max L2_STRIPING when MULTI_Q=1 exceeded. Changing to L2_STRIPING=%u.\n", fft.shape.width/128); + } + + // The striped launches split the WIDTH/16 stripes into groups of L2_STRIPING and pair group i with its + // Hermitian partner WIDTH - L2_STRIPING*16 - base_lo, so the number of groups must be even (a multiple of + // four with MULTI_Q, which further splits them across two queues). Otherwise whole stripes are never + // transformed and others are squared twice. WIDTH/16 is a power of two, so round down to one that divides. + l2_striping = args.value("L2_STRIPING", 0); + if (l2_striping) { + u32 const groupsNeeded = multi_q ? 4 : 2; + u32 valid = l2_striping; + while (valid && (fft.shape.width / 16) % (groupsNeeded * valid)) { --valid; } + if (valid != l2_striping) { + config["L2_STRIPING"] = to_string(valid); + args.flags["L2_STRIPING"] = to_string(valid); + log("L2_STRIPING must divide WIDTH/%u. Changing to L2_STRIPING=%u.\n", 16 * groupsNeeded, valid); + } + } + } + string defines = toDefine(config); if (doLog) { log("config: %s\n", defines.c_str()); } + defines += toDefine("EXP", E); defines += toDefine(initializer_list>{ - {"EXP", E}, {"WIDTH", fft.shape.width}, {"SMALL_HEIGHT", fft.shape.height}, {"MIDDLE", fft.shape.middle}, @@ -311,23 +421,25 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< if (isAmdGpu(id)) { defines += toDefine("AMDGPU", 1); } if (isNvidiaGpu(id)) { defines += toDefine("NVIDIAGPU", 1); } + if (isNvidiaGpu(id)) { defines += toDefine("CC", getNvidiaComputeCapability(id)); } if ((fft.carry == CARRY_AUTO && fft.shape.needsLargeCarry(E)) || (fft.carry == CARRY_64)) { if (doLog) { log("Using CARRY64\n"); } defines += toDefine("CARRY64", 1); } - u32 N = fft.shape.size(); + u32 const N = fft.shape.size(); defines += toDefine("FFT_VARIANT", fft.variant); defines += toDefine("MAXBPW", (u32)(fft.maxBpw() * 100.0f)); + defines += toDefine("STATS_SIZE", u32(std::min(ROE_SIZE, CARRY_SIZE))); - if (fft.FFT_FP64 | fft.FFT_FP32) { + if (fft.FFT_FP64 || fft.FFT_FP32) { defines += toDefine("WEIGHT_STEP", weightM1(N, E, fft.shape.height * fft.shape.middle, 0, 0, 1)); defines += toDefine("IWEIGHT_STEP", invWeightM1(N, E, fft.shape.height * fft.shape.middle, 0, 0, 1)); if (fft.FFT_FP64) defines += toDefine("TAILT", root1Fancy(fft.shape.height * 2, 1)); else defines += toDefine("TAILT", root1FancyFP32(fft.shape.height * 2, 1)); - TrigCoefs coefs = trigCoefs(fft.shape.size() / 4); + TrigCoefs const coefs = trigCoefs(fft.shape.size() / 4); defines += toDefine("TRIG_SCALE", int(coefs.scale)); defines += toDefine("TRIG_SIN", coefs.sinCoefs); defines += toDefine("TRIG_COS", coefs.cosCoefs); @@ -341,10 +453,6 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< // Send the FFT/NTT type and booleans that enable/disable code for each possible FP and NTT defines += toDefine("FFT_TYPE", (int) fft.shape.fft_type); - defines += toDefine("FFT_FP64", (int) fft.FFT_FP64); - defines += toDefine("FFT_FP32", (int) fft.FFT_FP32); - defines += toDefine("NTT_GF31", (int) fft.NTT_GF31); - defines += toDefine("NTT_GF61", (int) fft.NTT_GF61); defines += toDefine("WordSize", fft.WordSize); // When using multiple NTT primes or hybrid FFT/NTT, each FFT/NTT prime's data buffer and trig values are combined into one buffer. @@ -407,8 +515,8 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< } // Calculate fractional bits-per-word = (E % N) / N * 2^64 - u32 bpw_hi = (u64(E % N) << 32) / N; - u32 bpw_lo = (((u64(E % N) << 32) % N) << 32) / N; + u32 const bpw_hi = (u64(E % N) << 32) / N; + u32 const bpw_lo = (((u64(E % N) << 32) % N) << 32) / N; u64 bpw = (u64(bpw_hi) << 32) + bpw_lo; bpw--; // bpw must not be an exact value -- it must be less than exact value to get last biglit value right defines += toDefine("FRAC_BPW_HI", (u32) (bpw >> 32)); @@ -441,36 +549,43 @@ RoeInfo roeStat(const vector& roe) { double maxRoe = 0; for (auto xf : roe) { - double x = xf; + double const x = xf; assert(x >= 0); maxRoe = max(x, maxRoe); sumRoe += x; sum2Roe += x * x; } - u32 n = roe.size(); + u32 const n = u32(roe.size()); - double sdRoe = sqrt(n * sum2Roe - sumRoe * sumRoe) / n; - double meanRoe = sumRoe / n; + double const sdRoe = sqrt(n * sum2Roe - sumRoe * sumRoe) / n; + double const meanRoe = sumRoe / n; return {n, maxRoe, meanRoe, sdRoe}; } class IterationTimer { Timer timer; - u32 kStart; + u64 kStart; public: - explicit IterationTimer(u32 kStart) : kStart(kStart) { } + explicit IterationTimer(u64 kStart) : kStart(kStart) { } - float reset(u32 k) { - float secs = timer.reset(); + float reset(u64 k) { + float const secs = float(timer.reset()); - u32 its = max(1u, k - kStart); + u64 const its = max(u64(1), k - kStart); kStart = k; return secs / its; } }; +// The block sizes baseCheckStep() knows about. blockSize comes from the savefile, so it must be +// validated on load (see Gpu::loadPRP) -- a bad value here would otherwise produce checkStep == 0 +// in a release build, where the assert below is compiled out, and then "k % checkStep" divides by zero. +bool isValidBlockSize(u32 blockSize) { + return blockSize == 200 || blockSize == 400 || blockSize == 500 || blockSize == 1000; +} + u32 baseCheckStep(u32 blockSize) { switch (blockSize) { case 200: return 40'000; @@ -478,13 +593,13 @@ u32 baseCheckStep(u32 blockSize) { case 500: return 200'000; case 1000: return 1'000'000; default: - assert(false); - return 0; + log("Invalid blockSize %u\n", blockSize); + throw "invalid blockSize"; } } u32 checkStepForErrors(u32 blockSize, u32 nErrors) { - u32 step = baseCheckStep(blockSize); + u32 const step = baseCheckStep(blockSize); return nErrors ? step / 2 : step; } @@ -502,12 +617,23 @@ string toHex(const vector& v) { return s; } +string formatSecsPerIter(float secsPerIter) { + char buf[64]; + float const usecsPerIter = secsPerIter * 1.0e6f; // Convert to micro-seconds + if (usecsPerIter > 1000.0f) { + snprintf(buf, sizeof(buf), "%4.0f", usecsPerIter); + } else { + snprintf(buf, sizeof(buf), "%6.1f", usecsPerIter); + } + return string(buf); +} + } // namespace // -------- -unique_ptr Gpu::make(Queue* q, u32 E, GpuCommon shared, FFTConfig fftConfig, const vector& extraConf, bool logFftSize) { - return make_unique(q, shared, fftConfig, E, extraConf, logFftSize); +unique_ptr Gpu::make(u64 E, GpuCommon shared, FFTConfig fftConfig, const vector& extraConf, bool logFftSize) { + return make_unique(shared, fftConfig, E, extraConf, logFftSize); } Gpu::~Gpu() { @@ -515,79 +641,324 @@ Gpu::~Gpu() { background->waitEmpty(); } -#define ROE_SIZE 100000 -#define CARRY_SIZE 100000 +// Part of GPU initialization is to compute the default number of registers each kernel should target during compilation. +// Kernel register usage is critical for maximizing GPU occupancy. The default values can be overrriden with command line arguments. +// On CUDA this sets --maxrregcount (or launch bounds). On AMD the same REGxxxx options select waves per SIMD or a VGPR count, see amdRegisterOption below. +// Most kernels have occupancy limited by register usage. For reference, the following guidelines dictate where an "uptick" in occupancy occurs. +// If kernel threads=256, register crossovers are at 128, 80, 64, 48, 40 +// If kernel threads=128, register crossovers are at 128, 96, 80, 72, 64, 56, 48, 40 +// If kernel threads=64, register crossovers are at 128, 112, 96, 88, 80, 72, 64, 56, 48, 40 +string Gpu::numRegisters(enum WHICH_KERNEL which_kernel) { + [[maybe_unused]] int regs = 0; // Default CUDA maximum register count (the AMD path only uses the override value) + const char *use_override = ""; + // Allow command line to prefer the compiler's default number of registers + if (args.value("NOREG", 0)) return string(""); + // Determine a CUDA kernel specific default maximum number of GPU registers (values set to -1 have not been tuned for best default value). + // This switch also selects which REGxxxx option applies to the kernel, and that selection is needed on AMD too (see amdRegisterOption), + // so it must not be compiled out for non-CUDA backends. The default register counts below are only used by CUDA. + switch (which_kernel) { + case CARRYFUSED: // Register usage depends on NW, the FFT/NTT type, and perhaps the long carry setting + switch (fft.shape.fft_type) { + case FFT64: + regs = nW == 8 ? 80 : 64; + use_override = "REGCF64"; + break; + case FFT3161: + regs = nW == 8 ? 96 : 64; // Tested on 4090, nW=8, CUDA 13.0 (88 regs is possible without spilling but is slower) + use_override = "REGCF3161"; + break; + case FFT3261: + regs = nW == 8 ? 96 : 64; + use_override = "REGCF3261"; + break; + case FFT61: + regs = nW == 8 ? 80 : 64; + use_override = "REGCF61"; + break; + case FFT323161: + regs = nW == 8 ? 128 : 80; + use_override = "REGCF323161"; + break; + case FFT3231: + regs = -1; + use_override = "REGCF3231"; + break; + case FFT6431: + regs = nW == 8 ? -1 : -1; // Tested on TitanV, NW=8, CUDA 13.0. NW=4 not tested. + use_override = "REGCF6431"; + break; + case FFT31: + regs = -1; + use_override = "REGCF31"; + break; + case FFT32: + regs = -1; + use_override = "REGCF32"; + break; + } + break; + case MIDIN: // Register usage depends on MIDDLE and the FP32/FP64 + if (fft.FFT_FP64) { + if (fft.shape.middle >= 16) regs = 96; + else if (fft.shape.middle >= 14) regs = 88; + else if (fft.shape.middle >= 13) regs = 80; + else if (fft.shape.middle >= 10) regs = 72; + else if (fft.shape.middle >= 7) regs = 64; + else if (fft.shape.middle >= 5) regs = 56; + else if (fft.shape.middle >= 4) regs = 48; + else regs = -1; + use_override = "REGMI64"; + } else { + if (fft.shape.middle == 16) regs = 56; + else if (fft.shape.middle == 8) regs = 40; + else if (fft.shape.middle == 4) regs = 32; + else regs = -1; + use_override = "REGMI32"; + } + break; + case MIDIN31: // Register usage depends on MIDDLE + if (fft.shape.middle == 16) regs = 56; + else if (fft.shape.middle == 8) regs = 48; // Tested on 4090, CUDA 13.0 (40 regs is possible without spilling but is not measurably faster) + else if (fft.shape.middle == 4) regs = 32; // Tested on 5070Ti, CUDA 13.2. + else regs = -1; + use_override = "REGMI31"; + break; + case MIDIN61: // Register usage depends on MIDDLE + if (fft.shape.middle == 16) regs = 96; + else if (fft.shape.middle == 8) regs = 64; // Tested on 4090, CUDA 13.0 + else if (fft.shape.middle == 4) regs = -1; // Tested on 5070Ti, CUDA 13.2 (48 regs is possible without spilling but is slower), best is -1. + else regs = -1; + use_override = "REGMI61"; + break; + case TAIL: // Register usage depends on NH and the FP32/FP64 (assumes double-wide kernel) + if (fft.FFT_FP64) { + regs = nH == 8 ? 88 : 64; + use_override = "REGTS64"; + } else { + regs = nH == 8 ? 64 : 48; + use_override = "REGTS32"; + } + break; + case TAIL31: // Register usage depends on NH (assumes double-wide kernel) + regs = nH == 8 ? -1 : 48; // Tested on 4090, NH=8, CUDA 13.0. Occupancy is limited by LDS memory use, register usage of 48 is possible, best is 64. + // Tested on 5070Ti, NH=8, CUDA 13.2. Occupancy is limited by LDS memory use, register usage of 56 is possible, best is -1. + use_override = "REGTS31"; + break; + case TAIL61: // Register usage depends on NH (assumes double-wide kernel) + regs = nH == 8 ? 96 : 64; // Tested on 4090, nH=8, CUDA 13.0 (80 regs is possible without spilling but is slower) + use_override = "REGTS61"; + break; + case MIDOUT: // Register usage depends on MIDDLE and the FFT/NTT type + if (fft.FFT_FP64) { + if (fft.shape.middle >= 15) regs = 96; + else if (fft.shape.middle >= 14) regs = 88; + else if (fft.shape.middle >= 11) regs = 80; + else if (fft.shape.middle >= 10) regs = 72; + else if (fft.shape.middle >= 7) regs = 64; + else if (fft.shape.middle >= 5) regs = 56; + else if (fft.shape.middle >= 4) regs = 48; + else regs = -1; + use_override = "REGMO64"; + } else { + if (fft.shape.middle == 16) regs = 56; + else if (fft.shape.middle == 8) regs = 40; + else if (fft.shape.middle == 4) regs = 32; + else regs = -1; + use_override = "REGMO32"; + } + break; + case MIDOUT31: // Register usage depends on MIDDLE + if (fft.shape.middle == 16) regs = 48; + else if (fft.shape.middle == 8) regs = 40; // Tested on 4090, CUDA 13.0 + else if (fft.shape.middle == 4) regs = -1; // Tested on 5070Ti, CUDA 13.2 (32 regs is possible without spilling but is slower), best is -1. + else regs = -1; + use_override = "REGMO31"; + break; + case MIDOUT61: // Register usage depends on MIDDLE + if (fft.shape.middle == 16) regs = 96; + else if (fft.shape.middle == 8) regs = 64; // Tested on 4090, CUDA 13.0, best is 64. + // Tested on 5070Ti, CUDA 13.2, best is 72. + else if (fft.shape.middle == 4) regs = 64; // Tested on 5070Ti, CUDA 13.2 (48 regs is possible without spilling but is slower), best is 64. + else regs = -1; + use_override = "REGMO61"; + break; + } + // Get the optional override register count + int const override_regs = args.value(use_override, 0); +#if CUDA_BACKEND + // If a specified override is small, use the count as a CUDA launch_bounds rather than a maximum register count + if (override_regs && (override_regs > 0 && override_regs <= 16)) return string("-DCUDA_MIN_BLOCKS=") + to_string(override_regs) + " "; + // If specified, override the default maximum register count + if (override_regs) regs = override_regs; + // Sometimes the results using CUDA compiler's default launch_bounds without setting an explicit launch bounds or maxrrregcount can't be beat + if (regs == -1) return string(""); + // Format an explicit register count setting + return string("--maxrregcount=") + to_string(regs) + " "; +#else + return amdRegisterOption(which_kernel, override_regs); +#endif +} -Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u32 E, const vector& extraConf, bool logFftSize) : - queue(q), +// AMD analog of the CUDA register cap, driven by the same REGxxxx options. override_regs is the value of the kernel's option: +// 0 = not specified (use the default below), -1 = compiler default (no cap), +// 1..10 = minimum waves per SIMD (like CUDA's launch bounds; 10 is the GCN maximum), more than 10 = explicit VGPR count. +// A minimum-waves request caps VGPR usage. On gfx9 (256 VGPRs per lane, allocated in units of 4) the occupancy crossovers are: +// 128 VGPRs for 2 waves, 84 for 3, 64 for 4, 48 for 5. A kernel a few VGPRs above the 128 boundary runs with one wave per SIMD; +// capping it costs a few spills but doubles occupancy. Only that one-wave cliff is worth a default: on a Radeon VII / MI50 the in-place +// fftMiddleIn / fftMiddleOut kernels use 133 VGPRs, and requiring 2 waves per SIMD recovers most of their slowdown. Capping to reach 3 or more +// waves, or capping tailSquare / carryFused, was measured slower. An explicit VGPR count makes rocm generate its usual code and then spill to fit. +string Gpu::amdRegisterOption([[maybe_unused]] enum WHICH_KERNEL which_kernel, int override_regs) { + cl_device_id const id = shared.context->deviceId(); + if (!isAmdGpu(id)) return string(""); + if (override_regs < 0) return string(""); + if (override_regs > 10) return string("-DAMD_NUM_VGPR=") + to_string(override_regs) + " "; + if (override_regs > 0) return string("-DAMD_WAVES_PER_EU=") + to_string(override_regs) + " "; + // Default: 2 waves per SIMD for the in-place middle kernels on Vega class GPUs (gfx900/902/904/906/909/90c: 256 VGPRs per lane). + // Other architectures are untested. + bool const is_middle = which_kernel == MIDIN || which_kernel == MIDIN31 || which_kernel == MIDIN61 || + which_kernel == MIDOUT || which_kernel == MIDOUT31 || which_kernel == MIDOUT61; + string const name = getDeviceName(id); + bool const vega = name.rfind("gfx90", 0) == 0 && name.size() > 5 && string("02469c").find(name[5]) != string::npos; + return (in_place && is_middle && vega) ? string("-DAMD_WAVES_PER_EU=2 ") : string(""); +} + +// Kernels are compiled one at a time, but OpenCL source files contain multiple kernels. This routine set the #defines necessary so that only one kernel is compiled. +// While not strictly necessary, startup speed will be a bit faster if we do less compilations. +string Gpu::kernelDefines(enum WHICH_KERNEL_TYPE which_kernel) { + string defines; + // Determine the kernel specific #defines + switch (which_kernel) { + case KFP: // FP64 or FP32 kernel + defines += toDefine("FFT_FP64", (int) fft.FFT_FP64); + defines += toDefine("FFT_FP32", (int) fft.FFT_FP32); + defines += toDefine("NTT_GF31", 0); + defines += toDefine("NTT_GF61", 0); + break; + case K31: // GF1 kernel + defines += toDefine("FFT_FP64", 0); + defines += toDefine("FFT_FP32", 0); + defines += toDefine("NTT_GF31", (int) fft.NTT_GF31); + defines += toDefine("NTT_GF61", 0); + break; + case K61: // GF61 kernel + defines += toDefine("FFT_FP64", 0); + defines += toDefine("FFT_FP32", 0); + defines += toDefine("NTT_GF31", 0); + defines += toDefine("NTT_GF61", (int) fft.NTT_GF61); + break; + case KALL: // Kernels, like carryFused, that need all defines set properly + defines += toDefine("FFT_FP64", (int) fft.FFT_FP64); + defines += toDefine("FFT_FP32", (int) fft.FFT_FP32); + defines += toDefine("NTT_GF31", (int) fft.NTT_GF31); + defines += toDefine("NTT_GF61", (int) fft.NTT_GF61); + break; + } + return defines + " "; +} + + +Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, bool logFftSize) : + shared(s), background{shared.background}, args{*shared.args}, E(E), N(fft.shape.size()), - fft(fft), + fft(fft), WIDTH(fft.shape.width), SMALL_H(fft.shape.height), BIG_H(SMALL_H * fft.shape.middle), hN(N / 2), nW(fft.shape.nW()), nH(fft.shape.nH()), - useLongCarry{args.carry == Args::CARRY_LONG}, - compiler{args, queue->context, clDefines(args, queue->context->deviceId(), fft, extraConf, E, logFftSize, tail_single_wide, tail_single_kernel, in_place, pad_size)}, + useLongCarry{args.carry == CARRY_64}, + queue{*shared.context, args.profile}, + + compiler{args, shared.context, clDefines(args, shared.context->deviceId(), fft, extraConf, E, logFftSize, tail_single_wide, tail_single_kernel, in_place, pad_size, wmul)}, -#define K(name, ...) name(#name, &compiler, profile.make(#name), queue, __VA_ARGS__) +#define K(name, ...) name(#name, &compiler, profile.make(#name), &queue, __VA_ARGS__) - K(kfftMidIn, "fftmiddlein.cl", "fftMiddleIn", hN / (BIG_H / SMALL_H)), - K(kfftHin, "ffthin.cl", "fftHin", hN / nH), - K(ktailSquareZero, "tailsquare.cl", "tailSquareZero", SMALL_H / nH * 2), + K(kfftMidIn, "fftmiddlein.cl", "fftMiddleIn", hN / (BIG_H / SMALL_H), kernelDefines(KFP) + numRegisters(MIDIN)), + K(kfftHin, "ffthin.cl", "fftHin", hN / nH, kernelDefines(KFP)), + K(ktailSquareZero, "tailsquare.cl", "tailSquareZero", SMALL_H / nH * 2, kernelDefines(KFP)), K(ktailSquare, "tailsquare.cl", "tailSquare", !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2), // Single-wide tailSquare with one kernel - K(ktailMul, "tailmul.cl", "tailMul", hN / nH / 2), - K(ktailMulLow, "tailmul.cl", "tailMul", hN / nH / 2, "-DMUL_LOW=1"), - K(kfftMidOut, "fftmiddleout.cl", "fftMiddleOut", hN / (BIG_H / SMALL_H)), - K(kfftW, "fftw.cl", "fftW", hN / nW), - - K(kfftMidInGF31, "fftmiddlein.cl", "fftMiddleInGF31", hN / (BIG_H / SMALL_H)), - K(kfftHinGF31, "ffthin.cl", "fftHinGF31", hN / nH), - K(ktailSquareZeroGF31, "tailsquare.cl", "tailSquareZeroGF31", SMALL_H / nH * 2), + hN / nH / 2, kernelDefines(KFP) + numRegisters(TAIL)), // Single-wide tailSquare with one kernel + K(ktailMulZero, "tailmul.cl", "tailMulZero", SMALL_H / nH * 2, kernelDefines(KFP)), + K(ktailMulLowZero, "tailmul.cl", "tailMulZero", SMALL_H / nH * 2, kernelDefines(KFP) + "-DMUL_LOW=1"), + K(ktailMul, "tailmul.cl", "tailMul", + !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailMul with two kernels + !tail_single_wide ? hN / nH : // Double-wide tailMul with one kernel + !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailMul with two kernels + hN / nH / 2, kernelDefines(KFP)), // Single-wide tailMul with one kernel + K(ktailMulLow, "tailmul.cl", "tailMul", + !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailMul with two kernels + !tail_single_wide ? hN / nH : // Double-wide tailMul with one kernel + !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailMul with two kernels + hN / nH / 2, kernelDefines(KFP) + "-DMUL_LOW=1"), // Single-wide tailMul with one kernel + K(kfftMidOut, "fftmiddleout.cl", "fftMiddleOut", hN / (BIG_H / SMALL_H), kernelDefines(KFP) + numRegisters(MIDOUT)), + K(kfftW, "fftw.cl", "fftW", hN / nW, kernelDefines(KFP)), + + K(kfftMidInGF31, "fftmiddlein.cl", "fftMiddleInGF31", hN / (BIG_H / SMALL_H), kernelDefines(K31) + numRegisters(MIDIN31)), + K(kfftHinGF31, "ffthin.cl", "fftHinGF31", hN / nH, kernelDefines(K31)), + K(ktailSquareZeroGF31, "tailsquare.cl", "tailSquareZeroGF31", SMALL_H / nH * 2, kernelDefines(K31)), K(ktailSquareGF31, "tailsquare.cl", "tailSquareGF31", !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2), // Single-wide tailSquare with one kernel - K(ktailMulGF31, "tailmul.cl", "tailMulGF31", hN / nH / 2), - K(ktailMulLowGF31, "tailmul.cl", "tailMulGF31", hN / nH / 2, "-DMUL_LOW=1"), - K(kfftMidOutGF31, "fftmiddleout.cl", "fftMiddleOutGF31", hN / (BIG_H / SMALL_H)), - K(kfftWGF31, "fftw.cl", "fftWGF31", hN / nW), - - K(kfftMidInGF61, "fftmiddlein.cl", "fftMiddleInGF61", hN / (BIG_H / SMALL_H)), - K(kfftHinGF61, "ffthin.cl", "fftHinGF61", hN / nH), - K(ktailSquareZeroGF61, "tailsquare.cl", "tailSquareZeroGF61", SMALL_H / nH * 2), + hN / nH / 2, kernelDefines(K31) + numRegisters(TAIL31)), // Single-wide tailSquare with one kernel + K(ktailMulZeroGF31, "tailmul.cl", "tailMulZeroGF31", SMALL_H / nH * 2, kernelDefines(K31)), + K(ktailMulLowZeroGF31, "tailmul.cl", "tailMulZeroGF31", SMALL_H / nH * 2, kernelDefines(K31) + "-DMUL_LOW=1"), + K(ktailMulGF31, "tailmul.cl", "tailMulGF31", + !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailMul with two kernels + !tail_single_wide ? hN / nH : // Double-wide tailMul with one kernel + !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailMul with two kernels + hN / nH / 2, kernelDefines(K31)), // Single-wide tailMul with one kernel + K(ktailMulLowGF31, "tailmul.cl", "tailMulGF31", + !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailMul with two kernels + !tail_single_wide ? hN / nH : // Double-wide tailMul with one kernel + !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailMul with two kernels + hN / nH / 2, kernelDefines(K31) + "-DMUL_LOW=1"), // Single-wide tailMul with one kernel + K(kfftMidOutGF31, "fftmiddleout.cl", "fftMiddleOutGF31", hN / (BIG_H / SMALL_H), kernelDefines(K31) + numRegisters(MIDOUT31)), + K(kfftWGF31, "fftw.cl", "fftWGF31", hN / nW, kernelDefines(K31)), + + K(kfftMidInGF61, "fftmiddlein.cl", "fftMiddleInGF61", hN / (BIG_H / SMALL_H), kernelDefines(K61) + numRegisters(MIDIN61)), + K(kfftHinGF61, "ffthin.cl", "fftHinGF61", hN / nH, kernelDefines(K61)), + K(ktailSquareZeroGF61, "tailsquare.cl", "tailSquareZeroGF61", SMALL_H / nH * 2, kernelDefines(K61)), K(ktailSquareGF61, "tailsquare.cl", "tailSquareGF61", !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2), // Single-wide tailSquare with one kernel - K(ktailMulGF61, "tailmul.cl", "tailMulGF61", hN / nH / 2), - K(ktailMulLowGF61, "tailmul.cl", "tailMulGF61", hN / nH / 2, "-DMUL_LOW=1"), - K(kfftMidOutGF61, "fftmiddleout.cl", "fftMiddleOutGF61", hN / (BIG_H / SMALL_H)), - K(kfftWGF61, "fftw.cl", "fftWGF61", hN / nW), - - K(kfftP, "fftp.cl", "fftP", hN / nW), - K(kCarryA, "carry.cl", "carry", hN / CARRY_LEN), - K(kCarryAROE, "carry.cl", "carry", hN / CARRY_LEN, "-DROE=1"), - K(kCarryM, "carry.cl", "carry", hN / CARRY_LEN, "-DMUL3=1"), - K(kCarryMROE, "carry.cl", "carry", hN / CARRY_LEN, "-DMUL3=1 -DROE=1"), - K(kCarryLL, "carry.cl", "carry", hN / CARRY_LEN, "-DLL=1"), - K(kCarryFused, "carryfused.cl", "carryFused", WIDTH * (BIG_H + 1) / nW), - K(kCarryFusedROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + 1) / nW, "-DROE=1"), - K(kCarryFusedMul, "carryfused.cl", "carryFused", WIDTH * (BIG_H + 1) / nW, "-DMUL3=1"), - K(kCarryFusedMulROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + 1) / nW, "-DMUL3=1 -DROE=1"), - K(kCarryFusedLL, "carryfused.cl", "carryFused", WIDTH * (BIG_H + 1) / nW, "-DLL=1"), - - K(carryB, "carryb.cl", "carryB", hN / CARRY_LEN), + hN / nH / 2, kernelDefines(K61) + numRegisters(TAIL61)), // Single-wide tailSquare with one kernel + K(ktailMulZeroGF61, "tailmul.cl", "tailMulZeroGF61", SMALL_H / nH * 2, kernelDefines(K61)), + K(ktailMulLowZeroGF61, "tailmul.cl", "tailMulZeroGF61", SMALL_H / nH * 2, kernelDefines(K61) + "-DMUL_LOW=1"), + K(ktailMulGF61, "tailmul.cl", "tailMulGF61", + !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailMul with two kernels + !tail_single_wide ? hN / nH : // Double-wide tailMul with one kernel + !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailMul with two kernels + hN / nH / 2, kernelDefines(K61)), // Single-wide tailMul with one kernel + K(ktailMulLowGF61, "tailmul.cl", "tailMulGF61", + !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailMul with two kernels + !tail_single_wide ? hN / nH : // Double-wide tailMul with one kernel + !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailMul with two kernels + hN / nH / 2, kernelDefines(K61) + "-DMUL_LOW=1"), // Single-wide tailMul with one kernel + K(kfftMidOutGF61, "fftmiddleout.cl", "fftMiddleOutGF61", hN / (BIG_H / SMALL_H), kernelDefines(K61) + numRegisters(MIDOUT61)), + K(kfftWGF61, "fftw.cl", "fftWGF61", hN / nW, kernelDefines(K61)), + + K(kfftP, "fftp.cl", "fftP", hN / nW, kernelDefines(KALL)), + K(kCarryA, "carry.cl", "carry", hN / CARRY_LEN, kernelDefines(KALL)), + K(kCarryAROE, "carry.cl", "carry", hN / CARRY_LEN, kernelDefines(KALL) + "-DROE=1"), + K(kCarryM, "carry.cl", "carry", hN / CARRY_LEN, kernelDefines(KALL) + "-DMUL3=1"), + K(kCarryMROE, "carry.cl", "carry", hN / CARRY_LEN, kernelDefines(KALL) + "-DMUL3=1 -DROE=1"), + K(kCarryLL, "carry.cl", "carry", hN / CARRY_LEN, kernelDefines(KALL) + "-DLL=1"), + K(kCarryFused, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numRegisters(CARRYFUSED)), + K(kCarryFusedROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numRegisters(CARRYFUSED) + "-DROE=1"), + K(kCarryFusedMul, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numRegisters(CARRYFUSED) + "-DMUL3=1"), + K(kCarryFusedMulROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numRegisters(CARRYFUSED) + "-DMUL3=1 -DROE=1"), + K(kCarryFusedLL, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numRegisters(CARRYFUSED) + "-DLL=1"), + + K(carryB, "carryb.cl", "carryB", hN / CARRY_LEN, kernelDefines(KALL)), // 64 K(transpIn, "transpose.cl", "transposeIn", hN / 64), @@ -599,7 +970,7 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u32 E, const vector& K(kernIsEqual, "etc.cl", "isEqual", 256 * 256, "-DISEQUAL=1"), K(sum64, "etc.cl", "sum64", 256 * 256, "-DSUM64=1"), - K(testTrig, "selftest.cl", "testTrig", 256 * 256), + K(testTrig, "selftest.cl", "testTrig", 256 * 256), K(testFFT4, "selftest.cl", "testFFT4", 256), K(testFFT14, "selftest.cl", "testFFT14", 256), K(testFFT15, "selftest.cl", "testFFT15", 256), @@ -612,12 +983,11 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u32 E, const vector& bufTrigM{shared.bufCache->middleTrig(shared.args, fft, SMALL_H, BIG_H / SMALL_H, WIDTH)}, bufTrigW{shared.bufCache->smallTrig(shared.args, fft, WIDTH, nW, fft.shape.middle, SMALL_H, nH, tail_single_wide)}, - weights{genWeights(fft, E, WIDTH, BIG_H, nW, isAmdGpu(q->context->deviceId()))}, - bufConstWeights{q->context, std::move(weights.weightsConstIF)}, - bufWeights{q->context, std::move(weights.weightsIF)}, - bufBits{q->context, std::move(weights.bitsCF)}, + weights{genWeights(fft, E, WIDTH, BIG_H, nW, isNvidiaGpu(shared.context->deviceId()))}, + bufConstWeights{shared.context, std::move(weights.weightsConstIF)}, + bufWeights{shared.context, std::move(weights.weightsIF)}, -#define BUF(name, ...) name{profile.make(#name), queue, __VA_ARGS__} +#define BUF(name, ...) name{profile.make(#name), &queue, __VA_ARGS__} // GPU Buffers containing integer data. Since this buffer is type i64, if fft.WordSize < 8 then we need less memory allocated. BUF(bufData, N * fft.WordSize / sizeof(Word)), @@ -630,8 +1000,8 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u32 E, const vector& BUF(bufSmallOut, 256), BUF(bufSumOut, 1), BUF(bufTrue, 1), - BUF(bufROE, ROE_SIZE), - BUF(bufStatsCarry, CARRY_SIZE), + BUF(bufROE, ROE_SIZE + 2), + BUF(bufStatsCarry, CARRY_SIZE + 2), BUF(buf1, TOTAL_DATA_SIZE(fft, WIDTH, fft.shape.middle, SMALL_H, in_place, pad_size)), BUF(buf2, TOTAL_DATA_SIZE(fft, WIDTH, fft.shape.middle, SMALL_H, in_place, pad_size)), @@ -639,17 +1009,22 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u32 E, const vector& #undef BUF statsBits{u32(args.value("STATS", 0))}, - timeBufVect{profile.make("proofBufVect")} -{ + timeBufVect{profile.make("proofBufVect")}, + + recorded_kernels{}, + recorded_kernel_args{}, - float bitsPerWord = E / float(N); + use_graphs{}, + graph_square{} +{ + float const bitsPerWord = E / float(N); if (logFftSize) { log("FFT: %s %s (%.2f bpw)\n", numberK(N).c_str(), fft.spec().c_str(), bitsPerWord); // Sometimes we do want to run a FFT beyond a reasonable BPW (e.g. during -ztune), and these situations // coincide with logFftSize == false if (fft.maxExp() < E) { - log("Warning: %s (max %" PRIu64 ") may be too small for %u\n", fft.spec().c_str(), fft.maxExp(), E); + log("Warning: %s (max %" PRIu64 ") may be too small for %" PRIu64 "\n", fft.spec().c_str(), fft.maxExp(), E); } } @@ -663,48 +1038,54 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u32 E, const vector& if (useLongCarry) { log("Using long carry!\n"); } if (fft.FFT_FP64 || fft.FFT_FP32) { - kfftMidIn.setFixedArgs(2, bufTrigM); - kfftHin.setFixedArgs(2, bufTrigH); + kfftMidIn.setFixedArgs(3, bufTrigM); + kfftHin.setFixedArgs(3, bufTrigH); ktailSquareZero.setFixedArgs(2, bufTrigH); - ktailSquare.setFixedArgs(2, bufTrigH); - ktailMulLow.setFixedArgs(3, bufTrigH); - ktailMul.setFixedArgs(3, bufTrigH); - kfftMidOut.setFixedArgs(2, bufTrigM); + ktailSquare.setFixedArgs(3, bufTrigH); + ktailMulZero.setFixedArgs(3, bufTrigH); + ktailMulLowZero.setFixedArgs(3, bufTrigH); + ktailMulLow.setFixedArgs(4, bufTrigH); + ktailMul.setFixedArgs(4, bufTrigH); + kfftMidOut.setFixedArgs(3, bufTrigM); kfftW.setFixedArgs(2, bufTrigW); } if (fft.NTT_GF31) { - kfftMidInGF31.setFixedArgs(2, bufTrigM); - kfftHinGF31.setFixedArgs(2, bufTrigH); + kfftMidInGF31.setFixedArgs(3, bufTrigM); + kfftHinGF31.setFixedArgs(3, bufTrigH); ktailSquareZeroGF31.setFixedArgs(2, bufTrigH); - ktailSquareGF31.setFixedArgs(2, bufTrigH); - ktailMulLowGF31.setFixedArgs(3, bufTrigH); - ktailMulGF31.setFixedArgs(3, bufTrigH); - kfftMidOutGF31.setFixedArgs(2, bufTrigM); + ktailSquareGF31.setFixedArgs(3, bufTrigH); + ktailMulZeroGF31.setFixedArgs(3, bufTrigH); + ktailMulLowZeroGF31.setFixedArgs(3, bufTrigH); + ktailMulLowGF31.setFixedArgs(4, bufTrigH); + ktailMulGF31.setFixedArgs(4, bufTrigH); + kfftMidOutGF31.setFixedArgs(3, bufTrigM); kfftWGF31.setFixedArgs(2, bufTrigW); } if (fft.NTT_GF61) { - kfftMidInGF61.setFixedArgs(2, bufTrigM); - kfftHinGF61.setFixedArgs(2, bufTrigH); + kfftMidInGF61.setFixedArgs(3, bufTrigM); + kfftHinGF61.setFixedArgs(3, bufTrigH); ktailSquareZeroGF61.setFixedArgs(2, bufTrigH); - ktailSquareGF61.setFixedArgs(2, bufTrigH); - ktailMulLowGF61.setFixedArgs(3, bufTrigH); - ktailMulGF61.setFixedArgs(3, bufTrigH); - kfftMidOutGF61.setFixedArgs(2, bufTrigM); + ktailSquareGF61.setFixedArgs(3, bufTrigH); + ktailMulZeroGF61.setFixedArgs(3, bufTrigH); + ktailMulLowZeroGF61.setFixedArgs(3, bufTrigH); + ktailMulLowGF61.setFixedArgs(4, bufTrigH); + ktailMulGF61.setFixedArgs(4, bufTrigH); + kfftMidOutGF61.setFixedArgs(3, bufTrigM); kfftWGF61.setFixedArgs(2, bufTrigW); } - if (fft.FFT_FP64 || fft.FFT_FP32) { // The FP versions take bufWeight arguments (and bufBits which may be deleted) + if (fft.FFT_FP64 || fft.FFT_FP32) { // The FP versions take bufWeight arguments kfftP.setFixedArgs(2, bufTrigW, bufWeights); for (Kernel* k : {&kCarryA, &kCarryAROE, &kCarryM, &kCarryMROE, &kCarryLL}) { k->setFixedArgs(3, bufCarry, bufWeights); } for (Kernel* k : {&kCarryA, &kCarryM, &kCarryLL}) { k->setFixedArgs(5, bufStatsCarry); } for (Kernel* k : {&kCarryAROE, &kCarryMROE}) { k->setFixedArgs(5, bufROE); } for (Kernel* k : {&kCarryFused, &kCarryFusedROE, &kCarryFusedMul, &kCarryFusedMulROE, &kCarryFusedLL}) { - k->setFixedArgs(3, bufCarry, bufReady, bufTrigW, bufBits, bufConstWeights, bufWeights); + k->setFixedArgs(3, bufCarry, bufReady, bufTrigW, bufConstWeights, bufWeights); } - for (Kernel* k : {&kCarryFusedROE, &kCarryFusedMulROE}) { k->setFixedArgs(9, bufROE); } - for (Kernel* k : {&kCarryFused, &kCarryFusedMul, &kCarryFusedLL}) { k->setFixedArgs(9, bufStatsCarry); } + for (Kernel* k : {&kCarryFusedROE, &kCarryFusedMulROE}) { k->setFixedArgs(8, bufROE); } + for (Kernel* k : {&kCarryFused, &kCarryFusedMul, &kCarryFusedLL}) { k->setFixedArgs(8, bufStatsCarry); } } else { kfftP.setFixedArgs(2, bufTrigW); for (Kernel* k : {&kCarryA, &kCarryAROE, &kCarryM, &kCarryMROE, &kCarryLL}) { k->setFixedArgs(3, bufCarry); } @@ -726,66 +1107,594 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u32 E, const vector& bufStatsCarry.zero(); bufTrue.write({1}); - if (args.verbose) { + if (args.verbose >= 99) { selftestTrig(); } - queue->setSquareKernels(1 + 3 * (fft.FFT_FP64 + fft.FFT_FP32 + fft.NTT_GF31 + fft.NTT_GF61)); - queue->finish(); + // Create aux queues. For now, we only have one auxiliary queue. We could do more. + if (args.value("MULTI_Q", 0)) { + auxQueues.push_back(Queue{*shared.context, args.profile, true}); + } + + // Set flag indicating we're going to use CUDA graphs. + use_graphs = graph_square[0].isSupported(shared.context->deviceId()) && args.value("GRAPHS", 1); + + // Set L1 cache configuration. Really we should only do this once rather than once per worker. + // However, the current way PRPLL is organized would then make this option hard to tune. +#if CUDA_BACKEND + cudaSetL1Config(args.value("L1CUDA", 0)); +#endif + + // Process the queue. I don't know if this is really needed. + queue.finish(); } +// Optionallly split some of the MiddleIn/Tail/MiddleOut kernels off of executing on the main queue to run on an auxiliary queue. +// This will increase GPU occupancy but will negatively impact L2 cache coherency. +// If the L2 cache is large enough so that all FFT data fits in the cache, this ought to be a win. +// If the L2 cache is small enough such that L2 cache hits are very low anyway, this might be a win. +void Gpu::splitQueue() { + // Queue a sync event in the main queue. Have all auxiliary queues wait on the event. + EventHolder event = queue.createSyncEvent(); + for (auto & auxQueue : auxQueues) { + auxQueue.waitForSyncEvent(&event); + } +} -// Call the appropriate kernels to support hybrid FFTs and NTTs +void Gpu::mergeQueue() { + // Queue a sync event in each auxiliary queue(s). Wait on the event(s) in the main queue. + for (auto & auxQueue : auxQueues) { + EventHolder event = auxQueue.createSyncEvent(); + queue.waitForSyncEvent(&event); + } +} + +// We've finished the "bottom half" of a squaring or multiply. Replay the recorded bottom half kernel calls. +void Gpu::endBottomHalf() { + replay(); + // Increment the squaring count. The queue's squarings/multiplies count determines how long to sleep when the queue is full. + // We only do this for the main command queue. Auxiliary queues are not allowed to cause a CPU sleep. + queue.incSquareCount(); +} + +// Replay the recorded bottom half kernels in a cache friendly order. We support several options here using multiple openCl command queues. +void Gpu::replay() { + // If there are no recorded kernels to replay, we're done + if (recorded_kernels.size() == 0) return; + + // Get MULTI_Q and L2_STRIPING settings + bool multi_q = args.value("MULTI_Q", 0); + int l2_striping = args.value("L2_STRIPING", 0); + + // In the simplest case, we use one command queue and process one data type at a time. By processing one data type at a time, we reduce maximum L2 cache used. + // For example, a 4M GF61+GF31 NTT needs just 32MB L2 cache during GF61 processing of fftMiddleIn, tailSquare, and fftMiddleOut (and only 16MB duing GF31 processing). + // Without MULTI_Q, PRPLL needs 32MB + 16MB of L2 cache by processing both fftMiddleIns, then both tailSquares, then both fftMiddleOuts. + + if ((!multi_q || fft.shape.fft_type == FFT64 || fft.shape.fft_type == FFT61 || fft.shape.fft_type == FFT31 || fft.shape.fft_type == FFT32) && !l2_striping) { + for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { + // Check for irrelevant cache group + if (cache_group == 1 && !(fft.FFT_FP64 || fft.FFT_FP32)) continue; + if (cache_group == 2 && !fft.NTT_GF31) continue; + if (cache_group == 3 && !fft.NTT_GF61) continue; + + // Iterate over the recorded kernels. Execute each. + int arg = 0; + for (auto kern : recorded_kernels) { + replay_one(kern, cache_group, arg); + arg = replay_next_arg(kern, arg); + } + } + } + + // The next simple case, we use two command queues and process one data type in each queue. This works well for large L2 caches where all FFT data fits in the cache. + // The extra command queue can hide the latency in starting up kernels for each data type. Also occupancy may benefit as the queue may be executing kernels with different + // workgroup size, register usage, and local memory usage. This case requires an FFT using at least two data types. + + else if (multi_q && !l2_striping) { + // Switch tp using multiple command queues. + splitQueue(); + for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { + // Check for irrelevant cache group + if (cache_group == 1 && !(fft.FFT_FP64 || fft.FFT_FP32)) continue; + if (cache_group == 2 && !fft.NTT_GF31) continue; + if (cache_group == 3 && !fft.NTT_GF61) continue; + + // To better balance the load on the two command queues, put a 64-bit data type in one queue and two 32-bit data types in the other queue. + Queue *q; + if (cache_group == 1) q = &queue; + if (cache_group == 2) q = (fft.shape.fft_type == FFT323161 || fft.shape.fft_type == FFT3161) ? &queue : &auxQueues[0]; + if (cache_group == 3) q = &auxQueues[0]; + + // Iterate over the recorded kernels. Execute each. + int arg = 0; + for (auto kern : recorded_kernels) { + replay_one(kern, cache_group, arg, q); + arg = replay_next_arg(kern, arg); + } + } + // Using multiple command queues, go back to a single command queue + mergeQueue(); + } + + // The next case is L2 striping in one command queue. The hope is two stripe groups plus one stripe are small enough to fit in the L2 caches + // for the fftMiddleIn, tailSquare, and fftMiddleOut kernels. + // Sadly, testing thusfar shows the extra overhead of more kernel launches and events/syncs outweighs the benefit of more L2 cache hits. +#ifdef ORIGINAL_VERSION // Very readable, replaced by version below which merges the teo base_lo and base_hi kernel calls into one combined kernel call + else if (!multi_q && l2_striping) { + for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { + // Check for irrelevant cache group + if (cache_group == 1 && !(fft.FFT_FP64 || fft.FFT_FP32)) continue; + if (cache_group == 2 && !fft.NTT_GF31) continue; + if (cache_group == 3 && !fft.NTT_GF61) continue; + + // Allow larger caches to do several L2 stripes in a single "stripe group" at a time (increases occupancy, reduces kernel launch costs). + u32 stripe_group_size = l2_striping; + +//For now, only support INPLACE with its 16x16 transpose + + // Loop over all the stripes for this data type. Let's define a stripe as one "column" of data processed by fftMiddleIn producing 16*MIDDLE tailSquare lines. + // Since tailSquare operates on Hermetian pairs of lines, fftMiddleIn alternates operating on low stripes and high stripes. + u32 num_stripes = fft.shape.width / 16; + u32 num_stripe_groups = num_stripes / stripe_group_size; + for (u32 i = 0; i < num_stripe_groups / 2; ++i) { + // Base_lo refers to the starting fftMiddleIn column number (x coordinate). When fftMiddleIn uses a 16x16 transpose, base_lo advances 16 at a time. + // Base_hi refers to the fftMiddleIn column number that outputs the lines needed for Hermetian matching in tailSquare. + u32 base_lo = i * stripe_group_size * 16; + u32 base_hi = (num_stripe_groups - 1 - i) * stripe_group_size * 16; + bool last_block = (base_hi == fft.shape.width / 2); + + // Iterate over the recorded kernels. Execute each. + int arg = 0; + for (auto kern : recorded_kernels) { + + if (kern == KMIDIN) { + u32 oneStripeKernelsToExecute = fft.shape.height / 16; + // If MIDDLE is odd, the first fftMiddleIn call must be preceeded by a fftMiddleIn call to produce the special N/2 tailSquare line from the WIDTH/2 column. + if (base_lo == 0 && (fft.shape.middle & 1)) { + replay_one(kern, cache_group, arg, &queue, fft.shape.width / 2, oneStripeKernelsToExecute); + } + + // Produce one stripe group. The last base value must take into account that the N/2 stripe has already been done if MIDDLE is odd. + u32 kernelsToExecute = stripe_group_size * oneStripeKernelsToExecute; + replay_one(kern, cache_group, arg, &queue, base_lo, kernelsToExecute); + + u32 base = base_hi; // Read full base_hi stripe groups (usually) + if (last_block && (fft.shape.middle & 1)) base += 16, kernelsToExecute -= oneStripeKernelsToExecute; // Skip first stripe for last block if MIDDLE is odd + if (kernelsToExecute) replay_one(kern, cache_group, arg, &queue, base, kernelsToExecute); + } + + else if (kern == KFFTHIN) { + // fftMiddleIn produces 16 * MIDDLE lines + replay_one(kern, cache_group, arg, &queue, base_lo, stripe_group_size * 16 * fft.shape.middle); + replay_one(kern, cache_group, arg, &queue, base_hi, stripe_group_size * 16 * fft.shape.middle); + } + + else if (kern == KTAILSQUARE || kern == KTAILMUL || kern == KTAILMULLOW) { + // fftMiddleIn produces 16 * MIDDLE lines in base_lo and base_hi. tailSquare kernel processes 2 lines linked by Hermetian symmetry. + // Tail kernels use two dimensions, the X coordinate bumps line number by one, the Y coordinate bumps line number by WIDTH. + u32 kernelsToExecuteX = stripe_group_size * 16; + u32 kernelsToExecuteY = (fft.shape.middle + 1) / 2; // For base_lo, round odd middles up. + + // We can now completely process lines from the lower half of the base_lo stripe group (Hermetian mates are mostly in upper half of base_hi stripe group) + replay_one(kern, cache_group, arg, &queue, base_lo, kernelsToExecuteX, kernelsToExecuteY); + + // We can process most lines from the lower half of the base_hi stripe group (Hermetian mates are mostly in upper half of base_lo stripe group) + // The first line in the stripe group is the only line that is not ready for base_hi tail processing. + u32 base = base_hi + 1; // Skip first line in base_hi (usually) + if (base_lo == 0) kernelsToExecuteX--; // Do one fewer line for the first tail call. + if (base_hi == fft.shape.width / 2) base--, kernelsToExecuteX++; // Last tail call does not skip first line + if (fft.shape.middle & 1) kernelsToExecuteY--; // For base_hi, round odd middles down. + replay_one(kern, cache_group, arg, &queue, base, kernelsToExecuteX, kernelsToExecuteY); + } + + else if (kern == KMIDOUT) { + u32 oneStripeKernelsToExecute = fft.shape.height / 16; + u32 kernelsToExecute = stripe_group_size * oneStripeKernelsToExecute; + + // We've completely processed lines from the base_lo stripe group + replay_one(kern, cache_group, arg, &queue, base_lo, kernelsToExecute); + + // The first stripe in the base_hi stripe group is not ready for output. + u32 base = base_hi + 16; // Skip first stripe in base_hi (usually) + if (base_lo == 0) kernelsToExecute -= oneStripeKernelsToExecute; // Do one fewer stripe for the first midOut call. + if (base_hi == fft.shape.width / 2) base -= 16, kernelsToExecute += oneStripeKernelsToExecute; // Last midOut call does not skip first stripe + if (kernelsToExecute) replay_one(kern, cache_group, arg, &queue, base, kernelsToExecute); + } + + // Skip other kernels (KFFTW) + else; + + // Advance argument index + arg = replay_next_arg(kern, arg); + } + } + + // Iterate over the recorded kernels. Execute any not already executed (KFFTW). + // FFTW cannot benefit from L2 striping, it can only benefit from datatype grouping. + int arg = 0; + for (auto kern : recorded_kernels) { + if (kern == KFFTW) replay_one(kern, cache_group, arg, &queue); + arg = replay_next_arg(kern, arg); + } + } + } +#endif + + // The next case is L2 striping in one command queue. The hope is two stripe groups plus one stripe are small enough to fit in the L2 caches + // for the fftMiddleIn, tailSquare, and fftMiddleOut kernels. + // Sadly, testing thusfar shows the extra overhead of more kernel launches and events/syncs outweighs the benefit of more L2 cache hits. + + else if (!multi_q && l2_striping) { + for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { + // Check for irrelevant cache group + if (cache_group == 1 && !(fft.FFT_FP64 || fft.FFT_FP32)) continue; + if (cache_group == 2 && !fft.NTT_GF31) continue; + if (cache_group == 3 && !fft.NTT_GF61) continue; + + // Allow larger caches to do several L2 stripes in a single "stripe group" at a time (increases occupancy, reduces kernel launch costs). + u32 stripe_group_size = l2_striping; + +//For now, only support INPLACE with its 16x16 transpose + + // Loop over all the stripes for this data type. Let's define a stripe as one "column" of data processed by fftMiddleIn producing 16*MIDDLE tailSquare lines. + // Since tailSquare operates on Hermetian pairs of lines, fftMiddleIn alternates operating on low stripes and high stripes. + u32 num_stripes = fft.shape.width / 16; + u32 num_stripe_groups = num_stripes / stripe_group_size; + for (u32 i = 0; i < num_stripe_groups / 2; ++i) { + bool last_block = (i == num_stripe_groups / 2 - 1); + // Base_lo refers to the starting fftMiddleIn column number (x coordinate). When fftMiddleIn uses a 16x16 transpose, base_lo advances 16 at a time. + // Base_hi refers to the fftMiddleIn column number that outputs the lines needed for Hermetian matching in tailSquare. + u32 base_lo = i * stripe_group_size * 16; + // u32 base_hi = (num_stripe_groups - 1 - i) * stripe_group_size * 16; + + // Iterate over the recorded kernels. Execute each. + int arg = 0; + for (auto kern : recorded_kernels) { + + if (kern == KMIDIN) { + // Do midIn on base_lo and base_hi. If MIDDLE is odd, the first midIn call must be preceeded by a midIn call to produce the special N/2 tailSquare + // line from the WIDTH/2 column. The last base pair must take into account that the N/2 stripe has already been done if MIDDLE is odd. + u32 oneStripeKernelsToExecute = fft.shape.height / 16; + u32 kernelsToExecute = 2 * stripe_group_size * oneStripeKernelsToExecute; + if (base_lo == 0 && (fft.shape.middle & 1)) kernelsToExecute += oneStripeKernelsToExecute; + if (last_block && (fft.shape.middle & 1)) kernelsToExecute -= oneStripeKernelsToExecute; + replay_one(kern, cache_group, arg, &queue, base_lo, kernelsToExecute); + } + + else if (kern == KFFTHIN) { + // fftMiddleIn produces 2 * stripe_group_size * 16 * MIDDLE lines + replay_one(kern, cache_group, arg, &queue, base_lo, 2 * stripe_group_size * 16 * fft.shape.middle); + } + + else if (kern == KTAILSQUARE || kern == KTAILMUL || kern == KTAILMULLOW) { + // We can now completely process lines from the lower half of the base_lo stripe group (Hermetian mates are mostly in upper half of base_hi stripe group) + u32 half_size = (fft.shape.middle + 1) / 2; // For base_lo, round odd middles up. + u32 stripeGroupLines = stripe_group_size * 16; + u32 kernelsToExecute = half_size * stripeGroupLines; + // We can process most lines from the lower half of the base_hi stripe group (Hermetian mates are mostly in upper half of base_lo stripe group) + // The first line in the stripe group is the only line that is not ready for base_hi tail processing. + if (base_lo == 0) stripeGroupLines--; // Do one fewer line for the first tail call. + if (last_block) stripeGroupLines++; // Last tail call does not skip first line + if (fft.shape.middle & 1) half_size--; // For base_hi, round odd middles down. + kernelsToExecute += half_size * stripeGroupLines; + // Call the kernel + replay_one(kern, cache_group, arg, &queue, base_lo, kernelsToExecute); + } + + else if (kern == KMIDOUT) { + u32 oneStripeKernelsToExecute = fft.shape.height / 16; + u32 kernelsToExecute = 2 * stripe_group_size * oneStripeKernelsToExecute; + // We've completely processed lines from the base_lo stripe group. + // The first stripe in the base_hi stripe group is not ready for output. + // The last stripe group will does an extra stripe. + if (base_lo == 0) kernelsToExecute -= oneStripeKernelsToExecute; // Do one fewer stripe for the first midOut call + if (last_block) kernelsToExecute += oneStripeKernelsToExecute; // Last midOut call does not skip first stripe + replay_one(kern, cache_group, arg, &queue, base_lo, kernelsToExecute); + } + + // Skip other kernels (KFFTW) + else {} + + // Advance argument index + arg = replay_next_arg(kern, arg); + } + } -void Gpu::fftP(Buffer& out, Buffer& in) { - kfftP(out, in); + // Iterate over the recorded kernels. Execute any not already executed (KFFTW). + // FFTW cannot benefit from L2 striping, it can only benefit from datatype grouping. + int arg = 0; + for (auto kern : recorded_kernels) { + if (kern == KFFTW) replay_one(kern, cache_group, arg, &queue); + arg = replay_next_arg(kern, arg); + } + } + } + + // The last case is L2 striping in two command queues. The hope is four stripe groups plus two stripes are small enough to fit in the L2 caches for the + // fftMiddleIn, tailSquare, and fftMiddleOut kernels. The hope also is the dual queue approach hides the overhead introduced by more kernel launches. + // Sadly, testing thusfar shows the extra overhead of more kernel launches and events/syncs outweighs the benefit of more L2 cache hits. + + else if (multi_q && l2_striping) { + Queue *queues[2] = {&queue, &auxQueues[0]}; + EventHolder midInEvents[2]; + EventHolder tailEvents[2]; + + splitQueue(); + for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { + bool has_unexecuted_kernels = false; + + // Check for irrelevant cache group + if (cache_group == 1 && !(fft.FFT_FP64 || fft.FFT_FP32)) continue; + if (cache_group == 2 && !fft.NTT_GF31) continue; + if (cache_group == 3 && !fft.NTT_GF61) continue; + + // Allow larger caches to do several L2 stripes at a time (increases occupancy, reduces kernel launch costs). + u32 stripe_group_size = l2_striping; + +//For now, only support INPLACE with its 16x16 transpose + + // Loop over all the stripes for this data type. Let's define a stripe as one "column" of data processed by fftMiddleIn producing 16*MIDDLE tailSquare lines. + // Since tailSquare operates on Hermetian pairs of lines, fftMiddleIn alternates operating on low stripes and high stripes. + u32 num_stripes = fft.shape.width / 16; + u32 num_stripe_groups = num_stripes / stripe_group_size; + for (u32 i = 0; i < num_stripe_groups / 2; ++i) { + int q = (i & 1); // Index into which command queue to use + u32 i_within_queue = i >> 1; + bool last_i_within_queue = (i_within_queue == num_stripe_groups / 4 - 1); + // Base_lo refers to the starting fftMiddleIn column number (x coordinate). When fftMiddleIn uses a 16x16 transpose, base_lo advances 16 at a time. + // Base_hi refers to the fftMiddleIn column number that outputs the lines needed for Hermetian matching in tailSquare. + u32 base_lo = (i_within_queue + q * num_stripe_groups / 4) * stripe_group_size * 16; + //u32 base_hi = fft.shape.width - stripe_group_size * 16 - base_lo; + + // Iterate over the recorded kernels. Execute each. + int arg = 0; + for (auto kern : recorded_kernels) { + + if (kern == KMIDIN) { + // Do midIn on base_lo and base_hi. If MIDDLE is odd, the first midIn call must be preceeded by a midIn call to produce the special N/2 tailSquare + // line from the WIDTH/2 column. The first midIn call in the second command queue also must be preceeded by a midIn call to produce one L2 stripe. + // The last base_hi in each queue must take into account pre-read stripes in the other queue. +#define q_requires_preread(q) (((q) == 0 && fft.shape.middle & 1) || ((q) == 1)) + u32 oneStripeKernelsToExecute = fft.shape.height / 16; + u32 kernelsToExecute = 2 * stripe_group_size * oneStripeKernelsToExecute; + if (i_within_queue == 0 && q_requires_preread(q)) kernelsToExecute += oneStripeKernelsToExecute; + if (last_i_within_queue && q_requires_preread(!q)) kernelsToExecute -= oneStripeKernelsToExecute; + replay_one(kern, cache_group, arg, queues[q], base_lo, kernelsToExecute); + if (i_within_queue == 0 && q_requires_preread(q)) midInEvents[q] = queues[q]->createSyncEvent(); + // The last block must sync with a pre-read from the other command queue + // BUG - if block is both i_within_queue == 0 and last_block_in_queue, then midInEvents[1] does not exist! + // Sanity checking L2_STRIPING setting to a max of WIDTH/128 at startup eliminates this bug. + if (last_i_within_queue && q_requires_preread(!q)) queues[q]->waitForSyncEvent(&midInEvents[!q]); + } + + else if (kern == KFFTHIN) { + // fftMiddleIn produces 2 * stripe_group_size * 16 * MIDDLE lines + replay_one(kern, cache_group, arg, queues[q], base_lo, 2 * stripe_group_size * 16 * fft.shape.middle); + } + + else if (kern == KTAILSQUARE || kern == KTAILMUL || kern == KTAILMULLOW) { + // We can now completely process lines from the lower half of the base_lo stripe group (Hermetian mates are mostly in upper half of base_hi stripe group) + u32 half_size = (fft.shape.middle + 1) / 2; // For base_lo, round odd middles up. + u32 stripeGroupLines = stripe_group_size * 16; + u32 kernelsToExecute = half_size * stripeGroupLines; + // We can process most lines from the lower half of the base_hi stripe group (Hermetian mates are mostly in upper half of base_lo stripe group) + // The first line in the stripe group is the only line that is not ready for base_hi tail processing. + if (i == 0) stripeGroupLines--; // Do one fewer line for the first tail call. + if (last_i_within_queue && q == 1) stripeGroupLines++; // Last tail call does not skip first line + half_size = fft.shape.middle / 2; // For base_hi, round odd middles down. + kernelsToExecute += half_size * stripeGroupLines; + // Call the kernel + replay_one(kern, cache_group, arg, queues[q], base_lo, kernelsToExecute); + if (i_within_queue == 0 && q_requires_preread(q)) tailEvents[q] = queues[q]->createSyncEvent(); +// We could eliminate two events and syncs by having the last midIn wait on the first tailsquare. It exposes a little less parallellism, but perhaps that is irrelevant. +// For even middles, we only save one event and sync. + // The last block must sync with the first tailSquare in the other command queue + // BUG - if block is both i_within_queue == 0 and last_block_in_queue, then tailEvents[1] does not exist! + // Sanity checking L2_STRIPING setting to a max of WIDTH/128 at startup eliminates this bug. + if (last_i_within_queue && q_requires_preread(!q)) queues[q]->waitForSyncEvent(&tailEvents[!q]); + } + + else if (kern == KMIDOUT) { + u32 oneStripeKernelsToExecute = fft.shape.height / 16; + u32 kernelsToExecute = 2 * stripe_group_size * oneStripeKernelsToExecute; + // We've completely processed lines from the base_lo stripe group. + // The first stripe in the base_hi stripe group is not ready for output. + // The last stripe group does an extra stripe. + if (i_within_queue == 0) kernelsToExecute -= oneStripeKernelsToExecute; // Do one fewer stripe for the first midOut call + if (last_i_within_queue) kernelsToExecute += oneStripeKernelsToExecute; // Last midOut call does not skip first stripe + replay_one(kern, cache_group, arg, queues[q], base_lo, kernelsToExecute); + } + + // Skip other kernels (KFFTW) + else + has_unexecuted_kernels = true; + + // Advance argument index + arg = replay_next_arg(kern, arg); + } + } + + // Iterate over the recorded kernels again. Execute any not already executed (KFFTW). + // FFTW cannot benefit from L2 striping, it can only benefit from datatype grouping. + if (has_unexecuted_kernels) { + int arg = 0; + mergeQueue(); + for (auto kern : recorded_kernels) { + if (kern == KFFTW) replay_one(kern, cache_group, arg, &queue); + arg = replay_next_arg(kern, arg); + } + splitQueue(); + } + } + mergeQueue(); + } + + // Empty the recorded kernels queue + recorded_kernels.clear(); + recorded_kernel_args.clear(); +} + +// Replay one recorded kernel on the specified queue, with specified base and kernelsToExecute. Two dimensional work groups are supported for some kernels. +// A kernelsToExecuteX of zero is permitted - used for the default kernelsToExecute (a.k.a. workSize) set at kernel creation that operates on all the FFT data. +void Gpu::replay_one(enum BOTTOM_HALF_KERNELS kern, int cache_group, int arg, Queue *q, int base, int kernelsToExecuteX, int kernelsToExecuteY) { + + // Call the appropriate kernel + if (kern == KMIDIN) { + Buffer const *buf = recorded_kernel_args[arg++]; + // If not in place, the input is from the scratch buffer + Buffer const *in = in_place ? buf : &buf3; + Buffer const *out = buf; + if (cache_group == 1) { kfftMidIn.setQueue(q); kfftMidIn.setKernelsToExecute(kernelsToExecuteX); kfftMidIn(*out, *in, base); } + if (cache_group == 2) { kfftMidInGF31.setQueue(q); kfftMidInGF31.setKernelsToExecute(kernelsToExecuteX); kfftMidInGF31(*out, *in, base); } + if (cache_group == 3) { kfftMidInGF61.setQueue(q); kfftMidInGF61.setKernelsToExecute(kernelsToExecuteX); kfftMidInGF61(*out, *in, base); } + } + + if (kern == KFFTHIN) { + Buffer const *out = recorded_kernel_args[arg++]; + Buffer const *in = recorded_kernel_args[arg++]; + if (cache_group == 1) { kfftHin.setQueue(q); kfftHin.setKernelsToExecute(kernelsToExecuteX); kfftHin(*out, *in, base); } + if (cache_group == 2) { kfftHinGF31.setQueue(q); kfftHinGF31.setKernelsToExecute(kernelsToExecuteX); kfftHinGF31(*out, *in, base); } + if (cache_group == 3) { kfftHinGF61.setQueue(q); kfftHinGF61.setKernelsToExecute(kernelsToExecuteX); kfftHinGF61(*out, *in, base); } + } + + if (kern == KTAILSQUARE) { + Buffer const *buf = recorded_kernel_args[arg++]; + // If not in place, the output is to the scratch buffer + Buffer const *in = buf; + Buffer const *out = in_place ? buf : &buf3; + if (!tail_single_kernel && base == 0) { + if (cache_group == 1) { ktailSquareZero.setQueue(q); ktailSquareZero(*out, *in); } + if (cache_group == 2) { ktailSquareZeroGF31.setQueue(q); ktailSquareZeroGF31(*out, *in); } + if (cache_group == 3) { ktailSquareZeroGF61.setQueue(q); ktailSquareZeroGF61(*out, *in); } + if (kernelsToExecuteX) kernelsToExecuteX--; + } + if (cache_group == 1) { ktailSquare.setQueue(q); ktailSquare.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailSquare(*out, *in, base); } + if (cache_group == 2) { ktailSquareGF31.setQueue(q); ktailSquareGF31.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailSquareGF31(*out, *in, base); } + if (cache_group == 3) { ktailSquareGF61.setQueue(q); ktailSquareGF61.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailSquareGF61(*out, *in, base); } + } + + if (kern == KTAILMUL) { + Buffer const *buf = recorded_kernel_args[arg++]; + Buffer const *in2 = recorded_kernel_args[arg++]; + // If not in place, the output is to the scratch buffer + Buffer const *in1 = buf; + Buffer const *out = in_place ? buf : &buf3; + if (!tail_single_kernel && base == 0) { + if (cache_group == 1) { ktailMulZero.setQueue(q); ktailMulZero(*out, *in1, *in2); } + if (cache_group == 2) { ktailMulZeroGF31.setQueue(q); ktailMulZeroGF31(*out, *in1, *in2); } + if (cache_group == 3) { ktailMulZeroGF61.setQueue(q); ktailMulZeroGF61(*out, *in1, *in2); } + if (kernelsToExecuteX) kernelsToExecuteX--; + } + if (cache_group == 1) { ktailMul.setQueue(q); ktailMul.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailMul(*out, *in1, *in2, base); } + if (cache_group == 2) { ktailMulGF31.setQueue(q); ktailMulGF31.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailMulGF31(*out, *in1, *in2, base); } + if (cache_group == 3) { ktailMulGF61.setQueue(q); ktailMulGF61.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailMulGF61(*out, *in1, *in2, base); } + } + + if (kern == KTAILMULLOW) { + Buffer const *buf = recorded_kernel_args[arg++]; + Buffer const *in2 = recorded_kernel_args[arg++]; + // If not in place, the output is to the scratch buffer + Buffer const *in1 = buf; + Buffer const *out = in_place ? buf : &buf3; + if (!tail_single_kernel && base == 0) { + if (cache_group == 1) { ktailMulLowZero.setQueue(q); ktailMulLowZero(*out, *in1, *in2); } + if (cache_group == 2) { ktailMulLowZeroGF31.setQueue(q); ktailMulLowZeroGF31(*out, *in1, *in2); } + if (cache_group == 3) { ktailMulLowZeroGF61.setQueue(q); ktailMulLowZeroGF61(*out, *in1, *in2); } + if (kernelsToExecuteX) kernelsToExecuteX--; + } + if (cache_group == 1) { ktailMulLow.setQueue(q); ktailMulLow.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailMulLow(*out, *in1, *in2, base); } + if (cache_group == 2) { ktailMulLowGF31.setQueue(q); ktailMulLowGF31.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailMulLowGF31(*out, *in1, *in2, base); } + if (cache_group == 3) { ktailMulLowGF61.setQueue(q); ktailMulLowGF61.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailMulLowGF61(*out, *in1, *in2, base); } + } + + if (kern == KMIDOUT) { + Buffer const *buf = recorded_kernel_args[arg++]; + // If not in place, the input is from the scratch buffer + Buffer const *in = in_place ? buf : &buf3; + Buffer const *out = buf; + if (cache_group == 1) { kfftMidOut.setQueue(q); kfftMidOut.setKernelsToExecute(kernelsToExecuteX); kfftMidOut(*out, *in, base); } + if (cache_group == 2) { kfftMidOutGF31.setQueue(q); kfftMidOutGF31.setKernelsToExecute(kernelsToExecuteX); kfftMidOutGF31(*out, *in, base); } + if (cache_group == 3) { kfftMidOutGF61.setQueue(q); kfftMidOutGF61.setKernelsToExecute(kernelsToExecuteX); kfftMidOutGF61(*out, *in, base); } + } + + if (kern == KFFTW) { + Buffer const *out = recorded_kernel_args[arg++]; + Buffer const *in = recorded_kernel_args[arg++]; + if (cache_group == 1) { kfftW.setQueue(q); kfftW(*out, *in); } + if (cache_group == 2) { kfftWGF31.setQueue(q); kfftWGF31(*out, *in); } + if (cache_group == 3) { kfftWGF61.setQueue(q); kfftWGF61(*out, *in); } + } } -void Gpu::fftW(Buffer& out, Buffer& in, int cache_group) { - if ((cache_group == 0 || cache_group == 1) && (fft.FFT_FP64 || fft.FFT_FP32)) kfftW(out, in); - if ((cache_group == 0 || cache_group == 2) && fft.NTT_GF31) kfftWGF31(out, in); - if ((cache_group == 0 || cache_group == 3) && fft.NTT_GF61) kfftWGF61(out, in); +// Advance the index into the array of kernel arguments +int Gpu::replay_next_arg(enum BOTTOM_HALF_KERNELS kern, int arg) { + + if (kern == KMIDIN || kern == KTAILSQUARE || kern == KMIDOUT) { + return arg + 1; + } + + else { //if (kern == KFFTHIN || kern == KTAILMUL || kern == KTAILMULLOW || kern == KFFTW) { + return arg + 2; + } } -void Gpu::fftMidIn(Buffer& out, Buffer& in, int cache_group) { - if ((cache_group == 0 || cache_group == 1) && (fft.FFT_FP64 || fft.FFT_FP32)) kfftMidIn(out, in); - if ((cache_group == 0 || cache_group == 2) && fft.NTT_GF31) kfftMidInGF31(out, in); - if ((cache_group == 0 || cache_group == 3) && fft.NTT_GF61) kfftMidInGF61(out, in); +// Call the appropriate kernels to support hybrid FFTs and NTTs + +void Gpu::fftP(Buffer& buf, Buffer& in) { + // Work around a troublesome oddball case. ModMul calls fftP and fftMidIn on one multiplication argument. If !in_place, fftP writes to buf3, and fftMidIn is queued. + // Modmul then calls fftP on the other multiplication argument. If we don't replay now, fftP overwrite buf3. + replay(); + // If not in place, instead write the output to the scratch buffer + Buffer const*out = in_place ? &buf : &buf3; + kfftP(*out, in); } -void Gpu::fftMidOut(Buffer& out, Buffer& in, int cache_group) { - if ((cache_group == 0 || cache_group == 1) && (fft.FFT_FP64 || fft.FFT_FP32)) kfftMidOut(out, in); - if ((cache_group == 0 || cache_group == 2) && fft.NTT_GF31) kfftMidOutGF31(out, in); - if ((cache_group == 0 || cache_group == 3) && fft.NTT_GF61) kfftMidOutGF61(out, in); +void Gpu::fftMidIn(Buffer& buf) { + // Record this call for later playback + recorded_kernels.push_back(KMIDIN); + recorded_kernel_args.push_back(&buf); } void Gpu::fftHin(Buffer& out, Buffer& in) { - if (fft.FFT_FP64 || fft.FFT_FP32) kfftHin(out, in); - if (fft.NTT_GF31) kfftHinGF31(out, in); - if (fft.NTT_GF61) kfftHinGF61(out, in); + // Record this call for later playback + recorded_kernels.push_back(KFFTHIN); + recorded_kernel_args.push_back(&out); + recorded_kernel_args.push_back(&in); } -void Gpu::tailSquare(Buffer& out, Buffer& in, int cache_group) { - if (!tail_single_kernel) { - if ((cache_group == 0 || cache_group == 1) && (fft.FFT_FP64 || fft.FFT_FP32)) ktailSquareZero(out, in); - if ((cache_group == 0 || cache_group == 2) && fft.NTT_GF31) ktailSquareZeroGF31(out, in); - if ((cache_group == 0 || cache_group == 3) && fft.NTT_GF61) ktailSquareZeroGF61(out, in); - } - if ((cache_group == 0 || cache_group == 1) && (fft.FFT_FP64 || fft.FFT_FP32)) ktailSquare(out, in); - if ((cache_group == 0 || cache_group == 2) && fft.NTT_GF31) ktailSquareGF31(out, in); - if ((cache_group == 0 || cache_group == 3) && fft.NTT_GF61) ktailSquareGF61(out, in); +void Gpu::tailSquare(Buffer& buf) { + // Record this call for later playback + recorded_kernels.push_back(KTAILSQUARE); + recorded_kernel_args.push_back(&buf); +} + +void Gpu::tailMul(Buffer& buf, Buffer& in2) { + // Record this call for later playback + recorded_kernels.push_back(KTAILMUL); + recorded_kernel_args.push_back(&buf); + recorded_kernel_args.push_back(&in2); +} + +void Gpu::tailMulLow(Buffer& buf, Buffer& in2) { + // Record this call for later playback + recorded_kernels.push_back(KTAILMULLOW); + recorded_kernel_args.push_back(&buf); + recorded_kernel_args.push_back(&in2); } -void Gpu::tailMul(Buffer& out, Buffer& in1, Buffer& in2, int cache_group) { - if ((cache_group == 0 || cache_group == 1) && (fft.FFT_FP64 || fft.FFT_FP32)) ktailMul(out, in1, in2); - if ((cache_group == 0 || cache_group == 2) && fft.NTT_GF31) ktailMulGF31(out, in1, in2); - if ((cache_group == 0 || cache_group == 3) && fft.NTT_GF61) ktailMulGF61(out, in1, in2); +void Gpu::fftMidOut(Buffer& buf) { + // Record this call for later playback + recorded_kernels.push_back(KMIDOUT); + recorded_kernel_args.push_back(&buf); } -void Gpu::tailMulLow(Buffer& out, Buffer& in1, Buffer& in2, int cache_group) { - if ((cache_group == 0 || cache_group == 1) && (fft.FFT_FP64 || fft.FFT_FP32)) ktailMulLow(out, in1, in2); - if ((cache_group == 0 || cache_group == 2) && fft.NTT_GF31) ktailMulLowGF31(out, in1, in2); - if ((cache_group == 0 || cache_group == 3) && fft.NTT_GF61) ktailMulLowGF61(out, in1, in2); +void Gpu::fftW(Buffer& out, Buffer& in) { + // Record this call for later playback + recorded_kernels.push_back(KFFTW); + recorded_kernel_args.push_back(&out); + recorded_kernel_args.push_back(&in); + // This kernel always ends the "bottom half". Replay the recorded kernel calls. + endBottomHalf(); } void Gpu::carryA(Buffer& out, Buffer& in) { @@ -804,20 +1713,35 @@ void Gpu::carryLL(Buffer& out, Buffer& in) { kCarryLL(out, in, updateCarryPos(1 << 2)); } -void Gpu::carryFused(Buffer& out, Buffer& in) { +void Gpu::carryFused(Buffer& buf) { + // This kernel always ends the "bottom half". Replay the recorded kernel calls. + endBottomHalf(); + // Like fftP, if not in place write the output to the scratch buffer + Buffer const *in = &buf; + Buffer const *out = in_place ? &buf : &buf3; assert(roePos <= ROE_SIZE); - roePos < wantROE ? kCarryFusedROE(out, in, roePos++) - : kCarryFused(out, in, updateCarryPos(1 << 0)); + roePos < wantROE ? kCarryFusedROE(*out, *in, roePos++) + : kCarryFused(*out, *in, updateCarryPos(1 << 0)); } -void Gpu::carryFusedMul(Buffer& out, Buffer& in) { +void Gpu::carryFusedMul(Buffer& buf) { + // This kernel always ends the "bottom half". Replay the recorded kernel calls. + endBottomHalf(); + // Like fftP, if not in place write the output to the scratch buffer + Buffer const *in = &buf; + Buffer const *out = in_place ? &buf : &buf3; assert(roePos <= ROE_SIZE); - roePos < wantROE ? kCarryFusedMulROE(out, in, roePos++) - : kCarryFusedMul(out, in, updateCarryPos(1 << 1)); + roePos < wantROE ? kCarryFusedMulROE(*out, *in, roePos++) + : kCarryFusedMul(*out, *in, updateCarryPos(1 << 1)); } -void Gpu::carryFusedLL(Buffer& out, Buffer& in) { - kCarryFusedLL(out, in, updateCarryPos(1 << 0)); +void Gpu::carryFusedLL(Buffer& buf) { + // This kernel always ends the "bottom half". Replay the recorded kernel calls. + endBottomHalf(); + // Like fftP, if not in place write the output to the scratch buffer + Buffer const *in = &buf; + Buffer const *out = in_place ? &buf : &buf3; + kCarryFusedLL(*out, *in, updateCarryPos(1 << 0)); } @@ -825,7 +1749,7 @@ void Gpu::carryFusedLL(Buffer& out, Buffer& in) { void Gpu::measureTransferSpeed() { u32 SIZE_MB = 16; vector data(SIZE_MB * 1024 * 1024, 1); - Buffer buf{profile.make("DMA"), queue, SIZE}; + Buffer buf{profile.make("DMA"), &queue, SIZE}; Timer t; for (int i = 0; i < 4; ++i) { @@ -835,11 +1759,11 @@ void Gpu::measureTransferSpeed() { for (int i = 0; i < 4; ++i) { buf.read(data); - // queue->finish(); + // queue.finish(); log("buffer READ : %f GB/s\n", double(SIZE / 1024 / 1024) * sizeof(double) / (1024 * t.reset())); } - queue->finish(); + queue.finish(); } #endif @@ -849,21 +1773,31 @@ u32 Gpu::updateCarryPos(u32 bit) { vector> Gpu::makeBufVector(u32 size) { vector> r; - for (u32 i = 0; i < size; ++i) { r.emplace_back(timeBufVect, queue, N); } + r.reserve(size); +for (u32 i = 0; i < size; ++i) { r.emplace_back(timeBufVect, &queue, N); } return r; } pair Gpu::readROE() { assert(roePos <= ROE_SIZE); if (roePos) { - vector roe = bufROE.read(roePos); - assert(roe.size() == roePos); - bufROE.zero(roePos); - roePos = 0; + vector roe = bufROE.read(roePos + 2); + assert(roe.size() == roePos + 2); + // Split the roe buffer into two. One for squarings and one for multiplications. This is likely overkill as the multiplication ROE is not used - though + // it could be useful for debugging (in which case we could support getting roe for squarings or multipplications, but not both). auto [squareRoe, mulRoe] = split(roe, mulRoePos); + // Delete first two used to calculate roePos on the GPU. Do this after splitting the vector (mulRoePos recorded indices in "+ 2" format). + u32 squareRoeSize = u32(squareRoe.size()) - 2; + roe[0] = squareRoe[squareRoeSize]; + roe[1] = squareRoe[squareRoeSize+1]; + squareRoe.resize(squareRoeSize); + // Clear the ROE buffer and mulRoePos vector + bufROE.zero(roePos + 2); + roePos = 0; mulRoePos.clear(); return {roeStat(squareRoe), roeStat(mulRoe)}; } else { + mulRoePos.clear(); // indices recorded while ROE sampling was off must not tag the next window return {}; } } @@ -871,9 +1805,14 @@ pair Gpu::readROE() { RoeInfo Gpu::readCarryStats() { assert(carryPos <= CARRY_SIZE); if (carryPos == 0) { return {}; } - vector carry = bufStatsCarry.read(carryPos); - assert(carry.size() == carryPos); - bufStatsCarry.zero(carryPos); + vector carry = bufStatsCarry.read(carryPos + 2); + assert(carry.size() == carryPos + 2); + // Delete first two used to calculate carryPos on the GPU. + carry[0] = carry[carryPos]; + carry[1] = carry[carryPos+1]; + carry.resize(carryPos); + // Clear the GPU buffer + bufStatsCarry.zero(carryPos + 2); carryPos = 0; RoeInfo ret = roeStat(carry); @@ -908,22 +1847,16 @@ static bool isAllZero(vector v) { return std::all_of(v.begin(), v.end(), [](T // Read from GPU, verifying the transfer with a sum, and retry on failure. vector Gpu::readChecked(Buffer& buf) { for (int nRetry = 0; nRetry < 3; ++nRetry) { - sum64(bufSumOut, u32(buf.size * sizeof(Word)), buf); - + bufSumOut.zero(); + sum64(bufSumOut, N, buf); vector expectedVect(1); - bufSumOut.readAsync(expectedVect); - vector data = readOut(buf); - u64 gpuSum = expectedVect[0]; + vector data = readOut(buf); u64 hostSum = 0; + for (auto it = data.begin(), end = data.end(); it < end; ++it) hostSum += u64(*it); - int even = 1; - for (auto it = data.begin(), end = data.end(); it < end; ++it, even = !even) { - if (fft.WordSize == 4) hostSum += even ? u64(u32(*it)) : (u64(*it) << 32); - if (fft.WordSize == 8) hostSum += u64(*it); - } - + u64 const gpuSum = expectedVect[0]; if (hostSum == gpuSum) { // A buffer containing all-zero is exceptional, so mark that through the empty vector. if (gpuSum == 0 && isAllZero(data)) { @@ -942,57 +1875,35 @@ Words Gpu::readAndCompress(Buffer& buf) { return compactBits(readChecked( vector Gpu::readCheck() { return readAndCompress(bufCheck); } vector Gpu::readData() { return readAndCompress(bufData); } -// out := inA * inB; inB is preserved -void Gpu::mul(Buffer& ioA, Buffer& inB, Buffer& tmp1, Buffer& tmp2, bool mul3) { - if (!in_place) { - fftP(tmp2, ioA); - for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { - fftMidIn(tmp1, tmp2, cache_group); - tailMul(tmp2, inB, tmp1, cache_group); - fftMidOut(tmp1, tmp2, cache_group); - fftW(tmp2, tmp1, cache_group); - } - } - else { - fftP(tmp1, ioA); - for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { - fftMidIn(tmp1, tmp1, cache_group); - tailMul(tmp1, inB, tmp1, cache_group); - fftMidOut(tmp1, tmp1, cache_group); - fftW(tmp2, tmp1, cache_group); - } - } +// ioA := ioA * inB; inB must be the output of fftMidIn; inB is preserved +void Gpu::mul(Buffer& ioA, Buffer& inB, Buffer& tmp1, bool mul3) { + fftP(tmp1, ioA); + fftMidIn(tmp1); + tailMul(tmp1, inB); + fftMidOut(tmp1); + fftW(buf3, tmp1); // Register the current ROE pos as multiplication (vs. a squaring) - if (mulRoePos.empty() || mulRoePos.back() < roePos) { mulRoePos.push_back(roePos); } + // mulRoePos holds indices in the "+ 2" format of the raw bufROE vector, so compare in that format too. + if (mulRoePos.empty() || mulRoePos.back() != roePos + 2) { mulRoePos.push_back(roePos + 2); } - if (mul3) { carryM(ioA, tmp2); } else { carryA(ioA, tmp2); } + if (mul3) { carryM(ioA, buf3); } else { carryA(ioA, buf3); } carryB(ioA); } -void Gpu::mul(Buffer& io, Buffer& buf1) { - // We know that mul() stores double output in buf1; so we're going to use buf2 & buf3 for temps. - mul(io, buf1, buf2, buf3, false); -} - -// out := inA * inB; +// ioA := ioA * inB; inB will end up in buf1 in the LEAD_MIDDLE state void Gpu::modMul(Buffer& ioA, Buffer& inB, bool mul3) { modMul(ioA, inB, LEAD_NONE, mul3); }; -// out := inA * inB; inB will end up in buf1 in the LEAD_MIDDLE state +// ioA := ioA * inB; inB will end up in buf1 in the LEAD_MIDDLE state void Gpu::modMul(Buffer& ioA, Buffer& inB, enum LEAD_TYPE leadInB, bool mul3) { - if (!in_place) { - if (leadInB == LEAD_NONE) fftP(buf2, inB); - if (leadInB != LEAD_MIDDLE) fftMidIn(buf1, buf2); - } else { - if (leadInB == LEAD_NONE) fftP(buf1, inB); - if (leadInB != LEAD_MIDDLE) fftMidIn(buf1, buf1); - } - mul(ioA, buf1, buf2, buf3, mul3); + if (leadInB == LEAD_NONE) fftP(buf1, inB); + if (leadInB != LEAD_MIDDLE) fftMidIn(buf1); + mul(ioA, buf1, buf2, mul3); }; -void Gpu::writeState(u32 k, const vector& check, u32 blockSize) { +void Gpu::writeState(u64 k, const vector& check, u32 blockSize) { assert(blockSize > 0); writeIn(bufCheck, check); @@ -1022,7 +1933,7 @@ void Gpu::writeState(u32 k, const vector& check, u32 blockSize) { } modMul(bufData, bufAux, true); } - + bool Gpu::doCheck(u32 blockSize) { squareLoop(bufAux, bufCheck, 0, blockSize, true); modMul(bufCheck, bufData); @@ -1040,14 +1951,14 @@ void Gpu::logTimeKernels() { string s = "Profile:\n"; for (const TimeInfo* p : prof) { - u32 n = p->n; + u32 const n = p->n; assert(n); - double f = 1e-3 / n; - double percent = 100.0 / total * p->times[2]; + double const f = 1e-3 / n; + double const percent = 100.0 / total * p->times[2]; if (!args.verbose && percent < 0.2) { break; } snprintf(buf, sizeof(buf), - args.verbose ? "%s %5.2f%% %-11s : %6.0f us/call x %5d calls (%.3f %.0f)\n" - : "%s %5.2f%% %-11s %4.0f x%6d %.3f %.0f\n", + args.verbose ? "%s %5.2f%% %-18s : %6.1f us/call x %5d calls (%.3f %.0f)\n" + : "%s %5.2f%% %-18s %6.1f x%6d %.3f %.0f\n", logContext().c_str(), percent, p->name.c_str(), p->times[2] * f, n, p->times[0] * (f * 1e-3), p->times[1] * (f * 1e-3)); s += buf; @@ -1073,7 +1984,7 @@ vector Gpu::readWords(Buffer &buf) { void Gpu::writeWords(Buffer& buf, vector &words) { // GPU is expecting either 4-byte or 8-byte integers. C++ code is using 8-byte integers. Handle the "no conversion" case. - if (fft.WordSize == 8) buf.write(std::move(words)); + if (fft.WordSize == 8) buf.write(words); // Convert 64-bit C++ Words into 32-bit GPU Words else { vector GPUdata; @@ -1082,7 +1993,7 @@ void Gpu::writeWords(Buffer& buf, vector &words) { for (u32 i = 0; i < words.size(); i += 2) { GPUdata[i/2] = ((i64) words[i+1] << 32) | (u32) words[i]; } - buf.write(std::move(GPUdata)); + buf.write(GPUdata); } } @@ -1099,20 +2010,20 @@ void Gpu::writeIn(Buffer& buf, vector&& words) { } Words Gpu::expExp2(const Words& A, u32 n) { - u32 logStep = 10000; - u32 blockSize = 100; + u32 const logStep = 10000; + u32 const blockSize = 100; - writeIn(bufData, std::move(A)); + writeIn(bufData, A); IterationTimer timer{0}; u32 k = 0; while (k < n) { - u32 its = std::min(blockSize, n - k); + u32 const its = std::min(blockSize, n - k); squareLoop(bufData, 0, its); k += its; - queue->finish(); + queue.finish(); if (k % logStep == 0) { - float secsPerIt = timer.reset(k); - log("%u / %u, %.0f us/it\n", k, n, secsPerIt * 1'000'000); + float const secsPerIt = timer.reset(k); + log("%u / %u, %s us/it\n", k, n, formatSecsPerIter(secsPerIt).c_str()); } } return readData(); @@ -1120,7 +2031,7 @@ Words Gpu::expExp2(const Words& A, u32 n) { // A:= A^h * B void Gpu::expMul(Buffer& A, u64 h, Buffer& B) { - exponentiate(A, h, buf1, buf2, buf3); + exponentiate(A, h); modMul(A, B); } @@ -1137,131 +2048,101 @@ Words Gpu::expMul(const Words& A, u64 h, const Words& B, bool doSquareB) { static bool testBit(u64 x, int bit) { return x & (u64(1) << bit); } // See "left-to-right binary exponentiation" on wikipedia -void Gpu::exponentiate(Buffer& bufInOut, u64 exp, Buffer& buf1, Buffer& buf2, Buffer& buf3) { +void Gpu::exponentiate(Buffer& bufInOut, u64 exp) { if (exp == 0) { bufInOut.set(1); } else if (exp > 1) { - if (!in_place) { - fftP(buf3, bufInOut); - fftMidIn(buf2, buf3); - } else { - fftP(buf2, bufInOut); - fftMidIn(buf2, buf2); - } - fftHin(buf1, buf2); // save "base" to buf1 - bool midInAlreadyDone = 1; + fftP(buf1, bufInOut); + fftMidIn(buf1); + fftHin(buf2, buf1); // save fully FFTed "base" to buf2 + bool midInAlreadyDone = true; int p = 63; while (!testBit(exp, p)) { --p; } for (--p; ; --p) { - for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { - if (!in_place) { - if (!midInAlreadyDone) fftMidIn(buf2, buf3, cache_group); - tailSquare(buf3, buf2, cache_group); - fftMidOut(buf2, buf3, cache_group); - } else { - if (!midInAlreadyDone) fftMidIn(buf2, buf2, cache_group); - tailSquare(buf2, buf2, cache_group); - fftMidOut(buf2, buf2, cache_group); - } - } - midInAlreadyDone = 0; + if (!midInAlreadyDone) fftMidIn(buf1); + tailSquare(buf1); + fftMidOut(buf1); + midInAlreadyDone = false; if (testBit(exp, p)) { - doCarry(buf3, buf2, bufInOut); - for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { - if (!in_place) { - fftMidIn(buf2, buf3, cache_group); - tailMulLow(buf3, buf2, buf1, cache_group); - fftMidOut(buf2, buf3, cache_group); - } else { - fftMidIn(buf2, buf2, cache_group); - tailMulLow(buf2, buf2, buf1, cache_group); - fftMidOut(buf2, buf2, cache_group); - } - } + doCarry(buf1, bufInOut); + fftMidIn(buf1); + tailMulLow(buf1, buf2); + fftMidOut(buf1); } if (!p) { break; } - doCarry(buf3, buf2, bufInOut); + doCarry(buf1, bufInOut); } - fftW(buf3, buf2); + fftW(buf3, buf1); carryA(bufInOut, buf3); carryB(bufInOut); } } // does either carryFused() or the expanded version depending on useLongCarry -void Gpu::doCarry(Buffer& out, Buffer& in, Buffer& tmp) { - if (!in_place) { - if (useLongCarry) { - fftW(out, in); - carryA(tmp, out); - carryB(tmp); - fftP(out, tmp); - } else { - carryFused(out, in); - } +void Gpu::doCarry(Buffer& in, Buffer& wordBuf) { + if (useLongCarry) { + fftW(buf3, in); + carryA(wordBuf, buf3); + carryB(wordBuf); + fftP(in, wordBuf); } else { - if (useLongCarry) { - fftW(out, in); - carryA(tmp, out); - carryB(tmp); - fftP(in, tmp); - } else { - carryFused(in, in); - } + carryFused(in); } } -// Use buf1 and buf2 to do a single squaring. +// Use buf1 (and buf23 if not in place) to do a single squaring. void Gpu::square(Buffer& out, Buffer& in, enum LEAD_TYPE leadIn, enum LEAD_TYPE leadOut, bool doMul3, bool doLL) { // leadOut = LEAD_MIDDLE is not supported (slower than LEAD_WIDTH) assert(leadOut != LEAD_MIDDLE); // LL does not do Mul3 assert(!(doMul3 && doLL)); - // Not in place FFTs use buf1 and buf2 in a "ping pong" fashion. - // If leadIn is LEAD_NONE, in contains the input data, squaring starts at fftP - // If leadIn is LEAD_WIDTH, buf2 contains the input data, squaring starts at fftMidIn - // If leadIn is LEAD_MIDDLE, buf1 contains the input data, squaring starts at tailSquare - // If leadOut is LEAD_WIDTH, then will buf2 contain the output of carryFused -- to be used as input to the next squaring. - if (!in_place) { - if (leadIn == LEAD_NONE) fftP(buf2, in); - for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { - if (leadIn != LEAD_MIDDLE) fftMidIn(buf1, buf2, cache_group); - tailSquare(buf2, buf1, cache_group); - fftMidOut(buf1, buf2, cache_group); - if (leadOut == LEAD_NONE) fftW(buf2, buf1, cache_group); + // Use CUDA graphs for some common squarings + // NOTE: assumes that if doLL is set, it will always be set + bool graph_recording = false; + Graph *graph = NULL; + if (use_graphs && (&out == &bufData || &out == &bufAux) && &in == &out && leadIn == LEAD_WIDTH && leadOut == LEAD_WIDTH && !doMul3) { + // We have one graph for ROE and one for no-ROE and one for bufData and one for bufAux + bool roe = (roePos < wantROE); + bool srcData = (&out == &bufData); + graph = &graph_square[2 * roe + srcData]; + // Execute an already recorded graph + if (graph->isRecorded()) { + graph->launch(&queue); + queue.incSquareCount(); + if (roe) roePos++; // WARNING: If we ever graph Gpu::Mul, we'll need to also maintain mulRoePos vector. + return; } + // Otherwise, record a new graph + graph->beginRecording(&queue); + graph_recording = true; } - // In place FFTs use buf1. + // In place FFTs use buf1. Not in place FFTs also use buf3. // If leadIn is LEAD_NONE, in contains the input data, squaring starts at fftP - // If leadIn is LEAD_WIDTH, buf1 contains the input data, squaring starts at fftMidIn + // If leadIn is LEAD_WIDTH, buf1 (or buf3 if not in place) contains the input data, squaring starts at fftMidIn // If leadIn is LEAD_MIDDLE, buf1 contains the input data, squaring starts at tailSquare - // If leadOut is LEAD_WIDTH, then buf1 will contain the output of carryFused -- to be used as input to the next squaring. - else { - if (leadIn == LEAD_NONE) fftP(buf1, in); - for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { - if (leadIn != LEAD_MIDDLE) fftMidIn(buf1, buf1, cache_group); - tailSquare(buf1, buf1, cache_group); - fftMidOut(buf1, buf1, cache_group); - if (leadOut == LEAD_NONE) fftW(buf2, buf1, cache_group); - } - } + // If leadOut is LEAD_WIDTH, then buf1 (or buf3 if not in place) will contain the output of carryFused -- to be used as input to the next squaring. + if (leadIn == LEAD_NONE) fftP(buf1, in); + if (leadIn != LEAD_MIDDLE) fftMidIn(buf1); + tailSquare(buf1); + fftMidOut(buf1); // If leadOut is not allowed then we cannot use the faster carryFused kernel if (leadOut == LEAD_NONE) { + fftW(buf3, buf1); if (!doLL && !doMul3) { - carryA(out, buf2); + carryA(out, buf3); } else if (doLL) { - carryLL(out, buf2); + carryLL(out, buf3); } else { - carryM(out, buf2); + carryM(out, buf3); } carryB(out); } @@ -1271,18 +2152,24 @@ void Gpu::square(Buffer& out, Buffer& in, enum LEAD_TYPE leadIn, enu assert(!useLongCarry); assert(!doMul3); if (doLL) { - carryFusedLL(in_place ? buf1 : buf2, buf1); + carryFusedLL(buf1); } else { - carryFused(in_place ? buf1 : buf2, buf1); + carryFused(buf1); } } + + // End CUDA graph recording (and execute the just recorded graph) + if (graph_recording) { + graph->endRecording(&queue); + graph->launch(&queue); + } } -u32 Gpu::squareLoop(Buffer& out, Buffer& in, u32 from, u32 to, bool doTailMul3) { +u64 Gpu::squareLoop(Buffer& out, Buffer& in, u64 from, u64 to, bool doTailMul3) { assert(from < to); enum LEAD_TYPE leadIn = LEAD_NONE; - for (u32 k = from; k < to; ++k) { - enum LEAD_TYPE leadOut = useLongCarry || (k == to - 1) ? LEAD_NONE : LEAD_WIDTH; + for (u64 k = from; k < to; ++k) { + enum LEAD_TYPE const leadOut = useLongCarry || (k == to - 1) ? LEAD_NONE : LEAD_WIDTH; square(out, (k==from) ? in : out, leadIn, leadOut, doTailMul3 && (k == to - 1)); leadIn = leadOut; } @@ -1303,18 +2190,18 @@ u64 Gpu::bufResidue(Buffer &buf) { int carry = 0; for (int i = 0; i < 32; ++i) { - u32 len = bitlen(N, E, N - 32 + i); - i64 w = (i64) words[i] + carry; + u32 const len = bitlen(N, E, N - 32 + i); + i64 const w = (i64) words[i] + carry; carry = (int) (w >> len); } u64 res = 0; int hasBits = 0; for (int k = 0; k < 32 && hasBits < 64; ++k) { - u32 len = bitlen(N, E, k); - i64 tmp = (i64) words[32 + k] + carry; + u32 const len = bitlen(N, E, k); + i64 const tmp = (i64) words[32 + k] + carry; carry = (int) (tmp >> len); - u64 w = tmp - ((i64) carry << len); + u64 const w = tmp - ((i64) carry << len); assert(w < (1ULL << len)); res += w << hasBits; hasBits += len; @@ -1323,10 +2210,10 @@ u64 Gpu::bufResidue(Buffer &buf) { } static string formatETA(u32 secs) { - u32 etaMins = (secs + 30) / 60; - int days = etaMins / (24 * 60); - int hours = etaMins / 60 % 24; - int mins = etaMins % 60; + u32 const etaMins = (secs + 30) / 60; + int const days = etaMins / (24 * 60); + int const hours = etaMins / 60 % 24; + int const mins = etaMins % 60; char buf[64]; if (days) { snprintf(buf, sizeof(buf), "%dd %02d:%02d", days, hours, mins); @@ -1336,8 +2223,8 @@ static string formatETA(u32 secs) { return string(buf); } -static string getETA(u32 step, u32 total, float secsPerStep) { - u32 etaSecs = max(0u, u32((total - step) * secsPerStep)); +static string getETA(u64 step, u64 total, float secsPerStep) { + u32 const etaSecs = max(0u, u32((total - step) * secsPerStep)); return formatETA(etaSecs); } @@ -1350,18 +2237,18 @@ string RoeInfo::toString() const { return buf; } -static string makeLogStr(const string& status, u32 k, u64 res, float secsPerIt, u32 nIters) { +static string makeLogStr(const string& status, u64 k, u64 res, float secsPerIt, u64 nIters) { char buf[256]; - snprintf(buf, sizeof(buf), "%2s %9u %016" PRIx64 " %4.0f ETA %s; ", + snprintf(buf, sizeof(buf), "%2s %9" PRIu64 " %016" PRIx64 " %s ETA %s; ", status.c_str(), k, res, /* k / float(nIters) * 100, */ - secsPerIt * 1'000'000, getETA(k, nIters, secsPerIt).c_str()); + formatSecsPerIter(secsPerIt).c_str(), getETA(k, nIters, secsPerIt).c_str()); return buf; } -void Gpu::doBigLog(u32 k, u64 res, bool checkOK, float secsPerIt, u32 nIters, u32 nErrors) { +void Gpu::doBigLog(u64 k, u64 res, bool checkOK, float secsPerIt, u64 nIters, u32 nErrors) { auto [roeSq, roeMul] = readROE(); - double z = roeSq.z(); + double const z = roeSq.z(); zAvg.update(z, roeSq.N); if (roeSq.max > 0.005) log("%sZ=%.0f (avg %.1f), ROEmax=%.3f, ROEavg=%.3f. %s\n", makeLogStr(checkOK ? "OK" : "EE", k, res, secsPerIt, nIters).c_str(), @@ -1377,10 +2264,10 @@ void Gpu::doBigLog(u32 k, u64 res, bool checkOK, float secsPerIt, u32 nIters, u3 // Unless ROE log is not explicitly requested, measure only a few iterations to minimize overhead wantROE = args.logROE ? ROE_SIZE : 400; - RoeInfo carryStats = readCarryStats(); + RoeInfo const carryStats = readCarryStats(); if (carryStats.N > 2) { - u32 m = ldexp(carryStats.max, 32); - double z = carryStats.z(); + u32 const m = u32(ldexp(carryStats.max, 32)); + double const z = carryStats.z(); log("Carry: %x Z(%u)=%.1f\n", m, carryStats.N, z); } } @@ -1391,20 +2278,20 @@ bool Gpu::equals9(const Words& a) { return true; } -int ulps(double a, double b) { +[[maybe_unused]] static int ulps(double a, double b) { if (a == 0 && b == 0) { return 0; } - u64 aa = as(a); - u64 bb = as(b); - bool sameSign = (aa >> 63) == (bb >> 63); - int delta = sameSign ? bb - aa : bb + aa; + u64 const aa = as(a); + u64 const bb = as(b); + bool const sameSign = (aa >> 63) == (bb >> 63); + int const delta = int(sameSign ? bb - aa : bb + aa); return delta; } [[maybe_unused]] static double trigNorm(double c, double s) { - double c2 = c * c; - double err = fma(c, c, -c2); - double norm = c2 + fma(s, s, err); + double const c2 = c * c; + double const err = fma(c, c, -c2); + double const norm = c2 + fma(s, s, err); return norm; } @@ -1447,24 +2334,24 @@ void Gpu::selftestTrig() { log("TRIG norm: up %d, down %d\n", oneUp, oneDown); #endif - if (isAmdGpu(queue->context->deviceId())) { + if (isAmdGpu(shared.context->deviceId())) { vector WHATS {"V_NOP", "V_ADD_I32", "V_FMA_F32", "V_ADD_F64", "V_FMA_F64", "V_MUL_F64", "V_MAD_U64_U32"}; - for (int w = 0; w < int(WHATS.size()); ++w) { + for (int w = 0; std::cmp_less(w, WHATS.size()); ++w) { const int what = w; testTime(what, bufCarry); - vector times = bufCarry.read(4096 * 2); - [[maybe_unused]] i64 prev = 0; + vector const times = bufCarry.read(4096 * 2); + [[maybe_unused]] i64 const prev = 0; u64 min = -1; u64 sum = 0; - for (int i = 0; i < int(times.size()); ++i) { - i64 x = times[i]; + for (i64 const x : times) { + #if 0 if (x != prev) { log("%4d : %ld\n", i, x); prev = x; } #endif - if (x > 0 && u64(x) < min) { min = x; } + if (x > 0 && std::cmp_less(x, min)) { min = x; } if (x > 0) { sum += x; } } log("%-15s : %.2f cycles latency; time min: %d; avg %.0f\n", @@ -1476,47 +2363,57 @@ void Gpu::selftestTrig() { static u32 mod3(const std::vector &words) { u32 r = 0; // uses the fact that 2**32 % 3 == 1. - for (u32 w : words) { r += w % 3; } + for (u32 const w : words) { r += w % 3; } return r % 3; } -static void doDiv3(u32 E, Words& words) { +static void doDiv3(u64 E, Words& words) { u32 r = (3 - mod3(words)) % 3; assert(r < 3); - int topBits = E % 32; + int const topBits = E % 32; assert(topBits > 0 && topBits < 32); { - u64 w = (u64(r) << topBits) + words.back(); - words.back() = w / 3; + u64 const w = (u64(r) << topBits) + words.back(); + words.back() = u32(w / 3); r = w % 3; } for (auto it = words.rbegin() + 1, end = words.rend(); it != end; ++it) { - u64 w = (u64(r) << 32) + *it; - *it = w / 3; + u64 const w = (u64(r) << 32) + *it; + *it = u32(w / 3); r = w % 3; } } -void Gpu::doDiv9(u32 E, Words& words) { +void Gpu::doDiv9(u64 E, Words& words) { doDiv3(E, words); doDiv3(E, words); } -fs::path Gpu::saveProof(const Args& args, const ProofSet& proofSet) { - for (int retry = 0; retry < 2; ++retry) { - auto [proof, hashes] = proofSet.computeProof(this); - fs::path tmpFile = proof.file(args.proofToVerifyDir); - proof.save(tmpFile); - - fs::path proofFile = proof.file(args.proofResultDir); - - bool ok = Proof::load(tmpFile).verify(this, hashes); - log("Proof '%s' verification %s\n", tmpFile.string().c_str(), ok ? "OK" : "FAILED"); - if (ok) { - fancyRename(tmpFile, proofFile); - log("Proof '%s' generated\n", proofFile.string().c_str()); - return proofFile; +fs::path Gpu::saveProof(const Args& args, ProofSet& proofSet) { + bool problem_proof = false; + for ( ; ; ) { + for (int retry = 0; retry == 0 || (retry == 1 && !problem_proof); ++retry) { + try { + auto [proof, hashes] = proofSet.computeProof(this); + fs::path const tmpFile = proof.file(args.proofToVerifyDir); + proof.save(tmpFile); + + fs::path proofFile = proof.file(args.proofResultDir); + + bool const ok = Proof::load(tmpFile).verify(this, hashes); + log("Proof '%s' verification %s\n", tmpFile.string().c_str(), ok ? "OK" : "FAILED"); + if (ok) { + fancyRename(tmpFile, proofFile); + log("Proof '%s' generated\n", proofFile.string().c_str()); + return proofFile; + } + } catch (const CRCError&) { + break; + } } + problem_proof = true; + proofSet.reducePower(); + if (proofSet.power < 4) break; } throw "bad proof generation"; } @@ -1528,16 +2425,26 @@ PRPState Gpu::loadPRP(Saver& saver) { } PRPState state = saver.load(); + + // blockSize is read straight out of the savefile, and the v12 CRC covers the check data but not + // the header. Reject an out-of-range value the same way a residue mismatch is rejected, so an + // earlier savefile gets a chance, rather than letting it reach baseCheckStep(). + if (!isValidBlockSize(state.blockSize)) { + log("EE %9" PRIu64 " on-load: invalid blockSize %u\n", state.k, state.blockSize); + if (!state.k) { break; } + continue; + } + writeState(state.k, state.check, state.blockSize); - u64 res = dataResidue(); + u64 const res = dataResidue(); if (res == state.res64) { - log("OK %9u on-load: blockSize %d, %016" PRIx64 "\n", state.k, state.blockSize, res); + log("OK %9" PRIu64 " on-load: blockSize %d, %016" PRIx64 "\n", state.k, state.blockSize, res); return state; // return {loaded.k, loaded.blockSize, loaded.nErrors}; } - log("EE %9u on-load: %016" PRIx64 " vs. %016" PRIx64 "\n", state.k, res, state.res64); + log("EE %9" PRIu64 " on-load: %016" PRIx64 " vs. %016" PRIx64 "\n", state.k, res, state.res64); if (!state.k) { break; } // We failed on PRP start } @@ -1545,8 +2452,8 @@ PRPState Gpu::loadPRP(Saver& saver) { throw "Error on load"; } -u32 Gpu::getProofPower(u32 k) { - u32 power = ProofSet::effectivePower(E, args.getProofPow(E), k); +u32 Gpu::getProofPower(u64 k) { + u32 const power = ProofSet::effectivePower(E, args.getProofPow(E), k); if (power != args.getProofPow(E)) { log("Proof using power %u (vs %u)\n", power, args.getProofPow(E)); @@ -1570,10 +2477,10 @@ tuple Gpu::measureCarry() { assert(iters % blockSize == 0); u32 k = 0; - PRPState state{E, 0, blockSize, 3, makeWords(E, 1), 0}; + PRPState const state{.exponent=E, .k=0, .blockSize=blockSize, .res64=3, .check=makeWords(E, 1), .nErrors=0}; writeState(state.k, state.check, state.blockSize); { - u64 res = dataResidue(); + u64 const res = dataResidue(); if (res != state.res64) { log("residue expected %016" PRIx64 " found %016" PRIx64 "\n", state.res64, res); } @@ -1584,7 +2491,7 @@ tuple Gpu::measureCarry() { modMul(bufCheck, bufData, leadIn); leadIn = LEAD_MIDDLE; - enum LEAD_TYPE leadOut = useLongCarry ? LEAD_NONE : LEAD_WIDTH; + enum LEAD_TYPE const leadOut = useLongCarry ? LEAD_NONE : LEAD_WIDTH; square(bufData, bufData, leadIn, leadOut); leadIn = leadOut; ++k; @@ -1616,27 +2523,23 @@ tuple Gpu::measureCarry() { if (Signal::stopRequested()) { throw "stop requested"; } } - [[maybe_unused]] u64 res = dataResidue(); + [[maybe_unused]] u64 const res = dataResidue(); if (Signal::stopRequested()) { throw "stop requested"; } - bool ok = doCheck(blockSize); + bool const ok = doCheck(blockSize); auto stats = readCarryStats(); // log("%s %016" PRIx64 " %s\n", ok ? "OK" : "EE", res, roe.toString(statsBits).c_str()); return {ok, stats}; } -tuple Gpu::measureROE(bool quick) { +tuple Gpu::measureROE(bool /*quick*/) { u32 blockSize{}, iters{}, warmup{}; - if (true) { + { blockSize = 200; iters = 2000; warmup = 50; - } else { - blockSize = 500; - iters = 10'000; - warmup = 100; } assert(iters % blockSize == 0); @@ -1644,10 +2547,10 @@ tuple Gpu::measureROE(bool quick) { wantROE = ROE_SIZE; // should be large enough to capture fully this measureROE() u32 k = 0; - PRPState state{E, 0, blockSize, 3, makeWords(E, 1), 0}; + PRPState const state{.exponent=E, .k=0, .blockSize=blockSize, .res64=3, .check=makeWords(E, 1), .nErrors=0}; writeState(state.k, state.check, state.blockSize); { - u64 res = dataResidue(); + u64 const res = dataResidue(); if (res != state.res64) { log("residue expected %016" PRIx64 " found %016" PRIx64 "\n", state.res64, res); } @@ -1658,7 +2561,7 @@ tuple Gpu::measureROE(bool quick) { modMul(bufCheck, bufData, leadIn); leadIn = LEAD_MIDDLE; - enum LEAD_TYPE leadOut = useLongCarry ? LEAD_NONE : LEAD_WIDTH; + enum LEAD_TYPE const leadOut = useLongCarry ? LEAD_NONE : LEAD_WIDTH; square(bufData, bufData, leadIn, leadOut); leadIn = leadOut; ++k; @@ -1690,10 +2593,10 @@ tuple Gpu::measureROE(bool quick) { if (Signal::stopRequested()) { throw "stop requested"; } } - [[maybe_unused]] u64 res = dataResidue(); + [[maybe_unused]] u64 const res = dataResidue(); if (Signal::stopRequested()) { throw "stop requested"; } - bool ok = doCheck(blockSize); + bool const ok = doCheck(blockSize); auto roes = readROE(); wantROE = 0; @@ -1719,7 +2622,7 @@ double Gpu::timePRP(int quick) { // Quick varies from 1 (slowest, longest assert(iters % blockSize == 0); u32 k = 0; - PRPState state{E, 0, blockSize, 3, makeWords(E, 1), 0}; + PRPState const state{.exponent=E, .k=0, .blockSize=blockSize, .res64=3, .check=makeWords(E, 1), .nErrors=0}; writeState(state.k, state.check, state.blockSize); assert(dataResidue() == state.res64); @@ -1727,7 +2630,7 @@ double Gpu::timePRP(int quick) { // Quick varies from 1 (slowest, longest modMul(bufCheck, bufData, leadIn); leadIn = LEAD_MIDDLE; - enum LEAD_TYPE leadOut = useLongCarry ? LEAD_NONE : LEAD_WIDTH; + enum LEAD_TYPE const leadOut = useLongCarry ? LEAD_NONE : LEAD_WIDTH; square(bufData, bufData, leadIn, leadOut); leadIn = leadOut; ++k; @@ -1737,11 +2640,11 @@ double Gpu::timePRP(int quick) { // Quick varies from 1 (slowest, longest leadIn = leadOut; ++k; } - queue->finish(); + queue.finish(); if (Signal::stopRequested()) { throw "stop requested"; } Timer t; - queue->setSquareTime(0); // Busy wait on nVidia to get the most accurate timings while tuning + queue.setSquareTime(0); // Busy wait on nVidia to get the most accurate timings while tuning while (true) { while (k % blockSize < blockSize-1) { square(bufData, bufData, leadIn, leadOut); @@ -1758,13 +2661,13 @@ double Gpu::timePRP(int quick) { // Quick varies from 1 (slowest, longest leadIn = LEAD_MIDDLE; if (Signal::stopRequested()) { throw "stop requested"; } } - queue->finish(); + queue.finish(); double secsPerIt = t.reset() / (iters - warmup); if (Signal::stopRequested()) { throw "stop requested"; } - u64 res = dataResidue(); - bool ok = doCheck(blockSize); + u64 const res = dataResidue(); + bool const ok = doCheck(blockSize); if (!ok) { log("Error %016" PRIx64 "\n", res); secsPerIt = 0.1; // a large value to mark the error @@ -1772,7 +2675,7 @@ double Gpu::timePRP(int quick) { // Quick varies from 1 (slowest, longest return secsPerIt * 1e6; } -PRPResult Gpu::isPrimePRP(const Task& task) { +PRPResult Gpu::isPrimePRP([[maybe_unused]] const Task& task) { assert(E == task.exponent); // This timer is used to measure total elapsed time to be written to the savefile. @@ -1781,15 +2684,16 @@ PRPResult Gpu::isPrimePRP(const Task& task) { u32 nErrors = 0; int nSeqErrors = 0; u64 lastFailedRes64 = 0; - u32 logStep = args.logStep; + u32 const logStep = args.logStep; reload: elapsedTimer.reset(); - u32 blockSize{}, k{}; + u32 blockSize{}; + u64 k{}; double elapsedBefore = 0; { - PRPState state = loadPRP(*getSaver()); + PRPState const state = loadPRP(*getSaver()); nErrors = std::max(nErrors, state.nErrors); blockSize = state.blockSize; k = state.k; @@ -1799,11 +2703,26 @@ PRPResult Gpu::isPrimePRP(const Task& task) { assert(blockSize > 0 && logStep % blockSize == 0); u32 checkStep = checkStepForErrors(blockSize, nErrors); + + // A verified savefile is only written when k % checkStep == 0; every other check writes the single + // rolling "unverified" savefile instead. When checkStep exceeds the exponent that condition is + // never met, so the only durable savefile is the one from the very first check at k = 2 * blockSize + // and one error costs the whole run. Scale the step down so a short run still gets several + // checkpoints. Staying on a multiple of logStep preserves both invariants asserted here, because + // logStep % blockSize == 0 was asserted just above. + if (checkStep > E / 8) { + checkStep = std::max(logStep, u32(std::min(checkStep, E / 8) / logStep) * logStep); + } + assert(checkStep % logStep == 0); + assert(checkStep % blockSize == 0); - u32 power = getProofPower(k); - - ProofSet proofSet{E, power}; + u32 const power = getProofPower(k); + + // power == 0 means "proof generation disabled" (no complete set of residues can be built from here on). ProofSet does + // not accept 0, so only construct one when there is a proof to make. + std::optional proofSet; + if (power) { proofSet.emplace(E, power); } bool isPrime = false; @@ -1814,28 +2733,28 @@ PRPResult Gpu::isPrimePRP(const Task& task) { // For M=2^E-1, residue "type-3" == 3^(M+1), and residue "type-1" == type-3 / 9, // See http://www.mersenneforum.org/showpost.php?p=468378&postcount=209 // For both type-1 and type-3 we need to do E squarings (as M+1==2^E). - const u32 kEnd = E; + const u64 kEnd = E; assert(k < kEnd); // We continue beyound kEnd: to the next multiple of blockSize, to do a check there - u32 kEndEnd = roundUp(kEnd, blockSize); + u64 const kEndEnd = roundUp(kEnd, blockSize); bool skipNextCheckUpdate = false; - u32 persistK = proofSet.next(k); + u64 persistK = proofSet ? proofSet->next(k) : u64(-1); enum LEAD_TYPE leadIn = LEAD_NONE; assert(k % blockSize == 0); assert(checkStep % blockSize == 0); - const u32 startK = k; + const u64 startK = k; IterationTimer iterationTimer{k}; wantROE = 0; // skip the initial iterations while (true) { assert(k < kEndEnd); - + if (!wantROE && k - startK > 30) { wantROE = args.logROE ? ROE_SIZE : 2'000; } if (skipNextCheckUpdate) { @@ -1847,10 +2766,10 @@ PRPResult Gpu::isPrimePRP(const Task& task) { ++k; // !! early inc - bool doStop = (k % blockSize == 0) && (Signal::stopRequested() || (args.iters && k - startK >= args.iters)); - bool doCheck = doStop || (k % checkStep == 0) || (k >= kEndEnd) || (k - startK == 2 * blockSize); - bool doLog = k % logStep == 0; - enum LEAD_TYPE leadOut = doCheck || doLog || k == persistK || k == kEnd || useLongCarry ? LEAD_NONE : LEAD_WIDTH; + bool const doStop = (k % blockSize == 0) && (Signal::stopRequested() || (args.iters && k - startK >= args.iters)); + bool const doCheck = doStop || (k % checkStep == 0) || (k >= kEndEnd) || (k - startK == 2 * blockSize); + bool const doLog = k % logStep == 0; + enum LEAD_TYPE const leadOut = doCheck || doLog || k == persistK || k == kEnd || useLongCarry ? LEAD_NONE : LEAD_WIDTH; if (doStop) { log("Stopping, please wait..\n"); } @@ -1858,14 +2777,14 @@ PRPResult Gpu::isPrimePRP(const Task& task) { leadIn = leadOut; if (k == persistK) { - vector rawData = readChecked(bufData); + vector const rawData = readChecked(bufData); if (rawData.empty()) { log("Data error ZERO\n"); ++nErrors; goto reload; } (*background)([=, E=this->E] { ProofSet::save(E, power, k, compactBits(rawData, E)); }); - persistK = proofSet.next(k); + persistK = proofSet->next(k); } if (k == kEnd) { @@ -1876,39 +2795,39 @@ PRPResult Gpu::isPrimePRP(const Task& task) { res2048.clear(); assert(words.size() >= 64); res2048.insert(res2048.end(), words.begin(), std::next(words.begin(), 64)); - log("%s %8d / %d, %s\n", isPrime ? "PP" : "CC", kEnd, E, hex(finalRes64).c_str()); + log("%s %8" PRIu64 " / %" PRIu64 ", %s\n", isPrime ? "PP" : "CC", kEnd, E, hex(finalRes64).c_str()); } if (!doCheck && !doLog) continue; - u64 res = dataResidue(); - float secsPerIt = iterationTimer.reset(k); - queue->setSquareTime((int) (secsPerIt * 1'000'000)); + u64 const res = dataResidue(); + float const secsPerIt = iterationTimer.reset(k); + queue.setSquareTime((int) (secsPerIt * 1'000'000)); vector rawCheck = readChecked(bufCheck); if (rawCheck.empty()) { ++nErrors; - log("%9u %016" PRIx64 " read NULL check\n", k, res); + log("%9" PRIu64 " %016" PRIx64 " read NULL check\n", k, res); if (++nSeqErrors > 2) { throw "sequential errors"; } goto reload; } if (!doCheck) { (*background)([=, this] { - getSaver()->saveUnverified({E, k, blockSize, res, compactBits(rawCheck, E), nErrors, - elapsedBefore + elapsedTimer.at()}); + getSaver()->saveUnverified({.exponent=E, .k=k, .blockSize=blockSize, .res64=res, .check=compactBits(rawCheck, E), .nErrors=nErrors, + .elapsed=elapsedBefore + elapsedTimer.at()}); }); - log(" %9u %016" PRIx64 " %4.0f\n", k, res, /*k / float(kEndEnd) * 100*,*/ secsPerIt * 1'000'000); - RoeInfo carryStats = readCarryStats(); + log(" %9" PRIu64 " %016" PRIx64 " %s\n", k, res, formatSecsPerIter(secsPerIt).c_str()); + RoeInfo const carryStats = readCarryStats(); if (carryStats.N) { - u32 m = ldexp(carryStats.max, 32); - double z = carryStats.z(); + u32 const m = u32(ldexp(carryStats.max, 32)); + double const z = carryStats.z(); log("Carry: %x Z(%u)=%.1f\n", m, carryStats.N, z); } } else { - bool ok = this->doCheck(blockSize); - [[maybe_unused]] float secsCheck = iterationTimer.reset(k); + bool const ok = this->doCheck(blockSize); + [[maybe_unused]] float const secsCheck = iterationTimer.reset(k); if (ok) { nSeqErrors = 0; @@ -1917,16 +2836,28 @@ PRPResult Gpu::isPrimePRP(const Task& task) { if (k < kEnd) { (*background)([=, this, rawCheck = std::move(rawCheck)] { - getSaver()->save({E, k, blockSize, res, compactBits(rawCheck, E), nErrors, elapsedBefore + elapsedTimer.at()}); + getSaver()->save({.exponent=E, .k=k, .blockSize=blockSize, .res64=res, .check=compactBits(rawCheck, E), .nErrors=nErrors, .elapsed=elapsedBefore + elapsedTimer.at()}); }); } doBigLog(k, res, ok, secsPerIt, kEndEnd, nErrors); - + if (k >= kEndEnd) { - fs::path proofFile = saveProof(args, proofSet); - return {isPrime, finalRes64, nErrors, proofFile.string(), toHex(res2048)}; - } + // The test is complete; nothing after this point may lose the result. Make sure the final proof residue, + // written by the background thread at k == E, is on disk before computeProof reads it back, and if the proof + // cannot be generated report the result without one rather than throw it away. + fs::path proofFile; + if (proofSet) { + background->waitEmpty(); + try { + proofFile = saveProof(args, *proofSet); + } catch (...) { + if (Signal::stopRequested()) { throw; } + log("Proof generation failed; reporting the result without a proof\n"); + } + } + return {.isPrime=isPrime, .res64=finalRes64, .nErrors=nErrors, .proofPath=proofFile.string(), .res2048=toHex(res2048)}; + } } else { ++nErrors; doBigLog(k, res, ok, secsPerIt, kEndEnd, nErrors); @@ -1941,11 +2872,11 @@ PRPResult Gpu::isPrimePRP(const Task& task) { lastFailedRes64 = res; if (!doStop) { goto reload; } } - + logTimeKernels(); - + if (doStop) { - queue->finish(); + queue.finish(); throw "stop requested"; } @@ -1954,7 +2885,7 @@ PRPResult Gpu::isPrimePRP(const Task& task) { } } -LLResult Gpu::isPrimeLL(const Task& task) { +LLResult Gpu::isPrimeLL([[maybe_unused]] const Task& task) { assert(E == task.exponent); wantROE = 0; @@ -1965,25 +2896,25 @@ LLResult Gpu::isPrimeLL(const Task& task) { reload: elapsedTimer.reset(); - u32 startK = 0; + u64 startK = 0; double elapsedBefore = 0; { LLState state = saver.load(); elapsedBefore = state.elapsed; startK = state.k; - u64 expectedRes = (u64(state.data[1]) << 32) | state.data[0]; - writeIn(bufData, std::move(state.data)); - u64 res = dataResidue(); + u64 const expectedRes = (u64(state.data[1]) << 32) | state.data[0]; + writeIn(bufData, state.data); + u64 const res = dataResidue(); if (res != expectedRes) { throw "Invalid savefile (res64)"; } assert(res == expectedRes); - log("LL loaded @ %u : %016" PRIx64 "\n", startK, res); + log("LL loaded @ %" PRIu64 " : %016" PRIx64 "\n", startK, res); } IterationTimer iterationTimer{startK}; - u32 k = startK; - u32 kEnd = E - 2; + u64 k = startK; + u64 const kEnd = E - 2; enum LEAD_TYPE leadIn = LEAD_NONE; while (true) { @@ -1995,8 +2926,8 @@ LLResult Gpu::isPrimeLL(const Task& task) { log("Stopping, please wait..\n"); } - bool doLog = (k % args.logStep == 0) || doStop; - enum LEAD_TYPE leadOut = doLog || useLongCarry ? LEAD_NONE : LEAD_WIDTH; + bool const doLog = (k % args.logStep == 0) || doStop; + enum LEAD_TYPE const leadOut = doLog || useLongCarry ? LEAD_NONE : LEAD_WIDTH; squareLL(bufData, leadIn, leadOut); leadIn = leadOut; @@ -2005,29 +2936,29 @@ LLResult Gpu::isPrimeLL(const Task& task) { u64 res64 = 0; auto data = readData(); - bool isAllZero = data.empty(); + bool const isAllZero = data.empty(); if (isAllZero) { if (k < kEnd) { - log("Error: early ZERO @ %u\n", k); + log("Error: early ZERO @ %" PRIu64 "\n", k); if (doStop) { throw "stop requested"; - } else { + } goto reload; - } + } res64 = 0; } else { assert(data.size() >= 2); res64 = (u64(data[1]) << 32) | data[0]; - saver.save({E, k, std::move(data), elapsedBefore + elapsedTimer.at()}); + saver.save({.exponent=E, .k=k, .data=std::move(data), .elapsed=elapsedBefore + elapsedTimer.at()}); } - float secsPerIt = iterationTimer.reset(k); - queue->setSquareTime((int) (secsPerIt * 1'000'000)); - log("%9u %016" PRIx64 " %4.0f\n", k, res64, secsPerIt * 1'000'000); + float const secsPerIt = iterationTimer.reset(k); + queue.setSquareTime((int) (secsPerIt * 1'000'000)); + log("%9" PRIu64 " %016" PRIx64 " %s ETA %s\n", k, res64, formatSecsPerIter(secsPerIt).c_str(), getETA(k, kEnd, secsPerIt).c_str()); - if (k >= kEnd) { return {isAllZero, res64}; } + if (k >= kEnd) { return {.isPrime=isAllZero, .res64=res64}; } if (doStop) { throw "stop requested"; } } @@ -2039,27 +2970,39 @@ array Gpu::isCERT(const Task& task) { // Get CERT start value char fname[32]; - sprintf(fname, "M%u.cert", E); + sprintf(fname, "M%" PRIu64 ".cert", E); -// Autoprimenet.py does not add the cert entry to worktodo.txt until it has successfully downloaded the .cert file. +// AutoPrimenet.py does not add the cert entry to worktodo.txt until it has successfully downloaded the .cert file. + + // Resume from a checkpoint if there is one for this assignment; otherwise start from the .cert file. + Saver saver{E, 1000, args.nSavefiles}; + CERTState state = saver.load(); + if (!state.data.empty() && state.squarings != task.squarings) { + log("CERT checkpoint is for %" PRIu64 " squarings, assignment says %u; starting over\n", state.squarings, task.squarings); + state = CERTState{.exponent=E, .k=0, .squarings=0, .data={}, .elapsed=0}; + } - { // Enclosing this code in braces ensures the file will be closed by the File destructor. The later file deletion requires the file be closed in Windows. + if (!state.data.empty()) { + writeIn(bufData, state.data); + log("CERT loaded @ %" PRIu64 "\n", state.k); + } else { // Enclosing this code in braces ensures the file will be closed by the File destructor. The later file deletion requires the file be closed in Windows. File fi = File::openReadThrow(fname); - u32 nBytes = (E - 1) / 8 + 1; - Words B = fi.readBytesLE(nBytes); - writeIn(bufData, std::move(B)); + u32 const nBytes = u32((E - 1) / 8 + 1); + Words const B = fi.readBytesLE(nBytes); + writeIn(bufData, B); } + double const elapsedBefore = state.elapsed; Timer elapsedTimer; elapsedTimer.reset(); - u32 startK = 0; + u32 const startK = u32(state.k); IterationTimer iterationTimer{startK}; - u32 k = 0; - u32 kEnd = task.squarings; + u32 k = startK; + u32 const kEnd = task.squarings; enum LEAD_TYPE leadIn = LEAD_NONE; while (true) { @@ -2071,8 +3014,8 @@ array Gpu::isCERT(const Task& task) { log("Stopping, please wait..\n"); } - bool doLog = (k % 100'000 == 0) || doStop; - enum LEAD_TYPE leadOut = doLog || useLongCarry ? LEAD_NONE : LEAD_WIDTH; + bool const doLog = (k % args.logStep == 0) || doStop; // same cadence as LL; every log point is also a checkpoint + enum LEAD_TYPE const leadOut = doLog || useLongCarry ? LEAD_NONE : LEAD_WIDTH; squareCERT(bufData, leadIn, leadOut); leadIn = leadOut; @@ -2081,17 +3024,20 @@ array Gpu::isCERT(const Task& task) { Words data = readData(); assert(data.size() >= 2); - u64 res64 = (u64(data[1]) << 32) | data[0]; + u64 const res64 = (u64(data[1]) << 32) | data[0]; - float secsPerIt = iterationTimer.reset(k); - queue->setSquareTime((int) (secsPerIt * 1'000'000)); - log("%9u %016" PRIx64 " %4.0f\n", k, res64, secsPerIt * 1'000'000); + float const secsPerIt = iterationTimer.reset(k); + queue.setSquareTime((int) (secsPerIt * 1'000'000)); + log("%7u / %7u %016" PRIx64 " %s ETA %s\n", k, kEnd, res64, formatSecsPerIter(secsPerIt).c_str(), getETA(k, kEnd, secsPerIt).c_str()); if (k >= kEnd) { fs::remove (fname); - return std::move(SHA3{}.update(data.data(), (E-1)/8+1)).finish(); + Saver::clear(E); + return std::move(SHA3{}.update(data.data(), u32((E-1)/8+1))).finish(); } + saver.save({.exponent=E, .k=k, .squarings=kEnd, .data=std::move(data), .elapsed=elapsedBefore + elapsedTimer.at()}); + if (doStop) { throw "stop requested"; } } } diff --git a/src/Gpu.h b/src/Gpu.h index fc5166f3..0427d871 100644 --- a/src/Gpu.h +++ b/src/Gpu.h @@ -15,13 +15,14 @@ #include "GpuCommon.h" #include "FFTConfig.h" +#include #include #include #include #include struct PRPResult; -struct Task; +class Task; class Signal; class ProofSet; @@ -35,7 +36,7 @@ struct PRPResult { bool isPrime{}; u64 res64 = 0; u32 nErrors = 0; - fs::path proofPath{}; + fs::path proofPath; std::string res2048; }; @@ -62,15 +63,15 @@ class RoeInfo { RoeInfo(u32 n, double max, double mean, double sd) : N{n}, max{max}, mean{mean}, sd{sd} { // https://en.wikipedia.org/wiki/Gumbel_distribution gumbelBeta = sd * 0.779696801233676; // sqrt(6)/pi - gumbelMiu = mean - gumbelBeta * 0.577215664901533; // Euler-Mascheroni + gumbelMiu = mean - gumbelBeta * std::numbers::egamma; // Euler-Mascheroni } - double z(double x = 0.5) const { return N ? (x - gumbelMiu) / gumbelBeta : 0.0; } + [[nodiscard]] double z(double x = 0.5) const { return N ? (x - gumbelMiu) / gumbelBeta : 0.0; } - double gumbelCDF(double x) const { return exp(-exp(-z(x))); } - double gumbelRightCDF(double x) const { return -expm1(-exp(-z(x))); } + [[nodiscard]] double gumbelCDF(double x) const { return exp(-exp(-z(x))); } + [[nodiscard]] double gumbelRightCDF(double x) const { return -expm1(-exp(-z(x))); } - std::string toString() const; + [[nodiscard]] std::string toString() const; u32 N{}; double max{}, mean{}, sd{}; @@ -80,20 +81,19 @@ class RoeInfo { struct Weights { vector weightsConstIF; vector weightsIF; - vector bitsCF; }; class Gpu { - Queue* queue; + GpuCommon shared; Background* background; public: - const Args& args; + Args& args; private: std::unique_ptr> saver; - u32 E; + u64 E; u32 N; FFTConfig fft; @@ -107,6 +107,8 @@ class Gpu { Profile profile{}; + Queue queue; + vector auxQueues; KernelCompiler compiler; /* Kernels for FFT_FP64 or FFT_FP32 */ @@ -114,6 +116,8 @@ class Gpu { Kernel kfftHin; Kernel ktailSquareZero; Kernel ktailSquare; + Kernel ktailMulZero; + Kernel ktailMulLowZero; Kernel ktailMul; Kernel ktailMulLow; Kernel kfftMidOut; @@ -124,6 +128,8 @@ class Gpu { Kernel kfftHinGF31; Kernel ktailSquareZeroGF31; Kernel ktailSquareGF31; + Kernel ktailMulZeroGF31; + Kernel ktailMulLowZeroGF31; Kernel ktailMulGF31; Kernel ktailMulLowGF31; Kernel kfftMidOutGF31; @@ -134,6 +140,8 @@ class Gpu { Kernel kfftHinGF61; Kernel ktailSquareZeroGF61; Kernel ktailSquareGF61; + Kernel ktailMulZeroGF61; + Kernel ktailMulLowZeroGF61; Kernel ktailMulGF61; Kernel ktailMulLowGF61; Kernel kfftMidOutGF61; @@ -172,6 +180,7 @@ class Gpu { bool tail_single_wide; // TailSquare processes one line at a time bool tail_single_kernel; // TailSquare does not use a separate kernel for line zero u32 in_place; // Should GPU perform transform in-place. 1 = nVidia friendly memory layout, 2 = AMD friendly. + u32 wmul; // Number of workgroups carryFused kernel should process ("width multiplier"). u32 pad_size; // Pad size in bytes as specified on the command line or config.txt. Maximum value is 512. // Twiddles: trigonometry constant buffers, used in FFTs. @@ -185,7 +194,6 @@ class Gpu { Weights weights; Buffer bufConstWeights; Buffer bufWeights; - Buffer bufBits; // bigWord bits aligned for CarryFused/fftP // "integer word" buffers. These are "small buffers": N x int. Buffer bufData; // Main int buffer with the words. @@ -218,24 +226,37 @@ class Gpu { TimeInfo* timeBufVect; ZAvg zAvg; - int NUM_CACHE_GROUPS = 3; + enum BOTTOM_HALF_KERNELS {KMIDIN, KFFTHIN, KTAILSQUARE, KTAILMUL, KTAILMULLOW, KMIDOUT, KFFTW}; + vector recorded_kernels; + vector *> recorded_kernel_args; + + bool use_graphs; + Graph graph_square[4]; + + const int NUM_CACHE_GROUPS = 3; + void splitQueue(); + void mergeQueue(); + void endBottomHalf(); + void replay(); + void replay_one(enum BOTTOM_HALF_KERNELS kern, int cache_group, int arg, Queue *q = nullptr, int base = 0, int kernelsToExecuteX = 0, int kernelsToExecuteY = 1); + int replay_next_arg(enum BOTTOM_HALF_KERNELS kern, int arg); void fftP(Buffer& out, Buffer& in) { fftP(out, reinterpret_cast&>(in)); } void fftP(Buffer& out, Buffer& in); - void fftMidIn(Buffer& out, Buffer& in, int cache_group = 0); - void fftMidOut(Buffer& out, Buffer& in, int cache_group = 0); + void fftMidIn(Buffer& buf); + void fftMidOut(Buffer& buf); void fftHin(Buffer& out, Buffer& in); - void tailSquare(Buffer& out, Buffer& in, int cache_group = 0); - void tailMul(Buffer& out, Buffer& in1, Buffer& in2, int cache_group = 0); - void tailMulLow(Buffer& out, Buffer& in1, Buffer& in2, int cache_group = 0); - void fftW(Buffer& out, Buffer& in, int cache_group = 0); + void tailSquare(Buffer& buf); + void tailMul(Buffer& buf, Buffer& in2); + void tailMulLow(Buffer& buf, Buffer& in2); + void fftW(Buffer& out, Buffer& in); void carryA(Buffer& out, Buffer& in) { carryA(reinterpret_cast&>(out), in); } void carryA(Buffer& out, Buffer& in); void carryM(Buffer& out, Buffer& in); void carryLL(Buffer& out, Buffer& in); - void carryFused(Buffer& out, Buffer& in); - void carryFusedMul(Buffer& out, Buffer& in); - void carryFusedLL(Buffer& out, Buffer& in); + void carryFused(Buffer& buf); + void carryFusedMul(Buffer& buf); + void carryFusedLL(Buffer& buf); vector readWords(Buffer &buf); void writeWords(Buffer& buf, vector &words); @@ -250,28 +271,27 @@ class Gpu { void squareCERT(Buffer& io, enum LEAD_TYPE leadIn, enum LEAD_TYPE leadOut) { square(io, io, leadIn, leadOut, false, false); } void squareLL(Buffer& io, enum LEAD_TYPE leadIn, enum LEAD_TYPE leadOut) { square(io, io, leadIn, leadOut, false, true); } - u32 squareLoop(Buffer& out, Buffer& in, u32 from, u32 to, bool doTailMul3); - u32 squareLoop(Buffer& io, u32 from, u32 to) { return squareLoop(io, io, from, to, false); } + u64 squareLoop(Buffer& out, Buffer& in, u64 from, u64 to, bool doTailMul3); + u64 squareLoop(Buffer& io, u64 from, u64 to) { return squareLoop(io, io, from, to, false); } bool isEqual(Buffer& bufCheck, Buffer& bufAux); u64 bufResidue(Buffer& buf); vector writeBase(const vector &v); - void exponentiate(Buffer& bufInOut, u64 exp, Buffer& buf1, Buffer& buf2, Buffer& buf3); + void exponentiate(Buffer& bufInOut, u64 exp); - void writeState(u32 k, const vector& check, u32 blockSize); + void writeState(u64 k, const vector& check, u32 blockSize); // does either carrryFused() or the expanded version depending on useLongCarry - void doCarry(Buffer& out, Buffer& in, Buffer& tmp); + void doCarry(Buffer& in, Buffer& wordBuf); - void mul(Buffer& ioA, Buffer& inB, Buffer& tmp1, Buffer& tmp2, bool mul3 = false); - void mul(Buffer& io, Buffer& inB); + void mul(Buffer& ioA, Buffer& inB, Buffer& tmp1, bool mul3 = false); void modMul(Buffer& ioA, Buffer& inB, bool mul3 = false); void modMul(Buffer& ioA, Buffer& inB, enum LEAD_TYPE leadInB, bool mul3 = false); - fs::path saveProof(const Args& args, const ProofSet& proofSet); + fs::path saveProof(const Args& args, ProofSet& proofSet); std::pair readROE(); RoeInfo readCarryStats(); @@ -283,14 +303,13 @@ class Gpu { // void measureTransferSpeed(); - static void doDiv9(u32 E, Words& words); + static void doDiv9(u64 E, Words& words); static bool equals9(const Words& words); void selftestTrig(); public: - Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u32 E, const vector& extraConf, bool logFftSize); - static unique_ptr make(Queue* q, u32 E, GpuCommon shared, FFTConfig fft, - const vector& extraConf = {}, bool logFftSize = true); + Gpu(GpuCommon shared, FFTConfig fft, u64 E, const vector& extraConf, bool logFftSize); + static unique_ptr make(u64 E, GpuCommon shared, FFTConfig fft, const vector& extraConf = {}, bool logFftSize = true); ~Gpu(); @@ -306,7 +325,7 @@ class Gpu { Saver *getSaver(); void writeIn(Buffer& buf, const vector &words); - + u64 dataResidue() { return bufResidue(bufData); } u64 checkResidue() { return bufResidue(bufCheck); } @@ -318,7 +337,6 @@ class Gpu { vector readCheck(); vector readData(); - u32 getFFTSize() { return N; } // return A^h * B @@ -337,19 +355,24 @@ class Gpu { void clear(bool isPRP); private: - u32 getProofPower(u32 k); - void doBigLog(u32 k, u64 res, bool checkOK, float secsPerIt, u32 nIters, u32 nErrors); + u32 getProofPower(u64 k); + void doBigLog(u64 k, u64 res, bool checkOK, float secsPerIt, u64 nIters, u32 nErrors); + enum WHICH_KERNEL {CARRYFUSED=0, MIDIN=1, MIDIN31=2, MIDIN61=3, TAIL=4, TAIL31=5, TAIL61=6, MIDOUT=7, MIDOUT31=8, MIDOUT61=9}; + string numRegisters(enum WHICH_KERNEL which_kernel); + string amdRegisterOption(enum WHICH_KERNEL which_kernel, int override_regs); + enum WHICH_KERNEL_TYPE {KFP=0, K31=1, K61=2, KALL=3}; + string kernelDefines(enum WHICH_KERNEL_TYPE which_kernel); }; // Compute the size of an FFT/NTT data buffer depending on the FFT/NTT float/prime. Size is returned in units of sizeof(double). // Data buffers require extra space for padding. We can probably tighten up the amount of extra memory allocated. // The worst case seems to be !INPLACE, MIDDLE=4, PAD_SIZE=512. -#define MID_ADJUST(size,M,pad) ((pad == 0 || M != 4) ? (size) : (size) * 5/4) -#define PAD_ADJUST(N,M,inplace,pad) (inplace ? 3*N/2 : MID_ADJUST(pad == 0 ? N : pad <= 128 ? 9*N/8 : pad <= 256 ? 5*N/4 : 3*N/2, M, pad)) -#define FP64_DATA_SIZE(W,M,H,inplace,pad) PAD_ADJUST(W*M*H*2, M, inplace, pad) -#define FP32_DATA_SIZE(W,M,H,inplace,pad) PAD_ADJUST(W*M*H*2, M, inplace, pad) * sizeof(float) / sizeof(double) -#define GF31_DATA_SIZE(W,M,H,inplace,pad) PAD_ADJUST(W*M*H*2, M, inplace, pad) * sizeof(uint) / sizeof(double) -#define GF61_DATA_SIZE(W,M,H,inplace,pad) PAD_ADJUST(W*M*H*2, M, inplace, pad) * sizeof(ulong) / sizeof(double) -#define TOTAL_DATA_SIZE(fft,W,M,H,inplace,pad) (int)fft.FFT_FP64 * FP64_DATA_SIZE(W,M,H,inplace,pad) + (int)fft.FFT_FP32 * FP32_DATA_SIZE(W,M,H,inplace,pad) + \ - (int)fft.NTT_GF31 * GF31_DATA_SIZE(W,M,H,inplace,pad) + (int)fft.NTT_GF61 * GF61_DATA_SIZE(W,M,H,inplace,pad) +#define MID_ADJUST(size,M,pad) (((pad) == 0 || (M) != 4) ? (size) : (size) * 5/4) +#define PAD_ADJUST(N,M,inplace,pad) ((inplace) ? 3*(N)/2 : MID_ADJUST((pad) == 0 ? (N) : (pad) <= 128 ? 9*(N)/8 : (pad) <= 256 ? 5*(N)/4 : 3*(N)/2, M, pad)) +#define FP64_DATA_SIZE(W,M,H,inplace,pad) PAD_ADJUST((W)*(M)*(H)*2, M, inplace, pad) +#define FP32_DATA_SIZE(W,M,H,inplace,pad) PAD_ADJUST((W)*(M)*(H)*2, M, inplace, pad) * sizeof(float) / sizeof(double) +#define GF31_DATA_SIZE(W,M,H,inplace,pad) PAD_ADJUST((W)*(M)*(H)*2, M, inplace, pad) * sizeof(uint) / sizeof(double) +#define GF61_DATA_SIZE(W,M,H,inplace,pad) PAD_ADJUST((W)*(M)*(H)*2, M, inplace, pad) * sizeof(ulong) / sizeof(double) +#define TOTAL_DATA_SIZE(fft,W,M,H,inplace,pad) ((int)(fft).FFT_FP64 * FP64_DATA_SIZE(W,M,H,inplace,pad) + (int)(fft).FFT_FP32 * FP32_DATA_SIZE(W,M,H,inplace,pad) + \ + (int)(fft).NTT_GF31 * GF31_DATA_SIZE(W,M,H,inplace,pad) + (int)(fft).NTT_GF61 * GF61_DATA_SIZE(W,M,H,inplace,pad)) diff --git a/src/GpuCommon.h b/src/GpuCommon.h index 5a9f4756..76ba072b 100644 --- a/src/GpuCommon.h +++ b/src/GpuCommon.h @@ -2,6 +2,7 @@ #pragma once +class Context; class Args; class TrigBufCache; class Background; @@ -9,6 +10,7 @@ class Background; // Data that's normally shared between Gpu instances class GpuCommon { public: + Context* context; Args* args; TrigBufCache* bufCache; Background* background; diff --git a/src/Hash.h b/src/Hash.h index ddeb9c2e..87a9aa25 100644 --- a/src/Hash.h +++ b/src/Hash.h @@ -11,7 +11,7 @@ class Hash { public: template - static auto hash(Ts... data) { + static auto hash(const Ts&... data) { Hash hash; (hash.update(data),...); return std::move(hash).finish(); @@ -20,15 +20,15 @@ class Hash { Hash& update(const void* data, u32 size) { h.update(data, size); return *this; } template - Hash&& update(const array& v) && { h.update(v.data(), N * sizeof(T)); return std::move(*this); } + Hash&& update(const array& v) && { h.update(v.data(), u32(N * sizeof(T))); return std::move(*this); } - void update(u32 x) { h.update(&x, sizeof(x)); } - void update(u64 x) { h.update(&x, sizeof(x)); } + void update(u32 x) { h.update(&x, u32(sizeof(x))); } + void update(u64 x) { h.update(&x, u32(sizeof(x))); } template - void update(const vector& v) { h.update(v.data(), v.size() * sizeof(T)); } + void update(const vector& v) { h.update(v.data(), u32(v.size() * sizeof(T))); } - void update(const string& s) {h.update(s.c_str(), s.size()); } + void update(const string& s) {h.update(s.c_str(), u32(s.size())); } auto finish() && { return std::move(h).finish(); } }; diff --git a/src/Kernel.cpp b/src/Kernel.cpp index 5da47088..850096ec 100644 --- a/src/Kernel.cpp +++ b/src/Kernel.cpp @@ -3,7 +3,6 @@ #include "Kernel.h" #include "KernelCompiler.h" -#include Kernel::Kernel(string_view name, KernelCompiler* compiler, TimeInfo* timeInfo, Queue* queue, string_view fileName, string_view nameInFile, @@ -15,7 +14,8 @@ Kernel::Kernel(string_view name, KernelCompiler* compiler, TimeInfo* timeInfo, Q defines{defines}, timeInfo{timeInfo}, queue{queue}, - workSize{workSize} + workSizeX{workSize}, + workSizeY{1} {} Kernel::~Kernel() = default; @@ -33,12 +33,27 @@ void Kernel::finishLoad() { assert(kernel); groupSize = getWorkGroupSize(kernel.get(), deviceId, name.c_str()); assert(groupSize); - assert(workSize % groupSize == 0); + assert(workSizeX % groupSize == 0); for (auto [pos, arg] : pendingArgs) { setArgs(pos, arg); } } +void Kernel::setKernelsToExecute(size_t nX, size_t nY) { // The rare two-dimensional kernel execution + if (nX == 0) return; // Use the default work size set at object creation. + + // Make sure kernel is loaded so that we have the groupSize + if (!kernel) { + startLoad(compiler); + finishLoad(); + } + if (!kernel) { throw std::runtime_error("OpenCL kernel "s + name + " not found"); } + + // For 2D kernels, we only support groupSizeY of 1. Setting workSizeY to more than one indicates a 2D kernel execution. + workSizeX = nX * groupSize; + workSizeY = nY; +} + void Kernel::run() { assert(kernel); - queue->run(kernel.get(), groupSize, workSize, timeInfo); + queue->run(kernel.get(), groupSize, workSizeX, workSizeY, timeInfo); } diff --git a/src/Kernel.h b/src/Kernel.h index a1505d4d..79f54242 100644 --- a/src/Kernel.h +++ b/src/Kernel.h @@ -24,10 +24,11 @@ class Kernel { TimeInfo *timeInfo; Queue* queue; - size_t workSize; + size_t workSizeX; + size_t workSizeY; u32 groupSize = 0; - KernelHolder kernel{}; + KernelHolder kernel; std::future pendingKernel; cl_device_id deviceId; std::vector> pendingArgs; @@ -42,9 +43,16 @@ class Kernel { void startLoad(KernelCompiler* compiler); void finishLoad(); - + + // Change which queue is used to run a kernel + void setQueue(Queue *q) { if (q != NULL) queue = q; } + + // Change number of kernels to execute. Usually this is set by Gpu.cpp at object creation (by setting the total number of work-items in workSizeX). + // L2 striping requires the ability to change this setting on-the-fly. One and two dimensional kernels are supported. + void setKernelsToExecute(size_t nX, size_t nY = 1); + template void setFixedArgs(int pos, const Args &...tail) { setArgs(pos, tail...); } - + template void operator()(const Args &...args) { if (!kernel) { startLoad(compiler); @@ -64,7 +72,7 @@ class Kernel { if (kernel) { ::setArg(kernel.get(), pos, arg, name); } else { - pendingArgs.push_back({pos, arg}); + pendingArgs.emplace_back(pos, arg); } } diff --git a/src/KernelCompiler.cpp b/src/KernelCompiler.cpp index cbf50c16..33337762 100644 --- a/src/KernelCompiler.cpp +++ b/src/KernelCompiler.cpp @@ -8,6 +8,10 @@ #include #include #include +#include +#include +#include +#include using namespace std; @@ -23,18 +27,38 @@ static_assert(sizeof(Program) == sizeof(cl_program)); // * -fno-bin-llvmir // * various: -fno-bin-source -fno-bin-amdil +// Does the device's compiler accept this -cl-std? Compiles an empty kernel with just that option. +static bool acceptsClStd(cl_context context, cl_device_id deviceId, const string& clStd) { + Program probe = loadSource(context, "kernel void probe() {}\n"); + if (!probe) { return false; } + string const opts = "-cl-std=" + clStd; + return clCompileProgram(probe.get(), 1, &deviceId, opts.c_str(), 0, nullptr, nullptr, nullptr, nullptr) == CL_SUCCESS; +} + KernelCompiler::KernelCompiler(const Args& args, const Context* context, const string& clArgs) : cacheDir{args.cacheDir.string()}, context{context->get()}, - linkArgs{"-cl-finite-math-only " }, - baseArgs{linkArgs + "-cl-std=CL2.0 " + clArgs}, + linkArgs{}, // no compile-only options here: clLinkProgram accepts only linker options (POCL enforces it) + baseArgs{}, dump{args.dump}, useCache{args.useCache}, verbose{args.verbose}, deviceId{context->deviceId()} { - string hw = getDriverVersion(deviceId) + ':' + getDeviceName(deviceId); + // Every GPU driver we run on accepts -cl-std=CL2.0. Some OpenCL 3.0 implementations (POCL; likely Mesa rusticl) + // offer no OpenCL C 2.0 at all and reject the option, but do offer OpenCL C 3.0, whose optional features cover what + // the kernels use from 2.0 (generic address space, memory-order atomics). Probe once and fall back. + string clStd = "CL2.0"; +#ifndef CUDA_BACKEND + if (!acceptsClStd(context->get(), deviceId, "CL2.0") && acceptsClStd(context->get(), deviceId, "CL3.0")) { + clStd = "CL3.0"; + log("OpenCL C 2.0 is not available on this device; compiling the kernels as OpenCL C 3.0\n"); + } +#endif + baseArgs = "-cl-finite-math-only -cl-std=" + clStd + ' ' + clArgs; + + string const hw = getDriverVersion(deviceId) + ':' + getDeviceName(deviceId); if (args.verbose) { log("OpenCL: %s, args %s\n", hw.c_str(), baseArgs.c_str()); } SHA3 hasher; @@ -44,10 +68,10 @@ KernelCompiler::KernelCompiler(const Args& args, const Context* context, const s auto& clNames = getClFileNames(); auto& clFiles = getClFiles(); assert(clNames.size() == clFiles.size()); - int n = clNames.size(); + int const n = int(clNames.size()); for (int i = 0; i < n; ++i) { auto &src = clFiles[i]; - files.push_back({clNames[i], src}); + files.emplace_back(clNames[i], src); clSources.push_back(loadSource(context->get(), src)); hasher.update(clNames[i]); @@ -65,10 +89,17 @@ Program KernelCompiler::compile(const string& fileName, const string& extraArgs) if (!dump.empty()) { args += " -save-temps="s + dump + "/" + fileName; } +#ifdef CUDA_BACKEND int err = clCompileProgram(p1.get(), 1, &deviceId, args.c_str(), - clSources.size(), (const cl_program*) (clSources.data()), getClFileNames().data(), + u32(clSources.size()), (const cl_program*) (clSources.data()), getClFileNames().data(), nullptr, nullptr); - if (string mes = getBuildLog(p1.get(), deviceId); !mes.empty()) { log("%s\n", mes.c_str()); } +#else + // Skip first file (opencl_compat.cuh) if this is a standard openCL application rather than a CUDA translation + int err = clCompileProgram(p1.get(), 1, &deviceId, args.c_str(), + u32(clSources.size())-1, (const cl_program*) (clSources.data()+1), getClFileNames().data()+1, + nullptr, nullptr); +#endif + if (string const mes = getBuildLog(p1.get(), deviceId); !mes.empty()) { log("%s\n", mes.c_str()); } if (err != CL_SUCCESS) { log("Compiling '%s' error %s (args %s)\n", fileName.c_str(), errMes(err).c_str(), args.c_str()); return {}; @@ -76,7 +107,10 @@ Program KernelCompiler::compile(const string& fileName, const string& extraArgs) Program p2{clLinkProgram(context, 1, &deviceId, linkArgs.c_str(), 1, (cl_program *) &p1, nullptr, nullptr, &err)}; - if (string mes = getBuildLog(p1.get(), deviceId); !mes.empty()) { log("%s\n", mes.c_str()); } + // The linker's diagnostics live on the linked program. Asking p1 again instead says nothing about the link + // -- and repeats the compile log that was already printed above. A failed clLinkProgram may hand back no + // program at all, and then there is nothing to query. + if (p2) { if (string const mes = getBuildLog(p2.get(), deviceId); !mes.empty()) { log("%s\n", mes.c_str()); } } if (err != CL_SUCCESS) { log("Linking '%s' error %s (args %s)\n", fileName.c_str(), errMes(err).c_str(), linkArgs.c_str()); } @@ -90,14 +124,14 @@ static string to_hex(u64 d) { } KernelHolder KernelCompiler::loadAux(const string& fileName, const string& kernelName, const string& args) const { - Timer timer; + Timer const timer; bool fromCache = true; Program program; string cacheFile; if (useCache) { - string f = kernelName + '-' + to_hex(SHA3::hash(contextHash, fileName, kernelName, args)[0]); + string const f = kernelName + '-' + to_hex(SHA3::hash(contextHash, fileName, kernelName, args)[0]); cacheFile = cacheDir + '/' + f; program = loadBinary(context, deviceId, cacheFile); } @@ -130,11 +164,28 @@ KernelHolder KernelCompiler::loadAux(const string& fileName, const string& kerne } std::future KernelCompiler::load(const string& fileName, const string& kernelName, const string& args) const { -#if 0 - // Do the compilation in parallel on a separate thread. - // Unfortunatelly no benefit on ROCm (the compiler serializes). - return async(std::launch::async, &KernelCompiler::loadAux, this, fileName, kernelName, args); +#ifdef CUDA_BACKEND + // NVRTC compiles independently per thread, so the ~36 kernels of a Gpu + // compile in parallel — bounded to the core count and to eight: each + // NVRTC instance holds a few hundred MB while it runs, so 36 at once, or + // one per thread of a 32-thread machine, would take gigabytes of host + // memory for a speedup that eight threads over 36 kernels already give + // most of. The CUDA shim makes the context current per thread and guards + // its shared module counts. The thread logs through this worker's log + // file and context (LogLink): the log is thread-local, and a fresh thread + // would otherwise print its "Loaded" lines to stdout alone, unprefixed. + static std::counting_semaphore<8> slots{std::max(1u, std::min(8u, std::thread::hardware_concurrency()))}; + LogLink const link = logLink(); + return async(std::launch::async, [this, fileName, kernelName, args, link] { + slots.acquire(); + struct Release { std::counting_semaphore<8>& s; ~Release() { s.release(); } } release{slots}; + LogLinkScope const logScope{link}; + return loadAux(fileName, kernelName, args); + }); #else + // Serial: the ROCm compiler serializes parallel builds anyway (no benefit + // measured), and the OpenCL runtime's thread-safety for concurrent + // clCompileProgram varies by vendor. std::promise promise; promise.set_value(loadAux(fileName, kernelName, args)); return promise.get_future(); diff --git a/src/KernelCompiler.h b/src/KernelCompiler.h index 0d07a7ee..a5588331 100644 --- a/src/KernelCompiler.h +++ b/src/KernelCompiler.h @@ -18,20 +18,20 @@ class KernelCompiler { std::string baseArgs; std::string dump; const bool useCache; - const bool verbose; + const int verbose; std::vector clSources; std::vector> files; u64 contextHash{}; - Program compile(const string& fileName, const string& args) const; - KernelHolder loadAux(const string& fileName, const string& kernelName, const string& args) const; + [[nodiscard]] Program compile(const string& fileName, const string& args) const; + [[nodiscard]] KernelHolder loadAux(const string& fileName, const string& kernelName, const string& args) const; public: const cl_device_id deviceId; KernelCompiler(const Args& args, const Context* context, const string& clArgs); - std::future load(const string& fileName, const string& kernelName, const string& args) const; + [[nodiscard]] std::future load(const string& fileName, const string& kernelName, const string& args) const; }; diff --git a/src/MD5.h b/src/MD5.h index 433d1b64..c0b19437 100644 --- a/src/MD5.h +++ b/src/MD5.h @@ -22,10 +22,10 @@ class MD5Hash { unsigned char digest[16]; MD5Final(digest, &context); string s; - char hex[] = "0123456789abcdef"; - for (int i = 0; i < 16; ++i) { - s.push_back(hex[digest[i] >> 4]); - s.push_back(hex[digest[i] & 0xf]); + char const hex[] = "0123456789abcdef"; + for (unsigned char const i : digest) { + s.push_back(hex[i >> 4]); + s.push_back(hex[i & 0xf]); } return s; } diff --git a/src/PRPState.cpp b/src/PRPState.cpp index bdc2c9c8..02b9ae68 100644 --- a/src/PRPState.cpp +++ b/src/PRPState.cpp @@ -8,7 +8,8 @@ PRPState::PRPState(File&& fi) { string header = fi.readLine(); - u32 fileE, fileK, blockSize, nErrors, crc; + u64 fileE, fileK; + u32 blockSize, nErrors, crc; u64 res64; vector check; u32 b1, nBits, start, nextK; diff --git a/src/PRPState.h b/src/PRPState.h index edf0ccf9..1eab5085 100644 --- a/src/PRPState.h +++ b/src/PRPState.h @@ -10,23 +10,23 @@ class File; class PRPState { // E, k, block-size, res64, nErrors - static constexpr const char *PRP_v10 = "OWL PRP 10 %u %u %u %016" SCNx64 " %u\n"; + static constexpr const char *PRP_v10 = "OWL PRP 10 %" PRIu64 " %" PRIu64 " %u %016" SCNx64 " %u\n"; // Exponent, iteration, block-size, res64, nErrors // B1, nBits, start, nextK, crc - static constexpr const char *PRP_v11 = "OWL PRP 11 %u %u %u %016" SCNx64 " %u %u %u %u %u %u\n"; + static constexpr const char *PRP_v11 = "OWL PRP 11 %" PRIu64 " %" PRIu64 " %u %016" SCNx64 " %u %u %u %u %u %u\n"; // E, k, block-size, res64, nErrors, CRC - static constexpr const char *PRP_v12 = "OWL PRP 12 %u %u %u %016" SCNx64 " %u %u\n"; + static constexpr const char *PRP_v12 = "OWL PRP 12 %" PRIu64 " %" PRIu64 " %u %016" SCNx64 " %u %u\n"; public: - u32 k{}; + u64 k{}; u32 blockSize{}; u64 res64{}; vector check; u32 nErrors{}; - // PRPState(u32 k, u32 blockSize, u64 res64, vector<) + // PRPState(u64 k, u32 blockSize, u64 res64, vector<) PRPState(File&& f); void saveTo(const File& f); }; diff --git a/src/Primes.cpp b/src/Primes.cpp index 865cfaec..665ee251 100644 --- a/src/Primes.cpp +++ b/src/Primes.cpp @@ -9,20 +9,20 @@ Primes::Primes() { for (u32 i = 0; i < sieve.size(); ++i) { if (sieve[i]) { - u32 n = 2 * i + 3; + u32 const n = 2 * i + 3; for (u32 k = i + n; k < sieve.size(); k += n) { sieve.reset(k); } } } } -bool Primes::isPrimeOdd(u32 n) const { +bool Primes::isPrimeOdd(u64 n) const { assert(n % 2); // must be odd to call here if (n < 3) { return false; } for (u32 k = 0; k < sieve.size(); ++k) { if (sieve[k]) { - u32 p = k * 2 + 3; - if (p * p > n) { return true; } + u32 const p = k * 2 + 3; + if (u64(p) * u64(p) > n) { return true; } if (n % p == 0) { return false; } } } @@ -30,11 +30,11 @@ bool Primes::isPrimeOdd(u32 n) const { return false; } -bool Primes::isPrime(u32 n) const { +bool Primes::isPrime(u64 n) const { return (n%2 && isPrimeOdd(n)) || (n == 2); } -u32 Primes::prevPrime(u32 n) const { +u64 Primes::prevPrime(u64 n) const { --n; if (n % 2 == 0) { --n; } @@ -43,7 +43,7 @@ u32 Primes::prevPrime(u32 n) const { return 0; } -u32 Primes::nextPrime(u32 n) const { +u64 Primes::nextPrime(u64 n) const { ++n; if (n % 2 == 0) { ++n; } for (; ; n += 2) { if (isPrimeOdd(n)) { return n; }} @@ -51,10 +51,10 @@ u32 Primes::nextPrime(u32 n) const { return 0; } -u32 Primes::nearestPrime(u32 n) const { +u64 Primes::nearestPrime(u64 n) const { if (isPrime(n)) { return n; } - u32 a = prevPrime(n); - u32 b = nextPrime(n); + u64 const a = prevPrime(n); + u64 const b = nextPrime(n); assert(a < n && n < b); return n-a < b-n ? a : b; } diff --git a/src/Primes.h b/src/Primes.h index b951c02d..aee511f4 100644 --- a/src/Primes.h +++ b/src/Primes.h @@ -6,14 +6,14 @@ #include "common.h" class Primes { - std::bitset<50000> sieve; - bool isPrimeOdd(u32 n) const; + std::bitset<50000> sieve; // Allows for testing primes up to 10 billion + [[nodiscard]] bool isPrimeOdd(u64 n) const; public: Primes(); - bool isPrime(u32 n) const; - u32 prevPrime(u32 n) const; - u32 nextPrime(u32 n) const; - u32 nearestPrime(u32 n) const; + [[nodiscard]] bool isPrime(u64 n) const; + [[nodiscard]] u64 prevPrime(u64 n) const; + [[nodiscard]] u64 nextPrime(u64 n) const; + [[nodiscard]] u64 nearestPrime(u64 n) const; }; diff --git a/src/Profile.cpp b/src/Profile.cpp index dcb543d2..4437abde 100644 --- a/src/Profile.cpp +++ b/src/Profile.cpp @@ -13,7 +13,7 @@ TimeInfo* Profile::make(string_view s) { vector Profile::get() const { vector ret; for (auto& t : entries) { if (t->n) { ret.push_back(t.get()); } } - std::sort(ret.begin(), ret.end(), [](auto p1, auto p2) { return p1->times[2] > p2->times[2]; }); + std::ranges::sort(ret, [](auto p1, auto p2) { return p1->times[2] > p2->times[2]; }); return ret; } diff --git a/src/Profile.h b/src/Profile.h index 3c919142..46040201 100644 --- a/src/Profile.h +++ b/src/Profile.h @@ -16,7 +16,7 @@ class Profile { public: TimeInfo *make(std::string_view s); - std::vector get() const; + [[nodiscard]] std::vector get() const; void reset(); }; diff --git a/src/Proof.cpp b/src/Proof.cpp index 31c488f8..a869d5cc 100644 --- a/src/Proof.cpp +++ b/src/Proof.cpp @@ -6,6 +6,7 @@ #include "Gpu.h" #include +#include #include #include #include @@ -19,33 +20,34 @@ namespace proof { -array hashWords(u32 E, const Words& words) { - return std::move(SHA3{}.update(words.data(), (E-1)/8+1)).finish(); +array hashWords(u64 E, const Words& words) { + return std::move(SHA3{}.update(words.data(), u32((E-1)/8+1))).finish(); } -array hashWords(u32 E, array prefix, const Words& words) { - return std::move(SHA3{}.update(prefix).update(words.data(), (E-1)/8+1)).finish(); +array hashWords(u64 E, array prefix, const Words& words) { + return std::move(SHA3{}.update(prefix).update(words.data(), u32((E-1)/8+1))).finish(); } string fileHash(const fs::path& filePath) { File fi = File::openReadThrow(filePath); char buf[64 * 1024]; MD5 h; - u32 size = 0; - while ((size = fi.readUpTo(buf, sizeof(buf)))) { h.update(buf, size); } + size_t size = 0; + while ((size = fi.readUpTo(buf, sizeof(buf)))) { h.update(buf, u32(size)); } return std::move(h).finish(); } ProofInfo getInfo(const fs::path& proofFile) { - string hash = proof::fileHash(proofFile); + string const hash = proof::fileHash(proofFile); File fi = File::openReadThrow(proofFile); - u32 E = 0, power = 0; + u64 E = 0; + u32 power = 0; char c = 0; if (fi.scanf(Proof::HEADER_v2, &power, &E, &c) != 3 || c != '\n') { log("Proof file '%s' has invalid header\n", proofFile.string().c_str()); throw "Invalid proof header"; } - return {power, E, hash}; + return {.power=power, .exp=E, .md5=hash}; } } @@ -53,14 +55,14 @@ ProofInfo getInfo(const fs::path& proofFile) { // ---- Proof ---- fs::path Proof::file(const fs::path& proofDir) const { - string strE = to_string(E); - u32 power = middles.size(); + string const strE = to_string(E); + u32 const power = u32(middles.size()); return proofDir / (strE + '-' + to_string(power) + ".proof"); } void Proof::save(const fs::path& proofFile) const { - File fo = File::openWrite(proofFile); - u32 power = middles.size(); + File const fo = File::openWrite(proofFile); + u32 const power = u32(middles.size()); fo.printf(HEADER_v2, power, E, '\n'); fo.write(B.data(), (E-1)/8+1); for (const Words& w : middles) { fo.write(w.data(), (E-1)/8+1); } @@ -68,57 +70,59 @@ void Proof::save(const fs::path& proofFile) const { Proof Proof::load(const fs::path& path) { File fi = File::openReadThrow(path); - u32 E = 0, power = 0; + u64 E = 0; + u32 power = 0; char c = 0; if (fi.scanf(HEADER_v2, &power, &E, &c) != 3 || c != '\n') { log("Proof file '%s' has invalid header\n", path.string().c_str()); throw "Invalid proof header"; } - u32 nBytes = (E - 1) / 8 + 1; - Words B = fi.readBytesLE(nBytes); + u32 const nBytes = u32((E - 1) / 8 + 1); + Words const B = fi.readBytesLE(nBytes); vector middles; - for (u32 i = 0; i < power; ++i) { middles.push_back(fi.readBytesLE(nBytes)); } - return {E, B, middles}; + middles.reserve(power); +for (u32 i = 0; i < power; ++i) { middles.push_back(fi.readBytesLE(nBytes)); } + return {.E=E, .B=B, .middles=middles}; } bool Proof::verify(Gpu *gpu, const vector& hashes) const { // log("B %016" PRIx64 "\n", res64(B)); // for (u32 i = 0; i < middles.size(); ++i) { log("Middle[%u] %016" PRIx64 "\n", i, res64(middles[i])); } - - u32 power = middles.size(); + + u32 const power = u32(middles.size()); assert(power > 0); - bool isPrime = (B == makeWords(E, 9)); + bool const isPrime = (B == makeWords(E, 9)); Words A{makeWords(E, 3)}; Words B{this->B}; - + auto hash = proof::hashWords(E, B); - u32 span = E; + u64 span = E; for (u32 i = 0; i < power; ++i, span = (span + 1) / 2) { const Words& M = middles[i]; hash = proof::hashWords(E, hash, M); - u64 h = hash[0]; + u64 const h = hash[0]; if (hashes.size() > i && h != hashes.at(i)) { log("proof [%u] : hash expected %016" PRIx64 " != %016" PRIx64 "\n", i, hashes[i], h); return false; } - bool doSquareB = span % 2; + bool const doSquareB = span % 2; B = gpu->expMul(M, h, B, doSquareB); A = gpu->expMul(A, h, M, false); if (gpu->args.verbose) { log("proof [%u] : A %016" PRIx64 ", B %016" PRIx64 ", h %016" PRIx64 "\n", i, res64(A), res64(B), h); } } - log("proof verification: doing %d iterations\n", span); - A = gpu->expExp2(A, span); + log("proof verification: doing %" PRIu64 " iterations\n", span); + A = gpu->expExp2(A, u32(span)); - bool ok = (A == B); + bool const ok = (A == B); if (ok) { - log("proof: %u proved %s\n", E, isPrime ? "probable prime" : "composite"); + log("proof: %" PRIu64 " proved %s\n", E, isPrime ? "probable prime" : "composite"); } else { log("proof: invalid (%016" PRIx64 " expected %016" PRIx64 ")\n", res64(A), res64(B)); } @@ -127,48 +131,61 @@ bool Proof::verify(Gpu *gpu, const vector& hashes) const { // ---- ProofSet ---- -ProofSet::ProofSet(u32 E, u32 power) +ProofSet::ProofSet(u64 E, u32 power) : E{E}, power{power} { - + assert(E & 1); // E is supposed to be prime - if (power <= 0 || power > 12) { + if (power <= 0 || power > 13) { log("Invalid proof power: %u\n", power); throw "Invalid proof power"; } fs::create_directories(proofPath(E)); - vector spans; - for (u32 span = (E + 1) / 2; spans.size() < power; span = (span + 1) / 2) { spans.push_back(span); } + rebuildPoints(); +} + +void ProofSet::rebuildPoints() { + points.clear(); points.push_back(0); - for (u32 p = 0, span = (E + 1) / 2; p < power; ++p, span = (span + 1) / 2) { - for (u32 i = 0, end = points.size(); i < end; ++i) { + u32 p; + u64 span; + for (p = 0, span = (E + 1) / 2; p < power; ++p, span = (span + 1) / 2) { + for (u32 i = 0, end = u32(points.size()); i < end; ++i) { points.push_back(points[i] + span); } } - assert(points.size() == (1u << power)); + assert(u32(points.size()) == (1u << power)); assert(points.front() == 0); points.front() = E; - std::sort(points.begin(), points.end()); + std::ranges::sort(points); - assert(points.size() == (1u << power)); + assert(u32(points.size()) == (1u << power)); assert(points.back() == E); - points.push_back(u32(-1)); // guard element + points.push_back(u64(-1LL)); // guard element cacheIt = points.begin(); - for ([[maybe_unused]] u32 p : points) { + for ([[maybe_unused]] u64 const p : points) { assert(p > E || isInPoints(E, power, p)); } } -bool ProofSet::isInPoints(u32 E, u32 power, u32 k) { +void ProofSet::reducePower() { + assert(power > 0); + --power; + rebuildPoints(); +} + +bool ProofSet::isInPoints(u64 E, u32 power, u64 k) { if (k == E) { return true; } // special-case E - u32 start = 0; - for (u32 p = 0, span = (E + 1) / 2; p < power; ++p, span = (span + 1) / 2) { + u64 start = 0; + u32 p; + u64 span; + for (p = 0, span = (E + 1) / 2; p < power; ++p, span = (span + 1) / 2) { assert(k >= start); if (k > start + span) { start += span; @@ -179,12 +196,12 @@ bool ProofSet::isInPoints(u32 E, u32 power, u32 k) { return false; } -bool ProofSet::canDo(u32 E, u32 power, u32 currentK) { - assert(power > 0 && power <= 12); +bool ProofSet::canDo(u64 E, u32 power, u64 currentK) { + assert(power > 0 && power <= 13); return ProofSet{E, power}.isValidTo(currentK); } -u32 ProofSet::bestPower(u32 E) { +u32 ProofSet::bestPower(u64 E) { // Best proof powers assuming no disk space concern. // We increment power by 1 for each fourfold increase of the exponent. // The values below produce power=10 at wavefront, and power=11 at 100Mdigits: @@ -192,12 +209,13 @@ u32 ProofSet::bestPower(u32 E) { assert(E > 0); // log2(x)/2 is log4(x) - int power = 10 + floor(log2(E / 60e6) / 2); + int power = int(10 + floor(log2(double(E) / 60e6) / 2)); + if (power > 13) power = 13; assert(power >= 2); return power; } -double ProofSet::diskUsageGB(u32 E, u32 power) { +double ProofSet::diskUsageGB(u64 E, u32 power) { // -3 because convert exponent bits to bytes // -30 because convert bytes to GB // +power because needs 2^power residues for proof generation @@ -205,7 +223,7 @@ double ProofSet::diskUsageGB(u32 E, u32 power) { return power ? ldexp(E, -33 + int(power)) * 1.05 : 0.0; } -u32 ProofSet::effectivePower(u32 E, u32 power, u32 currentK) { +u32 ProofSet::effectivePower(u64 E, u32 power, u64 currentK) { for (u32 p = power; p > 0; --p) { // log("validating proof residues for power %u\n", p); if (canDo(E, p, currentK)) { return p; } @@ -213,12 +231,12 @@ u32 ProofSet::effectivePower(u32 E, u32 power, u32 currentK) { return 0; } -bool ProofSet::fileExists(u32 k) const { - return File::size(proofPath(E) / to_string(k)) == i64(E / 32 + 2) * 4; +bool ProofSet::fileExists(u64 k) const { + return File::size(proofPath(E) / to_string(k)) == (E / 32 + 2) * 4; } -bool ProofSet::isValidTo(u32 limitK) const { - auto it = upper_bound(points.begin(), points.end(), limitK); +bool ProofSet::isValidTo(u64 limitK) const { + auto it = std::ranges::upper_bound(points, limitK); if (it == points.begin()) { return true; @@ -238,14 +256,14 @@ bool ProofSet::isValidTo(u32 limitK) const { return true; } -u32 ProofSet::next(u32 k) const { +u64 ProofSet::next(u64 k) const { if (*cacheIt <= k || (cacheIt > points.begin() && *prev(cacheIt) > k)) { - cacheIt = upper_bound(points.begin(), points.end(), k); + cacheIt = std::ranges::upper_bound(points, k); } return *cacheIt; } -void ProofSet::save(u32 E, u32 power, u32 k, const Words& words) { +void ProofSet::save(u64 E, [[maybe_unused]] u32 power, u64 k, const Words& words) { assert(k && k <= E); assert(isInPoints(E, power, k)); @@ -253,7 +271,7 @@ void ProofSet::save(u32 E, u32 power, u32 k, const Words& words) { assert(load(E, power, k) == words); } -Words ProofSet::load(u32 E, u32 power, u32 k) { +Words ProofSet::load(u64 E, [[maybe_unused]] u32 power, u64 k) { assert(k && k <= E); assert(isInPoints(E, power, k)); return File::openReadThrow(proofPath(E) / to_string(k)).readChecked(E/32 + 1); @@ -261,7 +279,7 @@ Words ProofSet::load(u32 E, u32 power, u32 k) { std::pair> ProofSet::computeProof(Gpu *gpu) const { Words B = load(E); - Words A = makeWords(E, 3); + Words const A = makeWords(E, 3); vector middles; vector hashes; @@ -273,19 +291,19 @@ std::pair> ProofSet::computeProof(Gpu *gpu) const { for (u32 p = 0; p < power; ++p) { auto bufIt = bufVect.begin(); assert(p == hashes.size()); - u32 s = (1u << (power - p - 1)); + u32 const s = (1u << (power - p - 1)); for (u32 i = 0; i < (1u << p); ++i) { - Words w = load(points[s * (i * 2 + 1) - 1]); + Words const w = load(points[s * (i * 2 + 1) - 1]); gpu->writeIn(*bufIt++, w); for (u32 k = 0; i & (1u << k); ++k) { assert(k <= p - 1); --bufIt; - u64 h = hashes[p - 1 - k]; + u64 const h = hashes[p - 1 - k]; gpu->expMul(*(bufIt - 1), h, *bufIt); } } assert(bufIt == bufVect.begin() + 1); - Words w = gpu->readAndCompress(bufVect.front()); + Words const w = gpu->readAndCompress(bufVect.front()); if (w.empty()) { throw "Read ZERO during proof generation"; } middles.push_back(w); hash = proof::hashWords(E, hash, middles.back()); @@ -293,5 +311,5 @@ std::pair> ProofSet::computeProof(Gpu *gpu) const { log("proof [%u] : M %016" PRIx64 ", h %016" PRIx64 "\n", p, res64(middles.back()), hashes.back()); } - return {Proof{E, std::move(B), std::move(middles)}, hashes}; + return {Proof{.E=E, .B=std::move(B), .middles=std::move(middles)}, hashes}; } diff --git a/src/Proof.h b/src/Proof.h index 97e9056d..383eaef9 100644 --- a/src/Proof.h +++ b/src/Proof.h @@ -4,6 +4,7 @@ #include "File.h" #include "common.h" +#include namespace fs = std::filesystem; @@ -11,15 +12,15 @@ class Gpu; struct ProofInfo { u32 power; - u32 exp; + u64 exp; string md5; }; namespace proof { -array hashWords(u32 E, const Words& words); +array hashWords(u64 E, const Words& words); -array hashWords(u32 E, array prefix, const Words& words); +array hashWords(u64 E, array prefix, const Words& words); string fileHash(const fs::path& filePath); @@ -29,7 +30,7 @@ ProofInfo getInfo(const fs::path& proofFile); class Proof { public: - const u32 E; + const u64 E; const Words B; const vector middles; @@ -40,51 +41,53 @@ class Proof { POWER=8\n NUMBER=M216091\n */ - static const constexpr char* HEADER_v2 = "PRP PROOF\nVERSION=2\nHASHSIZE=64\nPOWER=%u\nNUMBER=M%u%c"; + static const constexpr char* HEADER_v2 = "PRP PROOF\nVERSION=2\nHASHSIZE=64\nPOWER=%u\nNUMBER=M%" PRIu64 "%c"; static Proof load(const fs::path& path); void save(const fs::path& proofResultDir) const; - fs::path file(const fs::path& proofDir) const; + [[nodiscard]] fs::path file(const fs::path& proofDir) const; bool verify(Gpu *gpu, const vector& hashes = {}) const; }; class ProofSet { public: - const u32 E; - const u32 power; + const u64 E; + u32 power; private: - vector points; + vector points; + + void rebuildPoints(); - bool isValidTo(u32 limitK) const; + bool isValidTo(u64 limitK) const; - static bool canDo(u32 E, u32 power, u32 currentK); + static bool canDo(u64 E, u32 power, u64 currentK); - mutable decltype(points)::const_iterator cacheIt{}; + mutable decltype(points)::const_iterator cacheIt; - bool fileExists(u32 k) const; + bool fileExists(u64 k) const; - static fs::path proofPath(u32 E) { return fs::path(to_string(E)) / "proof"; } + static fs::path proofPath(u64 E) { return fs::path(to_string(E)) / "proof"; } public: - static u32 bestPower(u32 E); - static u32 effectivePower(u32 E, u32 power, u32 currentK); - static double diskUsageGB(u32 E, u32 power); - static bool isInPoints(u32 E, u32 power, u32 k); + static u32 bestPower(u64 E); + static u32 effectivePower(u64 E, u32 power, u64 currentK); + static double diskUsageGB(u64 E, u32 power); + static bool isInPoints(u64 E, u32 power, u64 k); - ProofSet(u32 E, u32 power); - - u32 next(u32 k) const; + ProofSet(u64 E, u32 power); + + u64 next(u64 k) const; - static void save(u32 E, u32 power, u32 k, const Words& words); - static Words load(u32 E, u32 power, u32 k); - - void save(u32 k, const Words& words) const { return save(E, power, k, words); } - Words load(u32 k) const { return load(E, power, k); } + static void save(u64 E, u32 power, u64 k, const Words& words); + static Words load(u64 E, u32 power, u64 k); + void save(u64 k, const Words& words) const { return save(E, power, k, words); } + Words load(u64 k) const { return load(E, power, k); } + void reducePower(); std::pair> computeProof(Gpu *gpu) const; }; diff --git a/src/Queue.cpp b/src/Queue.cpp index 27f461ad..1fbd79d6 100644 --- a/src/Queue.cpp +++ b/src/Queue.cpp @@ -1,14 +1,13 @@ // Copyright (C) Mihai Preda #include "Queue.h" -#include "Args.h" #include "TimeInfo.h" -#include "timeutil.h" #include "log.h" #include #include #include +#include void Events::clearCompleted() { while (!empty() && front().isComplete()) { pop_front(); } } @@ -17,16 +16,18 @@ void Events::synced() { assert(empty()); } -Queue::Queue(const Context& context, bool profile) : +Queue::Queue(const Context& context, bool profile, bool auxQueue) : QueueHolder{makeQueue(context.deviceId(), context.get(), profile)}, hasEvents{profile}, + isAuxQueue(auxQueue), context{&context}, markerEvent{}, markerQueued(false), queueCount(0), + squareCount(0), squareTime(50), - squareKernels(4), - firstSetTime(true) + firstSetTime(true), + graphRecording(false) { // Formerly a constant (thus the CAPS). nVidia is 3% CPU load at 400 or 500, and 35% load at 800 on my Linux machine. // AMD is just over 2% load at 1600 and 3200 on the same Linux machine. Marginally better timings(?) at 3200. @@ -38,11 +39,11 @@ void Queue::writeTE(cl_mem buf, u64 size, const void* data, TimeInfo* tInfo) { events.synced(); } -void Queue::fillBufTE(cl_mem buf, u32 patSize, const void* pattern, u64 size, TimeInfo* tInfo) { +void Queue::fillBufTE(cl_mem buf, size_t patSize, const void* pattern, size_t size, TimeInfo* tInfo) { add(::fillBuf(get(), {}, buf, pattern, patSize, size, hasEvents), tInfo); } -string status(Events& events) { +static string status(Events& events) { if (events.empty()) { return ""; } Event& f = events.front(); return f.isComplete() ? "C" : f.isQueued() ? "Q" : f.isRunning() ? "R" : f.isSubmitted() ? "S" : "?"; @@ -55,35 +56,39 @@ void Queue::print() { void Queue::add(EventHolder&& e, TimeInfo* ti) { if (hasEvents) { events.emplace_back(std::move(e), ti); } + if (isAuxQueue || graphRecording) return; queueCount++; - if (queueCount == MAX_QUEUE_COUNT) queueMarkerEvent(); + if (queueCount >= MAX_QUEUE_COUNT) queueMarkerEvent(); } -void Queue::readSync(cl_mem buf, u32 size, void* out, TimeInfo* tInfo) { +void Queue::readSync(cl_mem buf, size_t size, void* out, TimeInfo* tInfo) { add(read(get(), {}, true, buf, size, out, hasEvents), tInfo); events.synced(); } -void Queue::readAsync(cl_mem buf, u32 size, void* out, TimeInfo* tInfo) { +void Queue::readAsync(cl_mem buf, size_t size, void* out, TimeInfo* tInfo) { add(read(get(), {}, false, buf, size, out, hasEvents), tInfo); } -void Queue::copyBuf(cl_mem src, cl_mem dst, u32 size, TimeInfo* tInfo) { +void Queue::copyBuf(cl_mem src, cl_mem dst, size_t size, TimeInfo* tInfo) { add(::copyBuf(get(), {}, src, dst, size, hasEvents), tInfo); } -void Queue::run(cl_kernel kernel, size_t groupSize, size_t workSize, TimeInfo* tInfo) { - add(::run(get(), kernel, groupSize, workSize, {}, tInfo->name, hasEvents), tInfo); +void Queue::run(cl_kernel kernel, size_t groupSizeX, size_t workSizeX, size_t workSizeY, TimeInfo* tInfo) { + add(::run(get(), kernel, groupSizeX, workSizeX, workSizeY, {}, tInfo->name, hasEvents), tInfo); } void Queue::finish() { + assert(!isAuxQueue); waitForMarkerEvent(); ::finish(get()); events.synced(); queueCount = 0; + squareCount = 0; } void Queue::queueMarkerEvent() { + assert(!isAuxQueue); waitForMarkerEvent(); if (queueCount) { // AMD GPUs have no trouble waiting for a finish without a CPU busy wait. So, instead of markers and events, simply run finish every now and then. @@ -95,27 +100,31 @@ void Queue::queueMarkerEvent() { markerEvent = enqueueMarker(get()); markerQueued = true; queueCount = 0; + squareCount = 0; } } } void Queue::waitForMarkerEvent() { + assert(!isAuxQueue); if (!markerQueued) return; // By default, nVidia finish causes a CPU busy wait. Instead, sleep for a while. Since we know how many items are enqueued after the marker we can make an // educated guess of how long to sleep to keep CPU overhead low. while (getEventInfo(markerEvent.get()) != CL_COMPLETE) { - // There are 4, 7, or 10 kernels per squaring. Don't overestimate sleep time. Divide by much more than the number of kernels. - std::this_thread::sleep_for(std::chrono::microseconds(1 + queueCount * squareTime / squareKernels / 2)); + // There are usually 4, 7, or 10 kernels per squaring. Use a rolling average to create a very accurate kernel count. + // Don't overestimate sleep time. Divide by much more than the number of kernels. + std::this_thread::sleep_for(std::chrono::microseconds((squareCount + 1) * squareTime / 2)); } markerQueued = false; } void Queue::setSquareTime(int time) { + assert(!isAuxQueue); if (firstSetTime) { // Ignore first setSquareTime call. First measured times are wrong because of startup costs firstSetTime = false; return; } - if (time < 30) time = 30; // Assume a minimum square time of 30us - if (time > 3000) time = 3000; // Assume a maximum square time of 3000us + time = std::max(time, 30); // Assume a minimum square time of 30us + time = std::min(time, 3000); // Assume a maximum square time of 3000us squareTime = time; } diff --git a/src/Queue.h b/src/Queue.h index be8341d6..65643251 100644 --- a/src/Queue.h +++ b/src/Queue.h @@ -23,9 +23,10 @@ class Events : public std::deque { class Queue : public QueueHolder { Events events; bool hasEvents; + bool isAuxQueue; void writeTE(cl_mem buf, u64 size, const void* data, TimeInfo *tInfo); - void fillBufTE(cl_mem buf, u32 patSize, const void* pattern, u64 size, TimeInfo* tInfo); + void fillBufTE(cl_mem buf, size_t patSize, const void* pattern, size_t size, TimeInfo* tInfo); void flush(); void print(); void add(EventHolder &&e, TimeInfo* ti); @@ -33,7 +34,7 @@ class Queue : public QueueHolder { public: const Context* context; - Queue(const Context& context, bool profile); + Queue(const Context& context, bool profile, bool auxQueue = false); static int registerThread(); static int tid(); @@ -42,25 +43,54 @@ class Queue : public QueueHolder { void write(cl_mem buf, const vector& v, TimeInfo* tInfo) { writeTE(buf, v.size() * sizeof(T), v.data(), tInfo); } template - void fillBuf(cl_mem buf, T pattern, u32 size, TimeInfo* tInfo) { fillBufTE(buf, sizeof(T), &pattern, size, tInfo); } + void fillBuf(cl_mem buf, T pattern, size_t size, TimeInfo* tInfo) { fillBufTE(buf, sizeof(T), &pattern, size, tInfo); } - void run(cl_kernel kernel, size_t groupSize, size_t workSize, TimeInfo* tInfo); - void readSync(cl_mem buf, u32 size, void* out, TimeInfo* tInfo); - void readAsync(cl_mem buf, u32 size, void* out, TimeInfo* tInfo); - void copyBuf(cl_mem src, cl_mem dst, u32 size, TimeInfo* tInfo); + void run(cl_kernel kernel, size_t groupSizeX, size_t workSizeX, size_t workSizeY, TimeInfo* tInfo); + void readSync(cl_mem buf, size_t size, void* out, TimeInfo* tInfo); + void readAsync(cl_mem buf, size_t size, void* out, TimeInfo* tInfo); + void copyBuf(cl_mem src, cl_mem dst, size_t size, TimeInfo* tInfo); void finish(); + EventHolder createSyncEvent() { if (!isAuxQueue && !graphRecording) queueCount++; return enqueueMarker(get()); } // Enqueue a synchronization event. Used to sync work among multiple queues. + void waitForSyncEvent(EventHolder* e) { if (!isAuxQueue && !graphRecording) queueCount++; enqueueMarkerWithWaits(get(), {e->get()}); } // Wait for a synchronization event to complete. + + void incSquareCount(int n = 1) { squareCount += n; } void setSquareTime(int); // Update the time to do one squaring (in microseconds) - void setSquareKernels(int n) { squareKernels = n; firstSetTime = true; } + + void beginRecording(void) { graphRecording = true; CHECK1(clGraphBeginRecording(get())); } + void endRecording(cl_graph *graph) { graphRecording = false; CHECK1(clGraphEndRecording(get(), graph)); } + void playRecording(cl_graph graph) { CHECK1(clGraphLaunch(graph)); add(EventHolder{}, NULL); } private: // This replaces the "call queue->finish every 400 squarings" code in Gpu.cpp. Solves the busy wait on nVidia GPUs. int MAX_QUEUE_COUNT; // Queue size before a marker will be enqueued. Typically, 100 to 1000 squarings. EventHolder markerEvent; // Event associated with an enqueued marker placed in the queue every MAX_QUEUE_COUNT entries and before r/w operations. - bool markerQueued; // TRUE if a marker and event have been queued - int queueCount; // Count of items added to the queue since last marker - int squareTime; // Time to do one squaring (in microseconds) - int squareKernels; // Number of kernels in one squaring - bool firstSetTime; // Flag so we can ignore first setSquareTime call (which is inaccurate because of all the initial openCL compiles) + bool markerQueued{false}; // TRUE if a marker and event have been queued + int queueCount{0}; // Count of items added to the queue since last marker + int squareCount{0}; // Count of squarings/multiplies since last marker queued + int squareTime{50}; // Time to do one squaring (in microseconds) + bool firstSetTime{true}; // Flag so we can ignore first setSquareTime call (which is inaccurate because of all the initial openCL compiles) + bool graphRecording{false}; // Graph recording in progress. waitForMarkerEvent and enqueueMarker must be avoided. void queueMarkerEvent(); // Queue the marker event void waitForMarkerEvent(); // Wait for marker event to complete }; + + + +// Wrapper class for our OpenCL-like extensions invented to provide a clean interface to some nVidia CUDA graphs feature + +class Graph { + +public: + Graph() : graph{} {} + ~Graph() { if (graph) release(graph); } + + bool isSupported(cl_device_id id) { return clIsGraphSupported(id); } + void beginRecording(Queue *q) { q->beginRecording(); } + void endRecording(Queue *q) { q->endRecording(&graph); } + bool isRecorded() { return graph != NULL; } + void launch(Queue *q) { q->playRecording(graph); } + +private: + cl_graph graph; +}; + diff --git a/src/Saver.cpp b/src/Saver.cpp index 118d1eae..72926a3b 100644 --- a/src/Saver.cpp +++ b/src/Saver.cpp @@ -11,41 +11,54 @@ #include #include #include +#include namespace { // E, k, block-size, res64, nErrors, CRC -static constexpr const char *PRP_v12 = "OWL PRP 12 %u %u %u %016" SCNx64 " %u %u\n"; +static constexpr const char *PRP_v12 = "OWL PRP 12 %" PRIu64 " %" PRIu64 " %u %016" SCNx64 " %u %u\n"; // Anticipated next version of the header. // Has general number form N=k*b^E+c, and labels for values. -static constexpr const char *PRP_v13 = "OWL PRP 13 N=1*2^%u-1 k=%u block=%u res64=%016" SCNx64 " err=%u time=%lf\n"; +static constexpr const char *PRP_v13 = "OWL PRP 13 N=1*2^%" PRIu64 "-1 k=%" PRIu64 " block=%u res64=%016" SCNx64 " err=%u time=%lf\n"; // static constexpr const char *PRP_v13_PRI = "OWL PRP 13 N=1*2^%u-1 k=%u block=%u res64=%016" PRIx64 " err=%u time=%.0lf\n"; // E, k, CRC -static constexpr const char *LL_v1 = "OWL LL 1 E=%u k=%u CRC=%u\n"; +static constexpr const char *LL_v1 = "OWL LL 1 E=%" PRIu64 " k=%" PRIu64 " CRC=%u\n"; // Anticipated next version. // Push version number to sync it with PRP. -static constexpr const char *LL_v13 = "OWL LL 13 N=1*2^%u-1 k=%u time=%lf\n"; - -struct BadHeaderError { string name; }; +static constexpr const char *LL_v13 = "OWL LL 13 N=1*2^%" PRIu64 "-1 k=%" PRIu64 " time=%lf\n"; +static constexpr const char *CERT_v1 = "OWL CERT 1 N=1*2^%" PRIu64 "-1 k=%" PRIu64 " squarings=%" PRIu64 " time=%lf\n"; + +struct BadHeaderError : std::runtime_error { + string name; + explicit BadHeaderError(string n) : std::runtime_error("bad savefile header: " + n), name(std::move(n)) {} +}; + +// The header's exponent sizes the residue read that follows; a corrupt header must not be allowed to size it. +void checkExponent(u64 got, u64 want, const string& header) { + if (got != want) { + log("savefile header exponent %" PRIu64 " does not match %" PRIu64 ": \"%s\"\n", got, want, rstripNewline(header).c_str()); + throw BadHeaderError{header}; + } +} bool startsWith(const string& s, const string& prefix) { - return s.rfind(prefix, 0) == 0; + return s.starts_with(prefix); } -vector savefiles(fs::path dir, const string& prefix, const string& kind) { - vector v; +vector savefiles(const fs::path& dir, const string& prefix, const string& kind) { + vector v; for (const auto& entry: fs::directory_iterator(dir)) { if (entry.is_regular_file()) { - string filename = entry.path().filename().string(); + string const filename = entry.path().filename().string(); auto dot = filename.find('.'); if (dot != string::npos && startsWith(filename, prefix) && filename.substr(dot + 1) == kind) { assert(dot > prefix.size()); string id = filename.substr(prefix.size(), dot - prefix.size()); if (id == "unverified") { continue; } - u32 k = 0; + u64 k = 0; const char* first = id.data(); const char* end = first + id.size(); auto res = from_chars(first, end, k); @@ -57,69 +70,74 @@ vector savefiles(fs::path dir, const string& prefix, const string& kind) { } } } - std::sort(v.begin(), v.end()); + std::ranges::sort(v); return v; } -string str9(u32 k) { +string str9(u64 k) { char buf[32]; - snprintf(buf, sizeof(buf), "%09u", k); + snprintf(buf, sizeof(buf), "%09" PRIu64, k); return buf; } -fs::path pathFor(fs::path base, const string& prefix, const string& kind, u32 k) { +fs::path pathFor(const fs::path& base, const string& prefix, const string& kind, u64 k) { return base / (prefix + str9(k) + '.' + kind); } -fs::path pathUnverified(fs::path base, const string& prefix) { +fs::path pathUnverified(const fs::path& base, const string& prefix) { return base / (prefix + "unverified.prp"); } // find the "most advanced" file in dir with a name of the form // . // e.g.: 125784077-010000000.prp -fs::path findLast(fs::path dir, const string& prefix, const string& kind) { - vector v = savefiles(dir, prefix, kind); +fs::path findLast(const fs::path& dir, const string& prefix, const string& kind) { + vector v = savefiles(dir, prefix, kind); if (v.empty()) { return {}; } - u32 lastK = v.back(); + u64 const lastK = v.back(); fs::path path = pathFor(dir, prefix, kind, lastK); assert(is_regular_file(path)); return path; } -PRPState readState(const PRPState& dummy, File fi) { - u32 exponent{}, k{}, blockSize{}, nErrors{}; +PRPState readState(const PRPState& expected, File fi) { + u64 exponent{}, k{}; + u32 blockSize{}, nErrors{}; u64 res64{}; double elapsed{}; - string header = fi.readLine(); + string const header = fi.readLine(); if (sscanf(header.c_str(), PRP_v13, &exponent, &k, &blockSize, &res64, &nErrors, &elapsed) == 6) { - return {exponent, k, blockSize, res64, fi.readChecked(nWords(exponent)), nErrors, elapsed}; + checkExponent(exponent, expected.exponent, header); + return {.exponent=exponent, .k=k, .blockSize=blockSize, .res64=res64, .check=fi.readChecked(nWords(exponent)), .nErrors=nErrors, .elapsed=elapsed}; } u32 crc{}; if (sscanf(header.c_str(), PRP_v12, &exponent, &k, &blockSize, &res64, &nErrors, &crc) == 6) { - return {exponent, k, blockSize, res64, fi.readWithCRC(nWords(exponent), crc), nErrors, 0}; + checkExponent(exponent, expected.exponent, header); + return {.exponent=exponent, .k=k, .blockSize=blockSize, .res64=res64, .check=fi.readWithCRC(nWords(exponent), crc), .nErrors=nErrors, .elapsed=0}; } log("Loading PRP from '%s': bad header '%s'\n", fi.name.c_str(), header.c_str()); throw BadHeaderError{fi.name}; } -LLState readState(const LLState& dummy, File fi) { - u32 exponent{}, k{}; +LLState readState(const LLState& expected, File fi) { + u64 exponent{}, k{}; double elapsed{}; - string header = fi.readLine(); + string const header = fi.readLine(); if (sscanf(header.c_str(), LL_v13, &exponent, &k, &elapsed) == 3) { - return {exponent, k, fi.readChecked(nWords(exponent)), elapsed}; + checkExponent(exponent, expected.exponent, header); + return {.exponent=exponent, .k=k, .data=fi.readChecked(nWords(exponent)), .elapsed=elapsed}; } u32 crc{}; if (sscanf(header.c_str(), LL_v1, &exponent, &k, &crc) == 3) { - return {exponent, k, fi.readWithCRC(nWords(exponent), crc), 0}; + checkExponent(exponent, expected.exponent, header); + return {.exponent=exponent, .k=k, .data=fi.readWithCRC(nWords(exponent), crc), .elapsed=0}; } log("Loading LL from '%s': bad header '%s'\n", fi.name.c_str(), header.c_str()); @@ -134,6 +152,28 @@ void writeState(const File& fo, const PRPState& state) { fo.writeChecked(state.check); } +CERTState readState([[maybe_unused]] const CERTState& dummy, File fi) { + u64 exponent{}, k{}, squarings{}; + double elapsed{}; + + string const header = fi.readLine(); + + if (sscanf(header.c_str(), CERT_v1, &exponent, &k, &squarings, &elapsed) == 4) { + return {.exponent=exponent, .k=k, .squarings=squarings, .data=fi.readChecked(nWords(exponent)), .elapsed=elapsed}; + } + + log("Loading CERT from '%s': bad header '%s'\n", fi.name.c_str(), header.c_str()); + throw BadHeaderError{fi.name}; +} + +void writeState(const File& fo, const CERTState& state) { + assert(state.data.size() == nWords(state.exponent)); + if (fo.printf(CERT_v1, state.exponent, state.k, state.squarings, state.elapsed) <= 0) { + throw WriteError{fo.name}; + } + fo.writeChecked(state.data); +} + void writeState(const File& fo, const LLState& state) { assert(state.data.size() == nWords(state.exponent)); if (fo.printf(LL_v13, state.exponent, state.k, state.elapsed) <= 0) { @@ -142,7 +182,7 @@ void writeState(const File& fo, const LLState& state) { fo.writeChecked(state.data); } -double roundNumberScore(u32 x) { +double roundNumberScore(u64 x) { if (x == 0) { return 1; } double score = 0; @@ -158,18 +198,22 @@ double roundNumberScore(u32 x) { } // namespace template<> PRPState Saver::initState() { - return {exponent, 0, blockSize, 3, makeWords(exponent, 1), 0, 0}; + return {.exponent=exponent, .k=0, .blockSize=blockSize, .res64=3, .check=makeWords(exponent, 1), .nErrors=0, .elapsed=0}; } template<> LLState Saver::initState() { - return {exponent, 0, makeWords(exponent, 4), 0}; + return {.exponent=exponent, .k=0, .data=makeWords(exponent, 4), .elapsed=0}; +} + +template<> CERTState Saver::initState() { + return {.exponent=exponent, .k=0, .squarings=0, .data={}, .elapsed=0}; // no checkpoint: caller starts from the .cert file } // ---- Saver ---- template -Saver::Saver(u32 exponent, u32 blockSize, u32 nSavefiles) : +Saver::Saver(u64 exponent, u32 blockSize, u32 nSavefiles) : exponent{exponent}, blockSize{blockSize}, prefix{to_string(exponent) + '-'}, @@ -188,16 +232,16 @@ template Saver::~Saver() = default; template -void Saver::clear(u32 exponent) { +void Saver::clear(u64 exponent) { error_code dummy; - fs::path base = std::is_same_v ? + fs::path const base = std::is_same_v ? fs::current_path() / to_string(exponent) : fs::current_path() / (string(State::KIND) + '-' + to_string(exponent)); fs::remove_all(base, dummy); } template -void Saver::moveToTrash(fs::path src) { +void Saver::moveToTrash(const fs::path& src) { log("Removing bad savefile '%s'\n", src.string().c_str()); fancyRename(src, src + ".bad"s); } @@ -215,7 +259,7 @@ fs::path Saver::mostRecentSavefile() { template State Saver::load() { for (int i = 0; i < 2; ++i) { - fs::path path = mostRecentSavefile(); + fs::path const path = mostRecentSavefile(); if (path.empty()) { // no savefiles at all @@ -224,7 +268,9 @@ State Saver::load() { if (File fi{File::openRead(path)}; fi) { try { - State state = readState(State{}, std::move(fi)); + State expected{}; + expected.exponent = exponent; + State state = readState(expected, std::move(fi)); assert(state.exponent == exponent); if (state.exponent == exponent) { return state; @@ -244,18 +290,18 @@ State Saver::load() { template void Saver::trimFiles() { - vector v = savefiles(base, prefix, State::KIND); + vector v = savefiles(base, prefix, State::KIND); assert(nSavefiles > 0); while (v.size() > nSavefiles) { int bestIdx = -1; double bestSpan = 1e20; - u32 prevK = 0; + u64 prevK = 0; for (u32 i = 0; i < v.size() - 1; ++i) { - u32 k = v[i]; - double niceBias = std::min(1.0, roundNumberScore(k) - 4); - double span = (v[i + 1] - prevK) * niceBias; + u64 const k = v[i]; + double const niceBias = std::min(1.0, roundNumberScore(k) - 4); + double const span = (v[i + 1] - prevK) * niceBias; prevK = k; if (span < bestSpan) { bestSpan = span; @@ -263,9 +309,9 @@ void Saver::trimFiles() { } } assert(bestIdx >= 0); - u32 k = v[bestIdx]; - // log("Deleting savefile %u\n", k); - fs::path path = pathFor(base, prefix, State::KIND, k); + u64 const k = v[bestIdx]; + // log("Deleting savefile %" PRIu64 "\n", k); + fs::path const path = pathFor(base, prefix, State::KIND, k); fs::remove(path); v.erase(v.begin() + bestIdx); } @@ -273,7 +319,7 @@ void Saver::trimFiles() { template void Saver::save(const State& state) { - fs::path path = pathFor(base, to_string(exponent) + '-', State::KIND, state.k); + fs::path const path = pathFor(base, to_string(exponent) + '-', State::KIND, state.k); ::writeState(*CycleFile{path}, state); trimFiles(); // log("rm '%s'\n", pathUnverified(base, prefix).string().c_str()); @@ -287,7 +333,7 @@ void Saver::saveUnverified(const PRPState& state) const { template void Saver::dropMostRecent() { - fs::path path = mostRecentSavefile(); + fs::path const path = mostRecentSavefile(); assert(!path.empty()); if (!path.empty()) { moveToTrash(path); } } @@ -295,3 +341,4 @@ void Saver::dropMostRecent() { template class Saver; template class Saver; +template class Saver; diff --git a/src/Saver.h b/src/Saver.h index 3bf8e6e7..359552fc 100644 --- a/src/Saver.h +++ b/src/Saver.h @@ -13,39 +13,51 @@ class SaveMan; struct PRPState { static const constexpr char* KIND = "prp"; - u32 exponent; - u32 k; + u64 exponent; + u64 k; u32 blockSize; u64 res64; vector check; u32 nErrors; - double elapsed; + double elapsed{}; }; struct LLState { static const constexpr char* KIND = "ll"; - u32 exponent; - u32 k; + u64 exponent; + u64 k; + vector data; + double elapsed{}; +}; + +// A CERT (certification) in progress: the current residue after k of `squarings` squarings. An empty `data` +// means "no checkpoint": start from the M.cert file. +struct CERTState { + static const constexpr char* KIND = "cert"; + + u64 exponent; + u64 k; + u64 squarings; vector data; double elapsed{}; }; template class Saver { - u32 exponent; + u64 exponent; u32 blockSize; fs::path base; string prefix; u32 nSavefiles; State initState(); - void moveToTrash(fs::path file); + void moveToTrash(const fs::path& file); void trimFiles(); fs::path mostRecentSavefile(); public: - Saver(u32 exponent, u32 blockSize, u32 nSavefiles); + Saver(u64 exponent, u32 blockSize, u32 nSavefiles); ~Saver(); State load(); @@ -53,7 +65,7 @@ class Saver { void dropMostRecent(); - static void clear(u32 exponent); + static void clear(u64 exponent); // For PRP, we can save a verified save (see save() above) or an unverified save. void saveUnverified(const PRPState& s) const; diff --git a/src/Sha3Hash.h b/src/Sha3Hash.h index f47fc74b..f02b7707 100644 --- a/src/Sha3Hash.h +++ b/src/Sha3Hash.h @@ -18,7 +18,7 @@ class Sha3Hash { void update(const void* data, u32 size) { SHA3Update(&context, reinterpret_cast(data), size); } array finish() && { - u64 *p = reinterpret_cast(SHA3Final(&context)); + u64 const*p = reinterpret_cast(SHA3Final(&context)); return {p[0], p[1], p[2], p[3]}; } }; diff --git a/src/Signal.cpp b/src/Signal.cpp index 77192bd2..f18b532b 100644 --- a/src/Signal.cpp +++ b/src/Signal.cpp @@ -1,32 +1,60 @@ // Copyright (C) Mihai Preda. #include "Signal.h" +#include "log.h" #include +#include +#include using namespace std; static volatile sig_atomic_t signalled = 0; -static void (* volatile oldHandler)(int) = 0; +static void (* volatile oldIntHandler)(int) = nullptr; +static void (* volatile oldTermHandler)(int) = nullptr; static void signalHandler(int signal) { signalled = signal; } +// A file named "stop" in the run directory asks for the same graceful stop +// a SIGINT does: finish the block, verify it, write the savefile, exit. +// It is the stop a launcher can request where no signal reaches the +// process — a hidden Windows child has no console to deliver a Ctrl-C to, +// and TerminateProcess forfeits the work since the last savefile. The file +// is removed once seen, so the next start is not a stop. +static const char *STOP_FILE = "stop"; + +static bool stopFileSeen() { + error_code ec; + if (!filesystem::exists(STOP_FILE, ec) || ec) { return false; } + filesystem::remove(STOP_FILE, ec); + log("Stop requested by the '%s' file\n", STOP_FILE); + return true; +} + Signal::Signal() { - if (!oldHandler) { - oldHandler = signal(SIGINT, signalHandler); + if (!oldIntHandler) { + oldIntHandler = signal(SIGINT, signalHandler); + // SIGTERM is what service managers and supervisors send first; unhandled + // it ends the process at once. + oldTermHandler = signal(SIGTERM, signalHandler); isOwner = true; } } Signal::~Signal() { release(); } -unsigned Signal::stopRequested() { return signalled; } +unsigned Signal::stopRequested() { + if (!signalled && stopFileSeen()) { signalled = SIGTERM; } + return signalled; +} void Signal::release() { if (isOwner) { isOwner = false; - signal(SIGINT, oldHandler); - oldHandler = 0; + signal(SIGINT, oldIntHandler); + signal(SIGTERM, oldTermHandler); + oldIntHandler = nullptr; + oldTermHandler = nullptr; } } diff --git a/src/Task.cpp b/src/Task.cpp index d72f4db2..8c1726e6 100644 --- a/src/Task.cpp +++ b/src/Task.cpp @@ -15,6 +15,8 @@ #include #include +#include +#include namespace { @@ -63,9 +65,9 @@ struct OsInfo { }; [[maybe_unused]] OsInfo getOsInfoMinimum() { - int plat = platform(); - string os = plat == LINUX_64 || plat == LINUX_32 ? "Linux" : plat == 4 ? "Windows" : plat == MACOSX_64 ? "MacOS" : ""; - return {os, "", ""}; + int const plat = platform(); + string const os = plat == LINUX_64 || plat == LINUX_32 ? "Linux" : plat == 4 ? "Windows" : plat == MACOSX_64 ? "MacOS" : ""; + return {.os=os, .release="", .arch=""}; } #if __has_include() @@ -75,7 +77,7 @@ struct OsInfo { OsInfo getOsInfo() { utsname buf{}; uname(&buf); - return OsInfo{buf.sysname, buf.release, buf.machine}; + return OsInfo{.os=buf.sysname, .release=buf.release, .arch=buf.machine}; } #else @@ -102,8 +104,20 @@ string json(const vector& v) { return {isFirst ? ""s : (s + '}')}; } -string json(const string& s) { return '"' + s + '"'; } +// JSON string literal: escape the quote, the backslash and control characters, so a user name or OS string +// containing them cannot produce an unparseable results line. +string json(const string& s) { + string out = "\""; + for (unsigned char c : s) { + if (c == '"' || c == '\\') { out += '\\'; out += char(c); } + else if (c < 0x20) { char buf[8]; snprintf(buf, sizeof(buf), "\\u%04x", c); out += buf; } + else { out += char(c); } + } + return out + '"'; +} +string json(int x) { return to_string(x); } string json(u32 x) { return to_string(x); } +string json(u64 x) { return to_string(x); } template string json(const string& key, const T& value) { return json(key) + ':' + json(value); } @@ -112,7 +126,7 @@ string maybe(const string& key, const string& value) { return value.empty() ? "" template void operator+=(vector& a, const vector& b) { a.insert(a.end(), b.begin(), b.end()); } -vector commonFields(u32 E, const char *worktype, const string &status) { +vector commonFields(u64 E, const char *worktype, const string &status) { return { json("status", status), json("exponent", E), @@ -122,7 +136,7 @@ vector commonFields(u32 E, const char *worktype, const string &status) { vector tailFields(const std::string &AID, const Args &args) { assert(*VERSION); // version string isn't empty - OsInfo os = getOsInfo(); + OsInfo const os = getOsInfo(); return {json("program", vector{ json("name", "prpll"), json("version", (VERSION[0] == 'v') ? VERSION + 1 : VERSION), // skip leading "v" from version @@ -140,13 +154,13 @@ vector tailFields(const std::string &AID, const Args &args) { }; } -void writeResult(u32 instance, u32 E, const char *workType, const string &status, const std::string &AID, const Args &args, +void writeResult(u32 instance, u64 E, const char *workType, const string &status, const std::string &AID, const Args &args, const vector& extras) { - fs::path resultsFile = "results-" + to_string(instance) + ".txt"; + fs::path const resultsFile = "results-" + to_string(instance) + ".txt"; vector fields = commonFields(E, workType, status); fields += extras; fields += tailFields(AID, args); - string s = json(std::move(fields)); + string const s = json(fields); log("%s\n", s.c_str()); File::append(resultsFile, s + '\n'); } @@ -164,7 +178,7 @@ void Task::writeResultPRP(FFTConfig fft, const Args &args, u32 instance, bool is // "proof":{"version":1, "power":6, "hashsize":64, "md5":"0123456789ABCDEF"}, if (!proofPath.empty()) { - ProofInfo info = proof::getInfo(proofPath); + ProofInfo const info = proof::getInfo(proofPath); if (info.power > 0) { fields.push_back(json("proof", vector{ json("version", 1), @@ -179,7 +193,7 @@ void Task::writeResultPRP(FFTConfig fft, const Args &args, u32 instance, bool is } void Task::writeResultLL(FFTConfig fft, const Args &args, u32 instance, bool isPrime, u64 res64) const { - vector fields{json("res64", hex(res64)), + vector const fields{json("res64", hex(res64)), json("fft-type", ffttype(fft)), json("fft-length", fft.size()), json("shift-count", 0), @@ -190,7 +204,7 @@ void Task::writeResultLL(FFTConfig fft, const Args &args, u32 instance, bool isP } void Task::writeResultCERT(FFTConfig fft, const Args &args, u32 instance, array hash, u32 squarings) const { - string hexhash = hex(hash[3]) + hex(hash[2]) + hex(hash[1]) + hex(hash[0]); + string const hexhash = hex(hash[3]) + hex(hash[2]) + hex(hash[1]) + hex(hash[0]); vector fields{json("worktype", "Cert"), json("exponent", exponent), json("sha3-hash", hexhash.c_str()), @@ -201,42 +215,46 @@ void Task::writeResultCERT(FFTConfig fft, const Args &args, u32 instance, array json("error-code", "00000000"), // I don't know the meaning of this }; fields += tailFields(AID, args); - string s = json(std::move(fields)); + string const s = json(fields); log("%s\n", s.c_str()); - fs::path resultsFile = "results-" + to_string(instance) + ".txt"; + fs::path const resultsFile = "results-" + to_string(instance) + ".txt"; File::append(resultsFile, s + '\n'); } -void Task::execute(GpuCommon shared, Queue *q, u32 instance) { +void Task::execute(GpuCommon shared, u32 instance) { if (kind == VERIFY) { exponent = proof::getInfo(verifyPath).exp; } assert(exponent); // Testing exponent 140000001 using FFT 512:15:512 fails with severe round off errors. - // I'm guessing this is because bot the exponent and FFT size are divisible by 3. + // I'm guessing this is because both the exponent and FFT size are divisible by 3. // Here we make sure the exponent is prime. If not we do not raise an error because it // is very common to use command line argument "-prp some-random-exponent" to get a quick // timing. Instead, we output a warning and test a smaller prime exponent. { - Primes primes; + Primes const primes; if (!primes.isPrime(exponent)) { - u32 new_exponent = primes.prevPrime(exponent); - log("Warning: Exponent %u is not prime. Using exponent %u instead.\n", exponent, new_exponent); + u64 const new_exponent = primes.prevPrime(exponent); + log("Warning: Exponent %" PRIu64 " is not prime. Using exponent %" PRIu64 " instead.\n", exponent, new_exponent); exponent = new_exponent; } } - LogContext pushContext(std::to_string(exponent)); + LogContext const pushContext(std::to_string(exponent)); - FFTConfig fft = FFTConfig::bestFit(*shared.args, exponent, shared.args->fftSpec); + FFTConfig const fft = FFTConfig::bestFit(*shared.args, exponent, shared.args->fftSpec); - auto gpu = Gpu::make(q, exponent, shared, fft); + auto gpu = Gpu::make(exponent, shared, fft); if (kind == VERIFY) { - Proof proof{Proof::load(verifyPath)}; + Proof const proof{Proof::load(verifyPath)}; assert(proof.E == exponent); - bool ok = proof.verify(gpu.get()); + bool const ok = proof.verify(gpu.get()); log("proof '%s' %s\n", verifyPath.c_str(), ok ? "verified" : "failed"); + // -verify is a one-shot job (VERIFY tasks come only from the command line), so a proof that does not + // check out is a failed run and has to be visible as one: without this the process exits 0 and a + // caller cannot tell a good proof from a bad one except by reading the log. + if (!ok) { throw "proof verification failed"; } } else if (kind == PRP || kind == LL) { bool isPrime; @@ -253,7 +271,7 @@ void Task::execute(GpuCommon shared, Queue *q, u32 instance) { Worktodo::deleteTask(*this, instance); if (isPrime) { - log("%u is PRIME!\n", exponent); + log("%" PRIu64 " is PRIME!\n", exponent); } else if (shared.args->clean) { gpu->clear(kind == PRP); } diff --git a/src/Task.h b/src/Task.h index 95f08024..945172a5 100644 --- a/src/Task.h +++ b/src/Task.h @@ -12,7 +12,6 @@ class Args; class Result; class Context; -class Queue; class TrigBufCache; class Task { @@ -20,13 +19,13 @@ class Task { enum Kind {PRP, VERIFY, LL, CERT}; Kind kind; - u32 exponent; - string AID; // Assignment ID - string line; // the verbatim worktodo line, used in deleteTask(). - u32 squarings; // For CERTs + u64 exponent{}; + string AID{}; // Assignment ID + string line{}; // the verbatim worktodo line, used in deleteTask(). + u32 squarings{}; // For CERTs + string verifyPath{}; // For Verify - string verifyPath; // For Verify - void execute(GpuCommon shared, Queue* q, u32 instance); + void execute(GpuCommon shared, u32 instance); void writeResultPRP(FFTConfig fft, const Args&, u32 instance, bool isPrime, u64 res64, const std::string& res2048, u32 nErrors, const fs::path& proofPath) const; void writeResultLL(FFTConfig fft, const Args&, u32 instance, bool isPrime, u64 res64) const; diff --git a/src/TimeInfo.h b/src/TimeInfo.h index 872d1945..b99ad4b1 100644 --- a/src/TimeInfo.h +++ b/src/TimeInfo.h @@ -29,7 +29,7 @@ class TimeInfo { bool operator<(const TimeInfo& rhs) const { return times[2] > rhs.times[2]; } - auto secs() const { + [[nodiscard]] auto secs() const { std::array ret{}; for (int i = 0; i < 3; ++i) { ret[i] = times[i] * 1e-9; } return ret; diff --git a/src/Trig.cpp b/src/Trig.cpp index 917d748f..1359d5c4 100644 --- a/src/Trig.cpp +++ b/src/Trig.cpp @@ -80,10 +80,10 @@ TrigCoefs trigCoefs(u32 n) { assert(mid % 2 == 1); assert(mid <= 15 || (mid % 625 == 0 && mid / 625 <= 13)); - double scale = 1.0 / (twos / 4); + double const scale = 1.0 / (twos / 4); for (u32 i = 0; i < MUL_TAB.size(); ++i) { if (MUL_TAB[i] % mid == 0) { - return {MUL_TAB[i] / mid, scaleSin(SIN[i], scale), scaleCos(COS[i], scale)}; + return {.scale=MUL_TAB[i] / mid, .sinCoefs=scaleSin(SIN[i], scale), .cosCoefs=scaleCos(COS[i], scale)}; } } log("Trig tab not found for %u (%u * %u)\n", n, mid, twos); diff --git a/src/TrigBufCache.cpp b/src/TrigBufCache.cpp index 03c8e1e6..85586786 100644 --- a/src/TrigBufCache.cpp +++ b/src/TrigBufCache.cpp @@ -3,8 +3,10 @@ #include #include "TrigBufCache.h" -#define SAVE_ONE_MORE_WIDTH_MUL 0 // I want to make saving the only option -- but rocm optimizer is inexplicably making it slower in carryfused -#define SAVE_ONE_MORE_HEIGHT_MUL 1 // In tailSquare this is the fastest option +enum { +SAVE_ONE_MORE_WIDTH_MUL = 0, // I want to make saving the only option -- but rocm optimizer is inexplicably making it slower in carryfused +SAVE_ONE_MORE_HEIGHT_MUL = 1 // In tailSquare this is the fastest option +}; #define _USE_MATH_DEFINES #include @@ -25,7 +27,7 @@ double2 root1Fancy(u32 N, u32 k) { assert(k < N); assert(k < N/4); - long double angle = M_PIl * k / (N / 2); + long double const angle = M_PIl * k / (N / 2); return {double(cosl(angle) - 1), double(sinl(angle))}; } @@ -34,15 +36,15 @@ static double trigError(double c, double s) { return abs(trigNorm(c, s) - 1.0); // Round trig long double to double as to satisfy c^2 + s^2 == 1 as best as possible static double2 roundTrig(long double lc, long double ls) { - double c1 = lc; - double c2 = nexttoward(c1, lc); - double s1 = ls; - double s2 = nexttoward(s1, ls); + double const c1 = lc; + double const c2 = nexttoward(c1, lc); + double const s1 = ls; + double const s2 = nexttoward(s1, ls); double c = c1; double s = s1; - for (double tryC : {c1, c2}) { - for (double tryS : {s1, s2}) { + for (double const tryC : {c1, c2}) { + for (double const tryS : {s1, s2}) { if (trigError(tryC, tryS) < trigError(c, s)) { c = tryC; s = tryS; @@ -58,13 +60,13 @@ double2 root1(u32 N, u32 k) { if (k >= N/2) { auto [c, s] = root1(N, k - N/2); return {-c, -s}; - } else if (k > N/4) { + } if (k > N/4) { auto [c, s] = root1(N, N/2 - k); return {-c, s}; - } else if (k > N/8) { + } if (k > N/8) { auto [c, s] = root1(N, N/4 - k); return {s, c}; - } else { + } assert(k <= N/8); long double angle = M_PIl * k / (N / 2); @@ -79,17 +81,17 @@ double2 root1(u32 N, u32 k) { return {double(cosl(angle)), double(sinl(angle))}; } #endif - } + } // Epsilon value, 2^-250, should have an exact representation as a double. Used to avoid divide-by-zero in root1over. const double epsilon = 5.5271478752604445602472651921923E-76; // Protect against divide by zero // Returns the primitive root of unity of order N, to the power k. Returned format is cosine, sine/cosine. -double2 root1over(u32 N, u32 k) { +static double2 root1over(u32 N, u32 k) { assert(k < N); - long double angle = M_PIl * k / (N / 2); + long double const angle = M_PIl * k / (N / 2); double c = cos(angle); long double s = sinl(angle); @@ -99,10 +101,10 @@ double2 root1over(u32 N, u32 k) { } // Returns the primitive root of unity of order N, to the power k. Returns only the cosine value. -double root1cos(u32 N, u32 k) { +static double root1cos(u32 N, u32 k) { assert(k < N); - long double angle = M_PIl * k / (N / 2); + long double const angle = M_PIl * k / (N / 2); double c = cos(angle); if (c > -1.0e-15 && c < 1.0e-15) c = epsilon; @@ -110,10 +112,10 @@ double root1cos(u32 N, u32 k) { } // Returns the primitive root of unity of order N, to the power k. Returns only the cosine value divided by another cosine value. -double root1cosover(u32 N, u32 k, double over) { +static double root1cosover(u32 N, u32 k, double over) { assert(k < N); - long double angle = M_PIl * k / (N / 2); + long double const angle = M_PIl * k / (N / 2); long double c = cosl(angle); if (c > -1.0e-15 && c < 1.0e-15) c = epsilon; @@ -123,9 +125,9 @@ double root1cosover(u32 N, u32 k, double over) { static const constexpr bool LOG_TRIG_ALLOC = false; // Interleave two lines of trig values so that AMD GPUs can use global_load_dwordx4 instructions -void T2shuffle(u32 size, u32 radix, u32 line, vector &tab) { +static void T2shuffle(u32 size, u32 radix, u32 line, vector &tab) { vector line1, line2; - u32 line_size = size / radix; + u32 const line_size = size / radix; for (u32 col = 0; col < line_size; ++col) { line1.push_back(tab[line*line_size + col]); line2.push_back(tab[(line+1)*line_size + col]); @@ -136,15 +138,31 @@ void T2shuffle(u32 size, u32 radix, u32 line, vector &tab) { } } -vector genSmallTrigFP64(u32 size, u32 radix) { +static vector genSmallTrigFP64(u32 size, u32 radix) { if (LOG_TRIG_ALLOC) { log("genSmallTrigFP64(%u, %u)\n", size, radix); } - u32 WG = size / radix; + u32 const WG = size / radix; vector tab; -// old fft_WIDTH and fft_HEIGHT - for (u32 line = 1; line < radix; ++line) { - for (u32 col = 0; col < WG; ++col) { - tab.push_back(radix / line >= 8 ? root1Fancy(size, col * line) : root1(size, col * line)); + // New SIZE=256, RADIX=8 which is really mixed radix-4 and radix-8. This is for a 4 * 8 * 8 implementation. + if (size == 256 && radix == 8) { + for (u32 line = 1; line < radix/2; ++line) { + for (u32 col = 0; col < WG*2; ++col) { + tab.push_back(0 && radix / line >= 8 ? root1Fancy(size, col * line) : root1(size, col * line)); + } + } + for (u32 line = 1; line < radix; ++line) { + for (u32 col = 0; col < WG; col += 4) { + tab.push_back(0 && radix / line >= 8 ? root1Fancy(size, col * line) : root1(size, col * line)); + } + } + } + + // original fft_WIDTH and fft_HEIGHT + else { + for (u32 line = 1; line < radix; ++line) { + for (u32 col = 0; col < WG; ++col) { + tab.push_back(radix / line >= 8 ? root1Fancy(size, col * line) : root1(size, col * line)); + } } } tab.resize(size); @@ -160,7 +178,7 @@ vector genSmallTrigFP64(u32 size, u32 radix) { // Sine/cosine values for first fft4 or fft8 for (u32 line = 1; line < radix; ++line) { for (u32 col = 0; col < WG; ++col) { - double2 root = root1over(size, col * line); + double2 const root = root1over(size, col * line); tab1.push_back(root.second); } } @@ -168,7 +186,7 @@ vector genSmallTrigFP64(u32 size, u32 radix) { // Sine/cosine values for later fft4 or fft8 for (u32 line = 0; line < radix; ++line) { for (u32 col = 0; col < WG; col += radix) { - double2 root = root1over(size, col * line); + double2 const root = root1over(size, col * line); tab1.push_back(root.second); } } @@ -176,7 +194,7 @@ vector genSmallTrigFP64(u32 size, u32 radix) { // Cosine values for first fft4 or fft8 (output in post-shufl order) //TODO: Examine why when sine is 0.0 cosine is not 1.0 or -1.0 (printf is outputting 0.999... and -0.999...) for (u32 grp = 0; grp < WG; ++grp) { - u32 line = grp / (WG/radix); // Output "line" number, where each line multiplies a different u[i]. There are radix lines. Each line has WG values. + u32 const line = grp / (WG/radix); // Output "line" number, where each line multiplies a different u[i]. There are radix lines. Each line has WG values. for (u32 col = 0; col < radix; ++col) { double divide_by = 1.0; // Compute cosine3 / cosine1 @@ -194,7 +212,7 @@ vector genSmallTrigFP64(u32 size, u32 radix) { // Cosine values for later fft4 or fft8 (output in post-shufl order). Similar to cosines above but output every radix-th value. for (u32 grp = 0; grp < radix; ++grp) { for (u32 col = 0; col < WG; col += radix) { - u32 line = col / (WG/radix); + u32 const line = col / (WG/radix); double divide_by = 1.0; // Compute cosine3 / cosine1 if ((radix == 4 && line == 3) || (radix == 8 && save_one_more_mul && line == 3)) { @@ -213,7 +231,7 @@ vector genSmallTrigFP64(u32 size, u32 radix) { for (u32 i = radix; i < 2*radix; i += 2) T2shuffle(size, radix, i, tab1); // Convert to a vector of double2 - for (u32 i = 0; i < tab1.size(); i += 2) tab.push_back({tab1[i], tab1[i+1]}); + for (u32 i = 0; i < tab1.size(); i += 2) tab.emplace_back(tab1[i], tab1[i+1]); } tab.resize(5*size); @@ -221,16 +239,16 @@ vector genSmallTrigFP64(u32 size, u32 radix) { } // Generate the small trig values for fft_HEIGHT plus optionally trig values used in pairSq. -vector genSmallTrigComboFP64(Args *args, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { +static vector genSmallTrigComboFP64(Args *args, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { if (LOG_TRIG_ALLOC) { log("genSmallTrigComboFP64(%u, %u)\n", size, radix); } vector tab = genSmallTrigFP64(size, radix); - u32 tail_trigs = args->value("TAIL_TRIGS", 2); // Default is calculating from scratch, no memory accesses + u32 const tail_trigs = args->value("TAIL_TRIGS", 2); // Default is calculating from scratch, no memory accesses // From tailSquare pre-calculate some or all of these: T2 trig = slowTrig_N(line + H * lowMe, ND / NH * 2); if (tail_trigs == 1) { // Some trig values in memory, some are computed with a complex multiply. Best option on a Radeon VII. - u32 height = size; + u32 const height = size; // Output line 0 trig values to be read by every u,v pair of lines for (u32 me = 0; me < height / radix; ++me) { tab.push_back(root1(width * middle * height, width * middle * me)); @@ -242,10 +260,10 @@ vector genSmallTrigComboFP64(Args *args, u32 width, u32 middle, u32 siz } } if (tail_trigs == 0) { // All trig values read from memory. Best option for GPUs with lousy DP performance. - u32 height = size; + u32 const height = size; for (u32 u = 0; u <= width * middle / 2; ++u) { - for (u32 v = 0; v < (tail_single_wide ? 1 : 2); ++v) { - u32 line = (v == 0) ? u : (u ? width * middle - u : width * middle / 2); + for (u32 v = 0; std::cmp_less(v , (tail_single_wide ? 1 : 2)); ++v) { + u32 const line = (v == 0) ? u : (u ? width * middle - u : width * middle / 2); for (u32 me = 0; me < height / radix; ++me) { tab.push_back(root1(width * middle * height, line + width * middle * me)); } @@ -258,9 +276,11 @@ vector genSmallTrigComboFP64(Args *args, u32 width, u32 middle, u32 siz // starting from a MIDDLE of 5 we consider angles in [0, 2Pi/MIDDLE] as worth storing with the // cos-1 "fancy" trick. -#define SHARP_MIDDLE 5 +enum { +SHARP_MIDDLE = 5 +}; -vector genMiddleTrigFP64(u32 smallH, u32 middle, u32 width) { +static vector genMiddleTrigFP64(u32 smallH, u32 middle, u32 width) { if (LOG_TRIG_ALLOC) { log("genMiddleTrigFP64(%u, %u, %u)\n", smallH, middle, width); } vector tab; if (middle == 1) { @@ -290,7 +310,7 @@ float2 root1FancyFP32(u32 N, u32 k) { assert(k < N); assert(k < N/4); - double angle = M_PI * k / (N / 2); + double const angle = M_PI * k / (N / 2); return {float(cos(angle) - 1), float(sin(angle))}; } @@ -299,15 +319,15 @@ static float trigError(float c, float s) { return abs(trigNorm(c, s) - 1.0f); } // Round trig double to float as to satisfy c^2 + s^2 == 1 as best as possible static float2 roundTrig(double lc, double ls) { - float c1 = lc; - float c2 = nexttoward(c1, lc); - float s1 = ls; - float s2 = nexttoward(s1, ls); + float const c1 = float(lc); + float const c2 = nexttoward(c1, lc); + float const s1 = float(ls); + float const s2 = nexttoward(s1, ls); float c = c1; float s = s1; - for (float tryC : {c1, c2}) { - for (float tryS : {s1, s2}) { + for (float const tryC : {c1, c2}) { + for (float const tryS : {s1, s2}) { if (trigError(tryC, tryS) < trigError(c, s)) { c = tryC; s = tryS; @@ -323,43 +343,179 @@ float2 root1FP32(u32 N, u32 k) { if (k >= N/2) { auto [c, s] = root1FP32(N, k - N/2); return {-c, -s}; - } else if (k > N/4) { + } if (k > N/4) { auto [c, s] = root1FP32(N, N/2 - k); return {-c, s}; - } else if (k > N/8) { + } if (k > N/8) { auto [c, s] = root1FP32(N, N/4 - k); return {s, c}; - } else { + } assert(k <= N/8); double angle = M_PI * k / (N / 2); return roundTrig(cos(angle), sin(angle)); + +} + +// Epsilon value, 2^-50, should have an exact representation as a float. Used to avoid divide-by-zero in root1overFP32. +const double epsilonFP32 = 8.8817841970012523233890533447266e-16; // Protect against divide by zero + +// Returns the primitive root of unity of order N, to the power k. Returned format is cosine, sine/cosine. +static float2 root1overFP32(u32 N, u32 k) { + assert(k < N); + + double const angle = M_PI * k / (N / 2); + double c = cos(angle); + double s = sin(angle); + + if (c > -1.0e-15 && c < 1.0e-15) c = epsilonFP32; + s = s / c; + return {float(c), float(s)}; +} + +// Returns the primitive root of unity of order N, to the power k. Returns only the cosine value. +static float root1cosFP32(u32 N, u32 k) { + assert(k < N); + + double const angle = M_PI * k / (N / 2); + double c = cos(angle); + + if (c > -1.0e-15 && c < 1.0e-15) c = epsilonFP32; + return float(c); +} + +// Returns the primitive root of unity of order N, to the power k. Returns only the cosine value divided by another cosine value. +static float root1cosoverFP32(u32 N, u32 k, double over) { + assert(k < N); + + double const angle = M_PI * k / (N / 2); + double c = cos(angle); + + if (c > -1.0e-15 && c < 1.0e-15) c = epsilonFP32; + return float(c / over); +} + +// Interleave two lines of trig values so that AMD GPUs can use global_load_dwordx4 instructions +static void F2shuffle(u32 size, u32 radix, u32 line, vector &tab) { + vector line1, line2; + u32 const line_size = size / radix; + for (u32 col = 0; col < line_size; ++col) { + line1.push_back(tab[line*line_size + col]); + line2.push_back(tab[(line+1)*line_size + col]); + } + for (u32 col = 0; col < line_size; ++col) { + tab[line*line_size + 2*col] = line1[col]; + tab[line*line_size + 2*col + 1] = line2[col]; } } -vector genSmallTrigFP32(u32 size, u32 radix) { - u32 WG = size / radix; +static vector genSmallTrigFP32(u32 size, u32 radix) { + u32 const WG = size / radix; vector tab; -// old fft_WIDTH and fft_HEIGHT - for (u32 line = 1; line < radix; ++line) { - for (u32 col = 0; col < WG; ++col) { - tab.push_back(radix / line >= 8 ? root1FancyFP32(size, col * line) : root1FP32(size, col * line)); + // New SIZE=256, RADIX=8 which is really mixed radix-4 and radix-8. This is for a 4 * 8 * 8 implementation. + if (size == 256 && radix == 8) { + for (u32 line = 1; line < radix/2; ++line) { + for (u32 col = 0; col < WG*2; ++col) { + tab.push_back(0 && radix / line >= 8 ? root1FancyFP32(size, col * line) : root1FP32(size, col * line)); + } + } + for (u32 line = 1; line < radix; ++line) { + for (u32 col = 0; col < WG; col += 4) { + tab.push_back(0 && radix / line >= 8 ? root1FancyFP32(size, col * line) : root1FP32(size, col * line)); + } + } + } + + // original fft_WIDTH and fft_HEIGHT + else { + for (u32 line = 1; line < radix; ++line) { + for (u32 col = 0; col < WG; ++col) { + tab.push_back(radix / line >= 8 ? root1FancyFP32(size, col * line) : root1FP32(size, col * line)); + } } } tab.resize(size); + +// New fft_WIDTH and fft_HEIGHT +// We need two versions of trig values. One where we save one more mul and one where we don't. +// In theory, we should always use save one more mul but the rocm optimizer is doing something weird in fft_WIDTH. + + for (u32 save_one_more_mul = 0; save_one_more_mul <= 1; ++save_one_more_mul) { + vector tab1; + if (save_one_more_mul) tab.resize(3*size); + + // Sine/cosine values for first fft4 or fft8 + for (u32 line = 1; line < radix; ++line) { + for (u32 col = 0; col < WG; ++col) { + float2 const root = root1overFP32(size, col * line); + tab1.push_back(root.second); + } + } + + // Sine/cosine values for later fft4 or fft8 + for (u32 line = 0; line < radix; ++line) { + for (u32 col = 0; col < WG; col += radix) { + float2 const root = root1overFP32(size, col * line); + tab1.push_back(root.second); + } + } + + // Cosine values for first fft4 or fft8 (output in post-shufl order) + for (u32 grp = 0; grp < WG; ++grp) { + u32 const line = grp / (WG/radix); // Output "line" number, where each line multiplies a different u[i]. There are radix lines. Each line has WG values. + for (u32 col = 0; col < radix; ++col) { + float divide_by = 1.0; + // Compute cosine3 / cosine1 + if ((radix == 4 && line == 3) || (radix == 8 && save_one_more_mul && line == 3)) { + divide_by = root1cosFP32(size, col * (grp - 2*(WG/radix))); + } + // Compute cosine5 / cosine1, cosine6 / cosine2, cosine7 / cosine3 + if (radix == 8 && ((save_one_more_mul && line == 5) || line == 6 || line == 7)) { + divide_by = root1cosFP32(size, col * (grp - 4*(WG/radix))); + } + tab1.push_back(root1cosoverFP32(size, col * grp, divide_by)); + } + } + + // Cosine values for later fft4 or fft8 (output in post-shufl order). Similar to cosines above but output every radix-th value. + for (u32 grp = 0; grp < radix; ++grp) { + for (u32 col = 0; col < WG; col += radix) { + u32 const line = col / (WG/radix); + double divide_by = 1.0; + // Compute cosine3 / cosine1 + if ((radix == 4 && line == 3) || (radix == 8 && save_one_more_mul && line == 3)) { + divide_by = root1cosFP32(size, grp * (col - 2*(WG/radix))); + } + // Compute cosine5 / cosine1, cosine6 / cosine2, cosine7 / cosine3 + if (radix == 8 && ((save_one_more_mul && line == 5) || line == 6 || line == 7)) { + divide_by = root1cosFP32(size, grp * (col - 4*(WG/radix))); + } + tab1.push_back(root1cosoverFP32(size, grp * col, divide_by)); + } + } + + // Interleave first fft4 or fft8 trig values for faster AMD GPU access + for (u32 i = 0; i < radix-2; i += 2) F2shuffle(size, radix, i, tab1); + for (u32 i = radix; i < 2*radix; i += 2) F2shuffle(size, radix, i, tab1); + + // Convert to a vector of float2 + for (u32 i = 0; i < tab1.size(); i += 2) tab.emplace_back(tab1[i], tab1[i+1]); + } + + tab.resize(5*size); return tab; } // Generate the small trig values for fft_HEIGHT plus optionally trig values used in pairSq. -vector genSmallTrigComboFP32(Args *args, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { +static vector genSmallTrigComboFP32(Args *args, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { vector tab = genSmallTrigFP32(size, radix); - u32 tail_trigs = args->value("TAIL_TRIGS32", 2); // Default is calculating from scratch, no memory accesses + u32 const tail_trigs = args->value("TAIL_TRIGS32", 2); // Default is calculating from scratch, no memory accesses // From tailSquare pre-calculate some or all of these: F2 trig = slowTrig_N(line + H * lowMe, ND / NH * 2); if (tail_trigs == 1) { // Some trig values in memory, some are computed with a complex multiply. - u32 height = size; + u32 const height = size; // Output line 0 trig values to be read by every u,v pair of lines for (u32 me = 0; me < height / radix; ++me) { tab.push_back(root1FP32(width * middle * height, width * middle * me)); @@ -371,10 +527,10 @@ vector genSmallTrigComboFP32(Args *args, u32 width, u32 middle, u32 size } } if (tail_trigs == 0) { // All trig values read from memory. Best option for GPUs with lousy FP performance? - u32 height = size; + u32 const height = size; for (u32 u = 0; u <= width * middle / 2; ++u) { - for (u32 v = 0; v < (tail_single_wide ? 1 : 2); ++v) { - u32 line = (v == 0) ? u : (u ? width * middle - u : width * middle / 2); + for (u32 v = 0; std::cmp_less(v , (tail_single_wide ? 1 : 2)); ++v) { + u32 const line = (v == 0) ? u : (u ? width * middle - u : width * middle / 2); for (u32 me = 0; me < height / radix; ++me) { tab.push_back(root1FP32(width * middle * height, line + width * middle * me)); } @@ -385,7 +541,7 @@ vector genSmallTrigComboFP32(Args *args, u32 width, u32 middle, u32 size return tab; } -vector genMiddleTrigFP32(u32 smallH, u32 middle, u32 width) { +static vector genMiddleTrigFP32(u32 smallH, u32 middle, u32 width) { vector tab; if (middle == 1) { tab.resize(1); @@ -436,10 +592,10 @@ class Z31 } public: - Z31() {} + Z31() = default; explicit Z31(const uint32_t n) : _n(n) {} - uint32_t get() const { return _n; } + [[nodiscard]] uint32_t get() const { return _n; } bool operator!=(const Z31 & rhs) const { return (_n != rhs._n); } @@ -450,7 +606,7 @@ class Z31 Z31 operator-(const Z31 & rhs) const { return Z31(_sub(_n, rhs._n)); } Z31 operator*(const Z31 & rhs) const { return Z31(_mul(_n, rhs._n)); } - Z31 sqr() const { return Z31(_mul(_n, _n)); } + [[nodiscard]] Z31 sqr() const { return Z31(_mul(_n, _n)); } }; @@ -464,20 +620,20 @@ class GF31 static const uint32_t _h_0 = 7735u, _h_1 = 748621u; public: - GF31() {} + GF31() = default; explicit GF31(const Z31 & s0, const Z31 & s1) : _s0(s0), _s1(s1) {} explicit GF31(const uint32_t n0, const uint32_t n1) : _s0(n0), _s1(n1) {} - const Z31 & s0() const { return _s0; } - const Z31 & s1() const { return _s1; } + [[nodiscard]] const Z31 & s0() const { return _s0; } + [[nodiscard]] const Z31 & s1() const { return _s1; } GF31 operator+(const GF31 & rhs) const { return GF31(_s0 + rhs._s0, _s1 + rhs._s1); } GF31 operator-(const GF31 & rhs) const { return GF31(_s0 - rhs._s0, _s1 - rhs._s1); } - GF31 sqr() const { const Z31 t = _s0 * _s1; return GF31(_s0.sqr() - _s1.sqr(), t + t); } - GF31 mul(const GF31 & rhs) const { return GF31(_s0 * rhs._s0 - _s1 * rhs._s1, _s1 * rhs._s0 + _s0 * rhs._s1); } + [[nodiscard]] GF31 sqr() const { const Z31 t = _s0 * _s1; return GF31(_s0.sqr() - _s1.sqr(), t + t); } + [[nodiscard]] GF31 mul(const GF31 & rhs) const { return GF31(_s0 * rhs._s0 - _s1 * rhs._s1, _s1 * rhs._s0 + _s0 * rhs._s1); } - GF31 pow(const uint64_t e) const + [[nodiscard]] GF31 pow(const uint64_t e) const { if (e == 0) return GF31(1u, 0u); GF31 r = GF31(1u, 0u), y = *this; @@ -485,44 +641,62 @@ class GF31 return r.mul(y); } - static const GF31 root_one(const size_t n) { return GF31(Z31(_h_0), Z31(_h_1)).pow(_h_order / n); } + static GF31 root_one(const size_t n) { return GF31(Z31(_h_0), Z31(_h_1)).pow(_h_order / n); } static uint8_t log2_root_two(const size_t n) { return uint8_t(((uint64_t(1) << 30) / n) % 31); } }; // Returns the primitive root of unity of order N, to the power k. -uint2 root1GF31(GF31 root1N, u32 k) { - GF31 x = root1N.pow(k); +static uint2 root1GF31(GF31 root1N, u32 k) { + GF31 const x = root1N.pow(k); return { x.s0().get(), x.s1().get() }; } uint2 root1GF31(u32 N, u32 k) { assert(k < N); - GF31 root1N = GF31::root_one(N); + GF31 const root1N = GF31::root_one(N); return root1GF31(root1N, k); } -vector genSmallTrigGF31(u32 size, u32 radix) { - u32 WG = size / radix; +static vector genSmallTrigGF31(u32 size, u32 radix) { + u32 const WG = size / radix; vector tab; + GF31 const root1size = GF31::root_one(size); + + // New SIZE=256, RADIX=8 which is really mixed radix-4 and radix-8. This is for a 4 * 8 * 8 implementation. + if (size == 256 && radix == 8) { + for (u32 line = 1; line < radix/2; ++line) { + for (u32 col = 0; col < WG*2; ++col) { + tab.push_back(root1GF31(root1size, col * line)); + } + } + for (u32 line = 1; line < radix; ++line) { + for (u32 col = 0; col < WG; col += 4) { + tab.push_back(root1GF31(root1size, col * line)); + } + } + } - GF31 root1size = GF31::root_one(size); - for (u32 line = 1; line < radix; ++line) { - for (u32 col = 0; col < WG; ++col) { - tab.push_back(root1GF31(root1size, col * line)); + // Standard roots + else { + for (u32 line = 1; line < radix; ++line) { + for (u32 col = 0; col < WG; ++col) { + tab.push_back(root1GF31(root1size, col * line)); + } } } + tab.resize(size); return tab; } // Generate the small trig values for fft_HEIGHT plus optionally trig values used in pairSq. -vector genSmallTrigComboGF31(Args *args, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { +static vector genSmallTrigComboGF31(Args *args, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { vector tab = genSmallTrigGF31(size, radix); - u32 tail_trigs = args->value("TAIL_TRIGS31", 0); // Default is reading all trigs from memory + u32 const tail_trigs = args->value("TAIL_TRIGS31", 0); // Default is reading all trigs from memory // From tailSquareGF31 pre-calculate some or all of these: GF31 trig = slowTrigGF31(line + H * lowMe, ND / NH * 2); - u32 height = size; - GF31 root1wmh = GF31::root_one(width * middle * height); + u32 const height = size; + GF31 const root1wmh = GF31::root_one(width * middle * height); if (tail_trigs >= 1) { // Some trig values in memory, some are computed with a complex multiply. Best option on a Radeon VII. // Output line 0 trig values to be read by every u,v pair of lines for (u32 me = 0; me < height / radix; ++me) { @@ -536,8 +710,8 @@ vector genSmallTrigComboGF31(Args *args, u32 width, u32 middle, u32 size, } if (tail_trigs == 0) { // All trig values read from memory. Best option for GPUs with great memory performance. for (u32 u = 0; u <= width * middle / 2; ++u) { - for (u32 v = 0; v < (tail_single_wide ? 1 : 2); ++v) { - u32 line = (v == 0) ? u : (u ? width * middle - u : width * middle / 2); + for (u32 v = 0; std::cmp_less(v , (tail_single_wide ? 1 : 2)); ++v) { + u32 const line = (v == 0) ? u : (u ? width * middle - u : width * middle / 2); for (u32 me = 0; me < height / radix; ++me) { tab.push_back(root1GF31(root1wmh, line + width * middle * me)); } @@ -548,18 +722,18 @@ vector genSmallTrigComboGF31(Args *args, u32 width, u32 middle, u32 size, return tab; } -vector genMiddleTrigGF31(u32 smallH, u32 middle, u32 width) { +static vector genMiddleTrigGF31(u32 smallH, u32 middle, u32 width) { vector tab; if (middle == 1) { tab.resize(1); } else { - GF31 root1hm = GF31::root_one(smallH * middle); + GF31 const root1hm = GF31::root_one(smallH * middle); for (u32 m = 1; m < middle; ++m) { for (u32 k = 0; k < smallH; ++k) { tab.push_back(root1GF31(root1hm, k * m)); } } - GF31 root1mw = GF31::root_one(middle * width); + GF31 const root1mw = GF31::root_one(middle * width); for (u32 k = 0; k < width; ++k) { tab.push_back(root1GF31(root1mw, k)); } - GF31 root1wmh = GF31::root_one(width * middle * smallH); + GF31 const root1wmh = GF31::root_one(width * middle * smallH); for (u32 k = 0; k < smallH; ++k) { tab.push_back(root1GF31(root1wmh, k)); } } return tab; @@ -593,17 +767,17 @@ class Z61 static uint64_t _mul(const uint64_t a, const uint64_t b) { - const __uint128_t t = a * __uint128_t(b); - const uint64_t lo = uint64_t(t), hi = uint64_t(t >> 64); + const u128 t = a * u128(b); + const auto lo = uint64_t(t), hi = uint64_t(t >> 64); const uint64_t lo61 = lo & _p, hi61 = (lo >> 61) | (hi << 3); return _add(lo61, hi61); } public: - Z61() {} + Z61() = default; explicit Z61(const uint64_t n) : _n(n) {} - uint64_t get() const { return _n; } + [[nodiscard]] uint64_t get() const { return _n; } bool operator!=(const Z61 & rhs) const { return (_n != rhs._n); } @@ -611,7 +785,7 @@ class Z61 Z61 operator-(const Z61 & rhs) const { return Z61(_sub(_n, rhs._n)); } Z61 operator*(const Z61 & rhs) const { return Z61(_mul(_n, rhs._n)); } - Z61 sqr() const { return Z61(_mul(_n, _n)); } + [[nodiscard]] Z61 sqr() const { return Z61(_mul(_n, _n)); } }; // GF((2^61 - 1)^2): the prime field of order p^2, p = 2^61 - 1 @@ -626,20 +800,20 @@ class GF61 static const uint64_t _h_order = uint64_t(1) << 62; public: - GF61() {} + GF61() = default; explicit GF61(const Z61 & s0, const Z61 & s1) : _s0(s0), _s1(s1) {} explicit GF61(const uint64_t n0, const uint64_t n1) : _s0(n0), _s1(n1) {} - const Z61 & s0() const { return _s0; } - const Z61 & s1() const { return _s1; } + [[nodiscard]] const Z61 & s0() const { return _s0; } + [[nodiscard]] const Z61 & s1() const { return _s1; } GF61 operator+(const GF61 & rhs) const { return GF61(_s0 + rhs._s0, _s1 + rhs._s1); } GF61 operator-(const GF61 & rhs) const { return GF61(_s0 - rhs._s0, _s1 - rhs._s1); } - GF61 sqr() const { const Z61 t = _s0 * _s1; return GF61(_s0.sqr() - _s1.sqr(), t + t); } - GF61 mul(const GF61 & rhs) const { return GF61(_s0 * rhs._s0 - _s1 * rhs._s1, _s1 * rhs._s0 + _s0 * rhs._s1); } + [[nodiscard]] GF61 sqr() const { const Z61 t = _s0 * _s1; return GF61(_s0.sqr() - _s1.sqr(), t + t); } + [[nodiscard]] GF61 mul(const GF61 & rhs) const { return GF61(_s0 * rhs._s0 - _s1 * rhs._s1, _s1 * rhs._s0 + _s0 * rhs._s1); } - GF61 pow(const uint64_t e) const + [[nodiscard]] GF61 pow(const uint64_t e) const { if (e == 0) return GF61(1u, 0u); GF61 r = GF61(1u, 0u), y = *this; @@ -647,44 +821,62 @@ class GF61 return r.mul(y); } - static const GF61 root_one(const size_t n) { return GF61(Z61(_h_0), Z61(_h_1)).pow(_h_order / n); } + static GF61 root_one(const size_t n) { return GF61(Z61(_h_0), Z61(_h_1)).pow(_h_order / n); } static uint8_t log2_root_two(const size_t n) { return uint8_t(((uint64_t(1) << 60) / n) % 61); } }; // Returns the primitive root of unity of order N, to the power k. -ulong2 root1GF61(GF61 root1N, u32 k) { - GF61 x = root1N.pow(k); +static ulong2 root1GF61(GF61 root1N, u32 k) { + GF61 const x = root1N.pow(k); return { x.s0().get(), x.s1().get() }; } ulong2 root1GF61(u32 N, u32 k) { assert(k < N); - GF61 root1N = GF61::root_one(N); + GF61 const root1N = GF61::root_one(N); return root1GF61(root1N, k); } -vector genSmallTrigGF61(u32 size, u32 radix) { - u32 WG = size / radix; +static vector genSmallTrigGF61(u32 size, u32 radix) { + u32 const WG = size / radix; vector tab; + GF61 const root1size = GF61::root_one(size); + + // New SIZE=256, RADIX=8 which is really mixed radix-4 and radix-8. This is for a 4 * 8 * 8 implementation. + if (size == 256 && radix == 8) { + for (u32 line = 1; line < radix/2; ++line) { + for (u32 col = 0; col < WG*2; ++col) { + tab.push_back(root1GF61(root1size, col * line)); + } + } + for (u32 line = 1; line < radix; ++line) { + for (u32 col = 0; col < WG; col += 4) { + tab.push_back(root1GF61(root1size, col * line)); + } + } + } - GF61 root1size = GF61::root_one(size); - for (u32 line = 1; line < radix; ++line) { - for (u32 col = 0; col < WG; ++col) { - tab.push_back(root1GF61(root1size, col * line)); + // Standard roots + else { + for (u32 line = 1; line < radix; ++line) { + for (u32 col = 0; col < WG; ++col) { + tab.push_back(root1GF61(root1size, col * line)); + } } } + tab.resize(size); return tab; } // Generate the small trig values for fft_HEIGHT plus optionally trig values used in pairSq. -vector genSmallTrigComboGF61(Args *args, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { +static vector genSmallTrigComboGF61(Args *args, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { vector tab = genSmallTrigGF61(size, radix); - u32 tail_trigs = args->value("TAIL_TRIGS61", 0); // Default is reading all trigs from memory + u32 const tail_trigs = args->value("TAIL_TRIGS61", 0); // Default is reading all trigs from memory // From tailSquareGF61 pre-calculate some or all of these: GF61 trig = slowTrigGF61(line + H * lowMe, ND / NH * 2); - u32 height = size; - GF61 root1wmh = GF61::root_one(width * middle * height); + u32 const height = size; + GF61 const root1wmh = GF61::root_one(width * middle * height); if (tail_trigs >= 1) { // Some trig values in memory, some are computed with a complex multiply. Best option on a Radeon VII. // Output line 0 trig values to be read by every u,v pair of lines for (u32 me = 0; me < height / radix; ++me) { @@ -698,8 +890,8 @@ vector genSmallTrigComboGF61(Args *args, u32 width, u32 middle, u32 size } if (tail_trigs == 0) { // All trig values read from memory. Best option for GPUs with great memory performance. for (u32 u = 0; u <= width * middle / 2; ++u) { - for (u32 v = 0; v < (tail_single_wide ? 1 : 2); ++v) { - u32 line = (v == 0) ? u : (u ? width * middle - u : width * middle / 2); + for (u32 v = 0; std::cmp_less(v , (tail_single_wide ? 1 : 2)); ++v) { + u32 const line = (v == 0) ? u : (u ? width * middle - u : width * middle / 2); for (u32 me = 0; me < height / radix; ++me) { tab.push_back(root1GF61(root1wmh, line + width * middle * me)); } @@ -710,18 +902,18 @@ vector genSmallTrigComboGF61(Args *args, u32 width, u32 middle, u32 size return tab; } -vector genMiddleTrigGF61(u32 smallH, u32 middle, u32 width) { +static vector genMiddleTrigGF61(u32 smallH, u32 middle, u32 width) { vector tab; if (middle == 1) { tab.resize(1); } else { - GF61 root1hm = GF61::root_one(smallH * middle); + GF61 const root1hm = GF61::root_one(smallH * middle); for (u32 m = 1; m < middle; ++m) { for (u32 k = 0; k < smallH; ++k) { tab.push_back(root1GF61(root1hm, k * m)); } } - GF61 root1mw = GF61::root_one(middle * width); + GF61 const root1mw = GF61::root_one(middle * width); for (u32 k = 0; k < width; ++k) { tab.push_back(root1GF61(root1mw, k)); } - GF61 root1wmh = GF61::root_one(width * middle * smallH); + GF61 const root1wmh = GF61::root_one(width * middle * smallH); for (u32 k = 0; k < smallH; ++k) { tab.push_back(root1GF61(root1wmh, k)); } } return tab; @@ -732,9 +924,9 @@ vector genMiddleTrigGF61(u32 smallH, u32 middle, u32 width) { /* Build all the needed trig values into one big buffer */ /**********************************************************/ -vector genSmallTrig(FFTConfig fft, u32 size, u32 radix) { +static vector genSmallTrig(FFTConfig fft, u32 size, u32 radix) { vector tab; - u32 tabsize; + size_t tabsize; if (fft.FFT_FP64) { tab = genSmallTrigFP64(size, radix); @@ -771,9 +963,9 @@ vector genSmallTrig(FFTConfig fft, u32 size, u32 radix) { return tab; } -vector genSmallTrigCombo(Args *args, FFTConfig fft, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { +static vector genSmallTrigCombo(Args *args, FFTConfig fft, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { vector tab; - u32 tabsize; + size_t tabsize; if (fft.FFT_FP64) { tab = genSmallTrigComboFP64(args, width, middle, size, radix, tail_single_wide); @@ -810,9 +1002,9 @@ vector genSmallTrigCombo(Args *args, FFTConfig fft, u32 width, u32 midd return tab; } -vector genMiddleTrig(FFTConfig fft, u32 smallH, u32 middle, u32 width) { +static vector genMiddleTrig(FFTConfig fft, u32 smallH, u32 middle, u32 width) { vector tab; - u32 tabsize; + size_t tabsize; if (fft.FFT_FP64) { tab = genMiddleTrigFP64(smallH, middle, width); @@ -854,32 +1046,39 @@ vector genMiddleTrig(FFTConfig fft, u32 smallH, u32 middle, u32 width) /* Code to manage a cache of trigBuffers */ /********************************************************/ -#define make_key_part(b,tt,b31,tt31,b32,tt32,b61,tt61,tk) ((((((((b+tt) << 2) + b31+tt31) << 2) + b32+tt32) << 2) + b61+tt61) << 2) + tk +// Each field of the key is one "is this number type in use" flag and that type's TAIL_TRIGS setting. They +// need separate bits: added together, as they used to be, a type in use with TAIL_TRIGS=n is indistinguishable +// from that type unused with TAIL_TRIGS=n+1, and a TAIL_TRIGS of 3 or more carries into the neighbouring +// field. Two Gpus in one process can then share a cached table generated for the other one's number type, +// which is silently wrong twiddles -- caught, if at all, only by the Gerbicz check. +#define make_key_field(b, tt) (((((b) != 0) << 3) | ((tt) & 7))) +#define make_key_part(b,tt,b31,tt31,b32,tt32,b61,tt61,tk) \ + ((((((((make_key_field(b, tt) << 4) | make_key_field(b31, tt31)) << 4) | make_key_field(b32, tt32)) << 4) | make_key_field(b61, tt61)) << 1) | ((tk) != 0)) TrigBufCache::~TrigBufCache() = default; TrigPtr TrigBufCache::smallTrig(Args *args, FFTConfig fft, u32 width, u32 nW, u32 middle, u32 height, u32 nH, bool tail_single_wide) { - lock_guard lock{mut}; + std::scoped_lock const lock{mut}; auto& m = small; TrigPtr p{}; - u32 tail_trigs = args->value("TAIL_TRIGS", 2); // Default is calculating FP64 trigs from scratch, no memory accesses - u32 tail_trigs31 = args->value("TAIL_TRIGS31", 2); // Default is reading GF31 trigs from memory - u32 tail_trigs32 = args->value("TAIL_TRIGS32", 2); // Default is calculating FP32 trigs from scratch, no memory accesses - u32 tail_trigs61 = args->value("TAIL_TRIGS61", 2); // Default is reading GF61 trigs from memory - u32 key_part = make_key_part(fft.FFT_FP64, tail_trigs, fft.NTT_GF31, tail_trigs31, fft.FFT_FP32, tail_trigs32, fft.NTT_GF61, tail_trigs61, tail_single_wide); + u32 const tail_trigs = args->value("TAIL_TRIGS", 2); // Default is calculating FP64 trigs from scratch, no memory accesses + u32 const tail_trigs31 = args->value("TAIL_TRIGS31", 2); // Default is reading GF31 trigs from memory + u32 const tail_trigs32 = args->value("TAIL_TRIGS32", 2); // Default is calculating FP32 trigs from scratch, no memory accesses + u32 const tail_trigs61 = args->value("TAIL_TRIGS61", 2); // Default is reading GF61 trigs from memory + u32 const key_part = make_key_part(fft.FFT_FP64, tail_trigs, fft.NTT_GF31, tail_trigs31, fft.FFT_FP32, tail_trigs32, fft.NTT_GF61, tail_trigs61, tail_single_wide); // See if there is an existing smallTrigCombo that we can return (using only a subset of the data) // In theory, we could match any smallTrigCombo where width matches. However, SMALLTRIG_GF31_SIZE wouldn't be able to figure out the size. // In practice, those cases will likely never arise. if (width == height && nW == nH) { - decay_t::key_type key{height, nH, width, middle, key_part}; + decay_t::key_type const key{height, nH, width, middle, key_part}; auto it = m.find(key); if (it != m.end() && (p = it->second.lock())) return p; } // See if there is an existing non-combo smallTrig that we can return - decay_t::key_type key{width, nW, 0, 0, key_part}; + decay_t::key_type const key{width, nW, 0, 0, key_part}; auto it = m.find(key); if (it != m.end() && (p = it->second.lock())) return p; @@ -891,19 +1090,19 @@ TrigPtr TrigBufCache::smallTrig(Args *args, FFTConfig fft, u32 width, u32 nW, u3 } TrigPtr TrigBufCache::smallTrigCombo(Args *args, FFTConfig fft, u32 width, u32 middle, u32 height, u32 nH, bool tail_single_wide) { - u32 tail_trigs = args->value("TAIL_TRIGS", 2); // Default is calculating FP64 trigs from scratch, no memory accesses - u32 tail_trigs31 = args->value("TAIL_TRIGS31", 2); // Default is reading GF31 trigs from memory - u32 tail_trigs32 = args->value("TAIL_TRIGS32", 2); // Default is calculating FP32 trigs from scratch, no memory accesses - u32 tail_trigs61 = args->value("TAIL_TRIGS61", 2); // Default is reading GF61 trigs from memory - u32 key_part = make_key_part(fft.FFT_FP64, tail_trigs, fft.NTT_GF31, tail_trigs31, fft.FFT_FP32, tail_trigs32, fft.NTT_GF61, tail_trigs61, tail_single_wide); + u32 const tail_trigs = args->value("TAIL_TRIGS", 2); // Default is calculating FP64 trigs from scratch, no memory accesses + u32 const tail_trigs31 = args->value("TAIL_TRIGS31", 2); // Default is reading GF31 trigs from memory + u32 const tail_trigs32 = args->value("TAIL_TRIGS32", 2); // Default is calculating FP32 trigs from scratch, no memory accesses + u32 const tail_trigs61 = args->value("TAIL_TRIGS61", 2); // Default is reading GF61 trigs from memory + u32 const key_part = make_key_part(fft.FFT_FP64, tail_trigs, fft.NTT_GF31, tail_trigs31, fft.FFT_FP32, tail_trigs32, fft.NTT_GF61, tail_trigs61, tail_single_wide); // If there are no pre-computed trig values we might be able to share this trig table with fft_WIDTH if (((tail_trigs == 2 && fft.FFT_FP64) || (tail_trigs32 == 2 && fft.FFT_FP32)) && !fft.NTT_GF31 && !fft.NTT_GF61) return smallTrig(args, fft, height, nH, middle, height, nH, tail_single_wide); - lock_guard lock{mut}; + std::scoped_lock const lock{mut}; auto& m = small; - decay_t::key_type key{height, nH, width, middle, key_part}; + decay_t::key_type const key{height, nH, width, middle, key_part}; TrigPtr p{}; auto it = m.find(key); @@ -915,11 +1114,11 @@ TrigPtr TrigBufCache::smallTrigCombo(Args *args, FFTConfig fft, u32 width, u32 m return p; } -TrigPtr TrigBufCache::middleTrig(Args *args, FFTConfig fft, u32 SMALL_H, u32 MIDDLE, u32 width) { - lock_guard lock{mut}; +TrigPtr TrigBufCache::middleTrig(Args * /*args*/, FFTConfig fft, u32 SMALL_H, u32 MIDDLE, u32 width) { + std::scoped_lock const lock{mut}; auto& m = middle; - u32 key_part = make_key_part(fft.FFT_FP64, 0, fft.NTT_GF31, 0, fft.FFT_FP32, 0, fft.NTT_GF61, 0, 0); - decay_t::key_type key{SMALL_H, MIDDLE, width, key_part}; + u32 const key_part = make_key_part(fft.FFT_FP64, 0, fft.NTT_GF31, 0, fft.FFT_FP32, 0, fft.NTT_GF61, 0, 0); + decay_t::key_type const key{SMALL_H, MIDDLE, width, key_part}; TrigPtr p{}; auto it = m.find(key); diff --git a/src/TrigBufCache.h b/src/TrigBufCache.h index d5e5317a..35482084 100644 --- a/src/TrigBufCache.h +++ b/src/TrigBufCache.h @@ -6,6 +6,7 @@ #include "FFTConfig.h" #include +#include using TrigBuf = Buffer; using TrigPtr = shared_ptr; @@ -18,7 +19,7 @@ class StrongCache { explicit StrongCache(u32 size) : ptrs(size) {} void add(TrigPtr ptr) { - ptrs.at(pos) = ptr; + ptrs.at(pos) = std::move(ptr); if (++pos >= ptrs.size()) { pos = 0; } } }; @@ -57,25 +58,25 @@ float2 root1FP32(u32 N, u32 k); uint2 root1GF31(u32 N, u32 k); ulong2 root1GF61(u32 N, u32 k); -// Compute the size of the largest possible trig buffer given width, middle, height (in number of float2 values) -#define SMALLTRIG_FP64_SIZE(W,M,H,nH) (W != H || H == 0 ? W * 5 : SMALLTRIGCOMBO_FP64_SIZE(W,M,H,nH)) // See genSmallTrigFP64 -#define SMALLTRIGCOMBO_FP64_SIZE(W,M,H,nH) (H * 5 + (W * M / 2 + 1) * 2 * H / nH) // See genSmallTrigComboFP64 -#define MIDDLETRIG_FP64_SIZE(W,M,H) (H + W + H) // See genMiddleTrigFP64 +// Compute the size of the largest possible trig buffer given width, middle, height (in number of double2 values) +#define SMALLTRIG_FP64_SIZE(W,M,H,nH) ((W) != (H) || (H) == 0 ? (W) * 5 : SMALLTRIGCOMBO_FP64_SIZE(W,M,H,nH)) // See genSmallTrigFP64 +#define SMALLTRIGCOMBO_FP64_SIZE(W,M,H,nH) ((H) * 5 + ((W) * (M) / 2 + 1) * 2 * (H) / (nH)) // See genSmallTrigComboFP64 +#define MIDDLETRIG_FP64_SIZE(W,M,H) ((H) + (W) + (H)) // See genMiddleTrigFP64 // Compute the size of the largest possible trig buffer given width, middle, height (in number of float2 values) -#define SMALLTRIG_FP32_SIZE(W,M,H,nH) (W != H || H == 0 ? W : SMALLTRIGCOMBO_FP32_SIZE(W,M,H,nH)) // See genSmallTrigFP32 -#define SMALLTRIGCOMBO_FP32_SIZE(W,M,H,nH) (H + (W * M / 2 + 1) * 2 * H / nH) // See genSmallTrigComboFP32 -#define MIDDLETRIG_FP32_SIZE(W,M,H) (H + W + H) // See genMiddleTrigFP32 +#define SMALLTRIG_FP32_SIZE(W,M,H,nH) ((W) != (H) || (H) == 0 ? (W) * 5 : SMALLTRIGCOMBO_FP32_SIZE(W,M,H,nH)) // See genSmallTrigFP32 +#define SMALLTRIGCOMBO_FP32_SIZE(W,M,H,nH) ((H) * 5 + ((W) * (M) / 2 + 1) * 2 * (H) / (nH)) // See genSmallTrigComboFP32 +#define MIDDLETRIG_FP32_SIZE(W,M,H) ((H) + (W) + (H)) // See genMiddleTrigFP32 // Compute the size of the largest possible trig buffer given width, middle, height (in number of uint2 values) -#define SMALLTRIG_GF31_SIZE(W,M,H,nH) (W != H || H == 0 ? W : SMALLTRIGCOMBO_GF31_SIZE(W,M,H,nH)) // See genSmallTrigGF31 -#define SMALLTRIGCOMBO_GF31_SIZE(W,M,H,nH) (H + (W * M / 2 + 1) * 2 * H / nH) // See genSmallTrigComboGF31 -#define MIDDLETRIG_GF31_SIZE(W,M,H) (H * (M - 1) + W + H) // See genMiddleTrigGF31 +#define SMALLTRIG_GF31_SIZE(W,M,H,nH) ((W) != (H) || (H) == 0 ? (W) : SMALLTRIGCOMBO_GF31_SIZE(W,M,H,nH)) // See genSmallTrigGF31 +#define SMALLTRIGCOMBO_GF31_SIZE(W,M,H,nH) ((H) + ((W) * (M) / 2 + 1) * 2 * (H) / (nH)) // See genSmallTrigComboGF31 +#define MIDDLETRIG_GF31_SIZE(W,M,H) ((H) * ((M) - 1) + (W) + (H)) // See genMiddleTrigGF31 // Compute the size of the largest possible trig buffer given width, middle, height (in number of ulong2 values) -#define SMALLTRIG_GF61_SIZE(W,M,H,nH) (W != H || H == 0 ? W : SMALLTRIGCOMBO_GF61_SIZE(W,M,H,nH)) // See genSmallTrigGF61 -#define SMALLTRIGCOMBO_GF61_SIZE(W,M,H,nH) (H + (W * M / 2 + 1) * 2 * H / nH) // See genSmallTrigComboGF61 -#define MIDDLETRIG_GF61_SIZE(W,M,H) (H * (M - 1) + W + H) // See genMiddleTrigGF61 +#define SMALLTRIG_GF61_SIZE(W,M,H,nH) ((W) != (H) || (H) == 0 ? (W) : SMALLTRIGCOMBO_GF61_SIZE(W,M,H,nH)) // See genSmallTrigGF61 +#define SMALLTRIGCOMBO_GF61_SIZE(W,M,H,nH) ((H) + ((W) * (M) / 2 + 1) * 2 * (H) / (nH)) // See genSmallTrigComboGF61 +#define MIDDLETRIG_GF61_SIZE(W,M,H) ((H) * ((M) - 1) + (W) + (H)) // See genMiddleTrigGF61 // Convert above sizes to distances (in units of double2) #define SMALLTRIG_FP64_DIST(W,M,H,nH) SMALLTRIG_FP64_SIZE(W,M,H,nH) diff --git a/src/TuneEntry.cpp b/src/TuneEntry.cpp index c3288d24..dd741105 100644 --- a/src/TuneEntry.cpp +++ b/src/TuneEntry.cpp @@ -3,14 +3,15 @@ #include "CycleFile.h" #include +#include // Returns whether *results* was updated. bool TuneEntry::update(vector& results) const { - u32 maxExp = fft.maxExp(); + u64 const maxExp = fft.maxExp(); [[maybe_unused]] bool didErase = false; int i{}; - for (i = results.size() - 1; i >= 0 && results[i].cost > cost; --i) { + for (i = int(results.size()) - 1; i >= 0 && results[i].cost > cost; --i) { if (results[i].fft.maxExp() <= maxExp) { results.erase(std::next(results.begin(), i)); didErase = true; @@ -28,11 +29,11 @@ bool TuneEntry::update(vector& results) const { // Returns whether entry *e* represents an improvement over *results* (i.e. would update the results). bool TuneEntry::willUpdate(const vector& results) const { - u32 maxExp = fft.maxExp(); + u64 const maxExp = fft.maxExp(); for (const auto& r : results) { if (r.cost > cost) { break; - } else if (r.fft.maxExp() >= maxExp) { + } if (r.fft.maxExp() >= maxExp) { return false; } } @@ -51,7 +52,7 @@ vector TuneEntry::readTuneFile(const Args& args) { File fi = File::openRead(tuneFile); if (!fi) { return {}; } - [[maybe_unused]] u32 prevMaxExp{}; + [[maybe_unused]] u64 prevMaxExp{}; [[maybe_unused]] double prevCost{}; for (const string& line : fi) { @@ -59,8 +60,9 @@ vector TuneEntry::readTuneFile(const Args& args) { double cost{}; if (sscanf(line.c_str(), "%lf %31s", &cost, specBuf) < 2) { log("tune.txt line '%s' ignored\n", line.c_str()); + continue; // otherwise specBuf below is uninitialised } - FFTConfig fft{specBuf}; + FFTConfig const fft{specBuf}; assert(cost >= prevCost && fft.maxExp() > prevMaxExp); prevCost = cost; prevMaxExp = fft.maxExp(); @@ -71,14 +73,14 @@ vector TuneEntry::readTuneFile(const Args& args) { } void TuneEntry::writeTuneFile(const vector& results) { - [[maybe_unused]] u32 prevMaxExp{}; + [[maybe_unused]] u64 prevMaxExp{}; [[maybe_unused]] double prevCost{}; CycleFile tune{"tune.txt"}; for (const TuneEntry& r : results) { - u32 maxExp = r.fft.maxExp(); + u64 const maxExp = r.fft.maxExp(); assert(r.cost >= prevCost && maxExp > prevMaxExp); prevCost = r.cost; prevMaxExp = maxExp; - tune->printf("%6.1f %14s # %u\n", r.cost, r.fft.spec().c_str(), maxExp); + tune->printf("%6.1f %14s # %" PRIu64 "\n", r.cost, r.fft.spec().c_str(), maxExp); } } diff --git a/src/TuneEntry.h b/src/TuneEntry.h index 79a0d06a..73a86e9e 100644 --- a/src/TuneEntry.h +++ b/src/TuneEntry.h @@ -14,7 +14,7 @@ class TuneEntry { FFTConfig fft; bool update(std::vector&) const; - bool willUpdate(const vector&) const; + [[nodiscard]] bool willUpdate(const vector&) const; static vector readTuneFile(const Args& args); static void writeTuneFile(const vector&); diff --git a/src/U128.cpp b/src/U128.cpp new file mode 100644 index 00000000..6b2b858d --- /dev/null +++ b/src/U128.cpp @@ -0,0 +1,285 @@ +/******************************************************************* +* +* Author: Kareem Omar +* kareem.h.omar@gmail.com +* https://github.com/komrad36 +* +* Last updated Feb 15, 2021 +*******************************************************************/ + +#include +#include +#include +#include + +#include "U128.h" + +using I8 = int8_t; +using I16 = int16_t; +using I32 = int32_t; +using I64 = int64_t; + +using U8 = uint8_t; +using U16 = uint16_t; +using U32 = uint32_t; +using U64 = uint64_t; + +static inline bool FitsHardwareDivL(U64 nHi, U64 nLo, U64 d) +{ + return !(nHi | (d >> 32)) && nLo < (d << 32); +} + +static inline U64 HardwareDivL(U64 n, U64 d, U64& rem) +{ + U32 rLo; + const U32 qLo = _udiv64(n, U32(d), &rLo); + rem = rLo; + return qLo; +} + +static inline U64 HardwareDivQ(U64 nHi, U64 nLo, U64 d, U64& rem) +{ + nLo = _udiv128(nHi, nLo, d, &nHi); + rem = nHi; + return nLo; +} + +static inline bool IsPow2(U64 hi, U64 lo) +{ + const U64 T = hi | lo; + return !((hi & lo) | (T & (T - 1))); +} + +static inline U64 CountTrailingZeros(U64 hi, U64 lo) +{ + const U64 nLo = _tzcnt_u64(lo); + const U64 nHi = 64ULL + _tzcnt_u64(hi); + return lo ? nLo : nHi; +} + +static inline U64 CountLeadingZeros(U64 hi, U64 lo) +{ + const U64 nLo = 64ULL + _lzcnt_u64(lo); + const U64 nHi = _lzcnt_u64(hi); + return hi ? nHi : nLo; +} + +static inline U128 MaskBitsBelow(U64 hi, U64 lo, U64 n) +{ + return U128(_bzhi_u64(hi, U32(n < 64 ? 0 : n - 64)), _bzhi_u64(lo, U32(n))); +} + +U128 DivMod(U128 N, U128 D, U128& rem) +{ + if (D > N) + { + rem = N; + return 0; + } + + U64 nHi = N.m_hi; + U64 nLo = N.m_lo; + U64 dHi = D.m_hi; + U64 dLo = D.m_lo; + + if (IsPow2(dHi, dLo)) + { + const U64 n = CountTrailingZeros(dHi, dLo); + rem = MaskBitsBelow(nHi, nLo, n); + return N >> n; + } + + if (!dHi) + { + if (nHi < dLo) + { + U64 remLo; + U64 Q; + if (FitsHardwareDivL(nHi, nLo, dLo)) + Q = HardwareDivL(nLo, dLo, remLo); + else + Q = HardwareDivQ(nHi, nLo, dLo, remLo); + rem = remLo; + return Q; + } + + U64 remLo; + const U64 qHi = HardwareDivQ(0, nHi, dLo, remLo); + const U64 qLo = HardwareDivQ(remLo, nLo, dLo, remLo); + rem = remLo; + return U128(qHi, qLo); + } + + U64 n = _lzcnt_u64(dHi) - _lzcnt_u64(nHi); + + dHi = __shiftleft128(dLo, dHi, U8(n)); + dLo <<= n; + + U64 Q = 0; + ++n; + + do + { + U64 tLo, tHi; + unsigned char carry = _subborrow_u64(_subborrow_u64(0, nLo, dLo, &tLo), nHi, dHi, &tHi); + nLo = !carry ? tLo : nLo; + nHi = !carry ? tHi : nHi; + Q = (Q << 1) + !carry; + dLo = __shiftright128(dLo, dHi, 1); + dHi >>= 1; + } while (--n); + + rem = U128(nHi, nLo); + return Q; +} + +U128::U128(float x) +{ + const U32 bits = U32(_mm_cvtsi128_si32(_mm_castps_si128(_mm_set_ss(x)))); + const U32 s = bits >> 31; + + // technically UB but let's be nice + if (s) + { + m_hi = m_lo = 0ULL; + return; + } + + const U32 e = (bits >> 23) - 127; + const U32 m = (bits & ((1U << 23) - 1U)) | (1U << 23); + + // again, technically UB but let's be nice + if (e >= 128) + { + m_hi = m_lo = ~0ULL; + return; + } + + if (e >= 23) + *this = U128(m) << (e - 23); + else + *this = m >> (23 - e); +} + +U128::U128(double x) +{ + const U64 bits = U64(_mm_cvtsi128_si64(_mm_castpd_si128(_mm_set_sd(x)))); + const U64 s = bits >> 63; + + // technically UB but let's be nice + if (s) + { + m_hi = m_lo = 0ULL; + return; + } + + const U64 e = (bits >> 52) - 1023; + const U64 m = (bits & ((1ULL << 52) - 1ULL)) | (1ULL << 52); + + // again, technically UB but let's be nice + if (e >= 128) + { + m_hi = m_lo = ~0ULL; + return; + } + + if (e >= 52) + *this = U128(m) << (e - 52); + else + *this = m >> (52 - e); +} + +U128::operator float() const +{ + if (!*this) + return 0.0f; + + const U32 numBits = 128U - U32(CountLeadingZeros(m_hi, m_lo)); + + U32 bits; + + if (numBits <= 24) + { + const U32 m = (U32(m_lo) << (24 - numBits)) & ~(1U << 23); + const U32 e = numBits + 126; + bits = (e << 23) | m; + } + else + { + const U32 s = numBits - 24; + const U32 m = U32(*this >> s) & ~(1U << 23); + const U32 G = U32(*this >> (s - 1)); + const U32 R = U32(bool(MaskBitsBelow(m_hi, m_lo, s < 2 ? 0 : s - 2))); + const U32 e = numBits + 126; + bits = ((e << 23) | m) + (G & (R | m) & 1U); + } + + return _mm_cvtss_f32(_mm_castsi128_ps(_mm_cvtsi32_si128((I32)bits))); +} + +U128::operator double() const +{ + if (!*this) + return 0.0; + + const U64 numBits = 128ULL - CountLeadingZeros(m_hi, m_lo); + + U64 bits; + + if (numBits <= 53) + { + const U64 m = (m_lo << (53 - numBits)) & ~(1ULL << 52); + const U64 e = numBits + 1022; + bits = (e << 52) | m; + } + else + { + const U64 s = numBits - 53; + const U64 m = U64(*this >> s) & ~(1ULL << 52); + const U64 G = U64(*this >> (s - 1)); + const U64 R = U64(bool(MaskBitsBelow(m_hi, m_lo, s < 2 ? 0 : s - 2))); + const U64 e = numBits + 1022; + bits = ((e << 52) | m) + (G & (R | m) & 1ULL); + } + + return _mm_cvtsd_f64(_mm_castsi128_pd(_mm_cvtsi64_si128((I64)bits))); +} + +void U128::ToString(char* buf, U64 base/* = 10*/) const +{ + U64 i = 0; + if (base >= 2 && base <= 36) + { + U128 n = *this; + U128 r, b = base; + do + { + n = DivMod(n, b, r); + const char c(r); + buf[i++] = c + (c >= 10 ? '7' : '0'); + } while (n); + + for (U64 j = 0; j < (i >> 1); ++j) + { + const char t = buf[j]; + buf[j] = buf[i - j - 1]; + buf[i - j - 1] = t; + } + } + buf[i] = '\0'; +} + +std::ostream& operator<<(std::ostream& os, const U128& x) +{ + char buf[40]; + x.ToString(buf); + os << buf; + return os; +} + +const char* NatVisStr_DebugOnly(const U128& x) +{ + static char buf[40]; + x.ToString(buf); + return buf; +} diff --git a/src/U128.h b/src/U128.h new file mode 100644 index 00000000..2379a8f1 --- /dev/null +++ b/src/U128.h @@ -0,0 +1,390 @@ +/******************************************************************* +* +* Author: Kareem Omar +* kareem.h.omar@gmail.com +* https://github.com/komrad36 +* +* Last updated Feb 15, 2021 +*******************************************************************/ + +#pragma once + +#include +#include +#include + +using I8 = int8_t; +using I16 = int16_t; +using I32 = int32_t; +using I64 = int64_t; + +using U8 = uint8_t; +using U16 = uint16_t; +using U32 = uint32_t; +using U64 = uint64_t; + +#define MAKE_BINARY_OP_HELPERS(op) \ +friend auto operator op(const U128& x, U8 y) { return operator op(x, (U128)y); } \ +friend auto operator op(const U128& x, U16 y) { return operator op(x, (U128)y); } \ +friend auto operator op(const U128& x, U32 y) { return operator op(x, (U128)y); } \ +friend auto operator op(const U128& x, U64 y) { return operator op(x, (U128)y); } \ +friend auto operator op(const U128& x, I8 y) { return operator op(x, (U128)y); } \ +friend auto operator op(const U128& x, I16 y) { return operator op(x, (U128)y); } \ +friend auto operator op(const U128& x, I32 y) { return operator op(x, (U128)y); } \ +friend auto operator op(const U128& x, I64 y) { return operator op(x, (U128)y); } \ +friend auto operator op(const U128& x, char y) { return operator op(x, (U128)y); } \ +friend auto operator op(U8 x, const U128& y) { return operator op((U128)x, y); } \ +friend auto operator op(U16 x, const U128& y) { return operator op((U128)x, y); } \ +friend auto operator op(U32 x, const U128& y) { return operator op((U128)x, y); } \ +friend auto operator op(U64 x, const U128& y) { return operator op((U128)x, y); } \ +friend auto operator op(I8 x, const U128& y) { return operator op((U128)x, y); } \ +friend auto operator op(I16 x, const U128& y) { return operator op((U128)x, y); } \ +friend auto operator op(I32 x, const U128& y) { return operator op((U128)x, y); } \ +friend auto operator op(I64 x, const U128& y) { return operator op((U128)x, y); } \ +friend auto operator op(char x, const U128& y) { return operator op((U128)x, y); } + +#define MAKE_BINARY_OP_HELPERS_FLOAT(op) \ +friend auto operator op(const U128& x, float y) { return (float)x op y; } \ +friend auto operator op(const U128& x, double y) { return (double)x op y; } \ +friend auto operator op(float x, const U128& y) { return x op (float)y; } \ +friend auto operator op(double x, const U128& y) { return x op (double)y; } + +#define MAKE_BINARY_OP_HELPERS_U64(op) \ +friend U128 operator op(const U128& x, U8 n) { return operator op(x, (U64)n); } \ +friend U128 operator op(const U128& x, U16 n) { return operator op(x, (U64)n); } \ +friend U128 operator op(const U128& x, U32 n) { return operator op(x, (U64)n); } \ +friend U128 operator op(const U128& x, I8 n) { return operator op(x, (U64)n); } \ +friend U128 operator op(const U128& x, I16 n) { return operator op(x, (U64)n); } \ +friend U128 operator op(const U128& x, I32 n) { return operator op(x, (U64)n); } \ +friend U128 operator op(const U128& x, I64 n) { return operator op(x, (U64)n); } \ +friend U128 operator op(const U128& x, const U128& n) { return operator op(x, (U64)n); } + +class U128 +{ +public: + friend U128 DivMod(U128 n, U128 d, U128& rem); + + U128() = default; + U128(U8 x) : m_lo(x), m_hi(0) {} + U128(U16 x) : m_lo(x), m_hi(0) {} + U128(U32 x) : m_lo(x), m_hi(0) {} + U128(U64 x) : m_lo(x), m_hi(0) {} + U128(I8 x) : m_lo(I64(x)), m_hi(I64(x) >> 63) {} + U128(I16 x) : m_lo(I64(x)), m_hi(I64(x) >> 63) {} + U128(I32 x) : m_lo(I64(x)), m_hi(I64(x) >> 63) {} + U128(I64 x) : m_lo(I64(x)), m_hi(I64(x) >> 63) {} + U128(U64 hi, U64 lo) : m_lo(lo), m_hi(hi) {} + + // inexact values truncate, as per the Standard [conv.fpint] + // passing values unrepresentable in the destination format is undefined behavior, + // as per the Standard, but this implementation saturates + U128(float x); + + // inexact values truncate, as per the Standard [conv.fpint] + // passing values unrepresentable in the destination format is undefined behavior, + // as per the Standard, but this implementation saturates + U128(double x); + + U128& operator+=(const U128& x) + { + static_cast(_addcarry_u64(_addcarry_u64(0, m_lo, x.m_lo, &m_lo), m_hi, x.m_hi, &m_hi)); + return *this; + } + + friend U128 operator+(const U128& x, const U128& y) + { + U128 ret; + static_cast(_addcarry_u64(_addcarry_u64(0, x.m_lo, y.m_lo, &ret.m_lo), x.m_hi, y.m_hi, &ret.m_hi)); + return ret; + } + + MAKE_BINARY_OP_HELPERS(+); + MAKE_BINARY_OP_HELPERS_FLOAT(+); + + U128& operator-=(const U128& x) + { + static_cast(_subborrow_u64(_subborrow_u64(0, m_lo, x.m_lo, &m_lo), m_hi, x.m_hi, &m_hi)); + return *this; + } + + friend U128 operator-(const U128& x, const U128& y) + { + U128 ret; + static_cast(_subborrow_u64(_subborrow_u64(0, x.m_lo, y.m_lo, &ret.m_lo), x.m_hi, y.m_hi, &ret.m_hi)); + return ret; + } + + MAKE_BINARY_OP_HELPERS(-); + MAKE_BINARY_OP_HELPERS_FLOAT(-); + + U128& operator*=(const U128& x) + { + // ab * cd + // == + // (2^64*a + b) * (2^64*c + d) + // if a*c == e, a*d == f, b*c == g, b*d == h + // |ee|ee| | | + // | |fg|fg| | + // | | |hh|hh| + + U64 hHi; + const U64 hLo = _umul128(m_lo, x.m_lo, &hHi); + m_hi = hHi + m_hi * x.m_lo + m_lo * x.m_hi; + m_lo = hLo; + return *this; + } + + friend U128 operator*(const U128& x, const U128& y) + { + U128 ret; + U64 hHi; + ret.m_lo = _umul128(x.m_lo, y.m_lo, &hHi); + ret.m_hi = hHi + y.m_hi * x.m_lo + y.m_lo * x.m_hi; + return ret; + } + + MAKE_BINARY_OP_HELPERS(*); + MAKE_BINARY_OP_HELPERS_FLOAT(*); + + U128& operator/=(const U128& x) + { + U128 rem; + *this = DivMod(*this, x, rem); + return *this; + } + + friend U128 operator/(const U128& x, const U128& y) + { + U128 rem; + return DivMod(x, y, rem); + } + + MAKE_BINARY_OP_HELPERS(/); + MAKE_BINARY_OP_HELPERS_FLOAT(/); + + U128& operator%=(const U128& x) + { + static_cast(DivMod(*this, x, *this)); + return *this; + } + + friend U128 operator%(const U128& x, const U128& y) + { + U128 ret; + static_cast(DivMod(x, y, ret)); + return ret; + } + + MAKE_BINARY_OP_HELPERS(%); + + U128& operator&=(const U128& x) + { + m_hi &= x.m_hi; + m_lo &= x.m_lo; + return *this; + } + + friend U128 operator&(const U128& x, const U128& y) + { + return U128(x.m_hi & y.m_hi, x.m_lo & y.m_lo); + } + + MAKE_BINARY_OP_HELPERS(&); + + U128& operator|=(const U128& x) + { + m_hi |= x.m_hi; + m_lo |= x.m_lo; + return *this; + } + + friend U128 operator|(const U128& x, const U128& y) + { + return U128(x.m_hi | y.m_hi, x.m_lo | y.m_lo); + } + + MAKE_BINARY_OP_HELPERS(|); + + U128& operator^=(const U128& x) + { + m_hi ^= x.m_hi; + m_lo ^= x.m_lo; + return *this; + } + + friend U128 operator^(const U128& x, const U128& y) + { + return U128(x.m_hi ^ y.m_hi, x.m_lo ^ y.m_lo); + } + + MAKE_BINARY_OP_HELPERS(^); + + U128& operator>>=(U64 n) + { + const U64 lo = __shiftright128(m_lo, m_hi, (U8)n); + const U64 hi = m_hi >> (n & 63ULL); + + m_lo = n & 64 ? hi : lo; + m_hi = n & 64 ? 0 : hi; + + return *this; + } + + friend U128 operator>>(const U128& x, U64 n) + { + U128 ret; + + const U64 lo = __shiftright128(x.m_lo, x.m_hi, (U8)n); + const U64 hi = x.m_hi >> (n & 63ULL); + + ret.m_lo = n & 64 ? hi : lo; + ret.m_hi = n & 64 ? 0 : hi; + + return ret; + } + + MAKE_BINARY_OP_HELPERS_U64(>>); + + U128& operator<<=(U64 n) + { + const U64 hi = __shiftleft128(m_lo, m_hi, (U8)n); + const U64 lo = m_lo << (n & 63ULL); + + m_hi = n & 64 ? lo : hi; + m_lo = n & 64 ? 0 : lo; + + return *this; + } + + friend U128 operator<<(const U128& x, U64 n) + { + U128 ret; + + const U64 hi = __shiftleft128(x.m_lo, x.m_hi, (U8)n); + const U64 lo = x.m_lo << (n & 63ULL); + + ret.m_hi = n & 64 ? lo : hi; + ret.m_lo = n & 64 ? 0 : lo; + + return ret; + } + + MAKE_BINARY_OP_HELPERS_U64(<<); + + friend U128 operator~(const U128& x) + { + return U128(~x.m_hi, ~x.m_lo); + } + + friend U128 operator+(const U128& x) + { + return x; + } + + friend U128 operator-(const U128& x) + { + U128 ret; + static_cast(_subborrow_u64(_subborrow_u64(0, 0, x.m_lo, &ret.m_lo), 0, x.m_hi, &ret.m_hi)); + return ret; + } + + U128& operator++() + { + operator+=(1); + return *this; + } + + U128 operator++(int) + { + const U128 x = *this; + operator++(); + return x; + } + + U128& operator--() + { + operator-=(1); + return *this; + } + + U128 operator--(int) + { + const U128 x = *this; + operator--(); + return x; + } + + friend bool operator<(const U128& x, const U128& y) + { + U64 unusedLo, unusedHi; + return _subborrow_u64(_subborrow_u64(0, x.m_lo, y.m_lo, &unusedLo), x.m_hi, y.m_hi, &unusedHi); + } + MAKE_BINARY_OP_HELPERS(<); + MAKE_BINARY_OP_HELPERS_FLOAT(<); + + friend bool operator>(const U128& x, const U128& y) { return y < x; } + MAKE_BINARY_OP_HELPERS(>); + MAKE_BINARY_OP_HELPERS_FLOAT(>); + + friend bool operator<=(const U128& x, const U128& y) { return !(x > y); } + MAKE_BINARY_OP_HELPERS(<=); + MAKE_BINARY_OP_HELPERS_FLOAT(<=); + + friend bool operator>=(const U128& x, const U128& y) { return !(x < y); } + MAKE_BINARY_OP_HELPERS(>=); + MAKE_BINARY_OP_HELPERS_FLOAT(>=); + + friend bool operator==(const U128& x, const U128& y) + { + return !((x.m_hi ^ y.m_hi) | (x.m_lo ^ y.m_lo)); + } + MAKE_BINARY_OP_HELPERS(==); + MAKE_BINARY_OP_HELPERS_FLOAT(==); + + friend bool operator!=(const U128& x, const U128& y) { return !(x == y); } + MAKE_BINARY_OP_HELPERS(!=); + MAKE_BINARY_OP_HELPERS_FLOAT(!=); + + explicit operator bool() const { return m_hi | m_lo; } + + operator U8 () const { return (U8) m_lo; } + operator U16() const { return (U16)m_lo; } + operator U32() const { return (U32)m_lo; } + operator U64() const { return (U64)m_lo; } + + operator I8 () const { return (I8) m_lo; } + operator I16() const { return (I16)m_lo; } + operator I32() const { return (I32)m_lo; } + operator I64() const { return (I64)m_lo; } + + operator char() const { return (char)m_lo; } + + // rounding method is implementation-defined as per the Standard [conv.fpint] + // this implementation performs IEEE 754-compliant "round half to even" rounding to nearest, + // regardless of the current FPU rounding mode, which matches the behavior of clang and GCC + operator float() const; + + // rounding method is implementation-defined as per the Standard [conv.fpint] + // this implementation performs IEEE 754-compliant "round half to even" rounding to nearest, + // regardless of the current FPU rounding mode, which matches the behavior of clang and GCC + operator double() const; + + // caller is responsible for ensuring that buf has space for the U128 AND the null terminator + // that follows, in the given output base. + // Common bases and worst-case size requirements: + // Base 2: 129 bytes (128 + null terminator) + // Base 8: 44 bytes ( 43 + null terminator) + // Base 10: 40 bytes ( 39 + null terminator) + // Base 16: 33 bytes ( 32 + null terminator) + void ToString(char* buf, U64 base = 10) const; + +private: + U64 m_lo; + U64 m_hi; +}; + +#undef MAKE_BINARY_OP_HELPERS +#undef MAKE_BINARY_OP_HELPERS_FLOAT +#undef MAKE_BINARY_OP_HELPERS_U64 + +std::ostream& operator<<(std::ostream& os, const U128& x); diff --git a/src/Worktodo.cpp b/src/Worktodo.cpp index 0a981a39..7d0239ba 100644 --- a/src/Worktodo.cpp +++ b/src/Worktodo.cpp @@ -7,11 +7,15 @@ #include "common.h" #include "Args.h" #include "fs.h" +#include "Primes.h" #include #include #include #include +#include +#include +#include namespace { @@ -36,7 +40,7 @@ std::optional parse(const std::string& line) { bool isCERT = false; if (topParts.size() == 2) { - string kind = topParts.front(); + const string& kind = topParts.front(); if (kind == "PRP" || kind == "PRPDC") { isPRP = true; } else if (kind == "Test" || kind == "DoubleCheck") { @@ -58,14 +62,23 @@ std::optional parse(const std::string& line) { parts.erase(parts.begin()); } - string s = (parts.size() >= 4 && parts[0] == "1" && parts[1] == "2" && parts[3] == "-1") ? parts[2] - : (!parts.empty() ? parts[0] : ""); + // PRP lines are "k,b,n,c,..." and only k=1, b=2, c=-1 is a Mersenne number; anything else (k*2^n-1, 2^n+1, base 3) + // is not ours and must not be run as exponent k. The bare "E,..." form is only used by Test=/DoubleCheck= lines. + bool const mersenne = parts.size() >= 4 && parts[0] == "1" && parts[1] == "2" && (parts[3] == "-1" || parts[3] == "-1\n"); + string const s = mersenne ? parts[2] : ((isLL && !parts.empty()) ? parts[0] : ""); const char *end = s.c_str() + s.size(); u64 exp{}; auto [ptr, _] = from_chars(s.c_str(), end, exp, 10); if (ptr != end) { exp = 0; } - if (exp > 1000) { return {{isPRP ? Task::PRP : Task::LL, u32(exp), AID, line, 0}}; } + // Task::execute silently retargets a composite exponent to the previous prime. That is a convenience for + // "-prp " timing runs; an assignment line with a composite exponent is a mistake, and running a + // different exponent under its AID would report the wrong result. Ignore the line instead. + if (exp > 1000 && !Primes{}.isPrime(exp)) { + log("worktodo.txt line ignored, exponent %" PRIu64 " is not prime: \"%s\"\n", exp, rstripNewline(line).c_str()); + return {}; + } + if (exp > 1000) { return {{.kind=isPRP ? Task::PRP : Task::LL, .exponent=exp, .AID=AID, .line=line, .squarings=0}}; } } if (isCERT) { vector parts = split(topParts.back(), ','); @@ -84,7 +97,7 @@ std::optional parse(const std::string& line) { u64 squarings{0}; from_chars(s.c_str(), end, squarings, 10); //printf ("Exec cert %d %d \n", (int) exp, (int) squarings); - if (exp > 1000 && squarings > 100) { return {{Task::CERT, u32(exp), AID, line, u32(squarings) }}; } + if (exp > 1000 && squarings > 100) { return {{.kind=Task::CERT, .exponent=exp, .AID=AID, .line=line, .squarings=u32(squarings) }}; } } } } @@ -97,6 +110,12 @@ static std::optional bestTask(const fs::path& fileName, bool smallest) { optional best; for (const string& line : File::openRead(fileName)) { optional task = parse(line); + // A Cert line whose start-value file is not here cannot run: isCERT would throw and end the worker, and since + // Cert lines take priority over PRP/LL the worker would be wedged for good. Skip the line until the file appears. + if (task && task->kind == Task::CERT && !std::filesystem::exists("M" + to_string(task->exponent) + ".cert")) { + log("Cert start file M%" PRIu64 ".cert not found; skipping that worktodo line for now\n", task->exponent); + continue; + } if (task && (!best || (best->kind != Task::CERT && task->kind == Task::CERT) || ((best->kind != Task::CERT || task->kind == Task::CERT) && smallest && task->exponent < best->exponent))) { @@ -109,14 +128,16 @@ static std::optional bestTask(const fs::path& fileName, bool smallest) { string workName(i32 instance) { return "worktodo-" + to_string(instance) + ".txt"; } optional getWork(Args& args, i32 instance) { - fs::path localWork = workName(instance); + string filename = workName(instance); // Used for printf statements. Using fd::path is problematic because it 8-bit char in Linux and 16-bit char in Windows. + fs::path const localWork = filename; // Try to get a task from the local worktodo- file. if (optional task = bestTask(localWork, args.smallest)) { return task; } - if (args.masterDir.empty()) { return {}; } + if (args.masterDir.empty()) { log("No work to do found. Add work to %s.\n", filename.c_str()); return {}; } - fs::path worktodo = args.masterDir / "worktodo.txt"; + filename = "worktodo.txt"; + fs::path const worktodo = args.masterDir / filename; /* We need to aquire a task from the global worktodo.txt, and "atomically" @@ -136,14 +157,19 @@ optional getWork(Args& args, i32 instance) { 8. start again (from step 1) */ + // The size heuristic below guards against other processes. Within this process the workers start together and + // would all read the same file and pick the same task, so serialize the claim itself. + static std::mutex claimMutex; + std::lock_guard const claimLock(claimMutex); + for (int retry = 0; retry < 2; ++retry) { - u64 initialSize = fileSize(worktodo); + u64 const initialSize = fileSize(worktodo); if (!initialSize) { return {}; } optional task = bestTask(worktodo, args.smallest); if (!task) { return {}; } - string workLine = task->line; + string const workLine = task->line; File::append(localWork, workLine); if (deleteLine(worktodo, workLine, initialSize)) { @@ -151,12 +177,12 @@ optional getWork(Args& args, i32 instance) { } // Undo add to local worktodo. Attempt twice. - bool found = deleteLine(localWork, workLine) || deleteLine(localWork, workLine); + bool const found = deleteLine(localWork, workLine) || deleteLine(localWork, workLine); assert(found); if (!found) { return {}; } } - log("Could not extract a task from '%s'\n", worktodo.string().c_str()); + log("Could not extract a task from '%s'\n", filename.c_str()); // must be tough luck to be preempted twice while mutating the global worktodo assert(false); return {}; @@ -167,14 +193,14 @@ optional getWork(Args& args, i32 instance) { std::optional Worktodo::getTask(Args &args, i32 instance) { if (instance == 0) { if (args.prpExp) { - u32 exp = args.prpExp; + u64 const exp = args.prpExp; args.prpExp = 0; - return Task{Task::PRP, exp}; - } else if (args.llExp) { - u32 exp = args.llExp; + return Task{.kind=Task::PRP, .exponent=exp}; + } if (args.llExp) { + u64 const exp = args.llExp; args.llExp = 0; - return Task{Task::LL, exp}; - } else if (!args.verifyPath.empty()) { + return Task{.kind=Task::LL, .exponent=exp}; + } if (!args.verifyPath.empty()) { auto path = args.verifyPath; args.verifyPath.clear(); return Task{.kind=Task::VERIFY, .verifyPath=path}; diff --git a/src/cl/base.cl b/src/cl/base.cl index df1ef02b..02ffceeb 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -34,6 +34,8 @@ G_H "group height" == SMALL_HEIGHT / NH #define STR(x) XSTR(x) #define XSTR(x) #x +#pragma clang diagnostic ignored "-Wconstant-logical-operand" + #define OVERLOAD __attribute__((overloadable)) #pragma OPENCL FP_CONTRACT ON @@ -58,6 +60,13 @@ G_H "group height" == SMALL_HEIGHT / NH //__builtin_assume(condition) #endif // DEBUG +#ifndef AMDGPU +#define AMDGPU 0 +#endif +#ifndef NVIDIAGPU +#define NVIDIAGPU 0 +#endif + #if NO_ASM #define HAS_ASM 0 #define HAS_PTX 0 @@ -66,7 +75,7 @@ G_H "group height" == SMALL_HEIGHT / NH #define HAS_PTX 0 #elif NVIDIAGPU #define HAS_ASM 0 -#define HAS_PTX 1200 // Assume CUDA 12.00 support until we can figure out how to automatically determine this at runtime +#define HAS_PTX CC // C code computed the nVidia GPU's compute capability #else #define HAS_ASM 0 #define HAS_PTX 0 @@ -82,6 +91,16 @@ G_H "group height" == SMALL_HEIGHT / NH #define OLD_FENCE 1 #endif +// The default is the in-place FFT data layout for nVidia GPUs, not in-place otherwise. +// This must match the in_place default in clDefines() in Gpu.cpp. +#if !defined(INPLACE) +#if NVIDIAGPU +#define INPLACE 1 +#else +#define INPLACE 0 +#endif +#endif + // Nontemporal reads and writes might be a little bit faster on many GPUs by keeping more reusable data in the caches. // However, on those GPUs with large caches there should be a significant speed gain from keeping FFT data in the caches. // Default to the big win when caching is beneficial rather than the tiny gain when non-temporal is better. @@ -128,8 +147,31 @@ G_H "group height" == SMALL_HEIGHT / NH #endif #endif -#if !defined(BIGLIT) -#define BIGLIT 1 +// Shufl width in bytes (can be 4, 8, or 16). See fftbase.cl. Allow different shufl widths for fft_width and fft_height. +// Default is 8 bytes (one double). Historically best for Radeon VII and TitanV. This setting will affect how much LDS +// memory is needed which in turn may affect occupancy and thus performance. +#if !defined(SHUFL_BYTES_W) +#define SHUFL_BYTES_W 8 +#endif +#if !defined(SHUFL_BYTES_H) +#define SHUFL_BYTES_H 8 +#endif + +// Shufl can pad (or swizzle) to avoid LDS bank conflicts. See fftbase.cl. You would think this option would be good (or bad) +// for both fft_width and fft_height, but the rocm optimizer is super-finicky. Default to using LDS padding. +#if !defined(LDSPAD_W) +#define LDSPAD_W 1 +#endif +#if !defined(LDSPAD_H) +#define LDSPAD_H 1 +#endif + +// By default, LDS access is not shared among workgroups. +#if !defined(LDSMUL_W) +#define LDSMUL_W 1 +#endif +#if !defined(LDSMUL_H) +#define LDSMUL_H 1 #endif #if !defined(TABMUL_CHAIN) @@ -176,12 +218,21 @@ G_H "group height" == SMALL_HEIGHT / NH #define ZEROHACK_H 1 #endif +#if !defined(MULTI_Q) +#define MULTI_Q 0 +#endif + +#if !defined(L2_STRIPING) +#define L2_STRIPING 0 +#endif + // Expected defines: EXP the exponent. // WIDTH, SMALL_HEIGHT, MIDDLE. #define BIG_HEIGHT (SMALL_HEIGHT * MIDDLE) #define ND (WIDTH * BIG_HEIGHT) #define NWORDS (ND * 2u) +#define NWORDS_IS_POWER_OF_TWO !(NWORDS & (NWORDS - 1)) #if (NW != 4 && NW != 8) || (NH != 4 && NH != 8) #error NW and NH must be passed in, expected value 4 or 8. @@ -235,9 +286,10 @@ error - unsupported integer WordSize double2 OVERLOAD U2(double a, double b) { return (double2) (a, b); } float2 OVERLOAD U2(float a, float b) { return (float2) (a, b); } int2 OVERLOAD U2(int a, int b) { return (int2) (a, b); } -long2 OVERLOAD U2(long a, long b) { return (long2) (a, b); } +long2 OVERLOAD U2(i64 a, i64 b) { return (long2) (a, b); } uint2 OVERLOAD U2(uint a, uint b) { return (uint2) (a, b); } -ulong2 OVERLOAD U2(ulong a, ulong b) { return (ulong2) (a, b); } +ulong2 OVERLOAD U2(unsigned long a, unsigned long b) { return (ulong2) ((ulong)a, (ulong)b); } // Two versions dealing with longs to handle TAILTGF61 constant +ulong2 OVERLOAD U2(unsigned long long a, unsigned long long b) { return (ulong2) ((ulong)a, (ulong)b); } // Other handy macros #define RE(a) (a.x) @@ -246,108 +298,500 @@ ulong2 OVERLOAD U2(ulong a, ulong b) { return (ulong2) (a, b); } #define P(x) global x * restrict #define CP(x) const P(x) -// Macros for non-temporal load and store. The theory behind only non-temporal reads (option 2) is that with alternating buffers, -// read buffers will not be needed for quite a while, but write buffers will be needed soon. -#if NONTEMPORAL == 1 && defined(__has_builtin) && __has_builtin(__builtin_nontemporal_load) && __has_builtin(__builtin_nontemporal_store) -#define NTLOAD(mem) __builtin_nontemporal_load(&(mem)) -#define NTSTORE(mem,val) __builtin_nontemporal_store(val, &(mem)) -#elif NONTEMPORAL == 2 && defined(__has_builtin) && __has_builtin(__builtin_nontemporal_load) -#define NTLOAD(mem) __builtin_nontemporal_load(&(mem)) -#define NTSTORE(mem,val) (mem) = val +#define KERNEL(x) kernel __attribute__((reqd_work_group_size(x, 1, 1))) void + +// AMD only: Gpu.cpp can pass -DAMD_WAVES_PER_EU=n (ask for at least n waves per SIMD) or -DAMD_NUM_VGPR=n (explicit VGPR count), either of which caps +// the kernel's VGPR usage. Used to avoid the one-wave-per-SIMD occupancy cliff (more than 128 VGPRs on gfx9). See Gpu::amdRegisterOption. +#if AMDGPU && defined(AMD_NUM_VGPR) +#define KERNEL_CAP(x) kernel __attribute__((reqd_work_group_size(x, 1, 1), amdgpu_num_vgpr(AMD_NUM_VGPR))) void +#elif AMDGPU && defined(AMD_WAVES_PER_EU) +#define KERNEL_CAP(x) kernel __attribute__((reqd_work_group_size(x, 1, 1), amdgpu_waves_per_eu(AMD_WAVES_PER_EU))) void #else -#define NTLOAD(mem) (mem) -#define NTSTORE(mem,val) (mem) = val +#define KERNEL_CAP(x) KERNEL(x) #endif -// Prefetch macros. Unused at present, I tried using them in fftMiddleInGF61 on a 5080 with no benefit. -void PREFETCHL1(const __global void *addr) { -#if HAS_PTX >= 200 // Prefetch instruction requires sm_20 support or higher - __asm("prefetch.global.L1 [%0];" : : "l"(addr)); +// ENABLE_RESTRICT=1 marks the trig and weight table pointers below as restrict. That lets the compiler hoist their loads (on nVidia they become ld.global.nc), +// which is sometimes faster but can cost many more registers. Off by default. +#ifndef ENABLE_RESTRICT +#define ENABLE_RESTRICT 0 #endif -} -void PREFETCHL2(const __global void *addr) { -#if HAS_PTX >= 200 // Prefetch instruction requires sm_20 support or higher - __asm("prefetch.global.L2 [%0];" : : "l"(addr)); +#if ENABLE_RESTRICT +#define TABLE_RESTRICT restrict +#else +#define TABLE_RESTRICT #endif -} // For reasons unknown, loading trig values into nVidia's constant cache has terrible performance #if AMDGPU -typedef constant const T2* Trig; -typedef constant const T* TrigSingle; -typedef constant const F2* TrigFP32; -typedef constant const GF31* TrigGF31; -typedef constant const GF61* TrigGF61; +typedef constant const T2* TABLE_RESTRICT Trig; +typedef constant const T* TABLE_RESTRICT TrigSingle; +typedef constant const F2* TABLE_RESTRICT TrigFP32; +typedef constant const F* TABLE_RESTRICT TrigSingleFP32; +typedef constant const GF31* TABLE_RESTRICT TrigGF31; +typedef constant const GF61* TABLE_RESTRICT TrigGF61; #else -typedef global const T2* Trig; -typedef global const T* TrigSingle; -typedef global const F2* TrigFP32; -typedef global const GF31* TrigGF31; -typedef global const GF61* TrigGF61; +typedef global const T2* TABLE_RESTRICT Trig; +typedef global const T* TABLE_RESTRICT TrigSingle; +typedef global const F2* TABLE_RESTRICT TrigFP32; +typedef global const F* TABLE_RESTRICT TrigSingleFP32; +typedef global const GF31* TABLE_RESTRICT TrigGF31; +typedef global const GF61* TABLE_RESTRICT TrigGF61; #endif // However, caching weights in nVidia's constant cache improves performance. // Even better is to not pollute the constant cache with weights that are used only once. // This requires two typedefs depending on how we want to use the BigTab pointer. // For AMD we can declare BigTab as constant or global - it doesn't really matter. -typedef constant const double2* ConstBigTab; -typedef constant const float2* ConstBigTabFP32; +typedef constant const double2* TABLE_RESTRICT ConstBigTab; +typedef constant const float2* TABLE_RESTRICT ConstBigTabFP32; #if AMDGPU -typedef constant const double2* BigTab; -typedef constant const float2* BigTabFP32; +typedef constant const double2* TABLE_RESTRICT BigTab; +typedef constant const float2* TABLE_RESTRICT BigTabFP32; #else -typedef global const double2* BigTab; -typedef global const float2* BigTabFP32; +typedef global const double2* TABLE_RESTRICT BigTab; +typedef global const float2* TABLE_RESTRICT BigTabFP32; #endif -#define KERNEL(x) kernel __attribute__((reqd_work_group_size(x, 1, 1))) void +// +// nVidia GPUs have lots of different caching options for loads and stores. +// AMD GPUs have have far fewer options for loads and stores. +// These routines and macros let us try the different options. +// + +// Basic load and store. Presumably stored in all caches using a standard LRU algorithm. + +#define LOAD(mem) *(mem) +#define STORE(mem,val) *(mem) = val + +// Non-temporal load and store. + +#if defined(__has_builtin) && __has_builtin(__builtin_nontemporal_load) +#define NTLOAD(mem) __builtin_nontemporal_load(mem) +#else +#define NTLOAD LOAD +#endif + +#if defined(__has_builtin) && __has_builtin(__builtin_nontemporal_store) +#define NTSTORE(mem,val) __builtin_nontemporal_store(val, mem) +#else +#define NTSTORE STORE +#endif + +// Routines for loading data from memory into the L2 cache but not the L1 cache. -#if FFT_FP64 -void OVERLOAD read(u32 WG, u32 N, T2 *u, const global T2 *in, u32 base) { - in += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { u[i] = in[i * WG]; } +#if HAS_PTX >= 200 // Cache hints requires sm_20 support or higher +T2 OVERLOAD L2LOAD(CP(T2) mem) { + T2 retval; + __asm("ld.global.cg.v2.f64 {%0, %1}, [%2];" : "=d"(retval.x), "=d"(retval.y) : "l"(mem)); + return retval; +} +T OVERLOAD L2LOAD(TrigSingle mem) { + T retval; + __asm("ld.global.cg.f64 %0, [%1];" : "=d"(retval) : "l"(mem)); + return retval; +} +F2 OVERLOAD L2LOAD(CP(F2) mem) { + F2 retval; + __asm("ld.global.cg.v2.f32 {%0, %1}, [%2];" : "=f"(retval.x), "=f"(retval.y) : "l"(mem)); + return retval; +} +F OVERLOAD L2LOAD(TrigSingleFP32 mem) { + F retval; + __asm("ld.global.cg.f32 %0, [%1];" : "=f"(retval) : "l"(mem)); + return retval; +} +i64 OVERLOAD L2LOAD(i64 *mem) { + i64 retval; + __asm("ld.global.cg.b64 %0, [%1];" : "=l"(retval) : "l"(mem)); + return retval; +} +GF61 OVERLOAD L2LOAD(TrigGF61 mem) { + GF61 retval; + __asm("ld.global.cg.v2.b64 {%0, %1}, [%2];" : "=l"(retval.x), "=l"(retval.y) : "l"(mem)); + return retval; } +i32 OVERLOAD L2LOAD(i32 *mem) { + i32 retval; + __asm("ld.global.cg.b32 %0, [%1];" : "=r"(retval) : "l"(mem)); + return retval; +} +GF31 OVERLOAD L2LOAD(TrigGF31 mem) { + GF31 retval; + __asm("ld.global.cg.v2.b32 {%0, %1}, [%2];" : "=r"(retval.x), "=r"(retval.y) : "l"(mem)); + return retval; +} +#else +#define L2LOAD LOAD +#endif + +// Routines for storing to L2 cache bypassing L1 cache. -void OVERLOAD write(u32 WG, u32 N, T2 *u, global T2 *out, u32 base) { - out += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { out[i * WG] = u[i]; } +#if HAS_PTX >= 200 // Cache hints requires sm_20 support or higher +void OVERLOAD L2STORE(P(T2) mem, T2 val) { + __asm("st.global.cg.v2.f64 [%0], {%1, %2};" : : "l"(mem), "d"(val.x), "d"(val.y)); +} +void OVERLOAD L2STORE(P(F2) mem, F2 val) { + __asm("st.global.cg.v2.f32 [%0], {%1, %2};" : : "l"(mem), "f"(val.x), "f"(val.y)); +} +void OVERLOAD L2STORE(P(GF61) mem, GF61 val) { + __asm("st.global.cg.v2.b64 [%0], {%1, %2};" : : "l"(mem), "l"(val.x), "l"(val.y)); +} +void OVERLOAD L2STORE(P(GF31) mem, GF31 val) { + __asm("st.global.cg.v2.b32 [%0], {%1, %2};" : : "l"(mem), "r"(val.x), "r"(val.y)); +} +void OVERLOAD L2STORE(i64 *mem, i64 val) { + __asm("st.global.cg.b64 [%0], %1;" : : "l"(mem), "l"(val)); } +void OVERLOAD L2STORE(i32 *mem, i32 val) { + __asm("st.global.cg.b32 [%0], %1;" : : "l"(mem), "r"(val)); +} +#else +#define L2STORE STORE #endif -#if FFT_FP32 -void OVERLOAD read(u32 WG, u32 N, F2 *u, const global F2 *in, u32 base) { - in += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { u[i] = in[i * WG]; } +// Routines for loading data from memory into the L1 and L2 caches, but cache line is marked evict first to limit cache pollution. + +#if HAS_PTX >= 200 // Cache hints requires sm_20 support or higher +T2 OVERLOAD EFLOAD(CP(T2) mem) { + T2 retval; + __asm("ld.global.cs.v2.f64 {%0, %1}, [%2];" : "=d"(retval.x), "=d"(retval.y) : "l"(mem)); + return retval; +} +T OVERLOAD EFLOAD(TrigSingle mem) { + T retval; + __asm("ld.global.cs.f64 %0, [%1];" : "=d"(retval) : "l"(mem)); + return retval; +} +F2 OVERLOAD EFLOAD(CP(F2) mem) { + F2 retval; + __asm("ld.global.cs.v2.f32 {%0, %1}, [%2];" : "=f"(retval.x), "=f"(retval.y) : "l"(mem)); + return retval; +} +F OVERLOAD EFLOAD(TrigSingleFP32 mem) { + F retval; + __asm("ld.global.cs.f32 %0, [%1];" : "=f"(retval) : "l"(mem)); + return retval; +} +i64 OVERLOAD EFLOAD(i64 *mem) { + i64 retval; + __asm("ld.global.cs.b64 %0, [%1];" : "=l"(retval) : "l"(mem)); + return retval; +} +GF61 OVERLOAD EFLOAD(TrigGF61 mem) { + GF61 retval; + __asm("ld.global.cs.v2.b64 {%0, %1}, [%2];" : "=l"(retval.x), "=l"(retval.y) : "l"(mem)); + return retval; +} +i32 OVERLOAD EFLOAD(i32 *mem) { + i32 retval; + __asm("ld.global.cs.b32 %0, [%1];" : "=r"(retval) : "l"(mem)); + return retval; } +GF31 OVERLOAD EFLOAD(TrigGF31 mem) { + GF31 retval; + __asm("ld.global.cs.v2.b32 {%0, %1}, [%2];" : "=r"(retval.x), "=r"(retval.y) : "l"(mem)); + return retval; +} +#else +#define EFLOAD LOAD +#endif -void OVERLOAD write(u32 WG, u32 N, F2 *u, global F2 *out, u32 base) { - out += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { out[i * WG] = u[i]; } +// Routines for storing to L1 and L2 caches with cache line marked evict first. + +#if HAS_PTX >= 200 // Cache hints requires sm_20 support or higher +void OVERLOAD EFSTORE(P(T2) mem, T2 val) { + __asm("st.global.cs.v2.f64 [%0], {%1, %2};" : : "l"(mem), "d"(val.x), "d"(val.y)); +} +void OVERLOAD EFSTORE(P(F2) mem, F2 val) { + __asm("st.global.cs.v2.f32 [%0], {%1, %2};" : : "l"(mem), "f"(val.x), "f"(val.y)); +} +void OVERLOAD EFSTORE(P(GF61) mem, GF61 val) { + __asm("st.global.cs.v2.b64 [%0], {%1, %2};" : : "l"(mem), "l"(val.x), "l"(val.y)); +} +void OVERLOAD EFSTORE(P(GF31) mem, GF31 val) { + __asm("st.global.cs.v2.b32 [%0], {%1, %2};" : : "l"(mem), "r"(val.x), "r"(val.y)); +} +void OVERLOAD EFSTORE(i64 *mem, i64 val) { + __asm("st.global.cs.b64 [%0], %1;" : : "l"(mem), "l"(val)); +} +void OVERLOAD EFSTORE(i32 *mem, i32 val) { + __asm("st.global.cs.b32 [%0], %1;" : : "l"(mem), "r"(val)); } +#else +#define EFSTORE STORE #endif -#if NTT_GF31 -void OVERLOAD read(u32 WG, u32 N, GF31 *u, const global GF31 *in, u32 base) { - in += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { u[i] = in[i * WG]; } +// Routines for loading a value and marking it for "last use". + +#if HAS_PTX >= 200 // Cache hints requires sm_20 support or higher +T2 OVERLOAD LULOAD(Trig mem) { + T2 retval; + __asm("ld.global.lu.v2.f64 {%0, %1}, [%2];" : "=d"(retval.x), "=d"(retval.y) : "l"(mem)); + return retval; +} +T OVERLOAD LULOAD(TrigSingle mem) { + T retval; + __asm("ld.global.lu.f64 %0, [%1];" : "=d"(retval) : "l"(mem)); + return retval; +} +F2 OVERLOAD LULOAD(TrigFP32 mem) { + F2 retval; + __asm("ld.global.lu.v2.f32 {%0, %1}, [%2];" : "=f"(retval.x), "=f"(retval.y) : "l"(mem)); + return retval; +} +F OVERLOAD LULOAD(TrigSingleFP32 mem) { + F retval; + __asm("ld.global.lu.f32 %0, [%1];" : "=f"(retval) : "l"(mem)); + return retval; } +i64 OVERLOAD LULOAD(i64 *mem) { + i64 retval; + __asm("ld.global.lu.b64 %0, [%1];" : "=l"(retval) : "l"(mem)); + return retval; +} +GF61 OVERLOAD LULOAD(TrigGF61 mem) { + GF61 retval; + __asm("ld.global.lu.v2.b64 {%0, %1}, [%2];" : "=l"(retval.x), "=l"(retval.y) : "l"(mem)); + return retval; +} +i32 OVERLOAD LULOAD(i32 *mem) { + i32 retval; + __asm("ld.global.lu.b32 %0, [%1];" : "=r"(retval) : "l"(mem)); + return retval; +} +GF31 OVERLOAD LULOAD(TrigGF31 mem) { + GF31 retval; + __asm("ld.global.lu.v2.b32 {%0, %1}, [%2];" : "=r"(retval.x), "=r"(retval.y) : "l"(mem)); + return retval; +} +#else +#define LULOAD LOAD +#endif + +// Routines for loading a read-only value and placing it in the non-coherent texture cache. -void OVERLOAD write(u32 WG, u32 N, GF31 *u, global GF31 *out, u32 base) { - out += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { out[i * WG] = u[i]; } +#if HAS_PTX >= 500 // Texture cache requires sm_50 support or higher +T2 OVERLOAD NCLOAD(Trig mem) { + T2 retval; + __asm("ld.global.nc.v2.f64 {%0, %1}, [%2];" : "=d"(retval.x), "=d"(retval.y) : "l"(mem)); + return retval; +} +T OVERLOAD NCLOAD(TrigSingle mem) { + T retval; + __asm("ld.global.nc.f64 %0, [%1];" : "=d"(retval) : "l"(mem)); + return retval; +} +F2 OVERLOAD NCLOAD(TrigFP32 mem) { + F2 retval; + __asm("ld.global.nc.v2.f32 {%0, %1}, [%2];" : "=f"(retval.x), "=f"(retval.y) : "l"(mem)); + return retval; +} +F OVERLOAD NCLOAD(TrigSingleFP32 mem) { + F retval; + __asm("ld.global.nc.f32 %0, [%1];" : "=f"(retval) : "l"(mem)); + return retval; +} +i64 OVERLOAD NCLOAD(i64 *mem) { + i64 retval; + __asm("ld.global.nc.b64 %0, [%1];" : "=l"(retval) : "l"(mem)); + return retval; +} +GF61 OVERLOAD NCLOAD(TrigGF61 mem) { + GF61 retval; + __asm("ld.global.nc.v2.b64 {%0, %1}, [%2];" : "=l"(retval.x), "=l"(retval.y) : "l"(mem)); + return retval; +} +i32 OVERLOAD NCLOAD(i32 *mem) { + i32 retval; + __asm("ld.global.nc.b32 %0, [%1];" : "=r"(retval) : "l"(mem)); + return retval; +} +GF31 OVERLOAD NCLOAD(TrigGF31 mem) { + GF31 retval; + __asm("ld.global.nc.v2.b32 {%0, %1}, [%2];" : "=r"(retval.x), "=r"(retval.y) : "l"(mem)); + return retval; } +#else +#define NCLOAD LOAD #endif -#if NTT_GF61 -void OVERLOAD read(u32 WG, u32 N, GF61 *u, const global GF61 *in, u32 base) { - in += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { u[i] = in[i * WG]; } +// Routines for loading data from memory into the L1 and L2 caches. This should be same as the default LOAD macro. + +#if HAS_PTX >= 200 // Cache hints requires sm_20 support or higher +T2 OVERLOAD CALOAD(CP(T2) mem) { + T2 retval; + __asm("ld.global.ca.v2.f64 {%0, %1}, [%2];" : "=d"(retval.x), "=d"(retval.y) : "l"(mem)); + return retval; +} +T OVERLOAD CALOAD(TrigSingle mem) { + T retval; + __asm("ld.global.ca.f64 %0, [%1];" : "=d"(retval) : "l"(mem)); + return retval; +} +F2 OVERLOAD CALOAD(CP(F2) mem) { + F2 retval; + __asm("ld.global.ca.v2.f32 {%0, %1}, [%2];" : "=f"(retval.x), "=f"(retval.y) : "l"(mem)); + return retval; +} +F OVERLOAD CALOAD(TrigSingleFP32 mem) { + F retval; + __asm("ld.global.ca.f32 %0, [%1];" : "=f"(retval) : "l"(mem)); + return retval; +} +i64 OVERLOAD CALOAD(i64 *mem) { + i64 retval; + __asm("ld.global.ca.b64 %0, [%1];" : "=l"(retval) : "l"(mem)); + return retval; +} +GF61 OVERLOAD CALOAD(TrigGF61 mem) { + GF61 retval; + __asm("ld.global.ca.v2.b64 {%0, %1}, [%2];" : "=l"(retval.x), "=l"(retval.y) : "l"(mem)); + return retval; +} +i32 OVERLOAD CALOAD(i32 *mem) { + i32 retval; + __asm("ld.global.ca.b32 %0, [%1];" : "=r"(retval) : "l"(mem)); + return retval; +} +GF31 OVERLOAD CALOAD(TrigGF31 mem) { + GF31 retval; + __asm("ld.global.ca.v2.b32 {%0, %1}, [%2];" : "=r"(retval.x), "=r"(retval.y) : "l"(mem)); + return retval; } +#else +#define CALOAD LOAD +#endif + +// +// These macros map various types of data accesses to one of the load/store routines above +// + +// Routines for loading/storing FFT data. Lots of data, kernels read it once, write it once. If possible, data should not be written to L1 cache. +// If L2 cache is "small", we should look for ways to prioritize keeping data that is re-used in the L2 cache. + +#define FFTLOAD_TYPE LOADS % 10 +#define CSLOAD_TYPE (LOADS / 10) % 10 +#define TFLOAD_TYPE (LOADS / 100) % 10 +#define TSLOAD_TYPE (LOADS / 1000) % 10 +#define TOLOAD_TYPE (LOADS / 10000) % 10 + +#define FFTSTORE_TYPE STORES % 10 +#define CSSTORE_TYPE (STORES / 10) % 10 + +#if FFTLOAD_TYPE == 1 +#define FFTLOAD NTLOAD +#elif FFTLOAD_TYPE == 2 +#define FFTLOAD L2LOAD +#elif FFTLOAD_TYPE == 3 +#define FFTLOAD EFLOAD +#elif FFTLOAD_TYPE == 4 +#define FFTLOAD LULOAD +#elif FFTLOAD_TYPE == 5 +#define FFTLOAD NCLOAD +#else +#define FFTLOAD LOAD +#endif -void OVERLOAD write(u32 WG, u32 N, GF61 *u, global GF61 *out, u32 base) { - out += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { out[i * WG] = u[i]; } +#if FFTSTORE_TYPE == 1 +#define FFTSTORE NTSTORE +#elif FFTSTORE_TYPE == 2 +#define FFTSTORE L2STORE +#elif FFTSTORE_TYPE == 3 +#define FFTSTORE EFSTORE +#else +#define FFTSTORE STORE +#endif + +// Routines for loading/storing carryShuttle data. CarryFused writes it once, and reads it once. The data is never used again. +// If possible, data should not be written to L1 cache and not written to memory after it is read. + +#if CSLOAD_TYPE == 1 +#define CSLOAD NTLOAD +#elif CSLOAD_TYPE == 2 +#define CSLOAD L2LOAD +#elif CSLOAD_TYPE == 3 +#define CSLOAD EFLOAD +#elif CSLOAD_TYPE == 4 +#define CSLOAD LULOAD +#elif CSLOAD_TYPE == 5 +#define CSLOAD NCLOAD +#else +#define CSLOAD LOAD +#endif + +#if CSSTORE_TYPE == 1 +#define CSSTORE NTSTORE +#elif CSSTORE_TYPE == 2 +#define CSSTORE L2STORE +#elif CSSTORE_TYPE == 3 +#define CSSTORE EFSTORE +#else +#define CSSTORE STORE +#endif + +// Routines for loading trig data that is frequently reused. If possible, data should saved in L1 and L2 caches and perhaps marked evict last. +// TF stands for "Trig Frequently reused". It is highly unlikely that any option other than the default LOAD makes sense. + +#if TFLOAD_TYPE == 1 +#define TFLOAD NTLOAD +#elif TFLOAD_TYPE == 2 +#define TFLOAD L2LOAD +#elif TFLOAD_TYPE == 3 +#define TFLOAD EFLOAD +#elif TFLOAD_TYPE == 4 +#define TFLOAD LULOAD +#elif TFLOAD_TYPE == 5 +#define TFLOAD NCLOAD +#else +#define TFLOAD LOAD +#endif + +// Routines for loading trig data that is used once but is smaller than a cache line. The rest of the cache line will be needed soon. +// If possible, data should be saved in L1(?) and L2 caches and perhaps marked evict first. +// TS stands for "Trig Several reuses". + +#if TSLOAD_TYPE == 1 +#define TSLOAD NTLOAD +#elif TSLOAD_TYPE == 2 +#define TSLOAD L2LOAD +#elif TSLOAD_TYPE == 3 +#define TSLOAD EFLOAD +#elif TSLOAD_TYPE == 4 +#define TSLOAD LULOAD +#elif TSLOAD_TYPE == 5 +#define TSLOAD NCLOAD +#else +#define TSLOAD LOAD +#endif + +// Routines for loading trig data that is used once and is a cache line or larger. +// If possible, data should saved in L2 caches if the L2 cache is very large. +// TO stands for "Trig used Once". + +#if TOLOAD_TYPE == 1 +#define TOLOAD NTLOAD +#elif TOLOAD_TYPE == 2 +#define TOLOAD L2LOAD +#elif TOLOAD_TYPE == 3 +#define TOLOAD EFLOAD +#elif TOLOAD_TYPE == 4 +#define TOLOAD LULOAD +#elif TOLOAD_TYPE == 5 +#define TOLOAD NCLOAD +#else +#define TOLOAD LOAD +#endif + +// Prefetch macros. Unused at present, I tried using them in fftMiddleInGF61 on a 5080 with no benefit. +void PREFETCHL1(const __global void *addr) { +#if HAS_PTX >= 200 // Prefetch instruction requires sm_20 support or higher + __asm("prefetch.global.L1 [%0];" : : "l"(addr)); +#endif } +void PREFETCHL2(const __global void *addr) { +#if HAS_PTX >= 200 // Prefetch instruction requires sm_20 support or higher + __asm("prefetch.global.L2 [%0];" : : "l"(addr)); #endif +} // On "classic" AMD GCN GPUs such as Radeon VII, the wavefront size was always 64. On RDNA GPUs the wavefront can // be configured to be either 64 or 32. We use the FAST_BARRIER define as an indicator for GCN GPUs. @@ -360,6 +804,25 @@ void OVERLOAD write(u32 WG, u32 N, GF61 *u, global GF61 *out, u32 base) { #endif #endif +// Default settings for USE_REGISTER_BARSYNC. OpenCL on nVidia has compiler issues when USE_REGISTER_BARSYNC=0. Annoying, as register bar.sync is slower in many cases. +#ifndef USE_REGISTER_BARSYNC +#if CUDA_BACKEND +#define USE_REGISTER_BARSYNC 0 +#else +#define USE_REGISTER_BARSYNC 1 +#endif +#endif + +// Force divergent threads in a warp to converge. AMD GCN does not require this, all threads in a WAVEFRONT operate in lockstep. Early CUDA versions did also. +// The sync is needed in cases where one thread is setting a flag or state on behalf of all the threads in a WAVEFRONT. For example, carryFused has thread 0 set +// the carries-are-ready flag on behalf of all 32 threads in a warp. +void OVERLOAD sync() { +#if HAS_PTX >= 600 // bar.warp.sync requires sm_60 support or higher + __asm("bar.warp.sync 0xffffffff;" : : ); +#endif +} + +// Create a barrier across all threads. void OVERLOAD bar(void) { // barrier(CLK_LOCAL_MEM_FENCE) is correct, but it turns out that on some GPUs // (in particular on Radeon VII and Radeon PRO VII) barrier(0) works as well and is faster. @@ -371,8 +834,85 @@ void OVERLOAD bar(void) { #endif } -void OVERLOAD bar(u32 WG) { if (WG > WAVEFRONT) { bar(); } } +// Create a barrier across a subset of threads OR across all threads if that is faster. +void OVERLOAD bar(const u32 WG) { + // A group no larger than a wavefront can skip the barrier only where the hardware really does run a whole + // wavefront in lock-step. AMD GCN does, and so did nVidia before Volta. Volta and later do not: + // Independent Thread Scheduling lets the threads of a warp drift apart, and every caller of bar(WG) + // exchanges data through LDS right afterwards, so the warp has to be reconverged and its LDS traffic + // ordered -- which is what sync() plus the fence do, for a fraction of the cost of a barrier. Anything + // else (Intel, a CPU device under POCL) offers no lock-step guarantee at all: use a real barrier there. + if (WG <= WAVEFRONT) { +#if HAS_PTX >= 600 // bar.warp.sync requires sm_60 or higher; ITS arrived in sm_70 + sync(); + mem_fence(CLK_LOCAL_MEM_FENCE); + return; +#elif AMDGPU || HAS_PTX >= 200 + return; +#endif + } +#if ENABLE_BARSYNC && HAS_PTX >= 200 // bar.sync with thread count requires sm_20 support or higher. Slower on TitanV, need to try on later nVidia GPUs. + __asm("bar.sync %0, %1;" : : "r"(get_local_id(0) / WG + 1), "n"(WG)); +// The above is GROSSLY slow on an RTX 5070Ti. The code below is much faster (may need to be expanded to handle more than four named barriers). +// WARNING, WARNING, WARNING: On TitanV using CUDA 12.9 tools and driver 580, similar code in LDSbar does not work in openCL (but works in CUDA build). +// if (get_local_id(0) / WG + 1 == 1) __asm("bar.sync 1, %0;" : : "n"(WG)); +// else if (get_local_id(0) / WG + 1 == 2) __asm("bar.sync 2, %0;" : : "n"(WG)); +// else if (get_local_id(0) / WG + 1 == 2) __asm("bar.sync 3, %0;" : : "n"(WG)); +// else __asm("bar.sync 4, %0;" : : "n"(WG)); +#else + bar(); +#endif +} + +// Create a barrier across a subset of threads. Substituting a barrier on all threads is not permitted, so this is only +// defined where the hardware can do it (PTX bar.sync with a thread count, sm_20 or higher). On any other GPU a call to +// barsync() fails to compile at the call site instead of the whole of base.cl failing whether or not it is used. +#if HAS_PTX >= 200 +void OVERLOAD barsync(const u32 numWG, const u32 WG) { + // As in bar(WG) above, except that substituting a barrier over all threads is not allowed here, so on + // Volta and later the warp-wide sync is the only option. (This routine is nVidia-only to begin with.) + if (WG <= WAVEFRONT) { +#if HAS_PTX >= 600 + sync(); + mem_fence(CLK_LOCAL_MEM_FENCE); +#endif + return; + } +#if USE_REGISTER_BARSYNC // bar.sync with a register is horribly slow on an RTX 5070Ti. + __asm("bar.sync %0, %1;" : : "r"(get_local_id(0) / WG + 1), "n"(WG)); +#else // WARNING, WARNING, WARNING: On TitanV using CUDA 12.9 tools and driver 580, this branch does not work in openCL (but works in CUDA build). + for (u32 i = 1; i <= numWG; i++) { + if (i == get_local_id(0) / WG + 1) { + __asm("bar.sync %0, %1;" : : "n"(i), "n"(WG)); + break; + } + } +#endif +} +#endif + +// nVidia GPUs (Hopper architecture sm 9.0 and later) support Programatic Dependent Launch where the tail end execution of one kernel can overlap +// with the beginning of the next kernel. This requires a special launch kernel command that is only available in CUDA 12.0 and later. +// These routines let us take advantage of this CUDA feature. These routines do nothing in OpenCL. + +// Switched on per run with -use PDL=1. The CUDA shim launches a kernel with +// programmatic stream serialization exactly when its compiled code contains +// the wait below (it reads the PTX), so a kernel that never waits is never +// allowed to start early. Off, both routines compile to nothing and every +// launch is ordinary. +#ifndef PDL +#define PDL 0 +#endif + +void dependentLaunch() { +#if CUDA_BACKEND && HAS_PTX >= 900 && PDL + __asm volatile("griddepcontrol.launch_dependents;"); // same as cudaTriggerProgrammaticLaunchCompletion(); +#endif +} + +void dependentLaunchWait() { +#if CUDA_BACKEND && HAS_PTX >= 900 && PDL + __asm volatile("griddepcontrol.wait;"); // same as cudaGridDependencySynchronize(); +#endif +} -// A half-barrier is only needed when half-a-workgroup needs a barrier. -// This is used e.g. by the double-wide tailSquare, where LDS is split between the halves. -void halfBar() { if (get_enqueued_local_size(0) / 2 > WAVEFRONT) { bar(); } } diff --git a/src/cl/carry.cl b/src/cl/carry.cl index 28863dd8..3c356cae 100644 --- a/src/cl/carry.cl +++ b/src/cl/carry.cl @@ -1,8 +1,17 @@ // Copyright (C) Mihai Preda +#include "base.cl" +#include "math.cl" +#include "trig.cl" #include "carryutil.cl" #include "weight.cl" +// Number of workgroups this kernel is launched with: it is enqueued with hN / CARRY_LEN work-items +// and a workgroup of G_W, so hN / (CARRY_LEN * G_W) = BIG_HEIGHT * NW / CARRY_LEN. updateStats needs +// this exact count -- passing BIG_HEIGHT is only correct when NW == CARRY_LEN, which fails for WIDTH=256 +// where nW() is 4. CARRY_LEN divides BIG_HEIGHT since BIG_HEIGHT = MIDDLE * SMALL_HEIGHT. +#define CARRY_GROUPS (NW * (BIG_HEIGHT / CARRY_LEN)) + #if FFT_TYPE == FFT64 // Carry propagation with optional MUL-3, over CARRY_LEN words. @@ -22,8 +31,8 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big // Calculate the most significant 32-bits of FRAC_BPW * the index of the FFT word. Also add FRAC_BPW_HI to test first biglit flag. u32 line = gy * CARRY_LEN; - u32 fft_word_index = (gx * G_W * H + me * H + line) * 2; - u32 frac_bits = fft_word_index * FRAC_BPW_HI + mad_hi (fft_word_index, FRAC_BPW_LO, FRAC_BPW_HI); + u32 word_index = (gx * G_W * H + me * H + line) * 2; + u32 frac_bits = fracBits(word_index) + FRAC_BPW_HI; T base = optionalDouble(fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx))); @@ -38,9 +47,11 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big carryOut[G_W * g + me] = carry; #if ROE - updateStats(bufROE, posROE, roundMax); + local u32 lds[G_W]; + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) - updateStats(bufROE, posROE, carryMax); + local u32 lds[G_W]; + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, carryMax); #endif } @@ -65,27 +76,41 @@ KERNEL(G_W) carry(P(Word2) out, CP(F2) in, u32 posROE, P(CarryABM) carryOut, Big float roundMax = 0; float carryMax = 0; - // Calculate the most significant 32-bits of FRAC_BPW * the index of the FFT word. Also add FRAC_BPW_HI to test first biglit flag. + // Calculate the most significant 32-bits of FRAC_BPW * the index of the FFT word. u32 line = gy * CARRY_LEN; - u32 fft_word_index = (gx * G_W * H + me * H + line) * 2; - u32 frac_bits = fft_word_index * FRAC_BPW_HI + mad_hi (fft_word_index, FRAC_BPW_LO, FRAC_BPW_HI); + u32 word_index = (gx * G_W * H + me * H + line) * 2; - F base = optionalDouble(fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx))); + F base = fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx)); + u32 me_frac_bits = fracBits(me * H * 2); + u32 step_frac_bits = weightStepFracBits(gx); + u32 base_frac_bits = me_frac_bits + step_frac_bits; + base = optionalDouble(base, base_frac_bits > step_frac_bits); + + u32 frac_bits = fracBits(word_index); + + // Base_frac_bits and frac_bits are inexact values. We only want to trigger an optional double when it is clear to do so. + // Fudge base_frac_bits to make it harder to trigger a double when the two inexact values are equal. + base_frac_bits++; for (i32 i = 0; i < CARRY_LEN; ++i) { - u32 p = G_W * gx + WIDTH * (CARRY_LEN * gy + i) + me; - F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + gy * CARRY_LEN + i].x)); - F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP)); - bool biglit0 = frac_bits + (2*i) * FRAC_BPW_HI <= FRAC_BPW_HI; - bool biglit1 = frac_bits + (2*i) * FRAC_BPW_HI >= -FRAC_BPW_HI; // Same as frac_bits + (2*i) * FRAC_BPW_HI + FRAC_BPW_HI <= FRAC_BPW_HI; + u32 p = G_W * gx + WIDTH * (line + i) + me; + F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + line + i].x), frac_bits > base_frac_bits); + F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); + frac_bits += FRAC_BPW_HI; + bool biglit0 = frac_bits <= FRAC_BPW_HI; + bool biglit1 = frac_bits >= -FRAC_BPW_HI; // Same as frac_bits + FRAC_BPW_HI <= FRAC_BPW_HI; out[p] = weightAndCarryPair(SWAP_XY(in[p]), U2(w1, w2), carry, biglit0, biglit1, &carry, &roundMax, &carryMax); + // Generate frac_bits for next pair + frac_bits += FRAC_BPW_HI; } carryOut[G_W * g + me] = carry; #if ROE - updateStats(bufROE, posROE, roundMax); + local u32 lds[G_W]; + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) - updateStats(bufROE, posROE, carryMax); + local u32 lds[G_W]; + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, carryMax); #endif } @@ -125,8 +150,8 @@ KERNEL(G_W) carry(P(Word2) out, CP(GF31) in, u32 posROE, P(CarryABM) carryOut, P #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -156,10 +181,12 @@ KERNEL(G_W) carry(P(Word2) out, CP(GF31) in, u32 posROE, P(CarryABM) carryOut, P carryOut[G_W * g + me] = carry; #if ROE + local u32 lds[G_W]; float fltRoundMax = (float) roundMax / (float) M31; // For speed, roundoff was computed as 32-bit integer. Convert to float. - updateStats(bufROE, posROE, fltRoundMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, fltRoundMax); #elif (STATS & (1 << (2 + MUL3))) - updateStats(bufROE, posROE, carryMax); + local u32 lds[G_W]; + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, carryMax); #endif } @@ -199,8 +226,8 @@ KERNEL(G_W) carry(P(Word2) out, CP(GF61) in, u32 posROE, P(CarryABM) carryOut, P #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -230,10 +257,12 @@ KERNEL(G_W) carry(P(Word2) out, CP(GF61) in, u32 posROE, P(CarryABM) carryOut, P carryOut[G_W * g + me] = carry; #if ROE + local u32 lds[G_W]; float fltRoundMax = (float) roundMax / (float) (M61 >> 32); // For speed, roundoff was computed as 32-bit integer. Convert to float. - updateStats(bufROE, posROE, fltRoundMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, fltRoundMax); #elif (STATS & (1 << (2 + MUL3))) - updateStats(bufROE, posROE, carryMax); + local u32 lds[G_W]; + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, carryMax); #endif } @@ -274,8 +303,8 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -309,9 +338,11 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big carryOut[G_W * g + me] = carry; #if ROE - updateStats(bufROE, posROE, roundMax); + local u32 lds[G_W]; + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) - updateStats(bufROE, posROE, carryMax); + local u32 lds[G_W]; + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, carryMax); #endif } @@ -339,7 +370,11 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big u32 word_index = (gx * G_W * H + me * H + line) * 2; - F base = optionalDouble(fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx))); + F base = fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx)); + u32 me_frac_bits = fracBits(me * H * 2); + u32 step_frac_bits = weightStepFracBits(gx); + u32 base_frac_bits = me_frac_bits + step_frac_bits; + base = optionalDouble(base, base_frac_bits > step_frac_bits); // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. const u32 log2_root_two = (u32) (((1ULL << 30) / NWORDS) % 31); @@ -353,8 +388,8 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -363,11 +398,11 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big weight_shift = (weight_shift + log2_NWORDS + 1) % 31; for (i32 i = 0; i < CARRY_LEN; ++i) { - u32 p = G_W * gx + WIDTH * (CARRY_LEN * gy + i) + me; + u32 p = G_W * gx + WIDTH * (line + i) + me; // Generate the FP32 and second GF31 weight shift - F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + gy * CARRY_LEN + i].x)); - F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP)); + F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + line + i].x), frac_bits > base_frac_bits); + F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 weight_shift0 = weight_shift; combo_counter += combo_step; if (weight_shift > 31) weight_shift -= 31; @@ -388,9 +423,11 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big carryOut[G_W * g + me] = carry; #if ROE - updateStats(bufROE, posROE, roundMax); + local u32 lds[G_W]; + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) - updateStats(bufROE, posROE, carryMax); + local u32 lds[G_W]; + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, carryMax); #endif } @@ -418,7 +455,11 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big u32 word_index = (gx * G_W * H + me * H + line) * 2; - F base = optionalDouble(fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx))); + F base = fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx)); + u32 me_frac_bits = fracBits(me * H * 2); + u32 step_frac_bits = weightStepFracBits(gx); + u32 base_frac_bits = me_frac_bits + step_frac_bits; + base = optionalDouble(base, base_frac_bits > step_frac_bits); // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. const u32 log2_root_two = (u32) (((1ULL << 60) / NWORDS) % 61); @@ -432,8 +473,8 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -442,11 +483,11 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big weight_shift = (weight_shift + log2_NWORDS + 1) % 61; for (i32 i = 0; i < CARRY_LEN; ++i) { - u32 p = G_W * gx + WIDTH * (CARRY_LEN * gy + i) + me; + u32 p = G_W * gx + WIDTH * (line + i) + me; // Generate the FP32 and second GF61 weight shift - F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + gy * CARRY_LEN + i].x)); - F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP)); + F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + line + i].x), frac_bits > base_frac_bits); + F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 weight_shift0 = weight_shift; combo_counter += combo_step; if (weight_shift > 61) weight_shift -= 61; @@ -467,9 +508,11 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big carryOut[G_W * g + me] = carry; #if ROE - updateStats(bufROE, posROE, roundMax); + local u32 lds[G_W]; + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) - updateStats(bufROE, posROE, carryMax); + local u32 lds[G_W]; + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, carryMax); #endif } @@ -515,10 +558,10 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, P(u #define m61_weight_shift m61_combo.a[1] #define m61_combo_counter m61_combo.b - const u64 m31_combo_step = ((u64) m31_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - m31_combo_counter = word_index * m31_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; - const u64 m61_combo_step = ((u64) m61_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - m61_combo_counter = word_index * m61_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 m31_combo_step = make_u64(m31_bigword_weight_shift_minus1, FRAC_BPW_HI); + m31_combo_counter = comboFracBits(word_index) + make_u64(word_index * m31_bigword_weight_shift_minus1, 0xFFFFFFFF); + const u64 m61_combo_step = make_u64(m61_bigword_weight_shift_minus1, FRAC_BPW_HI); + m61_combo_counter = comboFracBits(word_index) + make_u64(word_index * m61_bigword_weight_shift_minus1, 0xFFFFFFFF); // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -557,10 +600,12 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, P(u carryOut[G_W * g + me] = carry; #if ROE + local u32 lds[G_W]; float fltRoundMax = (float) roundMax / (float) 0x1FFFFFFF; // For speed, roundoff was computed as 32-bit integer. Convert to float. - updateStats(bufROE, posROE, fltRoundMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, fltRoundMax); #elif (STATS & (1 << (2 + MUL3))) - updateStats(bufROE, posROE, carryMax); + local u32 lds[G_W]; + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, carryMax); #endif } @@ -590,7 +635,11 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big u32 word_index = (gx * G_W * H + me * H + line) * 2; - F base = optionalDouble(fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx))); + F base = fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx)); + u32 me_frac_bits = fracBits(me * H * 2); + u32 step_frac_bits = weightStepFracBits(gx); + u32 base_frac_bits = me_frac_bits + step_frac_bits; + base = optionalDouble(base, base_frac_bits > step_frac_bits); // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. const u32 m31_log2_root_two = (u32) (((1ULL << 30) / NWORDS) % 31); @@ -610,9 +659,9 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big #define m61_combo_counter m61_combo.b const u64 m31_combo_step = ((u64) m31_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - m31_combo_counter = word_index * m31_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + m31_combo_counter = comboFracBits(word_index) + make_u64(word_index * m31_bigword_weight_shift_minus1, 0xFFFFFFFF); const u64 m61_combo_step = ((u64) m61_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - m61_combo_counter = word_index * m61_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + m61_combo_counter = comboFracBits(word_index) + make_u64(word_index * m61_bigword_weight_shift_minus1, 0xFFFFFFFF); // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -625,8 +674,8 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big u32 p = G_W * gx + WIDTH * (CARRY_LEN * gy + i) + me; // Generate the FP32 and second GF31 and GF61 weight shift - F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + gy * CARRY_LEN + i].x)); - F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP)); + F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + line + i].x), frac_bits > base_frac_bits); + F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 m31_weight_shift0 = m31_weight_shift; m31_combo_counter += m31_combo_step; m31_weight_shift = adjust_m31_weight_shift(m31_weight_shift); @@ -653,9 +702,11 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big carryOut[G_W * g + me] = carry; #if ROE - updateStats(bufROE, posROE, roundMax); + local u32 lds[G_W]; + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) - updateStats(bufROE, posROE, carryMax); + local u32 lds[G_W]; + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, carryMax); #endif } diff --git a/src/cl/carryb.cl b/src/cl/carryb.cl index d6033e19..c675267a 100644 --- a/src/cl/carryb.cl +++ b/src/cl/carryb.cl @@ -1,6 +1,9 @@ // Copyright (C) Mihai Preda +#include "base.cl" +#include "math.cl" #include "carryutil.cl" +#include "weight.cl" KERNEL(G_W) carryB(P(Word2) io, CP(CarryABM) carryIn) { u32 g = get_group_id(0); @@ -12,8 +15,8 @@ KERNEL(G_W) carryB(P(Word2) io, CP(CarryABM) carryIn) { // Derive the big vs. little flags from the fractional number of bits in each FFT word rather read the flags from memory. // Calculate the most significant 32-bits of FRAC_BPW * the index of the FFT word. Also add FRAC_BPW_HI to test first biglit flag. u32 line = gy * CARRY_LEN; - u32 fft_word_index = (gx * G_W * H + me * H + line) * 2; - u32 frac_bits = fft_word_index * FRAC_BPW_HI + mad_hi (fft_word_index, FRAC_BPW_LO, FRAC_BPW_HI); + u32 word_index = (gx * G_W * H + me * H + line) * 2; + u32 frac_bits = fracBits(word_index) + FRAC_BPW_HI; io += G_W * gx + WIDTH * CARRY_LEN * gy; @@ -29,6 +32,18 @@ KERNEL(G_W) carryB(P(Word2) io, CP(CarryABM) carryIn) { u32 p = i * WIDTH + me; bool biglit0 = frac_bits + (2*i) * FRAC_BPW_HI <= FRAC_BPW_HI; bool biglit1 = frac_bits + (2*i) * FRAC_BPW_HI >= -FRAC_BPW_HI; // Same as frac_bits + (2*i) * FRAC_BPW_HI + FRAC_BPW_HI <= FRAC_BPW_HI; + // carryB has no carry-out: a carry leaving the last word of this group would be dropped, silently + // losing 1 ulp at the first word of the next group. On the last word pair, add the carry into the + // high word without normalizing it -- as carryFinal does at the end of the fused carry chain -- so + // that nothing can escape the group. An un-normalized word holds the same value and is normalized + // by the next iteration. + if (i == CARRY_LEN - 1) { + Word2 a = io[p]; + a.x = carryStep(a.x + carry, &carry, biglit0); + a.y += carry; + io[p] = a; + return; + } io[p] = carryWord(io[p], &carry, biglit0, biglit1); if (!carry) { return; } } diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index 05e4ca4c..1f949333 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -1,9 +1,12 @@ // Copyright (C) Mihai Preda +#include "base.cl" +#include "fftwidth.cl" #include "carryutil.cl" #include "weight.cl" -#include "fftwidth.cl" -#include "middle.cl" + +#define INCLUDE_FILE "middle.cl" +#include "expand.cl" void spin() { #if defined(__has_builtin) && __has_builtin(__builtin_amdgcn_s_sleep) @@ -16,18 +19,113 @@ void spin() { #endif } +// Increasing WMUL to 2 reduces carryShuttle activity. This led to a 1% speedup on Titan V. Testing on other GPUs is needed. +#ifndef WMUL +#define WMUL 2 +#endif + +#if AMDGPU +#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions +//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 +#else +#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better +#endif + +// The last WMUL workgroup's carries have been written to global memory. Now we shuffle WMUL-1 workgroups carries up using local memory. +void OVERLOAD shufl_carries_up(local void *lds2, i64 *carry, u32 me, u32 lowMe) { + // If WMUL is one, there is no shuffling of carries + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead of the clean looking if statement below we use the uglier #if + //if (WMUL == 1) return; +#if WMUL > 1 + + const u32 lds_i64s = LDS_SHUFL_BYTES(WMUL) / sizeof(i64); // Number of i64s in LDS used by shufl for each WMUL workgroup + local i64 *lds = (local i64 *) lds2; + + // Handle nasty case where we are writing 8-byte quantities but SHUFL_BYTES_W is only 4 bytes + if (SHUFL_BYTES_W == 4) { + if (WMUL == 2) { + // Full barrier needed as we are using the entire LDS buffer. + bar(); + // Write the carries. This will use the entire LDS buffer. + if (me < G_W) for (i32 i = 0; i < NW; ++i) lds[i * G_W + lowMe] = carry[i]; + // Read carries from previous WMUL workgroup + bar(); + if (me >= G_W) for (i32 i = 0; i < NW; ++i) carry[i] = lds[i * G_W + lowMe]; + // Full barrier needed as one workgroup just read data from two workgroups LDS buffer. Not compatible with shufl(). + bar(); + } + + // The really nasty case where all the carries will not fit in LDS memory + else { + lds += (me / G_W) * lds_i64s + lowMe; // This WMUL workgroup's LDS area + // Write half the carries to next WMUL's workgroup LDS area + bar(); + if (me < (WMUL-1) * G_W) for (i32 i = 0; i < NW/2; ++i) lds[lds_i64s + i * G_W] = carry[i]; + // Read carries from our WMUL workgroup LDS area + bar(); + if (me >= G_W) for (i32 i = 0; i < NW/2; ++i) carry[i] = lds[i * G_W]; + // Write the other half of the carries + bar(); + if (me < (WMUL-1) * G_W) for (i32 i = 0; i < NW/2; ++i) lds[lds_i64s + i * G_W] = carry[i + NW/2]; + // Read carries from our WMUL workgroup LDS area. Compatible with shufl when no trailing bar() needed. + bar(); + if (me >= G_W) for (i32 i = 0; i < NW/2; ++i) carry[i + NW/2] = lds[i * G_W]; + // One last bar() is needed when sharing LDS memory. This is because when sharing a workgroup will write to more than its own LDS area. + if (SHARING_LDS(WMUL)) bar(); + } + } + + // Easy case. Write carries to local memory (except last WMUL workgroup which was written to global memory). + else { + lds += (me / G_W) * lds_i64s + lowMe; // This WMUL workgroup's LDS area + // Full barrier needed as we are moving data to next WMUL workgroup's LDS area + bar(); + if (me < (WMUL-1) * G_W) for (i32 i = 0; i < NW; ++i) lds[lds_i64s + i * G_W] = carry[i]; + // Full barrier needed as we just moved data from one WMUL workgroup LDS area to the another WMUL workgroup's LDS area + bar(); + // Read carries from our WMUL workgroup's LDS area. This is compatible with shufl when no trailing bar() is required. + if (me >= G_W) for (i32 i = 0; i < NW; ++i) carry[i] = lds[i * G_W]; + // One last bar() is needed when sharing LDS memory. This is because when sharing a workgroup will write to more than its own LDS area. + if (SHARING_LDS(WMUL)) bar(); + } + +#endif +} + +// The last WMUL workgroup's carries have been written to global memory. Now we shuffle WMUL-1 workgroup carries up using local memory. +void OVERLOAD shufl_carries_up(local void *lds2, i32 *carry, u32 me, u32 lowMe) { + // If WMUL is one, there is no shuffling of carries + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead of the clean looking if statement below we use the uglier #if + //if (WMUL == 1) return; +#if WMUL > 1 + + const u32 lds_i32s = LDS_SHUFL_BYTES(WMUL) / sizeof(i32); // Number of i32s in LDS used by shufl for each WMUL workgroup + local i32 *lds = (local i32 *) lds2; + lds += (me / G_W) * lds_i32s + lowMe; // This WMUL workgroup's LDS area + + // Write carries to local memory (except last WMUL workgroup which was written to global memory) + // Full barrier needed as we are moving data to next WMUL workgroup's LDS area + bar(); + if (me < (WMUL-1) * G_W) for (i32 i = 0; i < NW; ++i) lds[lds_i32s + i * G_W] = carry[i]; + // Full barrier needed as we just moved data from one WMUL workgroup LDS area to the another WMUL workgroup's LDS area + bar(); + // Read carries from our WMUL workgroup's LDS area. This is compatible with shufl when no trailing bar() is required. + if (me >= G_W) for (i32 i = 0; i < NW; ++i) carry[i] = lds[i * G_W]; + // One last bar() is needed when sharing LDS memory. This is because when sharing a workgroup will write to more than its own LDS area. + if (SHARING_LDS(WMUL)) bar(); + +#endif +} + + #if FFT_TYPE == FFT64 // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, - CP(u32) bits, ConstBigTab CONST_THREAD_WEIGHTS, BigTab THREAD_WEIGHTS, P(uint) bufROE) { - -#if 0 // fft_WIDTH uses shufl_int instead of shufl - local T2 lds[WIDTH / 4]; -#else - local T2 lds[WIDTH / 2]; -#endif +KERNEL_CAP(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, + ConstBigTab CONST_THREAD_WEIGHTS, BigTab THREAD_WEIGHTS, P(uint) bufROE) { + local T2 lds[LDS_BYTES(WMUL) / sizeof(T2)]; + LDSinit(lds, WMUL); T2 u[NW]; @@ -35,36 +133,37 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( u32 me = get_local_id(0); u32 H = BIG_HEIGHT; - u32 line = gr % H; +#if WMUL == 1 + u32 lowMe = me; + u32 line = gr; +#else + u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. + u32 line = gr * WMUL + me / G_W; +#endif + if (line >= H) line -= H; #if HAS_ASM __asm("s_setprio 3"); #endif - readCarryFusedLine(in, u, line); + dependentLaunchWait(); // Previous kernel was fftMiddleOutFP64 -// Split 32 bits into NW groups of 2 bits. See later for different way to do this. -#if !BIGLIT -#define GPW (16 / NW) - u32 b = NTLOAD(bits[(G_W * line + me) / GPW]) >> (me % GPW * (2 * NW)); -#undef GPW -#endif + readCarryFusedLine(in, u, line, lowMe); // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. -#if ZEROHACK_W - u32 zerohack = get_group_id(0) / 131072; - new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack); -#else - new_fft_WIDTH1(lds, u, smallTrig); -#endif + u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, lowMe); Word2 wu[NW]; -#if AMDGPU - T2 weights = fancyMul(THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); +#if !NVIDIAGPU || CUDA_BACKEND + T2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), TSLOAD(&THREAD_WEIGHTS[G_W + line])); #else - T2 weights = fancyMul(CONST_THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights + T2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), CONST_THREAD_WEIGHTS[line % 64]); + weights.x = optionalDouble(weights.x); + weights.y = optionalHalve(weights.y); + weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); #endif #if MUL3 @@ -75,41 +174,25 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( CFcarry carry[NW+1]; #endif -#if AMDGPU -#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions -//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 -#else -#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better -#endif - float roundMax = 0; float carryMax = 0; - // On Titan V it is faster to derive the big vs. little flags from the fractional number of bits in each FFT word rather than read the flags from memory. - // On Radeon VII this code is about the same speed. Not sure which is better on other GPUs. -#if BIGLIT // Calculate the most significant 32-bits of FRAC_BPW * the word index. Also add FRAC_BPW_HI to test first biglit flag. - u32 word_index = (me * H + line) * 2; - u32 frac_bits = word_index * FRAC_BPW_HI + mad_hi (word_index, FRAC_BPW_LO, FRAC_BPW_HI); - const u32 frac_bits_bigstep = ((G_W * H * 2) * FRAC_BPW_HI + (u32)(((u64)(G_W * H * 2) * FRAC_BPW_LO) >> 32)); -#endif + u32 word_index = (lowMe * H + line) * 2; + u32 frac_bits = fracBits(word_index) + FRAC_BPW_HI; + const u32 frac_bits_bigstep = fracBits(G_W * H * 2); + u32 starting_frac_bits = frac_bits; // Apply the inverse weights and carry propagate pairs to generate the output carries T invBase = optionalDouble(weights.x); - for (u32 i = 0; i < NW; ++i) { T invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i))); T invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP)); // Generate big-word/little-word flags -#if BIGLIT - bool biglit0 = frac_bits + i * frac_bits_bigstep <= FRAC_BPW_HI; - bool biglit1 = frac_bits + i * frac_bits_bigstep >= -FRAC_BPW_HI; // Same as frac_bits + i * frac_bits_bigstep + FRAC_BPW_HI <= FRAC_BPW_HI; -#else - bool biglit0 = test(b, 2 * i); - bool biglit1 = test(b, 2 * i + 1); -#endif + bool biglit0 = frac_bits <= FRAC_BPW_HI; + bool biglit1 = frac_bits >= -FRAC_BPW_HI; // Same as frac_bits + FRAC_BPW_HI <= FRAC_BPW_HI; // Apply the inverse weights, optionally compute roundoff error, and convert to integer. Also apply MUL3 here. // Then propagate carries through two words (the first carry does not have to be accurately calculated because it will @@ -118,36 +201,49 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( // For an LL test, add -2 as the very initial "carry in" // We'd normally use logical &&, but the compiler whines with warning and bitwise fixes it (LL & (i == 0) & (line==0) & (me == 0)) ? -2 : 0, biglit0, biglit1, &carry[i], &roundMax, &carryMax); - } -#if ROE - updateStats(bufROE, posROE, roundMax); -#elif STATS & (1 << MUL3) - updateStats(bufROE, posROE, carryMax); -#endif + // Generate frac_bits for next pair + frac_bits += frac_bits_bigstep; + } + frac_bits = starting_frac_bits; // Restore starting frac_bits for applying weights after carry propagation - // Write out our carries. Only groups 0 to H-1 need to write carries out. - // Group H is a duplicate of group 0 (producing the same results) so we don't care about group H writing out, + // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. + // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. - if (gr < H) { for (i32 i = 0; i < NW; ++i) { carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(me, i)] = carry[i]; } } - - // Tell next line that its carries are ready + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead an #if is required +#if WMUL == 1 if (gr < H) { -#if OLD_FENCE - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(); - if (me == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else + if (gr < H / WMUL && me >= (WMUL-1) * G_W) { +#endif + for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } + + // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (me % WAVEFRONT == 0) { - u32 pos = gr * (G_W / WAVEFRONT) + me / WAVEFRONT; - atomic_store((atomic_uint *) &ready[pos], 1); +#if !OLD_FENCE + sync(); // Make sure all lanes have completed the CSSTORE + if (lowMe % WAVEFRONT == 0) { + u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; + atomic_store((global atomic_uint *) &ready[pos], 1); } #endif } - // Line zero will be redone when gr == H +#if OLD_FENCE + // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the + // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow + // is undefined. No barrier is needed when a sub-workgroup is a single wavefront. + if (gr < H / WMUL) { + bar(G_W); +#if WMUL == 1 + if (lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } +#else + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } +#endif + } +#endif + + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } // Do some work while our carries may not be ready @@ -155,6 +251,12 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( __asm("s_setprio 0"); #endif +#if ROE + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, roundMax); +#elif STATS & (1 << MUL3) + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, carryMax); +#endif + // Calculate inverse weights T base = optionalHalve(weights.y); for (u32 i = 0; i < NW; ++i) { @@ -163,68 +265,78 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( u[i] = U2(weight1, weight2); } - // Wait until our carries are ready + // Shuffle carries up + shufl_carries_up(lds, carry, me, lowMe); + + // Wait until our carries are ready. The barrier below must be reached by every work-item of the + // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + if (me == 0) { do { spin(); } while(!atomic_load_explicit((global atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } bar(); - read_mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me == 0) ready[gr - 1] = 0; +#endif + if (me < G_W) { +#if OLD_FENCE + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me == 0) ready[gr - 1] = 0; #else - u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; - if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); - } - mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (me % WAVEFRONT == 0) { + do { spin(); } while(atomic_load_explicit((global atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + } + sync(); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; #endif #if HAS_ASM - __asm("s_setprio 1"); + __asm("s_setprio 1"); #endif - - // Read from the carryShuttle carries produced by the previous WIDTH row. Rotate carries from the last WIDTH row. - // The new carry layout lets the compiler generate global_load_dwordx4 instructions. - if (gr < H) { - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]; - } - } else { + } #if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); -#endif - - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]; - } - - if (me == 0) { - carry[NW] = carry[NW-1]; - for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } - carry[0] = carry[NW]; + // For the last group the carry reading is shifted, so the per-wavefront ready flags are not enough and a + // barrier is needed. gr is uniform but "me < G_W" is not, so the barrier is taken outside that guard and + // the shuttle reads resume in a second "me < G_W" block. + if (gr >= H / WMUL) { bar(); } +#endif + + if (me < G_W) { + + // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. + // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. + if (gr < H / WMUL) { + for (i32 i = 0; i < NW; ++i) { + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + } + } else { + + for (i32 i = 0; i < NW; ++i) { + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + } + + if (me == 0) { + carry[NW] = carry[NW-1]; + for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } + carry[0] = carry[NW]; + } } } // Apply each 32 or 64 bit carry to the 2 words for (i32 i = 0; i < NW; ++i) { -#if BIGLIT - bool biglit0 = frac_bits + i * frac_bits_bigstep <= FRAC_BPW_HI; -#else - bool biglit0 = test(b, 2 * i); -#endif + bool biglit0 = frac_bits <= FRAC_BPW_HI; wu[i] = carryFinal(wu[i], carry[i], biglit0); u[i] = U2(u[i].x * wu[i].x, u[i].y * wu[i].y); - } - bar(); + // Generate frac_bits for next pair + frac_bits += frac_bits_bigstep; + } -// fft_WIDTH(lds, u, smallTrig); - new_fft_WIDTH2(lds, u, smallTrig); + dependentLaunch(); // Next kernel will be fftMiddleInFP64 - writeCarryFusedLine(u, out, line); + fft_WIDTH2(lds, u, smallTrig, WMUL, lowMe); + writeCarryFusedLine(u, out, line, lowMe); } @@ -236,14 +348,10 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigFP32 smallTrig, - CP(u32) bits, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - -#if 0 // fft_WIDTH uses shufl_int instead of shufl - local F2 lds[WIDTH / 4]; -#else - local F2 lds[WIDTH / 2]; -#endif +KERNEL_CAP(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigFP32 smallTrig, + ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { + local F2 lds[LDS_BYTES(WMUL) / sizeof(F2)]; + LDSinit(lds, WMUL); F2 u[NW]; @@ -251,60 +359,72 @@ KERNEL(G_W) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShuttle, P( u32 me = get_local_id(0); u32 H = BIG_HEIGHT; - u32 line = gr % H; +#if WMUL == 1 + u32 lowMe = me; + u32 line = gr; +#else + u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. + u32 line = gr * WMUL + me / G_W; +#endif + if (line >= H) line -= H; #if HAS_ASM __asm("s_setprio 3"); #endif - readCarryFusedLine(in, u, line); + dependentLaunchWait(); // Previous kernel was fftMiddleOutFP32 + + readCarryFusedLine(in, u, line, lowMe); // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. -#if ZEROHACK_W - u32 zerohack = get_group_id(0) / 131072; - new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack); -#else - new_fft_WIDTH1(lds, u, smallTrig); -#endif + u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, lowMe); Word2 wu[NW]; -#if AMDGPU - F2 weights = fancyMul(THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); + u32 me_frac_bits = fracBits(lowMe * H * 2); +#if !NVIDIAGPU || CUDA_BACKEND + F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), TSLOAD(&THREAD_WEIGHTS[G_W + line])); + u32 line_frac_bits = fracBits(line * 2); + u32 base_frac_bits = me_frac_bits + line_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > line_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > line_frac_bits); #else - F2 weights = fancyMul(CONST_THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights + F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), CONST_THREAD_WEIGHTS[line % 64]); + u32 partialLine_frac_bits = fracBits((line % 64) * 2); + u32 base_frac_bits = me_frac_bits + partialLine_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); + weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); + partialLine_frac_bits = fracBits(((line / 64) * 64) * 2); + base_frac_bits = base_frac_bits + partialLine_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); #endif P(CFcarry) carryShuttlePtr = (P(CFcarry)) carryShuttle; CFcarry carry[NW+1]; -#if AMDGPU -#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions -//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 -#else -#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better -#endif - float roundMax = 0; float carryMax = 0; - // Calculate the most significant 32-bits of FRAC_BPW * the word index. Also add FRAC_BPW_HI to test first biglit flag. - u32 word_index = (me * H + line) * 2; - u32 frac_bits = word_index * FRAC_BPW_HI + mad_hi (word_index, FRAC_BPW_LO, FRAC_BPW_HI); - const u32 frac_bits_bigstep = ((G_W * H * 2) * FRAC_BPW_HI + (u32)(((u64)(G_W * H * 2) * FRAC_BPW_LO) >> 32)); + // Calculate the most significant 32-bits of FRAC_BPW * the word index (it's the same as base_frac_bits). + u32 word_index = (lowMe * H + line) * 2; + const u32 frac_bits_bigstep = fracBits(G_W * H * 2 - 1); // Apply the inverse weights and carry propagate pairs to generate the output carries - F invBase = optionalDouble(weights.x); - + F invBase = weights.x; + u32 frac_bits = base_frac_bits; for (u32 i = 0; i < NW; ++i) { - F invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i))); - F invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP)); + F invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i)), frac_bits > base_frac_bits); + F invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); + frac_bits += FRAC_BPW_HI; // Generate big-word/little-word flags - bool biglit0 = frac_bits + i * frac_bits_bigstep <= FRAC_BPW_HI; - bool biglit1 = frac_bits + i * frac_bits_bigstep >= -FRAC_BPW_HI; // Same as frac_bits + i * frac_bits_bigstep + FRAC_BPW_HI <= FRAC_BPW_HI; + bool biglit0 = frac_bits <= FRAC_BPW_HI; + bool biglit1 = frac_bits >= -FRAC_BPW_HI; // Same as frac_bits + FRAC_BPW_HI <= FRAC_BPW_HI; // Apply the inverse weights, optionally compute roundoff error, and convert to integer. Also apply MUL3 here. // Then propagate carries through two words (the first carry does not have to be accurately calculated because it will @@ -313,36 +433,48 @@ KERNEL(G_W) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShuttle, P( // For an LL test, add -2 as the very initial "carry in" // We'd normally use logical &&, but the compiler whines with warning and bitwise fixes it (LL & (i == 0) & (line==0) & (me == 0)) ? -2 : 0, biglit0, biglit1, &carry[i], &roundMax, &carryMax); - } -#if ROE - updateStats(bufROE, posROE, roundMax); -#elif STATS & (1 << MUL3) - updateStats(bufROE, posROE, carryMax); -#endif + // Generate frac_bits for next pair + frac_bits += frac_bits_bigstep; + } - // Write out our carries. Only groups 0 to H-1 need to write carries out. - // Group H is a duplicate of group 0 (producing the same results) so we don't care about group H writing out, + // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. + // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. - if (gr < H) { for (i32 i = 0; i < NW; ++i) { carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(me, i)] = carry[i]; } } - - // Tell next line that its carries are ready + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead an #if is required +#if WMUL == 1 if (gr < H) { -#if OLD_FENCE - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(); - if (me == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else + if (gr < H / WMUL && me >= (WMUL-1) * G_W) { +#endif + for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } + + // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (me % WAVEFRONT == 0) { - u32 pos = gr * (G_W / WAVEFRONT) + me / WAVEFRONT; - atomic_store((atomic_uint *) &ready[pos], 1); +#if !OLD_FENCE + sync(); // Make sure all lanes have completed the CSSTORE + if (lowMe % WAVEFRONT == 0) { + u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; + atomic_store((global atomic_uint *) &ready[pos], 1); } #endif } - // Line zero will be redone when gr == H +#if OLD_FENCE + // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the + // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow + // is undefined. No barrier is needed when a sub-workgroup is a single wavefront. + if (gr < H / WMUL) { + bar(G_W); +#if WMUL == 1 + if (lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } +#else + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } +#endif + } +#endif + + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } // Do some work while our carries may not be ready @@ -350,72 +482,90 @@ KERNEL(G_W) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShuttle, P( __asm("s_setprio 0"); #endif - // Calculate inverse weights - F base = optionalHalve(weights.y); - for (u32 i = 0; i < NW; ++i) { - F weight1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i))); - F weight2 = optionalHalve(fancyMul(weight1, WEIGHT_STEP)); - u[i] = U2(weight1, weight2); - } +#if ROE + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, roundMax); +#elif STATS & (1 << MUL3) + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, carryMax); +#endif - // Wait until our carries are ready + // Shuffle carries up + shufl_carries_up(lds, carry, me, lowMe); + + // Wait until our carries are ready. The barrier below must be reached by every work-item of the + // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + if (me == 0) { do { spin(); } while(!atomic_load_explicit((global atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } bar(); - read_mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me == 0) ready[gr - 1] = 0; +#endif + if (me < G_W) { +#if OLD_FENCE + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me == 0) ready[gr - 1] = 0; #else - u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; - if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); - } - mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (me % WAVEFRONT == 0) { + do { spin(); } while(atomic_load_explicit((global atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + } + sync(); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; #endif #if HAS_ASM - __asm("s_setprio 1"); + __asm("s_setprio 1"); #endif - - // Read from the carryShuttle carries produced by the previous WIDTH row. Rotate carries from the last WIDTH row. - // The new carry layout lets the compiler generate global_load_dwordx4 instructions. - if (gr < H) { - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]; - } - } else { + } #if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); -#endif - - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]; - } - - if (me == 0) { - carry[NW] = carry[NW-1]; - for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } - carry[0] = carry[NW]; + // For the last group the carry reading is shifted, so the per-wavefront ready flags are not enough and a + // barrier is needed. gr is uniform but "me < G_W" is not, so the barrier is taken outside that guard and + // the shuttle reads resume in a second "me < G_W" block. + if (gr >= H / WMUL) { bar(); } +#endif + + if (me < G_W) { + + // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. + // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. + if (gr < H / WMUL) { + for (i32 i = 0; i < NW; ++i) { + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + } + } else { + + for (i32 i = 0; i < NW; ++i) { + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + } + + if (me == 0) { + carry[NW] = carry[NW-1]; + for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } + carry[0] = carry[NW]; + } } } // Apply each 32 or 64 bit carry to the 2 words + F base = weights.y; + frac_bits = base_frac_bits; for (i32 i = 0; i < NW; ++i) { - bool biglit0 = frac_bits + i * frac_bits_bigstep <= FRAC_BPW_HI; + // Calculate inverse weights + F weight1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); + F weight2 = optionalHalve(fancyMul(weight1, WEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); + frac_bits += FRAC_BPW_HI; + bool biglit0 = frac_bits <= FRAC_BPW_HI; wu[i] = carryFinal(wu[i], carry[i], biglit0); - u[i] = U2(u[i].x * wu[i].x, u[i].y * wu[i].y); - } + u[i] = U2(weight1 * wu[i].x, weight2 * wu[i].y); - bar(); + // Generate frac_bits for next pair + frac_bits += frac_bits_bigstep; + } -// fft_WIDTH(lds, u, smallTrig); - new_fft_WIDTH2(lds, u, smallTrig); + dependentLaunch(); // Next kernel will be fftMiddleInFP32 - writeCarryFusedLine(u, out, line); + fft_WIDTH2(lds, u, smallTrig, WMUL, lowMe); + writeCarryFusedLine(u, out, line, lowMe); } @@ -427,13 +577,9 @@ KERNEL(G_W) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShuttle, P( // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigGF31 smallTrig, P(uint) bufROE) { - -#if 0 // fft_WIDTH uses shufl_int instead of shufl - local GF31 lds[WIDTH / 4]; -#else - local GF31 lds[WIDTH / 2]; -#endif +KERNEL_CAP(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigGF31 smallTrig, P(uint) bufROE) { + local GF31 lds[LDS_BYTES(WMUL) / sizeof(GF31)]; + LDSinit(lds, WMUL); GF31 u[NW]; @@ -441,40 +587,38 @@ KERNEL(G_W) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle u32 me = get_local_id(0); u32 H = BIG_HEIGHT; - u32 line = gr % H; +#if WMUL == 1 + u32 lowMe = me; + u32 line = gr; +#else + u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. + u32 line = gr * WMUL + me / G_W; +#endif + if (line >= H) line -= H; #if HAS_ASM __asm("s_setprio 3"); #endif - readCarryFusedLine(in, u, line); + dependentLaunchWait(); // Previous kernel was fftMiddleOutGF31 + + readCarryFusedLine(in, u, line, lowMe); // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. -#if ZEROHACK_W - u32 zerohack = get_group_id(0) / 131072; - new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack); -#else - new_fft_WIDTH1(lds, u, smallTrig); -#endif + u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, lowMe); Word2 wu[NW]; P(CFcarry) carryShuttlePtr = (P(CFcarry)) carryShuttle; CFcarry carry[NW+1]; -#if AMDGPU -#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions -//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 -#else -#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better -#endif - u32 roundMax = 0; float carryMax = 0; - u32 word_index = (me * H + line) * 2; + u32 word_index = (lowMe * H + line) * 2; // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Weights can be applied with shifts because 2 is the 60th root GF31. @@ -490,9 +634,9 @@ KERNEL(G_W) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * H * 2 - 1) * combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * H * 2 - 1) + make_u64((G_W * H * 2 - 1) * bigword_weight_shift_minus1, 0)) % (31ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 31; u64 starting_combo_counter = combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -530,35 +674,43 @@ KERNEL(G_W) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle } combo_counter = starting_combo_counter; // Restore starting counter for applying weights after carry propagation -#if ROE - float fltRoundMax = (float) roundMax / (float) M31; // For speed, roundoff was computed as 32-bit integer. Convert to float. - updateStats(bufROE, posROE, fltRoundMax); -#elif STATS & (1 << MUL3) - updateStats(bufROE, posROE, carryMax); -#endif - - // Write out our carries. Only groups 0 to H-1 need to write carries out. - // Group H is a duplicate of group 0 (producing the same results) so we don't care about group H writing out, + // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. + // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. - if (gr < H) { for (i32 i = 0; i < NW; ++i) { carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(me, i)] = carry[i]; } } - - // Tell next line that its carries are ready + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead an #if is required +#if WMUL == 1 if (gr < H) { -#if OLD_FENCE - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(); - if (me == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else + if (gr < H / WMUL && me >= (WMUL-1) * G_W) { +#endif + for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } + + // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (me % WAVEFRONT == 0) { - u32 pos = gr * (G_W / WAVEFRONT) + me / WAVEFRONT; - atomic_store((atomic_uint *) &ready[pos], 1); +#if !OLD_FENCE + sync(); // Make sure all lanes have completed the CSSTORE + if (lowMe % WAVEFRONT == 0) { + u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; + atomic_store((global atomic_uint *) &ready[pos], 1); } #endif } - // Line zero will be redone when gr == H +#if OLD_FENCE + // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the + // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow + // is undefined. No barrier is needed when a sub-workgroup is a single wavefront. + if (gr < H / WMUL) { + bar(G_W); +#if WMUL == 1 + if (lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } +#else + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } +#endif + } +#endif + + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } // Do some work while our carries may not be ready @@ -566,48 +718,67 @@ KERNEL(G_W) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle __asm("s_setprio 0"); #endif - // Wait until our carries are ready +#if ROE + float fltRoundMax = (float) roundMax / (float) M31; // For speed, roundoff was computed as 32-bit integer. Convert to float. + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, fltRoundMax); +#elif STATS & (1 << MUL3) + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, carryMax); +#endif + + // Shuffle carries up + shufl_carries_up(lds, carry, me, lowMe); + + // Wait until our carries are ready. The barrier below must be reached by every work-item of the + // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + if (me == 0) { do { spin(); } while(!atomic_load_explicit((global atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } bar(); - read_mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me == 0) ready[gr - 1] = 0; +#endif + if (me < G_W) { +#if OLD_FENCE + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me == 0) ready[gr - 1] = 0; #else - u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; - if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); - } - mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (me % WAVEFRONT == 0) { + do { spin(); } while(atomic_load_explicit((global atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + } + sync(); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; #endif #if HAS_ASM - __asm("s_setprio 1"); + __asm("s_setprio 1"); #endif - - // Read from the carryShuttle carries produced by the previous WIDTH row. Rotate carries from the last WIDTH row. - // The new carry layout lets the compiler generate global_load_dwordx4 instructions. - if (gr < H) { - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]; - } - } else { + } #if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); -#endif - - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]; - } - - if (me == 0) { - carry[NW] = carry[NW-1]; - for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } - carry[0] = carry[NW]; + // For the last group the carry reading is shifted, so the per-wavefront ready flags are not enough and a + // barrier is needed. gr is uniform but "me < G_W" is not, so the barrier is taken outside that guard and + // the shuttle reads resume in a second "me < G_W" block. + if (gr >= H / WMUL) { bar(); } +#endif + + if (me < G_W) { + + // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. + if (gr < H / WMUL) { + for (i32 i = 0; i < NW; ++i) { + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + } + } else { + + for (i32 i = 0; i < NW; ++i) { + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + } + + if (me == 0) { + carry[NW] = carry[NW-1]; + for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } + carry[0] = carry[NW]; + } } } @@ -627,11 +798,10 @@ KERNEL(G_W) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle if (weight_shift > 31) weight_shift -= 31; } - bar(); - - new_fft_WIDTH2(lds, u, smallTrig); + dependentLaunch(); // Next kernel will be fftMiddleInGF31 - writeCarryFusedLine(u, out, line); + fft_WIDTH2(lds, u, smallTrig, WMUL, lowMe); + writeCarryFusedLine(u, out, line, lowMe); } @@ -643,13 +813,9 @@ KERNEL(G_W) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigGF61 smallTrig, P(uint) bufROE) { - -#if 0 // fft_WIDTH uses shufl_int instead of shufl - local GF61 lds[WIDTH / 4]; -#else - local GF61 lds[WIDTH / 2]; -#endif +KERNEL_CAP(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigGF61 smallTrig, P(uint) bufROE) { + local GF61 lds[LDS_BYTES(WMUL) / sizeof(GF61)]; + LDSinit(lds, WMUL); GF61 u[NW]; @@ -657,23 +823,28 @@ KERNEL(G_W) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle u32 me = get_local_id(0); u32 H = BIG_HEIGHT; - u32 line = gr % H; +#if WMUL == 1 + u32 lowMe = me; + u32 line = gr; +#else + u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. + u32 line = gr * WMUL + me / G_W; +#endif + if (line >= H) line -= H; #if HAS_ASM __asm("s_setprio 3"); #endif - readCarryFusedLine(in, u, line); + dependentLaunchWait(); // Previous kernel was fftMiddleOutGF61 + + readCarryFusedLine(in, u, line, lowMe); // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. -#if ZEROHACK_W - u32 zerohack = get_group_id(0) / 131072; - new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack); -#else - new_fft_WIDTH1(lds, u, smallTrig); -#endif + u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, lowMe); Word2 wu[NW]; @@ -685,17 +856,10 @@ KERNEL(G_W) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle CFcarry carry[NW+1]; #endif -#if AMDGPU -#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions -//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 -#else -#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better -#endif - u32 roundMax = 0; float carryMax = 0; - u32 word_index = (me * H + line) * 2; + u32 word_index = (lowMe * H + line) * 2; // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Weights can be applied with shifts because 2 is the 60th root GF61. @@ -711,9 +875,9 @@ KERNEL(G_W) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * H * 2 - 1) * combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * H * 2 - 1) + make_u64((G_W * H * 2 - 1) * bigword_weight_shift_minus1, 0)) % (61ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 61; u64 starting_combo_counter = combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -751,35 +915,43 @@ KERNEL(G_W) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle } combo_counter = starting_combo_counter; // Restore starting counter for applying weights after carry propagation -#if ROE - float fltRoundMax = (float) roundMax / (float) (M61 >> 32); // For speed, roundoff was computed as 32-bit integer. Convert to float. - updateStats(bufROE, posROE, fltRoundMax); -#elif STATS & (1 << MUL3) - updateStats(bufROE, posROE, carryMax); -#endif - - // Write out our carries. Only groups 0 to H-1 need to write carries out. - // Group H is a duplicate of group 0 (producing the same results) so we don't care about group H writing out, + // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. + // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. - if (gr < H) { for (i32 i = 0; i < NW; ++i) { carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(me, i)] = carry[i]; } } - - // Tell next line that its carries are ready + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead an #if is required +#if WMUL == 1 if (gr < H) { -#if OLD_FENCE - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(); - if (me == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else + if (gr < H / WMUL && me >= (WMUL-1) * G_W) { +#endif + for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } + + // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (me % WAVEFRONT == 0) { - u32 pos = gr * (G_W / WAVEFRONT) + me / WAVEFRONT; - atomic_store((atomic_uint *) &ready[pos], 1); +#if !OLD_FENCE + sync(); // Make sure all lanes have completed the CSSTORE + if (lowMe % WAVEFRONT == 0) { + u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; + atomic_store((global atomic_uint *) &ready[pos], 1); } #endif } - // Line zero will be redone when gr == H +#if OLD_FENCE + // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the + // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow + // is undefined. No barrier is needed when a sub-workgroup is a single wavefront. + if (gr < H / WMUL) { + bar(G_W); +#if WMUL == 1 + if (lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } +#else + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } +#endif + } +#endif + + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } // Do some work while our carries may not be ready @@ -787,48 +959,68 @@ KERNEL(G_W) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle __asm("s_setprio 0"); #endif - // Wait until our carries are ready +#if ROE + float fltRoundMax = (float) roundMax / (float) (M61 >> 32); // For speed, roundoff was computed as 32-bit integer. Convert to float. + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, fltRoundMax); +#elif STATS & (1 << MUL3) + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, carryMax); +#endif + + // Shuffle carries up + shufl_carries_up(lds, carry, me, lowMe); + + // Wait until our carries are ready. The barrier below must be reached by every work-item of the + // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + if (me == 0) { do { spin(); } while(!atomic_load_explicit((global atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } bar(); - read_mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me == 0) ready[gr - 1] = 0; +#endif + if (me < G_W) { +#if OLD_FENCE + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me == 0) ready[gr - 1] = 0; #else - u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; - if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); - } - mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (me % WAVEFRONT == 0) { + do { spin(); } while(atomic_load_explicit((global atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + } + sync(); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; #endif #if HAS_ASM - __asm("s_setprio 1"); + __asm("s_setprio 1"); #endif - - // Read from the carryShuttle carries produced by the previous WIDTH row. Rotate carries from the last WIDTH row. - // The new carry layout lets the compiler generate global_load_dwordx4 instructions. - if (gr < H) { - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]; - } - } else { + } #if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); -#endif - - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]; - } - - if (me == 0) { - carry[NW] = carry[NW-1]; - for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } - carry[0] = carry[NW]; + // For the last group the carry reading is shifted, so the per-wavefront ready flags are not enough and a + // barrier is needed. gr is uniform but "me < G_W" is not, so the barrier is taken outside that guard and + // the shuttle reads resume in a second "me < G_W" block. + if (gr >= H / WMUL) { bar(); } +#endif + + if (me < G_W) { + + // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. + // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. + if (gr < H / WMUL) { + for (i32 i = 0; i < NW; ++i) { + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + } + } else { + + for (i32 i = 0; i < NW; ++i) { + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + } + + if (me == 0) { + carry[NW] = carry[NW-1]; + for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } + carry[0] = carry[NW]; + } } } @@ -848,11 +1040,10 @@ KERNEL(G_W) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle if (weight_shift > 61) weight_shift -= 61; } - bar(); - - new_fft_WIDTH2(lds, u, smallTrig); + dependentLaunch(); // Next kernel will be fftMiddleInGF61 - writeCarryFusedLine(u, out, line); + fft_WIDTH2(lds, u, smallTrig, WMUL, lowMe); + writeCarryFusedLine(u, out, line, lowMe); } @@ -864,11 +1055,11 @@ KERNEL(G_W) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, - CP(u32) bits, ConstBigTab CONST_THREAD_WEIGHTS, BigTab THREAD_WEIGHTS, P(uint) bufROE) { - - local T2 lds[WIDTH / 2]; +KERNEL_CAP(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, + ConstBigTab CONST_THREAD_WEIGHTS, BigTab THREAD_WEIGHTS, P(uint) bufROE) { + local T2 lds[LDS_BYTES(WMUL) / sizeof(T2)]; local GF31 *lds31 = (local GF31 *) lds; + LDSinit(lds, WMUL); T2 u[NW]; GF31 u31[NW]; @@ -877,7 +1068,14 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( u32 me = get_local_id(0); u32 H = BIG_HEIGHT; - u32 line = gr % H; +#if WMUL == 1 + u32 lowMe = me; + u32 line = gr; +#else + u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. + u32 line = gr * WMUL + me / G_W; +#endif + if (line >= H) line -= H; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -887,43 +1085,36 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( __asm("s_setprio 3"); #endif - readCarryFusedLine(in, u, line); - readCarryFusedLine(in31, u31, line); - // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. -#if ZEROHACK_W - u32 zerohack = get_group_id(0) / 131072; - new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack); - bar(); - new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack); -#else - new_fft_WIDTH1(lds, u, smallTrig); - bar(); - new_fft_WIDTH1(lds31, u31, smallTrig31); -#endif + u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + + readCarryFusedLine(in, u, line, lowMe); + fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, lowMe); + + dependentLaunchWait(); // Previous kernel was fftMiddleOutGF31 + + readCarryFusedLine(in31, u31, line, lowMe); + fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, lowMe); Word2 wu[NW]; -#if AMDGPU - T2 weights = fancyMul(THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); +#if !NVIDIAGPU || CUDA_BACKEND + T2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), TSLOAD(&THREAD_WEIGHTS[G_W + line])); #else - T2 weights = fancyMul(CONST_THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights + T2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), CONST_THREAD_WEIGHTS[line % 64]); + weights.x = optionalDouble(weights.x); + weights.y = optionalHalve(weights.y); + weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); #endif + P(i64) carryShuttlePtr = (P(i64)) carryShuttle; i64 carry[NW+1]; -#if AMDGPU -#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions -//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 -#else -#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better -#endif - float roundMax = 0; float carryMax = 0; - u32 word_index = (me * H + line) * 2; + u32 word_index = (lowMe * H + line) * 2; // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Let s be the shift amount for word 1. The shift amount for word x is ceil(x * (s - 1) + num_big_words_less_than_x) % 31. @@ -938,9 +1129,9 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * H * 2 - 1) * combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * H * 2 - 1) + make_u64((G_W * H * 2 - 1) * bigword_weight_shift_minus1, 0)) % (31ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 31; u64 starting_combo_counter = combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -981,34 +1172,43 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( } combo_counter = starting_combo_counter; // Restore starting counter for applying weights after carry propagation -#if ROE - updateStats(bufROE, posROE, roundMax); -#elif STATS & (1 << MUL3) - updateStats(bufROE, posROE, carryMax); -#endif - - // Write out our carries. Only groups 0 to H-1 need to write carries out. - // Group H is a duplicate of group 0 (producing the same results) so we don't care about group H writing out, + // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. + // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. - if (gr < H) { for (i32 i = 0; i < NW; ++i) { carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(me, i)] = carry[i]; } } - - // Tell next line that its carries are ready + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead an #if is required +#if WMUL == 1 if (gr < H) { -#if OLD_FENCE - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(); - if (me == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else + if (gr < H / WMUL && me >= (WMUL-1) * G_W) { +#endif + for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } + + // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (me % WAVEFRONT == 0) { - u32 pos = gr * (G_W / WAVEFRONT) + me / WAVEFRONT; - atomic_store((atomic_uint *) &ready[pos], 1); +#if !OLD_FENCE + sync(); // Make sure all lanes have completed the CSSTORE + if (lowMe % WAVEFRONT == 0) { + u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; + atomic_store((global atomic_uint *) &ready[pos], 1); } #endif } - // Line zero will be redone when gr == H +#if OLD_FENCE + // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the + // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow + // is undefined. No barrier is needed when a sub-workgroup is a single wavefront. + if (gr < H / WMUL) { + bar(G_W); +#if WMUL == 1 + if (lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } +#else + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } +#endif + } +#endif + + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } // Do some work while our carries may not be ready @@ -1016,6 +1216,12 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( __asm("s_setprio 0"); #endif +#if ROE + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, roundMax); +#elif STATS & (1 << MUL3) + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, carryMax); +#endif + // Calculate inverse weights T base = optionalHalve(weights.y); for (u32 i = 0; i < NW; ++i) { @@ -1024,48 +1230,61 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( u[i] = U2(weight1, weight2); } - // Wait until our carries are ready + // Shuffle carries up + shufl_carries_up(lds, carry, me, lowMe); + + // Wait until our carries are ready. The barrier below must be reached by every work-item of the + // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + if (me == 0) { do { spin(); } while(!atomic_load_explicit((global atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } bar(); - read_mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me == 0) ready[gr - 1] = 0; +#endif + if (me < G_W) { +#if OLD_FENCE + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me == 0) ready[gr - 1] = 0; #else - u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; - if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); - } - mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (me % WAVEFRONT == 0) { + do { spin(); } while(atomic_load_explicit((global atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + } + sync(); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; #endif #if HAS_ASM - __asm("s_setprio 1"); -#endif - - // Read from the carryShuttle carries produced by the previous WIDTH row. Rotate carries from the last WIDTH row. - // The new carry layout lets the compiler generate global_load_dwordx4 instructions. - if (gr < H) { - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]; - } - } else { - -#if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); + __asm("s_setprio 1"); #endif + } - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]; - } - - if (me == 0) { - carry[NW] = carry[NW-1]; - for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } - carry[0] = carry[NW]; +#if !OLD_FENCE + // For the last group the carry reading is shifted, so the per-wavefront ready flags are not enough and a + // barrier is needed. gr is uniform but "me < G_W" is not, so the barrier is taken outside that guard and + // the shuttle reads resume in a second "me < G_W" block. + if (gr >= H / WMUL) { bar(); } +#endif + + if (me < G_W) { + + // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. + // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. + if (gr < H / WMUL) { + for (i32 i = 0; i < NW; ++i) { + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + } + } else { + + for (i32 i = 0; i < NW; ++i) { + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + } + + if (me == 0) { + carry[NW] = carry[NW-1]; + for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } + carry[0] = carry[NW]; + } } } @@ -1087,15 +1306,13 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( if (weight_shift > 31) weight_shift -= 31; } - bar(); - - new_fft_WIDTH2(lds, u, smallTrig); - writeCarryFusedLine(u, out, line); + fft_WIDTH2(lds, u, smallTrig, WMUL, lowMe); + writeCarryFusedLine(u, out, line, lowMe); - bar(); + dependentLaunch(); // Next kernel will be fftMiddleInFP32 - new_fft_WIDTH2(lds31, u31, smallTrig31); - writeCarryFusedLine(u31, out31, line); + fft_WIDTH2(lds31, u31, smallTrig31, WMUL, lowMe); + writeCarryFusedLine(u31, out31, line, lowMe); } @@ -1107,11 +1324,11 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, - CP(u32) bits, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - - local F2 ldsF2[WIDTH / 2]; +KERNEL_CAP(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, + ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { + local F2 ldsF2[LDS_BYTES(WMUL) / sizeof(F2)]; local GF31 *lds31 = (local GF31 *) ldsF2; + LDSinit(ldsF2, WMUL); F2 uF2[NW]; GF31 u31[NW]; @@ -1120,7 +1337,14 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( u32 me = get_local_id(0); u32 H = BIG_HEIGHT; - u32 line = gr % H; +#if WMUL == 1 + u32 lowMe = me; + u32 line = gr; +#else + u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. + u32 line = gr * WMUL + me / G_W; +#endif + if (line >= H) line -= H; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -1133,43 +1357,47 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( __asm("s_setprio 3"); #endif - readCarryFusedLine(inF2, uF2, line); - readCarryFusedLine(in31, u31, line); - // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. -#if ZEROHACK_W - u32 zerohack = get_group_id(0) / 131072; - new_fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack); - bar(); - new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack); -#else - new_fft_WIDTH1(ldsF2, uF2, smallTrigF2); - bar(); - new_fft_WIDTH1(lds31, u31, smallTrig31); -#endif + u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + + readCarryFusedLine(inF2, uF2, line, lowMe); + fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack, WMUL, lowMe); + + dependentLaunchWait(); // Previous kernel was fftMiddleOutGF31 + + readCarryFusedLine(in31, u31, line, lowMe); + fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, lowMe); Word2 wu[NW]; -#if AMDGPU - F2 weights = fancyMul(THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); + u32 me_frac_bits = fracBits(lowMe * H * 2); +#if !NVIDIAGPU || CUDA_BACKEND + F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), TSLOAD(&THREAD_WEIGHTS[G_W + line])); + u32 line_frac_bits = fracBits(line * 2); + u32 base_frac_bits = me_frac_bits + line_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > line_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > line_frac_bits); #else - F2 weights = fancyMul(CONST_THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights + F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), CONST_THREAD_WEIGHTS[line % 64]); + u32 partialLine_frac_bits = fracBits((line % 64) * 2); + u32 base_frac_bits = me_frac_bits + partialLine_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); + weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); + partialLine_frac_bits = fracBits(((line / 64) * 64) * 2); + base_frac_bits = base_frac_bits + partialLine_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); #endif + P(i32) carryShuttlePtr = (P(i32)) carryShuttle; i32 carry[NW+1]; -#if AMDGPU -#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions -//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 -#else -#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better -#endif - float roundMax = 0; float carryMax = 0; - u32 word_index = (me * H + line) * 2; + u32 word_index = (lowMe * H + line) * 2; // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Let s be the shift amount for word 1. The shift amount for word x is ceil(x * (s - 1) + num_big_words_less_than_x) % 31. @@ -1184,9 +1412,9 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * H * 2 - 1) * combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * H * 2 - 1) + make_u64((G_W * H * 2 - 1) * bigword_weight_shift_minus1, 0)) % (61ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 31; u64 starting_combo_counter = combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -1199,11 +1427,11 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( // Apply the inverse weights and carry propagate pairs to generate the output carries - F invBase = optionalDouble(weights.x); + F invBase = weights.x; for (u32 i = 0; i < NW; ++i) { // Generate the FP32 weights and second GF31 weight shift - F invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i))); - F invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP)); + F invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i)), frac_bits > base_frac_bits); + F invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 weight_shift0 = weight_shift; combo_counter += combo_step; if (weight_shift > 31) weight_shift -= 31; @@ -1227,34 +1455,43 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( } combo_counter = starting_combo_counter; // Restore starting counter for applying weights after carry propagation -#if ROE - updateStats(bufROE, posROE, roundMax); -#elif STATS & (1 << MUL3) - updateStats(bufROE, posROE, carryMax); -#endif - - // Write out our carries. Only groups 0 to H-1 need to write carries out. - // Group H is a duplicate of group 0 (producing the same results) so we don't care about group H writing out, + // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. + // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. - if (gr < H) { for (i32 i = 0; i < NW; ++i) { carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(me, i)] = carry[i]; } } - - // Tell next line that its carries are ready + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead an #if is required +#if WMUL == 1 if (gr < H) { -#if OLD_FENCE - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(); - if (me == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else + if (gr < H / WMUL && me >= (WMUL-1) * G_W) { +#endif + for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } + + // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (me % WAVEFRONT == 0) { - u32 pos = gr * (G_W / WAVEFRONT) + me / WAVEFRONT; - atomic_store((atomic_uint *) &ready[pos], 1); +#if !OLD_FENCE + sync(); // Make sure all lanes have completed the CSSTORE + if (lowMe % WAVEFRONT == 0) { + u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; + atomic_store((global atomic_uint *) &ready[pos], 1); } #endif } - // Line zero will be redone when gr == H +#if OLD_FENCE + // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the + // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow + // is undefined. No barrier is needed when a sub-workgroup is a single wavefront. + if (gr < H / WMUL) { + bar(G_W); +#if WMUL == 1 + if (lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } +#else + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } +#endif + } +#endif + + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } // Do some work while our carries may not be ready @@ -1262,61 +1499,75 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( __asm("s_setprio 0"); #endif - // Calculate inverse weights - F base = optionalHalve(weights.y); - for (u32 i = 0; i < NW; ++i) { - F weight1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i))); - F weight2 = optionalHalve(fancyMul(weight1, WEIGHT_STEP)); - uF2[i] = U2(weight1, weight2); - } +#if ROE + updateStats((local u32 *) ldsF2, G_W * WMUL, H / WMUL, bufROE, posROE, roundMax); +#elif STATS & (1 << MUL3) + updateStats((local u32 *) ldsF2, G_W * WMUL, H / WMUL, bufROE, posROE, carryMax); +#endif - // Wait until our carries are ready + // Shuffle carries up + shufl_carries_up(ldsF2, carry, me, lowMe); + + // Wait until our carries are ready. The barrier below must be reached by every work-item of the + // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + if (me == 0) { do { spin(); } while(!atomic_load_explicit((global atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } bar(); - read_mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me == 0) ready[gr - 1] = 0; +#endif + if (me < G_W) { +#if OLD_FENCE + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me == 0) ready[gr - 1] = 0; #else - u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; - if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); - } - mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (me % WAVEFRONT == 0) { + do { spin(); } while(atomic_load_explicit((global atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + } + sync(); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; #endif #if HAS_ASM - __asm("s_setprio 1"); + __asm("s_setprio 1"); #endif - - // Read from the carryShuttle carries produced by the previous WIDTH row. Rotate carries from the last WIDTH row. - // The new carry layout lets the compiler generate global_load_dwordx4 instructions. - if (gr < H) { - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]; - } - } else { + } #if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); -#endif - - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]; - } - - if (me == 0) { - carry[NW] = carry[NW-1]; - for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } - carry[0] = carry[NW]; + // For the last group the carry reading is shifted, so the per-wavefront ready flags are not enough and a + // barrier is needed. gr is uniform but "me < G_W" is not, so the barrier is taken outside that guard and + // the shuttle reads resume in a second "me < G_W" block. + if (gr >= H / WMUL) { bar(); } +#endif + + if (me < G_W) { + + // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. + // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. + if (gr < H / WMUL) { + for (i32 i = 0; i < NW; ++i) { + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + } + } else { + + for (i32 i = 0; i < NW; ++i) { + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + } + + if (me == 0) { + carry[NW] = carry[NW-1]; + for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } + carry[0] = carry[NW]; + } } } // Apply each 32 or 64 bit carry to the 2 words. Apply weights. + F base = weights.y; for (i32 i = 0; i < NW; ++i) { + F weight1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); + F weight2 = optionalHalve(fancyMul(weight1, WEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); // Generate the second weight shift u32 weight_shift0 = weight_shift; combo_counter += combo_step; @@ -1325,7 +1576,7 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( // Generate big-word/little-word flag, propagate final carry bool biglit0 = frac_bits <= FRAC_BPW_HI; wu[i] = carryFinal(wu[i], carry[i], biglit0); - uF2[i] = U2(uF2[i].x * wu[i].x, uF2[i].y * wu[i].y); + uF2[i] = U2(weight1 * wu[i].x, weight2 * wu[i].y); u31[i] = U2(shl(make_Z31(wu[i].x), weight_shift0), shl(make_Z31(wu[i].y), weight_shift1)); // Generate weight shifts and frac_bits for next pair @@ -1333,15 +1584,13 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( if (weight_shift > 31) weight_shift -= 31; } - bar(); - - new_fft_WIDTH2(ldsF2, uF2, smallTrigF2); - writeCarryFusedLine(uF2, outF2, line); + fft_WIDTH2(ldsF2, uF2, smallTrigF2, WMUL, lowMe); + writeCarryFusedLine(uF2, outF2, line, lowMe); - bar(); + dependentLaunch(); // Next kernel will be fftMiddleInFP32 - new_fft_WIDTH2(lds31, u31, smallTrig31); - writeCarryFusedLine(u31, out31, line); + fft_WIDTH2(lds31, u31, smallTrig31, WMUL, lowMe); + writeCarryFusedLine(u31, out31, line, lowMe); } @@ -1353,11 +1602,11 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, - CP(u32) bits, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - - local GF61 lds61[WIDTH / 2]; +KERNEL_CAP(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, + ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { + local GF61 lds61[LDS_BYTES(WMUL) / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; + LDSinit(lds61, WMUL); F2 uF2[NW]; GF61 u61[NW]; @@ -1366,7 +1615,14 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( u32 me = get_local_id(0); u32 H = BIG_HEIGHT; - u32 line = gr % H; +#if WMUL == 1 + u32 lowMe = me; + u32 line = gr; +#else + u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. + u32 line = gr * WMUL + me / G_W; +#endif + if (line >= H) line -= H; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -1379,43 +1635,47 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( __asm("s_setprio 3"); #endif - readCarryFusedLine(inF2, uF2, line); - readCarryFusedLine(in61, u61, line); - // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. -#if ZEROHACK_W - u32 zerohack = get_group_id(0) / 131072; - new_fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack); - bar(); - new_fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack); -#else - new_fft_WIDTH1(ldsF2, uF2, smallTrigF2); - bar(); - new_fft_WIDTH1(lds61, u61, smallTrig61); -#endif + u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + + readCarryFusedLine(inF2, uF2, line, lowMe); + fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack, WMUL, lowMe); + + dependentLaunchWait(); // Previous kernel was fftMiddleOutGF61 + + readCarryFusedLine(in61, u61, line, lowMe); + fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, lowMe); Word2 wu[NW]; -#if AMDGPU - F2 weights = fancyMul(THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); + u32 me_frac_bits = fracBits(lowMe * H * 2); +#if !NVIDIAGPU || CUDA_BACKEND + F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), TSLOAD(&THREAD_WEIGHTS[G_W + line])); + u32 line_frac_bits = fracBits(line * 2); + u32 base_frac_bits = me_frac_bits + line_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > line_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > line_frac_bits); #else - F2 weights = fancyMul(CONST_THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights + F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), CONST_THREAD_WEIGHTS[line % 64]); + u32 partialLine_frac_bits = fracBits((line % 64) * 2); + u32 base_frac_bits = me_frac_bits + partialLine_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); + weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); + partialLine_frac_bits = fracBits(((line / 64) * 64) * 2); + base_frac_bits = base_frac_bits + partialLine_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); #endif + P(i64) carryShuttlePtr = (P(i64)) carryShuttle; i64 carry[NW+1]; -#if AMDGPU -#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions -//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 -#else -#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better -#endif - float roundMax = 0; float carryMax = 0; - u32 word_index = (me * H + line) * 2; + u32 word_index = (lowMe * H + line) * 2; // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Let s be the shift amount for word 1. The shift amount for word x is ceil(x * (s - 1) + num_big_words_less_than_x) % 61. @@ -1430,9 +1690,9 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * H * 2 - 1) * combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * H * 2 - 1) + make_u64((G_W * H * 2 - 1) * bigword_weight_shift_minus1, 0)) % (61ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 61; u64 starting_combo_counter = combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -1445,11 +1705,12 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( // Apply the inverse weights and carry propagate pairs to generate the output carries - F invBase = optionalDouble(weights.x); + F invBase = weights.x; for (u32 i = 0; i < NW; ++i) { // Generate the FP32 weights and second GF61 weight shift - F invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i))); - F invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP)); + F invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i)), frac_bits > base_frac_bits); + F invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); + u32 weight_shift0 = weight_shift; combo_counter += combo_step; if (weight_shift > 61) weight_shift -= 61; @@ -1473,34 +1734,43 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( } combo_counter = starting_combo_counter; // Restore starting counter for applying weights after carry propagation -#if ROE - updateStats(bufROE, posROE, roundMax); -#elif STATS & (1 << MUL3) - updateStats(bufROE, posROE, carryMax); -#endif - - // Write out our carries. Only groups 0 to H-1 need to write carries out. - // Group H is a duplicate of group 0 (producing the same results) so we don't care about group H writing out, + // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. + // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. - if (gr < H) { for (i32 i = 0; i < NW; ++i) { carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(me, i)] = carry[i]; } } - - // Tell next line that its carries are ready + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead an #if is required +#if WMUL == 1 if (gr < H) { -#if OLD_FENCE - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(); - if (me == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else + if (gr < H / WMUL && me >= (WMUL-1) * G_W) { +#endif + for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } + + // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (me % WAVEFRONT == 0) { - u32 pos = gr * (G_W / WAVEFRONT) + me / WAVEFRONT; - atomic_store((atomic_uint *) &ready[pos], 1); +#if !OLD_FENCE + sync(); // Make sure all lanes have completed the CSSTORE + if (lowMe % WAVEFRONT == 0) { + u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; + atomic_store((global atomic_uint *) &ready[pos], 1); } #endif } - // Line zero will be redone when gr == H +#if OLD_FENCE + // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the + // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow + // is undefined. No barrier is needed when a sub-workgroup is a single wavefront. + if (gr < H / WMUL) { + bar(G_W); +#if WMUL == 1 + if (lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } +#else + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } +#endif + } +#endif + + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } // Do some work while our carries may not be ready @@ -1508,61 +1778,76 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( __asm("s_setprio 0"); #endif - // Calculate inverse weights - F base = optionalHalve(weights.y); - for (u32 i = 0; i < NW; ++i) { - F weight1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i))); - F weight2 = optionalHalve(fancyMul(weight1, WEIGHT_STEP)); - uF2[i] = U2(weight1, weight2); - } +#if ROE + updateStats((local u32 *) lds61, G_W * WMUL, H / WMUL, bufROE, posROE, roundMax); +#elif STATS & (1 << MUL3) + updateStats((local u32 *) lds61, G_W * WMUL, H / WMUL, bufROE, posROE, carryMax); +#endif + + // Shuffle carries up + shufl_carries_up(lds61, carry, me, lowMe); - // Wait until our carries are ready + // Wait until our carries are ready. The barrier below must be reached by every work-item of the + // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + if (me == 0) { do { spin(); } while(!atomic_load_explicit((global atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } bar(); - read_mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me == 0) ready[gr - 1] = 0; +#endif + if (me < G_W) { +#if OLD_FENCE + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me == 0) ready[gr - 1] = 0; #else - u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; - if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); - } - mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (me % WAVEFRONT == 0) { + do { spin(); } while(atomic_load_explicit((global atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + } + sync(); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; #endif #if HAS_ASM - __asm("s_setprio 1"); + __asm("s_setprio 1"); #endif - - // Read from the carryShuttle carries produced by the previous WIDTH row. Rotate carries from the last WIDTH row. - // The new carry layout lets the compiler generate global_load_dwordx4 instructions. - if (gr < H) { - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]; - } - } else { + } #if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); -#endif - - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]; - } - - if (me == 0) { - carry[NW] = carry[NW-1]; - for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } - carry[0] = carry[NW]; + // For the last group the carry reading is shifted, so the per-wavefront ready flags are not enough and a + // barrier is needed. gr is uniform but "me < G_W" is not, so the barrier is taken outside that guard and + // the shuttle reads resume in a second "me < G_W" block. + if (gr >= H / WMUL) { bar(); } +#endif + + if (me < G_W) { + + // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. + // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. + if (gr < H / WMUL) { + for (i32 i = 0; i < NW; ++i) { + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + } + } else { + + for (i32 i = 0; i < NW; ++i) { + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + } + + if (me == 0) { + carry[NW] = carry[NW-1]; + for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } + carry[0] = carry[NW]; + } } } // Apply each 32 or 64 bit carry to the 2 words. Apply weights. + F base = weights.y; for (i32 i = 0; i < NW; ++i) { + // Calculate inverse weights + F weight1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); + F weight2 = optionalHalve(fancyMul(weight1, WEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); // Generate the second weight shift u32 weight_shift0 = weight_shift; combo_counter += combo_step; @@ -1571,7 +1856,7 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( // Generate big-word/little-word flag, propagate final carry bool biglit0 = frac_bits <= FRAC_BPW_HI; wu[i] = carryFinal(wu[i], carry[i], biglit0); - uF2[i] = U2(uF2[i].x * wu[i].x, uF2[i].y * wu[i].y); + uF2[i] = U2(weight1 * wu[i].x, weight2 * wu[i].y); u61[i] = U2(shl(make_Z61(wu[i].x), weight_shift0), shl(make_Z61(wu[i].y), weight_shift1)); // Generate weight shifts and frac_bits for next pair @@ -1579,15 +1864,13 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( if (weight_shift > 61) weight_shift -= 61; } - bar(); - - new_fft_WIDTH2(ldsF2, uF2, smallTrigF2); - writeCarryFusedLine(uF2, outF2, line); + fft_WIDTH2(ldsF2, uF2, smallTrigF2, WMUL, lowMe); + writeCarryFusedLine(uF2, outF2, line, lowMe); - bar(); + dependentLaunch(); // Next kernel will be fftMiddleInFP32 - new_fft_WIDTH2(lds61, u61, smallTrig61); - writeCarryFusedLine(u61, out61, line); + fft_WIDTH2(lds61, u61, smallTrig61, WMUL, lowMe); + writeCarryFusedLine(u61, out61, line, lowMe); } @@ -1599,14 +1882,10 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, P(uint) bufROE) { - -#if 0 // fft_WIDTH uses shufl_int instead of shufl - local GF61 lds61[WIDTH / 4]; -#else - local GF61 lds61[WIDTH / 2]; -#endif +KERNEL_CAP(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, P(uint) bufROE) { + local GF61 lds61[LDS_BYTES(WMUL) / sizeof(GF61)]; local GF31 *lds31 = (local GF31 *) lds61; + LDSinit(lds61, WMUL); GF31 u31[NW]; GF61 u61[NW]; @@ -1615,7 +1894,14 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( u32 me = get_local_id(0); u32 H = BIG_HEIGHT; - u32 line = gr % H; +#if WMUL == 1 + u32 lowMe = me; + u32 line = gr; +#else + u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. + u32 line = gr * WMUL + me / G_W; +#endif + if (line >= H) line -= H; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -1628,38 +1914,28 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( __asm("s_setprio 3"); #endif - readCarryFusedLine(in31, u31, line); - readCarryFusedLine(in61, u61, line); - // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. -#if ZEROHACK_W - u32 zerohack = get_group_id(0) / 131072; - new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack); - bar(); - new_fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack); -#else - new_fft_WIDTH1(lds31, u31, smallTrig31); - bar(); - new_fft_WIDTH1(lds61, u61, smallTrig61); -#endif + u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + + readCarryFusedLine(in31, u31, line, lowMe); + fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, lowMe); + + dependentLaunchWait(); // Previous kernel was fftMiddleOutGF61 + + readCarryFusedLine(in61, u61, line, lowMe); + fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, lowMe); Word2 wu[NW]; + P(i64) carryShuttlePtr = (P(i64)) carryShuttle; i64 carry[NW+1]; -#if AMDGPU -#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions -//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 -#else -#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better -#endif - u32 roundMax = 0; float carryMax = 0; - u32 word_index = (me * H + line) * 2; + u32 word_index = (lowMe * H + line) * 2; // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Let s be the shift amount for word 1. The shift amount for word x is ceil(x * (s - 1) + num_big_words_less_than_x) % 31. @@ -1679,14 +1955,14 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( #define m61_weight_shift m61_combo.a[1] #define m61_combo_counter m61_combo.b - const u64 m31_combo_step = ((u64) m31_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 m31_combo_bigstep = ((G_W * H * 2 - 1) * m31_combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - m31_combo_counter = word_index * m31_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 m31_combo_step = make_u64(m31_bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 m31_combo_bigstep = (comboFracBits(G_W * H * 2 - 1) + make_u64((G_W * H * 2 - 1) * m31_bigword_weight_shift_minus1, 0)) % (31ULL << 32); + m31_combo_counter = comboFracBits(word_index) + make_u64(word_index * m31_bigword_weight_shift_minus1, 0xFFFFFFFF); m31_weight_shift = m31_weight_shift % 31; u64 m31_starting_combo_counter = m31_combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation - const u64 m61_combo_step = ((u64) m61_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 m61_combo_bigstep = ((G_W * H * 2 - 1) * m61_combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - m61_combo_counter = word_index * m61_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 m61_combo_step = make_u64(m61_bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 m61_combo_bigstep = (comboFracBits(G_W * H * 2 - 1) + make_u64((G_W * H * 2 - 1) * m61_bigword_weight_shift_minus1, 0)) % (61ULL << 32); + m61_combo_counter = comboFracBits(word_index) + make_u64(word_index * m61_bigword_weight_shift_minus1, 0xFFFFFFFF); m61_weight_shift = m61_weight_shift % 61; u64 m61_starting_combo_counter = m61_combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -1731,35 +2007,43 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( m31_combo_counter = m31_starting_combo_counter; // Restore starting counter for applying weights after carry propagation m61_combo_counter = m61_starting_combo_counter; -#if ROE - float fltRoundMax = (float) roundMax / (float) 0x1FFFFFFF; // For speed, roundoff was computed as 32-bit integer. Convert to float - divide by M61. - updateStats(bufROE, posROE, fltRoundMax); -#elif STATS & (1 << MUL3) - updateStats(bufROE, posROE, carryMax); -#endif - - // Write out our carries. Only groups 0 to H-1 need to write carries out. - // Group H is a duplicate of group 0 (producing the same results) so we don't care about group H writing out, + // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. + // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. - if (gr < H) { for (i32 i = 0; i < NW; ++i) { carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(me, i)] = carry[i]; } } - - // Tell next line that its carries are ready + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead an #if is required +#if WMUL == 1 if (gr < H) { -#if OLD_FENCE - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(); - if (me == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else + if (gr < H / WMUL && me >= (WMUL-1) * G_W) { +#endif + for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } + + // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (me % WAVEFRONT == 0) { - u32 pos = gr * (G_W / WAVEFRONT) + me / WAVEFRONT; - atomic_store((atomic_uint *) &ready[pos], 1); +#if !OLD_FENCE + sync(); // Make sure all lanes have completed the CSSTORE + if (lowMe % WAVEFRONT == 0) { + u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; + atomic_store((global atomic_uint *) &ready[pos], 1); } #endif } - // Line zero will be redone when gr == H +#if OLD_FENCE + // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the + // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow + // is undefined. No barrier is needed when a sub-workgroup is a single wavefront. + if (gr < H / WMUL) { + bar(G_W); +#if WMUL == 1 + if (lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } +#else + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } +#endif + } +#endif + + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } // Do some work while our carries may not be ready @@ -1767,48 +2051,68 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( __asm("s_setprio 0"); #endif - // Wait until our carries are ready +#if ROE + float fltRoundMax = (float) roundMax / (float) 0x1FFFFFFF; // For speed, roundoff was computed as 32-bit integer. Convert to float - divide by M61. + updateStats((local u32 *) lds61, G_W * WMUL, H / WMUL, bufROE, posROE, fltRoundMax); +#elif STATS & (1 << MUL3) + updateStats((local u32 *) lds61, G_W * WMUL, H / WMUL, bufROE, posROE, carryMax); +#endif + + // Shuffle carries up + shufl_carries_up(lds61, carry, me, lowMe); + + // Wait until our carries are ready. The barrier below must be reached by every work-item of the + // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + if (me == 0) { do { spin(); } while(!atomic_load_explicit((global atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } bar(); - read_mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me == 0) ready[gr - 1] = 0; +#endif + if (me < G_W) { +#if OLD_FENCE + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me == 0) ready[gr - 1] = 0; #else - u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; - if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); - } - mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (me % WAVEFRONT == 0) { + do { spin(); } while(atomic_load_explicit((global atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + } + sync(); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; #endif #if HAS_ASM - __asm("s_setprio 1"); + __asm("s_setprio 1"); #endif - - // Read from the carryShuttle carries produced by the previous WIDTH row. Rotate carries from the last WIDTH row. - // The new carry layout lets the compiler generate global_load_dwordx4 instructions. - if (gr < H) { - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]; - } - } else { + } #if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); -#endif - - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]; - } - - if (me == 0) { - carry[NW] = carry[NW-1]; - for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } - carry[0] = carry[NW]; + // For the last group the carry reading is shifted, so the per-wavefront ready flags are not enough and a + // barrier is needed. gr is uniform but "me < G_W" is not, so the barrier is taken outside that guard and + // the shuttle reads resume in a second "me < G_W" block. + if (gr >= H / WMUL) { bar(); } +#endif + + if (me < G_W) { + + // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. + // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. + if (gr < H / WMUL) { + for (i32 i = 0; i < NW; ++i) { + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + } + } else { + + for (i32 i = 0; i < NW; ++i) { + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + } + + if (me == 0) { + carry[NW] = carry[NW-1]; + for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } + carry[0] = carry[NW]; + } } } @@ -1836,15 +2140,13 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( m61_weight_shift = adjust_m61_weight_shift(m61_weight_shift); } - bar(); - - new_fft_WIDTH2(lds31, u31, smallTrig31); - writeCarryFusedLine(u31, out31, line); + fft_WIDTH2(lds31, u31, smallTrig31, WMUL, lowMe); + writeCarryFusedLine(u31, out31, line, lowMe); - bar(); + dependentLaunch(); // Next kernel will be fftMiddleInGF31 - new_fft_WIDTH2(lds61, u61, smallTrig61); - writeCarryFusedLine(u61, out61, line); + fft_WIDTH2(lds61, u61, smallTrig61, WMUL, lowMe); + writeCarryFusedLine(u61, out61, line, lowMe); } @@ -1856,16 +2158,12 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, - CP(u32) bits, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - -#if 0 // fft_WIDTH uses shufl_int instead of shufl - local GF61 lds61[WIDTH / 4]; -#else - local GF61 lds61[WIDTH / 2]; -#endif +KERNEL_CAP(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, + ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { + local GF61 lds61[LDS_BYTES(WMUL) / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; local GF31 *lds31 = (local GF31 *) lds61; + LDSinit(lds61, WMUL); F2 uF2[NW]; GF31 u31[NW]; @@ -1875,7 +2173,14 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( u32 me = get_local_id(0); u32 H = BIG_HEIGHT; - u32 line = gr % H; +#if WMUL == 1 + u32 lowMe = me; + u32 line = gr; +#else + u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. + u32 line = gr * WMUL + me / G_W; +#endif + if (line >= H) line -= H; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -1891,48 +2196,50 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( __asm("s_setprio 3"); #endif - readCarryFusedLine(inF2, uF2, line); - readCarryFusedLine(in31, u31, line); - readCarryFusedLine(in61, u61, line); - // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. -#if ZEROHACK_W - u32 zerohack = get_group_id(0) / 131072; - new_fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack); - bar(); - new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack); - bar(); - new_fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack); -#else - new_fft_WIDTH1(ldsF2, uF2, smallTrigF2); - bar(); - new_fft_WIDTH1(lds31, u31, smallTrig31); - bar(); - new_fft_WIDTH1(lds61, u61, smallTrig61); -#endif + u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + + readCarryFusedLine(inF2, uF2, line, lowMe); + fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack, WMUL, lowMe); + + readCarryFusedLine(in31, u31, line, lowMe); + fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, lowMe); + + dependentLaunchWait(); // Previous kernel was fftMiddleOutGF61 + + readCarryFusedLine(in61, u61, line, lowMe); + fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, lowMe); Word2 wu[NW]; -#if AMDGPU - F2 weights = fancyMul(THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); + u32 me_frac_bits = fracBits(lowMe * H * 2); +#if !NVIDIAGPU || CUDA_BACKEND + F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), TSLOAD(&THREAD_WEIGHTS[G_W + line])); + u32 line_frac_bits = fracBits(line * 2); + u32 base_frac_bits = me_frac_bits + line_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > line_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > line_frac_bits); #else - F2 weights = fancyMul(CONST_THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights + F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), CONST_THREAD_WEIGHTS[line % 64]); + u32 partialLine_frac_bits = fracBits((line % 64) * 2); + u32 base_frac_bits = me_frac_bits + partialLine_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); + weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); + partialLine_frac_bits = fracBits(((line / 64) * 64) * 2); + base_frac_bits = base_frac_bits + partialLine_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); #endif + P(i64) carryShuttlePtr = (P(i64)) carryShuttle; i64 carry[NW+1]; -#if AMDGPU -#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions -//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 -#else -#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better -#endif - float roundMax = 0; float carryMax = 0; - u32 word_index = (me * H + line) * 2; + u32 word_index = (lowMe * H + line) * 2; // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Let s be the shift amount for word 1. The shift amount for word x is ceil(x * (s - 1) + num_big_words_less_than_x) % 31. @@ -1952,14 +2259,14 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( #define m61_weight_shift m61_combo.a[1] #define m61_combo_counter m61_combo.b - const u64 m31_combo_step = ((u64) m31_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 m31_combo_bigstep = ((G_W * H * 2 - 1) * m31_combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - m31_combo_counter = word_index * m31_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 m31_combo_step = make_u64(m31_bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 m31_combo_bigstep = (comboFracBits(G_W * H * 2 - 1) + make_u64((G_W * H * 2 - 1) * m31_bigword_weight_shift_minus1, 0)) % (31ULL << 32); + m31_combo_counter = comboFracBits(word_index) + make_u64(word_index * m31_bigword_weight_shift_minus1, 0xFFFFFFFF); m31_weight_shift = m31_weight_shift % 31; u64 m31_starting_combo_counter = m31_combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation - const u64 m61_combo_step = ((u64) m61_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 m61_combo_bigstep = ((G_W * H * 2 - 1) * m61_combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - m61_combo_counter = word_index * m61_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 m61_combo_step = make_u64(m61_bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 m61_combo_bigstep = (comboFracBits(G_W * H * 2 - 1) + make_u64((G_W * H * 2 - 1) * m61_bigword_weight_shift_minus1, 0)) % (61ULL << 32); + m61_combo_counter = comboFracBits(word_index) + make_u64(word_index * m61_bigword_weight_shift_minus1, 0xFFFFFFFF); m61_weight_shift = m61_weight_shift % 61; u64 m61_starting_combo_counter = m61_combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -1972,11 +2279,11 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( // Apply the inverse weights and carry propagate pairs to generate the output carries - F invBase = optionalDouble(weights.x); + F invBase = weights.x; for (u32 i = 0; i < NW; ++i) { // Generate the FP32 weights and second GF31 and GF61 weight shift - F invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i))); - F invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP)); + F invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i)), frac_bits > base_frac_bits); + F invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 m31_weight_shift0 = m31_weight_shift; m31_combo_counter += m31_combo_step; m31_weight_shift = adjust_m31_weight_shift(m31_weight_shift); @@ -2007,34 +2314,43 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( m31_combo_counter = m31_starting_combo_counter; // Restore starting counter for applying weights after carry propagation m61_combo_counter = m61_starting_combo_counter; -#if ROE - updateStats(bufROE, posROE, roundMax); -#elif STATS & (1 << MUL3) - updateStats(bufROE, posROE, carryMax); -#endif - - // Write out our carries. Only groups 0 to H-1 need to write carries out. - // Group H is a duplicate of group 0 (producing the same results) so we don't care about group H writing out, + // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. + // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. - if (gr < H) { for (i32 i = 0; i < NW; ++i) { carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(me, i)] = carry[i]; } } - - // Tell next line that its carries are ready + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead an #if is required +#if WMUL == 1 if (gr < H) { -#if OLD_FENCE - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(); - if (me == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else + if (gr < H / WMUL && me >= (WMUL-1) * G_W) { +#endif + for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } + + // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (me % WAVEFRONT == 0) { - u32 pos = gr * (G_W / WAVEFRONT) + me / WAVEFRONT; - atomic_store((atomic_uint *) &ready[pos], 1); +#if !OLD_FENCE + sync(); // Make sure all lanes have completed the CSSTORE + if (lowMe % WAVEFRONT == 0) { + u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; + atomic_store((global atomic_uint *) &ready[pos], 1); } #endif } - // Line zero will be redone when gr == H +#if OLD_FENCE + // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the + // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow + // is undefined. No barrier is needed when a sub-workgroup is a single wavefront. + if (gr < H / WMUL) { + bar(G_W); +#if WMUL == 1 + if (lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } +#else + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } +#endif + } +#endif + + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } // Do some work while our carries may not be ready @@ -2042,61 +2358,76 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( __asm("s_setprio 0"); #endif - // Calculate inverse weights - F base = optionalHalve(weights.y); - for (u32 i = 0; i < NW; ++i) { - F weight1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i))); - F weight2 = optionalHalve(fancyMul(weight1, WEIGHT_STEP)); - uF2[i] = U2(weight1, weight2); - } +#if ROE + updateStats((local u32 *) lds61, G_W * WMUL, H / WMUL, bufROE, posROE, roundMax); +#elif STATS & (1 << MUL3) + updateStats((local u32 *) lds61, G_W * WMUL, H / WMUL, bufROE, posROE, carryMax); +#endif + + // Shuffle carries up + shufl_carries_up(lds61, carry, me, lowMe); - // Wait until our carries are ready + // Wait until our carries are ready. The barrier below must be reached by every work-item of the + // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + if (me == 0) { do { spin(); } while(!atomic_load_explicit((global atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } bar(); - read_mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me == 0) ready[gr - 1] = 0; +#endif + if (me < G_W) { +#if OLD_FENCE + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me == 0) ready[gr - 1] = 0; #else - u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; - if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); - } - mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (me % WAVEFRONT == 0) { + do { spin(); } while(atomic_load_explicit((global atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + } + sync(); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; #endif #if HAS_ASM - __asm("s_setprio 1"); + __asm("s_setprio 1"); #endif - - // Read from the carryShuttle carries produced by the previous WIDTH row. Rotate carries from the last WIDTH row. - // The new carry layout lets the compiler generate global_load_dwordx4 instructions. - if (gr < H) { - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]; - } - } else { + } #if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); -#endif - - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]; - } - - if (me == 0) { - carry[NW] = carry[NW-1]; - for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } - carry[0] = carry[NW]; + // For the last group the carry reading is shifted, so the per-wavefront ready flags are not enough and a + // barrier is needed. gr is uniform but "me < G_W" is not, so the barrier is taken outside that guard and + // the shuttle reads resume in a second "me < G_W" block. + if (gr >= H / WMUL) { bar(); } +#endif + + if (me < G_W) { + + // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. + // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. + if (gr < H / WMUL) { + for (i32 i = 0; i < NW; ++i) { + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + } + } else { + + for (i32 i = 0; i < NW; ++i) { + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + } + + if (me == 0) { + carry[NW] = carry[NW-1]; + for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } + carry[0] = carry[NW]; + } } } // Apply each 32 or 64 bit carry to the 2 words. Apply weights. + F base = weights.y; for (i32 i = 0; i < NW; ++i) { + // Calculate inverse weights + F weight1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); + F weight2 = optionalHalve(fancyMul(weight1, WEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); // Generate the second weight shifts u32 m31_weight_shift0 = m31_weight_shift; m31_combo_counter += m31_combo_step; @@ -2109,7 +2440,7 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( // Generate big-word/little-word flag, propagate final carry bool biglit0 = frac_bits <= FRAC_BPW_HI; wu[i] = carryFinal(wu[i], carry[i], biglit0); - uF2[i] = U2(uF2[i].x * wu[i].x, uF2[i].y * wu[i].y); + uF2[i] = U2(weight1 * wu[i].x, weight2 * wu[i].y); u31[i] = U2(shl(make_Z31(wu[i].x), m31_weight_shift0), shl(make_Z31(wu[i].y), m31_weight_shift1)); u61[i] = U2(shl(make_Z61(wu[i].x), m61_weight_shift0), shl(make_Z61(wu[i].y), m61_weight_shift1)); @@ -2120,20 +2451,16 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( m61_weight_shift = adjust_m61_weight_shift(m61_weight_shift); } - bar(); - - new_fft_WIDTH2(ldsF2, uF2, smallTrigF2); - writeCarryFusedLine(uF2, outF2, line); - - bar(); + fft_WIDTH2(ldsF2, uF2, smallTrigF2, WMUL, lowMe); + writeCarryFusedLine(uF2, outF2, line, lowMe); - new_fft_WIDTH2(lds31, u31, smallTrig31); - writeCarryFusedLine(u31, out31, line); + dependentLaunch(); // Next kernel will be fftMiddleInFP32 - bar(); + fft_WIDTH2(lds31, u31, smallTrig31, WMUL, lowMe); + writeCarryFusedLine(u31, out31, line, lowMe); - new_fft_WIDTH2(lds61, u61, smallTrig61); - writeCarryFusedLine(u61, out61, line); + fft_WIDTH2(lds61, u61, smallTrig61, WMUL, lowMe); + writeCarryFusedLine(u61, out61, line, lowMe); } diff --git a/src/cl/carryutil.cl b/src/cl/carryutil.cl index 6cdff372..c9c88a76 100644 --- a/src/cl/carryutil.cl +++ b/src/cl/carryutil.cl @@ -1,8 +1,5 @@ // Copyright (C) Mihai Preda -#include "base.cl" -#include "math.cl" - #if CARRY64 typedef i64 CFcarry; #else @@ -97,7 +94,7 @@ bool test(u32 bits, u32 pos) { return (bits >> pos) & 1; } #if FFT_FP64 // Rounding constant: 3 * 2^51, See https://stackoverflow.com/questions/17035464 -#define RNDVAL (3.0 * (1l << 51)) +#define RNDVAL (3.0 * (1ull << 51)) // Convert a double to long efficiently. Double must be in RNDVAL+integer format. i64 RNDVALdoubleToLong(double d) { @@ -138,14 +135,52 @@ float OVERLOAD boundCarry(i32 c) { return ldexp(fabs((float) c), -32); } float OVERLOAD boundCarry(i64 c) { return ldexp(fabs((float) (i32) (c >> 8)), -24); } #if STATS || ROE -void updateStats(global uint *bufROE, u32 posROE, float roundMax) { +void updateStats(local u32 *lds, u32 num_threads, u32 num_blocks, global uint *bufROE, u32 posROE, float roundMax) { assert(roundMax >= 0); - // work_group_reduce_max() allocates an additional 256Bytes LDS for a 64lane workgroup, so avoid it. - // u32 groupRound = work_group_reduce_max(as_uint(roundMax)); - // if (get_local_id(0) == 0) { atomic_max(bufROE + posROE, groupRound); } - - // Do the reduction directly over global mem. - atomic_max(bufROE + posROE, as_uint(roundMax)); + u32 me = get_local_id(0); + u32 u32RoundMax = as_uint(roundMax); + + // Reduce to a handful of roundMax values + // We could use shfl_down_sync (and AMD's equivalent) instead of LDS memory once num_threads < WAVEFRONT + // (see https://github.com/mahmoudmaftah/MaxReduction-Cuda/blob/main/code/reduction_benchmarks.cu) + while (num_threads > 8) { + // Write roundMax for high half of threads to local memory. Ignore threads not participating in the reduction. + // bar(num_threads) rather than a hand-rolled "only if it is wider than a wavefront": that test assumes a + // wavefront advances in lock-step, which holds on AMD but not on nVidia Volta and later, and nowhere else + // at all. bar() decides that by what the hardware guarantees, and with G_W == 64 and a 32-lane wavefront + // two of the three reduction steps here were running with no barrier and no fence. num_threads is a + // compile-time workgroup size, so every thread makes the same number of passes and reaches both calls. + bar(num_threads); + if (me >= num_threads / 2 && me < num_threads) lds[me - num_threads / 2] = u32RoundMax; + bar(num_threads); + // Low half of threads do a max + if (me < num_threads / 2) { + u32 highHalfMax = lds[me]; + if (u32RoundMax < highHalfMax) u32RoundMax = highHalfMax; + } + // Cut num threads in half, loop + num_threads /= 2; + } + + // The bufROE entry to update is stored in the first bufROE entry. This value used to be passed into carryFused as an argument. + // CUDA graphs don't allow arguments to change. Thus, calculating posROE and storing it in bufROE works better. + if (me < num_threads) { + posROE = bufROE[0]; + // The buffer holds STATS_SIZE samples. The host resets the position only when it reads the samples, and the LL and + // CERT loops never read the carry statistics, so once the buffer is full stop recording rather than write past it. + if (posROE < STATS_SIZE) { + atomic_max(bufROE + posROE + 2, u32RoundMax); + + // The second bufRoe entry is a count of the number atomic_maxes performed. When the last atomic_max is done, increment posROE and clear the counter. + if (me == 0) { + u32 old_value = atomic_add(bufROE + 1, 1); + if (old_value == num_blocks - 1) { + bufROE[0] = posROE + 1; + bufROE[1] = 0; + } + } + } + } } #endif @@ -185,7 +220,9 @@ i64 weightAndCarryOne(T u, T invWeight, i64 inCarry, float* maxROE, int sloppy_r float roundoff = fabs((float) fma(u, invWeight, RNDVALCarry - d)); *maxROE = max(*maxROE, roundoff); - // Convert to long (for CARRY32 case we don't need to strip off the RNDVAL bits) + // Convert to long (for CARRY32 case we don't need to strip off the RNDVAL bits). + // Leaving RNDVAL in place is only safe while the carry extraction window stays below bit 51 -- see the + // #error below and FFTShape::carry32BPW. if (sloppy_result_is_acceptable) return as_long(d); else return RNDVALdoubleToLong(d); @@ -285,7 +322,7 @@ i64 weightAndCarryOne(Z61 u, u32 invWeight, i64 inCarry, u32* maxROE) { i64 value = get_balanced_Z61(u); // Optionally calculate roundoff error as proximity to M61/2. 28 bits of accuracy should be sufficient. - u32 roundoff = (u32) abs((i32) (value >> 32)); + u32 roundoff = (u32) abs((i32) hi32(value)); *maxROE = max(*maxROE, roundoff); // Mul by 3 and add carry @@ -375,9 +412,9 @@ i96 weightAndCarryOne(float uF2, Z61 u61, float F2_invWeight, u32 m61_invWeight, u61 = shr(u61, m61_invWeight); u64 n61 = get_Z61(u61); - // The final result must be n61 mod M61. Use FP32 data to calculate this value. - float n61f = (float)((u32)(n61 >> 32)) * -4294967296.0f; // Conversion from u64 to float might be slow, this might be faster - uF2 = fma(uF2, F2_invWeight, n61f); // This should be close to a multiple of M61 + // The final result mod M61 must be n61. Use FP32 data to calculate how many multiples of M61 need to be added to n61. + float n61f = (float)hi32(n61) * -4294967296.0f; // Estimate -n61 as a float. + uF2 = fma(uF2, F2_invWeight, n61f); // This should be close to an integer multiple of M61 float uF2int = fma(uF2, 4.3368086899420177360298112034798e-19f, RNDVAL); // Divide by M61 and round to int i32 nF2 = RNDVALfloatToInt(uF2int); @@ -414,21 +451,21 @@ i96 weightAndCarryOne(Z31 u31, Z61 u61, u32 m31_invWeight, u32 m61_invWeight, bo // Use chinese remainder theorem to create a 92-bit result. Loosely copied from Yves Gallot's mersenne2 program. u32 n31 = get_Z31(u31); - u61 = subq(u61, make_Z61(n31), 2); // u61 - u31 - u61 = add(u61, shl(u61, 31)); // u61 + (u61 << 31) + u61 += make_u64(hi32(M61), lo32(M61) - n31); // u61 - u31 + u61 += shl(u61, 31); // u61 + (u61 << 31) // The resulting value will be get_Z61(u61) * M31 + n31 and if larger than ~M31*M61/2 return a negative value by subtracting M31 * M61. // We can save a little work by determining if the result will be large using just u61 and returning (get_Z61(u61) - M61) * M31 + n31. // This simplifies to get_balanced_Z61(u61) * M31 + n31. - i64 n61 = get_balanced_Z61(u61); + i64 n61 = get_balanced_Z61(modM61(u61)); // Optionally calculate roundoff error as proximity to M61/2. 28 bits of accuracy should be sufficient. - u32 roundoff = (u32) abs((i32)(n61 >> 32)); + u32 roundoff = (u32) abs((i32)hi32(n61)); *maxROE = max(*maxROE, roundoff); // Compute the value using i96 math - i64 vhi = n61 >> 33; - u64 vlo = ((u64)n61 << 31) | n31; + i64 vhi = n61 >> 1; + u32 vlo = ((u32)n61 << 31) | n31; i96 value = make_i96(vhi, vlo); // (n61 << 31) + n31 value = sub(value, n61); // n61 * M31 + n31 @@ -452,26 +489,39 @@ i128 weightAndCarryOne(float uF2, Z31 u31, Z61 u61, float F2_invWeight, u32 m31_ // Apply inverse weights u31 = shr(u31, m31_invWeight); u61 = shr(u61, m61_invWeight); - // Use chinese remainder theorem to create a 92-bit result. Loosely copied from Yves Gallot's mersenne2 program. u32 n31 = get_Z31(u31); - u61 = subq(u61, make_Z61(n31), 2); // u61 - u31 - u61 = add(u61, shl(u61, 31)); // u61 + (u61 << 31) - u64 n61 = get_Z61(u61); - - i128 n3161 = make_i128(n61 >> 33, (n61 << 31) | n31); // n61 << 31 + n31 - n3161 = sub(n3161, n61); // n61 * M31 + n31 - - // The final result must be n3161 mod M31*M61. Use FP32 data to calculate this value. - float n3161f = (float)((u32)(n61 >> 32)) * -9223372036854775808.0f; // Converting n3161 from i128 to float might be slow, this might be faster - uF2 = fma(uF2, F2_invWeight, n3161f); // This should be close to a multiple of M31*M61 + u61 += make_u64(hi32(M61), lo32(M61) - n31); // u61 - u31 + u61 += shl(u61, 31); // u61 + (u61 << 31) + u64 n61 = get_Z61(modM61(u61)); + // Let's call the 92-bit CRT result n3161. At this point, n3161 = n61 * M31 + n31. + + // The final result mod M31*M61 must be n3161. Use FP32 data to calculate how many multiples of M31*M61 need to be added to n3161. + float n3161f = (float)hi32(n61) * -9223372036854775808.0f; // Estimate -n3161 as a float. -n61 << 31 should be close enough. + uF2 = fma(uF2, F2_invWeight, n3161f); // This should be close to an integer multiple of M31*M61 float uF2int = fma(uF2, 2.0194839183061857038255724444152e-28f, RNDVAL); // Divide by M31*M61 and round to int i32 nF2 = RNDVALfloatToInt(uF2int); - i64 nF2m31 = ((i64)nF2 << 31) - nF2; // nF2 * M31 - i128 v = make_i128(nF2m31 >> 3, (u64)nF2m31 << 61); // nF2m31 << 61 - v = sub(v, nF2m31); // nF2m31 * M61 - v = add(v, n3161); // nF2m31 * M61 + n3161 + // The final result will be nF2 * M31*M61 + n3161. Rearranging to use as few 128-bit and 64-bit ops as possible: + // = nF2 * M61 * M31 + n61 * M31 + n31 + // = (nF2 * M61 + n61) * M31 + n31 + // = ((nF2 << 61) - nF2 + n61) * M31 + n31 + // = (((nF2 << 61) - nF2 + n61) << 31) - ((nF2 << 61) - nF2 + n61) + n31 + // = (nF2 << 92) + ((n61 - nF2) << 31) - (nF2 << 61) - (n61 - nF2) + n31 + // = (nF2 << 92) - (nF2 << 61) + ((n61 - nF2) << 31) - (n61 - nF2) + n31 + // = (((nF2 << 31) - nF2) << 61) + ((n61 - nF2) << 31) - (n61 - nF2) + n31 + // = (((nF2 << 32) - nF2*2) << 60) + ((n61 - nF2) << 31) - (n61 - nF2) + n31 + + // Compute x = (n61 - nF2) + i64 x = (i64)n61 - nF2; + // Compute y = ((n61 - nF2) << 31) + n31 + i128 y = make_i128(x >> 33, (x << 31) | n31); + // Compute z = ((nF2 << 32) - nF2*2) << 60 + i64 tmp = make_i64(nF2, 0) - (i64)(nF2 + nF2); + i128 z = make_i128(tmp >> 4, tmp << 60); + + // Put the parts together + i128 v = sub(add(z, y), x); // Optionally calculate roundoff error float roundoff = fabs(fma(uF2, 2.0194839183061857038255724444152e-28f, RNDVAL - uF2int)); @@ -538,7 +588,7 @@ Word OVERLOAD carryStep(i64 x, i64 *outCarry, bool isBigWord) { #elif EXP / NWORDS == 32 i32 xhi = hi32(x); i64 w = lowBits(x, nBits); - xhi -= (i32)(w >> 32); + xhi -= (i32)hi32(w); *outCarry = xhi >> (nBits - 32); return w; #elif EXP / NWORDS == 31 @@ -675,18 +725,18 @@ Word OVERLOAD carryStepSignedSloppy(i96 x, i64 *outCarry, bool isBigWord) { const u32 bigwordBits = EXP / NWORDS + 1; u32 nBits = bitlen(isBigWord); #if EXP / NWORDS >= 32 // nBits is 32 or more - return carryStep(x, outCarry, isBigWord); // Should be just as fast as code below + return carryStep(x, outCarry, isBigWord); // Should be just as fast as code below // u32 xmid_topbit = i96_mid32(x) & (1 << (bigwordBits - 32 - 1)); // i32 whi = ulowFixedBits(i96_mid32(x), bigwordBits - 32 - 1) - xmid_topbit; // i64 xhi = i96_hi64(x) + xmid_topbit; // *outCarry = xhi >> (nBits - 32); // return as_long((int2)(i96_lo32(x), whi)); -#elif EXP / NWORDS == 31 || SLOPPY_MAXBPW >= 3200 // nBits = 31 or 32, bigwordBits = 32 (or allowed to create 32-bit word for better performance) +#elif EXP / NWORDS == 31 || (SLOPPY_MAXBPW >= 3200 && EXP / NWORDS >= 22) // nBits = 31 or 32, bigwordBits = 32 (or allowed to create 32-bit word for better performance) i32 w = i96_lo32(x); // lowBits(x, bigwordBits = 32); *outCarry = (i96_hi64(x) + (w < 0)) << (32 - nBits); return w; #else // nBits less than 32 - return carryStep(x, outCarry, isBigWord); // Should be faster than code below + return carryStep(x, outCarry, isBigWord); // Should be faster than code below // i32 w = lowFixedBits(i96_lo32(x), bigwordBits); // *outCarry = (as_long((int2)(xtract32(i96_lo64(x), bigwordBits), xtract32(i96_hi64(x), bigwordBits))) + (w < 0)) << (bigwordBits - nBits); // return w; @@ -717,7 +767,7 @@ Word OVERLOAD carryStepSignedSloppy(i64 x, i32 *outCarry, bool isBigWord) { #if EXP / NWORDS >= 32 // nBits is 32 or more u64 x_topbit = x & ((u64)1 << (bigwordBits - 1)); i64 w = ulowFixedBits(x, bigwordBits - 1) - x_topbit; - i32 xhi = (i32)(x >> 32) + (i32)(x_topbit >> 32); + i32 xhi = (i32)hi32(x) + (i32)hi32(x_topbit); *outCarry = xhi >> (nBits - 32); return w; // nBits = 31 or 32, bigwordBits = 32 (or allowed to create 32-bit word for better performance). For reasons I don't fully understand the sloppy @@ -725,7 +775,7 @@ Word OVERLOAD carryStepSignedSloppy(i64 x, i32 *outCarry, bool isBigWord) { // Not a major concern as end users should avoid small BPW as there is probably a more efficient NTT that could be used. #elif EXP / NWORDS == 31 || (EXP / NWORDS >= 23 && SLOPPY_MAXBPW >= 3200) i32 w = x; // lowBits(x, bigwordBits = 32); - *outCarry = ((i32)(x >> 32) + (w < 0)) << (32 - nBits); + *outCarry = ((i32)hi32(x) + (w < 0)) << (32 - nBits); return w; #else // nBits less than 32 //GWBUG - is there a faster version? Is this faster than plain old carryStep? No // u32 x_topbit = (u32) x & (1 << (bigwordBits - 1)); @@ -757,6 +807,13 @@ Word2 carryWord(Word2 a, CarryABM* carry, bool b1, bool b2) { /* Support both 32-bit and 64-bit carries */ #if WordSize <= 4 +// A 32-bit carry means weightAndCarryOne returns RNDVAL + value un-stripped, and carryStep(i64, i32*) reads +// the carry from bits [nBits, nBits+32). That window must stay strictly below the RNDVAL bit 51, so +// nBits <= 19; a big word has nBits = EXP / NWORDS + 1. The host is supposed to select CARRY64 before this +// point (FFTShape::needsLargeCarry); fail loudly rather than compute wrong carries if it ever does not. +#if !CARRY64 && FFT_TYPE == FFT64 && EXP / NWORDS >= 19 +#error "CARRY32 requires EXP / NWORDS <= 18; this exponent needs CARRY64 (-carry long)" +#endif #define iCARRY i32 #include "carryinc.cl" #undef iCARRY diff --git a/src/cl/etc.cl b/src/cl/etc.cl index ae4fd857..e3e0f7c9 100644 --- a/src/cl/etc.cl +++ b/src/cl/etc.cl @@ -16,11 +16,9 @@ KERNEL(32) readResidue(P(Word2) out, CP(Word2) in) { #endif #if SUM64 -KERNEL(64) sum64(global ulong* out, u32 sizeBytes, global ulong* in) { - if (get_global_id(0) == 0) { out[0] = 0; } - +KERNEL(64) sum64(global ulong* out, u32 count, CP(Word) in) { ulong sum = 0; - for (i32 p = get_global_id(0); p < sizeBytes / sizeof(u64); p += get_global_size(0)) { + for (i32 p = get_global_id(0); p < count; p += get_global_size(0)) { sum += in[p]; } u32 prev = atomic_add((global u32*)out, (u32) sum); diff --git a/src/cl/expand.cl b/src/cl/expand.cl new file mode 100644 index 00000000..eb097eaa --- /dev/null +++ b/src/cl/expand.cl @@ -0,0 +1,57 @@ +// Copyright (C) Mihai Preda + +// Some routines can be written for any 64-bit data type (T2 or GF61). Same for 32-bit data types (F2 or GF31). +// Some routines can be written to work 32-bit and 64-bit data types. +// These #defines make it easy to write those routines. This used to be done with type-casting, but +// this method generates better PTX code (not sure if that results in any better run times). + +#if FFT_FP64 +#define T_Z61 T +#define T2_GF61 T2 +#define T_F_Z31_Z61 T +#define T2_F2_GF31_GF61 T2 +#define as_T2_GF61 as_double2 +#include INCLUDE_FILE +#undef T_Z61 +#undef T2_GF61 +#undef T2_F2_GF31_GF61 +#undef as_T2_GF61 +#endif + +#if NTT_GF61 +#define T_Z61 Z61 +#define T2_GF61 GF61 +#define T_F_Z31_Z61 Z61 +#define T2_F2_GF31_GF61 GF61 +#define as_T2_GF61 as_ulong2 +#include INCLUDE_FILE +#undef T_Z61 +#undef T2_GF61 +#undef T2_F2_GF31_GF61 +#undef as_T2_GF61 +#endif + +#if FFT_FP32 +#define F_Z31 F +#define F2_GF31 F2 +#define T_F_Z31_Z61 F +#define T2_F2_GF31_GF61 F2 +#include INCLUDE_FILE +#undef F_Z31 +#undef F2_GF31 +#undef T2_F2_GF31_GF61 +#endif + +#if NTT_GF31 +#define F_Z31 Z31 +#define F2_GF31 GF31 +#define T_F_Z31_Z61 Z31 +#define T2_F2_GF31_GF61 GF31 +#include INCLUDE_FILE +#undef F_Z31 +#undef F2_GF31 +#undef T2_F2_GF31_GF61 +#endif + +#undef INCLUDE_FILE + diff --git a/src/cl/fft-middle.cl b/src/cl/fft-middle.cl index 31db8bfc..c0d96ad5 100644 --- a/src/cl/fft-middle.cl +++ b/src/cl/fft-middle.cl @@ -1,5 +1,6 @@ // Copyright (C) Mihai Preda +#include "math.cl" #include "trig.cl" #if MIDDLE == 3 @@ -100,7 +101,7 @@ void OVERLOAD middleMul(T2 *u, u32 s, Trig trig) { if (MIDDLE == 1) return; if (WIDTH == SMALL_HEIGHT) trig += SMALL_HEIGHT; // In this case we can share the MiddleMul2 trig table. Skip over the MiddleMul trig table. - T2 w = trig[s]; // s / BIG_HEIGHT + T2 w = TFLOAD(&trig[s]); // s / BIG_HEIGHT if (MIDDLE < SHARP_MIDDLE) { WADD(1, w); @@ -191,8 +192,8 @@ void OVERLOAD middleMul2(T2 *u, u32 x, u32 y, double factor, Trig trig) { return; } - trig += SMALL_HEIGHT; // Skip over the MiddleMul trig table - T2 w = trig[x]; // x / (MIDDLE * WIDTH) + trig += SMALL_HEIGHT; // Skip over the MiddleMul trig table + T2 w = TFLOAD(&trig[x]); // x / (MIDDLE * WIDTH) if (MIDDLE < SHARP_MIDDLE) { T2 base = slowTrig_N(x * y + x * SMALL_HEIGHT, ND / MIDDLE * 2) * factor; @@ -208,7 +209,7 @@ void OVERLOAD middleMul2(T2 *u, u32 x, u32 y, double factor, Trig trig) { Trig trig2 = trig + WIDTH; // Skip over the fist MiddleMul2 trig table u32 desired_root = x * y; - T2 base = cmulFancy(trig2[desired_root % SMALL_HEIGHT], trig[desired_root / SMALL_HEIGHT]) * factor; //Optimization to do: put multiply by factor in trig2 table + T2 base = cmulFancy(TFLOAD(&trig2[desired_root % SMALL_HEIGHT]), TFLOAD(&trig[desired_root / SMALL_HEIGHT])) * factor; //Optimization to do: put multiply by factor in trig2 table WADD(0, base); for (u32 k = 1; k < MIDDLE; ++k) { @@ -352,12 +353,13 @@ void OVERLOAD middleShuffle(local T2 *lds, T2 *u) { u32 x = me % 16; for (int i = 0; i < MIDDLE; ++i) { -// lds[x * 16 + y] = u[i]; - lds[x * 16 + y ^ x] = u[i]; // Swizzling with XOR should reduce LDS bank conflicts + lds[x * 16 + y ^ x] = u[i]; // Swizzling with XOR should reduce LDS bank conflicts, formerly "lds[x * 16 + y] = u[i];" bar(); -// u[i] = lds[me]; - u[i] = lds[y * 16 + x ^ y]; + u[i] = lds[y * 16 + x ^ y]; // Formerly "u[i] = lds[me];" + if (++i == MIDDLE) break; + lds[y * 16 + x ^ y] = u[i]; bar(); + u[i] = lds[x * 16 + y ^ x]; } } @@ -396,8 +398,8 @@ void OVERLOAD middleMul(F2 *u, u32 s, TrigFP32 trig) { if (MIDDLE == 1) return; if (WIDTH == SMALL_HEIGHT) trig += SMALL_HEIGHT; // In this case we can share the MiddleMul2 trig table. Skip over the MiddleMul trig table. - F2 w = trig[s]; // s / BIG_HEIGHT - + F2 w = TFLOAD(&trig[s]); // s / BIG_HEIGHT + if (MIDDLE < SHARP_MIDDLE) { WADD(1, w); #if MM_CHAIN == 0 @@ -488,7 +490,7 @@ void OVERLOAD middleMul2(F2 *u, u32 x, u32 y, float factor, TrigFP32 trig) { } trig += SMALL_HEIGHT; // Skip over the MiddleMul trig table - F2 w = trig[x]; // x / (MIDDLE * WIDTH) + F2 w = TFLOAD(&trig[x]); // x / (MIDDLE * WIDTH) if (MIDDLE < SHARP_MIDDLE) { F2 base = slowTrig_N(x * y + x * SMALL_HEIGHT, ND / MIDDLE * 2) * factor; @@ -504,7 +506,7 @@ void OVERLOAD middleMul2(F2 *u, u32 x, u32 y, float factor, TrigFP32 trig) { TrigFP32 trig2 = trig + WIDTH; // Skip over the fist MiddleMul2 trig table u32 desired_root = x * y; - F2 base = cmulFancy(trig2[desired_root % SMALL_HEIGHT], trig[desired_root / SMALL_HEIGHT]) * factor; //Optimization to do: put multiply by factor in trig2 table + F2 base = cmulFancy(TFLOAD(&trig2[desired_root % SMALL_HEIGHT]), TFLOAD(&trig[desired_root / SMALL_HEIGHT])) * factor; //Optimization to do: put multiply by factor in trig2 table WADD(0, base); for (u32 k = 1; k < MIDDLE; ++k) { @@ -626,12 +628,13 @@ void OVERLOAD middleShuffle(local F2 *lds, F2 *u) { u32 y = me / 16; u32 x = me % 16; for (int i = 0; i < MIDDLE; ++i) { -// lds[x * 16 + y] = u[i]; - lds[x * 16 + y ^ x] = u[i]; // Swizzling with XOR should reduce LDS bank conflicts + lds[x * 16 + y ^ x] = u[i]; // Swizzling with XOR should reduce LDS bank conflicts, formerly "lds[x * 16 + y] = u[i];" bar(); -// u[i] = lds[me]; - u[i] = lds[y * 16 + x ^ y]; + u[i] = lds[y * 16 + x ^ y]; // Formerly "u[i] = lds[me];" + if (++i == MIDDLE) break; + lds[y * 16 + x ^ y] = u[i]; bar(); + u[i] = lds[x * 16 + y ^ x]; } } #endif @@ -668,13 +671,13 @@ void OVERLOAD middleMul(GF31 *u, u32 s, TrigGF31 trig) { #if !MIDDLE_CHAIN // Read all trig values from memory for (u32 k = 1; k < MIDDLE; ++k) { - WADD(k, trig[s]); + WADD(k, TFLOAD(&trig[s])); s += SMALL_HEIGHT; } #else - GF31 w = trig[s]; // s / BIG_HEIGHT + GF31 w = TFLOAD(&trig[s]); // s / BIG_HEIGHT WADD(1, w); if (MIDDLE == 2) return; @@ -705,9 +708,9 @@ void OVERLOAD middleMul2(GF31 *u, u32 x, u32 y, TrigGF31 trig) { // The first trig table can be shared with MiddleMul trig table if WIDTH = HEIGHT. if (WIDTH == SMALL_HEIGHT) trig1 = trig; - GF31 w = trig1[x]; // x / (MIDDLE * WIDTH) + GF31 w = TFLOAD(&trig1[x]); // x / (MIDDLE * WIDTH) u32 desired_root = x * y; - GF31 base = cmul(trig2[desired_root % SMALL_HEIGHT], trig1[desired_root / SMALL_HEIGHT]); + GF31 base = cmul(TFLOAD(&trig2[desired_root % SMALL_HEIGHT]), TFLOAD(&trig1[desired_root / SMALL_HEIGHT])); WADD(0, base); for (u32 k = 1; k < MIDDLE; ++k) { @@ -745,12 +748,13 @@ void OVERLOAD middleShuffle(local GF31 *lds, GF31 *u) { u32 y = me / 16; u32 x = me % 16; for (int i = 0; i < MIDDLE; ++i) { -// lds[x * 16 + y] = u[i]; - lds[x * 16 + y ^ x] = u[i]; // Swizzling with XOR should reduce LDS bank conflicts + lds[x * 16 + y ^ x] = u[i]; // Swizzling with XOR should reduce LDS bank conflicts, formerly "lds[x * 16 + y] = u[i];" bar(); -// u[i] = lds[me]; - u[i] = lds[y * 16 + x ^ y]; + u[i] = lds[y * 16 + x ^ y]; // Formerly "u[i] = lds[me];" + if (++i == MIDDLE) break; + lds[y * 16 + x ^ y] = u[i]; bar(); + u[i] = lds[x * 16 + y ^ x]; } } @@ -788,13 +792,13 @@ void OVERLOAD middleMul(GF61 *u, u32 s, TrigGF61 trig) { #if !MIDDLE_CHAIN // Read all trig values from memory for (u32 k = 1; k < MIDDLE; ++k) { - WADD(k, trig[s]); + WADD(k, TFLOAD(&trig[s])); s += SMALL_HEIGHT; } #else - GF61 w = trig[s]; // s / BIG_HEIGHT + GF61 w = TFLOAD(&trig[s]); // s / BIG_HEIGHT WADD(1, w); if (MIDDLE == 2) return; @@ -825,9 +829,9 @@ void OVERLOAD middleMul2(GF61 *u, u32 x, u32 y, TrigGF61 trig) { // The first trig table can be shared with MiddleMul trig table if WIDTH = HEIGHT. if (WIDTH == SMALL_HEIGHT) trig1 = trig; - GF61 w = trig1[x]; // x / (MIDDLE * WIDTH) + GF61 w = TFLOAD(&trig1[x]); // x / (MIDDLE * WIDTH) u32 desired_root = x * y; - GF61 base = cmul(trig2[desired_root % SMALL_HEIGHT], trig1[desired_root / SMALL_HEIGHT]); + GF61 base = cmul(TFLOAD(&trig2[desired_root % SMALL_HEIGHT]), TFLOAD(&trig1[desired_root / SMALL_HEIGHT])); WADD(0, base); for (u32 k = 1; k < MIDDLE; ++k) { @@ -886,12 +890,13 @@ void OVERLOAD middleShuffle(local GF61 *lds, GF61 *u) { u32 y = me / 16; u32 x = me % 16; for (int i = 0; i < MIDDLE; ++i) { -// lds[x * 16 + y] = u[i]; - lds[x * 16 + y ^ x] = u[i]; // Swizzling with XOR should reduce LDS bank conflicts + lds[x * 16 + y ^ x] = u[i]; // Swizzling with XOR should reduce LDS bank conflicts, formerly "lds[x * 16 + y] = u[i];" bar(); -// u[i] = lds[me]; - u[i] = lds[y * 16 + x ^ y]; + u[i] = lds[y * 16 + x ^ y]; // Formerly "u[i] = lds[me];" + if (++i == MIDDLE) break; + lds[y * 16 + x ^ y] = u[i]; bar(); + u[i] = lds[x * 16 + y ^ x]; } } diff --git a/src/cl/fft16.cl b/src/cl/fft16.cl index 7cbbb24b..edb494be 100644 --- a/src/cl/fft16.cl +++ b/src/cl/fft16.cl @@ -2,33 +2,53 @@ #if FFT_FP64 -#if 0 +#if 1 #if 1 #include "fft4.cl" -// 24 FMA (of which 16 MUL) + 136 ADD +// 56 FMA + 96 ADD void OVERLOAD fft16(T2 *u) { double C1 = 0.92387953251128674, // cos(tau/16) - S1 = 0.38268343236508978; // sin(tau/16) + S1 = 0.38268343236508978, // sin(tau/16) + S1_over_C1 = 0.4142135623730950488017, + C1_over_S1 = 2.4142135623730950488017; for (int i = 0; i < 4; ++i) { fft4by(u, i, 4, 16); } - u[5] = cmul(u[ 5], U2(C1, S1)); - u[7] = cmul(u[ 7], U2(S1, C1)); - u[13] = cmul(u[13], U2(S1, C1)); - u[15] = cmul(u[15], -U2(C1, S1)); - - u[6] = mul_t8(u[6]); - u[9] = mul_t8(u[9]); - u[11] = mul_3t8(u[11]); - u[14] = mul_3t8(u[14]); - - u[10] = mul_t4(u[10]); - - for (int i = 0; i < 4; ++i) { fft4by(u, 4 * i, 1, 16); } + X2(u[0], u[2]); + X2_mul_t4(u[1], u[3]); + X2(u[0], u[1]); + X2(u[2], u[3]); + SWAP(u[1], u[2]); + + u[9] = mul_t8_delayed(u[9]); // delays a mul by M_SQRT1_2 + u[11] = mul_t8_delayed(u[11]); // delays a mul by i*M_SQRT1_2 (a negation cheaper than mul_3t8_delayed) + X2t4(u[8], u[10]); + X2t4_mul_t4(u[9], u[11]); + X2ad(u[8], u[9], M_SQRT1_2); + X2ad(u[10], u[11], M_SQRT1_2); + SWAP(u[9], u[10]); + + u[5] = partial_cmul(u[5], S1_over_C1); // delays a mul by C1 + u[6] = mul_t8_delayed(u[6]); // delays a mul by M_SQRT1_2 + u[7] = partial_cmul(u[7], C1_over_S1); // delays a mul by S1 + X2ad(u[4], u[6], M_SQRT1_2); + X2ad_mul_t4(u[5], u[7], S1_over_C1); // mul by S1/C1, now both are delaying a mul by C1 + X2ad(u[4], u[5], C1); // apply delayed mul by C1 + X2ad(u[6], u[7], C1); // apply delayed mul by C1 + SWAP(u[5], u[6]); + + u[13] = partial_cmul(u[13], C1_over_S1); // delays a mul by S1 + u[14] = mul_t8_delayed(u[14]); // delays a mul by i*M_SQRT1_2 (a negation cheaper than mul_3t8_delayed) + u[15] = partial_cmul(u[15], S1_over_C1); // delays a mul by -C1 + X2t4ad(u[12], u[14], M_SQRT1_2); + X2ad_mul_t4(u[13], u[15], -C1_over_S1); // mul by -C1/S1, now both are delaying a mul by S1 + X2ad(u[12], u[13], S1); // apply delayed mul by S1 + X2ad(u[14], u[15], S1); // apply delayed mul by S1 + SWAP(u[13], u[14]); SWAP(u[1], u[4]); SWAP(u[2], u[8]); @@ -36,8 +56,6 @@ void OVERLOAD fft16(T2 *u) { SWAP(u[6], u[9]); SWAP(u[7], u[13]); SWAP(u[11], u[14]); - - // for (int i = 0; i < 4; ++i) { fft4by(u, i, 4, 16); } } #else @@ -207,6 +225,8 @@ void OVERLOAD fft16(F2 *u) { void OVERLOAD fft16(GF31 *u) { const Z31 C1 = 1556715293; const Z31 S1 = 978592373; + const Z31 negC1 = M31 - C1; + const Z31 negS1 = M31 - S1; X2(u[0], u[8]); X2(u[1], u[9]); @@ -217,10 +237,10 @@ void OVERLOAD fft16(GF31 *u) { X2_mul_3t8(u[6], u[14]); X2(u[7], u[15]); - u[ 9] = cmul(u[ 9], U2( C1, S1)); // 1t16 - u[11] = cmul(u[11], U2( S1, C1)); // 3t16 - u[13] = cmul(u[13], U2(neg(S1), C1)); // 5t16 //GWBUG - check if optimizer is eliminating the neg (or better yet perhaps tweak follow up code to expect a negative) - u[15] = cmul(u[15], U2(neg(C1), S1)); // 7t16 + u[ 9] = cmul_const(u[ 9], U2( C1, S1)); // 1t16 + u[11] = cmul_const(u[11], U2( S1, C1)); // 3t16 + u[13] = cmul_const(u[13], U2(negS1, C1)); // 5t16 + u[15] = cmul_const(u[15], U2(negC1, S1)); // 7t16 fft8Core(u); fft8Core(u + 8); @@ -247,9 +267,6 @@ void OVERLOAD fft16(GF31 *u) { #include "fft8.cl" void OVERLOAD fft16(GF61 *u) { - const Z61 C1 = 22027337052962166ULL; - const Z61 S1 = 1693317751237720973ULL; - X2(u[0], u[8]); X2(u[1], u[9]); X2_mul_t8(u[2], u[10]); @@ -259,10 +276,10 @@ void OVERLOAD fft16(GF61 *u) { X2_mul_3t8(u[6], u[14]); X2(u[7], u[15]); - u[ 9] = cmul(u[ 9], U2( C1, S1)); // 1t16 - u[11] = cmul(u[11], U2( S1, C1)); // 3t16 - u[13] = cmul(u[13], U2(neg(S1), C1)); // 5t16 //GWBUG - check if optimizer is eliminating the neg (or better yet perhaps tweak follow up code to expect a negative) - u[15] = cmul(u[15], U2(neg(C1), S1)); // 7t16 + u[9] = mul_t16(u[9]); + u[11] = mul_3t16(u[11]); + u[13] = mul_5t16(u[13]); + u[15] = mul_7t16(u[15]); fft8Core(u); fft8Core(u + 8); diff --git a/src/cl/fft4.cl b/src/cl/fft4.cl index 8b422ca9..80b2c3af 100644 --- a/src/cl/fft4.cl +++ b/src/cl/fft4.cl @@ -6,7 +6,7 @@ void OVERLOAD fft4Core(T2 *u) { X2(u[0], u[2]); - X2(u[1], u[3]); u[3] = mul_t4(u[3]); + X2_mul_t4(u[1], u[3]); X2(u[0], u[1]); X2(u[2], u[3]); @@ -26,7 +26,7 @@ void OVERLOAD fft4by(T2 *u, u32 base, u32 step, u32 M) { double x1 = A(1).x + A(3).x; double y3 = A(1).x - A(3).x; double y1 = A(1).y + A(3).y; - double x3 = -(A(1).y - A(3).y); + double x3 = A(3).y - A(1).y; double a0 = x0 + x1; double a1 = x0 - x1; @@ -48,10 +48,9 @@ void OVERLOAD fft4by(T2 *u, u32 base, u32 step, u32 M) { #else X2(A(0), A(2)); - X2(A(1), A(3)); + X2_mul_t4(A(1), A(3)); X2(A(0), A(1)); - A(3) = mul_t4(A(3)); X2(A(2), A(3)); SWAP(A(1), A(2)); @@ -74,7 +73,7 @@ void OVERLOAD fft4(T2 *u) { fft4by(u, 0, 1, 4); } void OVERLOAD fft4Core(F2 *u) { X2(u[0], u[2]); - X2(u[1], u[3]); u[3] = mul_t4(u[3]); + X2_mul_t4(u[1], u[3]); X2(u[0], u[1]); X2(u[2], u[3]); @@ -94,7 +93,7 @@ void OVERLOAD fft4by(F2 *u, u32 base, u32 step, u32 M) { float x1 = A(1).x + A(3).x; float y3 = A(1).x - A(3).x; float y1 = A(1).y + A(3).y; - float x3 = -(A(1).y - A(3).y); + float x3 = A(3).y - A(1).y; float a0 = x0 + x1; float a1 = x0 - x1; @@ -116,10 +115,9 @@ void OVERLOAD fft4by(F2 *u, u32 base, u32 step, u32 M) { #else X2(A(0), A(2)); - X2(A(1), A(3)); + X2_mul_t4(A(1), A(3)); X2(A(0), A(1)); - A(3) = mul_t4(A(3)); X2(A(2), A(3)); SWAP(A(1), A(2)); @@ -194,11 +192,13 @@ void OVERLOAD fft4(GF31 *u) { fft4by(u, 0, 1, 4); } #if NTT_GF61 -void OVERLOAD fft4Core(GF61 *u) { // Starts with all u[i] having maximum values of M61+epsilon. - X2q(&u[0], &u[2], 2); // X2(u[0], u[2]); No reductions mod M61. Will require 3 M61s additions to make positives. - X2q_mul_t4(&u[1], &u[3], 2); // X2(u[1], u[3]); u[3] = mul_t4(u[3]); - X2s(&u[0], &u[1], 3); - X2s(&u[2], &u[3], 3); +void OVERLOAD fft4Core(GF61 *u) { + X2q(&u[0], &u[2]); // u[0] = 0..2+, u[2] = -1-..1+ + X2q_mul_t4(&u[1], &u[3]); // u[1] = 0..2+, u[3] = -1-..1+ + X2q(&u[0], &u[1]); // u[0] = 0..4+, u[1] = -2-..2+ + X2q(&u[2], &u[3]); // u[2] = -2..2+, u[0] = -2-..2+ + u[0] = modM61q(u[0], 0); + for (u32 i = 1; i <= 3; ++i) u[i] = modM61q(u[i], 3); } // 16 ADD @@ -206,32 +206,32 @@ void OVERLOAD fft4by(GF61 *u, u32 base, u32 step, u32 M) { #define A(k) u[(base + step * k) % M] - Z61 x0 = addq(A(0).x, A(2).x); // Max value is 2*M61+epsilon - Z61 x2 = subq(A(0).x, A(2).x, 2); // Max value is 3*M61+epsilon - Z61 y0 = addq(A(0).y, A(2).y); - Z61 y2 = subq(A(0).y, A(2).y, 2); + Z61 x0 = A(0).x + A(2).x; // 0..2+ + Z61 x2 = A(0).x - A(2).x; // -1-..1+ + Z61 y0 = A(0).y + A(2).y; // 0..2+ + Z61 y2 = A(0).y - A(2).y; // -1-..1+ - Z61 x1 = addq(A(1).x, A(3).x); - Z61 y3 = subq(A(1).x, A(3).x, 2); - Z61 y1 = addq(A(1).y, A(3).y); - Z61 x3 = subq(A(3).y, A(1).y, 2); + Z61 x1 = A(1).x + A(3).x; // 0..2+ + Z61 y3 = A(1).x - A(3).x; // -1-..1+ + Z61 y1 = A(1).y + A(3).y; // 0..2+ + Z61 x3 = A(3).y - A(1).y; // -1-..1+ - Z61 a0 = add(x0, x1); - Z61 a1 = subs(x0, x1, 3); + Z61 a0 = x0 + x1; // 0..4+ + Z61 a1 = x0 - x1; // -2-..2+ - Z61 b0 = add(y0, y1); - Z61 b1 = subs(y0, y1, 3); + Z61 b0 = y0 + y1; // 0..4+ + Z61 b1 = y0 - y1; // -2-..2+ - Z61 a2 = add(x2, x3); - Z61 a3 = subs(x2, x3, 4); + Z61 a2 = x2 + x3; // -2-..2+ + Z61 a3 = x2 - x3; // -2-..2+ - Z61 b2 = add(y2, y3); - Z61 b3 = subs(y2, y3, 4); + Z61 b2 = y2 + y3; // -2-..2+ + Z61 b3 = y2 - y3; // -2-..2+ - A(0) = U2(a0, b0); - A(1) = U2(a2, b2); - A(2) = U2(a1, b1); - A(3) = U2(a3, b3); + A(0) = modM61q(U2(a0, b0), 0); + A(1) = modM61q(U2(a2, b2), 3); + A(2) = modM61q(U2(a1, b1), 3); + A(3) = modM61q(U2(a3, b3), 3); #undef A diff --git a/src/cl/fft7.cl b/src/cl/fft7.cl index 450cfa7d..96e2827e 100644 --- a/src/cl/fft7.cl +++ b/src/cl/fft7.cl @@ -2,8 +2,6 @@ #pragma once -#include "base.cl" - #if FFT_FP64 #define A(i) u[(base + i * step) % M] diff --git a/src/cl/fft8.cl b/src/cl/fft8.cl index 56e2fc94..6a9bb104 100644 --- a/src/cl/fft8.cl +++ b/src/cl/fft8.cl @@ -6,23 +6,18 @@ #if FFT_FP64 -T2 mul_t8_delayed(T2 a) { return U2(a.x - a.y, a.x + a.y); } -T2 mul_3t8_delayed(T2 a) { return U2(-(a.x + a.y), a.x - a.y); } -//#define X2_apply_delay(a, b) { T2 t = a; a = t + M_SQRT1_2 * b; b = t - M_SQRT1_2 * b; } -#define X2_apply_delay(a, b) { T2 t = a; a.x = fma(b.x, M_SQRT1_2, a.x); a.y = fma(b.y, M_SQRT1_2, a.y); b.x = fma(-M_SQRT1_2, b.x, t.x); b.y = fma(-M_SQRT1_2, b.y, t.y); } - void OVERLOAD fft4CoreSpecial(T2 *u) { X2(u[0], u[2]); - X2_mul_t4(u[1], u[3]); // X2(u[1], u[3]); u[3] = mul_t4(u[3]); - X2_apply_delay(u[0], u[1]); - X2_apply_delay(u[2], u[3]); + X2t4_mul_t4(u[1], u[3]); // u[3] = mul_t4(u[3]); X2(u[1], u[3]); u[3] = mul_t4(u[3]); + X2ad(u[0], u[1], M_SQRT1_2); + X2ad(u[2], u[3], M_SQRT1_2); } void OVERLOAD fft8Core(T2 *u) { X2(u[0], u[4]); - X2(u[1], u[5]); u[5] = mul_t8_delayed(u[5]); - X2_mul_t4(u[2], u[6]); // X2(u[2], u[6]); u[6] = mul_t4(u[6]); - X2(u[3], u[7]); u[7] = mul_3t8_delayed(u[7]); + X2(u[1], u[5]); u[5] = mul_t8_delayed(u[5]); // Delays a mul by M_SQRT1_2 + X2_mul_t4(u[2], u[6]); // X2(u[2], u[6]); u[6] = mul_t4(u[6]); + X2(u[3], u[7]); u[7] = mul_t8_delayed(u[7]); // Delays a mul by i*M_SQRT1_2 (cheaper than calling mul_3t8_delayed) fft4Core(u); fft4CoreSpecial(u + 4); } @@ -44,23 +39,18 @@ void OVERLOAD fft8(T2 *u) { #if FFT_FP32 -F2 mul_t8_delayed(F2 a) { return U2(a.x - a.y, a.x + a.y); } -F2 mul_3t8_delayed(F2 a) { return U2(-(a.x + a.y), a.x - a.y); } -//#define X2_apply_delay(a, b) { F2 t = a; a = t + M_SQRT1_2 * b; b = t - M_SQRT1_2 * b; } -#define X2_apply_delay(a, b) { F2 t = a; a.x = fma(b.x, (float) M_SQRT1_2, a.x); a.y = fma(b.y, (float) M_SQRT1_2, a.y); b.x = fma((float) -M_SQRT1_2, b.x, t.x); b.y = fma((float) -M_SQRT1_2, b.y, t.y); } - void OVERLOAD fft4CoreSpecial(F2 *u) { X2(u[0], u[2]); - X2_mul_t4(u[1], u[3]); // X2(u[1], u[3]); u[3] = mul_t4(u[3]); - X2_apply_delay(u[0], u[1]); - X2_apply_delay(u[2], u[3]); + X2t4_mul_t4(u[1], u[3]); // u[3] = mul_t4(u[3]); X2(u[1], u[3]); u[3] = mul_t4(u[3]); + X2ad(u[0], u[1], M_SQRT1_2); + X2ad(u[2], u[3], M_SQRT1_2); } void OVERLOAD fft8Core(F2 *u) { X2(u[0], u[4]); - X2(u[1], u[5]); u[5] = mul_t8_delayed(u[5]); - X2_mul_t4(u[2], u[6]); // X2(u[2], u[6]); u[6] = mul_t4(u[6]); - X2(u[3], u[7]); u[7] = mul_3t8_delayed(u[7]); + X2(u[1], u[5]); u[5] = mul_t8_delayed(u[5]); // Delays a mul by M_SQRT1_2 + X2_mul_t4(u[2], u[6]); // X2(u[2], u[6]); u[6] = mul_t4(u[6]); + X2(u[3], u[7]); u[7] = mul_t8_delayed(u[7]); // Delays a mul by i*M_SQRT1_2 (cheaper than calling mul_3t8_delayed) fft4Core(u); fft4CoreSpecial(u + 4); } @@ -108,15 +98,39 @@ void OVERLOAD fft8(GF31 *u) { #if NTT_GF61 -#if 0 // Working code. +void OVERLOAD fft4CoreSpecial1(GF61 *u) { // Starts with u[0,1,2,3] in range of 0..2*M61+epsilon. + X2q(&u[0], &u[2]); // X2(u[0], u[2]); No reductions mod M61. u[0,2] range is 0..4+, -2-..2+ + X2q_mul_t4(&u[1], &u[3]); // X2(u[1], u[3]); u[3] = mul_t4(u[3]); u[1,3] range is 0..4+, -2-..2+ + u[1] = optsubqu(u[1], 2, 2); // Partially reduce. If u[1] > 2*M61, sub 2*M61. u[1] now has range 0..2+ + u[3] = optsubqs(u[3], 0, 2); // Partially reduce. If u[3] > 0*M61, sub 2*M61. u[3] now has range -2-..0+ + X2q(&u[0], &u[1]); // X2(u[0], u[1]); u[0,1] range is 0..6+, -2-..4+ + X2q(&u[2], &u[3]); // X2(u[2], u[3]); u[2,3] range is -4-..2+, -2-..4+ + u[0] = modM61q(u[0], 0); + u[1] = modM61q(u[1], 3); + u[2] = modM61q(u[2], 5); + u[3] = modM61q(u[3], 3); +} -void OVERLOAD fft8Core(GF61 *u) { - X2(u[0], u[4]); //GWBUG: Delay some mods using extra 3 bits of Z61 - X2_mul_t8(u[1], u[5]); // X2(u[1], u[5]); u[5] = mul_t8(u[5]); - X2_mul_t4(u[2], u[6]); // X2(u[2], u[6]); u[6] = mul_t4(u[6]); - X2_mul_3t8(u[3], u[7]); // X2(u[3], u[7]); u[7] = mul_3t8(u[7]); - fft4Core(u); - fft4Core(u + 4); +void OVERLOAD fft4CoreSpecial2(GF61 *u) { // Bottom half of an fft8. Starts with u[0,1,2,3] in range of -1*M61-epsilon..1*M61+epsilon + X2q(&u[0], &u[2]); // X2(u[0], u[2]); No reductions mod M61. u[0,2] range is -2-..2+, -2-..2+ + u[1] = mul_t8q(u[1], 3); // Perform delayed mul_t8. u[1] range is 0..1+ + u[3] = mul_t8q(u[3], 3); // Perform delayed mul_t8. u[3] range is 0..1+ + X2q_mul_t4(&u[1], &u[3]); // X2(u[1], u[3]); u[3] = mul_t4(u[3]); u[1,3] range is 0..2+, -1-..1+ + X2q(&u[0], &u[1]); // X2(u[0], u[1]); u[0,1] range is -2-..4+, -4-..2+ + X2q(&u[2], &u[3]); // X2(u[2], u[3]); u[2,3] range is -3-..3+, -3-..3+ + u[0] = modM61q(u[0], 3); + u[1] = modM61q(u[1], 5); + u[2] = modM61q(u[2], 4); + u[3] = modM61q(u[3], 4); +} + +void OVERLOAD fft8Core(GF61 *u) { // Starts with all u[i] values in range of 0..M61+epsilon (shorthand notation is 0..1+) + X2q(&u[0], &u[4]); // X2(u[0], u[4]); No reductions mod M61. u[0,4] range is 0..2+, -1-..1+ + X2q(&u[1], &u[5]); // X2(u[1], u[5]); Delay mul_t8 on u[5]. u[1,5] range is 0..2+, -1-..1+ + X2q_mul_t4(&u[2], &u[6]); // X2(u[2], u[6]); u[6] = mul_t4(u[6]); u[2,6] range is 0..2+, -1-..1+ + X2q_mul_t4(&u[3], &u[7]); // X2(u[3], u[7]); u[7] = mul_t4(u[7]); u[3,7] range is 0..2+, -1-..1+ Delay mul_t8 on u[7]. + fft4CoreSpecial1(u); + fft4CoreSpecial2(u + 4); } void OVERLOAD fft8(GF61 *u) { @@ -126,40 +140,215 @@ void OVERLOAD fft8(GF61 *u) { SWAP(u[3], u[6]); } -#else // Carefully track the size of numbers to reduce the number of mod M61 reductions +#endif + + + + +//*********************************************************************************************************************** +// In a primarily radix 8 FFT, support other some other options such as radix-4 and radix-16 +//*********************************************************************************************************************** + +#if FFT_FP64 + +// Do two fft4s on eight FFT values +void OVERLOAD fft8_4(T2 *u) { + fft4by(u, 0, 2, 8); + fft4by(u, 1, 2, 8); +} + +// Perform the last three levels of a radix-16 butterfly. The initial radix-2 has already been performed and shufl'ed. +// This is used by SIZE=1K, RADIX=8 fft. There are two versions, one for the first eight radix-16 values and one for the second eight radix-16 values. +void OVERLOAD fft8_16a(T2 *u) { + fft8(u); +} +void OVERLOAD fft8_16b(T2 *u) { + const double C1 = 0.92387953251128674, // cos(tau/16) + S1 = 0.38268343236508978, // sin(tau/16) + S1_over_C1 = 0.4142135623730950488017, + C1_over_S1 = 2.4142135623730950488017; + + X2t4(u[0], u[4]); + X2t4(u[1], u[5]); + X2t4(u[2], u[6]); + X2t4(u[3], u[7]); + + u[1] = partial_cmul(u[1], S1_over_C1); // delays a mul by C1 + u[2] = mul_t8_delayed(u[2]); // delays a mul by M_SQRT1_2 + u[3] = partial_cmul(u[3], C1_over_S1); // delays a mul by S1 + X2ad(u[0], u[2], M_SQRT1_2); + X2ad_mul_t4(u[1], u[3], S1_over_C1); // mul by S1/C1, now both are delaying a mul by C1 + X2ad(u[0], u[1], C1); // apply delayed mul by C1 + X2ad(u[2], u[3], C1); // apply delayed mul by C1 + + u[5] = partial_cmul(u[5], C1_over_S1); // delays a mul by S1 + u[6] = mul_t8_delayed(u[6]); // delays a mul by i*M_SQRT1_2 (a negation cheaper than mul_3t8_delayed) + u[7] = partial_cmul(u[7], S1_over_C1); // delays a mul by -C1 + X2t4ad(u[4], u[6], M_SQRT1_2); + X2ad_mul_t4(u[5], u[7], -C1_over_S1); // mul by -C1/S1, now both are delaying a mul by S1 + X2ad(u[4], u[5], S1); // apply delayed mul by S1 + X2ad(u[6], u[7], S1); // apply delayed mul by S1 -void OVERLOAD fft4CoreSpecial1(GF61 *u) { // Starts with u[0,1,2,3] having maximum values of (2,2,3,2)*M61+epsilon. - X2q(&u[0], &u[2], 4); // X2(u[0], u[2]); No reductions mod M61. u[0,2] max value is 5,6*M61+epsilon. - X2q_mul_t4(&u[1], &u[3], 3); // X2(u[1], u[3]); u[3] = mul_t4(u[3]); u[1,3] max value is 5,4*M61+epsilon. - u[1] = modM61(u[1]); u[2] = modM61(u[2]); // Reduce the worst offenders. u[0,1,2,3] have maximum values of (5,1,1,4)*M61+epsilon. - X2s(&u[0], &u[1], 2); // u[0,1] max value before reduction is 6,7*M61+epsilon - X2s(&u[2], &u[3], 5); // u[2,3] max value before reduction is 5,6*M61+epsilon + SWAP(u[1], u[4]); + SWAP(u[3], u[6]); } -void OVERLOAD fft4CoreSpecial2(GF61 *u) { // Similar to above. Starts with u[0,1,2,3] having maximum values of (3,1,2,1)*M61+epsilon. - X2q(&u[0], &u[2], 3); // u[0,2] max value is 5,6*M61+epsilon. - X2q_mul_t4(&u[1], &u[3], 2); // X2(u[1], u[3]); u[3] = mul_t4(u[3]); u[1,3] max value is 3,2*M61+epsilon. - u[0] = modM61(u[0]); u[2] = modM61(u[2]); // Reduce the worst offenders u[0,1,2,3] have maximum values of (1,3,1,2)*M61+epsilon. - X2s(&u[0], &u[1], 4); // u[0,1] max value before reduction is 4,5*M61+epsilon - X2s(&u[2], &u[3], 3); // u[2,3] max value before reduction is 3,4*M61+epsilon +#endif + +#if FFT_FP32 + +// Do two fft4s on eight FFT values +void OVERLOAD fft8_4(F2 *u) { + fft4by(u, 0, 2, 8); + fft4by(u, 1, 2, 8); } -void OVERLOAD fft8Core(GF61 *u) { // Starts with all u[i] having maximum values of M61+epsilon. - X2q(&u[0], &u[4], 2); // X2(u[0], u[4]); No reductions mod M61. u[0,4] max value is 2,3*M61+epsilon. - X2q_mul_t8(&u[1], &u[5], 2); // X2(u[1], u[5]); u[5] = mul_t8(u[5]); u[1,5] max value is 2,1*M61+epsilon. - X2q_mul_t4(&u[2], &u[6], 2); // X2(u[2], u[6]); u[6] = mul_t4(u[6]); u[2,6] max value is 3,2*M61+epsilon. - X2q_mul_3t8(&u[3], &u[7], 2); // X2(u[3], u[7]); u[7] = mul_3t8(u[7]); u[3,7] max value is 2,1*M61+epsilon. - fft4CoreSpecial1(u); - fft4CoreSpecial2(u + 4); +// Perform the last three levels of a radix-16 butterfly. The initial radix-2 has already been performed and shufl'ed. +// This is used by SIZE=1K, RADIX=8 fft. There are two versions, one for the first eight radix-16 values and one for the second eight radix-16 values. +void OVERLOAD fft8_16a(F2 *u) { + fft8(u); } +void OVERLOAD fft8_16b(F2 *u) { + const float C1 = 0.92387953251128674, // cos(tau/16) + S1 = 0.38268343236508978, // sin(tau/16) + S1_over_C1 = 0.4142135623730950488017, + C1_over_S1 = 2.4142135623730950488017; + + X2t4(u[0], u[4]); + X2t4(u[1], u[5]); + X2t4(u[2], u[6]); + X2t4(u[3], u[7]); + + u[1] = partial_cmul(u[1], S1_over_C1); // delays a mul by C1 + u[2] = mul_t8_delayed(u[2]); // delays a mul by M_SQRT1_2 + u[3] = partial_cmul(u[3], C1_over_S1); // delays a mul by S1 + X2ad(u[0], u[2], M_SQRT1_2); + X2ad_mul_t4(u[1], u[3], S1_over_C1); // mul by S1/C1, now both are delaying a mul by C1 + X2ad(u[0], u[1], C1); // apply delayed mul by C1 + X2ad(u[2], u[3], C1); // apply delayed mul by C1 + + u[5] = partial_cmul(u[5], C1_over_S1); // delays a mul by S1 + u[6] = mul_t8_delayed(u[6]); // delays a mul by i*M_SQRT1_2 (a negation cheaper than mul_3t8_delayed) + u[7] = partial_cmul(u[7], S1_over_C1); // delays a mul by -C1 + X2t4ad(u[4], u[6], M_SQRT1_2); + X2ad_mul_t4(u[5], u[7], -C1_over_S1); // mul by -C1/S1, now both are delaying a mul by S1 + X2ad(u[4], u[5], S1); // apply delayed mul by S1 + X2ad(u[6], u[7], S1); // apply delayed mul by S1 + + SWAP(u[1], u[4]); + SWAP(u[3], u[6]); +} + +#endif + +#if NTT_GF31 + +// Do two fft4s on eight FFT values +void OVERLOAD fft8_4(GF31 *u) { + fft4by(u, 0, 2, 8); + fft4by(u, 1, 2, 8); +} + +// Perform the last three levels of a radix-16 butterfly. The initial radix-2 has already been performed and shufl'ed. +// This is used by SIZE=1K, RADIX=8 fft. There are two versions, one for the first eight radix-16 values and one for the second eight radix-16 values. +void OVERLOAD fft8_16a(GF31 *u) { + fft8(u); +} +void OVERLOAD fft8_16b(GF31 *u) { + const Z31 C1 = 1556715293; + const Z31 S1 = 978592373; + const Z31 negC1 = M31 - C1; + const Z31 negS1 = M31 - S1; + + X2t4(u[0], u[4]); + X2t4(u[1], u[5]); + X2t4(u[2], u[6]); + X2t4(u[3], u[7]); + + u[1] = cmul_const(u[1], U2(C1, S1)); + u[2] = mul_t8(u[2]); + u[3] = cmul_const(u[3], U2(S1, C1)); + X2(u[0], u[2]); + X2_mul_t4(u[1], u[3]); + X2(u[0], u[1]); + X2(u[2], u[3]); + + u[5] = cmul_const(u[5], U2(S1, C1)); + u[6] = mul_3t8(u[6]); + u[7] = cmul_const(u[7], U2(negC1, negS1)); + X2(u[4], u[6]); + X2_mul_t4(u[5], u[7]); + X2(u[4], u[5]); + X2(u[6], u[7]); -void OVERLOAD fft8(GF61 *u) { - fft8Core(u); - // revbin [0, 4, 2, 6, 1, 5, 3, 7] undo SWAP(u[1], u[4]); SWAP(u[3], u[6]); } #endif +#if NTT_GF61 + +// Do two fft4s on eight FFT values +void OVERLOAD fft8_4(GF61 *u) { + fft4by(u, 0, 2, 8); + fft4by(u, 1, 2, 8); +} + +// Perform the last three levels of a radix-16 butterfly. The initial radix-2 has already been performed and shufl'ed. +// This is used by SIZE=1K, RADIX=8 fft. There are two versions, one for the first eight radix-16 values and one for the second eight radix-16 values. +void OVERLOAD fft8_16a(GF61 *u) { + // shufl_and_fft2 performed "quick" adds, u[0-7] are in range 0..2+ + X2q(&u[0], &u[4]); // X2(u[0], u[4]); No reductions mod M61. u[0,4] range is 0..4+, -2-..2+ + X2q(&u[1], &u[5]); // X2(u[1], u[5]); Delay mul_t8 on u[5]. u[1,5] range is 0..4+, -2-..2+ + X2q_mul_t4(&u[2], &u[6]); // X2(u[2], u[6]); u[6] = mul_t4(u[6]); u[2,6] range is 0..4+, -2-..2+ + X2q_mul_t4(&u[3], &u[7]); // X2(u[3], u[7]); u[7] = mul_t4(u[7]); u[3,7] range is 0..4+, -2-..2+ Delay mul_t8 on u[7]. + + // Must normalize values. The delayed mul_t8s can help with that (half of the complex number needs normalizing before mul_t8). + for (u32 i = 0; i <= 3; ++i) u[i] = modM61q(u[i], 0); + u[4] = modM61q(u[4], 3); + u[5].x = optional_add((i64)u[5].x, 2*M61); // u[5] now 0-..2+, -2..2+ + u[5] = mul_t8q(u[5], 5); // Perform delayed mul_t8 (count of 5 based on u[5].y - u[5].x range of -4-..2+ + u[6] = modM61q(u[6], 3); + u[7].x = optional_add((i64)u[7].x, 2*M61); // u[7] now 0-..2+, -2..2+ + u[7] = mul_t8q(u[7], 5); // Perform delayed mul_t8. + + fft4Core(u); + fft4Core(u + 4); + + // revbin [0, 4, 2, 6, 1, 5, 3, 7] undo + SWAP(u[1], u[4]); + SWAP(u[3], u[6]); +} +void OVERLOAD fft8_16b(GF61 *u) { + // shufl_and_fft2 performed "quick" subtracts, u[0-7] are in range -1-..1+ + X2qt4(&u[0], &u[4]); // -2..2+ + X2qt4(&u[1], &u[5]); + X2qt4(&u[2], &u[6]); + X2qt4(&u[3], &u[7]); + + // Must normalize values. Some mul_t8s can help with that (only half of the complex number needs normalizing before mul_t8). + u[0] = modM61q(u[0], 3); + u[1] = modM61q(u[1], 3); + u[2].x = optional_add((i64)u[2].x, 2*M61); // u[2] now 0-..2+, -2..2+ + u[3] = modM61q(u[3], 3); + u[4] = modM61q(u[4], 3); + u[5] = modM61q(u[5], 3); + u[6].y = optional_add((i64)u[6].y, 2*M61); // u[6] now -2-..2+, 0-..2+ + u[7] = modM61q(u[7], 3); + + u[1] = mul_t16(u[1]); + u[2] = mul_t8q(u[2], 5); // Perform mul_t8 (count of 5 based on u[2].y - u[2].x range of -4-..2+, -(u[2].x + u[2].y) range of -4-..2+ + u[3] = mul_3t16(u[3]); + fft4Core(u); + + u[5] = mul_3t16(u[5]); + u[6] = mul_3t8q(u[6], 3); // Perform mul_3t8 (count of 5 based on u[6].y - u[6].x range of -2-..4+, u[6].y + u[6].x range of -2-..4+ + u[7] = mul_9t16(u[7]); + fft4Core(u + 4); + + SWAP(u[1], u[4]); + SWAP(u[3], u[6]); +} + #endif diff --git a/src/cl/fftbase.cl b/src/cl/fftbase.cl index c955e803..25a29be5 100644 --- a/src/cl/fftbase.cl +++ b/src/cl/fftbase.cl @@ -2,8 +2,178 @@ #include "fft4.cl" #include "fft8.cl" -#include "trig.cl" -// #include "math.cl" + +// NOTE: tailSquare, with its ability to optionally define tailSquareZero, does not allow us to know numWG at #include time. +// Thus, we must define macros that take numWG as in input argument. This could be rectified by making tailSquareZero obey the TAIL_KERNELS setting. + +// This section is not necessary. On TitanV, CUDA 12.9, I see a 0.5% slowdown when not LDS sharing but compiled with the LDS sharing code. + +#if LDSMUL == 1 // Not sharing LDS memory, use simplified code. + +#define SHARING_LDS(numWG) 0 +#define SBMUL(numWG) 1 +#define LDSPAD_COUNT(numWG) (!LDSPAD ? 0 : RADIX == 4 ? 12 : SHUFL_BYTES >= 16 ? 7 : 56) +#define LDS_SHUFL_BYTES(numWG) ((WG * RADIX + LDSPAD_COUNT(numWG)) * SHUFL_BYTES) +#define LDS_BYTES(numWG) (numWG * LDS_SHUFL_BYTES(numWG)) + +void OVERLOAD LDSinit(void local *lds, const u32 numWG) { +} + +local void * OVERLOAD LDSptr(local void *lds, const u32 numWG) { + return (local char *)lds + ((u32)get_local_id(0) / WG) * LDS_SHUFL_BYTES(numWG); +} + +local void * OVERLOAD LDSsharing_ptr(local void *lds, const u32 numWG) { + return LDSptr(lds, numWG); +} + +void OVERLOAD LDSbar(const u32 numWG) { + bar(WG); +} + +void OVERLOAD LDStx_start(local void *lds, const u32 numWG) { + LDSbar(numWG); +} + +void OVERLOAD LDStx_end(local void *lds, const u32 numWG) { +} + + +// This section handles both cases of sharing and not sharing LDS memory + +#else + +// LDS access is shared if the kernel processes multiple independent workgroups, the user settable LDSMUL is more than one, and the GPU allows barriers on a subset of threads +#define SHARING_LDS(numWG) (numWG > 1 && LDSMUL > 1 && (NVIDIAGPU || WG <= WAVEFRONT)) +// If sharing LDS access, LDSMUL sets a limit on how many workgroups share the same LDS memory. Sharing LDS allow shufl to use a multiple of SHUFL_BYTES. +#define SBMUL(numWG) (!SHARING_LDS(numWG) ? 1 : numWG >= LDSMUL ? LDSMUL : numWG) +// Calculate the LDS padding used by shufl +#define LDSPAD_COUNT(numWG) (!LDSPAD ? 0 : RADIX == 4 ? 12 : SBMUL(numWG) * SHUFL_BYTES >= 16 ? 7 : 56) +// LDS_SHUFL_BYTES is the number of LDS bytes *allocated* for each workgroup (SBMUL > 1 means the workgroup can *access* some multiple of LDS_SHUFL_BYTES) +#define LDS_SHUFL_BYTES(numWG) ((WG * RADIX + LDSPAD_COUNT(numWG)) * SHUFL_BYTES) +// The workgroups are partitioned into groups of SBMUL that share one LDS region and one semaphore. +// SBMUL need not divide numWG, so round up: the last group is short but still spans SBMUL regions and +// owns a semaphore of its own. Allocating only numWG regions, and only ever four semaphores, is not +// enough then -- LDSsharing_ptr hands out a region past the end of the array and LDSinit leaves the +// last semaphore uninitialised. With SBMUL == 1, which is every configuration that does not opt into +// sharing, all of this is numWG regions and no semaphores, exactly as before. +#define LDS_GROUPS(numWG) ((numWG + SBMUL(numWG) - 1) / SBMUL(numWG)) +#define LDS_REGIONS(numWG) (LDS_GROUPS(numWG) * SBMUL(numWG)) +#define LDS_SEM_OFFSET(numWG) (LDS_REGIONS(numWG) * LDS_SHUFL_BYTES(numWG)) +#define LDS_BYTES(numWG) (LDS_SEM_OFFSET(numWG) + (SHARING_LDS(numWG) ? LDS_GROUPS(numWG) * 4 : 0)) + +// Variant 2 keeps its own pointer into the shared region (partitioned_lds) and both reads and writes it +// in partial_tabMul4/8 outside the LDStx lock, with only a bar(WG) that does not cover the other +// workgroups sharing that memory. That is what the "partitioned_LDS is a nightmare" note was about, so +// refuse the combination rather than corrupt quietly. This also keeps LDSptr's truncating divide out of +// reach: it is only inexact when sharing rounds LDS_SHUFL_BYTES off a 16-byte boundary, and variant 2 is +// its only caller. +#if VARIANT == 2 +#error LDSMUL > 1 is not supported with FFT variant 2 (partial_tabMul touches the shared LDS region outside the lock) +#endif + +// shufl dispatches on SBMUL * SHUFL_BYTES with branches for >= 16, == 8 and == 4 and no fallback, so a +// product of 12 would return with the data unexchanged and no diagnostic. Only SBMUL == 3 can produce +// it. Rejecting on LDSMUL is slightly stronger than necessary -- a call with numWG < 3 would have come +// out at SBMUL < 3 -- but numWG is a runtime argument, and a build error beats silently wrong results. +#if LDSMUL >= 3 && SHUFL_BYTES == 4 +#error LDSMUL >= 3 with SHUFL_BYTES == 4 gives SBMUL * SHUFL_BYTES == 12, which no shufl branch handles +#endif + +// Initialize access to LDS memory. It may be advantageous to have independent workgroups share access to LDS memory via a lock controlling a critical section. +// This may let a kernel use less LDS memory, or have each workgroup use more LDS memory to perform fewer passes of writing and reading LDS memory. +void OVERLOAD LDSinit(void local *lds, const u32 numWG) { + // Init semaphores to unlocked state + if (SHARING_LDS(numWG)) { + if (get_local_id(0) == 0) { + volatile local int *semaphores = (volatile local int *)(((local char *) lds) + LDS_SEM_OFFSET(numWG)); + // One per sharing group, and every group that exists: the highest index in use is + // (numWG - 1) / SBMUL, which the old numWG / SBMUL bound missed whenever SBMUL did not divide numWG. + for (u32 i = 0; i < LDS_GROUPS(numWG); i++) semaphores[i] = 0; + } + bar(); + } +} + +// Return a pointer to the LDS memory allocated for this workgroup. If SBMUL is greater than 1, workgroup may use additional memory by sharing +// with other workgroups and using locks to control access. +local void * OVERLOAD LDSptr(local void *lds, const u32 numWG) { + return (local char *)lds + ((u32)get_local_id(0) / WG) * LDS_SHUFL_BYTES(numWG); +} + +// Return a pointer to the LDS memory this workgroup is allowed to access when sharing with other workgroups. +local void * OVERLOAD LDSsharing_ptr(local void *lds, const u32 numWG) { + if (!SHARING_LDS(numWG)) return LDSptr(lds, numWG); + return (local char *)lds + ((u32)get_local_id(0) / WG / SBMUL(numWG)) * SBMUL(numWG) * LDS_SHUFL_BYTES(numWG); +} + +// Wait for all of a workgroup's threads to arrive. +// NOTE: A "workgroup" is an independent group of threads doing FFT work (see WMUL in carryFused or TAIL_KERNELS=2). +void OVERLOAD LDSbar(const u32 numWG) { + + // No early return for WG <= WAVEFRONT here: bar(WG) and barsync() below both decide that for themselves, + // by what the hardware guarantees rather than by size alone, and returning early would skip the warp + // reconvergence and LDS fence they do on hardware that is not in lock-step. + + // If were not using semaphores to share LDS access, perform a standard bar. The standard bar is free to implement a full bar across + // all threads if that is more efficient than a bar across a subset of threads. + if (!SHARING_LDS(numWG)) { + bar(WG); + return; + } + + // Barrier on a subset of threads. + barsync(numWG, WG); +} + +// Start a new LDS access transaction. This is required for sharing LDS memory with other workgroups. +// Historically, each workgroup had its own LDS area, and shufl routines performed a bar(WG) at the start of accessing LDS but not at the end. +// After calling shufl, a bar(WG) was required before next LDS memory usage. All routines that use LDS memory OBEYED THIS PROTOCOL +// of bar(WG) before LDS use (full bar() if writing outside the workgroup's LDS area) and no bar(WG) after last use (full bar() if reading from +// outside the workgroup's LDS area). If we're not sharing LDS access among multiple workgroups, maintain this historical implementation. +// If sharing LDS we have NEW REQUIREMENTS. LDStx_end performs an LDSbar because workgroups write to more than just their own LDS area. The LDSbar +// ensures the reads have completed before any future writes. When accessing LDS memory without LDStx calls (see shufl_carries_up in carryFused and +// reverseLines in tailutil) they too must perform a bar() after the last read from LDS. +// NOTE: Pass in the original LDS pointer, not the pointer returned by LDSptr or LDSsharing_ptr. +void OVERLOAD LDStx_start(local void *lds, const u32 numWG) { + // If each workgroup has its own LDS area, then no locks are needed to access shared memory. Use the historical model of requiring a barrier before LDS access. + if (!SHARING_LDS(numWG)) { + LDSbar(numWG); + return; + } + // Have first thread in a workgroup lock the semaphore controlling access to LDS memory + if (get_local_id(0) % WG == 0) { + volatile local int *semaphores = (volatile local int *)(((local char *) lds) + LDS_SEM_OFFSET(numWG)); + + // Lock semaphore (set to one) to gain access to critical section + // Spin until the semaphore was observed unlocked: cmpxchg returns the old value, so anything other + // than 0 means the lock was not taken. Testing for 1 alone would let any other value through here + // without the lock, and LDStx_end would then clear a semaphore this workgroup never owned. + while (atomic_cmpxchg(&semaphores[get_local_id(0) / WG / SBMUL(numWG)], 0, 1) != 0); + } + LDSbar(numWG); +} + +// End an LDS access transaction +// NOTE: Pass in the original LDS pointer, not the pointer returned by LDSptr or LDSsharing_ptr. +void OVERLOAD LDStx_end(local void *lds, const u32 numWG) { + // Historically, no trailing LDSbar is required when not sharing LDS memory. + if (!SHARING_LDS(numWG)) return; + + // Since we are sharing LDS areas among multiple workgroups, we must wait for all of a workgroup's threads to finish their LDS access. + LDSbar(numWG); + // Unlock the semaphore + if (get_local_id(0) % WG == 0) { + volatile local int *semaphores = (volatile local int *)(((local char *) lds) + LDS_SEM_OFFSET(numWG)); + semaphores[get_local_id(0) / WG / SBMUL(numWG)] = 0; + } +} + +#endif + + +#define INCLUDE_FILE "shufl.cl" +#include "expand.cl" #if FFT_FP64 @@ -21,12 +191,12 @@ void OVERLOAD chainMul4(T2 *u, T2 w) { #if 1 // This version of chainMul8 tries to minimize roundoff error even if more F64 ops are used. // Trial and error looking at Z values on a WIDTH=512 FFT was used to determine when to switch from fancy to non-fancy powers of w. -void OVERLOAD chainMul8(T2 *u, T2 w, u32 tailSquareBcast) { +void OVERLOAD chainMul8(T2 *u, T2 w) { u[1] = cmulFancy(u[1], w); T2 w2; - // Rocm optimizer behaves weirdly. Using multiple mul2s instead of one mul2 in csqTrigFancy makes double-wide single-kernel tailSquare inexplicably slower - if (!tailSquareBcast) { + // Rocm optimizer behaves weirdly. Using multiple mul2s instead of one mul2 in csqTrigFancy makes double-wide single-kernel variant 0 tailSquare inexplicably slower + if (DOING_WIDTH || VARIANT != 0) { w2 = csqTrigFancy(w); } else { w2 = U2(mulminus2(w.y) * w.y, mul2(fma(w.x, w.y, w.y))); @@ -36,7 +206,7 @@ void OVERLOAD chainMul8(T2 *u, T2 w, u32 tailSquareBcast) { T2 w3; // Rocm optimizer behaves weirdly yet again. Using mul2 instead of 2.0* makes double-wide single-kernel tailSquare inexplicably slower // even though it is one fewer F64 op. - if (!tailSquareBcast) { + if (DOING_WIDTH || VARIANT != 0) { w3 = ccubeTrigFancy(w2, w); } else { double a = 2*w2.y; @@ -45,7 +215,7 @@ void OVERLOAD chainMul8(T2 *u, T2 w, u32 tailSquareBcast) { u[3] = cmulFancy(u[3], w3); w3.x += 1; - T2 base = cmulFancy (w3, w); + T2 base = cmulFancy(w3, w); for (int i = 4; i < 8; ++i) { u[i] = cmul(u[i], base); base = cmulFancy(base, w); @@ -54,12 +224,12 @@ void OVERLOAD chainMul8(T2 *u, T2 w, u32 tailSquareBcast) { #else // This version of chainMul8 minimizes F64 ops even if that increases roundoff error. -// This version is faster on a Radeon 7 with worse roundoff. However, new_FFT_width is even faster with better roundoff. +// This version is faster on a Radeon 7 with worse roundoff. However, FFT_width is even faster with better roundoff. // This version is the same speed on a TitanV probably due to its great F64 throughput. // This version is slower on R7Pro due to a rocm optimizer issue in double-wide single-kernel tailSquare using BCAST. I could not find a work-around. // Other GPUs??? This version might be useful. If we decide to make this available, it will need a new width and height fft spec number. // Consequently, an increase in the BPW table and increase work for -ztune and -tune. -void OVERLOAD chainMul8(T2 *u, T2 w, u32 tailSquareBcast) { +void OVERLOAD chainMul8(T2 *u, T2 w) { u[1] = cmulFancy(u[1], w); T2 w2 = csqTrigFancy(w); @@ -79,15 +249,15 @@ void OVERLOAD chainMul8(T2 *u, T2 w, u32 tailSquareBcast) { } #endif -void OVERLOAD chainMul(u32 len, T2 *u, T2 w, u32 tailSquareBcast) { +void OVERLOAD chainMul(T2 *u, T2 w) { // Do a length 4 chain mul, w must not be in Fancy format - if (len == 4) chainMul4(u, w); + if (RADIX == 4) chainMul4(u, w); // Do a length 8 chain mul, w must be in Fancy format - if (len == 8) chainMul8(u, w, tailSquareBcast); + if (RADIX == 8) chainMul8(u, w); } -#if AMDGPU && (FFT_VARIANT_W == 0 || FFT_VARIANT_H == 0) +#if AMDGPU && VARIANT == 0 int bcast4(int x) { return __builtin_amdgcn_mov_dpp(x, 0, 0xf, 0xf, false); } int bcast8(int x) { return __builtin_amdgcn_ds_swizzle(x, 0x0018); } @@ -106,86 +276,19 @@ T2 bcast(T2 src, u32 span) { #endif -void OVERLOAD shuflBigLDS(u32 WG, local T2 *lds, T2 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i]; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i] = lds[i * WG + me]; } -} - -void OVERLOAD shufl(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - local T* lds = (local T*) lds2; - - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].x; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + me]; } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].y; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + me]; } -} - -// Same as shufl but use ints instead of doubles to reduce LDS memory requirements. -// Lower LDS requirements should let the optimizer use fewer VGPRs and increase occupancy for WIDTHs >= 1024. -// Alas, the increased occupancy does not offset extra code needed for shufl_int (the assembly -// code generated is not pretty). This might not be true for nVidia or future ROCm optimizers. -void OVERLOAD shufl_int(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - local int* lds = (local int*) lds2; - - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = as_int4(u[i]).x; } - bar(); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + me]; u[i] = as_double2(tmp); } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = as_int4(u[i]).y; } - bar(); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + me]; u[i] = as_double2(tmp); } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = as_int4(u[i]).z; } - bar(); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + me]; u[i] = as_double2(tmp); } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = as_int4(u[i]).w; } - bar(); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + me]; u[i] = as_double2(tmp); } - bar(); // I'm not sure why this barrier call is needed -} - -// Shufl two simultaneous FFT_HEIGHTs. Needed for tailSquared where u and v are computed simultaneously in different threads. -// NOTE: It is very important for this routine to use lds memory in coordination with reverseLine2 and unreverseLine2. -// Failure to do so would result in the need for more bar() calls. Specifically, the u values are stored in the upper half -// of lds memory (first SMALL_HEIGHT T2 values). The v values are stored in the lower half of lds memory (next SMALL_HEIGHT T2 values). -void OVERLOAD shufl2(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - - // Partition lds memory into upper and lower halves - assert(WG == G_H); - - // Accessing lds memory as doubles is faster than T2 accesses - local T* lds = ((local T*) lds2) + (me / WG) * SMALL_HEIGHT; - - me = me % WG; - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].x; } - bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + me]; } - bar(WG); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].y; } - bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + me]; } +void OVERLOAD fft_RADIX(T2 *u) { +#if RADIX == 4 + fft4(u); +#elif RADIX == 5 + fft5(u); +#elif RADIX == 8 + fft8(u); +#else +#error RADIX +#endif } -void OVERLOAD tabMul(u32 WG, Trig trig, T2 *u, u32 n, u32 f, u32 me) { +void OVERLOAD tabMul(Trig trig, T2 *u, u32 f, u32 me) { #if 0 u32 p = me / f * f; #else @@ -194,9 +297,9 @@ void OVERLOAD tabMul(u32 WG, Trig trig, T2 *u, u32 n, u32 f, u32 me) { // Compute trigs from scratch every time. This can't possibly be a good idea on any GPUs. #if 0 - T2 w = slowTrig_N(ND / n / WG * p, ND / n); + T2 w = slowTrig_N(ND / RADIX / WG * p, ND / RADIX); T2 base = w; - for (int i = 1; i < n; ++i) { + for (int i = 1; i < RADIX; ++i) { u[i] = cmul(u[i], w); w = cmul(w, base); } @@ -208,8 +311,8 @@ void OVERLOAD tabMul(u32 WG, Trig trig, T2 *u, u32 n, u32 f, u32 me) { // Apparently, chained Fancy muls at these short n=4 and n=8 lengths are very accurate. if (TABMUL_CHAIN) { - T2 w = trig[p]; - chainMul (n, u, w, 0); + T2 w = TFLOAD(&trig[p]); + chainMul(u, w); return; } @@ -217,65 +320,174 @@ void OVERLOAD tabMul(u32 WG, Trig trig, T2 *u, u32 n, u32 f, u32 me) { // Radeon VII loves this case, it is faster than the chainmul case. nVidia Titan V hates this case. if (!TABMUL_CHAIN) { - T2 w = trig[p]; + T2 w = TFLOAD(&trig[p]); - if (n >= 8) { + if (RADIX >= 8) { u[1] = cmulFancy(u[1], w); } else { u[1] = cmul(u[1], w); } - for (u32 i = 2; i < n; ++i) { - T2 base = trig[(i-1)*WG + p]; - u[i] = cmul(u[i], base); + for (u32 i = 2; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*WG + p])); } return; } } +// Tabmul after doing an fft4 when RADIX=8. See the SIZE=256 code for example memory and trig layout. +void OVERLOAD tabMul8_4a(Trig trig, T2 *u, u32 f, u32 me) { + + if (f == 1) { // fft8_4 is performed first + u32 p = me; + +// This code uses chained complex multiplies which could be faster on GPUs with great DP throughput or poor memory bandwidth or caching. +// This ought to be the least accurate version of Tabmul. In practice, this is just as accurate as reading precomputed values from memory. +// Perform two length=4 chain muls. + + if (TABMUL_CHAIN) { + T2 w = TFLOAD(&trig[p]); + T2 w2 = TFLOAD(&trig[WG + p]); + u[2] = cmul(u[2], w); + u[3] = cmul(u[3], w2); + T2 base = csqTrig(w); + T2 base2 = csqTrig(w2); + u[4] = cmul(u[4], base); + u[5] = cmul(u[5], base2); + base = ccubeTrig(base, w); + base2 = ccubeTrig(base2, w2); + u[6] = cmul(u[6], base); + u[7] = cmul(u[7], base2); + } + +// Theoretically, maximum accuracy. Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. +// Radeon VII loves this case, it is faster than the chainmul case. nVidia Titan V hates this case. + + if (!TABMUL_CHAIN) { + for (u32 i = 2; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-2)*WG + p])); + } + } + } + + else { // fft8_4 is performed after an initial fft8 + +// This code uses chained complex multiplies which could be faster on GPUs with great DP throughput or poor memory bandwidth or caching. +// This ought to be the least accurate version of Tabmul. In practice, this is just as accurate as reading precomputed values from memory. +// Perform two length=4 chain muls. + + u32 p = me / 8; // Generate index into condensed trig table that does not have duplicated trig values + trig += 7 * WG; // Skip over the trig values used in the first tabmul + if (TABMUL_CHAIN) { + T2 w = TFLOAD(&trig[p]); + T2 w2 = TFLOAD(&trig[WG/8 + p]); + u[2] = cmul(u[2], w); + u[3] = cmul(u[3], w2); + T2 base = csqTrig(w); + T2 base2 = csqTrig(w2); + u[4] = cmul(u[4], base); + u[5] = cmul(u[5], base2); + base = ccubeTrig(base, w); + base2 = ccubeTrig(base2, w2); + u[6] = cmul(u[6], base); + u[7] = cmul(u[7], base2); + } + +// Theoretically, maximum accuracy. Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. +// Radeon VII loves this case, it is faster than the chainmul case. nVidia Titan V hates this case. + + if (!TABMUL_CHAIN) { + for (u32 i = 2; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-2)*WG/8 + p])); + } + } + } +} + +// Later tabmuls after starting with an fft4 when RADIX=8. See the SIZE=256 code for example memory and trig layout. +void OVERLOAD tabMul8_4b(Trig trig, T2 *u, u32 f, u32 me) { + +// This code uses chained complex multiplies which could be faster on GPUs with great DP throughput or poor memory bandwidth or caching. +// This ought to be the least accurate version of Tabmul. In practice, this is just as accurate as reading precomputed values from memory. +// Apparently, chained Fancy muls at n=8 lengths are very accurate. + + if (TABMUL_CHAIN) { + u32 p = me & ~(f - 1); + T2 w = TFLOAD(&trig[p]); + +// u[1] = cmulFancy(u[1], w); // GW: - this should use Fancy, but tabmul8_4a does not and it could for half of the data +// T2 w2 = csqTrigFancy(w); +// u[2] = cmulFancy(u[2], w2); +// T2 w3 = ccubeTrigFancy(w2, w); +// u[3] = cmulFancy(u[3], w3); +// w3.x += 1; +// T2 base = cmulFancy(w3, w); +// for (int i = 4; i < 8; ++i) { +// u[i] = cmul(u[i], base); +// base = cmulFancy(base, w); +// } + + u[1] = cmul(u[1], w); // GW: - this should use Fancy, but tabmul8_4a does not and it could for half of the data + T2 w2 = csqTrig(w); + u[2] = cmul(u[2], w2); + T2 w3 = ccubeTrig(w2, w); + u[3] = cmul(u[3], w3); + T2 base = cmul(w3, w); + for (int i = 4; i < 8; ++i) { + u[i] = cmul(u[i], base); + base = cmul(base, w); + } + } + +// Theoretically, maximum accuracy. Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. +// Radeon VII loves this case, it is faster than the chainmul case. nVidia Titan V hates this case. + + if (!TABMUL_CHAIN) { + u32 p = (me/4) & ~(f/4 - 1); // Generate index into condensed trig table that does not have duplicated trig values + trig += 6 * WG; // Skip over the trig values used in tabmul8_4a + +//GW: Can any of these be Fancy? Yes, u[1] and u[2] + for (u32 i = 1; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*(WG/4) + p])); + } + } +} //************************************************************************************ // New fft WIDTH and HEIGHT macros to support radix-4 FFTs with more FMA instructions //************************************************************************************ -// Partial complex-multiply that delays the mul-by-cosine so it can be part of an FMA. -// We're trying to calculate u * U2(cosine,sine). -// real = (u.x - u.y*sine_over_cosine) * cosine -// imag = (u.x*sine_over_cosine + u.y) * cosine -T2 partial_cmul(T2 u, T sine_over_cosine) { - return U2(fma(-u.y, sine_over_cosine, u.x), fma(u.x, sine_over_cosine, u.y)); -} - // Copy of macro from fft4 and fft8 with FMAs added #define X2_via_FMA(a, c, b) { T2 t = a; a = fma(c, b, t); b = fma(-c, b, t); } // Preload trig values for the first partial tabMul. We load the sine/cosine values early so that F64 ops can hide the read latency. -void preload_tabMul4_trig(u32 WG, Trig trig, T *preloads, u32 f, u32 me) { +void preload_tabMul4_trig(Trig trig, T *preloads, u32 f, u32 numWG, u32 me) { TrigSingle trig1 = (TrigSingle) trig; // Read 3 lines of sine/cosine values for the first fft4. Read two of the lines as a pair as AMD likes T2 global memory reads Trig trig2 = (Trig) trig1; - T2 sine_over_cosines = trig2[me]; + T2 sine_over_cosines = TFLOAD(&trig2[me]); preloads[0] = sine_over_cosines.x; preloads[1] = sine_over_cosines.y; // Read 3rd line - preloads[2] = trig1[2*WG + me]; + preloads[2] = TFLOAD(&trig1[2*WG + me]); } // Do a partial tabMul. Save the mul-by-cosine for later FMA instructions. -void partial_tabMul4(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f, u32 me) { +void partial_tabMul4(local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG, u32 me) { local T *lds1 = (local T *) lds; TrigSingle trig1 = (TrigSingle) trig; trig1 += 4*WG; // Skip past sine_over_cosine values // Use LDS memory to distribute preloaded trig values. if (f > 1) { + bar(WG); lds1[me] = preloads[4]; // Preloaded sine/cosine values lds1[WG+me] = preloads[5]; // Preloaded cosine values - bar(WG); } // Apply sine/cosines + bar(WG); for (u32 i = 1; i < 4; ++i) { T sine_over_cosine; if (f == 1) sine_over_cosine = preloads[i-1]; @@ -288,7 +500,7 @@ void partial_tabMul4(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f // Read pairs of lines to make AMD happy with T2 global memory loads for (u32 i = 0; i < 4; i += 2) { Trig trig2 = (Trig) (trig1 + i*WG); - T2 cosines = trig2[me]; + T2 cosines = TFLOAD(&trig2[me]); preloads[i] = cosines.x; preloads[i+1] = cosines.y; } @@ -299,13 +511,11 @@ void partial_tabMul4(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f preloads[2] = lds1[WG + ((me/f) & 3) * WG/4 + (2 * WG + me)/(4*f) * f/4]; preloads[3] = lds1[WG + ((me/f) & 3) * WG/4 + (3 * WG + me)/(4*f) * f/4]; preloads[1] = lds1[WG + ((me/f) & 3) * WG/4 + (1 * WG + me)/(4*f) * f/4]; - bar(WG); } } // Finish off a partial tabMul while doing next fft4 making more use of FMA. -void finish_tabMul4_fft4(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f, u32 me, u32 save_one_more_mul) { - local T *lds1 = (local T *) lds; +void finish_tabMul4_fft4(Trig trig, T *preloads, T2 *u, u32 f, u32 numWG, u32 me, u32 save_one_more_mul) { TrigSingle trig1 = (TrigSingle) trig; // @@ -321,8 +531,8 @@ void finish_tabMul4_fft4(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u // Preload one line of sine/cosines and one line of cosines for later tabMuls. We'll later broadcast these values as needed using LDS. if (f == 1) { - preloads[4] = trig1[3*WG + me]; // Sine/cosines for later tabMuls - preloads[5] = trig1[4*WG + 4*WG + me]; // Cosines for later tabMuls + preloads[4] = TFLOAD(&trig1[3*WG + me]); // Sine/cosines for later tabMuls + preloads[5] = TFLOAD(&trig1[4*WG + 4*WG + me]); // Cosines for later tabMuls } // Do the last level of fft4 applying cosine1 @@ -338,34 +548,35 @@ void finish_tabMul4_fft4(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u //************************************************************************************ // Preload trig values for the first partial tabMul. We load the sine/cosine values early so that F64 ops can hide the read latency. -void preload_tabMul8_trig(u32 WG, Trig trig, T *preloads, u32 f, u32 me) { +void preload_tabMul8_trig(Trig trig, T *preloads, u32 f, u32 numWG, u32 me) { TrigSingle trig1 = (TrigSingle) trig; // Read 7 lines of sine/cosine values for the first fft8. Read six of the lines as pairs as AMD likes T2 global memory reads for (u32 i = 1; i < 7; i += 2) { Trig trig2 = (Trig) (trig1 + (i-1)*WG); - T2 sine_over_cosines = trig2[me]; + T2 sine_over_cosines = TFLOAD(&trig2[me]); preloads[i-1] = sine_over_cosines.x; preloads[i] = sine_over_cosines.y; } // Read 7th line - preloads[6] = trig1[6*WG + me]; + preloads[6] = TFLOAD(&trig1[6*WG + me]); } // Do a partial tabMul. Save the mul-by-cosine for later FMA instructions. -void partial_tabMul8(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f, u32 me) { +void partial_tabMul8(local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG, u32 me) { local T *lds1 = (local T *) lds; TrigSingle trig1 = (TrigSingle) trig; trig1 += 8*WG; // Skip past sine_over_cosine values // Use LDS memory to distribute preloaded trig values. if (f > 1) { + bar(WG); lds1[me] = preloads[8]; // Preloaded sine/cosine values lds1[WG+me] = preloads[9]; // Preloaded cosine values - bar(WG); } // Apply sine/cosines + bar(WG); for (u32 i = 1; i < 8; ++i) { T sine_over_cosine; if (f == 1) sine_over_cosine = preloads[i-1]; @@ -378,7 +589,7 @@ void partial_tabMul8(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f // Read pairs of lines to make AMD happy with T2 global memory loads for (u32 i = 0; i < 8; i += 2) { Trig trig2 = (Trig) (trig1 + i*WG); - T2 cosines = trig2[me]; + T2 cosines = TFLOAD(&trig2[me]); preloads[i] = cosines.x; preloads[i+1] = cosines.y; } @@ -394,13 +605,11 @@ void partial_tabMul8(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f preloads[7] = lds1[WG + ((me/f) & 7) * WG/8 + (7 * WG + me)/(8*f) * f/8]; preloads[2] = lds1[WG + ((me/f) & 7) * WG/8 + (2 * WG + me)/(8*f) * f/8]; preloads[3] = lds1[WG + ((me/f) & 7) * WG/8 + (3 * WG + me)/(8*f) * f/8]; - bar(WG); } } // Finish off a partial tabMul while doing next fft8 making more use of FMA. -void finish_tabMul8_fft8(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f, u32 me, u32 save_one_more_mul) { - local T *lds1 = (local T *) lds; +void finish_tabMul8_fft8(Trig trig, T *preloads, T2 *u, u32 f, u32 numWG, u32 me, u32 save_one_more_mul) { TrigSingle trig1 = (TrigSingle) trig; // @@ -410,7 +619,7 @@ void finish_tabMul8_fft8(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u // Apply cosine0 to u[0] if (f < WG/8) u[0] = u[0] * preloads[0]; - if (save_one_more_mul) { // This should always be the best option. ROCm optimizer is doing something weird in new_fft_WIDTH case. + if (save_one_more_mul) { // This should always be the best option. ROCm optimizer is doing something weird in fft_WIDTH case. // Apply cosine4, cosine5/cosine1, cosine6/cosine2, cosine7/cosine3 to u[4] through u[7] using FMA X2_via_FMA(u[0], preloads[4], u[4]); @@ -420,8 +629,8 @@ void finish_tabMul8_fft8(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u // Preload one line of sine/cosines and one line of cosines for second tabMul. We'll later broadcast these values as needed using LDS. if (f == 1) { - preloads[8] = trig1[7*WG + me]; // Sine/cosines for second tabMul - preloads[9] = trig1[8*WG + 8*WG + me]; // Cosines for second tabMul + preloads[8] = TFLOAD(&trig1[7*WG + me]); // Sine/cosines for second tabMul + preloads[9] = TFLOAD(&trig1[8*WG + 8*WG + me]); // Cosines for second tabMul } // Do the fft4Core and fft4CoreSpecial applying cosine2, cosine3/cosine1 @@ -451,8 +660,8 @@ void finish_tabMul8_fft8(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u // Preload one line of sine/cosines and one line of cosines for second tabMul. We'll later broadcast these values as needed using LDS. if (f == 1) { - preloads[8] = trig1[7*WG + me]; // Sine/cosines for second tabMul - preloads[9] = trig1[8*WG + 8*WG + me]; // Cosines for second tabMul + preloads[8] = TFLOAD(&trig1[7*WG + me]); // Sine/cosines for second tabMul + preloads[9] = TFLOAD(&trig1[8*WG + 8*WG + me]); // Cosines for second tabMul } // Do the fft4Core and fft4CoreSpecial applying cosine2, cosine3 @@ -464,8 +673,8 @@ void finish_tabMul8_fft8(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u // Do last level of fft8 X2(u[0], u[1]); X2(u[2], u[3]); - X2_apply_delay(u[4], u[5]); - X2_apply_delay(u[6], u[7]); + X2ad(u[4], u[5], M_SQRT1_2); + X2ad(u[6], u[7], M_SQRT1_2); } // revbin [0, 4, 2, 6, 1, 5, 3, 7] undo @@ -473,6 +682,293 @@ void finish_tabMul8_fft8(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u SWAP(u[3], u[6]); } + +void OVERLOAD fft_common(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, u32 lowMe, int callnum) { + + // This line mimics shufl -- partition lds for variant 2 + local T2* partitioned_lds = LDSptr(lds, numWG); + +// Variant 0 uses broadcast instructions. Only available on AMD GPUs. + +#if VARIANT == 0 + +#if WG * RADIX > 1024 +#error VARIANT == 0 only supported for FFT size <= 1024 +#endif +#if !AMDGPU +#error VARIANT == 0 only supported by AMD GPUs +#endif + +// There is a slight difference between fft_WIDTH and fft_HEIGHT. Tail square computes the trig values +// to be broadcast, while fft_WIDTH does not. Compute the trig values now for fft_WIDTH, +#if DOING_WIDTH +#if RADIX == 8 + w = fancyTrig_N(ND / (WG * RADIX) * lowMe); +#else + w = slowTrig_N(ND / (WG * RADIX) * lowMe, ND / RADIX); +#endif +#endif + + for (u32 s = 1; s < WG; s *= RADIX) { + fft_RADIX(u); + w = bcast(w, s); + chainMul(u, w); + shufl(lds, u, s, numWG, lowMe); + } + fft_RADIX(u); + +// Variant 2 uses more FMA instructions than the original FFT code. +// The tabMul after fft8 only does a partial complex multiply, saving a mul-by-cosine for the next fft8 using FMA instructions. +// To maximize FMA opportunities we precompute trig values as cosine and sine/cosine rather than cosine and sine. +// The downside is sine/cosine cannot be computed with chained multiplies. + +// Variant 2 code for SIZE=256, RADIX=4 +#elif WG == 64 && RADIX == 4 && VARIANT == 2 + + T preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul4_trig(trig, preloads, 1, numWG, lowMe); + + // Do first fft4, partial tabMul, and shufl. + fft4(u); + partial_tabMul4(partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 1, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 4, numWG, lowMe); + shufl(lds, u, 4, numWG, lowMe); + + // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 4, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 16, numWG, lowMe); + shufl(lds, u, 16, numWG, lowMe); + + // Finish third tabMul and perform final fft4. + finish_tabMul4_fft4(trig, preloads, u, 16, numWG, lowMe, 1); + +// Variant 2 code for SIZE=512, RADIX=8 +#elif WG == 64 && RADIX == 8 && VARIANT == 2 + + T preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*8 + SAVE_ONE_MUL*2*WG*8; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul8_trig(trig, preloads, 1, numWG, lowMe); + + // Do first fft8, partial tabMul, and shufl. + fft8(u); + partial_tabMul8(partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. + finish_tabMul8_fft8(trig, preloads, u, 1, numWG, lowMe, SAVE_ONE_MUL); // We'd rather set save_one_more_mul to 1 + partial_tabMul8(partitioned_lds, trig, preloads, u, 8, numWG, lowMe); + shufl(lds, u, 8, numWG, lowMe); + + // Finish second tabMul and perform final fft8. + finish_tabMul8_fft8(trig, preloads, u, 8, numWG, lowMe, SAVE_ONE_MUL); // We'd rather set save_one_more_mul to 1 + +// Variant 2 code for SIZE=1024, RADIX=4 +#elif WG == 256 && RADIX == 4 && VARIANT == 2 + + T preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul4_trig(trig, preloads, 1, numWG, lowMe); + + // Do first fft4, partial tabMul, and shufl. + fft4(u); + partial_tabMul4(partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 1, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 4, numWG, lowMe); + shufl(lds, u, 4, numWG, lowMe); + + // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 4, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 16, numWG, lowMe); + shufl(lds, u, 16, numWG, lowMe); + + // Finish the third tabMul and perform fourth fft4. Do fourth partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 16, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 64, numWG, lowMe); + shufl(lds, u, 64, numWG, lowMe); + + // Finish fourth tabMul and perform final fft4. + finish_tabMul4_fft4(trig, preloads, u, 64, numWG, lowMe, 1); + +// Custom code for SIZE=4K, RADIX=8 +#elif WG == 512 && RADIX == 8 && VARIANT == 2 + + T preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*8; // Skip past old FFT_width trig values to the !save_one_more_mul trig values + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul8_trig(trig, preloads, 1, numWG, lowMe); + + // Do first fft8, partial tabMul, and shufl. + fft8(u); + partial_tabMul8(partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. + finish_tabMul8_fft8(trig, preloads, u, 1, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + partial_tabMul8(partitioned_lds, trig, preloads, u, 8, numWG, lowMe); + shufl(lds, u, 8, numWG, lowMe); + + // Finish the second tabMul and perform third fft8. Do third partial tabMul and shufl. + finish_tabMul8_fft8(trig, preloads, u, 8, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + partial_tabMul8(partitioned_lds, trig, preloads, u, 64, numWG, lowMe); + shufl(lds, u, 64, numWG, lowMe); + + // Finish third tabMul and perform final fft8. + finish_tabMul8_fft8(trig, preloads, u, 64, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + + +// Custom code for SIZE=256, RADIX=8, threads=32. Performed as 4 * 8 * 8. Radix-8 allows fewer +// shufls and tabmuls than radix-4. Fewer instructions, but more registers. +// Uses only 32 threads which is fine on nVidia, lousy on radeon VII (use WMUL=2, TAIL_KERNELS=2). +// +// Details for memory layout, trig data, and shufls: +// Mem: 0 1... 31 +// 32 +// ... +// 196 +// 224 ... 255 +// Only do a radix-4 fft. Non-standard TABMUL: (64 3/4 cmuls in blocks of 1 duplicated trig values) +// trig powers are: 0*0 0*1 .. 0*31 +// 0*32 .. 0*63 +// 1*0 1*1 .. 1*31 +// 1*32 .. 1*63 +// 2*0 2*1 .. 2*31 +// 2*32 .. 2*63 +// 3*0 3*1 .. 3*31 +// 3*32 .. 3*63 total trig data (6*32*16=3KB) +// non-standard shufl out: +// 0 64 .. 192 1... 7... +// 8 +// 16 +// ... +// 48 +// 56 +// standard TABMUL: (8 7/8 cmuls in blocks of 4 duplicated trig values) +// trig powers are: 0000 0000 .. 0000*7 +// 0000 4444 .. 4444*7 +// 0000 8888 .. 8888*7 +// 0000 12 ... +// 0000 16 ... +// 0000 20 ... +// 0000 24 ... +// 0000 28 ... total trig data (7*8*16=896B) +// standard shufl out: +// 0 64 .. 192 8... 56... +// 1 +// 2 +// ... +// 6 +// 7 +// +// FP64 and FP32 could benefit by starting the next fft8 after the radix-4 tabmul (easy FMA opportunities). + +// Code for SIZE=256, RADIX=8 +#elif WG == 32 && NW == 8 + + fft8_4(u); + tabMul8_4a(trig, u, 1, lowMe); + shufl(lds, u, 1, 4, numWG, lowMe); + + fft8(u); + tabMul8_4b(trig, u, 4, lowMe); + shufl(lds, u, 4, numWG, lowMe); + + fft8(u); + +// Custom code for SIZE=1024, RADIX=8, threads=128. Performed as 8 * 8 * 2 * 8. Radix-8 allows fewer +// shufls and tabmuls than radix-4. Fewer instructions, but more registers. If we process 4 (or 8) +// independent width lines then we should be able to avoid the mul by w^0 in the next to last radix-8 step. +// +// Details for memory layout, trig data, and shufls: +// Mem: 0 1 ... 127 +// 128 +// 256 +// 384 +// 512 +// 640 +// 768 +// 896 ... 1023 +// standard TABMUL: (128 7/8 cmuls in blocks of 1 duplicated trig values) +// trig powers are: 0 0 0 .. 0*127 +// 0 1 2 .. 1*127 +// 0 2 4 .. 2*127 +// 0 3 6 .. 3*127 +// 0 4 8 .. 4*127 +// 0 5 10 .. 5*127 +// 0 6 12 .. 6*127 +// 0 7 14 .. 7*127 total trig data (7*128*16=14KB) +// standard shufl out: +// 0 128 .. 896 1... 15... +// 16 +// 32 +// 48 +// 64 +// 80 +// 96 +// 112 +// standard TABMUL: (16 7/8 cmuls in blocks of 8 duplicated trig values) +// trig powers are: 00000000 00000000 .. 00000000*15 +// 00000000 11111111 .. 11111111*15 +// 00000000 22222222 .. 22222222*15 +// 00000000 33333333 .. 33333333*15 +// 00000000 44444444 .. 44444444*15 +// 00000000 55555555 .. 55555555*15 +// 00000000 66666666 .. 66666666*15 +// 00000000 77777777 .. 77777777*15 total trig data (7*16*16=1.75KB) +// shufl out with an fft2: +// 0 128 .. 896 16 .. 112... 8... +// 1 +// 2 +// 3 +// 4 +// 5 +// 6 +// 7 + +// Code for SIZE=1024, RADIX=8 +#elif WG == 128 && RADIX == 8 + + fft8(u); + tabMul(trig, u, 1, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + fft8(u); + tabMul(trig, u, 8, lowMe); + shufl_and_fft2(lds, u, 8, numWG, lowMe); + + if (lowMe < WG / 2) fft8_16a(u); else fft8_16b(u); + +#else + + // Old / original version + +#if !UNROLL + __attribute__((opencl_unroll_hint(1))) +#endif + for (u32 s = 1; s < WG; s *= RADIX) { + fft_RADIX(u); + tabMul(trig, u, s, lowMe); + shufl(lds, u, s, numWG, lowMe); + } + fft_RADIX(u); + +#endif +} + #endif @@ -482,6 +978,16 @@ void finish_tabMul8_fft8(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u #if FFT_FP32 +void OVERLOAD fft_RADIX(F2 *u) { +#if RADIX == 4 + fft4(u); +#elif RADIX == 8 + fft8(u); +#else +#error RADIX +#endif +} + void OVERLOAD chainMul4(F2 *u, F2 w) { u[1] = cmul(u[1], w); @@ -493,9 +999,9 @@ void OVERLOAD chainMul4(F2 *u, F2 w) { u[3] = cmul(u[3], base); } -void OVERLOAD chainMul8(F2 *u, F2 w, u32 tailSquareBcast) { +void OVERLOAD chainMul8(F2 *u, F2 w) { u[1] = cmulFancy(u[1], w); - //GWBUG - see FP64 version for many possible optimizations + //GWBUG - see FP64 version for many possible optimizations F2 w2 = csqTrigFancy(w); u[2] = cmulFancy(u[2], w2); @@ -503,208 +1009,770 @@ void OVERLOAD chainMul8(F2 *u, F2 w, u32 tailSquareBcast) { u[3] = cmulFancy(u[3], w3); w3.x += 1; - F2 base = cmulFancy (w3, w); + F2 base = cmulFancy(w3, w); for (int i = 4; i < 8; ++i) { u[i] = cmul(u[i], base); base = cmulFancy(base, w); } } -void OVERLOAD chainMul(u32 len, F2 *u, F2 w, u32 tailSquareBcast) { +void OVERLOAD chainMul(F2 *u, F2 w) { // Do a length 4 chain mul - if (len == 4) chainMul4(u, w); + if (RADIX == 4) chainMul4(u, w); // Do a length 8 chain mul - if (len == 8) chainMul8(u, w, tailSquareBcast); -} - -void OVERLOAD shuflBigLDS(u32 WG, local F2 *lds, F2 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i]; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i] = lds[i * WG + me]; } -} - -void OVERLOAD shufl(u32 WG, local F2 *lds2, F2 *u, u32 n, u32 f) { //GWBUG - is shufl of int2 faster (BigLDS)? - u32 me = get_local_id(0); - local F* lds = (local F*) lds2; - - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].x; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + me]; } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].y; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + me]; } -} - -// Shufl two simultaneous FFT_HEIGHTs. Needed for tailSquared where u and v are computed simultaneously in different threads. -// NOTE: It is very important for this routine to use lds memory in coordination with reverseLine2 and unreverseLine2. -// Failure to do so would result in the need for more bar() calls. Specifically, the u values are stored in the upper half -// of lds memory (first SMALL_HEIGHT GF31 values). The v values are stored in the lower half of lds memory (next SMALL_HEIGHT GF31 values). -void OVERLOAD shufl2(u32 WG, local F2 *lds2, F2 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - - // Partition lds memory into upper and lower halves - assert(WG == G_H); - - // Accessing lds memory as F is faster than F2 accesses //GWBUG??? - local F* lds = ((local F*) lds2) + (me / WG) * SMALL_HEIGHT; - - me = me % WG; - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].x; } - bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + me]; } - bar(WG); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].y; } - bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + me]; } + if (RADIX == 8) chainMul8(u, w); } -void OVERLOAD tabMul(u32 WG, TrigFP32 trig, F2 *u, u32 n, u32 f, u32 me) { +void OVERLOAD tabMul(TrigFP32 trig, F2 *u, u32 f, u32 me) { u32 p = me & ~(f - 1); // This code uses chained complex multiplies which could be faster on GPUs with great mul throughput or poor memory bandwidth or caching. if (TABMUL_CHAIN32) { - chainMul (n, u, trig[p], 0); + chainMul(u, TFLOAD(&trig[p])); return; } // Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. if (!TABMUL_CHAIN32) { - if (n >= 8) { - u[1] = cmulFancy(u[1], trig[p]); + if (RADIX >= 8) { + u[1] = cmulFancy(u[1], TFLOAD(&trig[p])); } else { - u[1] = cmul(u[1], trig[p]); + u[1] = cmul(u[1], TFLOAD(&trig[p])); } - for (u32 i = 2; i < n; ++i) { - u[i] = cmul(u[i], trig[(i-1)*WG + p]); + for (u32 i = 2; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*WG + p])); } return; } } -#endif - +// Tabmul after doing an fft4 when RADIX=8. See the SIZE=256 code for example memory and trig layout. +void OVERLOAD tabMul8_4a(TrigFP32 trig, F2 *u, u32 f, u32 me) { -/**************************************************************************/ -/* Similar to above, but for an NTT based on GF(M31^2) */ -/**************************************************************************/ + if (f == 1) { // fft8_4 is performed first + u32 p = me; -#if NTT_GF31 +// This code uses chained complex multiplies which could be faster on GPUs with great SP throughput or poor memory bandwidth or caching. +// This ought to be the least accurate version of Tabmul. In practice, this is just as accurate as reading precomputed values from memory. +// Perform two length=4 chain muls. + + if (TABMUL_CHAIN32) { + F2 w = TFLOAD(&trig[p]); + F2 w2 = TFLOAD(&trig[WG + p]); + u[2] = cmul(u[2], w); + u[3] = cmul(u[3], w2); + F2 base = csqTrig(w); + F2 base2 = csqTrig(w2); + u[4] = cmul(u[4], base); + u[5] = cmul(u[5], base2); + base = ccubeTrig(base, w); + base2 = ccubeTrig(base2, w2); + u[6] = cmul(u[6], base); + u[7] = cmul(u[7], base2); + } -void OVERLOAD chainMul4(GF31 *u, GF31 w) { - u[1] = cmul(u[1], w); +// Theoretically, maximum accuracy. Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. - GF31 base = csqTrig(w); - u[2] = cmul(u[2], base); + if (!TABMUL_CHAIN32) { + for (u32 i = 2; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-2)*WG + p])); + } + } + } - base = ccubeTrig(base, w); - u[3] = cmul(u[3], base); -} + else { // fft8_4 is performed after an initial fft8 -void OVERLOAD chainMul8(GF31 *u, GF31 w) { - u[1] = cmul(u[1], w); +// This code uses chained complex multiplies which could be faster on GPUs with great SP throughput or poor memory bandwidth or caching. +// This ought to be the least accurate version of Tabmul. In practice, this is just as accurate as reading precomputed values from memory. +// Perform two length=4 chain muls. + + u32 p = me / 8; // Generate index into condensed trig table that does not have duplicated trig values + trig += 7 * WG; // Skip over the trig values used in the first tabmul + if (TABMUL_CHAIN32) { + F2 w = TFLOAD(&trig[p]); + F2 w2 = TFLOAD(&trig[WG/8 + p]); + u[2] = cmul(u[2], w); + u[3] = cmul(u[3], w2); + F2 base = csqTrig(w); + F2 base2 = csqTrig(w2); + u[4] = cmul(u[4], base); + u[5] = cmul(u[5], base2); + base = ccubeTrig(base, w); + base2 = ccubeTrig(base2, w2); + u[6] = cmul(u[6], base); + u[7] = cmul(u[7], base2); + } - GF31 base = csqTrig(w); - u[2] = cmul(u[2], base); +// Theoretically, maximum accuracy. Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. - base = ccubeTrig(base, w); - for (int i = 3; i < 8; ++i) { - u[i] = cmul(u[i], base); - base = cmul(base, w); + if (!TABMUL_CHAIN32) { + for (u32 i = 2; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-2)*WG/8 + p])); + } + } } } -void OVERLOAD chainMul(u32 len, GF31 *u, GF31 w) { - // Do a length 4 chain mul - if (len == 4) chainMul4(u, w); - // Do a length 8 chain mul - if (len == 8) chainMul8(u, w); -} +// Later tabmuls after starting with an fft4 when RADIX=8. See the SIZE=256 code for example memory and trig layout. +void OVERLOAD tabMul8_4b(TrigFP32 trig, F2 *u, u32 f, u32 me) { -void OVERLOAD shuflBigLDS(u32 WG, local GF31 *lds, GF31 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); +// This code uses chained complex multiplies which could be faster on GPUs with great SP throughput or poor memory bandwidth or caching. +// This ought to be the least accurate version of Tabmul. In practice, this is just as accurate as reading precomputed values from memory. +// Apparently, chained Fancy muls at n=8 lengths are very accurate. - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i]; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i] = lds[i * WG + me]; } -} + if (TABMUL_CHAIN32) { + u32 p = me & ~(f - 1); + F2 w = TFLOAD(&trig[p]); + +// u[1] = cmulFancy(u[1], w); // GW: - this should use Fancy, but tabmul8_4a does not and it could for half of the data +// T2 w2 = csqTrigFancy(w); +// u[2] = cmulFancy(u[2], w2); +// T2 w3 = ccubeTrigFancy(w2, w); +// u[3] = cmulFancy(u[3], w3); +// w3.x += 1; +// T2 base = cmulFancy(w3, w); +// for (int i = 4; i < 8; ++i) { +// u[i] = cmul(u[i], base); +// base = cmulFancy(base, w); +// } + + u[1] = cmul(u[1], w); // GW: - this should use Fancy, but tabmul8_4a does not and it could for half of the data + F2 w2 = csqTrig(w); + u[2] = cmul(u[2], w2); + F2 w3 = ccubeTrig(w2, w); + u[3] = cmul(u[3], w3); + F2 base = cmul(w3, w); + for (int i = 4; i < 8; ++i) { + u[i] = cmul(u[i], base); + base = cmul(base, w); + } + } -void OVERLOAD shufl(u32 WG, local GF31 *lds2, GF31 *u, u32 n, u32 f) { //GWBUG - is shufl of int2 faster (BigLDS)? - u32 me = get_local_id(0); - local Z31* lds = (local Z31*) lds2; +// Theoretically, maximum accuracy. Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. + + if (!TABMUL_CHAIN32) { + u32 p = (me/4) & ~(f/4 - 1); // Generate index into condensed trig table that does not have duplicated trig values + trig += 6 * WG; // Skip over the trig values used in tabmul8_4a - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].x; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + me]; } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].y; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + me]; } +//GW: Can any of these be Fancy? Yes, u[1] and u[2] + for (u32 i = 1; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*(WG/4) + p])); + } + } } -// Shufl two simultaneous FFT_HEIGHTs. Needed for tailSquared where u and v are computed simultaneously in different threads. -// NOTE: It is very important for this routine to use lds memory in coordination with reverseLine2 and unreverseLine2. -// Failure to do so would result in the need for more bar() calls. Specifically, the u values are stored in the upper half -// of lds memory (first SMALL_HEIGHT GF31 values). The v values are stored in the lower half of lds memory (next SMALL_HEIGHT GF31 values). -void OVERLOAD shufl2(u32 WG, local GF31 *lds2, GF31 *u, u32 n, u32 f) { - u32 me = get_local_id(0); +//************************************************************************************ +// New fft WIDTH and HEIGHT macros to support radix-4 FFTs with more FMA instructions +//************************************************************************************ + +// Some OpenCL compilers are having trouble with fma on floats. Specifically, line "X2_via_FMA(u[3], preloads[7], u[7]); u[7] = mul_3t8_delayed(u[7]);". +// Since we're not enabling FP32 variant 2 by default, don't include these "more FMA" routines. - // Partition lds memory into upper and lower halves - assert(WG == G_H); +#if ENABLE_FP32_VARIANT_2 - // Accessing lds memory as Z31s is faster than GF31 accesses //GWBUG??? - local Z31* lds = ((local Z31*) lds2) + (me / WG) * SMALL_HEIGHT; +// Copy of macro from fft4 and fft8 with FMAs added +#define X2_via_FMA(a, c, b) { F2 t = a; a = fma(c, b, t); b = fma(-c, b, t); } - me = me % WG; - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); +// Preload trig values for the first partial tabMul. We load the sine/cosine values early so that F64 ops can hide the read latency. +void preload_tabMul4_trig(TrigFP32 trig, F *preloads, u32 f, u32 numWG, u32 me) { + TrigSingleFP32 trig1 = (TrigSingleFP32) trig; - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].x; } - bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + me]; } - bar(WG); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].y; } - bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + me]; } + // Read 3 lines of sine/cosine values for the first fft4. Read two of the lines as a pair as AMD likes T2 global memory reads + TrigFP32 trig2 = (TrigFP32) trig1; + F2 sine_over_cosines = TFLOAD(&trig2[me]); + preloads[0] = sine_over_cosines.x; + preloads[1] = sine_over_cosines.y; + // Read 3rd line + preloads[2] = TFLOAD(&trig1[2*WG + me]); } -void OVERLOAD tabMul(u32 WG, TrigGF31 trig, GF31 *u, u32 n, u32 f, u32 me) { - u32 p = me & ~(f - 1); - -// This code uses chained complex multiplies which could be faster on GPUs with great mul throughput or poor memory bandwidth or caching. +// Do a partial tabMul. Save the mul-by-cosine for later FMA instructions. +void partial_tabMul4(local F2 *lds, TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u32 me) { + local F *lds1 = (local F *) lds; + TrigSingleFP32 trig1 = (TrigSingleFP32) trig; + trig1 += 4*WG; // Skip past sine_over_cosine values - if (TABMUL_CHAIN31) { - chainMul (n, u, trig[p]); - return; + // Use LDS memory to distribute preloaded trig values. + if (f > 1) { + bar(WG); + lds1[me] = preloads[4]; // Preloaded sine/cosine values + lds1[WG+me] = preloads[5]; // Preloaded cosine values } -// Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. + // Apply sine/cosines + bar(WG); + for (u32 i = 1; i < 4; ++i) { + F sine_over_cosine; + if (f == 1) sine_over_cosine = preloads[i-1]; + else sine_over_cosine = lds1[i*(WG/4) + (me/f)*(f/4)]; + u[i] = partial_cmul(u[i], sine_over_cosine); + } - if (!TABMUL_CHAIN31) { - for (u32 i = 1; i < n; ++i) { - u[i] = cmul(u[i], trig[(i-1)*WG + p]); + // Preload cosines for finishing first tabMul (done after using up preloaded sine/cosine values). Hopefully, shufl will hide the latency. + if (f == 1) { + // Read pairs of lines to make AMD happy with T2 global memory loads + for (u32 i = 0; i < 4; i += 2) { + TrigFP32 trig2 = (TrigFP32) (trig1 + i*WG); + F2 cosines = TFLOAD(&trig2[me]); + preloads[i] = cosines.x; + preloads[i+1] = cosines.y; } - return; + } + else { + // Load cosine1, cosine2, cosine3/cosine1 + if (f < WG/4) preloads[0] = lds1[WG + ((me/f) & 3) * WG/4 + (0 * WG + me)/(4*f) * f/4]; + preloads[2] = lds1[WG + ((me/f) & 3) * WG/4 + (2 * WG + me)/(4*f) * f/4]; + preloads[3] = lds1[WG + ((me/f) & 3) * WG/4 + (3 * WG + me)/(4*f) * f/4]; + preloads[1] = lds1[WG + ((me/f) & 3) * WG/4 + (1 * WG + me)/(4*f) * f/4]; } } -#endif - +// Finish off a partial tabMul while doing next fft4 making more use of FMA. +void finish_tabMul4_fft4(TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u32 me, u32 save_one_more_mul) { + TrigSingleFP32 trig1 = (TrigSingleFP32) trig; + + // + // Mimic a traditional fft4 but use FMA instructions to apply the cosine multiplies. + // + + // Apply cosine0 to u[0] + if (f < WG/4) u[0] = u[0] * preloads[0]; + + // Apply cosine2, cosine3/cosine1 to u[2] and u[3] using FMA + X2_via_FMA(u[0], preloads[2], u[2]); + X2_via_FMA(u[1], preloads[3], u[3]); u[3] = mul_t4(u[3]); + + // Preload one line of sine/cosines and one line of cosines for later tabMuls. We'll later broadcast these values as needed using LDS. + if (f == 1) { + preloads[4] = TFLOAD(&trig1[3*WG + me]); // Sine/cosines for later tabMuls + preloads[5] = TFLOAD(&trig1[4*WG + 4*WG + me]); // Cosines for later tabMuls + } + + // Do the last level of fft4 applying cosine1 + X2_via_FMA(u[0], preloads[1], u[1]); + X2_via_FMA(u[2], preloads[1], u[3]); + + // revbin [0, 2, 1, 3] undo + SWAP(u[1], u[2]); +} + +//************************************************************************************ +// New fft WIDTH and HEIGHT macros to support radix-8 FFTs with more FMA instructions +//************************************************************************************ + +// Preload trig values for the first partial tabMul. We load the sine/cosine values early so that F64 ops can hide the read latency. +void preload_tabMul8_trig(TrigFP32 trig, F *preloads, u32 f, u32 numWG, u32 me) { + TrigSingleFP32 trig1 = (TrigSingleFP32) trig; + + // Read 7 lines of sine/cosine values for the first fft8. Read six of the lines as pairs as AMD likes T2 global memory reads + for (u32 i = 1; i < 7; i += 2) { + TrigFP32 trig2 = (TrigFP32) (trig1 + (i-1)*WG); + F2 sine_over_cosines = TFLOAD(&trig2[me]); + preloads[i-1] = sine_over_cosines.x; + preloads[i] = sine_over_cosines.y; + } + // Read 7th line + preloads[6] = TFLOAD(&trig1[6*WG + me]); +} + +// Do a partial tabMul. Save the mul-by-cosine for later FMA instructions. +void partial_tabMul8(local F2 *lds, TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u32 me) { + local F *lds1 = (local F *) lds; + TrigSingleFP32 trig1 = (TrigSingleFP32) trig; + trig1 += 8*WG; // Skip past sine_over_cosine values + + // Use LDS memory to distribute preloaded trig values. + if (f > 1) { + bar(WG); + lds1[me] = preloads[8]; // Preloaded sine/cosine values + lds1[WG+me] = preloads[9]; // Preloaded cosine values + } + + // Apply sine/cosines + bar(WG); + for (u32 i = 1; i < 8; ++i) { + F sine_over_cosine; + if (f == 1) sine_over_cosine = preloads[i-1]; + else sine_over_cosine = lds1[i*(WG/8) + (me/f)*(f/8)]; + u[i] = partial_cmul(u[i], sine_over_cosine); + } + + // Preload cosines for finishing first tabMul (done after using up preloaded sine/cosine values). Hopefully, shufl will hide the latency. + if (f == 1) { + // Read pairs of lines to make AMD happy with T2 global memory loads + for (u32 i = 0; i < 8; i += 2) { + TrigFP32 trig2 = (TrigFP32) (trig1 + i*WG); + F2 cosines = TFLOAD(&trig2[me]); + preloads[i] = cosines.x; + preloads[i+1] = cosines.y; + } + } + else { + // Load cosine4, cosine5/cosine1, cosine6/cosine2, cosine7/cosine3, cosine2, cosine3/cosine1, cosine1 + // Load them in the order they will be used, though it probably won't matter. + if (f < WG/8) preloads[0] = lds1[WG + ((me/f) & 7) * WG/8 + (0 * WG + me)/(8*f) * f/8]; + preloads[1] = lds1[WG + ((me/f) & 7) * WG/8 + (1 * WG + me)/(8*f) * f/8]; + preloads[4] = lds1[WG + ((me/f) & 7) * WG/8 + (4 * WG + me)/(8*f) * f/8]; + preloads[5] = lds1[WG + ((me/f) & 7) * WG/8 + (5 * WG + me)/(8*f) * f/8]; + preloads[6] = lds1[WG + ((me/f) & 7) * WG/8 + (6 * WG + me)/(8*f) * f/8]; + preloads[7] = lds1[WG + ((me/f) & 7) * WG/8 + (7 * WG + me)/(8*f) * f/8]; + preloads[2] = lds1[WG + ((me/f) & 7) * WG/8 + (2 * WG + me)/(8*f) * f/8]; + preloads[3] = lds1[WG + ((me/f) & 7) * WG/8 + (3 * WG + me)/(8*f) * f/8]; + } +} + +// Finish off a partial tabMul while doing next fft8 making more use of FMA. +void finish_tabMul8_fft8(TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u32 me, u32 save_one_more_mul) { + TrigSingleFP32 trig1 = (TrigSingleFP32) trig; + + // + // Mimic a traditional fft8 but use FMA instructions to apply the cosine multiplies. + // + + // Apply cosine0 to u[0] + if (f < WG/8) u[0] = u[0] * preloads[0]; + + if (save_one_more_mul) { // This should always be the best option. ROCm optimizer is doing something weird in fft_WIDTH case. + + // Apply cosine4, cosine5/cosine1, cosine6/cosine2, cosine7/cosine3 to u[4] through u[7] using FMA + X2_via_FMA(u[0], preloads[4], u[4]); + X2_via_FMA(u[1], preloads[5], u[5]); u[5] = mul_t8_delayed(u[5]); + X2_via_FMA(u[2], preloads[6], u[6]); u[6] = mul_t4(u[6]); + X2_via_FMA(u[3], preloads[7], u[7]); u[7] = mul_3t8_delayed(u[7]); + + // Preload one line of sine/cosines and one line of cosines for second tabMul. We'll later broadcast these values as needed using LDS. + if (f == 1) { + preloads[8] = TFLOAD(&trig1[7*WG + me]); // Sine/cosines for second tabMul + preloads[9] = TFLOAD(&trig1[8*WG + 8*WG + me]); // Cosines for second tabMul + } + + // Do the fft4Core and fft4CoreSpecial applying cosine2, cosine3/cosine1 + X2_via_FMA(u[0], preloads[2], u[2]); + X2_via_FMA(u[4], preloads[2], u[6]); + X2_via_FMA(u[1], preloads[3], u[3]); u[3] = mul_t4(u[3]); + X2_via_FMA(u[5], preloads[3], u[7]); u[7] = mul_t4(u[7]); + + // Do last level of fft8 applying cosine1 +//TODO: Save this MUL by SQRT(1/2) by pre-computing cosine1*SQRTHALF + F cosine1_SQRT1_2 = preloads[1] * (float) M_SQRT1_2; + X2_via_FMA(u[0], preloads[1], u[1]); + X2_via_FMA(u[2], preloads[1], u[3]); + X2_via_FMA(u[4], cosine1_SQRT1_2, u[5]); + X2_via_FMA(u[6], cosine1_SQRT1_2, u[7]); + + } else { + + // Apply cosine to u[1] + u[1] = u[1] * preloads[1]; + + // Apply cosine4, cosine5, cosine6/cosine2, cosine7/cosine3 to u[4] through u[7] using FMA + X2_via_FMA(u[0], preloads[4], u[4]); + X2_via_FMA(u[1], preloads[5], u[5]); u[5] = mul_t8_delayed(u[5]); + X2_via_FMA(u[2], preloads[6], u[6]); u[6] = mul_t4(u[6]); + X2_via_FMA(u[3], preloads[7], u[7]); u[7] = mul_3t8_delayed(u[7]); + + // Preload one line of sine/cosines and one line of cosines for second tabMul. We'll later broadcast these values as needed using LDS. + if (f == 1) { + preloads[8] = TFLOAD(&trig1[7*WG + me]); // Sine/cosines for second tabMul + preloads[9] = TFLOAD(&trig1[8*WG + 8*WG + me]); // Cosines for second tabMul + } + + // Do the fft4Core and fft4CoreSpecial applying cosine2, cosine3 + X2_via_FMA(u[0], preloads[2], u[2]); + X2_via_FMA(u[4], preloads[2], u[6]); + X2_via_FMA(u[1], preloads[3], u[3]); u[3] = mul_t4(u[3]); + X2_via_FMA(u[5], preloads[3], u[7]); u[7] = mul_t4(u[7]); + + // Do last level of fft8 + X2(u[0], u[1]); + X2(u[2], u[3]); + X2ad(u[4], u[5], M_SQRT1_2); + X2ad(u[6], u[7], M_SQRT1_2); + } + + // revbin [0, 4, 2, 6, 1, 5, 3, 7] undo + SWAP(u[1], u[4]); + SWAP(u[3], u[6]); +} + +#endif + +// Variant 2 code uses more FMA instructions than the original fft version. +// The tabMul after fft8 only does a partial complex multiply, saving a mul-by-cosine for the next fft8 using FMA instructions. +// To maximize FMA opportunities we precompute trig values as cosine and sine/cosine rather than cosine and sine. +// The downside is sine/cosine cannot be computed with chained multiplies. + +void OVERLOAD fft_common(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, u32 lowMe, int callnum) { + + // This line mimics shufl -- partition lds + local F2* partitioned_lds = LDSptr(lds, numWG); + +// Variant 2 code for SIZE=256, RADIX=4 +#if ENABLE_FP32_VARIANT_2 && WG == 64 && RADIX == 4 && VARIANT == 2 + + F preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul4_trig(trig, preloads, 1, numWG, lowMe); + + // Do first fft4, partial tabMul, and shufl. + fft4(u); + partial_tabMul4(partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 1, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 4, numWG, lowMe); + shufl(lds, u, 4, numWG, lowMe); + + // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 4, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 16, numWG, lowMe); + shufl(lds, u, 16, numWG, lowMe); + + // Finish third tabMul and perform final fft4. + finish_tabMul4_fft4(trig, preloads, u, 16, numWG, lowMe, 1); + +// Variant 2 code for SIZE=512, RADIX=8 +#elif ENABLE_FP32_VARIANT_2 && WG == 64 && RADIX == 8 && VARIANT == 2 + + F preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*8 + SAVE_ONE_MUL*2*WG*8; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul8_trig(trig, preloads, 1, numWG, lowMe); + + // Do first fft8, partial tabMul, and shufl. + fft8(u); + partial_tabMul8(partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. + finish_tabMul8_fft8(trig, preloads, u, 1, numWG, lowMe, SAVE_ONE_MUL); + partial_tabMul8(partitioned_lds, trig, preloads, u, 8, numWG, lowMe); + shufl(lds, u, 8, numWG, lowMe); + + // Finish second tabMul and perform final fft8. + finish_tabMul8_fft8(trig, preloads, u, 8, numWG, lowMe, SAVE_ONE_MUL); + +// Variant 2 code for SIZE=1024, RADIX=4 +#elif ENABLE_FP32_VARIANT_2 && WG == 256 && RADIX == 4 && VARIANT == 2 + + F preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul4_trig(trig, preloads, 1, numWG, lowMe); + + // Do first fft4, partial tabMul, and shufl. + fft4(u); + partial_tabMul4(partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 1, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 4, numWG, lowMe); + shufl(lds, u, 4, numWG, lowMe); + + // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 4, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 16, numWG, lowMe); + shufl(lds, u, 16, numWG, lowMe); + + // Finish the third tabMul and perform fourth fft4. Do fourth partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 16, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 64, numWG, lowMe); + shufl(lds, u, 64, numWG, lowMe); + + // Finish fourth tabMul and perform final fft4. + finish_tabMul4_fft4(trig, preloads, u, 64, numWG, lowMe, 1); + +// Variant 2 code for SIZE=4K, RADIX=8 +#elif ENABLE_FP32_VARIANT_2 && WG == 512 && RADIX == 8 && VARIANT == 2 + + F preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*8; // Skip past old FFT_width trig values to the !save_one_more_mul trig values + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul8_trig(trig, preloads, 1, numWG, lowMe); + + // Do first fft8, partial tabMul, and shufl. + fft8(u); + partial_tabMul8(partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. + finish_tabMul8_fft8(trig, preloads, u, 1, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + partial_tabMul8(partitioned_lds, trig, preloads, u, 8, numWG, lowMe); + shufl(lds, u, 8, numWG, lowMe); + + // Finish the second tabMul and perform third fft8. Do third partial tabMul and shufl. + finish_tabMul8_fft8(trig, preloads, u, 8, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + partial_tabMul8(partitioned_lds, trig, preloads, u, 64, numWG, lowMe); + shufl(lds, u, 64, numWG, lowMe); + + // Finish third tabMul and perform final fft8. + finish_tabMul8_fft8(trig, preloads, u, 64, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + +// Code for SIZE=256, RADIX=8 +#elif WG == 32 && NW == 8 + + fft8_4(u); + tabMul8_4a(trig, u, 1, lowMe); + shufl(lds, u, 1, 4, numWG, lowMe); + + fft8(u); + tabMul8_4b(trig, u, 4, lowMe); + shufl(lds, u, 4, numWG, lowMe); + + fft8(u); + +// Code for SIZE=1024, RADIX=8 +#elif WG == 128 && RADIX == 8 + + fft8(u); + tabMul(trig, u, 1, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + fft8(u); + tabMul(trig, u, 8, lowMe); + shufl_and_fft2(lds, u, 8, numWG, lowMe); + + if (lowMe < WG / 2) fft8_16a(u); else fft8_16b(u); + +#else + + // Old / original version + +#if !UNROLL + __attribute__((opencl_unroll_hint(1))) +#endif + for (u32 s = 1; s < WG; s *= RADIX) { + fft_RADIX(u); + tabMul(trig, u, s, lowMe); + shufl(lds, u, s, numWG, lowMe); + } + fft_RADIX(u); + +#endif +} + +#endif + + +/**************************************************************************/ +/* Similar to above, but for an NTT based on GF(M31^2) */ +/**************************************************************************/ + +#if NTT_GF31 + +void OVERLOAD fft_RADIX(GF31 *u) { +#if RADIX == 4 + fft4(u); +#elif RADIX == 8 + fft8(u); +#else +#error RADIX +#endif +} + +void OVERLOAD chainMul4(GF31 *u, GF31 w) { + u[1] = cmul(u[1], w); + + GF31 base = csqTrig(w); + u[2] = cmul(u[2], base); + + base = ccubeTrig(base, w); + u[3] = cmul(u[3], base); +} + +void OVERLOAD chainMul8(GF31 *u, GF31 w) { + u[1] = cmul(u[1], w); + + GF31 base = csqTrig(w); + u[2] = cmul(u[2], base); + + base = ccubeTrig(base, w); + for (int i = 3; i < 8; ++i) { + u[i] = cmul(u[i], base); + base = cmul(base, w); + } +} + +void OVERLOAD chainMul(GF31 *u, GF31 w) { + // Do a length 4 chain mul + if (RADIX == 4) chainMul4(u, w); + // Do a length 8 chain mul + if (RADIX == 8) chainMul8(u, w); +} + +void OVERLOAD tabMul(TrigGF31 trig, GF31 *u, u32 f, u32 me) { + u32 p = me & ~(f - 1); + +// This code uses chained complex multiplies which could be faster on GPUs with great mul throughput or poor memory bandwidth or caching. + + if (TABMUL_CHAIN31) { + chainMul(u, TFLOAD(&trig[p])); + return; + } + +// Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. + + if (!TABMUL_CHAIN31) { + for (u32 i = 1; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*WG + p])); + } + return; + } +} + +// Tabmul after doing an fft4 when RADIX=8. See the SIZE=256 code for example memory and trig layout. +void OVERLOAD tabMul8_4a(TrigGF31 trig, GF31 *u, u32 f, u32 me) { + + if (f == 1) { // fft8_4 is performed first + u32 p = me; + +// This code uses chained complex multiplies which could be faster on GPUs with great throughput or poor memory bandwidth or caching. +// Perform two length=4 chain muls. + + if (TABMUL_CHAIN31) { + GF31 w = TFLOAD(&trig[p]); + GF31 w2 = TFLOAD(&trig[WG + p]); + u[2] = cmul(u[2], w); + u[3] = cmul(u[3], w2); + GF31 base = csqTrig(w); + GF31 base2 = csqTrig(w2); + u[4] = cmul(u[4], base); + u[5] = cmul(u[5], base2); + base = ccubeTrig(base, w); + base2 = ccubeTrig(base2, w2); + u[6] = cmul(u[6], base); + u[7] = cmul(u[7], base2); + } + +// Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. + + if (!TABMUL_CHAIN31) { + for (u32 i = 2; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-2)*WG + p])); + } + } + } + + else { // fft8_4 is performed after an initial fft8 + +// This code uses chained complex multiplies which could be faster on GPUs with great throughput or poor memory bandwidth or caching. +// Perform two length=4 chain muls. + + u32 p = me / 8; // Generate index into condensed trig table that does not have duplicated trig values + trig += 7 * WG; // Skip over the trig values used in the first tabmul + if (TABMUL_CHAIN31) { + GF31 w = TFLOAD(&trig[p]); + GF31 w2 = TFLOAD(&trig[WG/8 + p]); + u[2] = cmul(u[2], w); + u[3] = cmul(u[3], w2); + GF31 base = csqTrig(w); + GF31 base2 = csqTrig(w2); + u[4] = cmul(u[4], base); + u[5] = cmul(u[5], base2); + base = ccubeTrig(base, w); + base2 = ccubeTrig(base2, w2); + u[6] = cmul(u[6], base); + u[7] = cmul(u[7], base2); + } + +// Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. + + if (!TABMUL_CHAIN31) { + for (u32 i = 2; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-2)*WG/8 + p])); + } + } + } +} + +// Later tabmuls after starting with an fft4 when RADIX=8. See the SIZE=256 code for example memory and trig layout. +void OVERLOAD tabMul8_4b(TrigGF31 trig, GF31 *u, u32 f, u32 me) { + +// This code uses chained complex multiplies which could be faster on GPUs with great throughput or poor memory bandwidth or caching. + + if (TABMUL_CHAIN31) { + u32 p = me & ~(f - 1); + GF31 w = TFLOAD(&trig[p]); + + u[1] = cmul(u[1], w); + GF31 w2 = csqTrig(w); + u[2] = cmul(u[2], w2); + GF31 w3 = ccubeTrig(w2, w); + u[3] = cmul(u[3], w3); + GF31 base = cmul(w3, w); + for (int i = 4; i < 8; ++i) { + u[i] = cmul(u[i], base); + base = cmul(base, w); + } + } + +// Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. + + if (!TABMUL_CHAIN31) { + u32 p = (me/4) & ~(f/4 - 1); // Generate index into condensed trig table that does not have duplicated trig values + trig += 6 * WG; // Skip over the trig values used in tabmul8_4a + + for (u32 i = 1; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*(WG/4) + p])); + } + } +} + +void OVERLOAD fft_common(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, u32 lowMe) { + +// Code for SIZE=256, RADIX=8 +#if WG == 32 && NW == 8 + + fft8_4(u); + tabMul8_4a(trig, u, 1, lowMe); + shufl(lds, u, 1, 4, numWG, lowMe); + + fft8(u); + tabMul8_4b(trig, u, 4, lowMe); + shufl(lds, u, 4, numWG, lowMe); + + fft8(u); + +// Code for SIZE=1024, RADIX=8 +#elif WG == 128 && RADIX == 8 + + fft8(u); + tabMul(trig, u, 1, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + fft8(u); + tabMul(trig, u, 8, lowMe); + shufl_and_fft2(lds, u, 8, numWG, lowMe); + + if (lowMe < WG / 2) fft8_16a(u); else fft8_16b(u); + +#else + +#if !UNROLL + __attribute__((opencl_unroll_hint(1))) +#endif + for (u32 s = 1; s < WG; s *= RADIX) { + fft_RADIX(u); + tabMul(trig, u, s, lowMe); + shufl(lds, u, s, numWG, lowMe); + } + fft_RADIX(u); + +#endif + +} + +#endif + /**************************************************************************/ /* Similar to above, but for an NTT based on GF(M61^2) */ @@ -712,133 +1780,207 @@ void OVERLOAD tabMul(u32 WG, TrigGF31 trig, GF31 *u, u32 n, u32 f, u32 me) { #if NTT_GF61 +void OVERLOAD fft_RADIX(GF61 *u) { +#if RADIX == 4 + fft4(u); +#elif RADIX == 8 + fft8(u); +#else +#error RADIX +#endif +} + void OVERLOAD chainMul4(GF61 *u, GF61 w) { u[1] = cmul(u[1], w); GF61 base = csq(w); u[2] = cmul(u[2], base); - base = cmul(base, w); //GWBUG - see FP64 version for possible optimization + base = cmul(base, w); //GWBUG - see FP64 version for possible optimization u[3] = cmul(u[3], base); } -void OVERLOAD chainMul8(GF61 *u, GF61 w, u32 tailSquareBcast) { +void OVERLOAD chainMul8(GF61 *u, GF61 w) { u[1] = cmul(u[1], w); GF61 w2 = csq(w); u[2] = cmul(u[2], w2); - GF61 base = cmul (w2, w); //GWBUG - see FP64 version for many possible optimizations + GF61 base = cmul(w2, w); //GWBUG - see FP64 version for many possible optimizations for (int i = 3; i < 8; ++i) { u[i] = cmul(u[i], base); base = cmul(base, w); } } -void OVERLOAD chainMul(u32 len, GF61 *u, GF61 w, u32 tailSquareBcast) { +void OVERLOAD chainMul(GF61 *u, GF61 w) { // Do a length 4 chain mul - if (len == 4) chainMul4(u, w); + if (RADIX == 4) chainMul4(u, w); // Do a length 8 chain mul - if (len == 8) chainMul8(u, w, tailSquareBcast); -} - -void OVERLOAD shuflBigLDS(u32 WG, local GF61 *lds, GF61 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i]; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i] = lds[i * WG + me]; } -} - -void OVERLOAD shufl(u32 WG, local GF61 *lds2, GF61 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - local Z61* lds = (local Z61*) lds2; - - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].x; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + me]; } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].y; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + me]; } -} - -// Same as shufl but use ints instead of Z61s to reduce LDS memory requirements. -// Lower LDS requirements should let the optimizer use fewer VGPRs and increase occupancy for WIDTHs >= 1024. -// Alas, the increased occupancy does not offset extra code needed for shufl_int (the assembly -// code generated is not pretty). This might not be true for nVidia or future ROCm optimizers. -void OVERLOAD shufl_int(u32 WG, local GF61 *lds2, GF61 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - local int* lds = (local int*) lds2; - - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = as_int4(u[i]).x; } - bar(); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + me]; u[i] = as_ulong2(tmp); } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = as_int4(u[i]).y; } - bar(); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + me]; u[i] = as_ulong2(tmp); } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = as_int4(u[i]).z; } - bar(); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + me]; u[i] = as_ulong2(tmp); } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = as_int4(u[i]).w; } - bar(); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + me]; u[i] = as_ulong2(tmp); } - bar(); // I'm not sure why this barrier call is needed -} - -// Shufl two simultaneous FFT_HEIGHTs. Needed for tailSquared where u and v are computed simultaneously in different threads. -// NOTE: It is very important for this routine to use lds memory in coordination with reverseLine2 and unreverseLine2. -// Failure to do so would result in the need for more bar() calls. Specifically, the u values are stored in the upper half -// of lds memory (first SMALL_HEIGHT GF61 values). The v values are stored in the lower half of lds memory (next SMALL_HEIGHT GF61 values). -void OVERLOAD shufl2(u32 WG, local GF61 *lds2, GF61 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - - // Partition lds memory into upper and lower halves - assert(WG == G_H); - - // Accessing lds memory as Z61s is faster than GF61 accesses - local Z61* lds = ((local Z61*) lds2) + (me / WG) * SMALL_HEIGHT; - - me = me % WG; - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].x; } - bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + me]; } - bar(WG); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].y; } - bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + me]; } + if (RADIX == 8) chainMul8(u, w); } -void OVERLOAD tabMul(u32 WG, TrigGF61 trig, GF61 *u, u32 n, u32 f, u32 me) { +void OVERLOAD tabMul(TrigGF61 trig, GF61 *u, u32 f, u32 me) { u32 p = me & ~(f - 1); // This code uses chained complex multiplies which could be faster on GPUs with great mul throughput or poor memory bandwidth or caching. if (TABMUL_CHAIN61) { - chainMul (n, u, trig[p], 0); + chainMul(u, TFLOAD(&trig[p])); return; } // Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. if (!TABMUL_CHAIN61) { - for (u32 i = 1; i < n; ++i) { - u[i] = cmul(u[i], trig[(i-1)*WG + p]); + for (u32 i = 1; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*WG + p])); } return; } } +// Tabmul after doing an fft4 when RADIX=8. See the SIZE=256 code for example memory and trig layout. +void OVERLOAD tabMul8_4a(TrigGF61 trig, GF61 *u, u32 f, u32 me) { + + if (f == 1) { // fft8_4 is performed first + u32 p = me; + +// This code uses chained complex multiplies which could be faster on GPUs with great throughput or poor memory bandwidth or caching. +// Perform two length=4 chain muls. + + if (TABMUL_CHAIN61) { + GF61 w = TFLOAD(&trig[p]); + GF61 w2 = TFLOAD(&trig[WG + p]); + u[2] = cmul(u[2], w); + u[3] = cmul(u[3], w2); + GF61 base = csqTrig(w); + GF61 base2 = csqTrig(w2); + u[4] = cmul(u[4], base); + u[5] = cmul(u[5], base2); + base = ccubeTrig(base, w); + base2 = ccubeTrig(base2, w2); + u[6] = cmul(u[6], base); + u[7] = cmul(u[7], base2); + } + +// Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. + + if (!TABMUL_CHAIN61) { + for (u32 i = 2; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-2)*WG + p])); + } + } + } + + else { // fft8_4 is performed after an initial fft8 + +// This code uses chained complex multiplies which could be faster on GPUs with great throughput or poor memory bandwidth or caching. +// Perform two length=4 chain muls. + + u32 p = me / 8; // Generate index into condensed trig table that does not have duplicated trig values + trig += 7 * WG; // Skip over the trig values used in the first tabmul + if (TABMUL_CHAIN61) { + GF61 w = TFLOAD(&trig[p]); + GF61 w2 = TFLOAD(&trig[WG/8 + p]); + u[2] = cmul(u[2], w); + u[3] = cmul(u[3], w2); + GF61 base = csqTrig(w); + GF61 base2 = csqTrig(w2); + u[4] = cmul(u[4], base); + u[5] = cmul(u[5], base2); + base = ccubeTrig(base, w); + base2 = ccubeTrig(base2, w2); + u[6] = cmul(u[6], base); + u[7] = cmul(u[7], base2); + } + +// Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. + + if (!TABMUL_CHAIN61) { + for (u32 i = 2; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-2)*WG/8 + p])); + } + } + } +} + +// Later tabmuls after starting with an fft4 when RADIX=8. See the SIZE=256 code for example memory and trig layout. +void OVERLOAD tabMul8_4b(TrigGF61 trig, GF61 *u, u32 f, u32 me) { + +// This code uses chained complex multiplies which could be faster on GPUs with great throughput or poor memory bandwidth or caching. + + if (TABMUL_CHAIN61) { + u32 p = me & ~(f - 1); + GF61 w = TFLOAD(&trig[p]); + + u[1] = cmul(u[1], w); + GF61 w2 = csqTrig(w); + u[2] = cmul(u[2], w2); + GF61 w3 = ccubeTrig(w2, w); + u[3] = cmul(u[3], w3); + GF61 base = cmul(w3, w); + for (int i = 4; i < 8; ++i) { + u[i] = cmul(u[i], base); + base = cmul(base, w); + } + } + +// Theoretically, maximum accuracy. Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. +// Radeon VII loves this case, it is faster than the chainmul case. nVidia Titan V hates this case. + + if (!TABMUL_CHAIN61) { + u32 p = (me/4) & ~(f/4 - 1); // Generate index into condensed trig table that does not have duplicated trig values + trig += 6 * WG; // Skip over the trig values used in tabmul8_4a + + for (u32 i = 1; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*(WG/4) + p])); + } + } +} + +void OVERLOAD fft_common(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, u32 lowMe) { + +// Code for SIZE=256, RADIX=8 +#if WG == 32 && NW == 8 + + fft8_4(u); + tabMul8_4a(trig, u, 1, lowMe); + shufl(lds, u, 1, 4, numWG, lowMe); + + fft8(u); + tabMul8_4b(trig, u, 4, lowMe); + shufl(lds, u, 4, numWG, lowMe); + + fft8(u); + +// Code for SIZE=1024, RADIX=8 +#elif WG == 128 && RADIX == 8 + + fft8(u); + tabMul(trig, u, 1, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + fft8(u); + tabMul(trig, u, 8, lowMe); + shufl_and_fft2(lds, u, 8, numWG, lowMe); + + if (lowMe < WG / 2) fft8_16a(u); else fft8_16b(u); + +#else + +#if !UNROLL + __attribute__((opencl_unroll_hint(1))) +#endif + for (u32 s = 1; s < WG; s *= RADIX) { + fft_RADIX(u); + tabMul(trig, u, s, lowMe); + shufl(lds, u, s, numWG, lowMe); + } + fft_RADIX(u); + +#endif + +} + #endif diff --git a/src/cl/fftheight.cl b/src/cl/fftheight.cl index 69fa75b8..a19d7b2c 100644 --- a/src/cl/fftheight.cl +++ b/src/cl/fftheight.cl @@ -1,11 +1,24 @@ // Copyright (C) Mihai Preda -#include "base.cl" +// #defines that allow fft_height and fft_width share common code in fftbase.cl +#define WG G_H +#define RADIX NH +#define VARIANT FFT_VARIANT_H +#define LDSPAD LDSPAD_H +#define LDSSWIZ LDSSWIZ_H +#define SHUFL_BYTES SHUFL_BYTES_H +#define LDSMUL LDSMUL_H +#define UNROLL UNROLL_H +#define SAVE_ONE_MUL 1 // Radeon VII weirdness where saving one mul was slower (needs retesting!) +#define DOING_HEIGHT 1 // Flags to work around any optimizer weirdness where common code performs better in fft_WIDTH and worse in fft_HEIGHT or vice versa +#define DOING_WIDTH 0 + +#include "math.cl" +#include "trig.cl" #include "fftbase.cl" -#include "middle.cl" -#if SMALL_HEIGHT != 256 && SMALL_HEIGHT != 512 && SMALL_HEIGHT != 1024 && SMALL_HEIGHT != 4096 -#error SMALL_HEIGHT must be one of: 256, 512, 1024, 4096 +#if SMALL_HEIGHT != 256 && SMALL_HEIGHT != 512 && SMALL_HEIGHT != 1024 +#error SMALL_HEIGHT must be one of: 256, 512, 1024 #endif #if !INPLACE @@ -16,198 +29,10 @@ u32 transPos(u32 k, u32 middle, u32 width) { return k; } #if FFT_FP64 -void OVERLOAD fft_NH(T2 *u) { -#if NH == 4 - fft4(u); -#elif NH == 8 - fft8(u); -#else -#error NH -#endif -} - -#if FFT_VARIANT_H == 0 - -#if HEIGHT > 1024 -#error FFT_VARIANT_H == 0 only supports HEIGHT <= 1024 -#endif -#if !AMDGPU -#error FFT_VARIANT_H == 0 only supported by AMD GPUs -#endif - -void OVERLOAD fft_HEIGHT(local T2 *lds, T2 *u, Trig trig, T2 w) { - for (u32 s = 1; s < SMALL_HEIGHT / NH; s *= NH) { - if (s > 1) { bar(); } - fft_NH(u); - w = bcast(w, s); - - chainMul(NH, u, w, 1); - - shufl(SMALL_HEIGHT / NH, lds, u, NH, s); - } - fft_NH(u); -} - -void OVERLOAD fft_HEIGHT2(local T2 *lds, T2 *u, Trig trig, T2 w) { - u32 WG = SMALL_HEIGHT / NH; - for (u32 s = 1; s < SMALL_HEIGHT / NH; s *= NH) { - if (s > 1) { bar(WG); } - fft_NH(u); - w = bcast(w, s); - - chainMul(NH, u, w, 1); - - shufl2(SMALL_HEIGHT / NH, lds, u, NH, s); - } - fft_NH(u); -} - -#else - -void OVERLOAD fft_HEIGHT(local T2 *lds, T2 *u, Trig trig, T2 w) { - u32 me = get_local_id(0); - -#if !UNROLL_H - __attribute__((opencl_unroll_hint(1))) -#endif - - for (u32 s = 1; s < SMALL_HEIGHT / NH; s *= NH) { - if (s > 1) { bar(); } - fft_NH(u); - tabMul(SMALL_HEIGHT / NH, trig, u, NH, s, me); - shufl(SMALL_HEIGHT / NH, lds, u, NH, s); - } - fft_NH(u); -} - -void OVERLOAD fft_HEIGHT2(local T2 *lds, T2 *u, Trig trig, T2 w) { - u32 me = get_local_id(0); - u32 WG = SMALL_HEIGHT / NH; - -#if !UNROLL_H - __attribute__((opencl_unroll_hint(1))) -#endif - - for (u32 s = 1; s < WG; s *= NH) { - if (s > 1) { bar(WG); } - fft_NH(u); - tabMul(WG, trig, u, NH, s, me % WG); - shufl2(WG, lds, u, NH, s); - } - fft_NH(u); -} - -#endif - -void OVERLOAD new_fft_HEIGHT2(local T2 *lds, T2 *u, Trig trig, T2 w, int callnum) { - u32 WG = SMALL_HEIGHT / NH; - u32 me = get_local_id(0); - // This line mimics shufl2 -- partition lds into halves - local T2* partitioned_lds = lds + (me / WG) * SMALL_HEIGHT / 2; - me = me % WG; - -// Custom code for various SMALL_HEIGHT values - -#if SMALL_HEIGHT == 256 && NH == 4 && FFT_VARIANT_H == 2 - -// Custom code for SMALL_HEIGHT=256, NH=4 - - T preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul4_trig(WG, trig, preloads, 1, me); - - // Do first fft4, partial tabMul, and shufl. - fft4(u); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, me); - shufl2(WG, lds, u, NH, 1); - - // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. - finish_tabMul4_fft4(WG, partitioned_lds, trig, preloads, u, 1, me, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, me); - bar(WG); - shufl2(WG, lds, u, NH, 4); - - // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. - finish_tabMul4_fft4(WG, partitioned_lds, trig, preloads, u, 4, me, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, me); - bar(WG); - shufl2(WG, lds, u, NH, 16); - - // Finish third tabMul and perform final fft4. - finish_tabMul4_fft4(WG, partitioned_lds, trig, preloads, u, 16, me, 1); - -#elif SMALL_HEIGHT == 512 && NH == 8 && FFT_VARIANT_H == 2 - -// Custom code for SMALL_HEIGHT=512, NH=8 - - T preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*8 + 2*WG*8; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul8_trig(WG, trig, preloads, 1, me); - - // Do first fft8, partial tabMul, and shufl. - fft8(u); - partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 1, me); - shufl2(WG, lds, u, NH, 1); - - // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. - finish_tabMul8_fft8(WG, partitioned_lds, trig, preloads, u, 1, me, 1); - partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 8, me); - bar(WG); - shufl2(WG, lds, u, NH, 8); - - // Finish second tabMul and perform final fft8. - finish_tabMul8_fft8(WG, partitioned_lds, trig, preloads, u, 8, me, 1); - -#elif SMALL_HEIGHT == 1024 && NH == 4 && FFT_VARIANT_H == 2 - -// Custom code for SMALL_HEIGHT=1024, NH=4 - - T preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul4_trig(WG, trig, preloads, 1, me); - - // Do first fft4, partial tabMul, and shufl. - fft4(u); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, me); - shufl2(WG, lds, u, NH, 1); - - // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. - finish_tabMul4_fft4(WG, partitioned_lds, trig, preloads, u, 1, me, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, me); - bar(WG); - shufl2(WG, lds, u, NH, 4); - - // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. - finish_tabMul4_fft4(WG, partitioned_lds, trig, preloads, u, 4, me, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, me); - bar(WG); - shufl2(WG, lds, u, NH, 16); - - // Finish the third tabMul and perform fourth fft4. Do fourth partial tabMul and shufl. - finish_tabMul4_fft4(WG, partitioned_lds, trig, preloads, u, 16, me, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 64, me); - bar(WG); - shufl2(WG, lds, u, NH, 64); - - // Finish fourth tabMul and perform final fft4. - finish_tabMul4_fft4(WG, partitioned_lds, trig, preloads, u, 64, me, 1); - -#else - - // Old version - fft_HEIGHT2(lds, u, trig, w); - -#endif -} - -void new_fft_HEIGHT2_1(local T2 *lds, T2 *u, Trig trig, T2 w) { new_fft_HEIGHT2(lds, u, trig, w, 1); } -void new_fft_HEIGHT2_2(local T2 *lds, T2 *u, Trig trig, T2 w) { new_fft_HEIGHT2(lds, u, trig, w, 2); } +// Three versions. fft_HEIGHT1 and fft_HEIGHT2 are for the two tailSquare calls where a future version might save some data from call 1 for use in call 2. +void fft_HEIGHT(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, w, numWG, lowMe, 0); } +void fft_HEIGHT1(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, w, numWG, lowMe, 1); } +void fft_HEIGHT2(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, w, numWG, lowMe, 2); } #endif @@ -218,51 +43,10 @@ void new_fft_HEIGHT2_2(local T2 *lds, T2 *u, Trig trig, T2 w) { new_fft_HEIGHT2 #if FFT_FP32 -void OVERLOAD fft_NH(F2 *u) { -#if NH == 4 - fft4(u); -#elif NH == 8 - fft8(u); -#else -#error NH -#endif -} - -void OVERLOAD fft_HEIGHT(local F2 *lds, F2 *u, TrigFP32 trig) { - u32 me = get_local_id(0); - -#if !UNROLL_H - __attribute__((opencl_unroll_hint(1))) -#endif - - for (u32 s = 1; s < SMALL_HEIGHT / NH; s *= NH) { - if (s > 1) { bar(); } - fft_NH(u); - tabMul(SMALL_HEIGHT / NH, trig, u, NH, s, me); - shufl(SMALL_HEIGHT / NH, lds, u, NH, s); - } - fft_NH(u); -} - -void OVERLOAD fft_HEIGHT2(local F2 *lds, F2 *u, TrigFP32 trig) { - u32 me = get_local_id(0); - u32 WG = SMALL_HEIGHT / NH; - -#if !UNROLL_H - __attribute__((opencl_unroll_hint(1))) -#endif - - for (u32 s = 1; s < WG; s *= NH) { - if (s > 1) { bar(WG); } - fft_NH(u); - tabMul(WG, trig, u, NH, s, me % WG); - shufl2(WG, lds, u, NH, s); - } - fft_NH(u); -} - -void new_fft_HEIGHT2_1(local F2 *lds, F2 *u, TrigFP32 trig) { fft_HEIGHT2(lds, u, trig); } -void new_fft_HEIGHT2_2(local F2 *lds, F2 *u, TrigFP32 trig) { fft_HEIGHT2(lds, u, trig); } +// Three versions. fft_HEIGHT1 and fft_HEIGHT2 are for the two tailSquare calls where a future version might save some data from call 1 for use in call 2. +void fft_HEIGHT(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe, 0); } +void fft_HEIGHT1(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe, 1); } +void fft_HEIGHT2(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe, 2); } #endif @@ -273,51 +57,10 @@ void new_fft_HEIGHT2_2(local F2 *lds, F2 *u, TrigFP32 trig) { fft_HEIGHT2(lds, #if NTT_GF31 -void OVERLOAD fft_NH(GF31 *u) { -#if NH == 4 - fft4(u); -#elif NH == 8 - fft8(u); -#else -#error NH -#endif -} - -void OVERLOAD fft_HEIGHT(local GF31 *lds, GF31 *u, TrigGF31 trig) { - u32 me = get_local_id(0); - -#if !UNROLL_H - __attribute__((opencl_unroll_hint(1))) -#endif - - for (u32 s = 1; s < SMALL_HEIGHT / NH; s *= NH) { - if (s > 1) { bar(); } - fft_NH(u); - tabMul(SMALL_HEIGHT / NH, trig, u, NH, s, me); - shufl(SMALL_HEIGHT / NH, lds, u, NH, s); - } - fft_NH(u); -} - -void OVERLOAD fft_HEIGHT2(local GF31 *lds, GF31 *u, TrigGF31 trig) { - u32 me = get_local_id(0); - u32 WG = SMALL_HEIGHT / NH; - -#if !UNROLL_H - __attribute__((opencl_unroll_hint(1))) -#endif - - for (u32 s = 1; s < WG; s *= NH) { - if (s > 1) { bar(WG); } - fft_NH(u); - tabMul(WG, trig, u, NH, s, me % WG); - shufl2(WG, lds, u, NH, s); - } - fft_NH(u); -} - -void OVERLOAD new_fft_HEIGHT2_1(local GF31 *lds, GF31 *u, TrigGF31 trig) { fft_HEIGHT2(lds, u, trig); } -void OVERLOAD new_fft_HEIGHT2_2(local GF31 *lds, GF31 *u, TrigGF31 trig) { fft_HEIGHT2(lds, u, trig); } +// Three versions. fft_HEIGHT1 and fft_HEIGHT2 are for the two tailSquare calls where a future version might save some data from call 1 for use in call 2. +void OVERLOAD fft_HEIGHT(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } +void OVERLOAD fft_HEIGHT1(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } +void OVERLOAD fft_HEIGHT2(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } #endif @@ -328,50 +71,9 @@ void OVERLOAD new_fft_HEIGHT2_2(local GF31 *lds, GF31 *u, TrigGF31 trig) { fft_ #if NTT_GF61 -void OVERLOAD fft_NH(GF61 *u) { -#if NH == 4 - fft4(u); -#elif NH == 8 - fft8(u); -#else -#error NH -#endif -} - -void OVERLOAD fft_HEIGHT(local GF61 *lds, GF61 *u, TrigGF61 trig) { - u32 me = get_local_id(0); - -#if !UNROLL_H - __attribute__((opencl_unroll_hint(1))) -#endif - - for (u32 s = 1; s < SMALL_HEIGHT / NH; s *= NH) { - if (s > 1) { bar(); } - fft_NH(u); - tabMul(SMALL_HEIGHT / NH, trig, u, NH, s, me); - shufl(SMALL_HEIGHT / NH, lds, u, NH, s); - } - fft_NH(u); -} - -void OVERLOAD fft_HEIGHT2(local GF61 *lds, GF61 *u, TrigGF61 trig) { - u32 me = get_local_id(0); - u32 WG = SMALL_HEIGHT / NH; - -#if !UNROLL_H - __attribute__((opencl_unroll_hint(1))) -#endif - - for (u32 s = 1; s < WG; s *= NH) { - if (s > 1) { bar(WG); } - fft_NH(u); - tabMul(WG, trig, u, NH, s, me % WG); - shufl2(WG, lds, u, NH, s); - } - fft_NH(u); -} - -void OVERLOAD new_fft_HEIGHT2_1(local GF61 *lds, GF61 *u, TrigGF61 trig) { fft_HEIGHT2(lds, u, trig); } -void OVERLOAD new_fft_HEIGHT2_2(local GF61 *lds, GF61 *u, TrigGF61 trig) { fft_HEIGHT2(lds, u, trig); } +// Three versions. fft_HEIGHT1 and fft_HEIGHT2 are for the two tailSquare calls where a future version might save some data from call 1 for use in call 2. +void OVERLOAD fft_HEIGHT(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } +void OVERLOAD fft_HEIGHT1(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } +void OVERLOAD fft_HEIGHT2(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } #endif diff --git a/src/cl/ffthin.cl b/src/cl/ffthin.cl index ae14ec53..84f6f9d6 100644 --- a/src/cl/ffthin.cl +++ b/src/cl/ffthin.cl @@ -1,31 +1,60 @@ // Copyright (C) Mihai Preda #include "base.cl" -#include "math.cl" #include "fftheight.cl" +#define INCLUDE_FILE "middle.cl" +#include "expand.cl" + +// If not doing L2 stripes, process the lines in any order. +// If L2 striping, process lines output by fftMiddleIn. fftMiddleIn outputs 2 * stripe_group_size * 16 * MIDDLE tailSquare lines. +u32 get_line_number(u32 base_lo) { + u32 g = get_group_id(0); +#if L2_STRIPING + // Old, simple L2 striping code + // return g / (L2_STRIPING * 16) * WIDTH + base + g % (L2_STRIPING * 16); + + // Process stripe group base_lo or base_hi. Unlike tailSquare, there is no Hermitian-pair readiness rule here: + // by the time fftHin runs for a block, fftMiddleIn has produced every line of both stripe groups. + u32 stripe_group_size = L2_STRIPING; + u32 base_hi = WIDTH - stripe_group_size * 16 - base_lo; + u32 linesInOneStripe = 16 * MIDDLE; + u32 linesInOneStripeGroup = stripe_group_size * linesInOneStripe; + u32 base; + if (g < linesInOneStripeGroup) base = base_lo; + else base = base_hi, g -= linesInOneStripeGroup; + return g / (L2_STRIPING * 16) * WIDTH + base + g % (L2_STRIPING * 16); +#else + return g; +#endif +} + #if FFT_FP64 // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) -KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { - local T2 lds[SMALL_HEIGHT / 2]; - - T2 u[NH]; - u32 g = get_group_id(0); +KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { + local T2 lds[LDS_BYTES(1) / sizeof(T2)]; + LDSinit(lds, 1); + const u32 H = ND / SMALL_HEIGHT; + + T2 u[NH]; + u32 line = get_line_number(base); u32 me = get_local_id(0); - readTailFusedLine(in, u, g, me); + readTailFusedLine(in, u, line, me); -#if NH == 8 - T2 w = fancyTrig_N(ND / SMALL_HEIGHT * me); +#if FFT_VARIANT_H != 0 + T2 w; +#elif NH == 8 + T2 w = fancyTrig_N(H * me); #else - T2 w = slowTrig_N(ND / SMALL_HEIGHT * me, ND / NH); + T2 w = slowTrig_N(H * me, ND / NH); #endif - fft_HEIGHT(lds, u, smallTrig, w); + fft_HEIGHT(lds, u, smallTrig, w, 1, me); - write(G_H, NH, u, out, SMALL_HEIGHT * transPos(g, MIDDLE, WIDTH)); + write(G_H, NH, u, out, SMALL_HEIGHT * transPos(line, MIDDLE, WIDTH)); } #endif @@ -38,28 +67,33 @@ KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { #if FFT_FP32 // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) -KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { - local F2 lds[SMALL_HEIGHT / 2]; +KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { + local F2 lds[LDS_BYTES(1) / sizeof(F2)]; + LDSinit(lds, 1); + + const u32 H = ND / SMALL_HEIGHT; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; TrigFP32 smallTrigF2 = (TrigFP32) smallTrig; F2 u[NH]; - u32 g = get_group_id(0); + u32 line = get_line_number(base); u32 me = get_local_id(0); - readTailFusedLine(inF2, u, g, me); + readTailFusedLine(inF2, u, line, me); -#if NH == 8 - F2 w = fancyTrig_N(ND / SMALL_HEIGHT * me); +#if FFT_VARIANT_H != 0 + T2 w; +#elif NH == 8 + F2 w = fancyTrig_N(H * me); #else - F2 w = slowTrig_N(ND / SMALL_HEIGHT * me, ND / NH); + F2 w = slowTrig_N(H * me, ND / NH); #endif - fft_HEIGHT(lds, u, smallTrigF2); + fft_HEIGHT(lds, u, smallTrigF2, 1, me); - write(G_H, NH, u, outF2, SMALL_HEIGHT * transPos(g, MIDDLE, WIDTH)); + write(G_H, NH, u, outF2, SMALL_HEIGHT * transPos(line, MIDDLE, WIDTH)); } #endif @@ -72,23 +106,23 @@ KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { #if NTT_GF31 // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) -KERNEL(G_H) fftHinGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF31 lds[SMALL_HEIGHT / 2]; +KERNEL(G_H) fftHinGF31(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { + local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; + LDSinit(lds, 1); CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); TrigGF31 smallTrig31 = (TrigGF31) (smallTrig + DISTHTRIGGF31); GF31 u[NH]; - u32 g = get_group_id(0); - + u32 line = get_line_number(base); u32 me = get_local_id(0); - readTailFusedLine(in31, u, g, me); + readTailFusedLine(in31, u, line, me); - fft_HEIGHT(lds, u, smallTrig31); + fft_HEIGHT(lds, u, smallTrig31, 1, me); - write(G_H, NH, u, out31, SMALL_HEIGHT * transPos(g, MIDDLE, WIDTH)); + write(G_H, NH, u, out31, SMALL_HEIGHT * transPos(line, MIDDLE, WIDTH)); } #endif @@ -101,23 +135,23 @@ KERNEL(G_H) fftHinGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #if NTT_GF61 // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) -KERNEL(G_H) fftHinGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF61 lds[SMALL_HEIGHT / 2]; +KERNEL(G_H) fftHinGF61(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { + local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; + LDSinit(lds, 1); CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); TrigGF61 smallTrig61 = (TrigGF61) (smallTrig + DISTHTRIGGF61); GF61 u[NH]; - u32 g = get_group_id(0); - + u32 line = get_line_number(base); u32 me = get_local_id(0); - readTailFusedLine(in61, u, g, me); + readTailFusedLine(in61, u, line, me); - fft_HEIGHT(lds, u, smallTrig61); + fft_HEIGHT(lds, u, smallTrig61, 1, me); - write(G_H, NH, u, out61, SMALL_HEIGHT * transPos(g, MIDDLE, WIDTH)); + write(G_H, NH, u, out61, SMALL_HEIGHT * transPos(line, MIDDLE, WIDTH)); } #endif diff --git a/src/cl/fftmiddlein.cl b/src/cl/fftmiddlein.cl index d1788799..aa5d28fe 100644 --- a/src/cl/fftmiddlein.cl +++ b/src/cl/fftmiddlein.cl @@ -1,15 +1,16 @@ // Copyright (C) Mihai Preda and George Woltman #include "base.cl" -#include "math.cl" #include "fft-middle.cl" -#include "middle.cl" + +#define INCLUDE_FILE "middle.cl" +#include "expand.cl" #if !INPLACE // Original implementation (not in place) #if FFT_FP64 -KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, Trig trig) { +KERNEL_CAP(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, u32 base, Trig trig) { T2 u[MIDDLE]; u32 SIZEY = IN_WG / IN_SIZEX; @@ -30,6 +31,10 @@ KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, Trig trig) { u32 x = startx + mx; u32 y = starty + my; +#if FFT_TYPE == FFT64 + dependentLaunchWait(); // Previous kernel was carryfused that launched dependents before writing FP64 data +#endif + readMiddleInLine(u, in, y, x); middleMul2(u, x, y, 1, trig); @@ -38,6 +43,8 @@ KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, Trig trig) { middleMul(u, y, trig); + dependentLaunch(); // Next kernel will be tailSquareFP64 which must dependentLaunchWait before reading data + #if MIDDLE_IN_LDS_TRANSPOSE // Transpose the x and y values local T lds[IN_WG / 2 * (MIDDLE <= 8 ? 2 * MIDDLE : MIDDLE)]; @@ -60,7 +67,7 @@ KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, Trig trig) { #if FFT_FP32 -KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, Trig trig) { +KERNEL_CAP(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, u32 base, Trig trig) { F2 u[MIDDLE]; CP(F2) inF2 = (CP(F2)) in; @@ -85,6 +92,10 @@ KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, Trig trig) { u32 x = startx + mx; u32 y = starty + my; +#if FFT_TYPE == FFT32 + dependentLaunchWait(); // Previous kernel was carryfused that launched dependents before writing FP32 data +#endif + readMiddleInLine(u, inF2, y, x); middleMul2(u, x, y, 1, trigF2); @@ -93,6 +104,8 @@ KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, Trig trig) { middleMul(u, y, trigF2); + dependentLaunch(); // Next kernel will be tailSquareFP32 which must dependentLaunchWait before reading data + #if MIDDLE_IN_LDS_TRANSPOSE // Transpose the x and y values local F lds[IN_WG / 2 * (MIDDLE <= 16 ? 2 * MIDDLE : MIDDLE)]; @@ -115,7 +128,7 @@ KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, Trig trig) { #if NTT_GF31 -KERNEL(IN_WG) fftMiddleInGF31(P(T2) out, CP(T2) in, Trig trig) { +KERNEL_CAP(IN_WG) fftMiddleInGF31(P(T2) out, CP(T2) in, u32 base, Trig trig) { GF31 u[MIDDLE]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); @@ -140,6 +153,10 @@ KERNEL(IN_WG) fftMiddleInGF31(P(T2) out, CP(T2) in, Trig trig) { u32 x = startx + mx; u32 y = starty + my; +#if FFT_TYPE == FFT31 + dependentLaunchWait(); // Previous kernel was carryfused that launched dependents before writing GF31 data +#endif + readMiddleInLine(u, in31, y, x); middleMul2(u, x, y, trig31); @@ -148,6 +165,8 @@ KERNEL(IN_WG) fftMiddleInGF31(P(T2) out, CP(T2) in, Trig trig) { middleMul(u, y, trig31); + dependentLaunch(); // Next kernel will be tailSquareGF31 which must dependentLaunchWait before reading data + #if MIDDLE_IN_LDS_TRANSPOSE // Transpose the x and y values local Z31 lds[IN_WG / 2 * (MIDDLE <= 16 ? 2 * MIDDLE : MIDDLE)]; @@ -170,7 +189,7 @@ KERNEL(IN_WG) fftMiddleInGF31(P(T2) out, CP(T2) in, Trig trig) { #if NTT_GF61 -KERNEL(IN_WG) fftMiddleInGF61(P(T2) out, CP(T2) in, Trig trig) { +KERNEL_CAP(IN_WG) fftMiddleInGF61(P(T2) out, CP(T2) in, u32 base, Trig trig) { GF61 u[MIDDLE]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); @@ -180,7 +199,7 @@ KERNEL(IN_WG) fftMiddleInGF61(P(T2) out, CP(T2) in, Trig trig) { u32 SIZEY = IN_WG / IN_SIZEX; u32 N = WIDTH / IN_SIZEX; - + u32 g = get_group_id(0); u32 gx = g % N; u32 gy = g / N; @@ -195,6 +214,10 @@ KERNEL(IN_WG) fftMiddleInGF61(P(T2) out, CP(T2) in, Trig trig) { u32 x = startx + mx; u32 y = starty + my; +#if FFT_TYPE == FFT61 + dependentLaunchWait(); // Previous kernel was carryfused that launched dependents before writing GF31 data +#endif + readMiddleInLine(u, in61, y, x); middleMul2(u, x, y, trig61); @@ -203,6 +226,8 @@ KERNEL(IN_WG) fftMiddleInGF61(P(T2) out, CP(T2) in, Trig trig) { middleMul(u, y, trig61); + dependentLaunch(); // Next kernel will be tailSquareGF61 which must dependentLaunchWait before reading data + #if MIDDLE_IN_LDS_TRANSPOSE // Transpose the x and y values local Z61 lds[IN_WG / 2 * (MIDDLE <= 8 ? 2 * MIDDLE : MIDDLE)]; @@ -221,18 +246,68 @@ KERNEL(IN_WG) fftMiddleInGF61(P(T2) out, CP(T2) in, Trig trig) { - +// fftMiddleIn processes lines output by fftP or carryFused. Call this the x coordinate with range 0..WIDTH-1. The y coordinate ranges from 0..MIDDLE*SMALL_HEIGHT-1. +// In place transpose processeses blocks of 16 x coordinates by 16 y coordinates. fftMiddleIn processes processes MIDDLE blocks at a time. +// fftMiddleIn outputs lines for tailSquare. The y coordinate from 0..SMALL_HEIGHT-1 is transposed into the x coordinate for tailSquare. +// +// fftMiddleIn can work on all the FFT data, in which case we can process blocks in any order. Sequentially through memory by increasing x coordinates first might be best. +// More interestingly, fftMiddleIn can be configured to work on smaller amounts of FFT data in hopes that the data will stay in the L2 cache during the tailSquare and +// fftMiddleOut kernels. I call this L2 striping. In this case we process "columns of FFT data" by processing all the y coordinates to create complete lines for tailSquare. +// We must also output lines x and N-x for tailSquare handling of Hermetian symmetry. #else // in place transpose +// L2 striping processes both base_lo and base_hi (see Gpu.cpp) in one kernel call to reduce kernel launch overhead. There is also some special handling required for the +// first and last stripe groups. This results in a more complicated map group_id to compute the startx and starty coordinates. +#if L2_STRIPING +void map_striping_group_id(u32 base_lo, u32 g, u32 *startx, u32 *starty) { + // Old, simple L2 striping code + // u32 N = SMALL_HEIGHT / 16; + // u32 starty = g % N * 16; + // u32 startx = base + g / N * 16; + + // If MIDDLE is odd, the first fftMiddleIn must process the special N/2 tailSquare line from the WIDTH/2 stripe. + // If MULTI_Q, the first fftMiddleIn in the second queue must also process the 3*WIDTH/4 stripe. + u32 oneStripeKernelsToExecute = SMALL_HEIGHT / 16; + if ((base_lo == 0 && (MIDDLE & 1)) || (MULTI_Q && base_lo == WIDTH / 4)) { + if (g < oneStripeKernelsToExecute) { + *startx = base_lo + WIDTH / 2; + *starty = g * 16; + return; + } + g -= oneStripeKernelsToExecute; + } + + // Process stripe group base_lo. + u32 stripe_group_size = L2_STRIPING; + u32 kernelsToExecute = stripe_group_size * oneStripeKernelsToExecute; + if (g < kernelsToExecute) { + *startx = base_lo + g / oneStripeKernelsToExecute * 16; + *starty = g % oneStripeKernelsToExecute * 16; + return; + } + g -= kernelsToExecute; + + // Process stripe group base_hi. The last group must take into account that the first stripe may have already been processed. + u32 base_hi = WIDTH - stripe_group_size * 16 - base_lo; + if ((base_hi == WIDTH / 2 && (MIDDLE & 1)) || (MULTI_Q && base_hi == 3 * WIDTH / 4)) base_hi += 16; + *startx = base_hi + g / oneStripeKernelsToExecute * 16; + *starty = g % oneStripeKernelsToExecute * 16; +} +#endif + #if FFT_FP64 -KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, Trig trig) { +KERNEL_CAP(256) fftMiddleIn(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); T2 u[MIDDLE]; u32 g = get_group_id(0); -#if INPLACE == 1 // nVidia friendly padding +#if L2_STRIPING + u32 startx, starty; + map_striping_group_id(base, g, &startx, &starty); + u32 zerohack = (MIDDLE >= 16) ? 0 : g / 131072; // Rocm optimizer goes bonkers if zerohack used when MIDDLE=16 +#elif INPLACE == 1 // nVidia friendly padding u32 N = SMALL_HEIGHT / 16; u32 starty = g % N * 16; u32 startx = g / N * 16; @@ -248,6 +323,10 @@ KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, Trig trig) { u32 x = startx + me % 16; u32 y = starty + me / 16; +#if FFT_TYPE == FFT64 + dependentLaunchWait(); // Previous kernel was carryfused that launched dependents before writing FP64 data +#endif + readMiddleInLine(u, in, y, x); middleMul2(u, x, y, 1, trig); @@ -256,6 +335,8 @@ KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, Trig trig) { middleMul(u, y, trig); + dependentLaunch(); // Next kernel will be tailSquareFP64 which must dependentLaunchWait before reading data + // Transpose the x and y values local T2 lds[256]; middleShuffle(lds, u); @@ -272,7 +353,7 @@ KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, Trig trig) { #if FFT_FP32 -KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, Trig trig) { +KERNEL_CAP(256) fftMiddleIn(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); F2 u[MIDDLE]; @@ -281,7 +362,11 @@ KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, Trig trig) { TrigFP32 trigF2 = (TrigFP32) trig; u32 g = get_group_id(0); -#if INPLACE == 1 // nVidia friendly padding +#if L2_STRIPING + u32 startx, starty; + map_striping_group_id(base, g, &startx, &starty); + u32 zerohack = 0; // Need to test if g / 131072 is of any benefit +#elif INPLACE == 1 // nVidia friendly padding u32 N = SMALL_HEIGHT / 16; u32 starty = g % N * 16; u32 startx = g / N * 16; @@ -297,6 +382,10 @@ KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, Trig trig) { u32 x = startx + me % 16; u32 y = starty + me / 16; +#if FFT_TYPE == FFT32 + dependentLaunchWait(); // Previous kernel was carryfused that launched dependents before writing FP32 data +#endif + readMiddleInLine(u, inF2, y, x); middleMul2(u, x, y, 1, trigF2); @@ -305,6 +394,8 @@ KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, Trig trig) { middleMul(u, y, trigF2); + dependentLaunch(); // Next kernel will be tailSquareFP32 which must dependentLaunchWait before reading data + // Transpose the x and y values local F2 lds[256]; middleShuffle(lds, u); @@ -321,7 +412,7 @@ KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, Trig trig) { #if NTT_GF31 -KERNEL(256) fftMiddleInGF31(P(T2) out, P(T2) in, Trig trig) { +KERNEL_CAP(256) fftMiddleInGF31(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); GF31 u[MIDDLE]; @@ -330,7 +421,11 @@ KERNEL(256) fftMiddleInGF31(P(T2) out, P(T2) in, Trig trig) { TrigGF31 trig31 = (TrigGF31) (trig + DISTMTRIGGF31); u32 g = get_group_id(0); -#if INPLACE == 1 // nVidia friendly padding +#if L2_STRIPING + u32 startx, starty; + map_striping_group_id(base, g, &startx, &starty); + u32 zerohack = 0; // Need to test if g / 131072 is of any benefit +#elif INPLACE == 1 // nVidia friendly padding u32 N = SMALL_HEIGHT / 16; u32 starty = g % N * 16; u32 startx = g / N * 16; @@ -346,6 +441,10 @@ KERNEL(256) fftMiddleInGF31(P(T2) out, P(T2) in, Trig trig) { u32 x = startx + me % 16; u32 y = starty + me / 16; +#if FFT_TYPE == FFT31 + dependentLaunchWait(); // Previous kernel was carryfused that launched dependents before writing GF31 data +#endif + readMiddleInLine(u, in31, y, x); middleMul2(u, x, y, trig31); @@ -354,6 +453,8 @@ KERNEL(256) fftMiddleInGF31(P(T2) out, P(T2) in, Trig trig) { middleMul(u, y, trig31); + dependentLaunch(); // Next kernel will be tailSquareGF31 which must dependentLaunchWait before reading data + // Transpose the x and y values local GF31 lds[256]; middleShuffle(lds, u); @@ -370,7 +471,7 @@ KERNEL(256) fftMiddleInGF31(P(T2) out, P(T2) in, Trig trig) { #if NTT_GF61 -KERNEL(256) fftMiddleInGF61(P(T2) out, P(T2) in, Trig trig) { +KERNEL_CAP(256) fftMiddleInGF61(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); GF61 u[MIDDLE]; @@ -379,7 +480,11 @@ KERNEL(256) fftMiddleInGF61(P(T2) out, P(T2) in, Trig trig) { TrigGF61 trig61 = (TrigGF61) (trig + DISTMTRIGGF61); u32 g = get_group_id(0); -#if INPLACE == 1 // nVidia friendly padding +#if L2_STRIPING + u32 startx, starty; + map_striping_group_id(base, g, &startx, &starty); + u32 zerohack = 0; // Need to test if g / 131072 is of any benefit +#elif INPLACE == 1 // nVidia friendly padding u32 N = SMALL_HEIGHT / 16; u32 starty = g % N * 16; u32 startx = g / N * 16; @@ -395,6 +500,10 @@ KERNEL(256) fftMiddleInGF61(P(T2) out, P(T2) in, Trig trig) { u32 x = startx + me % 16; u32 y = starty + me / 16; +#if FFT_TYPE == FFT61 + dependentLaunchWait(); // Previous kernel was carryfused that launched dependents before writing GF31 data +#endif + readMiddleInLine(u, in61, y, x); middleMul2(u, x, y, trig61); @@ -403,6 +512,8 @@ KERNEL(256) fftMiddleInGF61(P(T2) out, P(T2) in, Trig trig) { middleMul(u, y, trig61); + dependentLaunch(); // Next kernel will be tailSquareGF61 which must dependentLaunchWait before reading data + // Transpose the x and y values local GF61 lds[256]; middleShuffle(lds, u); diff --git a/src/cl/fftmiddleout.cl b/src/cl/fftmiddleout.cl index a7263dcd..78b589a3 100644 --- a/src/cl/fftmiddleout.cl +++ b/src/cl/fftmiddleout.cl @@ -1,15 +1,16 @@ // Copyright (C) Mihai Preda and George Woltman #include "base.cl" -#include "math.cl" #include "fft-middle.cl" -#include "middle.cl" + +#define INCLUDE_FILE "middle.cl" +#include "expand.cl" #if !INPLACE // Original implementation (not in place) #if FFT_FP64 -KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, Trig trig) { +KERNEL_CAP(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, u32 base, Trig trig) { T2 u[MIDDLE]; u32 SIZEY = OUT_WG / OUT_SIZEX; @@ -34,6 +35,8 @@ KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, Trig trig) { u32 x = startx + mx; u32 y = starty + my; + dependentLaunchWait(); // Previous kernel was tailSquareFP64 that launched dependents before writing FP64 data + readMiddleOutLine(u, in, y, x); middleMul(u, x, trig); @@ -48,6 +51,8 @@ KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, Trig trig) { middleMul2(u, y, x, factor, trig); + dependentLaunch(); // Next kernel will be carryFused which must dependentLaunchWait before reading data + #if MIDDLE_OUT_LDS_TRANSPOSE // Transpose the x and y values local T lds[OUT_WG / 2 * (MIDDLE <= 8 ? 2 * MIDDLE : MIDDLE)]; @@ -70,7 +75,7 @@ KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, Trig trig) { #if FFT_FP32 -KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, Trig trig) { +KERNEL_CAP(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, u32 base, Trig trig) { F2 u[MIDDLE]; CP(F2) inF2 = (CP(F2)) in; @@ -99,20 +104,21 @@ KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, Trig trig) { u32 x = startx + mx; u32 y = starty + my; + dependentLaunchWait(); // Previous kernel was tailSquareFP32 that launched dependents before writing FP64 data + readMiddleOutLine(u, inF2, y, x); middleMul(u, x, trigF2); fft_MIDDLE(u); - // FFT results come out multiplied by the FFT length (NWORDS). Also, for performance reasons - // weights and invweights are doubled meaning we need to divide by another 2^2 and 2^2. - // Finally, roundoff errors are sometimes improved if we use the next lower double precision - // number. This may be due to roundoff errors introduced by applying inexact TWO_TO_N_8TH weights. - double factor = 1.0 / (4 * 4 * NWORDS); + // FFT results come out multiplied by the FFT length (NWORDS * 2). + const float factor = 1.0f / (NWORDS * 2); middleMul2(u, y, x, factor, trigF2); + dependentLaunch(); // Next kernel will be carryFused which must dependentLaunchWait before reading data + #if MIDDLE_OUT_LDS_TRANSPOSE // Transpose the x and y values local F lds[OUT_WG / 2 * (MIDDLE <= 16 ? 2 * MIDDLE : MIDDLE)]; @@ -135,7 +141,7 @@ KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, Trig trig) { #if NTT_GF31 -KERNEL(OUT_WG) fftMiddleOutGF31(P(T2) out, CP(T2) in, Trig trig) { +KERNEL_CAP(OUT_WG) fftMiddleOutGF31(P(T2) out, CP(T2) in, u32 base, Trig trig) { GF31 u[MIDDLE]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); @@ -164,6 +170,8 @@ KERNEL(OUT_WG) fftMiddleOutGF31(P(T2) out, CP(T2) in, Trig trig) { u32 x = startx + mx; u32 y = starty + my; + dependentLaunchWait(); // Previous kernel was tailSquareGF31 that launched dependents before writing GF31 data + readMiddleOutLine(u, in31, y, x); middleMul(u, x, trig31); @@ -172,6 +180,8 @@ KERNEL(OUT_WG) fftMiddleOutGF31(P(T2) out, CP(T2) in, Trig trig) { middleMul2(u, y, x, trig31); + dependentLaunch(); // Next kernel will be carryFused which must dependentLaunchWait before reading data + #if MIDDLE_OUT_LDS_TRANSPOSE // Transpose the x and y values local Z31 lds[OUT_WG / 2 * (MIDDLE <= 16 ? 2 * MIDDLE : MIDDLE)]; @@ -194,7 +204,7 @@ KERNEL(OUT_WG) fftMiddleOutGF31(P(T2) out, CP(T2) in, Trig trig) { #if NTT_GF61 -KERNEL(OUT_WG) fftMiddleOutGF61(P(T2) out, CP(T2) in, Trig trig) { +KERNEL_CAP(OUT_WG) fftMiddleOutGF61(P(T2) out, CP(T2) in, u32 base, Trig trig) { GF61 u[MIDDLE]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); @@ -223,6 +233,8 @@ KERNEL(OUT_WG) fftMiddleOutGF61(P(T2) out, CP(T2) in, Trig trig) { u32 x = startx + mx; u32 y = starty + my; + dependentLaunchWait(); // Previous kernel was tailSquare61 that launched dependents before writing GF61 data + readMiddleOutLine(u, in61, y, x); middleMul(u, x, trig61); @@ -231,6 +243,8 @@ KERNEL(OUT_WG) fftMiddleOutGF61(P(T2) out, CP(T2) in, Trig trig) { middleMul2(u, y, x, trig61); + dependentLaunch(); // Next kernel will be carryfused which must dependentLaunchWait before reading data + #if MIDDLE_OUT_LDS_TRANSPOSE // Transpose the x and y values local Z61 lds[OUT_WG / 2 * (MIDDLE <= 8 ? 2 * MIDDLE : MIDDLE)]; @@ -248,16 +262,58 @@ KERNEL(OUT_WG) fftMiddleOutGF61(P(T2) out, CP(T2) in, Trig trig) { +// fftMiddleOut processes lines output by tailSquare or tailMul. Call this the x coordinate with range 0..SMALL_HEIGHT-1 +// fftMiddleOut outputs lines for carryFused or fftW. Call this the y coordinate with range 0..WIDTH-1 +// In place transpose processeses blocks of 16 x coordinates by 16 y coordinates. +// fftMiddleOut processes processes MIDDLE blocks at a time. +// +// fftMiddleOut can work on all the FFT data, in which case we can process blocks in any order. Sequentially through memory by increasing x coordinates first might be best. +// More interestingly, fftMiddleOut can work on smaller amounts of FFT data in hopes that the data has stayed in the L2 cache during the fftMiddleIn/tailSquare/fftMiddleOut kernels. +// In this case we must work through all the x coordinates to read complete lines from tailSquare. We must also read y and N-y due to Hermetian symmetry. + + #else // in place transpose +// L2 striping processes both base_lo and base_hi (see Gpu.cpp) in one kernel call to reduce kernel launch overhead. There is also some special handling required for the +// first and last stripe groups. This results in some more complicated to map group_id into startx and starty coordinates. +#if L2_STRIPING +void map_striping_group_id(u32 base_lo, u32 g, u32 *startx, u32 *starty) { + // Old, simple L2 striping code + // u32 N = SMALL_HEIGHT / 16; + // u32 startx = g % N * 16; + // u32 starty = base + g / N * 16; + + // Process stripe group from base_lo. + u32 oneStripeKernelsToExecute = SMALL_HEIGHT / 16; + u32 stripe_group_size = L2_STRIPING; + u32 kernelsToExecute = stripe_group_size * oneStripeKernelsToExecute; + if (g < kernelsToExecute) { + *startx = g % oneStripeKernelsToExecute * 16; + *starty = base_lo + g / oneStripeKernelsToExecute * 16; + return; + } + g -= kernelsToExecute; + + // Process stripe group from base_hi. The first stripe in the base_hi stripe group is not ready for output (except in the last group). + // The last group processes the stripe that was skipped in the first base_hi group. + u32 base_hi = WIDTH - stripe_group_size * 16 - base_lo; + u32 base = (base_hi == WIDTH / 2 || (MULTI_Q && base_hi == 3 * WIDTH / 4)) ? base_hi : base_hi + 16; // Skip first stripe in base_hi (usually) + *startx = g % oneStripeKernelsToExecute * 16; + *starty = base + g / oneStripeKernelsToExecute * 16; +} +#endif + #if FFT_FP64 -KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, Trig trig) { +KERNEL_CAP(256) fftMiddleOut(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); T2 u[MIDDLE]; u32 g = get_group_id(0); -#if INPLACE == 1 // nVidia friendly padding +#if L2_STRIPING + u32 startx, starty; + map_striping_group_id(base, g, &startx, &starty); +#elif INPLACE == 1 // nVidia friendly padding u32 N = SMALL_HEIGHT / 16; u32 startx = g % N * 16; u32 starty = g / N * 16; @@ -271,6 +327,8 @@ KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, Trig trig) { u32 x = startx + me % 16; u32 y = starty + me / 16; + dependentLaunchWait(); // Previous kernel was tailSquareFP64 that launched dependents before writing FP64 data + readMiddleOutLine(u, in, y, x); middleMul(u, x, trig); @@ -285,6 +343,8 @@ KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, Trig trig) { middleMul2(u, y, x, factor, trig); + dependentLaunch(); // Next kernel will be carryFused which must dependentLaunchWait before reading data + // Transpose the x and y values local T2 lds[256]; middleShuffle(lds, u); @@ -301,7 +361,7 @@ KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, Trig trig) { #if FFT_FP32 -KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, Trig trig) { +KERNEL_CAP(256) fftMiddleOut(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); F2 u[MIDDLE]; @@ -310,7 +370,10 @@ KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, Trig trig) { TrigFP32 trigF2 = (TrigFP32) trig; u32 g = get_group_id(0); -#if INPLACE == 1 // nVidia friendly padding +#if L2_STRIPING + u32 startx, starty; + map_striping_group_id(base, g, &startx, &starty); +#elif INPLACE == 1 // nVidia friendly padding u32 N = SMALL_HEIGHT / 16; u32 startx = g % N * 16; u32 starty = g / N * 16; @@ -324,20 +387,21 @@ KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, Trig trig) { u32 x = startx + me % 16; u32 y = starty + me / 16; + dependentLaunchWait(); // Previous kernel was tailSquareFP32 that launched dependents before writing FP64 data + readMiddleOutLine(u, inF2, y, x); middleMul(u, x, trigF2); fft_MIDDLE(u); - // FFT results come out multiplied by the FFT length (NWORDS). Also, for performance reasons - // weights and invweights are doubled meaning we need to divide by another 2^2 and 2^2. - // Finally, roundoff errors are sometimes improved if we use the next lower double precision - // number. This may be due to roundoff errors introduced by applying inexact TWO_TO_N_8TH weights. - double factor = 1.0 / (4 * 4 * NWORDS); + // FFT results come out multiplied by the FFT length (NWORDS * 2). + const float factor = 1.0f / (NWORDS * 2); middleMul2(u, y, x, factor, trigF2); + dependentLaunch(); // Next kernel will be carryFused which must dependentLaunchWait before reading data + // Transpose the x and y values local F2 lds[256]; middleShuffle(lds, u); @@ -354,7 +418,7 @@ KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, Trig trig) { #if NTT_GF31 -KERNEL(256) fftMiddleOutGF31(P(T2) out, P(T2) in, Trig trig) { +KERNEL_CAP(256) fftMiddleOutGF31(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); GF31 u[MIDDLE]; @@ -363,7 +427,10 @@ KERNEL(256) fftMiddleOutGF31(P(T2) out, P(T2) in, Trig trig) { TrigGF31 trig31 = (TrigGF31) (trig + DISTMTRIGGF31); u32 g = get_group_id(0); -#if INPLACE == 1 // nVidia friendly padding +#if L2_STRIPING + u32 startx, starty; + map_striping_group_id(base, g, &startx, &starty); +#elif INPLACE == 1 // nVidia friendly padding u32 N = SMALL_HEIGHT / 16; u32 startx = g % N * 16; u32 starty = g / N * 16; @@ -377,6 +444,8 @@ KERNEL(256) fftMiddleOutGF31(P(T2) out, P(T2) in, Trig trig) { u32 x = startx + me % 16; u32 y = starty + me / 16; + dependentLaunchWait(); // Previous kernel was tailSquareGF31 that launched dependents before writing GF31 data + readMiddleOutLine(u, in31, y, x); middleMul(u, x, trig31); @@ -385,6 +454,8 @@ KERNEL(256) fftMiddleOutGF31(P(T2) out, P(T2) in, Trig trig) { middleMul2(u, y, x, trig31); + dependentLaunch(); // Next kernel will be carryFused which must dependentLaunchWait before reading data + // Transpose the x and y values local GF31 lds[256]; middleShuffle(lds, u); @@ -401,7 +472,7 @@ KERNEL(256) fftMiddleOutGF31(P(T2) out, P(T2) in, Trig trig) { #if NTT_GF61 -KERNEL(256) fftMiddleOutGF61(P(T2) out, P(T2) in, Trig trig) { +KERNEL_CAP(256) fftMiddleOutGF61(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); GF61 u[MIDDLE]; @@ -410,7 +481,10 @@ KERNEL(256) fftMiddleOutGF61(P(T2) out, P(T2) in, Trig trig) { TrigGF61 trig61 = (TrigGF61) (trig + DISTMTRIGGF61); u32 g = get_group_id(0); -#if INPLACE == 1 // nVidia friendly padding +#if L2_STRIPING + u32 startx, starty; + map_striping_group_id(base, g, &startx, &starty); +#elif INPLACE == 1 // nVidia friendly padding u32 N = SMALL_HEIGHT / 16; u32 startx = g % N * 16; u32 starty = g / N * 16; @@ -424,6 +498,8 @@ KERNEL(256) fftMiddleOutGF61(P(T2) out, P(T2) in, Trig trig) { u32 x = startx + me % 16; u32 y = starty + me / 16; + dependentLaunchWait(); // Previous kernel was tailSquareGF61 that launched dependents before writing GF61 data + readMiddleOutLine(u, in61, y, x); middleMul(u, x, trig61); @@ -432,6 +508,8 @@ KERNEL(256) fftMiddleOutGF61(P(T2) out, P(T2) in, Trig trig) { middleMul2(u, y, x, trig61); + dependentLaunch(); // Next kernel will be carryfused which must dependentLaunchWait before reading data + // Transpose the x and y values local GF61 lds[256]; middleShuffle(lds, u); diff --git a/src/cl/fftp.cl b/src/cl/fftp.cl index 2fe0d7d4..29c90854 100644 --- a/src/cl/fftp.cl +++ b/src/cl/fftp.cl @@ -1,16 +1,19 @@ // Copyright (C) Mihai Preda #include "base.cl" -#include "math.cl" -#include "weight.cl" #include "fftwidth.cl" -#include "middle.cl" +#include "weight.cl" + +#define INCLUDE_FILE "middle.cl" +#include "expand.cl" #if FFT_TYPE == FFT64 // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) { - local T2 lds[WIDTH / 2]; + local T2 lds[LDS_BYTES(1) / sizeof(T2)]; + LDSinit(lds, 1); + T2 u[NW]; u32 g = get_group_id(0); @@ -27,9 +30,9 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) u[i] = U2(in[p].x * w1, in[p].y * w2); } - fft_WIDTH(lds, u, smallTrig); + fft_WIDTH(lds, u, smallTrig, 1, me); - writeCarryFusedLine(u, out, g); + writeCarryFusedLine(u, out, g, me); } @@ -41,7 +44,9 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(F2) out, CP(Word2) in, TrigFP32 smallTrig, BigTabFP32 THREAD_WEIGHTS) { - local F2 lds[WIDTH / 2]; + local F2 lds[LDS_BYTES(1) / sizeof(F2)]; + LDSinit(lds, 1); + F2 u[NW]; u32 g = get_group_id(0); @@ -49,18 +54,28 @@ KERNEL(G_W) fftP(P(F2) out, CP(Word2) in, TrigFP32 smallTrig, BigTabFP32 THREAD_ in += g * WIDTH; - F base = optionalHalve(fancyMul(THREAD_WEIGHTS[me].y, THREAD_WEIGHTS[G_W + g].y)); + u32 word_index = (me * BIG_HEIGHT + g) * 2; + + u32 me_frac_bits = fracBits(me * BIG_HEIGHT * 2); + u32 line_frac_bits = fracBits(g * 2); + u32 base_frac_bits = me_frac_bits + line_frac_bits; + F base = optionalHalve(fancyMul(THREAD_WEIGHTS[me].y, THREAD_WEIGHTS[G_W + g].y), base_frac_bits > line_frac_bits); + const u32 frac_bits_bigstep = fracBits(G_W * BIG_HEIGHT * 2); + + u32 frac_bits = base_frac_bits; for (u32 i = 0; i < NW; ++i) { - F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i))); - F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP)); + F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); + F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 p = G_W * i + me; u[i] = U2(in[p].x * w1, in[p].y * w2); + // Generate frac_bits for next pair + frac_bits += frac_bits_bigstep; } - fft_WIDTH(lds, u, smallTrig); + fft_WIDTH(lds, u, smallTrig, 1, me); - writeCarryFusedLine(u, out, g); + writeCarryFusedLine(u, out, g, me); } @@ -72,7 +87,9 @@ KERNEL(G_W) fftP(P(F2) out, CP(Word2) in, TrigFP32 smallTrig, BigTabFP32 THREAD_ // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(GF31) out, CP(Word2) in, TrigGF31 smallTrig) { - local GF31 lds[WIDTH / 2]; + local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; + LDSinit(lds, 1); + GF31 u[NW]; u32 g = get_group_id(0); @@ -94,9 +111,9 @@ KERNEL(G_W) fftP(P(GF31) out, CP(Word2) in, TrigGF31 smallTrig) { #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * BIG_HEIGHT * 2 - 1) + make_u64((G_W * BIG_HEIGHT * 2 - 1) * bigword_weight_shift_minus1, 0)) % (31ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 31; for (u32 i = 0; i < NW; ++i) { @@ -113,9 +130,9 @@ KERNEL(G_W) fftP(P(GF31) out, CP(Word2) in, TrigGF31 smallTrig) { if (weight_shift > 31) weight_shift -= 31; } - fft_WIDTH(lds, u, smallTrig); + fft_WIDTH(lds, u, smallTrig, 1, me); - writeCarryFusedLine(u, out, g); + writeCarryFusedLine(u, out, g, me); } @@ -127,7 +144,9 @@ KERNEL(G_W) fftP(P(GF31) out, CP(Word2) in, TrigGF31 smallTrig) { // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(GF61) out, CP(Word2) in, TrigGF61 smallTrig) { - local GF61 lds[WIDTH / 2]; + local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; + LDSinit(lds, 1); + GF61 u[NW]; u32 g = get_group_id(0); @@ -151,9 +170,9 @@ KERNEL(G_W) fftP(P(GF61) out, CP(Word2) in, TrigGF61 smallTrig) { #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * BIG_HEIGHT * 2 - 1) + make_u64((G_W * BIG_HEIGHT * 2 - 1) * bigword_weight_shift_minus1, 0)) % (61ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 61; for (u32 i = 0; i < NW; ++i) { @@ -170,9 +189,9 @@ KERNEL(G_W) fftP(P(GF61) out, CP(Word2) in, TrigGF61 smallTrig) { if (weight_shift > 61) weight_shift -= 61; } - fft_WIDTH(lds, u, smallTrig); + fft_WIDTH(lds, u, smallTrig, 1, me); - writeCarryFusedLine(u, out, g); + writeCarryFusedLine(u, out, g, me); } @@ -184,8 +203,10 @@ KERNEL(G_W) fftP(P(GF61) out, CP(Word2) in, TrigGF61 smallTrig) { // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) { - local T2 lds[WIDTH / 2]; + local T2 lds[LDS_BYTES(1) / sizeof(T2)]; local GF31 *lds31 = (local GF31 *) lds; + LDSinit(lds, 1); + T2 u[NW]; GF31 u31[NW]; @@ -214,9 +235,9 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * BIG_HEIGHT * 2 - 1) + make_u64((G_W * BIG_HEIGHT * 2 - 1) * bigword_weight_shift_minus1, 0)) % (31ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 31; for (u32 i = 0; i < NW; ++i) { @@ -236,11 +257,11 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) if (weight_shift > 31) weight_shift -= 31; } - fft_WIDTH(lds, u, smallTrig); - writeCarryFusedLine(u, out, g); - bar(); - fft_WIDTH(lds31, u31, smallTrig31); - writeCarryFusedLine(u31, out31, g); + fft_WIDTH(lds, u, smallTrig, 1, me); + writeCarryFusedLine(u, out, g, me); + + fft_WIDTH(lds31, u31, smallTrig31, 1, me); + writeCarryFusedLine(u31, out31, g, me); } @@ -252,8 +273,10 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIGHTS) { - local F2 ldsF2[WIDTH / 2]; + local F2 ldsF2[LDS_BYTES(1) / sizeof(F2)]; local GF31 *lds31 = (local GF31 *) ldsF2; + LDSinit(ldsF2, 1); + F2 uF2[NW]; GF31 u31[NW]; @@ -267,10 +290,13 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG in += g * WIDTH; - F base = optionalHalve(fancyMul(THREAD_WEIGHTS[me].y, THREAD_WEIGHTS[G_W + g].y)); - u32 word_index = (me * BIG_HEIGHT + g) * 2; + u32 me_frac_bits = fracBits(me * BIG_HEIGHT * 2); + u32 line_frac_bits = fracBits(g * 2); + u32 base_frac_bits = me_frac_bits + line_frac_bits; + F base = optionalHalve(fancyMul(THREAD_WEIGHTS[me].y, THREAD_WEIGHTS[G_W + g].y), base_frac_bits > line_frac_bits); + // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Let s be the shift amount for word 1. The shift amount for word x is ceil(x * (s - 1) + num_big_words_less_than_x) % 31. const u32 log2_root_two = (u32) (((1ULL << 30) / NWORDS) % 31); @@ -284,16 +310,16 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * BIG_HEIGHT * 2 - 1) + make_u64((G_W * BIG_HEIGHT * 2 - 1) * bigword_weight_shift_minus1, 0)) % (31ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 31; for (u32 i = 0; i < NW; ++i) { u32 p = G_W * i + me; // Generate the FP32 weights and the second GF31 weight shift - F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i))); - F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP)); + F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); + F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 weight_shift0 = weight_shift; combo_counter += combo_step; if (weight_shift > 31) weight_shift -= 31; @@ -301,16 +327,17 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG // Convert and weight input uF2[i] = U2(in[p].x * w1, in[p].y * w2); u31[i] = U2(shl(make_Z31(in[p].x), weight_shift0), shl(make_Z31(in[p].y), weight_shift1)); // Form a GF31 from each pair of input words + // Generate weight shifts and frac_bits for next pair combo_counter += combo_bigstep; if (weight_shift > 31) weight_shift -= 31; } - fft_WIDTH(ldsF2, uF2, smallTrigF2); - writeCarryFusedLine(uF2, outF2, g); - bar(); - fft_WIDTH(lds31, u31, smallTrig31); - writeCarryFusedLine(u31, out31, g); + fft_WIDTH(ldsF2, uF2, smallTrigF2, 1, me); + writeCarryFusedLine(uF2, outF2, g, me); + + fft_WIDTH(lds31, u31, smallTrig31, 1, me); + writeCarryFusedLine(u31, out31, g, me); } @@ -322,8 +349,10 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIGHTS) { - local GF61 lds61[WIDTH / 2]; + local GF61 lds61[LDS_BYTES(1) / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; + LDSinit(lds61, 1); + F2 uF2[NW]; GF61 u61[NW]; @@ -337,10 +366,13 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG in += g * WIDTH; - F base = optionalHalve(fancyMul(THREAD_WEIGHTS[me].y, THREAD_WEIGHTS[G_W + g].y)); - u32 word_index = (me * BIG_HEIGHT + g) * 2; + u32 me_frac_bits = fracBits(me * BIG_HEIGHT * 2); + u32 line_frac_bits = fracBits(g * 2); + u32 base_frac_bits = me_frac_bits + line_frac_bits; + F base = optionalHalve(fancyMul(THREAD_WEIGHTS[me].y, THREAD_WEIGHTS[G_W + g].y), base_frac_bits > line_frac_bits); + // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Let s be the shift amount for word 1. The shift amount for word x is ceil(x * (s - 1) + num_big_words_less_than_x) % 61. const u32 log2_root_two = (u32) (((1ULL << 60) / NWORDS) % 61); @@ -354,16 +386,16 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * BIG_HEIGHT * 2 - 1) + make_u64((G_W * BIG_HEIGHT * 2 - 1) * bigword_weight_shift_minus1, 0)) % (61ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 61; for (u32 i = 0; i < NW; ++i) { u32 p = G_W * i + me; // Generate the FP32 weights and the second GF61 weight shift - F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i))); - F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP)); + F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); + F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 weight_shift0 = weight_shift; combo_counter += combo_step; if (weight_shift > 61) weight_shift -= 61; @@ -376,11 +408,11 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG if (weight_shift > 61) weight_shift -= 61; } - fft_WIDTH(ldsF2, uF2, smallTrigF2); - writeCarryFusedLine(uF2, outF2, g); - bar(); - fft_WIDTH(lds61, u61, smallTrig61); - writeCarryFusedLine(u61, out61, g); + fft_WIDTH(ldsF2, uF2, smallTrigF2, 1, me); + writeCarryFusedLine(uF2, outF2, g, me); + + fft_WIDTH(lds61, u61, smallTrig61, 1, me); + writeCarryFusedLine(u61, out61, g, me); } @@ -392,8 +424,10 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig) { - local GF61 lds61[WIDTH / 2]; + local GF61 lds61[LDS_BYTES(1) / sizeof(GF61)]; local GF31 *lds31 = (local GF31 *) lds61; + LDSinit(lds61, 1); + GF31 u31[NW]; GF61 u61[NW]; @@ -428,13 +462,13 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig) { #define m61_weight_shift m61_combo.a[1] #define m61_combo_counter m61_combo.b - const u64 m31_combo_step = ((u64) m31_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 m31_combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * m31_combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - m31_combo_counter = word_index * m31_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 m31_combo_step = make_u64(m31_bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 m31_combo_bigstep = (comboFracBits(G_W * BIG_HEIGHT * 2 - 1) + make_u64((G_W * BIG_HEIGHT * 2 - 1) * m31_bigword_weight_shift_minus1, 0)) % (31ULL << 32); + m31_combo_counter = comboFracBits(word_index) + make_u64(word_index * m31_bigword_weight_shift_minus1, 0xFFFFFFFF); m31_weight_shift = m31_weight_shift % 31; - const u64 m61_combo_step = ((u64) m61_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 m61_combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * m61_combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - m61_combo_counter = word_index * m61_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 m61_combo_step = make_u64(m61_bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 m61_combo_bigstep = (comboFracBits(G_W * BIG_HEIGHT * 2 - 1) + make_u64((G_W * BIG_HEIGHT * 2 - 1) * m61_bigword_weight_shift_minus1, 0)) % (61ULL << 32); + m61_combo_counter = comboFracBits(word_index) + make_u64(word_index * m61_bigword_weight_shift_minus1, 0xFFFFFFFF); m61_weight_shift = m61_weight_shift % 61; for (u32 i = 0; i < NW; ++i) { @@ -459,11 +493,11 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig) { m61_weight_shift = adjust_m61_weight_shift(m61_weight_shift); } - fft_WIDTH(lds31, u31, smallTrig31); - writeCarryFusedLine(u31, out31, g); - bar(); - fft_WIDTH(lds61, u61, smallTrig61); - writeCarryFusedLine(u61, out61, g); + fft_WIDTH(lds31, u31, smallTrig31, 1, me); + writeCarryFusedLine(u31, out31, g, me); + + fft_WIDTH(lds61, u61, smallTrig61, 1, me); + writeCarryFusedLine(u61, out61, g, me); } @@ -475,9 +509,11 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig) { // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIGHTS) { - local GF61 lds61[WIDTH / 2]; + local GF61 lds61[LDS_BYTES(1) / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; local GF31 *lds31 = (local GF31 *) lds61; + LDSinit(lds61, 1); + F2 uF2[NW]; GF31 u31[NW]; GF61 u61[NW]; @@ -494,10 +530,13 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG in += g * WIDTH; - F base = optionalHalve(fancyMul(THREAD_WEIGHTS[me].y, THREAD_WEIGHTS[G_W + g].y)); - u32 word_index = (me * BIG_HEIGHT + g) * 2; + u32 me_frac_bits = fracBits(me * BIG_HEIGHT * 2); + u32 line_frac_bits = fracBits(g * 2); + u32 base_frac_bits = me_frac_bits + line_frac_bits; + F base = optionalHalve(fancyMul(THREAD_WEIGHTS[me].y, THREAD_WEIGHTS[G_W + g].y), base_frac_bits > line_frac_bits); + // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Weights can be applied with shifts because 2 is the 60th root GF61. // Let s be the shift amount for word 1. The shift amount for word x is ceil(x * (s - 1) + num_big_words_less_than_x) % 61. @@ -517,20 +556,20 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG #define m61_weight_shift m61_combo.a[1] #define m61_combo_counter m61_combo.b - const u64 m31_combo_step = ((u64) m31_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 m31_combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * m31_combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - m31_combo_counter = word_index * m31_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 m31_combo_step = make_u64(m31_bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 m31_combo_bigstep = (comboFracBits(G_W * BIG_HEIGHT * 2 - 1) + make_u64((G_W * BIG_HEIGHT * 2 - 1) * m31_bigword_weight_shift_minus1, 0)) % (31ULL << 32); + m31_combo_counter = comboFracBits(word_index) + make_u64(word_index * m31_bigword_weight_shift_minus1, 0xFFFFFFFF); m31_weight_shift = m31_weight_shift % 31; - const u64 m61_combo_step = ((u64) m61_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 m61_combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * m61_combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - m61_combo_counter = word_index * m61_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 m61_combo_step = make_u64(m61_bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 m61_combo_bigstep = (comboFracBits(G_W * BIG_HEIGHT * 2 - 1) + make_u64((G_W * BIG_HEIGHT * 2 - 1) * m61_bigword_weight_shift_minus1, 0)) % (61ULL << 32); + m61_combo_counter = comboFracBits(word_index) + make_u64(word_index * m61_bigword_weight_shift_minus1, 0xFFFFFFFF); m61_weight_shift = m61_weight_shift % 61; for (u32 i = 0; i < NW; ++i) { u32 p = G_W * i + me; // Generate the FP32 weights and the second GF31 and GF61 weight shift - F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i))); - F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP)); + F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); + F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 m31_weight_shift0 = m31_weight_shift; m31_combo_counter += m31_combo_step; m31_weight_shift = adjust_m31_weight_shift(m31_weight_shift); @@ -551,14 +590,14 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG m61_weight_shift = adjust_m61_weight_shift(m61_weight_shift); } - fft_WIDTH(ldsF2, uF2, smallTrigF2); - writeCarryFusedLine(uF2, outF2, g); - bar(); - fft_WIDTH(lds31, u31, smallTrig31); - writeCarryFusedLine(u31, out31, g); - bar(); - fft_WIDTH(lds61, u61, smallTrig61); - writeCarryFusedLine(u61, out61, g); + fft_WIDTH(ldsF2, uF2, smallTrigF2, 1, me); + writeCarryFusedLine(uF2, outF2, g, me); + + fft_WIDTH(lds31, u31, smallTrig31, 1, me); + writeCarryFusedLine(u31, out31, g, me); + + fft_WIDTH(lds61, u61, smallTrig61, 1, me); + writeCarryFusedLine(u61, out61, g, me); } diff --git a/src/cl/fftw.cl b/src/cl/fftw.cl index a19b26d0..43926016 100644 --- a/src/cl/fftw.cl +++ b/src/cl/fftw.cl @@ -1,21 +1,26 @@ // Copyright (C) Mihai Preda #include "base.cl" -#include "math.cl" #include "fftwidth.cl" -#include "middle.cl" + +#define INCLUDE_FILE "middle.cl" +#include "expand.cl" #if FFT_FP64 // Do the ending fft_WIDTH after an fftMiddleOut. This is the same as the first half of carryFused. KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { - local T2 lds[WIDTH / 2]; + local T2 lds[LDS_BYTES(1) / sizeof(T2)]; + LDSinit(lds, 1); T2 u[NW]; u32 g = get_group_id(0); + u32 me = get_local_id(0); + + dependentLaunchWait(); // Previous kernel was fftMiddleOut - readCarryFusedLine(in, u, g); - fft_WIDTH(lds, u, smallTrig); + readCarryFusedLine(in, u, g, me); + fft_WIDTH(lds, u, smallTrig, 1, me); out += WIDTH * g; write(G_W, NW, u, out, 0); } @@ -31,7 +36,8 @@ KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { // Do the ending fft_WIDTH after an fftMiddleOut. This is the same as the first half of carryFused. KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { - local F2 lds[WIDTH / 2]; + local F2 lds[LDS_BYTES(1) / sizeof(F2)]; + LDSinit(lds, 1); CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -39,9 +45,12 @@ KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { F2 u[NW]; u32 g = get_group_id(0); + u32 me = get_local_id(0); + + dependentLaunchWait(); // Previous kernel was fftMiddleOut - readCarryFusedLine(inF2, u, g); - fft_WIDTH(lds, u, smallTrigF2); + readCarryFusedLine(inF2, u, g, me); + fft_WIDTH(lds, u, smallTrigF2, 1, me); outF2 += WIDTH * g; write(G_W, NW, u, outF2, 0); } @@ -56,7 +65,8 @@ KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { #if NTT_GF31 KERNEL(G_W) fftWGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF31 lds[WIDTH / 2]; + local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; + LDSinit(lds, 1); CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -64,9 +74,12 @@ KERNEL(G_W) fftWGF31(P(T2) out, CP(T2) in, Trig smallTrig) { GF31 u[NW]; u32 g = get_group_id(0); + u32 me = get_local_id(0); - readCarryFusedLine(in31, u, g); - fft_WIDTH(lds, u, smallTrig31); + dependentLaunchWait(); // Previous kernel was fftMiddleOut + + readCarryFusedLine(in31, u, g, me); + fft_WIDTH(lds, u, smallTrig31, 1, me); out31 += WIDTH * g; write(G_W, NW, u, out31, 0); } @@ -81,7 +94,8 @@ KERNEL(G_W) fftWGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #if NTT_GF61 KERNEL(G_W) fftWGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF61 lds[WIDTH / 2]; + local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; + LDSinit(lds, 1); CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); @@ -89,9 +103,12 @@ KERNEL(G_W) fftWGF61(P(T2) out, CP(T2) in, Trig smallTrig) { GF61 u[NW]; u32 g = get_group_id(0); + u32 me = get_local_id(0); + + dependentLaunchWait(); // Previous kernel was fftMiddleOut - readCarryFusedLine(in61, u, g); - fft_WIDTH(lds, u, smallTrig61); + readCarryFusedLine(in61, u, g, me); + fft_WIDTH(lds, u, smallTrig61, 1, me); out61 += WIDTH * g; write(G_W, NW, u, out61, 0); } diff --git a/src/cl/fftwidth.cl b/src/cl/fftwidth.cl index d0589ab0..5dff7e6c 100644 --- a/src/cl/fftwidth.cl +++ b/src/cl/fftwidth.cl @@ -1,5 +1,20 @@ // Copyright (C) Mihai Preda +// #defines that allow fft_height and fft_width share common code in fftbase.cl +#define WG G_W +#define RADIX NW +#define VARIANT FFT_VARIANT_W +#define LDSPAD LDSPAD_W +#define LDSSWIZ LDSSWIZ_W +#define SHUFL_BYTES SHUFL_BYTES_W +#define LDSMUL LDSMUL_W +#define UNROLL UNROLL_W +#define SAVE_ONE_MUL 0 // Radeon VII weirdness where saving one mul in width variant 2 was slower +#define DOING_HEIGHT 0 // Flags to work around any optimizer weirdness where common code performs better in fft_WIDTH and worse in fft_HEIGHT or vice versa +#define DOING_WIDTH 1 + +#include "math.cl" +#include "trig.cl" #include "fftbase.cl" #if WIDTH != 256 && WIDTH != 512 && WIDTH != 1024 && WIDTH != 4096 && WIDTH != 625 @@ -8,209 +23,10 @@ #if FFT_FP64 -void OVERLOAD fft_NW(T2 *u) { -#if NW == 4 - fft4(u); -#elif NW == 5 - fft5(u); -#elif NW == 8 - fft8(u); -#else -#error NW -#endif -} - -#if FFT_VARIANT_W == 0 - -#if WIDTH > 1024 -#error FFT_VARIANT_W == 0 only supports WIDTH <= 1024 -#endif -#if !AMDGPU -#error FFT_VARIANT_W == 0 only supported by AMD GPUs -#endif - -void OVERLOAD fft_WIDTH(local T2 *lds, T2 *u, Trig trig) { - u32 me = get_local_id(0); -#if NW == 8 - T2 w = fancyTrig_N(ND / WIDTH * me); -#else - T2 w = slowTrig_N(ND / WIDTH * me, ND / NW); -#endif - - for (u32 s = 1; s < WIDTH / NW; s *= NW) { - if (s > 1) { bar(); } - fft_NW(u); - w = bcast(w, s); - - chainMul(NW, u, w, 0); - - shufl( WIDTH / NW, lds, u, NW, s); - } - fft_NW(u); -} - -#else - -void OVERLOAD fft_WIDTH(local T2 *lds, T2 *u, Trig trig) { - u32 me = get_local_id(0); - -#if !UNROLL_W - __attribute__((opencl_unroll_hint(1))) -#endif - for (u32 s = 1; s < WIDTH / NW; s *= NW) { - if (s > 1) { bar(); } - fft_NW(u); - tabMul(WIDTH / NW, trig, u, NW, s, me); - shufl(WIDTH / NW, lds, u, NW, s); - } - fft_NW(u); -} - -#endif - - -// New fft_WIDTH that uses more FMA instructions than the old fft_WIDTH. -// The tabMul after fft8 only does a partial complex multiply, saving a mul-by-cosine for the next fft8 using FMA instructions. -// To maximize FMA opportunities we precompute trig values as cosine and sine/cosine rather than cosine and sine. -// The downside is sine/cosine cannot be computed with chained multiplies. - -void OVERLOAD new_fft_WIDTH(local T2 *lds, T2 *u, Trig trig, int callnum) { - u32 WG = WIDTH / NW; - u32 me = get_local_id(0); - -// Custom code for various WIDTH values - -#if WIDTH == 256 && NW == 4 && FFT_VARIANT_W == 2 - -// Custom code for WIDTH=256, NW=4 - - T preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul4_trig(WG, trig, preloads, 1, me); - - // Do first fft4, partial tabMul, and shufl. - fft4(u); - partial_tabMul4(WG, lds, trig, preloads, u, 1, me); - shufl(WG, lds, u, NW, 1); - - // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. - finish_tabMul4_fft4(WG, lds, trig, preloads, u, 1, me, 1); - partial_tabMul4(WG, lds, trig, preloads, u, 4, me); - bar(WG); - shufl(WG, lds, u, NW, 4); - - // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. - finish_tabMul4_fft4(WG, lds, trig, preloads, u, 4, me, 1); - partial_tabMul4(WG, lds, trig, preloads, u, 16, me); - bar(WG); - shufl(WG, lds, u, NW, 16); - - // Finish third tabMul and perform final fft4. - finish_tabMul4_fft4(WG, lds, trig, preloads, u, 16, me, 1); - -#elif WIDTH == 512 && NW == 8 && FFT_VARIANT_W == 2 - -// Custom code for WIDTH=512, NW=8 - - T preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*8; // Skip past old FFT_width trig values. - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul8_trig(WG, trig, preloads, 1, me); - - // Do first fft8, partial tabMul, and shufl. - fft8(u); - partial_tabMul8(WG, lds, trig, preloads, u, 1, me); - shufl(WG, lds, u, NW, 1); - - // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. - finish_tabMul8_fft8(WG, lds, trig, preloads, u, 1, me, 0); // We'd rather set save_one_more_mul to 1 - partial_tabMul8(WG, lds, trig, preloads, u, 8, me); - bar(); - shufl(WG, lds, u, NW, 8); - - // Finish second tabMul and perform final fft8. - finish_tabMul8_fft8(WG, lds, trig, preloads, u, 8, me, 0); // We'd rather set save_one_more_mul to 1 - -#elif WIDTH == 1024 && NW == 4 && FFT_VARIANT_W == 2 - -// Custom code for WIDTH=1024, NW=4 - - T preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul4_trig(WG, trig, preloads, 1, me); - - // Do first fft4, partial tabMul, and shufl. - fft4(u); - partial_tabMul4(WG, lds, trig, preloads, u, 1, me); - shufl(WG, lds, u, NW, 1); - - // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. - finish_tabMul4_fft4(WG, lds, trig, preloads, u, 1, me, 1); - partial_tabMul4(WG, lds, trig, preloads, u, 4, me); - bar(WG); - shufl(WG, lds, u, NW, 4); - - // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. - finish_tabMul4_fft4(WG, lds, trig, preloads, u, 4, me, 1); - partial_tabMul4(WG, lds, trig, preloads, u, 16, me); - bar(WG); - shufl(WG, lds, u, NW, 16); - - // Finish the third tabMul and perform fourth fft4. Do fourth partial tabMul and shufl. - finish_tabMul4_fft4(WG, lds, trig, preloads, u, 16, me, 1); - partial_tabMul4(WG, lds, trig, preloads, u, 64, me); - bar(WG); - shufl(WG, lds, u, NW, 64); - - // Finish fourth tabMul and perform final fft4. - finish_tabMul4_fft4(WG, lds, trig, preloads, u, 64, me, 1); - -#elif WIDTH == 4096 && NW == 8 && FFT_VARIANT_W == 2 - -// Custom code for WIDTH=4K, NW=8 - - T preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*8; // Skip past old FFT_width trig values to the !save_one_more_mul trig values - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul8_trig(WG, trig, preloads, 1, me); - - // Do first fft8, partial tabMul, and shufl. - fft8(u); - partial_tabMul8(WG, lds, trig, preloads, u, 1, me); - shufl(WG, lds, u, NW, 1); - - // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. - finish_tabMul8_fft8(WG, lds, trig, preloads, u, 1, me, 0); // We'd rather set save_one_more_mul to 1 - partial_tabMul8(WG, lds, trig, preloads, u, 8, me); - bar(); - shufl(WG, lds, u, NW, 8); - - // Finish the second tabMul and perform third fft8. Do third partial tabMul and shufl. - finish_tabMul8_fft8(WG, lds, trig, preloads, u, 8, me, 0); // We'd rather set save_one_more_mul to 1 - partial_tabMul8(WG, lds, trig, preloads, u, 64, me); - bar(); - shufl(WG, lds, u, NW, 64); - - // Finish third tabMul and perform final fft8. - finish_tabMul8_fft8(WG, lds, trig, preloads, u, 64, me, 0); // We'd rather set save_one_more_mul to 1 - -#else - - // Old version - fft_WIDTH(lds, u, trig); - -#endif -} - -// There are two version of new_fft_WIDTH in case we want to try saving some trig values from new_fft_WIDTH1 in LDS memory for later use in new_fft_WIDTH2. -void OVERLOAD new_fft_WIDTH1(local T2 *lds, T2 *u, Trig trig) { new_fft_WIDTH(lds, u, trig, 1); } -void OVERLOAD new_fft_WIDTH2(local T2 *lds, T2 *u, Trig trig) { new_fft_WIDTH(lds, u, trig, 2); } +// Three versions. fft_WIDTH1 and fft_WIDTH2 are for the two carryFused calls where a future version might save some data from call 1 for use in call 2. +void OVERLOAD fft_WIDTH(local T2 *lds, T2 *u, Trig trig, u32 numWG, u32 lowMe) { T2 dummy; fft_common(lds, u, trig, dummy, numWG, lowMe, 0); } +void OVERLOAD fft_WIDTH1(local T2 *lds, T2 *u, Trig trig, u32 numWG, u32 lowMe) { T2 dummy; fft_common(lds, u, trig, dummy, numWG, lowMe, 1); } +void OVERLOAD fft_WIDTH2(local T2 *lds, T2 *u, Trig trig, u32 numWG, u32 lowMe) { T2 dummy; fft_common(lds, u, trig, dummy, numWG, lowMe, 2); } #endif @@ -221,33 +37,10 @@ void OVERLOAD new_fft_WIDTH2(local T2 *lds, T2 *u, Trig trig) { new_fft_WIDTH(ld #if FFT_FP32 -void OVERLOAD fft_NW(F2 *u) { -#if NW == 4 - fft4(u); -#elif NW == 8 - fft8(u); -#else -#error NW -#endif -} - -void OVERLOAD fft_WIDTH(local F2 *lds, F2 *u, TrigFP32 trig) { - u32 me = get_local_id(0); - -#if !UNROLL_W - __attribute__((opencl_unroll_hint(1))) -#endif - for (u32 s = 1; s < WIDTH / NW; s *= NW) { - if (s > 1) { bar(); } - fft_NW(u); - tabMul(WIDTH / NW, trig, u, NW, s, me); - shufl(WIDTH / NW, lds, u, NW, s); - } - fft_NW(u); -} - -void OVERLOAD new_fft_WIDTH1(local F2 *lds, F2 *u, TrigFP32 trig) { fft_WIDTH(lds, u, trig); } -void OVERLOAD new_fft_WIDTH2(local F2 *lds, F2 *u, TrigFP32 trig) { fft_WIDTH(lds, u, trig); } +// Three versions. fft_WIDTH1 and fft_WIDTH2 are for the two carryFused calls where a future version might save some data from call 1 for use in call 2. +void OVERLOAD fft_WIDTH(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, u32 lowMe) { fft_common(lds , u, trig, numWG, lowMe, 0); } +void OVERLOAD fft_WIDTH1(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe, 1); } +void OVERLOAD fft_WIDTH2(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe, 2); } #endif @@ -258,33 +51,10 @@ void OVERLOAD new_fft_WIDTH2(local F2 *lds, F2 *u, TrigFP32 trig) { fft_WIDTH(ld #if NTT_GF31 -void OVERLOAD fft_NW(GF31 *u) { -#if NW == 4 - fft4(u); -#elif NW == 8 - fft8(u); -#else -#error NW -#endif -} - -void OVERLOAD fft_WIDTH(local GF31 *lds, GF31 *u, TrigGF31 trig) { - u32 me = get_local_id(0); - -#if !UNROLL_W - __attribute__((opencl_unroll_hint(1))) -#endif - for (u32 s = 1; s < WIDTH / NW; s *= NW) { - if (s > 1) { bar(); } - fft_NW(u); - tabMul(WIDTH / NW, trig, u, NW, s, me); - shufl(WIDTH / NW, lds, u, NW, s); - } - fft_NW(u); -} - -void OVERLOAD new_fft_WIDTH1(local GF31 *lds, GF31 *u, TrigGF31 trig) { fft_WIDTH(lds, u, trig); } -void OVERLOAD new_fft_WIDTH2(local GF31 *lds, GF31 *u, TrigGF31 trig) { fft_WIDTH(lds, u, trig); } +// Three versions. fft_WIDTH1 and fft_WIDTH2 are for the two carryFused calls where a future version might save some data from call 1 for use in call 2. +void OVERLOAD fft_WIDTH(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } +void OVERLOAD fft_WIDTH1(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } +void OVERLOAD fft_WIDTH2(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } #endif @@ -295,32 +65,9 @@ void OVERLOAD new_fft_WIDTH2(local GF31 *lds, GF31 *u, TrigGF31 trig) { fft_WIDT #if NTT_GF61 -void OVERLOAD fft_NW(GF61 *u) { -#if NW == 4 - fft4(u); -#elif NW == 8 - fft8(u); -#else -#error NW -#endif -} - -void OVERLOAD fft_WIDTH(local GF61 *lds, GF61 *u, TrigGF61 trig) { - u32 me = get_local_id(0); - -#if !UNROLL_W - __attribute__((opencl_unroll_hint(1))) -#endif - for (u32 s = 1; s < WIDTH / NW; s *= NW) { - if (s > 1) { bar(); } - fft_NW(u); - tabMul(WIDTH / NW, trig, u, NW, s, me); - shufl(WIDTH / NW, lds, u, NW, s); - } - fft_NW(u); -} - -void OVERLOAD new_fft_WIDTH1(local GF61 *lds, GF61 *u, TrigGF61 trig) { fft_WIDTH(lds, u, trig); } -void OVERLOAD new_fft_WIDTH2(local GF61 *lds, GF61 *u, TrigGF61 trig) { fft_WIDTH(lds, u, trig); } +// Three versions. fft_WIDTH1 and fft_WIDTH2 are for the two carryFused calls where a future version might save some data from call 1 for use in call 2. +void OVERLOAD fft_WIDTH(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } +void OVERLOAD fft_WIDTH1(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } +void OVERLOAD fft_WIDTH2(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } #endif diff --git a/src/cl/math.cl b/src/cl/math.cl index 479238c6..9c7de28d 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -2,22 +2,29 @@ #pragma once -#include "base.cl" - // Access parts of a 64-bit value -u32 OVERLOAD lo32(u64 x) { uint2 x2 = as_uint2(x); return (u32)x2.x; } -u32 OVERLOAD hi32(u64 x) { uint2 x2 = as_uint2(x); return (u32)x2.y; } -u32 OVERLOAD lo32(i64 x) { uint2 x2 = as_uint2(x); return (u32)x2.x; } -i32 OVERLOAD hi32(i64 x) { uint2 x2 = as_uint2(x); return (i32)x2.y; } +u32 OVERLOAD lo32(u64 x) { return (u32)x; } +u32 OVERLOAD hi32(u64 x) { union { uint2 ui2; u64 ul; } u; u.ul = x; return u.ui2.y; } +u32 OVERLOAD lo32(i64 x) { return (u32)x; } +u32 OVERLOAD hi32(i64 x) { union { uint2 ui2; u64 ul; } u; u.ul = x; return u.ui2.y; } + +u64 OVERLOAD make_u64(u32 hi, u32 lo) { union { uint2 ui2; u64 ul; } u; u.ui2.x = lo; u.ui2.y = hi; return u.ul; } +i64 OVERLOAD make_i64(i32 hi, u32 lo) { union { uint2 ui2; u64 ul; } u; u.ui2.x = lo; u.ui2.y = hi; return u.ul; } // A primitive partial implementation of an i96 integer type +// NOTE: the two-argument constructors use two different conventions, distinguished only by the type of "lo": +// make_i96(hi, u32 lo) places hi at bit 32 (value = hi * 2^32 + lo) +// make_i96(hi, u64 lo) places hi at bit 64 (value = hi * 2^64 + lo) +// Every implementation below must provide both, or a caller passing a u32 "lo" silently +// widens to the bit-64 form. #if 1 // An all 32-bit implementation. The add and subtract routines desperately need to use ASM with add.cc and sub.cc PTX instructions. // This version might be best on AMD and Intel if we can generate add-with-carry instructions. typedef struct { i32 hi32; u32 mid32; u32 lo32; } i96; i96 OVERLOAD make_i96(i64 v) { i96 val; val.lo32 = lo32(v); val.mid32 = hi32(v); val.hi32 = (i32)val.mid32 >> 31; return val; } i96 OVERLOAD make_i96(i32 v) { i96 val; val.lo32 = v; val.mid32 = val.hi32 = (i32)val.lo32 >> 31; return val; } +i96 OVERLOAD make_i96(i64 hi, u32 lo) { i96 val; val.hi32 = hi32(hi); val.mid32 = lo32(hi); val.lo32 = lo; return val; } i96 OVERLOAD make_i96(i64 hi, u64 lo) { i96 val; val.hi32 = hi; val.mid32 = hi32(lo); val.lo32 = lo32(lo); return val; } i96 OVERLOAD make_i96(i32 hi, u64 lo) { i96 val; val.hi32 = hi; val.mid32 = hi32(lo); val.lo32 = lo32(lo); return val; } u32 i96_hi32(i96 val) { return val.hi32; } @@ -61,10 +68,11 @@ i96 OVERLOAD sub(i96 a, i32 b) { return sub(a, make_i96(b)); } typedef struct { __int128 x; } i96; i96 OVERLOAD make_i96(i64 v) { i96 val; val.x = v; return val; } i96 OVERLOAD make_i96(i32 v) { i96 val; val.x = v; return val; } +i96 OVERLOAD make_i96(i64 hi, u32 lo) { i96 val; val.x = ((__int128)hi << 32) + lo; return val; } i96 OVERLOAD make_i96(i64 hi, u64 lo) { i96 val; val.x = ((unsigned __int128)hi << 64) + lo; return val; } i96 OVERLOAD make_i96(i32 hi, u64 lo) { return make_i96((i64)hi, lo); } u32 i96_hi32(i96 val) { return (unsigned __int128)val.x >> 64; } -u32 i96_mid32(i96 val) { return (u64)val.x >> 32; } +u32 i96_mid32(i96 val) { return hi32((u64)val.x); } u32 i96_lo32(i96 val) { return val.x; } u64 i96_lo64(i96 val) { return val.x; } u64 i96_hi64(i96 val) { return (unsigned __int128)val.x >> 32; } @@ -79,13 +87,14 @@ i96 OVERLOAD sub(i96 a, i32 b) { return sub(a, make_i96(b)); } typedef struct { u64 lo64; u32 hi32; } i96; i96 OVERLOAD make_i96(i64 v) { i96 val; val.hi32 = v >> 63, val.lo64 = v; return val; } i96 OVERLOAD make_i96(i32 v) { return make_i96((i64)v); } +i96 OVERLOAD make_i96(i64 hi, u32 lo) { i96 val; val.hi32 = (u64)hi >> 32, val.lo64 = ((u64)hi << 32) | lo; return val; } i96 OVERLOAD make_i96(i64 hi, u64 lo) { i96 val; val.hi32 = hi, val.lo64 = lo; return val; } i96 OVERLOAD make_i96(i32 hi, u64 lo) { i96 val; val.hi32 = hi, val.lo64 = lo; return val; } u32 i96_hi32(i96 val) { return val.hi32; } u32 i96_mid32(i96 val) { return hi32(val.lo64); } u32 i96_lo32(i96 val) { return val.lo64; } u64 i96_lo64(i96 val) { return val.lo64; } -u64 i96_hi64(i96 val) { return ((u64) val.hi32 << 32) | i96_mid32(val); } +u64 i96_hi64(i96 val) { return make_64(val.hi32, i96_mid32(val)); } i96 OVERLOAD add(i96 a, i96 b) { i96 val; val.lo64 = a.lo64 + b.lo64; val.hi32 = a.hi32 + b.hi32 + (val.lo64 < a.lo64); return val; } i96 OVERLOAD add(i96 a, i64 b) { return add(a, make_i96(b)); } i96 OVERLOAD sub(i96 a, i96 b) { i96 val; val.lo64 = a.lo64 - b.lo64; val.hi32 = a.hi32 - b.hi32 - (val.lo64 > a.lo64); return val; } @@ -110,9 +119,9 @@ i128 OVERLOAD sub(i128 a, i64 b) { i128 val; val.x = a.x - (__int128)b; return v u128 OVERLOAD make_u128(u64 hi, u64 lo) { u128 val; val.x = ((unsigned __int128)hi << 64) | lo; return val; } u64 u128_lo64(u128 val) { return val.x; } u64 u128_hi64(u128 val) { return val.x >> 64; } -u128 mul64(u64 a, u64 b) { u128 val; val.x = (unsigned __int128)a * (unsigned __int128)b; return val; } +u64 u128_shrlo64(u128 val, u32 bits) { return val.x >> bits; } u128 OVERLOAD add(u128 a, u128 b) { u128 val; val.x = a.x + b.x; return val; } -#else // UNTESTED! The mul64 macro causes clang to hang! +#else // UNTESTED! typedef struct { i64 hi64; u64 lo64; } i128; typedef struct { u64 hi64; u64 lo64; } u128; i128 OVERLOAD make_i128(i64 hi, u64 lo) { i128 val; val.hi64 = hi; val.lo64 = lo; return val; } @@ -128,7 +137,7 @@ i128 OVERLOAD sub(i128 a, i64 b) { i128 val; val.lo64 = a.lo64 - (u64)b; val.hi6 u128 OVERLOAD make_u128(u64 hi, u64 lo) { u128 val; val.hi64 = hi; val.lo64 = lo; return val; } u64 u128_lo64(u128 val) { return val.lo64; } u64 u128_hi64(u128 val) { return val.hi64; } -u128 mul64(u64 a, u64 b) { u128 val; val.lo64 = a * b; val.hi64 = mul_hi(a, b); return val; } +u64 u128_shrlo64(u128 val, u32 bits) { return (val.hi64 << (64 - bits)) | (val.lo64 >> bits); } u128 OVERLOAD add(u128 a, u128 b) { u128 val; val.lo64 = a.lo64 + b.lo64; val.hi64 = a.hi64 + b.hi64 + (val.lo64 < a.lo64); return val; } #endif @@ -143,8 +152,8 @@ i32 select32(i32 a, i32 b, i32 c) { #endif } -// Optionally add a value if first arg is negative. -i32 optional_add(i32 a, const i32 b) { +// Optionally add a constant value if first arg is negative. +i32 OVERLOAD optional_add(i32 a, const i32 b) { #if HAS_PTX >= 100 // setp/add instruction requires sm_10 support or higher __asm("{.reg .pred %%p;\n\t" " setp.lt.s32 %%p, %0, 0;\n\t" // a < 0 @@ -156,8 +165,8 @@ i32 optional_add(i32 a, const i32 b) { return a; } -// Optionally subtract a value if first arg is negative. -i32 optional_sub(i32 a, const i32 b) { +// Optionally subtract a constant value if first arg is negative. +i32 OVERLOAD optional_sub(i32 a, const i32 b) { #if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher __asm("{.reg .pred %%p;\n\t" " setp.lt.s32 %%p, %0, 0;\n\t" // a < 0 @@ -169,9 +178,9 @@ i32 optional_sub(i32 a, const i32 b) { return a; } -// Optionally subtract a value if first arg is greater than value. -i32 optional_mod(i32 a, const i32 b) { -#if 0 //HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher // Not faster on 5xxx GPUs (not sure why) +// Optionally subtract a constant value if first arg is greater than value. +i32 OVERLOAD optional_mod(i32 a, const i32 b) { +#if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher // Not faster on 5xxx GPUs (too small a gain to measure??) __asm("{.reg .pred %%p;\n\t" " setp.ge.s32 %%p, %0, %1;\n\t" // a > b " @%%p sub.s32 %0, %0, %1;}" // if (a > b) a = a - b @@ -182,8 +191,90 @@ i32 optional_mod(i32 a, const i32 b) { return a; } +#define M61 ((((Z61) 1) << 61) - 1) + +// Optionally add a constant value if first arg is negative. +i64 OVERLOAD optional_addM61(i64 a) { +#if HAS_PTX >= 100 // setp/add instruction requires sm_10 support or higher + __asm("{.reg .pred %%p;\n\t" + " setp.lt.s64 %%p, %0, 0;\n\t" // a < 0 + " @%%p add.s64 %0, %0, 2305843009213693951;}" // if (a < 0) a = a + M61 + : "+l"(a)); +#else + if (a < 0) a = a + M61; +#endif + return a; +} + +// Optionally add a constant value if first arg is negative. +i64 OVERLOAD optional_add(i64 a, const i64 b) { +#if HAS_PTX >= 100 // setp/add instruction requires sm_10 support or higher + __asm("{.reg .pred %%p;\n\t" + " setp.lt.s64 %%p, %0, 0;\n\t" // a < 0 + " @%%p add.s64 %0, %0, %1;}" // if (a < 0) a = a + b + : "+l"(a) : "l"(b)); // It would be nice if there was an asm constraint to indicate a 64-bit constant +#else + if (a < 0) a = a + b; +#endif + return a; +} + +// Optionally subtract constant c from a if a >= b. +u64 OVERLOAD optional_sub(u64 a, const u64 b, const u64 c) { +#if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher + __asm("{.reg .pred %%p;\n\t" + " setp.ge.u64 %%p, %0, %1;\n\t" // a >= b + " @%%p sub.u64 %0, %0, %2;}" // if (a >= b) a = a - c + : "+l"(a) : "l"(b), "l"(c)); // It would be nice if there was an asm constraint to indicate a 64-bit constant +#else + if (a >= b) a = a - c; +#endif + return a; +} +i64 OVERLOAD optional_sub(i64 a, const i64 b, const i64 c) { +#if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher + __asm("{.reg .pred %%p;\n\t" + " setp.ge.s64 %%p, %0, %1;\n\t" // a >= b + " @%%p sub.s64 %0, %0, %2;}" // if (a >= b) a = a - c + : "+l"(a) : "l"(b), "l"(c)); // It would be nice if there was an asm constraint to indicate a 64-bit constant +#else + if (a >= b) a = a - c; +#endif + return a; +} + +// Optionally subtract constant c from a if hi32(a) >= b. +u64 OVERLOAD optional_sub(u64 a, const u32 b, const u64 c) { +#if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher + __asm("{.reg .pred %%p;\n\t" + " setp.ge.u32 %%p, %1, %2;\n\t" // hi32(a) >= b + " @%%p sub.u64 %0, %0, %3;}" // if (hi32(a) >= b) a = a - c + : "+l"(a) : "r"((u32)hi32(a)), "n"(b), "l"(c)); +#else + if ((u32)hi32(a) >= b) a = a - c; +#endif + return a; +} +i64 OVERLOAD optional_sub(i64 a, const i32 b, const i64 c) { +#if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher + __asm("{.reg .pred %%p;\n\t" + " setp.ge.s32 %%p, %1, %2;\n\t" // hi32(a) >= b + " @%%p sub.s64 %0, %0, %3;}" // if (hi32(a) >= b) a = a - c + : "+l"(a) : "r"((i32)hi32(a)), "n"(b), "l"(c)); +#else + if ((i32)hi32(a) >= b) a = a - c; +#endif + return a; +} + // Multiply and add primitives +u64 OVERLOAD mul3264(u32 a, u64 b) { // 3 32-bit multiplies (instead of 4 for a u32 * u64 multiply) + u32 blo = lo32(b); + u32 bhi = hi32(b); + return make_u64(mul_hi(a, blo) + a * bhi, a * blo); +} + u64 OVERLOAD mad32(u32 a, u32 b, u32 c) { #if HAS_PTX >= 200 // mad instruction requires sm_20 support or higher // Same speed on TitanV, any gain may be too small to measure u32 reslo, reshi; @@ -206,11 +297,41 @@ u64 OVERLOAD mad32(u32 a, u32 b, u64 c) { #endif } +// Multiply to u64s creating a u128. This inline mysteriously speeds up PRPLL by 2%. Verified on RTX 3xxx through RTX 5xxx GPUs, both CUDA 12 and CUDA 13. +// The generated PTX code is nearly identical with DISABLE_MUL64. It must somehow trigger different/weird PTXAS optimization decisions. +// Future CUDA releases may make this inline unnecessary. +u128 OVERLOAD mul64(u64 a, u64 b) { +#if !DISABLE_MUL64 && HAS_PTX >= 100 // mul instruction requires sm_10 support or higher + u64 reslo, reshi; + __asm("mul.lo.u64 %0, %2, %3;\n\t" + "mul.hi.u64 %1, %2, %3;" : "=l"(reslo), "=l"(reshi) : "l"(a), "l"(b)); + return make_u128(reshi, reslo); +#elif ENABLE_ALT_MUL64 && HAS_PTX >= 200 // mad instruction requires sm_20 support or higher. This is slower than the above. + uint2 a2 = as_uint2(a); + uint2 b2 = as_uint2(b); + uint2 rlo2, rhi2; + __asm("mul.lo.u32 %0, %4, %6;\n\t" + "mul.hi.u32 %1, %4, %6;\n\t" + "mul.lo.u32 %2, %5, %7;\n\t" + "mad.lo.cc.u32 %1, %5, %6, %1;\n\t" + "madc.hi.cc.u32 %2, %5, %6, %2;\n\t" + "madc.hi.u32 %3, %5, %7, 0;\n\t" + "mad.lo.cc.u32 %1, %4, %7, %1;\n\t" + "madc.hi.cc.u32 %2, %4, %7, %2;\n\t" + "addc.u32 %3, %3, 0;" + : "=r"(rlo2.x), "=r"(rlo2.y), "=r"(rhi2.x), "=r"(rhi2.y) + : "r"(a2.x), "r"(a2.y), "r"(b2.x), "r"(b2.y)); + return make_u128((u64)as_ulong(rhi2), (u64)as_ulong(rlo2)); +#else // May cause clang to hang! + return make_u128(mul_hi(a, b), a * b); +#endif +} + u128 OVERLOAD mad64(u64 a, u64 b, u64 c) { -#if 0 && HAS_PTX >= 200 // mad instruction requires sm_20 support or higher // Slower on TitanV and mobile 4070, don't understand why +#if ENABLE_MAD64 && HAS_PTX >= 200 // mad instruction requires sm_20 support or higher // Slower on TitanV and mobile 4070, don't understand why u64 reslo, reshi; __asm("mad.lo.cc.u64 %0, %2, %3, %4;\n\t" - "madc.hi.u64 %1, %2, %3, 0;" : "=l"(reslo), "=l"(reshi) : "l"(a), "l"(b), "l"(u128_lo64(c))); + "madc.hi.u64 %1, %2, %3, 0;" : "=l"(reslo), "=l"(reshi) : "l"(a), "l"(b), "l"(c)); return make_u128(reshi, reslo); #elif HAS_PTX >= 200 // mad instruction requires sm_20 support or higher // Faster on TitanV. No difference on mobile 4070. Much cleaner PTX code generated. uint2 a2 = as_uint2(a); @@ -236,7 +357,7 @@ u128 OVERLOAD mad64(u64 a, u64 b, u64 c) { } u128 OVERLOAD mad64(u64 a, u64 b, u128 c) { -#if 0 && HAS_PTX >= 200 // mad instruction requires sm_20 support or higher // Slower on TitanV and mobile 4070, don't understand why +#if ENABLE_MAD64 && HAS_PTX >= 200 // mad instruction requires sm_20 support or higher // Slower on TitanV and mobile 4070, don't understand why u64 reslo, reshi; __asm("mad.lo.cc.u64 %0, %2, %3, %4;\n\t" "madc.hi.u64 %1, %2, %3, %5;" : "=l"(reslo), "=l"(reshi) : "l"(a), "l"(b), "l"(u128_lo64(c)), "l"(u128_hi64(c))); @@ -265,10 +386,11 @@ u128 OVERLOAD mad64(u64 a, u64 b, u128 c) { #endif } - // The X2 family of macros and SWAP are #defines because OpenCL does not allow pass by reference. // With NTT support added, we need to turn these macros into overloaded routines. #define X2(a, b) X2_internal(&(a), &(b)) // a = a + b, b = a - b +#define X2t4(a, b) X2t4_internal(&(a), &(b)) // X2(a, mul_t4(b)) +#define X2t4_mul_t4(a, b) X2t4_mul_t4_internal(&(a), &(b)) // X2(a, mul_t4(b)), b = mul_t4(b) #define X2conjb(a, b) X2conjb_internal(&(a), &(b)) // X2(a, conjugate(b)) #define X2_mul_t4(a, b) X2_mul_t4_internal(&(a), &(b)) // X2(a, b), b = mul_t4(b) #define X2_mul_t8(a, b) X2_mul_t8_internal(&(a), &(b)) // X2(a, b), b = mul_t8(b) @@ -276,9 +398,18 @@ u128 OVERLOAD mad64(u64 a, u64 b, u128 c) { #define X2_conjb(a, b) X2_conjb_internal(&(a), &(b)) // X2(a, b), b = conjugate(b) #define SWAP(a, b) SWAP_internal(&(a), &(b)) // a = b, b = a #define SWAP_XY(a) U2((a).y, (a).x) // Swap real and imaginary components of a +// Macros for FP64 and FP32 only. They allow some optimizations using FMA. NOTE: "ad" stands for "apply delayed" mul (see partial_cmul). +#define X2ad(a, b, d) X2ad_internal(&(a), &(b), d) // b = d * b, X2(a, b) +#define X2t4ad(a, b, d) X2t4ad_internal(&(a), &(b), d) // b = mul_t4(b), b = d * b, X2(a, b) +#define X2ad_mul_t4(a, b, d) X2ad_mul_t4_internal(&(a), &(b), d) // b = d * b, X2(a, b), b = mul_t4(b) #if FFT_FP64 +T OVERLOAD add(T a, T b) { return a + b; } +T2 OVERLOAD add(T2 a, T2 b) { return U2(add(a.x, b.x), add(a.y, b.y)); } +T OVERLOAD sub(T a, T b) { return a - b; } +T2 OVERLOAD sub(T2 a, T2 b) { return U2(sub(a.x, b.x), sub(a.y, b.y)); } + T2 OVERLOAD conjugate(T2 a) { return U2(a.x, -a.y); } // Multiply by 2 without using floating point instructions. This is a little sloppy as an input of zero returns 2^-1022. @@ -359,12 +490,18 @@ T2 OVERLOAD mul_3t8(T2 a) { // mul(a, U2(-1, 1)) * (T)(M_SQRT1_2); } // Return a+b and a-b void OVERLOAD X2_internal(T2 *a, T2 *b) { T2 t = *a; *a = t + *b; *b = t - *b; } -// Same as X2(a, b), b = mul_t4(b) -void OVERLOAD X2_mul_t4_internal(T2 *a, T2 *b) { T2 t = *a; *a = *a + *b; t.x = t.x - b->x; b->x = b->y - t.y; b->y = t.x; } +// Same as X2(a, mul_t4(b)) +void OVERLOAD X2t4_internal(T2 *a, T2 *b) { T by = b->y; b->y = sub(a->y, b->x); a->y = add(a->y, b->x); b->x = add(a->x, by); a->x = sub(a->x, by); } + +// Same as X2(a, mul_t4(b)), b = mul_t4(b) +void OVERLOAD X2t4_mul_t4_internal(T2 *a, T2 *b) { T2 t = *a; a->x = sub(a->x, b->y); a->y = add(a->y, b->x); b->x = sub(b->x, t.y); b->y = add(t.x, b->y); } // Same as X2(a, conjugate(b)) void OVERLOAD X2conjb_internal(T2 *a, T2 *b) { T2 t = *a; a->x = a->x + b->x; a->y = a->y - b->y; b->x = t.x - b->x; b->y = t.y + b->y; } +// Same as X2(a, b), b = mul_t4(b) +void OVERLOAD X2_mul_t4_internal(T2 *a, T2 *b) { T by = b->y; b->y = sub(a->x, b->x); a->x = add(a->x, b->x); b->x = sub(by, a->y); a->y = add(a->y, by); } + // Same as X2(a, b), b = conjugate(b) void OVERLOAD X2_conjb_internal(T2 *a, T2 *b) { T2 t = *a; *a = t + *b; b->x = t.x - b->x; b->y = b->y - t.y; } @@ -384,6 +521,29 @@ T2 OVERLOAD foo2(T2 a, T2 b) { a = addsub(a); b = addsub(b); return addsub(U2(RE // computes 2*[x^2+y^2 + i*(2*x*y)]. i.e. 2 * cyclical autoconvolution of (x, y) T2 OVERLOAD foo(T2 a) { return foo2(a, a); } +// Partial complex-multiply that delays the mul-by-cosine so it can be part of an FMA. +// We're trying to calculate u * U2(cosine,sine). Instead calculate u * U2(1,sine/cosine). +// real = (u.x - u.y*sine_over_cosine) * cosine +// imag = (u.x*sine_over_cosine + u.y) * cosine +T2 partial_cmul(T2 u, T sine_over_cosine) { + return U2(fma(-u.y, sine_over_cosine, u.x), fma(u.x, sine_over_cosine, u.y)); +} + +T2 mul_t8_delayed(T2 a) { return U2(a.x - a.y, a.x + a.y); } // Apply mul by M_SQRT1_2 later +T2 mul_3t8_delayed(T2 a) { return U2(-(a.x + a.y), a.x - a.y); } // Apply mul by M_SQRT1_2 later. Alternatively, use mul_t8_delayed and mul by i*M_SQRT1_2 later. + +// Compute a + d * b and a - d * b +void X2ad_internal(T2 *a, T2 *b, T d) { T2 t = *a; a->x = fma(b->x, d, a->x); a->y = fma(b->y, d, a->y); b->x = fma(-d, b->x, t.x); b->y = fma(-d, b->y, t.y); } +void X2t4ad_internal(T2 *a, T2 *b, T d) { T bx = b->x; b->x = fma(d, b->y, a->x); a->x = fma(b->y, -d, a->x); b->y = fma(-d, bx, a->y); a->y = fma(bx, d, a->y); } +void X2ad_mul_t4_internal(T2 *a, T2 *b, T d) { T by = b->y; b->y = fma(-d, b->x, a->x); a->x = fma(b->x, d, a->x); b->x = -fma(-d, by, a->y); a->y = fma(by, d, a->y); } + +// Create "quick" routines for compatibility with shufl_and_fft2 and the GF61 data type. + +T OVERLOAD addq(T a, T b) { return add(a, b); } +T OVERLOAD subq(T a, T b) { return sub(a, b); } +T2 OVERLOAD addq(T2 a, T2 b) { return add(a, b); } +T2 OVERLOAD subq(T2 a, T2 b) { return sub(a, b); } + #endif @@ -393,6 +553,11 @@ T2 OVERLOAD foo(T2 a) { return foo2(a, a); } #if FFT_FP32 +F OVERLOAD add(F a, F b) { return a + b; } +F2 OVERLOAD add(F2 a, F2 b) { return U2(add(a.x, b.x), add(a.y, b.y)); } +F OVERLOAD sub(F a, F b) { return a - b; } +F2 OVERLOAD sub(F2 a, F2 b) { return U2(sub(a.x, b.x), sub(a.y, b.y)); } + F2 OVERLOAD conjugate(F2 a) { return U2(a.x, -a.y); } // Multiply by 2 without using floating point instructions. This is a little sloppy as an input of zero returns 2^-126. @@ -454,7 +619,7 @@ void cmul_a_by_fancyb_and_conjfancyb(F2 *res1, F2 *res2, F2 a, F2 b) { F2 OVERLOAD mul_t4(F2 a) { return U2(-a.y, a.x); } // i.e. a * i -F2 OVERLOAD mul_t8(F2 a) { // mul(a, U2(1, 1)) * (T)(M_SQRT1_2); } +F2 OVERLOAD mul_t8(F2 a) { // mul(a, U2(1, 1)) * (F)(M_SQRT1_2); } // One mul, two FMAs F ay = a.y * (float) M_SQRT1_2; return U2(fma(a.x, (float) M_SQRT1_2, -ay), fma(a.x, (float) M_SQRT1_2, ay)); @@ -462,7 +627,7 @@ F2 OVERLOAD mul_t8(F2 a) { // mul(a, U2(1, 1)) * (T)(M_SQRT1_2); } // return U2(a.x - a.y, a.x + a.y) * M_SQRT1_2; } -F2 OVERLOAD mul_3t8(F2 a) { // mul(a, U2(-1, 1)) * (T)(M_SQRT1_2); } +F2 OVERLOAD mul_3t8(F2 a) { // mul(a, U2(-1, 1)) * (F)(M_SQRT1_2); } // One mul, two FMAs F ay = a.y * (float) M_SQRT1_2; return U2(fma(-a.x, (float) M_SQRT1_2, -ay), fma(a.x, (float) M_SQRT1_2, -ay)); @@ -473,12 +638,18 @@ F2 OVERLOAD mul_3t8(F2 a) { // mul(a, U2(-1, 1)) * (T)(M_SQRT1_2); } // Return a+b and a-b void OVERLOAD X2_internal(F2 *a, F2 *b) { F2 t = *a; *a = t + *b; *b = t - *b; } -// Same as X2(a, b), b = mul_t4(b) -void OVERLOAD X2_mul_t4_internal(F2 *a, F2 *b) { F2 t = *a; *a = *a + *b; t.x = t.x - b->x; b->x = b->y - t.y; b->y = t.x; } +// Same as X2(a, mul_t4(b)) +void OVERLOAD X2t4_internal(F2 *a, F2 *b) { F by = b->y; b->y = sub(a->y, b->x); a->y = add(a->y, b->x); b->x = add(a->x, by); a->x = sub(a->x, by); } + +// Same as X2(a, mul_t4(b)), b = mul_t4(b) +void OVERLOAD X2t4_mul_t4_internal(F2 *a, F2 *b) { F2 t = *a; a->x = sub(a->x, b->y); a->y = add(a->y, b->x); b->x = sub(b->x, t.y); b->y = add(t.x, b->y); } // Same as X2(a, conjugate(b)) void OVERLOAD X2conjb_internal(F2 *a, F2 *b) { F2 t = *a; a->x = a->x + b->x; a->y = a->y - b->y; b->x = t.x - b->x; b->y = t.y + b->y; } +// Same as X2(a, b), b = mul_t4(b) +void OVERLOAD X2_mul_t4_internal(F2 *a, F2 *b) { F by = b->y; b->y = sub(a->x, b->x); a->x = add(a->x, b->x); b->x = sub(by, a->y); a->y = add(a->y, by); } + // Same as X2(a, b), b = conjugate(b) void OVERLOAD X2_conjb_internal(F2 *a, F2 *b) { F2 t = *a; *a = t + *b; b->x = t.x - b->x; b->y = b->y - t.y; } @@ -498,6 +669,29 @@ F2 OVERLOAD foo2(F2 a, F2 b) { a = addsub(a); b = addsub(b); return addsub(U2(RE // computes 2*[x^2+y^2 + i*(2*x*y)]. i.e. 2 * cyclical autoconvolution of (x, y) F2 OVERLOAD foo(F2 a) { return foo2(a, a); } +// Partial complex-multiply that delays the mul-by-cosine so it can be part of an FMA. +// We're trying to calculate u * U2(cosine,sine). Instead calculate u * U2(1,sine/cosine). +// real = (u.x - u.y*sine_over_cosine) * cosine +// imag = (u.x*sine_over_cosine + u.y) * cosine +F2 partial_cmul(F2 u, F sine_over_cosine) { + return U2(fma(-u.y, sine_over_cosine, u.x), fma(u.x, sine_over_cosine, u.y)); +} + +F2 mul_t8_delayed(F2 a) { return U2(a.x - a.y, a.x + a.y); } // Apply mul by M_SQRT1_2 later +F2 mul_3t8_delayed(F2 a) { return U2(-(a.x + a.y), a.x - a.y); } // Apply mul by M_SQRT1_2 later. Alternatively, use mul_t8_delayed and mul by i*M_SQRT1_2 later. + +// Compute a + d * b and a - d * b +void X2ad_internal(F2 *a, F2 *b, F d) { F2 t = *a; a->x = fma(b->x, d, a->x); a->y = fma(b->y, d, a->y); b->x = fma(-d, b->x, t.x); b->y = fma(-d, b->y, t.y); } +void X2t4ad_internal(F2 *a, F2 *b, F d) { F bx = b->x; b->x = fma(d, b->y, a->x); a->x = fma(b->y, -d, a->x); b->y = fma(-d, bx, a->y); a->y = fma(bx, d, a->y); } +void X2ad_mul_t4_internal(F2 *a, F2 *b, F d) { F by = b->y; b->y = fma(-d, b->x, a->x); a->x = fma(b->x, d, a->x); b->x = -fma(-d, by, a->y); a->y = fma(by, d, a->y); } + +// Create "quick" routines for compatibility with shufl_and_fft2 and the GF61 data type. + +F OVERLOAD addq(F a, F b) { return add(a, b); } +F OVERLOAD subq(F a, F b) { return sub(a, b); } +F2 OVERLOAD addq(F2 a, F2 b) { return add(a, b); } +F2 OVERLOAD subq(F2 a, F2 b) { return sub(a, b); } + #endif @@ -572,11 +766,14 @@ GF31 OVERLOAD mul_3t8(GF31 a) { return U2(shl(neg(add(a.x, a.y)), 15), shl(sub(a // Return a+b and a-b void OVERLOAD X2_internal(GF31 *a, GF31 *b) { GF31 t = *a; *a = add(t, *b); *b = sub(t, *b); } +// Same as X2(a, mul_t4(b)) +void OVERLOAD X2t4_internal(GF31 *a, GF31 *b) { Z31 by = b->y; b->y = sub(a->y, b->x); a->y = add(a->y, b->x); b->x = add(a->x, by); a->x = sub(a->x, by); } + // Same as X2(a, conjugate(b)) void OVERLOAD X2conjb_internal(GF31 *a, GF31 *b) { GF31 t = *a; a->x = add(a->x, b->x); a->y = sub(a->y, b->y); b->x = sub(t.x, b->x); b->y = add(t.y, b->y); } // Same as X2(a, b), b = mul_t4(b) -void OVERLOAD X2_mul_t4_internal(GF31 *a, GF31 *b) { GF31 t = *a; *a = add(*a, *b); t.x = sub(t.x, b->x); b->x = sub(b->y, t.y); b->y = t.x; } +void OVERLOAD X2_mul_t4_internal(GF31 *a, GF31 *b) { Z31 by = b->y; b->y = sub(a->x, b->x); a->x = add(a->x, b->x); b->x = sub(by, a->y); a->y = add(a->y, by); } // Same as X2(a, b), b = mul_t8(b) void OVERLOAD X2_mul_t8_internal(GF31 *a, GF31 *b) { X2(*a, *b); *b = mul_t8(*b); } @@ -607,7 +804,7 @@ Z31 OVERLOAD modM31(i32 a) { return (a & M31) + (a >> 31); } Z31 OVERLOAD modM31(Z31 a) { i32 alt = a + 0x80000001; return select32(a, a, alt); } // Assumes a is not 0xFFFFFFFF (which would return 0x80000000) Z31 OVERLOAD modM31(i32 a) { i32 alt = a - 0x80000001; return select32(a, a, alt); } // Assumes a is not 0x80000000 (which would return 0xFFFFFFFF) #else -Z31 OVERLOAD modM31(Z31 a) { return optional_add(a, 0x80000001); } // Assumes a is not 0xFFFFFFFF (which would return 0x80000000) +Z31 OVERLOAD modM31(Z31 a) { return optional_add((i32)a, 0x80000001); } // Assumes a is not 0xFFFFFFFF (which would return 0x80000000) Z31 OVERLOAD modM31(i32 a) { return optional_sub(a, 0x80000001); } // Assumes a is not 0x80000000 (which would return 0xFFFFFFFF) #endif @@ -617,27 +814,25 @@ Z31 OVERLOAD modM31(u64 a) { // a must u32 ahi = a >> 62; return modM31(ahi + amid + alo); // 32-bit overflow does not occur due to restrictions on input } +Z31 OVERLOAD modM31(u64 a, u32 maxbits) { + if (maxbits <= 62) { + u32 alo = lo32(a) & M31; // 31 bits + u32 ahi = hi32(a + a); // 31 bits + return modM31(ahi + alo); + } + else if (maxbits == 63) { + u32 alo = lo32(a) & M31; // 31 bits + u32 ahi = hi32(a + a); // 32 bits + return modM31(modM31(ahi) + alo); + } + return modM31(a); +} Z31 OVERLOAD modM31(i64 a) { // abs(a) must be less than 0x7FFFFFFF80000000 u32 alo = a & M31; u32 amid = ((u64) a >> 31) & M31; // Unsigned shift might be faster than signed shift u32 ahi = a >> 62; // Sign extend the top bits return modM31(ahi + amid + alo); // This is where caller must assure a 32-bit overflow does not occur } -Z31 OVERLOAD modM31q(u64 a) { // Quick version, a < 2^62 - u32 alo = a & M31; - u32 ahi = a >> 31; - return modM31(ahi + alo); -} -#if 0 // GWBUG - which is faster? -Z31 OVERLOAD modM31q(i64 a) { // Quick version, abs(a) must be 61 bits - u32 alo = a & M31; - i32 ahi = a >> 31; // Sign extend the top bits - if (ahi < 0) ahi = ahi + M31; - return modM31((u32) ahi + alo); -} -#else -Z31 OVERLOAD modM31q(i64 a) { return modM31(a); } // Quick version, abs(a) must be 61 bits -#endif Z31 OVERLOAD neg(Z31 a) { return M31 - a; } // GWBUG: Examine all callers to see if neg call can be avoided GF31 OVERLOAD neg(GF31 a) { return U2(neg(a.x), neg(a.y)); } @@ -645,12 +840,17 @@ GF31 OVERLOAD neg(GF31 a) { return U2(neg(a.x), neg(a.y)); } Z31 OVERLOAD add(Z31 a, Z31 b) { return modM31(a + b); } GF31 OVERLOAD add(GF31 a, GF31 b) { return U2(add(a.x, b.x), add(a.y, b.y)); } -Z31 OVERLOAD sub(Z31 a, Z31 b) { i32 t = a - b; return (t & M31) + (t >> 31); } +Z31 OVERLOAD sub(Z31 a, Z31 b) { return modM31((i32)(a - b)); } GF31 OVERLOAD sub(GF31 a, GF31 b) { return U2(sub(a.x, b.x), sub(a.y, b.y)); } Z31 OVERLOAD make_Z31(i32 a) { return (Z31) (a < 0 ? a + M31 : a); } // Handles signed values of a Z31 OVERLOAD make_Z31(u32 a) { return (Z31) (a); } // a must be in range of 0 .. M31-1 -Z31 OVERLOAD make_Z31(i64 a) { return modM31q(a); } // Handles range -2^61..2^61 +Z31 OVERLOAD make_Z31(i64 a) { // Handles range -2^61..2^61 + u32 alo = (u32)a & M31; + i32 ahi = (i32)((u64)a >> 31); // Unsigned shift might be faster than signed shift + ahi = optional_add(ahi, M31); // Make ahi positive + return modM31((u32)ahi + alo); +} u32 get_Z31(Z31 a) { return a == M31 ? 0 : a; } // Get value in range 0 to M31-1 i32 get_balanced_Z31(Z31 a) { return (a & 0xC0000000) ? (i32) a - M31 : (i32) a; } // Get balanced value in range -M31/2 to M31/2 @@ -661,7 +861,7 @@ GF31 OVERLOAD shr(GF31 a, u32 k) { return U2(shr(a.x, k), shr(a.y, k)); } Z31 OVERLOAD shl(Z31 a, u32 k) { return shr(a, 31 - k); } GF31 OVERLOAD shl(GF31 a, u32 k) { return U2(shl(a.x, k), shl(a.y, k)); } -Z31 OVERLOAD mul(Z31 a, Z31 b) { u64 t = a * (u64) b; return modM31(add((Z31)(t & M31), (Z31)(t >> 31))); } +Z31 OVERLOAD mul(Z31 a, Z31 b) { u64 t = a * (u64) b; return modM31(t, 62); } // Multiply by 2 Z31 OVERLOAD mul2(Z31 a) { return add(a, a); } @@ -671,39 +871,79 @@ GF31 OVERLOAD mul2(GF31 a) { return U2(mul2(a.x), mul2(a.y)); } GF31 OVERLOAD conjugate(GF31 a) { return U2(a.x, neg(a.y)); } // Complex square. input, output 31 bits. Uses (a + i*b)^2 == ((a+b)*(a-b) + i*2*a*b). +#if 0 GF31 OVERLOAD csq(GF31 a) { - u64 r = (a.x + a.y) * (u64) (a.x + neg(a.y)); // 64-bit value, max = FFFF FFFE 0000 0004 (actually cannot exceed 9000 0000 0000 0000) - u64 i = (a.x + a.x) * (u64) a.y; // 63-bit value, max = 7FFF FFFE 0000 0002 - return U2(modM31(r), modM31(i)); + u64 r = (a.x + a.y) * (u64) (a.x + neg(a.y)); // 64-bit value, max = FFFF FFFE 8000 0003 (actually cannot exceed 9000 0000 0000 0000) + u64 i = (a.x + a.x) * (u64) a.y; // 63-bit value, max = 7FFF FFFE 8000 0001 + return U2(modM31(r), modM31(i, 63)); } +#else +GF31 OVERLOAD csq(GF31 a) { + u64 r = mad32(a.x, a.x, neg(a.y) * (u64) a.y); // Max value is 2*M31^2 = 7FFF FFFE 0000 0002 + u64 i = (a.x + a.x) * (u64) a.y; // Max value is 2*M31^2 = 7FFF FFFE 0000 0002 + return U2(modM31(r, 63), modM31(i, 63)); +} +#endif // a^2 + c +#if 0 GF31 OVERLOAD csq_add(GF31 a, GF31 c) { - u64 r = mad32(a.x + a.y, a.x + neg(a.y), c.x); // 64-bit value, mul max = FFFF FFFE 0000 0004 (actually cannot exceed 9000 0000 0000 0000) - u64 i = mad32(a.x + a.x, a.y, c.y); // 63-bit value, mul max = 7FFF FFFE 0000 0002 - return U2(modM31(r), modM31(i)); + u64 r = mad32(a.x + a.y, a.x + neg(a.y), c.x); // 64-bit value, mul max = FFFF FFFE 8000 0003 (actually cannot exceed 9000 0000 0000 0000) + u64 i = mad32(a.x + a.x, a.y, c.y); // 63-bit value, mul max = 7FFF FFFE 8000 0001 + return U2(modM31(r), modM31(i, 63)); } +#else +GF31 OVERLOAD csq_add(GF31 a, GF31 c) { + u64 r = mad32(a.x, a.x, mad32(neg(a.y), a.y, c.x)); // Max value is 2*M31^2+M31 = 7FFF FFFE 8000 0001 + u64 i = mad32(a.x + a.x, a.y, c.y); // Max value is 2*M31^2+M31 = 7FFF FFFE 8000 0001 + return U2(modM31(r, 63), modM31(i, 63)); +} +#endif // a^2 - c +#if 0 +GF31 OVERLOAD csq_sub(GF31 a, GF31 c) { + u64 r = mad32(a.x + a.y, a.x + neg(a.y), neg(c.x)); // 64-bit value, mul max = FFFF FFFE 8000 0003 (actually cannot exceed 9000 0000 0000 0000) + u64 i = mad32(a.x + a.x, a.y, neg(c.y)); // 63-bit value, mul max = 7FFF FFFE 8000 0001 + return U2(modM31(r), modM31(i, 63)); +} +#else GF31 OVERLOAD csq_sub(GF31 a, GF31 c) { - u64 r = mad32(a.x + a.y, a.x + neg(a.y), neg(c.x)); // 64-bit value, mul max = FFFF FFFE 0000 0004 (actually cannot exceed 9000 0000 0000 0000) - u64 i = mad32(a.x + a.x, a.y, neg(c.y)); // 63-bit value, mul max = 7FFF FFFE 0000 0002 - return U2(modM31(r), modM31(i)); + u64 r = mad32(a.x, a.x, mad32(neg(a.y), a.y, neg(c.x))); // Max value is 2*M31^2+M31 = 7FFF FFFE 8000 0001 + u64 i = mad32(a.x + a.x, a.y, neg(c.y)); // Max value is 2*M31^2+M31 = 7FFF FFFE 8000 0001 + return U2(modM31(r, 63), modM31(i, 63)); } +#endif // a^2 + i*c +#if 0 +GF31 OVERLOAD csq_addi(GF31 a, GF31 c) { + u64 r = mad32(a.x + a.y, a.x + neg(a.y), neg(c.y)); // 64-bit value, mul max = FFFF FFFE 8000 0003 (actually cannot exceed 9000 0000 0000 0000) + u64 i = mad32(a.x + a.x, a.y, c.x); // 63-bit value, mul max = 7FFF FFFE 8000 0001 + return U2(modM31(r), modM31(i, 63)); +} +#else GF31 OVERLOAD csq_addi(GF31 a, GF31 c) { - u64 r = mad32(a.x + a.y, a.x + neg(a.y), neg(c.y)); // 64-bit value, mul max = FFFF FFFE 0000 0004 (actually cannot exceed 9000 0000 0000 0000) - u64 i = mad32(a.x + a.x, a.y, c.x); // 63-bit value, mul max = 7FFF FFFE 0000 0002 - return U2(modM31(r), modM31(i)); + u64 r = mad32(a.x, a.x, mad32(neg(a.y), a.y, neg(c.y))); // Max value is 2*M31^2+M31 = 7FFF FFFE 8000 0001 + u64 i = mad32(a.x + a.x, a.y, c.x); // Max value is 2*M31^2+M31 = 7FFF FFFE 8000 0001 + return U2(modM31(r, 63), modM31(i, 63)); } +#endif // a^2 - i*c +#if 0 +GF31 OVERLOAD csq_subi(GF31 a, GF31 c) { + u64 r = mad32(a.x + a.y, a.x + neg(a.y), c.y); // 64-bit value, mul max = FFFF FFFE 8000 0003 (actually cannot exceed 9000 0000 0000 0000) + u64 i = mad32(a.x + a.x, a.y, neg(c.x)); // 63-bit value, max = 7FFF FFFE 8000 0001 + return U2(modM31(r), modM31(i, 63)); +} +#else GF31 OVERLOAD csq_subi(GF31 a, GF31 c) { - u64 r = mad32(a.x + a.y, a.x + neg(a.y), c.y); // 64-bit value, mul max = FFFF FFFE 0000 0004 (actually cannot exceed 9000 0000 0000 0000) - u64 i = mad32(a.x + a.x, a.y, neg(c.x)); // 63-bit value, max = 7FFF FFFE 0000 0002 - return U2(modM31(r), modM31(i)); + u64 r = mad32(a.x, a.x, mad32(neg(a.y), a.y, c.y)); // Max value is 2*M31^2+M31 = 7FFF FFFE 8000 0001 + u64 i = mad32(a.x + a.x, a.y, neg(c.x)); // Max value is 2*M31^2+M31 = 7FFF FFFE 8000 0001 + return U2(modM31(r, 63), modM31(i, 63)); } +#endif // Complex mul #if 0 // One less negation, requires signed shifts. Seems microscopically faster on TitanV. @@ -715,7 +955,7 @@ GF31 OVERLOAD cmul(GF31 a, GF31 b) { u64 k1k2 = k1 + k2; // unsigned 64-bit value, max = FFFF FFFC 0000 0004 return U2(modM31(k1k3), modM31(k1k2)); } -#else +#elif 0 GF31 OVERLOAD cmul(GF31 a, GF31 b) { u32 negbx = neg(b.x); // Negate and add b values as much as possible in case b is used several times (as in a chainmul) u64 k1 = b.x * (u64)(a.x + a.y); // 63-bit value, max = 7FFF FFFE 0000 0002 @@ -723,7 +963,28 @@ GF31 OVERLOAD cmul(GF31 a, GF31 b) { u64 k1k3 = mad32(a.y, neg(b.y) + negbx, k1); // unsigned 64-bit value, max = FFFF FFFC 0000 0004 return U2(modM31(k1k3), modM31(k1k2)); } +#else // Straight forward 4 multiply version +GF31 OVERLOAD cmul(GF31 a, GF31 b) { + u64 ayby = (u64) a.y * (u64) neg(b.y); + u64 aybx = (u64) a.y * (u64) b.x; + u64 r = mad32(a.x, b.x, ayby); // Max value is 2*M31^2 = 7FFF FFFE 0000 0002 + u64 i = mad32(a.x, b.y, aybx); // Max value is 2*M31^2 = 7FFF FFFE 0000 0002 + return U2(modM31(r, 63), modM31(i, 63)); +} +#endif + +// Complex mul where 2nd argument is a constant. Allows cheaper modM31 in some cases. +GF31 OVERLOAD cmul_const(GF31 a, GF31 b) { +#if 0 + return cmul(a, b); +#else + u64 ayby = (u64) a.y * (u64) neg(b.y); + u64 aybx = (u64) a.y * (u64) b.x; + u64 r = mad32(a.x, b.x, ayby); // Max value is (M31 - b.y + b.x)*M31 + u64 i = mad32(a.x, b.y, aybx); // Max value is (b.x + b.y)*M31 + return U2(modM31(r, (M31 - b.y + b.x <= M31) ? 62 : 63), modM31(i, (b.x + b.y <= M31) ? 62 : 63)); #endif +} // Square a root of unity complex number GF31 OVERLOAD csqTrig(GF31 a) { u32 two_ay = a.y + a.y; return U2(modM31(mad32(two_ay, neg(a.y), (u32)1)), modM31(a.x * (u64)two_ay)); } @@ -743,11 +1004,14 @@ GF31 OVERLOAD mul_3t8(GF31 a) { return U2(shl(neg(add(a.x, a.y)), 15), shl(sub(a // Return a+b and a-b void OVERLOAD X2_internal(GF31 *a, GF31 *b) { GF31 t = *a; *a = add(t, *b); *b = sub(t, *b); } +// Same as X2(a, mul_t4(b)) +void OVERLOAD X2t4_internal(GF31 *a, GF31 *b) { Z31 by = b->y; b->y = sub(a->y, b->x); a->y = add(a->y, b->x); b->x = add(a->x, by); a->x = sub(a->x, by); } + // Same as X2(a, conjugate(b)) void OVERLOAD X2conjb_internal(GF31 *a, GF31 *b) { GF31 t = *a; a->x = add(a->x, b->x); a->y = sub(a->y, b->y); b->x = sub(t.x, b->x); b->y = add(t.y, b->y); } // Same as X2(a, b), b = mul_t4(b) -void OVERLOAD X2_mul_t4_internal(GF31 *a, GF31 *b) { GF31 t = *a; *a = add(*a, *b); t.x = sub(t.x, b->x); b->x = sub(b->y, t.y); b->y = t.x; } +void OVERLOAD X2_mul_t4_internal(GF31 *a, GF31 *b) { Z31 by = b->y; b->y = sub(a->x, b->x); a->x = add(a->x, b->x); b->x = sub(by, a->y); a->y = add(a->y, by); } // Same as X2(a, b), b = mul_t8(b) void OVERLOAD X2_mul_t8_internal(GF31 *a, GF31 *b) { X2(*a, *b); *b = mul_t8(*b); } @@ -766,6 +1030,13 @@ GF31 OVERLOAD foo(GF31 a) { return foo2(a, a); } #endif +// Create "quick" routines for compatibility with shufl_and_fft2 and the GF61 data type. + +Z31 OVERLOAD addq(Z31 a, Z31 b) { return add(a, b); } +Z31 OVERLOAD subq(Z31 a, Z31 b) { return sub(a, b); } +GF31 OVERLOAD addq(GF31 a, GF31 b) { return add(a, b); } +GF31 OVERLOAD subq(GF31 a, GF31 b) { return sub(a, b); } + #endif @@ -775,144 +1046,18 @@ GF31 OVERLOAD foo(GF31 a) { return foo2(a, a); } #if NTT_GF61 -// bits in reduced mod M. -#define M61 ((((Z61) 1) << 61) - 1) - Z61 OVERLOAD make_Z61(i32 a) { return (Z61) (a < 0 ? (i64) a + M61 : (i64) a); } // Handles all values of a -Z61 OVERLOAD make_Z61(i64 a) { return (Z61) (a < 0 ? a + M61 : a); } // a must be in range of -M61 .. M61-1 +Z61 OVERLOAD make_Z61(i64 a) { return (Z61) optional_addM61(a); } // a must be in range of -M61 .. M61-1 Z61 OVERLOAD make_Z61(u32 a) { return (Z61) (a); } // Handles all values of a Z61 OVERLOAD make_Z61(u64 a) { return (Z61) (a); } // a must be in range of 0 .. M61-1 -#if 0 // Slower version that keeps results strictly in the range 0 .. M61-1 - -u64 OVERLOAD get_Z61(Z61 a) { return a; } // Get value in range 0 to M61-1 -i64 OVERLOAD get_balanced_Z61(Z61 a) { return (hi32(a) & 0xF0000000) ? (i64) a - (i64) M61 : (i64) a; } // Get balanced value in range -M61/2 to M61/2 - -Z61 OVERLOAD neg(Z61 a) { return a == 0 ? 0 : M61 - a; } // GWBUG: Examine all callers to see if neg call can be avoided -GF61 OVERLOAD neg(GF61 a) { return U2(neg(a.x), neg(a.y)); } - -Z61 OVERLOAD add(Z61 a, Z61 b) { Z61 t = a + b; Z61 m = t - M61; return (m & 0x8000000000000000ULL) ? t : m; } -//Z61 OVERLOAD add(Z61 a, Z61 b) { Z61 t = a + b; Z61 m = t - M61; return t < m ? t : m; } // Slower on TitanV -//Z61 OVERLOAD add(Z61 a, Z61 b) { Z61 t = a + b; return t - (t >= M61 ? M61 : 0); } // Slower on TitanV -GF61 OVERLOAD add(GF61 a, GF61 b) { return U2(add(a.x, b.x), add(a.y, b.y)); } - -Z61 OVERLOAD sub(Z61 a, Z61 b) { Z61 t = a - b; return t + (((i64) t >> 63) & 0x1FFFFFFFFFFFFFFFULL); } -//Z61 OVERLOAD sub(Z61 a, Z61 b) { Z61 t = a - b; Z61 p = t + M61; return (t & 0x8000000000000000ULL) ? p : t; } // Better??? -//Z61 OVERLOAD sub(Z61 a, Z61 b) { Z61 t = a - b; return t + (t >= M61 ? M61 : 0); } // Slower on TitanV -// BETTER???: t = a - b; carry_mask = sbb x, x; (generates 32 bits of 0 or 1; return t + make_carry_mask_64_bits -GF61 OVERLOAD sub(GF61 a, GF61 b) { return U2(sub(a.x, b.x), sub(a.y, b.y)); } - -// Assumes k reduced mod 61. -Z61 OVERLOAD shl(Z61 a, u32 k) { return ((a << k) + (a >> (61 - k))) & M61; } //GWBUG: Make sure & M61 operates on just one u32 -GF61 OVERLOAD shl(GF61 a, u32 k) { return U2(shl(a.x, k), shl(a.y, k)); } -Z61 OVERLOAD shr(Z61 a, u32 k) { return ((a >> k) + (a << (61 - k))) & M61; } //GWBUG: Make sure & M61 operates on just one u32. & M61 not needed? -GF61 OVERLOAD shr(GF61 a, u32 k) { return U2(shr(a.x, k), shr(a.y, k)); } - -ulong2 wideMul(u64 ab, u64 cd) { - u128 r = mul64(ab, cd); - return U2(u128_lo64(r), u128_hi64(r)); -} - -Z61 OVERLOAD mul(Z61 a, Z61 b) { - ulong2 ab = wideMul(a, b); - u64 lo = ab.x, hi = ab.y; - u64 lo61 = lo & M61, hi61 = (hi << 3) + (lo >> 61); - return add(lo61, hi61); -} - -Z61 OVERLOAD fma(Z61 a, Z61 b, Z61 c) { return add(mul(a, b), c); } // GWBUG: Can we do better? - -// Multiply by 2 -Z61 OVERLOAD mul2(Z61 a) { return ((a + a) + (a >> 60)) & M61; } // GWBUG: Make sure "+ a>>60" does an add to lower u32 without a followup adc. -GF61 OVERLOAD mul2(GF61 a) { return U2(mul2(a.x), mul2(a.y)); } - -// Return conjugate of a -GF61 OVERLOAD conjugate(GF61 a) { return U2(a.x, neg(a.y)); } - -// Complex square. input, output 61 bits. Uses (a + i*b)^2 == ((a+b)*(a-b) + i*2*a*b). -GF61 OVERLOAD csq(GF61 a) { return U2(mul(add(a.x, a.y), sub(a.x, a.y)), mul2(mul(a.x, a.y))); } //GWBUG: Probably faster to double a.y and have a mul that takes non-normalized inputs - -// a^2 + c -GF61 OVERLOAD csqa(GF61 a, GF61 c) { return add(csq(a), c); } // GWBUG: inline csq so we only "mod" after adding c?? Find a way to use fma instructions - -// Complex mul -//GF61 OVERLOAD cmul(GF61 a, GF61 b) { return U2(sub(mul(a.x, b.x), mul(a.y, b.y)), add(mul(a.x, b.y), mul(a.y, b.x)));} // GWBUG: Is a 3 multiply complex mul faster? See above -GF61 OVERLOAD cmul(GF61 a, GF61 b) { - Z61 k1 = mul(b.x, add(a.x, a.y)); - Z61 k2 = mul(a.x, sub(b.y, b.x)); - Z61 k3 = mul(a.y, add(b.y, b.x)); - return U2(sub(k1, k3), add(k1, k2)); -} - -// mul with (0, 1). (twiddle of tau/4, sqrt(-1) aka "i"). -GF61 OVERLOAD mul_t4(GF61 a) { return U2(neg(a.y), a.x); } // GWBUG: Can caller use a version that does not negate real? - -// mul with (-2^30, -2^30). (twiddle of tau/8 aka sqrt(i)). Note: 2 * (+/-2^30)^2 == 1 (mod M61). -GF61 OVERLOAD mul_t8(GF61 a) { return shl(U2(sub(a.y, a.x), neg(add(a.x, a.y))), 30); } // GWBUG: Can caller use a version that does not negate real? - -// mul with (2^30, -2^30). (twiddle of 3*tau/8). -GF61 OVERLOAD mul_3t8(GF61 a) { return shl(U2(add(a.x, a.y), sub(a.y, a.x)), 30); } - -// Return a+b and a-b -void OVERLOAD X2_internal(GF61 *a, GF61 *b) { GF61 t = *a; *a = add(t, *b); *b = sub(t, *b); } - -// Same as X2(a, conjugate(b)) -void OVERLOAD X2conjb_internal(GF61 *a, GF61 *b) { GF61 t = *a; a->x = add(a->x, b->x); a->y = sub(a->y, b->y); b->x = sub(t.x, b->x); b->y = add(t.y, b->y); } - -// Same as X2(a, b), b = mul_t4(b) -void OVERLOAD X2_mul_t4_internal(GF61 *a, GF61 *b) { GF61 t = *a; *a = add(*a, *b); t.x = sub(t.x, b->x); b->x = sub(b->y, t.y); b->y = t.x; } - -// Same as X2(a, b), b = mul_t8(b) -void OVERLOAD X2_mul_t8_internal(GF61 *a, GF61 *b) { X2(*a, *b); *b = mul_t8(*b); } - -// Same as X2(a, b), b = mul_3t8(b) -void OVERLOAD X2_mul_3t8_internal(GF61 *a, GF61 *b) { X2(*a, *b); *b = mul_3t8(*b); } - -// Same as X2(a, b), b = conjugate(b) -void OVERLOAD X2_conjb_internal(GF61 *a, GF61 *b) { GF61 t = *a; *a = add(t, *b); b->x = sub(t.x, b->x); b->y = sub(b->y, t.y); } - -void OVERLOAD SWAP_internal(GF61 *a, GF61 *b) { GF61 t = *a; *a = *b; *b = t; } - -GF61 OVERLOAD addsub(GF61 a) { return U2(add(a.x, a.y), sub(a.x, a.y)); } -GF61 OVERLOAD foo2(GF61 a, GF61 b) { a = addsub(a); b = addsub(b); return addsub(U2(mul(RE(a), RE(b)), mul(IM(a), IM(b)))); } -GF61 OVERLOAD foo(GF61 a) { return foo2(a, a); } - -// The following routines can be used to reduce mod M61 operations (in the other Z61 implementations). -// Caller must track how many M61s need to be added to make positive values for subtractions. -// In function names, "q" stands for quick, "s" stands for slow (i.e. does mod). -// These functions are untested with this strict Z61 implementation. Callers need to eliminate all uses of + or - operators. - -Z61 OVERLOAD modM61(Z61 a) { return a; } -GF61 OVERLOAD modM61(GF61 a) { return a; } -Z61 OVERLOAD neg(Z61 a, u32 m61_count) { return neg(a); } -GF61 OVERLOAD neg(GF61 a, u32 m61_count) { return neg(a); } -Z61 OVERLOAD addq(Z61 a, Z61 b) { return add(a, b); } -GF61 OVERLOAD addq(GF61 a, GF61 b) { return add(a, b); } -Z61 OVERLOAD subq(Z61 a, Z61 b, u32 m61_count) { return sub(a, b); } -GF61 OVERLOAD subq(GF61 a, GF61 b, u32 m61_count) { return sub(a, b); } -Z61 OVERLOAD subs(Z61 a, Z61 b, u32 m61_count) { return sub(a, b); } -GF61 OVERLOAD subs(GF61 a, GF61 b, u32 m61_count) { return sub(a, b); } -void OVERLOAD X2q(GF61 *a, GF61 *b, u32 m61_count) { X2_internal(a, b); } -void OVERLOAD X2q_mul_t4(GF61 *a, GF61 *b, u32 m61_count) { X2_mul_t4_internal(a, b); } -void OVERLOAD X2s(GF61 *a, GF61 *b, u32 m61_count) { X2_internal(a, b); } -void OVERLOAD X2s_conjb(GF61 *a, GF61 *b, u32 m61_count) { X2_conjb_internal(a, b); } - - - - // Philosophy: This Z61/GF61 implementation uses faster, sloppier mod M61 reduction where the end result is in the range 0..M61+epsilon. -// This implementation also handles subtractions by adding enough M61s to make a value positive. This allows us to always deal with positive -// intermediate results. The downside is that a caller using the sloppy/quick routines must keep track of how large unreduced values can get. -// An alternative implementation is to have Z61 be an i64 (costs us a precious bit of precision) but is surprisingly slower (at least on TitanV) because -// mod(a - b), where the mod routinue uses a signed right shift is slower than -// mod(a + (M61*2 - b)) where the mod routine uses an unsigned shift right. -// However, a long string of subtracts (example, fft8 does 3 subtracts before mod M61 might be better off using negative intermediate results. -// The mul routine (and obviously csq and cmul) must use only positive values as __int128 multiply is very slow. +// This implementation also handles subtractions by adding enough M61s to make a value positive. This allows us to always deal with positive intermediate results. +// However, a long string of subtracts (example, fft8 does 3 subtracts before mod M61 will be better off using the quick routines and negative intermediate results). +// The mul routine (and csq and cmul) must use only positive values as __int128 multiply is very slow. -#elif 1 // Faster version that keeps results in the range 0 .. M61+epsilon - -u64 OVERLOAD get_Z61(Z61 a) { Z61 m = a - M61; return (m & 0x8000000000000000ULL) ? a : m; } // Get value in range 0 to M61-1 -i64 OVERLOAD get_balanced_Z61(Z61 a) { return (a >= 0x1000000000000000ULL) ? (i64) a - (i64) M61 : (i64) a; } // Get balanced value in range -M61/2 to M61/2 +u64 OVERLOAD get_Z61(Z61 a) { Z61 m = a - M61; return (m & 0x8000000000000000ULL) ? a : m; } // Get value in range 0 to M61-1 +i64 OVERLOAD get_balanced_Z61(Z61 a) { return (hi32(a) >= 0x10000000) ? (i64)(a - M61) : (i64)a; } // Get balanced value in range -M61/2 to M61/2 // Internal routine to bring Z61 value into the range 0..M61+epsilon Z61 OVERLOAD modM61(Z61 a) { return (a & M61) + (a >> 61); } @@ -938,31 +1083,121 @@ Z61 OVERLOAD shl(Z61 a, u32 k) { return shr(a, 61 - k); } // Return rang //Z61 OVERLOAD shl(Z61 a, u32 k) { return modM61((a << k) + ((a >> (64 - k)) << 3)); } // Return range 0..M61+epsilon, input must be M61+epsilon a full 62-bit value can overflow GF61 OVERLOAD shl(GF61 a, u32 k) { return U2(shl(a.x, k), shl(a.y, k)); } -ulong2 wideMul(u64 ab, u64 cd) { - u128 r = mul64(ab, cd); - return U2(u128_lo64(r), u128_hi64(r)); +// Maybe we can make shl faster +Z61 OVERLOAD shl30(Z61 a) { +#if TRY_SHL30 && HAS_PTX >= 320 // shf instruction requires sm_32 support or higher + uint2 b = as_uint2(a); + uint2 r, tmp; + __asm("shf.r.clamp.b32 %0, %4, %5, 31;\n\t" // High 32 bits (wrap to LSW) + "shr.b32 %1, %5, 31;\n\t" // This is needed unless we can assume highest bit is zero (wrap to MSW) + "and.b32 %2, %4, 2147483647;\n\t" // Low 31 bits + "shr.b32 %3, %2, 2;\n\t" // Shift left 30 (29 bits end up in MSW) + "shl.b32 %2, %2, 30;\n\t" // Shift left 30 (2 bits end up in LSW) + "add.cc.u32 %0, %0, %2;\n\t" + "addc.u32 %1, %1, %3;" // Use if highest bit may not be zero + //"addc.u32 %1, 0, %3;" // Use if highest bit is assumed to be zero + : "=r"(r.x), "=r"(r.y), "=r"(tmp.x), "=r"(tmp.y) : "r"(b.x), "r"(b.y) : ); + return as_ulong(r); +#else + return shl(a, 30); +#endif +} +GF61 OVERLOAD shl30(GF61 a) { return U2(shl30(a.x), shl30(a.y)); } + +// Maybe we can make shl faster +Z61 OVERLOAD shl31(Z61 a) { +#if TRY_SHL31 && HAS_PTX >= 320 // shf instruction requires sm_32 support or higher + uint2 b = as_uint2(a); + uint2 r, tmp; + __asm("shf.r.clamp.b32 %0, %4, %5, 30;\n\t" // High 32 bits (wrap to LSW) + //"shr.b32 %1, %5, 30;\n\t" // Assume highest 2 bits are zero (wrap to MSW) + "and.b32 %2, %4, 1073741823;\n\t" // Low 30 bits + "shr.b32 %3, %2, 1;\n\t" // Shift left 31 (29 bits end up in MSW) + "shl.b32 %2, %2, 31;\n\t" // Shift left 31 (1 bit ends up in LSW) + "add.cc.u32 %0, %0, %2;\n\t" + //"addc.u32 %1, %1, %3;" // Use if highest 2 bits may not be zero + "addc.u32 %1, 0, %3;" + : "=r"(r.x), "=r"(r.y), "=r"(tmp.x), "=r"(tmp.y) : "r"(b.x), "r"(b.y) : ); + return as_ulong(r); +#else + return shl(a, 31); +#endif +} +GF61 OVERLOAD shl31(GF61 a) { return U2(shl31(a.x), shl31(a.y)); } + +u64 OVERLOAD weakModM61(u128 a, u32 num_bits) { +// This is faster on TitanV, CUDA 13.0. No difference on 5070Ti. +#if HAS_PTX >= 320 // shf instruction requires sm_32 support or higher + uint2 alo = as_uint2(u128_lo64(a)); + uint2 ahi = as_uint2(u128_hi64(a)); + if (num_bits <= 125) { + __asm("shf.r.clamp.b32 %3, %2, %3, 29;\n\t" + "shf.r.clamp.b32 %2, %1, %2, 29;\n\t" + "and.b32 %1, %1, 536870911;" + : "+r"(alo.x), "+r"(alo.y), "+r"(ahi.x), "+r"(ahi.y) : ); + return (u64)as_ulong(ahi) + (u64)as_ulong(alo); + } else { + uint top6; + __asm("shr.u32 %4, %3, 26;\n\t" + "shf.r.clamp.b32 %3, %2, %3, 29;\n\t" + "shf.r.clamp.b32 %2, %1, %2, 29;\n\t" + "and.b32 %1, %1, 536870911;\n\t" + "and.b32 %3, %3, 536870911;\n\t" + "add.cc.u32 %0, %0, %4;\n\t" + "addc.u32 %1, %1, 0;" + : "+r"(alo.x), "+r"(alo.y), "+r"(ahi.x), "+r"(ahi.y), "=r"(top6) : ); + return (u64)as_ulong(ahi) + (u64)as_ulong(alo); + } +#else + u64 lo = u128_lo64(a), hi = u128_hi64(a); + u64 lo61 = lo & M61; // Max value is M61 + if (num_bits <= 125) { + hi = (hi << 3) + (lo >> 61); + return lo61 + hi; // Caller must insure this does not overflow + } else { + u64 hi61 = ((hi << 3) + (lo >> 61)) & M61; // Max value is M61 + return lo61 + hi61 + (hi >> 58); // Max value is 2*M61 + epsilon + } +#endif } // Returns a * b not modded by M61. Max value of result depends on the m61_counts of the inputs. // Let n = (a_m61_count - 1) * (b_m61_count - 1). This is the maximum value in the highest 6 bits of a * b. -// If n <= 4 result will be at most (n+1)*M61+epsilon. -// If n > 4 result will be at most 2*M61+epsilon. +// If n <= 6 result will be at most (n+1)*M61+epsilon. +// If n > 6 result will be at most 2*M61+epsilon. Z61 OVERLOAD weakMul(Z61 a, Z61 b, const u32 a_m61_count, const u32 b_m61_count) { - ulong2 ab = wideMul(a, b); - u64 lo = ab.x, hi = ab.y; - u64 lo61 = lo & M61; // Max value is M61 - if ((a_m61_count - 1) * (b_m61_count - 1) <= 4) { - hi = (hi << 3) + (lo >> 61); // Max value is (a_m61_count - 1) * (b_m61_count - 1) * M61 + epsilon - return lo61 + hi; // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 1) * M61 + epsilon + u128 ab = mul64(a, b); // Max value is (a_m61_count - 1) * (b_m61_count - 1) * M61^2 + epsilon + if ((a_m61_count - 1) * (b_m61_count - 1) <= 6) { + return weakModM61(ab, 125); // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 1) * M61 + epsilon } else { - u64 hi61 = ((hi << 3) + (lo >> 61)) & M61; // Max value is M61 - return lo61 + hi61 + (hi >> 58); // Max value is 2*M61 + epsilon + return weakModM61(ab, 128); // Max value is 2*M61 + epsilon + } +} +Z61 OVERLOAD weakMulAdd(Z61 a, Z61 b, u64 c, const u32 a_m61_count, const u32 b_m61_count) { + u128 ab = mad64(a, b, c); // Max value is (a_m61_count - 1) * (b_m61_count - 1) * M61^2 + epsilon + if ((a_m61_count - 1) * (b_m61_count - 1) <= 6) { + return weakModM61(ab, 125); // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 1) * M61 + epsilon + } else { + return weakModM61(ab, 128); // Max value is 2*M61 + epsilon + } +} +Z61 OVERLOAD weakMulAdd(Z61 a, Z61 b, u128 c, const u32 a_m61_count, const u32 b_m61_count) { // Max c value assumed to be 2*M61^2+epsilon + u128 ab = mad64(a, b, c); // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 2) * M61^2 + epsilon + if ((a_m61_count - 1) * (b_m61_count - 1) + 2 <= 6) { + return weakModM61(ab, 125); // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 3) * M61 + epsilon + } else { + return weakModM61(ab, 128); // Max value is 2*M61 + epsilon } } Z61 OVERLOAD mul(Z61 a, Z61 b) { return modM61(weakMul(a, b, 2, 2)); } -Z61 OVERLOAD fma(Z61 a, Z61 b, Z61 c) { return modM61(weakMul(a, b, 2, 2) + c); } // GWBUG: Can we do better? +// Not named fma(): declaring a user overload of fma() hides the OpenCL builtin +// fma(float,float,float), so in a build with both FFT_FP32 and NTT_GF61 the float call sites +// (chainMul4() in fftbase.cl, for one) resolve to this overload instead, convert their float +// arguments to ulong, and return ulong2 where float2 is expected. This overload has no call +// sites, so the name is free to change. +Z61 OVERLOAD fmaZ61(Z61 a, Z61 b, Z61 c) { return modM61(weakMulAdd(a, b, c, 2, 2)); } // Multiply by 2 Z61 OVERLOAD mul2(Z61 a) { return add(a, a); } @@ -972,45 +1207,47 @@ GF61 OVERLOAD mul2(GF61 a) { return U2(mul2(a.x), mul2(a.y)); } GF61 OVERLOAD conjugate(GF61 a) { return U2(a.x, neg(a.y)); } // Complex square. Uses (a + i*b)^2 == ((a+b)*(a-b) + i*2*a*b). -GF61 OVERLOAD csqq(GF61 a, const u32 m61_count) { - if (m61_count > 4) return csqq(modM61(a), 2); - Z61 re = weakMul(a.x + a.y, a.x + neg(a.y, m61_count), 2 * m61_count - 1, 2 * m61_count); - Z61 im = weakMul(a.x + a.x, a.y, 2 * m61_count - 1, m61_count); +GF61 OVERLOAD csqq(GF61 a, const u32 x_m61_count, const u32 y_m61_count) { + if (x_m61_count + y_m61_count >= 9) return csqq(modM61(a), 2, 2); + Z61 re = weakMul(a.x + a.y, a.x + neg(a.y, y_m61_count), x_m61_count + y_m61_count - 1, x_m61_count + y_m61_count); + Z61 im = (x_m61_count <= y_m61_count) ? weakMul(a.x + a.x, a.y, x_m61_count + x_m61_count - 1, y_m61_count) : + weakMul(a.x, a.y + a.y, x_m61_count, y_m61_count + y_m61_count - 1); return U2(re, im); } -GF61 OVERLOAD csqs(GF61 a, const u32 m61_count) { return modM61(csqq(a, m61_count)); } -GF61 OVERLOAD csq(GF61 a) { return csqs(a, 2); } +GF61 OVERLOAD csqq(GF61 a, const u32 m61_count) { return csqq(a, m61_count, m61_count); } +GF61 OVERLOAD csq(GF61 a, const u32 x_m61_count, const u32 y_m61_count) { return modM61(csqq(a, x_m61_count, y_m61_count)); } +GF61 OVERLOAD csq(GF61 a, const u32 m61_count) { return csq(a, m61_count, m61_count); } +GF61 OVERLOAD csq(GF61 a) { return csq(a, 2); } // a^2 + c -GF61 OVERLOAD csqa(GF61 a, GF61 c) { return U2(modM61(weakMul(a.x + a.y, a.x + neg(a.y, 2), 3, 4) + c.x), modM61(weakMul(a.x + a.x, a.y, 3, 2) + c.y)); } +GF61 OVERLOAD csqaq(GF61 a, GF61 c, const u32 x_m61_count, const u32 y_m61_count) { + if (x_m61_count + y_m61_count >= 9) return csqaq(modM61(a), c, 2, 2); + Z61 re = weakMulAdd(a.x + a.y, a.x + neg(a.y, y_m61_count), c.x, x_m61_count + y_m61_count - 1, x_m61_count + y_m61_count); + Z61 im = (x_m61_count <= y_m61_count) ? weakMulAdd(a.x + a.x, a.y, c.y, x_m61_count + x_m61_count - 1, y_m61_count) : + weakMulAdd(a.x, a.y + a.y, c.y, x_m61_count, y_m61_count + y_m61_count - 1); + return U2(re, im); +} +GF61 OVERLOAD csqaq(GF61 a, GF61 c, const u32 m61_count) { return csqaq(a, c, m61_count, m61_count); } +GF61 OVERLOAD csqa(GF61 a, GF61 c, const u32 x_m61_count, const u32 y_m61_count) { return modM61(csqaq(a, c, x_m61_count, y_m61_count)); } +GF61 OVERLOAD csqa(GF61 a, GF61 c, const u32 m61_count) { return csqa(a, c, m61_count, m61_count); } +GF61 OVERLOAD csqa(GF61 a, GF61 c) { return csqa(a, c, 2); } // Complex mul -#if 0 -GF61 OVERLOAD cmul(GF61 a, GF61 b) { // Use 3-epsilon extra bits in u64 - Z61 k1 = weakMul(b.x, a.x + a.y, 2, 3); // max value is 3*M61+epsilon - Z61 k2 = weakMul(a.x, b.y + neg(b.x, 2), 2, 3); // max value is 3*M61+epsilon - Z61 k3 = weakMul(a.y, b.y + b.x, 2, 3); // max value is 3*M61+epsilon - return U2(modM61(k1 + neg(k3, 4)), modM61(k1 + k2)); -} -#else -Z61 OVERLOAD weakMulAdd(Z61 a, Z61 b, u128 c, const u32 a_m61_count, const u32 b_m61_count) { // Max c value assumed to be 2*M61^2+epsilon - u128 ab = mad64(a, b, c); // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 2) * M61^2 + epsilon - u64 lo = u128_lo64(ab), hi = u128_hi64(ab); - u64 lo61 = lo & M61; // Max value is M61 - if ((a_m61_count - 1) * (b_m61_count - 1) + 2 <= 6) { - hi = (hi << 3) + (lo >> 61); // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 2) * M61 + epsilon - return lo61 + hi; // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 3) * M61 + epsilon - } else { - u64 hi61 = ((hi << 3) + (lo >> 61)) & M61; // Max value is M61 - return lo61 + hi61 + (hi >> 58); // Max value is 2*M61 + epsilon - } -} +#if 1 GF61 OVERLOAD cmul(GF61 a, GF61 b) { u128 k1 = mul64(b.x, a.x + a.y); // max value is 2*M61^2+epsilon Z61 k1k2 = weakMulAdd(a.x, b.y + neg(b.x, 2), k1, 2, 4); // max value is 6*M61+epsilon Z61 k1k3 = weakMulAdd(a.y, neg(b.y + b.x, 3), k1, 2, 4); // max value is 6*M61+epsilon return U2(modM61(k1k3), modM61(k1k2)); } +#else +GF61 OVERLOAD cmul(GF61 a, GF61 b) { + u128 ayby = mul64(a.y, neg(b.y, 2)); // max value is 2*M61^2+epsilon + u128 r = mad64(a.x, b.x, ayby); // max value is 3*M61^2+epsilon + u128 aybx = mul64(a.y, b.x); // max value is 1*M61^2+epsilon + u128 i = mad64(a.x, b.y, aybx); // max value is 2*M61^2+epsilon + return U2(add(u128_lo64(r) & M61, u128_shrlo64(r, 61)), add(u128_lo64(i) & M61, u128_shrlo64(i, 61))); +} #endif // Square a root of unity complex number (the second version may be faster if the compiler optimizes the u128 squaring). @@ -1029,24 +1266,77 @@ GF61 OVERLOAD subi(GF61 a, GF61 b) { return U2(add(a.x, b.y), sub(a.y, b.x)); } GF61 OVERLOAD mul_t4(GF61 a) { return U2(neg(a.y), a.x); } // GWBUG: Can caller use a version that does not negate real? // mul with (-2^30, -2^30). (twiddle of tau/8 aka sqrt(i)). Note: 2 * (+/-2^30)^2 == 1 (mod M61). -GF61 OVERLOAD mul_t8(GF61 a, const u32 m61_count) { return shl(U2(a.y + neg(a.x, m61_count), neg(a.x + a.y, 2 * m61_count - 1)), 30); } +GF61 OVERLOAD mul_t8(GF61 a, const u32 m61_count) { return shl30(U2(a.y + neg(a.x, m61_count), neg(a.x + a.y, 2 * m61_count - 1))); } GF61 OVERLOAD mul_t8(GF61 a) { return mul_t8(a, 2); } // mul with (2^30, -2^30). (twiddle of 3*tau/8). -GF61 OVERLOAD mul_3t8(GF61 a, const u32 m61_count) { return shl(U2(a.x + a.y, a.y + neg(a.x, m61_count)), 30); } +GF61 OVERLOAD mul_3t8(GF61 a, const u32 m61_count) { return shl30(U2(a.x + a.y, a.y + neg(a.x, m61_count))); } GF61 OVERLOAD mul_3t8(GF61 a) { return mul_3t8(a, 2); } +// mul with twiddles of (1,3,5,7,9)*tau/16 +// Define C1 = 22027337052962166, S1 = 1693317751237720973, negC1 = M61 - C1, negS1 = M61 - S1. +// Twiddle for 1t16 is U2(C1, S1), 3t16 is U2(S1, C1), 5t16 is U2(negS1, C1), 7t16 is U2(negC1, S1), 9t16 is U2(negC1, negS1). +#if TRY_SQRT2 +// NOTE: C1/S1 = SQRT2 + 1 and S1/C1 = SQRT2 - 1. This lets us compute the cmul in two steps. For example, mul_t16 can be (a * U2(1, S1/C1)) * C1. +// Mul by SQRT2 is a mul by -2^31 which is done with a shift rather than a multiply. This reduces the total number of 64-bit multiplies (but shifts aren't cheap either). +// As a bonus, C1 is a 55-bit value which means a modM61 reduction after weakMul should not be necessary. +GF61 OVERLOAD mul_t16(GF61 a) { + // a * U2(1, S1/C1) = axbx - ayby, axby + aybx + // = ax - (ay * (SQRT2 - 1)), ax * (SQRT2 - 1) + ay + // = ax - ay * SQRT2 + ay, ax * SQRT2 - ax + ay + GF61 a_negsqrt2 = shl31(a); // Mul by (2^31 = -SQRT2) + return U2(weakMul(a.x + a_negsqrt2.y + a.y, 22027337052962166ULL, 2, 2), weakMul(neg(a_negsqrt2.x + a.x, 3) + a.y, 22027337052962166ULL, 2, 2)); +} +GF61 OVERLOAD mul_3t16(GF61 a) { + // a * U2(S1/C1, 1) = axbx - ayby, axby + aybx + // = ax * (SQRT2 - 1) - ay, ax + ay * (SQRT2 - 1) + // = ax * SQRT2 - ax - ay, ax + ay * SQRT2 - ay + GF61 a_negsqrt2 = shl31(a); // Mul by (2^31 = -SQRT2) + return U2(weakMul(neg(a_negsqrt2.x + a.x + a.y, 4), 22027337052962166ULL, 2, 2), weakMul(a.x + neg(a_negsqrt2.y + a.y, 3), 22027337052962166ULL, 2, 2)); +} +GF61 OVERLOAD mul_5t16(GF61 a) { + // a * U2(-S1/C1, 1) = axbx - ayby, axby + aybx + // = ax * -(SQRT2 - 1) - ay, ax + ay * -(SQRT2 - 1) + // = -ax * SQRT2 + ax - ay, ax - ay * SQRT2 + ay + GF61 a_negsqrt2 = shl31(a); // Mul by (2^31 = -SQRT2) + return U2(weakMul(a_negsqrt2.x + a.x + neg(a.y, 2), 22027337052962166ULL, 2, 2), weakMul(a.x + a_negsqrt2.y + a.y, 22027337052962166ULL, 2, 2)); +} +GF61 OVERLOAD mul_7t16(GF61 a) { + // a * U2(-1, S1/C1) = axbx - ayby, axby + aybx + // = -ax - (ay * (SQRT2 - 1)), ax * (SQRT2 - 1) - ay + // = -ax - ay * SQRT2 + ay, ax * SQRT2 - ax - ay + GF61 a_negsqrt2 = shl31(a); // Mul by (2^31 = -SQRT2) + return U2(weakMul(neg(a.x, 2) + a_negsqrt2.y + a.y, 22027337052962166ULL, 2, 2), weakMul(neg(a_negsqrt2.x + a.x + a.y, 4), 22027337052962166ULL, 2, 2)); +} +GF61 OVERLOAD mul_9t16(GF61 a) { + // a * U2(-1, -S1/C1) = axbx - ayby, axby + aybx + // = -ax - (ay * -(SQRT2 - 1)), ax * -(SQRT2 - 1) - ay + // = -ax + ay * SQRT2 - ay, -ax * SQRT2 + ax - ay + GF61 a_negsqrt2 = shl31(a); // Mul by (2^31 = -SQRT2) + return U2(weakMul(neg(a.x + a_negsqrt2.y + a.y, 4), 22027337052962166ULL, 2, 2), weakMul(a_negsqrt2.x + a.x + neg(a.y, 2), 22027337052962166ULL, 2, 2)); +} +#else +GF61 OVERLOAD mul_t16(GF61 a) { return cmul(a, U2(22027337052962166ULL, 1693317751237720973ULL)); } +GF61 OVERLOAD mul_3t16(GF61 a) { return cmul(a, U2(1693317751237720973ULL, 22027337052962166ULL)); } +GF61 OVERLOAD mul_5t16(GF61 a) { return cmul(a, U2(M61 - 1693317751237720973ULL, 22027337052962166ULL)); } +GF61 OVERLOAD mul_7t16(GF61 a) { return cmul(a, U2(M61 - 22027337052962166ULL, 1693317751237720973ULL)); } +GF61 OVERLOAD mul_9t16(GF61 a) { return cmul(a, U2(M61 - 22027337052962166ULL, M61 - 1693317751237720973ULL)); } +#endif + // Return a+b and a-b void OVERLOAD X2_internal(GF61 *a, GF61 *b) { GF61 t = *a; *a = add(t, *b); *b = sub(t, *b); } +// Same as X2(a, mul_t4(b)) +void OVERLOAD X2t4_internal(GF61 *a, GF61 *b) { Z61 by = b->y; b->y = sub(a->y, b->x); a->y = add(a->y, b->x); b->x = add(a->x, by); a->x = sub(a->x, by); } + // Same as X2(a, conjugate(b)) void OVERLOAD X2conjb_internal(GF61 *a, GF61 *b) { GF61 t = *a; a->x = add(a->x, b->x); a->y = sub(a->y, b->y); b->x = sub(t.x, b->x); b->y = add(t.y, b->y); } // Same as X2(a, b), b = mul_t4(b) -void OVERLOAD X2_mul_t4_internal(GF61 *a, GF61 *b) { GF61 t = *a; *a = add(*a, *b); t.x = sub(t.x, b->x); b->x = sub(b->y, t.y); b->y = t.x; } +void OVERLOAD X2_mul_t4_internal(GF61 *a, GF61 *b) { Z61 by = b->y; b->y = sub(a->x, b->x); a->x = add(a->x, b->x); b->x = sub(by, a->y); a->y = add(a->y, by); } // Same as X2(a, b), b = mul_t8(b) -void OVERLOAD X2_mul_t8_internal(GF61 *a, GF61 *b) { GF61 t = *a; *a = add(t, *b); t = *b + neg(t, 2); *b = shl(U2(t.x + neg(t.y, 4), t.x + t.y), 30); } +void OVERLOAD X2_mul_t8_internal(GF61 *a, GF61 *b) { GF61 t = *a; *a = add(t, *b); t = *b + neg(t, 2); *b = shl30(U2(t.x + neg(t.y, 4), t.x + t.y)); } // Same as X2(a, b), b = mul_3t8(b) void OVERLOAD X2_mul_3t8_internal(GF61 *a, GF61 *b) { GF61 t = *a; *a = add(t, *b); *b = t + neg(*b, 2); *b = mul_3t8(*b, 4); } @@ -1060,29 +1350,31 @@ GF61 OVERLOAD addsub(GF61 a) { return U2(add(a.x, a.y), sub(a.x, a.y)); } GF61 OVERLOAD foo2(GF61 a, GF61 b) { a = addsub(a); b = addsub(b); return addsub(U2(mul(RE(a), RE(b)), mul(IM(a), IM(b)))); } GF61 OVERLOAD foo(GF61 a) { return foo2(a, a); } -// The following routines can be used to reduce mod M61 operations. Caller must track how many M61s need to be added to make positive -// values for subtractions. In function names, "q" stands for quick (no modM61), "s" stands for slow (i.e. does modM61). +// The following routines can be used to reduce mod M61 operations by carefully tracking the range of intermediate results which can be negative. +// This reduces the number of m61_count*M61 addition operations too. By tracking ranges of intermediate results, the caller knows how many M61s +// need to be added to make result positive prior to the final modM61. In function names, "q" stands for quick (no modM61). Z61 OVERLOAD addq(Z61 a, Z61 b) { return a + b; } -GF61 OVERLOAD addq(GF61 a, GF61 b) { return U2(addq(a.x, b.x), addq(a.y, b.y)); } - -Z61 OVERLOAD subq(Z61 a, Z61 b, const u32 m61_count) { return a + neg(b, m61_count); } -GF61 OVERLOAD subq(GF61 a, GF61 b, const u32 m61_count) { return U2(subq(a.x, b.x, m61_count), subq(a.y, b.y, m61_count)); } - -Z61 OVERLOAD subs(Z61 a, Z61 b, const u32 m61_count) { return modM61(a + neg(b, m61_count)); } -GF61 OVERLOAD subs(GF61 a, GF61 b, const u32 m61_count) { return U2(subs(a.x, b.x, m61_count), subs(a.y, b.y, m61_count)); } - -GF61 OVERLOAD addiq(GF61 a, GF61 b, const u32 m61_count) { return U2(subq(a.x, b.y, m61_count), addq(a.y, b.x)); } -GF61 OVERLOAD subiq(GF61 a, GF61 b, const u32 m61_count) { return U2(addq(a.x, b.y), subq(a.y, b.x, m61_count)); } - -void OVERLOAD X2q(GF61 *a, GF61 *b, const u32 m61_count) { GF61 t = *a; *a = t + *b; *b = t + neg(*b, m61_count); } -void OVERLOAD X2q_mul_t4(GF61 *a, GF61 *b, const u32 m61_count) { GF61 t = *a; *a = t + *b; t.x = t.x + neg(b->x, m61_count); b->x = b->y + neg(t.y, m61_count); b->y = t.x; } -void OVERLOAD X2q_mul_t8(GF61 *a, GF61 *b, const u32 m61_count) { GF61 t = *a; *a = t + *b; t = *b + neg(t, m61_count); *b = shl(U2(t.x + neg(t.y, m61_count * 2), t.x + t.y), 30); } -void OVERLOAD X2q_mul_3t8(GF61 *a, GF61 *b, const u32 m61_count) { GF61 t = *a; *a = t + *b; t = t + neg(*b, m61_count); *b = mul_3t8(t, m61_count * 2); } - -void OVERLOAD X2s(GF61 *a, GF61 *b, const u32 m61_count) { GF61 t = *a; *a = add(t, *b); *b = subs(t, *b, m61_count); } -void OVERLOAD X2s_conjb(GF61 *a, GF61 *b, const u32 a_m61_count, const u32 b_m61_count) { GF61 t = *a; *a = add(t, *b); b->x = subs(t.x, b->x, b_m61_count); b->y = subs(b->y, t.y, a_m61_count); } - -#endif +Z61 OVERLOAD subq(Z61 a, Z61 b) { return a - b; } +GF61 OVERLOAD addq(GF61 a, GF61 b) { return a + b; } +GF61 OVERLOAD subq(GF61 a, GF61 b) { return a - b; } +GF61 OVERLOAD addiq(GF61 a, GF61 b) { return U2(a.x - b.y, a.y + b.x); } +GF61 OVERLOAD subiq(GF61 a, GF61 b) { return U2(a.x + b.y, a.y - b.x); } + +void OVERLOAD X2q(GF61 *a, GF61 *b) { GF61 t = *a; *a = t + *b; *b = t - *b; } +void OVERLOAD X2qt4(GF61 *a, GF61 *b) { Z61 by = b->y; b->y = a->y - b->x; a->y = a->y + b->x; b->x = a->x + by; a->x = a->x - by; } +void OVERLOAD X2qconjb(GF61 *a, GF61 *b) { GF61 t = *a; a->x += b->x; a->y -= b->y; b->x = t.x - b->x; b->y = t.y + b->y; } +void OVERLOAD X2q_mul_t4(GF61 *a, GF61 *b) { Z61 by = b->y; b->y = a->x - b->x; a->x = a->x + b->x; b->x = by - a->y; a->y = a->y + by; } +void OVERLOAD X2q_conjb(GF61 *a, GF61 *b) { GF61 t = *a; *a = t + *b; b->x = t.x - b->x; b->y = b->y - t.y; } + +GF61 OVERLOAD mul_t8q(GF61 a, const u32 m61_count) { return shl30(U2(m61_count * M61 + (a.y - a.x), m61_count * M61 - (a.x + a.y))); } +GF61 OVERLOAD mul_3t8q(GF61 a, const u32 m61_count) { return shl30(U2(m61_count * M61 + a.x + a.y, m61_count * M61 + (a.y - a.x))); } + +Z61 OVERLOAD optsubqu(Z61 a, const u32 m61_limit, const u32 m61_count) { return optional_sub((u64)a, (u32)(m61_limit << (61 - 32)), (u64)(m61_count * M61)); } +GF61 OVERLOAD optsubqu(GF61 a, const u32 m61_limit, const u32 m61_count) { return U2(optsubqu(a.x, m61_limit, m61_count), optsubqu(a.y, m61_limit, m61_count)); } +Z61 OVERLOAD optsubqs(Z61 a, const u32 m61_limit, const u32 m61_count) { return optional_sub((i64)a, (i32)(m61_limit << (61 - 32)), (i64)(m61_count * M61)); } +GF61 OVERLOAD optsubqs(GF61 a, const u32 m61_limit, const u32 m61_count) { return U2(optsubqs(a.x, m61_limit, m61_count), optsubqs(a.y, m61_limit, m61_count)); } +GF61 OVERLOAD modM61q(GF61 a, const u32 m61_count) { if (m61_count) { a.x += m61_count * M61; a.y += m61_count * M61; } return modM61(a); } +GF61 OVERLOAD modM61q(GF61 a, const u32 m61_count_x, const u32 m61_count_y) { if (m61_count_x) a.x += m61_count_x * M61; if (m61_count_y) a.y += m61_count_y * M61; return modM61(a); } #endif diff --git a/src/cl/middle.cl b/src/cl/middle.cl index 263937f3..b52cea27 100644 --- a/src/cl/middle.cl +++ b/src/cl/middle.cl @@ -34,9 +34,30 @@ #define MIDDLE_OUT_LDS_TRANSPOSE 1 #endif +// These were the original read/write routines for accessing FFT data. I'm not sure if they are used anymore. + +#ifdef T2_F2_GF31_GF61 +void OVERLOAD read(u32 WG_SZ, u32 N, T2_F2_GF31_GF61 *u, const global T2_F2_GF31_GF61 *in, u32 base) { + in += base + (u32) get_local_id(0); + for (u32 i = 0; i < N; ++i) { u[i] = FFTLOAD(&in[i * WG_SZ]); } +} + +// Same, but for a kernel whose workgroup spans more than one line: get_local_id(0) is then not the lane +// within the line, and the caller has to say which lane it means. +void OVERLOAD read(u32 WG_SZ, u32 N, T2_F2_GF31_GF61 *u, const global T2_F2_GF31_GF61 *in, u32 base, u32 lane) { + in += base + lane; + for (u32 i = 0; i < N; ++i) { u[i] = FFTLOAD(&in[i * WG_SZ]); } +} + +void OVERLOAD write(u32 WG_SZ, u32 N, T2_F2_GF31_GF61 *u, global T2_F2_GF31_GF61 *out, u32 base) { + out += base + (u32) get_local_id(0); + for (u32 i = 0; i < N; ++i) { FFTSTORE(&out[i * WG_SZ], u[i]); } +} +#endif + #if !INPLACE // Original implementation (not in place) -#if FFT_FP64 || NTT_GF61 +#ifdef T2_GF61 //**************************************************************************************** // Pair of routines to write data from carryFused and read data into fftMiddleIn @@ -54,27 +75,27 @@ // u[i] i ranges 0...MIDDLE-1 (multiples of SMALL_HEIGHT) // y ranges 0...SMALL_HEIGHT-1 (multiples of one) -void OVERLOAD writeCarryFusedLine(T2 *u, P(T2) out, u32 line) { +void OVERLOAD writeCarryFusedLine(T2_GF61 *u, P(T2_GF61) out, u32 line, u32 me) { #if PAD_SIZE > 0 u32 BIG_PAD_SIZE = (PAD_SIZE/2+1)*PAD_SIZE; - out += line * WIDTH + line * PAD_SIZE + line / SMALL_HEIGHT * BIG_PAD_SIZE + (u32) get_local_id(0); // One pad every line + a big pad every SMALL_HEIGHT lines - for (u32 i = 0; i < NW; ++i) { NTSTORE(out[i * G_W], u[i]); } + out += line * WIDTH + line * PAD_SIZE + line / SMALL_HEIGHT * BIG_PAD_SIZE + me; // One pad every line + a big pad every SMALL_HEIGHT lines + for (u32 i = 0; i < NW; ++i) { FFTSTORE(&out[i * G_W], u[i]); } #else - out += line * WIDTH + (u32) get_local_id(0); - for (u32 i = 0; i < NW; ++i) { NTSTORE(out[i * G_W], u[i]); } + out += line * WIDTH + me; + for (u32 i = 0; i < NW; ++i) { FFTSTORE(&out[i * G_W], u[i]); } #endif } -void OVERLOAD readMiddleInLine(T2 *u, CP(T2) in, u32 y, u32 x) { +void OVERLOAD readMiddleInLine(T2_GF61 *u, CP(T2_GF61) in, u32 y, u32 x) { #if PAD_SIZE > 0 // Each work group reads successive y's which increments by one pad size. // Rather than having u[i] also increment by one, we choose a larger pad increment u32 BIG_PAD_SIZE = (PAD_SIZE/2+1)*PAD_SIZE; in += y * WIDTH + y * PAD_SIZE + (y / SMALL_HEIGHT) * BIG_PAD_SIZE + x; - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * (SMALL_HEIGHT * (WIDTH + PAD_SIZE) + BIG_PAD_SIZE)]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * (SMALL_HEIGHT * (WIDTH + PAD_SIZE) + BIG_PAD_SIZE)]); } #else in += y * WIDTH + x; - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * SMALL_HEIGHT * WIDTH]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SMALL_HEIGHT * WIDTH]); } #endif } @@ -90,7 +111,7 @@ void OVERLOAD readMiddleInLine(T2 *u, CP(T2) in, u32 y, u32 x) { // x ranges 0...SMALL_HEIGHT-1 (multiples of one) (also known as 0...G_H-1 and 0...NH-1) // y ranges 0...MIDDLE*WIDTH-1 (multiples of SMALL_HEIGHT) -void OVERLOAD writeMiddleInLine (P(T2) out, T2 *u, u32 chunk_y, u32 chunk_x) +void OVERLOAD writeMiddleInLine(P(T2_GF61) out, T2_GF61 *u, u32 chunk_y, u32 chunk_x) { //u32 SIZEY = IN_WG / IN_SIZEX; //u32 num_x_chunks = WIDTH / IN_SIZEX; // Number of x chunks @@ -108,7 +129,7 @@ void OVERLOAD writeMiddleInLine (P(T2) out, T2 *u, u32 chunk_y, u32 chunk_x) // = SMALL_HEIGHT / (IN_WG / IN_SIZEX) * (MIDDLE * IN_WG + PAD_SIZE) // = SMALL_HEIGHT * MIDDLE * IN_SIZEX + SMALL_HEIGHT / SIZEY * PAD_SIZE // Write each u[i] sequentially - for (int i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * IN_WG], u[i]); } + for (int i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * IN_WG], u[i]); } #else @@ -118,14 +139,14 @@ void OVERLOAD writeMiddleInLine (P(T2) out, T2 *u, u32 chunk_y, u32 chunk_x) // = MIDDLE * SMALL_HEIGHT / (IN_WG / IN_SIZEX) * IN_WG // = MIDDLE * SMALL_HEIGHT * IN_SIZEX // Write each u[i] sequentially - for (int i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * IN_WG], u[i]); } + for (int i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * IN_WG], u[i]); } #endif } // Read a line for tailFused or fftHin // This reads partially transposed data as written by fftMiddleIn -void OVERLOAD readTailFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { +void OVERLOAD readTailFusedLine(CP(T2_GF61) in, T2_GF61 *u, u32 line, u32 me) { u32 SIZEY = IN_WG / IN_SIZEX; #if PAD_SIZE > 0 @@ -151,7 +172,7 @@ void OVERLOAD readTailFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { for (i32 i = 0; i < NH; ++i) { // u32 fftMiddleIn_y = i * G_H + me; // The fftMiddleIn y value // u32 chunk_y = fftMiddleIn_y / SIZEY; // The fftMiddleIn chunk_y value - u[i] = NTLOAD(in[chunk_y * (MIDDLE * IN_WG + PAD_SIZE)]); // Adjust in pointer the same way writeMiddleInLine did + u[i] = FFTLOAD(&in[chunk_y * (MIDDLE * IN_WG + PAD_SIZE)]); // Adjust in pointer the same way writeMiddleInLine did chunk_y += chunk_y_incr; } @@ -177,7 +198,7 @@ void OVERLOAD readTailFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { for (i32 i = 0; i < NH; ++i) { u32 fftMiddleIn_y = i * G_H + me; // The fftMiddleIn y value u32 chunk_y = fftMiddleIn_y / SIZEY; // The fftMiddleIn chunk_y value - u[i] = NTLOAD(in[chunk_y * (MIDDLE * IN_WG)]); // Adjust in pointer the same way writeMiddleInLine did + u[i] = FFTLOAD(&in[chunk_y * (MIDDLE * IN_WG)]); // Adjust in pointer the same way writeMiddleInLine did chunk_y += chunk_y_incr; } @@ -200,7 +221,7 @@ void OVERLOAD readTailFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { // i in u[i] ranges 0...MIDDLE-1 (multiples of SMALL_HEIGHT) // y ranges 0...WIDTH-1 (multiples of BIG_HEIGHT) (processed in batches of OUT_WG/OUT_SIZEX) -void OVERLOAD writeTailFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { +void OVERLOAD writeTailFusedLine(T2_GF61 *u, P(T2_GF61) out, u32 line, u32 me) { #if PAD_SIZE > 0 #if MIDDLE == 4 || MIDDLE == 8 || MIDDLE == 16 u32 BIG_PAD_SIZE = (PAD_SIZE/2+1)*PAD_SIZE; @@ -208,14 +229,14 @@ void OVERLOAD writeTailFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { #else out += line * (SMALL_HEIGHT + PAD_SIZE) + me; // Pad every output line #endif - for (u32 i = 0; i < NH; ++i) { NTSTORE(out[i * G_H], u[i]); } + for (u32 i = 0; i < NH; ++i) { FFTSTORE(&out[i * G_H], u[i]); } #else // No padding out += line * SMALL_HEIGHT + me; - for (u32 i = 0; i < NH; ++i) { NTSTORE(out[i * G_H], u[i]); } + for (u32 i = 0; i < NH; ++i) { FFTSTORE(&out[i * G_H], u[i]); } #endif } -void OVERLOAD readMiddleOutLine(T2 *u, CP(T2) in, u32 y, u32 x) { +void OVERLOAD readMiddleOutLine(T2_GF61 *u, CP(T2_GF61) in, u32 y, u32 x) { #if PAD_SIZE > 0 #if MIDDLE == 4 || MIDDLE == 8 || MIDDLE == 16 // Each u[i] increments by one pad size. @@ -225,10 +246,10 @@ void OVERLOAD readMiddleOutLine(T2 *u, CP(T2) in, u32 y, u32 x) { #else in += y * MIDDLE * (SMALL_HEIGHT + PAD_SIZE) + x; #endif - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * (SMALL_HEIGHT + PAD_SIZE)]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * (SMALL_HEIGHT + PAD_SIZE)]); } #else // No rotation, might be better on nVidia cards in += y * MIDDLE * SMALL_HEIGHT + x; - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * SMALL_HEIGHT]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SMALL_HEIGHT]); } #endif } @@ -278,7 +299,7 @@ void OVERLOAD readMiddleOutLine(T2 *u, CP(T2) in, u32 y, u32 x) { // adjusted to effect a transpose. Or caller must transpose the x and y values and send us an out pointer with thread_id added in. // In other words, caller is responsible for deciding the best way to transpose x and y values. -void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 chunk_y, u32 chunk_x) +void OVERLOAD writeMiddleOutLine(P(T2_GF61) out, T2_GF61 *u, u32 chunk_y, u32 chunk_x) { //u32 SIZEY = OUT_WG / OUT_SIZEX; //u32 num_x_chunks = SMALL_HEIGHT / OUT_SIZEX; // Number of x chunks @@ -295,7 +316,7 @@ void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 chunk_y, u32 chunk_x) // = WIDTH / (OUT_WG / OUT_SIZEX) * (MIDDLE * OUT_WG + PAD_SIZE) // = WIDTH * MIDDLE * OUT_SIZEX + WIDTH / SIZEY * PAD_SIZE // Write each u[i] sequentially - for (int i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * OUT_WG], u[i]); } + for (int i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * OUT_WG], u[i]); } #else @@ -305,14 +326,13 @@ void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 chunk_y, u32 chunk_x) // = MIDDLE * WIDTH / (OUT_WG / OUT_SIZEX) * OUT_WG // = MIDDLE * WIDTH * OUT_SIZEX // Write each u[i] sequentially - for (int i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * OUT_WG], u[i]); } + for (int i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * OUT_WG], u[i]); } #endif } // Read a line for carryFused or FFTW. This line was written by writeMiddleOutLine above. -void OVERLOAD readCarryFusedLine(CP(T2) in, T2 *u, u32 line) { - u32 me = get_local_id(0); +void OVERLOAD readCarryFusedLine(CP(T2_GF61) in, T2_GF61 *u, u32 line, u32 me) { u32 SIZEY = OUT_WG / OUT_SIZEX; #if PAD_SIZE > 0 @@ -338,7 +358,7 @@ void OVERLOAD readCarryFusedLine(CP(T2) in, T2 *u, u32 line) { for (i32 i = 0; i < NW; ++i) { // u32 fftMiddleOut_y = i * G_W + me; // The fftMiddleOut y value // u32 chunk_y = fftMiddleOut_y / SIZEY; // The fftMiddleOut chunk_y value - u[i] = NTLOAD(in[chunk_y * (MIDDLE * OUT_WG + PAD_SIZE)]); // Adjust in pointer the same way writeMiddleOutLine did + u[i] = FFTLOAD(&in[chunk_y * (MIDDLE * OUT_WG + PAD_SIZE)]); // Adjust in pointer the same way writeMiddleOutLine did chunk_y += chunk_y_incr; } @@ -364,7 +384,7 @@ void OVERLOAD readCarryFusedLine(CP(T2) in, T2 *u, u32 line) { for (i32 i = 0; i < NW; ++i) { // u32 fftMiddleOut_y = i * G_W + me; // The fftMiddleOut y value // u32 chunk_y = fftMiddleOut_y / SIZEY; // The fftMiddleOut chunk_y value - u[i] = NTLOAD(in[chunk_y * MIDDLE * OUT_WG]); // Adjust in pointer the same way writeMiddleOutLine did + u[i] = FFTLOAD(&in[chunk_y * MIDDLE * OUT_WG]); // Adjust in pointer the same way writeMiddleOutLine did chunk_y += chunk_y_incr; } @@ -376,36 +396,36 @@ void OVERLOAD readCarryFusedLine(CP(T2) in, T2 *u, u32 line) { /**************************************************************************/ -/* Similar to above, but for an FFT based on FP32 */ +/* Similar to above, but for an FFT based on FP32 or GF31 */ /**************************************************************************/ -#if FFT_FP32 || NTT_GF31 +#ifdef F2_GF31 -void OVERLOAD writeCarryFusedLine(F2 *u, P(F2) out, u32 line) { +void OVERLOAD writeCarryFusedLine(F2_GF31 *u, P(F2_GF31) out, u32 line, u32 me) { #if PAD_SIZE > 0 u32 BIG_PAD_SIZE = (PAD_SIZE/2+1)*PAD_SIZE; - out += line * WIDTH + line * PAD_SIZE + line / SMALL_HEIGHT * BIG_PAD_SIZE + (u32) get_local_id(0); // One pad every line + a big pad every SMALL_HEIGHT lines - for (u32 i = 0; i < NW; ++i) { NTSTORE(out[i * G_W], u[i]); } + out += line * WIDTH + line * PAD_SIZE + line / SMALL_HEIGHT * BIG_PAD_SIZE + me; // One pad every line + a big pad every SMALL_HEIGHT lines + for (u32 i = 0; i < NW; ++i) { FFTSTORE(&out[i * G_W], u[i]); } #else - out += line * WIDTH + (u32) get_local_id(0); - for (u32 i = 0; i < NW; ++i) { NTSTORE(out[i * G_W], u[i]); } + out += line * WIDTH + me; + for (u32 i = 0; i < NW; ++i) { FFTSTORE(&out[i * G_W], u[i]); } #endif } -void OVERLOAD readMiddleInLine(F2 *u, CP(F2) in, u32 y, u32 x) { +void OVERLOAD readMiddleInLine(F2_GF31 *u, CP(F2_GF31) in, u32 y, u32 x) { #if PAD_SIZE > 0 // Each work group reads successive y's which increments by one pad size. // Rather than having u[i] also increment by one, we choose a larger pad increment u32 BIG_PAD_SIZE = (PAD_SIZE/2+1)*PAD_SIZE; in += y * WIDTH + y * PAD_SIZE + (y / SMALL_HEIGHT) * BIG_PAD_SIZE + x; - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * (SMALL_HEIGHT * (WIDTH + PAD_SIZE) + BIG_PAD_SIZE)]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * (SMALL_HEIGHT * (WIDTH + PAD_SIZE) + BIG_PAD_SIZE)]); } #else in += y * WIDTH + x; - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * SMALL_HEIGHT * WIDTH]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SMALL_HEIGHT * WIDTH]); } #endif } -void OVERLOAD writeMiddleInLine (P(F2) out, F2 *u, u32 chunk_y, u32 chunk_x) +void OVERLOAD writeMiddleInLine(P(F2_GF31) out, F2_GF31 *u, u32 chunk_y, u32 chunk_x) { #if PAD_SIZE > 0 u32 SIZEY = IN_WG / IN_SIZEX; @@ -418,7 +438,7 @@ void OVERLOAD writeMiddleInLine (P(F2) out, F2 *u, u32 chunk_y, u32 chunk_x) // = SMALL_HEIGHT / (IN_WG / IN_SIZEX) * (MIDDLE * IN_WG + PAD_SIZE) // = SMALL_HEIGHT * MIDDLE * IN_SIZEX + SMALL_HEIGHT / SIZEY * PAD_SIZE // Write each u[i] sequentially - for (int i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * IN_WG], u[i]); } + for (int i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * IN_WG], u[i]); } #else // Output data such that readCarryFused lines are packed tightly together. No padding. out += chunk_y * MIDDLE * IN_WG + // Write y chunks after middles @@ -426,13 +446,13 @@ void OVERLOAD writeMiddleInLine (P(F2) out, F2 *u, u32 chunk_y, u32 chunk_x) // = MIDDLE * SMALL_HEIGHT / (IN_WG / IN_SIZEX) * IN_WG // = MIDDLE * SMALL_HEIGHT * IN_SIZEX // Write each u[i] sequentially - for (int i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * IN_WG], u[i]); } + for (int i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * IN_WG], u[i]); } #endif } // Read a line for tailFused or fftHin // This reads partially transposed data as written by fftMiddleIn -void OVERLOAD readTailFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { +void OVERLOAD readTailFusedLine(CP(F2_GF31) in, F2_GF31 *u, u32 line, u32 me) { u32 SIZEY = IN_WG / IN_SIZEX; #if PAD_SIZE > 0 // Adjust in pointer based on the x value used in writeMiddleInLine @@ -453,7 +473,7 @@ void OVERLOAD readTailFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { u32 fftMiddleIn_y_incr = G_H; // The increment to next fftMiddleIn y value u32 chunk_y_incr = fftMiddleIn_y_incr / SIZEY; // The increment to next fftMiddleIn chunk_y value for (i32 i = 0; i < NH; ++i) { - u[i] = NTLOAD(in[chunk_y * (MIDDLE * IN_WG + PAD_SIZE)]); // Adjust in pointer the same way writeMiddleInLine did + u[i] = FFTLOAD(&in[chunk_y * (MIDDLE * IN_WG + PAD_SIZE)]); // Adjust in pointer the same way writeMiddleInLine did chunk_y += chunk_y_incr; } #else // Read data that was not rotated or padded @@ -475,13 +495,13 @@ void OVERLOAD readTailFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { for (i32 i = 0; i < NH; ++i) { u32 fftMiddleIn_y = i * G_H + me; // The fftMiddleIn y value u32 chunk_y = fftMiddleIn_y / SIZEY; // The fftMiddleIn chunk_y value - u[i] = NTLOAD(in[chunk_y * (MIDDLE * IN_WG)]); // Adjust in pointer the same way writeMiddleInLine did + u[i] = FFTLOAD(&in[chunk_y * (MIDDLE * IN_WG)]); // Adjust in pointer the same way writeMiddleInLine did chunk_y += chunk_y_incr; } #endif } -void OVERLOAD writeTailFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { +void OVERLOAD writeTailFusedLine(F2_GF31 *u, P(F2_GF31) out, u32 line, u32 me) { #if PAD_SIZE > 0 #if MIDDLE == 4 || MIDDLE == 8 || MIDDLE == 16 u32 BIG_PAD_SIZE = (PAD_SIZE/2+1)*PAD_SIZE; @@ -489,14 +509,14 @@ void OVERLOAD writeTailFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { #else out += line * (SMALL_HEIGHT + PAD_SIZE) + me; // Pad every output line #endif - for (u32 i = 0; i < NH; ++i) { NTSTORE(out[i * G_H], u[i]); } + for (u32 i = 0; i < NH; ++i) { FFTSTORE(&out[i * G_H], u[i]); } #else // No padding out += line * SMALL_HEIGHT + me; - for (u32 i = 0; i < NH; ++i) { NTSTORE(out[i * G_H], u[i]); } + for (u32 i = 0; i < NH; ++i) { FFTSTORE(&out[i * G_H], u[i]); } #endif } -void OVERLOAD readMiddleOutLine(F2 *u, CP(F2) in, u32 y, u32 x) { +void OVERLOAD readMiddleOutLine(F2_GF31 *u, CP(F2_GF31) in, u32 y, u32 x) { #if PAD_SIZE > 0 #if MIDDLE == 4 || MIDDLE == 8 || MIDDLE == 16 // Each u[i] increments by one pad size. @@ -506,14 +526,14 @@ void OVERLOAD readMiddleOutLine(F2 *u, CP(F2) in, u32 y, u32 x) { #else in += y * MIDDLE * (SMALL_HEIGHT + PAD_SIZE) + x; #endif - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * (SMALL_HEIGHT + PAD_SIZE)]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * (SMALL_HEIGHT + PAD_SIZE)]); } #else // No rotation, might be better on nVidia cards in += y * MIDDLE * SMALL_HEIGHT + x; - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * SMALL_HEIGHT]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SMALL_HEIGHT]); } #endif } -void OVERLOAD writeMiddleOutLine (P(F2) out, F2 *u, u32 chunk_y, u32 chunk_x) +void OVERLOAD writeMiddleOutLine(P(F2_GF31) out, F2_GF31 *u, u32 chunk_y, u32 chunk_x) { #if PAD_SIZE > 0 u32 SIZEY = OUT_WG / OUT_SIZEX; @@ -525,7 +545,7 @@ void OVERLOAD writeMiddleOutLine (P(F2) out, F2 *u, u32 chunk_y, u32 chunk_x) // = WIDTH / (OUT_WG / OUT_SIZEX) * (MIDDLE * OUT_WG + PAD_SIZE) // = WIDTH * MIDDLE * OUT_SIZEX + WIDTH / SIZEY * PAD_SIZE // Write each u[i] sequentially - for (int i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * OUT_WG], u[i]); } + for (int i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * OUT_WG], u[i]); } #else // Output data such that readCarryFused lines are packed tightly together. No padding. out += chunk_y * MIDDLE * OUT_WG + // Write y chunks after middles @@ -533,12 +553,11 @@ void OVERLOAD writeMiddleOutLine (P(F2) out, F2 *u, u32 chunk_y, u32 chunk_x) // = MIDDLE * WIDTH / (OUT_WG / OUT_SIZEX) * OUT_WG // = MIDDLE * WIDTH * OUT_SIZEX // Write each u[i] sequentially - for (int i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * OUT_WG], u[i]); } + for (int i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * OUT_WG], u[i]); } #endif } -void OVERLOAD readCarryFusedLine(CP(F2) in, F2 *u, u32 line) { - u32 me = get_local_id(0); +void OVERLOAD readCarryFusedLine(CP(F2_GF31) in, F2_GF31 *u, u32 line, u32 me) { u32 SIZEY = OUT_WG / OUT_SIZEX; #if PAD_SIZE > 0 // Adjust in pointer based on the x value used in writeMiddleOutLine @@ -558,7 +577,7 @@ void OVERLOAD readCarryFusedLine(CP(F2) in, F2 *u, u32 line) { u32 fftMiddleOut_y_incr = G_W; // The increment to next fftMiddleOut y value u32 chunk_y_incr = fftMiddleOut_y_incr / SIZEY; // The increment to next fftMiddleOut chunk_y value for (i32 i = 0; i < NW; ++i) { - u[i] = NTLOAD(in[chunk_y * (MIDDLE * OUT_WG + PAD_SIZE)]); // Adjust in pointer the same way writeMiddleOutLine did + u[i] = FFTLOAD(&in[chunk_y * (MIDDLE * OUT_WG + PAD_SIZE)]); // Adjust in pointer the same way writeMiddleOutLine did chunk_y += chunk_y_incr; } #else // Read data that was not rotated or padded @@ -578,7 +597,7 @@ void OVERLOAD readCarryFusedLine(CP(F2) in, F2 *u, u32 line) { u32 fftMiddleOut_y_incr = G_W; // The increment to next fftMiddleOut y value u32 chunk_y_incr = fftMiddleOut_y_incr / SIZEY; // The increment to next fftMiddleOut chunk_y value for (i32 i = 0; i < NW; ++i) { - u[i] = NTLOAD(in[chunk_y * MIDDLE * OUT_WG]); // Adjust in pointer the same way writeMiddleOutLine did + u[i] = FFTLOAD(&in[chunk_y * MIDDLE * OUT_WG]); // Adjust in pointer the same way writeMiddleOutLine did chunk_y += chunk_y_incr; } #endif @@ -587,90 +606,6 @@ void OVERLOAD readCarryFusedLine(CP(F2) in, F2 *u, u32 line) { #endif -/**************************************************************************/ -/* Similar to above, but for an NTT based on GF(M31^2) */ -/**************************************************************************/ - -#if NTT_GF31 - -// Since F2 and GF31 are the same size we can simply call the floats based code - -void OVERLOAD writeCarryFusedLine(GF31 *u, P(GF31) out, u32 line) { - writeCarryFusedLine((F2 *) u, (P(F2)) out, line); -} - -void OVERLOAD readMiddleInLine(GF31 *u, CP(GF31) in, u32 y, u32 x) { - readMiddleInLine((F2 *) u, (CP(F2)) in, y, x); -} - -void OVERLOAD writeMiddleInLine (P(GF31) out, GF31 *u, u32 chunk_y, u32 chunk_x) { - writeMiddleInLine ((P(F2)) out, (F2 *) u, chunk_y, chunk_x); -} - -void OVERLOAD readTailFusedLine(CP(GF31) in, GF31 *u, u32 line, u32 me) { - readTailFusedLine((CP(F2)) in, (F2 *) u, line, me); -} - -void OVERLOAD writeTailFusedLine(GF31 *u, P(GF31) out, u32 line, u32 me) { - writeTailFusedLine((F2 *) u, (P(F2)) out, line, me); -} - -void OVERLOAD readMiddleOutLine(GF31 *u, CP(GF31) in, u32 y, u32 x) { - readMiddleOutLine((F2 *) u, (CP(F2)) in, y, x); -} - -void OVERLOAD writeMiddleOutLine (P(GF31) out, GF31 *u, u32 chunk_y, u32 chunk_x) { - writeMiddleOutLine ((P(F2)) out, (F2 *) u, chunk_y, chunk_x); -} - -void OVERLOAD readCarryFusedLine(CP(GF31) in, GF31 *u, u32 line) { - readCarryFusedLine((CP(F2)) in, (F2 *) u, line); -} - -#endif - - -/**************************************************************************/ -/* Similar to above, but for an NTT based on GF(M61^2) */ -/**************************************************************************/ - -#if NTT_GF61 - -// Since T2 and GF61 are the same size we can simply call the doubles based code - -void OVERLOAD writeCarryFusedLine(GF61 *u, P(GF61) out, u32 line) { - writeCarryFusedLine((T2 *) u, (P(T2)) out, line); -} - -void OVERLOAD readMiddleInLine(GF61 *u, CP(GF61) in, u32 y, u32 x) { - readMiddleInLine((T2 *) u, (CP(T2)) in, y, x); -} - -void OVERLOAD writeMiddleInLine (P(GF61) out, GF61 *u, u32 chunk_y, u32 chunk_x) { - writeMiddleInLine ((P(T2)) out, (T2 *) u, chunk_y, chunk_x); -} - -void OVERLOAD readTailFusedLine(CP(GF61) in, GF61 *u, u32 line, u32 me) { - readTailFusedLine((CP(T2)) in, (T2 *) u, line, me); -} - -void OVERLOAD writeTailFusedLine(GF61 *u, P(GF61) out, u32 line, u32 me) { - writeTailFusedLine((T2 *) u, (P(T2)) out, line, me); -} - -void OVERLOAD readMiddleOutLine(GF61 *u, CP(GF61) in, u32 y, u32 x) { - readMiddleOutLine((T2 *) u, (CP(T2)) in, y, x); -} - -void OVERLOAD writeMiddleOutLine (P(GF61) out, GF61 *u, u32 chunk_y, u32 chunk_x) { - writeMiddleOutLine ((P(T2)) out, (T2 *) u, chunk_y, chunk_x); -} - -void OVERLOAD readCarryFusedLine(CP(GF61) in, GF61 *u, u32 line) { - readCarryFusedLine((CP(T2)) in, (T2 *) u, line); -} - -#endif @@ -682,11 +617,11 @@ void OVERLOAD readCarryFusedLine(CP(GF61) in, GF61 *u, u32 line) { // Goals: // 1) In-place transpose. Rather than "ping-pong"ing buffers, an in-place transpose uses half as much memory. This may allow // the entire FFT/NTT data set to reside in the L2 cache on upper end consumer GPUs (circa 2025) which can have 64MB or larger L2 caches. -// 2) We want to have distribute the carryFused and/or tailSquare memory in the L2 cache with minimal cache line collisions. The hope is to (one day) do +// 2) We want to distribute the carryFused and/or tailSquare memory in the L2 cache with minimal cache line collisions. The hope is to (one day) do // fftMiddleOut/carryFused/fftMiddleIn or fftMiddleIn/tailSquare/fftMiddleOut in L2 cache-sized chunks to minimize the slowest memory accesses. // The cost of extra kernel launches may negate any L2 cache benefits. // 3) We use swizzling and/or modest padding to reduce carryFused L2 cache line collisions. Several different memory layouts and padding were tried -// on nVidia Titan V and AMD Radeon VII to find the fastest in-place layout and padding scheme. Hopefully, these will schemes will work well +// on nVidia Titan V and AMD Radeon VII to find the fastest in-place layout and padding scheme. Hopefully, these schemes will work well // on later generation GPUs with different L2 cache dimensions (size and "number-of-ways"). // 4) Apparently cache line collisions in the L1 cache also adversely affect timings. The L1 cache may have a different cache line size and number-of-ways // which makes padding tuning a bit difficult. This is especially true on AMD which has a very strange channel & banks partitioning of memory accesses. @@ -745,7 +680,7 @@ void OVERLOAD readCarryFusedLine(CP(GF61) in, GF61 *u, u32 line) { // This leaves the "columns" starting at +1KB unused - suggesting a pad of +1KB before the 64Ks would yield a better distribution in the L2 cache. // However, if we have say an 8-way 16MB L2 cache then each way contains 2MB. If so, we'd want to pad 1KB before the 16th 64K FFT data value. -#if FFT_FP64 || NTT_GF61 +#ifdef T2_GF61 //**************************************************************************************** // Pair of routines to read/write data to/from carryFused @@ -778,21 +713,19 @@ void OVERLOAD readCarryFusedLine(CP(GF61) in, GF61 *u, u32 line) { // line ranges 0...BIG_HEIGHT-1 (multiples of one) // Read a line for carryFused or FFTW. This line was written by writeMiddleOutLine above. -void OVERLOAD readCarryFusedLine(CP(T2) in, T2 *u, u32 line) { - u32 me = get_local_id(0); // Multiples of BIG_HEIGHT +void OVERLOAD readCarryFusedLine(CP(T2_GF61) in, T2_GF61 *u, u32 line, u32 me) { u32 middle = line / SMALL_HEIGHT; // Multiples of SMALL_HEIGHT line = line % SMALL_HEIGHT; // Multiples of one in += (me / 16 * SIZEW) + (middle * SIZEM) + (line % 16 * SIZEBLK) + SWIZ(line % 16, line / 16) * 16 + (me % 16); - for (u32 i = 0; i < NW; ++i) { u[i] = NTLOAD(in[i * G_W / 16 * SIZEW]); } + for (u32 i = 0; i < NW; ++i) { u[i] = FFTLOAD(&in[i * G_W / 16 * SIZEW]); } } // Write a line from carryFused. This data will be read by fftMiddleIn. -void OVERLOAD writeCarryFusedLine(T2 *u, P(T2) out, u32 line) { - u32 me = get_local_id(0); // Multiples of BIG_HEIGHT +void OVERLOAD writeCarryFusedLine(T2_GF61 *u, P(T2_GF61) out, u32 line, u32 me) { // me is multiples of BIG_HEIGHT u32 middle = line / SMALL_HEIGHT; // Multiples of SMALL_HEIGHT line = line % SMALL_HEIGHT; // Multiples of one out += (me / 16 * SIZEW) + (middle * SIZEM) + (line % 16 * SIZEBLK) + SWIZ(line % 16, line / 16) * 16 + (me % 16); - for (i32 i = 0; i < NW; ++i) { NTSTORE(out[i * G_W / 16 * SIZEW], u[i]); } + for (i32 i = 0; i < NW; ++i) { FFTSTORE(&out[i * G_W / 16 * SIZEW], u[i]); } } //**************************************************************************************** @@ -803,16 +736,16 @@ void OVERLOAD writeCarryFusedLine(T2 *u, P(T2) out, u32 line) { // u[i] ranges 0...MIDDLE-1 (multiples of SMALL_HEIGHT) // y ranges 0...SMALL_HEIGHT-1 (multiples of one) -void OVERLOAD readMiddleInLine(T2 *u, CP(T2) in, u32 y, u32 x) { +void OVERLOAD readMiddleInLine(T2_GF61 *u, CP(T2_GF61) in, u32 y, u32 x) { in += (x / 16 * SIZEW) + (y % 16 * SIZEBLK) + (SWIZ(y % 16, y / 16) * 16) + (x % 16); - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * SIZEM]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SIZEM]); } } // NOTE: writeMiddleInLine uses the same definition of x,y as readMiddleInLine. Caller transposes 16x16 blocks of FFT data before calling writeMiddleInLine. -void OVERLOAD writeMiddleInLine (P(T2) out, T2 *u, u32 y, u32 x) +void OVERLOAD writeMiddleInLine(P(T2_GF61) out, T2_GF61 *u, u32 y, u32 x) { out += (x / 16 * SIZEW) + (y % 16 * SIZEBLK) + (SWIZ(y % 16, y / 16) * 16) + (x % 16); - for (i32 i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * SIZEM], u[i]); } + for (i32 i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * SIZEM], u[i]); } } //**************************************************************************************** @@ -824,18 +757,18 @@ void OVERLOAD writeMiddleInLine (P(T2) out, T2 *u, u32 y, u32 x) // line ranges 0...MIDDLE*WIDTH-1 (multiples of SMALL_HEIGHT) // Read a line for tailSquare/Mul or fftHin -void OVERLOAD readTailFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { +void OVERLOAD readTailFusedLine(CP(T2_GF61) in, T2_GF61 *u, u32 line, u32 me) { u32 width = line % WIDTH; // Multiples of BIG_HEIGHT u32 middle = line / WIDTH; // Multiples of SMALL_HEIGHT in += (width / 16 * SIZEW) + (middle * SIZEM) + (width % 16 * SIZEBLK) + (me % 16); - for (i32 i = 0; i < NH; ++i) { u[i] = NTLOAD(in[SWIZ(width % 16, (i * SMALL_HEIGHT / NH + me) / 16) * 16]); } + for (i32 i = 0; i < NH; ++i) { u[i] = FFTLOAD(&in[SWIZ(width % 16, (i * SMALL_HEIGHT / NH + me) / 16) * 16]); } } -void OVERLOAD writeTailFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { +void OVERLOAD writeTailFusedLine(T2_GF61 *u, P(T2_GF61) out, u32 line, u32 me) { u32 width = line % WIDTH; // Multiples of BIG_HEIGHT u32 middle = line / WIDTH; // Multiples of SMALL_HEIGHT out += (width / 16 * SIZEW) + (middle * SIZEM) + (width % 16 * SIZEBLK) + (me % 16); - for (i32 i = 0; i < NH; ++i) { NTSTORE(out[SWIZ(width % 16, (i * SMALL_HEIGHT / NH + me) / 16) * 16], u[i]); } + for (i32 i = 0; i < NH; ++i) { FFTSTORE(&out[SWIZ(width % 16, (i * SMALL_HEIGHT / NH + me) / 16) * 16], u[i]); } } //**************************************************************************************** @@ -846,26 +779,26 @@ void OVERLOAD writeTailFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { // u[i] ranges 0...MIDDLE-1 (multiples of SMALL_HEIGHT) // y ranges 0...WIDTH-1 (multiples of BIG_HEIGHT) -void OVERLOAD readMiddleOutLine(T2 *u, CP(T2) in, u32 y, u32 x) { +void OVERLOAD readMiddleOutLine(T2_GF61 *u, CP(T2_GF61) in, u32 y, u32 x) { in += (y / 16 * SIZEW) + (y % 16 * SIZEBLK) + (SWIZ(y % 16, x / 16) * 16) + (x % 16); - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * SIZEM]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SIZEM]); } } // NOTE: writeMiddleOutLine uses the same definition of x,y as readMiddleOutLine. Caller transposes 16x16 blocks of FFT data before calling writeMiddleOutLine. -void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 y, u32 x) +void OVERLOAD writeMiddleOutLine(P(T2_GF61) out, T2_GF61 *u, u32 y, u32 x) { out += (y / 16 * SIZEW) + (y % 16 * SIZEBLK) + (SWIZ(y % 16, x / 16) * 16) + (x % 16); - for (i32 i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * SIZEM], u[i]); } + for (i32 i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * SIZEM], u[i]); } } #endif /**************************************************************************/ -/* Similar to above, but for an FFT based on FP32 */ +/* Similar to above, but for an FFT based on FP32 or GF31 */ /**************************************************************************/ -#if FFT_FP32 || NTT_GF31 +#ifdef F2_GF31 //**************************************************************************************** // Pair of routines to read/write data to/from carryFused @@ -879,8 +812,8 @@ void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 y, u32 x) //#define SIZEM32 (MIDDLE * SIZEB + (1 - (MIDDLE & 1)) * 16) // Pad 128 bytes if MIDDLE is even // Place middle rows after all width rows #define SIZEBLK32 (SMALL_HEIGHT + 0) // No pad needed when swizzling -#define SIZEW32 (16 * SIZEBLK + 16) // Pad 128 bytes -#define SIZEM32 (WIDTH / 16 * SIZEW + 16) // Pad 128 bytes +#define SIZEW32 (16 * SIZEBLK32 + 16) // Pad 128 bytes +#define SIZEM32 (WIDTH / 16 * SIZEW32 + 16) // Pad 128 bytes #define SWIZ32(a,m) ((m) ^ (a)) // Swizzle 16 rows (remove "^ (a)" to turn swizzling off) #else // AMD friendly padding // Place middle rows after first 16 rows @@ -889,8 +822,8 @@ void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 y, u32 x) //#define SIZEW32 (MIDDLE * SIZEM + (1 - (MIDDLE & 1)) * 16) // Pad 128 bytes if MIDDLE is even // Place middle rows after all width rows #define SIZEBLK32 (SMALL_HEIGHT + 0) // No pad needed when swizzling -#define SIZEW32 (16 * SIZEBLK + 16) // Pad 128 bytes -#define SIZEM32 (WIDTH / 16 * SIZEW + 0) // Pad 0 bytes +#define SIZEW32 (16 * SIZEBLK32 + 16) // Pad 128 bytes +#define SIZEM32 (WIDTH / 16 * SIZEW32 + 0) // Pad 0 bytes #define SWIZ32(a,m) ((m) ^ (a)) // Swizzle 16 rows (remove "^ (a)" to turn swizzling off) #endif @@ -899,21 +832,19 @@ void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 y, u32 x) // line ranges 0...BIG_HEIGHT-1 (multiples of one) // Read a line for carryFused or FFTW. This line was written by writeMiddleOutLine above. -void OVERLOAD readCarryFusedLine(CP(F2) in, F2 *u, u32 line) { - u32 me = get_local_id(0); // Multiples of BIG_HEIGHT +void OVERLOAD readCarryFusedLine(CP(F2_GF31) in, F2_GF31 *u, u32 line, u32 me) { u32 middle = line / SMALL_HEIGHT; // Multiples of SMALL_HEIGHT line = line % SMALL_HEIGHT; // Multiples of one in += (me / 16 * SIZEW32) + (middle * SIZEM32) + (line % 16 * SIZEBLK32) + SWIZ32(line % 16, line / 16) * 16 + (me % 16); - for (u32 i = 0; i < NW; ++i) { u[i] = NTLOAD(in[i * G_W / 16 * SIZEW32]); } + for (u32 i = 0; i < NW; ++i) { u[i] = FFTLOAD(&in[i * G_W / 16 * SIZEW32]); } } // Write a line from carryFused. This data will be read by fftMiddleIn. -void OVERLOAD writeCarryFusedLine(F2 *u, P(F2) out, u32 line) { - u32 me = get_local_id(0); // Multiples of BIG_HEIGHT +void OVERLOAD writeCarryFusedLine(F2_GF31 *u, P(F2_GF31) out, u32 line, u32 me) { // me is multiples of BIG_HEIGHT u32 middle = line / SMALL_HEIGHT; // Multiples of SMALL_HEIGHT line = line % SMALL_HEIGHT; // Multiples of one out += (me / 16 * SIZEW32) + (middle * SIZEM32) + (line % 16 * SIZEBLK32) + SWIZ32(line % 16, line / 16) * 16 + (me % 16); - for (i32 i = 0; i < NW; ++i) { NTSTORE(out[i * G_W / 16 * SIZEW32], u[i]); } + for (i32 i = 0; i < NW; ++i) { FFTSTORE(&out[i * G_W / 16 * SIZEW32], u[i]); } } //**************************************************************************************** @@ -924,16 +855,16 @@ void OVERLOAD writeCarryFusedLine(F2 *u, P(F2) out, u32 line) { // u[i] ranges 0...MIDDLE-1 (multiples of SMALL_HEIGHT) // y ranges 0...SMALL_HEIGHT-1 (multiples of one) -void OVERLOAD readMiddleInLine(F2 *u, CP(F2) in, u32 y, u32 x) { +void OVERLOAD readMiddleInLine(F2_GF31 *u, CP(F2_GF31) in, u32 y, u32 x) { in += (x / 16 * SIZEW32) + (y % 16 * SIZEBLK32) + (SWIZ32(y % 16, y / 16) * 16) + (x % 16); - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * SIZEM32]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SIZEM32]); } } // NOTE: writeMiddleInLine uses the same definition of x,y as readMiddleInLine. Caller transposes 16x16 blocks of FFT data before calling writeMiddleInLine. -void OVERLOAD writeMiddleInLine (P(F2) out, F2 *u, u32 y, u32 x) +void OVERLOAD writeMiddleInLine(P(F2_GF31) out, F2_GF31 *u, u32 y, u32 x) { out += (x / 16 * SIZEW32) + (y % 16 * SIZEBLK32) + (SWIZ32(y % 16, y / 16) * 16) + (x % 16); - for (i32 i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * SIZEM32], u[i]); } + for (i32 i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * SIZEM32], u[i]); } } //**************************************************************************************** @@ -945,18 +876,18 @@ void OVERLOAD writeMiddleInLine (P(F2) out, F2 *u, u32 y, u32 x) // line ranges 0...MIDDLE*WIDTH-1 (multiples of SMALL_HEIGHT) // Read a line for tailSquare/Mul or fftHin -void OVERLOAD readTailFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { +void OVERLOAD readTailFusedLine(CP(F2_GF31) in, F2_GF31 *u, u32 line, u32 me) { u32 width = line % WIDTH; // Multiples of BIG_HEIGHT u32 middle = line / WIDTH; // Multiples of SMALL_HEIGHT in += (width / 16 * SIZEW32) + (middle * SIZEM32) + (width % 16 * SIZEBLK32) + (me % 16); - for (i32 i = 0; i < NH; ++i) { u[i] = NTLOAD(in[SWIZ32(width % 16, (i * SMALL_HEIGHT / NH + me) / 16) * 16]); } + for (i32 i = 0; i < NH; ++i) { u[i] = FFTLOAD(&in[SWIZ32(width % 16, (i * SMALL_HEIGHT / NH + me) / 16) * 16]); } } -void OVERLOAD writeTailFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { +void OVERLOAD writeTailFusedLine(F2_GF31 *u, P(F2_GF31) out, u32 line, u32 me) { u32 width = line % WIDTH; // Multiples of BIG_HEIGHT u32 middle = line / WIDTH; // Multiples of SMALL_HEIGHT out += (width / 16 * SIZEW32) + (middle * SIZEM32) + (width % 16 * SIZEBLK32) + (me % 16); - for (i32 i = 0; i < NH; ++i) { NTSTORE(out[SWIZ32(width % 16, (i * SMALL_HEIGHT / NH + me) / 16) * 16], u[i]); } + for (i32 i = 0; i < NH; ++i) { FFTSTORE(&out[SWIZ32(width % 16, (i * SMALL_HEIGHT / NH + me) / 16) * 16], u[i]); } } //**************************************************************************************** @@ -967,104 +898,19 @@ void OVERLOAD writeTailFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { // u[i] ranges 0...MIDDLE-1 (multiples of SMALL_HEIGHT) // y ranges 0...WIDTH-1 (multiples of BIG_HEIGHT) -void OVERLOAD readMiddleOutLine(F2 *u, CP(F2) in, u32 y, u32 x) { +void OVERLOAD readMiddleOutLine(F2_GF31 *u, CP(F2_GF31) in, u32 y, u32 x) { in += (y / 16 * SIZEW32) + (y % 16 * SIZEBLK32) + (SWIZ32(y % 16, x / 16) * 16) + (x % 16); - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * SIZEM32]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SIZEM32]); } } // NOTE: writeMiddleOutLine uses the same definition of x,y as readMiddleOutLine. Caller transposes 16x16 blocks of FFT data before calling writeMiddleOutLine. -void OVERLOAD writeMiddleOutLine (P(F2) out, F2 *u, u32 y, u32 x) +void OVERLOAD writeMiddleOutLine(P(F2_GF31) out, F2_GF31 *u, u32 y, u32 x) { out += (y / 16 * SIZEW32) + (y % 16 * SIZEBLK32) + (SWIZ32(y % 16, x / 16) * 16) + (x % 16); - for (i32 i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * SIZEM32], u[i]); } -} - -#endif - - -/**************************************************************************/ -/* Similar to above, but for an NTT based on GF(M31^2) */ -/**************************************************************************/ - -#if NTT_GF31 - -// Since F2 and GF31 are the same size we can simply call the floats based code - -void OVERLOAD readCarryFusedLine(CP(GF31) in, GF31 *u, u32 line) { - readCarryFusedLine((CP(F2)) in, (F2 *) u, line); -} - -void OVERLOAD writeCarryFusedLine(GF31 *u, P(GF31) out, u32 line) { - writeCarryFusedLine((F2 *) u, (P(F2)) out, line); -} - -void OVERLOAD readMiddleInLine(GF31 *u, CP(GF31) in, u32 y, u32 x) { - readMiddleInLine((F2 *) u, (CP(F2)) in, y, x); -} - -void OVERLOAD writeMiddleInLine (P(GF31) out, GF31 *u, u32 y, u32 x) { - writeMiddleInLine ((P(F2)) out, (F2 *) u, y, x); -} - -void OVERLOAD readTailFusedLine(CP(GF31) in, GF31 *u, u32 line, u32 me) { - readTailFusedLine((CP(F2)) in, (F2 *) u, line, me); -} - -void OVERLOAD writeTailFusedLine(GF31 *u, P(GF31) out, u32 line, u32 me) { - writeTailFusedLine((F2 *) u, (P(F2)) out, line, me); -} - -void OVERLOAD readMiddleOutLine(GF31 *u, CP(GF31) in, u32 y, u32 x) { - readMiddleOutLine((F2 *) u, (CP(F2)) in, y, x); -} - -void OVERLOAD writeMiddleOutLine (P(GF31) out, GF31 *u, u32 y, u32 x) { - writeMiddleOutLine ((P(F2)) out, (F2 *) u, y, x); + for (i32 i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * SIZEM32], u[i]); } } #endif -/**************************************************************************/ -/* Similar to above, but for an NTT based on GF(M61^2) */ -/**************************************************************************/ - -#if NTT_GF61 - -// Since T2 and GF61 are the same size we can simply call the doubles based code - -void OVERLOAD readCarryFusedLine(CP(GF61) in, GF61 *u, u32 line) { - readCarryFusedLine((CP(T2)) in, (T2 *) u, line); -} - -void OVERLOAD writeCarryFusedLine(GF61 *u, P(GF61) out, u32 line) { - writeCarryFusedLine((T2 *) u, (P(T2)) out, line); -} - -void OVERLOAD readMiddleInLine(GF61 *u, CP(GF61) in, u32 y, u32 x) { - readMiddleInLine((T2 *) u, (CP(T2)) in, y, x); -} - -void OVERLOAD writeMiddleInLine (P(GF61) out, GF61 *u, u32 y, u32 x) { - writeMiddleInLine ((P(T2)) out, (T2 *) u, y, x); -} - -void OVERLOAD readTailFusedLine(CP(GF61) in, GF61 *u, u32 line, u32 me) { - readTailFusedLine((CP(T2)) in, (T2 *) u, line, me); -} - -void OVERLOAD writeTailFusedLine(GF61 *u, P(GF61) out, u32 line, u32 me) { - writeTailFusedLine((T2 *) u, (P(T2)) out, line, me); -} - -void OVERLOAD readMiddleOutLine(GF61 *u, CP(GF61) in, u32 y, u32 x) { - readMiddleOutLine((T2 *) u, (CP(T2)) in, y, x); -} - -void OVERLOAD writeMiddleOutLine (P(GF61) out, GF61 *u, u32 y, u32 x) { - writeMiddleOutLine ((P(T2)) out, (T2 *) u, y, x); -} - -#endif - #endif diff --git a/src/cl/selftest.cl b/src/cl/selftest.cl index 3564f131..ecf5f063 100644 --- a/src/cl/selftest.cl +++ b/src/cl/selftest.cl @@ -1,6 +1,7 @@ // Copyright (C) Mihai Preda #include "base.cl" +#include "math.cl" #include "trig.cl" #include "fft3.cl" #include "fft4.cl" diff --git a/src/cl/shufl.cl b/src/cl/shufl.cl new file mode 100644 index 00000000..4e3f3839 --- /dev/null +++ b/src/cl/shufl.cl @@ -0,0 +1,984 @@ +// Copyright (C) Mihai Preda + + +// The LDSSWIZ swizzle masks below are sized for a workgroup of at least 64: at WG 32 the XOR patterns +// (lowMe & 7), (lowMe & 15), ((lowMe / 8) & 15) and friends fold several rows onto each other, and five +// of the cases then return the wrong data -- 64 to 192 elements of 256, depending on the case. No +// dispatch reaches them there today (the WG == 32 branch of fft_common asks for f=1,r=4 at RADIX 8 and +// f=4,r=8, and no swizzle case matches either), so like the padded cases above this is a constraint to +// record rather than code to rewrite. +#if LDSSWIZ && WG < 64 +#error LDSSWIZ needs a workgroup of at least 64: its swizzle masks fold rows together below that +#endif + +// The LDSPAD RADIX == 4 reads in this file choose between a WG == 64 form and an else arm whose +// i * 64 and (lowMe / 64) * 16 terms only balance at WG == 256. Both workgroup sizes RADIX 4 can +// have today are therefore handled -- a 256-wide/high shape gives WG 64, and a 1024 one would give +// WG 256 if the commented-out clause in FFTConfig::nW()/nH() were re-enabled -- so the code is +// correct as it stands, and generalising it would add index arithmetic for no present benefit. +// Any other WG would read slots that were never written, silently and with the wrong residue as the +// only symptom, so refuse to build it instead. Generalising is easy when it is needed: i * (WG / 4) +// in place of i * 64 is index-identical at both 64 and 256. +#if LDSPAD && RADIX == 4 && WG != 64 && WG != 256 +#error RADIX == 4 with this workgroup size needs the LDSPAD reads in shufl.cl generalised first (they assume WG is 64 or 256) +#endif + +// Strongly typed versions of LDSptr and LDSsharing_ptr. On TitanV, CUDA 12.9, this is 1% faster. +local T_F_Z31_Z61 * OVERLOAD LDSptr(local T_F_Z31_Z61 *lds, const u32 numWG) { + return lds + ((u32)get_local_id(0) / WG) * LDS_SHUFL_BYTES(numWG) / sizeof(T_F_Z31_Z61); +} +local T2_F2_GF31_GF61 * OVERLOAD LDSptr(local T2_F2_GF31_GF61 *lds, const u32 numWG) { + return lds + ((u32)get_local_id(0) / WG) * LDS_SHUFL_BYTES(numWG) / sizeof(T2_F2_GF31_GF61); +} +local T_F_Z31_Z61 * OVERLOAD LDSsharing_ptr(local T_F_Z31_Z61 *lds, const u32 numWG) { + if (!SHARING_LDS(numWG)) return LDSptr(lds, numWG); + return lds + ((u32)get_local_id(0) / WG / SBMUL(numWG)) * SBMUL(numWG) * LDS_SHUFL_BYTES(numWG) / sizeof(T_F_Z31_Z61); +} +local T2_F2_GF31_GF61 * OVERLOAD LDSsharing_ptr(local T2_F2_GF31_GF61 *lds, const u32 numWG) { + if (!SHARING_LDS(numWG)) return LDSptr(lds, numWG); + return lds + ((u32)get_local_id(0) / WG / SBMUL(numWG)) * SBMUL(numWG) * LDS_SHUFL_BYTES(numWG) / sizeof(T2_F2_GF31_GF61); +} + + +#ifdef T2_GF61 + +// Shufl two or more fft_WIDTHs or FFT_HEIGHTs operating on 64-bit values using LDS_BYTES of LDS memory. +// Care is taken that each simultaneous workgroup does not interfere with the LDS memory of other simultaneous workgroups -- +// even when operating on differernt sized data elements as can happen in an M31+M61 NTT. +// WG = workgroup size of a single fft_WIDTH or fft_HEIGHT +// n = sizeof array u (nW or nH). n * WG = WIDTH or HEIGHT +// r usually equals RADIX if a full fft_RADIX step was just performed. On occasion u[8] values may do less than an fft8 step. +// numWG = number of fft_WIDTHs or fft_HEIGHTs being processed simultaneously +// lowMe = me % WG +void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u32 lowMe) { + + u32 mask = f - 1; + assert((mask & (mask + 1)) == 0); + + // If SHUFL_BYTES is 16 we can write the complete T2 value to LDS memory with one instruction. + // We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. + if (SBMUL(numWG) * SHUFL_BYTES >= 16) { + local T2_GF61* lds = LDSsharing_ptr(lds2, numWG); + +#if LDSPAD + // Special case first RADIX == 8 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...448, 8, 72..., 16... lds[64..127] = +1 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 + // Pad 1 value every row to eliminate bank conflicts. + if (0 && f == 1 && r == 8 && RADIX == 8) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe & 7) * (WG + 1) + (lowMe / 8) * 8 + i] = u[i]; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG / 64) * 8 + ((lowMe / 8) & 7) * (WG + 1) + (lowMe & 7)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 1) + (lowMe & 7)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case first RADIX == 8 to eliminate LDS bank conflicts. + // Input values are in order and written straight to LDS memory. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that uses a little padding. Pad one value after every row to eliminate bank conflicts. + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 + if (f == 1 && r == 8 && RADIX == 8) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 1) + lowMe] = u[i]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG / 8) + (lowMe / 8) + (lowMe & 7) * (WG + 1)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...192, 1, 65..., 16... lds[64..127] = +2 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 + // Pad 1 value every row to eliminate bank conflicts. + if (f == 1 && r == 4 && RADIX == 4) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 3) * (WG + 1) + (lowMe / 8) * 8 + (lowMe & 1) * 4 + i] = u[i]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG / 4 + (lowMe / 32) * 8 + ((lowMe / 8) & 3) * (WG + 1) + (lowMe & 7)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16... 4.. lds[64..127] = +1 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 16... 1.. output[64..127] = +4 + // Pad 4 values after every row to eliminate bank conflicts. + if (f == 4 && r == 4 && RADIX == 4) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i]; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } + LDStx_end(lds2, numWG); + return; + } +#endif + +#if LDSSWIZ + // Special case first RADIX == 8 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 + // Swizzle LDS blocks to eliminate bank conflicts. Swizzle on the first 8 threads written to LDS (multiples of 1) and the first 8 threads read from LDS (multiples of 64). + if (f == 1 && r == 8 && RADIX == 8) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 7)] = u[i]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 7)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // No swizzle of LDS blocks is needed to eliminate bank conflicts. The first 8 threads written to LDS (multiples of 64) and + // the first 8 threads read from LDS (multiples of 64) are already in separate LDS banks. + // The read index must be the inverse of the natural write for every WG, not just WG == 64; at WG == 64 it + // reduces to lowMe / 8 * 64 + i * 8 + (lowMe & 7). + // We can however save a bar() by writing to same locations that previous shufl wrote to. + if (f == 8 && r == 8 && RADIX == 8) { + LDStx_start(lds2, numWG); //GRRR.... LDStx_start will do the bar we are trying to save + for (u32 i = 0; i < RADIX; ++i) { lds[i * WG + lowMe] = u[i]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[((lowMe / 8) & 7) * WG + i * (WG / 8) + (lowMe / 64) * 8 + (lowMe & 7)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 8 threads written to LDS (4 multiples of 1 and 2 multiples of 4) and the first 8 threads read from LDS (4 multiples of 64 and 2 multiples of 1). + if (f == 1 && r == 4 && RADIX == 4) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 7)] = u[i]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 7)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 8 threads written to LDS (4 multiples of 64 and 2 multiples of 1) and the first 8 threads read from LDS (4 multiples of 64 and 2 multiples of 4). + if (f == 4 && r == 4 && RADIX == 4) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 4)] = u[i]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 4)]; } + LDStx_end(lds2, numWG); + return; + } +#endif + + // Otherwise, execute the original shufl code modified to handle case where a full RADIX fft was not done + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = u[i]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG + lowMe]; } + LDStx_end(lds2, numWG); + return; + } + + // If SHUFL_BYTES is 8 we split the T2 values into two T values. These are written to LDS memory with two instructions. + // We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + else if (SBMUL(numWG) * SHUFL_BYTES == 8) { + local T_Z61* lds = LDSsharing_ptr((local T_Z61 *)lds2, numWG); + +#if LDSPAD + // Special case first RADIX == 8 code to eliminate LDS bank conflicts. + // Input values are in order and written straight to LDS memory. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that uses a little padding. Pad two values after every row to eliminate bank conflicts. + // Read from LDS in the desired output order. In the example: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + if (f == 1 && r == 8 && RADIX == 8) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 2) + lowMe] = u[i].x; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG / 8 + (lowMe / 8) + (lowMe & 7) * (WG + 2)]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 2) + lowMe] = u[i].y; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG / 8 + (lowMe / 8) + (lowMe & 7) * (WG + 2)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS with 8 pads after each row. + // Read from LDS in output order. In the example: u[0] = 0, 64, ... 448, 8, 72... u[1] = +1 + if (f == 8 && r == 8 && RADIX == 8) { + LDStx_start(lds2, numWG); + // One expression for every WG. The old pair of arms laid the data out as exactly eight padded rows, + // which only inverts when WG/64 == 8, and read a row of 64 regardless of WG: correct at WG 64 and 512, + // wrong everywhere between (896 of 1024 elements at WG 128, 1792 of 2048 at WG 256, some of them + // reading slots nothing had written). This is the form the 64-bit sibling above already uses. + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i].x; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG / 64) * 8 + (lowMe / 8) * (WG + 8) + (lowMe & 7)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i].y; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG / 64) * 8 + (lowMe / 8) * (WG + 8) + (lowMe & 7)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case alternate first RADIX == 8 code to eliminate LDS bank conflicts (only a radix-4 step was performed). + // Input values are in order and written straight to LDS memory. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +32... + // Output to LDS that uses a little padding. Pad four values after every other row to eliminate bank conflicts. + // Read from LDS in the desired output order. In the example: u[0] = 0, 64, 128, 192, 1, 65... u[1] = +8 + if (f == 1 && r == 4 && RADIX == 8) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / 2 * (2 * WG + 4) + (i % 2) * WG + lowMe] = u[i].x; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG / 4 + (lowMe / 4) + (lowMe & 3) * (2 * WG + 4)]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / 2 * (2 * WG + 4) + (i % 2) * WG + lowMe] = u[i].y; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG / 4 + (lowMe / 4) + (lowMe & 3) * (2 * WG + 4)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case alternate second RADIX == 8 to eliminate LDS bank conflicts (first shufl was partial after a radix-4 step). + // Input values are the output from a previous shufl. For example, WIDTH=256: u[0] = 0, 64, 128, 192, 1, 65... u[1] = +8 + // Output to LDS with 4 pads after each row. + // Read from LDS in output order. In the example: u[0] = 0, 64, 128, 192, 8, 72... u[1] = +1 + if (f == 4 && r == 8 && RADIX == 8) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 4) + lowMe] = u[i].x; } + LDSbar(numWG); + if (WG == 32) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG / 32) * 4 + (lowMe / 4) * (WG + 4) + (lowMe & 3)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG / 32) * 4 + (lowMe / 32) * 4 + ((lowMe / 4) & 7) * (WG + 4) + (lowMe & 3)]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 4) + lowMe] = u[i].y; } + LDSbar(numWG); + if (WG == 32) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG / 32) * 4 + (lowMe / 4) * (WG + 4) + (lowMe & 3)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG / 32) * 4 + (lowMe / 32) * 4 + ((lowMe / 4) & 7) * (WG + 4) + (lowMe & 3)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...192, 1.., 2.., 3.., 16... lds[64..127] = +4 + // Read from LDS in the desired output order. In the example: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Pad one value after every row to eliminate bank conflicts. + if (1 && f == 1 && r == 4 && RADIX == 4) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i].x; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i].y; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. + // Input values are in order and written straight to LDS memory. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS using a little padding. Pad four values after every row to eliminate bank conflicts. + // Read from LDS in the desired output order. In the example: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + if (0 && f == 1 && r == 4 && RADIX == 4) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 4) + lowMe] = u[i].x; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG / 4 + (lowMe / 4) + (lowMe & 3) * (WG + 4)]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 4) + lowMe] = u[i].y; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG / 4 + (lowMe / 4) + (lowMe & 3) * (WG + 4)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0...192, 16..., 32..., 48..., 4... lds[64..127] = +1 + // Output to LDS in the order we expect to read. In the example: u[0] = 0...192, 16... 32.. 48.. 1... u[1] = +4 + // Pad 4 values after every row to eliminate bank conflicts. + if (0 && f == 4 && r == 4 && RADIX == 4) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i].x; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i].y; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS with 4 pads after each row. + // Read from LDS in output order. In the example: u[0] = 0...192, 16... 32.. 48.. 1... u[1] = +4 + if (1 && f == 4 && r == 4 && RADIX == 4) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 4) + lowMe] = u[i].x; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG / 16) * 4 + (lowMe / 16) * 4 + ((lowMe / 4) & 3) * (WG + 4) + (lowMe & 3)]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 4) + lowMe] = u[i].y; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG / 16) * 4 + (lowMe / 16) * 4 + ((lowMe / 4) & 3) * (WG + 4) + (lowMe & 3)]; } + LDStx_end(lds2, numWG); + return; + } +#endif + +#if LDSSWIZ + // Special case first RADIX == 8 code to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (8 multiples of 1 and 2 multiples of 8) and the first 16 threads read from LDS (8 multiples of 64 and 2 multiples of 1). + if (f == 1 && r == 8 && RADIX == 8) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i].x; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i].y; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (8 multiples of 64 and 2 multiples of 1) and the first 16 threads read from LDS (8 multiples of 64 and 2 multiples of 8). + if (f == 8 && r == 8 && RADIX == 8) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i].x; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i].y; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (4 multiples of 1 and 4 multiples of 4) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 1). + if (f == 1 && r == 4 && RADIX == 4) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i].x; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i].y; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (4 multiples of 64 and 4 multiples of 1) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 16). + if (f == 4 && r == 4 && RADIX == 4) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].x; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].y; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } + LDStx_end(lds2, numWG); + return; + } +#endif + + // Otherwise, execute the original shufl code modified to handle case where a full RADIX fft was not done + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = u[i].x; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG + lowMe]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = u[i].y; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG + lowMe]; } + LDStx_end(lds2, numWG); + return; + } + + // If SHUFL_BYTES is 4 we split the T2 values into 4 int values. These are written to LDS memory using four instructions. + // NOT OPTIMIZED TO REDUCE LDS BANK CONFLICTS!! + else if (SBMUL(numWG) * SHUFL_BYTES == 4) { + // Lower LDS requirements may let the optimizer use fewer VGPRs and increase occupancy for WIDTHs >= 1024. + // Alas, the increased occupancy does not offset extra code needed for shufl_int (the assembly + // code generated is not pretty). This might not be true for nVidia or future ROCm optimizers. + local int* lds = (local int*)LDSsharing_ptr(lds2, numWG); + + // Use the same write index as the 8- and 16-byte paths: it honours r, which is smaller than RADIX when + // the caller has done only a partial fft_RADIX step (fft8_4 on the SIZE=256/RADIX=8 path). For r == RADIX + // this is identical to the i * f + (lowMe & ~mask) * RADIX + (lowMe & mask) it replaces. + + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = as_int4(u[i]).x; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = as_int4(u[i]).y; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = as_int4(u[i]).z; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = as_int4(u[i]).w; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + LDStx_end(lds2, numWG); + return; + } +} + +// Shortcut for the most common case where caller did a full RADIX step (as opposed to the oddball cases where we have a u[8] but only did a radix 2 or 4 step). +void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 numWG, u32 lowMe) { + shufl(lds2, u, f, RADIX, numWG, lowMe); +} + + +// NEEDS TONS OF WORK!!! SWIZ NOT CODED, MOST PAD CASES NOT CODED, SHUFL_BYTES = 4 needs differernt algorithm. +// At present, this is only used by WIDTH or HEIGHT = 1K with RADIX=8 and f=8. + +// Shufl two or more fft_WIDTHs or fft_HEIGHTs operating on 64-bit values using LDS_BYTES of LDS memory. An fft2 is also performed. +void OVERLOAD shufl_and_fft2(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 numWG, u32 lowMe) { + assert(RADIX == 8); + + u32 mask = f - 1; + assert((mask & (mask + 1)) == 0); + + // Start by doing the writes of a standard shufl. + // Next, each thread reads a pair of values. The lower threads add the two values, the higher threads subtract the two values. + // val1 is read from i * WG/2 + // val2 is read from 4 * WG + i * WG/2 + + // If SHUFL_BYTES is 16 we can write the complete T2 value to LDS memory with one instruction. + if (SBMUL(numWG) * SHUFL_BYTES >= 16) { + local T2_GF61* lds = LDSsharing_ptr(lds2, numWG); + + // Execute the original shufl code with an fft2 add-on. + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { + T2_GF61 val1 = lds[ i * (WG / 2) + lowMe % (WG / 2)]; + T2_GF61 val2 = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; + if (lowMe < WG / 2) u[i] = addq(val1, val2); + else u[i] = subq(val1, val2); + } + LDStx_end(lds2, numWG); + return; + } + + // If SHUFL_BYTES is 8 we split the T2 values into two T values. These are written to LDS memory with two instructions. + else if (SBMUL(numWG) * SHUFL_BYTES == 8) { + local T_Z61* lds = LDSsharing_ptr((local T_Z61 *)lds2, numWG); + +#if LDSPAD + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS with 8 pads after each row. + // Read from LDS in output order. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + if (f == 8 && RADIX == 8) { + local T_Z61 *ldsIn; + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i].x; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { + // Read val1 from the standard shufl's i = i/2, lowMe = lowMe % WG/2 + (i&1) * WG/2 + // Read val2 from the standard shufl's i = i/2 + 4, lowMe = lowMe % WG/2 + (i&1) * WG/2 + T_Z61 val1 = lds[(i / 2) * (WG / 64) * 8 + (((i & 1) * (WG / 2)) / 64) * 8 + ((lowMe % (WG / 2)) / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; + T_Z61 val2 = lds[(i / 2 + 4) * (WG / 64) * 8 + (((i & 1) * (WG / 2)) / 64) * 8 + ((lowMe % (WG / 2)) / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; + if (lowMe < WG / 2) u[i].x = addq(val1, val2); + else u[i].x = subq(val1, val2); + } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i].y; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { + // Read val1 from the standard shufl's i = i/2, lowMe = lowMe % WG/2 + (i&1) * WG/2 + // Read val2 from the standard shufl's i = i/2 + 4, lowMe = lowMe % WG/2 + (i&1) * WG/2 + T_Z61 val1 = lds[(i / 2) * (WG / 64) * 8 + (((i & 1) * (WG / 2)) / 64) * 8 + ((lowMe % (WG / 2)) / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; + T_Z61 val2 = lds[(i / 2 + 4) * (WG / 64) * 8 + (((i & 1) * (WG / 2)) / 64) * 8 + ((lowMe % (WG / 2)) / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; + if (lowMe < WG / 2) u[i].y = addq(val1, val2); + else u[i].y = subq(val1, val2); + } + LDStx_end(lds2, numWG); + return; + } +#endif + + // Execute the original shufl code with an fft2 add-on. + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].x; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { + T_Z61 val1 = lds[ i * (WG / 2) + lowMe % (WG / 2)]; + T_Z61 val2 = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; + if (lowMe < WG / 2) u[i].x = addq(val1, val2); + else u[i].x = subq(val1, val2); + } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].y; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { + T_Z61 val1 = lds[ i * (WG / 2) + lowMe % (WG / 2)]; + T_Z61 val2 = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; + if (lowMe < WG / 2) u[i].y = addq(val1, val2); + else u[i].y = subq(val1, val2); + } + LDStx_end(lds2, numWG); + return; + } + + // If SHUFL_BYTES is 4 we split the T2 values into 4 int values. These are written to LDS memory using four instructions. + // NOT OPTIMIZED TO REDUCE LDS BANK CONFLICTS!! + else if (SBMUL(numWG) * SHUFL_BYTES == 4) { + + // Lower LDS requirements may let the optimizer use fewer VGPRs and increase occupancy for WIDTHs >= 1024. + // Alas, the increased occupancy does not offset extra code needed for shufl_int (the assembly + // code generated is not pretty). This might not be true for nVidia or future ROCm optimizers. + local int* lds = (local int*)LDSsharing_ptr(lds2, numWG); + + // The fft2 has to add and subtract whole 64-bit values, so first gather all four 32-bit pieces of val1 and val2 + // (same LDS locations as the 16- and 8-byte paths above), then combine. u[] stays intact as the source of the + // four write passes. + int4 v1[RADIX], v2[RADIX]; + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).x; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { v1[i].x = lds[i * (WG / 2) + lowMe % (WG / 2)]; v2[i].x = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).y; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { v1[i].y = lds[i * (WG / 2) + lowMe % (WG / 2)]; v2[i].y = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).z; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { v1[i].z = lds[i * (WG / 2) + lowMe % (WG / 2)]; v2[i].z = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).w; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { v1[i].w = lds[i * (WG / 2) + lowMe % (WG / 2)]; v2[i].w = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; } + LDStx_end(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { + T2_GF61 val1 = as_T2_GF61(v1[i]); + T2_GF61 val2 = as_T2_GF61(v2[i]); + if (lowMe < WG / 2) u[i] = addq(val1, val2); + else u[i] = subq(val1, val2); + } + return; + } +} + +#endif + + +#ifdef F2_GF31 + +// Shufl two or more fft_WIDTHs or FFT_HEIGHTs using two 4-byte floats or Z31s. +void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u32 lowMe) { + + u32 mask = f - 1; + assert((mask & (mask + 1)) == 0); + + //GW - would a 16 byte implementation be useful? Less LDS conflict work? + + // If SHUFL_BYTES is 8 or more we can write the complete F2 value to LDS memory with one instruction. + // We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + if (SBMUL(numWG) * SHUFL_BYTES >= 8) { + local F2_GF31* lds = LDSsharing_ptr(lds2, numWG); + +#if LDSPAD + // Special case first RADIX == 8 to eliminate LDS bank conflicts. + // Input values are in order and written straight to LDS memory. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that uses a little padding. Pad two values after every row to eliminate bank conflicts. + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 + if (f == 1 && r == 8 && RADIX == 8) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 2) + lowMe] = u[i]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG / 8 + (lowMe / 8) + (lowMe & 7) * (WG + 2)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // Pad 8 values after every 64 values to eliminate bank conflicts. + if (1 && f == 8 && r == 8 && RADIX == 8) { + LDStx_start(lds2, numWG); + // One expression for every WG. The old pair of arms laid the data out as exactly eight padded rows, + // which only inverts when WG/64 == 8, and read a row of 64 regardless of WG: correct at WG 64 and 512, + // wrong everywhere between (896 of 1024 elements at WG 128, 1792 of 2048 at WG 256, some of them + // reading slots nothing had written). This is the form the 64-bit sibling above already uses. + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS with 8 pads after each row. + // Read from LDS in output order. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + if (0 && f == 8 && r == 8 && RADIX == 8) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i]; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...192, 1.., 2.., 3.., 16... lds[64..127] = +4 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 + // Pad one value after every row to eliminate bank conflicts. + if (f == 1 && r == 4 && RADIX == 4) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i]; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 1) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0...192, 16..., 32..., 48..., 4... lds[64..127] = +1 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0...192, 16... 32.. 48.. 1... lds[64..127] = +4 + // Pad 4 values after every row to eliminate bank conflicts. + if (f == 4 && r == 4 && RADIX == 4) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i]; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } + LDStx_end(lds2, numWG); + return; + } +#endif + +#if LDSSWIZ + // Special case first RADIX == 8 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (8 multiples of 1 and 2 multiples of 8) and the first 26 threads read from LDS (multiples of 64 and two multiples of 1). + if (f == 1 && r == 8 && RADIX == 8) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i]; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (8 multiples of 64 and 2 multiples of 1) and the first 16 threads read from LDS (8 multiples of 64 and two multiples of 8). + if (f == 8 && r == 8 && RADIX == 8) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i]; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (4 multiples of 1 and 4 multiples of 4) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 1). + if (f == 1 && r == 4 && RADIX == 4) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80 ... lds[64..127] = +4 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (4 multiples of 64 and 4 multiples of 1) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 16). + if (f == 4 && r == 4 && RADIX == 4) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } + LDStx_end(lds2, numWG); + return; + } +#endif + + // Otherwise, execute the original shufl code modified to handle case where a full RADIX fft was not done + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = u[i]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG + lowMe]; } + LDStx_end(lds2, numWG); + return; + } + + // If SHUFL_BYTES is 4 we split the F2 values into two F values. These are written to LDS memory using two instructions. + // We're writing 4 bytes at a time, which means groups of 32 must have unique LDS banks. + else if (SBMUL(numWG) * SHUFL_BYTES == 4) { + local F_Z31* lds = LDSsharing_ptr((local F_Z31 *)lds2, numWG); + +#if LDSPAD + // Special case first RADIX == 8 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...448, 1, 65..., 2, 66..., 3, 67..., 32, 96... lds[64..127] = +4 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 + // Pad one value after every row to eliminate bank conflicts. + if (f == 1 && r == 8 && RADIX == 8) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 7) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 3) * 8 + i] = u[i].x; } + LDSbar(numWG); + // Read back in the generic shufl's output order. The write above stores (i', me') at + // ((me'/4)&7)*(WG+1) + (me'/32)*32 + (me'&3)*8 + i', and output (i, lowMe) needs i' = lowMe & 7, + // me' = i*WG/8 + lowMe/8; the per-WG forms below are that inverse with the constants folded. + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 4) * 32 + (i & 3) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else if (WG == 128) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 32 + (4 * (i & 1) + lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else if (WG == 512) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 256) * 32 + ((lowMe / 32) & 7) * (WG + 1) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u32 mep = i * (WG / 8) + lowMe / 8; u[i].x = lds[((mep / 4) & 7) * (WG + 1) + (mep / 32) * 32 + (mep & 3) * 8 + (lowMe & 7)]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 7) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 3) * 8 + i] = u[i].y; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 4) * 32 + (i & 3) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else if (WG == 128) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 32 + (4 * (i & 1) + lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else if (WG == 512) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 256) * 32 + ((lowMe / 32) & 7) * (WG + 1) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u32 mep = i * (WG / 8) + lowMe / 8; u[i].y = lds[((mep / 4) & 7) * (WG + 1) + (mep / 32) * 32 + (mep & 3) * 8 + (lowMe & 7)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // Pad 8 values after every 64 values to eliminate bank conflicts. + if (f == 8 && r == 8 && RADIX == 8) { + LDStx_start(lds2, numWG); + // One expression for every WG. The old pair of arms laid the data out as exactly eight padded rows, + // which only inverts when WG/64 == 8, and read a row of 64 regardless of WG: correct at WG 64 and 512, + // wrong everywhere between (896 of 1024 elements at WG 128, 1792 of 2048 at WG 256, some of them + // reading slots nothing had written). This is the form the 64-bit sibling above already uses. + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i].x; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i].y; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...192, 1.., 2.., 7.., 32... lds[64..127] = +8 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 + // Pad one value after every row to eliminate bank conflicts. + if (f == 1 && r == 4 && RADIX == 4) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 7) * 4 + i] = u[i].x; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 1) + (lowMe & 31)]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 7) * 4 + i] = u[i].y; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 1) + (lowMe & 31)]; } + LDStx_end(lds2, numWG); + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0...192, 16..., 32..., 48..., 1.... ... 8... lds[64..127] = +2 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0...192, 16... 32.. 48.. 1... lds[64..127] = +4 + // Pad 4 values after every row to eliminate bank conflicts. + if (f == 4 && r == 4 && RADIX == 4) { + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 4) + (lowMe / 32) * 32 + ((lowMe / 4) & 1) * 16 + i * 4 + (lowMe & 3)] = u[i].x; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 4)) + (lowMe / 32) * (WG + 4) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 4) + (lowMe & 31)]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 4) + (lowMe / 32) * 32 + ((lowMe / 4) & 1) * 16 + i * 4 + (lowMe & 3)] = u[i].y; } + LDSbar(numWG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 4)) + (lowMe / 32) * (WG + 4) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 4) + (lowMe & 31)]; } + LDStx_end(lds2, numWG); + return; + } +#endif + + // Otherwise, execute the original shufl code modified to handle case where a full RADIX fft was not done + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = u[i].x; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG + lowMe]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = u[i].y; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG + lowMe]; } + LDStx_end(lds2, numWG); + return; + } +} + +// Shortcut for the most common case where caller did a full RADIX step (as opposed to the oddball cases where we have a u[8] but only did a radix 2 or 4 step). +void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 numWG, u32 lowMe) { + shufl(lds2, u, f, RADIX, numWG, lowMe); +} + + +// NEEDS TONS OF WORK!!! SWIZ NOT CODED, MOST PAD CASES NOT CODED. +// At present, this is only used by WIDTH or HEIGHT = 1K with RADIX=8 and f=8. + +// Shufl two or more fft_WIDTHs or fft_HEIGHTs operating on 32-bit values using LDS_BYTES of LDS memory. An fft2 is also performed. +void OVERLOAD shufl_and_fft2(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 numWG, u32 lowMe) { + assert(RADIX == 8); + + u32 mask = f - 1; + assert((mask & (mask + 1)) == 0); + + // Start by doing the writes of a standard shufl. + // Next, each thread reads a pair of values. The lower threads add the two values, the higher threads subtract the two values. + // val1 is read from i * WG/2 + // val2 is read from 4 * WG + i * WG/2 + + // If SHUFL_BYTES is 8 or more we can write the complete F2 value to LDS memory with one instruction. + if (SBMUL(numWG) * SHUFL_BYTES >= 8) { + local F2_GF31* lds = LDSsharing_ptr(lds2, numWG); + +#if LDSPAD + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS with 8 pads after each row. + // Read from LDS in output order. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + if (f == 8 && RADIX == 8) { + local F2_GF31 *ldsIn; + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { + // Read val1 from the standard shufl's i = i/2, lowMe = lowMe % WG/2 + (i&1) * WG/2 + // Read val2 from the standard shufl's i = i/2 + 4, lowMe = lowMe % WG/2 + (i&1) * WG/2 + F2_GF31 val1 = lds[(i / 2) * (WG / 64) * 8 + (((i & 1) * (WG / 2)) / 64) * 8 + ((lowMe % (WG / 2)) / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; + F2_GF31 val2 = lds[(i / 2 + 4) * (WG / 64) * 8 + (((i & 1) * (WG / 2)) / 64) * 8 + ((lowMe % (WG / 2)) / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; + if (lowMe < WG / 2) u[i] = addq(val1, val2); + else u[i] = subq(val1, val2); + } + LDStx_end(lds2, numWG); + return; + } +#endif + + // Execute the original shufl code with an fft2 add-on. + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i]; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { + F2_GF31 val1 = lds[ i * (WG / 2) + lowMe % (WG / 2)]; + F2_GF31 val2 = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; + if (lowMe < WG / 2) u[i] = addq(val1, val2); + else u[i] = subq(val1, val2); + } + LDStx_end(lds2, numWG); + return; + } + + // If SHUFL_BYTES is 4 we split the F2 values into two F values. These are written to LDS memory using two instructions. + else if (SBMUL(numWG) * SHUFL_BYTES == 4) { + local F_Z31* lds = LDSsharing_ptr((local F_Z31 *)lds2, numWG); + + // Execute the original shufl code with an fft2 add-on. + LDStx_start(lds2, numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].x; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { + F_Z31 val1 = lds[ i * (WG / 2) + lowMe % (WG / 2)]; + F_Z31 val2 = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; + if (lowMe < WG / 2) u[i].x = addq(val1, val2); + else u[i].x = subq(val1, val2); + } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].y; } + LDSbar(numWG); + for (u32 i = 0; i < RADIX; ++i) { + F_Z31 val1 = lds[ i * (WG / 2) + lowMe % (WG / 2)]; + F_Z31 val2 = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; + if (lowMe < WG / 2) u[i].y = addq(val1, val2); + else u[i].y = subq(val1, val2); + } + LDStx_end(lds2, numWG); + return; + } +} + +#endif diff --git a/src/cl/tailmul.cl b/src/cl/tailmul.cl index 1cdd5db0..12826691 100644 --- a/src/cl/tailmul.cl +++ b/src/cl/tailmul.cl @@ -1,10 +1,47 @@ // Copyright (C) Mihai Preda and George Woltman #include "base.cl" -#include "tailutil.cl" -#include "trig.cl" #include "fftheight.cl" +#define INCLUDE_FILE "tailutil.cl" +#include "expand.cl" +#define INCLUDE_FILE "middle.cl" +#include "expand.cl" + +// If not doing L2 stripes, process the lines in any order. +// If L2 striping, process lines output by fftMiddleIn. fftMiddleIn outputs 16 * MIDDLE tailSquare lines. +u32 get_line_number(u32 base) { + u32 g = get_group_id(0); +#if !SINGLE_KERNEL +#if L2_STRIPING + if (base == 0) g = g + 1; +#else + g = g + 1; +#endif +#endif +#if L2_STRIPING + // Old, simple L2 striping code + // return get_group_id(1) * WIDTH + base + g; + + // Process all lines from low half of base_lo stripe group. One stripe group is stripe_group_size * 16 * MIDDLE lines. + u32 base_lo = base; + u32 stripe_group_size = L2_STRIPING; + u32 half_size = (MIDDLE + 1) / 2; // For base_lo, round odd middles up. + u32 kernelsToExecute = half_size * stripe_group_size * 16; + if (g < kernelsToExecute) return g / (stripe_group_size * 16) * WIDTH + base_lo + g % (stripe_group_size * 16); + g -= kernelsToExecute; + + // Process lines from low half of base_hi stripe group. One stripe group is stripe_group_size * 16 * MIDDLE lines. + // The first line in the base_hi stripe group is not ready for processing (except for the last group). + u32 base_hi = WIDTH - stripe_group_size * 16 - base_lo; + if (base_hi != WIDTH / 2) base_hi++; // Skip first line in base_hi (usually) + half_size = MIDDLE / 2; // For base_hi, round odd middles up. + return g % half_size * WIDTH + base_hi + g / half_size; +#else + return g; +#endif +} + #if FFT_FP64 // Handle the final multiplication step on a pair of complex numbers. Swap real and imaginary results for the inverse FFT. @@ -48,75 +85,229 @@ void OVERLOAD pairMul(u32 N, T2 *u, T2 *v, T2 *p, T2 *q, T2 base_squared, bool s } } -KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { - local T2 lds[SMALL_HEIGHT]; +#if !SINGLE_KERNEL +// The kernel tailMulZero handles the special cases in tailMul, i.e. the lines 0 and H/2 +// This kernel is launched with 2 workgroups (handling line 0, resp. H/2) +KERNEL(G_H) tailMulZero(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { + local T2 lds[LDS_BYTES(1) / sizeof(T2)]; + LDSinit(lds, 1); + + T2 u[NH], p[NH]; + const u32 H = ND / SMALL_HEIGHT; + + // This kernel in executed in two workgroups. + u32 which = get_group_id(0); + assert(which < 2); + + u32 line = which ? (H/2) : 0; + u32 memline = transPos(line, MIDDLE, WIDTH); + u32 me = get_local_id(0); + + dependentLaunch(); // Next kernel will be tailMulFP64 which must dependentLaunchWait before reading data from fftMiddleInFP64 + dependentLaunchWait(); // Previous kernel was fftMiddleInFP64 that launched dependents before writing FP64 data + + readTailFusedLine(in, u, line, me); + +#if FFT_VARIANT_H != 0 + T2 w; +#elif NH == 8 + T2 w = fancyTrig_N(H * me); +#else + T2 w = slowTrig_N(H * me, ND / NH); +#endif + +#if MUL_LOW + read(G_H, NH, p, a, memline * SMALL_HEIGHT); + fft_HEIGHT1(lds, u, smallTrig, w, 1, me); +#else + readTailFusedLine(a, p, line, me); + fft_HEIGHT1(lds, u, smallTrig, w, 1, me); + fft_HEIGHT1(lds, p, smallTrig, w, 1, me); +#endif + + T2 trig = slowTrig_N(line + me * H, ND / NH); + + reverse(lds, u + NH/2, !which); + reverse(lds, p + NH/2, !which); + pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, !which); + reverse(lds, u + NH/2, !which); + + fft_HEIGHT1(lds, u, smallTrig, w, 1, me); + writeTailFusedLine(u, out, memline, me); +} +#endif + +#if SINGLE_WIDE + +KERNEL_CAP(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { + local T2 lds[LDS_BYTES(1) / sizeof(T2)]; + LDSinit(lds, 1); + + const u32 H = ND / SMALL_HEIGHT; T2 u[NH], v[NH]; T2 p[NH], q[NH]; - u32 H = ND / SMALL_HEIGHT; - - u32 line1 = get_group_id(0); + u32 line1 = get_line_number(base); u32 line2 = line1 ? H - line1 : (H / 2); u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); + dependentLaunchWait(); // Previous kernel was fftMiddleInFP64 that launched dependents before writing FP64 data + u32 me = get_local_id(0); readTailFusedLine(in, u, line1, me); readTailFusedLine(in, v, line2, me); -#if NH == 8 - T2 w = fancyTrig_N(ND / SMALL_HEIGHT * me); +#if FFT_VARIANT_H != 0 + T2 w; +#elif NH == 8 + T2 w = fancyTrig_N(H * me); #else - T2 w = slowTrig_N(ND / SMALL_HEIGHT * me, ND / NH); + T2 w = slowTrig_N(H * me, ND / NH); #endif #if MUL_LOW read(G_H, NH, p, a, memline1 * SMALL_HEIGHT); read(G_H, NH, q, a, memline2 * SMALL_HEIGHT); - fft_HEIGHT(lds, u, smallTrig, w); - bar(); - fft_HEIGHT(lds, v, smallTrig, w); + fft_HEIGHT1(lds, u, smallTrig, w, 1, me); + fft_HEIGHT1(lds, v, smallTrig, w, 1, me); #else readTailFusedLine(a, p, line1, me); readTailFusedLine(a, q, line2, me); - fft_HEIGHT(lds, u, smallTrig, w); - bar(); - fft_HEIGHT(lds, v, smallTrig, w); - bar(); - fft_HEIGHT(lds, p, smallTrig, w); - bar(); - fft_HEIGHT(lds, q, smallTrig, w); + fft_HEIGHT1(lds, u, smallTrig, w, 1, me); + fft_HEIGHT1(lds, v, smallTrig, w, 1, me); + fft_HEIGHT1(lds, p, smallTrig, w, 1, me); + fft_HEIGHT1(lds, q, smallTrig, w, 1, me); #endif T2 trig = slowTrig_N(line1 + me * H, ND / NH); +#if SINGLE_KERNEL if (line1 == 0) { - reverse(G_H, lds, u + NH/2, true); - reverse(G_H, lds, p + NH/2, true); + reverse(lds, u + NH/2, true); + reverse(lds, p + NH/2, true); pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, true); - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); T2 trig2 = cmulFancy(trig, TAILT); - reverse(G_H, lds, v + NH/2, false); - reverse(G_H, lds, q + NH/2, false); + reverse(lds, v + NH/2, false); + reverse(lds, q + NH/2, false); pairMul(NH/2, v, v + NH/2, q, q + NH/2, trig2, false); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); } else { - reverseLine(G_H, lds, v); - reverseLine(G_H, lds, q); +#else + if (1) { +#endif + reverseLine(lds, v); + reverseLine(lds, q); pairMul(NH, u, v, p, q, trig, false); - reverseLine(G_H, lds, v); + reverseLine(lds, v); } - bar(); - fft_HEIGHT(lds, v, smallTrig, w); - bar(); - fft_HEIGHT(lds, u, smallTrig, w); + dependentLaunch(); // Next kernel will be fftMiddleOutFP64 which must dependentLaunchWait before reading data + + fft_HEIGHT2(lds, v, smallTrig, w, 1, me); + fft_HEIGHT2(lds, u, smallTrig, w, 1, me); writeTailFusedLine(v, out, memline2, me); writeTailFusedLine(u, out, memline1, me); } + +// +// Create a kernel that uses a double-wide workgroup (u in half the workgroup, v in the other half) +// We hope to get better occupancy with the reduced register usage +// + +#else + +// Special pairMul for double-wide line 0: both halves compute their own self-pairing (u with p), +// there is no cross-half data since line_u == 0 and line_v == H/2 both pair with themselves. +void OVERLOAD pairMul2_special(T2 *u, T2 *p, T2 base_squared) { + u32 me = get_local_id(0); + for (i32 i = 0; i < NH / 4; ++i, base_squared = mul_t8(base_squared)) { + if (i == 0 && me == 0) { + u[0] = SWAP_XY(2 * foo2(u[0], p[0])); + u[NH/2] = SWAP_XY(4 * cmul(u[NH/2], p[NH/2])); + } else { + onePairMul(&u[i], &u[NH/2+i], &p[i], &p[NH/2+i], base_squared); + } + T2 new_base_squared = mul_t4(base_squared); + onePairMul(&u[i+NH/4], &u[NH/2+i+NH/4], &p[i+NH/4], &p[NH/2+i+NH/4], new_base_squared); + } +} + +KERNEL_CAP(G_H * 2) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { + local T2 lds[LDS_BYTES(2) / sizeof(T2)]; + LDSinit(lds, 2); + + const u32 H = ND / SMALL_HEIGHT; + + T2 u[NH], p[NH]; + + u32 line_u = get_line_number(base); + u32 line_v = line_u ? H - line_u : (H / 2); + u32 me = get_local_id(0); + u32 lowMe = me % G_H; // lane-id in one of the two halves (half-workgroups). + + // We're going to call the halves "first-half" and "second-half". + bool isSecondHalf = me >= G_H; + + u32 line = !isSecondHalf ? line_u : line_v; + u32 memline = transPos(line, MIDDLE, WIDTH); + + dependentLaunchWait(); // Previous kernel was fftMiddleInFP64 that launched dependents before writing FP64 data + + // Read line u (own half's line) and p (own half's multiplier line) + readTailFusedLine(in, u, line, lowMe); + +#if FFT_VARIANT_H != 0 + T2 w; +#elif NH == 8 + T2 w = fancyTrig_N(H * lowMe); +#else + T2 w = slowTrig_N(H * lowMe, ND / NH); +#endif + +#if MUL_LOW + read(G_H, NH, p, a, memline * SMALL_HEIGHT, lowMe); + fft_HEIGHT1(lds, u, smallTrig, w, 2, lowMe); +#else + readTailFusedLine(a, p, line, lowMe); + fft_HEIGHT1(lds, u, smallTrig, w, 2, lowMe); + fft_HEIGHT1(lds, p, smallTrig, w, 2, lowMe); +#endif + + T2 trig = slowTrig_N(line + H * lowMe, ND / NH * 2); + +#if SINGLE_KERNEL + // Line 0 and H/2 are special: they pair with themselves, line 0 is offseted by 1. + if (line_u == 0) { + reverse2(lds, u); + reverse2(lds, p); + pairMul2_special(u, p, trig); + reverse2(lds, u); + } + else { +#else + if (1) { +#endif + revCrossLine(lds, u); + revCrossLine(lds, p); + pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, false); + revCrossLine(lds, u); + } + + dependentLaunch(); // Next kernel will be fftMiddleOutFP64 which must dependentLaunchWait before reading data + + fft_HEIGHT2(lds, u, smallTrig, w, 2, lowMe); + + // Write line u (own half's line) + writeTailFusedLine(u, out, memline, lowMe); +} + +#endif + #endif @@ -163,8 +354,62 @@ void OVERLOAD pairMul(u32 N, F2 *u, F2 *v, F2 *p, F2 *q, F2 base_squared, bool s } } -KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { - local F2 lds[SMALL_HEIGHT]; +#if !SINGLE_KERNEL +// The kernel tailMulZero handles the special cases in tailMul, i.e. the lines 0 and H/2 +// This kernel is launched with 2 workgroups (handling line 0, resp. H/2) +KERNEL(G_H) tailMulZero(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { + local F2 lds[LDS_BYTES(1) / sizeof(F2)]; + LDSinit(lds, 1); + + F2 u[NH], p[NH]; + const u32 H = ND / SMALL_HEIGHT; + + CP(F2) inF2 = (CP(F2)) in; + CP(F2) aF2 = (CP(F2)) a; + P(F2) outF2 = (P(F2)) out; + TrigFP32 smallTrigF2 = (TrigFP32) smallTrig; + + // This kernel in executed in two workgroups. + u32 which = get_group_id(0); + assert(which < 2); + + u32 line = which ? (H/2) : 0; + u32 memline = transPos(line, MIDDLE, WIDTH); + u32 me = get_local_id(0); + + dependentLaunch(); // Next kernel will be tailMulFP32 which must dependentLaunchWait before reading data from fftMiddleInFP32 + dependentLaunchWait(); // Previous kernel was fftMiddleInFP32 that launched dependents before writing FP32 data + + readTailFusedLine(inF2, u, line, me); + +#if MUL_LOW + read(G_H, NH, p, aF2, memline * SMALL_HEIGHT); + fft_HEIGHT1(lds, u, smallTrigF2, 1, me); +#else + readTailFusedLine(aF2, p, line, me); + fft_HEIGHT1(lds, u, smallTrigF2, 1, me); + fft_HEIGHT1(lds, p, smallTrigF2, 1, me); +#endif + + F2 trig = slowTrig_N(line + me * H, ND / NH); + + reverse(lds, u + NH/2, !which); + reverse(lds, p + NH/2, !which); + pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, !which); + reverse(lds, u + NH/2, !which); + + fft_HEIGHT1(lds, u, smallTrigF2, 1, me); + writeTailFusedLine(u, outF2, memline, me); +} +#endif + +#if SINGLE_WIDE + +KERNEL_CAP(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { + local F2 lds[LDS_BYTES(1) / sizeof(F2)]; + LDSinit(lds, 1); + + const u32 H = ND / SMALL_HEIGHT; CP(F2) inF2 = (CP(F2)) in; CP(F2) aF2 = (CP(F2)) a; @@ -174,13 +419,13 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { F2 u[NH], v[NH]; F2 p[NH], q[NH]; - u32 H = ND / SMALL_HEIGHT; - - u32 line1 = get_group_id(0); + u32 line1 = get_line_number(base); u32 line2 = line1 ? H - line1 : (H / 2); u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); + dependentLaunchWait(); // Previous kernel was fftMiddleInFP32 that launched dependents before writing FP32 data + u32 me = get_local_id(0); readTailFusedLine(inF2, u, line1, me); readTailFusedLine(inF2, v, line2, me); @@ -188,49 +433,141 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { #if MUL_LOW read(G_H, NH, p, aF2, memline1 * SMALL_HEIGHT); read(G_H, NH, q, aF2, memline2 * SMALL_HEIGHT); - fft_HEIGHT(lds, u, smallTrigF2); - bar(); - fft_HEIGHT(lds, v, smallTrigF2); + fft_HEIGHT1(lds, u, smallTrigF2, 1, me); + fft_HEIGHT1(lds, v, smallTrigF2, 1, me); #else readTailFusedLine(aF2, p, line1, me); readTailFusedLine(aF2, q, line2, me); - fft_HEIGHT(lds, u, smallTrigF2); - bar(); - fft_HEIGHT(lds, v, smallTrigF2); - bar(); - fft_HEIGHT(lds, p, smallTrigF2); - bar(); - fft_HEIGHT(lds, q, smallTrigF2); + fft_HEIGHT1(lds, u, smallTrigF2, 1, me); + fft_HEIGHT1(lds, v, smallTrigF2, 1, me); + fft_HEIGHT1(lds, p, smallTrigF2, 1, me); + fft_HEIGHT1(lds, q, smallTrigF2, 1, me); #endif F2 trig = slowTrig_N(line1 + me * H, ND / NH); +#if SINGLE_KERNEL if (line1 == 0) { - reverse(G_H, lds, u + NH/2, true); - reverse(G_H, lds, p + NH/2, true); + reverse(lds, u + NH/2, true); + reverse(lds, p + NH/2, true); pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, true); - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); F2 trig2 = cmulFancy(trig, TAILT); - reverse(G_H, lds, v + NH/2, false); - reverse(G_H, lds, q + NH/2, false); + reverse(lds, v + NH/2, false); + reverse(lds, q + NH/2, false); pairMul(NH/2, v, v + NH/2, q, q + NH/2, trig2, false); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); } else { - reverseLine(G_H, lds, v); - reverseLine(G_H, lds, q); +#else + if (1) { +#endif + reverseLine(lds, v); + reverseLine(lds, q); pairMul(NH, u, v, p, q, trig, false); - reverseLine(G_H, lds, v); + reverseLine(lds, v); } - bar(); - fft_HEIGHT(lds, v, smallTrigF2); - bar(); - fft_HEIGHT(lds, u, smallTrigF2); + dependentLaunch(); // Next kernel will be fftMiddleOutFP32 which must dependentLaunchWait before reading data + + fft_HEIGHT2(lds, v, smallTrigF2, 1, me); + fft_HEIGHT2(lds, u, smallTrigF2, 1, me); writeTailFusedLine(v, outF2, memline2, me); writeTailFusedLine(u, outF2, memline1, me); } + +// +// Create a kernel that uses a double-wide workgroup (u in half the workgroup, v in the other half) +// We hope to get better occupancy with the reduced register usage +// + +#else + +// Special pairMul for double-wide line 0: both halves compute their own self-pairing (u with p), +// there is no cross-half data since line_u == 0 and line_v == H/2 both pair with themselves. +void OVERLOAD pairMul2_special(F2 *u, F2 *p, F2 base_squared) { + u32 me = get_local_id(0); + for (i32 i = 0; i < NH / 4; ++i, base_squared = mul_t8(base_squared)) { + if (i == 0 && me == 0) { + u[0] = SWAP_XY(2 * foo2(u[0], p[0])); + u[NH/2] = SWAP_XY(4 * cmul(u[NH/2], p[NH/2])); + } else { + onePairMul(&u[i], &u[NH/2+i], &p[i], &p[NH/2+i], base_squared); + } + F2 new_base_squared = mul_t4(base_squared); + onePairMul(&u[i+NH/4], &u[NH/2+i+NH/4], &p[i+NH/4], &p[NH/2+i+NH/4], new_base_squared); + } +} + +KERNEL_CAP(G_H * 2) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { + local F2 lds[LDS_BYTES(2) / sizeof(F2)]; + LDSinit(lds, 2); + + const u32 H = ND / SMALL_HEIGHT; + + CP(F2) inF2 = (CP(F2)) in; + CP(F2) aF2 = (CP(F2)) a; + P(F2) outF2 = (P(F2)) out; + TrigFP32 smallTrigF2 = (TrigFP32) smallTrig; + + F2 u[NH], p[NH]; + + u32 line_u = get_line_number(base); + u32 line_v = line_u ? H - line_u : (H / 2); + u32 me = get_local_id(0); + u32 lowMe = me % G_H; // lane-id in one of the two halves (half-workgroups). + + // We're going to call the halves "first-half" and "second-half". + bool isSecondHalf = me >= G_H; + + u32 line = !isSecondHalf ? line_u : line_v; + u32 memline = transPos(line, MIDDLE, WIDTH); + + dependentLaunchWait(); // Previous kernel was fftMiddleInFP32 that launched dependents before writing FP32 data + + // Read line u (own half's line) and p (own half's multiplier line) + readTailFusedLine(inF2, u, line, lowMe); + +#if MUL_LOW + read(G_H, NH, p, aF2, memline * SMALL_HEIGHT, lowMe); + fft_HEIGHT1(lds, u, smallTrigF2, 2, lowMe); +#else + readTailFusedLine(aF2, p, line, lowMe); + fft_HEIGHT1(lds, u, smallTrigF2, 2, lowMe); + fft_HEIGHT1(lds, p, smallTrigF2, 2, lowMe); +#endif + + F2 trig = slowTrig_N(line + H * lowMe, ND / NH * 2); + +#if SINGLE_KERNEL + // Line 0 and H/2 are special: they pair with themselves, line 0 is offseted by 1. + if (line_u == 0) { + reverse2(lds, u); + reverse2(lds, p); + pairMul2_special(u, p, trig); + reverse2(lds, u); + } + else { +#else + if (1) { +#endif + revCrossLine(lds, u); + revCrossLine(lds, p); + pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, false); + revCrossLine(lds, u); + } + + dependentLaunch(); // Next kernel will be fftMiddleOutFP32 which must dependentLaunchWait before reading data + + fft_HEIGHT2(lds, u, smallTrigF2, 2, lowMe); + + // Write line u (own half's line) + writeTailFusedLine(u, outF2, memline, lowMe); +} + +#endif + #endif @@ -242,13 +579,12 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { void OVERLOAD onePairMul(GF31* pa, GF31* pb, GF31* pc, GF31* pd, GF31 t_squared) { GF31 a = *pa, b = *pb, c = *pc, d = *pd; - X2conjb(a, b); X2conjb(c, d); - - *pa = sub(cmul(a, c), cmul(cmul(b, d), t_squared)); - *pb = add(cmul(b, c), cmul(a, d)); - + GF31 ac = cmul(a, c); + GF31 bd = cmul(b, d); + *pa = sub(ac, cmul(bd, t_squared)); + *pb = sub(sub(cmul(add(a, b), add(c, d)), ac), bd); X2_conjb(*pa, *pb); *pa = SWAP_XY(*pa), *pb = SWAP_XY(*pb); } @@ -277,8 +613,79 @@ void OVERLOAD pairMul(u32 N, GF31 *u, GF31 *v, GF31 *p, GF31 *q, GF31 base_squar } } -KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { - local GF31 lds[SMALL_HEIGHT]; +#if !SINGLE_KERNEL +// The kernel tailMulZeroGF31 handles the special cases in tailMulGF31, i.e. the lines 0 and H/2 +// This kernel is launched with 2 workgroups (handling line 0, resp. H/2) +KERNEL(G_H) tailMulZeroGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { + local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; + LDSinit(lds, 1); + + GF31 u[NH], p[NH]; + const u32 H = ND / SMALL_HEIGHT; + + CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); + CP(GF31) a31 = (CP(GF31)) (a + DISTGF31); + P(GF31) out31 = (P(GF31)) (out + DISTGF31); + TrigGF31 smallTrig31 = (TrigGF31) (smallTrig + DISTHTRIGGF31); + + // This kernel in executed in two workgroups. + u32 which = get_group_id(0); + assert(which < 2); + + u32 line = which ? (H/2) : 0; + u32 memline = transPos(line, MIDDLE, WIDTH); + u32 me = get_local_id(0); + + dependentLaunch(); // Next kernel will be tailMulGF31 which must dependentLaunchWait before reading data from fftMiddleInGF31 + dependentLaunchWait(); // Previous kernel was fftMiddleInGF31 that launched dependents before writing GF31 data + + readTailFusedLine(in31, u, line, me); + +#if MUL_LOW + read(G_H, NH, p, a31, memline * SMALL_HEIGHT); + fft_HEIGHT1(lds, u, smallTrig31, 1, me); +#else + readTailFusedLine(a31, p, line, me); + fft_HEIGHT1(lds, u, smallTrig31, 1, me); + fft_HEIGHT1(lds, p, smallTrig31, 1, me); +#endif + + // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) + // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. + u32 height_trigs = SMALL_HEIGHT*1; +#if TAIL_TRIGS31 >= 1 + GF31 trig = TFLOAD(&smallTrig31[height_trigs + me]); +#if SINGLE_WIDE + GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + line]); +#else + GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + which]); +#endif + trig = cmul(trig, mult); +#else +#if SINGLE_WIDE + GF31 trig = TOLOAD(&smallTrig31[height_trigs + line*G_H + me]); +#else + GF31 trig = TOLOAD(&smallTrig31[height_trigs + which*G_H + me]); +#endif +#endif + + reverse(lds, u + NH/2, !which); + reverse(lds, p + NH/2, !which); + pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, !which); + reverse(lds, u + NH/2, !which); + + fft_HEIGHT2(lds, u, smallTrig31, 1, me); + writeTailFusedLine(u, out31, memline, me); +} +#endif + +#if SINGLE_WIDE + +KERNEL_CAP(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { + local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; + LDSinit(lds, 1); + + const u32 H = ND / SMALL_HEIGHT; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); CP(GF31) a31 = (CP(GF31)) (a + DISTGF31); @@ -288,13 +695,13 @@ KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { GF31 u[NH], v[NH]; GF31 p[NH], q[NH]; - u32 H = ND / SMALL_HEIGHT; - - u32 line1 = get_group_id(0); + u32 line1 = get_line_number(base); u32 line2 = line1 ? H - line1 : (H / 2); u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); + dependentLaunchWait(); // Previous kernel was fftMiddleInGF31 that launched dependents before writing GF31 data + u32 me = get_local_id(0); readTailFusedLine(in31, u, line1, me); readTailFusedLine(in31, v, line2, me); @@ -302,66 +709,158 @@ KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { #if MUL_LOW read(G_H, NH, p, a31, memline1 * SMALL_HEIGHT); read(G_H, NH, q, a31, memline2 * SMALL_HEIGHT); - fft_HEIGHT(lds, u, smallTrig31); - bar(); - fft_HEIGHT(lds, v, smallTrig31); + fft_HEIGHT1(lds, u, smallTrig31, 1, me); + fft_HEIGHT1(lds, v, smallTrig31, 1, me); #else readTailFusedLine(a31, p, line1, me); readTailFusedLine(a31, q, line2, me); - fft_HEIGHT(lds, u, smallTrig31); - bar(); - fft_HEIGHT(lds, v, smallTrig31); - bar(); - fft_HEIGHT(lds, p, smallTrig31); - bar(); - fft_HEIGHT(lds, q, smallTrig31); + fft_HEIGHT1(lds, u, smallTrig31, 1, me); + fft_HEIGHT1(lds, v, smallTrig31, 1, me); + fft_HEIGHT1(lds, p, smallTrig31, 1, me); + fft_HEIGHT1(lds, q, smallTrig31, 1, me); #endif // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; #if TAIL_TRIGS31 >= 1 - GF31 trig = smallTrig31[height_trigs + me]; // Trig values for line zero, should be cached -#if SINGLE_WIDE - GF31 mult = smallTrig31[height_trigs + G_H + line1]; -#else - GF31 mult = smallTrig31[height_trigs + G_H + line1 * 2]; -#endif + GF31 trig = TFLOAD(&smallTrig31[height_trigs + me]); // Trig values for line zero, should be cached + GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + line1]); trig = cmul(trig, mult); #else -#if SINGLE_WIDE - GF31 trig = NTLOAD(smallTrig31[height_trigs + line1*G_H + me]); -#else - GF31 trig = NTLOAD(smallTrig31[height_trigs + line1*2*G_H + me]); -#endif + GF31 trig = TOLOAD(&smallTrig31[height_trigs + line1*G_H + me]); #endif +#if SINGLE_KERNEL if (line1 == 0) { - reverse(G_H, lds, u + NH/2, true); - reverse(G_H, lds, p + NH/2, true); + reverse(lds, u + NH/2, true); + reverse(lds, p + NH/2, true); pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, true); - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); GF31 trig2 = cmul(trig, TAILTGF31); - reverse(G_H, lds, v + NH/2, false); - reverse(G_H, lds, q + NH/2, false); + reverse(lds, v + NH/2, false); + reverse(lds, q + NH/2, false); pairMul(NH/2, v, v + NH/2, q, q + NH/2, trig2, false); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); } else { - reverseLine(G_H, lds, v); - reverseLine(G_H, lds, q); +#else + if (1) { +#endif + reverseLine(lds, v); + reverseLine(lds, q); pairMul(NH, u, v, p, q, trig, false); - reverseLine(G_H, lds, v); + reverseLine(lds, v); } - bar(); - fft_HEIGHT(lds, v, smallTrig31); - bar(); - fft_HEIGHT(lds, u, smallTrig31); + dependentLaunch(); // Next kernel will be fftMiddleOutGF31 which must dependentLaunchWait before reading data + + fft_HEIGHT2(lds, v, smallTrig31, 1, me); + fft_HEIGHT2(lds, u, smallTrig31, 1, me); writeTailFusedLine(v, out31, memline2, me); writeTailFusedLine(u, out31, memline1, me); } + +// +// Create a kernel that uses a double-wide workgroup (u in half the workgroup, v in the other half) +// We hope to get better occupancy with the reduced register usage +// + +#else + +// Special pairMul for double-wide line 0: both halves compute their own self-pairing (u with p), +// there is no cross-half data since line_u == 0 and line_v == H/2 both pair with themselves. +void OVERLOAD pairMul2_special(GF31 *u, GF31 *p, GF31 base_squared) { + u32 me = get_local_id(0); + for (i32 i = 0; i < NH / 4; ++i, base_squared = mul_t8(base_squared)) { + if (i == 0 && me == 0) { + u[0] = SWAP_XY(mul2(foo2(u[0], p[0]))); + u[NH/2] = SWAP_XY(shl(cmul(u[NH/2], p[NH/2]), 2)); + } else { + onePairMul(&u[i], &u[NH/2+i], &p[i], &p[NH/2+i], base_squared); + } + onePairMul(&u[i+NH/4], &u[NH/2+i+NH/4], &p[i+NH/4], &p[NH/2+i+NH/4], mul_t4(base_squared)); + } +} + +KERNEL_CAP(G_H * 2) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { + local GF31 lds[LDS_BYTES(2) / sizeof(GF31)]; + LDSinit(lds, 2); + + const u32 H = ND / SMALL_HEIGHT; + + CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); + CP(GF31) a31 = (CP(GF31)) (a + DISTGF31); + P(GF31) out31 = (P(GF31)) (out + DISTGF31); + TrigGF31 smallTrig31 = (TrigGF31) (smallTrig + DISTHTRIGGF31); + + GF31 u[NH], p[NH]; + + u32 line_u = get_line_number(base); + u32 line_v = line_u ? H - line_u : (H / 2); + u32 me = get_local_id(0); + u32 lowMe = me % G_H; // lane-id in one of the two halves (half-workgroups). + + // We're going to call the halves "first-half" and "second-half". + bool isSecondHalf = me >= G_H; + + u32 line = !isSecondHalf ? line_u : line_v; + u32 memline = transPos(line, MIDDLE, WIDTH); + + dependentLaunchWait(); // Previous kernel was fftMiddleInGF31 that launched dependents before writing GF31 data + + // Read line u (own half's line) and p (own half's multiplier line) + readTailFusedLine(in31, u, line, lowMe); + +#if MUL_LOW + read(G_H, NH, p, a31, memline * SMALL_HEIGHT, lowMe); + fft_HEIGHT1(lds, u, smallTrig31, 2, lowMe); +#else + readTailFusedLine(a31, p, line, lowMe); + fft_HEIGHT1(lds, u, smallTrig31, 2, lowMe); + fft_HEIGHT1(lds, p, smallTrig31, 2, lowMe); +#endif + + // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) + // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. + u32 height_trigs = SMALL_HEIGHT*1; +#if TAIL_TRIGS31 >= 1 + GF31 trig = TFLOAD(&smallTrig31[height_trigs + lowMe]); // Trig values for line zero, should be cached + GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + line_u*2 + isSecondHalf]); // Two multipliers. One for line u, one for line v. + trig = cmul(trig, mult); +#else + GF31 trig = TOLOAD(&smallTrig31[height_trigs + line_u*G_H*2 + me]); +#endif + +#if SINGLE_KERNEL + // Line 0 and H/2 are special: they pair with themselves, line 0 is offseted by 1. + if (line_u == 0) { + reverse2(lds, u); + reverse2(lds, p); + pairMul2_special(u, p, trig); + reverse2(lds, u); + } + else { +#else + if (1) { +#endif + revCrossLine(lds, u); + revCrossLine(lds, p); + pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, false); + revCrossLine(lds, u); + } + + dependentLaunch(); // Next kernel will be fftMiddleOutGF31 which must dependentLaunchWait before reading data + + fft_HEIGHT2(lds, u, smallTrig31, 2, lowMe); + + // Write line u (own half's line) + writeTailFusedLine(u, out31, memline, lowMe); +} + +#endif + #endif @@ -373,12 +872,15 @@ KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { void OVERLOAD onePairMul(GF61* pa, GF61* pb, GF61* pc, GF61* pd, GF61 t_squared) { GF61 a = *pa, b = *pb, c = *pc, d = *pd; - X2conjb(a, b); X2conjb(c, d); - GF61 e = subq(cmul(a, c), cmul(cmul(b, d), t_squared), 2); // Max value is 3*M61+epsilon - GF61 f = addq(cmul(b, c), cmul(a, d)); // Max value is 2*M61+epsilon - X2s_conjb(&e, &f, 4, 3); + GF61 ac = cmul(a, c); + GF61 bd = cmul(b, d); + GF61 e = subq(ac, cmul(bd, t_squared)); // Range is -1-..1+ + GF61 f = subq(subq(cmul(add(a, b), add(c, d)), ac), bd); // Compute bc + ad. Range is -2-..1+ + X2q_conjb(&e, &f); // e range is -3-..2+, f.x range is -2-..3+, f.y range is -3-..2+ + e = modM61q(e, 4); + f = modM61q(f, 3, 4); *pa = SWAP_XY(e), *pb = SWAP_XY(f); } @@ -406,8 +908,79 @@ void OVERLOAD pairMul(u32 N, GF61 *u, GF61 *v, GF61 *p, GF61 *q, GF61 base_squar } } -KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { - local GF61 lds[SMALL_HEIGHT]; +#if !SINGLE_KERNEL +// The kernel tailMulZeroGF61 handles the special cases in tailMulGF61, i.e. the lines 0 and H/2 +// This kernel is launched with 2 workgroups (handling line 0, resp. H/2) +KERNEL(G_H) tailMulZeroGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { + local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; + LDSinit(lds, 1); + + GF61 u[NH], p[NH]; + const u32 H = ND / SMALL_HEIGHT; + + CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); + CP(GF61) a61 = (CP(GF61)) (a + DISTGF61); + P(GF61) out61 = (P(GF61)) (out + DISTGF61); + TrigGF61 smallTrig61 = (TrigGF61) (smallTrig + DISTHTRIGGF61); + + // This kernel in executed in two workgroups. + u32 which = get_group_id(0); + assert(which < 2); + + u32 line = which ? (H/2) : 0; + u32 memline = transPos(line, MIDDLE, WIDTH); + u32 me = get_local_id(0); + + dependentLaunch(); // Next kernel will be tailMulGF61 which must dependentLaunchWait before reading data from fftMiddleInGF61 + dependentLaunchWait(); // Previous kernel was fftMiddleInGF61 that launched dependents before writing GF61 data + + readTailFusedLine(in61, u, line, me); + +#if MUL_LOW + read(G_H, NH, p, a61, memline * SMALL_HEIGHT); + fft_HEIGHT1(lds, u, smallTrig61, 1, me); +#else + readTailFusedLine(a61, p, line, me); + fft_HEIGHT1(lds, u, smallTrig61, 1, me); + fft_HEIGHT1(lds, p, smallTrig61, 1, me); +#endif + + // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) + // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. + u32 height_trigs = SMALL_HEIGHT*1; +#if TAIL_TRIGS61 >= 1 + GF61 trig = TFLOAD(&smallTrig61[height_trigs + me]); +#if SINGLE_WIDE + GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + line]); +#else + GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + which]); +#endif + trig = cmul(trig, mult); +#else +#if SINGLE_WIDE + GF61 trig = TOLOAD(&smallTrig61[height_trigs + line*G_H + me]); +#else + GF61 trig = TOLOAD(&smallTrig61[height_trigs + which*G_H + me]); +#endif +#endif + + reverse(lds, u + NH/2, !which); + reverse(lds, p + NH/2, !which); + pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, !which); + reverse(lds, u + NH/2, !which); + + fft_HEIGHT2(lds, u, smallTrig61, 1, me); + writeTailFusedLine(u, out61, memline, me); +} +#endif + +#if SINGLE_WIDE + +KERNEL_CAP(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { + local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; + LDSinit(lds, 1); + + const u32 H = ND / SMALL_HEIGHT; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); CP(GF61) a61 = (CP(GF61)) (a + DISTGF61); @@ -417,13 +990,13 @@ KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { GF61 u[NH], v[NH]; GF61 p[NH], q[NH]; - u32 H = ND / SMALL_HEIGHT; - - u32 line1 = get_group_id(0); + u32 line1 = get_line_number(base); u32 line2 = line1 ? H - line1 : (H / 2); u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); + dependentLaunchWait(); // Previous kernel was fftMiddleInGF61 that launched dependents before writing GF61 data + u32 me = get_local_id(0); readTailFusedLine(in61, u, line1, me); readTailFusedLine(in61, v, line2, me); @@ -431,64 +1004,156 @@ KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { #if MUL_LOW read(G_H, NH, p, a61, memline1 * SMALL_HEIGHT); read(G_H, NH, q, a61, memline2 * SMALL_HEIGHT); - fft_HEIGHT(lds, u, smallTrig61); - bar(); - fft_HEIGHT(lds, v, smallTrig61); + fft_HEIGHT1(lds, u, smallTrig61, 1, me); + fft_HEIGHT1(lds, v, smallTrig61, 1, me); #else readTailFusedLine(a61, p, line1, me); readTailFusedLine(a61, q, line2, me); - fft_HEIGHT(lds, u, smallTrig61); - bar(); - fft_HEIGHT(lds, v, smallTrig61); - bar(); - fft_HEIGHT(lds, p, smallTrig61); - bar(); - fft_HEIGHT(lds, q, smallTrig61); + fft_HEIGHT1(lds, u, smallTrig61, 1, me); + fft_HEIGHT1(lds, v, smallTrig61, 1, me); + fft_HEIGHT1(lds, p, smallTrig61, 1, me); + fft_HEIGHT1(lds, q, smallTrig61, 1, me); #endif // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; #if TAIL_TRIGS61 >= 1 - GF61 trig = smallTrig61[height_trigs + me]; // Trig values for line zero, should be cached -#if SINGLE_WIDE - GF61 mult = smallTrig61[height_trigs + G_H + line1]; -#else - GF61 mult = smallTrig61[height_trigs + G_H + line1 * 2]; -#endif + GF61 trig = TFLOAD(&smallTrig61[height_trigs + me]); // Trig values for line zero, should be cached + GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + line1]); trig = cmul(trig, mult); #else -#if SINGLE_WIDE - GF61 trig = NTLOAD(smallTrig61[height_trigs + line1*G_H + me]); -#else - GF61 trig = NTLOAD(smallTrig61[height_trigs + line1*2*G_H + me]); -#endif + GF61 trig = TOLOAD(&smallTrig61[height_trigs + line1*G_H + me]); #endif +#if SINGLE_KERNEL if (line1 == 0) { - reverse(G_H, lds, u + NH/2, true); - reverse(G_H, lds, p + NH/2, true); + reverse(lds, u + NH/2, true); + reverse(lds, p + NH/2, true); pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, true); - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); GF61 trig2 = cmul(trig, TAILTGF61); - reverse(G_H, lds, v + NH/2, false); - reverse(G_H, lds, q + NH/2, false); + reverse(lds, v + NH/2, false); + reverse(lds, q + NH/2, false); pairMul(NH/2, v, v + NH/2, q, q + NH/2, trig2, false); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); } else { - reverseLine(G_H, lds, v); - reverseLine(G_H, lds, q); +#else + if (1) { +#endif + reverseLine(lds, v); + reverseLine(lds, q); pairMul(NH, u, v, p, q, trig, false); - reverseLine(G_H, lds, v); + reverseLine(lds, v); } - bar(); - fft_HEIGHT(lds, v, smallTrig61); - bar(); - fft_HEIGHT(lds, u, smallTrig61); + dependentLaunch(); // Next kernel will be fftMiddleOutGF61 which must dependentLaunchWait before reading data + + fft_HEIGHT2(lds, v, smallTrig61, 1, me); + fft_HEIGHT2(lds, u, smallTrig61, 1, me); writeTailFusedLine(v, out61, memline2, me); writeTailFusedLine(u, out61, memline1, me); } + +// +// Create a kernel that uses a double-wide workgroup (u in half the workgroup, v in the other half) +// We hope to get better occupancy with the reduced register usage +// + +#else + +// Special pairMul for double-wide line 0: both halves compute their own self-pairing (u with p), +// there is no cross-half data since line_u == 0 and line_v == H/2 both pair with themselves. +void OVERLOAD pairMul2_special(GF61 *u, GF61 *p, GF61 base_squared) { + u32 me = get_local_id(0); + for (i32 i = 0; i < NH / 4; ++i, base_squared = mul_t8(base_squared)) { + if (i == 0 && me == 0) { + u[0] = SWAP_XY(mul2(foo2(u[0], p[0]))); + u[NH/2] = SWAP_XY(shl(cmul(u[NH/2], p[NH/2]), 2)); + } else { + onePairMul(&u[i], &u[NH/2+i], &p[i], &p[NH/2+i], base_squared); + } + onePairMul(&u[i+NH/4], &u[NH/2+i+NH/4], &p[i+NH/4], &p[NH/2+i+NH/4], mul_t4(base_squared)); + } +} + +KERNEL_CAP(G_H * 2) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { + local GF61 lds[LDS_BYTES(2) / sizeof(GF61)]; + LDSinit(lds, 2); + + const u32 H = ND / SMALL_HEIGHT; + + CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); + CP(GF61) a61 = (CP(GF61)) (a + DISTGF61); + P(GF61) out61 = (P(GF61)) (out + DISTGF61); + TrigGF61 smallTrig61 = (TrigGF61) (smallTrig + DISTHTRIGGF61); + + GF61 u[NH], p[NH]; + + u32 line_u = get_line_number(base); + u32 line_v = line_u ? H - line_u : (H / 2); + u32 me = get_local_id(0); + u32 lowMe = me % G_H; // lane-id in one of the two halves (half-workgroups). + + // We're going to call the halves "first-half" and "second-half". + bool isSecondHalf = me >= G_H; + + u32 line = !isSecondHalf ? line_u : line_v; + u32 memline = transPos(line, MIDDLE, WIDTH); + + dependentLaunchWait(); // Previous kernel was fftMiddleInGF61 that launched dependents before writing GF61 data + + // Read line u (own half's line) and p (own half's multiplier line) + readTailFusedLine(in61, u, line, lowMe); + +#if MUL_LOW + read(G_H, NH, p, a61, memline * SMALL_HEIGHT, lowMe); + fft_HEIGHT1(lds, u, smallTrig61, 2, lowMe); +#else + readTailFusedLine(a61, p, line, lowMe); + fft_HEIGHT1(lds, u, smallTrig61, 2, lowMe); + fft_HEIGHT1(lds, p, smallTrig61, 2, lowMe); +#endif + + // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) + // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. + u32 height_trigs = SMALL_HEIGHT*1; +#if TAIL_TRIGS61 >= 1 + GF61 trig = TFLOAD(&smallTrig61[height_trigs + lowMe]); // Trig values for line zero, should be cached + GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + line_u*2 + isSecondHalf]); // Two multipliers. One for line u, one for line v. + trig = cmul(trig, mult); +#else + GF61 trig = TOLOAD(&smallTrig61[height_trigs + line_u*G_H*2 + me]); +#endif + +#if SINGLE_KERNEL + // Line 0 and H/2 are special: they pair with themselves, line 0 is offseted by 1. + if (line_u == 0) { + reverse2(lds, u); + reverse2(lds, p); + pairMul2_special(u, p, trig); + reverse2(lds, u); + } + else { +#else + if (1) { +#endif + revCrossLine(lds, u); + revCrossLine(lds, p); + pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, false); + revCrossLine(lds, u); + } + + dependentLaunch(); // Next kernel will be fftMiddleOutGF61 which must dependentLaunchWait before reading data + + fft_HEIGHT2(lds, u, smallTrig61, 2, lowMe); + + // Write line u (own half's line) + writeTailFusedLine(u, out61, memline, lowMe); +} + +#endif + #endif diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index bf960f1c..09ec76cf 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -1,9 +1,47 @@ // Copyright (C) Mihai Preda and George Woltman -#include "tailutil.cl" -#include "trig.cl" +#include "base.cl" #include "fftheight.cl" +#define INCLUDE_FILE "tailutil.cl" +#include "expand.cl" +#define INCLUDE_FILE "middle.cl" +#include "expand.cl" + +// If not doing L2 stripes, process the lines in any order. +// If L2 striping, process lines output by fftMiddleIn. fftMiddleIn outputs 16 * MIDDLE tailSquare lines. +u32 get_line_number(u32 base) { + u32 g = get_group_id(0); +#if !SINGLE_KERNEL +#if L2_STRIPING + if (base == 0) g = g + 1; +#else + g = g + 1; +#endif +#endif +#if L2_STRIPING + // Old, simple L2 striping code + // return get_group_id(1) * WIDTH + base + g; + + // Process all lines from low half of base_lo stripe group. One stripe group is stripe_group_size * 16 * MIDDLE lines. + u32 base_lo = base; + u32 stripe_group_size = L2_STRIPING; + u32 half_size = (MIDDLE + 1) / 2; // For base_lo, round odd middles up. + u32 kernelsToExecute = half_size * stripe_group_size * 16; + if (g < kernelsToExecute) return g / (stripe_group_size * 16) * WIDTH + base_lo + g % (stripe_group_size * 16); + g -= kernelsToExecute; + + // Process lines from low half of base_hi stripe group. One stripe group is stripe_group_size * 16 * MIDDLE lines. + // The first line in the base_hi stripe group is not ready for processing (except for the last group). + u32 base_hi = WIDTH - stripe_group_size * 16 - base_lo; + if (base_hi != WIDTH / 2) base_hi++; // Skip first line in base_hi (usually) + half_size = MIDDLE / 2; // For base_hi, round odd middles up. + return g % half_size * WIDTH + base_hi + g / half_size; +#else + return g; +#endif +} + #if FFT_FP64 // Handle the final squaring step on a pair of complex numbers. Swap real and imaginary results for the inverse FFT. @@ -51,12 +89,15 @@ void OVERLOAD pairSq(u32 N, T2 *u, T2 *v, T2 base_squared, bool special) { } } +#if !SINGLE_KERNEL // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { - local T2 lds[SMALL_HEIGHT / 2]; + local T2 lds[LDS_BYTES(1) / sizeof(T2)]; + LDSinit(lds, 1); + T2 u[NH]; - u32 H = ND / SMALL_HEIGHT; + const u32 H = ND / SMALL_HEIGHT; // This kernel in executed in two workgroups. u32 which = get_group_id(0); @@ -64,65 +105,65 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { u32 line = which ? (H/2) : 0; u32 me = get_local_id(0); + + dependentLaunch(); // Next kernel will be tailSquareFP64 which must dependentLaunchWait before reading data from fftMiddleInFP64 + dependentLaunchWait(); // Previous kernel was fftMiddleInFP64 that launched dependents before writing FP64 data + readTailFusedLine(in, u, line, me); -#if NH == 8 - T2 w = fancyTrig_N(ND / SMALL_HEIGHT * me); +#if FFT_VARIANT_H != 0 + T2 w; +#elif NH == 8 + T2 w = fancyTrig_N(H * me); #else - T2 w = slowTrig_N(ND / SMALL_HEIGHT * me, ND / NH); + T2 w = slowTrig_N(H * me, ND / NH); #endif T2 trig = slowTrig_N(line + me * H, ND / NH); - fft_HEIGHT(lds, u, smallTrig, w); - reverse(G_H, lds, u + NH/2, !which); + fft_HEIGHT1(lds, u, smallTrig, w, 1, me); + reverse(lds, u + NH/2, !which); pairSq(NH/2, u, u + NH/2, trig, !which); - reverse(G_H, lds, u + NH/2, !which); + reverse(lds, u + NH/2, !which); - bar(); - fft_HEIGHT(lds, u, smallTrig, w); + fft_HEIGHT1(lds, u, smallTrig, w, 1, me); writeTailFusedLine(u, out, transPos(line, MIDDLE, WIDTH), me); } +#endif #if SINGLE_WIDE -KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { - local T2 lds[SMALL_HEIGHT]; +KERNEL_CAP(G_H) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { + local T2 lds[LDS_BYTES(1) / sizeof(T2)]; + LDSinit(lds, 1); - T2 u[NH], v[NH]; + const u32 H = ND / SMALL_HEIGHT; - u32 H = ND / SMALL_HEIGHT; + T2 u[NH], v[NH]; -#if SINGLE_KERNEL - u32 line1 = get_group_id(0); + u32 line1 = get_line_number(base); u32 line2 = line1 ? H - line1 : (H / 2); -#else - u32 line1 = get_group_id(0) + 1; - u32 line2 = H - line1; -#endif u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); u32 me = get_local_id(0); + + dependentLaunchWait(); // Previous kernel was fftMiddleInFP64 that launched dependents before writing FP64 data + readTailFusedLine(in, u, line1, me); readTailFusedLine(in, v, line2, me); -#if NH == 8 - T2 w = fancyTrig_N(ND / SMALL_HEIGHT * me); +#if FFT_VARIANT_H != 0 + T2 w; +#elif NH == 8 + T2 w = fancyTrig_N(H * me); #else - T2 w = slowTrig_N(ND / SMALL_HEIGHT * me, ND / NH); + T2 w = slowTrig_N(H * me, ND / NH); #endif -#if ZEROHACK_H - u32 zerohack = (u32) get_group_id(0) / 131072; - fft_HEIGHT(lds + zerohack, u, smallTrig + zerohack, w); - bar(); - fft_HEIGHT(lds + zerohack, v, smallTrig + zerohack, w); -#else - fft_HEIGHT(lds, u, smallTrig, w); - bar(); - fft_HEIGHT(lds, v, smallTrig, w); -#endif + u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; + fft_HEIGHT1(lds + zerohack, u, smallTrig + zerohack, w, 1, me); + fft_HEIGHT1(lds + zerohack, v, smallTrig + zerohack, w, 1, me); // Compute trig values from scratch. Good on GPUs with high DP throughput. #if TAIL_TRIGS == 2 @@ -134,8 +175,8 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*5; // Read a hopefully cached line of data and one non-cached T2 per line - T2 trig = smallTrig[height_trigs + me]; // Trig values for line zero, should be cached - T2 mult = smallTrig[height_trigs + G_H + line1]; // Line multiplier + T2 trig = TFLOAD(&smallTrig[height_trigs + me]); // Trig values for line zero, should be cached + T2 mult = TSLOAD(&smallTrig[height_trigs + G_H + line1]); // Line multiplier trig = cmulFancy(trig, mult); // On consumer-grade GPUs, it is likely beneficial to read all trig values. @@ -144,35 +185,35 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*5; // Read pre-computed trig values - T2 trig = NTLOAD(smallTrig[height_trigs + line1*G_H + me]); + T2 trig = TOLOAD(&smallTrig[height_trigs + line1*G_H + me]); #endif #if SINGLE_KERNEL if (line1 == 0) { // Line 0 is special: it pairs with itself, offseted by 1. - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); pairSq(NH/2, u, u + NH/2, trig, true); - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); // Line H/2 also pairs with itself (but without offset). T2 trig2 = cmulFancy(trig, TAILT); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); pairSq(NH/2, v, v + NH/2, trig2, false); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); } else { #else if (1) { #endif - reverseLine(G_H, lds, v); + reverseLine(lds, v); pairSq(NH, u, v, trig, false); - reverseLine(G_H, lds, v); + reverseLine(lds, v); } - bar(); - fft_HEIGHT(lds, v, smallTrig, w); - bar(); - fft_HEIGHT(lds, u, smallTrig, w); + dependentLaunch(); // Next kernel will be fftMiddleOutFP64 which must dependentLaunchWait before reading data + + fft_HEIGHT2(lds, v, smallTrig, w, 1, me); + fft_HEIGHT2(lds, u, smallTrig, w, 1, me); writeTailFusedLine(v, out, memline2, me); writeTailFusedLine(u, out, memline1, me); @@ -201,21 +242,16 @@ void OVERLOAD pairSq2_special(T2 *u, T2 base_squared) { } } -KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { - local T2 lds[SMALL_HEIGHT]; +KERNEL_CAP(G_H * 2) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { + local T2 lds[LDS_BYTES(2) / sizeof(T2)]; + LDSinit(lds, 2); - T2 u[NH]; + const u32 H = ND / SMALL_HEIGHT; - u32 H = ND / SMALL_HEIGHT; + T2 u[NH]; -#if SINGLE_KERNEL - u32 line_u = get_group_id(0); + u32 line_u = get_line_number(base); u32 line_v = line_u ? H - line_u : (H / 2); -#else - u32 line_u = get_group_id(0) + 1; - u32 line_v = H - line_u; -#endif - u32 me = get_local_id(0); u32 lowMe = me % G_H; // lane-id in one of the two halves (half-workgroups). @@ -224,21 +260,21 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { u32 line = !isSecondHalf ? line_u : line_v; + dependentLaunchWait(); // Previous kernel was fftMiddleInFP64 that launched dependents before writing FP64 data + // Read lines u and v readTailFusedLine(in, u, line, lowMe); -#if NH == 8 +#if FFT_VARIANT_H != 0 + T2 w; +#elif NH == 8 T2 w = fancyTrig_N(H * lowMe); #else T2 w = slowTrig_N(H * lowMe, ND / NH); #endif -#if ZEROHACK_H - u32 zerohack = (u32) get_group_id(0) / 131072; - new_fft_HEIGHT2_1(lds + zerohack, u, smallTrig + zerohack, w); -#else - new_fft_HEIGHT2_1(lds, u, smallTrig, w); -#endif + u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; + fft_HEIGHT1(lds + zerohack, u, smallTrig + zerohack, w, 2, lowMe); // Compute trig values from scratch. Good on GPUs with high DP throughput. #if TAIL_TRIGS == 2 @@ -250,8 +286,8 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*5; // Read a hopefully cached line of data and one non-cached T2 per line - T2 trig = smallTrig[height_trigs + lowMe]; // Trig values for line zero, should be cached - T2 mult = smallTrig[height_trigs + G_H + line_u*2 + isSecondHalf]; // Two multipliers. One for line u, one for line v. + T2 trig = TFLOAD(&smallTrig[height_trigs + lowMe]); // Trig values for line zero, should be cached + T2 mult = TSLOAD(&smallTrig[height_trigs + G_H + line_u*2 + isSecondHalf]); // Two multipliers. One for line u, one for line v. trig = cmulFancy(trig, mult); // On consumer-grade GPUs, it is likely beneficial to read all trig values. @@ -260,11 +296,9 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*5; // Read pre-computed trig values - T2 trig = NTLOAD(smallTrig[height_trigs + line_u*G_H*2 + me]); + T2 trig = TOLOAD(&smallTrig[height_trigs + line_u*G_H*2 + me]); #endif - bar(G_H); - #if SINGLE_KERNEL // Line 0 and H/2 are special: they pair with themselves, line 0 is offseted by 1. if (line_u == 0) { @@ -276,15 +310,14 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { #else if (1) { #endif - revCrossLine(G_H, lds, u + NH/2, NH/2, isSecondHalf); + revCrossLine(lds, u); pairSq(NH/2, u, u + NH/2, trig, false); - bar(G_H); - revCrossLine(G_H, lds, u + NH/2, NH/2, !isSecondHalf); + revCrossLine(lds, u); } - bar(G_H); + dependentLaunch(); // Next kernel will be fftMiddleOutFP64 which must dependentLaunchWait before reading data - new_fft_HEIGHT2_2(lds, u, smallTrig, w); + fft_HEIGHT2(lds, u, smallTrig, w, 2, lowMe); // Write lines u and v writeTailFusedLine(u, out, transPos(line, MIDDLE, WIDTH), lowMe); @@ -339,12 +372,15 @@ void OVERLOAD pairSq(u32 N, F2 *u, F2 *v, F2 base_squared, bool special) { } } +#if !SINGLE_KERNEL // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { - local F2 lds[SMALL_HEIGHT / 2]; + local F2 lds[LDS_BYTES(1) / sizeof(F2)]; + LDSinit(lds, 1); + F2 u[NH]; - u32 H = ND / SMALL_HEIGHT; + const u32 H = ND / SMALL_HEIGHT; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -356,24 +392,31 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { u32 line = which ? (H/2) : 0; u32 me = get_local_id(0); + + dependentLaunch(); // Next kernel will be tailSquareFP32 which must dependentLaunchWait before reading data from fftMiddleInFP32 + dependentLaunchWait(); // Previous kernel was fftMiddleInFP32 that launched dependents before writing FP32 data + readTailFusedLine(inF2, u, line, me); F2 trig = slowTrig_N(line + me * H, ND / NH); - fft_HEIGHT(lds, u, smallTrigF2); - reverse(G_H, lds, u + NH/2, !which); + fft_HEIGHT1(lds, u, smallTrigF2, 1, me); + reverse(lds, u + NH/2, !which); pairSq(NH/2, u, u + NH/2, trig, !which); - reverse(G_H, lds, u + NH/2, !which); + reverse(lds, u + NH/2, !which); - bar(); - fft_HEIGHT(lds, u, smallTrigF2); + fft_HEIGHT1(lds, u, smallTrigF2, 1, me); writeTailFusedLine(u, outF2, transPos(line, MIDDLE, WIDTH), me); } +#endif #if SINGLE_WIDE -KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { - local F2 lds[SMALL_HEIGHT]; +KERNEL_CAP(G_H) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { + local F2 lds[LDS_BYTES(1) / sizeof(F2)]; + LDSinit(lds, 1); + + const u32 H = ND / SMALL_HEIGHT; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -381,82 +424,71 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { F2 u[NH], v[NH]; - u32 H = ND / SMALL_HEIGHT; - -#if SINGLE_KERNEL - u32 line1 = get_group_id(0); + u32 line1 = get_line_number(base); u32 line2 = line1 ? H - line1 : (H / 2); -#else - u32 line1 = get_group_id(0) + 1; - u32 line2 = H - line1; -#endif u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); u32 me = get_local_id(0); + + dependentLaunchWait(); // Previous kernel was fftMiddleInFP32 that launched dependents before writing FP32 data + readTailFusedLine(inF2, u, line1, me); readTailFusedLine(inF2, v, line2, me); -#if ZEROHACK_H - u32 zerohack = get_group_id(0) / 131072; - fft_HEIGHT(lds + zerohack, u, smallTrigF2 + zerohack); - bar(); - fft_HEIGHT(lds + zerohack, v, smallTrigF2 + zerohack); -#else - fft_HEIGHT(lds, u, smallTrigF2); - bar(); - fft_HEIGHT(lds, v, smallTrigF2); -#endif + u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; + fft_HEIGHT1(lds + zerohack, u, smallTrigF2 + zerohack, 1, me); + fft_HEIGHT1(lds + zerohack, v, smallTrigF2 + zerohack, 1, me); - // Compute trig values from scratch. Good on GPUs with high DP throughput. + // Compute trig values from scratch. Good on GPUs with high FP throughput. #if TAIL_TRIGS32 == 2 F2 trig = slowTrig_N(line1 + me * H, ND / NH); - // Do a little bit of memory access and a little bit of DP math. Good on a Radeon VII. + // Do a little bit of memory access and a little bit of FP math. #elif TAIL_TRIGS32 == 1 // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. - u32 height_trigs = SMALL_HEIGHT*1; + u32 height_trigs = SMALL_HEIGHT*5; // Read a hopefully cached line of data and one non-cached F2 per line - F2 trig = smallTrigF2[height_trigs + me]; // Trig values for line zero, should be cached - F2 mult = smallTrigF2[height_trigs + G_H + line1]; // Line multiplier + F2 trig = TFLOAD(&smallTrigF2[height_trigs + me]); // Trig values for line zero, should be cached + F2 mult = TSLOAD(&smallTrigF2[height_trigs + G_H + line1]); // Line multiplier trig = cmulFancy(trig, mult); // On consumer-grade GPUs, it is likely beneficial to read all trig values. #else // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. - u32 height_trigs = SMALL_HEIGHT*1; + u32 height_trigs = SMALL_HEIGHT*5; // Read pre-computed trig values - F2 trig = NTLOAD(smallTrigF2[height_trigs + line1*G_H + me]); + F2 trig = TOLOAD(&smallTrigF2[height_trigs + line1*G_H + me]); #endif #if SINGLE_KERNEL if (line1 == 0) { // Line 0 is special: it pairs with itself, offseted by 1. - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); pairSq(NH/2, u, u + NH/2, trig, true); - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); // Line H/2 also pairs with itself (but without offset). F2 trig2 = cmulFancy(trig, TAILT); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); pairSq(NH/2, v, v + NH/2, trig2, false); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); } else { #else if (1) { #endif - reverseLine(G_H, lds, v); + reverseLine(lds, v); pairSq(NH, u, v, trig, false); - reverseLine(G_H, lds, v); + reverseLine(lds, v); } - bar(); - fft_HEIGHT(lds, v, smallTrigF2); - bar(); - fft_HEIGHT(lds, u, smallTrigF2); + dependentLaunch(); // Next kernel will be fftMiddleOutFP32 which must dependentLaunchWait before reading data + + fft_HEIGHT2(lds, v, smallTrigF2, 1, me); + fft_HEIGHT2(lds, u, smallTrigF2, 1, me); writeTailFusedLine(v, outF2, memline2, me); writeTailFusedLine(u, outF2, memline1, me); @@ -485,8 +517,11 @@ void OVERLOAD pairSq2_special(F2 *u, F2 base_squared) { } } -KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { - local F2 lds[SMALL_HEIGHT]; +KERNEL_CAP(G_H * 2) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { + local F2 lds[LDS_BYTES(2) / sizeof(F2)]; + LDSinit(lds, 2); + + const u32 H = ND / SMALL_HEIGHT; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -494,16 +529,8 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { F2 u[NH]; - u32 H = ND / SMALL_HEIGHT; - -#if SINGLE_KERNEL - u32 line_u = get_group_id(0); + u32 line_u = get_line_number(base); u32 line_v = line_u ? H - line_u : (H / 2); -#else - u32 line_u = get_group_id(0) + 1; - u32 line_v = H - line_u; -#endif - u32 me = get_local_id(0); u32 lowMe = me % G_H; // lane-id in one of the two halves (half-workgroups). @@ -512,41 +539,37 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { u32 line = !isSecondHalf ? line_u : line_v; + dependentLaunchWait(); // Previous kernel was fftMiddleInFP32 that launched dependents before writing FP32 data + // Read lines u and v readTailFusedLine(inF2, u, line, lowMe); -#if ZEROHACK_H - u32 zerohack = (u32) get_group_id(0) / 131072; - new_fft_HEIGHT2_1(lds + zerohack, u, smallTrigF2 + zerohack); -#else - new_fft_HEIGHT2_1(lds, u, smallTrigF2); -#endif + u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; + fft_HEIGHT1(lds + zerohack, u, smallTrigF2 + zerohack, 2, lowMe); - // Compute trig values from scratch. Good on GPUs with high DP throughput. + // Compute trig values from scratch. Good on GPUs with high FP throughput. #if TAIL_TRIGS32 == 2 F2 trig = slowTrig_N(line + H * lowMe, ND / NH * 2); - // Do a little bit of memory access and a little bit of DP math. Good on a Radeon VII. + // Do a little bit of memory access and a little bit of FP math. #elif TAIL_TRIGS32 == 1 // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. - u32 height_trigs = SMALL_HEIGHT*1; + u32 height_trigs = SMALL_HEIGHT*5; // Read a hopefully cached line of data and one non-cached F2 per line - F2 trig = smallTrigF2[height_trigs + lowMe]; // Trig values for line zero, should be cached - F2 mult = smallTrigF2[height_trigs + G_H + line_u*2 + isSecondHalf]; // Two multipliers. One for line u, one for line v. + F2 trig = TFLOAD(&smallTrigF2[height_trigs + lowMe]); // Trig values for line zero, should be cached + F2 mult = TSLOAD(&smallTrigF2[height_trigs + G_H + line_u*2 + isSecondHalf]); // Two multipliers. One for line u, one for line v. trig = cmulFancy(trig, mult); // On consumer-grade GPUs, it is likely beneficial to read all trig values. #else // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. - u32 height_trigs = SMALL_HEIGHT*1; + u32 height_trigs = SMALL_HEIGHT*5; // Read pre-computed trig values - F2 trig = NTLOAD(smallTrigF2[height_trigs + line_u*G_H*2 + me]); + F2 trig = TOLOAD(&smallTrigF2[height_trigs + line_u*G_H*2 + me]); #endif - bar(G_H); - #if SINGLE_KERNEL // Line 0 and H/2 are special: they pair with themselves, line 0 is offseted by 1. if (line_u == 0) { @@ -558,15 +581,14 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { #else if (1) { #endif - revCrossLine(G_H, lds, u + NH/2, NH/2, isSecondHalf); + revCrossLine(lds, u); pairSq(NH/2, u, u + NH/2, trig, false); - bar(G_H); - revCrossLine(G_H, lds, u + NH/2, NH/2, !isSecondHalf); + revCrossLine(lds, u); } - bar(G_H); + dependentLaunch(); // Next kernel will be fftMiddleOutFP32 which must dependentLaunchWait before reading data - new_fft_HEIGHT2_2(lds, u, smallTrigF2); + fft_HEIGHT2(lds, u, smallTrigF2, 2, lowMe); // Write lines u and v writeTailFusedLine(u, outF2, transPos(line, MIDDLE, WIDTH), lowMe); @@ -585,17 +607,18 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { void OVERLOAD onePairSq(GF31* pa, GF31* pb, GF31 t_squared, const u32 t_squared_type) { GF31 a = *pa, b = *pb; - GF31 c, d; + GF31 b2t2, c, d; X2conjb(a, b); - if (t_squared_type == 0) // mul t_squared by 1 - c = csq_sub(a, cmul(csq(b), t_squared)); // a^2 - (b^2 * t_squared) - if (t_squared_type == 1) // mul t_squared by i - c = csq_subi(a, cmul(csq(b), t_squared)); // a^2 - i*(b^2 * t_squared) - if (t_squared_type == 2) // mul t_squared by -1 - c = csq_add(a, cmul(csq(b), t_squared)); // a^2 - -1*(b^2 * t_squared) - if (t_squared_type == 3) // mul t_squared by -i - c = csq_addi(a, cmul(csq(b), t_squared)); // a^2 - -i*(b^2 * t_squared) + b2t2 = cmul(csq(b), t_squared); // b2t2 = b^2 * t_squared + if (t_squared_type == 0) // mul t_squared by 1 + c = csq_sub(a, b2t2); // a^2 - (b^2 * t_squared) + if (t_squared_type == 1) // mul t_squared by i + c = csq_subi(a, b2t2); // a^2 - i*(b^2 * t_squared) + if (t_squared_type == 2) // mul t_squared by -1 + c = csq_add(a, b2t2); // a^2 - -1*(b^2 * t_squared) + if (t_squared_type == 3) // mul t_squared by -i + c = csq_addi(a, b2t2); // a^2 - -i*(b^2 * t_squared) d = mul2(cmul(a, b)); X2_conjb(c, d); *pa = SWAP_XY(c), *pb = SWAP_XY(d); @@ -624,17 +647,20 @@ void OVERLOAD pairSq(u32 N, GF31 *u, GF31 *v, GF31 base_squared, bool special) { } } +#if !SINGLE_KERNEL // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF31 lds[SMALL_HEIGHT / 2]; + local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; + LDSinit(lds, 1); + + const u32 H = ND / SMALL_HEIGHT; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); TrigGF31 smallTrig31 = (TrigGF31) (smallTrig + DISTHTRIGGF31); GF31 u[NH]; - u32 H = ND / SMALL_HEIGHT; // This kernel in executed in two workgroups. u32 which = get_group_id(0); @@ -642,40 +668,48 @@ KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { u32 line = which ? (H/2) : 0; u32 me = get_local_id(0); + + dependentLaunch(); // Next kernel will be tailSquareGF31 which must dependentLaunchWait before reading data from fftMiddleInGF31 + dependentLaunchWait(); // Previous kernel was fftMiddleInGF31 that launched dependents before writing GF31 data + readTailFusedLine(in31, u, line, me); // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; #if TAIL_TRIGS31 >= 1 - GF31 trig = smallTrig31[height_trigs + me]; + GF31 trig = TFLOAD(&smallTrig31[height_trigs + me]); #if SINGLE_WIDE - GF31 mult = smallTrig31[height_trigs + G_H + line]; + GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + line]); #else - GF31 mult = smallTrig31[height_trigs + G_H + which]; + GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + which]); #endif trig = cmul(trig, mult); #else #if SINGLE_WIDE - GF31 trig = NTLOAD(smallTrig31[height_trigs + line*G_H + me]); + GF31 trig = TOLOAD(&smallTrig31[height_trigs + line*G_H + me]); #else - GF31 trig = NTLOAD(smallTrig31[height_trigs + which*G_H + me]); + GF31 trig = TOLOAD(&smallTrig31[height_trigs + which*G_H + me]); #endif #endif - fft_HEIGHT(lds, u, smallTrig31); - reverse(G_H, lds, u + NH/2, !which); + fft_HEIGHT1(lds, u, smallTrig31, 1, me); + reverse(lds, u + NH/2, !which); pairSq(NH/2, u, u + NH/2, trig, !which); - reverse(G_H, lds, u + NH/2, !which); - bar(); - fft_HEIGHT(lds, u, smallTrig31); + reverse(lds, u + NH/2, !which); + + fft_HEIGHT2(lds, u, smallTrig31, 1, me); writeTailFusedLine(u, out31, transPos(line, MIDDLE, WIDTH), me); } +#endif #if SINGLE_WIDE -KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF31 lds[SMALL_HEIGHT]; +KERNEL_CAP(G_H) tailSquareGF31(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { + local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; + LDSinit(lds, 1); + + const u32 H = ND / SMALL_HEIGHT; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -683,32 +717,21 @@ KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { GF31 u[NH], v[NH]; - u32 H = ND / SMALL_HEIGHT; - -#if SINGLE_KERNEL - u32 line1 = get_group_id(0); + u32 line1 = get_line_number(base); u32 line2 = line1 ? H - line1 : (H / 2); -#else - u32 line1 = get_group_id(0) + 1; - u32 line2 = H - line1; -#endif u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); u32 me = get_local_id(0); + + dependentLaunchWait(); // Previous kernel was fftMiddleInGF31 that launched dependents before writing GF31 data + readTailFusedLine(in31, u, line1, me); readTailFusedLine(in31, v, line2, me); -#if ZEROHACK_H - u32 zerohack = (u32) get_group_id(0) / 131072; - fft_HEIGHT(lds + zerohack, u, smallTrig31 + zerohack); - bar(); - fft_HEIGHT(lds + zerohack, v, smallTrig31 + zerohack); -#else - fft_HEIGHT(lds, u, smallTrig31); - bar(); - fft_HEIGHT(lds, v, smallTrig31); -#endif + u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; + fft_HEIGHT1(lds + zerohack, u, smallTrig31 + zerohack, 1, me); + fft_HEIGHT1(lds + zerohack, v, smallTrig31 + zerohack, 1, me); // Do a little bit of memory access and a little bit of math. #if TAIL_TRIGS31 >= 1 @@ -716,8 +739,8 @@ KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read a hopefully cached line of data and one non-cached GF31 per line - GF31 trig = smallTrig31[height_trigs + me]; // Trig values for line zero, should be cached - GF31 mult = smallTrig31[height_trigs + G_H + line1]; // Line multiplier + GF31 trig = TFLOAD(&smallTrig31[height_trigs + me]); // Trig values for line zero, should be cached + GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + line1]); // Line multiplier trig = cmul(trig, mult); // On consumer-grade GPUs, it is likely beneficial to read all trig values. @@ -726,35 +749,35 @@ KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read pre-computed trig values - GF31 trig = NTLOAD(smallTrig31[height_trigs + line1*G_H + me]); + GF31 trig = TOLOAD(&smallTrig31[height_trigs + line1*G_H + me]); #endif #if SINGLE_KERNEL if (line1 == 0) { // Line 0 is special: it pairs with itself, offseted by 1. - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); pairSq(NH/2, u, u + NH/2, trig, true); - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); // Line H/2 also pairs with itself (but without offset). GF31 trig2 = cmul(trig, TAILTGF31); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); pairSq(NH/2, v, v + NH/2, trig2, false); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); } else { #else if (1) { #endif - reverseLine(G_H, lds, v); + reverseLine(lds, v); pairSq(NH, u, v, trig, false); - reverseLine(G_H, lds, v); + reverseLine(lds, v); } - bar(); - fft_HEIGHT(lds, v, smallTrig31); - bar(); - fft_HEIGHT(lds, u, smallTrig31); + dependentLaunch(); // Next kernel will be fftMiddleOutGF31 which must dependentLaunchWait before reading data + + fft_HEIGHT2(lds, v, smallTrig31, 1, me); + fft_HEIGHT2(lds, u, smallTrig31, 1, me); writeTailFusedLine(v, out31, memline2, me); writeTailFusedLine(u, out31, memline1, me); @@ -782,8 +805,11 @@ void OVERLOAD pairSq2_special(GF31 *u, GF31 base_squared) { } } -KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF31 lds[SMALL_HEIGHT]; +KERNEL_CAP(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { + local GF31 lds[LDS_BYTES(2) / sizeof(GF31)]; + LDSinit(lds, 2); + + const u32 H = ND / SMALL_HEIGHT; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -791,16 +817,8 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { GF31 u[NH]; - u32 H = ND / SMALL_HEIGHT; - -#if SINGLE_KERNEL - u32 line_u = get_group_id(0); + u32 line_u = get_line_number(base); u32 line_v = line_u ? H - line_u : (H / 2); -#else - u32 line_u = get_group_id(0) + 1; - u32 line_v = H - line_u; -#endif - u32 me = get_local_id(0); u32 lowMe = me % G_H; // lane-id in one of the two halves (half-workgroups). @@ -809,15 +827,13 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { u32 line = !isSecondHalf ? line_u : line_v; + dependentLaunchWait(); // Previous kernel was fftMiddleInGF31 that launched dependents before writing GF31 data + // Read lines u and v readTailFusedLine(in31, u, line, lowMe); -#if ZEROHACK_H - u32 zerohack = (u32) get_group_id(0) / 131072; - new_fft_HEIGHT2_1(lds + zerohack, u, smallTrig31 + zerohack); -#else - new_fft_HEIGHT2_1(lds, u, smallTrig31); -#endif + u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; + fft_HEIGHT1(lds + zerohack, u, smallTrig31 + zerohack, 2, lowMe); // Do a little bit of memory access and a little bit of math. Good on a Radeon VII. #if TAIL_TRIGS31 >= 1 @@ -825,8 +841,8 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read a hopefully cached line of data and one non-cached GF31 per line - GF31 trig = smallTrig31[height_trigs + lowMe]; // Trig values for line zero, should be cached - GF31 mult = smallTrig31[height_trigs + G_H + line_u*2 + isSecondHalf]; // Two multipliers. One for line u, one for line v. + GF31 trig = TFLOAD(&smallTrig31[height_trigs + lowMe]); // Trig values for line zero, should be cached + GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + line_u*2 + isSecondHalf]); // Two multipliers. One for line u, one for line v. trig = cmul(trig, mult); // On consumer-grade GPUs, it is likely beneficial to read all trig values. @@ -835,11 +851,9 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read pre-computed trig values - GF31 trig = NTLOAD(smallTrig31[height_trigs + line_u*G_H*2 + me]); + GF31 trig = TOLOAD(&smallTrig31[height_trigs + line_u*G_H*2 + me]); #endif - bar(G_H); - #if SINGLE_KERNEL // Line 0 and H/2 are special: they pair with themselves, line 0 is offseted by 1. if (line_u == 0) { @@ -851,15 +865,14 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #else if (1) { #endif - revCrossLine(G_H, lds, u + NH/2, NH/2, isSecondHalf); + revCrossLine(lds, u); pairSq(NH/2, u, u + NH/2, trig, false); - bar(G_H); - revCrossLine(G_H, lds, u + NH/2, NH/2, !isSecondHalf); + revCrossLine(lds, u); } - bar(G_H); + dependentLaunch(); // Next kernel will be fftMiddleOutGF31 which must dependentLaunchWait before reading data - new_fft_HEIGHT2_2(lds, u, smallTrig31); + fft_HEIGHT2(lds, u, smallTrig31, 2, lowMe); // Write lines u and v writeTailFusedLine(u, out31, transPos(line, MIDDLE, WIDTH), lowMe); @@ -868,7 +881,7 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #endif #endif - + /**************************************************************************/ /* Similar to above, but for an NTT based on GF(M61^2) */ @@ -878,19 +891,78 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { void OVERLOAD onePairSq(GF61* pa, GF61* pb, GF61 t_squared, const u32 t_squared_type) { GF61 a = *pa, b = *pb; - GF61 c, d; + GF61 a2, b2, b2t2, ab, addin, c, d; - X2conjb(a, b); - if (t_squared_type == 0) // mul t_squared by 1 - c = subq(csqq(a, 2), cmul(csq(b), t_squared), 2); // max c value is 4*M61+epsilon - if (t_squared_type == 1) // mul t_squared by i - c = subiq(csqq(a, 2), cmul(csq(b), t_squared), 2); // max c value is 4*M61+epsilon - if (t_squared_type == 2) // mul t_squared by -1 - c = addq(csqq(a, 2), cmul(csq(b), t_squared)); // max c value is 3*M61+epsilon - if (t_squared_type == 3) // mul t_squared by -i - c = addiq(csqq(a, 2), cmul(csq(b), t_squared), 2); // max c value is 4*M61+epsilon - d = 2 * cmul(a, b); // max d value is 2*M61+epsilon - X2s_conjb(&c, &d, 5, 3); +// This code should be faster (saves at least one wide mul) but the CUDA compiler makes poorer decisions regarding register usage resulting in local memory usage +#if ENABLE_BETTER_ONEPAIRSQ + X2qconjb(&a, &b); // X2(a, conjugate(b)). a.x range is 0..2+, a.y range is -1-..1+, b.x range is -1-..1+, b.y range is 0..2+ + a.y += 2*M61; // a range is 0..2+ / 1-..3+ + b.x += 2*M61; // b range is 1-..3+ / 0..2+ + + ab = addq(a, b); // Compute 2ab as (a + b)^2 - a^2 - b^2. ab range is 1-..5+ + a2 = csqq(a, 3, 4); // a2 = a^2, a2 range is 0..2+ + b2 = csq(b, 4, 3); // b2 = b^2, b2 range is 0..1+ + + addin = neg(addq(a2, b2), 4); // add this into the csq of a+b, addin range is 0..4 + d = csqa(ab, addin, 6); // d = 2ab, range is 0..1+ + + b2t2 = cmul(b2, t_squared); // b2t2 = b^2 * t_squared, b2t2 range is 0..1+ + + if (t_squared_type == 0) { // mul t_squared by 1 + c = subq(a2, b2t2); // c range is -1-..2+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c range is -1-..3+, d.x range is -1-..2+, d.y range is -2-..1+ + c = modM61q(c, 2); + d = modM61q(d, 3); + } + if (t_squared_type == 1) { // mul t_squared by i + c = subiq(a2, b2t2); // c.x range is 0..3+, c.y range is -1-..2+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is 0..4+, c.y range is -1..3+, d.x range is -1-..3+, d.y range is -2-..1+ + c = modM61q(c, 0, 2); + d = modM61q(d, 2, 3); + } + if (t_squared_type == 2) { // mul t_squared by -1 + c = addq(a2, b2t2); // c range is 0..3+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c range is 0..4+, d.x range is -1-..3+, d.y range is -3-..1+ + c = modM61q(c, 0); + d = modM61q(d, 2, 4); + } + if (t_squared_type == 3) { // mul t_squared by -i + c = addiq(a2, b2t2); // c.x range is -1-..2+, c.y range is 0..3+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is -1-..3+, c.y range is 0..4+, d.x range is -1-..2+, d.y range is -3-..1+ + c = modM61q(c, 2, 0); + d = modM61q(d, 2, 4); + } +#else + X2conjb(a, b); // X2(a, conjugate(b)) + a2 = csqq(a, 2); // a2 = a^2, a2.x range is 0..7+, a2.y range is 0..2+ + a2.x = modM61(a2.x); // a2.x range is 0..1+, a2.y range is 0..2+ + b2t2 = cmul(csq(b), t_squared); // b2t2 = b^2 * t_squared, b2t2 range is 0..1+ + d = cmul(a, b); d = d + d; // d = 2ab, d range is 0..2+ + if (t_squared_type == 0) { // mul t_squared by 1 + c = subq(a2, b2t2); // c.x range is -1..2+, c.y range is -1-..3+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is -1-..4+, c.y range is -1-..5+, d.x range is -3-..2+, d.y range is -3-..3+ + c = modM61q(c, 2); + d = modM61q(d, 4); + } + if (t_squared_type == 1) { // mul t_squared by i + c = subiq(a2, b2t2); // c.x range is 0..3+, c.y range is -1-..3+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is 0..5+, c.y range is -1..5+, d.x range is -2-..3+, d.y range is -3-..3+ + c = modM61q(c, 0, 2); + d = modM61q(d, 4); + } + if (t_squared_type == 2) { // mul t_squared by -1 + c = addq(a2, b2t2); // c.x range is 0..3+, c.y range is 0..4+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is 0..5+, c.y range is 0..6+, d.x range is -2-..3+, d.y range is -4-..2+ + c = modM61q(c, 0); + d = modM61q(d, 3, 5); + } + if (t_squared_type == 3) { // mul t_squared by -i + c = addiq(a2, b2t2); // c.x range is -1-..2+, c.y range is 0..4+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is -1-..4+, c.y range is 0..6+, d.x range is -3-..2+, d.y range is -4-..2+ + c = modM61q(c, 2, 0); + d = modM61q(d, 4, 5); + } +#endif *pa = SWAP_XY(c), *pb = SWAP_XY(d); } @@ -917,17 +989,20 @@ void OVERLOAD pairSq(u32 N, GF61 *u, GF61 *v, GF61 base_squared, bool special) { } } +#if !SINGLE_KERNEL // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF61 lds[SMALL_HEIGHT / 2]; + local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; + LDSinit(lds, 1); + + const u32 H = ND / SMALL_HEIGHT; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); TrigGF61 smallTrig61 = (TrigGF61) (smallTrig + DISTHTRIGGF61); GF61 u[NH]; - u32 H = ND / SMALL_HEIGHT; // This kernel in executed in two workgroups. u32 which = get_group_id(0); @@ -935,40 +1010,48 @@ KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { u32 line = which ? (H/2) : 0; u32 me = get_local_id(0); + + dependentLaunch(); // Next kernel will be tailSquareGF61 which must dependentLaunchWait before reading data from fftMiddleInGF61 + dependentLaunchWait(); // Previous kernel was fftMiddleInGF61 that launched dependents before writing GF61 data + readTailFusedLine(in61, u, line, me); // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; #if TAIL_TRIGS61 >= 1 - GF61 trig = smallTrig61[height_trigs + me]; + GF61 trig = TFLOAD(&smallTrig61[height_trigs + me]); #if SINGLE_WIDE - GF61 mult = smallTrig61[height_trigs + G_H + line]; + GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + line]); #else - GF61 mult = smallTrig61[height_trigs + G_H + which]; + GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + which]); #endif trig = cmul(trig, mult); #else #if SINGLE_WIDE - GF61 trig = NTLOAD(smallTrig61[height_trigs + line*G_H + me]); + GF61 trig = TOLOAD(&smallTrig61[height_trigs + line*G_H + me]); #else - GF61 trig = NTLOAD(smallTrig61[height_trigs + which*G_H + me]); + GF61 trig = TOLOAD(&smallTrig61[height_trigs + which*G_H + me]); #endif #endif - fft_HEIGHT(lds, u, smallTrig61); - reverse(G_H, lds, u + NH/2, !which); + fft_HEIGHT1(lds, u, smallTrig61, 1, me); + reverse(lds, u + NH/2, !which); pairSq(NH/2, u, u + NH/2, trig, !which); - reverse(G_H, lds, u + NH/2, !which); - bar(); - fft_HEIGHT(lds, u, smallTrig61); + reverse(lds, u + NH/2, !which); + + fft_HEIGHT2(lds, u, smallTrig61, 1, me); writeTailFusedLine(u, out61, transPos(line, MIDDLE, WIDTH), me); } +#endif #if SINGLE_WIDE -KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF61 lds[SMALL_HEIGHT]; +KERNEL_CAP(G_H) tailSquareGF61(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { + local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; + LDSinit(lds, 1); + + const u32 H = ND / SMALL_HEIGHT; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); @@ -976,32 +1059,21 @@ KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { GF61 u[NH], v[NH]; - u32 H = ND / SMALL_HEIGHT; - -#if SINGLE_KERNEL - u32 line1 = get_group_id(0); + u32 line1 = get_line_number(base); u32 line2 = line1 ? H - line1 : (H / 2); -#else - u32 line1 = get_group_id(0) + 1; - u32 line2 = H - line1; -#endif u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); u32 me = get_local_id(0); + + dependentLaunchWait(); // Previous kernel was fftMiddleInGF61 that launched dependents before writing GF61 data + readTailFusedLine(in61, u, line1, me); readTailFusedLine(in61, v, line2, me); -#if ZEROHACK_H - u32 zerohack = (u32) get_group_id(0) / 131072; - fft_HEIGHT(lds + zerohack, u, smallTrig61 + zerohack); - bar(); - fft_HEIGHT(lds + zerohack, v, smallTrig61 + zerohack); -#else - fft_HEIGHT(lds, u, smallTrig61); - bar(); - fft_HEIGHT(lds, v, smallTrig61); -#endif + u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; + fft_HEIGHT1(lds + zerohack, u, smallTrig61 + zerohack, 1, me); + fft_HEIGHT1(lds + zerohack, v, smallTrig61 + zerohack, 1, me); // Do a little bit of memory access and a little bit of math. #if TAIL_TRIGS61 >= 1 @@ -1009,8 +1081,8 @@ KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read a hopefully cached line of data and one non-cached GF61 per line - GF61 trig = smallTrig61[height_trigs + me]; // Trig values for line zero, should be cached - GF61 mult = smallTrig61[height_trigs + G_H + line1]; // Line multiplier + GF61 trig = TFLOAD(&smallTrig61[height_trigs + me]); // Trig values for line zero, should be cached + GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + line1]); // Line multiplier trig = cmul(trig, mult); // On consumer-grade GPUs, it is likely beneficial to read all trig values. @@ -1019,35 +1091,35 @@ KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read pre-computed trig values - GF61 trig = NTLOAD(smallTrig61[height_trigs + line1*G_H + me]); + GF61 trig = TOLOAD(&smallTrig61[height_trigs + line1*G_H + me]); #endif #if SINGLE_KERNEL if (line1 == 0) { // Line 0 is special: it pairs with itself, offseted by 1. - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); pairSq(NH/2, u, u + NH/2, trig, true); - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); // Line H/2 also pairs with itself (but without offset). GF61 trig2 = cmul(trig, TAILTGF61); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); pairSq(NH/2, v, v + NH/2, trig2, false); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); } else { #else if (1) { #endif - reverseLine(G_H, lds, v); + reverseLine(lds, v); pairSq(NH, u, v, trig, false); - reverseLine(G_H, lds, v); + reverseLine(lds, v); } - bar(); - fft_HEIGHT(lds, v, smallTrig61); - bar(); - fft_HEIGHT(lds, u, smallTrig61); + dependentLaunch(); // Next kernel will be fftMiddleOutGF61 which must dependentLaunchWait before reading data + + fft_HEIGHT2(lds, v, smallTrig61, 1, me); + fft_HEIGHT2(lds, u, smallTrig61, 1, me); writeTailFusedLine(v, out61, memline2, me); writeTailFusedLine(u, out61, memline1, me); @@ -1075,8 +1147,11 @@ void OVERLOAD pairSq2_special(GF61 *u, GF61 base_squared) { } } -KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF61 lds[SMALL_HEIGHT]; +KERNEL_CAP(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { + local GF61 lds[LDS_BYTES(2) / sizeof(GF61)]; + LDSinit(lds, 2); + + const u32 H = ND / SMALL_HEIGHT; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); @@ -1084,16 +1159,8 @@ KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { GF61 u[NH]; - u32 H = ND / SMALL_HEIGHT; - -#if SINGLE_KERNEL - u32 line_u = get_group_id(0); + u32 line_u = get_line_number(base); u32 line_v = line_u ? H - line_u : (H / 2); -#else - u32 line_u = get_group_id(0) + 1; - u32 line_v = H - line_u; -#endif - u32 me = get_local_id(0); u32 lowMe = me % G_H; // lane-id in one of the two halves (half-workgroups). @@ -1102,15 +1169,13 @@ KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { u32 line = !isSecondHalf ? line_u : line_v; + dependentLaunchWait(); // Previous kernel was fftMiddleInGF61 that launched dependents before writing GF61 data + // Read lines u and v readTailFusedLine(in61, u, line, lowMe); -#if ZEROHACK_H - u32 zerohack = (u32) get_group_id(0) / 131072; - new_fft_HEIGHT2_1(lds + zerohack, u, smallTrig61 + zerohack); -#else - new_fft_HEIGHT2_1(lds, u, smallTrig61); -#endif + u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; + fft_HEIGHT1(lds + zerohack, u, smallTrig61 + zerohack, 2, lowMe); // Do a little bit of memory access and a little bit of math. Good on a Radeon VII. #if TAIL_TRIGS61 >= 1 @@ -1118,8 +1183,8 @@ KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read a hopefully cached line of data and one non-cached GF61 per line - GF61 trig = smallTrig61[height_trigs + lowMe]; // Trig values for line zero, should be cached - GF61 mult = smallTrig61[height_trigs + G_H + line_u*2 + isSecondHalf]; // Two multipliers. One for line u, one for line v. + GF61 trig = TFLOAD(&smallTrig61[height_trigs + lowMe]); // Trig values for line zero, should be cached + GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + line_u*2 + isSecondHalf]); // Two multipliers. One for line u, one for line v. trig = cmul(trig, mult); // On consumer-grade GPUs, it is likely beneficial to read all trig values. @@ -1128,11 +1193,9 @@ KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read pre-computed trig values - GF61 trig = NTLOAD(smallTrig61[height_trigs + line_u*G_H*2 + me]); + GF61 trig = TOLOAD(&smallTrig61[height_trigs + line_u*G_H*2 + me]); #endif - bar(G_H); - #if SINGLE_KERNEL // Line 0 and H/2 are special: they pair with themselves, line 0 is offseted by 1. if (line_u == 0) { @@ -1144,15 +1207,14 @@ KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { #else if (1) { #endif - revCrossLine(G_H, lds, u + NH/2, NH/2, isSecondHalf); + revCrossLine(lds, u); pairSq(NH/2, u, u + NH/2, trig, false); - bar(G_H); - revCrossLine(G_H, lds, u + NH/2, NH/2, !isSecondHalf); + revCrossLine(lds, u); } - bar(G_H); + dependentLaunch(); // Next kernel will be fftMiddleOutGF61 which must dependentLaunchWait before reading data - new_fft_HEIGHT2_2(lds, u, smallTrig61); + fft_HEIGHT2(lds, u, smallTrig61, 2, lowMe); // Write lines u and v writeTailFusedLine(u, out61, transPos(line, MIDDLE, WIDTH), lowMe); diff --git a/src/cl/tailutil.cl b/src/cl/tailutil.cl index 01710cf1..5eb3fe4d 100644 --- a/src/cl/tailutil.cl +++ b/src/cl/tailutil.cl @@ -1,7 +1,5 @@ // Copyright (C) Mihai Preda -#include "math.cl" - // TAIL_TRIGS setting: // 2 = No memory accesses, trig values computed from scratch. Good for excellent DP GPUs such as Titan V or Radeon VII Pro. // 1 = Limited memory accesses and some DP computation. Tuned for Radeon VII a GPU with good DP performance. @@ -25,666 +23,458 @@ // 2 = double wide, single kernel // 3 = double wide, two kernels #if !defined(TAIL_KERNELS) -#define TAIL_KERNELS 2 // Default is double-wide tailSquare with two kernels +#define TAIL_KERNELS 2 // Default is double-wide tailSquare with a single kernel #endif -#define SINGLE_WIDE TAIL_KERNELS < 2 // Old single-wide tailSquare vs. new double-wide tailSquare -#define SINGLE_KERNEL (TAIL_KERNELS & 1) == 0 // TailSquare uses a single kernel vs. two kernels +#define SINGLE_WIDE (TAIL_KERNELS < 2) // Old single-wide tailSquare vs. new double-wide tailSquare +#define SINGLE_KERNEL ((TAIL_KERNELS & 1) == 0) // TailSquare uses a single kernel vs. two kernels + +// 64-bit implementations of reverse routines -#if FFT_FP64 +#if FFT_FP64 || NTT_GF61 -void OVERLOAD reverse(u32 WG, local T2 *lds, T2 *u, bool bump) { +void OVERLOAD reverse(local T2_GF61 *lds2, T2_GF61 *u, bool bump) { u32 me = get_local_id(0); u32 revMe = WG - 1 - me + bump; - bar(); - + if (SHUFL_BYTES_H >= 8) { + local T2_GF61 *lds = lds2; + bar(WG); #if NH == 8 - lds[revMe + 0 * WG] = u[3]; - lds[revMe + 1 * WG] = u[2]; - lds[revMe + 2 * WG] = u[1]; - lds[bump ? ((revMe + 3 * WG) % (4 * WG)) : (revMe + 3 * WG)] = u[0]; + lds[revMe + 0 * WG] = u[3]; + lds[revMe + 1 * WG] = u[2]; + lds[revMe + 2 * WG] = u[1]; + lds[bump ? ((revMe + 3 * WG) % (4 * WG)) : (revMe + 3 * WG)] = u[0]; #elif NH == 4 - lds[revMe + 0 * WG] = u[1]; - lds[bump ? ((revMe + WG) % (2 * WG)) : (revMe + WG)] = u[0]; -#else -#error + lds[revMe + 0 * WG] = u[1]; + lds[bump ? ((revMe + WG) % (2 * WG)) : (revMe + WG)] = u[0]; #endif - - bar(); - for (i32 i = 0; i < NH/2; ++i) { u[i] = lds[i * WG + me]; } -} - -void OVERLOAD reverseLine(u32 WG, local T2 *lds2, T2 *u) { - u32 me = get_local_id(0); - u32 revMe = WG - 1 - me; - - local T2 *lds = lds2 + revMe; - bar(); - for (u32 i = 0; i < NH; ++i) { lds[WG * (NH - 1 - i)] = u[i]; } - - lds = lds2 + me; - bar(); - for (u32 i = 0; i < NH; ++i) { u[i] = lds[WG * i]; } -} - -// This is used to reverse the second part of a line, and cross the reversed parts between the halves. -void OVERLOAD revCrossLine(u32 WG, local T2* lds2, T2 *u, u32 n, bool writeSecondHalf) { - u32 me = get_local_id(0); - u32 lowMe = me % WG; - - u32 revLowMe = WG - 1 - lowMe; - - for (u32 i = 0; i < n; ++i) { lds2[WG * n * writeSecondHalf + WG * (n - 1 - i) + revLowMe] = u[i]; } - - bar(); // we need a full bar because we're crossing halves - - for (u32 i = 0; i < n; ++i) { u[i] = lds2[WG * n * !writeSecondHalf + WG * i + lowMe]; } -} - -// -// These versions are for the kernel(s) that uses a double-wide workgroup (u in half the workgroup, v in the other half) -// - -void OVERLOAD reverse2(local T2 *lds, T2 *u) { - u32 me = get_local_id(0); - - // For NH=8, u[0] to u[3] are left unchanged. Write to lds: - // u[7]rev u[6]rev - // u[5]rev u[4]rev - // v[7]rev v[6]rev - // v[5]rev v[4]rev - bar(); - for (u32 i = 0; i < NH / 2; ++i) { - u32 j = (i * G_H + me % G_H); - lds[me < G_H ? ((NH/2)*G_H - j) % ((NH/2)*G_H) : NH*G_H-1 - j] = u[NH/2 + i]; + bar(WG); + for (i32 i = 0; i < NH/2; ++i) { u[i] = lds[i * WG + me]; } } - // For NH=8, read from lds into u[i]: - // u[4] = u[7]rev v[7]rev - // u[5] = u[6]rev v[6]rev - // u[6] = u[5]rev v[5]rev - // u[7] = u[4]rev v[4]rev - bar(); - lds += me % G_H + (me / G_H) * NH/2 * G_H; - for (u32 i = 0; i < NH / 2; ++i) { u[NH/2 + i] = lds[i * G_H]; } -} - -// Somewhat similar to reverseLine. -// The u values are in threads < G_H, the v values to reverse in threads >= G_H. -// Whereas reverseLine leaves u values alone. This reverseLine moves u values around -// so that pairSq2 can easily operate on pairs. This means for NH = 4, web output: -// u[0] u[1] // Returned in u[0] -// u[2] u[3] // Returned in u[1] -// v[3]rev v[2]rev // Returned in u[2] -// v[1]rev v[0]rev // Returned in u[3] -void OVERLOAD reverseLine2(local T2 *lds, T2 *u) { - u32 me = get_local_id(0); - -// NOTE: It is important that this routine use lds memory in coordination with shufl2. Failure to do so would require an -// unqualified bar() call here. Specifically, the u values are stored in the upper half of lds memory (SMALL_HEIGHT T2 values). -// The v values are stored in the lower half of lds memory (the next SMALL_HEIGHT T2 values). - - if (G_H > WAVEFRONT) bar(); - -// For NH=4, the lds indices (where to write each incoming u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H -// That means saving to lds using index: me < G_H ? me % G_H + i * G_H : 8*G_H-1 - me % G_H - i * G_H - -#if 1 - local T2 *ldsOut = lds + (me < G_H ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsOutInc = (me < G_H) ? G_H : -G_H; - for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { *ldsOut = u[i]; } - - lds += me; - bar(); - for (u32 i = 0; i < NH; ++i) { u[i] = lds[i * 2*G_H]; } -#else - local T *ldsOut = (local T *) lds + (me < G_H ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsOutInc = (me < G_H) ? G_H : -G_H; - for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { ldsOut[0] = u[i].x; ldsOut[NH*2*G_H] = u[i].y; } - - local T *ldsIn = (local T *) lds + me; - bar(); - for (u32 i = 0; i < NH; ++i) { u[i].x = ldsIn[i * 2*G_H]; u[i].y = ldsIn[NH*2*G_H + i * 2*G_H]; } -#endif -} - -// Undo a reverseLine2 -void OVERLOAD unreverseLine2(local T2 *lds, T2 *u) { - u32 me = get_local_id(0); - -// NOTE: It is important that this routine use lds memory in coordination with reverseLine2 and shufl2. By initially -// writing to the lds locations that reverseLine2 read from we do not need an initial bar() call here. Also, by reading -// from the lds locations that shufl2 will use (u values in the upper half of lds memory, v values in the lower half of -// lds memory) we can issue a qualified bar() call before calling FFT_HEIGHT2. - -#if 1 - local T2 *ldsOut = lds + me; - for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*G_H] = u[i]; } - -// For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H - lds += (me < G_H) ? me % G_H : (NH*2)*G_H-1 - me % G_H; - i32 ldsInc = (me < G_H) ? G_H : -G_H; - bar(); - for (u32 i = 0; i < NH; ++i, lds += ldsInc) { u[i] = *lds; } -#else - local T *ldsOut = (local T *) lds + me; - for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*G_H] = u[i].x; ldsOut[NH*2*G_H + i * 2*G_H] = u[i].y; } - -// For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H - local T *ldsIn = (local T *) lds + ((me < G_H) ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsInc = (me < G_H) ? G_H : -G_H; - bar(); - for (u32 i = 0; i < NH; ++i, ldsIn += ldsInc) { u[i].x = ldsIn[0]; u[i].y = ldsIn[NH*2*G_H]; } -#endif -} + else if (SHUFL_BYTES_H == 4) { + local T_Z61 *lds = (local T_Z61 *) lds2; + bar(WG); +#if NH == 8 + lds[revMe + 0 * WG] = u[3].x; + lds[revMe + 1 * WG] = u[2].x; + lds[revMe + 2 * WG] = u[1].x; + lds[bump ? ((revMe + 3 * WG) % (4 * WG)) : (revMe + 3 * WG)] = u[0].x; +#elif NH == 4 + lds[revMe + 0 * WG] = u[1].x; + lds[bump ? ((revMe + WG) % (2 * WG)) : (revMe + WG)] = u[0].x; #endif - - -/**************************************************************************/ -/* Similar to above, but for an FFT based on FP32 */ -/**************************************************************************/ - -#if FFT_FP32 - -void OVERLOAD reverse(u32 WG, local F2 *lds, F2 *u, bool bump) { - u32 me = get_local_id(0); - u32 revMe = WG - 1 - me + bump; - - bar(); - + bar(WG); + for (i32 i = 0; i < NH/2; ++i) { u[i].x = lds[i * WG + me]; } + bar(WG); #if NH == 8 - lds[revMe + 0 * WG] = u[3]; - lds[revMe + 1 * WG] = u[2]; - lds[revMe + 2 * WG] = u[1]; - lds[bump ? ((revMe + 3 * WG) % (4 * WG)) : (revMe + 3 * WG)] = u[0]; + lds[revMe + 0 * WG] = u[3].y; + lds[revMe + 1 * WG] = u[2].y; + lds[revMe + 2 * WG] = u[1].y; + lds[bump ? ((revMe + 3 * WG) % (4 * WG)) : (revMe + 3 * WG)] = u[0].y; #elif NH == 4 - lds[revMe + 0 * WG] = u[1]; - lds[bump ? ((revMe + WG) % (2 * WG)) : (revMe + WG)] = u[0]; -#else -#error + lds[revMe + 0 * WG] = u[1].y; + lds[bump ? ((revMe + WG) % (2 * WG)) : (revMe + WG)] = u[0].y; #endif - - bar(); - for (i32 i = 0; i < NH/2; ++i) { u[i] = lds[i * WG + me]; } + bar(WG); + for (i32 i = 0; i < NH/2; ++i) { u[i].y = lds[i * WG + me]; } + } } -void OVERLOAD reverseLine(u32 WG, local F2 *lds2, F2 *u) { +void OVERLOAD reverseLine(local T2_GF61 *lds, T2_GF61 *u) { u32 me = get_local_id(0); u32 revMe = WG - 1 - me; - local F2 *lds = lds2 + revMe; - bar(); - for (u32 i = 0; i < NH; ++i) { lds[WG * (NH - 1 - i)] = u[i]; } - - lds = lds2 + me; - bar(); - for (u32 i = 0; i < NH; ++i) { u[i] = lds[WG * i]; } -} - -// This is used to reverse the second part of a line, and cross the reversed parts between the halves. -void OVERLOAD revCrossLine(u32 WG, local F2* lds2, F2 *u, u32 n, bool writeSecondHalf) { - u32 me = get_local_id(0); - u32 lowMe = me % WG; - - u32 revLowMe = WG - 1 - lowMe; - - for (u32 i = 0; i < n; ++i) { lds2[WG * n * writeSecondHalf + WG * (n - 1 - i) + revLowMe] = u[i]; } + if (SHUFL_BYTES_H == 16) { + local T2_GF61 *ldsOut = lds + revMe; + local T2_GF61 *ldsIn = lds + me; + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = u[i]; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { u[i] = ldsIn[WG * i]; } + } - bar(); // we need a full bar because we're crossing halves + else if (SHUFL_BYTES_H == 8) { + local T_Z61 *ldsOut = (local T_Z61 *) lds + revMe; + local T_Z61 *ldsIn = (local T_Z61 *) lds + me; + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = u[i].x; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { u[i].x = ldsIn[WG * i]; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = u[i].y; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { u[i].y = ldsIn[WG * i]; } + } - for (u32 i = 0; i < n; ++i) { u[i] = lds2[WG * n * !writeSecondHalf + WG * i + lowMe]; } + else if (SHUFL_BYTES_H == 4) { + local int *ldsOut = (local int *) lds + revMe; + local int *ldsIn = (local int *) lds + me; + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = as_int4(u[i]).x; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { int4 tmp = as_int4(u[i]); tmp.x = ldsIn[WG * i]; u[i] = as_T2_GF61(tmp); } + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = as_int4(u[i]).y; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { int4 tmp = as_int4(u[i]); tmp.y = ldsIn[WG * i]; u[i] = as_T2_GF61(tmp); } + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = as_int4(u[i]).z; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { int4 tmp = as_int4(u[i]); tmp.z = ldsIn[WG * i]; u[i] = as_T2_GF61(tmp); } + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = as_int4(u[i]).w; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { int4 tmp = as_int4(u[i]); tmp.w = ldsIn[WG * i]; u[i] = as_T2_GF61(tmp); } + } } // -// These versions are for the kernel(s) that uses a double-wide workgroup (u in half the workgroup, v in the other half) +// These versions are for the kernel(s) that use a double-wide workgroup (u in half the workgroup, v in the other half) // -void OVERLOAD reverse2(local F2 *lds, F2 *u) { +void OVERLOAD reverse2(local T2_GF61 *lds2, T2_GF61 *u) { u32 me = get_local_id(0); + u32 lowMe = me % WG; - // For NH=8, u[0] to u[3] are left unchanged. Write to lds: - // u[7]rev u[6]rev - // u[5]rev u[4]rev - // v[7]rev v[6]rev - // v[5]rev v[4]rev - bar(); - for (u32 i = 0; i < NH / 2; ++i) { - u32 j = (i * G_H + me % G_H); - lds[me < G_H ? ((NH/2)*G_H - j) % ((NH/2)*G_H) : NH*G_H-1 - j] = u[NH/2 + i]; + if (SBMUL(2) * SHUFL_BYTES_H >= 8) { + local T2_GF61 *lds = LDSsharing_ptr(lds2, 2); + // For NH=8, u[0] to u[3] are left unchanged. Write to lds: + // u[7]rev u[6]rev u[5]rev u[4]rev + // v[7]rev v[6]rev v[5]rev v[4]rev + LDStx_start(lds2, 2); + for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * WG - (me >= WG ? 1 : 0) - lowMe) % (NH/2 * WG)] = u[NH/2 + i]; } + // For NH=8, read from lds into u[i]: + // u[4] = u[7]rev v[7]rev + // u[5] = u[6]rev v[6]rev + // u[6] = u[5]rev v[5]rev + // u[7] = u[4]rev v[4]rev + LDSbar(2); + for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i] = lds[i * WG + lowMe]; } + LDStx_end(lds2, 2); } - // For NH=8, read from lds into u[i]: - // u[4] = u[7]rev v[7]rev - // u[5] = u[6]rev v[6]rev - // u[6] = u[5]rev v[5]rev - // u[7] = u[4]rev v[4]rev - bar(); - lds += me % G_H + (me / G_H) * NH/2 * G_H; - for (u32 i = 0; i < NH / 2; ++i) { u[NH/2 + i] = lds[i * G_H]; } -} - -// Somewhat similar to reverseLine. -// The u values are in threads < G_H, the v values to reverse in threads >= G_H. -// Whereas reverseLine leaves u values alone. This reverseLine moves u values around -// so that pairSq2 can easily operate on pairs. This means for NH = 4, web output: -// u[0] u[1] // Returned in u[0] -// u[2] u[3] // Returned in u[1] -// v[3]rev v[2]rev // Returned in u[2] -// v[1]rev v[0]rev // Returned in u[3] -void OVERLOAD reverseLine2(local F2 *lds, F2 *u) { - u32 me = get_local_id(0); - -// NOTE: It is important that this routine use lds memory in coordination with shufl2. Failure to do so would require an -// unqualified bar() call here. Specifically, the u values are stored in the upper half of lds memory (SMALL_HEIGHT F2 values). -// The v values are stored in the lower half of lds memory (the next SMALL_HEIGHT F2 values). - - if (G_H > WAVEFRONT) bar(); - -// For NH=4, the lds indices (where to write each incoming u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H -// That means saving to lds using index: me < G_H ? me % G_H + i * G_H : 8*G_H-1 - me % G_H - i * G_H - -#if 1 - local F2 *ldsOut = lds + (me < G_H ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsOutInc = (me < G_H) ? G_H : -G_H; - for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { *ldsOut = u[i]; } - - lds += me; - bar(); - for (u32 i = 0; i < NH; ++i) { u[i] = lds[i * 2*G_H]; } -#else - local F *ldsOut = (local F *) lds + (me < G_H ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsOutInc = (me < G_H) ? G_H : -G_H; - for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { ldsOut[0] = u[i].x; ldsOut[NH*2*G_H] = u[i].y; } - - local F *ldsIn = (local F *) lds + me; - bar(); - for (u32 i = 0; i < NH; ++i) { u[i].x = ldsIn[i * 2*G_H]; u[i].y = ldsIn[NH*2*G_H + i * 2*G_H]; } -#endif -} - -// Undo a reverseLine2 -void OVERLOAD unreverseLine2(local F2 *lds, F2 *u) { - u32 me = get_local_id(0); - -// NOTE: It is important that this routine use lds memory in coordination with reverseLine2 and shufl2. By initially -// writing to the lds locations that reverseLine2 read from we do not need an initial bar() call here. Also, by reading -// from the lds locations that shufl2 will use (u values in the upper half of lds memory, v values in the lower half of -// lds memory) we can issue a qualified bar() call before calling FFT_HEIGHT2. - -#if 1 - local F2 *ldsOut = lds + me; - for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*G_H] = u[i]; } - -// For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H - lds += (me < G_H) ? me % G_H : (NH*2)*G_H-1 - me % G_H; - i32 ldsInc = (me < G_H) ? G_H : -G_H; - bar(); - for (u32 i = 0; i < NH; ++i, lds += ldsInc) { u[i] = *lds; } -#else - local F *ldsOut = (local F *) lds + me; - for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*G_H] = u[i].x; ldsOut[NH*2*G_H + i * 2*G_H] = u[i].y; } - -// For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H - local F *ldsIn = (local F *) lds + ((me < G_H) ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsInc = (me < G_H) ? G_H : -G_H; - bar(); - for (u32 i = 0; i < NH; ++i, ldsIn += ldsInc) { u[i].x = ldsIn[0]; u[i].y = ldsIn[NH*2*G_H]; } -#endif -} - -#endif - - -/**************************************************************************/ -/* Similar to above, but for an NTT based on GF(M31^2) */ -/**************************************************************************/ - -#if NTT_GF31 - -void OVERLOAD reverse(u32 WG, local GF31 *lds, GF31 *u, bool bump) { - u32 me = get_local_id(0); - u32 revMe = WG - 1 - me + bump; - - bar(); - -#if NH == 8 - lds[revMe + 0 * WG] = u[3]; - lds[revMe + 1 * WG] = u[2]; - lds[revMe + 2 * WG] = u[1]; - lds[bump ? ((revMe + 3 * WG) % (4 * WG)) : (revMe + 3 * WG)] = u[0]; -#elif NH == 4 - lds[revMe + 0 * WG] = u[1]; - lds[bump ? ((revMe + WG) % (2 * WG)) : (revMe + WG)] = u[0]; -#else -#error -#endif - - bar(); - for (i32 i = 0; i < NH/2; ++i) { u[i] = lds[i * WG + me]; } -} -void OVERLOAD reverseLine(u32 WG, local GF31 *lds2, GF31 *u) { - u32 me = get_local_id(0); - u32 revMe = WG - 1 - me; - - local GF31 *lds = lds2 + revMe; - bar(); - for (u32 i = 0; i < NH; ++i) { lds[WG * (NH - 1 - i)] = u[i]; } - - lds = lds2 + me; - bar(); - for (u32 i = 0; i < NH; ++i) { u[i] = lds[WG * i]; } + else if (SBMUL(2) * SHUFL_BYTES_H == 4) { + local T_Z61 *lds = LDSsharing_ptr((local T_Z61 *)lds2, 2); + LDStx_start(lds2, 2); + for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * WG - (me >= WG ? 1 : 0) - lowMe) % (NH/2 * WG)] = u[NH/2 + i].x; } + LDSbar(2); + for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i].x = lds[i * WG + lowMe]; } + LDSbar(2); + for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * WG - (me >= WG ? 1 : 0) - lowMe) % (NH/2 * WG)] = u[NH/2 + i].y; } + LDSbar(2); + for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i].y = lds[i * WG + lowMe]; } + LDStx_end(lds2, 2); + } } // This is used to reverse the second part of a line, and cross the reversed parts between the halves. -void OVERLOAD revCrossLine(u32 WG, local GF31* lds2, GF31 *u, u32 n, bool writeSecondHalf) { +void OVERLOAD revCrossLine(local T2_GF61 *lds2, T2_GF61 *u) { u32 me = get_local_id(0); u32 lowMe = me % WG; - u32 revLowMe = WG - 1 - lowMe; - for (u32 i = 0; i < n; ++i) { lds2[WG * n * writeSecondHalf + WG * (n - 1 - i) + revLowMe] = u[i]; } - - bar(); // we need a full bar because we're crossing halves - - for (u32 i = 0; i < n; ++i) { u[i] = lds2[WG * n * !writeSecondHalf + WG * i + lowMe]; } -} - -// -// These versions are for the kernel(s) that uses a double-wide workgroup (u in half the workgroup, v in the other half) -// + if (SHUFL_BYTES_H >= 8) { + local T2_GF61 *ldsOut = lds2; + local T2_GF61 *ldsIn = lds2; + if (me < WG) ldsOut += LDS_SHUFL_BYTES(2) / sizeof(T2_GF61); // Crossing LDS halves + else ldsIn += LDS_SHUFL_BYTES(2) / sizeof(T2_GF61); // Staying within LDS halves (just like shufl) + bar(); // we need a full bar because we're crossing halves + for (u32 i = 0; i < NH/2; ++i) { ldsOut[WG * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2]; } + bar(); // we need a full bar because we just crossed halves. LDS reads are compatible with future shufl calls. + for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2] = ldsIn[WG * i + lowMe]; } + // One last bar() is needed when sharing LDS memory. This is because when sharing a workgroup will write to more than its own LDS area. + if (SHARING_LDS(2)) bar(); + } -void OVERLOAD reverse2(local GF31 *lds, GF31 *u) { - u32 me = get_local_id(0); - - // For NH=8, u[0] to u[3] are left unchanged. Write to lds: - // u[7]rev u[6]rev - // u[5]rev u[4]rev - // v[7]rev v[6]rev - // v[5]rev v[4]rev - bar(); - for (u32 i = 0; i < NH / 2; ++i) { - u32 j = (i * G_H + me % G_H); - lds[me < G_H ? ((NH/2)*G_H - j) % ((NH/2)*G_H) : NH*G_H-1 - j] = u[NH/2 + i]; + else if (SHUFL_BYTES_H == 4) { + local T_Z61 *ldsOut = (local T_Z61 *) lds2; + local T_Z61 *ldsIn = (local T_Z61 *) lds2; + if (me < WG) ldsOut += LDS_SHUFL_BYTES(2) / sizeof(T_Z61); + else ldsIn += LDS_SHUFL_BYTES(2) / sizeof(T_Z61); + bar(); // we need a full bar because we're crossing halves + for (u32 i = 0; i < NH/2; ++i) { ldsOut[WG * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2].x; } + bar(); // we need a full bar because we just crossed halves + for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2].x = ldsIn[WG * i + lowMe]; } + bar(); // we need a full bar because we're crossing halves + for (u32 i = 0; i < NH/2; ++i) { ldsOut[WG * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2].y; } + bar(); // we need a full bar because we just crossed halves. LDS reads are compatible with future shufl calls. + for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2].y = ldsIn[WG * i + lowMe]; } + // One last bar() is needed when sharing LDS memory. This is because when sharing a workgroup will write to more than its own LDS area. + if (SHARING_LDS(2)) bar(); } - // For NH=8, read from lds into u[i]: - // u[4] = u[7]rev v[7]rev - // u[5] = u[6]rev v[6]rev - // u[6] = u[5]rev v[5]rev - // u[7] = u[4]rev v[4]rev - bar(); - lds += me % G_H + (me / G_H) * NH/2 * G_H; - for (u32 i = 0; i < NH / 2; ++i) { u[NH/2 + i] = lds[i * G_H]; } } +#if 0 // Unused + // Somewhat similar to reverseLine. -// The u values are in threads < G_H, the v values to reverse in threads >= G_H. +// The u values are in threads < WG, the v values to reverse in threads >= WG. // Whereas reverseLine leaves u values alone. This reverseLine moves u values around // so that pairSq2 can easily operate on pairs. This means for NH = 4, web output: // u[0] u[1] // Returned in u[0] // u[2] u[3] // Returned in u[1] // v[3]rev v[2]rev // Returned in u[2] // v[1]rev v[0]rev // Returned in u[3] -void OVERLOAD reverseLine2(local GF31 *lds, GF31 *u) { +void OVERLOAD reverseLine2(local T2_GF61 *lds, T2_GF61 *u) { u32 me = get_local_id(0); -// NOTE: It is important that this routine use lds memory in coordination with shufl2. Failure to do so would require an -// unqualified bar() call here. Specifically, the u values are stored in the upper half of lds memory (SMALL_HEIGHT GF31 values). -// The v values are stored in the lower half of lds memory (the next SMALL_HEIGHT GF31 values). +// NOTE: It is important that this routine use lds memory in coordination with shufl. Failure to do so would require an +// unqualified bar() call here. Specifically, the u values are stored in the upper half of lds memory (SMALL_HEIGHT T2 values). +// The v values are stored in the lower half of lds memory (the next SMALL_HEIGHT T2 values). - if (G_H > WAVEFRONT) bar(); + if (WG > WAVEFRONT) bar(); // For NH=4, the lds indices (where to write each incoming u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H -// That means saving to lds using index: me < G_H ? me % G_H + i * G_H : 8*G_H-1 - me % G_H - i * G_H +// 0..GH-1 +0*WG GH-1..0 +7*WG +// 0..GH-1 +1*WG GH-1..0 +6*WG +// 0..GH-1 +2*WG GH-1..0 +5*WG +// 0..GH-1 +3*WG GH-1..0 +4*WG +// That means saving to lds using index: me < WG ? me % WG + i * WG : 8*WG-1 - me % WG - i * WG #if 1 - local GF31 *ldsOut = lds + (me < G_H ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsOutInc = (me < G_H) ? G_H : -G_H; + local T2_GF61 *ldsOut = lds + (me < WG ? me % WG : (NH*2)*WG-1 - me % WG); + i32 ldsOutInc = (me < WG) ? WG : -WG; for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { *ldsOut = u[i]; } lds += me; bar(); - for (u32 i = 0; i < NH; ++i) { u[i] = lds[i * 2*G_H]; } + for (u32 i = 0; i < NH; ++i) { u[i] = lds[i * 2*WG]; } #else - local Z61 *ldsOut = (local Z61 *) lds + (me < G_H ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsOutInc = (me < G_H) ? G_H : -G_H; - for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { ldsOut[0] = u[i].x; ldsOut[NH*2*G_H] = u[i].y; } + local T_Z61 *ldsOut = (local T_Z61 *) lds + (me < WG ? me % WG : (NH*2)*WG-1 - me % WG); + i32 ldsOutInc = (me < WG) ? WG : -WG; + for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { ldsOut[0] = u[i].x; ldsOut[NH*2*WG] = u[i].y; } - local ZF61 *ldsIn = (local T *) lds + me; + local T_Z61 *ldsIn = (local T_Z61 *) lds + me; bar(); - for (u32 i = 0; i < NH; ++i) { u[i].x = ldsIn[i * 2*G_H]; u[i].y = ldsIn[NH*2*G_H + i * 2*G_H]; } + for (u32 i = 0; i < NH; ++i) { u[i].x = ldsIn[i * 2*WG]; u[i].y = ldsIn[NH*2*WG + i * 2*WG]; } #endif } // Undo a reverseLine2 -void OVERLOAD unreverseLine2(local GF31 *lds, GF31 *u) { +void OVERLOAD unreverseLine2(local T2_GF61 *lds, T2_GF61 *u) { u32 me = get_local_id(0); -// NOTE: It is important that this routine use lds memory in coordination with reverseLine2 and shufl2. By initially +// NOTE: It is important that this routine use lds memory in coordination with reverseLine2 and shufl. By initially // writing to the lds locations that reverseLine2 read from we do not need an initial bar() call here. Also, by reading -// from the lds locations that shufl2 will use (u values in the upper half of lds memory, v values in the lower half of +// from the lds locations that shufl will use (u values in the upper half of lds memory, v values in the lower half of // lds memory) we can issue a qualified bar() call before calling FFT_HEIGHT2. #if 1 - local GF31 *ldsOut = lds + me; - for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*G_H] = u[i]; } + local T2_GF61 *ldsOut = lds + me; + for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*WG] = u[i]; } // For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H - lds += (me < G_H) ? me % G_H : (NH*2)*G_H-1 - me % G_H; - i32 ldsInc = (me < G_H) ? G_H : -G_H; +// 0..GH-1 +0*WG GH-1..0 +7*WG +// 0..GH-1 +1*WG GH-1..0 +6*WG +// 0..GH-1 +2*WG GH-1..0 +5*WG +// 0..GH-1 +3*WG GH-1..0 +4*WG + lds += (me < WG) ? me % WG : (NH*2)*WG-1 - me % WG; + i32 ldsInc = (me < WG) ? WG : -WG; bar(); for (u32 i = 0; i < NH; ++i, lds += ldsInc) { u[i] = *lds; } #else - local Z61 *ldsOut = (local T *) lds + me; - for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*G_H] = u[i].x; ldsOut[NH*2*G_H + i * 2*G_H] = u[i].y; } + local T_Z61 *ldsOut = (local T_Z61 *) lds + me; + for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*WG] = u[i].x; ldsOut[NH*2*WG + i * 2*WG] = u[i].y; } // For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H - local Z61 *ldsIn = (local T *) lds + ((me < G_H) ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsInc = (me < G_H) ? G_H : -G_H; - bar(); - for (u32 i = 0; i < NH; ++i, ldsIn += ldsInc) { u[i].x = ldsIn[0]; u[i].y = ldsIn[NH*2*G_H]; } +// 0..GH-1 +0*WG GH-1..0 +7*WG +// 0..GH-1 +1*WG GH-1..0 +6*WG +// 0..GH-1 +2*WG GH-1..0 +5*WG +// 0..GH-1 +3*WG GH-1..0 +4*WG + local T_Z61 *ldsIn = (local T_Z61 *) lds + ((me < WG) ? me % WG : (NH*2)*WG-1 - me % WG); + i32 ldsInc = (me < WG) ? WG : -WG; + bar(); + for (u32 i = 0; i < NH; ++i, ldsIn += ldsInc) { u[i].x = ldsIn[0]; u[i].y = ldsIn[NH*2*WG]; } #endif } #endif +#endif + /**************************************************************************/ -/* Similar to above, but for an NTT based on GF(M61^2) */ +/* Similar to above, but for an FFT based on FP32 or GF31 */ /**************************************************************************/ -#if NTT_GF61 +#if FFT_FP32 || NTT_GF31 -void OVERLOAD reverse(u32 WG, local GF61 *lds, GF61 *u, bool bump) { +void OVERLOAD reverse(local F2_GF31 *lds, F2_GF31 *u, bool bump) { u32 me = get_local_id(0); u32 revMe = WG - 1 - me + bump; - bar(); - + if (SHUFL_BYTES_H >= 4) { + bar(WG); #if NH == 8 - lds[revMe + 0 * WG] = u[3]; - lds[revMe + 1 * WG] = u[2]; - lds[revMe + 2 * WG] = u[1]; - lds[bump ? ((revMe + 3 * WG) % (4 * WG)) : (revMe + 3 * WG)] = u[0]; + lds[revMe + 0 * WG] = u[3]; + lds[revMe + 1 * WG] = u[2]; + lds[revMe + 2 * WG] = u[1]; + lds[bump ? ((revMe + 3 * WG) % (4 * WG)) : (revMe + 3 * WG)] = u[0]; #elif NH == 4 - lds[revMe + 0 * WG] = u[1]; - lds[bump ? ((revMe + WG) % (2 * WG)) : (revMe + WG)] = u[0]; -#else -#error + lds[revMe + 0 * WG] = u[1]; + lds[bump ? ((revMe + WG) % (2 * WG)) : (revMe + WG)] = u[0]; #endif - - bar(); - for (i32 i = 0; i < NH/2; ++i) { u[i] = lds[i * WG + me]; } + bar(WG); + for (i32 i = 0; i < NH/2; ++i) { u[i] = lds[i * WG + me]; } + } } -void OVERLOAD reverseLine(u32 WG, local GF61 *lds2, GF61 *u) { +void OVERLOAD reverseLine(local F2_GF31 *lds, F2_GF31 *u) { u32 me = get_local_id(0); u32 revMe = WG - 1 - me; - local GF61 *lds = lds2 + revMe; - bar(); - for (u32 i = 0; i < NH; ++i) { lds[WG * (NH - 1 - i)] = u[i]; } + if (SHUFL_BYTES_H >= 8) { + local F2_GF31 *ldsOut = lds + revMe; + local F2_GF31 *ldsIn = lds + me; + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = u[i]; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { u[i] = ldsIn[WG * i]; } + } - lds = lds2 + me; - bar(); - for (u32 i = 0; i < NH; ++i) { u[i] = lds[WG * i]; } + else if (SHUFL_BYTES_H == 4) { + local F_Z31 *ldsOut = (local F_Z31 *) lds + revMe; + local F_Z31 *ldsIn = (local F_Z31 *) lds + me; + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = u[i].x; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { u[i].x = ldsIn[WG * i]; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = u[i].y; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { u[i].y = ldsIn[WG * i]; } + } } -// This is used to reverse the second part of a line, and cross the reversed parts between the halves. -void OVERLOAD revCrossLine(u32 WG, local GF61* lds2, GF61 *u, u32 n, bool writeSecondHalf) { +// +// These versions are for the kernel(s) that use a double-wide workgroup (u in half the workgroup, v in the other half) +// + +void OVERLOAD reverse2(local F2_GF31 *lds2, F2_GF31 *u) { u32 me = get_local_id(0); u32 lowMe = me % WG; - u32 revLowMe = WG - 1 - lowMe; - - for (u32 i = 0; i < n; ++i) { lds2[WG * n * writeSecondHalf + WG * (n - 1 - i) + revLowMe] = u[i]; } - - bar(); // we need a full bar because we're crossing halves - - for (u32 i = 0; i < n; ++i) { u[i] = lds2[WG * n * !writeSecondHalf + WG * i + lowMe]; } + if (SBMUL(2) * SHUFL_BYTES_H >= 4) { + local F2_GF31 *lds = LDSsharing_ptr(lds2, 2); + // For NH=8, u[0] to u[3] are left unchanged. Write to lds: + // u[7]rev u[6]rev u[5]rev u[4]rev + // v[7]rev v[6]rev v[5]rev v[4]rev + LDStx_start(lds2, 2); + for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * WG - (me >= WG ? 1 : 0) - lowMe) % (NH/2 * WG)] = u[NH/2 + i]; } + // For NH=8, read from lds into u[i]: + // u[4] = u[7]rev v[7]rev + // u[5] = u[6]rev v[6]rev + // u[6] = u[5]rev v[5]rev + // u[7] = u[4]rev v[4]rev + LDSbar(2); + for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i] = lds[i * WG + lowMe]; } + LDStx_end(lds2, 2); + } } -// -// These versions are for the kernel(s) that uses a double-wide workgroup (u in half the workgroup, v in the other half) -// - -void OVERLOAD reverse2(local GF61 *lds, GF61 *u) { +// This is used to reverse the second part of a line, and cross the reversed parts between the halves. +void OVERLOAD revCrossLine(local F2_GF31 *lds2, F2_GF31 *u) { u32 me = get_local_id(0); - - // For NH=8, u[0] to u[3] are left unchanged. Write to lds: - // u[7]rev u[6]rev - // u[5]rev u[4]rev - // v[7]rev v[6]rev - // v[5]rev v[4]rev - bar(); - for (u32 i = 0; i < NH / 2; ++i) { - u32 j = (i * G_H + me % G_H); - lds[me < G_H ? ((NH/2)*G_H - j) % ((NH/2)*G_H) : NH*G_H-1 - j] = u[NH/2 + i]; + u32 lowMe = me % WG; + u32 revLowMe = WG - 1 - lowMe; + + if (SHUFL_BYTES_H >= 4) { + local F2_GF31 *ldsOut = lds2; + local F2_GF31 *ldsIn = lds2; + if (me < WG) ldsOut += LDS_SHUFL_BYTES(2) / sizeof(F2); + else ldsIn += LDS_SHUFL_BYTES(2) / sizeof(F2); + bar(); // we need a full bar because we're crossing halves + for (u32 i = 0; i < NH/2; ++i) { ldsOut[WG * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2]; } + bar(); // we need a full bar because we just crossed halves. LDS reads are compatible with future shufl calls. + for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2] = ldsIn[WG * i + lowMe]; } + // One last bar() is needed when sharing LDS memory. This is because when sharing a workgroup will write to more than its own LDS area. + if (SHARING_LDS(2)) bar(); } - // For NH=8, read from lds into u[i]: - // u[4] = u[7]rev v[7]rev - // u[5] = u[6]rev v[6]rev - // u[6] = u[5]rev v[5]rev - // u[7] = u[4]rev v[4]rev - bar(); - lds += me % G_H + (me / G_H) * NH/2 * G_H; - for (u32 i = 0; i < NH / 2; ++i) { u[NH/2 + i] = lds[i * G_H]; } } +#if 0 // Unused + // Somewhat similar to reverseLine. -// The u values are in threads < G_H, the v values to reverse in threads >= G_H. +// The u values are in threads < WG, the v values to reverse in threads >= WG. // Whereas reverseLine leaves u values alone. This reverseLine moves u values around // so that pairSq2 can easily operate on pairs. This means for NH = 4, web output: // u[0] u[1] // Returned in u[0] // u[2] u[3] // Returned in u[1] // v[3]rev v[2]rev // Returned in u[2] // v[1]rev v[0]rev // Returned in u[3] -void OVERLOAD reverseLine2(local GF61 *lds, GF61 *u) { +void OVERLOAD reverseLine2(local F2_GF31 *lds, F2_GF31 *u) { u32 me = get_local_id(0); -// NOTE: It is important that this routine use lds memory in coordination with shufl2. Failure to do so would require an -// unqualified bar() call here. Specifically, the u values are stored in the upper half of lds memory (SMALL_HEIGHT GF61 values). -// The v values are stored in the lower half of lds memory (the next SMALL_HEIGHT GF61 values). +// NOTE: It is important that this routine use lds memory in coordination with shufl. Failure to do so would require an +// unqualified bar() call here. Specifically, the u values are stored in the upper half of lds memory (SMALL_HEIGHT F2 values). +// The v values are stored in the lower half of lds memory (the next SMALL_HEIGHT F2 values). - if (G_H > WAVEFRONT) bar(); + if (WG > WAVEFRONT) bar(); // For NH=4, the lds indices (where to write each incoming u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H -// That means saving to lds using index: me < G_H ? me % G_H + i * G_H : 8*G_H-1 - me % G_H - i * G_H +// 0..GH-1 +0*WG GH-1..0 +7*WG +// 0..GH-1 +1*WG GH-1..0 +6*WG +// 0..GH-1 +2*WG GH-1..0 +5*WG +// 0..GH-1 +3*WG GH-1..0 +4*WG +// That means saving to lds using index: me < WG ? me % WG + i * WG : 8*WG-1 - me % WG - i * WG #if 1 - local GF61 *ldsOut = lds + (me < G_H ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsOutInc = (me < G_H) ? G_H : -G_H; + local F2_GF31 *ldsOut = lds + (me < WG ? me % WG : (NH*2)*WG-1 - me % WG); + i32 ldsOutInc = (me < WG) ? WG : -WG; for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { *ldsOut = u[i]; } lds += me; bar(); - for (u32 i = 0; i < NH; ++i) { u[i] = lds[i * 2*G_H]; } + for (u32 i = 0; i < NH; ++i) { u[i] = lds[i * 2*WG]; } #else - local Z61 *ldsOut = (local Z61 *) lds + (me < G_H ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsOutInc = (me < G_H) ? G_H : -G_H; - for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { ldsOut[0] = u[i].x; ldsOut[NH*2*G_H] = u[i].y; } + local F_Z31 *ldsOut = (local F_Z31 *) lds + (me < WG ? me % WG : (NH*2)*WG-1 - me % WG); + i32 ldsOutInc = (me < WG) ? WG : -WG; + for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { ldsOut[0] = u[i].x; ldsOut[NH*2*WG] = u[i].y; } - local ZF61 *ldsIn = (local T *) lds + me; + local F_Z31 *ldsIn = (local F_Z31 *) lds + me; bar(); - for (u32 i = 0; i < NH; ++i) { u[i].x = ldsIn[i * 2*G_H]; u[i].y = ldsIn[NH*2*G_H + i * 2*G_H]; } + for (u32 i = 0; i < NH; ++i) { u[i].x = ldsIn[i * 2*WG]; u[i].y = ldsIn[NH*2*WG + i * 2*WG]; } #endif } // Undo a reverseLine2 -void OVERLOAD unreverseLine2(local GF61 *lds, GF61 *u) { +void OVERLOAD unreverseLine2(local F2_GF31 *lds, F2_GF31 *u) { u32 me = get_local_id(0); -// NOTE: It is important that this routine use lds memory in coordination with reverseLine2 and shufl2. By initially +// NOTE: It is important that this routine use lds memory in coordination with reverseLine2 and shufl. By initially // writing to the lds locations that reverseLine2 read from we do not need an initial bar() call here. Also, by reading -// from the lds locations that shufl2 will use (u values in the upper half of lds memory, v values in the lower half of +// from the lds locations that shufl will use (u values in the upper half of lds memory, v values in the lower half of // lds memory) we can issue a qualified bar() call before calling FFT_HEIGHT2. #if 1 - local GF61 *ldsOut = lds + me; - for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*G_H] = u[i]; } + local F2_GF31 *ldsOut = lds + me; + for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*WG] = u[i]; } // For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H - lds += (me < G_H) ? me % G_H : (NH*2)*G_H-1 - me % G_H; - i32 ldsInc = (me < G_H) ? G_H : -G_H; +// 0..GH-1 +0*WG GH-1..0 +7*WG +// 0..GH-1 +1*WG GH-1..0 +6*WG +// 0..GH-1 +2*WG GH-1..0 +5*WG +// 0..GH-1 +3*WG GH-1..0 +4*WG + lds += (me < WG) ? me % WG : (NH*2)*WG-1 - me % WG; + i32 ldsInc = (me < WG) ? WG : -WG; bar(); for (u32 i = 0; i < NH; ++i, lds += ldsInc) { u[i] = *lds; } #else - local Z61 *ldsOut = (local T *) lds + me; - for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*G_H] = u[i].x; ldsOut[NH*2*G_H + i * 2*G_H] = u[i].y; } + local F_Z31 *ldsOut = (local F_Z31 *) lds + me; + for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*WG] = u[i].x; ldsOut[NH*2*WG + i * 2*WG] = u[i].y; } // For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H - local Z61 *ldsIn = (local T *) lds + ((me < G_H) ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsInc = (me < G_H) ? G_H : -G_H; - bar(); - for (u32 i = 0; i < NH; ++i, ldsIn += ldsInc) { u[i].x = ldsIn[0]; u[i].y = ldsIn[NH*2*G_H]; } +// 0..GH-1 +0*WG GH-1..0 +7*WG +// 0..GH-1 +1*WG GH-1..0 +6*WG +// 0..GH-1 +2*WG GH-1..0 +5*WG +// 0..GH-1 +3*WG GH-1..0 +4*WG + local F_Z31 *ldsIn = (local F_Z31 *) lds + ((me < WG) ? me % WG : (NH*2)*WG-1 - me % WG); + i32 ldsInc = (me < WG) ? WG : -WG; + bar(); + for (u32 i = 0; i < NH; ++i, ldsIn += ldsInc) { u[i].x = ldsIn[0]; u[i].y = ldsIn[NH*2*WG]; } #endif } #endif + +#endif diff --git a/src/cl/trig.cl b/src/cl/trig.cl index ebdd2af9..c03b544e 100644 --- a/src/cl/trig.cl +++ b/src/cl/trig.cl @@ -2,8 +2,6 @@ #pragma once -#include "math.cl" - #if FFT_FP64 T2 reducedCosSin(int k, double cosBase) { diff --git a/src/cl/weight.cl b/src/cl/weight.cl index 66075bb7..ccad666a 100644 --- a/src/cl/weight.cl +++ b/src/cl/weight.cl @@ -3,6 +3,33 @@ #define STEP (NWORDS - (EXP % NWORDS)) // bool isBigWord(u32 extra) { return extra < NWORDS - STEP; } +// Determine the fractional-bits-per-word for a given FFT word. The fracbits value is (word * STEP % NWORDS) / NWORDS. +// The fracbits value is multiplied by 2^32 and truncated to make an integer. Fracbits can be used to determine weights and big-vs-little-word flags. +// Weight is 2^(1 - fracbits), but if fracbits is zero it is 2^0. +// Big word is true if fracbits + FRAC_BPW_HI does not overflow, but also true if fracbits is zero. +// To eliminate special logic for the FFT word 0, we subtract one from fracbits. +u32 fracBits(u32 i) { +#if NWORDS_IS_POWER_OF_TWO + return i * (FRAC_BPW_HI + 1) - 1; // We know FRAC_BPW_LO is -1 +#else + return i * FRAC_BPW_HI + mul_hi(i, FRAC_BPW_LO) - 1; +#endif +} + +// Somewhat similar to the above. Also returns the number of big words that occurred in getting to a given word. +u64 comboFracBits(u32 i) { +#if NWORDS_IS_POWER_OF_TWO + return (u64)i * (u64)(FRAC_BPW_HI + 1) - 1; // We know FRAC_BPW_LO is -1 +#else + return (u64)i * (u64)FRAC_BPW_HI + mul_hi(i, FRAC_BPW_LO) - 1; +#endif +} + +// Routines to acces the 8 precomputed step weights +u32 weightStepIndex(u32 i) { return i * STEP % NW * (8 / NW); } +u32 weightStepFracBits(u32 i) { return 0xFFFFFFFF - (weightStepIndex(i) << 29); } + + #if FFT_FP64 T fweightStep(u32 i) { @@ -17,7 +44,7 @@ T fweightStep(u32 i) { 0.68179283050742912, 0.83400808640934243, }; - return TWO_TO_NTH[i * STEP % NW * (8 / NW)]; + return TWO_TO_NTH[weightStepIndex(i)]; } T iweightStep(u32 i) { @@ -32,7 +59,7 @@ T iweightStep(u32 i) { -0.40539644249863949, -0.45474613366737116, }; - return TWO_TO_MINUS_NTH[i * STEP % NW * (8 / NW)]; + return TWO_TO_MINUS_NTH[weightStepIndex(i)]; } // This routine is not used. It forces "-use NO_ASM" in Windows. bfi should be replaced by a builtin if ever needed. @@ -93,7 +120,7 @@ F fweightStep(u32 i) { 0.68179283050742912, 0.83400808640934243, }; - return TWO_TO_NTH[i * STEP % NW * (8 / NW)]; + return TWO_TO_NTH[weightStepIndex(i)]; } F iweightStep(u32 i) { @@ -108,28 +135,20 @@ F iweightStep(u32 i) { -0.40539644249863949, -0.45474613366737116, }; - return TWO_TO_MINUS_NTH[i * STEP % NW * (8 / NW)]; + return TWO_TO_MINUS_NTH[weightStepIndex(i)]; } -F optionalDouble(F iw) { - // In a straightforward implementation, inverse weights are between 0.5 and 1.0. We use inverse weights between 1.0 and 2.0 - // because it allows us to implement this routine with a single OR instruction on the exponent. The original implementation - // where this routine took as input values from 0.25 to 1.0 required both an AND and an OR instruction on the exponent. - // return iw <= 1.0 ? iw * 2 : iw; - assert(iw > 0.5 && iw < 2); - uint u = as_uint(iw); - u |= 0x00800000; - return as_float(u); +F optionalDouble(F iw, int flag) { + // The 23 bits of precision in a float is not enough to handle doubling and halving the same way FP64 does. + // A straightforward implementation. Inverse weights are between > 0.5 and <= 1.0. + F doubled_iw = iw + iw; + return flag ? doubled_iw : iw; } -F optionalHalve(F w) { // return w >= 4 ? w / 2 : w; - // In a straightforward implementation, weights are between 1.0 and 2.0. We use weights between 2.0 and 4.0 because - // it allows us to implement this routine with a single AND instruction on the exponent. The original implementation - // where this routine took as input values from 1.0 to 4.0 required both an AND and an OR instruction on the exponent. - assert(w >= 2 && w < 8); - uint u = as_uint(w); - u &= 0xFF7FFFFF; - return as_float(u); +F optionalHalve(F w, int flag) { + // A straightforward implementation. Weights are between >= 1.0 and < 2.0. + F halved_w = w * 0.5f; + return flag ? halved_w : w; } #endif diff --git a/src/clwrap.cpp b/src/clwrap.cpp index 93385e64..8567c7b7 100644 --- a/src/clwrap.cpp +++ b/src/clwrap.cpp @@ -1,6 +1,5 @@ // Copyright (C) 2017-2024 Mihai Preda. -#include "timeutil.h" #include "File.h" #include "clwrap.h" @@ -16,7 +15,7 @@ using namespace std; // starting at 0 to -70 -array ERR_MES = { +static array ERR_MES = { "SUCCESS", "DEVICE_NOT_FOUND", "DEVICE_NOT_AVAILABLE", "COMPILER_NOT_AVAILABLE", "MEM_OBJECT_ALLOCATION_FAILURE", "OUT_OF_RESOURCES", "OUT_OF_HOST_MEMORY", "PROFILING_INFO_NOT_AVAILABLE", "MEM_COPY_OVERLAP", "IMAGE_FORMAT_MISMATCH", "IMAGE_FORMAT_NOT_SUPPORTED", "BUILD_PROGRAM_FAILURE", @@ -37,8 +36,8 @@ array ERR_MES = { }; string errMes(int err) { - string nb = " ("s + to_string(err) + ")"; - string mes = (err <= 0 && err >= -70) ? ERR_MES[-err] : + string const nb = " ("s + to_string(err) + ")"; + string const mes = (err <= 0 && err >= -70) ? ERR_MES[-err] : (err == -1001) ? "ICD_NOT_FOUND" : ""s; return mes + nb; } @@ -62,10 +61,10 @@ void check(int err, const char *file, int line, const char *func, string_view me } static void getInfo_(cl_device_id id, int what, size_t bufSize, void *buf, string_view whatStr) { - CHECK2(clGetDeviceInfo(id, what, bufSize, buf, NULL), whatStr); + CHECK2(clGetDeviceInfo(id, what, bufSize, buf, nullptr), whatStr); } -#define GET_INFO(id, what, where) getInfo_(id, what, sizeof(where), &where, #what) +#define GET_INFO(id, what, where) getInfo_(id, what, sizeof(where), &(where), #what) string getBdfFromDevice(cl_device_id id) { @@ -145,7 +144,7 @@ float getGpuRamGB(cl_device_id id) { try { u64 totSize = 0; GET_INFO(id, CL_DEVICE_GLOBAL_MEM_SIZE, totSize); - return ldexp(totSize, -30); // to GB + return float(ldexp(totSize, -30)); // to GB } catch (const gpu_error& err) { } return 0; @@ -178,6 +177,14 @@ bool isNvidiaGpu(cl_device_id id) { return pcieId == 0x10DE; } +u32 getNvidiaComputeCapability(cl_device_id id) { + u32 major = 0; + u32 minor = 0; + GET_INFO(id, CL_DEVICE_COMPUTE_CAPABILITY_MAJOR_NV, major); + GET_INFO(id, CL_DEVICE_COMPUTE_CAPABILITY_MINOR_NV, minor); + return major * 100 + minor; +} + /* static string getFreq(cl_device_id device) { unsigned computeUnits, frequency; @@ -207,7 +214,7 @@ cl_device_id getDevice(u32 argsDeviceId) { cl_context createContext(cl_device_id id) { int err; - cl_context context = clCreateContext(NULL, 1, &id, NULL, NULL, &err); + cl_context context = clCreateContext(nullptr, 1, &id, nullptr, nullptr, &err); CHECK2(err, "clCreateContext"); return context; } @@ -219,10 +226,11 @@ void release(cl_mem buf) { CHECK1(clReleaseMemObject(buf)); } void release(cl_queue queue) { CHECK1(clReleaseCommandQueue(queue)); } void release(cl_kernel k) { CHECK1(clReleaseKernel(k)); } void release(cl_event event) { CHECK1(clReleaseEvent(event)); } +void release(cl_graph graph) { CHECK1(clReleaseGraph(graph));} Program loadSource(cl_context context, const string &source) { const char *ptr = source.c_str(); - size_t size = source.size(); + size_t const size = source.size(); int err = 0; cl_program program = clCreateProgramWithSource(context, 1, &ptr, &size, &err); CHECK2(err, "clCreateProgramWithSource"); @@ -230,7 +238,7 @@ Program loadSource(cl_context context, const string &source) { } string getBuildLog(cl_program program, cl_device_id deviceId) { - size_t logSize; + size_t logSize = 0; const size_t maxLogSize = 64 * 1024; int err = clGetProgramBuildInfo(program, deviceId, CL_PROGRAM_BUILD_LOG, 0, nullptr, &logSize); CHECK2(err, "clGetProgramBuildInfo"); @@ -240,7 +248,7 @@ string getBuildLog(cl_program program, cl_device_id deviceId) { log("getBuildLog: log size is %lu bytes, not showing\n", (unsigned long) logSize); return {}; } - std::unique_ptr buf(new char[logSize + 1]); + std::unique_ptr const buf(new char[logSize + 1]); err = clGetProgramBuildInfo(program, deviceId, CL_PROGRAM_BUILD_LOG, logSize, buf.get(), &logSize); CHECK2(err, "clGetProgramBuildInfo"); buf.get()[logSize] = 0; @@ -252,28 +260,28 @@ string getBuildLog(cl_program program, cl_device_id deviceId) { Program loadBinary(cl_context context, cl_device_id id, string_view fileName) { File f = File::openRead(fileName); if (!f) { return {}; } - string bytes = f.readAll(); - size_t size = bytes.size(); - const unsigned char *ptr = reinterpret_cast(bytes.c_str()); + string const bytes = f.readAll(); + size_t const size = bytes.size(); + const auto *ptr = reinterpret_cast(bytes.c_str()); int err = 0; - cl_program program = clCreateProgramWithBinary(context, 1, &id, &size, &ptr, NULL, &err); + cl_program program = clCreateProgramWithBinary(context, 1, &id, &size, &ptr, nullptr, &err); if (err) { log("Load binary %s : %s\n", string(fileName).c_str(), errMes(err).c_str()); return {}; } - if ((err = clBuildProgram(program, 1, &id, NULL, NULL, NULL))) { + if ((err = clBuildProgram(program, 1, &id, nullptr, nullptr, nullptr))) { log("Build binary %s : %s\n", string(fileName).c_str(), errMes(err).c_str()); return {}; } return Program{program}; } -string getBinary(cl_program program) { +static string getBinary(cl_program program) { size_t size; - CHECK1(clGetProgramInfo(program, CL_PROGRAM_BINARY_SIZES, sizeof(size), &size, NULL)); + CHECK1(clGetProgramInfo(program, CL_PROGRAM_BINARY_SIZES, sizeof(size), &size, nullptr)); auto buf = make_unique(size + 1); - char *ptr = buf.get(); - CHECK1(clGetProgramInfo(program, CL_PROGRAM_BINARIES, sizeof(&buf), &ptr, NULL)); + char const*ptr = buf.get(); + CHECK1(clGetProgramInfo(program, CL_PROGRAM_BINARIES, sizeof(&buf), &ptr, nullptr)); return {buf.get(), size}; } @@ -318,31 +326,33 @@ void flush( cl_queue q) { CHECK1(clFlush(q)); } void finish(cl_queue q) { CHECK1(clFinish(q)); } EventHolder run(cl_queue queue, cl_kernel kernel, - size_t groupSize, size_t workSize, + size_t groupSizeX, size_t workSizeX, size_t workSizeY, vector&& waits, const string &name, bool genEvent) { cl_event event{}; - CHECK2(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, &workSize, &groupSize, - waits.size(), waits.empty() ? 0 : waits.data(), genEvent ? &event : nullptr), + size_t workSizes[2] = {workSizeX, workSizeY}; + size_t groupSizes[2] = {groupSizeX, 1}; + CHECK2(clEnqueueNDRangeKernel(queue, kernel, workSizeY == 1 ? 1 : 2, nullptr, workSizes, groupSizes, + u32(waits.size()), waits.empty() ? 0 : waits.data(), genEvent ? &event : nullptr), name.c_str()); return genEvent ? EventHolder{event} : EventHolder{}; } EventHolder read(cl_queue queue, vector&& waits, bool blocking, cl_mem buf, size_t size, void *data, bool genEvent) { - size_t start = 0; + size_t const start = 0; cl_event event{}; CHECK1(clEnqueueReadBuffer(queue, buf, blocking, start, size, data, - waits.size(), waits.empty() ? 0 : waits.data(), genEvent ? &event : nullptr)); + u32(waits.size()), waits.empty() ? nullptr : waits.data(), genEvent ? &event : nullptr)); return genEvent ? EventHolder{event} : EventHolder{}; } EventHolder write(cl_queue queue, vector&& waits, bool blocking, cl_mem buf, size_t size, const void *data, bool genEvent) { - size_t start = 0; + size_t const start = 0; cl_event event{}; CHECK1(clEnqueueWriteBuffer(queue, buf, blocking, start, size, data, - waits.size(), waits.empty() ? 0 : waits.data(), genEvent ? &event : nullptr)); + u32(waits.size()), waits.empty() ? nullptr : waits.data(), genEvent ? &event : nullptr)); return genEvent ? EventHolder{event} : EventHolder{}; } @@ -350,7 +360,7 @@ EventHolder copyBuf(cl_queue queue, vector&& waits, const cl_mem src, cl_mem dst, size_t size, bool genEvent) { cl_event event{}; CHECK1(clEnqueueCopyBuffer(queue, src, dst, 0, 0, size, - waits.size(), waits.empty() ? 0 : waits.data(), genEvent ? &event : nullptr)); + u32(waits.size()), waits.empty() ? nullptr : waits.data(), genEvent ? &event : nullptr)); return genEvent ? EventHolder{event} : EventHolder{}; } @@ -359,32 +369,38 @@ EventHolder fillBuf(cl_queue q, vector&& waits, assert(size); cl_event event{}; CHECK1(clEnqueueFillBuffer(q, buf, pat, patSize, 0 /*start*/, size, - waits.size(), waits.empty() ? 0 : waits.data(), genEvent ? &event : nullptr)); + u32(waits.size()), waits.empty() ? nullptr : waits.data(), genEvent ? &event : nullptr)); return genEvent ? EventHolder{event} : EventHolder{}; } EventHolder enqueueMarker(cl_queue q) { cl_event event{}; - CHECK1(clEnqueueMarkerWithWaitList(q, 0, 0, &event)); + CHECK1(clEnqueueMarkerWithWaitList(q, 0, nullptr, &event)); + return EventHolder{event}; +} + +EventHolder enqueueMarkerWithWaits(cl_queue q, vector&& waits) { + cl_event event{}; + CHECK1(clEnqueueMarkerWithWaitList(q, u32(waits.size()), waits.empty() ? nullptr : waits.data(), &event)); return EventHolder{event}; } void waitForEvents(vector&& waits) { if (!waits.empty()) { - CHECK1(clWaitForEvents(waits.size(), waits.data())); + CHECK1(clWaitForEvents(u32(waits.size()), waits.data())); } } int getKernelNumArgs(cl_kernel k) { int nArgs = 0; - CHECK1(clGetKernelInfo(k, CL_KERNEL_NUM_ARGS, sizeof(nArgs), &nArgs, NULL)); + CHECK1(clGetKernelInfo(k, CL_KERNEL_NUM_ARGS, sizeof(nArgs), &nArgs, nullptr)); return nArgs; } int getWorkGroupSize(cl_kernel k, cl_device_id device, const char *name) { - size_t size[3]; - CHECK2(clGetKernelWorkGroupInfo(k, device, CL_KERNEL_COMPILE_WORK_GROUP_SIZE, sizeof(size), &size, NULL), name); - return size[0]; + size_t size[3]{}; + CHECK2(clGetKernelWorkGroupInfo(k, device, CL_KERNEL_COMPILE_WORK_GROUP_SIZE, sizeof(size), &size, nullptr), name); + return int(size[0]); } std::string getKernelArgName(cl_kernel k, int pos) { @@ -398,7 +414,7 @@ std::string getKernelArgName(cl_kernel k, int pos) { u32 getEventInfo(cl_event event) { u32 status = -1; - CHECK1(clGetEventInfo(event, CL_EVENT_COMMAND_EXECUTION_STATUS, sizeof(status), &status, 0)); + CHECK1(clGetEventInfo(event, CL_EVENT_COMMAND_EXECUTION_STATUS, sizeof(status), &status, nullptr)); return status; } @@ -419,7 +435,7 @@ array getEventNanos(cl_event event) { for (int i = 0; i < 4; ++i) { u64 t{}; - CHECK1(clGetEventProfilingInfo(event, what[i], sizeof(t), &t, 0)); + CHECK1(clGetEventProfilingInfo(event, what[i], sizeof(t), &t, nullptr)); if (i) { ret[i - 1] = delta(prev, t); } prev = t; } @@ -428,12 +444,24 @@ array getEventNanos(cl_event event) { cl_context getQueueContext(cl_command_queue q) { cl_context ret; - CHECK1(clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ret, 0)); + CHECK1(clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ret, nullptr)); return ret; } -cl_device_id getQueueDevice(cl_command_queue q) { +[[maybe_unused]] static cl_device_id getQueueDevice(cl_command_queue q) { cl_device_id id; - CHECK1(clGetCommandQueueInfo(q, CL_QUEUE_DEVICE, sizeof(id), &id, 0)); + CHECK1(clGetCommandQueueInfo(q, CL_QUEUE_DEVICE, sizeof(id), &id, nullptr)); return id; } + +// OpenCL-like extensions invented to provide a clean interface to some nVidia CUDA features. +// These routines are defined in clwrap_cuda.cpp for the CUDA translation of openCL. +// The dummy implementation below is for the native openCL builds. + +#ifndef CUDA_BACKEND +bool clIsGraphSupported(cl_device_id) { return 0; } +int clGraphBeginRecording(cl_command_queue) { return CL_INVALID_VALUE; } +int clGraphEndRecording(cl_command_queue, cl_graph*) { return CL_INVALID_VALUE; } +int clGraphLaunch(cl_graph) { return CL_INVALID_VALUE; } +int clReleaseGraph(cl_graph) { return CL_INVALID_VALUE; } +#endif diff --git a/src/clwrap.h b/src/clwrap.h index 8b9bf9e4..3ef3fa20 100644 --- a/src/clwrap.h +++ b/src/clwrap.h @@ -2,7 +2,11 @@ #pragma once +#ifdef CUDA_BACKEND +#include "tinycuda.h" +#else #include "tinycl.h" +#endif #include #include @@ -11,13 +15,13 @@ using cl_queue = cl_command_queue; - void release(cl_context context); void release(cl_kernel k); void release(cl_mem buf); void release(cl_program program); void release(cl_queue queue); void release(cl_event event); +void release(cl_graph graph); template struct Deleter { @@ -32,6 +36,7 @@ template<> struct default_delete : public Deleter {}; template<> struct default_delete : public Deleter {}; template<> struct default_delete : public Deleter {}; template<> struct default_delete : public Deleter {}; +template<> struct default_delete : public Deleter {}; } template using Holder = std::unique_ptr >; @@ -39,6 +44,7 @@ template using Holder = std::unique_ptr >; using QueueHolder = std::unique_ptr; using KernelHolder = std::unique_ptr; using EventHolder = std::unique_ptr; +using GraphHolder = std::unique_ptr; using Program = std::unique_ptr; class Context; @@ -63,6 +69,7 @@ u64 getFreeMem(cl_device_id id); bool hasFreeMemInfo(cl_device_id id); bool isAmdGpu(cl_device_id id); bool isNvidiaGpu(cl_device_id id); +u32 getNvidiaComputeCapability(cl_device_id id); string getDriverVersion(cl_device_id id); string getDriverVersionByPos(int pos); @@ -79,7 +86,7 @@ void saveBinary(cl_program program, string_view fileName); template void setArg(cl_kernel k, int pos, const T &value, const string& name) { - CHECK2(clSetKernelArg(k, pos, sizeof(value), &value), (name + '[' + to_string(pos) + "] size " + to_string(sizeof(value))).c_str()); + CHECK2(clSetKernelArg(k, pos, sizeof(value), &value), name + '[' + to_string(pos) + "] size " + to_string(sizeof(value))); } /* @@ -87,13 +94,13 @@ template<> void setArg(cl_kernel k, int pos, const int &value, const string& name); */ -cl_mem makeBuf_(cl_context context, unsigned kind, size_t size, const void *ptr = 0); +cl_mem makeBuf_(cl_context context, unsigned kind, size_t size, const void *ptr = nullptr); cl_queue makeQueue(cl_device_id d, cl_context c, bool enableProfile); void flush( cl_queue q); void finish(cl_queue q); -EventHolder run(cl_queue queue, cl_kernel kernel, size_t groupSize, size_t workSize, +EventHolder run(cl_queue queue, cl_kernel kernel, size_t groupSizeX, size_t workSizeX, size_t workSizeY, vector&& waits, const string &name, bool genEvent); EventHolder read(cl_queue queue, vector&& waits, @@ -107,6 +114,7 @@ EventHolder copyBuf(cl_queue queue, vector&& waits, const cl_mem src, EventHolder fillBuf(cl_queue q, vector&& waits, cl_mem buf, const void *pat, size_t patSize, size_t size, bool genEvent); EventHolder enqueueMarker(cl_queue q); +EventHolder enqueueMarkerWithWaits(cl_queue q, vector&& waits); void waitForEvents(vector&& waits); @@ -123,3 +131,15 @@ std::array getEventNanos(cl_event event); u32 getEventInfo(cl_event event); cl_context getQueueContext(cl_command_queue q); + +#ifdef CUDA_BACKEND +// Set L1 cache configuration - 4 possibilities +void cudaSetL1Config(int x); + +// Set L2 cache persistence for multiple read-only buffers on the given stream. +// Computes the address span covering all buffers and sets a single access policy window. +// Buffers that are nullptr or zero-size are skipped. +void cudaSetL2Persistent(cl_command_queue q, const std::vector& buffers); +#endif + + diff --git a/src/common.cpp b/src/common.cpp index c3a1ef80..29409bdd 100644 --- a/src/common.cpp +++ b/src/common.cpp @@ -1,16 +1,10 @@ // GpuOwl Mersenne primality tester; Copyright (C) 2017-2018 Mihai Preda. #include "common.h" -#include "File.h" -#include "timeutil.h" -#include #include -#include -#include #include #include -#include string hex(u64 x) { ostringstream out{}; @@ -24,7 +18,7 @@ std::string rstripNewline(std::string s) { } u32 crc32(const void *data, size_t size) { - u32 tab[16] = { + u32 const tab[16] = { 0x00000000, 0x1DB71064, 0x3B6E20C8, 0x26D930AC, 0x76DC4190, 0x6B6B51F4, 0x4DB26158, 0x5005713C, 0xEDB88320, 0xF00F9344, 0xD6D6A3E8, 0xCB61B38C, @@ -41,11 +35,11 @@ u32 crc32(const void *data, size_t size) { string formatBound(u32 b) { if (b >= 1'000'000 && b % 1'000'000 == 0) { return to_string(b / 1'000'000) + 'M'; - } else if (b >= 500'000 && b % 100'000 == 0) { + } if (b >= 500'000 && b % 100'000 == 0) { char buf[32]; snprintf(buf, sizeof(buf), "%.1fM", float(b) / 1'000'000); return buf; - } else { + } return to_string(b); - } + } diff --git a/src/common.h b/src/common.h index 516b099d..13a010e9 100644 --- a/src/common.h +++ b/src/common.h @@ -6,14 +6,22 @@ #include #include +// This is a copy of the args.verbose level. It allows the CUDA wrapper to access the value. +extern int prpll_verbose; + using u8 = uint8_t; using i32 = int32_t; using u32 = uint32_t; using i64 = int64_t; using u64 = uint64_t; +#ifndef _MSC_VER // MSVC does not suppport 128-bit values using i128 = __int128; using u128 = unsigned __int128; -using f128 = __float128; +#else +#include "U128.h" +using u128 = U128; +#endif +// using f128 = __float128; static_assert(sizeof(u8) == 1, "size u8"); static_assert(sizeof(u32) == 4, "size u32"); @@ -25,7 +33,7 @@ namespace fs = std::filesystem; // When using multiple primes in an NTT the size of an integer FFT "word" can be 64 bits. Original FP64 FFT needs only 32 bits. // C code will use i64 integer data. The code that reads and writes GPU buffers will downsize the integers to 32 bits when required. -typedef i64 Word; +using Word = i64; // Create datatype names that mimic the ones used in OpenCL code using double2 = pair; @@ -46,15 +54,15 @@ using Words = vector; inline u64 res64(const Words& words) { return words.empty() ? 0 : ((u64(words[1]) << 32) | words[0]); } -inline u32 nWords(u32 E) { return (E - 1) / 32 + 1; } +inline u32 nWords(u64 E) { return u32((E - 1) / 32 + 1); } -inline Words makeWords(u32 E, u32 value) { +inline Words makeWords(u64 E, u32 value) { Words ret(nWords(E)); ret[0] = value; return ret; } -inline u32 roundUp(u32 x, u32 multiple) { return ((x - 1) / multiple + 1) * multiple; } +inline u64 roundUp(u64 x, u32 multiple) { return ((x - 1) / multiple + 1) * multiple; } u32 crc32(const void* data, size_t size); diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp new file mode 100644 index 00000000..6006278b --- /dev/null +++ b/src/cuda/clwrap_cuda.cpp @@ -0,0 +1,1442 @@ +// CUDA Driver API implementation of OpenCL API functions. +// This replaces clwrap.cpp when building with the native CUDA backend, +// mapping all cl* calls to cu* equivalents via the CUDA Driver API. + +#include "tinycuda.h" +#include "cudawrap.h" // For NvrtcProgram::preprocessOpenCL and compile + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#endif + +using namespace std; + +// Track allocated cl_mem objects so clSetKernelArg can distinguish buffer args from scalars. +// In OpenCL, buffer args are passed as &memobj where memobj is cl_mem (a pointer to _cl_mem). +// We need to convert these to CUdeviceptr for CUDA kernel launch. +// The set is shared by every worker thread (-workers N runs N Gpu instances against this one +// process-wide context), so all access goes through g_allocatedBuffersMutex. +static unordered_set g_allocatedBuffers; +static std::mutex g_allocatedBuffersMutex; + +// Global CUDA context — set once by clCreateContext, used to ensure current before CUDA calls +static CUcontext g_cudaContext = nullptr; + +static void ensureContextCurrent() { + if (g_cudaContext) { + cuCtxSetCurrent(g_cudaContext); + } +} + +// Reference-count CUmodules so they get unloaded once nothing uses them. +// +// In OpenCL, clCreateKernel retains the program, so the underlying code object +// stays alive until BOTH the program and every kernel derived from it are +// released. PRPLL relies on this: KernelCompiler::loadAux() creates a kernel, +// then releases the program while the kernel keeps running. We therefore cannot +// unload the module in clReleaseProgram — a live CUfunction would be invalidated. +// +// Instead we count references: the owning program holds one ref (set when the +// module is loaded), and each kernel created from it holds one more. The module +// is unloaded when the count reaches zero. This is essential because Gpu::make() +// builds a fresh set of ~36 kernels per work unit (and dozens of times during +// tuning), all into the single process-lifetime CUDA context. Without unloading, +// device memory grows unbounded and eventually cuLaunchKernel fails with +// CUDA_ERROR_OUT_OF_MEMORY. +// +// KernelCompiler compiles kernels on parallel threads under CUDA, each +// loading its module and creating its kernel, so the map is shared: one +// mutex around every access (the counts are the only state the compile +// threads share; the CUDA context is per-thread current, see +// ensureContextCurrent). +static std::map g_moduleRefCount; +static std::mutex g_moduleRefMutex; + +static void moduleRetain(CUmodule m) { + if (!m) return; + std::lock_guard lock(g_moduleRefMutex); + ++g_moduleRefCount[m]; +} + +static void moduleRelease(CUmodule m) { + if (!m) return; + bool unload = false; + { + std::lock_guard lock(g_moduleRefMutex); + auto it = g_moduleRefCount.find(m); + if (it == g_moduleRefCount.end()) return; // untracked module — leave as-is + if (--it->second <= 0) { + g_moduleRefCount.erase(it); + unload = true; + } + } + if (unload) { + ensureContextCurrent(); + cuModuleUnload(m); + } +} + +// Global state for CUDA initialization +static bool g_cudaInitialized = false; +static void ensureCudaInit() { + if (!g_cudaInitialized) { + CUresult const err = cuInit(0); + if (err != CUDA_SUCCESS) { + fprintf(stderr, "cuInit failed: %d\n", (int)err); + } + g_cudaInitialized = true; + } +} + +// Global device list (allocated once, never freed) +static vector<_cl_device_id> g_devices; +static bool g_devicesEnumerated = false; + +static void enumerateDevices() { + if (g_devicesEnumerated) return; + ensureCudaInit(); + int count = 0; + cuDeviceGetCount(&count); + g_devices.resize(count); + for (int i = 0; i < count; i++) { + cuDeviceGet(&g_devices[i].dev, i); + } + g_devicesEnumerated = true; +} + +// ---- OpenCL API implementations ---- + +extern "C" { + +unsigned clGetPlatformIDs(unsigned num, cl_platform_id* platforms, unsigned* numRet) { + // CUDA has no "platforms" concept — just return 1 dummy + if (numRet) *numRet = 1; + if (platforms && num >= 1) platforms[0] = nullptr; + return CL_SUCCESS; +} + +int clGetDeviceIDs(cl_platform_id, cl_device_type, unsigned num, cl_device_id* devices, unsigned* numRet) { + enumerateDevices(); + unsigned const n = u32(g_devices.size()); + if (numRet) *numRet = n; + if (devices) { + for (unsigned i = 0; i < min(num, n); i++) { + devices[i] = &g_devices[i]; + } + } + return n > 0 ? CL_SUCCESS : CL_DEVICE_NOT_FOUND; +} + +cl_context clCreateContext(const intptr_t*, unsigned nDevices, const cl_device_id* devices, + void (*)(const char*, const void*, size_t, void*), void*, int* err) { + if (!devices || nDevices == 0) { if (err) *err = CL_INVALID_DEVICE; return nullptr; } + auto* ctx = new _cl_context; + ctx->dev = devices[0]->dev; +#if CUDA_VERSION >= 13000 + CUctxCreateParams params{}; + CUresult r = cuCtxCreate_v4(&ctx->ctx, ¶ms, 0, ctx->dev); +#else + CUresult const r = cuCtxCreate(&ctx->ctx, 0, ctx->dev); +#endif + if (r != CUDA_SUCCESS) { + delete ctx; + if (err) *err = CL_OUT_OF_RESOURCES; + return nullptr; + } + g_cudaContext = ctx->ctx; // Track for ensureContextCurrent() + + // L2 persistence: no benefit measured for this workload. + // cuCtxSetLimit(CU_LIMIT_PERSISTING_L2_CACHE_SIZE, 16 * 1024 * 1024); + + if (err) *err = CL_SUCCESS; + return ctx; +} + +int clReleaseContext(cl_context ctx) { + if (ctx) { + cuCtxDestroy(ctx->ctx); + delete ctx; + } + return CL_SUCCESS; +} + +int clReleaseProgram(cl_program p) { + if (p) { + // Drop the program's reference to its module. The module is unloaded only once + // every kernel created from it has also been released (see moduleRelease and the + // refcount rationale near the top of this file). This lets loadAux() release the + // program while keeping the kernel's CUfunction valid, matching OpenCL semantics, + // without leaking a module per Gpu::make(). + if (p->moduleLoaded) { moduleRelease(p->module); } + delete p; + } + return CL_SUCCESS; +} + +int clReleaseCommandQueue(cl_command_queue q) { + if (q) { + cuStreamDestroy(q->stream); + delete q; + } + return CL_SUCCESS; +} + +// ---- Program compilation (NVRTC) ---- + +cl_program clCreateProgramWithSource(cl_context ctx, unsigned count, const char** strings, + const size_t* lengths, int* err) { + auto* prog = new _cl_program; + prog->context = ctx; + for (unsigned i = 0; i < count; i++) { + if (lengths && lengths[i]) { + prog->source.append(strings[i], lengths[i]); + } else { + prog->source.append(strings[i]); + } + } + if (err) *err = CL_SUCCESS; + return prog; +} + +// The "binary" the kernel cache stores (CL_PROGRAM_BINARIES) and hands back +// (clCreateProgramWithBinary): the PTX text, then — when NVRTC produced a +// CUBIN that this driver loaded — this marker and the CUBIN. The PTX stays, +// and stays first, because clCreateKernel reads each kernel's declared +// work-group size (.maxntid) and its PDL wait from the PTX text; a CUBIN +// carries neither as text. A blob without the marker is a plain PTX (the +// format before the CUBIN), and loads as before. +static const char CUBIN_MARKER[] = "\n// PRPLL-CUBIN\n"; +static const size_t CUBIN_MARKER_LEN = sizeof(CUBIN_MARKER) - 1; + +static string cacheBlob(const _cl_program* prog) { + if (prog->cubin.empty()) { return prog->ptx; } + string blob; + blob.reserve(prog->ptx.size() + CUBIN_MARKER_LEN + prog->cubin.size()); + blob += prog->ptx; + blob += CUBIN_MARKER; + blob += prog->cubin; + return blob; +} + +// Loads prog->cubin when there is one — SASS for this device, so no JIT and +// no PTX-version check to fail on a driver older than the toolkit — and +// otherwise, or when the driver rejects it (logged), the PTX through the JIT. +// A rejected CUBIN is dropped so the cache stores what loaded. +static CUresult loadModule(_cl_program* prog, unsigned nOpts, CUjit_option* opts, void** optVals) { + if (!prog->cubin.empty()) { + CUresult const r = cuModuleLoadDataEx(&prog->module, prog->cubin.data(), nOpts, opts, optVals); + if (r == CUDA_SUCCESS) { return r; } + const char* errName = nullptr; + cuGetErrorName(r, &errName); + fprintf(stderr, "CUBIN rejected by the driver: %s (%d) — loading the PTX through the JIT instead\n", errName ? errName : "?", (int)r); + prog->cubin.clear(); + } + return cuModuleLoadDataEx(&prog->module, prog->ptx.c_str(), nOpts, opts, optVals); +} + +cl_program clCreateProgramWithBinary(cl_context ctx, unsigned /*nDevices*/, const cl_device_id*, + const size_t* lengths, const unsigned char** binaries, + int* binaryStatus, int* err) { + auto* prog = new _cl_program; + prog->context = ctx; + if (lengths && binaries && lengths[0] > 0) { + string blob((const char*)binaries[0], lengths[0]); + // A bare ELF is a CUBIN without its PTX: nothing to read the kernels' + // work-group sizes from. Refuse it; the caller recompiles and overwrites. + if (blob.compare(0, 4, "\177ELF", 4) == 0) { + fprintf(stderr, "Cached kernel binary is a bare CUBIN (no PTX): recompiling\n"); + if (binaryStatus) binaryStatus[0] = CL_INVALID_BINARY; + if (err) *err = CL_INVALID_BINARY; + return prog; + } + size_t const mark = blob.find(CUBIN_MARKER); + if (mark == string::npos) { + prog->ptx = std::move(blob); + } else { + prog->ptx = blob.substr(0, mark); + prog->cubin = blob.substr(mark + CUBIN_MARKER_LEN); + } + prog->compiled = true; + ensureContextCurrent(); + CUresult const r = loadModule(prog, 0, nullptr, nullptr); + if (r == CUDA_SUCCESS) { + prog->moduleLoaded = true; + moduleRetain(prog->module); // program owns one reference + if (binaryStatus) binaryStatus[0] = CL_SUCCESS; + } else { + fprintf(stderr, "cuModuleLoadData from cache failed: %d, blob size=%zu\n", (int)r, lengths[0]); + prog->compiled = false; + if (binaryStatus) binaryStatus[0] = CL_INVALID_BINARY; + if (err) { *err = CL_INVALID_BINARY; return prog; } + } + } + if (err) *err = CL_SUCCESS; + return prog; +} + + +int clCompileProgram(cl_program prog, unsigned /*nDevices*/, const cl_device_id* devices, const char* options, + unsigned numHeaders, const cl_program* headers, const char* const* headerNames, + void (*)(cl_program, void*), void*) { + if (!prog) return CL_INVALID_PROGRAM; + + // Get device arch for NVRTC + CUdevice const dev = devices ? devices[0]->dev : g_devices[0].dev; + int major = 0, minor = 0; + cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, dev); + cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, dev); + + char archOpt[32]; + snprintf(archOpt, sizeof(archOpt), "--gpu-architecture=sm_%d%d", major, minor); + + // Parse OpenCL options string into NVRTC options + // Convert OpenCL build options to NVRTC equivalents: + // -cl-std=CL2.0 → -std=c++17 + // -cl-finite-math-only → --fmad=true (enable FMA contraction, the safe subset) + // -Dfoo=bar → -Dfoo=bar (pass through) + vector nvrtcOpts; + nvrtcOpts.emplace_back(archOpt); + nvrtcOpts.emplace_back("-default-device"); + nvrtcOpts.emplace_back("-std=c++17"); + nvrtcOpts.emplace_back("-w"); // Suppress NVRTC macro redefinition warnings + + // FMA contraction: OpenCL uses -cl-finite-math-only + #pragma OPENCL FP_CONTRACT ON + // to allow the compiler to contract a*b+c into FMA instructions. NVRTC's --fmad=true + // is the safe equivalent — it ONLY enables FMA contraction without the dangerous + // parts of -use_fast_math (no flush-to-zero, no reduced-precision division/sqrt). + // This is critical for FFT performance: every butterfly is multiply-add pairs. + nvrtcOpts.emplace_back("--fmad=true"); + + // NOTE: --restrict (all kernel pointers are __restrict__) was tested but causes GPU read + // errors — some PRPLL kernels use in-place operations where in/out buffers alias. + // Do NOT enable globally. The compiler still auto-uses __ldg() for const pointers on sm_35+. + + // Debug: dump full options string + { + static const char* dumpPrefix = getenv("PRPLL_DUMP_PTX"); + static bool dumpedOpts = false; + if (dumpPrefix && !dumpedOpts && options) { + dumpedOpts = true; + fprintf(stderr, "clCompileProgram options: [%s]\n", options); + FILE* optLog = fopen("kernel_regs.log", "a"); + if (optLog) { fprintf(optLog, "clCompileProgram options: [%s]\n", options); fclose(optLog); } + } + } + + int maxregcount = 0; + if (options) { + istringstream iss(options); + string tok; + while (iss >> tok) { + if (tok.starts_with("-D")) { + // Fix AMD-only FFT variants for NVIDIA: variant_W=0 and variant_H=0 require + // AMD builtins (__builtin_amdgcn_ds_bpermute etc). Replace with variant 2. + // FFT_VARIANT is a 3-digit number WMH: e.g. 000, 101, 202 + if (tok.find("FFT_VARIANT=") != string::npos) { + size_t const eqPos = tok.find('='); + string valStr = tok.substr(eqPos + 1); + // Strip trailing 'u' suffix + if (!valStr.empty() && valStr.back() == 'u') valStr.pop_back(); + int const val = atoi(valStr.c_str()); + int vW = val / 100; + int const vM = (val % 100) / 10; + int vH = val % 10; + if (vW == 0) vW = 2; // AMD BCAST → NVIDIA generic + if (vH == 0) vH = 2; + int const newVal = vW * 100 + vM * 10 + vH; + tok = "-DFFT_VARIANT=" + to_string(newVal) + "u"; + } + nvrtcOpts.push_back(tok); + } else if (tok == "-cl-finite-math-only" || tok == "-cl-fast-relaxed-math") { + // FMA contraction already enabled above via --fmad=true. + // Do NOT use -use_fast_math here — it enables flush-to-zero and + // reduced-precision division/sqrt which breaks tailMul accuracy. + } else if (tok.starts_with("--maxrregcount")) { + nvrtcOpts.push_back(tok); + maxregcount = atoi(tok.substr(15, 3).c_str()); + } + // Skip other -cl-* options (not applicable to NVRTC) + } + } + + // Build NVRTC headers from the cl_program header array + vector> nvrtcHeaders; + + // Add all OpenCL source headers + for (unsigned i = 0; i < numHeaders; i++) { + // First header: opencl_compat.cuh (inject as virtual NVRTC header) + if (i == 0) { + nvrtcHeaders.emplace_back("opencl_compat.cuh", headers[0]->source); + } + // Remaining headers need preprocessing + else if (headers[i] && headerNames[i]) { + // Preprocess OpenCL source for CUDA compatibility + string const processedSrc = NvrtcProgram::preprocessOpenCL(headers[i]->source); + // Debug: verify KERNEL macro replacement +// I'm not sure what Sherpa was trying to print out here. It prints out nothing useful. +// { +// static const char* dumpPrefix = getenv("PRPLL_DUMP_PTX"); +// if (dumpPrefix && string(headerNames[i]) == "base.cl") { +// auto pos = processedSrc.find("KERNEL"); +// if (pos != string::npos) { +// string ctx = processedSrc.substr(pos > 20 ? pos-20 : 0, 120); +// fprintf(stderr, "base.cl KERNEL context: [%s]\n", ctx.c_str()); +// } +// } +// } + nvrtcHeaders.emplace_back(headerNames[i], processedSrc); + } + } + + // Preprocess the main source + string processedSource = NvrtcProgram::preprocessOpenCL(prog->source); + + // Prepend opencl_compat.cuh include if not already there + if (processedSource.find("opencl_compat.cuh") == string::npos) { + processedSource = "#include \"opencl_compat.cuh\"\n" + processedSource; + } + + // Debug: dump preprocessed source when PRPLL_DUMP_PTX is set + { + static const char* dumpPrefix = getenv("PRPLL_DUMP_PTX"); + if (dumpPrefix) { + static int srcCount = 0; + char fname[512]; + snprintf(fname, sizeof(fname), "%s_src_%d.cu", dumpPrefix, srcCount++); + FILE* f = fopen(fname, "w"); + if (f) { + fwrite(processedSource.c_str(), 1, processedSource.size(), f); + fclose(f); + fprintf(stderr, "Source dumped to %s (%zu bytes)\n", fname, processedSource.size()); + } + } + } + + // Store preprocessed source for __launch_bounds__ parsing in clCreateKernel + prog->preprocessedSource = processedSource; + for (auto& [name, src] : nvrtcHeaders) { + prog->preprocessedSource += "\n"; + prog->preprocessedSource += src; + } + + // Debug: dump NVRTC options when dumping PTX + { + static const char* dumpPrefix = getenv("PRPLL_DUMP_PTX"); + static std::atomic dumpedOnce{false}; + if (dumpPrefix && !dumpedOnce.exchange(true)) { + fprintf(stderr, "NVRTC options (%zu):\n", nvrtcOpts.size()); + for (auto& o : nvrtcOpts) fprintf(stderr, " %s\n", o.c_str()); + } + } + + try { + auto images = NvrtcProgram::compileImages(processedSource, "prpll_kernel.cu", nvrtcOpts, nvrtcHeaders); + prog->ptx = std::move(images.ptx); + // PRPLL_PTX_ONLY=1 keeps the driver's JIT path, for comparing the two. + static const bool ptxOnly = getenv("PRPLL_PTX_ONLY") != nullptr; + prog->cubin = ptxOnly ? string{} : std::move(images.cubin); + prog->compiled = true; + prog->buildLog.clear(); + } catch (const exception& e) { + prog->buildLog = e.what(); + prog->compiled = false; + fprintf(stderr, "NVRTC COMPILE FAILED: %s\n", e.what()); + // Dump the full preprocessed source for debugging + { + char fname[64]; + static std::atomic failCount{0}; + snprintf(fname, sizeof(fname), "prpll_fail_%d.cu", failCount++); + FILE* f = fopen(fname, "w"); + if (f) { + fprintf(f, "// === FAILED Main source ===\n%s\n", processedSource.c_str()); + for (auto& [name, src] : nvrtcHeaders) { + fprintf(f, "\n// === Header: %s (%zu bytes) ===\n%s\n", name.c_str(), src.size(), src.c_str()); + } + fclose(f); + fprintf(stderr, "Dumped failed source to %s\n", fname); + } + } + return CL_COMPILE_PROGRAM_FAILURE; + } + + // --maxrregcount is supposed to be applied by NVRTC's own ptxas when it builds the CUBIN, with the PTX + // fallback getting a spliced-in .maxnreg directive for the driver JIT path instead. In practice (verified + // against this toolkit/driver: CUDA 13.0, NVRTC accepts --maxrregcount without warning but the resulting + // CUBIN's register usage is unaffected by it -- e.g. requesting 24 regs for carryFused still yields 64). + // The .maxnreg-spliced PTX + driver JIT does honor the cap correctly. So for any kernel that asked for a + // register cap, drop the (silently non-compliant) CUBIN and force the PTX path, which is known to work. + // Kernels with no cap requested keep using the CUBIN fast path this shim was added for. + + if (maxregcount) { + string const maxntidPattern = ".maxntid "; + string const maxnregPattern = ".maxnreg " + to_string(maxregcount) + "\n"; + for (size_t startpos = 0; ; ) { + size_t const pos = prog->ptx.find(maxntidPattern, startpos); + if (pos == string::npos) break; + prog->ptx.insert(pos, maxnregPattern); + startpos = pos + 20; + } + prog->cubin.clear(); + } + + return CL_SUCCESS; +} + +cl_program clLinkProgram(cl_context ctx, unsigned /*nDevices*/, const cl_device_id*, + const char* /*options*/, unsigned nProgs, const cl_program* progs, + void (*)(cl_program, void*), void*, int* err) { + ensureContextCurrent(); + // In CUDA, compilation produces PTX directly — no separate link step needed. + // Just load the PTX as a CUmodule. + if (!progs || nProgs == 0 || !progs[0] || !progs[0]->compiled) { + if (err) *err = CL_LINK_PROGRAM_FAILURE; + return nullptr; + } + + auto* linked = new _cl_program; + linked->context = ctx; + linked->ptx = progs[0]->ptx; + linked->cubin = progs[0]->cubin; + linked->compiled = true; + // Carry preprocessed source through for KERNEL(N) parsing in clCreateKernel + for (unsigned i = 0; i < nProgs; ++i) { + if (progs[i] && !progs[i]->preprocessedSource.empty()) { + linked->preprocessedSource += progs[i]->preprocessedSource; + linked->preprocessedSource += "\n"; + } + } + + // Use cuModuleLoadDataEx with error log to see JIT errors + char jitErrorLog[8192] = {}; + char jitInfoLog[4096] = {}; + CUjit_option jitOpts[] = { + CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES, CU_JIT_ERROR_LOG_BUFFER, + CU_JIT_INFO_LOG_BUFFER_SIZE_BYTES, CU_JIT_INFO_LOG_BUFFER + }; + void* jitOptVals[] = { + (void*)(size_t)sizeof(jitErrorLog), (void*)jitErrorLog, + (void*)(size_t)sizeof(jitInfoLog), (void*)jitInfoLog + }; + CUresult const r = loadModule(linked, 4, jitOpts, jitOptVals); + if (r != CUDA_SUCCESS) { + const char* errName = nullptr; + cuGetErrorName(r, &errName); + fprintf(stderr, "cuModuleLoadData FAILED: %s (%d)\n", errName ? errName : "?", (int)r); + if (jitErrorLog[0]) fprintf(stderr, "JIT error log: %s\n", jitErrorLog); + if (jitInfoLog[0]) fprintf(stderr, "JIT info log: %s\n", jitInfoLog); + // Dump first 2000 chars of PTX for debugging + fprintf(stderr, "PTX size: %zu bytes\n", linked->ptx.size()); + // Dump full PTX to file + { + FILE* ptxFile = fopen("failed_ptx.ptx", "w"); + if (ptxFile) { + fwrite(linked->ptx.c_str(), 1, linked->ptx.size(), ptxFile); + fclose(ptxFile); + fprintf(stderr, "Dumped failed PTX to failed_ptx.ptx\n"); + } + } + delete linked; + if (err) *err = CL_LINK_PROGRAM_FAILURE; + return nullptr; + } + linked->moduleLoaded = true; + moduleRetain(linked->module); // program owns one reference + + // Dump PTX to file when PRPLL_DUMP_PTX is set (e.g., PRPLL_DUMP_PTX=kernel) + // Creates files like kernel_0.ptx, kernel_1.ptx, etc. + { + static const char* dumpPrefix = getenv("PRPLL_DUMP_PTX"); + if (dumpPrefix) { + static int ptxCount = 0; + char fname[512]; + snprintf(fname, sizeof(fname), "%s_%d.ptx", dumpPrefix, ptxCount++); + FILE* f = fopen(fname, "w"); + if (f) { + fwrite(linked->ptx.c_str(), 1, linked->ptx.size(), f); + fclose(f); + fprintf(stderr, "PTX dumped to %s (%zu bytes)\n", fname, linked->ptx.size()); + } + } + } + + if (err) *err = CL_SUCCESS; + return linked; +} + +int clBuildProgram(cl_program prog, unsigned nDevices, const cl_device_id* devices, + const char* options, void (*)(cl_program, void*), void*) { + // If the program was loaded from binary (cached PTX) and already has a module, + // skip recompilation — the module is already JIT'd and ready. + if (prog && prog->moduleLoaded) { + return CL_SUCCESS; + } + + // clBuildProgram = compile + link in one step + int const err = clCompileProgram(prog, nDevices, devices, options, 0, nullptr, nullptr, nullptr, nullptr); + if (err != CL_SUCCESS) return err; + + CUresult const r = loadModule(prog, 0, nullptr, nullptr); + if (r != CUDA_SUCCESS) return CL_BUILD_PROGRAM_FAILURE; + prog->moduleLoaded = true; + moduleRetain(prog->module); // program owns one reference + return CL_SUCCESS; +} + +int clGetProgramBuildInfo(cl_program prog, cl_device_id, cl_program_build_info info, + size_t size, void* value, size_t* sizeRet) { + if (!prog) return CL_INVALID_PROGRAM; + if (info == CL_PROGRAM_BUILD_LOG) { + // This program's own log — a process-wide "last log" would report the + // diagnostics of whichever compile finished last on another thread. + size_t const len = prog->buildLog.size() + 1; + if (sizeRet) *sizeRet = len; + if (value && size >= len) { + memcpy(value, prog->buildLog.c_str(), len); + } + } + return CL_SUCCESS; +} + +int clGetProgramInfo(cl_program prog, cl_program_info info, size_t size, void* value, size_t* sizeRet) { + if (!prog) return CL_INVALID_PROGRAM; + // The cache blob — PTX, then the CUBIN behind CUBIN_MARKER when one + // loaded; see clCreateProgramWithBinary for the reading side. + if (info == CL_PROGRAM_BINARY_SIZES) { + size_t blobSize = cacheBlob(prog).size(); + if (sizeRet) *sizeRet = sizeof(size_t); + if (value && size >= sizeof(size_t)) memcpy(value, &blobSize, sizeof(size_t)); + } else if (info == CL_PROGRAM_BINARIES) { + if (sizeRet) *sizeRet = sizeof(unsigned char*); + if (value && size >= sizeof(unsigned char*)) { + auto* const* ptrs = (unsigned char**)value; + if (ptrs[0]) { + string const blob = cacheBlob(prog); + memcpy(ptrs[0], blob.data(), blob.size()); + } + } + } + return CL_SUCCESS; +} + +// ---- Kernel ---- + +cl_kernel clCreateKernel(cl_program prog, const char* name, int* err) { + ensureContextCurrent(); + if (!prog || !prog->moduleLoaded) { + if (err) *err = CL_INVALID_PROGRAM; + return nullptr; + } + auto* k = new _cl_kernel; + k->name = name; + k->parentModule = prog->module; + CUresult const r = cuModuleGetFunction(&k->func, prog->module, name); + if (r != CUDA_SUCCESS) { + fprintf(stderr, "cuModuleGetFunction('%s') failed: %d, moduleLoaded=%d, module=%p\n", + name, (int)r, prog->moduleLoaded, (void*)prog->module); + delete k; // never retained the module, so nothing to release + if (err) *err = CL_INVALID_KERNEL_NAME; + return nullptr; + } + moduleRetain(k->parentModule); // kernel keeps the module alive past clReleaseProgram + + // Shared memory carveout: default adaptive carveout is optimal for mixed kernel workloads. + // Remainder of memory will be used for L1 cache. +// This was not measurably faster on my 570Ti Laptop. We should try it on other GPUs. +// If MULTI_Q is set, we might need to set all the middleIn, tailSquare, and middleOut kernels to use the same carveout value (which +// negates the primary benefit since middleIn and middleOut are the kernels using low carveouts). For now, disable the capability by default. +if (getenv("TRY_LDS_CARVEOUT")) + { + int numRegs = 0, shmem = 0, maxThreads = 0, maxShared = 0; + cuFuncGetAttribute(&numRegs, CU_FUNC_ATTRIBUTE_NUM_REGS, k->func); + cuFuncGetAttribute(&shmem, CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, k->func); + cuFuncGetAttribute(&maxThreads, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, k->func); + cuDeviceGetAttribute(&maxShared, CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_MULTIPROCESSOR, prog->context->dev); + + int max_occupancy = 65536 / (numRegs * maxThreads); // Maximum occupancy due to register pressure + int carveout = 100 * (max_occupancy * shmem) / maxShared; // Percent of shared memory needed for max occupancy + if (carveout > 100) carveout = 100; + cuFuncSetAttribute(k->func, CU_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT, carveout); + } + + // Log register and shared memory usage per kernel when PRPLL_DUMP_PTX is set + { + static const char* dumpPrefix = getenv("PRPLL_DUMP_PTX"); + if (prpll_verbose || dumpPrefix) { + int numRegs = 0, shmem = 0, localmem = 0, maxThreads = 0; + cuFuncGetAttribute(&numRegs, CU_FUNC_ATTRIBUTE_NUM_REGS, k->func); + cuFuncGetAttribute(&shmem, CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, k->func); + cuFuncGetAttribute(&localmem, CU_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES , k->func); + cuFuncGetAttribute(&maxThreads, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, k->func); + fprintf(stderr, " %-25s: %3d regs, %5d shmem, %d localmem, maxThreads=%d\n", name, numRegs, shmem, localmem, maxThreads); + // Also write to file since WSL2+CUDA swallows stderr + if (dumpPrefix) { + FILE* regLog = fopen("kernel_regs.log", "a"); + if (regLog) { fprintf(regLog, " %-25s: %3d regs, %5d shmem, %d localmem, maxThreads=%d\n", name, numRegs, shmem, localmem, maxThreads); fclose(regLog); } + } + } + } + + // Parse .maxntid from PTX to get __launch_bounds__ value. + // PTX pattern: .visible .entry (...)\n.maxntid N, 1, 1 + k->reqWorkGroupSize = 256; // fallback + { + const string& ptx = prog->ptx; + string const entryPattern = ".entry " + string(name) + "("; + size_t const pos = ptx.find(entryPattern); + if (pos != string::npos) { + // Found the kernel entry. Its text runs to the next .entry or .func. + size_t searchEnd = min(ptx.find(".entry ", pos + 1), ptx.find(".func ", pos + 1)); + if (searchEnd == string::npos) searchEnd = ptx.size(); + string const maxntidPattern = ".maxntid "; + size_t const mpos = ptx.find(maxntidPattern, pos); + if (mpos != string::npos && mpos < searchEnd) { + int const val = atoi(ptx.c_str() + mpos + maxntidPattern.size()); + if (val > 0) { + k->reqWorkGroupSize = val; + } + } + // A kernel that waits for its predecessor (griddepcontrol.wait — + // compiled in by -use PDL=1 on sm_90+) is launched with programmatic + // stream serialization: it may begin while the predecessor's tail + // still runs, and its wait holds every read until the predecessor + // has completed. A kernel without the wait keeps an ordinary launch: + // it may follow one that triggered early, and nothing else would + // order the two. Read from the compiled code, this holds for a + // cached program as much as a fresh one. + k->pdl = ptx.find("griddepcontrol.wait", pos) < searchEnd; + } + } + + if (err) *err = CL_SUCCESS; + return k; +} + +int clReleaseKernel(cl_kernel k) { + if (k) { + moduleRelease(k->parentModule); + delete k; + } + return CL_SUCCESS; +} + +int clSetKernelArg(cl_kernel k, unsigned pos, size_t size, const void* value) { + if (!k) return CL_INVALID_KERNEL; + + // Detect cl_mem buffer arguments and convert to CUdeviceptr. + // In OpenCL, buffer args are set with clSetKernelArg(k, i, sizeof(cl_mem), &memobj). + // sizeof(cl_mem) == sizeof(void*) == 8 on 64-bit. The value at 'value' is a cl_mem pointer. + // We need to store the CUdeviceptr (GPU address) instead of the cl_mem (host pointer). + if (size == sizeof(cl_mem) && value) { + cl_mem mem = *(cl_mem*)value; + if (mem) { + // Look the buffer up and copy its device pointer under the lock: another worker may be + // creating or releasing buffers concurrently, and a release deletes the _cl_mem. + CUdeviceptr devPtr = 0; + bool isBuffer = false; + { + std::lock_guard lock(g_allocatedBuffersMutex); + if (g_allocatedBuffers.contains(mem)) { + isBuffer = true; + devPtr = mem->ptr; + } + } + if (isBuffer) { + k->setArg(pos, sizeof(CUdeviceptr), &devPtr); + return CL_SUCCESS; + } + } + // NULL cl_mem → pass a null device pointer + if (!mem) { + CUdeviceptr devPtr = 0; + k->setArg(pos, sizeof(CUdeviceptr), &devPtr); + return CL_SUCCESS; + } + } + + k->setArg(pos, size, value); + return CL_SUCCESS; +} + +// ---- Buffer ---- + +cl_mem clCreateBuffer(cl_context /*ctx*/, cl_mem_flags flags, size_t size, void* hostPtr, int* err) { + auto* buf = new _cl_mem; + buf->size = size; + ensureContextCurrent(); + CUresult const r = cuMemAlloc(&buf->ptr, size); + if (r != CUDA_SUCCESS) { + delete buf; + if (err) *err = CL_MEM_OBJECT_ALLOCATION_FAILURE; + return nullptr; + } + // Handle CL_MEM_COPY_HOST_PTR + if ((flags & CL_MEM_COPY_HOST_PTR) && hostPtr) { + cuMemcpyHtoD(buf->ptr, hostPtr, size); + } + { + std::lock_guard lock(g_allocatedBuffersMutex); + g_allocatedBuffers.insert(buf); + } + if (err) *err = CL_SUCCESS; + return buf; +} + +int clReleaseMemObject(cl_mem buf) { + if (buf) { + ensureContextCurrent(); + { + std::lock_guard lock(g_allocatedBuffersMutex); + g_allocatedBuffers.erase(buf); + } + cuMemFree(buf->ptr); + delete buf; + } + return CL_SUCCESS; +} + +// ---- Command Queue ---- + +cl_command_queue clCreateCommandQueueWithProperties(cl_context ctx, cl_device_id /*dev*/, + const cl_queue_properties* props, int* err) { + auto* q = new _cl_command_queue; + q->context = ctx; + q->profiling = false; + + // Check for profiling flag + if (props) { + for (int i = 0; props[i]; i += 2) { + if (props[i] == CL_QUEUE_PROPERTIES && (props[i+1] & CL_QUEUE_PROFILING_ENABLE)) { + q->profiling = true; + } + } + } + + // Make sure context is current + cuCtxSetCurrent(ctx->ctx); + CUresult const r = cuStreamCreate(&q->stream, CU_STREAM_NON_BLOCKING); + if (r != CUDA_SUCCESS) { + delete q; + if (err) *err = CL_OUT_OF_RESOURCES; + return nullptr; + } + if (err) *err = CL_SUCCESS; + return q; +} + +// ---- Enqueue operations ---- + +// One launch for the three paths below. For a kernel whose code waits on +// its predecessor (k->pdl, see clCreateKernel; CUDA 12+, where +// cuLaunchKernelEx exists), the launch carries +// CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION: this kernel may +// begin once the previous one in the stream signals +// griddepcontrol.launch_dependents (that one's tail overlapping this one's +// prologue), and this kernel's griddepcontrol.wait holds its reads until +// the previous one has completed. Every other kernel is launched as before. +static CUresult launchKernel(cl_kernel k, unsigned numBlocksX, unsigned numBlocksY, unsigned lsX, unsigned lsY, + CUstream stream, void** argPtrs) { +#if CUDA_VERSION >= 12000 + if (k->pdl) { + CUlaunchAttribute attr{}; + attr.id = CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION; + attr.value.programmaticStreamSerializationAllowed = 1; + CUlaunchConfig config{}; + config.gridDimX = numBlocksX; + config.gridDimY = numBlocksY; + config.gridDimZ = 1; + config.blockDimX = lsX; + config.blockDimY = lsY; + config.blockDimZ = 1; + config.sharedMemBytes = 0; + config.hStream = stream; + config.attrs = &attr; + config.numAttrs = 1; + return cuLaunchKernelEx(&config, k->func, argPtrs, nullptr); + } +#endif + return cuLaunchKernel(k->func, numBlocksX, numBlocksY, 1, lsX, lsY, 1, 0, stream, argPtrs, nullptr); +} + +int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned workDim, + const size_t* /*globalOffset*/, const size_t* globalSize, + const size_t* localSize, unsigned /*nWaits*/, + const cl_event* /*waits*/, cl_event* event) { + if (!q || !k) return CL_INVALID_VALUE; + ensureContextCurrent(); + + unsigned int const gsX = u32(globalSize[0]); + unsigned int const lsX = u32(localSize ? localSize[0] : 256); + unsigned int const numBlocksX = (gsX + lsX - 1) / lsX; + unsigned int const gsY = u32((workDim > 1) ? globalSize[1] : 1); + unsigned int const lsY = u32((workDim > 1) ? localSize[1] : 1); + unsigned int const numBlocksY = (gsY + lsY - 1) / lsY; + + // Build args array + void* argPtrs[_cl_kernel::MAX_ARGS]; + k->buildArgPointers(argPtrs); + + // Env-gated kernel profiling (PRPLL_PROFILE=1) — takes priority over event profiling + static bool const doProfile = (getenv("PRPLL_PROFILE") != nullptr); + + // Event handling (skipped when env profiler is active) + if (!doProfile && event && q->profiling) { + auto* ev = new _cl_event; + cuEventCreate(&ev->start, CU_EVENT_DEFAULT); + cuEventCreate(&ev->end, CU_EVENT_DEFAULT); + ev->hasTimings = true; + ev->commandType = CL_COMMAND_NDRANGE_KERNEL; + cuEventRecord(ev->start, q->stream); + CUresult const r = launchKernel(k, numBlocksX, numBlocksY, lsX, lsY, q->stream, argPtrs); + cuEventRecord(ev->end, q->stream); + *event = ev; + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; + } + if (doProfile) { + static std::map kTime; + static std::map kCount; + static std::map kRegs; + static std::map kShmem; + static int totalLaunches = 0; + static CUevent pStart = nullptr, pEnd = nullptr; + if (!pStart) { cuEventCreate(&pStart, CU_EVENT_DEFAULT); cuEventCreate(&pEnd, CU_EVENT_DEFAULT); } + + cuEventRecord(pStart, q->stream); + CUresult const r = launchKernel(k, numBlocksX, numBlocksY, lsX, lsY, q->stream, argPtrs); + cuEventRecord(pEnd, q->stream); + cuEventSynchronize(pEnd); + float ms = 0; + cuEventElapsedTime(&ms, pStart, pEnd); + kTime[k->name] += ms; + kCount[k->name]++; + if (!kRegs.contains(k->name)) { + int regs = 0, shmem = 0; + cuFuncGetAttribute(®s, CU_FUNC_ATTRIBUTE_NUM_REGS, k->func); + cuFuncGetAttribute(&shmem, CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, k->func); + kRegs[k->name] = regs; + kShmem[k->name] = shmem; + } + totalLaunches++; + + if (totalLaunches % 10000 == 0) { + fprintf(stderr, "\n=== NTT KERNEL PROFILE (%d launches) ===\n", totalLaunches); + std::vector> sorted; + double totalMs = 0; + for (auto& [n, t] : kTime) { sorted.emplace_back(t, n); totalMs += t; } + std::sort(sorted.rbegin(), sorted.rend()); + for (auto& [t, n] : sorted) { + fprintf(stderr, " %6.1f ms (%5.1f%%) %5d calls avg %.3f ms regs=%d shmem=%d %s\n", + t, 100.0*t/totalMs, kCount[n], t/kCount[n], kRegs[n], kShmem[n], n.c_str()); + } + fprintf(stderr, " TOTAL: %.1f ms\n===\n\n", totalMs); + } + if (event) *event = nullptr; + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; + } + + CUresult const r = launchKernel(k, numBlocksX, numBlocksY, lsX, lsY, q->stream, argPtrs); + if (r != CUDA_SUCCESS) { + const char* errName = nullptr; + cuGetErrorName(r, &errName); + fprintf(stderr, "cuLaunchKernel FAILED for '%s': %s (%d)\n", k->name.c_str(), errName ? errName : "?", (int)r); + } + + if (event) *event = nullptr; + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; +} + +int clEnqueueReadBuffer(cl_command_queue q, cl_mem buf, cl_bool blocking, + size_t offset, size_t size, void* ptr, + unsigned /*nWaits*/, const cl_event* /*waits*/, cl_event* event) { + // Must use stream-ordered copy because the stream was created with CU_STREAM_NON_BLOCKING, + // which means cuMemcpyDtoH (NULL stream) won't wait for pending kernels on this stream. + CUresult r = cuMemcpyDtoHAsync(ptr, buf->ptr + offset, size, q->stream); + if (r == CUDA_SUCCESS && blocking) { + r = cuStreamSynchronize(q->stream); + } + if (event) *event = nullptr; + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; +} + +int clEnqueueWriteBuffer(cl_command_queue q, cl_mem buf, cl_bool blocking, + size_t offset, size_t size, const void* ptr, + unsigned /*nWaits*/, const cl_event* /*waits*/, cl_event* event) { + // Must use stream-ordered copy (same reason as clEnqueueReadBuffer above) + CUresult r = cuMemcpyHtoDAsync(buf->ptr + offset, ptr, size, q->stream); + if (r == CUDA_SUCCESS && blocking) { + r = cuStreamSynchronize(q->stream); + } + if (event) *event = nullptr; + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; +} + +int clEnqueueCopyBuffer(cl_command_queue q, cl_mem src, cl_mem dst, + size_t srcOffset, size_t dstOffset, size_t size, + unsigned /*nWaits*/, const cl_event* /*waits*/, cl_event* event) { + CUresult const r = cuMemcpyDtoDAsync(dst->ptr + dstOffset, src->ptr + srcOffset, size, q->stream); + if (event) *event = nullptr; + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; +} + +int clEnqueueFillBuffer(cl_command_queue q, cl_mem buf, const void* pattern, + size_t patternSize, size_t offset, size_t size, + unsigned /*nWaits*/, const cl_event* /*waits*/, cl_event* event) { + CUresult r; + if (patternSize == 1) { + unsigned char val; + memcpy(&val, pattern, 1); + r = cuMemsetD8Async(buf->ptr + offset, val, size, q->stream); + } else if (patternSize == 2) { + unsigned short val; + memcpy(&val, pattern, 2); + r = cuMemsetD16Async(buf->ptr + offset, val, size / 2, q->stream); + } else if (patternSize == 4) { + unsigned int val; + memcpy(&val, pattern, 4); + r = cuMemsetD32Async(buf->ptr + offset, val, size / 4, q->stream); + } else if (patternSize % 4 == 0 && size % patternSize == 0) { + // A pattern of N 32-bit words (Buffer//: N == 2): one + // strided memset per word writes word i of every pattern-sized slot — + // pitch = the pattern, width = one element, height = the slot count. + // Two driver calls instead of a host-side pattern buffer and a copy. + r = CUDA_SUCCESS; + size_t const slots = size / patternSize; + for (size_t i = 0; i < patternSize / 4 && r == CUDA_SUCCESS; ++i) { + unsigned int word; + memcpy(&word, static_cast(pattern) + 4 * i, 4); + r = cuMemsetD2D32Async(buf->ptr + offset + 4 * i, patternSize, word, 1, slots, q->stream); + } + } else { + // Neither a memset width nor a whole number of word-multiple slots: the + // OpenCL contract has no answer here, and a silent zero-fill (the old + // fallback) would hand the caller data it did not ask for. + if (event) *event = nullptr; + return CL_INVALID_VALUE; + } + if (event) *event = nullptr; + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; +} + +int clEnqueueMarkerWithWaitList(cl_command_queue q, unsigned nWaits, const cl_event* waits, cl_event* event) { + if (nWaits) { + for (unsigned int i = 0; i < nWaits; ++i) { + cuStreamWaitEvent(q->stream, waits[i]->end, 0); + } + } + if (event) { + auto* ev = new _cl_event; + cuEventCreate(&ev->end, CU_EVENT_DISABLE_TIMING); + cuEventRecord(ev->end, q->stream); + ev->commandType = CL_COMMAND_MARKER; + *event = ev; + } + return CL_SUCCESS; +} + +int clFlush(cl_command_queue /*q*/) { + // CUDA streams auto-flush; no-op + return CL_SUCCESS; +} + +int clFinish(cl_command_queue q) { + if (q) cuStreamSynchronize(q->stream); + return CL_SUCCESS; +} + +// ---- Events ---- + +int clReleaseEvent(cl_event ev) { + delete ev; + return CL_SUCCESS; +} + +int clWaitForEvents(unsigned n, const cl_event* events) { + for (unsigned i = 0; i < n; i++) { + if (events[i] && events[i]->end) { + cuEventSynchronize(events[i]->end); + } + } + return CL_SUCCESS; +} + +int clGetEventInfo(cl_event ev, cl_event_info info, size_t size, void* value, size_t* sizeRet) { + if (!ev) return CL_INVALID_VALUE; + if (info == CL_EVENT_COMMAND_EXECUTION_STATUS) { + int status = CL_COMPLETE; + if (ev->end) { + CUresult const r = cuEventQuery(ev->end); + if (r == CUDA_ERROR_NOT_READY) status = CL_RUNNING; + } + if (sizeRet) *sizeRet = sizeof(int); + if (value && size >= sizeof(int)) memcpy(value, &status, sizeof(int)); + } else if (info == CL_EVENT_COMMAND_TYPE) { + u32 type = ev->commandType; + if (sizeRet) *sizeRet = sizeof(u32); + if (value && size >= sizeof(u32)) memcpy(value, &type, sizeof(u32)); + } + return CL_SUCCESS; +} + +int clGetEventProfilingInfo(cl_event ev, cl_profiling_info info, size_t size, void* value, size_t* sizeRet) { + if (!ev || !ev->hasTimings) return CL_PROFILING_INFO_NOT_AVAILABLE; + + // CUDA events give elapsed time between two events, not absolute timestamps. + // We fake absolute timestamps by using a base time. + u64 timestamp = 0; + if (info == CL_PROFILING_COMMAND_START || info == CL_PROFILING_COMMAND_SUBMIT || + info == CL_PROFILING_COMMAND_QUEUED) { + timestamp = 0; // Relative start + } else if (info == CL_PROFILING_COMMAND_END || info == CL_PROFILING_COMMAND_COMPLETE) { + float ms = 0; + cuEventElapsedTime(&ms, ev->start, ev->end); + timestamp = (u64)(ms * 1e6); // Convert ms to ns + } + + if (sizeRet) *sizeRet = sizeof(u64); + if (value && size >= sizeof(u64)) memcpy(value, ×tamp, sizeof(u64)); + return CL_SUCCESS; +} + +// ---- Device info ---- + +int clGetDeviceInfo(cl_device_id dev, cl_device_info info, size_t size, void* value, size_t* sizeRet) { + if (!dev) return CL_INVALID_DEVICE; + + switch (info) { + case CL_DEVICE_NAME: { + char name[256]; + cuDeviceGetName(name, sizeof(name), dev->dev); + size_t const len = strlen(name) + 1; + if (sizeRet) *sizeRet = len; + if (value && size >= len) memcpy(value, name, len); + break; + } + case CL_DEVICE_VENDOR_ID: { + // Return NVIDIA vendor ID + unsigned int vid = 0x10DE; + if (sizeRet) *sizeRet = sizeof(vid); + if (value && size >= sizeof(vid)) memcpy(value, &vid, sizeof(vid)); + break; + } + case CL_DEVICE_MAX_COMPUTE_UNITS: { + int units = 0; + cuDeviceGetAttribute(&units, CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, dev->dev); + unsigned int val = units; + if (sizeRet) *sizeRet = sizeof(val); + if (value && size >= sizeof(val)) memcpy(value, &val, sizeof(val)); + break; + } + case CL_DEVICE_MAX_CLOCK_FREQUENCY: { + int mhz = 0; + cuDeviceGetAttribute(&mhz, CU_DEVICE_ATTRIBUTE_CLOCK_RATE, dev->dev); + unsigned int val = mhz / 1000; // kHz to MHz + if (sizeRet) *sizeRet = sizeof(val); + if (value && size >= sizeof(val)) memcpy(value, &val, sizeof(val)); + break; + } + case CL_DEVICE_GLOBAL_MEM_SIZE: { + size_t mem = 0; + cuDeviceTotalMem(&mem, dev->dev); + u64 val = mem; + if (sizeRet) *sizeRet = sizeof(val); + if (value && size >= sizeof(val)) memcpy(value, &val, sizeof(val)); + break; + } + case CL_DRIVER_VERSION: + case CL_DEVICE_VERSION: { + int ver = 0; + cuDriverGetVersion(&ver); + char verStr[64]; + snprintf(verStr, sizeof(verStr), "CUDA %d.%d", ver / 1000, (ver % 1000) / 10); + size_t const len = strlen(verStr) + 1; + if (sizeRet) *sizeRet = len; + if (value && size >= len) memcpy(value, verStr, len); + break; + } + case CL_DEVICE_ERROR_CORRECTION_SUPPORT: { + int ecc = 0; + cuDeviceGetAttribute(&ecc, CU_DEVICE_ATTRIBUTE_ECC_ENABLED, dev->dev); + cl_bool val = ecc; + if (sizeRet) *sizeRet = sizeof(val); + if (value && size >= sizeof(val)) memcpy(value, &val, sizeof(val)); + break; + } + case CL_DEVICE_BUILT_IN_KERNELS: { + const char* empty = ""; + if (sizeRet) *sizeRet = 1; + if (value && size >= 1) memcpy(value, empty, 1); + break; + } + case CL_DEVICE_TOPOLOGY_AMD: { + // The device's PCIe position in the AMD extension's shape, so `-pci + // ` selects a CUDA device the way it selects an AMD + // one (gpuid.cpp getPosFromBdf → clwrap.cpp getBdfFromDevice). The + // enumeration ordinal `-device N` names is not stable — CUDA orders + // FASTEST_FIRST unless CUDA_DEVICE_ORDER says otherwise, and a second + // card or a driver update can renumber — while the bus id is what + // nvidia-smi and every launcher already know the card by. + char bdf[32] = {0}; + if (cuDeviceGetPCIBusId(bdf, sizeof(bdf), dev->dev) != CUDA_SUCCESS) return CL_INVALID_VALUE; + // "0000:6a:00.0" — domain:bus:device.function; tolerate a missing domain. + unsigned domain = 0, bus = 0, device = 0, function = 0; + if (sscanf(bdf, "%x:%x:%x.%x", &domain, &bus, &device, &function) != 4 && + sscanf(bdf, "%x:%x.%x", &bus, &device, &function) != 3) { + return CL_INVALID_VALUE; + } + cl_device_topology_amd top{}; + top.pcie.type = CL_DEVICE_TOPOLOGY_TYPE_PCIE_AMD; + top.pcie.bus = (char) bus; + top.pcie.device = (char) device; + top.pcie.function = (char) function; + if (sizeRet) *sizeRet = sizeof(top); + if (value && size >= sizeof(top)) memcpy(value, &top, sizeof(top)); + break; + } + case CL_DEVICE_BOARD_NAME_AMD: + case CL_DEVICE_PCIE_ID_AMD: { + // AMD-specific queries with no CUDA counterpart — return failure + return CL_INVALID_VALUE; + } + case CL_DEVICE_GLOBAL_FREE_MEMORY_AMD: { + size_t freeMem = 0, totalMem = 0; + cuMemGetInfo(&freeMem, &totalMem); + // AMD returns in KB + u64 freeKB = freeMem / 1024; + if (sizeRet) *sizeRet = sizeof(freeKB); + if (value && size >= sizeof(freeKB)) memcpy(value, &freeKB, sizeof(freeKB)); + break; + } + case CL_DEVICE_COMPUTE_CAPABILITY_MAJOR_NV: { + int major = 0; + cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, dev->dev); + if (sizeRet) *sizeRet = sizeof(major); + if (value && size >= sizeof(major)) memcpy(value, &major, sizeof(major)); + break; + } + case CL_DEVICE_COMPUTE_CAPABILITY_MINOR_NV: { + int minor = 0; + cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, dev->dev); + if (sizeRet) *sizeRet = sizeof(minor); + if (value && size >= sizeof(minor)) memcpy(value, &minor, sizeof(minor)); + break; + } + default: + return CL_INVALID_VALUE; + } + return CL_SUCCESS; +} + +int clGetPlatformInfo(cl_platform_id, cl_device_info info, size_t size, void* value, size_t* sizeRet) { + if (info == CL_PLATFORM_VERSION) { + const char* ver = "CUDA (via PRPLL CUDA backend)"; + size_t const len = strlen(ver) + 1; + if (sizeRet) *sizeRet = len; + if (value && size >= len) memcpy(value, ver, len); + return CL_SUCCESS; + } + return CL_INVALID_VALUE; +} + +int clGetCommandQueueInfo(cl_command_queue q, cl_command_queue_info info, + size_t size, void* value, size_t* sizeRet) { + if (info == CL_QUEUE_CONTEXT) { + if (sizeRet) *sizeRet = sizeof(cl_context); + if (value && size >= sizeof(cl_context)) memcpy(value, &q->context, sizeof(cl_context)); + return CL_SUCCESS; + } + return CL_INVALID_VALUE; +} + +// ---- Kernel info ---- + +int clGetKernelInfo(cl_kernel k, cl_kernel_info info, size_t size, void* value, size_t* sizeRet) { + if (!k) return CL_INVALID_KERNEL; + if (info == CL_KERNEL_NUM_ARGS) { + int n = k->numArgs; + if (sizeRet) *sizeRet = sizeof(n); + if (value && size >= sizeof(n)) memcpy(value, &n, sizeof(n)); + } else if (info == CL_KERNEL_ATTRIBUTES) { + const char* empty = ""; + if (sizeRet) *sizeRet = 1; + if (value && size >= 1) memcpy(value, empty, 1); + } + return CL_SUCCESS; +} + +int clGetKernelArgInfo(cl_kernel /*k*/, unsigned pos, cl_kernel_arg_info info, + size_t size, void* value, size_t* sizeRet) { + if (info == CL_KERNEL_ARG_NAME) { + char name[32]; + snprintf(name, sizeof(name), "arg%u", pos); + size_t const len = strlen(name) + 1; + if (sizeRet) *sizeRet = len; + if (value && size >= len) memcpy(value, name, len); + } + return CL_SUCCESS; +} + +int clGetKernelWorkGroupInfo(cl_kernel k, cl_device_id /*dev*/, cl_kernel_work_group_info info, + size_t size, void* value, size_t* sizeRet) { + if (!k) return CL_INVALID_KERNEL; + ensureContextCurrent(); + if (info == CL_KERNEL_COMPILE_WORK_GROUP_SIZE) { + // Return the __launch_bounds__ value parsed from source during clCreateKernel. + // This matches OpenCL's CL_KERNEL_COMPILE_WORK_GROUP_SIZE which returns reqd_work_group_size. + // Previously we used CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK which returns the hardware max + // based on register/shared memory usage — NOT the declared group size. This caused wrong + // block sizes for every kernel (e.g., tailMul expected 64 threads but got 1024). + int const wgSize = k->reqWorkGroupSize > 0 ? k->reqWorkGroupSize : 256; + size_t wgs[3] = { (size_t)wgSize, 1, 1 }; + if (sizeRet) *sizeRet = sizeof(wgs); + if (value && size >= sizeof(wgs)) memcpy(value, wgs, sizeof(wgs)); + } + return CL_SUCCESS; +} + +// ---- SVM (not used but must exist) ---- + +void* clSVMAlloc(cl_context, cl_svm_mem_flags, size_t size, unsigned) { + CUdeviceptr ptr; + ensureContextCurrent(); + cuMemAlloc(&ptr, size); + return (void*)(uintptr_t)ptr; +} + +void clSVMFree(cl_context, void* ptr) { + ensureContextCurrent(); + cuMemFree((CUdeviceptr)(uintptr_t)ptr); +} + +int clSetKernelArgSVMPointer(cl_kernel k, unsigned pos, const void* ptr) { + auto dp = (CUdeviceptr)(uintptr_t)ptr; + k->setArg(pos, sizeof(dp), &dp); + return CL_SUCCESS; +} + +} // extern "C" + +// C++ linkage — must be outside the extern "C" block above. + +// Set L1 cache configuration +void cudaSetL1Config(int x) { + ensureContextCurrent(); + cuCtxSetCacheConfig (x == 0 ? CU_FUNC_CACHE_PREFER_NONE : // no preference for shared memory or L1 (default) + x == 1 ? CU_FUNC_CACHE_PREFER_SHARED : // prefer larger shared memory and smaller L1 cache + x == 2 ? CU_FUNC_CACHE_PREFER_L1 : // prefer larger L1 cache and smaller shared memory + CU_FUNC_CACHE_PREFER_EQUAL); // prefer equal sized L1 cache and shared memory +} + +// Set L2 cache persistence for multiple read-only buffers on the given stream. +// Computes the minimum address span covering all buffers, then sets one access policy +// window with hitRatio sized so that only the actual buffer bytes get persisting treatment, +// not the gaps between non-contiguous allocations. +#if CUDA_VERSION >= 11000 +[[maybe_unused]] static void cudaSetL2Persistent(cl_command_queue q, const std::vector& buffers) { + if (!q) return; + + // Find address span and total data size + CUdeviceptr minAddr = ~(CUdeviceptr)0; + CUdeviceptr maxAddr = 0; + size_t totalDataBytes = 0; + + for (auto buf : buffers) { + if (!buf || buf->size == 0) continue; + CUdeviceptr const lo = buf->ptr; + CUdeviceptr const hi = buf->ptr + buf->size; + minAddr = std::min(lo, minAddr); + maxAddr = std::max(hi, maxAddr); + totalDataBytes += buf->size; + } + + if (totalDataBytes == 0 || maxAddr <= minAddr) return; + + auto spanBytes = (size_t)(maxAddr - minAddr); + + // Query the device's max access policy window size + int maxWindowSize = 0; + cuDeviceGetAttribute(&maxWindowSize, CU_DEVICE_ATTRIBUTE_MAX_ACCESS_POLICY_WINDOW_SIZE, 0); + if (maxWindowSize > 0 && std::cmp_greater(spanBytes, maxWindowSize)) { + fprintf(stderr, "L2 persist: span %zuMB exceeds max window %dMB, clamping\n", + spanBytes / (1024*1024), maxWindowSize / (1024*1024)); + spanBytes = maxWindowSize; + } + + // hitRatio = actual data / window span. This way only the real buffer data gets + // persisting treatment, and any gaps between allocations get streaming treatment. + float hitRatio = (float)totalDataBytes / (float)spanBytes; + hitRatio = std::min(hitRatio, 1.0f); + + CUstreamAttrValue attr; + memset(&attr, 0, sizeof(attr)); + attr.accessPolicyWindow.base_ptr = (void*)(uintptr_t)minAddr; + attr.accessPolicyWindow.num_bytes = spanBytes; + attr.accessPolicyWindow.hitRatio = hitRatio; + attr.accessPolicyWindow.hitProp = CU_ACCESS_PROPERTY_PERSISTING; + attr.accessPolicyWindow.missProp = CU_ACCESS_PROPERTY_STREAMING; + + CUresult const r = cuStreamSetAttribute(q->stream, CU_STREAM_ATTRIBUTE_ACCESS_POLICY_WINDOW, &attr); + if (r != CUDA_SUCCESS) { + fprintf(stderr, "L2 persist: cuStreamSetAttribute failed (%d)\n", (int)r); + } else { + fprintf(stderr, "L2 persist: window %zuMB (%.1f%% hit ratio), %zuMB actual data, %zu buffers\n", + spanBytes / (1024*1024), hitRatio * 100.0f, totalDataBytes / (1024*1024), + buffers.size()); + } +} +#endif + + +// OpenCL-like extensions invented to provide a clean interface to some nVidia CUDA features + +// Interface to nVidia CUDA graphs feature + +bool clIsGraphSupported(cl_device_id dev) { + int major = 0; + cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, dev->dev); + return (major >= 6); +} + +int clGraphBeginRecording(cl_command_queue q) { + ensureContextCurrent(); + CUresult r = cuStreamBeginCapture(q->stream, CU_STREAM_CAPTURE_MODE_THREAD_LOCAL); + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; +} + +int clGraphEndRecording(cl_command_queue q, cl_graph* graph) { + ensureContextCurrent(); + auto* g = new _cl_graph; + g->queue = q; + CUresult r = cuStreamEndCapture(q->stream, &g->graph); +#if CUDA_VERSION >= 12000 + if (r == CUDA_SUCCESS) r = cuGraphInstantiate(&g->graphExec, g->graph, 0); +#elif CUDA_VERSION >= 11040 + if (r == CUDA_SUCCESS) r = cuGraphInstantiateWithFlags(&g->graphExec, g->graph, 0); +#else + if (r == CUDA_SUCCESS) r = cuGraphInstantiate(&g->graphExec, g->graph, nullptr, nullptr, 0); +#endif + *graph = g; + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; +} + +int clGraphLaunch(cl_graph graph) { + ensureContextCurrent(); + CUresult r = cuGraphLaunch(graph->graphExec, graph->queue->stream); + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; +} + +int clReleaseGraph(cl_graph graph) { + delete graph; + return CL_SUCCESS; +} diff --git a/src/cuda/cudawrap.cpp b/src/cuda/cudawrap.cpp new file mode 100644 index 00000000..dbdab458 --- /dev/null +++ b/src/cuda/cudawrap.cpp @@ -0,0 +1,542 @@ +// CUDA Driver API wrappers implementation + +#include "cudawrap.h" + +#include +#include +#include +#include + +// Local log function — avoids dependency on PRPLL's log.h/File.h chain +static void cuda_log(const char* fmt, ...) { + va_list args; + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); +} + +// ---- Error checking ---- + +void checkCuda(CUresult err, const char* file, int line, const char* func, const char* expr) { + if (err != CUDA_SUCCESS) { + const char* errName = nullptr; + const char* errStr = nullptr; + cuGetErrorName(err, &errName); + cuGetErrorString(err, &errStr); + char buf[512]; + snprintf(buf, sizeof(buf), "CUDA error %d (%s): %s at %s:%d in %s: %s", + (int)err, errName ? errName : "?", errStr ? errStr : "?", + file, line, func, expr); + cuda_log("%s\n", buf); + throw std::runtime_error(buf); + } +} + +void checkNvrtc(nvrtcResult err, const char* file, int line, const char* func, const char* expr) { + if (err != NVRTC_SUCCESS) { + char buf[512]; + snprintf(buf, sizeof(buf), "NVRTC error %d (%s) at %s:%d in %s: %s", + (int)err, nvrtcGetErrorString(err), file, line, func, expr); + cuda_log("%s\n", buf); + throw std::runtime_error(buf); + } +} + +// ---- Device management ---- + +std::vector getAllDevices() { + CU_CHECK(cuInit(0)); + int count = 0; + CU_CHECK(cuDeviceGetCount(&count)); + std::vector devices(count); + for (int i = 0; i < count; ++i) { + CU_CHECK(cuDeviceGet(&devices[i], i)); + } + return devices; +} + +std::string getDeviceName(CUdevice dev) { + char name[256]; + CU_CHECK(cuDeviceGetName(name, sizeof(name), dev)); + return name; +} + +std::string getDriverVersion() { + int ver = 0; + CU_CHECK(cuDriverGetVersion(&ver)); + char buf[32]; + snprintf(buf, sizeof(buf), "%d.%d", ver / 1000, (ver % 1000) / 10); + return buf; +} + +float getGpuRamGB(CUdevice dev) { + size_t bytes = 0; + CU_CHECK(cuDeviceTotalMem(&bytes, dev)); + return bytes / (1024.0f * 1024.0f * 1024.0f); +} + +u64 getFreeMem(CUdevice /*dev*/) { + // Need a context to query free memory + size_t free_bytes = 0, total = 0; + CU_CHECK(cuMemGetInfo(&free_bytes, &total)); + return free_bytes; +} + +std::string getShortInfo(CUdevice dev) { + int major = 0, minor = 0; + cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, dev); + cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, dev); + char buf[256]; + snprintf(buf, sizeof(buf), "%s (sm_%d%d, %.1f GB)", getDeviceName(dev).c_str(), + major, minor, getGpuRamGB(dev)); + return buf; +} + +// ---- Context ---- + +CudaContext::CudaContext(CUdevice dev) : device(dev) { +#if CUDA_VERSION >= 13000 + CUctxCreateParams params{}; + CU_CHECK(cuCtxCreate_v4(&ctx, ¶ms, CU_CTX_SCHED_BLOCKING_SYNC, dev)); +#else + CU_CHECK(cuCtxCreate(&ctx, CU_CTX_SCHED_BLOCKING_SYNC, dev)); +#endif +} + +CudaContext::~CudaContext() { + if (ctx) cuCtxDestroy(ctx); +} + +void CudaContext::makeCurrent() { + CU_CHECK(cuCtxSetCurrent(ctx)); +} + +// ---- Module ---- + +CudaModule::CudaModule(const std::string& ptx, const std::string& name) { + CUjit_option options[] = { CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES, CU_JIT_ERROR_LOG_BUFFER }; + char errorLog[4096] = {}; + void* optionValues[] = { (void*)(size_t)sizeof(errorLog), (void*)errorLog }; + + CUresult const err = cuModuleLoadDataEx(&module, ptx.c_str(), 2, options, optionValues); + if (err != CUDA_SUCCESS) { + cuda_log("Module load error for '%s': %s\n", name.c_str(), errorLog); + checkCuda(err, __FILE__, __LINE__, __func__, "cuModuleLoadDataEx"); + } +} + +CudaModule::~CudaModule() { + if (module) cuModuleUnload(module); +} + +CUfunction CudaModule::getFunction(const char* name) const { + CUfunction func{}; + CU_CHECK(cuModuleGetFunction(&func, module, name)); + return func; +} + +// ---- Stream ---- + +CudaStream::CudaStream() { + CU_CHECK(cuStreamCreate(&stream, CU_STREAM_DEFAULT)); +} + +CudaStream::~CudaStream() { + if (stream) cuStreamDestroy(stream); +} + +void CudaStream::sync() { + CU_CHECK(cuStreamSynchronize(stream)); +} + +// ---- Buffer ---- + +CudaBuffer::CudaBuffer(size_t bytes) : bytes(bytes) { + if (bytes > 0) { + CU_CHECK(cuMemAlloc(&ptr, bytes)); + } +} + +CudaBuffer::~CudaBuffer() { + if (ptr) cuMemFree(ptr); +} + +void CudaBuffer::readSync(void* dst, size_t n) const { + assert(n <= bytes); + CU_CHECK(cuMemcpyDtoH(dst, ptr, n)); +} + +void CudaBuffer::writeSync(const void* src, size_t n) { + assert(n <= bytes); + CU_CHECK(cuMemcpyHtoD(ptr, src, n)); +} + +void CudaBuffer::zero() { + if (bytes > 0) { + CU_CHECK(cuMemsetD8(ptr, 0, bytes)); + } +} + +void CudaBuffer::copyFrom(const CudaBuffer& src) { + assert(bytes == src.bytes); + CU_CHECK(cuMemcpyDtoD(ptr, src.ptr, bytes)); +} + +void CudaBuffer::fillPattern(const void* pattern, size_t patternSize) { + if (patternSize == 4) { + u32 val; + memcpy(&val, pattern, 4); + CU_CHECK(cuMemsetD32(ptr, val, bytes / 4)); + } else if (patternSize == 1) { + u8 val; + memcpy(&val, pattern, 1); + CU_CHECK(cuMemsetD8(ptr, val, bytes)); + } else { + // For other sizes, use cuMemsetD32 with multiple passes or copy pattern manually + // This is rarely used + assert(false && "fillPattern: unsupported pattern size"); + } +} + +// ---- NVRTC Compilation ---- + +// Helper: skip balanced parentheses starting at '(' at position i +static size_t skipBalancedParens(const std::string& s, size_t i) { + if (i >= s.size() || s[i] != '(') return i; + int depth = 1; + i++; + while (i < s.size() && depth > 0) { + if (s[i] == '(') depth++; + else if (s[i] == ')') depth--; + i++; + } + return i; +} + +// Strip OpenCL-specific constructs that NVRTC can't handle: +// 1. __global/global pointer qualifiers (without breaking __global__) +// 2. #pragma OPENCL ... directives +// 3. __attribute__((overloadable)) and __attribute__((reqd_work_group_size(...))) +// 4. OpenCL vector cast syntax: (type2)(a, b) → make_type2(a, b) [deferred to compat header] +std::string NvrtcProgram::preprocessOpenCL(const std::string& source) { + std::string result; + result.reserve(source.size()); + size_t i = 0; + while (i < source.size()) { + // Strip #pragma OPENCL ... lines + if (i + 14 <= source.size() && source.compare(i, 14, "#pragma OPENCL") == 0) { + bool const atLineStart = (i == 0 || source[i-1] == '\n'); + if (atLineStart) { + while (i < source.size() && source[i] != '\n') i++; + result += "// [stripped pragma]"; + continue; + } + } + + // Replace base.cl's KERNEL macro with CUDA-compatible version + // OpenCL: #define KERNEL(x) kernel __attribute__((reqd_work_group_size(x, 1, 1))) void + // CUDA: #define KERNEL(x) extern "C" __global__ void __launch_bounds__(x) + if (i + 15 <= source.size() && source.compare(i, 15, "#define KERNEL(") == 0) { + bool const atLineStart = (i == 0 || source[i-1] == '\n'); + if (atLineStart) { + while (i < source.size() && source[i] != '\n') i++; + result += "#ifdef CUDA_MIN_BLOCKS\n"; + result += "#define KERNEL(x) extern \"C\" __global__ void __launch_bounds__(x, CUDA_MIN_BLOCKS)\n"; + result += "#else\n"; + result += "#define KERNEL(x) extern \"C\" __global__ void __launch_bounds__(x)\n"; + result += "#endif"; + continue; + } + } + + // Strip base.cl's OpenCL typedefs that conflict with opencl_compat.cuh + // Only strip exact OpenCL patterns (using OpenCL types like 'long', 'ulong', 'uint') + // NOT our compat header's versions (which use 'long long', 'unsigned long long', etc.) + if (i + 7 <= source.size() && source.compare(i, 7, "typedef") == 0) { + bool const atLineStart = (i == 0 || source[i-1] == '\n'); + if (atLineStart) { + size_t lineEnd = source.find('\n', i); + if (lineEnd == std::string::npos) lineEnd = source.size(); + std::string line = source.substr(i, lineEnd - i); + // Strip trailing whitespace/CR for comparison + while (!line.empty() && (line.back() == ' ' || line.back() == '\r')) line.pop_back(); + // Only strip the EXACT OpenCL patterns from base.cl: + if (line == "typedef int i32;" || + line == "typedef uint u32;" || + line == "typedef long i64;" || + line == "typedef ulong u64;") { + result += "// [stripped OpenCL typedef]"; + i = lineEnd; + continue; + } + } + } + + // Convert vector array access: .a[0] → .a.x, .a[1] → .a.y + // OpenCL allows vec2[i] indexing; CUDA uses .x/.y member access. + // Used in fftp.cl: union { uint2 a; u64 b; } m31_combo; #define frac_bits m31_combo.a[0] + if (i + 5 <= source.size() && source.compare(i, 5, ".a[0]") == 0) { + result += ".a.x"; + i += 5; + continue; + } + if (i + 5 <= source.size() && source.compare(i, 5, ".a[1]") == 0) { + result += ".a.y"; + i += 5; + continue; + } + + // Handle __attribute__((...)) for overloadable and reqd_work_group_size + if (i + 15 <= source.size() && source.compare(i, 15, "__attribute__((") == 0) { + size_t const nameStart = i + 15; + if (nameStart + 12 <= source.size() && source.compare(nameStart, 12, "overloadable") == 0) { + // Strip __attribute__((overloadable)) — C++ has native overloading + i = skipBalancedParens(source, i + 13); + while (i < source.size() && source[i] == ' ') i++; + continue; + } + if (nameStart + 20 <= source.size() && source.compare(nameStart, 20, "reqd_work_group_size") == 0) { + // Convert __attribute__((reqd_work_group_size(N, 1, 1))) → __launch_bounds__(N) + // Extract the first number from the args + size_t argsStart = nameStart + 20; + while (argsStart < source.size() && source[argsStart] == '(') argsStart++; + size_t numEnd = argsStart; + while (numEnd < source.size() && source[numEnd] >= '0' && source[numEnd] <= '9') numEnd++; + if (numEnd > argsStart) { + std::string const wgSize = source.substr(argsStart, numEnd - argsStart); + result += "__launch_bounds__(" + wgSize + ") "; + } + // Skip past the entire __attribute__((...)) + i = skipBalancedParens(source, i + 13); + while (i < source.size() && source[i] == ' ') i++; + continue; + } + } + + // Convert OpenCL vector cast syntax: (type2)(a, b) → make_type2(a, b) + // Matches: (double2), (float2), (int2), (uint2), (long2), (ulong2) + if (source[i] == '(' && i + 1 < source.size()) { + static const char* vecTypes[] = { + "double2)", "float2)", "int2)", "uint2)", "long2)", "ulong2)", "Word2)", nullptr + }; + static const char* makeNames[] = { + "make_double2", "make_float2", "make_int2", "make_uint2", "make_long2", "make_ulong2", "make_Word2" + }; + bool matched = false; + for (int vi = 0; vecTypes[vi]; vi++) { + size_t const tlen = strlen(vecTypes[vi]); + if (i + 1 + tlen <= source.size() && source.compare(i + 1, tlen, vecTypes[vi]) == 0) { + // Check what follows: should be whitespace then '(' for a cast constructor + size_t after = i + 1 + tlen; + while (after < source.size() && source[after] == ' ') after++; + if (after < source.size() && source[after] == '(') { + // It's (type2) (args) — replace with make_type2(args) + result += makeNames[vi]; + i = after; // now pointing at '(' of args + matched = true; + break; + } + // Also handle (type2){args} — less common but possible + if (after < source.size() && source[after] == '{') { + result += makeNames[vi]; + result += '('; + i = after + 1; // skip '{' + // Find matching '}' and replace with ')' + int depth = 1; + while (i < source.size() && depth > 0) { + if (source[i] == '{') depth++; + else if (source[i] == '}') { depth--; if (depth == 0) { result += ')'; i++; break; } } + else result += source[i]; + i++; + } + matched = true; + break; + } + } + } + if (matched) continue; + } + + // Convert "local TYPE NAME[" to "__shared__ TYPE NAME[" for shared memory declarations + // This handles: " local T2 lds[WIDTH / 4];" → " __shared__ T2 lds[WIDTH / 4];" + // Also handles: " local T lds[IN_WG / 2 * (MIDDLE <= 8 ? 2 * MIDDLE : MIDDLE)];" + // But NOT: "local T2 *lds" in function params (which becomes just "T2 *lds" via macro) + if (i + 6 <= source.size() && source.compare(i, 6, "local ") == 0) { + bool const preceded = (i > 0 && (isalnum(source[i-1]) || source[i-1] == '_')); + if (!preceded) { + // Check if this "local" is followed by a type then a name then '[' + // i.e., it's a shared memory array declaration + size_t lineEnd = source.find('\n', i); + if (lineEnd == std::string::npos) lineEnd = source.size(); + std::string const line = source.substr(i, lineEnd - i); + // Match: "local TYPE IDENT[" pattern — indicates array declaration + // Array declarations have '[' and end with ';'. They may also contain '(' in + // the array size expression (e.g., ternary operators). The key distinction is + // that function parameters don't have '['. + if (line.find('[') != std::string::npos) { + result += "__shared__ "; + i += 6; // skip "local " + continue; + } + // For everything else (params, casts), just skip "local " → empty + i += 6; + continue; + } + } + // Same for __local + if (i + 8 <= source.size() && source.compare(i, 8, "__local ") == 0) { + bool const preceded = (i > 0 && (isalnum(source[i-1]) || source[i-1] == '_')); + if (!preceded) { + size_t lineEnd = source.find('\n', i); + if (lineEnd == std::string::npos) lineEnd = source.size(); + std::string const line = source.substr(i, lineEnd - i); + if (line.find('[') != std::string::npos) { + result += "__shared__ "; + i += 8; + continue; + } + i += 8; + continue; + } + } + + // Replace "n" constraint with "r" in PTX asm (NVRTC requires true constants for "n") + // Match: "n"( → "r"( + if (i + 3 <= source.size() && source.compare(i, 3, "\"n\"") == 0) { + // Check we're inside an asm statement context (look for preceding ':') + // Simple heuristic: if the previous non-whitespace char is ':', ':' + space, or ',' + size_t back = i; + while (back > 0 && (source[back-1] == ' ' || source[back-1] == '\t')) back--; + if (back > 0 && (source[back-1] == ':' || source[back-1] == ',')) { + result += "\"r\""; + i += 3; + continue; + } + } + + // Match __global NOT followed by _ + if (i + 8 <= source.size() && source.compare(i, 8, "__global") == 0) { + if (i + 8 < source.size() && source[i + 8] == '_') { + result += source[i++]; + } else { + bool const preceded = (i > 0 && (isalnum(source[i-1]) || source[i-1] == '_')); + if (preceded) { + result += source[i++]; + } else { + i += 8; + } + } + } + // Match standalone "global" (not inside a word or PTX instruction) + else if (i + 6 <= source.size() && source.compare(i, 6, "global") == 0) { + bool const preceded = (i > 0 && (isalnum(source[i-1]) || source[i-1] == '_' || source[i-1] == '.')); + bool const followed = (i + 6 < source.size() && (isalnum(source[i + 6]) || source[i + 6] == '_')); + if (!preceded && !followed) { + i += 6; + } else { + result += source[i++]; + } + } else { + result += source[i++]; + } + } + return result; +} + +std::string NvrtcProgram::compile(const std::string& source, const std::string& name, + const std::vector& options, + const std::vector>& headers) { + return compileImages(source, name, options, headers).ptx; +} + +NvrtcProgram::Images NvrtcProgram::compileImages(const std::string& source, const std::string& name, + const std::vector& options, + const std::vector>& headers) { + // Prepare header arrays + std::vector headerSources, headerNames; + for (auto& [hName, hSource] : headers) { + headerNames.push_back(hName.c_str()); + headerSources.push_back(hSource.c_str()); + } + + nvrtcProgram prog; + NVRTC_CHECK(nvrtcCreateProgram(&prog, source.c_str(), name.c_str(), + (int)headers.size(), + headerSources.empty() ? nullptr : headerSources.data(), + headerNames.empty() ? nullptr : headerNames.data())); + + // Convert options to char* + std::vector opts; + opts.reserve(options.size()); +for (auto& o : options) opts.push_back(o.c_str()); + + nvrtcResult const compileResult = nvrtcCompileProgram(prog, (int)opts.size(), opts.data()); + + // Get compilation log + size_t logSize; + nvrtcGetProgramLogSize(prog, &logSize); + if (logSize > 1) { + std::string compileLog(logSize, '\0'); + nvrtcGetProgramLog(prog, compileLog.data()); + if (compileResult != NVRTC_SUCCESS) { + cuda_log("NVRTC compile error for '%s':\n%s\n", name.c_str(), compileLog.c_str()); + } + } + + if (compileResult != NVRTC_SUCCESS) { + nvrtcDestroyProgram(&prog); + throw std::runtime_error("NVRTC compilation failed for " + name); + } + + Images images; + size_t ptxSize; + NVRTC_CHECK(nvrtcGetPTXSize(prog, &ptxSize)); + images.ptx.assign(ptxSize, '\0'); + NVRTC_CHECK(nvrtcGetPTX(prog, images.ptx.data())); + +#if CUDA_VERSION >= 11010 + // The CUBIN exists only when the architecture was a real sm_XY (not + // compute_XY); a zero size means NVRTC has none to give, not an error. + size_t cubinSize = 0; + if (nvrtcGetCUBINSize(prog, &cubinSize) == NVRTC_SUCCESS && cubinSize > 0) { + images.cubin.assign(cubinSize, '\0'); + if (nvrtcGetCUBIN(prog, images.cubin.data()) != NVRTC_SUCCESS) { images.cubin.clear(); } + } +#endif + + nvrtcDestroyProgram(&prog); + return images; +} + +// ---- Kernel launcher ---- + +void CudaKernelLauncher::launch(CUstream stream, u32 gridSize, void** args, u32 sharedMem) { +#if CUDA_VERSION >= 12000 && defined(ENABLE_PDL) && ENABLE_PDL + // enable pdl in kernel launch attributes + CUlaunchAttribute attrs[1]; + attrs[0].id = CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION; + attrs[0].value.programmaticStreamSerializationAllowed = 1; + // set kernel launch configuration + CUlaunchConfig_st config = {0}; + config.gridDimX = gridSize; + config.gridDimY = 1; + config.gridDimZ = 1; + config.blockDimX = blockSize; + config.blockDimY = 1; + config.blockDimZ = 1; + config.hStream = stream; + config.sharedMemBytes = sharedMem; + config.attrs = attrs; + config.numAttrs = 1; + // launch the kernel + CU_CHECK(cuLaunchKernelEx(&config, func, args, nullptr)); +#else + CU_CHECK(cuLaunchKernel(func, + gridSize, 1, 1, // grid dimensions + blockSize, 1, 1, // block dimensions + sharedMem, // shared memory bytes + stream, // stream + args, // kernel arguments + nullptr)); // extra +#endif +} diff --git a/src/cuda/cudawrap.h b/src/cuda/cudawrap.h new file mode 100644 index 00000000..ac3639e3 --- /dev/null +++ b/src/cuda/cudawrap.h @@ -0,0 +1,148 @@ +// CUDA Driver API wrappers, parallel to clwrap.h for OpenCL. +// Uses the CUDA Driver API (cu*) for maximum control. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +// Types — when building standalone tests, define minimal types. +// When building as part of PRPLL, common.h is already included. +#if __has_include("../common.h") && !defined(CUDAWRAP_STANDALONE) +#include "../common.h" +#else +#ifndef CUDAWRAP_TYPES_DEFINED +#define CUDAWRAP_TYPES_DEFINED +typedef unsigned int u32; +typedef unsigned long long u64; +typedef unsigned char u8; +#endif +#endif + +// ---- Error checking ---- +void checkCuda(CUresult err, const char* file, int line, const char* func, const char* expr); +void checkNvrtc(nvrtcResult err, const char* file, int line, const char* func, const char* expr); + +#define CU_CHECK(expr) checkCuda((expr), __FILE__, __LINE__, __func__, #expr) +#define NVRTC_CHECK(expr) checkNvrtc((expr), __FILE__, __LINE__, __func__, #expr) + +// ---- Device management ---- +std::vector getAllDevices(); +std::string getDeviceName(CUdevice dev); +std::string getDriverVersion(); +float getGpuRamGB(CUdevice dev); +u64 getFreeMem(CUdevice dev); +std::string getShortInfo(CUdevice dev); + +// ---- Context ---- +class CudaContext { + CUcontext ctx{}; + CUdevice device{}; +public: + explicit CudaContext(CUdevice dev); + ~CudaContext(); + + [[nodiscard]] CUcontext get() const { return ctx; } + [[nodiscard]] CUdevice getDevice() const { return device; } + void makeCurrent(); +}; + +// ---- Module (compiled kernels) ---- +class CudaModule { + CUmodule module{}; +public: + CudaModule() = default; + explicit CudaModule(const std::string& ptx, const std::string& name = ""); + ~CudaModule(); + + CudaModule(CudaModule&& rhs) noexcept : module(rhs.module) { rhs.module = nullptr; } + CudaModule& operator=(CudaModule&& rhs) noexcept { + if (this != &rhs) { if (module) cuModuleUnload(module); module = rhs.module; rhs.module = nullptr; } + return *this; + } + + [[nodiscard]] CUmodule get() const { return module; } + CUfunction getFunction(const char* name) const; +}; + +// ---- Stream (equivalent to cl_command_queue) ---- +class CudaStream { + CUstream stream{}; +public: + CudaStream(); + ~CudaStream(); + + [[nodiscard]] CUstream get() const { return stream; } + void sync(); + + CudaStream(CudaStream&& rhs) noexcept : stream(rhs.stream) { rhs.stream = nullptr; } + CudaStream& operator=(CudaStream&&) = delete; +}; + +// ---- Memory buffer ---- +class CudaBuffer { + CUdeviceptr ptr{}; + size_t bytes{}; +public: + CudaBuffer() = default; + explicit CudaBuffer(size_t bytes); + ~CudaBuffer(); + + CudaBuffer(CudaBuffer&& rhs) noexcept : ptr(rhs.ptr), bytes(rhs.bytes) { rhs.ptr = 0; rhs.bytes = 0; } + CudaBuffer& operator=(CudaBuffer&& rhs) noexcept { + if (this != &rhs) { if (ptr) cuMemFree(ptr); ptr = rhs.ptr; bytes = rhs.bytes; rhs.ptr = 0; rhs.bytes = 0; } + return *this; + } + + [[nodiscard]] CUdeviceptr get() const { return ptr; } + [[nodiscard]] size_t size() const { return bytes; } + + void readSync(void* dst, size_t n) const; + void writeSync(const void* src, size_t n); + void zero(); + void copyFrom(const CudaBuffer& src); + void fillPattern(const void* pattern, size_t patternSize); +}; + +// ---- NVRTC Compilation ---- +struct NvrtcProgram { + // Preprocess OpenCL source for CUDA: strip __global/global pointer qualifiers + // without breaking __global__ (CUDA kernel qualifier). + static std::string preprocessOpenCL(const std::string& source); + + static std::string compile(const std::string& source, const std::string& name, + const std::vector& options, + const std::vector>& headers = {}); + + // PTX plus, when NVRTC ran its ptxas for a real --gpu-architecture=sm_XY + // (NVRTC >= 11.1), the CUBIN — SASS the driver loads without a JIT, with + // every option (--maxrregcount included) already applied. `cubin` is empty + // when the toolkit or the architecture flag gave none. + struct Images { std::string ptx; std::string cubin; }; + static Images compileImages(const std::string& source, const std::string& name, + const std::vector& options, + const std::vector>& headers = {}); +}; + +// ---- Kernel launcher ---- +class CudaKernelLauncher { + CUfunction func{}; + std::string name; + u32 blockSize{}; + +public: + CudaKernelLauncher() = default; + CudaKernelLauncher(CUfunction f, std::string name, u32 blockSize) + : func(f), name(std::move(name)), blockSize(blockSize) {} + + void launch(CUstream stream, u32 gridSize, void** args, u32 sharedMem = 0); + + [[nodiscard]] CUfunction get() const { return func; } + [[nodiscard]] const std::string& getName() const { return name; } +}; diff --git a/src/cuda/opencl_compat.cuh b/src/cuda/opencl_compat.cuh new file mode 100644 index 00000000..203607f5 --- /dev/null +++ b/src/cuda/opencl_compat.cuh @@ -0,0 +1,334 @@ +// OpenCL → CUDA compatibility header for PRPLL +// Allows .cl kernel files to compile under NVRTC with minimal changes. +// Used together with NvrtcProgram::preprocessOpenCL() which strips: +// - __global/global pointer qualifiers (can't #define without breaking __global__) +// - #pragma OPENCL ... directives +// - __attribute__((overloadable)) and __attribute__((reqd_work_group_size(...))) + +#pragma once + +// Set flag that OpenCL can access. Let's us add CUDA-only code to the OpenCL sources. +#define CUDA_BACKEND 1 + +// ---- Qualifiers ---- +// __kernel / kernel → extern "C" __global__ (CUDA kernel launch qualifier) +// extern "C" is needed so cuModuleGetFunction() can find kernels by unmangled name +#define __kernel extern "C" __global__ +#define kernel extern "C" __global__ + +// __local / local — OpenCL address space qualifier for shared memory. +// In CUDA, __shared__ can only be used on variable declarations, NOT on function parameters. +// The preprocessor handles this: it strips "local" from function parameter lists +// and adds __shared__ for variable declarations matching "local TYPE NAME[". +#define __local +#define local + +// __constant / constant → const (CUDA __constant__ is file-scope only, can't be used +// for kernel params). The compiler auto-uses __ldg() for const pointers on sm_35+. +#define __constant const +#define constant const + +// restrict → __restrict__ (different keyword in CUDA) +#define restrict __restrict__ + +// ---- Work-item functions ---- +#define get_local_id(d) ((unsigned int)threadIdx.x) +#define get_group_id(d) ((unsigned int)(d == 0 ? blockIdx.x : blockIdx.y)) +#define get_local_size(d) ((unsigned int)(d == 0 ? blockDim.x : blockDim.y)) +#define get_global_id(d) ((unsigned int)(blockIdx.x * blockDim.x + threadIdx.x)) +#define get_num_groups(d) ((unsigned int)gridDim.x) +#define get_global_size(d) ((unsigned int)(gridDim.x * blockDim.x)) +#define get_enqueued_local_size(d) get_local_size(d) + +// ---- Barriers and fences ---- +#define CLK_LOCAL_MEM_FENCE (1 << 0) +#define CLK_GLOBAL_MEM_FENCE (1 << 1) + +__device__ __forceinline__ void ocl_barrier(int flags) { + __syncthreads(); // execution barrier is unconditional regardless of flags + // __syncthreads() already fences both shared and global memory per CUDA's + // documented semantics, so no additional fence is needed here for either flag. +} +#define barrier(flags) ocl_barrier(flags) + +__device__ __forceinline__ void ocl_mem_fence(int flags) { + if (flags & CLK_GLOBAL_MEM_FENCE) __threadfence(); + else if (flags & CLK_LOCAL_MEM_FENCE) __threadfence_block(); +} +#define mem_fence(flags) ocl_mem_fence(flags) +#define write_mem_fence(flags) ocl_mem_fence(flags) +#define read_mem_fence(flags) ocl_mem_fence(flags) + +// ---- Overloadable ---- +// CUDA C++ supports function overloading natively +#define OVERLOAD + +// ---- OpenCL extension macros ---- +#define cl_khr_fp64 1 +#define cl_khr_subgroups 1 + +// ---- Kernel macro ---- +// PRPLL uses KERNEL(WG_SIZE) void kernelName(...) +// base.cl defines: #define KERNEL(x) kernel __attribute__((reqd_work_group_size(x, 1, 1))) void +// Our preprocessor replaces base.cl's KERNEL macro with a CUDA version. +// This fallback is only used if base.cl hasn't been included yet. +// KERNEL macro fallback — usually overridden by base.cl KERNEL macro replacement in cudawrap.cpp +#define KERNEL(x) extern "C" __global__ void __launch_bounds__(x) + +// ---- Pointer macros ---- +#define P(x) x* __restrict__ +#define CP(x) const x* __restrict__ + +// ---- OpenCL type aliases ---- + +// Standard PRPLL type aliases +typedef unsigned int uint; + +// These match OpenCL's types exactly. The preprocessor strips base.cl's re-definitions of i32/u32 to avoid redeclaration errors. +typedef int i32; +typedef unsigned int u32; + +// OpenCL defines long, ulong, long2, ulong2, etc. as 64-bits. CUDA defines them as 32-bits (MSVC) or 64-bits (Linux). +// CUDA defines longlong, ulonglong, longlong2, ulonglong2, etc. as 64-bits. Map OpenCL types to CUDA types. + +//#define long long long // Obviously, we can't uncomment this #define. Instead, we must make sure the opencl code never uses this data type. Use i64 instead. +#define long2 longlong2 +#define ulong unsigned long long +#define ulong2 ulonglong2 +#define make_long2 make_longlong2 +#define make_ulong2 make_ulonglong2 + +// These must match the 64-bit data types defined above. The preprocessor strips base.cl's re-definitions of i64/u64 to avoid redeclaration errors. +// Must use 'long long' / 'unsigned long long' to match CUDA vector type members so that Z61 (typedef'd from ulong) matches ulong2 member types. +// Otherwise overloaded functions like add(Z31,Z31) vs add(Z61,Z61) become ambiguous when called with ulong2 member values (which are 'unsigned long'). +typedef long long i64; +typedef unsigned long long u64; + +// ---- Math constants ---- +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif +#ifndef M_SQRT1_2 +#define M_SQRT1_2 0.70710678118654752440 +#endif + +// ---- Vector arithmetic operators ---- +// OpenCL supports +, -, *, / on vector types natively. CUDA does not. +// double2 +__device__ __forceinline__ double2 operator+(double2 a, double2 b) { return make_double2(a.x+b.x, a.y+b.y); } +__device__ __forceinline__ double2 operator-(double2 a, double2 b) { return make_double2(a.x-b.x, a.y-b.y); } +__device__ __forceinline__ double2 operator*(double2 a, double2 b) { return make_double2(a.x*b.x, a.y*b.y); } +__device__ __forceinline__ double2 operator-(double2 a) { return make_double2(-a.x, -a.y); } +__device__ __forceinline__ double2 operator*(double s, double2 a) { return make_double2(s*a.x, s*a.y); } +__device__ __forceinline__ double2 operator*(double2 a, double s) { return make_double2(a.x*s, a.y*s); } +__device__ __forceinline__ double2& operator+=(double2& a, double2 b) { a.x+=b.x; a.y+=b.y; return a; } +__device__ __forceinline__ double2& operator-=(double2& a, double2 b) { a.x-=b.x; a.y-=b.y; return a; } + +// float2 +__device__ __forceinline__ float2 operator+(float2 a, float2 b) { return make_float2(a.x+b.x, a.y+b.y); } +__device__ __forceinline__ float2 operator-(float2 a, float2 b) { return make_float2(a.x-b.x, a.y-b.y); } +__device__ __forceinline__ float2 operator*(float2 a, float2 b) { return make_float2(a.x*b.x, a.y*b.y); } +__device__ __forceinline__ float2 operator-(float2 a) { return make_float2(-a.x, -a.y); } +__device__ __forceinline__ float2 operator*(float s, float2 a) { return make_float2(s*a.x, s*a.y); } +__device__ __forceinline__ float2 operator*(float2 a, float s) { return make_float2(a.x*s, a.y*s); } +__device__ __forceinline__ float2& operator+=(float2& a, float2 b) { a.x+=b.x; a.y+=b.y; return a; } +__device__ __forceinline__ float2& operator-=(float2& a, float2 b) { a.x-=b.x; a.y-=b.y; return a; } + +// int2 +__device__ __forceinline__ int2 operator+(int2 a, int2 b) { return make_int2(a.x+b.x, a.y+b.y); } +__device__ __forceinline__ int2 operator-(int2 a, int2 b) { return make_int2(a.x-b.x, a.y-b.y); } +__device__ __forceinline__ int2 operator*(int2 a, int2 b) { return make_int2(a.x*b.x, a.y*b.y); } +__device__ __forceinline__ int2 operator-(int2 a) { return make_int2(-a.x, -a.y); } +__device__ __forceinline__ int2& operator+=(int2& a, int2 b) { a.x+=b.x; a.y+=b.y; return a; } +__device__ __forceinline__ int2& operator-=(int2& a, int2 b) { a.x-=b.x; a.y-=b.y; return a; } + +// uint2 +__device__ __forceinline__ uint2 operator+(uint2 a, uint2 b) { return make_uint2(a.x+b.x, a.y+b.y); } +__device__ __forceinline__ uint2 operator-(uint2 a, uint2 b) { return make_uint2(a.x-b.x, a.y-b.y); } +__device__ __forceinline__ uint2 operator*(uint2 a, uint2 b) { return make_uint2(a.x*b.x, a.y*b.y); } +__device__ __forceinline__ uint2& operator+=(uint2& a, uint2 b) { a.x+=b.x; a.y+=b.y; return a; } +__device__ __forceinline__ uint2& operator-=(uint2& a, uint2 b) { a.x-=b.x; a.y-=b.y; return a; } + +// long2 +__device__ __forceinline__ long2 operator+(long2 a, long2 b) { return {a.x+b.x, a.y+b.y}; } +__device__ __forceinline__ long2 operator-(long2 a, long2 b) { return {a.x-b.x, a.y-b.y}; } +__device__ __forceinline__ long2 operator*(long2 a, long2 b) { return {a.x*b.x, a.y*b.y}; } +__device__ __forceinline__ long2 operator-(long2 a) { return {-a.x, -a.y}; } +__device__ __forceinline__ long2& operator+=(long2& a, long2 b) { a.x+=b.x; a.y+=b.y; return a; } +__device__ __forceinline__ long2& operator-=(long2& a, long2 b) { a.x-=b.x; a.y-=b.y; return a; } + +// ulong2 +__device__ __forceinline__ ulong2 operator+(ulong2 a, ulong2 b) { return {a.x+b.x, a.y+b.y}; } +__device__ __forceinline__ ulong2 operator-(ulong2 a, ulong2 b) { return {a.x-b.x, a.y-b.y}; } +__device__ __forceinline__ ulong2 operator*(ulong2 a, ulong2 b) { return {a.x*b.x, a.y*b.y}; } +__device__ __forceinline__ ulong2& operator+=(ulong2& a, ulong2 b) { a.x+=b.x; a.y+=b.y; return a; } +__device__ __forceinline__ ulong2& operator-=(ulong2& a, ulong2 b) { a.x-=b.x; a.y-=b.y; return a; } + +// Scalar * vector operators for types not built-in to NVRTC +// (NVRTC already provides double2*double, int2*int, uint2*uint, etc.) +// These cover cross-type scalar*vector that OpenCL supports natively. +__device__ __forceinline__ long2 operator*(i64 s, long2 v) { return {s*v.x, s*v.y}; } +__device__ __forceinline__ long2 operator*(long2 v, i64 s) { return {v.x*s, v.y*s}; } +__device__ __forceinline__ ulong2 operator*(ulong s, ulong2 v) { return {s*v.x, s*v.y}; } +__device__ __forceinline__ ulong2 operator*(ulong2 v, ulong s) { return {v.x*s, v.y*s}; } +// int * ulong2 (common in NTT code: int literal * GF61) +__device__ __forceinline__ ulong2 operator*(int s, ulong2 v) { return {(ulong)s*v.x, (ulong)s*v.y}; } +__device__ __forceinline__ ulong2 operator*(ulong2 v, int s) { return {v.x*(ulong)s, v.y*(ulong)s}; } + +// ---- Vector constructors (U2) ---- +// OpenCL (type2)(a,b) cast syntax is converted to make_type2(a,b) by the preprocessor. +// base.cl defines U2() as overloaded functions — they'll work after preprocessing. +// NVRTC provides make_double2, make_float2, make_int2, make_uint2, make_long2, make_ulong2 built-in. + +// ---- Type reinterpretation (as_*) ---- +// Scalar ↔ vector bitwise reinterpretations (OpenCL as_type functions) + +// as_uint2: split 64-bit value into two 32-bit halves +__device__ __forceinline__ uint2 as_uint2(double v) { union { uint2 ui2; double d; } u; u.d = v; return u.ui2; } +__device__ __forceinline__ uint2 as_uint2(ulong v) { union { uint2 ui2; ulong ul; } u; u.ul = v; return u.ui2; } +__device__ __forceinline__ uint2 as_uint2(i64 v) { union { uint2 ui2; i64 l; } u; u.l = v; return u.ui2; } + +// as_int2: split 64-bit value into two signed 32-bit halves +__device__ __forceinline__ int2 as_int2(double v) { union { int2 i2; double d; } u; u.d = v; return u.i2; } +__device__ __forceinline__ int2 as_int2(i64 v) { union { int2 i2; i64 l; } u; u.l = v; return u.i2; } + +// as_double: reinterpret bits as double +__device__ __forceinline__ double as_double(int2 v) { union { int2 i2; double d; } u; u.i2 = v; return u.d; } +__device__ __forceinline__ double as_double(uint2 v) { union { uint2 ui2; double d; } u; u.ui2 = v; return u.d; } +__device__ __forceinline__ double as_double(ulong v) { return __longlong_as_double(v); } +__device__ __forceinline__ double as_double(i64 v) { return __longlong_as_double(v); } + +// as_ulong: reinterpret as unsigned 64-bit +__device__ __forceinline__ ulong as_ulong(uint2 v) { union { uint2 ui2; ulong ul; } u; u.ui2 = v; return u.ul; } +__device__ __forceinline__ ulong as_ulong(int2 v) { union { int2 i2; ulong ul; } u; u.i2 = v; return u.ul; } +__device__ __forceinline__ ulong as_ulong(double v) { return (ulong)__double_as_longlong(v); } + +// as_long: reinterpret as signed 64-bit +__device__ __forceinline__ i64 as_long(int2 v) { union { int2 i2; i64 l; } u; u.i2 = v; return u.l; } +__device__ __forceinline__ i64 as_long(uint2 v) { union { uint2 ui2; i64 l; } u; u.ui2 = v; return u.l; } +__device__ __forceinline__ i64 as_long(double v) { return (i64)__double_as_longlong(v); } + +// as_float / as_int / as_uint: 32-bit reinterprets +__device__ __forceinline__ float as_float(int v) { return __int_as_float(v); } +__device__ __forceinline__ float as_float(uint v) { return __int_as_float((int)v); } +__device__ __forceinline__ int as_int(float v) { return __float_as_int(v); } +__device__ __forceinline__ uint as_uint(float v) { return (uint)__float_as_int(v); } + +// 16-byte reinterprets: int4 ↔ double2 ↔ ulong2 +__device__ __forceinline__ int4 as_int4(double2 v) { + union { double2 d; int4 i; } u; + u.d = v; + return u.i; +} +__device__ __forceinline__ int4 as_int4(ulong2 v) { + union { ulong2 ul; int4 i; } u; + u.ul = v; + return u.i; +} +__device__ __forceinline__ double2 as_double2(int4 v) { + union { int4 i; double2 d; } u; + u.i = v; + return u.d; +} +__device__ __forceinline__ ulong2 as_ulong2(int4 v) { + union { int4 i; ulong2 ul; } u; + u.i = v; + return u.ul; +} +__device__ __forceinline__ double2 as_double2(ulong2 v) { + union { ulong2 ul; double2 d; } u; + u.ul = v; + return u.d; +} +__device__ __forceinline__ ulong2 as_ulong2(double2 v) { + union { double2 d; ulong2 ul; } u; + u.d = v; + return u.ul; +} + +// ---- Math builtins ---- +// fma for vector types (OpenCL supports element-wise fma on vector types) +__device__ __forceinline__ double2 fma(double2 a, double2 b, double2 c) { + return make_double2(fma(a.x, b.x, c.x), fma(a.y, b.y, c.y)); +} +__device__ __forceinline__ float2 fma(float2 a, float2 b, float2 c) { + return make_float2(fmaf(a.x, b.x, c.x), fmaf(a.y, b.y, c.y)); +} +// Mixed scalar-vector fma: fma(scalar, vec2, vec2) — broadcasts scalar +__device__ __forceinline__ double2 fma(double a, double2 b, double2 c) { + return make_double2(fma(a, b.x, c.x), fma(a, b.y, c.y)); +} +__device__ __forceinline__ float2 fma(float a, float2 b, float2 c) { + return make_float2(fmaf(a, b.x, c.x), fmaf(a, b.y, c.y)); +} + +// mul_hi: upper half of multiplication +__device__ __forceinline__ uint mul_hi(uint a, uint b) { + return __umulhi(a, b); +} +__device__ __forceinline__ ulong mul_hi(ulong a, ulong b) { + return __umul64hi(a, b); +} +__device__ __forceinline__ uint mad_hi(uint a, uint b, uint c) { + return __umulhi(a, b) + c; +} + +// ---- Atomic operations ---- +#define atomic_max(p, v) atomicMax((unsigned int*)(p), (unsigned int)(v)) +#define atomic_add(p, v) atomicAdd(p, v) +#define atomic_cmpxchg(p, old, new) atomicCAS((int *)(p), old, new) + +// OpenCL 2.0 C11-style atomics — optimized for CUDA carry stairway pattern. +// The carryFused kernel uses: producer writes data, threadfence, bar, atomic_store(flag, 1) +// then consumer does: atomic_load(flag) in spin loop, bar, threadfence, read data. +// We minimize redundant fences while maintaining correctness. +__device__ __forceinline__ void atomic_store_uint(volatile unsigned int* p, unsigned int v) { + // Volatile store only — no fence needed here. The caller always does + // write_mem_fence(CLK_GLOBAL_MEM_FENCE) [= __threadfence()] before calling + // atomic_store(), which already orders all prior writes before this store. + // Adding a second __threadfence() here was redundant but costly (~100-400 cycles). + *p = v; +} +__device__ __forceinline__ unsigned int atomic_load_uint(volatile unsigned int* p) { + // Acquire load: volatile ensures we re-read from memory, not from register. + // No fence needed here — the caller does read_mem_fence AFTER confirming the flag. + return *p; +} +#define atomic_store(p, v) atomic_store_uint((volatile unsigned int*)(p), (unsigned int)(v)) +#define atomic_load_explicit(p, order, scope) atomic_load_uint((volatile unsigned int*)(p)) +#define memory_order_relaxed 0 +#define memory_order_acquire 0 +#define memory_order_release 0 +#define memory_scope_device 0 +typedef volatile unsigned int atomic_uint; + +// ---- Inline assembly ---- +// OpenCL uses __asm(); NVRTC uses asm() +#define __asm asm + +// ---- sub_group / warp functions ---- +#define sub_group_broadcast(v, lane) __shfl_sync(0xFFFFFFFF, (v), (lane)) + +// ---- Mark as CUDA compilation ---- +#define CUDA_BACKEND 1 + +// ---- Word2 constructor (typedef for long2 or int2) ---- +// base.cl defines Word2 as long2 (WordSize==8) or int2 (WordSize==4). +// The preprocessor converts (Word2)(a, b) → make_Word2(a, b). +// Must key on WordSize, not CARRY64, because FFT3261 has WordSize=8 without CARRY64. +#if WordSize == 8 +__device__ __forceinline__ long2 make_Word2(i64 a, i64 b) { return make_long2(a, b); } +#else +__device__ __forceinline__ int2 make_Word2(int a, int b) { return make_int2(a, b); } +#endif + +// ---- Force NVIDIAGPU and HAS_PTX ---- +#ifndef NVIDIAGPU +#define NVIDIAGPU 1 +#endif +#ifndef HAS_PTX +#define HAS_PTX 1200 +#endif +#ifndef AMDGPU +#define AMDGPU 0 +#endif diff --git a/src/cuda/tinycuda.h b/src/cuda/tinycuda.h new file mode 100644 index 00000000..0efed571 --- /dev/null +++ b/src/cuda/tinycuda.h @@ -0,0 +1,343 @@ +// CUDA type shim — replaces tinycl.h for the native CUDA backend. +// Maps OpenCL types and constants to CUDA Driver API equivalents. +// Used together with clwrap_cuda.cpp which implements cl* functions via cu*. + +#pragma once + +#include "../common.h" +#include +#include +#include +#include +#include +#include +#include + +// ---- Opaque handle wrappers ---- +// OpenCL uses opaque pointer types (struct _cl_foo*). +// CUDA uses different representations (int, pointer, u64). +// We wrap CUDA handles in structs so they're pointer-like opaque types. + +// cl_device_id wraps CUdevice (int) +struct _cl_device_id { CUdevice dev; }; +using cl_device_id = _cl_device_id*; + +// cl_context wraps CUcontext +struct _cl_context { CUcontext ctx; CUdevice dev; }; +using cl_context = _cl_context*; + +// cl_command_queue wraps CUstream +struct _cl_command_queue { + CUstream stream; + cl_context context; + bool profiling; +}; +using cl_command_queue = _cl_command_queue*; + +// cl_mem wraps CUdeviceptr + size +struct _cl_mem { + CUdeviceptr ptr; + size_t size; +}; +using cl_mem = _cl_mem*; + +// cl_program: dual-purpose — stores either source string or compiled PTX/module +struct _cl_program { + cl_context context; + std::string source; // OpenCL source (before NVRTC compilation) + std::string preprocessedSource; // CUDA source after preprocessOpenCL (for parsing __launch_bounds__) + std::string ptx; // Compiled PTX (after NVRTC compilation); always the text — clCreateKernel reads .maxntid from it + std::string cubin; // NVRTC's CUBIN for this device's sm, when it produced one; loaded ahead of the PTX JIT + CUmodule module{}; // Loaded module (after cuModuleLoadData) + std::string buildLog; // This program's last compile/link diagnostics (clGetProgramBuildInfo) + bool compiled{false}; + bool moduleLoaded{false}; + + _cl_program() {} +}; +using cl_program = _cl_program*; + +// cl_kernel wraps CUfunction + accumulated arguments +struct _cl_kernel { + CUfunction func{}; + std::string name; + CUmodule parentModule{}; // Keep reference so module isn't unloaded + bool pdl{false}; // Its PTX waits on the predecessor (griddepcontrol.wait): launch with programmatic stream serialization + int numArgs{0}; + int reqWorkGroupSize{0}; // From __launch_bounds__(N) in source, matches OpenCL reqd_work_group_size + + // Argument accumulator for setArg/launch pattern + static constexpr int MAX_ARGS = 32; + static constexpr int MAX_ARG_BYTES = 512; + char argData[MAX_ARG_BYTES]; + size_t argSizes[MAX_ARGS]; + size_t argOffsets[MAX_ARGS]; + + _cl_kernel() { + memset(argData, 0, sizeof(argData)); + memset(argSizes, 0, sizeof(argSizes)); + memset(argOffsets, 0, sizeof(argOffsets)); + } + + void setArg(int pos, size_t size, const void* value) { + if (pos >= MAX_ARGS) return; + if (pos >= numArgs) numArgs = pos + 1; + + // Fixed 8-byte slots per arg position. CUDA kernel args are pointers (8 bytes) + // or small scalars (4 bytes). Using fixed slots avoids data corruption when + // args are set out of order (e.g., setFixedArgs(2,3) then operator()(0,1)). + size_t const offset = pos * 8; + argOffsets[pos] = offset; + argSizes[pos] = size; + if (offset + size <= MAX_ARG_BYTES && value) { + memcpy(argData + offset, value, size); + } + } + + // Build void* args[] array for cuLaunchKernel + void buildArgPointers(void** ptrs) const { + for (int i = 0; i < numArgs; i++) { + ptrs[i] = const_cast(argData + argOffsets[i]); + } + } +}; +using cl_kernel = _cl_kernel*; + +// cl_event wraps CUevent pair (start + end for profiling) +struct _cl_event { + CUevent start{}; + CUevent end{}; + bool hasTimings{false}; + u32 commandType{0}; + + _cl_event() {} + ~_cl_event() { + if (start) cuEventDestroy(start); + if (end) cuEventDestroy(end); + } +}; +using cl_event = _cl_event*; + +// Unused types — just need to exist for compilation +using cl_platform_id = struct _cl_platform_id*; +using cl_sampler = struct _cl_sampler*; + +using cl_bool = unsigned; +using cl_program_build_info = unsigned; +using cl_program_info = unsigned; +using cl_device_info = unsigned; +using cl_kernel_info = unsigned; +using cl_kernel_arg_info = unsigned; +using cl_kernel_work_group_info = unsigned; +using cl_profiling_info = unsigned; +using cl_event_info = unsigned; +using cl_command_queue_info = unsigned; + +using cl_mem_flags = u64; +using cl_svm_mem_flags = u64; +using cl_device_type = u64; +using cl_queue_properties = u64; + +using cl_queue = cl_command_queue; + +// ---- Constants ---- +#define CL_SUCCESS 0 +#define CL_DEVICE_TYPE_GPU (1 << 2) +#define CL_DEVICE_TYPE_ALL 0xFFFFFFFF + +#define CL_DEVICE_VENDOR_ID 0x1001 +#define CL_DEVICE_MAX_COMPUTE_UNITS 0x1002 +#define CL_DEVICE_MAX_CLOCK_FREQUENCY 0x100C +#define CL_DEVICE_GLOBAL_MEM_SIZE 0x101F +#define CL_DEVICE_ERROR_CORRECTION_SUPPORT 0x1024 +#define CL_DEVICE_NAME 0x102B +#define CL_DRIVER_VERSION 0x102D +#define CL_DEVICE_VERSION 0x102F +#define CL_DEVICE_BUILT_IN_KERNELS 0x103F +#define CL_PLATFORM_VERSION 0x0901 + +#define CL_PROGRAM_BINARY_SIZES 0x1165 +#define CL_PROGRAM_BINARIES 0x1166 +#define CL_PROGRAM_BUILD_LOG 0x1183 + +#define CL_MEM_READ_WRITE (1 << 0) +#define CL_MEM_WRITE_ONLY (1 << 1) +#define CL_MEM_READ_ONLY (1 << 2) +#define CL_MEM_USE_HOST_PTR (1 << 3) +#define CL_MEM_ALLOC_HOST_PTR (1 << 4) +#define CL_MEM_COPY_HOST_PTR (1 << 5) +#define CL_MEM_HOST_WRITE_ONLY (1 << 7) +#define CL_MEM_HOST_READ_ONLY (1 << 8) +#define CL_MEM_HOST_NO_ACCESS (1 << 9) +#define CL_MEM_SVM_FINE_GRAIN_BUFFER (1 << 10) +#define CL_MEM_SVM_ATOMICS (1 << 11) + +#define CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE (1 << 0) +#define CL_QUEUE_PROFILING_ENABLE (1 << 1) +#define CL_QUEUE_ON_DEVICE (1 << 2) +#define CL_QUEUE_ON_DEVICE_DEFAULT (1 << 3) + +#define CL_QUEUE_CONTEXT 0x1090 +#define CL_QUEUE_DEVICE 0x1091 +#define CL_QUEUE_REFERENCE_COUNT 0x1092 +#define CL_QUEUE_PROPERTIES 0x1093 + +#define CL_PROFILING_COMMAND_QUEUED 0x1280 +#define CL_PROFILING_COMMAND_SUBMIT 0x1281 +#define CL_PROFILING_COMMAND_START 0x1282 +#define CL_PROFILING_COMMAND_END 0x1283 +#define CL_PROFILING_COMMAND_COMPLETE 0x1284 + +#define CL_EVENT_COMMAND_QUEUE 0x11D0 +#define CL_EVENT_COMMAND_TYPE 0x11D1 +#define CL_EVENT_REFERENCE_COUNT 0x11D2 +#define CL_EVENT_COMMAND_EXECUTION_STATUS 0x11D3 +#define CL_EVENT_CONTEXT 0x11D4 + +#define CL_COMMAND_NDRANGE_KERNEL 0x11F0 +#define CL_COMMAND_READ_BUFFER 0x11F3 +#define CL_COMMAND_WRITE_BUFFER 0x11F4 +#define CL_COMMAND_COPY_BUFFER 0x11F5 +#define CL_COMMAND_FILL_BUFFER 0x1207 +#define CL_COMMAND_MARKER 0x11FE + +#define CL_COMPLETE 0x0 +#define CL_RUNNING 0x1 +#define CL_SUBMITTED 0x2 +#define CL_QUEUED 0x3 + +#define CL_KERNEL_NUM_ARGS 0x1191 +#define CL_KERNEL_ARG_NAME 0x119A +#define CL_KERNEL_ATTRIBUTES 0x1195 +#define CL_KERNEL_COMPILE_WORK_GROUP_SIZE 0x11B1 + +// nVidia +#define CL_DEVICE_COMPUTE_CAPABILITY_MAJOR_NV 0x4000 +#define CL_DEVICE_COMPUTE_CAPABILITY_MINOR_NV 0x4001 + +// AMD-specific; the shim answers CL_DEVICE_TOPOLOGY_AMD from the CUDA device's PCI bus id +#define CL_DEVICE_PCIE_ID_AMD 0x4034 +#define CL_DEVICE_TOPOLOGY_AMD 0x4037 +#define CL_DEVICE_TOPOLOGY_TYPE_PCIE_AMD 1 +#define CL_DEVICE_BOARD_NAME_AMD 0x4038 +#define CL_DEVICE_GLOBAL_FREE_MEMORY_AMD 0x4039 + +using cl_device_topology_amd = union { + struct { u32 type; u32 data[5]; } raw; + struct { u32 type; char unused[17]; char bus; char device; char function; } pcie; +}; + +// Error codes +#define CL_DEVICE_NOT_FOUND -1 +#define CL_DEVICE_NOT_AVAILABLE -2 +#define CL_COMPILER_NOT_AVAILABLE -3 +#define CL_MEM_OBJECT_ALLOCATION_FAILURE -4 +#define CL_OUT_OF_RESOURCES -5 +#define CL_OUT_OF_HOST_MEMORY -6 +#define CL_PROFILING_INFO_NOT_AVAILABLE -7 +#define CL_BUILD_PROGRAM_FAILURE -11 +#define CL_COMPILE_PROGRAM_FAILURE -15 +#define CL_LINK_PROGRAM_FAILURE -17 +#define CL_INVALID_VALUE -30 +#define CL_INVALID_DEVICE -33 +#define CL_INVALID_CONTEXT -34 +#define CL_INVALID_MEM_OBJECT -38 +#define CL_INVALID_BINARY -42 +#define CL_INVALID_BUILD_OPTIONS -43 +#define CL_INVALID_PROGRAM -44 +#define CL_INVALID_KERNEL_NAME -46 +#define CL_INVALID_KERNEL -48 +#define CL_INVALID_ARG_INDEX -49 +#define CL_INVALID_ARG_VALUE -50 +#define CL_INVALID_ARG_SIZE -51 +#define CL_INVALID_WORK_GROUP_SIZE -54 +#define CL_INVALID_GLOBAL_WORK_SIZE -63 + +// ---- OpenCL API function declarations ---- +// These are implemented in clwrap_cuda.cpp using CUDA Driver API. +// They match the signatures from tinycl.h so clwrap.h compiles unchanged. + +extern "C" { + +unsigned clGetPlatformIDs(unsigned, cl_platform_id*, unsigned*); +int clGetDeviceIDs(cl_platform_id, cl_device_type, unsigned, cl_device_id*, unsigned*); +cl_context clCreateContext(const intptr_t*, unsigned, const cl_device_id*, + void (*)(const char*, const void*, size_t, void*), void*, int*); +int clReleaseContext(cl_context); +int clReleaseProgram(cl_program); +int clReleaseCommandQueue(cl_command_queue); +int clEnqueueNDRangeKernel(cl_command_queue, cl_kernel, unsigned, const size_t*, + const size_t*, const size_t*, unsigned, const cl_event*, cl_event*); + +cl_program clCreateProgramWithSource(cl_context, unsigned, const char**, const size_t*, int*); +cl_program clCreateProgramWithBinary(cl_context, unsigned, const cl_device_id*, const size_t*, + const unsigned char**, int*, int*); + +int clBuildProgram(cl_program, unsigned, const cl_device_id*, const char*, + void (*)(cl_program, void*), void*); +int clCompileProgram(cl_program, unsigned, const cl_device_id*, const char*, + unsigned numHeaders, const cl_program* headers, const char* const* headerNames, + void (*)(cl_program, void*), void*); +cl_program clLinkProgram(cl_context, unsigned, const cl_device_id*, const char*, + unsigned nProgs, const cl_program* progs, + void (*)(cl_program, void*), void*, int* err); + +int clGetProgramBuildInfo(cl_program, cl_device_id, cl_program_build_info, size_t, void*, size_t*); +int clGetProgramInfo(cl_program, cl_program_info, size_t, void*, size_t*); +int clGetDeviceInfo(cl_device_id, cl_device_info, size_t, void*, size_t*); +int clGetPlatformInfo(cl_platform_id, cl_device_info, size_t, void*, size_t*); +int clGetCommandQueueInfo(cl_command_queue, cl_command_queue_info, size_t, void*, size_t*); + +cl_kernel clCreateKernel(cl_program, const char*, int*); +int clReleaseKernel(cl_kernel); +cl_mem clCreateBuffer(cl_context, cl_mem_flags, size_t, void*, int*); +int clReleaseMemObject(cl_mem); +cl_command_queue clCreateCommandQueueWithProperties(cl_context, cl_device_id, + const cl_queue_properties*, int*); + +int clEnqueueMarkerWithWaitList(cl_command_queue, unsigned, const cl_event*, cl_event*); +int clEnqueueReadBuffer(cl_command_queue, cl_mem, cl_bool, size_t, size_t, void*, + unsigned, const cl_event*, cl_event*); +int clEnqueueWriteBuffer(cl_command_queue, cl_mem, cl_bool, size_t, size_t, const void*, + unsigned, const cl_event*, cl_event*); +int clEnqueueCopyBuffer(cl_command_queue, cl_mem, cl_mem, size_t, size_t, size_t, + unsigned, const cl_event*, cl_event*); +int clEnqueueFillBuffer(cl_command_queue, cl_mem, const void*, size_t, size_t, size_t, + unsigned, const cl_event*, cl_event*); + +int clFlush(cl_command_queue); +int clFinish(cl_command_queue); +int clSetKernelArg(cl_kernel, unsigned, size_t, const void*); + +int clReleaseEvent(cl_event); +int clWaitForEvents(unsigned, const cl_event*); + +int clGetKernelInfo(cl_kernel, cl_kernel_info, size_t, void*, size_t*); +int clGetKernelArgInfo(cl_kernel, unsigned, cl_kernel_arg_info, size_t, void*, size_t*); +int clGetKernelWorkGroupInfo(cl_kernel, cl_device_id, cl_kernel_work_group_info, size_t, void*, size_t*); + +int clGetEventInfo(cl_event, cl_event_info, size_t, void*, size_t*); +int clGetEventProfilingInfo(cl_event, cl_profiling_info, size_t, void*, size_t*); + +void* clSVMAlloc(cl_context, cl_svm_mem_flags, size_t, unsigned); +void clSVMFree(cl_context, void*); +int clSetKernelArgSVMPointer(cl_kernel, unsigned, const void*); + +} + +// OpenCL-like extensions invented to provide a clean interface to some nVidia CUDA features + +// cl_graph provides an openCL-like interface for CUgraph and CUgraphExec pair +struct _cl_graph { + CUgraph graph; + CUgraphExec graphExec; + cl_command_queue queue; // Queue to launch the graph on (same as queue that graph was recorded on) + _cl_graph() : graph{}, graphExec{} {} + ~_cl_graph() { if (graph) cuGraphDestroy(graph); if (graphExec) cuGraphExecDestroy(graphExec); } +}; +typedef _cl_graph* cl_graph; +bool clIsGraphSupported(cl_device_id); +int clGraphBeginRecording(cl_command_queue); +int clGraphEndRecording(cl_command_queue, cl_graph*); +int clGraphLaunch(cl_graph); +int clReleaseGraph(cl_graph); diff --git a/src/fftbpw.h b/src/fftbpw.h index c62db530..5bab17d0 100644 --- a/src/fftbpw.h +++ b/src/fftbpw.h @@ -1,208 +1,208 @@ // FFT64 - Computed by targeting Z=28 -{ "256:2:256", {19.204, 19.547, 19.636, 19.204, 19.547, 19.636}}, -{ "256:3:256", {19.106, 19.386, 19.369, 19.106, 19.399, 19.361}}, -{ "256:4:256", {18.928, 19.236, 19.322, 18.954, 19.272, 19.367}}, -{ "256:5:256", {19.093, 19.094, 19.242, 19.147, 19.142, 19.314}}, -{ "256:6:256", {18.805, 19.065, 19.005, 18.854, 19.134, 19.101}}, -{ "256:7:256", {18.688, 18.995, 18.971, 18.763, 19.068, 19.071}}, -{ "256:8:256", {18.600, 18.909, 18.964, 18.679, 19.018, 19.120}}, -{ "512:4:256", {18.729, 18.913, 19.062, 18.770, 18.963, 19.120}}, -{ "256:9:256", {18.634, 18.871, 18.802, 18.688, 18.981, 18.961}}, -{"256:10:256", {18.748, 18.770, 18.906, 18.895, 18.896, 19.051}}, -{ "512:5:256", {18.751, 18.766, 18.891, 18.812, 18.832, 18.966}}, -{"256:11:256", {18.523, 18.782, 18.791, 18.594, 18.910, 18.946}}, -{"256:12:256", {18.558, 18.749, 18.669, 18.612, 18.880, 18.842}}, -{ "512:6:256", {18.641, 18.748, 18.838, 18.686, 18.809, 18.949}}, -{"256:13:256", {18.423, 18.693, 18.794, 18.497, 18.820, 18.938}}, -{"256:14:256", {18.450, 18.671, 18.639, 18.519, 18.808, 18.785}}, -{ "512:7:256", {18.547, 18.671, 18.782, 18.629, 18.738, 18.895}}, -{"256:15:256", {18.666, 18.652, 18.628, 18.798, 18.784, 18.791}}, -{"256:16:256", {18.277, 18.568, 18.649, 18.425, 18.741, 18.849}}, -{ "512:8:256", {18.455, 18.592, 18.682, 18.508, 18.673, 18.835}}, -{ "512:4:512", {18.565, 18.599, 18.642, 18.594, 18.629, 18.716}}, -{ "512:9:256", {18.491, 18.579, 18.648, 18.546, 18.661, 18.792}}, -{ "1K:5:256", {18.553, 18.547, 18.713, 18.622, 18.601, 18.762}}, -{"512:10:256", {18.461, 18.479, 18.570, 18.561, 18.572, 18.697}}, -{ "512:5:512", {18.450, 18.473, 18.513, 18.499, 18.513, 18.583}}, -{"512:11:256", {18.335, 18.481, 18.586, 18.444, 18.579, 18.744}}, -{ "1K:6:256", {18.294, 18.528, 18.476, 18.335, 18.596, 18.562}}, -{"512:12:256", {18.381, 18.462, 18.537, 18.448, 18.560, 18.678}}, -{ "512:6:512", {18.440, 18.462, 18.585, 18.479, 18.504, 18.644}}, -{"512:13:256", {18.247, 18.401, 18.482, 18.348, 18.497, 18.645}}, -{ "1K:7:256", {18.171, 18.455, 18.442, 18.238, 18.531, 18.521}}, -{"512:14:256", {18.266, 18.376, 18.467, 18.374, 18.490, 18.637}}, -{ "512:7:512", {18.372, 18.372, 18.462, 18.428, 18.432, 18.567}}, -{"512:15:256", {18.351, 18.355, 18.460, 18.453, 18.465, 18.619}}, -{ "1K:8:256", {18.093, 18.359, 18.435, 18.156, 18.461, 18.570}}, -{"512:16:256", {18.108, 18.260, 18.298, 18.223, 18.420, 18.570}}, -{ "512:8:512", {18.256, 18.280, 18.314, 18.319, 18.369, 18.444}}, -{ "1K:9:256", {18.123, 18.328, 18.245, 18.171, 18.443, 18.426}}, -{ "512:9:512", {18.243, 18.262, 18.343, 18.315, 18.342, 18.493}}, -{ "1K:10:256", {18.233, 18.228, 18.370, 18.357, 18.343, 18.516}}, -{ "1K:5:512", {18.237, 18.232, 18.385, 18.281, 18.272, 18.479}}, -{"512:10:512", {18.142, 18.160, 18.174, 18.226, 18.243, 18.314}}, -{ "1K:11:256", {18.001, 18.234, 18.243, 18.084, 18.362, 18.411}}, -{"512:11:512", {18.156, 18.170, 18.196, 18.242, 18.251, 18.356}}, -{ "1K:12:256", {18.052, 18.225, 18.151, 18.090, 18.332, 18.291}}, -{ "1K:6:512", {18.125, 18.227, 18.293, 18.160, 18.283, 18.401}}, -{"512:12:512", {18.141, 18.163, 18.245, 18.217, 18.237, 18.409}}, -{ "1K:13:256", {17.903, 18.155, 18.249, 17.980, 18.261, 18.380}}, -{"512:13:512", {18.097, 18.104, 18.135, 18.168, 18.171, 18.239}}, -{ "1K:14:256", {17.929, 18.143, 18.103, 18.005, 18.268, 18.244}}, -{ "1K:7:512", {18.043, 18.150, 18.237, 18.100, 18.208, 18.347}}, -{"512:14:512", {18.094, 18.091, 18.154, 18.171, 18.173, 18.311}}, -{ "1K:15:256", {18.131, 18.127, 18.115, 18.260, 18.236, 18.238}}, -{"512:15:512", {18.049, 18.061, 18.112, 18.130, 18.136, 18.236}}, -{ "1K:16:256", {17.752, 18.019, 18.128, 17.892, 18.183, 18.326}}, -{ "1K:8:512", {17.930, 18.069, 18.190, 17.982, 18.137, 18.333}}, -{"512:16:512", {17.971, 17.997, 17.976, 18.072, 18.100, 18.170}}, -{ "1K:9:512", {17.944, 18.049, 18.124, 18.012, 18.122, 18.247}}, -{ "1K:10:512", {17.940, 17.939, 18.044, 18.044, 18.043, 18.195}}, -{ "1K:5:1K", {18.033, 18.016, 18.180, 18.101, 18.072, 18.237}}, -{ "1K:11:512", {17.829, 17.956, 18.078, 17.922, 18.044, 18.219}}, -{ "1K:12:512", {17.829, 17.944, 18.002, 17.912, 18.034, 18.139}}, -{ "1K:6:1K", {17.741, 18.003, 17.931, 17.794, 18.065, 18.051}}, -{ "1K:13:512", {17.744, 17.869, 18.010, 17.822, 17.959, 18.133}}, -{ "1K:14:512", {17.748, 17.854, 17.933, 17.848, 17.968, 18.100}}, -{ "1K:7:1K", {17.647, 17.914, 17.865, 17.714, 17.995, 17.994}}, -{ "1K:15:512", {17.828, 17.824, 17.943, 17.926, 17.934, 18.100}}, -{ "1K:16:512", {17.613, 17.744, 17.813, 17.711, 17.875, 18.073}}, -{ "1K:8:1K", {17.572, 17.810, 17.914, 17.628, 17.923, 18.046}}, -{ "1K:9:1K", {17.600, 17.807, 17.734, 17.636, 17.908, 17.892}}, -{ "1K:10:1K", {17.709, 17.681, 17.841, 17.823, 17.795, 17.981}}, -{ "1K:11:1K", {17.458, 17.682, 17.712, 17.560, 17.811, 17.891}}, -{ "1K:12:1K", {17.474, 17.698, 17.635, 17.560, 17.807, 17.752}}, -{ "1K:13:1K", {17.388, 17.619, 17.706, 17.465, 17.716, 17.853}}, -{ "1K:14:1K", {17.417, 17.627, 17.582, 17.491, 17.734, 17.715}}, -{ "1K:15:1K", {17.612, 17.592, 17.582, 17.720, 17.697, 17.722}}, -{ "1K:16:1K", {17.220, 17.471, 17.615, 17.369, 17.641, 17.778}}, -{ "4K:9:512", {17.457, 17.468, 17.550, 17.519, 17.531, 17.648}}, -{ "4K:10:512", {17.336, 17.336, 17.364, 17.416, 17.433, 17.507}}, -{ "4K:11:512", {17.351, 17.356, 17.393, 17.437, 17.440, 17.548}}, -{ "4K:12:512", {17.351, 17.362, 17.447, 17.420, 17.437, 17.576}}, -{ "4K:13:512", {17.278, 17.271, 17.324, 17.349, 17.351, 17.441}}, -{ "4K:14:512", {17.267, 17.270, 17.342, 17.359, 17.359, 17.503}}, -{ "4K:15:512", {17.238, 17.239, 17.295, 17.305, 17.315, 17.432}}, -{ "4K:16:512", {17.149, 17.163, 17.143, 17.251, 17.271, 17.352}}, -{ "4K:9:1K", {17.130, 17.225, 17.346, 17.189, 17.291, 17.485}}, -{ "4K:10:1K", {17.110, 17.111, 17.209, 17.199, 17.188, 17.351}}, -{ "4K:11:1K", {16.993, 17.108, 17.214, 17.084, 17.196, 17.405}}, -{ "4K:12:1K", {17.006, 17.123, 17.219, 17.101, 17.201, 17.370}}, -{ "4K:13:1K", {16.932, 17.045, 17.154, 17.002, 17.116, 17.298}}, -{ "4K:14:1K", {16.942, 17.055, 17.160, 17.027, 17.127, 17.306}}, -{ "4K:15:1K", {17.021, 17.007, 17.137, 17.104, 17.087, 17.282}}, -{ "4K:16:1K", {16.744, 16.887, 16.966, 16.921, 17.048, 17.208}}, +{ "256:2:256", {19.204f, 19.547f, 19.636f, 19.204f, 19.547f, 19.636f}}, +{ "256:3:256", {19.106f, 19.386f, 19.369f, 19.106f, 19.399f, 19.361f}}, +{ "256:4:256", {18.928f, 19.236f, 19.322f, 18.954f, 19.272f, 19.367f}}, +{ "256:5:256", {19.093f, 19.094f, 19.242f, 19.147f, 19.142f, 19.314f}}, +{ "256:6:256", {18.805f, 19.065f, 19.005f, 18.854f, 19.134f, 19.101f}}, +{ "256:7:256", {18.688f, 18.995f, 18.971f, 18.763f, 19.068f, 19.071f}}, +{ "256:8:256", {18.600f, 18.909f, 18.964f, 18.679f, 19.018f, 19.120f}}, +{ "512:4:256", {18.729f, 18.913f, 19.062f, 18.770f, 18.963f, 19.120f}}, +{ "256:9:256", {18.634f, 18.871f, 18.802f, 18.688f, 18.981f, 18.961f}}, +{"256:10:256", {18.748f, 18.770f, 18.906f, 18.895f, 18.896f, 19.051f}}, +{ "512:5:256", {18.751f, 18.766f, 18.891f, 18.812f, 18.832f, 18.966f}}, +{"256:11:256", {18.523f, 18.782f, 18.791f, 18.594f, 18.910f, 18.946f}}, +{"256:12:256", {18.558f, 18.749f, 18.669f, 18.612f, 18.880f, 18.842f}}, +{ "512:6:256", {18.641f, 18.748f, 18.838f, 18.686f, 18.809f, 18.949f}}, +{"256:13:256", {18.423f, 18.693f, 18.794f, 18.497f, 18.820f, 18.938f}}, +{"256:14:256", {18.450f, 18.671f, 18.639f, 18.519f, 18.808f, 18.785f}}, +{ "512:7:256", {18.547f, 18.671f, 18.782f, 18.629f, 18.738f, 18.895f}}, +{"256:15:256", {18.666f, 18.652f, 18.628f, 18.798f, 18.784f, 18.791f}}, +{"256:16:256", {18.277f, 18.568f, 18.649f, 18.425f, 18.741f, 18.849f}}, +{ "512:8:256", {18.455f, 18.592f, 18.682f, 18.508f, 18.673f, 18.835f}}, +{ "512:4:512", {18.565f, 18.599f, 18.642f, 18.594f, 18.629f, 18.716f}}, +{ "512:9:256", {18.491f, 18.579f, 18.648f, 18.546f, 18.661f, 18.792f}}, +{ "1K:5:256", {18.553f, 18.547f, 18.713f, 18.622f, 18.601f, 18.762f}}, +{"512:10:256", {18.461f, 18.479f, 18.570f, 18.561f, 18.572f, 18.697f}}, +{ "512:5:512", {18.450f, 18.473f, 18.513f, 18.499f, 18.513f, 18.583f}}, +{"512:11:256", {18.335f, 18.481f, 18.586f, 18.444f, 18.579f, 18.744f}}, +{ "1K:6:256", {18.294f, 18.528f, 18.476f, 18.335f, 18.596f, 18.562f}}, +{"512:12:256", {18.381f, 18.462f, 18.537f, 18.448f, 18.560f, 18.678f}}, +{ "512:6:512", {18.440f, 18.462f, 18.585f, 18.479f, 18.504f, 18.644f}}, +{"512:13:256", {18.247f, 18.401f, 18.482f, 18.348f, 18.497f, 18.645f}}, +{ "1K:7:256", {18.171f, 18.455f, 18.442f, 18.238f, 18.531f, 18.521f}}, +{"512:14:256", {18.266f, 18.376f, 18.467f, 18.374f, 18.490f, 18.637f}}, +{ "512:7:512", {18.372f, 18.372f, 18.462f, 18.428f, 18.432f, 18.567f}}, +{"512:15:256", {18.351f, 18.355f, 18.460f, 18.453f, 18.465f, 18.619f}}, +{ "1K:8:256", {18.093f, 18.359f, 18.435f, 18.156f, 18.461f, 18.570f}}, +{"512:16:256", {18.108f, 18.260f, 18.298f, 18.223f, 18.420f, 18.570f}}, +{ "512:8:512", {18.256f, 18.280f, 18.314f, 18.319f, 18.369f, 18.444f}}, +{ "1K:9:256", {18.123f, 18.328f, 18.245f, 18.171f, 18.443f, 18.426f}}, +{ "512:9:512", {18.243f, 18.262f, 18.343f, 18.315f, 18.342f, 18.493f}}, +{ "1K:10:256", {18.233f, 18.228f, 18.370f, 18.357f, 18.343f, 18.516f}}, +{ "1K:5:512", {18.237f, 18.232f, 18.385f, 18.281f, 18.272f, 18.479f}}, +{"512:10:512", {18.142f, 18.160f, 18.174f, 18.226f, 18.243f, 18.314f}}, +{ "1K:11:256", {18.001f, 18.234f, 18.243f, 18.084f, 18.362f, 18.411f}}, +{"512:11:512", {18.156f, 18.170f, 18.196f, 18.242f, 18.251f, 18.356f}}, +{ "1K:12:256", {18.052f, 18.225f, 18.151f, 18.090f, 18.332f, 18.291f}}, +{ "1K:6:512", {18.125f, 18.227f, 18.293f, 18.160f, 18.283f, 18.401f}}, +{"512:12:512", {18.141f, 18.163f, 18.245f, 18.217f, 18.237f, 18.409f}}, +{ "1K:13:256", {17.903f, 18.155f, 18.249f, 17.980f, 18.261f, 18.380f}}, +{"512:13:512", {18.097f, 18.104f, 18.135f, 18.168f, 18.171f, 18.239f}}, +{ "1K:14:256", {17.929f, 18.143f, 18.103f, 18.005f, 18.268f, 18.244f}}, +{ "1K:7:512", {18.043f, 18.150f, 18.237f, 18.100f, 18.208f, 18.347f}}, +{"512:14:512", {18.094f, 18.091f, 18.154f, 18.171f, 18.173f, 18.311f}}, +{ "1K:15:256", {18.131f, 18.127f, 18.115f, 18.260f, 18.236f, 18.238f}}, +{"512:15:512", {18.049f, 18.061f, 18.112f, 18.130f, 18.136f, 18.236f}}, +{ "1K:16:256", {17.752f, 18.019f, 18.128f, 17.892f, 18.183f, 18.326f}}, +{ "1K:8:512", {17.930f, 18.069f, 18.190f, 17.982f, 18.137f, 18.333f}}, +{"512:16:512", {17.971f, 17.997f, 17.976f, 18.072f, 18.100f, 18.170f}}, +{ "1K:9:512", {17.944f, 18.049f, 18.124f, 18.012f, 18.122f, 18.247f}}, +{ "1K:10:512", {17.940f, 17.939f, 18.044f, 18.044f, 18.043f, 18.195f}}, +{ "1K:5:1K", {18.033f, 18.016f, 18.180f, 18.101f, 18.072f, 18.237f}}, +{ "1K:11:512", {17.829f, 17.956f, 18.078f, 17.922f, 18.044f, 18.219f}}, +{ "1K:12:512", {17.829f, 17.944f, 18.002f, 17.912f, 18.034f, 18.139f}}, +{ "1K:6:1K", {17.741f, 18.003f, 17.931f, 17.794f, 18.065f, 18.051f}}, +{ "1K:13:512", {17.744f, 17.869f, 18.010f, 17.822f, 17.959f, 18.133f}}, +{ "1K:14:512", {17.748f, 17.854f, 17.933f, 17.848f, 17.968f, 18.100f}}, +{ "1K:7:1K", {17.647f, 17.914f, 17.865f, 17.714f, 17.995f, 17.994f}}, +{ "1K:15:512", {17.828f, 17.824f, 17.943f, 17.926f, 17.934f, 18.100f}}, +{ "1K:16:512", {17.613f, 17.744f, 17.813f, 17.711f, 17.875f, 18.073f}}, +{ "1K:8:1K", {17.572f, 17.810f, 17.914f, 17.628f, 17.923f, 18.046f}}, +{ "1K:9:1K", {17.600f, 17.807f, 17.734f, 17.636f, 17.908f, 17.892f}}, +{ "1K:10:1K", {17.709f, 17.681f, 17.841f, 17.823f, 17.795f, 17.981f}}, +{ "1K:11:1K", {17.458f, 17.682f, 17.712f, 17.560f, 17.811f, 17.891f}}, +{ "1K:12:1K", {17.474f, 17.698f, 17.635f, 17.560f, 17.807f, 17.752f}}, +{ "1K:13:1K", {17.388f, 17.619f, 17.706f, 17.465f, 17.716f, 17.853f}}, +{ "1K:14:1K", {17.417f, 17.627f, 17.582f, 17.491f, 17.734f, 17.715f}}, +{ "1K:15:1K", {17.612f, 17.592f, 17.582f, 17.720f, 17.697f, 17.722f}}, +{ "1K:16:1K", {17.220f, 17.471f, 17.615f, 17.369f, 17.641f, 17.778f}}, +{ "4K:9:512", {17.457f, 17.468f, 17.550f, 17.519f, 17.531f, 17.648f}}, +{ "4K:10:512", {17.336f, 17.336f, 17.364f, 17.416f, 17.433f, 17.507f}}, +{ "4K:11:512", {17.351f, 17.356f, 17.393f, 17.437f, 17.440f, 17.548f}}, +{ "4K:12:512", {17.351f, 17.362f, 17.447f, 17.420f, 17.437f, 17.576f}}, +{ "4K:13:512", {17.278f, 17.271f, 17.324f, 17.349f, 17.351f, 17.441f}}, +{ "4K:14:512", {17.267f, 17.270f, 17.342f, 17.359f, 17.359f, 17.503f}}, +{ "4K:15:512", {17.238f, 17.239f, 17.295f, 17.305f, 17.315f, 17.432f}}, +{ "4K:16:512", {17.149f, 17.163f, 17.143f, 17.251f, 17.271f, 17.352f}}, +{ "4K:9:1K", {17.130f, 17.225f, 17.346f, 17.189f, 17.291f, 17.485f}}, +{ "4K:10:1K", {17.110f, 17.111f, 17.209f, 17.199f, 17.188f, 17.351f}}, +{ "4K:11:1K", {16.993f, 17.108f, 17.214f, 17.084f, 17.196f, 17.405f}}, +{ "4K:12:1K", {17.006f, 17.123f, 17.219f, 17.101f, 17.201f, 17.370f}}, +{ "4K:13:1K", {16.932f, 17.045f, 17.154f, 17.002f, 17.116f, 17.298f}}, +{ "4K:14:1K", {16.942f, 17.055f, 17.160f, 17.027f, 17.127f, 17.306f}}, +{ "4K:15:1K", {17.021f, 17.007f, 17.137f, 17.104f, 17.087f, 17.282f}}, +{ "4K:16:1K", {16.744f, 16.887f, 16.966f, 16.921f, 17.048f, 17.208f}}, // FFT3161 - Computed by targeting maxROE of ~0.35 over 1000 iterations, probably could go higher -{ "1:256:2:256", {40.54, 40.54, 40.54, 40.54, 40.54, 40.54}}, -{ "1:256:4:256", {40.19, 40.19, 40.19, 40.19, 40.19, 40.19}}, -{ "1:256:8:256", {39.98, 39.98, 39.98, 39.98, 39.98, 39.98}}, -{ "1:512:4:256", {39.98, 39.98, 39.98, 39.98, 39.98, 39.98}}, -{"1:256:16:256", {39.67, 39.67, 39.67, 39.67, 39.67, 39.67}}, -{ "1:512:8:256", {39.67, 39.67, 39.67, 39.67, 39.67, 39.67}}, -{ "1:512:4:512", {39.67, 39.67, 39.67, 39.67, 39.67, 39.67}}, -{ "1:1K:8:256", {39.46, 39.46, 39.46, 39.46, 39.46, 39.46}}, -{"1:512:16:256", {39.46, 39.46, 39.46, 39.46, 39.46, 39.46}}, -{ "1:512:8:512", {39.46, 39.46, 39.46, 39.46, 39.46, 39.46}}, -{ "1:1K:16:256", {39.15, 39.15, 39.15, 39.15, 39.15, 39.15}}, -{ "1:1K:8:512", {39.15, 39.15, 39.15, 39.15, 39.15, 39.15}}, -{"1:512:16:512", {39.15, 39.15, 39.15, 39.15, 39.15, 39.15}}, -{ "1:1K:16:512", {38.97, 38.97, 38.97, 38.97, 38.97, 38.97}}, -{ "1:1K:8:1K", {38.97, 38.97, 38.97, 38.97, 38.97, 38.97}}, -{ "1:1K:16:1K", {38.62, 38.62, 38.62, 38.62, 38.62, 38.62}}, -{ "1:4K:16:512", {38.37, 38.37, 38.37, 38.37, 38.37, 38.37}}, -{ "1:4K:16:1K", {38.12, 38.12, 38.12, 38.12, 38.12, 38.12}}, // Estimated +{ "1:256:2:256", {40.54f, 40.54f, 40.54f, 40.54f, 40.54f, 40.54f}}, +{ "1:256:4:256", {40.19f, 40.19f, 40.19f, 40.19f, 40.19f, 40.19f}}, +{ "1:256:8:256", {39.98f, 39.98f, 39.98f, 39.98f, 39.98f, 39.98f}}, +{ "1:512:4:256", {39.98f, 39.98f, 39.98f, 39.98f, 39.98f, 39.98f}}, +{"1:256:16:256", {39.67f, 39.67f, 39.67f, 39.67f, 39.67f, 39.67f}}, +{ "1:512:8:256", {39.67f, 39.67f, 39.67f, 39.67f, 39.67f, 39.67f}}, +{ "1:512:4:512", {39.67f, 39.67f, 39.67f, 39.67f, 39.67f, 39.67f}}, +{ "1:1K:8:256", {39.46f, 39.46f, 39.46f, 39.46f, 39.46f, 39.46f}}, +{"1:512:16:256", {39.46f, 39.46f, 39.46f, 39.46f, 39.46f, 39.46f}}, +{ "1:512:8:512", {39.46f, 39.46f, 39.46f, 39.46f, 39.46f, 39.46f}}, +{ "1:1K:16:256", {39.15f, 39.15f, 39.15f, 39.15f, 39.15f, 39.15f}}, +{ "1:1K:8:512", {39.15f, 39.15f, 39.15f, 39.15f, 39.15f, 39.15f}}, +{"1:512:16:512", {39.15f, 39.15f, 39.15f, 39.15f, 39.15f, 39.15f}}, +{ "1:1K:16:512", {38.97f, 38.97f, 38.97f, 38.97f, 38.97f, 38.97f}}, +{ "1:1K:8:1K", {38.97f, 38.97f, 38.97f, 38.97f, 38.97f, 38.97f}}, +{ "1:1K:16:1K", {38.62f, 38.62f, 38.62f, 38.62f, 38.62f, 38.62f}}, +{ "1:4K:16:512", {38.37f, 38.37f, 38.37f, 38.37f, 38.37f, 38.37f}}, +{ "1:4K:16:1K", {38.12f, 38.12f, 38.12f, 38.12f, 38.12f, 38.12f}}, // Estimated // FFT3261 - Computed with -use TABMUL_CHAIN32=0,TAIL_TRIGS32=0 and targeting maxROE of ~0.35 over 1000 iterations, probably could go higher -{ "2:256:2:256", {34.57, 34.57, 34.57, 34.57, 34.57, 34.57}}, -{ "2:256:4:256", {34.24, 34.24, 34.24, 34.24, 34.24, 34.24}}, -{ "2:256:8:256", {34.06, 34.06, 34.06, 34.06, 34.06, 34.06}}, -{ "2:512:4:256", {34.06, 34.06, 34.06, 34.06, 34.06, 34.06}}, -{"2:256:16:256", {32.07, 32.07, 32.07, 32.07, 32.07, 32.07}}, -{ "2:512:8:256", {32.07, 32.07, 32.07, 32.07, 32.07, 32.07}}, -{ "2:512:4:512", {32.07, 32.07, 32.07, 32.07, 32.07, 32.07}}, -{ "2:1K:8:256", {31.81, 31.81, 31.81, 31.81, 31.81, 31.81}}, -{"2:512:16:256", {31.81, 31.81, 31.81, 31.81, 31.81, 31.81}}, -{ "2:512:8:512", {31.81, 31.81, 31.81, 31.81, 31.81, 31.81}}, -{ "2:1K:16:256", {31.50, 31.50, 31.50, 31.50, 31.50, 31.50}}, -{ "2:1K:8:512", {31.50, 31.50, 31.50, 31.50, 31.50, 31.50}}, -{"2:512:16:512", {31.50, 31.50, 31.50, 31.50, 31.50, 31.50}}, -{ "2:1K:16:512", {28.68, 28.68, 28.68, 28.68, 28.68, 28.68}}, // Very strange. 481421001 has a maxROE of 0.180, 481422001 has a maxROE of 0.5 -{ "2:1K:8:1K", {28.68, 28.68, 28.68, 28.68, 28.68, 28.68}}, -{ "2:1K:16:1K", {25.37, 25.37, 25.37, 25.37, 25.37, 25.37}}, // Also strange. 851422001 ROEmax=0.273, ROEavg=0.003 -{ "2:4K:16:512", {23.27, 23.27, 23.27, 23.27, 23.27, 23.27}}, -{ "2:4K:16:1K", {21.15, 21.15, 21.15, 21.15, 21.15, 21.15}}, // Estimated +{ "2:256:2:256", {34.53f, 34.53f, 34.53f, 34.53f, 34.53f, 34.53f}}, +{ "2:256:4:256", {34.24f, 34.24f, 34.24f, 34.24f, 34.24f, 34.24f}}, +{ "2:256:8:256", {33.89f, 33.89f, 33.89f, 33.89f, 33.89f, 33.89f}}, +{ "2:512:4:256", {33.89f, 33.89f, 33.89f, 33.89f, 33.89f, 33.89f}}, +{"2:256:16:256", {31.95f, 31.95f, 31.95f, 31.95f, 31.95f, 31.95f}}, +{ "2:512:8:256", {31.95f, 31.95f, 31.95f, 31.95f, 31.95f, 31.95f}}, +{ "2:512:4:512", {31.95f, 31.95f, 31.95f, 31.95f, 31.95f, 31.95f}}, +{ "2:1K:8:256", {31.66f, 31.66f, 31.66f, 31.66f, 31.66f, 31.66f}}, +{"2:512:16:256", {31.66f, 31.66f, 31.66f, 31.66f, 31.66f, 31.66f}}, +{ "2:512:8:512", {31.66f, 31.66f, 31.66f, 31.66f, 31.66f, 31.66f}}, +{ "2:1K:16:256", {31.35f, 31.35f, 31.35f, 31.35f, 31.35f, 31.35f}}, +{ "2:1K:8:512", {31.35f, 31.35f, 31.35f, 31.35f, 31.35f, 31.35f}}, +{"2:512:16:512", {31.35f, 31.35f, 31.35f, 31.35f, 31.35f, 31.35f}}, +{ "2:1K:16:512", {29.48f, 29.48f, 29.48f, 29.48f, 29.48f, 29.48f}}, +{ "2:1K:8:1K", {29.53f, 29.53f, 29.53f, 29.53f, 29.53f, 29.53f}}, +{ "2:1K:16:1K", {29.21f, 29.21f, 29.21f, 29.21f, 29.21f, 29.21f}}, +{ "2:4K:16:512", {28.94f, 28.94f, 28.94f, 28.94f, 28.94f, 28.94f}}, +{ "2:4K:16:1K", {28.64f, 28.64f, 28.64f, 28.64f, 28.64f, 28.64f}}, // Estimated // FFT61 - Computed by targeting maxROE of ~0.35 over 1000 iterations, probably could go higher -{ "3:256:2:256", {25.02, 25.02, 25.02, 25.02, 25.02, 25.02}}, -{ "3:256:4:256", {24.72, 24.10, 24.10, 24.10, 24.10, 24.10}}, -{ "3:256:8:256", {24.46, 24.46, 24.46, 24.46, 24.46, 24.46}}, -{ "3:512:4:256", {24.46, 24.46, 24.46, 24.46, 24.46, 24.46}}, -{"3:256:16:256", {24.15, 24.15, 24.15, 24.15, 24.15, 24.15}}, -{ "3:512:8:256", {24.15, 24.15, 24.15, 24.15, 24.15, 24.15}}, -{ "3:512:4:512", {24.15, 24.15, 24.15, 24.15, 24.15, 24.15}}, -{ "3:1K:8:256", {23.94, 23.94, 23.94, 23.94, 23.94, 23.94}}, -{"3:512:16:256", {23.94, 23.94, 23.94, 23.94, 23.94, 23.94}}, -{ "3:512:8:512", {23.94, 23.94, 23.94, 23.94, 23.94, 23.94}}, -{ "3:1K:16:256", {23.65, 23.65, 23.65, 23.65, 23.65, 23.65}}, -{ "3:1K:8:512", {23.65, 23.65, 23.65, 23.65, 23.65, 23.65}}, -{"3:512:16:512", {23.65, 23.65, 23.65, 23.65, 23.65, 23.65}}, -{ "3:1K:16:512", {23.42, 23.42, 23.42, 23.42, 23.42, 23.42}}, -{ "3:1K:8:1K", {23.42, 23.42, 23.42, 23.42, 23.42, 23.42}}, -{ "3:1K:16:1K", {23.13, 23.13, 23.13, 23.13, 23.13, 23.13}}, -{ "3:4K:16:512", {22.92, 22.92, 22.92, 22.92, 22.92, 22.92}}, -{ "3:4K:16:1K", {22.72, 22.72, 22.72, 22.72, 22.72, 22.72}}, // Estimated +{ "3:256:2:256", {25.02f, 25.02f, 25.02f, 25.02f, 25.02f, 25.02f}}, +{ "3:256:4:256", {24.72f, 24.10f, 24.10f, 24.10f, 24.10f, 24.10f}}, +{ "3:256:8:256", {24.46f, 24.46f, 24.46f, 24.46f, 24.46f, 24.46f}}, +{ "3:512:4:256", {24.46f, 24.46f, 24.46f, 24.46f, 24.46f, 24.46f}}, +{"3:256:16:256", {24.15f, 24.15f, 24.15f, 24.15f, 24.15f, 24.15f}}, +{ "3:512:8:256", {24.15f, 24.15f, 24.15f, 24.15f, 24.15f, 24.15f}}, +{ "3:512:4:512", {24.15f, 24.15f, 24.15f, 24.15f, 24.15f, 24.15f}}, +{ "3:1K:8:256", {23.84f, 23.84f, 23.84f, 23.84f, 23.84f, 23.84f}}, // LL of 100028317 failed (ROEmax=0.294, ROEavg=0.247). Lowering bpw from 23.94 to 23.84. +{"3:512:16:256", {23.84f, 23.84f, 23.84f, 23.84f, 23.84f, 23.84f}}, +{ "3:512:8:512", {23.84f, 23.84f, 23.84f, 23.84f, 23.84f, 23.84f}}, +{ "3:1K:16:256", {23.65f, 23.65f, 23.65f, 23.65f, 23.65f, 23.65f}}, +{ "3:1K:8:512", {23.65f, 23.65f, 23.65f, 23.65f, 23.65f, 23.65f}}, +{"3:512:16:512", {23.65f, 23.65f, 23.65f, 23.65f, 23.65f, 23.65f}}, +{ "3:1K:16:512", {23.42f, 23.42f, 23.42f, 23.42f, 23.42f, 23.42f}}, +{ "3:1K:8:1K", {23.42f, 23.42f, 23.42f, 23.42f, 23.42f, 23.42f}}, +{ "3:1K:16:1K", {23.13f, 23.13f, 23.13f, 23.13f, 23.13f, 23.13f}}, +{ "3:4K:16:512", {22.92f, 22.92f, 22.92f, 22.92f, 22.92f, 22.92f}}, +{ "3:4K:16:1K", {22.72f, 22.72f, 22.72f, 22.72f, 22.72f, 22.72f}}, // Estimated // FFT323161 - Computed with -use TABMUL_CHAIN32=0,TAIL_TRIGS32=0 and targeting maxROE of ~0.35 over 1000 iterations, probably could go higher -{ "4:256:2:256", {50.05, 50.05, 50.05, 50.05, 50.05, 50.05}}, -{ "4:256:4:256", {49.76, 49.76, 49.76, 49.76, 49.76, 49.76}}, -{ "4:256:8:256", {49.59, 49.59, 49.59, 49.59, 49.59, 49.59}}, -{ "4:512:4:256", {49.59, 49.59, 49.59, 49.59, 49.59, 49.59}}, -{"4:256:16:256", {47.59, 47.59, 47.59, 47.59, 47.59, 47.59}}, -{ "4:512:8:256", {47.59, 47.59, 47.59, 47.59, 47.59, 47.59}}, -{ "4:512:4:512", {47.59, 47.59, 47.59, 47.59, 47.59, 47.59}}, -{ "4:1K:8:256", {47.33, 47.33, 47.33, 47.33, 47.33, 47.33}}, -{"4:512:16:256", {47.33, 47.33, 47.33, 47.33, 47.33, 47.33}}, -{ "4:512:8:512", {47.33, 47.33, 47.33, 47.33, 47.33, 47.33}}, -{ "4:1K:16:256", {47.00, 47.00, 47.00, 47.00, 47.00, 47.00}}, -{ "4:1K:8:512", {47.00, 47.00, 47.00, 47.00, 47.00, 47.00}}, -{"4:512:16:512", {47.00, 47.00, 47.00, 47.00, 47.00, 47.00}}, -{ "4:1K:16:512", {44.52, 44.52, 44.52, 44.52, 44.52, 44.52}}, -{ "4:1K:8:1K", {44.52, 44.52, 44.52, 44.52, 44.52, 44.52}}, -{ "4:1K:16:1K", {41.72, 41.72, 41.72, 41.72, 41.72, 41.72}}, // Strange 41.72 has tiny error, 41.75 is 0.5 -{ "4:4K:16:512", {39.50, 39.50, 39.50, 39.50, 39.50, 39.50}}, // Estimated -{ "4:4K:16:1K", {37.50, 37.50, 37.50, 37.50, 37.50, 37.50}}, // Estimated +{ "4:256:2:256", {50.01f, 50.01f, 50.01f, 50.01f, 50.01f, 50.01f}}, +{ "4:256:4:256", {49.71f, 49.71f, 49.71f, 49.71f, 49.71f, 49.71f}}, +{ "4:256:8:256", {49.52f, 49.52f, 49.52f, 49.52f, 49.52f, 49.52f}}, +{ "4:512:4:256", {49.52f, 49.52f, 49.52f, 49.52f, 49.52f, 49.52f}}, +{"4:256:16:256", {47.55f, 47.55f, 47.55f, 47.55f, 47.55f, 47.55f}}, +{ "4:512:8:256", {47.55f, 47.55f, 47.55f, 47.55f, 47.55f, 47.55f}}, +{ "4:512:4:512", {47.55f, 47.55f, 47.55f, 47.55f, 47.55f, 47.55f}}, +{ "4:1K:8:256", {47.29f, 47.29f, 47.29f, 47.29f, 47.29f, 47.29f}}, +{"4:512:16:256", {47.29f, 47.29f, 47.29f, 47.29f, 47.29f, 47.29f}}, +{ "4:512:8:512", {47.29f, 47.29f, 47.29f, 47.29f, 47.29f, 47.29f}}, +{ "4:1K:16:256", {47.00f, 47.00f, 47.00f, 47.00f, 47.00f, 47.00f}}, +{ "4:1K:8:512", {47.00f, 47.00f, 47.00f, 47.00f, 47.00f, 47.00f}}, +{"4:512:16:512", {47.00f, 47.00f, 47.00f, 47.00f, 47.00f, 47.00f}}, +{ "4:1K:16:512", {45.02f, 45.02f, 45.02f, 45.02f, 45.02f, 45.02f}}, +{ "4:1K:8:1K", {45.02f, 45.02f, 45.02f, 45.02f, 45.02f, 45.02f}}, +{ "4:1K:16:1K", {44.70f, 44.70f, 44.70f, 44.70f, 44.70f, 44.70f}}, +{ "4:4K:16:512", {44.42f, 44.42f, 44.42f, 44.42f, 44.42f, 44.42f}}, +{ "4:4K:16:1K", {44.12f, 44.12f, 44.12f, 44.12f, 44.12f, 44.12f}}, // Estimated // FFT3231 - Computed with -use TABMUL_CHAIN32=0,TAIL_TRIGS32=0 and targeting maxROE of ~0.35 over 1000 iterations, probably could go higher -{ "50:256:2:256", {19.57, 19.57, 19.57, 19.57, 19.57, 19.57}}, -{ "50:256:4:256", {19.23, 19.23, 19.23, 19.23, 19.23, 19.23}}, -{ "50:256:8:256", {19.07, 19.07, 19.07, 19.07, 19.07, 19.07}}, -{ "50:512:4:256", {19.07, 19.07, 19.07, 19.07, 19.07, 19.07}}, -{"50:256:16:256", {17.07, 17.07, 17.07, 17.07, 17.07, 17.07}}, -{ "50:512:8:256", {17.07, 17.07, 17.07, 17.07, 17.07, 17.07}}, -{ "50:512:4:512", {17.07, 17.07, 17.07, 17.07, 17.07, 17.07}}, -{ "50:1K:8:256", {16.78, 16.78, 16.78, 16.78, 16.78, 16.78}}, -{"50:512:16:256", {16.78, 16.78, 16.78, 16.78, 16.78, 16.78}}, -{ "50:512:8:512", {16.78, 16.78, 16.78, 16.78, 16.78, 16.78}}, -{ "50:1K:16:256", {16.52, 16.52, 16.52, 16.52, 16.52, 16.52}}, -{ "50:1K:8:512", {16.52, 16.52, 16.52, 16.52, 16.52, 16.52}}, -{"50:512:16:512", {16.52, 16.52, 16.52, 16.52, 16.52, 16.52}}, -{ "50:1K:16:512", {14.01, 14.01, 14.01, 14.01, 14.01, 14.01}}, -{ "50:1K:8:1K", {14.01, 14.01, 14.01, 14.01, 14.01, 14.01}}, -{ "50:1K:16:1K", {11.21, 11.21, 11.21, 11.21, 11.21, 11.21}}, // Estimated -{ "50:4K:16:512", {9.15, 9.15, 9.15, 9.15, 9.15, 9.15}}, // Estimated -{ "50:4K:16:1K", {7.05, 7.05, 7.05, 7.05, 7.05, 7.05}}, // Estimated +{ "50:256:2:256", {19.57f, 19.57f, 19.57f, 19.57f, 19.57f, 19.57f}}, +{ "50:256:4:256", {19.23f, 19.23f, 19.23f, 19.23f, 19.23f, 19.23f}}, +{ "50:256:8:256", {19.07f, 19.07f, 19.07f, 19.07f, 19.07f, 19.07f}}, +{ "50:512:4:256", {19.07f, 19.07f, 19.07f, 19.07f, 19.07f, 19.07f}}, +{"50:256:16:256", {17.07f, 17.07f, 17.07f, 17.07f, 17.07f, 17.07f}}, +{ "50:512:8:256", {17.07f, 17.07f, 17.07f, 17.07f, 17.07f, 17.07f}}, +{ "50:512:4:512", {17.07f, 17.07f, 17.07f, 17.07f, 17.07f, 17.07f}}, +{ "50:1K:8:256", {16.78f, 16.78f, 16.78f, 16.78f, 16.78f, 16.78f}}, +{"50:512:16:256", {16.78f, 16.78f, 16.78f, 16.78f, 16.78f, 16.78f}}, +{ "50:512:8:512", {16.78f, 16.78f, 16.78f, 16.78f, 16.78f, 16.78f}}, +{ "50:1K:16:256", {16.52f, 16.52f, 16.52f, 16.52f, 16.52f, 16.52f}}, +{ "50:1K:8:512", {16.52f, 16.52f, 16.52f, 16.52f, 16.52f, 16.52f}}, +{"50:512:16:512", {16.52f, 16.52f, 16.52f, 16.52f, 16.52f, 16.52f}}, +{ "50:1K:16:512", {14.01f, 14.01f, 14.01f, 14.01f, 14.01f, 14.01f}}, +{ "50:1K:8:1K", {14.01f, 14.01f, 14.01f, 14.01f, 14.01f, 14.01f}}, +{ "50:1K:16:1K", {11.21f, 11.21f, 11.21f, 11.21f, 11.21f, 11.21f}}, // Estimated +{ "50:4K:16:512", {9.15f, 9.15f, 9.15f, 9.15f, 9.15f, 9.15f}}, // Estimated +{ "50:4K:16:1K", {7.05f, 7.05f, 7.05f, 7.05f, 7.05f, 7.05f}}, // Estimated // FFT6431 - Computed with variant 202 and targeting maxROE of ~0.35 over 1000 iterations, probably could go higher -{ "51:256:2:256", {35.27, 35.27, 35.27, 35.27, 35.27, 35.27}}, -{ "51:256:4:256", {34.98, 34.98, 34.98, 34.98, 34.98, 34.98}}, -{ "51:256:8:256", {34.61, 34.61, 34.61, 34.61, 34.61, 34.61}}, -{ "51:512:4:256", {34.61, 34.61, 34.61, 34.61, 34.61, 34.61}}, -{"51:256:16:256", {34.33, 34.33, 34.33, 34.33, 34.33, 34.33}}, -{ "51:512:8:256", {34.33, 34.33, 34.33, 34.33, 34.33, 34.33}}, -{ "51:512:4:512", {34.33, 34.33, 34.33, 34.33, 34.33, 34.33}}, -{ "51:1K:8:256", {33.97, 33.97, 33.97, 33.97, 33.97, 33.97}}, -{"51:512:16:256", {33.97, 33.97, 33.97, 33.97, 33.97, 33.97}}, -{ "51:512:8:512", {33.97, 33.97, 33.97, 33.97, 33.97, 33.97}}, -{ "51:1K:16:256", {33.64, 33.64, 33.64, 33.64, 33.64, 33.64}}, -{ "51:1K:8:512", {33.64, 33.64, 33.64, 33.64, 33.64, 33.64}}, -{"51:512:16:512", {33.64, 33.64, 33.64, 33.64, 33.64, 33.64}}, -{ "51:1K:16:512", {33.45, 33.45, 33.45, 33.45, 33.45, 33.45}}, -{ "51:1K:8:1K", {33.45, 33.45, 33.45, 33.45, 33.45, 33.45}}, -{ "51:1K:16:1K", {33.23, 33.23, 33.23, 33.23, 33.23, 33.23}}, -{ "51:4K:16:512", {32.75, 32.75, 32.75, 32.75, 32.75, 32.75}}, -{ "51:4K:16:1K", {32.25, 32.25, 32.25, 32.25, 32.25, 32.25}}, // Estimated +{ "51:256:2:256", {35.27f, 35.27f, 35.27f, 35.27f, 35.27f, 35.27f}}, +{ "51:256:4:256", {34.98f, 34.98f, 34.98f, 34.98f, 34.98f, 34.98f}}, +{ "51:256:8:256", {34.61f, 34.61f, 34.61f, 34.61f, 34.61f, 34.61f}}, +{ "51:512:4:256", {34.61f, 34.61f, 34.61f, 34.61f, 34.61f, 34.61f}}, +{"51:256:16:256", {34.33f, 34.33f, 34.33f, 34.33f, 34.33f, 34.33f}}, +{ "51:512:8:256", {34.33f, 34.33f, 34.33f, 34.33f, 34.33f, 34.33f}}, +{ "51:512:4:512", {34.33f, 34.33f, 34.33f, 34.33f, 34.33f, 34.33f}}, +{ "51:1K:8:256", {33.97f, 33.97f, 33.97f, 33.97f, 33.97f, 33.97f}}, +{"51:512:16:256", {33.97f, 33.97f, 33.97f, 33.97f, 33.97f, 33.97f}}, +{ "51:512:8:512", {33.97f, 33.97f, 33.97f, 33.97f, 33.97f, 33.97f}}, +{ "51:1K:16:256", {33.64f, 33.64f, 33.64f, 33.64f, 33.64f, 33.64f}}, +{ "51:1K:8:512", {33.64f, 33.64f, 33.64f, 33.64f, 33.64f, 33.64f}}, +{"51:512:16:512", {33.64f, 33.64f, 33.64f, 33.64f, 33.64f, 33.64f}}, +{ "51:1K:16:512", {33.45f, 33.45f, 33.45f, 33.45f, 33.45f, 33.45f}}, +{ "51:1K:8:1K", {33.45f, 33.45f, 33.45f, 33.45f, 33.45f, 33.45f}}, +{ "51:1K:16:1K", {33.23f, 33.23f, 33.23f, 33.23f, 33.23f, 33.23f}}, +{ "51:4K:16:512", {32.75f, 32.75f, 32.75f, 32.75f, 32.75f, 32.75f}}, +{ "51:4K:16:1K", {32.25f, 32.25f, 32.25f, 32.25f, 32.25f, 32.25f}}, // Estimated diff --git a/src/fs.cpp b/src/fs.cpp index 7f7ab719..7a4c4d6f 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -4,6 +4,7 @@ #include "File.h" #include +#include namespace { @@ -53,16 +54,20 @@ void fancyRename(const fs::path& src, const fs::path& dst) { u64 fileSize(const fs::path& path) { error_code dummy; auto size = fs::file_size(path, dummy); - if (size == decltype(size)(-1)) { size = 0; } + if (std::cmp_equal(size, -1)) { size = 0; } return size; } bool deleteLine(const fs::path& path, const string& targetLine, u64 initialSize) { if (!initialSize) { initialSize = fileSize(path); } - fs::path tmp = path + ("-"s + toString(this_thread::get_id())); + fs::path const tmp = path + ("-"s + toString(this_thread::get_id())); - if (!copyWithout(targetLine, path, tmp) || !sizeMatches(path, initialSize)) { return false; } + if (!copyWithout(targetLine, path, tmp) || !sizeMatches(path, initialSize)) { + error_code ec; + fs::remove(tmp, ec); // do not leave "-" behind on every failed attempt + return false; + } fancyRename(tmp, path); return true; diff --git a/src/gpuid.cpp b/src/gpuid.cpp index 8a24590d..f27be4f3 100644 --- a/src/gpuid.cpp +++ b/src/gpuid.cpp @@ -1,13 +1,14 @@ // Copyright (C) 2017-2024 Mihai Preda. #include "gpuid.h" +#include #include "clwrap.h" #include "File.h" using namespace std; static bool startsWith(string_view a, string_view b) { - return a.substr(0, b.length()) == b; + return a.starts_with(b); } string getBdfFromSysfs(int pos) { @@ -58,18 +59,18 @@ string getUidFromSysfs(int pos) { /* BDF is PCIe Bus:Device.Function e.g. "6a:00.0" */ string getUidFromBdf(const string& bdf) { - int pos = getSysfsFromBdf(bdf); + int const pos = getSysfsFromBdf(bdf); return pos >= 0 ? getUidFromSysfs(pos) : ""; } string getBdfFromUid(const string& uid) { - int pos = getSysfsFromUid(uid); + int const pos = getSysfsFromUid(uid); return (pos >= 0) ? getBdfFromSysfs(pos) : ""; } int getPosFromBdf(const string& bdf) { auto openclIds = getAllDeviceIDs(); - for (int pos = 0; pos < int(openclIds.size()); ++pos) { + for (int pos = 0; std::cmp_less(pos, openclIds.size()); ++pos) { auto bdfAtPos = getBdfFromDevice(openclIds[pos]); // log("BDF '%s' at %d\n", bdfAtPos.c_str(), pos); if (bdf == bdfAtPos) { return pos; } @@ -85,7 +86,7 @@ string getBdfFromPos(int pos) { } int getPosFromUid(const string& uid) { - string bdf = getBdfFromUid(uid); + string const bdf = getBdfFromUid(uid); if (bdf.empty()) { return -1; } return getPosFromBdf(bdf); } diff --git a/src/log.cpp b/src/log.cpp index 1b93ee57..a5177de8 100644 --- a/src/log.cpp +++ b/src/log.cpp @@ -10,8 +10,10 @@ thread_local string context; thread_local vector contextParts; thread_local File logFile; +// A worker's log file adopted by a helper thread (LogLinkScope); borrowed, never owned. +thread_local File* linkedLogFile = nullptr; -File stdoutFile{stdout, "stdout"}; +static File stdoutFile{stdout, "stdout"}; string logContext() { return context; } @@ -20,7 +22,7 @@ void initLog(const char *logName) { logFile = File::openAppend(logName); } -string longTimeStr() { return timeStr("%Y-%m-%d %H:%M:%S %Z"); } +[[maybe_unused]] static string longTimeStr() { return timeStr("%Y-%m-%d %H:%M:%S %Z"); } string shortTimeStr() { return timeStr("%Y%m%d %H:%M:%S"); } static char logBuf[32 * 1024]; @@ -28,9 +30,9 @@ static char logBuf[32 * 1024]; void log(const char *fmt, ...) { static std::mutex logMutex; - string prefix = shortTimeStr() + ' ' + context; + string const prefix = shortTimeStr() + ' ' + context; - std::unique_lock lock(logMutex); + std::unique_lock const lock(logMutex); int pos = 0; snprintf(logBuf, sizeof(logBuf), "%s %n", prefix.c_str(), &pos); @@ -38,12 +40,18 @@ void log(const char *fmt, ...) { va_start(va, fmt); vsnprintf(logBuf + pos, sizeof(logBuf) - pos, fmt, va); va_end(va); - string_view s{logBuf}; + string_view const s{logBuf}; - if (logFile) { logFile.write(s); } + if (logFile) { logFile.write(s); } else if (linkedLogFile && *linkedLogFile) { linkedLogFile->write(s); } stdoutFile.write(s); } +LogLink logLink() { return {logFile ? &logFile : linkedLogFile, context}; } + +LogLinkScope::LogLinkScope(const LogLink& link) : previous{linkedLogFile}, context{link.context} { linkedLogFile = link.file; } + +LogLinkScope::~LogLinkScope() { linkedLogFile = previous; } + LogContext::LogContext(const string& s) : part{s} { contextParts.push_back(s); context += s; diff --git a/src/log.h b/src/log.h index a49f5808..f25dc5a9 100644 --- a/src/log.h +++ b/src/log.h @@ -22,3 +22,24 @@ struct LogContext { private: std::string part; }; + +// The log file and context are thread-local (one log per worker instance), +// so a helper thread starts with neither and its log() lines reach stdout +// alone, without the exponent prefix. A thread that works on a worker's +// behalf — KernelCompiler's parallel compile under CUDA — takes a LogLink +// from the thread that starts it and adopts it for the task's duration. +class File; +struct LogLink { + File* file; // the starting thread's log file, borrowed — it outlives the task + std::string context; +}; +LogLink logLink(); + +struct LogLinkScope { + explicit LogLinkScope(const LogLink& link); + ~LogLinkScope(); + +private: + File* previous; + LogContext context; +}; diff --git a/src/main.cpp b/src/main.cpp index b62f1b32..74ce6ad7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -16,11 +16,20 @@ #include "Gpu.h" #include "tune.h" +#include +#include #include #include +#include // #include from GCC-13 onwards -void gpuWorker(GpuCommon shared, Queue *q, i32 instance) { +// Set when a worker dies on an exception, so that main() can report the failure through the exit code +// even though the other workers (and the process) carry on to a normal end. +static std::atomic workerFailed{false}; + +static bool isCleanExit(const char* reason); + +static void gpuWorker(GpuCommon shared, i32 instance) { // LogContext context{(instance ? shared.args->tailDir() : ""s) + to_string(instance) + ' '}; // log("Starting worker %d\n", instance); if (instance > 0) { @@ -29,13 +38,16 @@ void gpuWorker(GpuCommon shared, Queue *q, i32 instance) { } try { - while (auto task = Worktodo::getTask(*shared.args, instance)) { task->execute(shared, q, instance); } + while (auto task = Worktodo::getTask(*shared.args, instance)) { task->execute(shared, instance); } } catch (const char *mes) { log("Exception \"%s\"\n", mes); + if (!isCleanExit(mes)) { workerFailed = true; } } catch (const string& mes) { log("Exception \"%s\"\n", mes.c_str()); + if (!isCleanExit(mes.c_str())) { workerFailed = true; } } catch (const std::exception& e) { log("Exception %s: %s\n", typeName(e), e.what()); + workerFailed = true; } } @@ -44,21 +56,37 @@ void gpuWorker(GpuCommon shared, Queue *q, i32 instance) { extern int putenv(char *); #endif +// The exceptions that end a run on purpose: the user's stop, and the two +// flags that only print. Everything else thrown to main() is a failure. +static bool isCleanExit(const char *reason) { + return !strcmp(reason, "stop requested") || !strcmp(reason, "help") || !strcmp(reason, "version"); +} + int main(int argc, char **argv) { +//!MSVC version support +#ifdef _MSC_VER + _set_printf_count_output(1); // I'm not sure what this does (it's from CrazeTheDragon) +#endif -#if defined(__MSYS__) +#ifdef __MSYS__ // I was unable to get putenv to link in MSYS2 #elif defined(__MINGW32__) || defined(__MINGW64__) putenv("ROC_SIGNAL_POOL_SIZE=32"); +#elif defined(_WIN32) + _putenv_s("ROC_SIGNAL_POOL_SIZE", "32"); // For MSVC #else // Required to work around a ROCm bug when using multiple queues setenv("ROC_SIGNAL_POOL_SIZE", "32", 0); #endif + // 0 for a normal end — the queue ran dry, a stop was requested, -h or + // -version — and 1 for an exception nobody else classified (a kernel that + // would not compile, a missing device, a bad argument), so a supervisor + // can tell "out of work" from "cannot run" without parsing the log. int exitCode = 0; try { - string mainLine = Args::mergeArgs(argc, argv); + string const mainLine = Args::mergeArgs(argc, argv); { Args args{true}; args.parse(mainLine); @@ -88,17 +116,17 @@ int main(int argc, char **argv) { if (args.maxAlloc) { AllocTrac::setMaxAlloc(args.maxAlloc); } Context context(getDevice(args.device)); - Signal signal; + Signal const signal; Background background; GpuCommon shared; + shared.context = &context; shared.args = &args; TrigBufCache bufCache{&context}; shared.bufCache = &bufCache; shared.background = &background; if (args.doCtune || args.doTune || args.doZtune || args.carryTune) { - Queue q(context, args.profile); - Tune tune{&q, shared}; + Tune tune{shared}; if (args.doCtune) { tune.ctune(); @@ -111,23 +139,28 @@ int main(int argc, char **argv) { } } else { { - vector queues; - for (int i = 0; i < int(args.workers); ++i) { queues.emplace_back(context, args.profile); } vector threads; - for (int i = 1; i < int(args.workers); ++i) { - threads.emplace_back(gpuWorker, shared, &queues[i], i); + for (int i = 1; std::cmp_less(i, args.workers); ++i) { + threads.emplace_back(gpuWorker, shared, i); } - gpuWorker(shared, &queues[0], 0); + gpuWorker(shared, 0); } // log("No more work. Add work to worktodo.txt , see -h for details.\n"); } } catch (const char *mes) { log("Exiting because \"%s\"\n", mes); + exitCode = isCleanExit(mes) ? 0 : 1; } catch (const string& mes) { log("Exiting because \"%s\"\n", mes.c_str()); + exitCode = isCleanExit(mes.c_str()) ? 0 : 1; + } catch (const std::exception& e) { + log("Exiting because of exception %s: %s\n", typeName(e), e.what()); + exitCode = 1; } + if (workerFailed && exitCode == 0) { exitCode = 1; } + log("Bye\n"); - return exitCode; // not used yet. + return exitCode; } diff --git a/src/md5.cpp b/src/md5.cpp index 2bf8329f..bbecb022 100644 --- a/src/md5.cpp +++ b/src/md5.cpp @@ -5,21 +5,21 @@ * This code is in the public domain; do with it what you wish. */ #include "MD5.h" -#include +#include #define byteReverse(A,B) /* The four core functions - F1 is optimized somewhat */ /* #define F1(x, y, z) (x & y | ~x & z) */ -#define F1(x, y, z) (z ^ (x & (y ^ z))) +#define F1(x, y, z) ((z) ^ ((x) & ((y) ^ (z)))) #define F2(x, y, z) F1(z, x, y) -#define F3(x, y, z) (x ^ y ^ z) -#define F4(x, y, z) (y ^ (x | ~z)) +#define F3(x, y, z) ((x) ^ (y) ^ (z)) +#define F4(x, y, z) ((y) ^ ((x) | ~(z))) /* This is the central step in the MD5 algorithm. */ #define MD5STEP(f, w, x, y, z, data, s) \ - ( w += f(x, y, z) + data, w = w<>(32-s), w += x ) + ( (w) += f(x, y, z) + (data), (w) = (w)<<(s) | (w)>>(32-(s)), (w) += (x) ) /* * The core of the MD5 algorithm, this alters an existing MD5 hash to @@ -132,7 +132,7 @@ void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){ /* Update bitcount */ t = ctx->bits[0]; - if ((ctx->bits[0] = t + ((unsigned)len << 3)) < t) + if ((ctx->bits[0] = t + (len << 3)) < t) ctx->bits[1]++; /* Carry from low to high */ ctx->bits[1] += len >> 29; diff --git a/src/sha3.cpp b/src/sha3.cpp index cf0d8b10..48c4d2c0 100644 --- a/src/sha3.cpp +++ b/src/sha3.cpp @@ -90,7 +90,7 @@ static void KeccakF1600Step(SHA3Context *p){ # define A42 (p->u.s[22]) # define A43 (p->u.s[23]) # define A44 (p->u.s[24]) -# define ROL64(a,x) ((a<>(64-x))) +# define ROL64(a,x) (((a)<<(x))|((a)>>(64-(x)))) for(i=0; i<24; i+=4){ C0 = A00^A10^A20^A30^A40; @@ -395,7 +395,7 @@ void SHA3Update( ){ unsigned int i = 0; #if SHA3_BYTEORDER==1234 - if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ + if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)nullptr)&7)==0 ){ for(; i+7u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; p->nLoaded += 8; diff --git a/src/shared.h b/src/shared.h index c2d90dbc..92899b23 100644 --- a/src/shared.h +++ b/src/shared.h @@ -1,4 +1,5 @@ // included from both C++ and OpenCL. -u32 bitposToWord(u32 E, u32 N, u32 offset) { return offset * ((u64) N) / E; } -u32 wordToBitpos(u32 E, u32 N, u32 word) { return (word * ((u64) E) + (N - 1)) / N; } +// NO LONGER USED -- and if it were would need recoding for 64-bit exponents +//inline u32 bitposToWord(u64 E, u32 N, u32 offset) { return offset * ((u64) N) / E; } +//inline u32 wordToBitpos(u64 E, u32 N, u32 word) { return (word * (E) + (N - 1)) / N; } diff --git a/src/state.cpp b/src/state.cpp index 4ac159d8..637aa1cf 100644 --- a/src/state.cpp +++ b/src/state.cpp @@ -1,19 +1,14 @@ // Copyright 2017 Mihai Preda. #include "state.h" -#include "shared.h" -#include "log.h" -#include "timeutil.h" - #include -#include static i64 lowBits(i64 u, int bits) { return (u << (64 - bits)) >> (64 - bits); } -std::vector compactBits(const vector &dataVect, u32 E) { +std::vector compactBits(const vector &dataVect, u64 E) { if (dataVect.empty()) { return {}; } // Indicating all zero - u32 N = dataVect.size(); + u32 const N = u32(dataVect.size()); const Word *data = dataVect.data(); std::vector out; @@ -29,14 +24,14 @@ std::vector compactBits(const vector &dataVect, u32 E) { assert(nBits > 0); // Be careful adding in the carry -- it could overflow a 32-bit word. Convert value into desired unsigned range. - i64 tmp = (i64) data[p] + carry; + i64 const tmp = (i64) data[p] + carry; carry = (int) (tmp >> nBits); u64 w = (u64) (tmp - ((i64) carry << nBits)); assert(w < (1ULL << nBits)); assert(haveBits < 32); while (nBits) { - int needBits = 32 - haveBits; + int const needBits = 32 - haveBits; outWord |= w << haveBits; if (nBits >= needBits) { w >>= needBits; @@ -56,7 +51,7 @@ std::vector compactBits(const vector &dataVect, u32 E) { out.push_back(outWord); for (int p = 0; carry; ++p) { - i64 v = i64(out[p]) + carry; + i64 const v = i64(out[p]) + carry; out[p] = v & 0xffffffff; carry = v >> 32; } @@ -66,10 +61,10 @@ std::vector compactBits(const vector &dataVect, u32 E) { } struct BitBucket { - u128 bits; - u32 size; + u128 bits{0}; + u32 size{0}; - BitBucket() : bits(0), size(0) {} + BitBucket() = default; void put32(u32 b) { assert(size <= 96); @@ -79,7 +74,7 @@ struct BitBucket { i64 popSigned(u32 n) { assert(size >= n); - i64 b = lowBits((i64) bits, n); + i64 const b = lowBits((i64) bits, n); size -= n; bits >>= n; bits += (b < 0); // carry fixup. @@ -87,7 +82,7 @@ struct BitBucket { } }; -vector expandBits(const vector &compactBits, u32 N, u32 E) { +vector expandBits(const vector &compactBits, u32 N, u64 E) { assert(E % 32 != 0); std::vector out(N); @@ -97,7 +92,7 @@ vector expandBits(const vector &compactBits, u32 N, u32 E) { auto it = compactBits.cbegin(); [[maybe_unused]] auto itEnd = compactBits.cend(); for (u32 p = 0; p < N; ++p) { - u32 len = bitlen(N, E, p); + u32 const len = bitlen(N, E, p); while (bucket.size < len) { assert(it != itEnd); bucket.put32(*it++); @@ -107,6 +102,6 @@ vector expandBits(const vector &compactBits, u32 N, u32 E) { assert(it == itEnd); assert(bucket.size == 32 - E % 32); assert(bucket.bits == 0 || bucket.bits == 1); - data[0] += bucket.bits; // carry wrap-around. + data[0] += u32(bucket.bits); // carry wrap-around. return out; } diff --git a/src/state.h b/src/state.h index 9b37c0fd..7254e05c 100644 --- a/src/state.h +++ b/src/state.h @@ -8,10 +8,10 @@ #include #include -vector compactBits(const vector &dataVect, u32 E); -vector expandBits(const vector &compactBits, u32 N, u32 E); +vector compactBits(const vector &dataVect, u64 E); +vector expandBits(const vector &compactBits, u32 N, u64 E); -constexpr u32 step(u32 N, u32 E) { return N - (E % N); } -constexpr u32 extra(u32 N, u32 E, u32 k) { return u64(step(N, E)) * k % N; } -constexpr bool isBigWord(u32 N, u32 E, u32 k) { return extra(N, E, k) + step(N, E) < N; } -constexpr u32 bitlen(u32 N, u32 E, u32 k) { return E / N + isBigWord(N, E, k); } +constexpr u32 step(u32 N, u64 E) { return N - (E % N); } +constexpr u32 extra(u32 N, u64 E, u32 k) { return u64(step(N, E)) * k % N; } +constexpr bool isBigWord(u32 N, u64 E, u32 k) { return extra(N, E, k) + step(N, E) < N; } +constexpr u32 bitlen(u32 N, u64 E, u32 k) { return u32(E / N) + isBigWord(N, E, k); } diff --git a/src/timeutil.cpp b/src/timeutil.cpp index f3102145..f5fa1f8b 100644 --- a/src/timeutil.cpp +++ b/src/timeutil.cpp @@ -5,14 +5,14 @@ #include std::string timeStr(const char *format) { - time_t t = time(NULL); + time_t const t = time(nullptr); char buf[64]; strftime(buf, sizeof(buf), format, localtime(&t)); return buf; } std::string timeStr() { - time_t t = time(NULL); + time_t const t = time(nullptr); char buf[64]; strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", gmtime(&t)); // equivalent to: "%F %T" return buf; diff --git a/src/timeutil.h b/src/timeutil.h index ecc21c72..e2bbe7db 100644 --- a/src/timeutil.h +++ b/src/timeutil.h @@ -18,11 +18,11 @@ class Timer { Timer() : start(clock::now()) {} - double at() const { return std::chrono::duration(clock::now() - start).count(); } + [[nodiscard]] double at() const { return std::chrono::duration(clock::now() - start).count(); } double reset() { auto now = clock::now(); - double ret = std::chrono::duration(now - start).count(); + double const ret = std::chrono::duration(now - start).count(); start = now; return ret; } diff --git a/src/tinycl.h b/src/tinycl.h index e3317657..bcca2634 100644 --- a/src/tinycl.h +++ b/src/tinycl.h @@ -7,31 +7,31 @@ #include #include -typedef struct _cl_platform_id * cl_platform_id; -typedef struct _cl_device_id * cl_device_id; -typedef struct _cl_context * cl_context; -typedef struct _cl_command_queue * cl_command_queue; -typedef struct _cl_mem * cl_mem; -typedef struct _cl_program * cl_program; -typedef struct _cl_kernel * cl_kernel; -typedef struct _cl_event * cl_event; -typedef struct _cl_sampler * cl_sampler; - -typedef unsigned cl_bool; -typedef unsigned cl_program_build_info; -typedef unsigned cl_program_info; -typedef unsigned cl_device_info; -typedef unsigned cl_kernel_info; -typedef unsigned cl_kernel_arg_info; -typedef unsigned cl_kernel_work_group_info; -typedef unsigned cl_profiling_info; -typedef unsigned cl_event_info; -typedef unsigned cl_command_queue_info; - -typedef u64 cl_mem_flags; -typedef u64 cl_svm_mem_flags; -typedef u64 cl_device_type; -typedef u64 cl_queue_properties; +using cl_platform_id = struct _cl_platform_id *; +using cl_device_id = struct _cl_device_id *; +using cl_context = struct _cl_context *; +using cl_command_queue = struct _cl_command_queue *; +using cl_mem = struct _cl_mem *; +using cl_program = struct _cl_program *; +using cl_kernel = struct _cl_kernel *; +using cl_event = struct _cl_event *; +using cl_sampler = struct _cl_sampler *; + +using cl_bool = unsigned; +using cl_program_build_info = unsigned; +using cl_program_info = unsigned; +using cl_device_info = unsigned; +using cl_kernel_info = unsigned; +using cl_kernel_arg_info = unsigned; +using cl_kernel_work_group_info = unsigned; +using cl_profiling_info = unsigned; +using cl_event_info = unsigned; +using cl_command_queue_info = unsigned; + +using cl_mem_flags = u64; +using cl_svm_mem_flags = u64; +using cl_device_type = u64; +using cl_queue_properties = u64; extern "C" { @@ -95,12 +95,12 @@ int clGetKernelWorkGroupInfo(cl_kernel, cl_device_id, cl_kernel_work_group_info, int clGetEventInfo(cl_event, cl_event_info paramName, size_t paramValueSize, void* paramValue, size_t* sizeRet); int clGetEventProfilingInfo(cl_event, cl_profiling_info, size_t, void*, size_t* sizeRet); - + void* clSVMAlloc(cl_context, cl_svm_mem_flags, size_t, unsigned alignment); void clSVMFree(cl_context, void*); int clSetKernelArgSVMPointer(cl_kernel, unsigned, const void *); - + } #define CL_SUCCESS 0 @@ -207,6 +207,10 @@ int clSetKernelArgSVMPointer(cl_kernel, unsigned, const void *); #define CL_KERNEL_COMPILE_WORK_GROUP_SIZE 0x11B1 +// nVidia +#define CL_DEVICE_COMPUTE_CAPABILITY_MAJOR_NV 0x4000 +#define CL_DEVICE_COMPUTE_CAPABILITY_MINOR_NV 0x4001 + // AMD #define CL_DEVICE_PCIE_ID_AMD 0x4034 #define CL_DEVICE_TOPOLOGY_AMD 0x4037 @@ -316,3 +320,15 @@ typedef union #define CL_INVALID_DEVICE_PARTITION_COUNT -68 #define CL_INVALID_PIPE_SIZE -69 #define CL_INVALID_DEVICE_QUEUE -70 + +// OpenCL-like extensions invented to provide a clean interface to some nVidia CUDA features +// This is only supported by our CUDA translation of openCL. Since this file is for the +// native openCL builds, these extension routines basicly do nothing. + +struct _cl_graph {}; +using cl_graph = struct _cl_graph *; +bool clIsGraphSupported(cl_device_id); +int clGraphBeginRecording(cl_command_queue); +int clGraphEndRecording(cl_command_queue, cl_graph*); +int clGraphLaunch(cl_graph); +int clReleaseGraph(cl_graph); diff --git a/src/tune.cpp b/src/tune.cpp index 3b773437..c23ff146 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -12,11 +12,11 @@ #include #include +#include #include #include #include -using std::accumulate; using namespace std; @@ -24,13 +24,13 @@ vector split(const string& s, char delim) { vector ret; size_t start = 0; while (true) { - size_t p = s.find(delim, start); + size_t const p = s.find(delim, start); if (p == string::npos) { ret.push_back(s.substr(start)); break; - } else { + } ret.push_back(s.substr(start, p - start)); - } + start = p + 1; } return ret; @@ -41,12 +41,12 @@ namespace { vector permute(const vector>>& params) { vector configs; - int n = params.size(); + int const n = int(params.size()); vector vpos(n); while (true) { TuneConfig config; for (int i = 0; i < n; ++i) { - config.push_back({params[i].first, params[i].second[vpos[i]]}); + config.emplace_back(params[i].first, params[i].second[vpos[i]]); } configs.push_back(config); @@ -55,9 +55,9 @@ vector permute(const vector>>& params) { if (vpos[i] < int(params[i].second.size()) - 1) { ++vpos[i]; break; - } else { + } vpos[i] = 0; - } + } if (i < 0) { return configs; } @@ -69,14 +69,14 @@ vector getTuneConfigs(const string& tune) { for (auto& part : split(tune, ';')) { auto keyVal = split(part, '='); assert(keyVal.size() == 2); - string key = keyVal.front(); - string val = keyVal.back(); - params.push_back({key, split(val, ',')}); + string const& key = keyVal.front(); + const string& val = keyVal.back(); + params.emplace_back(key, split(val, ',')); } return permute(params); } -string toString(TuneConfig config) { +string toString(const TuneConfig& config) { string s{}; for (const auto& [k, v] : config) { s += k + '=' + v + ','; } s.pop_back(); @@ -89,7 +89,7 @@ struct Entry { double cost; }; -string formatEntry(Entry e) { +string formatEntry(const Entry& e) { char buf[256]; snprintf(buf, sizeof(buf), "! %s %s # %.0f\n", e.shape.spec().c_str(), toString(e.config).c_str(), e.cost); @@ -115,7 +115,7 @@ float Tune::maxBpw(FFTConfig fft) { // This doesn't need to be a very accurate estimate. // This estimate comes from analyzing a 4M FFT and a 7.5M FFT. // The 4M FFT needed a .015 step, the 7.5M FFT needed a .012 step. - float bpw_step = .015 + (log2(fft.size()) - log2(4.0*1024*1024)) / (log2(7.5*1024*1024) - log2(4.0*1024*1024)) * (.012 - .015); + float bpw_step = float(.015 + (log2(fft.size()) - log2(4.0*1024*1024)) / (log2(7.5*1024*1024) - log2(4.0*1024*1024)) * (.012 - .015)); // Pick a bpw that might be close to Z=34, it is best to err on the high side of Z=34 float bpw1 = fft.maxBpw() - 9 * bpw_step; // Old bpw gave Z=28, we want Z=34 (or more) @@ -133,36 +133,36 @@ float Tune::maxBpw(FFTConfig fft) { // Fine tune our estimate for Z=34 float z1 = zForBpw(bpw1, fft, 1); printf ("Guess bpw for %s is %.2f first Z34 is %.2f\n", fft.spec().c_str(), bpw1, z1); - while (z1 < 31.0 || z1 > 37.0) { - float prev_bpw1 = bpw1; - float prev_z1 = z1; - bpw1 = bpw1 + (z1 - 34) * bpw_step; + while (z1 < 31.0f || z1 > 37.0f) { + float const prev_bpw1 = bpw1; + float const prev_z1 = z1; + bpw1 = bpw1 + (z1 - 34.0f) * bpw_step; z1 = zForBpw(bpw1, fft, 1); printf ("Reguess bpw for %s is %.2f first Z34 is %.2f\n", fft.spec().c_str(), bpw1, z1); bpw_step = - (bpw1 - prev_bpw1) / (z1 - prev_z1); - if (bpw_step < 0.005) bpw_step = 0.005; - if (bpw_step > 0.025) bpw_step = 0.025; + bpw_step = std::max(bpw_step, 0.005f); + bpw_step = std::min(bpw_step, 0.025f); } // Get more samples for this bpw -- average in the sample we already have z1 = (z1 + (sample_size - 1) * zForBpw(bpw1, fft, sample_size - 1)) / sample_size; // Pick a bpw somewhere near Z=22 then fine tune the guess - float bpw2 = bpw1 + (z1 - 22) * bpw_step; + float bpw2 = bpw1 + (z1 - 22.0f) * bpw_step; float z2 = zForBpw(bpw2, fft, 1); printf ("Guess bpw for %s is %.2f first Z22 is %.2f\n", fft.spec().c_str(), bpw2, z2); - while (z2 < 20.0 || z2 > 25.0) { - float prev_bpw2 = bpw2; - float prev_z2 = z2; + while (z2 < 20.0f || z2 > 25.0f) { + float const prev_bpw2 = bpw2; + float const prev_z2 = z2; // bool error_recovery = (z2 <= 0.0); // if (error_recovery) bpw2 -= bpw_step; else - bpw2 = bpw2 + (z2 - 21) * bpw_step; + bpw2 = bpw2 + (z2 - 21.0f) * bpw_step; z2 = zForBpw(bpw2, fft, 1); printf ("Reguess bpw for %s is %.2f first Z22 is %.2f\n", fft.spec().c_str(), bpw2, z2); // if (error_recovery) { if (z2 >= 20.0) break; else continue; } bpw_step = - (bpw2 - prev_bpw2) / (z2 - prev_z2); - if (bpw_step < 0.005) bpw_step = 0.005; - if (bpw_step > 0.025) bpw_step = 0.025; + bpw_step = std::max(bpw_step, 0.005f); + bpw_step = std::min(bpw_step, 0.025f); } // Get more samples for this bpw -- average in the sample we already have @@ -173,11 +173,11 @@ printf ("Reguess bpw for %s is %.2f first Z22 is %.2f\n", fft.spec().c_str(), bp } float Tune::zForBpw(float bpw, FFTConfig fft, u32 count) { - u32 exponent = (count == 1) ? primes.prevPrime(fft.size() * bpw) : primes.nextPrime(fft.size() * bpw); - float total_z = 0.0; + u64 exponent = (count == 1) ? primes.prevPrime(u64(fft.size() * bpw)) : primes.nextPrime(u64(fft.size() * bpw)); + float total_z = 0.0f; for (u32 i = 0; i < count; i++, exponent = primes.nextPrime (exponent + 1)) { - auto [ok, res, roeSq, roeMul] = Gpu::make(q, exponent, shared, fft, {}, false)->measureROE(true); - float z = roeSq.z(); + auto [ok, res, roeSq, roeMul] = Gpu::make(exponent, shared, fft, {}, false)->measureROE(true); + float const z = float(roeSq.z()); total_z += z; log("Zforbpw %.2f (z %.2f) : %s\n", bpw, z, fft.spec().c_str()); if (!ok) { log("Error at bpw %.2f (z %.2f) : %s\n", bpw, z, fft.spec().c_str()); continue; } @@ -187,17 +187,17 @@ log("Zforbpw %.2f (z %.2f) : %s\n", bpw, z, fft.spec().c_str()); } void Tune::ztune() { - File ztune = File::openAppend("ztune.txt"); + File const ztune = File::openAppend("ztune.txt"); ztune.printf("\n// %s\n\n", shortTimeStr().c_str()); // Study a specific shape and variant - if (0) { - FFTShape shape = FFTShape(512, 15, 512); - u32 variant = 202; - u32 sample_size = 5; - FFTConfig fft{shape, variant, CARRY_AUTO}; - for (float bpw = 18.18; bpw < 18.305; bpw += 0.02) { - float z = zForBpw(bpw, fft, sample_size); + if (false) { + FFTShape const shape = FFTShape(FFT64, 512, 15, 512); + u32 const variant = 202; + u32 const sample_size = 5; + FFTConfig const fft{shape, variant, CARRY_AUTO}; + for (float bpw = 18.18f; bpw < 18.305f; bpw += 0.02f) { + float const z = zForBpw(bpw, fft, sample_size); log ("Avg zForBpw %s %.2f %.2f\n", fft.spec().c_str(), bpw, z); } } @@ -207,7 +207,7 @@ void Tune::ztune() { // Over this narrow Z range, linear curve fit should work well. The Z data is noisy, so more samples is better. auto configs = FFTShape::multiSpec(shared.args->fftSpec); - for (FFTShape shape : configs) { + for (FFTShape const shape : configs) { // 4K widths store data on variants 100, 101, 202, 110, 111, 212 u32 bpw_variants[NUM_BPW_ENTRIES] = {000, 101, 202, 10, 111, 212}; @@ -224,10 +224,10 @@ void Tune::ztune() { // Test specific variants needed for the maximum bpw table in fftbpw.h for (u32 j = 0; j < NUM_BPW_ENTRIES; ++j) { - FFTConfig fft{shape, bpw_variants[j], CARRY_AUTO}; + FFTConfig const fft{shape, bpw_variants[j], CARRY_AUTO}; bpw[j] = maxBpw(fft); } - string s = "\""s + shape.spec() + "\""; + string const s = "\""s + shape.spec() + "\""; // ztune.printf("{%12s, {%.3f, %.3f, %.3f, %.3f, %.3f, %.3f}},\n", s.c_str(), bpw[0], bpw[1], bpw[2], bpw[3], bpw[4], bpw[5]); ztune.printf("{%12s, {", s.c_str()); for (u32 j = 0; j < NUM_BPW_ENTRIES; ++j) ztune.printf("%s%.3f", j ? ", " : "", bpw[j]); @@ -236,47 +236,48 @@ void Tune::ztune() { } void Tune::carryTune() { - File fo = File::openAppend("carrytune.txt"); + File const fo = File::openAppend("carrytune.txt"); fo.printf("\n// %s\n\n", shortTimeStr().c_str()); shared.args->flags["STATS"] = "1"; u32 prevSize = 0; - for (FFTShape shape : FFTShape::multiSpec(shared.args->fftSpec)) { - FFTConfig fft{shape, LAST_VARIANT, CARRY_AUTO}; + for (FFTShape const shape : FFTShape::multiSpec(shared.args->fftSpec)) { + FFTConfig const fft{shape, LAST_VARIANT, CARRY_AUTO}; if (prevSize == fft.size()) { continue; } prevSize = fft.size(); vector zv; double m = 0; const float mid = fft.shape.carry32BPW(); - for (float bpw : {mid - 0.05, mid + 0.05}) { - u32 exponent = primes.nearestPrime(fft.size() * bpw); - auto [ok, carry] = Gpu::make(q, exponent, shared, fft, {}, false)->measureCarry(); + for (float const bpw : {mid - 0.05f, mid + 0.05f}) { + u64 const exponent = primes.nearestPrime(u64(fft.size() * bpw)); + auto [ok, carry] = Gpu::make(exponent, shared, fft, {}, false)->measureCarry(); m = carry.max; if (!ok) { log("Error %s at %f\n", fft.spec().c_str(), bpw); } - zv.push_back(carry.z()); + zv.push_back(float(carry.z())); } - float avg = (zv[0] + zv[1]) / 2; - u32 exponent = fft.shape.carry32BPW() * fft.size(); - double pErr100 = -expm1(-exp(-avg) * exponent * 100); + float const avg = (zv[0] + zv[1]) / 2; + u64 const exponent = u64(fft.shape.carry32BPW() * fft.size()); + double const pErr100 = -expm1(-exp(-avg) * exponent * 100); log("%14s %.3f : %.3f (%.3f %.3f) %f %.0f%%\n", fft.spec().c_str(), mid, avg, zv[0], zv[1], m, pErr100 * 100); fo.printf("%f %f\n", log2(fft.size()), avg); } } template -void add(vector& a, const vector& b) { +static void add(vector& a, const vector& b) { a.insert(a.end(), b.begin(), b.end()); } void Tune::ctune() { - Args *args = shared.args; + Args const*args = shared.args; vector ctune = args->ctune; - if (ctune.empty()) { ctune.push_back("IN_WG=256,128,64;IN_SIZEX=32,16,8;OUT_WG=256,128,64;OUT_SIZEX=32,16,8"); } + if (ctune.empty()) { ctune.emplace_back("IN_WG=256,128,64;IN_SIZEX=32,16,8;OUT_WG=256,128,64;OUT_SIZEX=32,16,8"); } vector> configsVect; - for (const string& s : ctune) { + configsVect.reserve(ctune.size()); +for (const string& s : ctune) { configsVect.push_back(getTuneConfigs(s)); } @@ -290,13 +291,13 @@ void Tune::ctune() { log("FFTs: %s\n", str.c_str()); } - for (FFTShape shape : shapes) { - FFTConfig fft{shape, 101, CARRY_32}; - u32 exponent = primes.prevPrime(fft.maxExp()); - // log("tuning %10s with exponent %u\n", fft.shape.spec().c_str(), exponent); + for (FFTShape const shape : shapes) { + FFTConfig const fft{shape, 101, CARRY_32}; + u64 const exponent = primes.prevPrime(fft.maxExp()); + // log("tuning %10s with exponent %" PRIu64 "\n", fft.shape.spec().c_str(), exponent); vector bestPos(configsVect.size()); - Entry best{{1, 1, 1}, {}, 1e9}; + Entry best{.shape={}, .config={}, .cost=1e9}; for (u32 i = 0; i < configsVect.size(); ++i) { for (u32 pos = i ? 1 : 0; pos < configsVect[i].size(); ++pos) { @@ -309,12 +310,12 @@ void Tune::ctune() { for (u32 k = i + 1; k < configsVect.size(); ++k) { add(c, configsVect[k][bestPos[k]]); } - auto cost = Gpu::make(q, exponent, shared, fft, c, false)->timePRP(); + auto cost = Gpu::make(exponent, shared, fft, c, false)->timePRP(); - bool isBest = (cost < best.cost); + bool const isBest = (cost < best.cost); if (isBest) { bestPos[i] = pos; - best = {shape, c, cost}; + best = {.shape=shape, .config=c, .cost=cost}; } log("%c %6.0f : %s %s\n", isBest ? '*' : ' ', cost, shape.spec().c_str(), toString(c).c_str()); @@ -327,28 +328,32 @@ void Tune::ctune() { } // Add better -use settings to list of changes to be made to config.txt -void configsUpdate(double current_cost, double best_cost, double threshold, const char *key, u32 value, vector> &newConfigKeyVals, vector> &suggestedConfigKeyVals) { +static void configsUpdate(double current_cost, double best_cost, double threshold, const char *key, u32 value, vector> &newConfigKeyVals, vector> &suggestedConfigKeyVals) { if (best_cost == current_cost) return; // If best cost is better than current cost by a substantial margin (the threshold) then add the key value pair to suggestedConfigKeyVals if (best_cost < (1.0 - threshold) * current_cost) - newConfigKeyVals.push_back({key, value}); + newConfigKeyVals.emplace_back(key, value); // Otherwise, add the key value pair to newConfigKeyVals else - suggestedConfigKeyVals.push_back({key, value}); + suggestedConfigKeyVals.emplace_back(key, value); } void Tune::tune() { Args *args = shared.args; vector shapes = FFTShape::multiSpec(args->fftSpec); - - // There are some options and variants that are different based on GPU manufacturer - bool AMDGPU = isAmdGpu(q->context->deviceId()); - bool tune_config = 1; - bool time_FFTs = 0; - bool time_NTTs = 0; - bool time_FP32 = 1; - int quick = 7; // Run config from slowest (quick=1) to fastest (quick=10) + // There are some options and variants that are different based on GPU manufacturer + bool const AMDGPU = isAmdGpu(shared.context->deviceId()); + bool const NVIDIAGPU = isNvidiaGpu(shared.context->deviceId()); + int const NO_ASM = args->value("NO_ASM", 0); + + bool tune_config = true; + bool time_FFTs = false; + bool time_NTTs = false; + bool time_FP32 = true; + bool time_FFT6431 = false; + bool time_inplace_only = NVIDIAGPU; // Default is nVidia is better off with INPLACE=1, AMD GPUs need to time extra options used when INPLACE=0 + int quick = 7; // Run config from slowest (quick=1) to fastest (quick=10) u64 min_exponent = 75000000; u64 max_exponent = 350000000; if (!args->fftSpec.empty()) { min_exponent = 0; max_exponent = 1000000000000ull; } @@ -356,19 +361,21 @@ void Tune::tune() { // Parse input args for (const string& s : split(args->tune, ',')) { if (s.empty()) continue; - if (s == "noconfig") tune_config = 0; - if (s == "fp64") time_FFTs = 1; - if (s == "ntt") time_NTTs = 1; - if (s == "nofp32") time_FP32 = 0; + if (s == "noconfig") tune_config = false; + if (s == "fp64") time_FFTs = true; + if (s == "ntt") time_NTTs = true; + if (s == "fp6431") time_FFT6431 = true; // It is rare to have a GPU good at both FP64 and integer ops. TitanV is one. Allow tuning FFT6431. + if (s == "nofp32") time_FP32 = false; // Workaround bug in some openCL compilers that cannot compile our FP32 openCL code + if (s == "inplace") time_inplace_only = true; auto keyVal = split(s, '='); if (keyVal.size() == 2) { - if (keyVal.front() == "quick") quick = stod(keyVal.back()); + if (keyVal.front() == "quick") quick = stoi(keyVal.back()); if (keyVal.front() == "minexp") min_exponent = stoull(keyVal.back()); if (keyVal.front() == "maxexp") max_exponent = stoull(keyVal.back()); } } - if (quick < 1) quick = 1; - if (quick > 10) quick = 10; + quick = std::max(quick, 1); + quick = std::min(quick, 10); // Look for best settings of various options. Append best settings to config.txt. if (tune_config) { @@ -380,13 +387,13 @@ void Tune::tune() { // If user gave us an fft-spec, use that to time options if (!args->fftSpec.empty()) { - defaultShape = &shapes[0]; + defaultShape = shapes.data(); if (shapes[0].fft_type == FFT64) { defaultFFTShape = shapes[0]; - time_FFTs = 1; + time_FFTs = true; } else { defaultNTTShape = shapes[0]; - time_NTTs = 1; + time_NTTs = true; } } // If user specified FP64-timings, time a wavefront exponent using an 7.5M FFT @@ -405,30 +412,30 @@ void Tune::tune() { else { log("Checking whether this GPU is better suited for double-precision FFTs or integer NTTs.\n"); defaultFFTShape = FFTShape(FFT64, 512, 16, 512); - FFTConfig fft{defaultFFTShape, 101, CARRY_32}; - double fp64_time = Gpu::make(q, 141000001, shared, fft, {}, false)->timePRP(quick); + FFTConfig const fft{defaultFFTShape, 101, CARRY_32}; + double const fp64_time = Gpu::make(141000001, shared, fft, {}, false)->timePRP(quick); log("Time for FP64 FFT %12s is %6.1f\n", fft.spec().c_str(), fp64_time); defaultNTTShape = FFTShape(FFT3161, 512, 8, 512); - FFTConfig ntt{defaultNTTShape, 202, CARRY_AUTO}; - double ntt_time = Gpu::make(q, 141000001, shared, ntt, {}, false)->timePRP(quick); + FFTConfig const ntt{defaultNTTShape, 202, CARRY_AUTO}; + double const ntt_time = Gpu::make(141000001, shared, ntt, {}, false)->timePRP(quick); log("Time for M31*M61 NTT %12s is %6.1f\n", ntt.spec().c_str(), ntt_time); if (fp64_time < ntt_time) { defaultShape = &defaultFFTShape; - time_FFTs = 1; + time_FFTs = true; if (fp64_time < 0.80 * ntt_time) { log("FP64 FFTs are significantly faster than integer NTTs. No NTT tuning will be performed.\n"); } else { log("FP64 FFTs are not significantly faster than integer NTTs. NTT tuning will be performed.\n"); - time_NTTs = 1; + time_NTTs = true; } } else { defaultShape = &defaultNTTShape; - time_NTTs = 1; + time_NTTs = true; if (fp64_time > 1.20 * ntt_time) { log("FP64 FFTs are significantly slower than integer NTTs. No FP64 tuning will be performed.\n"); } else { log("FP64 FFTs are not significantly slower than integer NTTs. FP64 tuning will be performed.\n"); - time_FFTs = 1; + time_FFTs = true; } } } @@ -438,28 +445,28 @@ void Tune::tune() { log("Please read config.txt after -tune completes.\n"); log("\n"); - u32 variant = (defaultShape == &defaultFFTShape) ? 101 : 202; + u32 const variant = (defaultShape == &defaultFFTShape) ? 101 : 202; //GW: if fft spec on the command line specifies a variant then we should use that variant (I get some interesting results with 000 vs 101 vs 201 vs 202 likely due to rocm optimizer) // IN_WG/SIZEX, OUT_WG/SIZEX, PAD, MIDDLE_IN/OUT_LDS_TRANSPOSE apply only if INPLACE=0 - u32 current_inplace = args->value("INPLACE", 0); + u32 const current_inplace = args->value("INPLACE", 0); args->flags["INPLACE"] = to_string(0); // Find best IN_WG,IN_SIZEX,OUT_WG,OUT_SIZEX settings - if (1/*option to time IN/OUT settings*/) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + if (!time_inplace_only) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_in_wg = 0; u32 best_in_sizex = 0; - u32 current_in_wg = args->value("IN_WG", 128); - u32 current_in_sizex = args->value("IN_SIZEX", 16); + u32 const current_in_wg = args->value("IN_WG", 128); + u32 const current_in_sizex = args->value("IN_SIZEX", 16); double best_cost = -1.0; double current_cost = -1.0; - for (u32 in_wg : {64, 128, 256}) { - for (u32 in_sizex : {8, 16, 32}) { + for (u32 const in_wg : {64, 128, 256}) { + for (u32 const in_sizex : {8, 16, 32}) { args->flags["IN_WG"] = to_string(in_wg); args->flags["IN_SIZEX"] = to_string(in_sizex); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using IN_WG=%u, IN_SIZEX=%u is %6.1f\n", fft.spec().c_str(), in_wg, in_sizex, cost); if (in_wg == current_in_wg && in_sizex == current_in_sizex) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_in_wg = in_wg; best_in_sizex = in_sizex; } @@ -473,15 +480,15 @@ void Tune::tune() { u32 best_out_wg = 0; u32 best_out_sizex = 0; - u32 current_out_wg = args->value("OUT_WG", 128); - u32 current_out_sizex = args->value("OUT_SIZEX", 16); + u32 const current_out_wg = args->value("OUT_WG", 128); + u32 const current_out_sizex = args->value("OUT_SIZEX", 16); best_cost = -1.0; current_cost = -1.0; - for (u32 out_wg : {64, 128, 256}) { - for (u32 out_sizex : {8, 16, 32}) { + for (u32 const out_wg : {64, 128, 256}) { + for (u32 const out_sizex : {8, 16, 32}) { args->flags["OUT_WG"] = to_string(out_wg); args->flags["OUT_SIZEX"] = to_string(out_sizex); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using OUT_WG=%u, OUT_SIZEX=%u is %6.1f\n", fft.spec().c_str(), out_wg, out_sizex, cost); if (out_wg == current_out_wg && out_sizex == current_out_sizex) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_out_wg = out_wg; best_out_sizex = out_sizex; } @@ -495,16 +502,16 @@ void Tune::tune() { } // Find best PAD setting. Default is 256 bytes for AMD, 0 for all others. - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + if (!time_inplace_only) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_pad = 0; - u32 current_pad = args->value("PAD", AMDGPU ? 256 : 0); + u32 const current_pad = args->value("PAD", AMDGPU ? 256 : 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 pad : {0, 64, 128, 256, 512}) { + for (u32 const pad : {0, 64, 128, 256, 512}) { args->flags["PAD"] = to_string(pad); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using PAD=%u is %6.1f\n", fft.spec().c_str(), pad, cost); if (pad == current_pad) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_pad = pad; } @@ -515,16 +522,16 @@ void Tune::tune() { } // Find best MIDDLE_IN_LDS_TRANSPOSE setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + if (!time_inplace_only) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_middle_in_lds_transpose = 0; - u32 current_middle_in_lds_transpose = args->value("MIDDLE_IN_LDS_TRANSPOSE", 1); + u32 const current_middle_in_lds_transpose = args->value("MIDDLE_IN_LDS_TRANSPOSE", 1); double best_cost = -1.0; double current_cost = -1.0; - for (u32 middle_in_lds_transpose : {0, 1}) { + for (u32 const middle_in_lds_transpose : {0, 1}) { args->flags["MIDDLE_IN_LDS_TRANSPOSE"] = to_string(middle_in_lds_transpose); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using MIDDLE_IN_LDS_TRANSPOSE=%u is %6.1f\n", fft.spec().c_str(), middle_in_lds_transpose, cost); if (middle_in_lds_transpose == current_middle_in_lds_transpose) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_middle_in_lds_transpose = middle_in_lds_transpose; } @@ -535,16 +542,16 @@ void Tune::tune() { } // Find best MIDDLE_OUT_LDS_TRANSPOSE setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + if (!time_inplace_only) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_middle_out_lds_transpose = 0; - u32 current_middle_out_lds_transpose = args->value("MIDDLE_OUT_LDS_TRANSPOSE", 1); + u32 const current_middle_out_lds_transpose = args->value("MIDDLE_OUT_LDS_TRANSPOSE", 1); double best_cost = -1.0; double current_cost = -1.0; - for (u32 middle_out_lds_transpose : {0, 1}) { + for (u32 const middle_out_lds_transpose : {0, 1}) { args->flags["MIDDLE_OUT_LDS_TRANSPOSE"] = to_string(middle_out_lds_transpose); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using MIDDLE_OUT_LDS_TRANSPOSE=%u is %6.1f\n", fft.spec().c_str(), middle_out_lds_transpose, cost); if (middle_out_lds_transpose == current_middle_out_lds_transpose) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_middle_out_lds_transpose = middle_out_lds_transpose; } @@ -554,16 +561,21 @@ void Tune::tune() { args->flags["MIDDLE_OUT_LDS_TRANSPOSE"] = to_string(best_middle_out_lds_transpose); } + // If only timing INPLACE=1 options, then set INPLACE + if (time_inplace_only) { + args->flags["INPLACE"] = to_string(1); + newConfigKeyVals.emplace_back("INPLACE", 1); + } // Find best INPLACE setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + else { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_inplace = 0; double best_cost = -1.0; double current_cost = -1.0; - for (u32 inplace : {0, 1}) { + for (u32 const inplace : {0, 1}) { args->flags["INPLACE"] = to_string(inplace); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using INPLACE=%u is %6.1f\n", fft.spec().c_str(), inplace, cost); if (inplace == current_inplace) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_inplace = inplace; } @@ -573,37 +585,143 @@ void Tune::tune() { args->flags["INPLACE"] = to_string(best_inplace); } - // Find best NONTEMPORAL setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); - u32 best_nontemporal = 0; - u32 current_nontemporal = args->value("NONTEMPORAL", 0); - double best_cost = -1.0; - double current_cost = -1.0; - for (u32 nontemporal : {0, 1, 2}) { - args->flags["NONTEMPORAL"] = to_string(nontemporal); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); - log("Time for %12s using NONTEMPORAL=%u is %6.1f\n", fft.spec().c_str(), nontemporal, cost); - if (nontemporal == current_nontemporal) current_cost = cost; - if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_nontemporal = nontemporal; } + // Find best LOADS/STORES settings + if (true) { + u32 loads = args->value("LOADS", 0); + u32 stores = args->value("STORES", 0); + + // Find best FFT data LOADS setting + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); + u32 best_fft_load = 0; + double best_cost = -1.0; + for (u32 const fft_load : {0, 1, 2, 3, 4}) { + if (fft_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; + args->flags["LOADS"] = to_string(loads / 10 * 10 + fft_load); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using FFT load=%u is %6.1f\n", fft.spec().c_str(), fft_load, cost); + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_fft_load = fft_load; } + } + log("Best FFT load is %u. Default is 0.\n", best_fft_load); + loads = loads / 10 * 10 + best_fft_load; + args->flags["LOADS"] = to_string(loads); + } + + // Find best FFT data STORES setting + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); + u32 best_fft_store = 0; + double best_cost = -1.0; + for (u32 const fft_store : {0, 1, 2, 3}) { + if (fft_store >= 2 && (!NVIDIAGPU || NO_ASM)) continue; + args->flags["STORES"] = to_string(stores / 10 * 10 + fft_store); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using FFT store=%u is %6.1f\n", fft.spec().c_str(), fft_store, cost); + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_fft_store = fft_store; } + } + log("Best FFT store is %u. Default is 0.\n", best_fft_store); + stores = stores / 10 * 10 + best_fft_store; + args->flags["STORES"] = to_string(stores); + } + + // Find best carryShuttle LOADS/STORES settings + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); + u32 best_cs_load = 0, best_cs_store = 0; + double best_cost = -1.0; + for (u32 const cs : {0, 1, 2}) { // Test three combinations: Default load/store, non-temporal, last-use load with L2 store + if (cs >= 2 && (!NVIDIAGPU || NO_ASM)) continue; + u32 const cs_load = cs == 0 ? 0 : cs == 1 ? 1 : 4; + u32 const cs_store = cs == 0 ? 0 : cs == 1 ? 1 : 2; + args->flags["LOADS"] = to_string(loads / 100 * 100 + cs_load * 10 + loads % 10); + args->flags["STORES"] = to_string(stores / 100 * 100 + cs_store * 10 + stores % 10); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using carry shuttle load=%u, store=%u is %6.1f\n", fft.spec().c_str(), cs_load, cs_store, cost); + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_cs_load = cs_load; best_cs_store = cs_store; } + } + log("Best carry shuttle load/store is %u/%u. Default is 0/0.\n", best_cs_load, best_cs_store); + loads = loads / 100 * 100 + best_cs_load * 10 + loads % 10; + stores = stores / 100 * 100 + best_cs_store * 10 + stores % 10; + args->flags["LOADS" ] = to_string(loads); + args->flags["STORES"] = to_string(stores); + } + + // Find best TRIG frequently used data LOADS setting + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); + u32 best_trig_load = 0; + double best_cost = -1.0; + for (u32 const trig_load : {0, 5}) { + if (trig_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; + args->flags["LOADS"] = to_string(loads / 1000 * 1000 + trig_load * 100 + loads % 100); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using Trig frequently used load=%u is %6.1f\n", fft.spec().c_str(), trig_load, cost); + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_trig_load = trig_load; } + } + log("Best Trig frequently used load is %u. Default is 0.\n", best_trig_load); + loads = loads / 1000 * 1000 + best_trig_load * 100 + loads % 100; + args->flags["LOADS" ] = to_string(loads); } - log("Best NONTEMPORAL is %u. Default NONTEMPORAL is 0.\n", best_nontemporal); - configsUpdate(current_cost, best_cost, 0.000, "NONTEMPORAL", best_nontemporal, newConfigKeyVals, suggestedConfigKeyVals); - args->flags["NONTEMPORAL"] = to_string(best_nontemporal); + + // Find best TRIG several uses data LOADS setting + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); + u32 best_trig_load = 0; + double best_cost = -1.0; + for (u32 const trig_load : {0, 1, 2, 3, 4, 5}) { + if (trig_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; + args->flags["LOADS"] = to_string(loads / 10000 * 10000 + trig_load * 1000 + loads % 1000); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using Trig several uses load=%u is %6.1f\n", fft.spec().c_str(), trig_load, cost); + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_trig_load = trig_load; } + } + log("Best Trig several uses load is %u. Default is 0.\n", best_trig_load); + loads = loads / 10000 * 10000 + best_trig_load * 1000 + loads % 1000; + args->flags["LOADS" ] = to_string(loads); + } + + // Find best TRIG used once data LOADS setting + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); + u32 best_trig_load = 0; + double best_cost = -1.0; + for (u32 const trig_load : {0, 1, 2, 3, 4, 5}) { + if (trig_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; + args->flags["LOADS"] = to_string(trig_load * 10000 + loads % 10000); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using Trig used once load=%u is %6.1f\n", fft.spec().c_str(), trig_load, cost); + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_trig_load = trig_load; } + } + log("Best Trig used once load is %u. Default is 0.\n", best_trig_load); + loads = best_trig_load * 10000 + loads % 10000; + args->flags["LOADS" ] = to_string(loads); + } + + // Write accumulated LOADS/STORES settings + configsUpdate(1.000, 0.000, 0.000, "LOADS", loads, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["LOADS"] = to_string(loads); + configsUpdate(1.000, 0.000, 0.000, "STORES", stores, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["STORES"] = to_string(stores); } +#ifndef CUDA_BACKEND // Find best FAST_BARRIER setting - if (AMDGPU) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + if (true /*AMDGPU*/) { // FAST_BARRIER now works for nVidia GPUs too (from what I've seen) + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_fast_barrier = 0; - u32 current_fast_barrier = args->value("FAST_BARRIER", 0); + u32 const current_fast_barrier = args->value("FAST_BARRIER", 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 fast_barrier : {0, 1}) { + for (u32 const fast_barrier : {0, 1}) { args->flags["FAST_BARRIER"] = to_string(fast_barrier); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using FAST_BARRIER=%u is %6.1f\n", fft.spec().c_str(), fast_barrier, cost); if (fast_barrier == current_fast_barrier) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_fast_barrier = fast_barrier; } @@ -612,18 +730,19 @@ void Tune::tune() { configsUpdate(current_cost, best_cost, 0.000, "FAST_BARRIER", best_fast_barrier, newConfigKeyVals, suggestedConfigKeyVals); args->flags["FAST_BARRIER"] = to_string(best_fast_barrier); } +#endif // Find best TAIL_KERNELS setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_tail_kernels = 0; - u32 current_tail_kernels = args->value("TAIL_KERNELS", 2); + u32 const current_tail_kernels = args->value("TAIL_KERNELS", 2); double best_cost = -1.0; double current_cost = -1.0; - for (u32 tail_kernels : {0, 1, 2, 3}) { + for (u32 const tail_kernels : {0, 1, 2, 3}) { args->flags["TAIL_KERNELS"] = to_string(tail_kernels); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TAIL_KERNELS=%u is %6.1f\n", fft.spec().c_str(), tail_kernels, cost); if (tail_kernels == current_tail_kernels) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tail_kernels = tail_kernels; } @@ -638,15 +757,15 @@ void Tune::tune() { // Find best TAIL_TRIGS setting if (time_FFTs) { - FFTConfig fft{defaultFFTShape, 101, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + FFTConfig const fft{defaultFFTShape, 101, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_tail_trigs = 0; - u32 current_tail_trigs = args->value("TAIL_TRIGS", 2); + u32 const current_tail_trigs = args->value("TAIL_TRIGS", 2); double best_cost = -1.0; double current_cost = -1.0; - for (u32 tail_trigs : {0, 1, 2}) { + for (u32 const tail_trigs : {0, 1, 2}) { args->flags["TAIL_TRIGS"] = to_string(tail_trigs); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TAIL_TRIGS=%u is %6.1f\n", fft.spec().c_str(), tail_trigs, cost); if (tail_trigs == current_tail_trigs) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tail_trigs = tail_trigs; } @@ -660,14 +779,14 @@ void Tune::tune() { if (time_NTTs) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.NTT_GF31) fft = FFTConfig(FFTShape(FFT3161, 512, 8, 512), 202, CARRY_AUTO); - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_tail_trigs = 0; - u32 current_tail_trigs = args->value("TAIL_TRIGS31", 0); + u32 const current_tail_trigs = args->value("TAIL_TRIGS31", 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 tail_trigs : {0, 1}) { + for (u32 const tail_trigs : {0, 1}) { args->flags["TAIL_TRIGS31"] = to_string(tail_trigs); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TAIL_TRIGS31=%u is %6.1f\n", fft.spec().c_str(), tail_trigs, cost); if (tail_trigs == current_tail_trigs) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tail_trigs = tail_trigs; } @@ -681,14 +800,14 @@ void Tune::tune() { if (time_NTTs && time_FP32) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.FFT_FP32) fft = FFTConfig(FFTShape(FFT3261, 512, 8, 512), 202, CARRY_AUTO); - u32 exponent = primes.prevPrime(fft.maxBpw() * 0.95 * fft.shape.size()); // Back off the maxExp as different settings will have different maxBpw + u64 const exponent = primes.prevPrime(u64(fft.maxBpw() * 0.95 * fft.shape.size())); // Back off the maxExp as different settings will have different maxBpw u32 best_tail_trigs = 0; - u32 current_tail_trigs = args->value("TAIL_TRIGS32", 2); + u32 const current_tail_trigs = args->value("TAIL_TRIGS32", 2); double best_cost = -1.0; double current_cost = -1.0; - for (u32 tail_trigs : {0, 1, 2}) { + for (u32 const tail_trigs : {0, 1, 2}) { args->flags["TAIL_TRIGS32"] = to_string(tail_trigs); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TAIL_TRIGS32=%u is %6.1f\n", fft.spec().c_str(), tail_trigs, cost); if (tail_trigs == current_tail_trigs) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tail_trigs = tail_trigs; } @@ -702,14 +821,14 @@ void Tune::tune() { if (time_NTTs) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.NTT_GF61) fft = FFTConfig(FFTShape(FFT3161, 512, 8, 512), 202, CARRY_AUTO); - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_tail_trigs = 0; - u32 current_tail_trigs = args->value("TAIL_TRIGS61", 0); + u32 const current_tail_trigs = args->value("TAIL_TRIGS61", 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 tail_trigs : {0, 1}) { + for (u32 const tail_trigs : {0, 1}) { args->flags["TAIL_TRIGS61"] = to_string(tail_trigs); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TAIL_TRIGS61=%u is %6.1f\n", fft.spec().c_str(), tail_trigs, cost); if (tail_trigs == current_tail_trigs) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tail_trigs = tail_trigs; } @@ -721,15 +840,15 @@ void Tune::tune() { // Find best TABMUL_CHAIN setting if (time_FFTs) { - FFTConfig fft{defaultFFTShape, 101, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + FFTConfig const fft{defaultFFTShape, 101, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_tabmul_chain = 0; - u32 current_tabmul_chain = args->value("TABMUL_CHAIN", 0); + u32 const current_tabmul_chain = args->value("TABMUL_CHAIN", 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 tabmul_chain : {0, 1}) { + for (u32 const tabmul_chain : {0, 1}) { args->flags["TABMUL_CHAIN"] = to_string(tabmul_chain); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TABMUL_CHAIN=%u is %6.1f\n", fft.spec().c_str(), tabmul_chain, cost); if (tabmul_chain == current_tabmul_chain) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tabmul_chain = tabmul_chain; } @@ -743,14 +862,14 @@ void Tune::tune() { if (time_NTTs) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.NTT_GF31) fft = FFTConfig(FFTShape(FFT3161, 512, 8, 512), 202, CARRY_AUTO); - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_tabmul_chain = 0; - u32 current_tabmul_chain = args->value("TABMUL_CHAIN31", 0); + u32 const current_tabmul_chain = args->value("TABMUL_CHAIN31", 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 tabmul_chain : {0, 1}) { + for (u32 const tabmul_chain : {0, 1}) { args->flags["TABMUL_CHAIN31"] = to_string(tabmul_chain); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TABMUL_CHAIN31=%u is %6.1f\n", fft.spec().c_str(), tabmul_chain, cost); if (tabmul_chain == current_tabmul_chain) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tabmul_chain = tabmul_chain; } @@ -764,14 +883,14 @@ void Tune::tune() { if (time_NTTs && time_FP32) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.FFT_FP32) fft = FFTConfig(FFTShape(FFT3261, 512, 8, 512), 202, CARRY_AUTO); - u32 exponent = primes.prevPrime(fft.maxBpw() * 0.95 * fft.shape.size()); // Back off the maxExp as different settings will have different maxBpw + u64 const exponent = primes.prevPrime(u64(fft.maxBpw() * 0.95 * fft.shape.size())); // Back off the maxExp as different settings will have different maxBpw u32 best_tabmul_chain = 0; - u32 current_tabmul_chain = args->value("TABMUL_CHAIN32", 0); + u32 const current_tabmul_chain = args->value("TABMUL_CHAIN32", 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 tabmul_chain : {0, 1}) { + for (u32 const tabmul_chain : {0, 1}) { args->flags["TABMUL_CHAIN32"] = to_string(tabmul_chain); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TABMUL_CHAIN32=%u is %6.1f\n", fft.spec().c_str(), tabmul_chain, cost); if (tabmul_chain == current_tabmul_chain) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tabmul_chain = tabmul_chain; } @@ -785,14 +904,14 @@ void Tune::tune() { if (time_NTTs) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.NTT_GF61) fft = FFTConfig(FFTShape(FFT3161, 512, 8, 512), 202, CARRY_AUTO); - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_tabmul_chain = 0; - u32 current_tabmul_chain = args->value("TABMUL_CHAIN61", 0); + u32 const current_tabmul_chain = args->value("TABMUL_CHAIN61", 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 tabmul_chain : {0, 1}) { + for (u32 const tabmul_chain : {0, 1}) { args->flags["TABMUL_CHAIN61"] = to_string(tabmul_chain); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TABMUL_CHAIN61=%u is %6.1f\n", fft.spec().c_str(), tabmul_chain, cost); if (tabmul_chain == current_tabmul_chain) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tabmul_chain = tabmul_chain; } @@ -806,34 +925,34 @@ void Tune::tune() { if (time_NTTs) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.NTT_GF31) fft = FFTConfig(FFTShape(FFT3161, 512, 8, 512), 202, CARRY_AUTO); - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_modm31 = 0; - u32 current_modm31 = args->value("MODM31", 0); + u32 const current_modm31 = args->value("MODM31", 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 modm31 : {0, 1, 2}) { + for (u32 const modm31 : {0, 1, 2}) { args->flags["MODM31"] = to_string(modm31); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using MODM31=%u is %6.1f\n", fft.spec().c_str(), modm31, cost); if (modm31 == current_modm31) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_modm31 = modm31; } } log("Best MODM31 is %u. Default MODM31 is 0.\n", best_modm31); - configsUpdate(current_cost, best_cost, 0.003, "MODM31", best_modm31, newConfigKeyVals, suggestedConfigKeyVals); + configsUpdate(current_cost, best_cost, 0.000, "MODM31", best_modm31, newConfigKeyVals, suggestedConfigKeyVals); args->flags["MODM31"] = to_string(best_modm31); } // Find best UNROLL_W setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_unroll_w = 0; - u32 current_unroll_w = args->value("UNROLL_W", AMDGPU ? 0 : 1); + u32 const current_unroll_w = args->value("UNROLL_W", AMDGPU ? 0 : 1); double best_cost = -1.0; double current_cost = -1.0; - for (u32 unroll_w : {0, 1}) { + for (u32 const unroll_w : {0, 1}) { args->flags["UNROLL_W"] = to_string(unroll_w); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using UNROLL_W=%u is %6.1f\n", fft.spec().c_str(), unroll_w, cost); if (unroll_w == current_unroll_w) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_unroll_w = unroll_w; } @@ -844,16 +963,16 @@ void Tune::tune() { } // Find best UNROLL_H setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_unroll_h = 0; - u32 current_unroll_h = args->value("UNROLL_H", AMDGPU && defaultShape->height >= 1024 ? 0 : 1); + u32 const current_unroll_h = args->value("UNROLL_H", AMDGPU && defaultShape->height >= 1024 ? 0 : 1); double best_cost = -1.0; double current_cost = -1.0; - for (u32 unroll_h : {0, 1}) { + for (u32 const unroll_h : {0, 1}) { args->flags["UNROLL_H"] = to_string(unroll_h); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using UNROLL_H=%u is %6.1f\n", fft.spec().c_str(), unroll_h, cost); if (unroll_h == current_unroll_h) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_unroll_h = unroll_h; } @@ -864,16 +983,16 @@ void Tune::tune() { } // Find best ZEROHACK_W setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_zerohack_w = 0; - u32 current_zerohack_w = args->value("ZEROHACK_W", 1); + u32 const current_zerohack_w = args->value("ZEROHACK_W", 1); double best_cost = -1.0; double current_cost = -1.0; - for (u32 zerohack_w : {0, 1}) { + for (u32 const zerohack_w : {0, 1}) { args->flags["ZEROHACK_W"] = to_string(zerohack_w); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using ZEROHACK_W=%u is %6.1f\n", fft.spec().c_str(), zerohack_w, cost); if (zerohack_w == current_zerohack_w) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_zerohack_w = zerohack_w; } @@ -884,16 +1003,16 @@ void Tune::tune() { } // Find best ZEROHACK_H setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_zerohack_h = 0; - u32 current_zerohack_h = args->value("ZEROHACK_H", 1); + u32 const current_zerohack_h = args->value("ZEROHACK_H", 1); double best_cost = -1.0; double current_cost = -1.0; - for (u32 zerohack_h : {0, 1}) { + for (u32 const zerohack_h : {0, 1}) { args->flags["ZEROHACK_H"] = to_string(zerohack_h); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using ZEROHACK_H=%u is %6.1f\n", fft.spec().c_str(), zerohack_h, cost); if (zerohack_h == current_zerohack_h) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_zerohack_h = zerohack_h; } @@ -903,17 +1022,120 @@ void Tune::tune() { args->flags["ZEROHACK_H"] = to_string(best_zerohack_h); } + // Find best WMUL setting + if (true && defaultShape->width != 4096) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); + u32 best_wmul = 0; + u32 const current_wmul = args->value("WMUL", 2); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 const wmul : {1, 2, 4}) { + args->flags["WMUL"] = to_string(wmul); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using WMUL=%u is %6.1f\n", fft.spec().c_str(), wmul, cost); + if (wmul == current_wmul) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_wmul = wmul; } + } + log("Best WMUL is %u. Default WMUL is 2.\n", best_wmul); + configsUpdate(current_cost, best_cost, 0.000, "WMUL", best_wmul, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["WMUL"] = to_string(best_wmul); + } + + // Find best MULTI_Q setting + if (1) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_multi_q = 0; + u32 current_multi_q = args->value("MULTI_Q", 0); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 multi_q : {0, 1}) { + args->flags["MULTI_Q"] = to_string(multi_q); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using MULTI_Q=%u is %6.1f\n", fft.spec().c_str(), multi_q, cost); + if (multi_q == current_multi_q) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_multi_q = multi_q; } + } + log("Best MULTI_Q is %u. Default MULTI_Q is 0.\n", best_multi_q); + configsUpdate(current_cost, best_cost, 0.000, "MULTI_Q", best_multi_q, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["MULTI_Q"] = to_string(best_multi_q); + } + + // Find best CUDA compiler options +#if CUDA_BACKEND + // Find best L1CUDA setting. + if (true) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_l1cuda = 0; + u32 current_l1cuda = args->value("L1CUDA", 0); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 l1cuda : {0, 1, 2, 3}) { + args->flags["L1CUDA"] = to_string(l1cuda); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using L1CUDA=%u is %6.1f\n", fft.spec().c_str(), l1cuda, cost); + if (l1cuda == current_l1cuda) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_l1cuda = l1cuda; } + } + log("Best L1CUDA is %u. Default L1CUDA is 0.\n", best_l1cuda); + configsUpdate(current_cost, best_cost, 0.000, "L1CUDA", best_l1cuda, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["L1CUDA"] = to_string(best_l1cuda); + } + + // Find best GRAPHS setting. Require a clear advantage to override the default GRAPHS setting. GRAPHS=1 will use less CPU time. + if (true) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_graphs = 0; + u32 current_graphs = args->value("GRAPHS", 1); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 graphs : {0, 1}) { + args->flags["GRAPHS"] = to_string(graphs); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using GRAPHS=%u is %6.1f\n", fft.spec().c_str(), graphs, cost); + if (graphs == current_graphs) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_graphs = graphs; } + } + log("Best GRAPHS is %u. Default GRAPHS is 1.\n", best_graphs); + configsUpdate(current_cost, best_cost, 0.003, "GRAPHS", best_graphs, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["GRAPHS"] = to_string(best_graphs); + } + + // See if disabling the default register usage makes sense + if (true) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_noreg = 0; + u32 const current_noreg = args->value("NOREG", 0); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 const noreg : {0, 1}) { + args->flags["NOREG"] = to_string(noreg); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using NOREG=%u is %6.1f\n", fft.spec().c_str(), noreg, cost); + if (noreg == current_noreg) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_noreg = noreg; } + } + log("Best NOREG is %u. Default NOREG is 0.\n", best_noreg); + configsUpdate(current_cost, best_cost, 0.000, "NOREG", best_noreg, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["NOREG"] = to_string(best_noreg); + } +#endif + // Find best BIGLIT setting - if (time_FFTs) { - FFTConfig fft{*defaultShape, 101, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + if (false && time_FFTs) { // Deprecated + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_biglit = 0; - u32 current_biglit = args->value("BIGLIT", 1); + u32 const current_biglit = args->value("BIGLIT", 1); double best_cost = -1.0; double current_cost = -1.0; - for (u32 biglit : {0, 1}) { + for (u32 const biglit : {0, 1}) { args->flags["BIGLIT"] = to_string(biglit); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using BIGLIT=%u is %6.1f\n", fft.spec().c_str(), biglit, cost); if (biglit == current_biglit) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_biglit = biglit; } @@ -925,7 +1147,7 @@ void Tune::tune() { // Output new settings to config.txt File config = File::openAppend("config.txt"); - if (newConfigKeyVals.size()) { + if (!newConfigKeyVals.empty()) { config.write("\n# New settings based on a -tune run."); for (u32 i = 0; i < newConfigKeyVals.size(); ++i) { config.write(i == 0 ? "\n -use " : ","); @@ -933,7 +1155,7 @@ void Tune::tune() { } config.write("\n"); } - if (suggestedConfigKeyVals.size()) { + if (!suggestedConfigKeyVals.empty()) { config.write("\n# These settings were slightly faster in a -tune run."); config.write("\n# It is suggested that each setting be timed over a longer duration to see if the setting really is faster."); for (u32 i = 0; i < suggestedConfigKeyVals.size(); ++i) { @@ -947,7 +1169,7 @@ void Tune::tune() { config.write("\n -log 1000000\n"); } if (args->workers < 2) { - config.write("\n# Running two workers sometimes gives better throughput. Autoprimenet will need to create up a second worktodo file."); + config.write("\n# Running two workers sometimes gives better throughput. AutoPrimeNet will need to create a second worktodo file (use --num-workers 2)."); config.write("\n# -workers 2\n"); config.write("\n# Changing TAIL_KERNELS to 3 when running two workers may be better."); config.write("\n# -use TAIL_KERNELS=3\n"); @@ -962,11 +1184,11 @@ void Tune::tune() { int skip_some_WH_variants = 1; // 0 = skip nothing, 1 = skip slower widths/heights unless they have better Z, 2 = only run fastest widths/heights // The width = height = 512 FFT shape is so good, we probably don't need to time the width = 1024, height = 256 shape. - bool skip_1K_256 = 1; + bool skip_1K_256 = true; // make command line args for this? skip_some_WH_variants = 2; // should default be 1?? -skip_1K_256 = 0; +skip_1K_256 = false; // For each width, time the 001, 101, and 201 FP64 variants to find the fastest width variant. // In an ideal world we'd use the -time feature and look at the kCarryFused timing. Then we'd save this info in config.txt or tune.txt. @@ -978,43 +1200,48 @@ skip_1K_256 = 0; vector results = TuneEntry::readTuneFile(*args); + // Time FFT shapes smallest-to-largest exponent handled + std::ranges::stable_sort(shapes, [](const FFTShape& a, const FFTShape& b) { return a.maxExp() < b.maxExp(); }); + // Loop through all possible FFT shapes for (const FFTShape& shape : shapes) { // Skip some FFTs and NTTs if (shape.fft_type == FFT64 && !time_FFTs) continue; - if (shape.fft_type != FFT64 && !time_NTTs) continue; + if (shape.fft_type == FFT6431 && !time_FFT6431) continue; + if (shape.fft_type != FFT64 && shape.fft_type != FFT6431 && !time_NTTs) continue; if ((shape.fft_type == FFT3261 || shape.fft_type == FFT323161 || shape.fft_type == FFT3231 || shape.fft_type == FFT32) && !time_FP32) continue; // Time an exponent that's good for all variants and carry-config. - u32 exponent = primes.prevPrime(FFTConfig{shape, shape.width <= 1024 ? 0u : 100u, CARRY_32}.maxExp()); + u64 const exponent = primes.prevPrime(FFTConfig{shape, shape.width <= 1024 ? 0u : 100u, CARRY_32}.maxExp()); u32 adjusted_quick = (exponent < 50000000) ? quick - 1 : (exponent < 170000000) ? quick : (exponent < 350000000) ? quick + 1 : quick + 2; - if (adjusted_quick < 1) adjusted_quick = 1; - if (adjusted_quick > 10) adjusted_quick = 10; + adjusted_quick = std::max(adjusted_quick, 1); + adjusted_quick = std::min(adjusted_quick, 10); // Loop through all possible variants for (u32 variant = 0; variant <= LAST_VARIANT; variant = next_variant (variant)) { - // Only FP64 code supports variants + // Only FP64 code supports variants. For FFT6431, we've not worked out how variant_M = 1 affects max exp. if (variant != 202 && !FFTConfig{shape, variant, CARRY_AUTO}.FFT_FP64) continue; + if (shape.fft_type == FFT6431 && variant_M(variant) == 1) continue; // Only AMD GPUs support variant zero (BCAST) and only if width <= 1024. CLANG doesn't support builtins. Let NO_ASM bypass variant zero. if (variant_W(variant) == 0) { if (!AMDGPU) continue; if (shape.width > 1024) continue; - if (args->value("NO_ASM", 0)) continue; + if (args->value("NO_ASM", 0)) continue; } // Only AMD GPUs support variant zero (BCAST) and only if height <= 1024. if (variant_H(variant) == 0) { if (!AMDGPU) continue; if (shape.height > 1024) continue; - if (args->value("NO_ASM", 0)) continue; + if (args->value("NO_ASM", 0)) continue; } // Reject shapes that won't be used to test exponents in the user's desired range { - FFTConfig fft{shape, variant, CARRY_AUTO}; + FFTConfig const fft{shape, variant, CARRY_AUTO}; if (fft.maxExp() < min_exponent) continue; if (fft.maxExp() > 2*max_exponent) continue; if (shape.fft_type == FFT64 && fft.maxExp() > 1.2*max_exponent) continue; @@ -1036,13 +1263,13 @@ skip_1K_256 = 0; if (auto it = fastest_width_variants.find(shape.width); it != fastest_width_variants.end()) { fastest_width = it->second; } else { - FFTShape test = FFTShape(shape.width, 12, 256); + FFTShape const test = FFTShape(FFT64, shape.width, 12, 256); double cost, min_cost = -1.0; for (u32 w = 0; w < N_VARIANT_W; w++) { if (w == 0 && !AMDGPU) continue; if (w == 0 && test.width > 1024) continue; - FFTConfig fft{test, variant_WMH (w, 0, 1), CARRY_32}; - cost = Gpu::make(q, primes.prevPrime(fft.maxExp()), shared, fft, {}, false)->timePRP(adjusted_quick); + FFTConfig const fft{test, variant_WMH (w, 0, 1), CARRY_32}; + cost = Gpu::make(primes.prevPrime(fft.maxExp()), shared, fft, {}, false)->timePRP(adjusted_quick); log("Fast width search %6.1f %12s\n", cost, fft.spec().c_str()); if (min_cost < 0.0 || cost < min_cost) { min_cost = cost; fastest_width = w; } } @@ -1058,13 +1285,13 @@ skip_1K_256 = 0; if (auto it = fastest_height_variants.find(shape.height); it != fastest_height_variants.end()) { fastest_height = it->second; } else { - FFTShape test = FFTShape(shape.height, 12, shape.height); + FFTShape const test = FFTShape(FFT64, shape.height, 12, shape.height); double cost, min_cost = -1.0; for (u32 h = 0; h < N_VARIANT_H; h++) { if (h == 0 && !AMDGPU) continue; if (h == 0 && test.height > 1024) continue; - FFTConfig fft{test, variant_WMH (1, 0, h), CARRY_32}; - cost = Gpu::make(q, primes.prevPrime(fft.maxExp()), shared, fft, {}, false)->timePRP(quick); + FFTConfig const fft{test, variant_WMH (1, 0, h), CARRY_32}; + cost = Gpu::make(primes.prevPrime(fft.maxExp()), shared, fft, {}, false)->timePRP(quick); log("Fast height search %6.1f %12s\n", cost, fft.spec().c_str()); if (min_cost < 0.0 || cost < min_cost) { min_cost = cost; fastest_height = h; } } @@ -1089,15 +1316,15 @@ skip_1K_256 = 0; } for (auto carry : carryToTest) { - FFTConfig fft{shape, variant, carry}; + FFTConfig const fft{shape, variant, carry}; // Skip middle = 1, CARRY_32 if maximum exponent would be the same as middle = 0, CARRY_32 if (variant_M(variant) > 0 && carry == CARRY_32 && fft.maxExp() <= FFTConfig{shape, variant - 10, CARRY_32}.maxExp()) continue; - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); - bool isUseful = TuneEntry{cost, fft}.update(results); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + bool const isUseful = TuneEntry{.cost=cost, .fft=fft}.update(results); log("%c %6.1f %12s %9" PRIu64 "\n", isUseful ? '*' : ' ', cost, fft.spec().c_str(), fft.maxExp()); - if (isUseful) TuneEntry::writeTuneFile(results); + if (isUseful) TuneEntry::writeTuneFile(results); } } } diff --git a/src/tune.h b/src/tune.h index a64f2100..b3ce6119 100644 --- a/src/tune.h +++ b/src/tune.h @@ -9,7 +9,6 @@ #include #include -class Queue; class GpuCommon; class RoeInfo; class Gpu; @@ -18,7 +17,6 @@ using TuneConfig = vector; class Tune { private: - Queue *q; GpuCommon shared; Primes primes; @@ -26,7 +24,7 @@ class Tune { float zForBpw(float bpw, FFTConfig fft, u32); public: - Tune(Queue *q, GpuCommon shared) : q{q}, shared{shared} {} + Tune(GpuCommon shared) : shared{shared} {} // Find the max-BPW for each FFT void ztune(); diff --git a/src/typeName.h b/src/typeName.h index 6ca21042..457d878b 100644 --- a/src/typeName.h +++ b/src/typeName.h @@ -9,7 +9,7 @@ const char* typeName(T&& v) { const char* ret = typeid(v).name(); try { size_t pos = 0; - std::stoi(ret, &pos); + (void)std::stoi(ret, &pos); return ret + pos; } catch (...) { return ret; diff --git a/test/786433-10.proof b/test/786433-10.proof new file mode 100644 index 00000000..c0f96f99 Binary files /dev/null and b/test/786433-10.proof differ diff --git a/tools/primenet.py b/tools/primenet.py deleted file mode 100755 index e61e1a42..00000000 --- a/tools/primenet.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/python3 - -# Copyright (c) Mihai Preda. -# Inspired by mlucas-primenet.py , part of Mlucas by Ernst W. Mayer. - -import argparse -import time -import urllib -import requests -import os -import upload -import getpass - -from http import cookiejar -from urllib.parse import urlencode -from urllib.request import build_opener -from urllib.request import HTTPCookieProcessor -from datetime import datetime - -baseUrl = "https://www.mersenne.org/" -primenet = build_opener(HTTPCookieProcessor(cookiejar.CookieJar())) - -def login(user, password): - login = {"user_login": user, "user_password": password} - data = urlencode(login).encode('utf-8') - r = primenet.open(baseUrl, data).read().decode("utf-8") - if not user + "
logged in" in r: - print(r) - print("Login failed"); - raise(PermissionError("Login failed")) - -def loadLines(fileName): - try: - with open(fileName, 'r') as fi: - return set((line.strip().strip('\n') for line in fi)) - except FileNotFoundError as e: - return set() - -def sendOne(line): - print("Sending result: ", line) - data = urlencode({"data": line}).encode('utf-8') - res = primenet.open(baseUrl + "manual_result/default.php", data).read().decode("utf-8") - if "Error code" in res: - begin = res.find("Error code") - end = res.find("", begin) - text = res[begin:end] - print(text) - already = text.startswith('Error code: 40, error text: This computer has already sent in this PRP result') - if already: - print('Already sent, will not retry') - return already - else: - begin = res.find("CPU credit is") - end = res.find("", begin); - if begin >= 0 and end >= 0: - print(res[begin:end], '\n') - return True - else: - return False - -def appendLine(fileName, line): - with open(fileName, 'a') as fo: print(line, file = fo, end = '\n') - -def sendResults(results, sent, sentName, retryName): - for result in results: - ok = sendOne(result) - sent.add(result) - appendLine(sentName if ok else retryName, result) - -def fetch(what): - assignment = {"cores":1, "num_to_get":1, "pref":what} - # res = primenet.open(baseUrl + "manual_assignment/?" + urlencode(assignment)).read().decode("utf-8") - res = primenet.open(baseUrl + "manual_assignment/", data=urlencode(assignment).encode()).read().decode("utf-8") - # print(res) - - BEGIN_MARK = "" - # begin = res.find(BEGIN_MARK) - begin = res.find(">PRP=") - if begin == -1: begin = res.find(">LL=") - if begin == -1: - print(res) - raise(AssertionError("assignment no BEGIN mark")) - begin += 1 - # begin += len(BEGIN_MARK) - end = res.find("= end) - return True - -def getTask(userId): - url = f'http://mersenne.org/oneAssignment/&UserID={userId}&workpref=150' - print(url) - r = requests.get(url) - print(r) - print(r.json()) - -def uploadProof(userId, fileName, verbose=False): - exponent = headerExponent(fileName) - print(f'Uploading M{exponent} from "{fileName}"') - data = fileBytes(fileName) - return upload(userId, exponent, data, verbose) - -if __name__ == '__main__': - if len(sys.argv) < 3: - print(f'Usage: {sys.argv[0]} ') - exit(1) - - userId = sys.argv[1] - fileName = sys.argv[2] - if uploadProof(userId, fileName, verbose=True): - print('Success') - else: - exit(1)