diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3180f8..84bab8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,65 +1,102 @@ -name: QuadBLAS Testing +name: QBLAS CI on: push: - branches: [main] + branches: [main, overhaul-rewrite] pull_request: workflow_dispatch: jobs: - test-ubuntu: - name: Test Ubuntu - runs-on: ubuntu-latest + test: + name: ${{ matrix.os }} / ${{ matrix.compiler }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + compiler: gcc + cc: gcc + cxx: g++ + - os: ubuntu-latest + compiler: clang + cc: clang + cxx: clang++ + - os: macos-14 + compiler: clang + cc: clang + cxx: clang++ + - os: macos-15 + compiler: clang + cc: clang + cxx: clang++ steps: - uses: actions/checkout@v4 with: submodules: recursive - - name: Install dependencies + - name: Install dependencies (Linux) + if: runner.os == 'Linux' run: | sudo apt-get update - sudo apt-get install -y cmake build-essential libomp-dev + sudo apt-get install -y cmake ninja-build build-essential libomp-dev python3 python3-pip python3-venv + if [ "${{ matrix.compiler }}" = "clang" ]; then + sudo apt-get install -y clang libomp-dev + fi - - name: Build SLEEF + - name: Install dependencies (macOS) + if: runner.os == 'macOS' run: | - git clone --branch 3.8 --depth 1 https://github.com/shibatch/sleef.git - cd sleef - cmake -S . -B build -DSLEEF_BUILD_QUAD=ON -DSLEEF_BUILD_SHARED_LIBS=ON - cmake --build build --parallel $(nproc) - sudo cmake --install build --prefix /usr/local + brew install cmake ninja libomp python@3.12 - - name: Build QuadBLAS + - name: Install numpy >= 2.0 + run: | + python3 -m venv .venv + .venv/bin/pip install --upgrade pip + .venv/bin/pip install 'numpy>=2.0' + echo "$(pwd)/.venv/bin" >> $GITHUB_PATH + # Make cmake's find_package(Python3) prefer the venv on macOS, + # where it otherwise picks the framework python without numpy. + echo "VIRTUAL_ENV=$(pwd)/.venv" >> $GITHUB_ENV + + - name: Build SLEEF with full SIMD coverage env: - SLEEF_ROOT: /usr/local + CC: ${{ matrix.cc }} + CXX: ${{ matrix.cxx }} run: | - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - make -j$(nproc) + bash scripts/bootstrap_sleef.sh - - name: Run tests + - name: Configure qblas + env: + CC: ${{ matrix.cc }} + CXX: ${{ matrix.cxx }} run: | - cd build - ./quadblas_test + cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release - - name: Run benchmarks + - name: Build qblas + tests + bench + run: cmake --build build --parallel + + - name: Run correctness tests (C + Python-vs-numpy) + run: ctest --test-dir build --output-on-failure + + - name: Smoke-run the benchmark (small problem sizes) run: | - cd build - if [ -f "./quadblas_benchmark" ]; then - echo "Running Google Benchmark tests..." - ./quadblas_benchmark --benchmark_min_time=0.1s --benchmark_filter="BM_DotProduct|BM_AXPY|BM_VectorNorm" - else - echo "Google Benchmark not available, running legacy benchmark..." - ./quadblas_benchmark_legacy - fi + ./build/bench/qblas_bench --benchmark_min_time=0.05s \ + --benchmark_filter='BM_qdot/1024|BM_qaxpy/1024|BM_qgemm/64' - test-macos: - name: Test macOS ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [macos-14, macos-15] + - name: Force-test each dispatch tier (Linux only) + if: runner.os == 'Linux' + run: | + for tier in generic sse2 avx2; do + echo "==== Testing tier: $tier ====" + QBLAS_DISPATCH=$tier ctest --test-dir build --output-on-failure + done + bench-baseline: + name: benchmark / ubuntu-gcc + runs-on: ubuntu-latest + needs: test steps: - uses: actions/checkout@v4 with: @@ -67,49 +104,33 @@ jobs: - name: Install dependencies run: | - brew install cmake libomp + sudo apt-get update + sudo apt-get install -y cmake ninja-build build-essential libomp-dev - - name: Install SLEEF - env: - MACOSX_DEPLOYMENT_TARGET: "11.0" - run: | - git clone --branch 3.8 --depth 1 https://github.com/shibatch/sleef.git - cd sleef - cmake -S . -B build \ - -DSLEEF_BUILD_QUAD=ON \ - -DSLEEF_BUILD_SHARED_LIBS=ON \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 \ - -DCMAKE_INSTALL_RPATH="@loader_path/../lib" \ - -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON - cmake --build build --clean-first -j - sudo cmake --install build --prefix /usr/local - - - name: Build QuadBLAS - env: - SLEEF_ROOT: /usr/local - DYLD_LIBRARY_PATH: "/usr/local/lib:$DYLD_LIBRARY_PATH" - run: | - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - make -j$(sysctl -n hw.ncpu) + - name: Build SLEEF + run: bash scripts/bootstrap_sleef.sh - - name: Run tests - env: - DYLD_LIBRARY_PATH: "/usr/local/lib:$DYLD_LIBRARY_PATH" + - name: Build qblas run: | - cd build - ./quadblas_test + cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release + cmake --build build --parallel - name: Run benchmarks - env: - DYLD_LIBRARY_PATH: "/usr/local/lib:$DYLD_LIBRARY_PATH" run: | - cd build - if [ -f "./quadblas_benchmark" ]; then - echo "Running Google Benchmark tests..." - ./quadblas_benchmark --benchmark_min_time=0.1s --benchmark_filter="BM_DotProduct|BM_AXPY|BM_VectorNorm" - else - echo "Google Benchmark not available, running legacy benchmark..." - ./quadblas_benchmark_legacy - fi + mkdir -p bench-results + OMP_NUM_THREADS=$(nproc) OMP_PLACES=cores OMP_PROC_BIND=close \ + ./build/bench/qblas_bench_compare \ + --benchmark_min_time=0.2s \ + --benchmark_repetitions=3 \ + --benchmark_report_aggregates_only=true \ + --benchmark_format=json > bench-results/qblas_bench.json + ./build/bench/qblas_bench_compare \ + --benchmark_min_time=0.2s \ + --benchmark_repetitions=3 \ + --benchmark_report_aggregates_only=true | tee bench-results/qblas_bench.txt + + - uses: actions/upload-artifact@v4 + with: + name: qblas-bench-results + path: bench-results/ + retention-days: 30 diff --git a/.gitignore b/.gitignore index d163863..ee6d9dc 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,16 @@ -build/ \ No newline at end of file +build/ +build-*/ +.sleef-prefix/ +.cache/ +compile_commands.json +*.o +*.a +*.so +*.so.* +*.dylib +*.pyc +__pycache__/ +.DS_Store +.vscode/ +Testing/ +upstream-project/ diff --git a/.gitmodules b/.gitmodules index 55a47ce..65ede7e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "third_party/benchmark"] path = third_party/benchmark url = https://github.com/google/benchmark.git +[submodule "third_party/sleef"] + path = third_party/sleef + url = https://github.com/shibatch/sleef.git diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..56a7697 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,108 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What this is + +QBLAS is a BLAS Level 1/2/3 library for IEEE-754 binary128 (quad-precision, `Sleef_quad`) built on top of SLEEF. It ships a compiled shared library `libqblas.so` with a CBLAS-style C API (`cblas_qdot`, `cblas_qgemm`, …) and runtime CPU dispatch (scalar / SSE2 / AVX2 / AVX-512 / NEON). + +This repo was rewritten from scratch on the `overhaul-rewrite` branch; the older header-only template implementation on `main` is obsolete. + +## Build + +```bash +scripts/bootstrap_sleef.sh # builds SLEEF 3.8 with all SIMD paths + # and installs into ./.sleef-prefix/ +mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Release +cmake --build . --parallel +ctest # ~5 s, 5 test executables +./bench/qblas_bench --benchmark_min_time=0.1s +``` + +`scripts/bootstrap_sleef.sh` forces `SLEEF_ENABLE_{SSE2,SSE4,AVX,AVX2,AVX512F}=ON` so every dispatch tier the binary contains has its underlying SLEEF symbols available; SLEEF's default config only builds for the host CPU and leaves unresolved references for other tiers in `libsleefquad.so`. + +The top-level CMake auto-discovers SLEEF via `SLEEF_ROOT` (env var or cmake var), then `./.sleef-prefix/`, then pkg-config. The binary RPATH is set with `--disable-new-dtags` so `libtlfloat.so.1` (an indirect dep of `libsleefquad.so`) resolves; without that, GLIBC's DT_RUNPATH semantics drop the path for transitive deps. + +## Architecture + +The library is a single shared object with multiple internal "kernel tiers", each compiled as an OBJECT library with its own `-march` flags. Runtime dispatch picks the best available tier at library init time. + +``` +include/qblas/ Public C API. Each cblas_q* declaration ties to a + Sleef_quad function (no template metaprogramming). +src/common/ Internal helpers (dispatch table, threading, + aligned alloc, branch hints). +src/cpu/ CPUID-based feature detection + dispatch init. + Reads QBLAS_DISPATCH=avx2|sse2|generic to override. +src/kernels/ One TU per tier (generic/sse2/avx2/avx512/neon). + Each defines QV_WIDTH + QV_ISA_SUFFIX and includes + kernels_template.h, which contains the actual SIMD + kernels parametrised by width and ISA suffix. +src/level1/ Front-end for vec ops (delegates to dispatch + threads). +src/level2/ Front-end for gemv/ger/symv/trsv. +src/level3/ qgemm.c (Goto-style blocking) + qblas_level3_extra.c + (syrk/trmm/trsm; the last two are reference, not yet + blocked). +tests/ Smoke + per-level correctness suites. Each test compares + against a naive scalar reference across positive and + negative strides, row/col-major layouts, all transpose + combos, and edge tile sizes (m=1, k=1, prime, multi-block). +bench/ Google Benchmark suite (BM_qdot, BM_qgemm, …). + items/sec ≈ quad FMA throughput. +cmake/QBLASArch.cmake Per-tier compile flags. Edit here to add new ISAs. +``` + +### Kernel template ([src/kernels/kernels_template.h](src/kernels/kernels_template.h)) + +A single header parametrised over `QV_WIDTH` (1, 2, 4, 8) and `QV_ISA_SUFFIX` (`sse2`, `advsimd`, …). For widths 1/4/8 there is only one SLEEF symbol set so the suffix is implicit; for width 2 we have to pick between `sse2` (x86) and `advsimd` (ARM) at the call site, which is why `QV_ISA_SUFFIX` is its own macro. Each ISA TU defines those macros and `#include`s the template — no source duplication. + +The template implements: `qdot`, `qaxpy`, `qscal`, `qasum`, `qiamax`, `qgemv_n`, `qgemv_t`, and the packed `qgemm_kernel`. The corresponding `qblas_register_` function publishes its function pointers into the global dispatch table. + +### GEMM ([src/level3/qblas_gemm.c](src/level3/qblas_gemm.c)) + +Goto / van de Geijn 5-loop blocking around a register tile published by the dispatch tier (MR×NR = 4×4 currently). Key correctness trick: **column-major C is computed as row-major C^T by swapping M↔N and A↔B with original (not flipped) transpose flags** — the buffer addresses come out identical because additions commute. This eliminated a separate col-major code path and a class of off-by-something bugs. + +Parallelism is over the outermost `jc` loop. `nc` is auto-scaled down so each thread gets ≥2 blocks (`target_blocks = nthreads * 2`); without this, large square GEMMs ran single-threaded because `nc=QBLAS_NC_DEFAULT (512)` swallowed the whole problem. + +### Threading + +OpenMP. Each Level 1 entry point checks `qblas_resolve_threads()` (a work-vs-overhead heuristic) and only forks above `QBLAS_PARALLEL_THRESHOLD_L1 = 8192`. Reduction-style ops (`qdot`, `qasum`, `qnrm2`) sum per-thread partials into a stack array (max 256 threads); `qnrm2` is implemented as `sqrt(qdot(x,x))` so it inherits the threading for free. + +### Dispatch override + +Set `QBLAS_DISPATCH=generic|sse2|avx2|avx512|neon` to force a tier — useful for comparing per-tier throughput in the bench and for diagnosing tier-specific bugs. The auto-selected tier is printed by `qblas_get_dispatch_tier()`. + +## Common gotchas + +- **Negative strides**: BLAS lets `incx < 0` walk a vector in reverse. The convention here: the caller (in `cblas_q*` front-ends) shifts `x` to its logical first element via `neg_offset(incx, n)`. The kernel iterates `ix = 0; ix += incx` from there — **never re-applies** the offset. There's a class of bugs where the offset gets applied twice (caller + kernel), so always cross-check the kernel's strided branch. + +- **SLEEF naming**: ULP-bounded ops (add, fma, mul, sqrt) use `Sleef_q_u05` (no underscore before isa) — e.g. `Sleef_fmaq4_u05avx2`. Non-ULP ops (splat, load, store, get, fabs, neg) use `Sleef_q_` — e.g. `Sleef_storeq2_sse2`. The template's `QV_NAME` macro encodes this. + +- **`Sleef_storeq2` / `Sleef_loadq2` (no ISA suffix) are not exported** even though declared. Always use explicit-ISA suffixes for load/store. + +- **`tlfloat.so` is not installed by SLEEF's `cmake --install`** even when it builds. The bootstrap script copies it manually into `.sleef-prefix/lib/`. + +## Performance baseline (AMD EPYC 7V13, 96 cores, AVX2 tier) + +| Routine | Size | Throughput (items/s, multi-thread) | +| ------- | ---- | --------- | +| qdot | 1 M | 836 M | +| qaxpy | 4 M | 2.15 G | +| qscal | 1 M | 1.90 G | +| qnrm2 | 4 M | 1.29 G | +| qasum | 1 M | 1.85 G | +| qgemv | 4096² | 935 M | +| qgemv_t | 4096² | 822 M | +| qgemm | 1024³ | **1.61 G FMA** | +| qsyrk | 1024² | 1.60 G FMA | + +Single-thread GEMM peak is ~30 MFMA/s; the parallel speedup is ~55× across 96 cores (≈57 % efficiency). The remaining gap is mostly per-quad-FMA latency inside SLEEF's `Sleef_fmaq4_u05avx2` (software DD over AVX2 doubles), not our blocking — further gains would need a hand-written micro-kernel that interleaves the DD doubles across multiple FMAs. + +## Repo layout reference + +- [include/qblas/qblas.h](include/qblas/qblas.h) — umbrella public header (pulls in level1/2/3) +- [src/CMakeLists.txt](src/CMakeLists.txt) — `qblas_add_tier()` macro for per-ISA OBJECT libs +- [src/cpu/qblas_cpu.c](src/cpu/qblas_cpu.c) — CPUID detection + `__attribute__((constructor))` library init +- [src/kernels/kernels_template.h](src/kernels/kernels_template.h) — the generic SIMD kernel template +- [scripts/bootstrap_sleef.sh](scripts/bootstrap_sleef.sh) — required first-time setup diff --git a/CMakeLists.txt b/CMakeLists.txt index 462d3ea..8b10b69 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,149 +1,101 @@ -cmake_minimum_required(VERSION 3.15) -project(QuadBLAS) +cmake_minimum_required(VERSION 3.16) +project(qblas + VERSION 1.5.0 + LANGUAGES C CXX + DESCRIPTION "BLAS for IEEE-754 binary128 on top of SLEEF") -# Set build type to Release if not specified -if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) -endif() +option(QBLAS_BUILD_TESTS "Build correctness tests" ON) +option(QBLAS_BUILD_BENCH "Build Google Benchmark suite" ON) +option(QBLAS_BUILD_SHARED "Build shared library" ON) +option(QBLAS_USE_OPENMP "Enable OpenMP threading" ON) +option(QBLAS_NATIVE "Tune top-level code for the host CPU" OFF) -# Set C++ standard globally +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE) +endif() +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_C_VISIBILITY_PRESET hidden) +set(CMAKE_CXX_VISIBILITY_PRESET hidden) +set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) -# Find OpenMP (your existing code) -if(APPLE) - find_library(OpenMP_LIBRARY - NAMES omp gomp iomp5 - PATHS /opt/homebrew/lib /usr/local/lib /opt/homebrew/opt/libomp/lib - NO_DEFAULT_PATH - ) - find_path(OpenMP_INCLUDE_DIR - NAMES omp.h - PATHS /opt/homebrew/include /usr/local/include /opt/homebrew/opt/libomp/include - NO_DEFAULT_PATH - ) - - if(OpenMP_LIBRARY AND OpenMP_INCLUDE_DIR) - set(OpenMP_FOUND TRUE) - add_library(OpenMP::OpenMP_CXX UNKNOWN IMPORTED) - set_target_properties(OpenMP::OpenMP_CXX PROPERTIES - IMPORTED_LOCATION "${OpenMP_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${OpenMP_INCLUDE_DIR}" - INTERFACE_COMPILE_OPTIONS "-Xpreprocessor;-fopenmp" - ) - message(STATUS "Found OpenMP: ${OpenMP_LIBRARY}") - else() - set(OpenMP_FOUND FALSE) - message(WARNING "OpenMP not found. Install with: brew install libomp") - endif() -else() - find_package(OpenMP) -endif() +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") -# Handle SLEEF library (fixed for your environment) -set(SLEEF_CUSTOM_ROOT "$ENV{SLEEF_ROOT}") -if(SLEEF_CUSTOM_ROOT) - set(SLEEF_INCLUDE_DIR "${SLEEF_CUSTOM_ROOT}/include") - set(SLEEF_LIB_DIR "${SLEEF_CUSTOM_ROOT}/lib") - if(NOT EXISTS ${SLEEF_INCLUDE_DIR} OR NOT EXISTS ${SLEEF_LIB_DIR}) - message(FATAL_ERROR "SLEEF installation not found at: ${SLEEF_CUSTOM_ROOT}") - endif() - - find_library(SLEEF_LIBRARY - NAMES sleef - PATHS ${SLEEF_LIB_DIR} - NO_DEFAULT_PATH - ) - find_library(SLEEFQUAD_LIBRARY - NAMES sleefquad - PATHS ${SLEEF_LIB_DIR} - NO_DEFAULT_PATH - ) - - if(SLEEF_LIBRARY AND SLEEFQUAD_LIBRARY) - set(SLEEF_LIBRARIES ${SLEEF_LIBRARY} ${SLEEFQUAD_LIBRARY}) - message(STATUS "Using SLEEF from: ${SLEEF_CUSTOM_ROOT}") - message(STATUS "Found SLEEF libraries: ${SLEEF_LIBRARIES}") - else() - message(FATAL_ERROR "SLEEF libraries not found in ${SLEEF_LIB_DIR}") - endif() -else() - find_package(PkgConfig REQUIRED) - pkg_check_modules(SLEEF REQUIRED sleef) +include(CheckCSourceCompiles) +include(CheckCCompilerFlag) +include(QBLASArch) +qblas_detect_arch() +message(STATUS "QBLAS target arch: ${QBLAS_ARCH}") +message(STATUS "QBLAS ISA tiers: ${QBLAS_ISA_TIERS}") + +# SLEEF: SLEEF_ROOT env var > .sleef-prefix > system pkg-config. +if(DEFINED ENV{SLEEF_ROOT} AND NOT SLEEF_ROOT) + set(SLEEF_ROOT "$ENV{SLEEF_ROOT}") +endif() +if(NOT SLEEF_ROOT AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.sleef-prefix/include/sleefquad.h") + set(SLEEF_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/.sleef-prefix") endif() -# Add Google Benchmark with proper Release configuration -if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/third_party/benchmark/CMakeLists.txt") - # Configure Google Benchmark for Release mode and disable testing - set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Enable testing of the benchmark library.") - set(BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE BOOL "Enable Google Test in benchmark.") - set(BENCHMARK_DOWNLOAD_DEPENDENCIES ON CACHE BOOL "Allow benchmark to download dependencies.") - - # Ensure benchmark is built in Release mode - set(CMAKE_BUILD_TYPE_BACKUP ${CMAKE_BUILD_TYPE}) - set(CMAKE_BUILD_TYPE Release) - - add_subdirectory(third_party/benchmark) - - # Restore original build type - set(CMAKE_BUILD_TYPE ${CMAKE_BUILD_TYPE_BACKUP}) - - set(GOOGLE_BENCHMARK_AVAILABLE TRUE) - message(STATUS "Google Benchmark added (Release mode)") -else() - set(GOOGLE_BENCHMARK_AVAILABLE FALSE) - message(STATUS "Google Benchmark not found (run: git submodule add https://github.com/google/benchmark.git third_party/benchmark)") +find_path(SLEEF_INCLUDE_DIR sleefquad.h + HINTS "${SLEEF_ROOT}/include" /usr/local/include /usr/include) +find_library(SLEEF_LIBRARY NAMES sleef HINTS "${SLEEF_ROOT}/lib" /usr/local/lib /usr/lib) +find_library(SLEEFQUAD_LIBRARY NAMES sleefquad HINTS "${SLEEF_ROOT}/lib" /usr/local/lib /usr/lib) + +if(NOT SLEEF_INCLUDE_DIR OR NOT SLEEF_LIBRARY OR NOT SLEEFQUAD_LIBRARY) + message(FATAL_ERROR + "SLEEF (with quad support) not found. " + "Set SLEEF_ROOT or run scripts/bootstrap_sleef.sh.") endif() +message(STATUS "SLEEF: ${SLEEFQUAD_LIBRARY}") -# Create executables -add_executable(quadblas_test test_quadblas.cpp) -add_executable(quadblas_benchmark_legacy benchmarks/legacy_benchmark.cpp) +find_library(TLFLOAT_LIBRARY NAMES tlfloat HINTS "${SLEEF_ROOT}/lib" /usr/local/lib /usr/lib) + +add_library(qblas::sleef INTERFACE IMPORTED) +target_include_directories(qblas::sleef INTERFACE "${SLEEF_INCLUDE_DIR}") +target_link_libraries(qblas::sleef INTERFACE "${SLEEFQUAD_LIBRARY}" "${SLEEF_LIBRARY}") +if(TLFLOAT_LIBRARY) + target_link_libraries(qblas::sleef INTERFACE "${TLFLOAT_LIBRARY}") +endif() -# Add Google Benchmark executable if available -if(GOOGLE_BENCHMARK_AVAILABLE) - add_executable(quadblas_benchmark benchmarks/benchmark.cpp) +# Use DT_RPATH (not DT_RUNPATH) on glibc so libsleefquad's NEEDED libtlfloat +# resolves through the executable's rpath. +set(CMAKE_INSTALL_RPATH "${SLEEF_ROOT}/lib") +set(CMAKE_BUILD_RPATH "${SLEEF_ROOT}/lib") +set(CMAKE_BUILD_WITH_INSTALL_RPATH OFF) +if((CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") AND NOT APPLE) + add_link_options(-Wl,--disable-new-dtags) endif() -# Common configuration function -function(configure_target target_name) - target_compile_features(${target_name} PRIVATE cxx_std_17) - - # Set optimization flags based on build type - target_compile_options(${target_name} PRIVATE - $<$:-O3 -march=native -ffast-math -DNDEBUG> - $<$:-g -O0 -fsanitize=address> - $<$:-O2 -g -march=native -ffast-math> - ) - - target_include_directories(${target_name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) - - if(SLEEF_CUSTOM_ROOT) - target_include_directories(${target_name} PRIVATE ${SLEEF_INCLUDE_DIR}) +if(QBLAS_USE_OPENMP) + find_package(OpenMP) + if(OpenMP_C_FOUND) + message(STATUS "OpenMP: enabled (${OpenMP_C_VERSION})") else() - target_include_directories(${target_name} PRIVATE ${SLEEF_INCLUDE_DIRS}) - target_link_directories(${target_name} PRIVATE ${SLEEF_LIBRARY_DIRS}) - target_compile_definitions(${target_name} PRIVATE ${SLEEF_CFLAGS_OTHER}) + message(WARNING "OpenMP not found - building single-threaded") + set(QBLAS_USE_OPENMP OFF) endif() - - target_link_libraries(${target_name} - ${SLEEF_LIBRARIES} - $<$:OpenMP::OpenMP_CXX> - $<$:-fsanitize=address> - ) -endfunction() +endif() -# Configure targets -configure_target(quadblas_test) -configure_target(quadblas_benchmark_legacy) +add_subdirectory(src) + +if(QBLAS_BUILD_TESTS) + enable_testing() + add_subdirectory(tests) +endif() -# Configure Google Benchmark target with proper linking -if(GOOGLE_BENCHMARK_AVAILABLE) - configure_target(quadblas_benchmark) - target_link_libraries(quadblas_benchmark benchmark::benchmark) +if(QBLAS_BUILD_BENCH) + add_subdirectory(bench) endif() -message(STATUS "=== Configuration ===") -message(STATUS "Build Type: ${CMAKE_BUILD_TYPE}") -message(STATUS "OpenMP: ${OpenMP_FOUND}") -message(STATUS "SLEEF: ${SLEEF_LIBRARIES}") -message(STATUS "Google Benchmark: ${GOOGLE_BENCHMARK_AVAILABLE}") \ No newline at end of file +message(STATUS "") +message(STATUS "=== QBLAS configuration ===") +message(STATUS " Build type : ${CMAKE_BUILD_TYPE}") +message(STATUS " Arch / ISAs : ${QBLAS_ARCH} / ${QBLAS_ISA_TIERS}") +message(STATUS " OpenMP : ${QBLAS_USE_OPENMP}") +message(STATUS " Tests : ${QBLAS_BUILD_TESTS}") +message(STATUS " Benchmarks : ${QBLAS_BUILD_BENCH}") +message(STATUS " SLEEF root : ${SLEEF_ROOT}") +message(STATUS "") diff --git a/README.md b/README.md index 64d349a..2cdc621 100644 --- a/README.md +++ b/README.md @@ -1,202 +1,316 @@ -# QuadBLAS (QBLAS) +# QBLAS -QuadBLAS is a high-performance linear algebra library implementing BLAS-compliant routines for IEEE 754 quadruple precision (binary128) floating-point arithmetic. Built as a header-only templated library on top of the SLEEF vectorized mathematical library, QuadBLAS provides optimized implementations of fundamental linear algebra operations with significant performance improvements over naive implementations. +High-performance BLAS for IEEE 754 binary128 (quad-precision) floating point, built on top of [SLEEF](https://sleef.org). Compiled C library with runtime CPU dispatch (SSE2 / AVX2 / AVX-512 on x86, NEON on AArch64). Drop-in CBLAS-style C API. -## Technical Features +This repository is the rewrite on branch `overhaul-rewrite`. The legacy header-only implementation on `main` is preserved for reference but is no longer maintained. -- **Quadruple Precision Arithmetic**: Full IEEE 754 binary128 (128-bit) floating-point support -- **SIMD Vectorization**: Platform-optimized implementations for x86-64 SSE/AVX and ARM64 NEON instruction sets -- **Parallel Execution**: OpenMP-based multi-threading with configurable parallelization thresholds -- **Memory Hierarchy Optimization**: Multi-level cache blocking algorithms for optimal data locality -- **Header-Only Design**: Template-based implementation enabling compile-time optimizations -- **BLAS-Compliant Interface**: Standard Level 1, 2, and 3 BLAS routine signatures -- **Cross-Platform Compatibility**: Support for Windows, Linux, and macOS operating systems -- **Dual API Design**: Both C and C++ interfaces for integration flexibility +## What is included -## Performance Characteristics +Routines implemented and tested: -QuadBLAS demonstrates substantial performance improvements over naive implementations through algorithmic optimizations and parallel execution: +| Level | Routine | Description | +| ------- | ----------- | -------------------------------------------------------- | +| 1 | `cblas_qdot` | dot product | +| 1 | `cblas_qaxpy` | y = alpha*x + y | +| 1 | `cblas_qscal` | x = alpha*x | +| 1 | `cblas_qnrm2` | sqrt(sum(x_i^2)) | +| 1 | `cblas_qasum` | sum(|x_i|) | +| 1 | `cblas_iqamax` | argmax |x_i| | +| 1 | `cblas_qcopy` | y = x | +| 1 | `cblas_qswap` | swap x and y | +| 2 | `cblas_qgemv` | y = alpha*op(A)*x + beta*y | +| 2 | `cblas_qger` | A += alpha*x*y^T | +| 2 | `cblas_qsymv` | y = alpha*A*x + beta*y, A symmetric | +| 2 | `cblas_qtrsv` | solve op(A)*x = b in place | +| 3 | `cblas_qgemm` | C = alpha*op(A)*op(B) + beta*C | +| 3 | `cblas_qsyrk` | C = alpha*op(A)*op(A)^T + beta*C | +| 3 | `cblas_qtrmm` | B = alpha*op(A)*B or alpha*B*op(A) | +| 3 | `cblas_qtrsm` | solve op(A)*X = alpha*B or X*op(A) = alpha*B | -| Operation Category | Problem Scale | Performance Improvement | Throughput | -| -------------------- | ------------------ | ------------------------------- | --------------------- | -| Level 1 BLAS (DDOT) | 10^5 elements | 21× over serial implementation | Vectorized execution | -| Level 2 BLAS (DGEMV) | 1500×1500 matrix | 75× over serial implementation | 1.6 GFLOPS sustained | -| Level 3 BLAS (DGEMM) | 1000×1000 matrices | 2.8× over serial implementation | 0.06 GFLOPS sustained | +All routines accept both row-major and column-major matrices, all four `Trans` / `NoTrans` / `ConjTrans` combinations, `Upper` / `Lower`, `Unit` / `NonUnit`, and BLAS-standard positive or negative strides. -_Performance measurements conducted on multi-core x86-64 architecture with OpenMP threading enabled._ +Library control: -## Prerequisites +```c +const char *qblas_get_version(void); +const char *qblas_get_dispatch_tier(void); /* "generic" | "sse2" | "avx2" | "avx512" | "neon" */ +void qblas_set_num_threads(int n); /* qblas-internal cap; does NOT call omp_set_num_threads */ +int qblas_get_num_threads(void); /* effective: min(omp_get_max_threads(), qblas cap) */ +int qblas_get_max_threads(void); /* raw omp_get_max_threads() */ +``` + +**Threading contract.** qblas never modifies OpenMP's global thread count. `OMP_NUM_THREADS` and `omp_set_num_threads()` are the only knobs that change OpenMP's view; `qblas_set_num_threads(N)` only narrows qblas's own work distribution within whatever the host OpenMP setting is. This is intentional so that linking qblas into a larger application cannot silently change the application's other OpenMP regions. Per-region thread counts inside qblas come from the dynamic threading heuristic (work-cycles vs measured fork-cost) capped at the value of `qblas_get_num_threads()`. + +Environment overrides: + +- `QBLAS_DISPATCH=generic|sse2|avx2|avx512|neon` forces a specific SIMD tier. Useful to confirm cross-tier numerical equivalence. +- `OMP_NUM_THREADS=N` sets thread count (standard OpenMP). +- `OMP_PLACES=cores OMP_PROC_BIND=close` is recommended on many-core hosts to keep per-iteration timing stable. -- ISO C++17 compliant compiler (GCC ≥7.0, Clang ≥5.0, MSVC ≥2017) -- CMake build system (≥3.15) -- [SLEEF vectorized mathematical library](https://sleef.org/) (≥3.6) -- OpenMP runtime (optional, required for multi-threading) +## Dependencies + +- C11 + C++17 compilers (gcc, clang, AppleClang). +- CMake 3.16 or newer. +- OpenMP (libgomp, libomp). Optional but strongly recommended. +- SLEEF 3.8+ with quad-precision support, built with all SIMD paths enabled. The `scripts/bootstrap_sleef.sh` script builds it the right way. +- Google Benchmark (vendored as a submodule; pulled in automatically by `cmake`). ## Installation -We already provide a `build.sh` file, which can directly executes the steps detailed as follows: +### 1. Clone with submodules + +```bash +git clone --recursive https://github.com/SwayamInSync/QBLAS.git +cd QBLAS +``` -For systems with SLEEF built from source or non-standard installation paths: +If you cloned without `--recursive`: ```bash -# Configure SLEEF library path -export SLEEF_ROOT=/path/to/sleef/installation +git submodule update --init --recursive +``` + +### 2. Build SLEEF + +This must be done once, before the first qblas build: + +```bash +bash scripts/bootstrap_sleef.sh +``` + +The script builds SLEEF with every SIMD path turned on (`SSE2`, `AVX`, `AVX2`, `AVX512F` on x86; `AdvSIMD` on AArch64; inline headers enabled) and installs it into `.sleef-prefix/`. Without this, SLEEF only builds for the host CPU and binaries fail to link when run on a different CPU or to use a different ISA tier. + +The script also copies the vendored `libtlfloat` into the prefix, which is a runtime dependency of `libsleefquad`. -# Build QuadBLAS -git clone https://github.com/your-org/QuadBLAS.git -cd QuadBLAS -mkdir build && cd build -cmake -DCMAKE_BUILD_TYPE=Release .. -make -j$(nproc) +### 3. Build qblas -# Execute test suite -./quadblas_test -./quadblas_benchmark --benchmark_min_time=0.5s +```bash +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build --parallel ``` -## Usage +This produces: + +- `build/src/libqblas.so` (or `.dylib` on macOS) — the shared library. +- `build/tests/test_smoke`, `test_level1`, `test_level2`, `test_gemm`, `test_level3_extra` — correctness tests. +- `build/bench/qblas_bench` — Google Benchmark performance suite. +- `build/bench/qblas_bench_compare` — sized to match the legacy `main`-branch benchmark for direct before/after. -### Header-Only Integration +CMake options: -For direct integration into existing projects: +- `-DQBLAS_BUILD_SHARED=OFF` for a static library. +- `-DQBLAS_USE_OPENMP=OFF` to disable threading. +- `-DQBLAS_BUILD_TESTS=OFF` to skip test executables. +- `-DQBLAS_BUILD_BENCH=OFF` to skip benchmarks. +- `-DSLEEF_ROOT=/path/to/sleef-prefix` if your SLEEF lives somewhere other than `./.sleef-prefix/`. -```cpp -#include "include/quadblas/quadblas.hpp" +### 4. Install (optional) -// Link against SLEEF libraries during compilation -// g++ -O3 -fopenmp source.cpp -lsleef -lsleefquad +```bash +sudo cmake --install build ``` -### C++ Template Interface +Installs `libqblas` to `${CMAKE_INSTALL_PREFIX}/lib` and headers to `${CMAKE_INSTALL_PREFIX}/include/qblas/`. -```cpp -#include "include/quadblas/quadblas.hpp" +## Using qblas -int main() { - using namespace QuadBLAS; +Include the umbrella header: - // Instantiate quadruple precision containers - const size_t n = 1000; - DefaultVector<> x(n), y(n); - DefaultMatrix<> A(n, n); +```c +#include +``` - // Initialize with quadruple precision values - for (size_t i = 0; i < n; ++i) { - x[i] = Sleef_cast_from_doubleq1(static_cast(i + 1)); - y[i] = Sleef_cast_from_doubleq1(2.0); - for (size_t j = 0; j < n; ++j) { - A(i, j) = Sleef_cast_from_doubleq1((i == j) ? 2.0 : 0.1); - } - } +Compile and link: - // Execute optimized linear algebra operations - Sleef_quad dot_result = x.dot(y); // Level 1 BLAS: DDOT - Sleef_quad norm_result = x.norm(); // Level 1 BLAS: DNRM2 - A.gemv(SLEEF_QUAD_C(1.0), x, SLEEF_QUAD_C(0.0), y); // Level 2 BLAS: DGEMV +```bash +gcc -O3 my_app.c -lqblas -lsleefquad -lsleef -o my_app +``` +Minimal example: + +```c +#include +#include + +int main(void) { + enum { N = 4 }; + Sleef_quad x[N], y[N]; + for (int i = 0; i < N; ++i) { + x[i] = Sleef_cast_from_doubleq1((double)(i + 1)); + y[i] = Sleef_cast_from_doubleq1(2.0); + } + Sleef_quad d = cblas_qdot(N, x, 1, y, 1); + printf("dot = %f\n", (double)Sleef_cast_to_doubleq1(d)); /* 20.0 */ + printf("tier = %s\n", qblas_get_dispatch_tier()); return 0; } ``` -### C Interface for Language Interoperability +GEMM: ```c -#include "include/quadblas/quadblas.hpp" +#include + +void mul(const Sleef_quad *A, const Sleef_quad *B, Sleef_quad *C, + int m, int n, int k) { + Sleef_quad alpha = Sleef_cast_from_doubleq1(1.0); + Sleef_quad beta = Sleef_cast_from_doubleq1(0.0); + cblas_qgemm(QblasRowMajor, QblasNoTrans, QblasNoTrans, + m, n, k, alpha, A, k, B, n, beta, C, n); +} +``` + +The full public API is in `include/qblas/qblas.h` (level-1/2/3 headers are included transitively). + +## Correctness testing -// Level 1 BLAS: Dot product computation -double result = quadblas_qdot(n, x_ptr, 1, y_ptr, 1); +Every routine is checked against a naive scalar reference written in plain SLEEF `Sleef_*q1` calls. Tests cover all transpose / layout / uplo / diag combinations, positive and negative strides, and sizes that exercise edge tiles (m=1, k=1, primes, sizes both above and below the GEMM block size). -// Level 2 BLAS: Matrix-vector multiplication -// y := alpha*A*x + beta*y -quadblas_qgemv('R', 'N', m, n, 1.0, A_ptr, lda, x_ptr, 1, 0.0, y_ptr, 1); +Run the full suite: -// Level 3 BLAS: Matrix-matrix multiplication -// C := alpha*A*B + beta*C -quadblas_qgemm('R', 'N', 'N', m, n, k, 1.0, A_ptr, lda, B_ptr, ldb, 0.0, C_ptr, ldc); +```bash +ctest --test-dir build --output-on-failure +``` + +Run a single executable: -// Runtime configuration -quadblas_set_num_threads(16); -int active_threads = quadblas_get_num_threads(); +```bash +./build/tests/test_gemm +./build/tests/test_level3_extra +``` + +Tolerance is `1e-25` for vector operations (about 9 orders of magnitude tighter than double precision) and a backward-error bound for trsm. + +Cross-tier validation: + +```bash +QBLAS_DISPATCH=generic ctest --test-dir build +QBLAS_DISPATCH=sse2 ctest --test-dir build +QBLAS_DISPATCH=avx2 ctest --test-dir build ``` -### Advanced Configuration +All four should pass identically. This confirms the runtime CPU dispatch produces the same result across SIMD widths. If you set `QBLAS_DISPATCH` to a tier the CPU does not support (e.g. `avx512` on a Zen 3 box), the library prints a stderr warning and falls back to the auto-detected tier instead of executing illegal instructions. + +### numpy comparison test (`test_against_numpy`) + +In addition to the C tests against a naive scalar reference, there is a Python test that calls every QBLAS routine via ctypes, rounds the quad output to float64, and compares against numpy's float64 implementation of the same operation. This catches any issue that survives both the naive C reference and the quad-precision pathway but would surface when a user does `Sleef_cast_to_doubleq1(...)`. + +Requirements: Python 3 and `numpy >= 2.0`. + +```bash +python3 -m venv .venv +.venv/bin/pip install 'numpy>=2.0' +PATH=$(pwd)/.venv/bin:$PATH ctest --test-dir build --output-on-failure -R test_against_numpy +``` + +The test exercises all Level 1 / 2 / 3 routines at a range of sizes with random `~U(-1, 1)` inputs. Tolerances are picked per routine to allow for the double-precision rounding step (~1e-13 for elementwise, ~1e-10 for matrix-vector, scales with k for matrix-matrix). + +## Benchmarking -```cpp -// Thread pool configuration for OpenMP execution -QuadBLAS::set_num_threads(16); +There are two benchmark binaries. Both use Google Benchmark. -// Memory layout specification -QuadBLAS::MatrixRowMajor A_c_style(m, n); // Row-major storage (C convention) -QuadBLAS::MatrixColMajor A_fortran_style(m, n); // Column-major storage (Fortran convention) +### `qblas_bench` (full) -// Manual memory management with SIMD alignment -Sleef_quad* aligned_buffer = QuadBLAS::aligned_alloc(1000); -QuadBLAS::Vector custom_vector(aligned_buffer, 1000); -// ... computational work ... -QuadBLAS::aligned_free(aligned_buffer); +Sweeps powers of 2 / 4 from 1024 up to 4M elements for L1, 128 to 4096 for L2 and GEMV, 64 to 1024 for GEMM and SYRK. Useful for tuning. + +```bash +./build/bench/qblas_bench --benchmark_min_time=0.3s --benchmark_repetitions=3 ``` -### Numerical Precision Demonstration +### `qblas_bench_compare` (size-matched to legacy) -```cpp -// Demonstrate quadruple precision numerical stability -QuadBLAS::DefaultVector<> x(3), y(3); +Same sizes as the original `benchmarks/benchmark.cpp` on the legacy `main` branch, so before/after comparison is direct. -// Configure test case prone to catastrophic cancellation in double precision -x[0] = Sleef_cast_from_doubleq1(1e20); // Large positive value -x[1] = Sleef_cast_from_doubleq1(1.0); // Unit value -x[2] = Sleef_cast_from_doubleq1(-1e20); // Large negative value +```bash +./build/bench/qblas_bench_compare --benchmark_min_time=0.3s --benchmark_repetitions=3 \ + --benchmark_report_aggregates_only=true +``` -y[0] = y[1] = y[2] = Sleef_cast_from_doubleq1(1.0); +### Recommended environment -// Comparison of arithmetic precision -double double_precision_result = 1e20 * 1.0 + 1.0 * 1.0 + (-1e20) * 1.0; // → 0.0 (precision loss) -Sleef_quad quad_precision_result = x.dot(y); // → 1.0 (mathematically correct) +On many-core hosts, pin threads for stable timing: -std::cout << "Double precision computation: " << double_precision_result << std::endl; -std::cout << "Quadruple precision computation: " << Sleef_cast_to_doubleq1(quad_precision_result) << std::endl; +```bash +OMP_NUM_THREADS=16 OMP_PLACES=cores OMP_PROC_BIND=close \ + ./build/bench/qblas_bench --benchmark_min_time=0.3s ``` -## Contributing +Use `OMP_NUM_THREADS=1` to measure pure single-thread kernel quality. -We welcome contributions to QuadBLAS development. Please follow established contribution guidelines: +### Performance numbers -### Development Process +On AMD EPYC 7V13 (96 cores, AVX2 tier, OMP_NUM_THREADS=16, median of 3): -1. Fork the repository and create a feature branch -2. Implement changes following existing code style conventions -3. Add comprehensive tests for new functionality -4. Update documentation and performance benchmarks as appropriate -5. Submit pull requests with detailed change descriptions +| Routine | Size | Throughput (M items/sec) | +| ------- | ---- | -----------------------: | +| qdot | 1M | 429.6 | +| qaxpy | 64K | 387.8 | +| qnrm2 | 64K | 409.8 | +| qasum | 64K | 853.7 | +| qscal | 64K | 888.2 | +| qgemv | 1600 x 1600 | 431.2 | +| qgemm | 512 x 512 x 512 | 429.7 | +| qsyrk | 512 x 512 x 512 | 443.3 | +| qtrmm | 512 x 512 | 350.3 | +| qtrsm | 512 x 512 | 335.7 | -### Contribution Areas +For full before/after tables (including single-thread and 96-thread numbers), see [perf_comparison_with_old.md](perf_comparison_with_old.md). -- Algorithm optimization and new BLAS routine implementations -- Platform-specific optimizations and hardware support -- Performance analysis and benchmarking improvements -- Documentation enhancement and example development -- Integration support for additional programming languages +For remaining performance opportunities not yet taken, see [performance_bottlenecks.md](performance_bottlenecks.md). + +## Repository layout + +``` +include/qblas/ Public C headers. +src/ Library sources. + cpu/qblas_cpu.c Runtime CPU and cache detection, OMP overhead timing, + dispatch table population, threading helpers. + common/ Internal helpers shared by all kernels. + kernels/ Per-ISA kernel translation units. + generic/ Scalar fallback (always built). + sse2/ x86 SSE2 (q2 width). + avx2/ x86 AVX2 (q4 width). + avx512/ x86 AVX-512F (q8 width). + neon/ AArch64 AdvSIMD (q2 width). + kernels_template.h Shared template instantiated per ISA. + level1/ L1 entry points (dispatch + threading). + level2/ L2 entry points. + level3/ qgemm, qsyrk, qtrmm, qtrsm. +tests/ ctest binaries. + python/ ctypes wrapper + numpy-vs-qblas test. +bench/ Google Benchmark binaries. +cmake/ CMake helpers (architecture detection). +scripts/ bootstrap_sleef.sh. +third_party/ sleef, benchmark, OpenBLAS (for reference) submodules. +``` + +## Continuous integration + +The `.github/workflows/ci.yml` workflow runs on every push and pull request: + +- Ubuntu (gcc and clang) and macOS (clang). +- Builds SLEEF via `scripts/bootstrap_sleef.sh`. +- Builds qblas and runs `ctest`. +- Smoke-runs the benchmark. +- On Linux only, re-runs the test suite under each forced dispatch tier (generic / sse2 / avx2) to verify they all give the same results. +- A separate `bench-baseline` job runs the matched-size benchmark suite and uploads results as an artifact for tracking perf over time. + +## License + +Apache 2.0. See [LICENSE](LICENSE). ## Citation ```bibtex -@software{quadblas2025, - title = {QuadBLAS: High-Performance Linear Algebra for Quadruple Precision Computing}, +@software{qblas2025, + title = {QBLAS: High-Performance BLAS for IEEE-754 Quadruple Precision}, author = {Swayam Singh}, - year = {2025}, - url = {https://github.com/SwayamInSync/QBLAS}, - version = {1.0.0}, - note = {A header-only BLAS library for cross platform IEEE 754 quadruple precision arithmetic} + year = {2025}, + url = {https://github.com/SwayamInSync/QBLAS}, + note = {BLAS Levels 1, 2, and 3 for binary128 floating point, built on SLEEF} } ``` -## Acknowledgments +## Acknowledgements -``` -@software{sleef, - title = {SLEEF: A Portable Vectorized Library of Elementary Functions}, - author = {Naoki Shibata and contributors}, - url = {https://github.com/shibatch/sleef}, - note = {SIMD Library for Evaluating Elementary Functions} -} -``` +Built on [SLEEF](https://sleef.org) (Naoki Shibata and contributors). diff --git a/bench/CMakeLists.txt b/bench/CMakeLists.txt new file mode 100644 index 0000000..9736ba5 --- /dev/null +++ b/bench/CMakeLists.txt @@ -0,0 +1,22 @@ +# Google Benchmark (vendored under third_party/benchmark). +if(EXISTS "${CMAKE_SOURCE_DIR}/third_party/benchmark/CMakeLists.txt") + set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "") + set(BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE BOOL "") + set(BENCHMARK_DOWNLOAD_DEPENDENCIES OFF CACHE BOOL "") + set(BENCHMARK_INSTALL_DOCS OFF CACHE BOOL "") + if(NOT TARGET benchmark::benchmark) + add_subdirectory(${CMAKE_SOURCE_DIR}/third_party/benchmark + ${CMAKE_CURRENT_BINARY_DIR}/benchmark_build EXCLUDE_FROM_ALL) + endif() +else() + message(STATUS "Google Benchmark submodule missing - bench/ disabled.") + return() +endif() + +add_executable(qblas_bench bench_qblas.cc) +target_link_libraries(qblas_bench PRIVATE qblas benchmark::benchmark) +target_include_directories(qblas_bench PRIVATE ${CMAKE_SOURCE_DIR}/include) + +add_executable(qblas_bench_compare bench_compare.cc) +target_link_libraries(qblas_bench_compare PRIVATE qblas benchmark::benchmark) +target_include_directories(qblas_bench_compare PRIVATE ${CMAKE_SOURCE_DIR}/include) diff --git a/bench/bench_compare.cc b/bench/bench_compare.cc new file mode 100644 index 0000000..d5d0d81 --- /dev/null +++ b/bench/bench_compare.cc @@ -0,0 +1,158 @@ +/* Size grid matches the legacy benchmarks/benchmark.cpp on `main` so + * before/after numbers line up routine-by-routine. */ + +#include +#include +#include +#include + +namespace { +Sleef_quad qd(double x) { return Sleef_cast_from_doubleq1(x); } +template +void fill_random(std::vector &v, unsigned seed) { + std::mt19937 g(seed); + std::uniform_real_distribution d(-1.0, 1.0); + for (auto &q : v) q = qd(d(g)); +} +} + +static void BM_DotProduct(benchmark::State &state) { + size_t n = static_cast(state.range(0)); + std::vector x(n), y(n); + fill_random(x, 1); fill_random(y, 2); + Sleef_quad s = qd(0.0); + for (auto _ : state) { + s = cblas_qdot(static_cast(n), x.data(), 1, y.data(), 1); + benchmark::DoNotOptimize(s); + } + state.SetItemsProcessed(state.iterations() * static_cast(n)); +} +BENCHMARK(BM_DotProduct)->Arg(1000)->Arg(10000)->Arg(100000)->Arg(1000000); + +static void BM_AXPY(benchmark::State &state) { + size_t n = static_cast(state.range(0)); + std::vector x(n), y(n); + fill_random(x, 1); fill_random(y, 2); + Sleef_quad a = qd(1.25); + for (auto _ : state) { + cblas_qaxpy(static_cast(n), a, x.data(), 1, y.data(), 1); + benchmark::DoNotOptimize(y.data()); + } + state.SetItemsProcessed(state.iterations() * static_cast(n)); +} +BENCHMARK(BM_AXPY)->Arg(1000)->Arg(4000)->Arg(16000)->Arg(64000); + +static void BM_VectorNorm(benchmark::State &state) { + size_t n = static_cast(state.range(0)); + std::vector x(n); + fill_random(x, 1); + Sleef_quad s = qd(0.0); + for (auto _ : state) { + s = cblas_qnrm2(static_cast(n), x.data(), 1); + benchmark::DoNotOptimize(s); + } + state.SetItemsProcessed(state.iterations() * static_cast(n)); +} +BENCHMARK(BM_VectorNorm)->Arg(1000)->Arg(4000)->Arg(16000)->Arg(64000); + +static void BM_ASUM(benchmark::State &state) { + size_t n = static_cast(state.range(0)); + std::vector x(n); + fill_random(x, 1); + Sleef_quad s = qd(0.0); + for (auto _ : state) { + s = cblas_qasum(static_cast(n), x.data(), 1); + benchmark::DoNotOptimize(s); + } + state.SetItemsProcessed(state.iterations() * static_cast(n)); +} +BENCHMARK(BM_ASUM)->Arg(1000)->Arg(4000)->Arg(16000)->Arg(64000); + +static void BM_SCAL(benchmark::State &state) { + size_t n = static_cast(state.range(0)); + std::vector x(n); + fill_random(x, 1); + Sleef_quad a = qd(0.9999); + for (auto _ : state) { + cblas_qscal(static_cast(n), a, x.data(), 1); + benchmark::DoNotOptimize(x.data()); + } + state.SetItemsProcessed(state.iterations() * static_cast(n)); +} +BENCHMARK(BM_SCAL)->Arg(1000)->Arg(4000)->Arg(16000)->Arg(64000); + +static void BM_GEMV(benchmark::State &state) { + size_t m = static_cast(state.range(0)), n = m; + std::vector A(m*n), x(n), y(m); + fill_random(A, 1); fill_random(x, 2); fill_random(y, 3); + Sleef_quad alpha = qd(1.0), beta = qd(0.0); + for (auto _ : state) { + cblas_qgemv(QblasRowMajor, QblasNoTrans, m, n, alpha, + A.data(), n, x.data(), 1, beta, y.data(), 1); + benchmark::DoNotOptimize(y.data()); + } + state.SetItemsProcessed(state.iterations() * static_cast(m * n)); +} +BENCHMARK(BM_GEMV)->Arg(100)->Arg(200)->Arg(400)->Arg(800)->Arg(1600); + +static void BM_GEMM(benchmark::State &state) { + size_t n = static_cast(state.range(0)); + std::vector A(n*n), B(n*n), C(n*n); + fill_random(A, 1); fill_random(B, 2); fill_random(C, 3); + Sleef_quad alpha = qd(1.0), beta = qd(0.0); + for (auto _ : state) { + cblas_qgemm(QblasRowMajor, QblasNoTrans, QblasNoTrans, + n, n, n, alpha, A.data(), n, B.data(), n, + beta, C.data(), n); + benchmark::DoNotOptimize(C.data()); + } + state.SetItemsProcessed(state.iterations() * static_cast(n * n * n)); +} +BENCHMARK(BM_GEMM)->Arg(64)->Arg(128)->Arg(256)->Arg(512); + +static void BM_SYRK(benchmark::State &state) { + size_t n = static_cast(state.range(0)); + size_t k = n; + std::vector A(n*k), C(n*n); + fill_random(A, 1); fill_random(C, 3); + Sleef_quad alpha = qd(1.0), beta = qd(0.0); + for (auto _ : state) { + cblas_qsyrk(QblasRowMajor, QblasUpper, QblasNoTrans, n, k, + alpha, A.data(), k, beta, C.data(), n); + benchmark::DoNotOptimize(C.data()); + } + state.SetItemsProcessed(state.iterations() * static_cast(n * n * k)); +} +BENCHMARK(BM_SYRK)->Arg(64)->Arg(128)->Arg(256)->Arg(512); + +static void BM_TRMM(benchmark::State &state) { + size_t m = static_cast(state.range(0)), n = m; + std::vector A(m*m), B(m*n); + fill_random(A, 1); fill_random(B, 2); + Sleef_quad alpha = qd(1.0); + for (auto _ : state) { + cblas_qtrmm(QblasRowMajor, QblasLeft, QblasLower, QblasNoTrans, QblasNonUnit, + m, n, alpha, A.data(), m, B.data(), n); + benchmark::DoNotOptimize(B.data()); + } + state.SetItemsProcessed(state.iterations() * static_cast(m * m * n / 2)); +} +BENCHMARK(BM_TRMM)->Arg(64)->Arg(128)->Arg(256)->Arg(512); + +static void BM_TRSM(benchmark::State &state) { + size_t m = static_cast(state.range(0)), n = m; + std::vector A(m*m), B(m*n), B0(m*n); + fill_random(A, 1); fill_random(B0, 2); + for (size_t i = 0; i < m; ++i) A[i*m+i] = qd(2.0 + (double)i*0.001); + Sleef_quad alpha = qd(1.0); + for (auto _ : state) { + std::copy(B0.begin(), B0.end(), B.begin()); + cblas_qtrsm(QblasRowMajor, QblasLeft, QblasLower, QblasNoTrans, QblasNonUnit, + m, n, alpha, A.data(), m, B.data(), n); + benchmark::DoNotOptimize(B.data()); + } + state.SetItemsProcessed(state.iterations() * static_cast(m * m * n / 2)); +} +BENCHMARK(BM_TRSM)->Arg(64)->Arg(128)->Arg(256)->Arg(512); + +BENCHMARK_MAIN(); diff --git a/bench/bench_qblas.cc b/bench/bench_qblas.cc new file mode 100644 index 0000000..7e68d8e --- /dev/null +++ b/bench/bench_qblas.cc @@ -0,0 +1,163 @@ +/* Google Benchmark suite. `items_per_second` = quad-FMA throughput + * (items = n for dot/axpy, m*n for gemv, m*n*k for gemm/syrk). */ + +#include +#include +#include +#include + +namespace { + +Sleef_quad qd(double x) { return Sleef_cast_from_doubleq1(x); } + +template +void fill_random(std::vector &v, unsigned seed) { + std::mt19937 g(seed); + std::uniform_real_distribution d(-1.0, 1.0); + for (auto &q : v) q = qd(d(g)); +} + +void BM_qdot(benchmark::State &state) { + size_t n = static_cast(state.range(0)); + std::vector x(n), y(n); + fill_random(x, 1); fill_random(y, 2); + Sleef_quad s = qd(0.0); + for (auto _ : state) { + s = cblas_qdot(static_cast(n), x.data(), 1, y.data(), 1); + benchmark::DoNotOptimize(s); + } + state.SetItemsProcessed(state.iterations() * static_cast(n)); + state.SetLabel("FMA/iter=" + std::to_string(n)); +} +BENCHMARK(BM_qdot)->RangeMultiplier(4)->Range(1<<10, 1<<22)->UseRealTime(); + +void BM_qaxpy(benchmark::State &state) { + size_t n = static_cast(state.range(0)); + std::vector x(n), y(n); + fill_random(x, 1); fill_random(y, 2); + Sleef_quad a = qd(1.25); + for (auto _ : state) { + cblas_qaxpy(static_cast(n), a, x.data(), 1, y.data(), 1); + benchmark::DoNotOptimize(y.data()); + } + state.SetItemsProcessed(state.iterations() * static_cast(n)); +} +BENCHMARK(BM_qaxpy)->RangeMultiplier(4)->Range(1<<10, 1<<22)->UseRealTime(); + +void BM_qscal(benchmark::State &state) { + size_t n = static_cast(state.range(0)); + std::vector x(n); + fill_random(x, 1); + Sleef_quad a = qd(0.99); + for (auto _ : state) { + cblas_qscal(static_cast(n), a, x.data(), 1); + benchmark::DoNotOptimize(x.data()); + } + state.SetItemsProcessed(state.iterations() * static_cast(n)); +} +BENCHMARK(BM_qscal)->RangeMultiplier(4)->Range(1<<10, 1<<22)->UseRealTime(); + +void BM_qnrm2(benchmark::State &state) { + size_t n = static_cast(state.range(0)); + std::vector x(n); + fill_random(x, 1); + Sleef_quad s = qd(0.0); + for (auto _ : state) { + s = cblas_qnrm2(static_cast(n), x.data(), 1); + benchmark::DoNotOptimize(s); + } + state.SetItemsProcessed(state.iterations() * static_cast(n)); +} +BENCHMARK(BM_qnrm2)->RangeMultiplier(4)->Range(1<<10, 1<<22)->UseRealTime(); + +void BM_qasum(benchmark::State &state) { + size_t n = static_cast(state.range(0)); + std::vector x(n); + fill_random(x, 1); + Sleef_quad s = qd(0.0); + for (auto _ : state) { + s = cblas_qasum(static_cast(n), x.data(), 1); + benchmark::DoNotOptimize(s); + } + state.SetItemsProcessed(state.iterations() * static_cast(n)); +} +BENCHMARK(BM_qasum)->RangeMultiplier(4)->Range(1<<10, 1<<22)->UseRealTime(); + +void BM_qgemv(benchmark::State &state) { + size_t m = static_cast(state.range(0)); + size_t n = m; + std::vector A(m * n), x(n), y(m); + fill_random(A, 1); fill_random(x, 2); fill_random(y, 3); + Sleef_quad alpha = qd(1.0), beta = qd(0.5); + for (auto _ : state) { + cblas_qgemv(QblasRowMajor, QblasNoTrans, m, n, alpha, + A.data(), n, x.data(), 1, beta, y.data(), 1); + benchmark::DoNotOptimize(y.data()); + } + state.SetItemsProcessed(state.iterations() * static_cast(m * n)); +} +BENCHMARK(BM_qgemv)->RangeMultiplier(2)->Range(128, 4096)->UseRealTime(); + +void BM_qgemv_t(benchmark::State &state) { + size_t m = static_cast(state.range(0)); + size_t n = m; + std::vector A(m * n), x(m), y(n); + fill_random(A, 1); fill_random(x, 2); fill_random(y, 3); + Sleef_quad alpha = qd(1.0), beta = qd(0.5); + for (auto _ : state) { + cblas_qgemv(QblasRowMajor, QblasTrans, m, n, alpha, + A.data(), n, x.data(), 1, beta, y.data(), 1); + benchmark::DoNotOptimize(y.data()); + } + state.SetItemsProcessed(state.iterations() * static_cast(m * n)); +} +BENCHMARK(BM_qgemv_t)->RangeMultiplier(2)->Range(128, 4096)->UseRealTime(); + +void BM_qgemm(benchmark::State &state) { + size_t n = static_cast(state.range(0)); + std::vector A(n*n), B(n*n), C(n*n); + fill_random(A, 1); fill_random(B, 2); fill_random(C, 3); + Sleef_quad alpha = qd(1.0), beta = qd(0.0); + for (auto _ : state) { + cblas_qgemm(QblasRowMajor, QblasNoTrans, QblasNoTrans, + n, n, n, alpha, A.data(), n, B.data(), n, + beta, C.data(), n); + benchmark::DoNotOptimize(C.data()); + } + state.SetItemsProcessed(state.iterations() * static_cast(n * n * n)); +} +BENCHMARK(BM_qgemm)->RangeMultiplier(2)->Range(64, 1024)->UseRealTime(); + +void BM_qgemm_TN(benchmark::State &state) { + size_t n = static_cast(state.range(0)); + std::vector A(n*n), B(n*n), C(n*n); + fill_random(A, 1); fill_random(B, 2); fill_random(C, 3); + Sleef_quad alpha = qd(1.0), beta = qd(0.0); + for (auto _ : state) { + cblas_qgemm(QblasRowMajor, QblasTrans, QblasNoTrans, + n, n, n, alpha, A.data(), n, B.data(), n, + beta, C.data(), n); + benchmark::DoNotOptimize(C.data()); + } + state.SetItemsProcessed(state.iterations() * static_cast(n * n * n)); +} +BENCHMARK(BM_qgemm_TN)->RangeMultiplier(2)->Range(64, 1024)->UseRealTime(); + +void BM_qsyrk(benchmark::State &state) { + size_t n = static_cast(state.range(0)); + size_t k = n; + std::vector A(n*k), C(n*n); + fill_random(A, 1); fill_random(C, 3); + Sleef_quad alpha = qd(1.0), beta = qd(0.0); + for (auto _ : state) { + cblas_qsyrk(QblasRowMajor, QblasUpper, QblasNoTrans, n, k, + alpha, A.data(), k, beta, C.data(), n); + benchmark::DoNotOptimize(C.data()); + } + state.SetItemsProcessed(state.iterations() * static_cast(n * n * k)); +} +BENCHMARK(BM_qsyrk)->RangeMultiplier(2)->Range(64, 1024)->UseRealTime(); + +} /* namespace */ + +BENCHMARK_MAIN(); diff --git a/benchmarks/benchmark.cpp b/benchmarks/benchmark.cpp deleted file mode 100644 index 7423834..0000000 --- a/benchmarks/benchmark.cpp +++ /dev/null @@ -1,432 +0,0 @@ -#include "include/quadblas/quadblas.hpp" -#include -#include -#include -#include -#include - -class QuadRandom -{ -private: - std::mt19937 gen; - std::uniform_real_distribution dist; - -public: - QuadRandom(unsigned seed = 42) : gen(seed), dist(-1.0, 1.0) {} - - Sleef_quad next() - { - return Sleef_cast_from_doubleq1(dist(gen)); - } - - void fill_array(Sleef_quad *arr, size_t n) - { - for (size_t i = 0; i < n; ++i) - { - arr[i] = next(); - } - } -}; - -template -class AlignedArray -{ -private: - T *data_; - size_t size_; - -public: - AlignedArray(size_t size) : size_(size) - { - data_ = QuadBLAS::aligned_alloc(size); - if (!data_) - { - throw std::bad_alloc(); - } - } - - ~AlignedArray() - { - QuadBLAS::aligned_free(data_); - } - - T *data() { return data_; } - const T *data() const { return data_; } - size_t size() const { return size_; } - - T &operator[](size_t i) { return data_[i]; } - const T &operator[](size_t i) const { return data_[i]; } -}; - -// Level 1 BLAS Benchmarks - -static void BM_DotProduct(benchmark::State &state) -{ - const size_t n = state.range(0); - - AlignedArray x(n), y(n); - QuadRandom rng; - rng.fill_array(x.data(), n); - rng.fill_array(y.data(), n); - - Sleef_quad result; - - for (auto _ : state) - { - result = QuadBLAS::dot(n, x.data(), 1, y.data(), 1); - benchmark::DoNotOptimize(result); - } - - state.SetItemsProcessed(state.iterations() * n); - state.SetBytesProcessed(state.iterations() * n * 2 * sizeof(Sleef_quad)); - state.counters["GFLOPS"] = benchmark::Counter( - 2.0 * n * state.iterations(), - benchmark::Counter::kIsRate, - benchmark::Counter::kIs1000); -} - -static void BM_VectorNorm(benchmark::State &state) -{ - const size_t n = state.range(0); - - QuadBLAS::DefaultVector<> x(n); - QuadRandom rng; - for (size_t i = 0; i < n; ++i) - { - x[i] = rng.next(); - } - - Sleef_quad result; - - for (auto _ : state) - { - result = x.norm(); - benchmark::DoNotOptimize(result); - } - - state.SetItemsProcessed(state.iterations() * n); - state.counters["GFLOPS"] = benchmark::Counter( - 2.0 * n * state.iterations(), - benchmark::Counter::kIsRate, - benchmark::Counter::kIs1000); -} - -// y = alpha * x + y -static void BM_AXPY(benchmark::State &state) -{ - const size_t n = state.range(0); - - QuadBLAS::DefaultVector<> x(n), y(n); - QuadRandom rng; - Sleef_quad alpha = rng.next(); - - for (size_t i = 0; i < n; ++i) - { - x[i] = rng.next(); - y[i] = rng.next(); - } - - for (auto _ : state) - { - y.axpy(alpha, x); - benchmark::DoNotOptimize(y.data()); - } - - state.SetItemsProcessed(state.iterations() * n); - state.counters["GFLOPS"] = benchmark::Counter( - 2.0 * n * state.iterations(), - benchmark::Counter::kIsRate, - benchmark::Counter::kIs1000); -} - -// Level 2 BLAS Benchmarks - -static void BM_GEMV(benchmark::State &state) -{ - const size_t n = state.range(0); - const size_t m = n; - - AlignedArray A(m * n), x(n), y(m); - QuadRandom rng; - - rng.fill_array(A.data(), m * n); - rng.fill_array(x.data(), n); - rng.fill_array(y.data(), m); - - Sleef_quad alpha = Sleef_cast_from_doubleq1(1.0); - Sleef_quad beta = Sleef_cast_from_doubleq1(0.0); - - for (auto _ : state) - { - QuadBLAS::gemv(QuadBLAS::Layout::RowMajor, m, n, alpha, - A.data(), n, x.data(), 1, beta, y.data(), 1); - benchmark::DoNotOptimize(y.data()); - } - - state.SetItemsProcessed(state.iterations() * m * n); - state.counters["GFLOPS"] = benchmark::Counter( - 2.0 * m * n * state.iterations(), - benchmark::Counter::kIsRate, - benchmark::Counter::kIs1000); - state.counters["MatrixSize"] = n; -} - -// Level 3 BLAS Benchmarks - -static void BM_GEMM(benchmark::State &state) -{ - const size_t n = state.range(0); - const size_t m = n, k = n; - - AlignedArray A(m * k), B(k * n), C(m * n); - QuadRandom rng; - - rng.fill_array(A.data(), m * k); - rng.fill_array(B.data(), k * n); - rng.fill_array(C.data(), m * n); - - Sleef_quad alpha = Sleef_cast_from_doubleq1(1.0); - Sleef_quad beta = Sleef_cast_from_doubleq1(0.0); - - for (auto _ : state) - { - QuadBLAS::gemm(QuadBLAS::Layout::RowMajor, m, n, k, alpha, - A.data(), k, B.data(), n, beta, C.data(), n); - benchmark::DoNotOptimize(C.data()); - } - - state.SetItemsProcessed(state.iterations() * m * n * k); - state.counters["GFLOPS"] = benchmark::Counter( - 2.0 * m * n * k * state.iterations(), - benchmark::Counter::kIsRate, - benchmark::Counter::kIs1000); - state.counters["MatrixSize"] = n; -} - -// Threading Scalability Benchmarks - -static void BM_ThreadingScalability(benchmark::State &state) -{ - const int num_threads = state.range(0); - const size_t n = 10000; - - QuadBLAS::set_num_threads(num_threads); - - AlignedArray x(n), y(n); - QuadRandom rng; - rng.fill_array(x.data(), n); - rng.fill_array(y.data(), n); - - Sleef_quad result; - - for (auto _ : state) - { - result = QuadBLAS::dot(n, x.data(), 1, y.data(), 1); - benchmark::DoNotOptimize(result); - } - - state.counters["Threads"] = num_threads; - state.counters["GFLOPS"] = benchmark::Counter( - 2.0 * n * state.iterations(), - benchmark::Counter::kIsRate, - benchmark::Counter::kIs1000); - - static double baseline_time = 0.0; - if (num_threads == 1) - { - baseline_time = state.counters["GFLOPS"]; - } - else if (baseline_time > 0.0) - { - double current_gflops = state.counters["GFLOPS"]; - double speedup = current_gflops / baseline_time; - double efficiency = speedup / num_threads * 100.0; - state.counters["Efficiency%"] = efficiency; - } -} - -// Numerical Precision Test -static void BM_NumericalPrecision(benchmark::State &state) -{ - QuadBLAS::DefaultVector<> x(3), y(3); - - x[0] = Sleef_cast_from_doubleq1(1e20); - x[1] = Sleef_cast_from_doubleq1(1.0); - x[2] = Sleef_cast_from_doubleq1(-1e20); - - y[0] = y[1] = y[2] = Sleef_cast_from_doubleq1(1.0); - - Sleef_quad result; - - for (auto _ : state) - { - result = x.dot(y); - benchmark::DoNotOptimize(result); - } - - double final_result = Sleef_cast_to_doubleq1(result); - state.counters["Result"] = final_result; - state.counters["Error"] = std::abs(final_result - 1.0); -} - -// Memory Layout Benchmarks - -static void BM_RowMajorGEMV(benchmark::State &state) -{ - const size_t n = state.range(0); - QuadBLAS::MatrixRowMajor A(n, n); - QuadBLAS::VectorRowMajor x(n), y(n); - - QuadRandom rng; - for (size_t i = 0; i < n; ++i) - { - x[i] = rng.next(); - for (size_t j = 0; j < n; ++j) - { - A(i, j) = rng.next(); - } - } - - for (auto _ : state) - { - A.gemv(Sleef_cast_from_doubleq1(1.0), x, Sleef_cast_from_doubleq1(0.0), y); - benchmark::DoNotOptimize(y.data()); - } - - state.counters["Layout"] = 0; - state.counters["GFLOPS"] = benchmark::Counter( - 2.0 * n * n * state.iterations(), - benchmark::Counter::kIsRate, - benchmark::Counter::kIs1000); -} - -static void BM_ColMajorGEMV(benchmark::State &state) -{ - const size_t n = state.range(0); - QuadBLAS::MatrixColMajor A(n, n); - QuadBLAS::VectorColMajor x(n), y(n); - - QuadRandom rng; - for (size_t i = 0; i < n; ++i) - { - x[i] = rng.next(); - for (size_t j = 0; j < n; ++j) - { - A(i, j) = rng.next(); - } - } - - for (auto _ : state) - { - A.gemv(Sleef_cast_from_doubleq1(1.0), x, Sleef_cast_from_doubleq1(0.0), y); - benchmark::DoNotOptimize(y.data()); - } - - state.counters["Layout"] = 1; - state.counters["GFLOPS"] = benchmark::Counter( - 2.0 * n * n * state.iterations(), - benchmark::Counter::kIsRate, - benchmark::Counter::kIs1000); -} - -// =============================================================================== -// REGISTER BENCHMARKS WITH PARAMETER RANGES -// =============================================================================== - -// Level 1 BLAS - Vector sizes from 1K to 1M -BENCHMARK(BM_DotProduct) - ->RangeMultiplier(10) - ->Range(1000, 1000000) - ->Unit(benchmark::kMicrosecond); - -BENCHMARK(BM_VectorNorm) - ->RangeMultiplier(4) - ->Range(1000, 64000) - ->Unit(benchmark::kMicrosecond); - -BENCHMARK(BM_AXPY) - ->RangeMultiplier(4) - ->Range(1000, 64000) - ->Unit(benchmark::kMicrosecond); - -// Level 2 BLAS - Matrix sizes from 100x100 to 2000x2000 -BENCHMARK(BM_GEMV) - ->RangeMultiplier(2) - ->Range(100, 1600) - ->Unit(benchmark::kMillisecond); - -BENCHMARK(BM_RowMajorGEMV) - ->RangeMultiplier(2) - ->Range(100, 800) - ->Unit(benchmark::kMillisecond); - -BENCHMARK(BM_ColMajorGEMV) - ->RangeMultiplier(2) - ->Range(100, 800) - ->Unit(benchmark::kMillisecond); - -// Level 3 BLAS - Matrix sizes from 64x64 to 512x512 (practical for quad precision) -BENCHMARK(BM_GEMM) - ->RangeMultiplier(2) - ->Range(64, 512) - ->Unit(benchmark::kMillisecond) - ->MinTime(2.0); // Ensure sufficient measurement time - -// Threading scalability - range set in main() to avoid static init issues -static void RegisterThreadingBenchmarks() -{ - int max_threads = std::min(16, QuadBLAS::get_num_threads()); - ::benchmark::RegisterBenchmark("BM_ThreadingScalability", BM_ThreadingScalability) - ->DenseRange(1, max_threads, 1) - ->Unit(benchmark::kMicrosecond); -} - -// Numerical precision test -BENCHMARK(BM_NumericalPrecision) - ->Iterations(1000) - ->Unit(benchmark::kNanosecond); - -// Custom main function with QuadBLAS info -int main(int argc, char **argv) -{ - // Print system information - std::cout << "=== QuadBLAS Google Benchmark Suite ===" << std::endl; - std::cout << "Library Version: " << QuadBLAS::VERSION << std::endl; - std::cout << "Available Threads: " << QuadBLAS::get_num_threads() << std::endl; - -#ifdef QUADBLAS_X86_64 - std::cout << "Platform: x86-64 with SIMD vectorization" << std::endl; -#elif defined(QUADBLAS_AARCH64) - std::cout << "Platform: AArch64 with NEON vectorization" << std::endl; -#else - std::cout << "Platform: Generic (scalar fallback)" << std::endl; -#endif - -#ifdef _OPENMP - std::cout << "OpenMP: Enabled" << std::endl; -#else - std::cout << "OpenMP: Disabled" << std::endl; -#endif - - std::cout << "SIMD Vector Size: " << QuadBLAS::VECTOR_SIZE << " elements" << std::endl; - std::cout << "Memory Alignment: " << QuadBLAS::ALIGNMENT << " bytes" << std::endl; - - // Add macOS-specific system info -#ifdef __APPLE__ - std::cout << "Note: macOS warnings about CPU frequency/thread affinity are normal and don't affect timing accuracy" << std::endl; -#endif - - std::cout << "=========================================" << std::endl; - - // Register benchmarks that need runtime info - RegisterThreadingBenchmarks(); - - // Initialize and run benchmarks - ::benchmark::Initialize(&argc, argv); - if (::benchmark::ReportUnrecognizedArguments(argc, argv)) - return 1; - ::benchmark::RunSpecifiedBenchmarks(); - ::benchmark::Shutdown(); - - return 0; -} \ No newline at end of file diff --git a/benchmarks/legacy_benchmark.cpp b/benchmarks/legacy_benchmark.cpp deleted file mode 100644 index a348178..0000000 --- a/benchmarks/legacy_benchmark.cpp +++ /dev/null @@ -1,398 +0,0 @@ -#include "include/quadblas/quadblas.hpp" -#include -#include -#include -#include -#include -#include - -// Helper functions -double to_double(Sleef_quad q) { - return static_cast(Sleef_cast_to_doubleq1(q)); -} - -Sleef_quad from_double(double d) { - return Sleef_cast_from_doubleq1(d); -} - -// High-precision timing with warmup and averaging -class BenchTimer { -private: - std::chrono::high_resolution_clock::time_point start_time; -public: - void start() { - start_time = std::chrono::high_resolution_clock::now(); - } - - double stop_ms() { - auto end_time = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast(end_time - start_time); - return duration.count() / 1e6; - } - - double stop_s() { - return stop_ms() / 1000.0; - } -}; - -// Compiler barrier to prevent dead code elimination of results -template -inline void do_not_optimize(const T &value) { -#if defined(__GNUC__) || defined(__clang__) - asm volatile("" : : "r,m"(value) : "memory"); -#else - volatile T sink = value; - (void)sink; -#endif -} - -// Run a callable multiple times and return median time -template -double benchmark_median(Func &&fn, int warmup = 2, int trials = 5) { - // Warmup - for (int i = 0; i < warmup; ++i) { - fn(); - } - - // Timed trials - std::vector times(trials); - BenchTimer timer; - for (int i = 0; i < trials; ++i) { - timer.start(); - fn(); - times[i] = timer.stop_ms(); - } - - // Return median - std::sort(times.begin(), times.end()); - return times[trials / 2]; -} - -// Random number generator for consistent tests -class QuadRandom { -private: - std::mt19937 gen; - std::uniform_real_distribution dist; - -public: - QuadRandom(unsigned seed = 12345) : gen(seed), dist(-1.0, 1.0) {} - - Sleef_quad next() { - return from_double(dist(gen)); - } - - void fill_array(Sleef_quad* arr, size_t n) { - for (size_t i = 0; i < n; ++i) { - arr[i] = next(); - } - } -}; - -// ================================================================================ -// NAIVE IMPLEMENTATIONS (No optimizations - baseline for comparison) -// ================================================================================ - -Sleef_quad naive_dot(size_t n, const Sleef_quad* x, const Sleef_quad* y) { - Sleef_quad result = from_double(0.0); - for (size_t i = 0; i < n; ++i) { - result = Sleef_fmaq1_u05(x[i], y[i], result); - } - return result; -} - -void naive_gemv(size_t m, size_t n, Sleef_quad alpha, - const Sleef_quad* A, size_t lda, - const Sleef_quad* x, Sleef_quad beta, Sleef_quad* y) { - for (size_t i = 0; i < m; ++i) { - Sleef_quad sum = from_double(0.0); - for (size_t j = 0; j < n; ++j) { - sum = Sleef_fmaq1_u05(A[i * lda + j], x[j], sum); - } - y[i] = Sleef_fmaq1_u05(alpha, sum, Sleef_mulq1_u05(beta, y[i])); - } -} - -void naive_gemm(size_t m, size_t n, size_t k, Sleef_quad alpha, - const Sleef_quad* A, size_t lda, - const Sleef_quad* B, size_t ldb, - Sleef_quad beta, Sleef_quad* C, size_t ldc) { - for (size_t i = 0; i < m; ++i) { - for (size_t j = 0; j < n; ++j) { - Sleef_quad sum = from_double(0.0); - for (size_t l = 0; l < k; ++l) { - sum = Sleef_fmaq1_u05(A[i * lda + l], B[l * ldb + j], sum); - } - C[i * ldc + j] = Sleef_fmaq1_u05(alpha, sum, Sleef_mulq1_u05(beta, C[i * ldc + j])); - } - } -} - -// ================================================================================ -// BENCHMARK FUNCTIONS (with warmup and median timing) -// ================================================================================ - -void benchmark_dot_product() { - std::cout << "\n" << std::string(80, '=') << "\n"; - std::cout << "DOT PRODUCT BENCHMARK (Vector . Vector)\n"; - std::cout << std::string(80, '=') << "\n"; - - std::vector sizes = {1000, 5000, 10000, 50000, 100000}; - - std::cout << std::setw(10) << "Size" - << std::setw(15) << "Naive (ms)" - << std::setw(15) << "QuadBLAS (ms)" - << std::setw(12) << "Speedup" - << std::setw(15) << "Error" << "\n"; - std::cout << std::string(80, '-') << "\n"; - - QuadRandom rng; - - for (size_t n : sizes) { - Sleef_quad* x = QuadBLAS::aligned_alloc(n); - Sleef_quad* y = QuadBLAS::aligned_alloc(n); - rng.fill_array(x, n); - rng.fill_array(y, n); - - Sleef_quad naive_result, quadblas_result; - - double naive_time = benchmark_median([&]() { - naive_result = naive_dot(n, x, y); - do_not_optimize(naive_result); - }); - - double quadblas_time = benchmark_median([&]() { - quadblas_result = QuadBLAS::dot(n, x, 1, y, 1); - do_not_optimize(quadblas_result); - }); - - double speedup = naive_time / quadblas_time; - double error = std::abs(to_double(naive_result) - to_double(quadblas_result)); - - std::cout << std::setw(10) << n - << std::setw(15) << std::fixed << std::setprecision(3) << naive_time - << std::setw(15) << std::fixed << std::setprecision(3) << quadblas_time - << std::setw(12) << std::fixed << std::setprecision(2) << speedup - << std::setw(15) << std::scientific << std::setprecision(2) << error - << "\n"; - - QuadBLAS::aligned_free(x); - QuadBLAS::aligned_free(y); - } -} - -void benchmark_matrix_vector() { - std::cout << "\n" << std::string(80, '=') << "\n"; - std::cout << "MATRIX-VECTOR BENCHMARK (Matrix x Vector)\n"; - std::cout << std::string(80, '=') << "\n"; - - std::vector sizes = {100, 300, 500, 1000, 1500}; - - std::cout << std::setw(10) << "Size" - << std::setw(15) << "Naive (ms)" - << std::setw(15) << "QuadBLAS (ms)" - << std::setw(12) << "Speedup" - << std::setw(15) << "GFLOPS" << "\n"; - std::cout << std::string(80, '-') << "\n"; - - QuadRandom rng; - - for (size_t n : sizes) { - size_t m = n; - Sleef_quad* A = QuadBLAS::aligned_alloc(m * n); - Sleef_quad* x = QuadBLAS::aligned_alloc(n); - Sleef_quad* y = QuadBLAS::aligned_alloc(m); - - rng.fill_array(A, m * n); - rng.fill_array(x, n); - - Sleef_quad alpha = from_double(1.0); - Sleef_quad beta = from_double(0.0); - - double naive_time = benchmark_median([&]() { - for (size_t i = 0; i < m; ++i) y[i] = from_double(0.0); - naive_gemv(m, n, alpha, A, n, x, beta, y); - }); - - double quadblas_time = benchmark_median([&]() { - for (size_t i = 0; i < m; ++i) y[i] = from_double(0.0); - QuadBLAS::gemv(QuadBLAS::Layout::RowMajor, m, n, alpha, A, n, x, 1, beta, y, 1); - }); - - double speedup = naive_time / quadblas_time; - double gflops = (2.0 * m * n) / (quadblas_time * 1e6); - - std::cout << std::setw(10) << n - << std::setw(15) << std::fixed << std::setprecision(3) << naive_time - << std::setw(15) << std::fixed << std::setprecision(3) << quadblas_time - << std::setw(12) << std::fixed << std::setprecision(2) << speedup - << std::setw(15) << std::fixed << std::setprecision(3) << gflops - << "\n"; - - QuadBLAS::aligned_free(A); - QuadBLAS::aligned_free(x); - QuadBLAS::aligned_free(y); - } -} - -void benchmark_matrix_matrix() { - std::cout << "\n" << std::string(80, '=') << "\n"; - std::cout << "MATRIX-MATRIX BENCHMARK (Matrix x Matrix)\n"; - std::cout << std::string(80, '=') << "\n"; - - // Reduced to practical sizes for quad precision - // 500x500 GEMM = 250M quad FMAs which already takes seconds - std::vector sizes = {32, 64, 128, 200, 500}; - - std::cout << std::setw(10) << "Size" - << std::setw(15) << "Naive (s)" - << std::setw(15) << "QuadBLAS (s)" - << std::setw(12) << "Speedup" - << std::setw(15) << "GFLOPS" << "\n"; - std::cout << std::string(80, '-') << "\n"; - - QuadRandom rng; - - for (size_t n : sizes) { - size_t m = n, k = n; - - Sleef_quad* A = QuadBLAS::aligned_alloc(m * k); - Sleef_quad* B = QuadBLAS::aligned_alloc(k * n); - Sleef_quad* C = QuadBLAS::aligned_alloc(m * n); - - rng.fill_array(A, m * k); - rng.fill_array(B, k * n); - - Sleef_quad alpha = from_double(1.0); - Sleef_quad beta = from_double(0.0); - - // Fewer trials for large sizes - int trials = (n <= 128) ? 5 : 3; - int warmup = (n <= 128) ? 2 : 1; - - double naive_time = benchmark_median([&]() { - for (size_t i = 0; i < m * n; ++i) C[i] = from_double(0.0); - naive_gemm(m, n, k, alpha, A, k, B, n, beta, C, n); - }, warmup, trials) / 1000.0; // Convert to seconds - - double quadblas_time = benchmark_median([&]() { - for (size_t i = 0; i < m * n; ++i) C[i] = from_double(0.0); - QuadBLAS::gemm(QuadBLAS::Layout::RowMajor, m, n, k, alpha, A, k, B, n, beta, C, n); - }, warmup, trials) / 1000.0; - - double speedup = naive_time / quadblas_time; - double gflops = (2.0 * m * n * k) / (quadblas_time * 1e9); - - std::cout << std::setw(10) << n - << std::setw(15) << std::fixed << std::setprecision(3) << naive_time - << std::setw(15) << std::fixed << std::setprecision(3) << quadblas_time - << std::setw(12) << std::fixed << std::setprecision(2) << speedup - << std::setw(15) << std::fixed << std::setprecision(3) << gflops - << "\n"; - - QuadBLAS::aligned_free(A); - QuadBLAS::aligned_free(B); - QuadBLAS::aligned_free(C); - } -} - -void benchmark_threading_scaling() { - std::cout << "\n" << std::string(80, '=') << "\n"; - std::cout << "THREADING SCALABILITY BENCHMARK\n"; - std::cout << std::string(80, '=') << "\n"; - - const size_t n = 50000; // Large enough for threading to matter - - Sleef_quad* x = QuadBLAS::aligned_alloc(n); - Sleef_quad* y = QuadBLAS::aligned_alloc(n); - - QuadRandom rng; - rng.fill_array(x, n); - rng.fill_array(y, n); - - std::cout << std::setw(12) << "Threads" - << std::setw(15) << "Time (ms)" - << std::setw(12) << "Speedup" - << std::setw(15) << "Efficiency" << "\n"; - std::cout << std::string(60, '-') << "\n"; - - double baseline_time = 0.0; - int max_threads = QuadBLAS::get_num_threads(); - - for (int threads = 1; threads <= max_threads; threads *= 2) { - QuadBLAS::set_num_threads(threads); - - Sleef_quad result; - double avg_time = benchmark_median([&]() { - result = QuadBLAS::dot(n, x, 1, y, 1); - do_not_optimize(result); - }, 3, 10); - - (void)result; - - if (threads == 1) baseline_time = avg_time; - - double speedup = baseline_time / avg_time; - double efficiency = speedup / threads * 100.0; - - std::cout << std::setw(12) << threads - << std::setw(15) << std::fixed << std::setprecision(3) << avg_time - << std::setw(12) << std::fixed << std::setprecision(2) << speedup - << std::setw(14) << std::fixed << std::setprecision(1) << efficiency << "%" - << "\n"; - - if (threads >= 16) break; - } - - QuadBLAS::set_num_threads(max_threads); - - QuadBLAS::aligned_free(x); - QuadBLAS::aligned_free(y); -} - -void print_system_info() { - std::cout << "\n" << std::string(80, '=') << "\n"; - std::cout << "SYSTEM INFORMATION\n"; - std::cout << std::string(80, '=') << "\n"; - - std::cout << "QuadBLAS Version: " << quadblas_get_version() << "\n"; - std::cout << "Available CPU threads: " << QuadBLAS::get_num_threads() << "\n"; - -#ifdef QUADBLAS_X86_64 - std::cout << "Platform: x86-64 with SIMD vectorization\n"; -#elif defined(QUADBLAS_AARCH64) - std::cout << "Platform: AArch64 with NEON vectorization\n"; -#else - std::cout << "Platform: Generic (scalar fallback)\n"; -#endif - -#ifdef _OPENMP - std::cout << "OpenMP: Enabled\n"; -#else - std::cout << "OpenMP: Disabled\n"; -#endif - - std::cout << "SIMD Vector Size: " << QuadBLAS::VECTOR_SIZE << " quad elements\n"; - std::cout << "Memory Alignment: " << QuadBLAS::ALIGNMENT << " bytes\n"; - std::cout << "Quad precision size: " << sizeof(Sleef_quad) << " bytes\n"; -} - -int main() { - print_system_info(); - - try { - benchmark_dot_product(); - benchmark_matrix_vector(); - benchmark_matrix_matrix(); - benchmark_threading_scaling(); - - std::cout << "\n" << std::string(80, '=') << "\n"; - std::cout << "All benchmarks completed.\n"; - std::cout << std::string(80, '=') << "\n"; - - } catch (const std::exception& e) { - std::cerr << "Benchmark error: " << e.what() << std::endl; - return 1; - } - - return 0; -} diff --git a/build.sh b/build.sh deleted file mode 100644 index 2a703fc..0000000 --- a/build.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -set -e - -# Clean and setup -rm -rf build -mkdir build -cd build - -# Environment setup -export SLEEF_ROOT="/Users/swayam/Desktop/numpy_dtypes/sleef/build" - -# Detect available parallelism -if command -v nproc &> /dev/null; then - NJOBS=$(nproc) -elif command -v sysctl &> /dev/null; then - NJOBS=$(sysctl -n hw.ncpu) -else - NJOBS=4 -fi - -# Initialize submodules if needed -if [ ! -f "../third_party/benchmark/CMakeLists.txt" ]; then - echo "Initializing Google Benchmark submodule..." - (cd .. && git submodule update --init --recursive) -fi - -# Build -cmake -DCMAKE_BUILD_TYPE=Release .. -make -j"${NJOBS}" - -echo "=== Running Tests ===" -./quadblas_test - -# Google Benchmark -echo "" -echo "=== Running Google Benchmarks ===" -if [ -f "./quadblas_benchmark" ]; then - ./quadblas_benchmark --benchmark_min_time=0.5s -else - echo "Google Benchmark not available, running legacy..." - ./quadblas_benchmark_legacy -fi diff --git a/cmake/QBLASArch.cmake b/cmake/QBLASArch.cmake new file mode 100644 index 0000000..6c21c1d --- /dev/null +++ b/cmake/QBLASArch.cmake @@ -0,0 +1,32 @@ +# Populates QBLAS_ARCH, QBLAS_ISA_TIERS, and per-tier QBLAS__FLAGS. + +include_guard() + +include(CheckCCompilerFlag) + +function(qblas_detect_arch) + set(arch "generic") + set(tiers "generic") + + if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64|amd64") + set(arch "x86_64") + set(tiers "generic" "sse2" "avx2" "avx512") + + set(QBLAS_GENERIC_FLAGS "" CACHE INTERNAL "") + set(QBLAS_SSE2_FLAGS "-msse2" CACHE INTERNAL "") + set(QBLAS_AVX2_FLAGS "-mavx2;-mfma;-mbmi2" CACHE INTERNAL "") + set(QBLAS_AVX512_FLAGS "-mavx512f;-mavx512dq;-mavx512bw;-mfma;-mbmi2" CACHE INTERNAL "") + + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") + set(arch "aarch64") + set(tiers "generic" "neon") + + set(QBLAS_GENERIC_FLAGS "" CACHE INTERNAL "") + set(QBLAS_NEON_FLAGS "" CACHE INTERNAL "") + else() + message(STATUS "QBLAS: unknown CMAKE_SYSTEM_PROCESSOR='${CMAKE_SYSTEM_PROCESSOR}', falling back to generic only") + endif() + + set(QBLAS_ARCH "${arch}" CACHE INTERNAL "QBLAS target arch") + set(QBLAS_ISA_TIERS "${tiers}" CACHE INTERNAL "QBLAS ISA tiers") +endfunction() diff --git a/include/qblas/qblas.h b/include/qblas/qblas.h new file mode 100644 index 0000000..bab5334 --- /dev/null +++ b/include/qblas/qblas.h @@ -0,0 +1,65 @@ +/* QBLAS - quad-precision BLAS built on SLEEF. CBLAS-style C API. */ +#ifndef QBLAS_H +#define QBLAS_H + +#include +#include + +#if defined(_WIN32) && !defined(QBLAS_STATIC) +# ifdef QBLAS_BUILDING +# define QBLAS_API __declspec(dllexport) +# else +# define QBLAS_API __declspec(dllimport) +# endif +#else +# define QBLAS_API __attribute__((visibility("default"))) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + QblasRowMajor = 101, + QblasColMajor = 102 +} QBLAS_LAYOUT; + +typedef enum { + QblasNoTrans = 111, + QblasTrans = 112, + QblasConjTrans = 113 /* treated as Trans (quad is real) */ +} QBLAS_TRANSPOSE; + +typedef enum { + QblasUpper = 121, + QblasLower = 122 +} QBLAS_UPLO; + +typedef enum { + QblasNonUnit = 131, + QblasUnit = 132 +} QBLAS_DIAG; + +typedef enum { + QblasLeft = 141, + QblasRight = 142 +} QBLAS_SIDE; + +QBLAS_API const char *qblas_get_version(void); +QBLAS_API const char *qblas_get_dispatch_tier(void); + +/* qblas_set_num_threads() caps qblas's own work distribution only; + * it does not call omp_set_num_threads. */ +QBLAS_API void qblas_set_num_threads(int n); +QBLAS_API int qblas_get_num_threads(void); +QBLAS_API int qblas_get_max_threads(void); + +#include "qblas_level1.h" +#include "qblas_level2.h" +#include "qblas_level3.h" + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/qblas/qblas_level1.h b/include/qblas/qblas_level1.h new file mode 100644 index 0000000..ae05853 --- /dev/null +++ b/include/qblas/qblas_level1.h @@ -0,0 +1,41 @@ +/* Level 1: vector operations. Negative strides are accepted and walk + * the array in reverse per the BLAS convention; incx/incy of 0 are UB. */ +#ifndef QBLAS_LEVEL1_H +#define QBLAS_LEVEL1_H + +#ifndef QBLAS_H +# include "qblas.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +QBLAS_API Sleef_quad cblas_qdot(int n, + const Sleef_quad *x, int incx, + const Sleef_quad *y, int incy); + +QBLAS_API Sleef_quad cblas_qnrm2(int n, const Sleef_quad *x, int incx); +QBLAS_API Sleef_quad cblas_qasum(int n, const Sleef_quad *x, int incx); +QBLAS_API size_t cblas_iqamax(int n, const Sleef_quad *x, int incx); + +QBLAS_API void cblas_qaxpy(int n, Sleef_quad alpha, + const Sleef_quad *x, int incx, + Sleef_quad *y, int incy); + +QBLAS_API void cblas_qscal(int n, Sleef_quad alpha, + Sleef_quad *x, int incx); + +QBLAS_API void cblas_qcopy(int n, + const Sleef_quad *x, int incx, + Sleef_quad *y, int incy); + +QBLAS_API void cblas_qswap(int n, + Sleef_quad *x, int incx, + Sleef_quad *y, int incy); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/qblas/qblas_level2.h b/include/qblas/qblas_level2.h new file mode 100644 index 0000000..02a1f4b --- /dev/null +++ b/include/qblas/qblas_level2.h @@ -0,0 +1,54 @@ +/* Level 2: matrix-vector operations. `lda` is the leading dimension of + * A in its natural orientation (stride between rows for row-major, + * between columns for col-major). */ +#ifndef QBLAS_LEVEL2_H +#define QBLAS_LEVEL2_H + +#ifndef QBLAS_H +# include "qblas.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* y := alpha * op(A) * x + beta * y */ +QBLAS_API void cblas_qgemv(QBLAS_LAYOUT layout, QBLAS_TRANSPOSE trans, + int m, int n, + Sleef_quad alpha, + const Sleef_quad *A, int lda, + const Sleef_quad *x, int incx, + Sleef_quad beta, + Sleef_quad *y, int incy); + +/* A := alpha * x * y^T + A */ +QBLAS_API void cblas_qger(QBLAS_LAYOUT layout, + int m, int n, + Sleef_quad alpha, + const Sleef_quad *x, int incx, + const Sleef_quad *y, int incy, + Sleef_quad *A, int lda); + +/* y := alpha * A * x + beta * y, A symmetric; only `uplo` triangle is read. */ +QBLAS_API void cblas_qsymv(QBLAS_LAYOUT layout, QBLAS_UPLO uplo, + int n, + Sleef_quad alpha, + const Sleef_quad *A, int lda, + const Sleef_quad *x, int incx, + Sleef_quad beta, + Sleef_quad *y, int incy); + +/* Solve op(A) * x = b in place, A triangular. */ +QBLAS_API void cblas_qtrsv(QBLAS_LAYOUT layout, + QBLAS_UPLO uplo, + QBLAS_TRANSPOSE trans, + QBLAS_DIAG diag, + int n, + const Sleef_quad *A, int lda, + Sleef_quad *x, int incx); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/qblas/qblas_level3.h b/include/qblas/qblas_level3.h new file mode 100644 index 0000000..90debe1 --- /dev/null +++ b/include/qblas/qblas_level3.h @@ -0,0 +1,56 @@ +/* Level 3: matrix-matrix operations. */ +#ifndef QBLAS_LEVEL3_H +#define QBLAS_LEVEL3_H + +#ifndef QBLAS_H +# include "qblas.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* C := alpha * op(A) * op(B) + beta * C (op(A) is m x k, op(B) is k x n). */ +QBLAS_API void cblas_qgemm(QBLAS_LAYOUT layout, + QBLAS_TRANSPOSE transa, QBLAS_TRANSPOSE transb, + int m, int n, int k, + Sleef_quad alpha, + const Sleef_quad *A, int lda, + const Sleef_quad *B, int ldb, + Sleef_quad beta, + Sleef_quad *C, int ldc); + +/* C := alpha * op(A) * op(A)^T + beta * C, C symmetric (`uplo` triangle). */ +QBLAS_API void cblas_qsyrk(QBLAS_LAYOUT layout, + QBLAS_UPLO uplo, + QBLAS_TRANSPOSE trans, + int n, int k, + Sleef_quad alpha, + const Sleef_quad *A, int lda, + Sleef_quad beta, + Sleef_quad *C, int ldc); + +/* B := alpha * op(A) * B (Left) or alpha * B * op(A) (Right). */ +QBLAS_API void cblas_qtrmm(QBLAS_LAYOUT layout, + QBLAS_SIDE side, QBLAS_UPLO uplo, + QBLAS_TRANSPOSE transa, QBLAS_DIAG diag, + int m, int n, + Sleef_quad alpha, + const Sleef_quad *A, int lda, + Sleef_quad *B, int ldb); + +/* Solve op(A) * X = alpha * B (Left) or X * op(A) = alpha * B (Right); + * X overwrites B. */ +QBLAS_API void cblas_qtrsm(QBLAS_LAYOUT layout, + QBLAS_SIDE side, QBLAS_UPLO uplo, + QBLAS_TRANSPOSE transa, QBLAS_DIAG diag, + int m, int n, + Sleef_quad alpha, + const Sleef_quad *A, int lda, + Sleef_quad *B, int ldb); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/quadblas/algorithms/level1.hpp b/include/quadblas/algorithms/level1.hpp deleted file mode 100644 index fd0a5cd..0000000 --- a/include/quadblas/algorithms/level1.hpp +++ /dev/null @@ -1,227 +0,0 @@ -#ifndef QUADBLAS_ALGORITHMS_LEVEL1_HPP -#define QUADBLAS_ALGORITHMS_LEVEL1_HPP - -#include "../core/constants.hpp" -#include "../core/platform.hpp" -#include "../simd/quad_vector.hpp" -#include "../threading/openmp_utils.hpp" -#include - -namespace QuadBLAS -{ - - // Vectorized dot product kernel for contiguous data - inline Sleef_quad dot_kernel_vectorized(const Sleef_quad *x, const Sleef_quad *y, size_t n) - { - const size_t vec_n = n / VECTOR_SIZE; - - QuadVector sum_vec(SLEEF_QUAD_C(0.0)); - - for (size_t i = 0; i < vec_n; ++i) - { - QuadVector x_vec = QuadVector::load(&x[i * VECTOR_SIZE]); - QuadVector y_vec = QuadVector::load(&y[i * VECTOR_SIZE]); - sum_vec = x_vec.fma(y_vec, sum_vec); - } - - Sleef_quad result = sum_vec.horizontal_sum(); - - for (size_t i = vec_n * VECTOR_SIZE; i < n; ++i) - { - result = Sleef_fmaq1_u05(x[i], y[i], result); - } - - return result; - } - - // Parallel dot product for large vectors - inline Sleef_quad dot_parallel(const Sleef_quad *x, const Sleef_quad *y, size_t n) - { - if (n < PARALLEL_THRESHOLD) - { - return dot_kernel_vectorized(x, y, n); - } - -#ifdef _OPENMP - const int num_threads = get_num_threads(); - const size_t chunk_size = n / num_threads; - - std::vector partial_results(num_threads); - for (int i = 0; i < num_threads; ++i) - { - partial_results[i] = SLEEF_QUAD_C(0.0); - } - -#pragma omp parallel - { - int tid = omp_get_thread_num(); - size_t start = tid * chunk_size; - size_t end = (tid == num_threads - 1) ? n : start + chunk_size; - - if (start < end) - { - partial_results[tid] = dot_kernel_vectorized(&x[start], &y[start], end - start); - } - } - - Sleef_quad result = SLEEF_QUAD_C(0.0); - for (int i = 0; i < num_threads; ++i) - { - result = Sleef_addq1_u05(result, partial_results[i]); - } - - return result; -#else - return dot_kernel_vectorized(x, y, n); -#endif - } - - // Main dot product function with stride support - inline Sleef_quad dot(size_t n, const Sleef_quad *x, size_t incx, - const Sleef_quad *y, size_t incy) - { - if (n == 0) - return SLEEF_QUAD_C(0.0); - - if (incx == 1 && incy == 1) - { - return dot_parallel(x, y, n); - } - - Sleef_quad result = SLEEF_QUAD_C(0.0); - - if (n >= PARALLEL_THRESHOLD) - { -#ifdef _OPENMP - const int num_threads = get_num_threads(); - - std::vector partial_results(num_threads); - for (int i = 0; i < num_threads; ++i) - { - partial_results[i] = SLEEF_QUAD_C(0.0); - } - -#pragma omp parallel - { - int tid = omp_get_thread_num(); - size_t chunk_size = n / num_threads; - size_t start = tid * chunk_size; - size_t end = (tid == num_threads - 1) ? n : start + chunk_size; - - for (size_t i = start; i < end; ++i) - { - partial_results[tid] = Sleef_fmaq1_u05(x[i * incx], y[i * incy], partial_results[tid]); - } - } - - for (int i = 0; i < num_threads; ++i) - { - result = Sleef_addq1_u05(result, partial_results[i]); - } -#else - for (size_t i = 0; i < n; ++i) - { - result = Sleef_fmaq1_u05(x[i * incx], y[i * incy], result); - } -#endif - } - else - { - for (size_t i = 0; i < n; ++i) - { - result = Sleef_fmaq1_u05(x[i * incx], y[i * incy], result); - } - } - - return result; - } - - // Vectorized AXPY kernel (y = alpha * x + y) - inline void axpy_kernel_vectorized(Sleef_quad alpha, const Sleef_quad *x, Sleef_quad *y, size_t n) - { - const size_t vec_n = n / VECTOR_SIZE; - - QuadVector alpha_vec(alpha); - - for (size_t i = 0; i < vec_n; ++i) - { - QuadVector x_vec = QuadVector::load(&x[i * VECTOR_SIZE]); - QuadVector y_vec = QuadVector::load(&y[i * VECTOR_SIZE]); - QuadVector result_vec = x_vec.fma(alpha_vec, y_vec); - result_vec.store(&y[i * VECTOR_SIZE]); - } - - for (size_t i = vec_n * VECTOR_SIZE; i < n; ++i) - { - y[i] = Sleef_fmaq1_u05(alpha, x[i], y[i]); - } - } - - // Parallel AXPY for large vectors - inline void axpy_parallel(Sleef_quad alpha, const Sleef_quad *x, Sleef_quad *y, size_t n) - { - if (n < PARALLEL_THRESHOLD) - { - axpy_kernel_vectorized(alpha, x, y, n); - return; - } - -#ifdef _OPENMP - const int num_threads = get_num_threads(); - -#pragma omp parallel - { - int tid = omp_get_thread_num(); - size_t chunk_size = n / num_threads; - size_t start = tid * chunk_size; - size_t end = (tid == num_threads - 1) ? n : start + chunk_size; - - if (start < end) - { - axpy_kernel_vectorized(alpha, &x[start], &y[start], end - start); - } - } -#else - axpy_kernel_vectorized(alpha, x, y, n); -#endif - } - - // Main AXPY function with stride support - inline void axpy(size_t n, Sleef_quad alpha, const Sleef_quad *x, size_t incx, Sleef_quad *y, size_t incy) - { - if (n == 0) - return; - - if (incx == 1 && incy == 1) - { - axpy_parallel(alpha, x, y, n); - return; - } - - if (n >= PARALLEL_THRESHOLD) - { -#ifdef _OPENMP -#pragma omp parallel for - for (size_t i = 0; i < n; ++i) - { - y[i * incy] = Sleef_fmaq1_u05(alpha, x[i * incx], y[i * incy]); - } -#else - for (size_t i = 0; i < n; ++i) - { - y[i * incy] = Sleef_fmaq1_u05(alpha, x[i * incx], y[i * incy]); - } -#endif - } - else - { - for (size_t i = 0; i < n; ++i) - { - y[i * incy] = Sleef_fmaq1_u05(alpha, x[i * incx], y[i * incy]); - } - } - } - -} // namespace QuadBLAS - -#endif // QUADBLAS_ALGORITHMS_LEVEL1_HPP diff --git a/include/quadblas/algorithms/level2.hpp b/include/quadblas/algorithms/level2.hpp deleted file mode 100644 index 68c079d..0000000 --- a/include/quadblas/algorithms/level2.hpp +++ /dev/null @@ -1,103 +0,0 @@ -#ifndef QUADBLAS_ALGORITHMS_LEVEL2_HPP -#define QUADBLAS_ALGORITHMS_LEVEL2_HPP - -#include "../core/constants.hpp" -#include "../core/types.hpp" -#include "../core/platform.hpp" -#include "../threading/openmp_utils.hpp" -#include "level1.hpp" - -namespace QuadBLAS -{ - - // GEMV: y = alpha * A * x + beta * y - // Row-major implementation - inline void gemv_row_major(size_t m, size_t n, Sleef_quad alpha, - const Sleef_quad *A, size_t lda, - const Sleef_quad *x, size_t incx, - Sleef_quad beta, Sleef_quad *y, size_t incy) - { - - if (m == 0 || n == 0) - return; - - const bool use_parallel = m >= PARALLEL_THRESHOLD; - -#ifdef _OPENMP -#pragma omp parallel for if (use_parallel) -#endif - for (size_t i = 0; i < m; ++i) - { - Sleef_quad sum = SLEEF_QUAD_C(0.0); - - const Sleef_quad *row = &A[i * lda]; - - if (incx == 1) - { - sum = dot_kernel_vectorized(row, x, n); - } - else - { - for (size_t j = 0; j < n; ++j) - { - sum = Sleef_fmaq1_u05(row[j], x[j * incx], sum); - } - } - - size_t y_idx = i * incy; - y[y_idx] = Sleef_fmaq1_u05(alpha, sum, Sleef_mulq1_u05(beta, y[y_idx])); - } - } - - // Column-major implementation - inline void gemv_col_major(size_t m, size_t n, Sleef_quad alpha, - const Sleef_quad *A, size_t lda, - const Sleef_quad *x, size_t incx, - Sleef_quad beta, Sleef_quad *y, size_t incy) - { - - if (m == 0 || n == 0) - return; - - // Scale y by beta first - for (size_t i = 0; i < m; ++i) - { - y[i * incy] = Sleef_mulq1_u05(beta, y[i * incy]); - } - - // Add alpha * A * x - for (size_t j = 0; j < n; ++j) - { - Sleef_quad x_j = Sleef_mulq1_u05(alpha, x[j * incx]); - const Sleef_quad *col = &A[j * lda]; - -#ifdef _OPENMP -#pragma omp parallel for if (m >= PARALLEL_THRESHOLD) -#endif - for (size_t i = 0; i < m; ++i) - { - y[i * incy] = Sleef_fmaq1_u05(col[i], x_j, y[i * incy]); - } - } - } - - // Main GEMV function - inline void gemv(Layout layout, size_t m, size_t n, Sleef_quad alpha, - const Sleef_quad *A, size_t lda, - const Sleef_quad *x, size_t incx, - Sleef_quad beta, Sleef_quad *y, size_t incy) - { - - if (layout == Layout::RowMajor) - { - gemv_row_major(m, n, alpha, A, lda, x, incx, beta, y, incy); - } - else - { - gemv_col_major(m, n, alpha, A, lda, x, incx, beta, y, incy); - } - } - -} // namespace QuadBLAS - -#endif // QUADBLAS_ALGORITHMS_LEVEL2_HPP diff --git a/include/quadblas/algorithms/level3.hpp b/include/quadblas/algorithms/level3.hpp deleted file mode 100644 index b4b7bd1..0000000 --- a/include/quadblas/algorithms/level3.hpp +++ /dev/null @@ -1,340 +0,0 @@ -#ifndef QUADBLAS_ALGORITHMS_LEVEL3_HPP -#define QUADBLAS_ALGORITHMS_LEVEL3_HPP - -#include "../core/constants.hpp" -#include "../core/types.hpp" -#include "../core/platform.hpp" -#include "../simd/quad_vector.hpp" -#include "../memory/allocation.hpp" -#include "../detail/blocking.hpp" -#include "../threading/openmp_utils.hpp" -#include - -namespace QuadBLAS -{ - - // Micro-kernel tile sizes - constexpr size_t GEMM_MR = 4; - constexpr size_t GEMM_NR = 4; - - inline void gemm_micro_kernel_scalar(size_t mr, size_t nr, size_t kc, - Sleef_quad alpha, - const Sleef_quad *A_packed, const Sleef_quad *B_packed, - Sleef_quad beta, Sleef_quad *C, size_t ldc) - { - - for (size_t i = 0; i < mr; ++i) - { - for (size_t j = 0; j < nr; ++j) - { - Sleef_quad sum = SLEEF_QUAD_C(0.0); - - for (size_t k = 0; k < kc; ++k) - { - Sleef_quad a_val = A_packed[i * kc + k]; - Sleef_quad b_val = B_packed[k * nr + j]; - sum = Sleef_fmaq1_u05(a_val, b_val, sum); - } - - Sleef_quad c_old = C[i * ldc + j]; - C[i * ldc + j] = Sleef_fmaq1_u05(alpha, sum, Sleef_mulq1_u05(beta, c_old)); - } - } - } - - inline void gemm_micro_kernel_vectorized(size_t mr, size_t nr, size_t kc, - Sleef_quad alpha, - const Sleef_quad *A_packed, Sleef_quad *B_packed, - Sleef_quad beta, Sleef_quad *C, size_t ldc) - { - - if (mr % VECTOR_SIZE != 0 || nr % VECTOR_SIZE != 0 || mr < VECTOR_SIZE || nr < VECTOR_SIZE) - { - gemm_micro_kernel_scalar(mr, nr, kc, alpha, A_packed, B_packed, beta, C, ldc); - return; - } - - const size_t nr_vec = nr / VECTOR_SIZE; - - // Fixed-size accumulator (max GEMM_MR x GEMM_NR) - no VLA - Sleef_quad c_acc[GEMM_MR][GEMM_NR]; - - for (size_t i = 0; i < mr; ++i) - { - for (size_t j = 0; j < nr; ++j) - { - c_acc[i][j] = SLEEF_QUAD_C(0.0); - } - } - - for (size_t i = 0; i < mr; ++i) - { - for (size_t j_vec = 0; j_vec < nr_vec; ++j_vec) - { - size_t j_start = j_vec * VECTOR_SIZE; - QuadVector sum_vec(SLEEF_QUAD_C(0.0)); - - for (size_t k = 0; k < kc; ++k) - { - Sleef_quad a_val = A_packed[i * kc + k]; - QuadVector a_vec(a_val); - - QuadVector b_vec = QuadVector::load(&B_packed[k * nr + j_start]); - - sum_vec = a_vec.fma(b_vec, sum_vec); - } - - for (size_t lane = 0; lane < VECTOR_SIZE; ++lane) - { - c_acc[i][j_start + lane] = sum_vec.get(lane); - } - } - - for (size_t j = nr_vec * VECTOR_SIZE; j < nr; ++j) - { - for (size_t k = 0; k < kc; ++k) - { - c_acc[i][j] = Sleef_fmaq1_u05(A_packed[i * kc + k], B_packed[k * nr + j], c_acc[i][j]); - } - } - } - - for (size_t i = 0; i < mr; ++i) - { - for (size_t j = 0; j < nr; ++j) - { - Sleef_quad c_old = C[i * ldc + j]; - C[i * ldc + j] = Sleef_fmaq1_u05(alpha, c_acc[i][j], Sleef_mulq1_u05(beta, c_old)); - } - } - } - - inline void gemm_micro_kernel(size_t mr, size_t nr, size_t kc, - Sleef_quad alpha, - const Sleef_quad *A_packed, Sleef_quad *B_packed, - Sleef_quad beta, Sleef_quad *C, size_t ldc) - { - - if (mr >= VECTOR_SIZE && nr >= VECTOR_SIZE && (mr * nr >= 8)) - { - gemm_micro_kernel_vectorized(mr, nr, kc, alpha, A_packed, B_packed, beta, C, ldc); - } - else - { - gemm_micro_kernel_scalar(mr, nr, kc, alpha, A_packed, B_packed, beta, C, ldc); - } - } - - inline void gemm_macro_kernel(size_t mc, size_t nc, size_t kc, - Sleef_quad alpha, - const Sleef_quad *A_packed, Sleef_quad *B_packed, - Sleef_quad beta, Sleef_quad *C, size_t ldc) - { - // Allocate B_sub once outside the loop instead of per-tile - Sleef_quad *B_sub = aligned_alloc(kc * GEMM_NR); - const bool have_b_sub = (B_sub != nullptr); - - for (size_t i = 0; i < mc; i += GEMM_MR) - { - size_t mr = std::min(GEMM_MR, mc - i); - - for (size_t j = 0; j < nc; j += GEMM_NR) - { - size_t nr = std::min(GEMM_NR, nc - j); - - if (have_b_sub) - { - // Repack B sub-tile into contiguous buffer - for (size_t k = 0; k < kc; ++k) - { - for (size_t jj = 0; jj < nr; ++jj) - { - B_sub[k * nr + jj] = B_packed[k * nc + (j + jj)]; - } - } - - gemm_micro_kernel(mr, nr, kc, alpha, - &A_packed[i * kc], B_sub, - beta, &C[i * ldc + j], ldc); - } - else - { - // Fallback: work directly on B_packed with stride - for (size_t ii = 0; ii < mr; ++ii) - { - for (size_t jj = 0; jj < nr; ++jj) - { - Sleef_quad sum = SLEEF_QUAD_C(0.0); - for (size_t k = 0; k < kc; ++k) - { - sum = Sleef_fmaq1_u05(A_packed[(i + ii) * kc + k], - B_packed[k * nc + (j + jj)], sum); - } - Sleef_quad c_old = C[(i + ii) * ldc + (j + jj)]; - C[(i + ii) * ldc + (j + jj)] = Sleef_fmaq1_u05(alpha, sum, Sleef_mulq1_u05(beta, c_old)); - } - } - } - } - } - - if (have_b_sub) - { - aligned_free(B_sub); - } - } - - inline void gemm_simple(Layout layout, size_t m, size_t n, size_t k, - Sleef_quad alpha, - const Sleef_quad *A, size_t lda, - const Sleef_quad *B, size_t ldb, - Sleef_quad beta, Sleef_quad *C, size_t ldc) - { -#ifdef _OPENMP -#pragma omp parallel for collapse(2) if (m * n >= PARALLEL_THRESHOLD) -#endif - for (size_t i = 0; i < m; ++i) - { - for (size_t j = 0; j < n; ++j) - { - Sleef_quad sum = SLEEF_QUAD_C(0.0); - - for (size_t l = 0; l < k; ++l) - { - size_t a_idx = (layout == Layout::RowMajor) ? i * lda + l : l * lda + i; - size_t b_idx = (layout == Layout::RowMajor) ? l * ldb + j : j * ldb + l; - sum = Sleef_fmaq1_u05(A[a_idx], B[b_idx], sum); - } - - size_t c_idx = (layout == Layout::RowMajor) ? i * ldc + j : j * ldc + i; - C[c_idx] = Sleef_fmaq1_u05(alpha, sum, Sleef_mulq1_u05(beta, C[c_idx])); - } - } - } - - inline void gemm(Layout layout, size_t m, size_t n, size_t k, - Sleef_quad alpha, - const Sleef_quad *A, size_t lda, - const Sleef_quad *B, size_t ldb, - Sleef_quad beta, Sleef_quad *C, size_t ldc) - { - if (m == 0 || n == 0 || k == 0) - return; - - constexpr size_t SMALL_MATRIX_THRESHOLD = 64; - if (m <= SMALL_MATRIX_THRESHOLD && n <= SMALL_MATRIX_THRESHOLD && k <= SMALL_MATRIX_THRESHOLD) - { - gemm_simple(layout, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc); - return; - } - - BlockingParams params(m, n, k); - - Sleef_quad *A_packed = aligned_alloc(params.mc * params.kc); - Sleef_quad *B_packed = aligned_alloc(params.kc * params.nc); - - if (!A_packed || !B_packed) - { - aligned_free(A_packed); - aligned_free(B_packed); - gemm_simple(layout, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc); - return; - } - - for (size_t kk = 0; kk < k; kk += params.kc) - { - size_t kc = std::min(params.kc, k - kk); - - for (size_t nn = 0; nn < n; nn += params.nc) - { - size_t nc = std::min(params.nc, n - nn); - - // Pack B panel once per (kk, nn) block - for (size_t i = 0; i < kc; ++i) - { - for (size_t j = 0; j < nc; ++j) - { - size_t src_idx = (layout == Layout::RowMajor) ? (kk + i) * ldb + (nn + j) : (nn + j) * ldb + (kk + i); - B_packed[i * nc + j] = B[src_idx]; - } - } - - // Iterate over row panels of A with OpenMP parallelism -#ifdef _OPENMP -#pragma omp parallel if (m >= PARALLEL_THRESHOLD) - { - // Each thread allocates its own A buffer once - Sleef_quad *A_local = nullptr; - bool multi_thread = (omp_get_num_threads() > 1); - if (multi_thread) - { - A_local = aligned_alloc(params.mc * kc); - } - else - { - A_local = A_packed; - } - - if (A_local) - { -#pragma omp for schedule(static) - for (size_t mm = 0; mm < m; mm += params.mc) - { - size_t mc = std::min(params.mc, m - mm); - - // Pack A panel - for (size_t i = 0; i < mc; ++i) - { - for (size_t j = 0; j < kc; ++j) - { - size_t src_idx = (layout == Layout::RowMajor) ? (mm + i) * lda + (kk + j) : (kk + j) * lda + (mm + i); - A_local[i * kc + j] = A[src_idx]; - } - } - - Sleef_quad *C_block = &C[(layout == Layout::RowMajor) ? mm * ldc + nn : nn * ldc + mm]; - - gemm_macro_kernel(mc, nc, kc, alpha, - A_local, B_packed, - (kk == 0) ? beta : SLEEF_QUAD_C(1.0), - C_block, ldc); - } - } - - if (multi_thread && A_local) - { - aligned_free(A_local); - } - } -#else - for (size_t mm = 0; mm < m; mm += params.mc) - { - size_t mc = std::min(params.mc, m - mm); - - for (size_t i = 0; i < mc; ++i) - { - for (size_t j = 0; j < kc; ++j) - { - size_t src_idx = (layout == Layout::RowMajor) ? (mm + i) * lda + (kk + j) : (kk + j) * lda + (mm + i); - A_packed[i * kc + j] = A[src_idx]; - } - } - - Sleef_quad *C_block = &C[(layout == Layout::RowMajor) ? mm * ldc + nn : nn * ldc + mm]; - - gemm_macro_kernel(mc, nc, kc, alpha, - A_packed, B_packed, - (kk == 0) ? beta : SLEEF_QUAD_C(1.0), - C_block, ldc); - } -#endif - } - } - - aligned_free(A_packed); - aligned_free(B_packed); - } - -} - -#endif diff --git a/include/quadblas/core/constants.hpp b/include/quadblas/core/constants.hpp deleted file mode 100644 index 45b21b6..0000000 --- a/include/quadblas/core/constants.hpp +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef QUADBLAS_CORE_CONSTANTS_HPP -#define QUADBLAS_CORE_CONSTANTS_HPP - -#include - -namespace QuadBLAS -{ - - // Configuration constants - constexpr size_t VECTOR_SIZE = 2; // Sleef quad vector size - constexpr size_t CACHE_LINE_SIZE = 64; - constexpr size_t L1_CACHE_SIZE = 32768; - constexpr size_t L2_CACHE_SIZE = 262144; - constexpr size_t PARALLEL_THRESHOLD = 500; - constexpr size_t GEMM_BLOCK_SIZE = 64; - - // Memory alignment for SIMD operations - constexpr size_t ALIGNMENT = 32; - -} // namespace QuadBLAS - -#endif // QUADBLAS_CORE_CONSTANTS_HPP \ No newline at end of file diff --git a/include/quadblas/core/platform.hpp b/include/quadblas/core/platform.hpp deleted file mode 100644 index 5ba2409..0000000 --- a/include/quadblas/core/platform.hpp +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef QUADBLAS_CORE_PLATFORM_HPP -#define QUADBLAS_CORE_PLATFORM_HPP - -// Platform detection -#if defined(__x86_64__) || defined(_M_X64) - #define QUADBLAS_X86_64 - #include -#elif defined(__aarch64__) || defined(_M_ARM64) - #define QUADBLAS_AARCH64 - #include -#endif - -// OpenMP detection -#ifdef _OPENMP -#include -#endif - -// SLEEF includes -#include - -// Compatibility for SLEEF_QUAD_C macro -#ifndef SLEEF_QUAD_C - #define SLEEF_QUAD_C(x) Sleef_cast_from_doubleq1(x) -#endif - -#endif // QUADBLAS_CORE_PLATFORM_HPP \ No newline at end of file diff --git a/include/quadblas/core/types.hpp b/include/quadblas/core/types.hpp deleted file mode 100644 index bc257ef..0000000 --- a/include/quadblas/core/types.hpp +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef QUADBLAS_CORE_TYPES_HPP -#define QUADBLAS_CORE_TYPES_HPP - -namespace QuadBLAS -{ - - // Matrix layout types - enum class Layout - { - RowMajor, - ColMajor - }; - - // Forward declarations (no default arguments here to avoid redefinition) - template - class Matrix; - - template - class Vector; - -} // namespace QuadBLAS - -#endif // QUADBLAS_CORE_TYPES_HPP \ No newline at end of file diff --git a/include/quadblas/detail/blocking.hpp b/include/quadblas/detail/blocking.hpp deleted file mode 100644 index c281e8e..0000000 --- a/include/quadblas/detail/blocking.hpp +++ /dev/null @@ -1,71 +0,0 @@ -#ifndef QUADBLAS_DETAIL_BLOCKING_HPP -#define QUADBLAS_DETAIL_BLOCKING_HPP - -#include "../core/constants.hpp" -#include "../core/platform.hpp" -#include - -namespace QuadBLAS -{ - - // Cache-friendly blocking parameters for GEMM - // - // Memory hierarchy usage (standard BLAS approach): - // L1: Micro-kernel slices: A(MR x kc) + B(kc x NR) + C(MR x NR) - // L2: A panel (mc x kc) stays resident - // L3/Memory: B panel (kc x nc) streams through - struct BlockingParams - { - size_t mc, kc, nc; // Panel sizes for GEMM - - BlockingParams(size_t m = 0, size_t n = 0, size_t k = 0) - { - constexpr size_t MR = 4; // Must match GEMM_MR - constexpr size_t NR = 4; // Must match GEMM_NR - const size_t elem_size = sizeof(Sleef_quad); - - // Determine cache sizes (detect at compile time for known platforms) - size_t l1_size = L1_CACHE_SIZE; - size_t l2_size = L2_CACHE_SIZE; - -#if defined(__APPLE__) && defined(QUADBLAS_AARCH64) - // Apple Silicon has larger caches - l1_size = 131072; // 128KB L1D per core - l2_size = 4194304; // 4MB L2 per cluster -#endif - - // kc: chosen so micro-kernel data fits in L1 - // Need: (MR * kc + kc * NR + MR * NR) * elem_size <= L1 / 2 - size_t l1_budget = l1_size / (2 * elem_size); - size_t max_kc = (l1_budget - MR * NR) / (MR + NR); - kc = std::min(max_kc, static_cast(256)); - - // mc: A panel (mc x kc) fits in L2 / 2 - size_t l2_budget = l2_size / (2 * elem_size); - size_t max_mc = l2_budget / kc; - mc = std::min(max_mc, static_cast(256)); - - // nc: B panel (kc x nc) can be larger - fits in remaining L2 or streams from L3 - size_t remaining_l2 = l2_budget - mc * kc; - size_t max_nc = remaining_l2 / kc; - nc = std::min(max_nc, static_cast(512)); - - // Clamp to actual matrix dimensions - if (m > 0) mc = std::min(mc, m); - if (k > 0) kc = std::min(kc, k); - if (n > 0) nc = std::min(nc, n); - - // Align to micro-kernel tile sizes for clean tiling - mc = (mc / MR) * MR; - nc = (nc / NR) * NR; - - // Ensure minimum sizes - mc = std::max(mc, MR); - kc = std::max(kc, static_cast(4)); - nc = std::max(nc, NR); - } - }; - -} // namespace QuadBLAS - -#endif // QUADBLAS_DETAIL_BLOCKING_HPP diff --git a/include/quadblas/interface/c_interface.hpp b/include/quadblas/interface/c_interface.hpp deleted file mode 100644 index b9dde00..0000000 --- a/include/quadblas/interface/c_interface.hpp +++ /dev/null @@ -1,148 +0,0 @@ -#ifndef QUADBLAS_INTERFACE_C_INTERFACE_HPP -#define QUADBLAS_INTERFACE_C_INTERFACE_HPP - -#include "../core/platform.hpp" -#include "../core/types.hpp" -#include "../algorithms/level1.hpp" -#include "../algorithms/level2.hpp" -#include "../algorithms/level3.hpp" -#include "../threading/openmp_utils.hpp" -#include -#include - -// C Interface for easy integration with Python/numpy -// All functions are inline to support header-only inclusion in multiple TUs -extern "C" -{ - - // C API - LEVEL 1 BLAS (Vector operations) - - // QDOT: dot product of two vectors - inline double quadblas_qdot(int n, void *x, int incx, void *y, int incy) - { - Sleef_quad *qx = static_cast(x); - Sleef_quad *qy = static_cast(y); - - Sleef_quad result = QuadBLAS::dot(static_cast(n), qx, - static_cast(incx), qy, - static_cast(incy)); - - return static_cast(Sleef_cast_to_doubleq1(result)); - } - - // QNRM2: Euclidean norm of a vector - inline double quadblas_qnrm2(int n, void *x, int incx) - { - Sleef_quad *qx = static_cast(x); - - Sleef_quad result = QuadBLAS::dot(static_cast(n), qx, - static_cast(incx), qx, - static_cast(incx)); - - result = Sleef_sqrtq1_u05(result); - return static_cast(Sleef_cast_to_doubleq1(result)); - } - - // QAXPY: y := alpha*x + y (uses optimized SIMD/threaded implementation) - inline void quadblas_qaxpy(int n, double alpha, void *x, int incx, void *y, int incy) - { - if (n <= 0) - return; - - Sleef_quad *qx = static_cast(x); - Sleef_quad *qy = static_cast(y); - Sleef_quad qalpha = Sleef_cast_from_doubleq1(alpha); - - QuadBLAS::axpy(static_cast(n), qalpha, qx, - static_cast(incx), qy, - static_cast(incy)); - } - - // C API - LEVEL 2 BLAS (Matrix-vector operations) - - // QGEMV: matrix-vector multiplication - inline void quadblas_qgemv(char layout, char trans, int m, int n, double alpha, - void *A, int lda, void *x, int incx, - double beta, void *y, int incy) - { - - Sleef_quad *qA = static_cast(A); - Sleef_quad *qx = static_cast(x); - Sleef_quad *qy = static_cast(y); - - Sleef_quad qalpha = Sleef_cast_from_doubleq1(alpha); - Sleef_quad qbeta = Sleef_cast_from_doubleq1(beta); - - QuadBLAS::Layout quadblas_layout = (layout == 'C' || layout == 'c') ? QuadBLAS::Layout::ColMajor : QuadBLAS::Layout::RowMajor; - - // Handle transpose (swap dimensions and flip layout) - if (trans == 'T' || trans == 't' || trans == 'C' || trans == 'c') - { - std::swap(m, n); - quadblas_layout = (quadblas_layout == QuadBLAS::Layout::RowMajor) - ? QuadBLAS::Layout::ColMajor - : QuadBLAS::Layout::RowMajor; - } - - QuadBLAS::gemv(quadblas_layout, static_cast(m), static_cast(n), - qalpha, qA, static_cast(lda), qx, static_cast(incx), - qbeta, qy, static_cast(incy)); - } - - // C API - LEVEL 3 BLAS (Matrix-matrix operations) - - // QGEMM: matrix-matrix multiplication - inline void quadblas_qgemm(char layout, char transa, char transb, int m, int n, int k, - double alpha, void *A, int lda, void *B, int ldb, - double beta, void *C, int ldc) - { - - Sleef_quad *qA = static_cast(A); - Sleef_quad *qB = static_cast(B); - Sleef_quad *qC = static_cast(C); - - Sleef_quad qalpha = Sleef_cast_from_doubleq1(alpha); - Sleef_quad qbeta = Sleef_cast_from_doubleq1(beta); - - QuadBLAS::Layout quadblas_layout = (layout == 'C' || layout == 'c') ? QuadBLAS::Layout::ColMajor : QuadBLAS::Layout::RowMajor; - - // Note: transpose support requires additional matrix manipulation - // For now, silently proceed with NoTranspose semantics - (void)transa; - (void)transb; - - QuadBLAS::gemm(quadblas_layout, static_cast(m), static_cast(n), - static_cast(k), qalpha, qA, static_cast(lda), - qB, static_cast(ldb), qbeta, qC, static_cast(ldc)); - } - - // Utility functions - - // Set number of threads for OpenMP - inline void quadblas_set_num_threads(int num_threads) - { - QuadBLAS::set_num_threads(num_threads); - } - - // Get number of threads - inline int quadblas_get_num_threads(void) - { - return QuadBLAS::get_num_threads(); - } - - // Version information - inline const char *quadblas_get_version(void) - { - return "QuadBLAS 1.0.0 - High Performance Quad Precision BLAS"; - } - - // Memory alignment check - inline int quadblas_is_aligned(const void *ptr) - { - uintptr_t addr = reinterpret_cast(ptr); - return (addr % QuadBLAS::ALIGNMENT) == 0; - } - -} // extern "C" - -#endif // QUADBLAS_INTERFACE_C_INTERFACE_HPP diff --git a/include/quadblas/interface/cpp_classes.hpp b/include/quadblas/interface/cpp_classes.hpp deleted file mode 100644 index 0a5b210..0000000 --- a/include/quadblas/interface/cpp_classes.hpp +++ /dev/null @@ -1,172 +0,0 @@ -#ifndef QUADBLAS_INTERFACE_CPP_CLASSES_HPP -#define QUADBLAS_INTERFACE_CPP_CLASSES_HPP - -#include "../core/types.hpp" -#include "../core/platform.hpp" -#include "../memory/allocation.hpp" -#include "../algorithms/level1.hpp" -#include "../algorithms/level2.hpp" -#include "../algorithms/level3.hpp" -#include - -namespace QuadBLAS -{ - - // C++ Convenience Classes - - // Simple vector class for convenience - template - class Vector - { - private: - Sleef_quad *data_; - size_t size_; - size_t stride_; - bool owns_memory_; - - public: - Vector(size_t size) : size_(size), stride_(1), owns_memory_(true) - { - data_ = aligned_alloc(size); - std::fill(data_, data_ + size, SLEEF_QUAD_C(0.0)); - } - - Vector(Sleef_quad *data, size_t size, size_t stride = 1) - : data_(data), size_(size), stride_(stride), owns_memory_(false) {} - - ~Vector() - { - if (owns_memory_) - { - aligned_free(data_); - } - } - - // Move constructor - Vector(Vector &&other) noexcept - : data_(other.data_), size_(other.size_), stride_(other.stride_), - owns_memory_(other.owns_memory_) - { - other.owns_memory_ = false; - } - - // Disable copy constructor to avoid double-free - Vector(const Vector &) = delete; - Vector &operator=(const Vector &) = delete; - - Sleef_quad &operator[](size_t i) { return data_[i * stride_]; } - const Sleef_quad &operator[](size_t i) const { return data_[i * stride_]; } - - size_t size() const { return size_; } - size_t stride() const { return stride_; } - Sleef_quad *data() { return data_; } - const Sleef_quad *data() const { return data_; } - - // Dot product - Sleef_quad dot(const Vector &other) const - { - return QuadBLAS::dot(size_, data_, stride_, other.data_, other.stride_); - } - - // AXPY: this = alpha * other + this - void axpy(Sleef_quad alpha, const Vector &other) - { - QuadBLAS::axpy(size_, alpha, other.data_, other.stride_, data_, stride_); - } - - // Norm - Sleef_quad norm() const - { - return Sleef_sqrtq1_u05(dot(*this)); - } - }; - - // Simple matrix class for convenience - template - class Matrix - { - private: - Sleef_quad *data_; - size_t rows_, cols_; - size_t ld_; - bool owns_memory_; - - public: - Matrix(size_t rows, size_t cols) - : rows_(rows), cols_(cols), ld_(cols), owns_memory_(true) - { - data_ = aligned_alloc(rows * cols); - std::fill(data_, data_ + rows * cols, SLEEF_QUAD_C(0.0)); - } - - Matrix(Sleef_quad *data, size_t rows, size_t cols, size_t ld = 0) - : data_(data), rows_(rows), cols_(cols), - ld_(ld == 0 ? cols : ld), owns_memory_(false) {} - - ~Matrix() - { - if (owns_memory_) - { - aligned_free(data_); - } - } - - // Move constructor - Matrix(Matrix &&other) noexcept - : data_(other.data_), rows_(other.rows_), cols_(other.cols_), - ld_(other.ld_), owns_memory_(other.owns_memory_) - { - other.owns_memory_ = false; - } - - // Disable copy constructor - Matrix(const Matrix &) = delete; - Matrix &operator=(const Matrix &) = delete; - - Sleef_quad &operator()(size_t i, size_t j) - { - return layout == Layout::RowMajor ? data_[i * ld_ + j] : data_[j * ld_ + i]; - } - - const Sleef_quad &operator()(size_t i, size_t j) const - { - return layout == Layout::RowMajor ? data_[i * ld_ + j] : data_[j * ld_ + i]; - } - - size_t rows() const { return rows_; } - size_t cols() const { return cols_; } - size_t leading_dimension() const { return ld_; } - Sleef_quad *data() { return data_; } - const Sleef_quad *data() const { return data_; } - - // Matrix-vector multiplication - void gemv(Sleef_quad alpha, const Vector &x, Sleef_quad beta, Vector &y) const - { - QuadBLAS::gemv(layout, rows_, cols_, alpha, data_, ld_, - x.data(), x.stride(), beta, y.data(), y.stride()); - } - - // Matrix-matrix multiplication - void gemm(Sleef_quad alpha, const Matrix &B, Sleef_quad beta, Matrix &C) const - { - QuadBLAS::gemm(layout, rows_, B.cols(), cols_, alpha, data_, ld_, - B.data(), B.leading_dimension(), beta, C.data(), C.leading_dimension()); - } - }; - - // Convenient type aliases with default template arguments - using VectorRowMajor = Vector; - using VectorColMajor = Vector; - using MatrixRowMajor = Matrix; - using MatrixColMajor = Matrix; - - // Default to row major - template - using DefaultVector = Vector; - - template - using DefaultMatrix = Matrix; - -} // namespace QuadBLAS - -#endif // QUADBLAS_INTERFACE_CPP_CLASSES_HPP \ No newline at end of file diff --git a/include/quadblas/memory/allocation.hpp b/include/quadblas/memory/allocation.hpp deleted file mode 100644 index 128f9ac..0000000 --- a/include/quadblas/memory/allocation.hpp +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef QUADBLAS_MEMORY_ALLOCATION_HPP -#define QUADBLAS_MEMORY_ALLOCATION_HPP - -#include "../core/constants.hpp" -#include -#include - -#ifdef _WIN32 -#include -#else -#include -#endif - -namespace QuadBLAS -{ - - // Aligned memory allocation - template - inline T *aligned_alloc(size_t count) - { - void *ptr = nullptr; -#ifdef _WIN32 - ptr = _aligned_malloc(count * sizeof(T), ALIGNMENT); -#else - if (posix_memalign(&ptr, ALIGNMENT, count * sizeof(T)) != 0) - { - ptr = nullptr; - } -#endif - return static_cast(ptr); - } - - template - inline void aligned_free(T *ptr) - { -#ifdef _WIN32 - _aligned_free(ptr); -#else - free(ptr); -#endif - } - -} // namespace QuadBLAS - -#endif // QUADBLAS_MEMORY_ALLOCATION_HPP \ No newline at end of file diff --git a/include/quadblas/quadblas.hpp b/include/quadblas/quadblas.hpp deleted file mode 100644 index 945b7fa..0000000 --- a/include/quadblas/quadblas.hpp +++ /dev/null @@ -1,61 +0,0 @@ -#ifndef QUADBLAS_HPP -#define QUADBLAS_HPP - -/** - * QuadBLAS - High Performance Quad Precision BLAS Library - * - * A header-only library providing optimized linear algebra operations - * for IEEE 754 quadruple precision (128-bit) floating point numbers. - * - * Features: - * - SIMD vectorization (x86-64 SSE/AVX, ARM64 NEON) - * - OpenMP multi-threading - * - Cache-optimized algorithms - * - Both C and C++ interfaces - * - Header-only design for easy integration - * - * Built on top of the SLEEF mathematical library. - * - * Copyright (c) 2025 QuadBLAS Project - * Licensed under the Apache License, Version 2.0 - * See LICENSE file in the root directory or visit: - * http://www.apache.org/licenses/LICENSE-2.0 - */ - -// Include all public headers in dependency order - -// Core fundamentals -#include "core/platform.hpp" -#include "core/constants.hpp" -#include "core/types.hpp" - -// Memory management -#include "memory/allocation.hpp" - -// SIMD abstractions -#include "simd/quad_vector.hpp" - -// Threading utilities -#include "threading/openmp_utils.hpp" - -// Internal details -#include "detail/blocking.hpp" - -// Algorithm implementations (Level 1, 2, 3 BLAS) -#include "algorithms/level1.hpp" -#include "algorithms/level2.hpp" -#include "algorithms/level3.hpp" - -// Public interfaces -#include "interface/c_interface.hpp" -#include "interface/cpp_classes.hpp" - -// Version information -namespace QuadBLAS { - constexpr const char* VERSION = "1.0.0"; - constexpr int VERSION_MAJOR = 1; - constexpr int VERSION_MINOR = 0; - constexpr int VERSION_PATCH = 0; -} - -#endif // QUADBLAS_HPP \ No newline at end of file diff --git a/include/quadblas/simd/quad_vector.hpp b/include/quadblas/simd/quad_vector.hpp deleted file mode 100644 index 4e5fa4f..0000000 --- a/include/quadblas/simd/quad_vector.hpp +++ /dev/null @@ -1,155 +0,0 @@ -#ifndef QUADBLAS_SIMD_QUAD_VECTOR_HPP -#define QUADBLAS_SIMD_QUAD_VECTOR_HPP - -#include "../core/platform.hpp" -#include "../memory/allocation.hpp" - -namespace QuadBLAS -{ - - // SIMD wrapper for platform abstraction - class QuadVector - { - private: -#ifdef QUADBLAS_X86_64 - Sleef_quadx2 data_; -#elif defined(QUADBLAS_AARCH64) - Sleef_quadx2 data_; -#else - Sleef_quad data_[2]; -#endif - - public: - QuadVector() = default; - - explicit QuadVector(Sleef_quad value) - { -#ifdef QUADBLAS_X86_64 - data_ = Sleef_splatq2_sse2(value); -#elif defined(QUADBLAS_AARCH64) - data_ = Sleef_splatq2_advsimd(value); -#else - data_[0] = data_[1] = value; -#endif - } - - QuadVector(Sleef_quad a, Sleef_quad b) - { - // Use stack-aligned buffer instead of heap allocation - alignas(ALIGNMENT) Sleef_quad temp[2]; - temp[0] = a; - temp[1] = b; -#ifdef QUADBLAS_X86_64 - data_ = Sleef_loadq2_sse2(temp); -#elif defined(QUADBLAS_AARCH64) - data_ = Sleef_loadq2_advsimd(temp); -#else - data_[0] = a; - data_[1] = b; -#endif - } - - static QuadVector load(Sleef_quad *ptr) - { - QuadVector result; -#ifdef QUADBLAS_X86_64 - result.data_ = Sleef_loadq2_sse2(ptr); -#elif defined(QUADBLAS_AARCH64) - result.data_ = Sleef_loadq2_advsimd(ptr); -#else - result.data_[0] = ptr[0]; - result.data_[1] = ptr[1]; -#endif - return result; - } - - // Const overload: SLEEF load takes non-const but doesn't modify memory - static QuadVector load(const Sleef_quad *ptr) - { - return load(const_cast(ptr)); - } - - void store(Sleef_quad *ptr) const - { -#ifdef QUADBLAS_X86_64 - // Work around SLEEF store bug - use get instead - ptr[0] = Sleef_getq2_sse2(data_, 0); - ptr[1] = Sleef_getq2_sse2(data_, 1); -#elif defined(QUADBLAS_AARCH64) - // Work around SLEEF store bug - use get instead - ptr[0] = Sleef_getq2_advsimd(data_, 0); - ptr[1] = Sleef_getq2_advsimd(data_, 1); -#else - ptr[0] = data_[0]; - ptr[1] = data_[1]; -#endif - } - - Sleef_quad get(int index) const - { -#ifdef QUADBLAS_X86_64 - return Sleef_getq2_sse2(data_, index); -#elif defined(QUADBLAS_AARCH64) - return Sleef_getq2_advsimd(data_, index); -#else - return data_[index]; -#endif - } - - QuadVector operator+(const QuadVector &other) const - { - QuadVector result; -#ifdef QUADBLAS_X86_64 - result.data_ = Sleef_addq2_u05sse2(data_, other.data_); -#elif defined(QUADBLAS_AARCH64) - result.data_ = Sleef_addq2_u05advsimd(data_, other.data_); -#else - result.data_[0] = Sleef_addq1_u05(data_[0], other.data_[0]); - result.data_[1] = Sleef_addq1_u05(data_[1], other.data_[1]); -#endif - return result; - } - - QuadVector operator*(const QuadVector &other) const - { - QuadVector result; -#ifdef QUADBLAS_X86_64 - result.data_ = Sleef_mulq2_u05sse2(data_, other.data_); -#elif defined(QUADBLAS_AARCH64) - result.data_ = Sleef_mulq2_u05advsimd(data_, other.data_); -#else - result.data_[0] = Sleef_mulq1_u05(data_[0], other.data_[0]); - result.data_[1] = Sleef_mulq1_u05(data_[1], other.data_[1]); -#endif - return result; - } - - QuadVector fma(const QuadVector &b, const QuadVector &c) const - { - QuadVector result; -#ifdef QUADBLAS_X86_64 - result.data_ = Sleef_fmaq2_u05sse2(data_, b.data_, c.data_); -#elif defined(QUADBLAS_AARCH64) - result.data_ = Sleef_fmaq2_u05advsimd(data_, b.data_, c.data_); -#else - result.data_[0] = Sleef_fmaq1_u05(data_[0], b.data_[0], c.data_[0]); - result.data_[1] = Sleef_fmaq1_u05(data_[1], b.data_[1], c.data_[1]); -#endif - return result; - } - - Sleef_quad horizontal_sum() const - { -#ifdef QUADBLAS_X86_64 - return Sleef_addq1_u05(Sleef_getq2_sse2(data_, 0), Sleef_getq2_sse2(data_, 1)); -#elif defined(QUADBLAS_AARCH64) - return Sleef_addq1_u05(Sleef_getq2_advsimd(data_, 0), Sleef_getq2_advsimd(data_, 1)); -#else - return Sleef_addq1_u05(data_[0], data_[1]); -#endif - } - }; - -} // namespace QuadBLAS - -#endif // QUADBLAS_SIMD_QUAD_VECTOR_HPP \ No newline at end of file diff --git a/include/quadblas/threading/openmp_utils.hpp b/include/quadblas/threading/openmp_utils.hpp deleted file mode 100644 index 3be8963..0000000 --- a/include/quadblas/threading/openmp_utils.hpp +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef QUADBLAS_THREADING_OPENMP_UTILS_HPP -#define QUADBLAS_THREADING_OPENMP_UTILS_HPP - -#include "../core/platform.hpp" - -namespace QuadBLAS -{ - - // Utility functions for threading - inline int get_num_threads() - { -#ifdef _OPENMP - return omp_get_max_threads(); -#else - return 1; -#endif - } - - inline void set_num_threads(int num_threads) - { -#ifdef _OPENMP - omp_set_num_threads(num_threads); -#endif - } - -} // namespace QuadBLAS - -#endif // QUADBLAS_THREADING_OPENMP_UTILS_HPP \ No newline at end of file diff --git a/meson.build b/meson.build new file mode 100644 index 0000000..eefba5c --- /dev/null +++ b/meson.build @@ -0,0 +1,102 @@ +# Mirrors the CMake build for use as a Meson subproject by downstream +# packages (notably numpy-quaddtype). Produces libqblas.a + qblas_dep + +project('qblas', 'c', + version: '1.5.0', + default_options: ['c_std=c11', 'b_pie=true', 'b_ndebug=if-release'], + meson_version: '>=1.1') + +cc = meson.get_compiler('c') + +if meson.is_subproject() + sleef_subproj = subproject('sleef') + sleef_dep = sleef_subproj.get_variable('sleef_dep') + sleefquad_dep = sleef_subproj.get_variable('sleefquad_dep') +else + sleef_dep = dependency('sleef', required: false) + if not sleef_dep.found() + sleef_dep = cc.find_library('sleef', required: true) + endif + sleefquad_dep = dependency('sleefquad', required: false) + if not sleefquad_dep.found() + sleefquad_dep = cc.find_library('sleefquad', required: true) + endif +endif + +openmp_dep = dependency('openmp', required: false) +omp_args = openmp_dep.found() ? ['-DQBLAS_USE_OPENMP=1'] : [] + +arch = host_machine.cpu_family() +is_x86 = (arch == 'x86_64' or arch == 'x86') +is_arm = (arch == 'aarch64' or arch == 'arm64') + +isa_tiers = ['generic'] +if is_x86 + isa_tiers += ['sse2', 'avx2', 'avx512'] +elif is_arm + isa_tiers += ['neon'] +endif + +isa_flags = { + 'generic': [], + 'sse2' : ['-msse2'], + 'avx2' : ['-mavx2', '-mfma', '-mbmi2'], + 'avx512' : ['-mavx512f', '-mavx512dq', '-mavx512bw', '-mfma', '-mbmi2'], + 'neon' : [], +} +isa_has_defines = { + 'generic': '-DQBLAS_HAS_GENERIC=1', + 'sse2' : '-DQBLAS_HAS_SSE2=1', + 'avx2' : '-DQBLAS_HAS_AVX2=1', + 'avx512' : '-DQBLAS_HAS_AVX512=1', + 'neon' : '-DQBLAS_HAS_NEON=1', +} + +qblas_inc = include_directories('include', 'src') +qblas_core_args = ['-O3', '-D_POSIX_C_SOURCE=200809L'] + omp_args + +tier_libs = [] +foreach tier : isa_tiers + tier_lib = static_library( + 'qblas_kernels_@0@'.format(tier), + 'src/kernels/@0@/kernels_@0@.c'.format(tier), + c_args: qblas_core_args + isa_flags[tier] + [isa_has_defines[tier]], + include_directories: qblas_inc, + dependencies: [sleef_dep, sleefquad_dep], + install: false, + ) + tier_libs += tier_lib +endforeach + + +core_defines = ['-DQBLAS_HAS_GENERIC=1'] +if is_x86 + core_defines += ['-DQBLAS_HAS_SSE2=1', '-DQBLAS_HAS_AVX2=1', '-DQBLAS_HAS_AVX512=1'] +elif is_arm + core_defines += ['-DQBLAS_HAS_NEON=1'] +endif + +core_srcs = [ + 'src/cpu/qblas_cpu.c', + 'src/level1/qblas_level1.c', + 'src/level2/qblas_level2.c', + 'src/level3/qblas_gemm.c', + 'src/level3/qblas_level3_extra.c', +] + +qblas_lib = static_library( + 'qblas', + core_srcs, + c_args: qblas_core_args + core_defines, + include_directories: qblas_inc, + link_with: tier_libs, + dependencies: [sleef_dep, sleefquad_dep] + (openmp_dep.found() ? [openmp_dep] : []), + install: false, +) + +qblas_dep = declare_dependency( + link_with: qblas_lib, + include_directories: include_directories('include'), + dependencies: [sleef_dep, sleefquad_dep] + (openmp_dep.found() ? [openmp_dep] : []), + version: meson.project_version(), +) diff --git a/perf_compare.cpp b/perf_compare.cpp deleted file mode 100644 index a0992e2..0000000 --- a/perf_compare.cpp +++ /dev/null @@ -1,185 +0,0 @@ -#include "include/quadblas/quadblas.hpp" -#include -#include -#include -#include -#include -#include -#include - -static double to_double(Sleef_quad q) { - return static_cast(Sleef_cast_to_doubleq1(q)); -} - -static Sleef_quad from_double(double d) { - return Sleef_cast_from_doubleq1(d); -} - -// Compiler barrier to prevent dead-code elimination -template -static void do_not_optimize(T const &value) { - asm volatile("" : : "r,m"(value) : "memory"); -} - -static void fill_random(Sleef_quad *arr, size_t n, unsigned seed = 42) { - std::mt19937 gen(seed); - std::uniform_real_distribution dist(-1.0, 1.0); - for (size_t i = 0; i < n; ++i) - arr[i] = from_double(dist(gen)); -} - -// Returns median time in milliseconds over `trials` runs (with 2 warmup runs) -template -static double benchmark_median(Func fn, int trials = 7) { - // warmup - for (int i = 0; i < 2; ++i) fn(); - - std::vector times(trials); - for (int i = 0; i < trials; ++i) { - auto t0 = std::chrono::high_resolution_clock::now(); - fn(); - auto t1 = std::chrono::high_resolution_clock::now(); - times[i] = std::chrono::duration(t1 - t0).count(); - } - std::sort(times.begin(), times.end()); - return times[trials / 2]; // median -} - -// ---- Benchmarks ---- - -static void bench_dot() { - std::cout << "\n=== DOT PRODUCT ===\n"; - std::cout << std::setw(10) << "N" - << std::setw(15) << "Time (ms)" << "\n"; - std::cout << std::string(25, '-') << "\n"; - - for (size_t n : {1000, 5000, 10000, 50000}) { - Sleef_quad *x = QuadBLAS::aligned_alloc(n); - Sleef_quad *y = QuadBLAS::aligned_alloc(n); - fill_random(x, n, 1); - fill_random(y, n, 2); - - double ms = benchmark_median([&]() { - Sleef_quad r = QuadBLAS::dot(n, x, 1, y, 1); - do_not_optimize(r); - }); - - std::cout << std::setw(10) << n - << std::setw(15) << std::fixed << std::setprecision(3) << ms << "\n"; - - QuadBLAS::aligned_free(x); - QuadBLAS::aligned_free(y); - } -} - -static void bench_axpy() { - std::cout << "\n=== AXPY ===\n"; - std::cout << std::setw(10) << "N" - << std::setw(15) << "Time (ms)" << "\n"; - std::cout << std::string(25, '-') << "\n"; - - Sleef_quad alpha = from_double(2.5); - - for (size_t n : {1000, 5000, 10000, 50000}) { - Sleef_quad *x = QuadBLAS::aligned_alloc(n); - Sleef_quad *y = QuadBLAS::aligned_alloc(n); - Sleef_quad *y_backup = QuadBLAS::aligned_alloc(n); - fill_random(x, n, 3); - fill_random(y, n, 4); - std::copy(y, y + n, y_backup); - - double ms = benchmark_median([&]() { - std::copy(y_backup, y_backup + n, y); // restore y each run - QuadBLAS::axpy(n, alpha, x, 1, y, 1); - do_not_optimize(y[0]); - }); - - std::cout << std::setw(10) << n - << std::setw(15) << std::fixed << std::setprecision(3) << ms << "\n"; - - QuadBLAS::aligned_free(x); - QuadBLAS::aligned_free(y); - QuadBLAS::aligned_free(y_backup); - } -} - -static void bench_gemv() { - std::cout << "\n=== GEMV ===\n"; - std::cout << std::setw(10) << "N" - << std::setw(15) << "Time (ms)" << "\n"; - std::cout << std::string(25, '-') << "\n"; - - Sleef_quad alpha = from_double(1.0); - Sleef_quad beta = from_double(0.0); - - for (size_t n : {100, 300, 500, 1000}) { - Sleef_quad *A = QuadBLAS::aligned_alloc(n * n); - Sleef_quad *x = QuadBLAS::aligned_alloc(n); - Sleef_quad *y = QuadBLAS::aligned_alloc(n); - fill_random(A, n * n, 5); - fill_random(x, n, 6); - - double ms = benchmark_median([&]() { - for (size_t i = 0; i < n; ++i) - y[i] = from_double(0.0); - QuadBLAS::gemv(QuadBLAS::Layout::RowMajor, n, n, alpha, A, n, x, 1, beta, y, 1); - do_not_optimize(y[0]); - }); - - std::cout << std::setw(10) << n - << std::setw(15) << std::fixed << std::setprecision(3) << ms << "\n"; - - QuadBLAS::aligned_free(A); - QuadBLAS::aligned_free(x); - QuadBLAS::aligned_free(y); - } -} - -static void bench_gemm() { - std::cout << "\n=== GEMM ===\n"; - std::cout << std::setw(10) << "N" - << std::setw(15) << "Time (ms)" << "\n"; - std::cout << std::string(25, '-') << "\n"; - - Sleef_quad alpha = from_double(1.0); - Sleef_quad beta = from_double(0.0); - - for (size_t n : {32, 64, 128, 200, 300}) { - Sleef_quad *A = QuadBLAS::aligned_alloc(n * n); - Sleef_quad *B = QuadBLAS::aligned_alloc(n * n); - Sleef_quad *C = QuadBLAS::aligned_alloc(n * n); - fill_random(A, n * n, 7); - fill_random(B, n * n, 8); - - int trials = (n <= 128) ? 5 : 3; - - double ms = benchmark_median([&]() { - for (size_t i = 0; i < n * n; ++i) - C[i] = from_double(0.0); - QuadBLAS::gemm(QuadBLAS::Layout::RowMajor, n, n, n, alpha, A, n, B, n, beta, C, n); - do_not_optimize(C[0]); - }, trials); - - std::cout << std::setw(10) << n - << std::setw(15) << std::fixed << std::setprecision(3) << ms << "\n"; - - QuadBLAS::aligned_free(A); - QuadBLAS::aligned_free(B); - QuadBLAS::aligned_free(C); - } -} - -int main() { - std::cout << "QuadBLAS Performance Comparison Benchmark\n"; - std::cout << "==========================================\n"; - std::cout << "Version: " << quadblas_get_version() << "\n"; - std::cout << "Threads: " << QuadBLAS::get_num_threads() << "\n"; - - bench_dot(); - bench_axpy(); - bench_gemv(); - bench_gemm(); - - std::cout << "\nDone.\n"; - return 0; -} diff --git a/perf_comparison_with_old.md b/perf_comparison_with_old.md new file mode 100644 index 0000000..c63e2b3 --- /dev/null +++ b/perf_comparison_with_old.md @@ -0,0 +1,202 @@ +# QBLAS overhaul, comprehensive before/after benchmark + +## Setup + +Same machine, same compiler, same SLEEF build for both sides: + +- Host: AMD EPYC 7V13 (Zen 3, AVX2+FMA, no AVX-512), 96 cores. +- Compiler: gcc (`-O3 -march=native`). +- SLEEF: built from `third_party/sleef` with every SIMD path enabled + (`SLEEF_ENABLE_{SSE2,AVX,AVX2,AVX512F,INLINE_HEADERS}=ON`) and installed + to `.sleef-prefix/`. Both old and new binaries link the same shared + libraries. +- Bench harness: Google Benchmark, **median of 3 runs**, `--benchmark_min_time=0.3s`. + +Both sides are run with `OMP_NUM_THREADS=16` — that's the most defensible +apples-to-apples setup on this shared 96-core box. Single-thread numbers +(showing pure kernel quality) are also reported at the end. + +- **OLD** = branch `main`, commit `42126fd` (original `QuadBLAS`). +- **NEW** = branch `overhaul-rewrite`, current head. + +Source for the matched-size bench: [bench/bench_compare.cc](bench/bench_compare.cc). + +All numbers are `items_per_second` from Google Benchmark, where `items = +n` for L1, `m*n` for gemv, `m*n*k` for gemm/syrk, `m*m*n/2` for trmm/trsm. +For gemm that's exactly the FMA throughput. + +--- + +## Master comparison table — 16 threads + +### Level 1 — vector operations + +| routine | n | OLD (M items/s) | NEW (M items/s) | speedup | +| -------- | --------- | --------------: | --------------: | --------: | +| **dot** | 1 000 | 29.9 | 28.3 | 0.95× | +| dot | 10 000 | 37.2 | 319.8 | **8.6×** | +| dot | 100 000 | 38.1 | 415.7 | **10.9×** | +| dot | 1 000 000 | 38.7 | 429.6 | **11.1×** | +| **axpy** | 1 000 | 29.2 | 27.9 | 0.96× | +| axpy | 4 000 | 36.3 | 251.9 | **6.9×** | +| axpy | 16 000 | 38.8 | 349.0 | **9.0×** | +| axpy | 64 000 | 39.5 | 387.8 | **9.8×** | +| **nrm2** | 1 000 | 28.9 | 29.2 | 1.01× | +| nrm2 | 4 000 | 36.9 | 224.9 | **6.1×** | +| nrm2 | 16 000 | 38.7 | 355.7 | **9.2×** | +| nrm2 | 64 000 | 39.2 | 409.8 | **10.5×** | +| asum | 1 000 | n/a | 65.3 | — | +| asum | 4 000 | n/a | 343.0 | — | +| asum | 16 000 | n/a | 598.5 | — | +| asum | 64 000 | n/a | 853.7 | — | +| scal | 1 000 | n/a | 65.9 | — | +| scal | 4 000 | n/a | 386.9 | — | +| scal | 16 000 | n/a | 703.0 | — | +| scal | 64 000 | n/a | 888.2 | — | + +OLD was missing `asum` and `scal` entirely; rows are marked n/a. + +### Level 2 — matrix-vector + +| routine | dim | OLD (M items/s) | NEW (M items/s) | speedup | +| -------- | ------ | --------------: | --------------: | --------: | +| **gemv** | 100² | 2.5 | 296.8 | **117×** | +| gemv | 128² | 2.6 | — | — | +| gemv | 200² | — | 376.5 | — | +| gemv | 256² | 2.6 | — | — | +| gemv | 400² | — | 408.9 | — | +| gemv | 512² | 38.0 | — | — | +| gemv | 800² | — | 419.5 | — | +| gemv | 1 024² | 38.0 | — | — | +| gemv | 1 600² | 38.2 | 431.2 | **11.3×** | + +The two side's bench harnesses use different power-of-2 grids; only 100² +and 1 600² are common. + +### Level 3 — matrix-matrix + +| routine | n³ (dim) | OLD (M items/s) | NEW (M items/s) | speedup | +| -------- | -------- | --------------: | --------------: | -------: | +| **gemm** | 64³ | 148.1 | 407.4 | 2.8× | +| gemm | 128³ | 2.6 | 414.7 | **159×** | +| gemm | 256³ | 2.6 | 421.4 | **161×** | +| gemm | 512³ | 16.6 | 429.7 | **26×** | +| syrk | 64³ | n/a | 410.2 | — | +| syrk | 128³ | n/a | 411.2 | — | +| syrk | 256³ | n/a | 418.9 | — | +| syrk | 512³ | n/a | 443.3 | — | +| trmm | 64²·64 | n/a | 151.0 | — | +| trmm | 128²·128 | n/a | 215.2 | — | +| trmm | 256²·256 | n/a | 287.3 | — | +| trmm | 512²·512 | n/a | 350.3 | — | +| trsm | 64²·64 | n/a | 170.2 | — | +| trsm | 128²·128 | n/a | 221.6 | — | +| trsm | 256²·256 | n/a | 282.9 | — | +| trsm | 512²·512 | n/a | 335.7 | — | + +The 128³–256³ GEMM cliff in OLD (2.6 M/s, ~1/60 of single-thread!) was a +real regression in the legacy blocked path that this overhaul fixed. +`syrk`, `trmm`, `trsm` did not exist in OLD at all. + +--- + +## Single-thread numbers (`OMP_NUM_THREADS=1`) + +These reveal _pure kernel quality_ — no parallel speedup, just the cost +of one routine call against one core. Across every routine and every +size, OLD ran at the unvectorised SLEEF scalar speed (~2.6 M ops/s) and +NEW runs at SLEEF's vectorised AVX2 ceiling (~30 M ops/s): + +| routine | OLD (M items/s) | NEW (M items/s) | speedup | +| ------- | --------------: | --------------: | -------: | +| dot | 2.6 – 2.7 | 29 – 30 | ~11× | +| axpy | 2.7 – 2.7 | 28 | ~10× | +| nrm2 | 2.7 – 2.8 | 29 – 30 | ~11× | +| gemv | 2.6 – 2.7 | 27 – 30 | ~10× | +| gemm | 2.6 – 10.4 | 30 | ~3 – 12× | + +So the new code is ~11× faster _per core_ before parallelisation even +enters the picture. That's the SIMD that the old build never actually +benefited from. + +--- + +## 96-thread results (`OMP_NUM_THREADS=96 OMP_PLACES=cores OMP_PROC_BIND=close`) + +Full-machine numbers from the same EPYC 7V13 box. Threads are pinned so +each iteration's timing is stable enough for median-of-3 to be +meaningful. Compute-bound L3 routines scale nearly linearly to all 96 +cores; L1/L2 at moderate sizes are memory-bandwidth bound and saturate +earlier (which is why my dynamic threading heuristic correctly _does +not_ spawn 96 threads for them). + +| routine | n / dim | OLD M items/s | NEW M items/s | new × old | +| ------- | ------- | ------------: | ------------------: | ---------: | +| dot | 1 M | 176.8 | **882.9** | **5.0×** | +| axpy | 64 K | 151.2 | 83.3 (mem-BW bound) | 0.55× | +| nrm2 | 64 K | 169.1 | 30.0 (memory bound) | 0.18× | +| asum | 64 K | n/a | 66.7 | — | +| scal | 64 K | n/a | 68.3 | — | +| gemv | 1 600² | 188.2 | **2 202.4** | **11.7×** | +| gemm | 64³ | 771.5 | 242.8 | 0.31× | +| gemm | 128³ | 2.6 | **1 821.8** | **701×** | +| gemm | 256³ | 2.6 | **3 129.0** | **1 203×** | +| gemm | 512³ | 18.1 | **3 730.9** | **206×** | +| syrk | 64³ | n/a | 211.4 | — | +| syrk | 128³ | n/a | 1 498.8 | — | +| syrk | 256³ | n/a | 3 721.6 | — | +| syrk | 512³ | n/a | **3 773.7** | — | +| trmm | 64² | n/a | 81.2 | — | +| trmm | 128² | n/a | 233.5 | — | +| trmm | 256² | n/a | 802.1 | — | +| trmm | 512² | n/a | **2 706.5** | — | +| trsm | 64² | n/a | 87.8 | — | +| trsm | 128² | n/a | 262.9 | — | +| trsm | 256² | n/a | 795.3 | — | +| trsm | 512² | n/a | **2 529.8** | — | + +Observations: + +- **GEMM at 256³–512³ hits 3.1–3.7 G FMA/sec**, which is the peak this + machine produces for quad precision through SLEEF. At 512³ that's an + 8.7× scale-up over the 16-thread number, almost linear. +- **TRSM/TRMM at 512² reach 2.5–2.7 G items/sec** because the trailing + GEMM updates dominate at that size and inherit GEMM's threading. +- **`gemm/64` regressed at 96 threads** vs OLD (771 vs 243). At m=n=k=64 + the parallel-region overhead of 96 fork/joins dominates the 262 K FMAs + of actual work. Running with `OMP_NUM_THREADS=16` gives `gemm/64 = 407 +M/s`, which beats OLD. This is exactly the kind of thing the dynamic + threading heuristic should fix: cap threads to ~16 even when 96 are + available, for small problems. +- **`axpy/64K`, `nrm2/64K` regressed at 96 threads** vs OLD's 16-thread + measurement. Same root cause: at this size the working set (1 MB) + doesn't justify 96 cores' worth of memory traffic. The 16-thread + numbers in the previous section are the better comparison for L1. + +--- + +## Reproducing this exact run + +```bash +# Build the SLEEF that both binaries link +scripts/bootstrap_sleef.sh + +# OLD +git checkout main +SLEEF_ROOT=$(pwd)/.sleef-prefix cmake -S . -B build-old -DCMAKE_BUILD_TYPE=Release +cmake --build build-old --target quadblas_benchmark -j + +# NEW +git checkout overhaul-rewrite +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build --target qblas_bench_compare -j + +# Comparison (16 threads, median of 3) +OMP_NUM_THREADS=16 LD_LIBRARY_PATH=$(pwd)/.sleef-prefix/lib \ + ./build-old/quadblas_benchmark --benchmark_min_time=0.3s \ + --benchmark_repetitions=3 --benchmark_report_aggregates_only=true \ + --benchmark_filter='BM_DotProduct|BM_AXPY|BM_VectorNorm|BM_GEMV/|BM_GEMM/' +OMP_NUM_THREADS=16 ./build/bench/qblas_bench_compare \ + --benchmark_min_time=0.3s --benchmark_repetitions=3 \ + --benchmark_report_aggregates_only=true +``` diff --git a/performance_bottlenecks.md b/performance_bottlenecks.md new file mode 100644 index 0000000..aa1c8c0 --- /dev/null +++ b/performance_bottlenecks.md @@ -0,0 +1,205 @@ +# QBLAS — remaining performance headroom + +This file captures the perf opportunities the current rewrite did **not** +take. Each section names the win, the rough size, the cost (in +engineering and/or precision), and an honest recommendation. + +Use this as the decision sheet for the next perf push. + +--- + +## 1. Hand-tuned DD-FMA micro-kernel (the big one) + +**The ceiling we hit.** Single-thread GEMM peaks at ~30 M quad-FMAs/sec/core +on AVX2. A tight microbench with 8 independent FMAs in flight tops out at +the same 30.8 M/core, which is exactly the throughput of +`Sleef_fmaq4_u05avx2`. We're at the SLEEF symbol's ceiling. + +Measured: see `/tmp/sleef_peak.c` style microbench (Zen 3 @ ~3 GHz): +~97 cycles per quad-FMA call. + +**What it would unlock.** ~2–3× single-thread, multiplied across all +cores. Concretely, expectation: +- Single-thread: 30 M → ~80–100 M quad-FMAs/sec/core +- 16-thread GEMM: 430 M → ~1.2 G items/sec +- 96-thread GEMM: 1.6 G → ~5 G items/sec + +**Why we didn't do it.** Writing a u05-accurate quad FMA on AVX2 doubles +from scratch is real work. SLEEF's u05 uses **triple-double (TD, 3-term)** +internally, which is more involved than the classical 2-term DD. A +straightforward AVX2 DD-FMA implementation gives ~106-bit accuracy vs +quad's 113 bits — a ~7-bit (≈2 decimal-digit) precision loss. + +**Decision needed.** Three options: + +| option | accuracy | est. speedup vs current | effort | +|---|---|---|---| +| (a) Keep SLEEF u05 (status quo) | 0.5 ULP (~113 bits) | 1× | 0 | +| (b) Hand DD-FMA, u10-like (~106 bits, 1.5–2 ULP)| ~106 bits | 2.5–3× per core | ~1 week | +| (c) Inline-DD via SLEEF's `sleefquadinline_avx2.h` | 0.5 ULP (same) | ~1.05× | 2 days | +| (d) Hand TD-FMA replicating SLEEF u05 | 0.5 ULP (same) | hard to predict, ~1.2–1.5× best | ~2 weeks | + +Recommendation: **(c) first** as a near-zero-risk 5 % win, then **(b)** if +losing ~7 bits is acceptable for the perf jump. The IEEE-754 binary128 +guarantee in the README would need a footnote for (b). + +--- + +## 2. Header-only "inline" path (for small-N callers) + +**The gap.** Compiled-library dispatch + function-call overhead costs +the new build ~3–5 % at n ≤ 1 000 for L1 ops (dot, axpy, nrm2) compared +to the old header-only build. Single-call overhead is ~50 ns and below +~1 K elements that's a measurable fraction of total cost. + +**What it would unlock.** Close the small-N regression entirely; +possibly modest wins at medium-N too if the compiler can fuse user code +with our inner loops. + +**Why we didn't do it.** We chose compiled-library by default because +of per-ISA `-march` correctness and stable C ABI (see project notes). +A *companion* `qblas/qblas_inline.h` could keep both modes. + +**Decision needed.** Should we ship an inline header alongside? +- (a) No: keep compiled-only. Simpler. +- (b) Yes: ship `qblas_inline.h` that pulls in the chosen ISA tier + inline. Consumer chooses `-DQBLAS_INLINE` at build. +- (c) BLIS-style: split the C API into `qblas_*` (compiled) and + `qblas_inline_*` (header). + +Recommendation: **(b)**, only if a user actually asks for it. + +--- + +## 3. AVX-512 tier untested on hardware + +**The gap.** Code compiles `qblas_kernels_avx512.c` against `__m512` +SLEEF symbols (`Sleef_fmaq8_u05avx512f`), and the dispatcher *would* pick +it on a Sapphire Rapids / Skylake-X box, but it has never been run there. +There's a strong chance of small bugs in the q=8 width branch. + +**What it would unlock.** q8 vs q4 → 2× quad-FMA throughput per core if +SLEEF's q8 fma is fully 2-way wider than q4. Realistically expect 1.4– +1.7×. + +**Why we didn't do it.** The bench host is an AMD EPYC 7V13 (Zen 3 / +Milan). Confirmed via `/proc/cpuinfo` flags: `avx avx2 fma bmi1 bmi2 +vaes vpclmulqdq` are all present, but **no `avx512*` flag**. Zen 3 +does not implement AVX-512 (added in Zen 4 / EPYC Genoa). So the +binary contains the AVX-512 tier but the runtime dispatcher correctly +selects AVX2 from CPUID leaf 7. We've also added a guard so an +explicit `QBLAS_DISPATCH=avx512` on a non-AVX-512 host emits a stderr +warning and falls back to the auto-detected tier instead of SIGILL. + +**Decision needed.** Run on an AVX-512 host. Two paths: +- Spin up a Sapphire Rapids VM in Azure for one CI run. +- Skip until a user opens the issue. + +Recommendation: add an **AVX-512 host to CI** so the q8 path gets at +least built and tested on every push. + +--- + +## 4. Level-2: qsymv, qtrsv, qger micro-kernels + +**The gap.** qgemv (both N and T) is properly vectorised + threaded. +The other three Level-2 routines still drop through to scalar + +qblas_dispatch_qaxpy / qdot. + +**Per current bench (16 threads, n=1600):** +- qgemv: **431 M items/s** (peak) +- qger: not benched, expected ~250–350 M/s (uses qaxpy per row) +- qsymv: not benched, expected ~200–300 M/s (two passes) +- qtrsv: not benched, sequential (only the trailing axpy is SIMD) + +**What it would unlock.** qger directly enters LU factorisation hot +paths; qsymv is the heart of eigen / Cholesky. ~2× would be reasonable +for proper blocking. + +**Why we didn't do it.** Time. qger and qsymv are easy follow-ups +(panel + gemv pattern). qtrsv is inherently serial along one axis but +can be blocked the same way as qtrsm. + +**Decision needed.** Schedule the work after the bigger ceilings get +cracked. + +--- + +## 5. L1 small-N parallel break-even + +**The gap.** Below n ≈ 1 000 we run single-thread because OpenMP fork +costs ≈ 18 µs and processing 1 000 quad ops at 30 M/s = 33 µs of work — +the parallel speedup is consumed by the fork. + +**What it would unlock.** Not much, frankly. Maybe 1.2–1.5× at n=1 000. +The break-even is fundamental to OpenMP's region cost. + +**Why we didn't do it.** Marginal. + +**Decision needed.** If small-N matters for a specific user, switching +to a persistent thread pool (libomp's `LIBOMP_*` env vars + `OMP_PROC_BIND`) +would help. Currently we don't touch any of that. + +--- + +## 6. NUMA-aware allocation on multi-socket hosts + +**The gap.** `qblas_aligned_alloc` is just `posix_memalign` — no +first-touch policy, no node binding. On a 2-socket EPYC, GEMM panels +allocated in `qgemm()` may live on the wrong NUMA node for some threads, +costing 1.5–2× on the affected accesses. + +**What it would unlock.** ~1.3–1.5× on dual-socket GEMM, maybe more on +multi-thread Level 1 with large arrays. + +**Why we didn't do it.** Implementation cost vs uncertainty about how +much this actually helps on the bench machine. + +**Bench machine NUMA topology (verified):** EPYC 7V13 is a 2-socket box +with 4 NUMA nodes: +- node 0: cores 0-23 +- node 1: cores 24-47 +- node 2: cores 48-71 +- node 3: cores 72-95 +- L3 cache 32 MB per CCX × 12 CCXes = 384 MB total +With `OMP_PROC_BIND=close` we keep threads on the launch node, but +panel buffers allocated inside the parallel region first-touch on the +calling thread's node, which can be wrong for cross-node fetchers. +NUMA-aware allocation (libnuma + `numa_alloc_onnode`, or pre-allocated +per-thread arenas) is the right fix. + +**Decision needed.** Implement when the issue first surfaces on a real +2P box. `numa_alloc_onnode` + libnuma dependency. + +--- + +## 7. Persistent packed-buffer cache for GEMM + +**The gap.** Every `cblas_qgemm` call inside `cblas_qtrsm` etc. allocates +new `Ap` and `Bp` buffers inside its parallel region. For a trsm that +calls gemm O(m / NB) times, that's m/NB allocations × n_threads. + +**What it would unlock.** ~5–10 % on routines that loop on gemm +(trsm/trmm specifically). Negligible for one-shot gemm callers. + +**Why we didn't do it.** Adds lifecycle complexity (per-thread arena +needs a clean-up hook). + +**Decision needed.** Yes for the next perf push. + +--- + +## How to evaluate the impact of any of these + +Repro path for any change: + +```bash +cd build +ctest # correctness MUST stay green +OMP_NUM_THREADS=16 ./bench/qblas_bench_compare \ + --benchmark_min_time=0.3s --benchmark_repetitions=3 \ + --benchmark_report_aggregates_only=true +# Compare against perf_comparison_with_old.md numbers. +``` + +Single change at a time. Keep the bench harness untouched. diff --git a/scripts/bootstrap_sleef.sh b/scripts/bootstrap_sleef.sh new file mode 100755 index 0000000..7226b96 --- /dev/null +++ b/scripts/bootstrap_sleef.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Build SLEEF with all SIMD paths enabled and install into .sleef-prefix/. +# SLEEF's default build only emits host-CPU paths, which would crash on +# older CPUs at runtime once qblas dispatches to a lower tier. +set -euo pipefail + +repo=$(cd "$(dirname "$0")/.." && pwd) +sleef_src="${repo}/third_party/sleef" +sleef_build="${sleef_src}/build-local" +prefix="${repo}/.sleef-prefix" + +if [[ ! -f "${sleef_src}/CMakeLists.txt" ]]; then + echo "[bootstrap] initialising sleef submodule..." + (cd "${repo}" && git submodule update --init --recursive third_party/sleef) +fi + +mkdir -p "${sleef_build}" +cmake -S "${sleef_src}" -B "${sleef_build}" \ + -DSLEEF_BUILD_QUAD=ON \ + -DSLEEF_BUILD_SHARED_LIBS=ON \ + -DSLEEF_ENABLE_SSE2=ON \ + -DSLEEF_ENABLE_SSE4=ON \ + -DSLEEF_ENABLE_AVX=ON \ + -DSLEEF_ENABLE_AVX2=ON \ + -DSLEEF_ENABLE_AVX512F=ON \ + -DSLEEF_DISABLE_NATIVE_KEYWORD=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${prefix}" + +if command -v nproc >/dev/null 2>&1; then + NJOBS=$(nproc) +elif [[ "$(uname)" == "Darwin" ]]; then + NJOBS=$(sysctl -n hw.ncpu) +else + NJOBS=4 +fi +cmake --build "${sleef_build}" --parallel "${NJOBS}" + +rm -rf "${prefix}" +cmake --install "${sleef_build}" + +# libtlfloat is built by SLEEF but not installed; copy it in so rpath +# resolution finds it. +tl_dir="${sleef_build}/ext_tlfloat-prefix/src/ext_tlfloat-build/lib" +if [[ -d "${tl_dir}" ]]; then + if [[ "$(uname)" == "Darwin" ]]; then + cp -a "${tl_dir}"/libtlfloat.*.dylib "${prefix}/lib/" 2>/dev/null || true + cp -a "${tl_dir}"/libtlfloat.dylib "${prefix}/lib/" 2>/dev/null || true + else + cp -a "${tl_dir}"/libtlfloat.so* "${prefix}/lib/" + fi +fi + +echo "[bootstrap] SLEEF installed at ${prefix}" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..f527d48 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,98 @@ +set(QBLAS_CORE_SOURCES + cpu/qblas_cpu.c + level1/qblas_level1.c + level2/qblas_level2.c + level3/qblas_gemm.c + level3/qblas_level3_extra.c) + +if(QBLAS_BUILD_SHARED) + add_library(qblas SHARED ${QBLAS_CORE_SOURCES}) +else() + add_library(qblas STATIC ${QBLAS_CORE_SOURCES}) +endif() + +target_compile_definitions(qblas PRIVATE QBLAS_BUILDING) +target_include_directories(qblas + PUBLIC + $ + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}) + +target_link_libraries(qblas PUBLIC qblas::sleef) +if(QBLAS_USE_OPENMP) + target_link_libraries(qblas PUBLIC OpenMP::OpenMP_C) +endif() + +set_target_properties(qblas PROPERTIES + C_STANDARD 11 + C_STANDARD_REQUIRED ON + VERSION ${PROJECT_VERSION} + SOVERSION ${PROJECT_VERSION_MAJOR}) + +if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(qblas PRIVATE + -Wall -Wextra -Wno-unused-parameter + $<$:-O3> + $<$:-O3 -g> + $<$:-O0 -g>) + if(QBLAS_NATIVE) + target_compile_options(qblas PRIVATE -march=native) + endif() +endif() + +# Per-ISA OBJECT libraries built with their own -march flags and linked +# into qblas. qblas_cpu.c selects which one's register_* gets called. +function(qblas_add_tier name width flags has_define) + set(target qblas_kernels_${name}) + add_library(${target} OBJECT ${CMAKE_CURRENT_SOURCE_DIR}/kernels/${name}/kernels_${name}.c) + target_include_directories(${target} PRIVATE + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(${target} PRIVATE qblas::sleef) + target_compile_definitions(${target} PRIVATE + QBLAS_BUILDING + ${has_define}=1) + if(flags) + target_compile_options(${target} PRIVATE ${flags}) + endif() + set_target_properties(${target} PROPERTIES + POSITION_INDEPENDENT_CODE ON + C_STANDARD 11) +endfunction() + +qblas_add_tier(generic 1 "" QBLAS_HAS_GENERIC) + +if(QBLAS_ARCH STREQUAL "x86_64") + qblas_add_tier(sse2 2 "${QBLAS_SSE2_FLAGS}" QBLAS_HAS_SSE2) + qblas_add_tier(avx2 4 "${QBLAS_AVX2_FLAGS}" QBLAS_HAS_AVX2) + qblas_add_tier(avx512 8 "${QBLAS_AVX512_FLAGS}" QBLAS_HAS_AVX512) + + target_link_libraries(qblas PRIVATE + qblas_kernels_generic + qblas_kernels_sse2 + qblas_kernels_avx2 + qblas_kernels_avx512) + target_compile_definitions(qblas PRIVATE + QBLAS_HAS_GENERIC=1 + QBLAS_HAS_SSE2=1 + QBLAS_HAS_AVX2=1 + QBLAS_HAS_AVX512=1) +elseif(QBLAS_ARCH STREQUAL "aarch64") + qblas_add_tier(neon 2 "${QBLAS_NEON_FLAGS}" QBLAS_HAS_NEON) + target_link_libraries(qblas PRIVATE qblas_kernels_generic qblas_kernels_neon) + target_compile_definitions(qblas PRIVATE + QBLAS_HAS_GENERIC=1 + QBLAS_HAS_NEON=1) +else() + target_link_libraries(qblas PRIVATE qblas_kernels_generic) + target_compile_definitions(qblas PRIVATE QBLAS_HAS_GENERIC=1) +endif() + +add_library(qblas::qblas ALIAS qblas) + +include(GNUInstallDirs) +install(TARGETS qblas + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) +install(DIRECTORY ${CMAKE_SOURCE_DIR}/include/qblas + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) diff --git a/src/common/qblas_dispatch.h b/src/common/qblas_dispatch.h new file mode 100644 index 0000000..321e336 --- /dev/null +++ b/src/common/qblas_dispatch.h @@ -0,0 +1,61 @@ +/* Function pointers populated once at library init by qblas_dispatch_init(), + * then read lock-free on the hot path. */ +#ifndef QBLAS_DISPATCH_H +#define QBLAS_DISPATCH_H + +#include "qblas_internal.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef Sleef_quad (*qdot_fn)(size_t n, const Sleef_quad *x, ptrdiff_t incx, + const Sleef_quad *y, ptrdiff_t incy); +typedef void (*qaxpy_fn)(size_t n, Sleef_quad alpha, + const Sleef_quad *x, ptrdiff_t incx, + Sleef_quad *y, ptrdiff_t incy); +typedef void (*qscal_fn)(size_t n, Sleef_quad alpha, + Sleef_quad *x, ptrdiff_t incx); +typedef Sleef_quad (*qasum_fn)(size_t n, const Sleef_quad *x, ptrdiff_t incx); +typedef size_t (*qiamax_fn)(size_t n, const Sleef_quad *x, ptrdiff_t incx); + +extern qdot_fn qblas_dispatch_qdot; +extern qaxpy_fn qblas_dispatch_qaxpy; +extern qscal_fn qblas_dispatch_qscal; +extern qasum_fn qblas_dispatch_qasum; +extern qiamax_fn qblas_dispatch_qiamax; + +typedef void (*qgemv_n_fn)(size_t m, size_t k, + Sleef_quad alpha, + const Sleef_quad *A, size_t lda, + const Sleef_quad *x, ptrdiff_t incx, + Sleef_quad beta, + Sleef_quad *y, ptrdiff_t incy); +typedef void (*qgemv_t_fn)(size_t m, size_t k, + Sleef_quad alpha, + const Sleef_quad *A, size_t lda, + const Sleef_quad *x, ptrdiff_t incx, + Sleef_quad beta, + Sleef_quad *y, ptrdiff_t incy); + +extern qgemv_n_fn qblas_dispatch_qgemv_n; +extern qgemv_t_fn qblas_dispatch_qgemv_t; + +/* Caller pre-scales C by beta; kernel accumulates alpha * Ap * Bp into C. */ +typedef void (*qgemm_kernel_fn)(size_t kc, + Sleef_quad alpha, + const Sleef_quad *A_packed, + const Sleef_quad *B_packed, + Sleef_quad *C, size_t ldc); + +extern qgemm_kernel_fn qblas_dispatch_qgemm_kernel; +extern size_t qblas_dispatch_qgemm_MR; +extern size_t qblas_dispatch_qgemm_NR; + +void qblas_dispatch_init(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/common/qblas_internal.h b/src/common/qblas_internal.h new file mode 100644 index 0000000..7060fc2 --- /dev/null +++ b/src/common/qblas_internal.h @@ -0,0 +1,92 @@ +#ifndef QBLAS_INTERNAL_H +#define QBLAS_INTERNAL_H + +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define QBLAS_ZERO Sleef_cast_from_doubleq1(0.0) +#define QBLAS_ONE Sleef_cast_from_doubleq1(1.0) +#define QBLAS_NEGONE Sleef_cast_from_doubleq1(-1.0) + +static inline Sleef_quad qadd(Sleef_quad a, Sleef_quad b) { return Sleef_addq1_u05(a, b); } +static inline Sleef_quad qsub(Sleef_quad a, Sleef_quad b) { return Sleef_subq1_u05(a, b); } +static inline Sleef_quad qmul(Sleef_quad a, Sleef_quad b) { return Sleef_mulq1_u05(a, b); } +static inline Sleef_quad qdiv(Sleef_quad a, Sleef_quad b) { return Sleef_divq1_u05(a, b); } +static inline Sleef_quad qfma(Sleef_quad a, Sleef_quad b, Sleef_quad c) { + return Sleef_fmaq1_u05(a, b, c); +} +static inline Sleef_quad qneg(Sleef_quad a) { return Sleef_negq1(a); } +static inline Sleef_quad qfabs(Sleef_quad a) { return Sleef_fabsq1(a); } +static inline Sleef_quad qsqrt(Sleef_quad a) { return Sleef_sqrtq1_u05(a); } + +static inline bool qiszero(Sleef_quad a) { return Sleef_icmpeqq1(a, QBLAS_ZERO) != 0; } +static inline bool qisone(Sleef_quad a) { return Sleef_icmpeqq1(a, QBLAS_ONE) != 0; } + +/* For incx<0 the vector starts at the high end and walks back. */ +static inline ptrdiff_t qblas_stride_offset(int incx, int n) { + return (incx < 0) ? (ptrdiff_t)(n - 1) * (ptrdiff_t)(-incx) : 0; +} + +#define QBLAS_ALIGN 64 + +void *qblas_aligned_alloc(size_t bytes); +void qblas_aligned_free(void *p); + +typedef enum { + QBLAS_TIER_GENERIC = 0, + QBLAS_TIER_SSE2 = 1, + QBLAS_TIER_NEON = 2, + QBLAS_TIER_AVX2 = 3, + QBLAS_TIER_AVX512 = 4 +} qblas_cpu_tier_t; + +QBLAS_API qblas_cpu_tier_t qblas_cpu_tier(void); + +int qblas_resolve_threads(size_t work_units, size_t per_unit_cost); + +/* Runtime tunables populated once by qblas_dispatch_init() from CPUID / + * sysconf / a timed empty OpenMP region. */ +typedef struct { + size_t l1_data; + size_t l2; + size_t l3; + int cores; + size_t l1_thread_threshold; /* elements */ + size_t gemv_thread_threshold; /* m*n elements */ + size_t gemm_mc; + size_t gemm_kc; + size_t gemm_nc; + size_t omp_overhead_cycles; +} qblas_tune_t; + +QBLAS_API const qblas_tune_t *qblas_tune(void); + +#define QBLAS_PARALLEL_THRESHOLD_L1_DEFAULT 16384 +#define QBLAS_PARALLEL_THRESHOLD_GEMV_DEFAULT 16384 +#define QBLAS_PARALLEL_THRESHOLD_GEMM 64 + +#if defined(__GNUC__) || defined(__clang__) +# define QBLAS_LIKELY(x) __builtin_expect(!!(x), 1) +# define QBLAS_UNLIKELY(x) __builtin_expect(!!(x), 0) +# define QBLAS_INLINE static inline __attribute__((always_inline)) +# define QBLAS_NOINLINE __attribute__((noinline)) +#else +# define QBLAS_LIKELY(x) (x) +# define QBLAS_UNLIKELY(x) (x) +# define QBLAS_INLINE static inline +# define QBLAS_NOINLINE +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/cpu/qblas_cpu.c b/src/cpu/qblas_cpu.c new file mode 100644 index 0000000..66380a2 --- /dev/null +++ b/src/cpu/qblas_cpu.c @@ -0,0 +1,399 @@ +#include "common/qblas_internal.h" +#include "common/qblas_dispatch.h" + +#ifndef _POSIX_C_SOURCE +# define _POSIX_C_SOURCE 200809L +#endif +/* macOS hides _SC_NPROCESSORS_ONLN behind _DARWIN_C_SOURCE when + * _POSIX_C_SOURCE is defined (it is a BSD extension, not in POSIX). */ +#ifdef __APPLE__ +# ifndef _DARWIN_C_SOURCE +# define _DARWIN_C_SOURCE 1 +# endif +#endif + +#include +#include +#include +#include +#include + +#if defined(__x86_64__) || defined(_M_X64) +# include +# define QBLAS_X86_64 1 +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +# define QBLAS_AARCH64 1 +#endif + +#ifdef _OPENMP +# include +#endif + +static qblas_cpu_tier_t g_tier = QBLAS_TIER_GENERIC; +static int g_initialized = 0; +static qblas_tune_t g_tune = { + .l1_data = 32 * 1024, + .l2 = 256 * 1024, + .l3 = 8 * 1024 * 1024, + .cores = 1, + .l1_thread_threshold = QBLAS_PARALLEL_THRESHOLD_L1_DEFAULT, + .gemv_thread_threshold = QBLAS_PARALLEL_THRESHOLD_GEMV_DEFAULT, + .gemm_mc = 128, + .gemm_kc = 128, + .gemm_nc = 512, + .omp_overhead_cycles = 8192, +}; +const qblas_tune_t *qblas_tune(void) { return &g_tune; } + +#ifdef QBLAS_X86_64 +static int has_xgetbv_avx_state(void) { + unsigned int eax, edx; + __asm__ __volatile__("xgetbv" : "=a"(eax), "=d"(edx) : "c"(0)); + return (eax & 0x6) == 0x6; /* XMM | YMM */ +} +static int has_xgetbv_avx512_state(void) { + unsigned int eax, edx; + __asm__ __volatile__("xgetbv" : "=a"(eax), "=d"(edx) : "c"(0)); + return (eax & 0xE6) == 0xE6; /* + opmask | ZMM_Hi256 | Hi16_ZMM */ +} +#endif + +static qblas_cpu_tier_t detect_tier(void) { +#ifdef QBLAS_X86_64 + unsigned int eax, ebx, ecx, edx; + int has_sse2 = 0, has_avx = 0, has_avx2 = 0, has_fma = 0; + int has_avx512f = 0, has_avx512dq = 0; + int has_osxsave = 0; + + if (__get_cpuid(1, &eax, &ebx, &ecx, &edx)) { + has_sse2 = (edx & (1u << 26)) != 0; + has_fma = (ecx & (1u << 12)) != 0; + has_osxsave = (ecx & (1u << 27)) != 0; + has_avx = (ecx & (1u << 28)) != 0; + } + if (__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx)) { + has_avx2 = (ebx & (1u << 5)) != 0; + has_avx512f = (ebx & (1u << 16)) != 0; + has_avx512dq = (ebx & (1u << 17)) != 0; + } + + int avx_ok = has_osxsave && has_avx && has_xgetbv_avx_state(); + int avx512_ok = avx_ok && has_xgetbv_avx512_state(); + +#ifdef QBLAS_HAS_AVX512 + if (avx512_ok && has_avx512f && has_avx512dq) return QBLAS_TIER_AVX512; +#endif +#ifdef QBLAS_HAS_AVX2 + if (avx_ok && has_avx2 && has_fma) return QBLAS_TIER_AVX2; +#endif +#ifdef QBLAS_HAS_SSE2 + if (has_sse2) return QBLAS_TIER_SSE2; +#endif + (void)avx_ok; + return QBLAS_TIER_GENERIC; +#elif defined(QBLAS_AARCH64) +# ifdef QBLAS_HAS_NEON + return QBLAS_TIER_NEON; +# else + return QBLAS_TIER_GENERIC; +# endif +#else + return QBLAS_TIER_GENERIC; +#endif +} + +static void detect_caches(qblas_tune_t *t) { +#ifdef QBLAS_X86_64 + for (unsigned i = 0; i < 8; ++i) { + unsigned int a, b, c, d; + if (!__get_cpuid_count(4, i, &a, &b, &c, &d)) break; + unsigned type = a & 0x1F; + if (type == 0) break; + unsigned level = (a >> 5) & 0x7; + /* cache size = (ways+1) * (partitions+1) * (line_size+1) * (sets+1) */ + unsigned ways = ((b >> 22) & 0x3FF) + 1; + unsigned partitions = ((b >> 12) & 0x3FF) + 1; + unsigned line_size = (b & 0xFFF) + 1; + unsigned sets = c + 1; + size_t cache_bytes = (size_t)ways * partitions * line_size * sets; + if (level == 1 && type != 2) t->l1_data = cache_bytes; + else if (level == 2 && type != 2) t->l2 = cache_bytes; + else if (level == 3 && type != 2) t->l3 = cache_bytes; + } +#else +# ifdef _SC_LEVEL1_DCACHE_SIZE + long v = sysconf(_SC_LEVEL1_DCACHE_SIZE); if (v > 0) t->l1_data = (size_t)v; +# endif +# ifdef _SC_LEVEL2_CACHE_SIZE + v = sysconf(_SC_LEVEL2_CACHE_SIZE); if (v > 0) t->l2 = (size_t)v; +# endif +# ifdef _SC_LEVEL3_CACHE_SIZE + v = sysconf(_SC_LEVEL3_CACHE_SIZE); if (v > 0) t->l3 = (size_t)v; +# endif +#endif +#ifdef _SC_NPROCESSORS_ONLN + long ncpu = sysconf(_SC_NPROCESSORS_ONLN); + t->cores = (ncpu > 0) ? (int)ncpu : 1; +#else + t->cores = 1; +#endif +} + +#ifdef _OPENMP +static inline uint64_t ts_ns(void) { + struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec; +} +#endif + +/* Measures the cost of an empty parallel region so qblas_resolve_threads + * can decide if a piece of work justifies forking. */ +static size_t measure_omp_overhead_cycles(int max_threads) { +#ifdef _OPENMP + if (max_threads <= 1) return 1; + int warm = 0; + for (int i = 0; i < 4; ++i) { + #pragma omp parallel num_threads(max_threads) reduction(+:warm) + warm += 1; + } + (void)warm; + const int iters = 256; + uint64_t t0 = ts_ns(); + int sink = 0; + for (int i = 0; i < iters; ++i) { + #pragma omp parallel num_threads(max_threads) reduction(+:sink) + sink += 1; + } + uint64_t t1 = ts_ns(); + (void)sink; + double ns_per_spawn = (double)(t1 - t0) / iters; + size_t cycles = (size_t)(ns_per_spawn * 3.0); /* approx at 3 GHz */ + if (cycles < 1024) cycles = 1024; + if (cycles > 1u << 20) cycles = 1u << 20; + return cycles; +#else + (void)max_threads; + return 1; +#endif +} + +static void derive_tunables(qblas_tune_t *t) { + size_t l1_thr = t->omp_overhead_cycles / 16; + if (l1_thr < 1024) l1_thr = 1024; + if (l1_thr > 16384) l1_thr = 16384; + t->l1_thread_threshold = l1_thr; + t->gemv_thread_threshold = l1_thr * 2; + + /* GEMM blocking sized to fit the live micro-kernel set in L1, A panel + * in ~80% of L2, B panel in L3. quad = 16 bytes per element. */ + const size_t MR = 4, NR = 4, quad_bytes = 16; + size_t kc = (t->l1_data / 2 - MR*NR*quad_bytes) / ((MR + NR) * quad_bytes); + if (kc > 256) kc = 256; + if (kc < 32) kc = 32; + kc &= ~(size_t)7; + t->gemm_kc = kc; + + size_t mc = (t->l2 * 4) / (5 * kc * quad_bytes); + if (mc > 512) mc = 512; + if (mc < MR) mc = MR; + mc -= mc % MR; + t->gemm_mc = mc; + + size_t nc = t->l3 / (kc * quad_bytes); + if (nc > 4096) nc = 4096; + if (nc < NR) nc = NR; + nc -= nc % NR; + t->gemm_nc = nc; +} + +const char *qblas_get_dispatch_tier(void) { + switch (qblas_cpu_tier()) { + case QBLAS_TIER_AVX512: return "avx512"; + case QBLAS_TIER_AVX2: return "avx2"; + case QBLAS_TIER_SSE2: return "sse2"; + case QBLAS_TIER_NEON: return "neon"; + default: return "generic"; + } +} + +qblas_cpu_tier_t qblas_cpu_tier(void) { + if (!g_initialized) qblas_dispatch_init(); + return g_tier; +} + +qdot_fn qblas_dispatch_qdot = NULL; +qaxpy_fn qblas_dispatch_qaxpy = NULL; +qscal_fn qblas_dispatch_qscal = NULL; +qasum_fn qblas_dispatch_qasum = NULL; +qiamax_fn qblas_dispatch_qiamax = NULL; +qgemv_n_fn qblas_dispatch_qgemv_n = NULL; +qgemv_t_fn qblas_dispatch_qgemv_t = NULL; +qgemm_kernel_fn qblas_dispatch_qgemm_kernel = NULL; +size_t qblas_dispatch_qgemm_MR = 0; +size_t qblas_dispatch_qgemm_NR = 0; + +void qblas_register_generic(void); +#ifdef QBLAS_HAS_SSE2 +void qblas_register_sse2(void); +#endif +#ifdef QBLAS_HAS_AVX2 +void qblas_register_avx2(void); +#endif +#ifdef QBLAS_HAS_AVX512 +void qblas_register_avx512(void); +#endif +#ifdef QBLAS_HAS_NEON +void qblas_register_neon(void); +#endif + +void qblas_dispatch_init(void) { + if (g_initialized) return; + + qblas_register_generic(); + + g_tier = detect_tier(); + + detect_caches(&g_tune); +#ifdef _OPENMP + int max_threads = omp_get_max_threads(); +#else + int max_threads = 1; +#endif + g_tune.omp_overhead_cycles = measure_omp_overhead_cycles(max_threads); + derive_tunables(&g_tune); + +#ifdef QBLAS_HAS_SSE2 + if (g_tier >= QBLAS_TIER_SSE2) qblas_register_sse2(); +#endif +#ifdef QBLAS_HAS_NEON + if (g_tier >= QBLAS_TIER_NEON) qblas_register_neon(); +#endif +#ifdef QBLAS_HAS_AVX2 + if (g_tier >= QBLAS_TIER_AVX2) qblas_register_avx2(); +#endif +#ifdef QBLAS_HAS_AVX512 + if (g_tier >= QBLAS_TIER_AVX512) qblas_register_avx512(); +#endif + + /* QBLAS_DISPATCH=tier_name forces a tier the binary contains *and* the + * CPU supports; otherwise the override is reported and dropped. */ + qblas_cpu_tier_t auto_tier = g_tier; + const char *env = getenv("QBLAS_DISPATCH"); + if (env) { + qblas_cpu_tier_t want = (qblas_cpu_tier_t)(-1); + if (strcmp(env, "generic") == 0) want = QBLAS_TIER_GENERIC; + else if (strcmp(env, "sse2") == 0) want = QBLAS_TIER_SSE2; + else if (strcmp(env, "neon") == 0) want = QBLAS_TIER_NEON; + else if (strcmp(env, "avx2") == 0) want = QBLAS_TIER_AVX2; + else if (strcmp(env, "avx512") == 0) want = QBLAS_TIER_AVX512; + + if ((int)want >= 0 && (int)want > (int)auto_tier) { + const char *auto_name; + switch (auto_tier) { + case QBLAS_TIER_AVX512: auto_name = "avx512"; break; + case QBLAS_TIER_AVX2: auto_name = "avx2"; break; + case QBLAS_TIER_NEON: auto_name = "neon"; break; + case QBLAS_TIER_SSE2: auto_name = "sse2"; break; + default: auto_name = "generic"; break; + } + fprintf(stderr, + "qblas: QBLAS_DISPATCH=%s is higher than the CPU's detected " + "tier (%s); ignoring.\n", + env, auto_name); + want = (qblas_cpu_tier_t)(-1); + } + + if ((int)want >= 0) { + qblas_register_generic(); + switch (want) { + case QBLAS_TIER_GENERIC: g_tier = QBLAS_TIER_GENERIC; break; +#ifdef QBLAS_HAS_SSE2 + case QBLAS_TIER_SSE2: qblas_register_sse2(); g_tier = QBLAS_TIER_SSE2; break; +#endif +#ifdef QBLAS_HAS_NEON + case QBLAS_TIER_NEON: qblas_register_neon(); g_tier = QBLAS_TIER_NEON; break; +#endif +#ifdef QBLAS_HAS_AVX2 + case QBLAS_TIER_AVX2: qblas_register_avx2(); g_tier = QBLAS_TIER_AVX2; break; +#endif +#ifdef QBLAS_HAS_AVX512 + case QBLAS_TIER_AVX512: qblas_register_avx512(); g_tier = QBLAS_TIER_AVX512; break; +#endif + default: break; + } + } + } + + g_initialized = 1; +} + +__attribute__((constructor)) +static void qblas_lib_init(void) { qblas_dispatch_init(); } + +const char *qblas_get_version(void) { return "QBLAS 1.5.0"; } + +void *qblas_aligned_alloc(size_t bytes) { + size_t rounded = (bytes + QBLAS_ALIGN - 1) & ~(size_t)(QBLAS_ALIGN - 1); + void *p = NULL; + if (posix_memalign(&p, QBLAS_ALIGN, rounded == 0 ? QBLAS_ALIGN : rounded) != 0) return NULL; + return p; +} +void qblas_aligned_free(void *p) { free(p); } + +#ifdef _OPENMP +static int g_user_thread_cap = 0; +#endif + +/* qblas_set_num_threads only caps qblas-internal work distribution; it + * never calls omp_set_num_threads, so the host application's OpenMP + * state is left alone. */ +void qblas_set_num_threads(int n) { +#ifdef _OPENMP + g_user_thread_cap = n > 0 ? n : 0; +#else + (void)n; +#endif +} +int qblas_get_num_threads(void) { +#ifdef _OPENMP + int omp_max = omp_get_max_threads(); + if (g_user_thread_cap > 0 && g_user_thread_cap < omp_max) + return g_user_thread_cap; + return omp_max; +#else + return 1; +#endif +} +int qblas_get_max_threads(void) { +#ifdef _OPENMP + return omp_get_max_threads(); +#else + return 1; +#endif +} + +/* Returns a thread count where each thread has at least one omp_overhead + * worth of work; small problems and big machines naturally pick small + * thread counts instead of fanning out and paying for fork/join. */ +int qblas_resolve_threads(size_t work_units, size_t per_unit_cost) { +#ifdef _OPENMP + int max = omp_get_max_threads(); + if (max <= 1) return 1; + if (per_unit_cost == 0) per_unit_cost = 1; + const size_t quad_cycles_per_op = 32; + size_t overhead = g_tune.omp_overhead_cycles; + if (overhead < 1024) overhead = 1024; + size_t total_cycles = work_units * per_unit_cost * quad_cycles_per_op; + if (total_cycles < overhead * 2) return 1; + size_t t = total_cycles / overhead; + if ((int)t > max) t = (size_t)max; + if (g_user_thread_cap && (int)t > g_user_thread_cap) t = (size_t)g_user_thread_cap; + if (t < 1) t = 1; + return (int)t; +#else + (void)work_units; (void)per_unit_cost; + return 1; +#endif +} diff --git a/src/kernels/avx2/kernels_avx2.c b/src/kernels/avx2/kernels_avx2.c new file mode 100644 index 0000000..9b6fe47 --- /dev/null +++ b/src/kernels/avx2/kernels_avx2.c @@ -0,0 +1,6 @@ +#define QBLAS_HAS_AVX2 1 +#define QV_WIDTH 4 +#define QV_SUFFIX avx2 +#include "../kernels_template.h" + +void qblas_register_avx2(void) { qblas_register_kernels_avx2(); } diff --git a/src/kernels/avx512/kernels_avx512.c b/src/kernels/avx512/kernels_avx512.c new file mode 100644 index 0000000..1269ad8 --- /dev/null +++ b/src/kernels/avx512/kernels_avx512.c @@ -0,0 +1,6 @@ +#define QBLAS_HAS_AVX512 1 +#define QV_WIDTH 8 +#define QV_SUFFIX avx512 +#include "../kernels_template.h" + +void qblas_register_avx512(void) { qblas_register_kernels_avx512(); } diff --git a/src/kernels/generic/kernels_generic.c b/src/kernels/generic/kernels_generic.c new file mode 100644 index 0000000..945dba3 --- /dev/null +++ b/src/kernels/generic/kernels_generic.c @@ -0,0 +1,5 @@ +#define QV_WIDTH 1 +#define QV_SUFFIX generic +#include "../kernels_template.h" + +void qblas_register_generic(void) { qblas_register_kernels_generic(); } diff --git a/src/kernels/kernels_template.h b/src/kernels/kernels_template.h new file mode 100644 index 0000000..5e5bf62 --- /dev/null +++ b/src/kernels/kernels_template.h @@ -0,0 +1,449 @@ +/* Kernel template instantiated once per ISA tier. The including TU sets: + * QV_WIDTH 1, 2, 4, or 8 vector width in quads + * QV_SUFFIX suffix for symbol names (generic, sse2, avx2, ...) + * QV_ISA_SUFFIX SLEEF ISA suffix for width-2 (sse2 or advsimd) */ + +#ifndef QV_WIDTH +# error "kernels_template.h: define QV_WIDTH (1|2|4|8) and QV_SUFFIX first" +#endif +#ifndef QV_SUFFIX +# error "kernels_template.h: define QV_SUFFIX before include" +#endif + +#include "common/qblas_internal.h" +#include "common/qblas_dispatch.h" + +#include +#include + +#define QV_PASTE_(a, b) a##_##b +#define QV_PASTE(a, b) QV_PASTE_(a, b) +#define QV_FN(name) QV_PASTE(QV_PASTE(qblas, name), QV_SUFFIX) + +#if QV_WIDTH == 1 + typedef Sleef_quad qv_t; +# define QV_ADD(a,b) Sleef_addq1_u05((a),(b)) +# define QV_FMA(a,b,c) Sleef_fmaq1_u05((a),(b),(c)) +# define QV_MUL(a,b) Sleef_mulq1_u05((a),(b)) +# define QV_SPLAT(s) (s) +# define QV_LOADU(p) (*(p)) +# define QV_STOREU(p,v) (*(p) = (v)) +# define QV_FABS(a) Sleef_fabsq1((a)) +# define QV_LANE(v, i) (v) +#elif QV_WIDTH == 2 +# ifndef QV_ISA_SUFFIX +# error "width-2 kernels must define QV_ISA_SUFFIX (sse2 or advsimd)" +# endif +# define QV_NAME_(op,ulp,isa) Sleef_ ## op ## q2 ## ulp ## isa +# define QV_NAME(op,ulp,isa) QV_NAME_(op,ulp,isa) + typedef Sleef_quadx2 qv_t; +# define QV_ADD(a,b) QV_NAME(add, _u05, QV_ISA_SUFFIX)((a),(b)) +# define QV_FMA(a,b,c) QV_NAME(fma, _u05, QV_ISA_SUFFIX)((a),(b),(c)) +# define QV_MUL(a,b) QV_NAME(mul, _u05, QV_ISA_SUFFIX)((a),(b)) +# define QV_SPLAT(s) QV_NAME(splat, _, QV_ISA_SUFFIX)((s)) +# define QV_LOADU(p) QV_NAME(load, _, QV_ISA_SUFFIX)((Sleef_quad *)(p)) +# define QV_STOREU(p,v) QV_NAME(store, _, QV_ISA_SUFFIX)((Sleef_quad *)(p), (v)) +# define QV_FABS(a) QV_NAME(fabs, _, QV_ISA_SUFFIX)((a)) +# define QV_LANE(v, i) QV_NAME(get, _, QV_ISA_SUFFIX)((v), (i)) +#elif QV_WIDTH == 4 + typedef Sleef_quadx4 qv_t; +# define QV_ADD(a,b) Sleef_addq4_u05avx2((a),(b)) +# define QV_FMA(a,b,c) Sleef_fmaq4_u05avx2((a),(b),(c)) +# define QV_MUL(a,b) Sleef_mulq4_u05avx2((a),(b)) +# define QV_SPLAT(s) Sleef_splatq4_avx2((s)) +# define QV_LOADU(p) Sleef_loadq4_avx2((Sleef_quad *)(p)) +# define QV_STOREU(p,v) Sleef_storeq4_avx2((Sleef_quad *)(p), (v)) +# define QV_FABS(a) Sleef_fabsq4_avx2((a)) +# define QV_LANE(v, i) Sleef_getq4_avx2((v), (i)) +#elif QV_WIDTH == 8 + typedef Sleef_quadx8 qv_t; +# define QV_ADD(a,b) Sleef_addq8_u05avx512f((a),(b)) +# define QV_FMA(a,b,c) Sleef_fmaq8_u05avx512f((a),(b),(c)) +# define QV_MUL(a,b) Sleef_mulq8_u05avx512f((a),(b)) +# define QV_SPLAT(s) Sleef_splatq8_avx512f((s)) +# define QV_LOADU(p) Sleef_loadq8_avx512f((Sleef_quad *)(p)) +# define QV_STOREU(p,v) Sleef_storeq8_avx512f((Sleef_quad *)(p), (v)) +# define QV_FABS(a) Sleef_fabsq8_avx512f((a)) +# define QV_LANE(v, i) Sleef_getq8_avx512f((v), (i)) +#else +# error "QV_WIDTH must be 1, 2, 4, or 8" +#endif + +/* dot */ +static Sleef_quad QV_FN(qdot)(size_t n, + const Sleef_quad *x, ptrdiff_t incx, + const Sleef_quad *y, ptrdiff_t incy) { + Sleef_quad acc = QBLAS_ZERO; + + if (incx == 1 && incy == 1) { +#if QV_WIDTH > 1 + /* Four independent accumulators to overlap quad FMA latency. */ + const size_t W = (size_t)QV_WIDTH; + const size_t UNROLL = 4; + const size_t step = W * UNROLL; + qv_t a0 = QV_SPLAT(QBLAS_ZERO); + qv_t a1 = QV_SPLAT(QBLAS_ZERO); + qv_t a2 = QV_SPLAT(QBLAS_ZERO); + qv_t a3 = QV_SPLAT(QBLAS_ZERO); + + size_t i = 0; + for (; i + step <= n; i += step) { + qv_t xv0 = QV_LOADU(x + i + 0 * W); + qv_t yv0 = QV_LOADU(y + i + 0 * W); + qv_t xv1 = QV_LOADU(x + i + 1 * W); + qv_t yv1 = QV_LOADU(y + i + 1 * W); + qv_t xv2 = QV_LOADU(x + i + 2 * W); + qv_t yv2 = QV_LOADU(y + i + 2 * W); + qv_t xv3 = QV_LOADU(x + i + 3 * W); + qv_t yv3 = QV_LOADU(y + i + 3 * W); + a0 = QV_FMA(xv0, yv0, a0); + a1 = QV_FMA(xv1, yv1, a1); + a2 = QV_FMA(xv2, yv2, a2); + a3 = QV_FMA(xv3, yv3, a3); + } + for (; i + W <= n; i += W) { + qv_t xv = QV_LOADU(x + i); + qv_t yv = QV_LOADU(y + i); + a0 = QV_FMA(xv, yv, a0); + } + a0 = QV_ADD(a0, a2); + a1 = QV_ADD(a1, a3); + a0 = QV_ADD(a0, a1); + for (size_t lane = 0; lane < W; ++lane) + acc = qadd(acc, QV_LANE(a0, (int)lane)); + for (; i < n; ++i) + acc = qfma(x[i], y[i], acc); +#else + for (size_t i = 0; i < n; ++i) + acc = qfma(x[i], y[i], acc); +#endif + } else { + /* Caller pre-shifts x/y to the logical first element so signed + * strides can start at zero here. */ + ptrdiff_t ix = 0, iy = 0; + for (size_t i = 0; i < n; ++i) { + acc = qfma(x[ix], y[iy], acc); + ix += incx; + iy += incy; + } + } + return acc; +} + +/* axpy */ +static void QV_FN(qaxpy)(size_t n, Sleef_quad alpha, + const Sleef_quad *x, ptrdiff_t incx, + Sleef_quad *y, ptrdiff_t incy) { + if (qiszero(alpha)) return; + + if (incx == 1 && incy == 1) { +#if QV_WIDTH > 1 + const size_t W = (size_t)QV_WIDTH; + const qv_t va = QV_SPLAT(alpha); + const size_t UNROLL = 4; + const size_t step = W * UNROLL; + size_t i = 0; + for (; i + step <= n; i += step) { + qv_t x0 = QV_LOADU(x + i + 0 * W); + qv_t x1 = QV_LOADU(x + i + 1 * W); + qv_t x2 = QV_LOADU(x + i + 2 * W); + qv_t x3 = QV_LOADU(x + i + 3 * W); + qv_t y0 = QV_LOADU(y + i + 0 * W); + qv_t y1 = QV_LOADU(y + i + 1 * W); + qv_t y2 = QV_LOADU(y + i + 2 * W); + qv_t y3 = QV_LOADU(y + i + 3 * W); + y0 = QV_FMA(va, x0, y0); + y1 = QV_FMA(va, x1, y1); + y2 = QV_FMA(va, x2, y2); + y3 = QV_FMA(va, x3, y3); + QV_STOREU(y + i + 0 * W, y0); + QV_STOREU(y + i + 1 * W, y1); + QV_STOREU(y + i + 2 * W, y2); + QV_STOREU(y + i + 3 * W, y3); + } + for (; i + W <= n; i += W) { + qv_t xv = QV_LOADU(x + i); + qv_t yv = QV_LOADU(y + i); + yv = QV_FMA(va, xv, yv); + QV_STOREU(y + i, yv); + } + for (; i < n; ++i) + y[i] = qfma(alpha, x[i], y[i]); +#else + for (size_t i = 0; i < n; ++i) + y[i] = qfma(alpha, x[i], y[i]); +#endif + } else { + ptrdiff_t ix = 0, iy = 0; + for (size_t i = 0; i < n; ++i) { + y[iy] = qfma(alpha, x[ix], y[iy]); + ix += incx; + iy += incy; + } + } +} + +/* scal */ +static void QV_FN(qscal)(size_t n, Sleef_quad alpha, Sleef_quad *x, ptrdiff_t incx) { + if (qisone(alpha)) return; + if (qiszero(alpha)) { + if (incx == 1) { + for (size_t i = 0; i < n; ++i) x[i] = QBLAS_ZERO; + } else { + ptrdiff_t ix = 0; + for (size_t i = 0; i < n; ++i) { x[ix] = QBLAS_ZERO; ix += incx; } + } + return; + } + + if (incx == 1) { +#if QV_WIDTH > 1 + const size_t W = (size_t)QV_WIDTH; + const qv_t va = QV_SPLAT(alpha); + size_t i = 0; + const size_t UNROLL = 4; + const size_t step = W * UNROLL; + for (; i + step <= n; i += step) { + qv_t x0 = QV_LOADU(x + i + 0 * W); + qv_t x1 = QV_LOADU(x + i + 1 * W); + qv_t x2 = QV_LOADU(x + i + 2 * W); + qv_t x3 = QV_LOADU(x + i + 3 * W); + QV_STOREU(x + i + 0 * W, QV_MUL(va, x0)); + QV_STOREU(x + i + 1 * W, QV_MUL(va, x1)); + QV_STOREU(x + i + 2 * W, QV_MUL(va, x2)); + QV_STOREU(x + i + 3 * W, QV_MUL(va, x3)); + } + for (; i + W <= n; i += W) { + qv_t xv = QV_LOADU(x + i); + QV_STOREU(x + i, QV_MUL(va, xv)); + } + for (; i < n; ++i) x[i] = qmul(alpha, x[i]); +#else + for (size_t i = 0; i < n; ++i) x[i] = qmul(alpha, x[i]); +#endif + } else { + ptrdiff_t ix = 0; + for (size_t i = 0; i < n; ++i) { + x[ix] = qmul(alpha, x[ix]); + ix += incx; + } + } +} + +/* asum: sum of |x_i| */ +static Sleef_quad QV_FN(qasum)(size_t n, const Sleef_quad *x, ptrdiff_t incx) { + Sleef_quad acc = QBLAS_ZERO; + if (incx == 1) { +#if QV_WIDTH > 1 + const size_t W = (size_t)QV_WIDTH; + qv_t a0 = QV_SPLAT(QBLAS_ZERO); + qv_t a1 = QV_SPLAT(QBLAS_ZERO); + size_t i = 0; + for (; i + 2 * W <= n; i += 2 * W) { + qv_t v0 = QV_FABS(QV_LOADU(x + i)); + qv_t v1 = QV_FABS(QV_LOADU(x + i + W)); + a0 = QV_ADD(a0, v0); + a1 = QV_ADD(a1, v1); + } + for (; i + W <= n; i += W) { + qv_t v = QV_FABS(QV_LOADU(x + i)); + a0 = QV_ADD(a0, v); + } + a0 = QV_ADD(a0, a1); + for (size_t lane = 0; lane < W; ++lane) + acc = qadd(acc, QV_LANE(a0, (int)lane)); + for (; i < n; ++i) acc = qadd(acc, qfabs(x[i])); +#else + for (size_t i = 0; i < n; ++i) acc = qadd(acc, qfabs(x[i])); +#endif + } else { + ptrdiff_t ix = 0; + for (size_t i = 0; i < n; ++i) { + acc = qadd(acc, qfabs(x[ix])); + ix += incx; + } + } + return acc; +} + +/* iamax: scalar walk; memory-bound at the sizes we ship for. */ +static size_t QV_FN(qiamax)(size_t n, const Sleef_quad *x, ptrdiff_t incx) { + if (n == 0) return 0; + Sleef_quad best = qfabs(x[0]); + size_t arg = 0; + if (incx == 1) { + for (size_t i = 1; i < n; ++i) { + Sleef_quad v = qfabs(x[i]); + if (Sleef_icmpgtq1(v, best)) { best = v; arg = i; } + } + } else { + ptrdiff_t ix = 0; + best = qfabs(x[ix]); arg = 0; + ix += incx; + for (size_t i = 1; i < n; ++i) { + Sleef_quad v = qfabs(x[ix]); + if (Sleef_icmpgtq1(v, best)) { best = v; arg = i; } + ix += incx; + } + } + return arg; +} + +/* gemv with no transpose: each output row is a dot of one A row with x. */ +static void QV_FN(qgemv_n)(size_t m, size_t k, + Sleef_quad alpha, + const Sleef_quad *A, size_t lda, + const Sleef_quad *x, ptrdiff_t incx, + Sleef_quad beta, + Sleef_quad *y, ptrdiff_t incy) { + for (size_t i = 0; i < m; ++i) { + Sleef_quad s = QV_FN(qdot)(k, A + i * lda, 1, x, incx); + if (qiszero(beta)) { + y[(ptrdiff_t)i * incy] = qmul(alpha, s); + } else { + Sleef_quad prev = y[(ptrdiff_t)i * incy]; + y[(ptrdiff_t)i * incy] = qfma(alpha, s, qmul(beta, prev)); + } + } +} + +/* gemv with transpose: scale y by beta, then accumulate alpha*x[i]*A[i,:]. */ +static void QV_FN(qgemv_t)(size_t m, size_t k, + Sleef_quad alpha, + const Sleef_quad *A, size_t lda, + const Sleef_quad *x, ptrdiff_t incx, + Sleef_quad beta, + Sleef_quad *y, ptrdiff_t incy) { + if (qiszero(beta)) { + for (size_t j = 0; j < k; ++j) y[(ptrdiff_t)j * incy] = QBLAS_ZERO; + } else if (!qisone(beta)) { + for (size_t j = 0; j < k; ++j) + y[(ptrdiff_t)j * incy] = qmul(beta, y[(ptrdiff_t)j * incy]); + } + + for (size_t i = 0; i < m; ++i) { + Sleef_quad ax = qmul(alpha, x[(ptrdiff_t)i * incx]); + if (qiszero(ax)) continue; + if (incy == 1) { + QV_FN(qaxpy)(k, ax, A + i * lda, 1, y, 1); + } else { + const Sleef_quad *Arow = A + i * lda; + for (size_t j = 0; j < k; ++j) + y[(ptrdiff_t)j * incy] = qfma(ax, Arow[j], y[(ptrdiff_t)j * incy]); + } + } +} + +/* Packed GEMM micro-kernel: C(MR×NR) += alpha * A_packed(MR×kc) * B_packed(kc×NR). + * A_packed has MR scalars per k step; B_packed has NR scalars per k step. */ +#if QV_WIDTH == 1 +# define QV_MR 4 +# define QV_NR 4 +#elif QV_WIDTH == 2 +# define QV_MR 4 +# define QV_NR 4 +#elif QV_WIDTH == 4 +# define QV_MR 4 +# define QV_NR 4 +#elif QV_WIDTH == 8 +# define QV_MR 4 +# define QV_NR 8 +#endif + +static void QV_FN(qgemm_kernel)(size_t kc, + Sleef_quad alpha, + const Sleef_quad *A_packed, + const Sleef_quad *B_packed, + Sleef_quad *C, size_t ldc) { +#if QV_WIDTH > 1 + enum { W = QV_WIDTH, NR_VEC = QV_NR / QV_WIDTH }; + qv_t c00 = QV_SPLAT(QBLAS_ZERO), c01 = QV_SPLAT(QBLAS_ZERO); + qv_t c10 = QV_SPLAT(QBLAS_ZERO), c11 = QV_SPLAT(QBLAS_ZERO); + qv_t c20 = QV_SPLAT(QBLAS_ZERO), c21 = QV_SPLAT(QBLAS_ZERO); + qv_t c30 = QV_SPLAT(QBLAS_ZERO), c31 = QV_SPLAT(QBLAS_ZERO); +#if QV_MR >= 8 + qv_t c40 = QV_SPLAT(QBLAS_ZERO), c41 = QV_SPLAT(QBLAS_ZERO); + qv_t c50 = QV_SPLAT(QBLAS_ZERO), c51 = QV_SPLAT(QBLAS_ZERO); + qv_t c60 = QV_SPLAT(QBLAS_ZERO), c61 = QV_SPLAT(QBLAS_ZERO); + qv_t c70 = QV_SPLAT(QBLAS_ZERO), c71 = QV_SPLAT(QBLAS_ZERO); + (void)c41; (void)c51; (void)c61; (void)c71; +#endif + (void)c01; (void)c11; (void)c21; (void)c31; + + for (size_t p = 0; p < kc; ++p) { + qv_t b0 = QV_LOADU(B_packed + p * QV_NR + 0 * W); + qv_t va0 = QV_SPLAT(A_packed[p * QV_MR + 0]); + qv_t va1 = QV_SPLAT(A_packed[p * QV_MR + 1]); + qv_t va2 = QV_SPLAT(A_packed[p * QV_MR + 2]); + qv_t va3 = QV_SPLAT(A_packed[p * QV_MR + 3]); + c00 = QV_FMA(va0, b0, c00); + c10 = QV_FMA(va1, b0, c10); + c20 = QV_FMA(va2, b0, c20); + c30 = QV_FMA(va3, b0, c30); +#if QV_MR >= 8 + qv_t va4 = QV_SPLAT(A_packed[p * QV_MR + 4]); + qv_t va5 = QV_SPLAT(A_packed[p * QV_MR + 5]); + qv_t va6 = QV_SPLAT(A_packed[p * QV_MR + 6]); + qv_t va7 = QV_SPLAT(A_packed[p * QV_MR + 7]); + c40 = QV_FMA(va4, b0, c40); + c50 = QV_FMA(va5, b0, c50); + c60 = QV_FMA(va6, b0, c60); + c70 = QV_FMA(va7, b0, c70); +#endif + if (NR_VEC > 1) { + qv_t b1 = QV_LOADU(B_packed + p * QV_NR + 1 * W); + c01 = QV_FMA(va0, b1, c01); + c11 = QV_FMA(va1, b1, c11); + c21 = QV_FMA(va2, b1, c21); + c31 = QV_FMA(va3, b1, c31); + } + } + + qv_t va = QV_SPLAT(alpha); + QV_STOREU(C + 0 * ldc + 0 * W, QV_FMA(va, c00, QV_LOADU(C + 0 * ldc + 0 * W))); + QV_STOREU(C + 1 * ldc + 0 * W, QV_FMA(va, c10, QV_LOADU(C + 1 * ldc + 0 * W))); + QV_STOREU(C + 2 * ldc + 0 * W, QV_FMA(va, c20, QV_LOADU(C + 2 * ldc + 0 * W))); + QV_STOREU(C + 3 * ldc + 0 * W, QV_FMA(va, c30, QV_LOADU(C + 3 * ldc + 0 * W))); +#if QV_MR >= 8 + QV_STOREU(C + 4 * ldc + 0 * W, QV_FMA(va, c40, QV_LOADU(C + 4 * ldc + 0 * W))); + QV_STOREU(C + 5 * ldc + 0 * W, QV_FMA(va, c50, QV_LOADU(C + 5 * ldc + 0 * W))); + QV_STOREU(C + 6 * ldc + 0 * W, QV_FMA(va, c60, QV_LOADU(C + 6 * ldc + 0 * W))); + QV_STOREU(C + 7 * ldc + 0 * W, QV_FMA(va, c70, QV_LOADU(C + 7 * ldc + 0 * W))); +#endif + if (NR_VEC > 1) { + QV_STOREU(C + 0 * ldc + 1 * W, QV_FMA(va, c01, QV_LOADU(C + 0 * ldc + 1 * W))); + QV_STOREU(C + 1 * ldc + 1 * W, QV_FMA(va, c11, QV_LOADU(C + 1 * ldc + 1 * W))); + QV_STOREU(C + 2 * ldc + 1 * W, QV_FMA(va, c21, QV_LOADU(C + 2 * ldc + 1 * W))); + QV_STOREU(C + 3 * ldc + 1 * W, QV_FMA(va, c31, QV_LOADU(C + 3 * ldc + 1 * W))); + } +#else + Sleef_quad c[QV_MR][QV_NR]; + for (size_t i = 0; i < QV_MR; ++i) + for (size_t j = 0; j < QV_NR; ++j) + c[i][j] = QBLAS_ZERO; + for (size_t p = 0; p < kc; ++p) { + for (size_t i = 0; i < QV_MR; ++i) { + Sleef_quad a = A_packed[p * QV_MR + i]; + for (size_t j = 0; j < QV_NR; ++j) + c[i][j] = qfma(a, B_packed[p * QV_NR + j], c[i][j]); + } + } + for (size_t i = 0; i < QV_MR; ++i) + for (size_t j = 0; j < QV_NR; ++j) + C[i * ldc + j] = qfma(alpha, c[i][j], C[i * ldc + j]); +#endif +} + +void QV_FN(register_kernels)(void); +void QV_FN(register_kernels)(void) { + qblas_dispatch_qdot = QV_FN(qdot); + qblas_dispatch_qaxpy = QV_FN(qaxpy); + qblas_dispatch_qscal = QV_FN(qscal); + qblas_dispatch_qasum = QV_FN(qasum); + qblas_dispatch_qiamax = QV_FN(qiamax); + qblas_dispatch_qgemv_n = QV_FN(qgemv_n); + qblas_dispatch_qgemv_t = QV_FN(qgemv_t); + qblas_dispatch_qgemm_kernel = QV_FN(qgemm_kernel); + qblas_dispatch_qgemm_MR = QV_MR; + qblas_dispatch_qgemm_NR = QV_NR; +} diff --git a/src/kernels/neon/kernels_neon.c b/src/kernels/neon/kernels_neon.c new file mode 100644 index 0000000..059f2c8 --- /dev/null +++ b/src/kernels/neon/kernels_neon.c @@ -0,0 +1,7 @@ +#define QBLAS_HAS_NEON 1 +#define QV_WIDTH 2 +#define QV_SUFFIX neon +#define QV_ISA_SUFFIX advsimd +#include "../kernels_template.h" + +void qblas_register_neon(void) { qblas_register_kernels_neon(); } diff --git a/src/kernels/sse2/kernels_sse2.c b/src/kernels/sse2/kernels_sse2.c new file mode 100644 index 0000000..37b5d3e --- /dev/null +++ b/src/kernels/sse2/kernels_sse2.c @@ -0,0 +1,7 @@ +#define QBLAS_HAS_SSE2 1 +#define QV_WIDTH 2 +#define QV_SUFFIX sse2 +#define QV_ISA_SUFFIX sse2 +#include "../kernels_template.h" + +void qblas_register_sse2(void) { qblas_register_kernels_sse2(); } diff --git a/src/level1/qblas_level1.c b/src/level1/qblas_level1.c new file mode 100644 index 0000000..df9cb22 --- /dev/null +++ b/src/level1/qblas_level1.c @@ -0,0 +1,175 @@ +#include "common/qblas_internal.h" +#include "common/qblas_dispatch.h" + +#ifdef _OPENMP +# include +#endif + +static inline ptrdiff_t neg_offset(int inc, int n) { + return (inc < 0) ? (ptrdiff_t)(n - 1) * (ptrdiff_t)(-inc) : 0; +} + +Sleef_quad cblas_qdot(int n, + const Sleef_quad *x, int incx, + const Sleef_quad *y, int incy) { + if (n <= 0) return QBLAS_ZERO; + if (incx == 1 && incy == 1) { + int nthreads = qblas_resolve_threads((size_t)n, 1); + if (nthreads > 1 && (size_t)n >= qblas_tune()->l1_thread_threshold) { +#ifdef _OPENMP + Sleef_quad totals[256] = {0}; + if (nthreads > 256) nthreads = 256; + #pragma omp parallel num_threads(nthreads) + { + int tid = omp_get_thread_num(); + int nt = omp_get_num_threads(); + size_t chunk = (size_t)n / nt; + size_t rem = (size_t)n % nt; + size_t start = (size_t)tid * chunk + (tid < (int)rem ? (size_t)tid : rem); + size_t cnt = chunk + (tid < (int)rem ? 1u : 0u); + totals[tid] = qblas_dispatch_qdot(cnt, x + start, 1, y + start, 1); + } + Sleef_quad acc = QBLAS_ZERO; + for (int t = 0; t < nthreads; ++t) acc = qadd(acc, totals[t]); + return acc; +#endif + } + return qblas_dispatch_qdot((size_t)n, x, 1, y, 1); + } + ptrdiff_t ox = neg_offset(incx, n); + ptrdiff_t oy = neg_offset(incy, n); + return qblas_dispatch_qdot((size_t)n, x + ox, (ptrdiff_t)incx, + y + oy, (ptrdiff_t)incy); +} + +Sleef_quad cblas_qnrm2(int n, const Sleef_quad *x, int incx) { + if (n <= 0) return QBLAS_ZERO; + Sleef_quad s = cblas_qdot(n, x, incx, x, incx); + return qsqrt(s); +} + +Sleef_quad cblas_qasum(int n, const Sleef_quad *x, int incx) { + if (n <= 0) return QBLAS_ZERO; + if (incx == 1) { + int nthreads = qblas_resolve_threads((size_t)n, 1); + if (nthreads > 1 && (size_t)n >= qblas_tune()->l1_thread_threshold) { +#ifdef _OPENMP + Sleef_quad totals[256] = {0}; + if (nthreads > 256) nthreads = 256; + #pragma omp parallel num_threads(nthreads) + { + int tid = omp_get_thread_num(); + int nt = omp_get_num_threads(); + size_t chunk = (size_t)n / nt; + size_t rem = (size_t)n % nt; + size_t start = (size_t)tid * chunk + (tid < (int)rem ? (size_t)tid : rem); + size_t cnt = chunk + (tid < (int)rem ? 1u : 0u); + totals[tid] = qblas_dispatch_qasum(cnt, x + start, 1); + } + Sleef_quad acc = QBLAS_ZERO; + for (int t = 0; t < nthreads; ++t) acc = qadd(acc, totals[t]); + return acc; +#endif + } + return qblas_dispatch_qasum((size_t)n, x, 1); + } + ptrdiff_t ox = neg_offset(incx, n); + return qblas_dispatch_qasum((size_t)n, x + ox, (ptrdiff_t)incx); +} + +size_t cblas_iqamax(int n, const Sleef_quad *x, int incx) { + if (n <= 0) return 0; + ptrdiff_t ox = neg_offset(incx, n); + return qblas_dispatch_qiamax((size_t)n, x + ox, (ptrdiff_t)incx); +} + +void cblas_qaxpy(int n, Sleef_quad alpha, + const Sleef_quad *x, int incx, + Sleef_quad *y, int incy) { + if (n <= 0 || qiszero(alpha)) return; + if (incx == 1 && incy == 1) { + int nthreads = qblas_resolve_threads((size_t)n, 2); + if (nthreads > 1 && (size_t)n >= qblas_tune()->l1_thread_threshold) { +#ifdef _OPENMP + #pragma omp parallel num_threads(nthreads) + { + int tid = omp_get_thread_num(); + int nt = omp_get_num_threads(); + size_t chunk = (size_t)n / nt; + size_t rem = (size_t)n % nt; + size_t start = (size_t)tid * chunk + (tid < (int)rem ? (size_t)tid : rem); + size_t cnt = chunk + (tid < (int)rem ? 1u : 0u); + qblas_dispatch_qaxpy(cnt, alpha, x + start, 1, y + start, 1); + } + return; +#endif + } + qblas_dispatch_qaxpy((size_t)n, alpha, x, 1, y, 1); + return; + } + ptrdiff_t ox = neg_offset(incx, n); + ptrdiff_t oy = neg_offset(incy, n); + qblas_dispatch_qaxpy((size_t)n, alpha, x + ox, incx, y + oy, incy); +} + +void cblas_qscal(int n, Sleef_quad alpha, Sleef_quad *x, int incx) { + if (n <= 0) return; + if (incx == 1) { + int nthreads = qblas_resolve_threads((size_t)n, 1); + if (nthreads > 1 && (size_t)n >= qblas_tune()->l1_thread_threshold) { +#ifdef _OPENMP + #pragma omp parallel num_threads(nthreads) + { + int tid = omp_get_thread_num(); + int nt = omp_get_num_threads(); + size_t chunk = (size_t)n / nt; + size_t rem = (size_t)n % nt; + size_t start = (size_t)tid * chunk + (tid < (int)rem ? (size_t)tid : rem); + size_t cnt = chunk + (tid < (int)rem ? 1u : 0u); + qblas_dispatch_qscal(cnt, alpha, x + start, 1); + } + return; +#endif + } + qblas_dispatch_qscal((size_t)n, alpha, x, 1); + return; + } + ptrdiff_t ox = neg_offset(incx, n); + qblas_dispatch_qscal((size_t)n, alpha, x + ox, incx); +} + +void cblas_qcopy(int n, + const Sleef_quad *x, int incx, + Sleef_quad *y, int incy) { + if (n <= 0) return; + if (incx == 1 && incy == 1) { + memcpy(y, x, (size_t)n * sizeof(Sleef_quad)); + return; + } + ptrdiff_t ix = neg_offset(incx, n); + ptrdiff_t iy = neg_offset(incy, n); + for (int i = 0; i < n; ++i) { + y[iy] = x[ix]; + ix += incx; + iy += incy; + } +} + +void cblas_qswap(int n, + Sleef_quad *x, int incx, + Sleef_quad *y, int incy) { + if (n <= 0) return; + if (incx == 1 && incy == 1) { + for (int i = 0; i < n; ++i) { + Sleef_quad t = x[i]; x[i] = y[i]; y[i] = t; + } + return; + } + ptrdiff_t ix = neg_offset(incx, n); + ptrdiff_t iy = neg_offset(incy, n); + for (int i = 0; i < n; ++i) { + Sleef_quad t = x[ix]; x[ix] = y[iy]; y[iy] = t; + ix += incx; + iy += incy; + } +} diff --git a/src/level2/qblas_level2.c b/src/level2/qblas_level2.c new file mode 100644 index 0000000..2503913 --- /dev/null +++ b/src/level2/qblas_level2.c @@ -0,0 +1,243 @@ +#include "common/qblas_internal.h" +#include "common/qblas_dispatch.h" + +#ifdef _OPENMP +# include +#endif + +/* Collapse all four (layout, trans) gemv combos to a row-major view that + * dispatches to one of qgemv_n / qgemv_t. Col-major no-trans is identical + * to row-major trans with M and N swapped, and vice versa. */ +void cblas_qgemv(QBLAS_LAYOUT layout, QBLAS_TRANSPOSE trans, + int m, int n, + Sleef_quad alpha, + const Sleef_quad *A, int lda, + const Sleef_quad *x, int incx, + Sleef_quad beta, + Sleef_quad *y, int incy) { + if (m <= 0 || n <= 0) return; + + int doT = (trans == QblasTrans || trans == QblasConjTrans); + int row = (layout == QblasRowMajor); + + size_t rm_rows, rm_cols; + int do_kernel_T; + if (row && !doT) { rm_rows = (size_t)m; rm_cols = (size_t)n; do_kernel_T = 0; } + else if (row && doT) { rm_rows = (size_t)m; rm_cols = (size_t)n; do_kernel_T = 1; } + else if (!row && !doT) { rm_rows = (size_t)n; rm_cols = (size_t)m; do_kernel_T = 1; } + else { rm_rows = (size_t)n; rm_cols = (size_t)m; do_kernel_T = 0; } + + size_t y_len = do_kernel_T ? rm_cols : rm_rows; + size_t x_len = do_kernel_T ? rm_rows : rm_cols; + (void)x_len; + + ptrdiff_t ox = (incx < 0) ? (ptrdiff_t)(x_len - 1) * (-incx) : 0; + ptrdiff_t oy = (incy < 0) ? (ptrdiff_t)(y_len - 1) * (-incy) : 0; + + if (do_kernel_T) { + int nthreads = qblas_resolve_threads(rm_rows * rm_cols, 1); + if (nthreads > 1 && rm_rows * rm_cols >= qblas_tune()->gemv_thread_threshold) { +#ifdef _OPENMP + /* Parallelise over output columns: each y[j] is a strided dot + * of A's column j with x. Avoids needing per-thread y buffers. */ + #pragma omp parallel for num_threads(nthreads) schedule(static) + for (size_t j = 0; j < y_len; ++j) { + Sleef_quad s = qblas_dispatch_qdot(rm_rows, + A + j, (ptrdiff_t)lda, + x + ox, (ptrdiff_t)incx); + Sleef_quad scaled = qmul(alpha, s); + if (qiszero(beta)) y[oy + (ptrdiff_t)j * incy] = scaled; + else y[oy + (ptrdiff_t)j * incy] = + qfma(beta, y[oy + (ptrdiff_t)j * incy], scaled); + } + return; +#endif + } + qblas_dispatch_qgemv_t(rm_rows, rm_cols, alpha, A, (size_t)lda, + x + ox, incx, beta, y + oy, incy); + } else { + int nthreads = qblas_resolve_threads(rm_rows * rm_cols, 1); + if (nthreads > 1 && rm_rows * rm_cols >= qblas_tune()->gemv_thread_threshold) { +#ifdef _OPENMP + #pragma omp parallel num_threads(nthreads) + { + int tid = omp_get_thread_num(); + int nt = omp_get_num_threads(); + size_t chunk = rm_rows / nt; + size_t rem = rm_rows % nt; + size_t start = (size_t)tid * chunk + (tid < (int)rem ? (size_t)tid : rem); + size_t cnt = chunk + (tid < (int)rem ? 1u : 0u); + qblas_dispatch_qgemv_n(cnt, rm_cols, alpha, + A + start * (size_t)lda, (size_t)lda, + x + ox, incx, beta, + y + oy + (ptrdiff_t)start * incy, incy); + } + return; +#endif + } + qblas_dispatch_qgemv_n(rm_rows, rm_cols, alpha, A, (size_t)lda, + x + ox, incx, beta, y + oy, incy); + } +} + +void cblas_qger(QBLAS_LAYOUT layout, + int m, int n, + Sleef_quad alpha, + const Sleef_quad *x, int incx, + const Sleef_quad *y, int incy, + Sleef_quad *A, int lda) { + if (m <= 0 || n <= 0 || qiszero(alpha)) return; + ptrdiff_t ox = (incx < 0) ? (ptrdiff_t)(m - 1) * (-incx) : 0; + ptrdiff_t oy = (incy < 0) ? (ptrdiff_t)(n - 1) * (-incy) : 0; + if (layout == QblasRowMajor) { + #ifdef _OPENMP + int nthreads = qblas_resolve_threads((size_t)m, (size_t)n); + #pragma omp parallel for num_threads(nthreads) schedule(static) + #endif + for (int i = 0; i < m; ++i) { + Sleef_quad ax = qmul(alpha, x[ox + (ptrdiff_t)i * incx]); + if (qiszero(ax)) continue; + qblas_dispatch_qaxpy((size_t)n, ax, y + oy, incy, A + (size_t)i * lda, 1); + } + } else { + #ifdef _OPENMP + int nthreads = qblas_resolve_threads((size_t)n, (size_t)m); + #pragma omp parallel for num_threads(nthreads) schedule(static) + #endif + for (int j = 0; j < n; ++j) { + Sleef_quad ay = qmul(alpha, y[oy + (ptrdiff_t)j * incy]); + if (qiszero(ay)) continue; + qblas_dispatch_qaxpy((size_t)m, ay, x + ox, incx, A + (size_t)j * lda, 1); + } + } +} + +/* For each row i: y[i] += alpha * sum_j A[i,j] * x[j], using the indicated + * triangle of A and the symmetric pair on the other side. */ +void cblas_qsymv(QBLAS_LAYOUT layout, QBLAS_UPLO uplo, + int n, + Sleef_quad alpha, + const Sleef_quad *A, int lda, + const Sleef_quad *x, int incx, + Sleef_quad beta, + Sleef_quad *y, int incy) { + if (n <= 0) return; + + /* Symmetric col-major is row-major with the triangle flipped. */ + if (layout == QblasColMajor) { + uplo = (uplo == QblasUpper) ? QblasLower : QblasUpper; + } + + ptrdiff_t ox = (incx < 0) ? (ptrdiff_t)(n - 1) * (-incx) : 0; + ptrdiff_t oy = (incy < 0) ? (ptrdiff_t)(n - 1) * (-incy) : 0; + + if (qiszero(beta)) { + for (int i = 0; i < n; ++i) y[oy + (ptrdiff_t)i * incy] = QBLAS_ZERO; + } else if (!qisone(beta)) { + for (int i = 0; i < n; ++i) + y[oy + (ptrdiff_t)i * incy] = qmul(beta, y[oy + (ptrdiff_t)i * incy]); + } + + if (qiszero(alpha)) return; + + if (uplo == QblasUpper) { + for (int i = 0; i < n; ++i) { + Sleef_quad xi = x[ox + (ptrdiff_t)i * incx]; + Sleef_quad axi = qmul(alpha, xi); + Sleef_quad yi = y[oy + (ptrdiff_t)i * incy]; + yi = qfma(axi, A[(size_t)i * lda + i], yi); + for (int j = i + 1; j < n; ++j) { + Sleef_quad aij = A[(size_t)i * lda + j]; + yi = qfma(axi, aij, yi); + y[oy + (ptrdiff_t)j * incy] = + qfma(qmul(alpha, A[(size_t)i * lda + j]), + x[ox + (ptrdiff_t)i * incx], + y[oy + (ptrdiff_t)j * incy]); + } + y[oy + (ptrdiff_t)i * incy] = yi; + } + } else { + for (int i = 0; i < n; ++i) { + Sleef_quad xi = x[ox + (ptrdiff_t)i * incx]; + Sleef_quad axi = qmul(alpha, xi); + Sleef_quad yi = y[oy + (ptrdiff_t)i * incy]; + yi = qfma(axi, A[(size_t)i * lda + i], yi); + for (int j = 0; j < i; ++j) { + Sleef_quad aij = A[(size_t)i * lda + j]; + yi = qfma(axi, aij, yi); + y[oy + (ptrdiff_t)j * incy] = + qfma(qmul(alpha, A[(size_t)i * lda + j]), + x[ox + (ptrdiff_t)i * incx], + y[oy + (ptrdiff_t)j * incy]); + } + y[oy + (ptrdiff_t)i * incy] = yi; + } + } +} + +void cblas_qtrsv(QBLAS_LAYOUT layout, + QBLAS_UPLO uplo, + QBLAS_TRANSPOSE trans, + QBLAS_DIAG diag, + int n, + const Sleef_quad *A, int lda, + Sleef_quad *x, int incx) { + if (n <= 0) return; + + /* Reinterpret col-major + (uplo, trans) as row-major with both flipped. */ + int doT = (trans == QblasTrans || trans == QblasConjTrans); + if (layout == QblasColMajor) { + uplo = (uplo == QblasUpper) ? QblasLower : QblasUpper; + doT = !doT; + } + int upper = (uplo == QblasUpper); + int unit = (diag == QblasUnit); + + ptrdiff_t ox = (incx < 0) ? (ptrdiff_t)(n - 1) * (-incx) : 0; + + if (!doT) { + if (!upper) { + for (int i = 0; i < n; ++i) { + Sleef_quad sum = x[ox + (ptrdiff_t)i * incx]; + for (int j = 0; j < i; ++j) + sum = qsub(sum, + qmul(A[(size_t)i * lda + j], + x[ox + (ptrdiff_t)j * incx])); + if (!unit) sum = qdiv(sum, A[(size_t)i * lda + i]); + x[ox + (ptrdiff_t)i * incx] = sum; + } + } else { + for (int i = n - 1; i >= 0; --i) { + Sleef_quad sum = x[ox + (ptrdiff_t)i * incx]; + for (int j = i + 1; j < n; ++j) + sum = qsub(sum, + qmul(A[(size_t)i * lda + j], + x[ox + (ptrdiff_t)j * incx])); + if (!unit) sum = qdiv(sum, A[(size_t)i * lda + i]); + x[ox + (ptrdiff_t)i * incx] = sum; + } + } + } else { + if (!upper) { + for (int i = n - 1; i >= 0; --i) { + Sleef_quad sum = x[ox + (ptrdiff_t)i * incx]; + for (int j = i + 1; j < n; ++j) + sum = qsub(sum, + qmul(A[(size_t)j * lda + i], + x[ox + (ptrdiff_t)j * incx])); + if (!unit) sum = qdiv(sum, A[(size_t)i * lda + i]); + x[ox + (ptrdiff_t)i * incx] = sum; + } + } else { + for (int i = 0; i < n; ++i) { + Sleef_quad sum = x[ox + (ptrdiff_t)i * incx]; + for (int j = 0; j < i; ++j) + sum = qsub(sum, + qmul(A[(size_t)j * lda + i], + x[ox + (ptrdiff_t)j * incx])); + if (!unit) sum = qdiv(sum, A[(size_t)i * lda + i]); + x[ox + (ptrdiff_t)i * incx] = sum; + } + } + } +} diff --git a/src/level3/qblas_gemm.c b/src/level3/qblas_gemm.c new file mode 100644 index 0000000..36f34a5 --- /dev/null +++ b/src/level3/qblas_gemm.c @@ -0,0 +1,246 @@ +/* Goto / van de Geijn 5-loop blocked GEMM around the dispatch tier's + * packed micro-kernel. Loop order: jc(L3) -> pc(K) -> ic(L2) -> + * jr,ir(MRxNR register tile). A and B are packed into contiguous + * buffers per pc panel so the kernel sees a single uniform layout + * regardless of user-side transposes / leading dims. C is pre-scaled by + * beta once up front; the kernel only does additive FMAs. */ + +#include "common/qblas_internal.h" +#include "common/qblas_dispatch.h" + +#include +#include + +#ifdef _OPENMP +# include +#endif + +typedef struct { + const Sleef_quad *data; + size_t row_stride; + size_t col_stride; +} qmat_t; + +static inline Sleef_quad qmat_get(const qmat_t *M, size_t i, size_t j) { + return M->data[i * M->row_stride + j * M->col_stride]; +} + +/* Build a row-major virtual view of A under (layout, trans). */ +static qmat_t make_view(const Sleef_quad *A, int lda, + QBLAS_LAYOUT layout, QBLAS_TRANSPOSE trans) { + qmat_t v; + v.data = A; + int doT = (trans == QblasTrans || trans == QblasConjTrans); + if (layout == QblasRowMajor) { + if (!doT) { v.row_stride = (size_t)lda; v.col_stride = 1; } + else { v.row_stride = 1; v.col_stride = (size_t)lda; } + } else { + if (!doT) { v.row_stride = 1; v.col_stride = (size_t)lda; } + else { v.row_stride = (size_t)lda; v.col_stride = 1; } + } + return v; +} + +/* Pack into [mc/MR][kc][MR] panels. Tail rows zero-padded so the kernel + * always sees MR rows. */ +static void pack_A(size_t mc, size_t kc, + const qmat_t *A, size_t i0, size_t p0, + size_t MR, + Sleef_quad *restrict Ap) { + size_t pos = 0; + size_t i = 0; + for (; i + MR <= mc; i += MR) { + for (size_t p = 0; p < kc; ++p) { + for (size_t r = 0; r < MR; ++r) { + Ap[pos++] = qmat_get(A, i0 + i + r, p0 + p); + } + } + } + if (i < mc) { + size_t left = mc - i; + for (size_t p = 0; p < kc; ++p) { + for (size_t r = 0; r < MR; ++r) { + Ap[pos++] = (r < left) ? qmat_get(A, i0 + i + r, p0 + p) : QBLAS_ZERO; + } + } + } +} + +/* Pack into [nc/NR][kc][NR] panels. Tail cols zero-padded. */ +static void pack_B(size_t kc, size_t nc, + const qmat_t *B, size_t p0, size_t j0, + size_t NR, + Sleef_quad *restrict Bp) { + size_t pos = 0; + size_t j = 0; + for (; j + NR <= nc; j += NR) { + for (size_t p = 0; p < kc; ++p) { + for (size_t c = 0; c < NR; ++c) { + Bp[pos++] = qmat_get(B, p0 + p, j0 + j + c); + } + } + } + if (j < nc) { + size_t left = nc - j; + for (size_t p = 0; p < kc; ++p) { + for (size_t c = 0; c < NR; ++c) { + Bp[pos++] = (c < left) ? qmat_get(B, p0 + p, j0 + j + c) : QBLAS_ZERO; + } + } + } +} + +static void scale_C(int row_major, size_t m, size_t n, + Sleef_quad beta, Sleef_quad *C, size_t ldc) { + if (qisone(beta)) return; + if (qiszero(beta)) { + if (row_major) { + for (size_t i = 0; i < m; ++i) + for (size_t j = 0; j < n; ++j) C[i * ldc + j] = QBLAS_ZERO; + } else { + for (size_t j = 0; j < n; ++j) + for (size_t i = 0; i < m; ++i) C[j * ldc + i] = QBLAS_ZERO; + } + return; + } + if (row_major) { + for (size_t i = 0; i < m; ++i) + for (size_t j = 0; j < n; ++j) + C[i * ldc + j] = qmul(beta, C[i * ldc + j]); + } else { + for (size_t j = 0; j < n; ++j) + for (size_t i = 0; i < m; ++i) + C[j * ldc + i] = qmul(beta, C[j * ldc + i]); + } +} + +/* For a full MR×NR tile we call the kernel against C directly. For an + * edge tile the kernel writes the zero-padded MR×NR result into scratch + * and we add only the live m_real × n_real region into C. */ +static void apply_tile(size_t m_real, size_t n_real, + size_t MR, size_t NR, + size_t kc, + Sleef_quad alpha, + const Sleef_quad *Ap, const Sleef_quad *Bp, + Sleef_quad *C, size_t ldc) { + if (m_real == MR && n_real == NR) { + qblas_dispatch_qgemm_kernel(kc, alpha, Ap, Bp, C, ldc); + return; + } + Sleef_quad scratch[16 * 16]; /* MR * NR <= 256 for shipped tiers */ + for (size_t i = 0; i < MR * NR; ++i) scratch[i] = QBLAS_ZERO; + qblas_dispatch_qgemm_kernel(kc, alpha, Ap, Bp, scratch, NR); + for (size_t i = 0; i < m_real; ++i) + for (size_t j = 0; j < n_real; ++j) + C[i * ldc + j] = qadd(C[i * ldc + j], scratch[i * NR + j]); +} + +void cblas_qgemm(QBLAS_LAYOUT layout, + QBLAS_TRANSPOSE transa, QBLAS_TRANSPOSE transb, + int m_, int n_, int k_, + Sleef_quad alpha, + const Sleef_quad *A, int lda, + const Sleef_quad *B, int ldb, + Sleef_quad beta, + Sleef_quad *C, int ldc) { + if (m_ <= 0 || n_ <= 0) return; + size_t m = (size_t)m_, n = (size_t)n_, k = (size_t)k_; + + int row_major = (layout == QblasRowMajor); + + /* Compute in a row-major C view. Col-major C is the same buffer + * reinterpreted: addresses match because additions commute, so we + * just swap M<->N and A<->B (keeping their transpose flags). */ + qmat_t Av, Bv; + size_t mm, nn, kk = k; + size_t ldc_eff = (size_t)ldc; + if (row_major) { + mm = m; nn = n; + Av = make_view(A, lda, layout, transa); + Bv = make_view(B, ldb, layout, transb); + } else { + mm = n; nn = m; + Av = make_view(B, ldb, QblasRowMajor, transb); + Bv = make_view(A, lda, QblasRowMajor, transa); + } + + scale_C(1, mm, nn, beta, C, ldc_eff); + + if (qiszero(alpha) || kk == 0) return; + + qblas_dispatch_init(); + const size_t MR = qblas_dispatch_qgemm_MR; + const size_t NR = qblas_dispatch_qgemm_NR; + + const qblas_tune_t *tune = qblas_tune(); + size_t kc = tune->gemm_kc; + size_t mc = tune->gemm_mc; + size_t nc = tune->gemm_nc; + if (kc > kk) kc = kk; + if (mc > mm) mc = mm; + + int nthreads = qblas_resolve_threads(mm * nn, kk); + if (nthreads < 1) nthreads = 1; + + /* Shrink nc so every thread gets at least ~2 jc-blocks of work. */ + if (nthreads > 1) { + size_t target_blocks = (size_t)nthreads * 2; + size_t max_nc_per_block = (nn + target_blocks - 1) / target_blocks; + if (max_nc_per_block < nc) nc = max_nc_per_block; + if (nc < NR) nc = NR; + } + if (nc > nn) nc = nn; + size_t mc_pad = ((mc + MR - 1) / MR) * MR; + size_t nc_pad = ((nc + NR - 1) / NR) * NR; + + size_t Bp_size = kc * nc_pad; + size_t Ap_size = mc_pad * kc; + + size_t njc = (nn + nc - 1) / nc; + +#ifdef _OPENMP + #pragma omp parallel num_threads(nthreads) +#endif + { + Sleef_quad *Ap = (Sleef_quad *)qblas_aligned_alloc(Ap_size * sizeof(Sleef_quad)); + Sleef_quad *Bp = (Sleef_quad *)qblas_aligned_alloc(Bp_size * sizeof(Sleef_quad)); + +#ifdef _OPENMP + #pragma omp for schedule(dynamic) +#endif + for (size_t b = 0; b < njc; ++b) { + size_t jc = b * nc; + size_t nc_use = (jc + nc <= nn) ? nc : (nn - jc); + size_t nc_use_pad = ((nc_use + NR - 1) / NR) * NR; + + for (size_t pc = 0; pc < kk; pc += kc) { + size_t kc_use = (pc + kc <= kk) ? kc : (kk - pc); + + pack_B(kc_use, nc_use, &Bv, pc, jc, NR, Bp); + + for (size_t ic = 0; ic < mm; ic += mc) { + size_t mc_use = (ic + mc <= mm) ? mc : (mm - ic); + size_t mc_use_pad = ((mc_use + MR - 1) / MR) * MR; + + pack_A(mc_use, kc_use, &Av, ic, pc, MR, Ap); + + for (size_t jr = 0; jr < nc_use_pad; jr += NR) { + size_t n_real = (jc + jr + NR <= jc + nc_use) ? NR : (nc_use - jr); + const Sleef_quad *Bp_panel = Bp + (jr / NR) * (kc_use * NR); + for (size_t ir = 0; ir < mc_use_pad; ir += MR) { + size_t m_real = (ic + ir + MR <= ic + mc_use) ? MR : (mc_use - ir); + const Sleef_quad *Ap_panel = Ap + (ir / MR) * (kc_use * MR); + Sleef_quad *C_tile = C + (ic + ir) * ldc_eff + (jc + jr); + apply_tile(m_real, n_real, MR, NR, kc_use, + alpha, Ap_panel, Bp_panel, + C_tile, ldc_eff); + } + } + } + } + } + + qblas_aligned_free(Ap); + qblas_aligned_free(Bp); + } +} diff --git a/src/level3/qblas_level3_extra.c b/src/level3/qblas_level3_extra.c new file mode 100644 index 0000000..2d32e40 --- /dev/null +++ b/src/level3/qblas_level3_extra.c @@ -0,0 +1,328 @@ +/* syrk delegates to qgemm. trmm/trsm use Goto-style blocking: the + * diagonal block is solved/multiplied in-place (parallel over columns) + * and the trailing matrix update is a cblas_qgemm() call, so they + * inherit gemm's packed kernel, blocking, and threading. */ + +#include "common/qblas_internal.h" +#include "common/qblas_dispatch.h" + +#include + +#ifdef _OPENMP +# include +#endif + +static inline Sleef_quad get_A(const Sleef_quad *A, int lda, + QBLAS_LAYOUT layout, QBLAS_TRANSPOSE trans, + size_t i, size_t j) { + int doT = (trans == QblasTrans || trans == QblasConjTrans); + if (layout == QblasRowMajor) { + if (!doT) return A[i * (size_t)lda + j]; + else return A[j * (size_t)lda + i]; + } else { + if (!doT) return A[j * (size_t)lda + i]; + else return A[i * (size_t)lda + j]; + } +} + +static inline Sleef_quad *B_at(Sleef_quad *B, int ldb, QBLAS_LAYOUT layout, + size_t i, size_t j) { + return (layout == QblasRowMajor) ? B + i * (size_t)ldb + j + : B + j * (size_t)ldb + i; +} + +void cblas_qsyrk(QBLAS_LAYOUT layout, + QBLAS_UPLO uplo, + QBLAS_TRANSPOSE trans, + int n, int k, + Sleef_quad alpha, + const Sleef_quad *A, int lda, + Sleef_quad beta, + Sleef_quad *C, int ldc) { + (void)uplo; + if (n <= 0) return; + if (k == 0 || qiszero(alpha)) { + if (qisone(beta)) return; + for (int i = 0; i < n; ++i) + for (int j = 0; j < n; ++j) + *B_at(C, ldc, layout, (size_t)i, (size_t)j) = + qmul(beta, *B_at(C, ldc, layout, (size_t)i, (size_t)j)); + return; + } + QBLAS_TRANSPOSE tA = (trans == QblasNoTrans) ? QblasNoTrans : QblasTrans; + QBLAS_TRANSPOSE tB = (trans == QblasNoTrans) ? QblasTrans : QblasNoTrans; + cblas_qgemm(layout, tA, tB, n, n, k, alpha, A, lda, A, lda, beta, C, ldc); +} + +static void scale_B_inplace(QBLAS_LAYOUT layout, int m, int n, + Sleef_quad alpha, Sleef_quad *B, int ldb) { + if (qisone(alpha)) return; + if (qiszero(alpha)) { + for (int i = 0; i < m; ++i) + for (int j = 0; j < n; ++j) + *B_at(B, ldb, layout, (size_t)i, (size_t)j) = QBLAS_ZERO; + return; + } + for (int i = 0; i < m; ++i) + for (int j = 0; j < n; ++j) { + Sleef_quad *p = B_at(B, ldb, layout, (size_t)i, (size_t)j); + *p = qmul(alpha, *p); + } +} + +/* Diagonal block solve. Columns of B are independent so they fan out + * across threads via omp parallel for. */ +static void trsm_left_diag(QBLAS_LAYOUT layout, QBLAS_UPLO uplo, + QBLAS_TRANSPOSE transa, QBLAS_DIAG diag, + int bs, int n, + const Sleef_quad *A, int lda, + size_t i0, + Sleef_quad *B, int ldb, + size_t row_off) { + int doT = (transa == QblasTrans || transa == QblasConjTrans); + int upper = (uplo == QblasUpper); + int eff_upper = doT ? !upper : upper; + int unit = (diag == QblasUnit); + + int step = eff_upper ? -1 : 1; + int i_start = eff_upper ? bs - 1 : 0; + int i_end = eff_upper ? -1 : bs; + +#ifdef _OPENMP + int nthreads = qblas_resolve_threads((size_t)bs * (size_t)n, (size_t)bs); + #pragma omp parallel for num_threads(nthreads) schedule(static) +#endif + for (int j = 0; j < n; ++j) { + for (int i = i_start; i != i_end; i += step) { + int k_start = eff_upper ? i + 1 : 0; + int k_end = eff_upper ? bs : i; + Sleef_quad rhs = *B_at(B, ldb, layout, row_off + (size_t)i, (size_t)j); + for (int k = k_start; k < k_end; ++k) { + Sleef_quad aik = doT + ? get_A(A, lda, layout, QblasNoTrans, i0 + (size_t)k, i0 + (size_t)i) + : get_A(A, lda, layout, QblasNoTrans, i0 + (size_t)i, i0 + (size_t)k); + Sleef_quad xkj = *B_at(B, ldb, layout, row_off + (size_t)k, (size_t)j); + rhs = qsub(rhs, qmul(aik, xkj)); + } + if (!unit) { + Sleef_quad aii = get_A(A, lda, layout, QblasNoTrans, + i0 + (size_t)i, i0 + (size_t)i); + rhs = qdiv(rhs, aii); + } + *B_at(B, ldb, layout, row_off + (size_t)i, (size_t)j) = rhs; + } + } +} + +static void trmm_left_diag(QBLAS_LAYOUT layout, QBLAS_UPLO uplo, + QBLAS_TRANSPOSE transa, QBLAS_DIAG diag, + int bs, int n, + const Sleef_quad *A, int lda, + size_t i0, + Sleef_quad *B, int ldb, + size_t row_off) { + int doT = (transa == QblasTrans || transa == QblasConjTrans); + int upper = (uplo == QblasUpper); + int eff_upper = doT ? !upper : upper; + int unit = (diag == QblasUnit); + +#ifdef _OPENMP + int nthreads = qblas_resolve_threads((size_t)bs * (size_t)n, (size_t)bs); + #pragma omp parallel for num_threads(nthreads) schedule(static) +#endif + for (int j = 0; j < n; ++j) { + if (eff_upper) { + for (int i = 0; i < bs; ++i) { + Sleef_quad sum = QBLAS_ZERO; + for (int k = i; k < bs; ++k) { + Sleef_quad aik; + if (k == i && unit) aik = QBLAS_ONE; + else aik = doT + ? get_A(A, lda, layout, QblasNoTrans, i0 + (size_t)k, i0 + (size_t)i) + : get_A(A, lda, layout, QblasNoTrans, i0 + (size_t)i, i0 + (size_t)k); + Sleef_quad bkj = *B_at(B, ldb, layout, row_off + (size_t)k, (size_t)j); + sum = qfma(aik, bkj, sum); + } + *B_at(B, ldb, layout, row_off + (size_t)i, (size_t)j) = sum; + } + } else { + for (int i = bs - 1; i >= 0; --i) { + Sleef_quad sum = QBLAS_ZERO; + for (int k = 0; k <= i; ++k) { + Sleef_quad aik; + if (k == i && unit) aik = QBLAS_ONE; + else aik = doT + ? get_A(A, lda, layout, QblasNoTrans, i0 + (size_t)k, i0 + (size_t)i) + : get_A(A, lda, layout, QblasNoTrans, i0 + (size_t)i, i0 + (size_t)k); + Sleef_quad bkj = *B_at(B, ldb, layout, row_off + (size_t)k, (size_t)j); + sum = qfma(aik, bkj, sum); + } + *B_at(B, ldb, layout, row_off + (size_t)i, (size_t)j) = sum; + } + } + } +} + +/* Returns pointer to op(A)[r0:r0+rb, c0:c0+cb]; the trans flag stays the + * user's so cblas_qgemm interprets the buffer correctly. */ +static const Sleef_quad *opA_block_ptr(const Sleef_quad *A, int lda, + QBLAS_LAYOUT layout, + QBLAS_TRANSPOSE transa, + size_t r0, size_t c0) { + int doT = (transa == QblasTrans || transa == QblasConjTrans); + if (layout == QblasRowMajor) { + if (!doT) return A + r0 * (size_t)lda + c0; + else return A + c0 * (size_t)lda + r0; + } else { + if (!doT) return A + r0 + c0 * (size_t)lda; + else return A + c0 + r0 * (size_t)lda; + } +} + +#define QBLAS_TRSM_NB 64 + +void cblas_qtrsm(QBLAS_LAYOUT layout, + QBLAS_SIDE side, QBLAS_UPLO uplo, + QBLAS_TRANSPOSE transa, QBLAS_DIAG diag, + int m, int n, + Sleef_quad alpha, + const Sleef_quad *A, int lda, + Sleef_quad *B, int ldb) { + if (m <= 0 || n <= 0) return; + + scale_B_inplace(layout, m, n, alpha, B, ldb); + if (qiszero(alpha)) return; + + if (side == QblasLeft) { + int doT = (transa == QblasTrans || transa == QblasConjTrans); + int upper = (uplo == QblasUpper); + int eff_upper = doT ? !upper : upper; + int NB = QBLAS_TRSM_NB; + if (NB > m) NB = m; + + if (m <= NB) { + trsm_left_diag(layout, uplo, transa, diag, m, n, + A, lda, 0, B, ldb, 0); + return; + } + + Sleef_quad neg_one = qneg(QBLAS_ONE); + + if (!eff_upper) { + /* Forward substitution: solve top block, gemm-update rows below. */ + for (int ib = 0; ib < m; ib += NB) { + int bs = (ib + NB <= m) ? NB : m - ib; + trsm_left_diag(layout, uplo, transa, diag, bs, n, + A, lda, (size_t)ib, B, ldb, (size_t)ib); + int rest = m - (ib + bs); + if (rest <= 0) continue; + const Sleef_quad *Ablk = opA_block_ptr(A, lda, layout, transa, + (size_t)(ib + bs), (size_t)ib); + Sleef_quad *Bsub = B_at(B, ldb, layout, (size_t)(ib + bs), 0); + Sleef_quad *Bdiag = B_at(B, ldb, layout, (size_t)ib, 0); + cblas_qgemm(layout, transa, QblasNoTrans, + rest, n, bs, + neg_one, Ablk, lda, Bdiag, ldb, + QBLAS_ONE, Bsub, ldb); + } + } else { + /* Back substitution: walk bottom-up. Top block may be partial + * when m % NB != 0. */ + int ib = m; + while (ib > 0) { + int bs = ib >= NB ? NB : ib; + ib -= bs; + trsm_left_diag(layout, uplo, transa, diag, bs, n, + A, lda, (size_t)ib, B, ldb, (size_t)ib); + if (ib == 0) break; + const Sleef_quad *Ablk = opA_block_ptr(A, lda, layout, transa, + 0, (size_t)ib); + Sleef_quad *Bsub = B; + Sleef_quad *Bdiag = B_at(B, ldb, layout, (size_t)ib, 0); + cblas_qgemm(layout, transa, QblasNoTrans, + ib, n, bs, + neg_one, Ablk, lda, Bdiag, ldb, + QBLAS_ONE, Bsub, ldb); + } + } + } else { + /* X op(A) = alpha B <=> op(A)^T X^T = alpha B^T : flip layout + * and transa, swap m/n, recurse into Left. */ + QBLAS_LAYOUT new_layout = (layout == QblasRowMajor) ? QblasColMajor : QblasRowMajor; + QBLAS_TRANSPOSE new_ta = (transa == QblasNoTrans) ? QblasTrans : QblasNoTrans; + cblas_qtrsm(new_layout, QblasLeft, uplo, new_ta, diag, + n, m, QBLAS_ONE, A, lda, B, ldb); + } +} + +void cblas_qtrmm(QBLAS_LAYOUT layout, + QBLAS_SIDE side, QBLAS_UPLO uplo, + QBLAS_TRANSPOSE transa, QBLAS_DIAG diag, + int m, int n, + Sleef_quad alpha, + const Sleef_quad *A, int lda, + Sleef_quad *B, int ldb) { + if (m <= 0 || n <= 0) return; + + if (side == QblasRight) { + QBLAS_LAYOUT new_layout = (layout == QblasRowMajor) ? QblasColMajor : QblasRowMajor; + QBLAS_TRANSPOSE new_ta = (transa == QblasNoTrans) ? QblasTrans : QblasNoTrans; + cblas_qtrmm(new_layout, QblasLeft, uplo, new_ta, diag, + n, m, alpha, A, lda, B, ldb); + return; + } + + int doT = (transa == QblasTrans || transa == QblasConjTrans); + int upper = (uplo == QblasUpper); + int eff_upper = doT ? !upper : upper; + int NB = QBLAS_TRSM_NB; + if (NB > m) NB = m; + + if (m <= NB) { + trmm_left_diag(layout, uplo, transa, diag, m, n, A, lda, 0, B, ldb, 0); + scale_B_inplace(layout, m, n, alpha, B, ldb); + return; + } + + if (eff_upper) { + /* Row i depends on rows k >= i; process top-down with gemm-update + * from below before each diagonal block. */ + for (int ib = 0; ib < m; ib += NB) { + int bs = (ib + NB <= m) ? NB : m - ib; + int rest = m - (ib + bs); + if (rest > 0) { + const Sleef_quad *Ablk = opA_block_ptr(A, lda, layout, transa, + (size_t)ib, (size_t)(ib + bs)); + Sleef_quad *Brest = B_at(B, ldb, layout, (size_t)(ib + bs), 0); + Sleef_quad *Bblk = B_at(B, ldb, layout, (size_t)ib, 0); + cblas_qgemm(layout, transa, QblasNoTrans, + bs, n, rest, + QBLAS_ONE, Ablk, lda, Brest, ldb, + QBLAS_ONE, Bblk, ldb); + } + trmm_left_diag(layout, uplo, transa, diag, bs, n, + A, lda, (size_t)ib, B, ldb, (size_t)ib); + } + } else { + /* Row i depends on rows k <= i; process bottom-up. */ + int ib = m; + while (ib > 0) { + int bs = ib >= NB ? NB : ib; + ib -= bs; + if (ib > 0) { + const Sleef_quad *Ablk = opA_block_ptr(A, lda, layout, transa, + (size_t)ib, 0); + Sleef_quad *Brest = B; + Sleef_quad *Bblk = B_at(B, ldb, layout, (size_t)ib, 0); + cblas_qgemm(layout, transa, QblasNoTrans, + bs, n, ib, + QBLAS_ONE, Ablk, lda, Brest, ldb, + QBLAS_ONE, Bblk, ldb); + } + trmm_left_diag(layout, uplo, transa, diag, bs, n, + A, lda, (size_t)ib, B, ldb, (size_t)ib); + } + } + + scale_B_inplace(layout, m, n, alpha, B, ldb); +} diff --git a/test_quadblas.cpp b/test_quadblas.cpp deleted file mode 100644 index cb19a40..0000000 --- a/test_quadblas.cpp +++ /dev/null @@ -1,785 +0,0 @@ -#include "include/quadblas/quadblas.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// =============================================================================== -// HELPER FUNCTIONS AND UTILITIES -// =============================================================================== - -// Helper functions for quad precision conversion -double to_double(Sleef_quad q) { - return static_cast(Sleef_cast_to_doubleq1(q)); -} - -Sleef_quad from_double(double d) { - return Sleef_cast_from_doubleq1(d); -} - -// Test result tracking -struct TestResult { - std::string name; - bool passed; - double max_error; - double avg_error; - size_t error_count; - size_t total_elements; - std::string details; - - TestResult(const std::string& n) : name(n), passed(false), max_error(0), avg_error(0), - error_count(0), total_elements(0) {} -}; - -class TestSuite { -private: - std::vector results; - double tolerance = 1e-12; // Stricter tolerance for quad precision - -public: - void set_tolerance(double tol) { tolerance = tol; } - - void add_result(const TestResult& result) { - results.push_back(result); - std::cout << "[" << (result.passed ? "PASS" : "FAIL") << "] " << result.name; - if (!result.passed) { - std::cout << " (max_error: " << std::scientific << result.max_error << ")"; - } - std::cout << std::endl; - if (!result.details.empty()) { - std::cout << " " << result.details << std::endl; - } - } - - void print_summary() { - size_t passed = 0, failed = 0; - for (const auto& r : results) { - if (r.passed) passed++; else failed++; - } - - std::cout << "\n" << std::string(80, '=') << std::endl; - std::cout << "TEST SUMMARY" << std::endl; - std::cout << std::string(80, '=') << std::endl; - std::cout << "Total tests: " << results.size() << std::endl; - std::cout << "Passed: " << passed << std::endl; - std::cout << "Failed: " << failed << std::endl; - - if (failed > 0) { - std::cout << "\nFAILED TESTS:" << std::endl; - for (const auto& r : results) { - if (!r.passed) { - std::cout << " - " << r.name << " (error: " << std::scientific << r.max_error << ")" << std::endl; - } - } - } - - std::cout << "\nOverall result: " << (failed == 0 ? "✓ ALL TESTS PASSED" : "✗ SOME TESTS FAILED") << std::endl; - } - - double get_tolerance() const { return tolerance; } -}; - -// Random number generator for reproducible tests -class TestRandom { -private: - std::mt19937 gen; - std::uniform_real_distribution uniform_dist; - std::normal_distribution normal_dist; - -public: - TestRandom(unsigned seed = 42) : gen(seed), uniform_dist(-1.0, 1.0), normal_dist(0.0, 1.0) {} - - Sleef_quad uniform() { return from_double(uniform_dist(gen)); } - Sleef_quad normal() { return from_double(normal_dist(gen)); } - Sleef_quad range(double min, double max) { - std::uniform_real_distribution dist(min, max); - return from_double(dist(gen)); - } - - void fill_uniform(Sleef_quad* arr, size_t n) { - for (size_t i = 0; i < n; ++i) arr[i] = uniform(); - } - - void fill_normal(Sleef_quad* arr, size_t n) { - for (size_t i = 0; i < n; ++i) arr[i] = normal(); - } -}; - -// Utility to compare arrays and compute error statistics -TestResult compare_arrays(const std::string& test_name, const Sleef_quad* expected, - const Sleef_quad* actual, size_t n, double tolerance) { - TestResult result(test_name); - result.total_elements = n; - - double sum_error = 0.0; - for (size_t i = 0; i < n; ++i) { - double error = std::abs(to_double(expected[i]) - to_double(actual[i])); - result.max_error = std::max(result.max_error, error); - sum_error += error; - if (error > tolerance) { - result.error_count++; - } - } - - result.avg_error = sum_error / n; - result.passed = (result.error_count == 0); - - if (!result.passed) { - result.details = "Errors in " + std::to_string(result.error_count) + "/" + std::to_string(n) + " elements"; - } - - return result; -} - -// =============================================================================== -// REFERENCE IMPLEMENTATIONS (GUARANTEED CORRECT) -// =============================================================================== - -// Reference dot product -Sleef_quad reference_dot(size_t n, const Sleef_quad* x, size_t incx, - const Sleef_quad* y, size_t incy) { - Sleef_quad result = from_double(0.0); - for (size_t i = 0; i < n; ++i) { - result = Sleef_fmaq1_u05(x[i * incx], y[i * incy], result); - } - return result; -} - -// Reference AXPY: y = alpha * x + y -void reference_axpy(size_t n, Sleef_quad alpha, const Sleef_quad* x, size_t incx, - Sleef_quad* y, size_t incy) { - for (size_t i = 0; i < n; ++i) { - y[i * incy] = Sleef_fmaq1_u05(alpha, x[i * incx], y[i * incy]); - } -} - -// Reference GEMV: y = alpha * A * x + beta * y -void reference_gemv(bool row_major, size_t m, size_t n, Sleef_quad alpha, - const Sleef_quad* A, size_t lda, const Sleef_quad* x, size_t incx, - Sleef_quad beta, Sleef_quad* y, size_t incy) { - for (size_t i = 0; i < m; ++i) { - Sleef_quad sum = from_double(0.0); - for (size_t j = 0; j < n; ++j) { - size_t a_idx = row_major ? i * lda + j : j * lda + i; - sum = Sleef_fmaq1_u05(A[a_idx], x[j * incx], sum); - } - y[i * incy] = Sleef_fmaq1_u05(alpha, sum, Sleef_mulq1_u05(beta, y[i * incy])); - } -} - -// Reference GEMM: C = alpha * A * B + beta * C -void reference_gemm(bool row_major, size_t m, size_t n, size_t k, Sleef_quad alpha, - const Sleef_quad* A, size_t lda, const Sleef_quad* B, size_t ldb, - Sleef_quad beta, Sleef_quad* C, size_t ldc) { - for (size_t i = 0; i < m; ++i) { - for (size_t j = 0; j < n; ++j) { - Sleef_quad sum = from_double(0.0); - for (size_t l = 0; l < k; ++l) { - size_t a_idx = row_major ? i * lda + l : l * lda + i; - size_t b_idx = row_major ? l * ldb + j : j * ldb + l; - sum = Sleef_fmaq1_u05(A[a_idx], B[b_idx], sum); - } - size_t c_idx = row_major ? i * ldc + j : j * ldc + i; - C[c_idx] = Sleef_fmaq1_u05(alpha, sum, Sleef_mulq1_u05(beta, C[c_idx])); - } - } -} - -// =============================================================================== -// LEVEL 1 BLAS TESTS (Vector-Vector Operations) -// =============================================================================== - -void test_level1_blas(TestSuite& suite) { - std::cout << "\n" << std::string(60, '=') << std::endl; - std::cout << "LEVEL 1 BLAS TESTS (Vector-Vector Operations)" << std::endl; - std::cout << std::string(60, '=') << std::endl; - - TestRandom rng; - - // Test 1: Small dot product - { - const size_t n = 10; - std::vector x(n), y(n); - - // Simple test pattern - for (size_t i = 0; i < n; ++i) { - x[i] = from_double(i + 1); // 1, 2, 3, ..., 10 - y[i] = from_double(1.0); // 1, 1, 1, ..., 1 - } - - Sleef_quad expected = reference_dot(n, x.data(), 1, y.data(), 1); - Sleef_quad actual = QuadBLAS::dot(n, x.data(), 1, y.data(), 1); - - double error = std::abs(to_double(expected) - to_double(actual)); - TestResult result("DOT Small (n=10)"); - result.passed = error < suite.get_tolerance(); - result.max_error = error; - result.total_elements = 1; - suite.add_result(result); - } - - // Test 2: Large dot product (tests vectorization) - { - const size_t n = 100000; - std::vector x(n), y(n); - rng.fill_uniform(x.data(), n); - rng.fill_uniform(y.data(), n); - - Sleef_quad expected = reference_dot(n, x.data(), 1, y.data(), 1); - Sleef_quad actual = QuadBLAS::dot(n, x.data(), 1, y.data(), 1); - - double error = std::abs(to_double(expected) - to_double(actual)); - TestResult result("DOT Large (n=100k)"); - result.passed = error < suite.get_tolerance() * n; // Allow for accumulation error - result.max_error = error; - result.total_elements = 1; - suite.add_result(result); - } - - // Test 3: Dot product with strides - { - const size_t n = 1000; - const size_t incx = 3, incy = 2; - std::vector x(n * incx), y(n * incy); - rng.fill_uniform(x.data(), n * incx); - rng.fill_uniform(y.data(), n * incy); - - Sleef_quad expected = reference_dot(n, x.data(), incx, y.data(), incy); - Sleef_quad actual = QuadBLAS::dot(n, x.data(), incx, y.data(), incy); - - double error = std::abs(to_double(expected) - to_double(actual)); - TestResult result("DOT with strides"); - result.passed = error < suite.get_tolerance() * n; - result.max_error = error; - suite.add_result(result); - } - - // Test 4: AXPY operation - { - const size_t n = 5000; - Sleef_quad alpha = from_double(2.5); - std::vector x(n), y_expected(n), y_actual(n); - - rng.fill_uniform(x.data(), n); - rng.fill_uniform(y_expected.data(), n); - std::copy(y_expected.begin(), y_expected.end(), y_actual.begin()); - - reference_axpy(n, alpha, x.data(), 1, y_expected.data(), 1); - - // Test using C++ interface - QuadBLAS::DefaultVector<> qx(n), qy(n); - for (size_t i = 0; i < n; ++i) { - qx[i] = x[i]; - qy[i] = y_actual[i]; - } - qy.axpy(alpha, qx); - for (size_t i = 0; i < n; ++i) { - y_actual[i] = qy[i]; - } - - TestResult result = compare_arrays("AXPY operation", y_expected.data(), y_actual.data(), n, suite.get_tolerance()); - suite.add_result(result); - } - - // Test 5: Vector norm - { - const size_t n = 1000; - std::vector x(n); - - // Use a simple pattern for exact computation - for (size_t i = 0; i < n; ++i) { - x[i] = from_double(i + 1); - } - - // Expected norm = sqrt(1^2 + 2^2 + ... + n^2) = sqrt(n(n+1)(2n+1)/6) - double sum_squares = n * (n + 1) * (2 * n + 1) / 6.0; - Sleef_quad expected_norm = from_double(std::sqrt(sum_squares)); - - QuadBLAS::DefaultVector<> qx(n); - for (size_t i = 0; i < n; ++i) qx[i] = x[i]; - Sleef_quad actual_norm = qx.norm(); - - double error = std::abs(to_double(expected_norm) - to_double(actual_norm)); - TestResult result("Vector norm"); - result.passed = error < suite.get_tolerance() * std::sqrt(n); - result.max_error = error; - suite.add_result(result); - } -} - -// =============================================================================== -// LEVEL 2 BLAS TESTS (Matrix-Vector Operations) -// =============================================================================== - -void test_level2_blas(TestSuite& suite) { - std::cout << "\n" << std::string(60, '=') << std::endl; - std::cout << "LEVEL 2 BLAS TESTS (Matrix-Vector Operations)" << std::endl; - std::cout << std::string(60, '=') << std::endl; - - TestRandom rng; - - // Test 1: Small GEMV - { - const size_t m = 3, n = 3; - std::vector A = { - from_double(1), from_double(2), from_double(3), - from_double(4), from_double(5), from_double(6), - from_double(7), from_double(8), from_double(9) - }; - std::vector x = {from_double(1), from_double(1), from_double(1)}; - std::vector y_expected(m), y_actual(m); - - // Initialize y - for (size_t i = 0; i < m; ++i) { - y_expected[i] = y_actual[i] = from_double(0.0); - } - - reference_gemv(true, m, n, from_double(1.0), A.data(), n, x.data(), 1, - from_double(0.0), y_expected.data(), 1); - - QuadBLAS::gemv(QuadBLAS::Layout::RowMajor, m, n, from_double(1.0), A.data(), n, - x.data(), 1, from_double(0.0), y_actual.data(), 1); - - TestResult result = compare_arrays("GEMV Small (3x3)", y_expected.data(), y_actual.data(), m, suite.get_tolerance()); - suite.add_result(result); - } - - // Test 2: Medium GEMV (tests optimization paths) - { - const size_t m = 100, n = 100; - std::vector A(m * n), x(n), y_expected(m), y_actual(m); - - rng.fill_uniform(A.data(), m * n); - rng.fill_uniform(x.data(), n); - rng.fill_uniform(y_expected.data(), m); - std::copy(y_expected.begin(), y_expected.end(), y_actual.begin()); - - Sleef_quad alpha = from_double(1.5), beta = from_double(0.5); - - reference_gemv(true, m, n, alpha, A.data(), n, x.data(), 1, beta, y_expected.data(), 1); - QuadBLAS::gemv(QuadBLAS::Layout::RowMajor, m, n, alpha, A.data(), n, x.data(), 1, - beta, y_actual.data(), 1); - - TestResult result = compare_arrays("GEMV Medium (100x100)", y_expected.data(), y_actual.data(), m, suite.get_tolerance() * n); - suite.add_result(result); - } - - // Test 3: Large GEMV (tests parallel execution) - { - const size_t m = 2000, n = 2000; - std::vector A(m * n), x(n), y_expected(m), y_actual(m); - - rng.fill_normal(A.data(), m * n); - rng.fill_normal(x.data(), n); - - for (size_t i = 0; i < m; ++i) { - y_expected[i] = y_actual[i] = from_double(0.0); - } - - reference_gemv(true, m, n, from_double(1.0), A.data(), n, x.data(), 1, - from_double(0.0), y_expected.data(), 1); - QuadBLAS::gemv(QuadBLAS::Layout::RowMajor, m, n, from_double(1.0), A.data(), n, - x.data(), 1, from_double(0.0), y_actual.data(), 1); - - TestResult result = compare_arrays("GEMV Large (2000x2000)", y_expected.data(), y_actual.data(), m, suite.get_tolerance() * n); - suite.add_result(result); - } - - // Test 4: GEMV with C++ interface - { - const size_t m = 50, n = 50; - QuadBLAS::DefaultMatrix<> A(m, n); - QuadBLAS::DefaultVector<> x(n), y_expected(m), y_actual(m); - - // Initialize with specific pattern - for (size_t i = 0; i < m; ++i) { - for (size_t j = 0; j < n; ++j) { - A(i, j) = from_double((i + j + 1) * 0.1); - } - } - for (size_t i = 0; i < n; ++i) { - x[i] = from_double(i + 1); - } - - // Compute expected using reference - std::vector A_flat(m * n), x_flat(n), y_flat(m); - for (size_t i = 0; i < m; ++i) { - for (size_t j = 0; j < n; ++j) { - A_flat[i * n + j] = A(i, j); - } - } - for (size_t i = 0; i < n; ++i) x_flat[i] = x[i]; - for (size_t i = 0; i < m; ++i) y_flat[i] = from_double(0.0); - - reference_gemv(true, m, n, from_double(1.0), A_flat.data(), n, x_flat.data(), 1, - from_double(0.0), y_flat.data(), 1); - - // Test QuadBLAS C++ interface - for (size_t i = 0; i < m; ++i) y_actual[i] = from_double(0.0); - A.gemv(from_double(1.0), x, from_double(0.0), y_actual); - - std::vector y_actual_flat(m); - for (size_t i = 0; i < m; ++i) y_actual_flat[i] = y_actual[i]; - - TestResult result = compare_arrays("GEMV C++ interface", y_flat.data(), y_actual_flat.data(), m, suite.get_tolerance() * n); - suite.add_result(result); - } -} - -// =============================================================================== -// LEVEL 3 BLAS TESTS (Matrix-Matrix Operations) -// =============================================================================== - -void test_level3_blas(TestSuite& suite) { - std::cout << "\n" << std::string(60, '=') << std::endl; - std::cout << "LEVEL 3 BLAS TESTS (Matrix-Matrix Operations)" << std::endl; - std::cout << std::string(60, '=') << std::endl; - - TestRandom rng; - - // Test 1: Tiny GEMM (2x2) - tests basic correctness - { - const size_t m = 2, n = 2, k = 2; - std::vector A = {from_double(1), from_double(2), from_double(3), from_double(4)}; - std::vector B = {from_double(5), from_double(6), from_double(7), from_double(8)}; - std::vector C_expected(m * n), C_actual(m * n); - - // Initialize C to zero - for (size_t i = 0; i < m * n; ++i) { - C_expected[i] = C_actual[i] = from_double(0.0); - } - - reference_gemm(true, m, n, k, from_double(1.0), A.data(), k, B.data(), n, - from_double(0.0), C_expected.data(), n); - QuadBLAS::gemm(QuadBLAS::Layout::RowMajor, m, n, k, from_double(1.0), A.data(), k, - B.data(), n, from_double(0.0), C_actual.data(), n); - - TestResult result = compare_arrays("GEMM Tiny (2x2)", C_expected.data(), C_actual.data(), m * n, suite.get_tolerance()); - suite.add_result(result); - } - - // Test 2: Small GEMM (uses simple implementation) - { - const size_t m = 16, n = 16, k = 16; - std::vector A(m * k), B(k * n), C_expected(m * n), C_actual(m * n); - - rng.fill_uniform(A.data(), m * k); - rng.fill_uniform(B.data(), k * n); - - for (size_t i = 0; i < m * n; ++i) { - C_expected[i] = C_actual[i] = from_double(0.0); - } - - reference_gemm(true, m, n, k, from_double(1.0), A.data(), k, B.data(), n, - from_double(0.0), C_expected.data(), n); - QuadBLAS::gemm(QuadBLAS::Layout::RowMajor, m, n, k, from_double(1.0), A.data(), k, - B.data(), n, from_double(0.0), C_actual.data(), n); - - TestResult result = compare_arrays("GEMM Small (16x16)", C_expected.data(), C_actual.data(), m * n, suite.get_tolerance() * k); - suite.add_result(result); - } - - // Test 3: Boundary GEMM (at the threshold) - this is critical! - { - const size_t m = 32, n = 32, k = 32; // Exactly at threshold - std::vector A(m * k), B(k * n), C_expected(m * n), C_actual(m * n); - - rng.fill_uniform(A.data(), m * k); - rng.fill_uniform(B.data(), k * n); - - for (size_t i = 0; i < m * n; ++i) { - C_expected[i] = C_actual[i] = from_double(0.0); - } - - reference_gemm(true, m, n, k, from_double(1.0), A.data(), k, B.data(), n, - from_double(0.0), C_expected.data(), n); - QuadBLAS::gemm(QuadBLAS::Layout::RowMajor, m, n, k, from_double(1.0), A.data(), k, - B.data(), n, from_double(0.0), C_actual.data(), n); - - TestResult result = compare_arrays("GEMM Boundary (32x32)", C_expected.data(), C_actual.data(), m * n, suite.get_tolerance() * k); - suite.add_result(result); - } - - // Test 4: Medium GEMM (triggers blocked algorithm) - this should expose bugs! - { - const size_t m = 64, n = 64, k = 64; // Beyond threshold - uses blocked algorithm - std::vector A(m * k), B(k * n), C_expected(m * n), C_actual(m * n); - - // Use patterns that would expose indexing errors - for (size_t i = 0; i < m; ++i) { - for (size_t j = 0; j < k; ++j) { - A[i * k + j] = from_double(i + j + 1); // Simple pattern - } - } - for (size_t i = 0; i < k; ++i) { - for (size_t j = 0; j < n; ++j) { - B[i * n + j] = from_double((i * j + 1) % 7); // Another pattern - } - } - - for (size_t i = 0; i < m * n; ++i) { - C_expected[i] = C_actual[i] = from_double(0.0); - } - - reference_gemm(true, m, n, k, from_double(1.0), A.data(), k, B.data(), n, - from_double(0.0), C_expected.data(), n); - QuadBLAS::gemm(QuadBLAS::Layout::RowMajor, m, n, k, from_double(1.0), A.data(), k, - B.data(), n, from_double(0.0), C_actual.data(), n); - - TestResult result = compare_arrays("GEMM Medium (64x64) - BLOCKED ALGORITHM", C_expected.data(), C_actual.data(), m * n, suite.get_tolerance() * k); - suite.add_result(result); - } - - // Test 5: Large GEMM (tests scalability) - { - const size_t m = 200, n = 200, k = 200; - std::vector A(m * k), B(k * n), C_expected(m * n), C_actual(m * n); - - std::cout << " Generating large matrices (this may take a moment)..." << std::endl; - rng.fill_normal(A.data(), m * k); - rng.fill_normal(B.data(), k * n); - - for (size_t i = 0; i < m * n; ++i) { - C_expected[i] = C_actual[i] = from_double(0.0); - } - - std::cout << " Computing reference result..." << std::endl; - auto start = std::chrono::high_resolution_clock::now(); - reference_gemm(true, m, n, k, from_double(1.0), A.data(), k, B.data(), n, - from_double(0.0), C_expected.data(), n); - auto ref_time = std::chrono::high_resolution_clock::now() - start; - - std::cout << " Computing QuadBLAS result..." << std::endl; - start = std::chrono::high_resolution_clock::now(); - QuadBLAS::gemm(QuadBLAS::Layout::RowMajor, m, n, k, from_double(1.0), A.data(), k, - B.data(), n, from_double(0.0), C_actual.data(), n); - auto qblas_time = std::chrono::high_resolution_clock::now() - start; - - TestResult result = compare_arrays("GEMM Large (200x200)", C_expected.data(), C_actual.data(), m * n, suite.get_tolerance() * k); - - auto ref_ms = std::chrono::duration_cast(ref_time).count(); - auto qblas_ms = std::chrono::duration_cast(qblas_time).count(); - result.details += " | Time: ref=" + std::to_string(ref_ms) + "ms, qblas=" + std::to_string(qblas_ms) + "ms"; - - suite.add_result(result); - } - - // Test 6: GEMM with alpha and beta scaling - { - const size_t m = 20, n = 20, k = 20; - std::vector A(m * k), B(k * n), C_expected(m * n), C_actual(m * n); - - rng.fill_uniform(A.data(), m * k); - rng.fill_uniform(B.data(), k * n); - rng.fill_uniform(C_expected.data(), m * n); - std::copy(C_expected.begin(), C_expected.end(), C_actual.begin()); - - Sleef_quad alpha = from_double(2.5), beta = from_double(1.5); - - reference_gemm(true, m, n, k, alpha, A.data(), k, B.data(), n, beta, C_expected.data(), n); - QuadBLAS::gemm(QuadBLAS::Layout::RowMajor, m, n, k, alpha, A.data(), k, - B.data(), n, beta, C_actual.data(), n); - - TestResult result = compare_arrays("GEMM with alpha/beta scaling", C_expected.data(), C_actual.data(), m * n, suite.get_tolerance() * k); - suite.add_result(result); - } - - // Test 7: GEMM C++ interface - { - const size_t m = 25, n = 25, k = 25; - QuadBLAS::DefaultMatrix<> A(m, k), B(k, n), C_expected(m, n), C_actual(m, n); - - // Initialize matrices - for (size_t i = 0; i < m; ++i) { - for (size_t j = 0; j < k; ++j) { - A(i, j) = rng.uniform(); - } - } - for (size_t i = 0; i < k; ++i) { - for (size_t j = 0; j < n; ++j) { - B(i, j) = rng.uniform(); - } - } - - // Compute expected using reference - std::vector A_flat(m * k), B_flat(k * n), C_flat(m * n); - for (size_t i = 0; i < m; ++i) { - for (size_t j = 0; j < k; ++j) { - A_flat[i * k + j] = A(i, j); - } - } - for (size_t i = 0; i < k; ++i) { - for (size_t j = 0; j < n; ++j) { - B_flat[i * n + j] = B(i, j); - } - } - for (size_t i = 0; i < m * n; ++i) C_flat[i] = from_double(0.0); - - reference_gemm(true, m, n, k, from_double(1.0), A_flat.data(), k, B_flat.data(), n, - from_double(0.0), C_flat.data(), n); - - // Test C++ interface - for (size_t i = 0; i < m; ++i) { - for (size_t j = 0; j < n; ++j) { - C_actual(i, j) = from_double(0.0); - } - } - A.gemm(from_double(1.0), B, from_double(0.0), C_actual); - - std::vector C_actual_flat(m * n); - for (size_t i = 0; i < m; ++i) { - for (size_t j = 0; j < n; ++j) { - C_actual_flat[i * n + j] = C_actual(i, j); - } - } - - TestResult result = compare_arrays("GEMM C++ interface", C_flat.data(), C_actual_flat.data(), m * n, suite.get_tolerance() * k); - suite.add_result(result); - } -} - -// =============================================================================== -// EDGE CASE AND STRESS TESTS -// =============================================================================== - -void test_edge_cases(TestSuite& suite) { - std::cout << "\n" << std::string(60, '=') << std::endl; - std::cout << "EDGE CASE AND STRESS TESTS" << std::endl; - std::cout << std::string(60, '=') << std::endl; - - TestRandom rng; - - // Test 1: Non-square matrices - { - const size_t m = 47, n = 31, k = 23; // Prime numbers to stress algorithms - std::vector A(m * k), B(k * n), C_expected(m * n), C_actual(m * n); - - rng.fill_uniform(A.data(), m * k); - rng.fill_uniform(B.data(), k * n); - - for (size_t i = 0; i < m * n; ++i) { - C_expected[i] = C_actual[i] = from_double(0.0); - } - - reference_gemm(true, m, n, k, from_double(1.0), A.data(), k, B.data(), n, - from_double(0.0), C_expected.data(), n); - QuadBLAS::gemm(QuadBLAS::Layout::RowMajor, m, n, k, from_double(1.0), A.data(), k, - B.data(), n, from_double(0.0), C_actual.data(), n); - - TestResult result = compare_arrays("Non-square matrices (47x31x23)", C_expected.data(), C_actual.data(), m * n, suite.get_tolerance() * k); - suite.add_result(result); - } - - // Test 2: Very small matrices (1x1) - { - const size_t m = 1, n = 1, k = 1; - Sleef_quad A = from_double(3.0), B = from_double(4.0), C_expected = from_double(0.0), C_actual = from_double(0.0); - - reference_gemm(true, m, n, k, from_double(1.0), &A, k, &B, n, from_double(0.0), &C_expected, n); - QuadBLAS::gemm(QuadBLAS::Layout::RowMajor, m, n, k, from_double(1.0), &A, k, &B, n, from_double(0.0), &C_actual, n); - - TestResult result = compare_arrays("Minimal matrices (1x1)", &C_expected, &C_actual, 1, suite.get_tolerance()); - suite.add_result(result); - } - - // Test 3: Identity matrix multiplication - { - const size_t n = 50; - std::vector I(n * n), A(n * n), C_expected(n * n), C_actual(n * n); - - // Create identity matrix - for (size_t i = 0; i < n * n; ++i) I[i] = from_double(0.0); - for (size_t i = 0; i < n; ++i) I[i * n + i] = from_double(1.0); - - rng.fill_uniform(A.data(), n * n); - - // I * A should equal A - for (size_t i = 0; i < n * n; ++i) { - C_expected[i] = A[i]; - C_actual[i] = from_double(0.0); - } - - QuadBLAS::gemm(QuadBLAS::Layout::RowMajor, n, n, n, from_double(1.0), I.data(), n, - A.data(), n, from_double(0.0), C_actual.data(), n); - - TestResult result = compare_arrays("Identity matrix multiplication", C_expected.data(), C_actual.data(), n * n, suite.get_tolerance()); - suite.add_result(result); - } - - // Test 4: Numerical precision test - { - std::cout << " Testing numerical precision with extreme values..." << std::endl; - const size_t n = 10; - QuadBLAS::DefaultVector<> x(n), y(n); - - // Test case that would lose precision in double - x[0] = from_double(1e20); - x[1] = from_double(1.0); - x[2] = from_double(-1e20); - for (size_t i = 3; i < n; ++i) x[i] = from_double(0.0); - - for (size_t i = 0; i < n; ++i) y[i] = from_double(1.0); - - Sleef_quad result = x.dot(y); - double result_double = to_double(result); - - TestResult test_result("Numerical precision test"); - test_result.passed = std::abs(result_double - 1.0) < 1e-10; // Should be exactly 1.0 - test_result.max_error = std::abs(result_double - 1.0); - test_result.total_elements = 1; - if (!test_result.passed) { - test_result.details = "Expected 1.0, got " + std::to_string(result_double); - } - suite.add_result(test_result); - } -} - -// =============================================================================== -// MAIN TEST PROGRAM -// =============================================================================== - -int main() { - std::cout << "QuadBLAS Comprehensive Correctness Test Suite" << std::endl; - std::cout << "=============================================" << std::endl; - - std::cout << "System Information:" << std::endl; - std::cout << " " << quadblas_get_version() << std::endl; - std::cout << " Available threads: " << QuadBLAS::get_num_threads() << std::endl; - -#ifdef QUADBLAS_X86_64 - std::cout << " Platform: x86-64 with SIMD vectorization" << std::endl; -#elif defined(QUADBLAS_AARCH64) - std::cout << " Platform: AArch64 with NEON vectorization" << std::endl; -#else - std::cout << " Platform: Generic (scalar fallback)" << std::endl; -#endif - -#ifdef _OPENMP - std::cout << " OpenMP: Enabled" << std::endl; -#else - std::cout << " OpenMP: Disabled" << std::endl; -#endif - - TestSuite suite; - suite.set_tolerance(1e-12); // Stricter tolerance for quad precision - - try { - test_level1_blas(suite); - test_level2_blas(suite); - test_level3_blas(suite); - test_edge_cases(suite); - - suite.print_summary(); - - } catch (const std::exception& e) { - std::cerr << "\nFATAL ERROR: " << e.what() << std::endl; - return 1; - } - - return 0; -} \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000..eda4919 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,60 @@ +# Each test target is a C executable linking the qblas shared lib. + +set(QBLAS_TESTS + test_smoke + test_level1 + test_level2 + test_gemm + test_level3_extra) + +foreach(t ${QBLAS_TESTS}) + add_executable(${t} ${t}.c) + target_link_libraries(${t} PRIVATE qblas m) + target_include_directories(${t} PRIVATE + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}) + add_test(NAME ${t} COMMAND ${t}) +endforeach() + +# ----------------------------------------------------------------------- +# Python numpy-comparison test +# ----------------------------------------------------------------------- +# Builds a small shim library (`libqblas_shim.so`) that re-exposes the +# qblas entry points with `Sleef_quad *` instead of `Sleef_quad` so that +# Python's ctypes / libffi handles the calling convention reliably. +# The Python harness then calls every routine and compares the result +# (rounded back to double) against numpy's double-precision reference. +add_library(qblas_shim SHARED python/qblas_shim.c) +target_link_libraries(qblas_shim PRIVATE qblas) +target_include_directories(qblas_shim PRIVATE ${CMAKE_SOURCE_DIR}/include) +set_target_properties(qblas_shim PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") + +# Look for a Python 3 interpreter with numpy. Skipped if neither is found +# (e.g. environments without numpy installed). On macOS CI the workflow +# exports VIRTUAL_ENV so find_package(Python3) selects the venv. +find_package(Python3 COMPONENTS Interpreter) +if(Python3_Interpreter_FOUND) + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import numpy" + RESULT_VARIABLE _qblas_have_numpy + OUTPUT_QUIET ERROR_QUIET) + if(_qblas_have_numpy EQUAL 0) + # Put SLEEF and the build's shared libs on the dynamic loader's + # search path so ctypes.CDLL can resolve them by basename alone. + if(APPLE) + set(_qblas_ld_var "DYLD_LIBRARY_PATH") + else() + set(_qblas_ld_var "LD_LIBRARY_PATH") + endif() + set(_qblas_ld_path "${SLEEF_ROOT}/lib:$:$") + add_test(NAME test_against_numpy + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/python/test_against_numpy.py) + set_tests_properties(test_against_numpy PROPERTIES + DEPENDS qblas_shim + ENVIRONMENT "${_qblas_ld_var}=${_qblas_ld_path}") + else() + message(STATUS "QBLAS: numpy not importable from ${Python3_EXECUTABLE}; skipping test_against_numpy") + endif() +endif() diff --git a/tests/debug_test.cpp b/tests/debug_test.cpp deleted file mode 100644 index 436b456..0000000 --- a/tests/debug_test.cpp +++ /dev/null @@ -1,190 +0,0 @@ -#include "quadblas.hpp" -#include -#include - -// Helper functions -double to_double(Sleef_quad q) { - return static_cast(Sleef_cast_to_doubleq1(q)); -} - -Sleef_quad from_double(double d) { - return Sleef_cast_from_doubleq1(d); -} - -void test_sleef_basic() { - std::cout << "\n=== Testing SLEEF Basic Operations ===\n"; - - Sleef_quad a = from_double(2.0); - Sleef_quad b = from_double(3.0); - - Sleef_quad sum = Sleef_addq1_u05(a, b); - Sleef_quad prod = Sleef_mulq1_u05(a, b); - Sleef_quad fma_result = Sleef_fmaq1_u05(a, b, from_double(1.0)); // a*b + 1 - - std::cout << "a = " << to_double(a) << std::endl; - std::cout << "b = " << to_double(b) << std::endl; - std::cout << "a + b = " << to_double(sum) << " (expected 5)" << std::endl; - std::cout << "a * b = " << to_double(prod) << " (expected 6)" << std::endl; - std::cout << "a*b + 1 = " << to_double(fma_result) << " (expected 7)" << std::endl; -} - -void test_simd_wrapper() { - std::cout << "\n=== Testing SIMD Wrapper ===\n"; - - QuadBLAS::QuadVector v1(from_double(2.0), from_double(3.0)); - QuadBLAS::QuadVector v2(from_double(4.0), from_double(5.0)); - - std::cout << "v1: [" << to_double(v1.get(0)) << ", " << to_double(v1.get(1)) << "]" << std::endl; - std::cout << "v2: [" << to_double(v2.get(0)) << ", " << to_double(v2.get(1)) << "]" << std::endl; - - QuadBLAS::QuadVector sum = v1 + v2; - QuadBLAS::QuadVector prod = v1 * v2; - - std::cout << "v1 + v2: [" << to_double(sum.get(0)) << ", " << to_double(sum.get(1)) << "]" << std::endl; - std::cout << "v1 * v2: [" << to_double(prod.get(0)) << ", " << to_double(prod.get(1)) << "]" << std::endl; - - Sleef_quad h_sum = sum.horizontal_sum(); - std::cout << "Horizontal sum of v1+v2: " << to_double(h_sum) << " (expected 14)" << std::endl; -} - -void test_manual_dot() { - std::cout << "\n=== Testing Manual Dot Product ===\n"; - - const size_t n = 5; - Sleef_quad x[5] = {from_double(1), from_double(2), from_double(3), from_double(4), from_double(5)}; - Sleef_quad y[5] = {from_double(1), from_double(1), from_double(1), from_double(1), from_double(1)}; - - std::cout << "x: ["; - for (size_t i = 0; i < n; ++i) { - std::cout << to_double(x[i]); - if (i < n-1) std::cout << ", "; - } - std::cout << "]" << std::endl; - - std::cout << "y: ["; - for (size_t i = 0; i < n; ++i) { - std::cout << to_double(y[i]); - if (i < n-1) std::cout << ", "; - } - std::cout << "]" << std::endl; - - // Manual scalar dot product - Sleef_quad manual_result = from_double(0.0); - for (size_t i = 0; i < n; ++i) { - manual_result = Sleef_fmaq1_u05(x[i], y[i], manual_result); - } - std::cout << "Manual scalar dot: " << to_double(manual_result) << " (expected 15)" << std::endl; - - // Test vectorized kernel - Sleef_quad vectorized_result = QuadBLAS::dot_kernel_vectorized(x, y, n); - std::cout << "Vectorized dot: " << to_double(vectorized_result) << " (expected 15)" << std::endl; - - // Test full dot function - Sleef_quad full_result = QuadBLAS::dot(n, x, 1, y, 1); - std::cout << "Full dot function: " << to_double(full_result) << " (expected 15)" << std::endl; -} - -void test_vector_class() { - std::cout << "\n=== Testing Vector Class ===\n"; - - const size_t n = 5; - QuadBLAS::DefaultVector<> x(n), y(n); - - std::cout << "Initializing vectors..." << std::endl; - for (size_t i = 0; i < n; ++i) { - x[i] = from_double(i + 1); // 1, 2, 3, 4, 5 - y[i] = from_double(1.0); // 1, 1, 1, 1, 1 - } - - std::cout << "x: ["; - for (size_t i = 0; i < n; ++i) { - std::cout << to_double(x[i]); - if (i < n-1) std::cout << ", "; - } - std::cout << "]" << std::endl; - - std::cout << "y: ["; - for (size_t i = 0; i < n; ++i) { - std::cout << to_double(y[i]); - if (i < n-1) std::cout << ", "; - } - std::cout << "]" << std::endl; - - Sleef_quad result = x.dot(y); - std::cout << "Vector class dot: " << to_double(result) << " (expected 15)" << std::endl; -} - -void test_memory_alignment() { - std::cout << "\n=== Testing Memory Alignment ===\n"; - - Sleef_quad* aligned_ptr = QuadBLAS::aligned_alloc(10); - if (aligned_ptr) { - uintptr_t addr = reinterpret_cast(aligned_ptr); - std::cout << "Aligned pointer: " << std::hex << addr << std::dec << std::endl; - std::cout << "Alignment check: " << (addr % QuadBLAS::ALIGNMENT == 0 ? "PASS" : "FAIL") << std::endl; - - // Test basic operations with aligned memory - for (int i = 0; i < 5; ++i) { - aligned_ptr[i] = from_double(i + 1); - } - - std::cout << "Aligned array: ["; - for (int i = 0; i < 5; ++i) { - std::cout << to_double(aligned_ptr[i]); - if (i < 4) std::cout << ", "; - } - std::cout << "]" << std::endl; - - QuadBLAS::aligned_free(aligned_ptr); - } else { - std::cout << "ERROR: Could not allocate aligned memory!" << std::endl; - } -} - -void test_load_store() { - std::cout << "\n=== Testing SIMD Load/Store ===\n"; - - Sleef_quad data[4] = {from_double(1.0), from_double(2.0), from_double(3.0), from_double(4.0)}; - - std::cout << "Original data: [" << to_double(data[0]) << ", " << to_double(data[1]) - << ", " << to_double(data[2]) << ", " << to_double(data[3]) << "]" << std::endl; - - // Test loading first pair - QuadBLAS::QuadVector vec1 = QuadBLAS::QuadVector::load(&data[0]); - std::cout << "Loaded vec1: [" << to_double(vec1.get(0)) << ", " << to_double(vec1.get(1)) << "]" << std::endl; - - // Test loading second pair - QuadBLAS::QuadVector vec2 = QuadBLAS::QuadVector::load(&data[2]); - std::cout << "Loaded vec2: [" << to_double(vec2.get(0)) << ", " << to_double(vec2.get(1)) << "]" << std::endl; - - // Test store - Sleef_quad result[2]; - vec1.store(result); - std::cout << "Stored vec1: [" << to_double(result[0]) << ", " << to_double(result[1]) << "]" << std::endl; -} - -int main() { - std::cout << "=== QuadBLAS Debug Tests ===\n"; - std::cout << quadblas_get_version() << std::endl; - -#ifdef QUADBLAS_X86_64 - std::cout << "Platform: x86-64" << std::endl; -#else - std::cout << "Platform: Generic" << std::endl; -#endif - - try { - test_sleef_basic(); - test_simd_wrapper(); - test_memory_alignment(); - test_load_store(); - test_manual_dot(); - test_vector_class(); - - } catch (const std::exception& e) { - std::cerr << "Error: " << e.what() << std::endl; - return 1; - } - - return 0; -} \ No newline at end of file diff --git a/tests/python/qblas_ctypes.py b/tests/python/qblas_ctypes.py new file mode 100644 index 0000000..b68b422 --- /dev/null +++ b/tests/python/qblas_ctypes.py @@ -0,0 +1,211 @@ +"""ctypes wrapper around libqblas via qblas_shim.so. The shim re-exports +every entry point with Sleef_quad scalars passed by pointer, since +ctypes / libffi does not reliably handle 16-byte struct-by-value args.""" + +from __future__ import annotations + +import ctypes +import sys + +import numpy as np + +# ctest adds .sleef-prefix/lib and the build's library dirs to +# {LD,DYLD}_LIBRARY_PATH; ctypes.CDLL hands these basenames to the +# dynamic loader, which resolves them along with SLEEF transitively. +_EXT = ".dylib" if sys.platform == "darwin" else ".so" + +_qblas = ctypes.CDLL(f"libqblas{_EXT}", mode=ctypes.RTLD_GLOBAL) +_shim = ctypes.CDLL(f"libqblas_shim{_EXT}", mode=ctypes.RTLD_GLOBAL) + +class Quad(ctypes.Structure): + _fields_ = [("x", ctypes.c_uint64), ("y", ctypes.c_uint64)] + +QUAD_DTYPE = np.dtype([("x", " Quad: + q = Quad() + _shim.shim_d2q(ctypes.c_double(d), ctypes.byref(q)) + return q + +def q2d(q: Quad) -> float: + return float(_shim.shim_q2d(ctypes.byref(q))) + +def doubles_to_quads(arr: np.ndarray) -> np.ndarray: + arr = np.ascontiguousarray(arr, dtype=np.float64).ravel() + out = np.empty(arr.shape, dtype=QUAD_DTYPE) + for i, x in enumerate(arr): + q = d2q(float(x)) + out[i] = (q.x, q.y) + return out + +def quads_to_doubles(qs: np.ndarray) -> np.ndarray: + qs = qs.reshape(-1) + out = np.empty(qs.shape, dtype=np.float64) + q = Quad() + for i in range(qs.shape[0]): + q.x = int(qs[i]["x"]); q.y = int(qs[i]["y"]) + out[i] = q2d(q) + return out + +QblasRowMajor = 101 +QblasColMajor = 102 +QblasNoTrans = 111 +QblasTrans = 112 +QblasUpper = 121 +QblasLower = 122 +QblasNonUnit = 131 +QblasUnit = 132 +QblasLeft = 141 +QblasRight = 142 + +_qblas.qblas_get_dispatch_tier.restype = ctypes.c_char_p +def dispatch_tier() -> str: + return _qblas.qblas_get_dispatch_tier().decode() + +_qblas.qblas_get_num_threads.restype = ctypes.c_int +def num_threads() -> int: + return int(_qblas.qblas_get_num_threads()) + +PQ = ctypes.POINTER(Quad) +VP = ctypes.c_void_p +INT = ctypes.c_int +SZ = ctypes.c_size_t + +_shim.shim_qdot.argtypes = [INT, VP, INT, VP, INT, PQ] +_shim.shim_qdot.restype = None +_shim.shim_qnrm2.argtypes = [INT, VP, INT, PQ] +_shim.shim_qnrm2.restype = None +_shim.shim_qasum.argtypes = [INT, VP, INT, PQ] +_shim.shim_qasum.restype = None +_shim.shim_iqamax.argtypes = [INT, VP, INT] +_shim.shim_iqamax.restype = SZ +_shim.shim_qaxpy.argtypes = [INT, PQ, VP, INT, VP, INT] +_shim.shim_qaxpy.restype = None +_shim.shim_qscal.argtypes = [INT, PQ, VP, INT] +_shim.shim_qscal.restype = None +_shim.shim_qgemv.argtypes = [INT, INT, INT, INT, PQ, VP, INT, VP, INT, PQ, VP, INT] +_shim.shim_qgemv.restype = None +_shim.shim_qger.argtypes = [INT, INT, INT, PQ, VP, INT, VP, INT, VP, INT] +_shim.shim_qger.restype = None +_shim.shim_qgemm.argtypes = [INT, INT, INT, INT, INT, INT, PQ, VP, INT, VP, INT, PQ, VP, INT] +_shim.shim_qgemm.restype = None +_shim.shim_qsyrk.argtypes = [INT, INT, INT, INT, INT, PQ, VP, INT, PQ, VP, INT] +_shim.shim_qsyrk.restype = None +_shim.shim_qtrmm.argtypes = [INT, INT, INT, INT, INT, INT, INT, PQ, VP, INT, VP, INT] +_shim.shim_qtrmm.restype = None +_shim.shim_qtrsm.argtypes = [INT, INT, INT, INT, INT, INT, INT, PQ, VP, INT, VP, INT] +_shim.shim_qtrsm.restype = None + +def _quad_buf(arr_f64): + buf = doubles_to_quads(arr_f64) + return buf, buf.ctypes.data + +def _q_ptr(d: float): + q = d2q(d) + return q, ctypes.byref(q) + +def qdot(x, y) -> float: + xb, xp = _quad_buf(x); yb, yp = _quad_buf(y) + out = Quad() + _shim.shim_qdot(x.size, xp, 1, yp, 1, ctypes.byref(out)) + return q2d(out) + +def qnrm2(x) -> float: + xb, xp = _quad_buf(x); out = Quad() + _shim.shim_qnrm2(x.size, xp, 1, ctypes.byref(out)) + return q2d(out) + +def qasum(x) -> float: + xb, xp = _quad_buf(x); out = Quad() + _shim.shim_qasum(x.size, xp, 1, ctypes.byref(out)) + return q2d(out) + +def iqamax(x) -> int: + xb, xp = _quad_buf(x) + return int(_shim.shim_iqamax(x.size, xp, 1)) + +def qaxpy(alpha: float, x, y): + n = x.size + xb, xp = _quad_buf(x); yb, yp = _quad_buf(y) + a, ap = _q_ptr(alpha) + _shim.shim_qaxpy(n, ap, xp, 1, yp, 1) + return quads_to_doubles(yb) + +def qscal(alpha: float, x): + xb, xp = _quad_buf(x); a, ap = _q_ptr(alpha) + _shim.shim_qscal(x.size, ap, xp, 1) + return quads_to_doubles(xb) + +def qgemv(A, x, y, alpha=1.0, beta=0.0, + trans=QblasNoTrans, layout=QblasRowMajor): + A = np.ascontiguousarray(A, dtype=np.float64) + m, n = A.shape; lda = n + Ab, Ap = _quad_buf(A); xb, xp = _quad_buf(x); yb, yp = _quad_buf(y) + a, ap = _q_ptr(alpha); b, bp = _q_ptr(beta) + _shim.shim_qgemv(layout, trans, m, n, ap, Ap, lda, xp, 1, bp, yp, 1) + return quads_to_doubles(yb) + +def qger(A, x, y, alpha=1.0, layout=QblasRowMajor): + A = np.ascontiguousarray(A, dtype=np.float64) + m, n = A.shape; lda = n + Ab, Ap = _quad_buf(A); xb, xp = _quad_buf(x); yb, yp = _quad_buf(y) + a, ap = _q_ptr(alpha) + _shim.shim_qger(layout, m, n, ap, xp, 1, yp, 1, Ap, lda) + return quads_to_doubles(Ab).reshape(m, n) + +def qgemm(A, B, C, alpha=1.0, beta=0.0, + ta=QblasNoTrans, tb=QblasNoTrans, layout=QblasRowMajor): + A = np.ascontiguousarray(A, dtype=np.float64) + B = np.ascontiguousarray(B, dtype=np.float64) + C = np.ascontiguousarray(C, dtype=np.float64) + if ta == QblasNoTrans: m, k = A.shape + else: k, m = A.shape + if tb == QblasNoTrans: kk, n = B.shape + else: n, kk = B.shape + assert k == kk, f"k mismatch: {k} vs {kk}" + assert C.shape == (m, n) + lda = A.shape[1]; ldb = B.shape[1]; ldc = n + Ab, Ap = _quad_buf(A); Bb, Bp = _quad_buf(B); Cb, Cp = _quad_buf(C) + a, ap = _q_ptr(alpha); b, bp = _q_ptr(beta) + _shim.shim_qgemm(layout, ta, tb, m, n, k, ap, Ap, lda, Bp, ldb, bp, Cp, ldc) + return quads_to_doubles(Cb).reshape(m, n) + +def qsyrk(A, C, alpha=1.0, beta=0.0, + uplo=QblasUpper, trans=QblasNoTrans, layout=QblasRowMajor): + A = np.ascontiguousarray(A, dtype=np.float64) + C = np.ascontiguousarray(C, dtype=np.float64) + if trans == QblasNoTrans: n, k = A.shape + else: k, n = A.shape + assert C.shape == (n, n) + Ab, Ap = _quad_buf(A); Cb, Cp = _quad_buf(C) + a, ap = _q_ptr(alpha); b, bp = _q_ptr(beta) + _shim.shim_qsyrk(layout, uplo, trans, n, k, ap, Ap, A.shape[1], bp, Cp, n) + return quads_to_doubles(Cb).reshape(n, n) + +def qtrmm(A, B, alpha=1.0, + side=QblasLeft, uplo=QblasLower, trans=QblasNoTrans, diag=QblasNonUnit, + layout=QblasRowMajor): + A = np.ascontiguousarray(A, dtype=np.float64) + B = np.ascontiguousarray(B, dtype=np.float64) + m, n = B.shape; lda = A.shape[1] + Ab, Ap = _quad_buf(A); Bb, Bp = _quad_buf(B) + a, ap = _q_ptr(alpha) + _shim.shim_qtrmm(layout, side, uplo, trans, diag, m, n, ap, Ap, lda, Bp, n) + return quads_to_doubles(Bb).reshape(m, n) + +def qtrsm(A, B, alpha=1.0, + side=QblasLeft, uplo=QblasLower, trans=QblasNoTrans, diag=QblasNonUnit, + layout=QblasRowMajor): + A = np.ascontiguousarray(A, dtype=np.float64) + B = np.ascontiguousarray(B, dtype=np.float64) + m, n = B.shape; lda = A.shape[1] + Ab, Ap = _quad_buf(A); Bb, Bp = _quad_buf(B) + a, ap = _q_ptr(alpha) + _shim.shim_qtrsm(layout, side, uplo, trans, diag, m, n, ap, Ap, lda, Bp, n) + return quads_to_doubles(Bb).reshape(m, n) diff --git a/tests/python/qblas_shim.c b/tests/python/qblas_shim.c new file mode 100644 index 0000000..9c72b2c --- /dev/null +++ b/tests/python/qblas_shim.c @@ -0,0 +1,99 @@ +/* Wraps the qblas entry points so every Sleef_quad scalar is passed by + * pointer instead of by value. ctypes / libffi mishandles the SysV + * AMD64 rule for 16-byte struct-by-value arguments and bus-errors on + * the first call; this shim sidesteps that. */ +#include + +#define EXP __attribute__((visibility("default"))) + +EXP void shim_qaxpy(int n, const Sleef_quad *alpha, + const void *x, int incx, void *y, int incy) { + cblas_qaxpy(n, *alpha, (const Sleef_quad *)x, incx, + (Sleef_quad *)y, incy); +} +EXP void shim_qscal(int n, const Sleef_quad *alpha, void *x, int incx) { + cblas_qscal(n, *alpha, (Sleef_quad *)x, incx); +} + +EXP void shim_qgemv(int layout, int trans, int m, int n, + const Sleef_quad *alpha, + const void *A, int lda, const void *x, int incx, + const Sleef_quad *beta, void *y, int incy) { + cblas_qgemv((QBLAS_LAYOUT)layout, (QBLAS_TRANSPOSE)trans, m, n, + *alpha, (const Sleef_quad *)A, lda, + (const Sleef_quad *)x, incx, *beta, + (Sleef_quad *)y, incy); +} + +EXP void shim_qger(int layout, int m, int n, + const Sleef_quad *alpha, + const void *x, int incx, const void *y, int incy, + void *A, int lda) { + cblas_qger((QBLAS_LAYOUT)layout, m, n, *alpha, + (const Sleef_quad *)x, incx, + (const Sleef_quad *)y, incy, + (Sleef_quad *)A, lda); +} + +EXP void shim_qgemm(int layout, int ta, int tb, int m, int n, int k, + const Sleef_quad *alpha, + const void *A, int lda, const void *B, int ldb, + const Sleef_quad *beta, void *C, int ldc) { + cblas_qgemm((QBLAS_LAYOUT)layout, (QBLAS_TRANSPOSE)ta, + (QBLAS_TRANSPOSE)tb, m, n, k, + *alpha, (const Sleef_quad *)A, lda, + (const Sleef_quad *)B, ldb, *beta, + (Sleef_quad *)C, ldc); +} + +EXP void shim_qsyrk(int layout, int uplo, int trans, int n, int k, + const Sleef_quad *alpha, + const void *A, int lda, + const Sleef_quad *beta, void *C, int ldc) { + cblas_qsyrk((QBLAS_LAYOUT)layout, (QBLAS_UPLO)uplo, + (QBLAS_TRANSPOSE)trans, n, k, + *alpha, (const Sleef_quad *)A, lda, + *beta, (Sleef_quad *)C, ldc); +} + +EXP void shim_qtrmm(int layout, int side, int uplo, int trans, int diag, + int m, int n, const Sleef_quad *alpha, + const void *A, int lda, void *B, int ldb) { + cblas_qtrmm((QBLAS_LAYOUT)layout, (QBLAS_SIDE)side, (QBLAS_UPLO)uplo, + (QBLAS_TRANSPOSE)trans, (QBLAS_DIAG)diag, m, n, + *alpha, (const Sleef_quad *)A, lda, + (Sleef_quad *)B, ldb); +} + +EXP void shim_qtrsm(int layout, int side, int uplo, int trans, int diag, + int m, int n, const Sleef_quad *alpha, + const void *A, int lda, void *B, int ldb) { + cblas_qtrsm((QBLAS_LAYOUT)layout, (QBLAS_SIDE)side, (QBLAS_UPLO)uplo, + (QBLAS_TRANSPOSE)trans, (QBLAS_DIAG)diag, m, n, + *alpha, (const Sleef_quad *)A, lda, + (Sleef_quad *)B, ldb); +} + +/* Return Sleef_quad through an out pointer. */ +EXP void shim_qdot(int n, const void *x, int incx, + const void *y, int incy, Sleef_quad *out) { + *out = cblas_qdot(n, (const Sleef_quad *)x, incx, + (const Sleef_quad *)y, incy); +} +EXP void shim_qnrm2(int n, const void *x, int incx, Sleef_quad *out) { + *out = cblas_qnrm2(n, (const Sleef_quad *)x, incx); +} +EXP void shim_qasum(int n, const void *x, int incx, Sleef_quad *out) { + *out = cblas_qasum(n, (const Sleef_quad *)x, incx); +} +EXP size_t shim_iqamax(int n, const void *x, int incx) { + return cblas_iqamax(n, (const Sleef_quad *)x, incx); +} + +/* Scalar Sleef_quad constructors / casts via out pointers. */ +EXP void shim_d2q(double v, Sleef_quad *out) { + *out = Sleef_cast_from_doubleq1(v); +} +EXP double shim_q2d(const Sleef_quad *in) { + return (double)Sleef_cast_to_doubleq1(*in); +} diff --git a/tests/python/test_against_numpy.py b/tests/python/test_against_numpy.py new file mode 100644 index 0000000..30ef387 --- /dev/null +++ b/tests/python/test_against_numpy.py @@ -0,0 +1,206 @@ +"""Compare qblas results (cast back to float64) against numpy at +double precision. Tolerances are chosen per routine to allow for the +double-rounding step.""" + +from __future__ import annotations + +import sys +import numpy as np + +import qblas_ctypes as qb + +RNG = np.random.default_rng(2026_05_14) + +def rel_err(actual: np.ndarray | float, expected: np.ndarray | float) -> float: + a = np.asarray(actual, dtype=np.float64) + e = np.asarray(expected, dtype=np.float64) + num = np.linalg.norm((a - e).ravel(), ord=np.inf) + den = np.linalg.norm(e.ravel(), ord=np.inf) + 1e-300 + return float(num / den) + +def gen_vec(n: int, scale: float = 1.0) -> np.ndarray: + return RNG.uniform(-scale, scale, size=n) + +def gen_mat(m: int, n: int, scale: float = 1.0) -> np.ndarray: + return RNG.uniform(-scale, scale, size=(m, n)) + +PASSED = 0 +FAILED = 0 +FAILS: list[str] = [] + +def check(name: str, actual, expected, tol: float): + global PASSED, FAILED + try: + r = rel_err(actual, expected) + except Exception as e: + FAILED += 1 + FAILS.append(f"{name}: EXCEPTION {e}") + return + if r <= tol: + PASSED += 1 + else: + FAILED += 1 + FAILS.append(f"{name}: rel_err={r:.3e} > tol={tol:.0e}") + +def test_qdot(): + for n in (1, 7, 64, 1023, 4096, 16384): + x = gen_vec(n); y = gen_vec(n) + check(f"qdot n={n}", qb.qdot(x, y), x @ y, 1e-11) + +def test_qnrm2(): + for n in (1, 7, 64, 1023, 4096, 16384): + x = gen_vec(n) + check(f"qnrm2 n={n}", qb.qnrm2(x), np.linalg.norm(x), 1e-12) + +def test_qasum(): + for n in (1, 7, 64, 1023, 4096, 16384): + x = gen_vec(n) + check(f"qasum n={n}", qb.qasum(x), np.sum(np.abs(x)), 1e-12) + +def test_iqamax(): + for n in (1, 7, 64, 1023, 4096): + x = gen_vec(n) + if abs(np.max(np.abs(x)) - np.min(np.abs(x))) < 1e-12: + continue # ambiguous tie + got = qb.iqamax(x) + want = int(np.argmax(np.abs(x))) + check(f"iqamax n={n}", got, want, 0) # exact integer match + +def test_qaxpy(): + for n in (1, 7, 64, 1023, 4096, 16384): + x = gen_vec(n); y = gen_vec(n); alpha = float(RNG.uniform(-2, 2)) + got = qb.qaxpy(alpha, x, y) + want = alpha * x + y + check(f"qaxpy n={n}", got, want, 1e-13) + +def test_qscal(): + for n in (1, 7, 64, 1023, 4096, 16384): + x = gen_vec(n); alpha = float(RNG.uniform(-2, 2)) + got = qb.qscal(alpha, x) + want = alpha * x + check(f"qscal n={n}", got, want, 1e-13) + +def test_qgemv(): + for (m, n) in [(1, 1), (5, 7), (32, 32), (64, 33), (127, 65)]: + for trans in (qb.QblasNoTrans, qb.QblasTrans): + A = gen_mat(m, n) + xn = n if trans == qb.QblasNoTrans else m + yn = m if trans == qb.QblasNoTrans else n + x = gen_vec(xn); y = gen_vec(yn) + alpha = float(RNG.uniform(-2, 2)) + beta = float(RNG.uniform(-2, 2)) + got = qb.qgemv(A, x, y.copy(), alpha=alpha, beta=beta, trans=trans) + opA = A if trans == qb.QblasNoTrans else A.T + want = alpha * (opA @ x) + beta * y + check(f"qgemv m={m} n={n} trans={trans}", got, want, 1e-10) + +def test_qger(): + for (m, n) in [(1, 1), (5, 7), (32, 32), (64, 33)]: + A = gen_mat(m, n); x = gen_vec(m); y = gen_vec(n) + alpha = float(RNG.uniform(-2, 2)) + got = qb.qger(A.copy(), x, y, alpha=alpha) + want = A + alpha * np.outer(x, y) + check(f"qger m={m} n={n}", got, want, 1e-12) + +def test_qsyrk(): + for (n, k) in [(3, 3), (8, 5), (17, 11), (32, 32)]: + A = gen_mat(n, k) + C = gen_mat(n, n) + alpha = float(RNG.uniform(-2, 2)) + beta = float(RNG.uniform(-2, 2)) + got = qb.qsyrk(A, C.copy(), alpha=alpha, beta=beta, + uplo=qb.QblasUpper, trans=qb.QblasNoTrans) + want = alpha * (A @ A.T) + beta * C + tol = max(1e-12, 1e-14 * k) + check(f"qsyrk n={n} k={k}", got, want, tol) + +def test_qtrmm(): + for (m, n) in [(3, 3), (8, 5), (17, 11), (64, 33)]: + # Diagonally dominant for a well-conditioned solve. + A = np.tril(gen_mat(m, m, scale=0.5)) + for i in range(m): A[i, i] = float(RNG.uniform(1.5, 2.5)) + B = gen_mat(m, n) + alpha = float(RNG.uniform(-2, 2)) + got = qb.qtrmm(A, B.copy(), alpha=alpha, + side=qb.QblasLeft, uplo=qb.QblasLower, + trans=qb.QblasNoTrans, diag=qb.QblasNonUnit) + want = alpha * (A @ B) + tol = max(1e-12, 1e-14 * m) + check(f"qtrmm m={m} n={n}", got, want, tol) + +def test_qtrsm(): + for (m, n) in [(3, 3), (8, 5), (17, 11), (64, 33), (129, 11)]: + A = np.tril(gen_mat(m, m, scale=0.5)) + for i in range(m): A[i, i] = float(RNG.uniform(1.5, 2.5)) + B = gen_mat(m, n) + alpha = float(RNG.uniform(-2, 2)) + got = qb.qtrsm(A, B.copy(), alpha=alpha, + side=qb.QblasLeft, uplo=qb.QblasLower, + trans=qb.QblasNoTrans, diag=qb.QblasNonUnit) + # numpy.linalg.solve gives the reference X. Solver error scales + # like the condition number, so tol grows with m². + want = np.linalg.solve(A, alpha * B) + tol = max(1e-11, 1e-14 * m * m) + check(f"qtrsm m={m} n={n}", got, want, tol) + +def test_qgemm(): + for (m, n, k) in [(1, 1, 1), (3, 5, 7), (16, 16, 16), + (17, 19, 23), (64, 64, 64), (128, 65, 33)]: + for ta in (qb.QblasNoTrans, qb.QblasTrans): + for tb in (qb.QblasNoTrans, qb.QblasTrans): + A_shape = (m, k) if ta == qb.QblasNoTrans else (k, m) + B_shape = (k, n) if tb == qb.QblasNoTrans else (n, k) + A = gen_mat(*A_shape) + B = gen_mat(*B_shape) + C = gen_mat(m, n) + alpha = float(RNG.uniform(-2, 2)) + beta = float(RNG.uniform(-2, 2)) + got = qb.qgemm(A, B, C.copy(), + alpha=alpha, beta=beta, ta=ta, tb=tb) + opA = A if ta == qb.QblasNoTrans else A.T + opB = B if tb == qb.QblasNoTrans else B.T + want = alpha * (opA @ opB) + beta * C + tol = max(1e-12, 1e-14 * k) + check(f"qgemm m={m} n={n} k={k} ta={ta} tb={tb}", + got, want, tol) + +def main() -> int: + print(f"QBLAS dispatch tier : {qb.dispatch_tier()}") + print(f"QBLAS threads : {qb._qblas.qblas_get_num_threads()}") + print(f"numpy version : {np.__version__}") + print() + + tests = [ + ("qdot", test_qdot), + ("qnrm2", test_qnrm2), + ("qasum", test_qasum), + ("iqamax", test_iqamax), + ("qaxpy", test_qaxpy), + ("qscal", test_qscal), + ("qgemv", test_qgemv), + ("qger", test_qger), + ("qgemm", test_qgemm), + ("qsyrk", test_qsyrk), + ("qtrmm", test_qtrmm), + ("qtrsm", test_qtrsm), + ] + for name, fn in tests: + print(f" running {name} ...", end=" ", flush=True) + before = (PASSED, FAILED) + fn() + added_pass = PASSED - before[0] + added_fail = FAILED - before[1] + print(f"{added_pass} ok, {added_fail} fail") + + print() + print(f"Total: {PASSED} passed, {FAILED} failed") + if FAILED: + print() + print("Failures:") + for line in FAILS: + print(f" {line}") + return 1 + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/run_debug.sh b/tests/run_debug.sh deleted file mode 100644 index 6bdd17c..0000000 --- a/tests/run_debug.sh +++ /dev/null @@ -1,8 +0,0 @@ -g++ -O3 -fopenmp \ - -I/datadrive/guestuser/t-swsingh/temp/sleef/build/include \ - -L/datadrive/guestuser/t-swsingh/temp/sleef/build/lib \ - -Wl,-rpath,/datadrive/guestuser/t-swsingh/temp/sleef/build/lib \ - -o benchmark_threaded benchmark.cpp \ - -lsleef -lsleefquad - -./benchmark_threaded \ No newline at end of file diff --git a/tests/test_gemm.c b/tests/test_gemm.c new file mode 100644 index 0000000..1b5c0d3 --- /dev/null +++ b/tests/test_gemm.c @@ -0,0 +1,119 @@ +#include "test_helpers.h" + +static void ref_gemm_row(int doTa, int doTb, int m, int n, int k, + Sleef_quad alpha, + const Sleef_quad *A, int lda, + const Sleef_quad *B, int ldb, + Sleef_quad beta, + Sleef_quad *C, int ldc) { + for (int i = 0; i < m; ++i) { + for (int j = 0; j < n; ++j) { + Sleef_quad s = qd(0.0); + for (int p = 0; p < k; ++p) { + Sleef_quad a = doTa ? A[(size_t)p * lda + i] : A[(size_t)i * lda + p]; + Sleef_quad b = doTb ? B[(size_t)j * ldb + p] : B[(size_t)p * ldb + j]; + s = q_fma(a, b, s); + } + C[(size_t)i * ldc + j] = q_fma(beta, C[(size_t)i * ldc + j], + q_mul(alpha, s)); + } + } +} + +static void rowmajor_to_colmajor(const Sleef_quad *A, int rows, int cols, + Sleef_quad *B) { + for (int i = 0; i < rows; ++i) + for (int j = 0; j < cols; ++j) + B[(size_t)j * rows + i] = A[(size_t)i * cols + j]; +} + +static void test_gemm_case(int m, int n, int k, + QBLAS_LAYOUT layout, + QBLAS_TRANSPOSE ta, QBLAS_TRANSPOSE tb) { + int doTa = (ta == QblasTrans || ta == QblasConjTrans); + int doTb = (tb == QblasTrans || tb == QblasConjTrans); + + int Arows = doTa ? k : m, Acols = doTa ? m : k; + int Brows = doTb ? n : k, Bcols = doTb ? k : n; + + Sleef_quad *A_row = malloc((size_t)Arows * Acols * sizeof(Sleef_quad)); + Sleef_quad *B_row = malloc((size_t)Brows * Bcols * sizeof(Sleef_quad)); + Sleef_quad *C0 = malloc((size_t)m * n * sizeof(Sleef_quad)); + fill_mat(A_row, Arows, Acols, Acols); + fill_mat(B_row, Brows, Bcols, Bcols); + fill_mat(C0, m, n, n); + + Sleef_quad alpha = qd(0.6), beta = qd(-0.4); + + Sleef_quad *C_ref = malloc((size_t)m * n * sizeof(Sleef_quad)); + memcpy(C_ref, C0, (size_t)m * n * sizeof(Sleef_quad)); + ref_gemm_row(doTa, doTb, m, n, k, alpha, + A_row, Acols, B_row, Bcols, beta, C_ref, n); + + Sleef_quad *A_use, *B_use, *C_use; + int lda, ldb, ldc; + Sleef_quad *A_col=NULL, *B_col=NULL, *C_col=NULL; + if (layout == QblasRowMajor) { + A_use = A_row; lda = Acols; + B_use = B_row; ldb = Bcols; + C_use = malloc((size_t)m * n * sizeof(Sleef_quad)); + memcpy(C_use, C0, (size_t)m * n * sizeof(Sleef_quad)); + ldc = n; + } else { + A_col = malloc((size_t)Arows * Acols * sizeof(Sleef_quad)); + B_col = malloc((size_t)Brows * Bcols * sizeof(Sleef_quad)); + C_col = malloc((size_t)m * n * sizeof(Sleef_quad)); + rowmajor_to_colmajor(A_row, Arows, Acols, A_col); + rowmajor_to_colmajor(B_row, Brows, Bcols, B_col); + rowmajor_to_colmajor(C0, m, n, C_col); + A_use = A_col; lda = Arows; + B_use = B_col; ldb = Brows; + C_use = C_col; ldc = m; + } + + cblas_qgemm(layout, ta, tb, m, n, k, alpha, + A_use, lda, B_use, ldb, beta, C_use, ldc); + + Sleef_quad *C_got = malloc((size_t)m * n * sizeof(Sleef_quad)); + if (layout == QblasRowMajor) { + memcpy(C_got, C_use, (size_t)m * n * sizeof(Sleef_quad)); + } else { + for (int i = 0; i < m; ++i) + for (int j = 0; j < n; ++j) + C_got[(size_t)i * n + j] = C_use[(size_t)j * m + i]; + } + + char tag[160]; + snprintf(tag, sizeof tag, + "gemm m=%d n=%d k=%d layout=%d ta=%d tb=%d", + m, n, k, layout, ta, tb); + CHECK_ARR(C_got, C_ref, m * n, 1e-29, tag); + + free(C_got); free(C_ref); + if (layout == QblasRowMajor) free(C_use); + if (A_col) free(A_col); + if (B_col) free(B_col); + if (C_col) free(C_col); + free(C0); free(B_row); free(A_row); +} + +int main(void) { + + int sizes[][3] = { + {1, 1, 1}, {1, 3, 1}, {3, 1, 1}, + {4, 4, 4}, {5, 5, 5}, {7, 11, 13}, + {16, 16, 16}, {17, 19, 23}, + {32, 32, 32}, {64, 64, 64}, + {65, 65, 65}, {129, 33, 17}, + {128, 128, 128} + }; + for (size_t s = 0; s < sizeof sizes / sizeof sizes[0]; ++s) { + int m = sizes[s][0], n = sizes[s][1], k = sizes[s][2]; + for (int layout = QblasRowMajor; layout <= QblasColMajor; ++layout) + for (int ta = QblasNoTrans; ta <= QblasTrans; ++ta) + for (int tb = QblasNoTrans; tb <= QblasTrans; ++tb) + test_gemm_case(m, n, k, (QBLAS_LAYOUT)layout, + (QBLAS_TRANSPOSE)ta, (QBLAS_TRANSPOSE)tb); + } + REPORT(); +} diff --git a/tests/test_helpers.h b/tests/test_helpers.h new file mode 100644 index 0000000..1e0a140 --- /dev/null +++ b/tests/test_helpers.h @@ -0,0 +1,105 @@ +/* Shared helpers for the qblas test suite. */ +#ifndef QBLAS_TEST_HELPERS_H +#define QBLAS_TEST_HELPERS_H + +#include +#include +#include +#include +#include + +static unsigned int g_seed = 12345u; + +static inline unsigned int rnd(void) { + g_seed = g_seed * 1664525u + 1013904223u; + return g_seed; +} +static inline double frand(double lo, double hi) { + return lo + (hi - lo) * (rnd() / (double)0xFFFFFFFFu); +} + +static inline Sleef_quad qd(double x) { return Sleef_cast_from_doubleq1(x); } +static inline double dq(Sleef_quad x) { return (double)Sleef_cast_to_doubleq1(x); } + +static inline Sleef_quad q_add(Sleef_quad a, Sleef_quad b) { return Sleef_addq1_u05(a, b); } +static inline Sleef_quad q_sub(Sleef_quad a, Sleef_quad b) { return Sleef_subq1_u05(a, b); } +static inline Sleef_quad q_mul(Sleef_quad a, Sleef_quad b) { return Sleef_mulq1_u05(a, b); } +static inline Sleef_quad q_fma(Sleef_quad a, Sleef_quad b, Sleef_quad c) { + return Sleef_fmaq1_u05(a, b, c); +} +static inline Sleef_quad q_abs(Sleef_quad a) { return Sleef_fabsq1(a); } +static inline int q_lt (Sleef_quad a, Sleef_quad b) { return Sleef_icmpltq1(a, b); } +static inline int q_eq (Sleef_quad a, Sleef_quad b) { return Sleef_icmpeqq1(a, b); } + +static inline void fill_vec(Sleef_quad *v, int n, int stride) { + for (int i = 0; i < n; ++i) v[(size_t)i * stride] = qd(frand(-1.0, 1.0)); +} +static inline void fill_mat(Sleef_quad *A, int rows, int cols, int ld) { + for (int i = 0; i < rows; ++i) + for (int j = 0; j < cols; ++j) + A[(size_t)i * ld + j] = qd(frand(-1.0, 1.0)); +} + +/* Max relative error across two quad arrays, falling back to absolute + * diff when both values are essentially zero. */ +static inline double max_rel_err(const Sleef_quad *a, const Sleef_quad *b, + int n, int astride, int bstride) { + double worst = 0.0; + for (int i = 0; i < n; ++i) { + Sleef_quad av = a[(size_t)i * astride]; + Sleef_quad bv = b[(size_t)i * bstride]; + Sleef_quad d = q_abs(q_sub(av, bv)); + Sleef_quad s = q_add(q_abs(av), q_abs(bv)); + double dd = dq(d); + double ss = dq(s); + double r = (ss > 1e-20) ? (dd / ss) : dd; + if (r > worst) worst = r; + } + return worst; +} + +static int g_failures = 0; +static int g_checks = 0; + +#define CHECK(cond, ...) do { \ + ++g_checks; \ + if (!(cond)) { \ + ++g_failures; \ + fprintf(stderr, "FAIL %s:%d: ", __FILE__, __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fputc('\n', stderr); \ + } \ +} while (0) + +#define CHECK_NEAR(got, want, tol, what) do { \ + ++g_checks; \ + double _g = dq(got), _w = dq(want); \ + double _e = fabs(_g - _w); \ + if (_e > (tol) * (fabs(_w) + 1e-30)) { \ + ++g_failures; \ + fprintf(stderr, "FAIL %s:%d %s: got %.17g want %.17g err %.3e\n", \ + __FILE__, __LINE__, what, _g, _w, _e); \ + } \ +} while (0) + +#define CHECK_ARR(actual, expected, n, tol, what) do { \ + double _rel = max_rel_err((actual), (expected), (n), 1, 1); \ + ++g_checks; \ + if (_rel > (tol)) { \ + ++g_failures; \ + fprintf(stderr, "FAIL %s:%d %s: rel_err=%.3e (n=%d)\n", \ + __FILE__, __LINE__, what, _rel, (int)(n)); \ + } \ +} while (0) + +#define REPORT() do { \ + if (g_failures) { \ + fprintf(stderr, "%d of %d checks FAILED\n", g_failures, g_checks); \ + return 1; \ + } \ + printf("%d checks PASSED (tier=%s, threads=%d)\n", \ + g_checks, qblas_get_dispatch_tier(), qblas_get_num_threads()); \ + return 0; \ +} while (0) + +#endif diff --git a/tests/test_level1.c b/tests/test_level1.c new file mode 100644 index 0000000..249b29c --- /dev/null +++ b/tests/test_level1.c @@ -0,0 +1,152 @@ +#include "test_helpers.h" + +/* Naive scalar references. */ +static Sleef_quad ref_dot(int n, const Sleef_quad *x, int ix, + const Sleef_quad *y, int iy) { + int ox = (ix < 0) ? (n - 1) * (-ix) : 0; + int oy = (iy < 0) ? (n - 1) * (-iy) : 0; + Sleef_quad s = qd(0.0); + for (int i = 0; i < n; ++i) { + s = q_fma(x[ox + i * ix], y[oy + i * iy], s); + } + return s; +} +static Sleef_quad ref_asum(int n, const Sleef_quad *x, int ix) { + int ox = (ix < 0) ? (n - 1) * (-ix) : 0; + Sleef_quad s = qd(0.0); + for (int i = 0; i < n; ++i) s = q_add(s, q_abs(x[ox + i * ix])); + return s; +} +static int ref_iamax(int n, const Sleef_quad *x, int ix) { + int ox = (ix < 0) ? (n - 1) * (-ix) : 0; + Sleef_quad best = q_abs(x[ox]); + int arg = 0; + for (int i = 1; i < n; ++i) { + Sleef_quad v = q_abs(x[ox + i * ix]); + if (q_lt(best, v)) { best = v; arg = i; } + } + return arg; +} +static void ref_axpy(int n, Sleef_quad a, + const Sleef_quad *x, int ix, + Sleef_quad *y, int iy) { + int ox = (ix < 0) ? (n - 1) * (-ix) : 0; + int oy = (iy < 0) ? (n - 1) * (-iy) : 0; + for (int i = 0; i < n; ++i) + y[oy + i * iy] = q_fma(a, x[ox + i * ix], y[oy + i * iy]); +} +static void ref_scal(int n, Sleef_quad a, Sleef_quad *x, int ix) { + int ox = (ix < 0) ? (n - 1) * (-ix) : 0; + for (int i = 0; i < n; ++i) x[ox + i * ix] = q_mul(a, x[ox + i * ix]); +} + +static void test_dot(int n, int incx, int incy) { + int len_x = n * (incx < 0 ? -incx : incx); + int len_y = n * (incy < 0 ? -incy : incy); + Sleef_quad *x = (Sleef_quad *)malloc(len_x * sizeof(Sleef_quad)); + Sleef_quad *y = (Sleef_quad *)malloc(len_y * sizeof(Sleef_quad)); + fill_vec(x, len_x, 1); + fill_vec(y, len_y, 1); + + Sleef_quad got = cblas_qdot(n, x, incx, y, incy); + Sleef_quad want = ref_dot(n, x, incx, y, incy); + char tag[64]; snprintf(tag, sizeof tag, "dot n=%d incx=%d incy=%d", n, incx, incy); + CHECK_NEAR(got, want, 1e-30, tag); + + free(x); free(y); +} + +static void test_asum(int n, int incx) { + int len_x = n * (incx < 0 ? -incx : incx); + Sleef_quad *x = (Sleef_quad *)malloc(len_x * sizeof(Sleef_quad)); + fill_vec(x, len_x, 1); + Sleef_quad got = cblas_qasum(n, x, incx); + Sleef_quad want = ref_asum(n, x, incx); + char tag[64]; snprintf(tag, sizeof tag, "asum n=%d incx=%d", n, incx); + CHECK_NEAR(got, want, 1e-30, tag); + free(x); +} + +static void test_iamax(int n, int incx) { + int len_x = n * (incx < 0 ? -incx : incx); + Sleef_quad *x = (Sleef_quad *)malloc(len_x * sizeof(Sleef_quad)); + fill_vec(x, len_x, 1); + int got_ = (int)cblas_iqamax(n, x, incx); + int want = ref_iamax(n, x, incx); + char tag[64]; snprintf(tag, sizeof tag, "iamax n=%d incx=%d (got=%d want=%d)", n, incx, got_, want); + CHECK(got_ == want, "%s", tag); + free(x); +} + +static void test_axpy(int n, int incx, int incy) { + int len_x = n * (incx < 0 ? -incx : incx); + int len_y = n * (incy < 0 ? -incy : incy); + Sleef_quad *x = (Sleef_quad *)malloc(len_x * sizeof(Sleef_quad)); + Sleef_quad *y1 = (Sleef_quad *)malloc(len_y * sizeof(Sleef_quad)); + Sleef_quad *y2 = (Sleef_quad *)malloc(len_y * sizeof(Sleef_quad)); + fill_vec(x, len_x, 1); + fill_vec(y1, len_y, 1); + memcpy(y2, y1, len_y * sizeof(Sleef_quad)); + + Sleef_quad alpha = qd(0.75); + cblas_qaxpy(n, alpha, x, incx, y1, incy); + ref_axpy (n, alpha, x, incx, y2, incy); + char tag[64]; snprintf(tag, sizeof tag, "axpy n=%d incx=%d incy=%d", n, incx, incy); + CHECK_ARR(y1, y2, len_y, 1e-30, tag); + + free(x); free(y1); free(y2); +} + +static void test_scal(int n, int incx) { + int len_x = n * (incx < 0 ? -incx : incx); + Sleef_quad *x1 = (Sleef_quad *)malloc(len_x * sizeof(Sleef_quad)); + Sleef_quad *x2 = (Sleef_quad *)malloc(len_x * sizeof(Sleef_quad)); + fill_vec(x1, len_x, 1); + memcpy(x2, x1, len_x * sizeof(Sleef_quad)); + cblas_qscal(n, qd(-2.5), x1, incx); + ref_scal (n, qd(-2.5), x2, incx); + char tag[64]; snprintf(tag, sizeof tag, "scal n=%d incx=%d", n, incx); + CHECK_ARR(x1, x2, len_x, 1e-30, tag); + free(x1); free(x2); +} + +static void test_copy_swap(int n) { + Sleef_quad *x = (Sleef_quad *)malloc(n * sizeof(Sleef_quad)); + Sleef_quad *y = (Sleef_quad *)malloc(n * sizeof(Sleef_quad)); + Sleef_quad *xs = (Sleef_quad *)malloc(n * sizeof(Sleef_quad)); + fill_vec(x, n, 1); + fill_vec(y, n, 1); + memcpy(xs, x, n * sizeof(Sleef_quad)); + + Sleef_quad *yc = (Sleef_quad *)malloc(n * sizeof(Sleef_quad)); + cblas_qcopy(n, x, 1, yc, 1); + CHECK_ARR(yc, x, n, 1e-30, "copy"); + + cblas_qswap(n, x, 1, y, 1); + Sleef_quad *yorig = (Sleef_quad *)malloc(n * sizeof(Sleef_quad)); + memcpy(yorig, yc, n * sizeof(Sleef_quad)); + CHECK_ARR(y, xs, n, 1e-30, "swap-y"); + + free(yorig); free(yc); free(xs); free(x); free(y); +} + +int main(void) { + static int sizes[] = { 1, 7, 16, 31, 64, 100, 1023, 8192, 100000 }; + static int strides[] = { 1, 2, 3, -1, -2 }; + + for (size_t si = 0; si < sizeof sizes / sizeof sizes[0]; ++si) { + int n = sizes[si]; + for (size_t a = 0; a < sizeof strides / sizeof strides[0]; ++a) { + for (size_t b = 0; b < sizeof strides / sizeof strides[0]; ++b) { + test_dot (n, strides[a], strides[b]); + test_axpy(n, strides[a], strides[b]); + } + test_asum (n, strides[a]); + test_scal (n, strides[a]); + test_iamax(n, strides[a]); + } + test_copy_swap(n); + } + + REPORT(); +} diff --git a/tests/test_level2.c b/tests/test_level2.c new file mode 100644 index 0000000..fce4c6a --- /dev/null +++ b/tests/test_level2.c @@ -0,0 +1,231 @@ +#include "test_helpers.h" + +/* Row-major naive reference. */ +static void ref_gemv_row(int doT, int m, int n, + Sleef_quad alpha, + const Sleef_quad *A, int lda, + const Sleef_quad *x, int incx, + Sleef_quad beta, + Sleef_quad *y, int incy) { + int rows = doT ? n : m; + int cols = doT ? m : n; + int ox = (incx < 0) ? (cols - 1) * (-incx) : 0; + int oy = (incy < 0) ? (rows - 1) * (-incy) : 0; + for (int i = 0; i < rows; ++i) { + Sleef_quad s = qd(0.0); + for (int j = 0; j < cols; ++j) { + Sleef_quad aij = doT ? A[(size_t)j * lda + i] : A[(size_t)i * lda + j]; + s = q_fma(aij, x[ox + j * incx], s); + } + Sleef_quad scaled = q_mul(alpha, s); + Sleef_quad oldy = y[oy + i * incy]; + y[oy + i * incy] = q_fma(beta, oldy, scaled); + } +} + +static void to_colmajor(const Sleef_quad *Arow, int M, int N, int lda_row, + Sleef_quad *Bcol, int ldb_col) { + for (int i = 0; i < M; ++i) + for (int j = 0; j < N; ++j) + Bcol[(size_t)j * ldb_col + i] = Arow[(size_t)i * lda_row + j]; +} + +static void test_gemv_case(int m, int n, QBLAS_LAYOUT layout, QBLAS_TRANSPOSE trans, + int incx, int incy) { + + Sleef_quad *Arow = (Sleef_quad *)malloc((size_t)m * n * sizeof(Sleef_quad)); + fill_mat(Arow, m, n, n); + + int doT = (trans == QblasTrans || trans == QblasConjTrans); + int rows_out = doT ? n : m; + int cols_in = doT ? m : n; + + int len_x = cols_in * (incx < 0 ? -incx : incx); + int len_y = rows_out * (incy < 0 ? -incy : incy); + + Sleef_quad *x = (Sleef_quad *)malloc(len_x * sizeof(Sleef_quad)); + Sleef_quad *y0 = (Sleef_quad *)malloc(len_y * sizeof(Sleef_quad)); + fill_vec(x, len_x, 1); + fill_vec(y0, len_y, 1); + + Sleef_quad alpha = qd(0.75), beta = qd(-0.3); + Sleef_quad *y_ref = (Sleef_quad *)malloc(len_y * sizeof(Sleef_quad)); + memcpy(y_ref, y0, len_y * sizeof(Sleef_quad)); + ref_gemv_row(doT, m, n, alpha, Arow, n, x, incx, beta, y_ref, incy); + + Sleef_quad *A_use; + int lda_use; + Sleef_quad *Acol = NULL; + if (layout == QblasRowMajor) { + A_use = Arow; lda_use = n; + } else { + Acol = (Sleef_quad *)malloc((size_t)m * n * sizeof(Sleef_quad)); + to_colmajor(Arow, m, n, n, Acol, m); + A_use = Acol; lda_use = m; + } + + Sleef_quad *y_got = (Sleef_quad *)malloc(len_y * sizeof(Sleef_quad)); + memcpy(y_got, y0, len_y * sizeof(Sleef_quad)); + cblas_qgemv(layout, trans, m, n, alpha, A_use, lda_use, x, incx, beta, y_got, incy); + + char tag[128]; + snprintf(tag, sizeof tag, "gemv m=%d n=%d layout=%d trans=%d incx=%d incy=%d", + m, n, layout, trans, incx, incy); + CHECK_ARR(y_got, y_ref, len_y, 1e-29, tag); + + free(y_got); free(y_ref); free(y0); free(x); free(Arow); + if (Acol) free(Acol); +} + +static void test_gemv_all(void) { + int dims[][2] = { {1,1}, {3,5}, {17,1}, {1,32}, {64,33}, {7, 11}, {32, 32}, {127, 65} }; + int strides[] = { 1, 2, -1 }; + for (size_t d = 0; d < sizeof dims / sizeof dims[0]; ++d) { + for (size_t s1 = 0; s1 < sizeof strides / sizeof strides[0]; ++s1) { + for (size_t s2 = 0; s2 < sizeof strides / sizeof strides[0]; ++s2) { + for (int layout = QblasRowMajor; layout <= QblasColMajor; ++layout) { + for (int t = QblasNoTrans; t <= QblasTrans; ++t) { + test_gemv_case(dims[d][0], dims[d][1], + (QBLAS_LAYOUT)layout, (QBLAS_TRANSPOSE)t, + strides[s1], strides[s2]); + } + } + } + } + } +} + +static void ref_ger_row(int m, int n, + Sleef_quad alpha, + const Sleef_quad *x, int incx, + const Sleef_quad *y, int incy, + Sleef_quad *A, int lda) { + int ox = (incx < 0) ? (m - 1) * (-incx) : 0; + int oy = (incy < 0) ? (n - 1) * (-incy) : 0; + for (int i = 0; i < m; ++i) + for (int j = 0; j < n; ++j) + A[(size_t)i * lda + j] = q_fma(q_mul(alpha, x[ox + i * incx]), + y[oy + j * incy], + A[(size_t)i * lda + j]); +} + +static void test_ger_case(int m, int n, QBLAS_LAYOUT layout, int incx, int incy) { + int len_x = m * (incx < 0 ? -incx : incx); + int len_y = n * (incy < 0 ? -incy : incy); + Sleef_quad *x = (Sleef_quad *)malloc(len_x * sizeof(Sleef_quad)); + Sleef_quad *y = (Sleef_quad *)malloc(len_y * sizeof(Sleef_quad)); + Sleef_quad *Arow = (Sleef_quad *)malloc((size_t)m * n * sizeof(Sleef_quad)); + Sleef_quad *Arow0 = (Sleef_quad *)malloc((size_t)m * n * sizeof(Sleef_quad)); + fill_vec(x, len_x, 1); + fill_vec(y, len_y, 1); + fill_mat(Arow, m, n, n); + memcpy(Arow0, Arow, (size_t)m * n * sizeof(Sleef_quad)); + + Sleef_quad alpha = qd(0.5); + ref_ger_row(m, n, alpha, x, incx, y, incy, Arow0, n); + + Sleef_quad *A_use; int lda_use; Sleef_quad *Acol = NULL; + if (layout == QblasRowMajor) { + A_use = Arow; lda_use = n; + } else { + Acol = (Sleef_quad *)malloc((size_t)m * n * sizeof(Sleef_quad)); + to_colmajor(Arow, m, n, n, Acol, m); + A_use = Acol; lda_use = m; + } + cblas_qger(layout, m, n, alpha, x, incx, y, incy, A_use, lda_use); + + Sleef_quad *Arow_after = (Sleef_quad *)malloc((size_t)m * n * sizeof(Sleef_quad)); + if (layout == QblasRowMajor) { + memcpy(Arow_after, A_use, (size_t)m * n * sizeof(Sleef_quad)); + } else { + for (int i = 0; i < m; ++i) + for (int j = 0; j < n; ++j) + Arow_after[(size_t)i * n + j] = A_use[(size_t)j * lda_use + i]; + } + + char tag[128]; + snprintf(tag, sizeof tag, "ger m=%d n=%d layout=%d incx=%d incy=%d", m, n, layout, incx, incy); + CHECK_ARR(Arow_after, Arow0, m * n, 1e-29, tag); + + free(Arow_after); free(Arow0); free(Arow); free(x); free(y); + if (Acol) free(Acol); +} + +static void test_ger_all(void) { + int dims[][2] = { {1,1}, {3,5}, {17,8}, {32,32}, {64,17}, {7,11} }; + int strides[] = { 1, 2, -1 }; + for (size_t d = 0; d < sizeof dims / sizeof dims[0]; ++d) { + for (size_t s1 = 0; s1 < sizeof strides / sizeof strides[0]; ++s1) { + for (size_t s2 = 0; s2 < sizeof strides / sizeof strides[0]; ++s2) { + for (int layout = QblasRowMajor; layout <= QblasColMajor; ++layout) { + test_ger_case(dims[d][0], dims[d][1], + (QBLAS_LAYOUT)layout, strides[s1], strides[s2]); + } + } + } + } +} + +static void test_trsv_case(int n, QBLAS_LAYOUT layout, QBLAS_UPLO uplo, + QBLAS_TRANSPOSE trans, QBLAS_DIAG diag) { + Sleef_quad *Arow = (Sleef_quad *)malloc((size_t)n * n * sizeof(Sleef_quad)); + fill_mat(Arow, n, n, n); + /* Diagonally dominant: keeps the solve well-conditioned. */ + for (int i = 0; i < n; ++i) + Arow[(size_t)i * n + i] = qd(frand(2.0, 4.0)); + + Sleef_quad *x_orig = (Sleef_quad *)malloc(n * sizeof(Sleef_quad)); + fill_vec(x_orig, n, 1); + + int doT = (trans == QblasTrans || trans == QblasConjTrans); + int upper = (uplo == QblasUpper); + Sleef_quad *b = (Sleef_quad *)malloc(n * sizeof(Sleef_quad)); + for (int i = 0; i < n; ++i) { + Sleef_quad s = qd(0.0); + for (int j = 0; j < n; ++j) { + int ii = doT ? j : i; + int jj = doT ? i : j; + int in_triangle = upper ? (jj >= ii) : (jj <= ii); + if (!in_triangle) continue; + Sleef_quad aij; + if (diag == QblasUnit && ii == jj) aij = qd(1.0); + else aij = Arow[(size_t)ii * n + jj]; + s = q_fma(aij, x_orig[j], s); + } + b[i] = s; + } + + Sleef_quad *A_use; int lda_use; Sleef_quad *Acol = NULL; + if (layout == QblasRowMajor) { A_use = Arow; lda_use = n; } + else { Acol = malloc((size_t)n*n*sizeof(Sleef_quad)); to_colmajor(Arow, n, n, n, Acol, n); A_use = Acol; lda_use = n; } + + Sleef_quad *x = (Sleef_quad *)malloc(n * sizeof(Sleef_quad)); + memcpy(x, b, n * sizeof(Sleef_quad)); + cblas_qtrsv(layout, uplo, trans, diag, n, A_use, lda_use, x, 1); + + char tag[128]; + snprintf(tag, sizeof tag, "trsv n=%d layout=%d uplo=%d trans=%d diag=%d", n, layout, uplo, trans, diag); + CHECK_ARR(x, x_orig, n, 1e-25, tag); + + free(x); free(b); free(x_orig); free(Arow); + if (Acol) free(Acol); +} + +static void test_trsv_all(void) { + int sizes[] = { 1, 5, 16, 33, 64 }; + for (size_t s = 0; s < sizeof sizes / sizeof sizes[0]; ++s) { + int n = sizes[s]; + for (int layout = QblasRowMajor; layout <= QblasColMajor; ++layout) + for (int uplo = QblasUpper; uplo <= QblasLower; ++uplo) + for (int trans = QblasNoTrans; trans <= QblasTrans; ++trans) + for (int diag = QblasNonUnit; diag <= QblasUnit; ++diag) + test_trsv_case(n, layout, uplo, trans, diag); + } +} + +int main(void) { + test_gemv_all(); + test_ger_all(); + test_trsv_all(); + REPORT(); +} diff --git a/tests/test_level3_extra.c b/tests/test_level3_extra.c new file mode 100644 index 0000000..3b26d39 --- /dev/null +++ b/tests/test_level3_extra.c @@ -0,0 +1,205 @@ + +#include "test_helpers.h" + +static void ref_syrk_row(int doT, int n, int k, + Sleef_quad alpha, + const Sleef_quad *A, int lda, + Sleef_quad beta, Sleef_quad *C, int ldc) { + for (int i = 0; i < n; ++i) { + for (int j = 0; j < n; ++j) { + Sleef_quad s = qd(0.0); + for (int p = 0; p < k; ++p) { + Sleef_quad aip = doT ? A[(size_t)p * lda + i] : A[(size_t)i * lda + p]; + Sleef_quad ajp = doT ? A[(size_t)p * lda + j] : A[(size_t)j * lda + p]; + s = q_fma(aip, ajp, s); + } + C[(size_t)i * ldc + j] = q_fma(beta, C[(size_t)i * ldc + j], q_mul(alpha, s)); + } + } +} + +static void test_syrk(int n, int k, QBLAS_LAYOUT layout, QBLAS_UPLO uplo, QBLAS_TRANSPOSE trans) { + int doT = (trans == QblasTrans); + int Arows = doT ? k : n, Acols = doT ? n : k; + Sleef_quad *A_row = malloc((size_t)Arows * Acols * sizeof(Sleef_quad)); + Sleef_quad *C_row = malloc((size_t)n * n * sizeof(Sleef_quad)); + fill_mat(A_row, Arows, Acols, Acols); + fill_mat(C_row, n, n, n); + + Sleef_quad alpha = qd(0.7), beta = qd(0.3); + + Sleef_quad *C_ref = malloc((size_t)n * n * sizeof(Sleef_quad)); + memcpy(C_ref, C_row, (size_t)n * n * sizeof(Sleef_quad)); + ref_syrk_row(doT, n, k, alpha, A_row, Acols, beta, C_ref, n); + + Sleef_quad *A_use, *C_use; int lda, ldc; + Sleef_quad *A_col=NULL, *C_col=NULL; + if (layout == QblasRowMajor) { + A_use = A_row; lda = Acols; + C_use = malloc((size_t)n * n * sizeof(Sleef_quad)); + memcpy(C_use, C_row, (size_t)n * n * sizeof(Sleef_quad)); + ldc = n; + } else { + A_col = malloc((size_t)Arows * Acols * sizeof(Sleef_quad)); + C_col = malloc((size_t)n * n * sizeof(Sleef_quad)); + for (int i = 0; i < Arows; ++i) for (int j = 0; j < Acols; ++j) + A_col[(size_t)j * Arows + i] = A_row[(size_t)i * Acols + j]; + for (int i = 0; i < n; ++i) for (int j = 0; j < n; ++j) + C_col[(size_t)j * n + i] = C_row[(size_t)i * n + j]; + A_use = A_col; lda = Arows; + C_use = C_col; ldc = n; + } + + cblas_qsyrk(layout, uplo, trans, n, k, alpha, A_use, lda, beta, C_use, ldc); + + Sleef_quad *C_got = malloc((size_t)n * n * sizeof(Sleef_quad)); + if (layout == QblasRowMajor) memcpy(C_got, C_use, (size_t)n * n * sizeof(Sleef_quad)); + else for (int i = 0; i < n; ++i) for (int j = 0; j < n; ++j) + C_got[(size_t)i * n + j] = C_use[(size_t)j * n + i]; + + int fails = 0; + for (int i = 0; i < n; ++i) { + for (int j = 0; j < n; ++j) { + int in_tri = (uplo == QblasUpper) ? (j >= i) : (j <= i); + if (!in_tri) continue; + Sleef_quad d = q_abs(q_sub(C_got[(size_t)i*n+j], C_ref[(size_t)i*n+j])); + Sleef_quad s = q_add(q_abs(C_got[(size_t)i*n+j]), q_abs(C_ref[(size_t)i*n+j])); + double dd = dq(d), ss = dq(s); + double r = (ss > 1e-20) ? dd/ss : dd; + if (r > 1e-28) ++fails; + } + } + char tag[128]; + snprintf(tag, sizeof tag, "syrk n=%d k=%d layout=%d uplo=%d trans=%d", n, k, layout, uplo, trans); + CHECK(fails == 0, "%s (%d cells off)", tag, fails); + + free(C_got); + if (layout == QblasRowMajor) free(C_use); + if (A_col) free(A_col); + if (C_col) free(C_col); + free(C_ref); free(C_row); free(A_row); +} + +/* Verify trsm by multiplying op(A)*X and comparing to alpha*B_orig. + * A and B are kept row-major; col-major paths transpose into a scratch + * buffer before being passed to qblas. */ +static inline Sleef_quad rowmaj_A(const Sleef_quad *A_row, int m, int i, int j) { + return A_row[(size_t)i * m + j]; +} +static inline Sleef_quad rowmaj_B(const Sleef_quad *B_row, int n, int i, int j) { + return B_row[(size_t)i * n + j]; +} + +static void test_trsm_left(int m, int n, QBLAS_LAYOUT layout, + QBLAS_UPLO uplo, QBLAS_TRANSPOSE transa, QBLAS_DIAG diag) { + Sleef_quad *A_row = malloc((size_t)m * m * sizeof(Sleef_quad)); + Sleef_quad *B_row = malloc((size_t)m * n * sizeof(Sleef_quad)); + Sleef_quad *B0_row = malloc((size_t)m * n * sizeof(Sleef_quad)); + fill_mat(A_row, m, m, m); + for (int i = 0; i < m; ++i) A_row[(size_t)i * m + i] = qd(frand(2.0, 4.0)); + fill_mat(B_row, m, n, n); + memcpy(B0_row, B_row, (size_t)m * n * sizeof(Sleef_quad)); + + Sleef_quad *A_use, *B_use; int lda, ldb; + Sleef_quad *A_col = NULL, *B_col = NULL; + if (layout == QblasRowMajor) { + A_use = A_row; lda = m; + B_use = B_row; ldb = n; + } else { + A_col = malloc((size_t)m * m * sizeof(Sleef_quad)); + B_col = malloc((size_t)m * n * sizeof(Sleef_quad)); + for (int i = 0; i < m; ++i) for (int j = 0; j < m; ++j) + A_col[(size_t)j * m + i] = A_row[(size_t)i * m + j]; + for (int i = 0; i < m; ++i) for (int j = 0; j < n; ++j) + B_col[(size_t)j * m + i] = B_row[(size_t)i * n + j]; + A_use = A_col; lda = m; + B_use = B_col; ldb = m; + } + + Sleef_quad alpha = qd(0.5); + cblas_qtrsm(layout, QblasLeft, uplo, transa, diag, m, n, + alpha, A_use, lda, B_use, ldb); + + Sleef_quad *X_row = malloc((size_t)m * n * sizeof(Sleef_quad)); + if (layout == QblasRowMajor) memcpy(X_row, B_use, (size_t)m * n * sizeof(Sleef_quad)); + else for (int i = 0; i < m; ++i) for (int j = 0; j < n; ++j) + X_row[(size_t)i * n + j] = B_use[(size_t)j * m + i]; + + /* Backward error: ‖op(A)·X − alpha·B0‖_F / (‖A‖_F·‖X‖_F + ‖alpha·B0‖_F) + * — the standard stability check for triangular solves. */ + int doT = (transa == QblasTrans); + int upper = (uplo == QblasUpper); + double res_sq = 0.0, denom_sq = 0.0, A_fro = 0.0, X_fro = 0.0, B_fro = 0.0; + for (int i = 0; i < m; ++i) { + for (int p = 0; p < m; ++p) { + int in_tri_orig = upper ? (p >= i) : (p <= i); + if (!in_tri_orig) continue; + double a = (diag == QblasUnit && p == i) ? 1.0 + : dq(rowmaj_A(A_row, m, i, p)); + A_fro += a * a; + } + } + for (int i = 0; i < m; ++i) { + for (int j = 0; j < n; ++j) { + X_fro += dq(rowmaj_B(X_row, n, i, j)) * dq(rowmaj_B(X_row, n, i, j)); + B_fro += dq(q_mul(alpha, rowmaj_B(B0_row, n, i, j))) * dq(q_mul(alpha, rowmaj_B(B0_row, n, i, j))); + } + } + for (int i = 0; i < m; ++i) { + for (int j = 0; j < n; ++j) { + Sleef_quad s = qd(0.0); + for (int p = 0; p < m; ++p) { + int ii = doT ? p : i; + int pp = doT ? i : p; + int in_tri = upper ? (pp >= ii) : (pp <= ii); + if (!in_tri) continue; + Sleef_quad aip; + if (diag == QblasUnit && ii == pp) aip = qd(1.0); + else aip = rowmaj_A(A_row, m, ii, pp); + s = q_fma(aip, rowmaj_B(X_row, n, p, j), s); + } + Sleef_quad rhs = q_mul(alpha, rowmaj_B(B0_row, n, i, j)); + double r = dq(q_sub(s, rhs)); + res_sq += r * r; + } + } + A_fro = sqrt(A_fro); + X_fro = sqrt(X_fro); + B_fro = sqrt(B_fro); + denom_sq = A_fro * X_fro + B_fro + 1e-300; + double backward_err = sqrt(res_sq) / denom_sq; + char tag[128]; + snprintf(tag, sizeof tag, "trsm m=%d n=%d layout=%d uplo=%d ta=%d diag=%d", + m, n, layout, uplo, transa, diag); + /* 1e-25 ceiling is ~10¹⁰ tighter than double-precision rounding. */ + CHECK(backward_err < 1e-25, "%s backward_err=%.3e", tag, backward_err); + free(X_row); + if (A_col) free(A_col); + if (B_col) free(B_col); + free(B0_row); free(B_row); free(A_row); +} + +int main(void) { + int dims[][2] = { {3,3}, {8,5}, {17,11}, {32,32} }; + for (size_t d = 0; d < sizeof dims / sizeof dims[0]; ++d) { + for (int layout = QblasRowMajor; layout <= QblasColMajor; ++layout) + for (int uplo = QblasUpper; uplo <= QblasLower; ++uplo) + for (int trans = QblasNoTrans; trans <= QblasTrans; ++trans) { + test_syrk(dims[d][0], dims[d][1], (QBLAS_LAYOUT)layout, + (QBLAS_UPLO)uplo, (QBLAS_TRANSPOSE)trans); + } + } + + /* trsm size grid spans the NB=64 block-size boundary. */ + int td[][2] = { {3,3}, {8,5}, {17,11}, {63,17}, {64,33}, {129,11}, {200,40} }; + for (size_t d = 0; d < sizeof td / sizeof td[0]; ++d) { + for (int layout = QblasRowMajor; layout <= QblasColMajor; ++layout) + for (int uplo = QblasUpper; uplo <= QblasLower; ++uplo) + for (int trans = QblasNoTrans; trans <= QblasTrans; ++trans) + for (int diag = QblasNonUnit; diag <= QblasUnit; ++diag) + test_trsm_left(td[d][0], td[d][1], (QBLAS_LAYOUT)layout, + (QBLAS_UPLO)uplo, (QBLAS_TRANSPOSE)trans, + (QBLAS_DIAG)diag); + } + REPORT(); +} diff --git a/tests/test_sleef_simd.cpp b/tests/test_sleef_simd.cpp deleted file mode 100644 index be572d1..0000000 --- a/tests/test_sleef_simd.cpp +++ /dev/null @@ -1,130 +0,0 @@ -#include "quadblas.hpp" -#include -#include -#include - -// Helper functions -double to_double(Sleef_quad q) { - return static_cast(Sleef_cast_to_doubleq1(q)); -} - -Sleef_quad from_double(double d) { - return Sleef_cast_from_doubleq1(d); -} - -void test_quadVector_fixed() { - std::cout << "\n=== Testing Fixed QuadVector ===\n"; - - QuadBLAS::QuadVector v1(from_double(2.0), from_double(3.0)); - QuadBLAS::QuadVector v2(from_double(4.0), from_double(5.0)); - - std::cout << "v1: [" << to_double(v1.get(0)) << ", " << to_double(v1.get(1)) << "]" << std::endl; - std::cout << "v2: [" << to_double(v2.get(0)) << ", " << to_double(v2.get(1)) << "]" << std::endl; - - // Test store with workaround - Sleef_quad result1[2], result2[2]; - v1.store(result1); - v2.store(result2); - - std::cout << "Stored v1: [" << to_double(result1[0]) << ", " << to_double(result1[1]) << "]" << std::endl; - std::cout << "Stored v2: [" << to_double(result2[0]) << ", " << to_double(result2[1]) << "]" << std::endl; - - // Test arithmetic - QuadBLAS::QuadVector sum = v1 + v2; - QuadBLAS::QuadVector prod = v1 * v2; - - std::cout << "v1 + v2: [" << to_double(sum.get(0)) << ", " << to_double(sum.get(1)) << "]" << std::endl; - std::cout << "v1 * v2: [" << to_double(prod.get(0)) << ", " << to_double(prod.get(1)) << "]" << std::endl; - - // Test FMA: v1 * v2 + sum (should be [2*4+6, 3*5+8] = [14, 23]) - QuadBLAS::QuadVector fma_result = v1.fma(v2, sum); - std::cout << "v1*v2 + sum: [" << to_double(fma_result.get(0)) << ", " << to_double(fma_result.get(1)) - << "] (expected [14, 23])" << std::endl; -} - -void test_dot_fixed() { - std::cout << "\n=== Testing Fixed Dot Product ===\n"; - - const size_t n = 5; - Sleef_quad x[5] = {from_double(1), from_double(2), from_double(3), from_double(4), from_double(5)}; - Sleef_quad y[5] = {from_double(1), from_double(1), from_double(1), from_double(1), from_double(1)}; - - std::cout << "x: [1, 2, 3, 4, 5]" << std::endl; - std::cout << "y: [1, 1, 1, 1, 1]" << std::endl; - - // Test vectorized kernel - Sleef_quad vectorized_result = QuadBLAS::dot_kernel_vectorized(x, y, n); - std::cout << "Vectorized dot: " << to_double(vectorized_result) << " (expected 15)" << std::endl; - - // Test full dot function - Sleef_quad full_result = QuadBLAS::dot(n, x, 1, y, 1); - std::cout << "Full dot function: " << to_double(full_result) << " (expected 15)" << std::endl; -} - -void test_vector_class_fixed() { - std::cout << "\n=== Testing Fixed Vector Class ===\n"; - - const size_t n = 5; - QuadBLAS::DefaultVector<> x(n), y(n); - - for (size_t i = 0; i < n; ++i) { - x[i] = from_double(i + 1); // 1, 2, 3, 4, 5 - y[i] = from_double(1.0); // 1, 1, 1, 1, 1 - } - - std::cout << "x: [1, 2, 3, 4, 5]" << std::endl; - std::cout << "y: [1, 1, 1, 1, 1]" << std::endl; - - Sleef_quad result = x.dot(y); - std::cout << "Vector class dot: " << to_double(result) << " (expected 15)" << std::endl; - - // Test norm - Sleef_quad norm_result = x.norm(); - double expected_norm = std::sqrt(1 + 4 + 9 + 16 + 25); // sqrt(55) ≈ 7.416 - std::cout << "Vector norm: " << to_double(norm_result) << " (expected " << expected_norm << ")" << std::endl; -} - -void test_simple_gemv() { - std::cout << "\n=== Testing Simple GEMV ===\n"; - - // 2x2 matrix test - const size_t m = 2, n = 2; - QuadBLAS::DefaultMatrix<> A(m, n); - QuadBLAS::DefaultVector<> x(n), y(m); - - // A = [[1, 2], [3, 4]] - A(0, 0) = from_double(1.0); A(0, 1) = from_double(2.0); - A(1, 0) = from_double(3.0); A(1, 1) = from_double(4.0); - - // x = [1, 1] - x[0] = from_double(1.0); - x[1] = from_double(1.0); - - std::cout << "A = [[1, 2], [3, 4]]" << std::endl; - std::cout << "x = [1, 1]" << std::endl; - - A.gemv(from_double(1.0), x, from_double(0.0), y); - - std::cout << "A*x = [" << to_double(y[0]) << ", " << to_double(y[1]) - << "] (expected [3, 7])" << std::endl; -} - -int main() { - std::cout << "=== Testing QuadBLAS Bug Fixes ===\n"; - std::cout << quadblas_get_version() << std::endl; - - try { - test_quadVector_fixed(); - test_dot_fixed(); - test_vector_class_fixed(); - test_simple_gemv(); - - std::cout << "\n=== Fix verification completed! ===\n"; - - } catch (const std::exception& e) { - std::cerr << "Error: " << e.what() << std::endl; - return 1; - } - - return 0; -} \ No newline at end of file diff --git a/tests/test_smoke.c b/tests/test_smoke.c new file mode 100644 index 0000000..8e71ef4 --- /dev/null +++ b/tests/test_smoke.c @@ -0,0 +1,32 @@ +#include +#include +#include + +static Sleef_quad qd(double x) { return Sleef_cast_from_doubleq1(x); } +static double dq(Sleef_quad x) { return (double)Sleef_cast_to_doubleq1(x); } + +int main(void) { + printf("QBLAS smoke test\n"); + printf(" version : %s\n", qblas_get_version()); + printf(" tier : %s\n", qblas_get_dispatch_tier()); + printf(" threads : %d\n", qblas_get_num_threads()); + + enum { N = 8 }; + Sleef_quad x[N], y[N]; + for (int i = 0; i < N; ++i) { + x[i] = qd((double)(i + 1)); + y[i] = qd(2.0); + } + Sleef_quad dot = cblas_qdot(N, x, 1, y, 1); + printf(" dot : %f (expected 72)\n", dq(dot)); + if (dq(dot) != 72.0) { fprintf(stderr, "FAIL: dot\n"); return 1; } + + cblas_qaxpy(N, qd(0.5), x, 1, y, 1); + if (dq(y[0]) != 2.5 || dq(y[7]) != 6.0) { + fprintf(stderr, "FAIL: axpy y[0]=%f y[7]=%f\n", dq(y[0]), dq(y[7])); + return 1; + } + + printf(" smoke : OK\n"); + return 0; +} diff --git a/third_party/sleef b/third_party/sleef new file mode 160000 index 0000000..7623d6c --- /dev/null +++ b/third_party/sleef @@ -0,0 +1 @@ +Subproject commit 7623d6cfa2712462880fa63a4d0f0b5f775d1a83